Hiya,

I have a problem with a child HTTPD process hanging on my "Red Hat Enterprise Linux ES release 3 (Taroon Update 2)" server.

Basically a child process just "hangs" occaisionally and manages to use 100% CPU which causes the server to slow down..

"Top" shows the line:

Code:
20847 nobody    25   0 16940  10M  2552 R    95.4  2.0  11:00   0 httpd

If i look at what files/sockets this process is using, i get the following output from lsof..

Code:
[root@us1 root]# lsof | grep 20847
httpd     20847  nobody  cwd    DIR        3,3     12288        2 /
httpd     20847  nobody  rtd    DIR        3,3     12288        2 /
httpd     20847  nobody  txt    REG        3,3    495146    49375 /usr/local/apache/bin/httpd
httpd     20847  nobody  mem    REG        3,3     18632   540705 /lib/libnss_dns-2.3.2.so
httpd     20847  nobody  mem    CHR        1,5             105805 /dev/zero
httpd     20847  nobody  mem    REG        3,3     90924   540692 /lib/libnsl-2.3.2.so
httpd     20847  nobody  mem    REG        3,3     21436  4161734 /usr/lib/gconv/gconv-modules.cache
httpd     20847  nobody  mem    DEL        0,4             229376 /SYSV00000000
httpd     20847  nobody  mem    REG        3,3  32148976  3293448 /usr/lib/locale/locale-archive
httpd     20847  nobody  mem    REG        3,3     51888   540708 /lib/libnss_files-2.3.2.so
httpd     20847  nobody  mem    REG        3,3     76508   540720 /lib/libresolv-2.3.2.so
httpd     20847  nobody  mem    REG        3,3   3263177  3080420 /usr/local/apache/libexec/libphp4.so
httpd     20847  nobody  mem    REG        3,3   1567768  4423686 /lib/tls/libc-2.3.2.so
httpd     20847  nobody  mem    REG        3,3     14868   540688 /lib/libdl-2.3.2.so
httpd     20847  nobody  mem    REG        3,3    127288  1228836 /usr/lib/libexpat.so.0.4.0
httpd     20847  nobody  mem    REG        3,3     22504  1228838 /usr/lib/libgdbm.so.2.0.0
httpd     20847  nobody  mem    REG        3,3     23388   540686 /lib/libcrypt-2.3.2.so
httpd     20847  nobody  mem    REG        3,3    212884  4423689 /lib/tls/libm-2.3.2.so
httpd     20847  nobody  mem    REG        3,3    103840   540675 /lib/ld-2.3.2.so
httpd     20847  nobody    0r   CHR        1,3              99827 /dev/null
httpd     20847  nobody    1w   CHR        1,3              99827 /dev/null
httpd     20847  nobody    2w   REG        3,3   4067844  3080419 /usr/local/apache/logs/error_log
httpd     20847  nobody   15w   REG        3,3   4067844  3080419 /usr/local/apache/logs/error_log
httpd     20847  nobody   16u  IPv4   26918710                TCP *:http (LISTEN)
httpd     20847  nobody   17w   REG        3,3 204608166  3080422 /usr/local/apache/logs/access_log
httpd     20847  nobody   18w   REG        3,3     24469  3080423 /usr/local/apache/logs/injury.uk.com.log
httpd     20847  nobody   19w   REG        3,3         0  3080425 /usr/local/apache/logs/cred1t.com.log
httpd     20847  nobody   20w   REG        3,3   1396510  3080426 /usr/local/apache/logs/gamble.uk.com.log
httpd     20847  nobody   21w   REG        3,3     15709  3080403 /usr/local/apache/logs/namepowered.com.log
httpd     20847  nobody   22w   REG        3,3   1765196  3080401 /usr/local/apache/logs/linuxforums.org.log
One thing that did occur to me is that there is no connected TCP connection, its in a listening state..

Anyway, I attached to the process in GDB as per the following output:

Code:
[root@us1 root]# gdb
GNU gdb Red Hat Linux (6.0post-0.20040223.20rh)
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-redhat-linux-gnu".
(gdb) attach 20847
Attaching to process 20847
Reading symbols from /usr/local/apache/bin/httpd...(no debugging symbols found)...done.
Using host libthread_db library "/lib/tls/libthread_db.so.1".
Reading symbols from /lib/tls/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/tls/libm.so.6
Reading symbols from /lib/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/libcrypt.so.1
Reading symbols from /usr/lib/libgdbm.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libgdbm.so.2
Reading symbols from /usr/lib/libexpat.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libexpat.so.0
Reading symbols from /lib/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/libdl.so.2
Reading symbols from /lib/tls/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib/tls/libc.so.6
Reading symbols from /lib/ld-linux.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/ld-linux.so.2
Reading symbols from /lib/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib/libnss_files.so.2
Reading symbols from /usr/local/apache/libexec/libphp4.so...done.
Loaded symbols for /usr/local/apache/libexec/libphp4.so
Reading symbols from /lib/libresolv.so.2...done.
Loaded symbols for /lib/libresolv.so.2
Reading symbols from /lib/libnsl.so.1...done.
Loaded symbols for /lib/libnsl.so.1
Reading symbols from /lib/libnss_dns.so.2...done.
Loaded symbols for /lib/libnss_dns.so.2
0xb74a130b in malloc_consolidate () from /lib/tls/libc.so.6
(gdb) n
Single stepping until exit from function malloc_consolidate,
which has no line number information.

<<<<< I DONE A CTRL-C HERE AS IT HANGED AGAIN.. <<<<<

&#40;gdb&#41; bt
#0  0xb74a133a in malloc_consolidate &#40;&#41; from /lib/tls/libc.so.6
#1  0xb74a11e8 in _int_free &#40;&#41; from /lib/tls/libc.so.6
#2  0xb749fe68 in free &#40;&#41; from /lib/tls/libc.so.6
#3  0xb734abf7 in php_regfree &#40;preg=0x1&#41; at /usr/local/src/php-4.3.8/regex/regfree.c&#58;31
#4  0xb7376bd7 in _free_reg_cache &#40;rc=0x81e4930&#41; at /usr/local/src/php-4.3.8/ext/standard/reg.c&#58;66
#5  0xb73d8d10 in zend_hash_destroy &#40;ht=0xb74271c0&#41; at /usr/local/src/php-4.3.8/Zend/zend_hash.c&#58;556
#6  0xb7376c27 in php_reg_destroy_globals &#40;reg_globals=0xb74271c0&#41; at /usr/local/src/php-4.3.8/ext/standard/reg.c&#58;81
#7  0xb7376c72 in zm_shutdown_regex &#40;type=1, module_number=3&#41; at /usr/local/src/php-4.3.8/ext/standard/reg.c&#58;93
#8  0xb73526a5 in zm_shutdown_basic &#40;type=1, module_number=3&#41; at /usr/local/src/php-4.3.8/ext/standard/basic_functions.c&#58;1144
#9  0xb73d6f2d in module_destructor &#40;module=0x80efa48&#41; at /usr/local/src/php-4.3.8/Zend/zend_API.c&#58;1125
#10 0xb73d8f0c in zend_hash_apply_deleter &#40;ht=0xb742a780, p=0x80efa18&#41; at /usr/local/src/php-4.3.8/Zend/zend_hash.c&#58;611
#11 0xb73d8f9a in zend_hash_graceful_reverse_destroy &#40;ht=0xb742a780&#41; at /usr/local/src/php-4.3.8/Zend/zend_hash.c&#58;677
#12 0xb73d4445 in zend_shutdown &#40;&#41; at /usr/local/src/php-4.3.8/Zend/zend.c&#58;556
#13 0xb73aead3 in php_module_shutdown &#40;&#41; at /usr/local/src/php-4.3.8/main/main.c&#58;1286
#14 0xb73aea9a in php_module_shutdown_wrapper &#40;sapi_globals=0xb7423b00&#41; at /usr/local/src/php-4.3.8/main/main.c&#58;1263
#15 0xb73e6650 in php_child_exit_handler &#40;s=0x80b805c, p=0x818a434&#41; at /usr/local/src/php-4.3.8/sapi/apache/mod_php4.c&#58;868
#16 0x080742ad in ap_child_exit_modules &#40;&#41;
#17 0x0807ac15 in clean_child_exit &#40;&#41;
#18 0x0807db7f in child_main &#40;&#41;
#19 0x0807e21d in make_child &#40;&#41;
#20 0x0807e55b in perform_idle_server_maintenance &#40;&#41;
#21 0x0807eb92 in standalone_main &#40;&#41;
#22 0x0807f198 in main &#40;&#41;
&#40;gdb&#41;
It is definatly hanging in that libc malloc_consolidate() function..

Navigating back up the stack I was able to get a look at where the code called free()..:

Code:
&#40;gdb&#41; up
#1  0xb74a11e8 in _int_free &#40;&#41; from /lib/tls/libc.so.6
&#40;gdb&#41; up
#2  0xb749fe68 in free &#40;&#41; from /lib/tls/libc.so.6
&#40;gdb&#41; up
#3  0xb734abf7 in php_regfree &#40;preg=0x1&#41; at /usr/local/src/php-4.3.8/regex/regfree.c&#58;31
31                      free&#40;&#40;char *&#41;g->sets&#41;;
&#40;gdb&#41; list /usr/local/src/php-4.3.8/regex/regfree.c&#58;31
26              g->magic = 0;                   /* mark it invalid */
27
28              if &#40;g->strip != NULL&#41;
29                      free&#40;&#40;char *&#41;g->strip&#41;;
30              if &#40;g->sets != NULL&#41;
31                      free&#40;&#40;char *&#41;g->sets&#41;;
32              if &#40;g->setbits != NULL&#41;
33                      free&#40;&#40;char *&#41;g->setbits&#41;;
34              if &#40;g->must != NULL&#41;
35                      free&#40;g->must&#41;;
&#40;gdb&#41; p g
$3 = &#40;struct re_guts *&#41; 0xb7564380
&#40;gdb&#41; p *g
$4 = &#123;magic = 1, strip = 0x0, csetsize = 0, ncsets = 0, sets = 0x0, setbits = 0x49 <Address 0x49 out of bounds>, cflags = 0, nstates = 0, firststate = 0, laststate = 0, iflags = 135171408,
  nbol = 135167304, neol = 135168120, ncategories = 135146056, categories = 0x0, must = 0x0, mlen = 143906184, nsub = 149198688, backrefs = 0, nplus = 1, catspace = ""&#125;
&#40;gdb&#41;
Also see above that g->sets on as free()'ed on line 31 is NULL by the time I got to print out the contents of the struct.

Here is the full code of regfree.c from my copy of php-4.3.8:

Code:
#include <sys/types.h>
#include <stdio.h>
#include <stdlib.h>
#include <regex.h>

#include "utils.h"
#include "regex2.h"

/*
 - regfree - free everything
 = API_EXPORT&#40;void&#41; regfree&#40;regex_t *&#41;;
 */
API_EXPORT&#40;void&#41;
regfree&#40;preg&#41;
regex_t *preg;
&#123;
        register struct re_guts *g;

        if &#40;preg->re_magic != MAGIC1&#41;   /* oops */
                return;                 /* nice to complain, but hard */

        g = preg->re_g;
        if &#40;g == NULL || g->magic != MAGIC2&#41;    /* oops again */
                return;
        preg->re_magic = 0;             /* mark it invalid */
        g->magic = 0;                   /* mark it invalid */

        if &#40;g->strip != NULL&#41;
                free&#40;&#40;char *&#41;g->strip&#41;;
        if &#40;g->sets != NULL&#41;
                free&#40;&#40;char *&#41;g->sets&#41;;
        if &#40;g->setbits != NULL&#41;
                free&#40;&#40;char *&#41;g->setbits&#41;;
        if &#40;g->must != NULL&#41;
                free&#40;g->must&#41;;
        free&#40;&#40;char *&#41;g&#41;;
&#125;
Version of libc, Redhat, and kernel:
Code:
&#91;root@us1 root&#93;# rpm -qa | grep libc
glibc-kernheaders-2.4-8.34
glibc-utils-2.3.2-95.20
glibc-headers-2.3.2-95.20
libcap-1.10-15
libcap-devel-1.10-15
glibc-common-2.3.2-95.20
glibc-devel-2.3.2-95.20
glibc-2.3.2-95.20
glibc-profile-2.3.2-95.20
&#91;root@us1 root&#93;# cat /etc/redhat-release
Red Hat Enterprise Linux ES release 3 &#40;Taroon Update 2&#41;
&#91;root@us1 root&#93;# cat /proc/version
Linux version 2.4.21-15.0.3.EL &#40;bhcompile@daffy.perf.redhat.com&#41; &#40;gcc version 3.2.3 20030502 &#40;Red Hat Linux 3.2.3-37&#41;&#41; #1 Tue Jun 29 18&#58;17&#58;52 EDT 2004
&#91;root@us1 root&#93;#

Version of apache is: 1.3.31, compiled with the following:
./configure --prefix=/usr/local/apache/ --enable-module=so --enable-module=rewrite

Im using PHP 4.3.8 as a module, compiled with:
./configure --with-mysql --with-apxs=/usr/local/apache/bin/apxs


Does anyone have any idea of what caused this, or what can be done to fix the issue? Is it just a buggy libc im looking at here?

As far as i know there is no "pattern" to this happening, its "just every now and then".

TIA

Jason