Re: nginx: worker process: malloc(): memory corruption

[Борис Долгов <boris@xxxxxxxxxxx>]

А какая версия glibc?
Видел подобные баги на 2.10, когда на 2.5 все работает без нареканий.

20 июля 2009 г. 15:33 пользователь Konstantin A. Lepikhov
(lakostis@xxxxxxxxx) написал:
> Привет!
>
> Имеется nginx версии 0.7.61, сконфигуренный след. образом:
>
> user www-data;
> # падения напрямую связаны с кол-вом worker'ов - чем больше, тем меньше
> # падений
> worker_processes  10;
>
> ...
>
> worker_rlimit_core 100M;
> working_directory /var/tmp;
>
> events {
>    worker_connections  1024;
>    # изначально был epoll, с которым тоже были падения
>    use select;
> }
>
> http {
>    include       /etc/nginx/mime.types;
>    default_type  application/octet-stream;
>
>
>    #sendfile        on;
>
>    keepalive_timeout  0;
>    tcp_nodelay        on;
>    server_tokens off;
>
>    #client_body_buffer_size 1m;
>    client_max_body_size 50m;
>
> ...
>
> # HTTPS server
> #
> server {
>        listen   443;
>        listen   14443;
>
>        ssl  on;
>        ssl_certificate         /etc/nginx/ssl/server.crt;
>        ssl_certificate_key     /etc/nginx/ssl/server.key;
>
>        ssl_session_timeout             5m;
>        ssl_session_cache               shared:SSL:10m;
>
>        ssl_verify_client               on;
>        ssl_verify_depth                2;
>        ssl_client_certificate          /etc/nginx/ssl/cachain.pem;
> ...
>
> Периодически наблюдаем в логах сообщения вида:
>
> *** glibc detected *** nginx: worker process: malloc(): memory corruption:
> 0x0000000002264130 ***
> ======= Backtrace: =========
> /lib/libc.so.6[0x7f0502c99a14]
> /lib/libc.so.6(__libc_malloc+0x90)[0x7f0502c9b360]
> nginx: worker process[0x41af7e]
> nginx: worker process[0x40702d]
> nginx: worker process[0x4071b6]
> nginx: worker process[0x408e33]
> nginx: worker process[0x4572c2]
> nginx: worker process[0x43e7bf]
> nginx: worker process[0x435618]
> nginx: worker process[0x456797]
> nginx: worker process[0x42bd2c]
> nginx: worker process[0x4272bd]
> nginx: worker process[0x42ff02]
> nginx: worker process[0x4307e3]
> nginx: worker process[0x418e56]
> nginx: worker process[0x41e2fa]
> nginx: worker process[0x41cbe0]
> nginx: worker process[0x41eef3]
> nginx: worker process[0x40646b]
> /lib/libc.so.6(__libc_start_main+0xf4)[0x7f0502c431c4]
> nginx: worker process[0x404d09]
> ======= Memory map: ========
> 00400000-00476000 r-xp 00000000 09:01 306212
> /usr/sbin/nginx
> 00676000-00684000 rw-p 00076000 09:01 306212
> /usr/sbin/nginx
> 00684000-00692000 rw-p 00684000 00:00 0
> 02187000-02307000 rw-p 02187000 00:00 0
> [heap]
> 7f04fc000000-7f04fc021000 rw-p 7f04fc000000 00:00 0
> 7f04fc021000-7f0500000000 ---p 7f04fc021000 00:00 0
> 7f05017de000-7f05017eb000 r-xp 00000000 09:01 273151
> /lib/libgcc_s.so.1
> 7f05017eb000-7f05019eb000 ---p 0000d000 09:01 273151
> /lib/libgcc_s.so.1
> 7f05019eb000-7f05019ec000 rw-p 0000d000 09:01 273151
> /lib/libgcc_s.so.1
> 7f05019ec000-7f05023ec000 rw-s 00000000 00:09 10634
> /dev/zero (deleted)
> 7f05023ec000-7f05023f6000 r-xp 00000000 09:01 274714
> /lib/libnss_files-2.7.so
> 7f05023f6000-7f05025f6000 ---p 0000a000 09:01 274714
> /lib/libnss_files-2.7.so
> 7f05025f6000-7f05025f8000 rw-p 0000a000 09:01 274714
> /lib/libnss_files-2.7.so
> 7f05025f8000-7f0502602000 r-xp 00000000 09:01 274716
> /lib/libnss_nis-2.7.so
> 7f0502602000-7f0502801000 ---p 0000a000 09:01 274716
> /lib/libnss_nis-2.7.so
> 7f0502801000-7f0502803000 rw-p 00009000 09:01 274716
> /lib/libnss_nis-2.7.so
> 7f0502803000-7f0502819000 r-xp 00000000 09:01 274711
> /lib/libnsl-2.7.so
> 7f0502819000-7f0502a18000 ---p 00016000 09:01 274711
> /lib/libnsl-2.7.so
> 7f0502a18000-7f0502a1a000 rw-p 00015000 09:01 274711
> /lib/libnsl-2.7.so
> 7f0502a1a000-7f0502a1c000 rw-p 7f0502a1a000 00:00 0
> 7f0502a1c000-7f0502a24000 r-xp 00000000 09:01 274712
> /lib/libnss_compat-2.7.so
> 7f0502a24000-7f0502c23000 ---p 00008000 09:01 274712
> /lib/libnss_compat-2.7.so
> 7f0502c23000-7f0502c25000 rw-p 00007000 09:01 274712
> /lib/libnss_compat-2.7.so
> 7f0502c25000-7f0502d7d000 r-xp 00000000 09:01 274705
> /lib/libc-2.7.so
> 7f0502f82000-7f0502f87000 rw-p 7f0502f82000 00:00 0
> 7f0502f87000-7f0502f9d000 r-xp 00000000 09:01 307286
> /usr/lib/libz.so.1.2.3.3
> 7f0502f9d000-7f050319d000 ---p 00016000 09:01 307286
> /usr/lib/libz.so.1.2.3.3
> 7f050319d000-7f050319e000 rw-p 00016000 09:01 307286
> /usr/lib/libz.so.1.2.3.3
> 7f050319e000-7f05031a0000 r-xp 00000000 09:01 274708
> /lib/libdl-2.7.so
> 7f05031a0000-7f05033a0000 ---p 00002000 09:01 274708
> /lib/libdl-2.7.so
> 7f05033a0000-7f05033a2000 rw-p 00002000 09:01 274708
> /lib/libdl-2.7.so
> 7f05033a2000-7f05034fc000 r-xp 00000000 09:01 305318
> /usr/lib/libcrypto.so.0.9.8
> 7f05034fc000-7f05036fc000 ---p 0015a000 09:01 305318
> /usr/lib/libcrypto.so.0.9.8
> 7f05036fc000-7f050371f000 rw-p 0015a000 09:01 305318
> /usr/lib/libcrypto.so.0.9.8
> 7f050371f000-7f0503722000 rw-p 7f050371f000 00:00 0
> 7f0503722000-7f0503766000 r-xp 00000000 09:01 305319
> /usr/lib/libssl.so.0.9.8
> 7f0503766000-7f0503966000 ---p 00044000 09:01 305319
> /usr/lib/libssl.so.0.9.8
> 7f0503966000-7f050396c000 rw-p 00044000 09:01 305319
> /usr/lib/libssl.so.0.9.8
> 7f050396c000-7f0503991000 r-xp 00000000 09:01 310143
> /usr/lib/libpcre.so.3.12.1
> 7f0503991000-7f0503b91000 ---p 00025000 09:01 310143
> /usr/lib/libpcre.so.3.12.1
> 7f0503b91000-7f0503b92000 rw-p 00025000 09:01 310143
> /usr/lib/libpcre.so.3.12.1
> 7f0503b92000-7f0503b9b000 r-xp 00000000 09:01 274707
> /lib/libcrypt-2.7.so
> 7f0503b9b000-7f0503d9a000 ---p 00009000 09:01 274707
> /lib/libcrypt-2.7.so
> 7f0503d9a000-7f0503d9c000 rw-p 00008000 09:01 274707
> /lib/libcrypt-2.7.so
> 7f0503d9c000-7f0503dca000 rw-p 7f0503d9c000 00:00 0
> 7f0503dca000-7f0503de7000 r-xp 00000000 09:01 274700
> /lib/ld-2.7.so
> 7f0503fac000-7f0503fdf000 rw-p 7f0503fac000 00:00 0
> 7f0503fe3000-7f0503fe4000 rw-s 00000000 00:09 10639
> /dev/zero (deleted)
> 7f0503fe4000-7f0503fe7000 rw-p 7f0503fe4000 00:00 0
> 7f0503fe7000-7f0503fe9000 rw-p 0001d000 09:01 274700
> /lib/ld-2.7.so
> 7fff0bfd4000-7fff0bfe9000 rw-p 7ffffffea000 00:00 0
> [stack]
> 7fff0bfff000-7fff0c000000 r-xp 7fff0bfff000 00:00 0
> [vdso]
> ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0
> [vsyscall]
> 2009/07/20 07:44:46 [notice] 4311#0: signal 17 (SIGCHLD) received
> 2009/07/20 07:44:46 [alert] 4311#0: worker process 20515 exited on signal
> 6 (core dumped)
> 2009/07/20 07:44:46 [notice] 4311#0: start worker process 24049
> 2009/07/20 07:44:46 [notice] 4311#0: signal 29 (SIGIO) received
>
> если собрать без strip'а, то в backtrace на этот core  видно след.:
>
> # gdb /usr/sbin/nginx core
> GNU gdb 6.8-debian
> Copyright (C) 2008 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later
> <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "x86_64-linux-gnu"...
>
> warning: exec file is newer than core file.
>
> warning: Can't read pathname for load map: Input/output error.
> Loaded symbols for /lib/libcrypt.so.1
> Reading symbols from /usr/lib/libpcre.so.3...done.
> Loaded symbols for /usr/lib/libpcre.so.3
> Reading symbols from /usr/lib/libssl.so.0.9.8...done.
> Loaded symbols for /usr/lib/libssl.so.0.9.8
> Reading symbols from /usr/lib/libcrypto.so.0.9.8...done.
> Loaded symbols for /usr/lib/libcrypto.so.0.9.8
> Reading symbols from /lib/libdl.so.2...done.
> Loaded symbols for /lib/libdl.so.2
> Reading symbols from /usr/lib/libz.so.1...done.
> Loaded symbols for /usr/lib/libz.so.1
> Reading symbols from /lib/libc.so.6...done.
> Loaded symbols for /lib/libc.so.6
> Reading symbols from /lib/ld-linux-x86-64.so.2...done.
> Loaded symbols for /lib64/ld-linux-x86-64.so.2
> Reading symbols from /lib/libnss_compat.so.2...done.
> Loaded symbols for /lib/libnss_compat.so.2
> Reading symbols from /lib/libnsl.so.1...done.
> Loaded symbols for /lib/libnsl.so.1
> Reading symbols from /lib/libnss_nis.so.2...done.
> Loaded symbols for /lib/libnss_nis.so.2
> Reading symbols from /lib/libnss_files.so.2...done.
> Loaded symbols for /lib/libnss_files.so.2
> Reading symbols from /lib/libgcc_s.so.1...done.
> Loaded symbols for /lib/libgcc_s.so.1
> Core was generated by `nginx: worker process
> '.
> Program terminated with signal 6, Aborted.
> [New process 23609]
> #0  0x00007fea37ffc095 in raise () from /lib/libc.so.6
> (gdb) bt
> #0  0x00007fea37ffc095 in raise () from /lib/libc.so.6
> #1  0x00007fea37ffdaf0 in abort () from /lib/libc.so.6
> #2  0x00007fea38036a7b in ?? () from /lib/libc.so.6
> #3  0x00007fea3803e08a in ?? () from /lib/libc.so.6
> #4  0x00007fea38041c1c in free () from /lib/libc.so.6
> #5  0x0000000000406e4d in ngx_destroy_pool (pool=0x765db0) at
> src/core/ngx_palloc.c:86
> #6  0x000000000042b574 in ngx_http_request_done (r=0x796820, error=0) at
> src/http/ngx_http_request.c:2821
> #7  0x000000000042b6c1 in ngx_http_close_request (r=<value optimized out>,
> error=23609) at src/http/ngx_http_request.c:2746
> #8  0x000000000042ca3a in ngx_http_lingering_close_handler (rev=0x708de8)
> at src/http/ngx_http_request.c:2647
> #9  0x000000000041f694 in ngx_epoll_process_events (cycle=<value optimized
> out>, timer=<value optimized out>, flags=<value optimized out>)
> rc/event/modules/ngx_epoll_module.c:518
> 0x0000000000418aae in ngx_process_events_and_timers
> le=0x6a69d0) at src/event/ngx_event.c:245
> 0x000000000041e1aa in ngx_worker_process_cycle (cycle=0x6a69d0,
> =<value optimized out>) at src/os/unix/ngx_process_cycle.c:775
> 0x000000000041ca90 in ngx_spawn_process (cycle=0x6a69d0,
> =0x41e100 <ngx_worker_process_cycle>, data=0x0, name=0x45f00c
> ker process", respawn=1)
> at src/os/unix/ngx_process.c:194
> #13 0x000000000041eda3 in ngx_master_process_cycle
> (cycle=0x6a69d0) at src/os/unix/ngx_process_cycle.c:577
> #14 0x00000000004062cb in main (argc=3, argv=0x7fff4138d028) at
> src/core/nginx.c:382
> (gdb) f 5
> #5  0x0000000000406e4d in ngx_destroy_pool (pool=0x765db0) at
> src/core/ngx_palloc.c:86
> 86              ngx_free(p);
> (gdb) p p
> $1 = (ngx_pool_t *) 0x5c39
> (gdb) p->max
> Undefined command: "p->max".  Try "help".
> (gdb) p p->max
> Cannot access memory at address 0x5c59
>
> Игорь, есть идеи, как это можно исправить?
>
> --
> WBR et al.
>
>



-- 
С уважением, Борис Долгов.
icq 77556665
e-mail boris@xxxxxxxxxxx