I have some code which is trying to detect if a device is mobile or tablet from user agent. The code uses rex_pcre module for regular _expression_ matches and works perfectly when testing on a single machine with no traffic on it. Once i push it live on production server with high volume traffic, the server starts hanging in a few minutes and i see strange errors in the log.
The server is running latest version of openresty.
2017/08/30 17:24:52 [alert] 16163#16163: worker process 16394 exited on signal 11
*** Error in `nginx: worker process': corrupted double-linked list: 0x00000000029324e0 ***
======= Backtrace: =========
/lib64/libc.so.6(+0x7ada4)[0x7fd46d0e2da4]
/lib64/libc.so.6(+0x7de41)[0x7fd46d0e5e41]
/lib64/libc.so.6(__libc_malloc+0x4c)[0x7fd46d0e7fbc]
/lib64/libc.so.6(posix_memalign+0x4c)[0x7fd46d0ea55c]
nginx: worker process(ngx_memalign+0x21)[0x43d9b1]
nginx: worker process[0x41ea32]
nginx: worker process[0x4d0670]
/usr/local/openresty/luajit/lib/libluajit-5.1.so.2(+0xaae3)[0x7fd46e281ae3]
nginx: worker process(ngx_http_lua_run_thread+0x99)[0x4db529]
nginx: worker process(ngx_http_lua_content_by_chunk+0x106)[0x4de796]
nginx: worker process(ngx_http_lua_content_handler_file+0xcb)[0x4deb1b]
nginx: worker process(ngx_http_lua_content_handler+0x6c)[0x4de2ec]
nginx: worker process(ngx_http_core_content_phase+0x1e)[0x45374e]
nginx: worker process(ngx_http_core_run_phases+0x25)[0x44df85]
nginx: worker process(ngx_http_run_posted_requests+0x30)[0x456f90]
nginx: worker process[0x443217]
nginx: worker process(ngx_process_events_and_timers+0x6b)[0x43ab9b]
nginx: worker process[0x441141]
nginx: worker process(ngx_spawn_process+0x154)[0x43fc24]
nginx: worker process[0x441584]
nginx: worker process(ngx_master_process_cycle+0x2ab)[0x441fab]
nginx: worker process(main+0x7e9)[0x41c649]
/lib64/libc.so.6(__libc_start_main+0xf5)[0x7fd46d089b35]
nginx: worker process[0x41c99e]
======= Memory map: ========
00400000-00550000 r-xp 00000000 08:05 7740195 /usr/local/openresty/nginx/sbin/nginx
0074f000-00750000 r--p 0014f000 08:05 7740195 /usr/local/openresty/nginx/sbin/nginx
00750000-00772000 rw-p 00150000 08:05 7740195 /usr/local/openresty/nginx/sbin/nginx
00772000-00793000 rw-p 00000000 00:00 0
01911000-02396000 rw-p 00000000 00:00 0 [heap]
02396000-02e43000 rw-p 00000000 00:00 0 [heap]
4033d000-4037d000 rw-p 00000000 00:00 0
405af000-405cf000 rw-p 00000000 00:00 0
4065c000-4067c000 rw-p 00000000 00:00 0
406fa000-4071a000 rw-p 00000000 00:00 0
40766000-40786000 rw-p 00000000 00:00 0
40a03000-40a23000 rw-p 00000000 00:00 0
40e3c000-40e5c000 rw-p 00000000 00:00 0
40e5c000-40e7c000 rw-p 00000000 00:00 0
4111f000-4113f000 rw-p 00000000 00:00 0
41265000-41285000 rw-p 00000000 00:00 0
412a9000-412c9000 rw-p 00000000 00:00 0
413fe000-4141e000 rw-p 00000000 00:00 0
416cc000-416ec000 rw-p 00000000 00:00 0
4179f000-417bf000 rw-p 00000000 00:00 0
41aa2000-41ac2000 rw-p 00000000 00:00 0
41ba6000-41bc6000 rw-p 00000000 00:00 0
41c66000-41c86000 rw-p 00000000 00:00 0
41dde000-41dfe000 rw-p 00000000 00:00 0
41ec1000-41ee1000 rw-p 00000000 00:00 0
7fd464000000-7fd464021000 rw-p 00000000 00:00 0
7fd464021000-7fd468000000 ---p 00000000 00:00 0
7fd469885000-7fd46988d000 r-xp 00000000 08:05 7868937 /usr/local/openresty/luajit/lib/lua/5.1/rex_pcre.so
7fd46988d000-7fd469a8c000 ---p 00008000 08:05 7868937 /usr/local/openresty/luajit/lib/lua/5.1/rex_pcre.so
7fd469a8c000-7fd469a8d000 r--p 00007000 08:05 7868937 /usr/local/openresty/luajit/lib/lua/5.1/rex_pcre.so
7fd469a8d000-7fd469a8e000 rw-p 00008000 08:05 7868937 /usr/local/openresty/luajit/lib/lua/5.1/rex_pcre.so
7fd469a8e000-7fd469a98000 r-xp 00000000 08:05 7868936 /usr/local/openresty/luajit/lib/lua/5.1/lpeg.so
7fd469a98000-7fd469c97000 ---p 0000a000 08:05 7868936 /usr/local/openresty/luajit/lib/lua/5.1/lpeg.so
7fd469c97000-7fd469c98000 r--p 00009000 08:05 7868936 /usr/local/openresty/luajit/lib/lua/5.1/lpeg.so
7fd469c98000-7fd469c99000 rw-p 0000a000 08:05 7868936 /usr/local/openresty/luajit/lib/lua/5.1/lpeg.so
7fd469c99000-7fd469ca0000 r-xp 00000000 08:05 7606195 /usr/local/openresty/lualib/cjson.so
7fd469ca0000-7fd469ea0000 ---p 00007000 08:05 7606195 /usr/local/openresty/lualib/cjson.so
7fd469ea0000-7fd469ea1000 r--p 00007000 08:05 7606195 /usr/local/openresty/lualib/cjson.so
7fd469ea1000-7fd469ea2000 rw-p 00008000 08:05 7606195 /usr/local/openresty/lualib/cjson.so
7fd469ea2000-7fd46ba0d000 rw-p 00000000 00:00 0
7fd46ba0d000-7fd46c40d000 rw-s 00000000 00:04 2118029276 /dev/zero (deleted)
7fd46c40d000-7fd46c41b000 r-xp 00000000 08:05 8130303 /usr/local/openresty/luajit/lib/lua/5.1/socket/core.so
7fd46c41b000-7fd46c61a000 ---p 0000e000 08:05 8130303 /usr/local/openresty/luajit/lib/lua/5.1/socket/core.so
7fd46c61a000-7fd46c61b000 r--p 0000d000 08:05 8130303 /usr/local/openresty/luajit/lib/lua/5.1/socket/core.so
7fd46c61b000-7fd46c61c000 rw-p 0000e000 08:05 8130303 /usr/local/openresty/luajit/lib/lua/5.1/socket/core.so
7fd46c61c000-7fd46c632000 r-xp 00000000 08:05 7475049 /usr/lib64/libresolv-2.17.so
7fd46c632000-7fd46c832000 ---p 00016000 08:05 7475049 /usr/lib64/libresolv-2.17.so
7fd46c832000-7fd46c833000 r--p 00016000 08:05 7475049 /usr/lib64/libresolv-2.17.so
7fd46c833000-7fd46c834000 rw-p 00017000 08:05 7475049 /usr/lib64/libresolv-2.17.so
7fd46c834000-7fd46c836000 rw-p 00000000 00:00 0
7fd46c836000-7fd46c83b000 r-xp 00000000 08:05 7471705 /usr/lib64/libnss_dns-2.17.so
7fd46c83b000-7fd46ca3a000 ---p 00005000 08:05 7471705 /usr/lib64/libnss_dns-2.17.so
7fd46ca3a000-7fd46ca3b000 r--p 00004000 08:05 7471705 /usr/lib64/libnss_dns-2.17.so
7fd46ca3b000-7fd46ca3c000 rw-p 00005000 08:05 7471705 /usr/lib64/libnss_dns-2.17.so
7fd46ca3c000-7fd46ca48000 r-xp 00000000 08:05 7475045 /usr/lib64/libnss_files-2.17.so
7fd46ca48000-7fd46cc47000 ---p 0000c000 08:05 7475045 /usr/lib64/libnss_files-2.17.so
7fd46cc47000-7fd46cc48000 r--p 0000b000 08:05 7475045 /usr/lib64/libnss_files-2.17.so
7fd46cc48000-7fd46cc49000 rw-p 0000c000 08:05 7475045 /usr/lib64/libnss_files-2.17.so
7fd46cc49000-7fd46cc4f000 rw-p 00000000 00:00 0
7fd46cc4f000-7fd46cc64000 r-xp 00000000 08:05 7471123 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fd46cc64000-7fd46ce63000 ---p 00015000 08:05 7471123 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fd46ce63000-7fd46ce64000 r--p 00014000 08:05 7471123 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fd46ce64000-7fd46ce65000 rw-p 00015000 08:05 7471123 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fd46ce65000-7fd46ce67000 r-xp 00000000 08:05 7471682 /usr/lib64/libfreebl3.so
7fd46ce67000-7fd46d066000 ---p 00002000 08:05 7471682 /usr/lib64/libfreebl3.so
7fd46d066000-7fd46d067000 r--p 00001000 08:05 7471682 /usr/lib64/libfreebl3.so
7fd46d067000-7fd46d068000 rw-p 00002000 08:05 7471682 /usr/lib64/libfreebl3.so
7fd46d068000-7fd46d21f000 r-xp 00000000 08:05 7471689 /usr/lib64/libc-2.17.so
7fd46d21f000-7fd46d41e000 ---p 001b7000 08:05 7471689 /usr/lib64/libc-2.17.so
7fd46d41e000-7fd46d422000 r--p 001b6000 08:05 7471689 /usr/lib64/libc-2.17.so
7fd46d422000-7fd46d424000 rw-p 001ba000 08:05 7471689 /usr/lib64/libc-2.17.so
7fd46d424000-7fd46d429000 rw-p 00000000 00:00 0
7fd46d429000-7fd46d442000 r-xp 00000000 08:05 7474894 /usr/local/openresty/zlib/lib/libz.so.1.2.11
7fd46d442000-7fd46d642000 ---p 00019000 08:05 7474894 /usr/local/openresty/zlib/lib/libz.so.1.2.11
7fd46d642000-7fd46d643000 r--p 00019000 08:05 7474894 /usr/local/openresty/zlib/lib/libz.so.1.2.11
7fd46d643000-7fd46d644000 rw-p 0001a000 08:05 7474894 /usr/local/openresty/zlib/lib/libz.so.1.2.11
7fd46d644000-7fd46d86a000 r-xp 00000000 08:05 7606184 /usr/local/openresty/openssl/lib/libcrypto.so.1.0.0
7fd46d86a000-7fd46da6a000 ---p 00226000 08:05 7606184 /usr/local/openresty/openssl/lib/libcrypto.so.1.0.0
7fd46da6a000-7fd46da85000 r--p 00226000 08:05 7606184 /usr/local/openresty/openssl/lib/libcrypto.so.1.0.0
7fd46da85000-7fd46da91000 rw-p 00241000 08:05 7606184 /usr/local/openresty/openssl/lib/libcrypto.so.1.0.0
7fd46da91000-7fd46da94000 rw-p 00000000 00:00 0
7fd46da94000-7fd46dafb000 r-xp 00000000 08:05 7606186 /usr/local/openresty/openssl/lib/libssl.so.1.0.0
7fd46dafb000-7fd46dcfb000 ---p 00067000 08:05 7606186 /usr/local/openresty/openssl/lib/libssl.so.1.0.0
7fd46dcfb000-7fd46dcff000 r--p 00067000 08:05 7606186 /usr/local/openresty/openssl/lib/libssl.so.1.0.0
7fd46dcff000-7fd46dd06000 rw-p 0006b000 08:05 7606186 /usr/local/openresty/openssl/lib/libssl.so.1.0.0
7fd46dd06000-7fd46dd74000 r-xp 00000000 08:05 7606191 /usr/local/openresty/pcre/lib/libpcre.so.1.2.8
7fd46dd74000-7fd46df73000 ---p 0006e000 08:05 7606191 /usr/local/openresty/pcre/lib/libpcre.so.1.2.8
7fd46df73000-7fd46df74000 r--p 0006d000 08:05 7606191 /usr/local/openresty/pcre/lib/libpcre.so.1.2.8
7fd46df74000-7fd46df75000 rw-p 0006e000 08:05 7606191 /usr/local/openresty/pcre/lib/libpcre.so.1.2.8
7fd46df75000-7fd46e075000 r-xp 00000000 08:05 7472921 /usr/lib64/libm-2.17.so
7fd46e075000-7fd46e275000 ---p 00100000 08:05 7472921 /usr/lib64/libm-2.17.so
7fd46e275000-7fd46e276000 r--p 00100000 08:05 7472921 /usr/lib64/libm-2.17.so
7fd46e276000-7fd46e277000 rw-p 00101000 08:05 7472921 /usr/lib64/libm-2.17.so
7fd46e277000-7fd46e2f0000 r-xp 00000000 08:05 7740177 /usr/local/openresty/luajit/lib/libluajit-5.1.so.2.1.0
7fd46e2f0000-7fd46e4ef000 ---p 00079000 08:05 7740177 /usr/local/openresty/luajit/lib/libluajit-5.1.so.2.1.0
7fd46e4ef000-7fd46e4f1000 r--p 00078000 08:05 7740177 /usr/local/openresty/luajit/lib/libluajit-5.1.so.2.1.0
7fd46e4f1000-7fd46e4f2000 rw-p 0007a000 08:05 7740177 /usr/local/openresty/luajit/lib/libluajit-5.1.so.2.1.0
7fd46e4f2000-7fd46e4fa000 r-xp 00000000 08:05 7471693 /usr/lib64/libcrypt-2.17.so
7fd46e4fa000-7fd46e6f9000 ---p 00008000 08:05 7471693 /usr/lib64/libcrypt-2.17.so
7fd46e6f9000-7fd46e6fa000 r--p 00007000 08:05 7471693 /usr/lib64/libcrypt-2.17.so
7fd46e6fa000-7fd46e6fb000 rw-p 00008000 08:05 7471693 /usr/lib64/libcrypt-2.17.so
7fd46e6fb000-7fd46e729000 rw-p 00000000 00:00 0
7fd46e729000-7fd46e740000 r-xp 00000000 08:05 7471715 /usr/lib64/libpthread-2.17.so
7fd46e740000-7fd46e93f000 ---p 00017000 08:05 7471715 /usr/lib64/libpthread-2.17.so
7fd46e93f000-7fd46e940000 r--p 00016000 08:05 7471715 /usr/lib64/libpthread-2.17.so
7fd46e940000-7fd46e941000 rw-p 00017000 08:05 7471715 /usr/lib64/libpthread-2.17.so
7fd46e941000-7fd46e945000 rw-p 00000000 00:00 0
7fd46e945000-7fd46e947000 r-xp 00000000 08:05 7472916 /usr/lib64/libdl-2.17.so
7fd46e947000-7fd46eb47000 ---p 00002000 08:05 7472916 /usr/lib64/libdl-2.17.so
7fd46eb47000-7fd46eb48000 r--p 00002000 08:05 7472916 /usr/lib64/libdl-2.17.so
7fd46eb48000-7fd46eb49000 rw-p 00003000 08:05 7472916 /usr/lib64/libdl-2.17.so
7fd46eb49000-7fd46eb69000 r-xp 00000000 08:05 7471678 /usr/lib64/ld-2.17.so
7fd46ec49000-7fd46ec59000 r-xp 00000000 00:00 0
7fd46ec59000-7fd46ed59000 rw-s 00000000 00:04 2118042170 /dev/zero (deleted)
7fd46ed59000-7fd46ed61000 rw-p 00000000 00:00 0
7fd46ed65000-7fd46ed66000 rw-p 00000000 00:00 0
7fd46ed66000-7fd46ed67000 rw-s 00000000 00:0a 2118215690 /[aio] (deleted)
7fd46ed67000-7fd46ed68000 rw-s 00000000 00:04 2118060045 /dev/zero (deleted)
7fd46ed68000-7fd46ed69000 rw-p 00000000 00:00 0
7fd46ed69000-7fd46ed6a000 r--p 00020000 08:05 7471678 /usr/lib64/ld-2.17.so
7fd46ed6a000-7fd46ed6b000 rw-p 00021000 08:05 7471678 /usr/lib64/ld-2.17.so
7fd46ed6b000-7fd46ed6c000 rw-p 00000000 00:00 0
7ffc5e219000-7ffc5e23a000 rw-p 00000000 00:00 0 [stack]
7ffc5e2e4000-7ffc5e2e6000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
Can we trace out anything from these error logs or are there any known memory leak issues with rex_pcre?
