Hello,
3 crashes occurred today (we never seen crashes on this server before) during a period of 20 minutes.
I had a look to the coredumps. Here is a quick overview of the first entries of "bt full":
core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0) at h_table.c:175
core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0) at h_table.c:157
core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0, p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762
core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000, frag=0x7f19864d53e8) at mem/f_malloc.c:245
core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000, frag=0x7f19864f4cc8) at mem/f_malloc.c:208
I don't know how to proceed now. Can you help me to highlight the issue here? Thank you.
Regards,
Igor.
Hello,
We got some others crashes even after updated to 4.2.3.
From the last one, I got the following into "bt full":
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
f = 0x7f7d4206d188
hash = 4
#1 0x000000000061a2fe in fm_free (qm=0x7f7d4206d000, p=0x7f7d422ea3a0, file=0x7f7d53408ff0 "dialog: dlg_timer.c", func=0x7f7d53409d00 "destroy_dlg_timer", line=95)
at mem/f_malloc.c:614
f = 0x7f7d422ea368
__FUNCTION__ = "fm_free"
#2 0x00007f7d533e441f in destroy_dlg_timer () at dlg_timer.c:95
__FUNCTION__ = "destroy_dlg_timer"
#3 0x00007f7d53393268 in mod_destroy () at dialog.c:784
No locals.
#4 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d59e9d670
foo = 0x7f7d59e9d440
__FUNCTION__ = "destroy_modules"
#5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#7 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 1107776752
__FUNCTION__ = "handle_sigs"
#8 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4147
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260T\a B}\177\000\000\000\000\000\020\004\000\000\000\260T\aB}\177\000\000\060SA\00 0\000\000\000\000\260:\240\343\001\000\000\000@8\240\343\377\177\000\000\032 dN\000\000\000\000\000h\261\346Yz\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x00000000004ab8bf in main (argc=7, argv=0x7fffe3a03ab8) at main.c:2578
cfg_stream = 0x1a04010
c = -1
r = 0
tmp = 0x7fffe3a04f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3320376299
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fffe3a0398e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] Envoyé : lundi 20 juillet 2015 16:22 À : sr-users@lists.sip-router.org Objet : Multiple crashes of Kamailio 4.2.1
Hello,
3 crashes occurred today (we never seen crashes on this server before) during a period of 20 minutes.
I had a look to the coredumps. Here is a quick overview of the first entries of "bt full":
core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0) at h_table.c:175
core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0) at h_table.c:157
core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0, p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762
core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000, frag=0x7f19864d53e8) at mem/f_malloc.c:245
core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000, frag=0x7f19864f4cc8) at mem/f_malloc.c:208
I don't know how to proceed now. Can you help me to highlight the issue here? Thank you.
Regards,
Igor.
Hello,
there are newer releases in branch 4.2, was there any reason to stop at 4.2.3?
Back to the backtrace, this one is from shutdown, happening during the cleanup, not showing the reason of the crash at runtime, unless there was a manual shut down triggered at that moment. Do you have another core file?
Cheers, Daniel
On 10/09/15 12:36, Igor Potjevlesch wrote:
Hello,
We got some others crashes even after updated to 4.2.3.
From the last one, I got the following into "bt full":
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
f = 0x7f7d4206d188 hash = 4#1 0x000000000061a2fe in fm_free (qm=0x7f7d4206d000, p=0x7f7d422ea3a0, file=0x7f7d53408ff0 "dialog: dlg_timer.c", func=0x7f7d53409d00 "destroy_dlg_timer", line=95)
at mem/f_malloc.c:614 f = 0x7f7d422ea368 __FUNCTION__ = "fm_free"#2 0x00007f7d533e441f in destroy_dlg_timer () at dlg_timer.c:95
__FUNCTION__ = "destroy_dlg_timer"#3 0x00007f7d53393268 in mod_destroy () at dialog.c:784
No locals.
#4 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d59e9d670 foo = 0x7f7d59e9d440 __FUNCTION__ = "destroy_modules"#5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0 __FUNCTION__ = "cleanup"#6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"#7 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0 chld_status = 139 memlog = 1107776752 __FUNCTION__ = "handle_sigs"#8 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8 pid = 4147 si = 0x0 si_desc = "udp receiver child=7sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260T\aB}\177\000\000\000\000\000\020\004\000\000\000\260T\aB}\177\000\000\060SA\000\000\000\000\000\260:\240\343\001\000\000\000@8\240\343\377\177\000\000\032dN\000\000\000\000\000h\261\346Yz\000\000\000\276}p\000\000\000\000"
nrprocs = 8 __FUNCTION__ = "main_loop"#9 0x00000000004ab8bf in main (argc=7, argv=0x7fffe3a03ab8) at main.c:2578
cfg_stream = 0x1a04010 c = -1 r = 0 tmp = 0x7fffe3a04f70 "" tmp_len = 0 port = 0 proto = 32767 options = 0x6fcc00":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1 seed = 3320376299 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xc2 p = 0x7fffe3a0398e "" __FUNCTION__ = "main"Regards,
Igor.
*De :*Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] *Envoyé :* lundi 20 juillet 2015 16:22 *À :* sr-users@lists.sip-router.org *Objet :* Multiple crashes of Kamailio 4.2.1
Hello,
3 crashes occurred today (we never seen crashes on this server before) during a period of 20 minutes.
I had a look to the coredumps. Here is a quick overview of the first entries of "bt full":
core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0) at h_table.c:175
core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0) at h_table.c:157
core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0, p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762
core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000, frag=0x7f19864d53e8) at mem/f_malloc.c:245
core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000, frag=0x7f19864f4cc8) at mem/f_malloc.c:208
I don't know how to proceed now. Can you help me to highlight the issue here? Thank you.
Regards,
Igor.
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list sr-users@lists.sip-router.org http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
Hello Daniel,
There is no particular reason. It's just that our others Kamailio are running 4.2.3. So, we try to have the same version on all nodes.
Regarding the backtrace, I'm surprised, because there was no manual shutdown.
Unfortunately, for this occurrence of the crash, it's the only coredump available. There was another crash later this day with two coredumps and the following backtrace:
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d2d49f000, frag=0x7f7d2d769830) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d2d49f000, frag=0x7f7d2d769830) at mem/f_malloc.c:245
f = 0x7f7d2d49f758
hash = 97
#1 0x000000000061a2fe in fm_free (qm=0x7f7d2d49f000, p=0x7f7d2d769868, file=0x7f7d43d77f4d "tm: h_table.c", func=0x7f7d43d78228 "free_cell", line=159) at mem/f_malloc.c:614
f = 0x7f7d2d769830
__FUNCTION__ = "fm_free"
#2 0x00007f7d43cbbfa4 in free_cell (dead_cell=0x7f7d2d75e600) at h_table.c:159
b = 0x0
i = 1
rpl = 0x0
tt = 0x0
foo = 0x0
cbs = 0x0
cbs_tmp = 0x7f7d2d75d378
__FUNCTION__ = "free_cell"
#3 0x00007f7d43cbd655 in free_hash_table () at h_table.c:448
p_cell = 0x7f7d2d75e600
tmp_cell = 0x7f7d2d5e89e0
i = 31354
__FUNCTION__ = "free_hash_table"
#4 0x00007f7d43ce1906 in tm_shutdown () at t_funcs.c:122
__FUNCTION__ = "tm_shutdown"
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d452c0a80
foo = 0x7f7d452c04b0
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 759854320
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 3362
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260tJ- }\177\000\000\000\000\000\020\004\000\000\000\260tJ-}\177\000\000\060SA\000\ 000\000\000\000\220\351\330\364\001\000\000\000 \347\330\364\377\177\000\000\032dN\000\000\000\000\000h\321)Ez\000\000\000\2 76}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7ffff4d8e998) at main.c:2578
cfg_stream = 0x2019010
c = -1
r = 0
tmp = 0x7ffff4d8ff70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3024771980
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7ffff4d8e86e ""
__FUNCTION__ = "main"
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x000000000061743c in fm_extract_free (qm=0x7f7d2d49f000, frag=0x7f7d2d78d098) at mem/f_malloc.c:208
208 if(frag->u.nxt_free) frag->u.nxt_free->prv_free = pf;
#0 0x000000000061743c in fm_extract_free (qm=0x7f7d2d49f000, frag=0x7f7d2d78d098) at mem/f_malloc.c:208
pf = 0x7f7d2d49f758
hash = 2097
#1 0x0000000000618af4 in fm_malloc (qm=0x7f7d2d49f000, size=776, file=0x754600 "<core>: mem/shm_mem.c", func=0x7553bc "sh_realloc", line=89) at mem/f_malloc.c:490
f = 0x7f7d2d49f758
frag = 0x7f7d2d78d098
hash = 97
__FUNCTION__ = "fm_malloc"
#2 0x000000000061e8df in sh_realloc (p=0x7f7d2d763ff0, size=774) at mem/shm_mem.c:89
r = 0x149f4d8e990
__FUNCTION__ = "sh_realloc"
#3 0x000000000061eb97 in _shm_resize (p=0x7f7d2d763ff0, s=774, file=0x7f7d43d9d263 "tm: t_reply.c", func=0x7f7d43da1ef1 "relay_reply", line=1944) at mem/shm_mem.c:114
__FUNCTION__ = "_shm_resize"
#4 0x00007f7d43d4fdc7 in relay_reply (t=0x7f7d2d7711a8, p_msg=0x7f7d454170b0, branch=0, msg_status=180, cancel_data=0x7ffff4d8e150, do_put_on_wait=1) at t_reply.c:1943
relay = 0
save_clone = 0
buf = 0x7f7d4541de08 "SIP/2.0 180 Ringing\r\nCSeq: 1 INVITE\r\nCall-ID: 7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom: "0123456789"sip:0123456789@A.B.C.D;tag=95ffcd055e0f78f7d5d397020e89288dc 9eabbab\r\nTo: <sip:01"...
res_len = 614
relayed_code = 180
relayed_msg = 0x7f7d454170b0
reply_bak = 0x4000000
bm = {to_tag_val = {s = 0xfffffffff4d8df20 <Address 0xfffffffff4d8df20 out of bounds>, len = 1}}
totag_retr = 0
reply_status = RPS_PROVISIONAL
uas_rb = 0x7f7d2d771268
to_tag = 0x7f7d2d772740
reason = {s = 0x7ffff4d8e178 "", len = 1161916592}
onsend_params = {req = 0x7ffff4d8df50, rpl = 0x7f7d43d0b170, param = 0x415330, code = -187111024, flags = 3, branch = 0, t_rbuf = 0x0, dst = 0xa6fa1c, send_buf = {
s = 0x7ffff4d8e060 "\320\341\330\364\377\177", len = 6772964}}
__FUNCTION__ = "relay_reply"
#5 0x00007f7d43d53642 in reply_received (p_msg=0x7f7d454170b0) at t_reply.c:2493
msg_status = 180
last_uac_status = 183
ack = 0x7f7d4529d010 "\001"
ack_len = 0
branch = 0
reply_status = 1160368600
onreply_route = 1
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 10943004}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 10943004}}}}
uac = 0x7f7d2d771310
t = 0x7f7d2d7711a8
lack_dst = {send_sock = 0x4000000, to = {s = {sa_family = 63657, sa_data = "\246\000\000\000\000\000\035\370\246\000\000\000\000"}, sin = {sin_family = 63657,
sin_port = 166, sin_addr = {s_addr = 0}, sin_zero = "\035\370\246\000\000\000\000"}, sin6 = {sin6_family = 63657, sin6_port = 166, sin6_flowinfo = 0,
sin6_addr = {__in6_u = {__u6_addr8 = "\035\370\246\000\000\000\000\000\220\313/E}\177\000", __u6_addr16 = {63517, 166, 0, 0, 52112, 17711, 32637, 0},
__u6_addr32 = {10942493, 0, 1160760208, 32637}}}, sin6_scope_id = 1160759624}}, id = 32637, proto = 48 '0', send_flags = {f = 174 '\256',
blst_imask = 47 '/'}}
backup_user_from = 0xa7f790
backup_user_to = 0xa7f798
backup_domain_from = 0xa7f7a0
backup_domain_to = 0xa7f7a8
backup_uri_from = 0xa7f780
backup_uri_to = 0xa7f788
backup_xavps = 0xa7f8c0
replies_locked = 1
branch_ret = 0
prev_branch = 1161916624
blst_503_timeout = 32637
hf = 0x2bcf4d8e220
onsend_params = {req = 0x7ffff4d8e110, rpl = 0x47dd84, param = 0x0, code = 1160754848, flags = 32637, branch = 0, t_rbuf = 0xa6fa1c, dst = 0xa6f853, send_buf = {
s = 0x7ffff4d8e1d0 "\210", <incomplete sequence \367\247>, len = 6393479}}
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {140176009556648, 387096857427510471, 4281136, 140737301244304, 0, 0, 387096857536562375,
-387072352975780665}, __mask_was_saved = 0, __saved_mask = {__val = {0, 140737301242320, 6430864, 140737301242080, 140175994783450, 140737301242048, 0,
67108864, 65540256, 1285944, 1568608, 1574280, 8, 88, 140175994783450, 1473240891392}}}}}
__FUNCTION__ = "reply_received"
#6 0x000000000048bf5d in do_forward_reply (msg=0x7f7d454170b0, mode=0) at forward.c:783
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000',
send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
new_len = 1
r = 0
ip = {af = 4107854704, len = 32767, u = {addrl = {6457509, 280}, addr32 = {6457509, 0, 280, 0}, addr16 = {34981, 98, 0, 0, 280, 0, 0, 0},
addr = "\245\210b\000\000\000\000\000\030\001\000\000\000\000\000"}}
s = 0x4 <Address 0x4 out of bounds>
len = 0
__FUNCTION__ = "do_forward_reply"
#7 0x000000000048d598 in forward_reply (msg=0x7f7d454170b0) at forward.c:885
No locals.
#8 0x0000000000508610 in receive_msg (
buf=0xa6f760 "SIP/2.0 180 Ringing\r\nCSeq: 1 INVITE\r\nCall-ID: 7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom: "0987654321"sip:0987654321@A.B.C.D;tag=95f
fcd055e0f78f7d5d397020e89288dc9eabbab\r\nTo: <sip:09"..., len=700, rcv_info=0x7ffff4d8e570) at receive.c:275
msg = 0x7f7d454170b0
ctx = {rec_lev = 10224768, run_flags = 0, last_retcode = 0, jmp_env = {{__jmpbuf = {0, 0, 0, 272136986608, 1812476198913, 0, 272145363728, 272145384176},
__mask_was_saved = 0, __saved_mask = {__val = {140176008147952, 140737301243152, 1, 140175607493872, 272137013029, 50195, 1024, 8402822336, 140175607493872,
140737301243072, 6292521, 140737301243360, 140175607493872, 81, 6292649, 140737301243440}}}}}
ret = -187112176
inb = {
s = 0xa6f760 "SIP/2.0 180 Ringing\r\nCSeq: 1 INVITE\r\nCall-ID: 7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom: "0987654321"sip:0987654321@A.B.C.D;tag=95ffcd055e0f78f7d5d397020e89288dc 9eabbab\r\nTo: <sip:09"..., len = 700}
__FUNCTION__ = "receive_msg"
#9 0x0000000000607436 in udp_rcv_loop () at udp_server.c:521
len = 700
buf = "SIP/2.0 180 Ringing\r\nCSeq: 1 INVITE\r\nCall-ID: 7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom: "0987654321"sip:0987654321@A.B.C.D;tag=95ffcd055e0f78f7d5d397020e89288dc 9eabbab\r\nTo: <sip:09"...
tmp = 0x3f50636ee0 <Address 0x3f50636ee0 out of bounds>
from = 0x7f7d452fd4b0
fromlen = 16
ri = {src_ip = {af = 2, len = 4, u = {addrl = {546428251, 0}, addr32 = {546428251, 0, 0, 0}, addr16 = {54619, 8337, 0, 0, 0, 0, 0, 0},
addr = "[Õ ", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u = {addrl = {1016190299, 0}, addr32 = {1016190299, 0, 0, 0}, addr16 = {54619, 15505, 0,
0, 0, 0, 0, 0}, addr = "[Õ<", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {
sa_family = 2, sa_data = "\023\304[Õ \000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 546428251},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 546428251, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},
bind_address = 0x7f7d452bf2b0, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#10 0x00000000004a5f0b in main_loop () at main.c:1629
i = 3
pid = 0
si = 0x7f7d452bf2b0
si_desc = "udp receiver child=3 sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260tJ- }\177\000\000\000\000\000\020\004\000\000\000\260tJ-}\177\000\000\060SA\000\ 000\000\000\000\220\351\330\364\001\000\000\000 \347\330\364\377\177\000\000\032dN\000\000\000\000\000h\321)Ez\000\000\000\2 76}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#11 0x00000000004ab8bf in main (argc=7, argv=0x7ffff4d8e998) at main.c:2578
cfg_stream = 0x2019010
c = -1
r = 0
tmp = 0x7ffff4d8ff70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3024771980
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7ffff4d8e86e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : sr-users [mailto:sr-users-bounces@lists.sip-router.org] De la part de Daniel-Constantin Mierla Envoyé : jeudi 10 septembre 2015 12:57 À : Kamailio (SER) - Users Mailing List sr-users@lists.sip-router.org Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
there are newer releases in branch 4.2, was there any reason to stop at 4.2.3?
Back to the backtrace, this one is from shutdown, happening during the cleanup, not showing the reason of the crash at runtime, unless there was a manual shut down triggered at that moment. Do you have another core file?
Cheers, Daniel
On 10/09/15 12:36, Igor Potjevlesch wrote:
Hello,
We got some others crashes even after updated to 4.2.3.
From the last one, I got the following into "bt full":
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
f = 0x7f7d4206d188
hash = 4
#1 0x000000000061a2fe in fm_free (qm=0x7f7d4206d000, p=0x7f7d422ea3a0, file=0x7f7d53408ff0 "dialog: dlg_timer.c", func=0x7f7d53409d00 "destroy_dlg_timer", line=95)
at mem/f_malloc.c:614
f = 0x7f7d422ea368
__FUNCTION__ = "fm_free"
#2 0x00007f7d533e441f in destroy_dlg_timer () at dlg_timer.c:95
__FUNCTION__ = "destroy_dlg_timer"
#3 0x00007f7d53393268 in mod_destroy () at dialog.c:784
No locals.
#4 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d59e9d670
foo = 0x7f7d59e9d440
__FUNCTION__ = "destroy_modules"
#5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#7 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 1107776752
__FUNCTION__ = "handle_sigs"
#8 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4147
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260T\a B}\177\000\000\000\000\000\020\004\000\000\000\260T\aB}\177\000\000\060SA\00 0\000\000\000\000\260:\240\343\001\000\000\000@8\240\343\377\177\000\000\032 dN\000\000\000\000\000h\261\346Yz\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x00000000004ab8bf in main (argc=7, argv=0x7fffe3a03ab8) at main.c:2578
cfg_stream = 0x1a04010
c = -1
r = 0
tmp = 0x7fffe3a04f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3320376299
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fffe3a0398e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] Envoyé : lundi 20 juillet 2015 16:22 À : sr-users@lists.sip-router.org mailto:sr-users@lists.sip-router.org Objet : Multiple crashes of Kamailio 4.2.1
Hello,
3 crashes occurred today (we never seen crashes on this server before) during a period of 20 minutes.
I had a look to the coredumps. Here is a quick overview of the first entries of "bt full":
core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0) at h_table.c:175
core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0) at h_table.c:157
core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0, p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762
core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000, frag=0x7f19864d53e8) at mem/f_malloc.c:245
core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000, frag=0x7f19864f4cc8) at mem/f_malloc.c:208
I don't know how to proceed now. Can you help me to highlight the issue here? Thank you.
Regards,
Igor.
_______________________________________________ SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list sr-users@lists.sip-router.org mailto:sr-users@lists.sip-router.org http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
Hello,
I got two more crashes but they seem to be different.
The logs mentioned:
kernel: kamailio[27096] general protection ip:7f0b3e0bae55 sp:7fffdbce5fd0 error:0 in dialog.so[7f0b3e077000+8f000]
/usr/local/sbin/kamailio[27056]: ALERT: <core> [main.c:784]: handle_sigs(): child process 27096 exited by a signal 11
Both gdb are related to dialog module:
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"
#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0
var = 0x7fb6976154b0
__FUNCTION__ = "destroy_dlg"
#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001
l_dlg = 0x7fb6978e9008
i = 2087
__FUNCTION__ = "destroy_dlg_table"
#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670
foo = 0x7fb6af43d440
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = -1755228944
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\ 227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\00 0\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177 \000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\ 000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] Envoyé : jeudi 10 septembre 2015 16:43 À : miconda@gmail.com; 'Kamailio (SER) - Users Mailing List' sr-users@lists.sip-router.org Objet : RE: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello Daniel,
There is no particular reason. It's just that our others Kamailio are running 4.2.3. So, we try to have the same version on all nodes.
Regarding the backtrace, I'm surprised, because there was no manual shutdown.
Unfortunately, for this occurrence of the crash, it's the only coredump available. There was another crash later this day with two coredumps and the following backtrace:
[ ]
Regards,
Igor.
De : sr-users [mailto:sr-users-bounces@lists.sip-router.org] De la part de Daniel-Constantin Mierla Envoyé : jeudi 10 septembre 2015 12:57 À : Kamailio (SER) - Users Mailing List <sr-users@lists.sip-router.org mailto:sr-users@lists.sip-router.org > Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
there are newer releases in branch 4.2, was there any reason to stop at 4.2.3?
Back to the backtrace, this one is from shutdown, happening during the cleanup, not showing the reason of the crash at runtime, unless there was a manual shut down triggered at that moment. Do you have another core file?
Cheers, Daniel
On 10/09/15 12:36, Igor Potjevlesch wrote:
Hello,
We got some others crashes even after updated to 4.2.3.
From the last one, I got the following into "bt full":
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
f = 0x7f7d4206d188
hash = 4
#1 0x000000000061a2fe in fm_free (qm=0x7f7d4206d000, p=0x7f7d422ea3a0, file=0x7f7d53408ff0 "dialog: dlg_timer.c", func=0x7f7d53409d00 "destroy_dlg_timer", line=95)
at mem/f_malloc.c:614
f = 0x7f7d422ea368
__FUNCTION__ = "fm_free"
#2 0x00007f7d533e441f in destroy_dlg_timer () at dlg_timer.c:95
__FUNCTION__ = "destroy_dlg_timer"
#3 0x00007f7d53393268 in mod_destroy () at dialog.c:784
No locals.
#4 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d59e9d670
foo = 0x7f7d59e9d440
__FUNCTION__ = "destroy_modules"
#5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#7 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 1107776752
__FUNCTION__ = "handle_sigs"
#8 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4147
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260T\a B}\177\000\000\000\000\000\020\004\000\000\000\260T\aB}\177\000\000\060SA\00 0\000\000\000\000\260:\240\343\001\000\000\000@8\240\343\377\177\000\000\032 dN\000\000\000\000\000h\261\346Yz\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x00000000004ab8bf in main (argc=7, argv=0x7fffe3a03ab8) at main.c:2578
cfg_stream = 0x1a04010
c = -1
r = 0
tmp = 0x7fffe3a04f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3320376299
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fffe3a0398e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] Envoyé : lundi 20 juillet 2015 16:22 À : sr-users@lists.sip-router.org mailto:sr-users@lists.sip-router.org Objet : Multiple crashes of Kamailio 4.2.1
Hello,
3 crashes occurred today (we never seen crashes on this server before) during a period of 20 minutes.
I had a look to the coredumps. Here is a quick overview of the first entries of "bt full":
core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0) at h_table.c:175
core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0) at h_table.c:157
core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0, p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762
core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000, frag=0x7f19864d53e8) at mem/f_malloc.c:245
core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000, frag=0x7f19864f4cc8) at mem/f_malloc.c:208
I don't know how to proceed now. Can you help me to highlight the issue here? Thank you.
Regards,
Igor.
_______________________________________________ SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list sr-users@lists.sip-router.org mailto:sr-users@lists.sip-router.org http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
Well, not many relevant changes were done for quite some time to core components, so those should be very stable, but it happened that we discovered bugs even after more than 5 years, because they can be triggered by corner cases that don't show up.
Then, we get always new modules, those take time to get more mature -- the case of issues caught in tsilo or cnxcc modules for example.
The hard part is that people don't use the latest version in same release series, which has only fixes compared with previous version in same series. It doesn't require config or database changes to upgrade.
There were a bunch of issue reported now that summer holidays are over, not first time having more activity in this respect, only 2-3 are still unknown, however, reported from single sources, so not something very common situation, I guess.
Cheers, Daniel
On 11/09/15 16:07, Gary Wallis wrote:
Is Kamailio getting too complicated? Trying to do everything?
Kind regards,
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list sr-users@lists.sip-router.org http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
Hello,
this one is also from the cleanup process at shut down -- if you look at the backtrace, if frames are executing shutdown_children(), cleanup(), destory_modules()... then it is the shutdown. There should be another core that is the reason for the runtime problem -- if there was something wrong with the memory structures at runtime, then shutdown can end in it as well.
Do you have other core files from where to extract the bt full?
Just to rule it out, are you having any private modules or patches?
Cheers, Daniel
On 11/09/15 16:03, Igor Potjevlesch wrote:
Hello,
I got two more crashes but they seem to be different.
The logs mentioned:
kernel: kamailio[27096] general protection ip:7f0b3e0bae55 sp:7fffdbce5fd0 error:0 in dialog.so[7f0b3e077000+8f000]
/usr/local/sbin/kamailio[27056]: ALERT: <core> [main.c:784]: handle_sigs(): child process 27096 exited by a signal 11
Both gdb are related to dialog module:
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0 var = 0x7fb6976154b0 __FUNCTION__ = "destroy_dlg"#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001 l_dlg = 0x7fb6978e9008 i = 2087 __FUNCTION__ = "destroy_dlg_table"#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670 foo = 0x7fb6af43d440 __FUNCTION__ = "destroy_modules"#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0 __FUNCTION__ = "cleanup"#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0 chld_status = 139 memlog = -1755228944 __FUNCTION__ = "handle_sigs"#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8 pid = 4424 si = 0x0 si_desc = "udp receiver child=7sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8 __FUNCTION__ = "main_loop"#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010 c = -1 r = 0 tmp = 0x7fff08879f70 "" tmp_len = 0 port = 0 proto = 32767 options = 0x6fcc00":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1 seed = 2249241156 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xc2 p = 0x7fff08877e7e "" __FUNCTION__ = "main"Regards,
Igor.
*De :*Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] *Envoyé :* jeudi 10 septembre 2015 16:43 *À :* miconda@gmail.com; 'Kamailio (SER) - Users Mailing List' sr-users@lists.sip-router.org *Objet :* RE: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello Daniel,
There is no particular reason. It's just that our others Kamailio are running 4.2.3. So, we try to have the same version on all nodes.
Regarding the backtrace, I'm surprised, because there was no manual shutdown.
Unfortunately, for this occurrence of the crash, it's the only coredump available. There was another crash later this day with two coredumps and the following backtrace:
[…]
Regards,
Igor.
*De :*sr-users [mailto:sr-users-bounces@lists.sip-router.org] *De la part de* Daniel-Constantin Mierla *Envoyé :* jeudi 10 septembre 2015 12:57 *À :* Kamailio (SER) - Users Mailing List <sr-users@lists.sip-router.org mailto:sr-users@lists.sip-router.org> *Objet :* Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
there are newer releases in branch 4.2, was there any reason to stop at 4.2.3?
Back to the backtrace, this one is from shutdown, happening during the cleanup, not showing the reason of the crash at runtime, unless there was a manual shut down triggered at that moment. Do you have another core file?
Cheers, Daniel
On 10/09/15 12:36, Igor Potjevlesch wrote:
Hello, We got some others crashes even after updated to 4.2.3. From the last one, I got the following into "bt full": Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'. Program terminated with signal 11, Segmentation fault. #0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245 245 if (*f) (*f)->prv_free = &(frag->u.nxt_free); Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64 (gdb) bt full #0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245 f = 0x7f7d4206d188 hash = 4 #1 0x000000000061a2fe in fm_free (qm=0x7f7d4206d000, p=0x7f7d422ea3a0, file=0x7f7d53408ff0 "dialog: dlg_timer.c", func=0x7f7d53409d00 "destroy_dlg_timer", line=95) at mem/f_malloc.c:614 f = 0x7f7d422ea368 __FUNCTION__ = "fm_free" #2 0x00007f7d533e441f in destroy_dlg_timer () at dlg_timer.c:95 __FUNCTION__ = "destroy_dlg_timer" #3 0x00007f7d53393268 in mod_destroy () at dialog.c:784 No locals. #4 0x0000000000590d79 in destroy_modules () at sr_module.c:811 t = 0x7f7d59e9d670 foo = 0x7f7d59e9d440 __FUNCTION__ = "destroy_modules" #5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569 memlog = 0 __FUNCTION__ = "cleanup" #6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711 __FUNCTION__ = "shutdown_children" #7 0x000000000049f6e1 in handle_sigs () at main.c:802 chld = 0 chld_status = 139 memlog = 1107776752 __FUNCTION__ = "handle_sigs" #8 0x00000000004a6fbf in main_loop () at main.c:1757 i = 8 pid = 4147 si = 0x0 si_desc = "udp receiver child=7 sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260T\aB}\177\000\000\000\000\000\020\004\000\000\000\260T\aB}\177\000\000\060SA\000\000\000\000\000\260:\240\343\001\000\000\000@8\240\343\377\177\000\000\032dN\000\000\000\000\000h\261\346Yz\000\000\000\276}p\000\000\000\000" nrprocs = 8 __FUNCTION__ = "main_loop" #9 0x00000000004ab8bf in main (argc=7, argv=0x7fffe3a03ab8) at main.c:2578 cfg_stream = 0x1a04010 c = -1 r = 0 tmp = 0x7fffe3a04f70 "" tmp_len = 0 port = 0 proto = 32767 options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:" ret = -1 seed = 3320376299 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xc2 p = 0x7fffe3a0398e "" __FUNCTION__ = "main" Regards, Igor. *De :*Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] *Envoyé :* lundi 20 juillet 2015 16:22 *À :* sr-users@lists.sip-router.org <mailto:sr-users@lists.sip-router.org> *Objet :* Multiple crashes of Kamailio 4.2.1 Hello, 3 crashes occurred today (we never seen crashes on this server before) during a period of 20 minutes. I had a look to the coredumps. Here is a quick overview of the first entries of "bt full": core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286 core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0) at h_table.c:175 core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286 core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286 core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286 core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0) at h_table.c:157 core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0, p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762 core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000, frag=0x7f19864d53e8) at mem/f_malloc.c:245 core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000, frag=0x7f19864f4cc8) at mem/f_malloc.c:208 I don't know how to proceed now. Can you help me to highlight the issue here? Thank you. Regards, Igor. _______________________________________________ SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list sr-users@lists.sip-router.org <mailto:sr-users@lists.sip-router.org> http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users-- Daniel-Constantin Mierla http://twitter.com/#!/miconda http://twitter.com/#%21/miconda - http://www.linkedin.com/in/miconda Book: SIP Routing With Kamailio - http://www.asipto.com Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
Hello Daniel,
From the two crashes occurred today, I got 2 coredump. So I copy/past the
result from these 4 backtraces:
No privates modules or patches. It's a regular 4.2.3.
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"
#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0
var = 0x7fb6976154b0
__FUNCTION__ = "destroy_dlg"
#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001
l_dlg = 0x7fb6978e9008
i = 2087
__FUNCTION__ = "destroy_dlg_table"
#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670
foo = 0x7fb6af43d440
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = -1755228944
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
(gdb) bt full
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
#1 0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0) at dialog.c:1260
No locals.
#2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1 "Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>, param=0x0,
interval=90000) at timer_proc.c:235
pid = 0
ts1 = 373077626
ts2 = 90000
#3 0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740
__FUNCTION__ = "child_init"
#4 0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at sr_module.c:921
__FUNCTION__ = "init_mod_child"
#5 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#6 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#7 0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#8 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#9 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947
No locals.
#15 0x00000000004a64c4 in main_loop () at main.c:1706
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
(gdb) bt full
#0 0x00007f0b3e0945d3 in destroy_dlg_callbacks_list (cb=0x43203532313a7061) at dlg_cb.c:75
cb_t = 0x43203532313a7061
__FUNCTION__ = "destroy_dlg_callbacks_list"
#1 0x00007f0b3e0bd67c in destroy_dlg (dlg=0x7f0b2d04c118) at dlg_hash.c:382
ret = 0
var = 0x7f0b2cd6b4b0
__FUNCTION__ = "destroy_dlg"
#2 0x00007f0b3e0bdb35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0x30a012010010a0d
l_dlg = 0x7f0b2d04c118
i = 1751
__FUNCTION__ = "destroy_dlg_table"
#3 0x00007f0b3e089263 in mod_destroy () at dialog.c:783
No locals.
#4 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f0b44b93670
foo = 0x7f0b44b93440
__FUNCTION__ = "destroy_modules"
#5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#7 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 752268528
__FUNCTION__ = "handle_sigs"
#8 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 27087
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260\264\326, \v\177\000\000\000\000\000\020\004\000\000\000\260\264\326,\v\177\000\000\06 0SA\000\000\000\000\000\260i\316\333\001\000\000\000@g\316\333\377\177\000\0 00\032dN\000\000\000\000\000h\021\266Dz\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x00000000004ab8bf in main (argc=7, argv=0x7fffdbce69b8) at main.c:2578
cfg_stream = 0x1f28010
c = -1
r = 0
tmp = 0x7fffdbce6f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1996811280
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fffdbce688e ""
__FUNCTION__ = "main"
(gdb) bt full
#0 0x00007f0b3e0bae55 in dlg_clean_run (ti=105036513) at dlg_hash.c:244
i = 1751
tm = 1441979491
dlg = 0x30a012010010a0d
tdlg = 0x30a012010010a0d
__FUNCTION__ = "dlg_clean_run"
#1 0x00007f0b3e08edd6 in dlg_clean_timer_exec (ticks=105036513, param=0x0) at dialog.c:1260
No locals.
#2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7f0b3e0ed0f1 "Dialog Clean Timer", make_sock=1, f=0x7f0b3e08edbd <dlg_clean_timer_exec>, param=0x0,
interval=90000) at timer_proc.c:235
pid = 0
ts1 = 1680584210
ts2 = 90000
#3 0x00007f0b3e088b50 in child_init (rank=0) at dialog.c:740
__FUNCTION__ = "child_init"
#4 0x0000000000591129 in init_mod_child (m=0x7f0b44b93670, rank=0) at sr_module.c:921
__FUNCTION__ = "init_mod_child"
#5 0x0000000000590e64 in init_mod_child (m=0x7f0b44b941b0, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#6 0x0000000000590e64 in init_mod_child (m=0x7f0b44b94728, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#7 0x0000000000590e64 in init_mod_child (m=0x7f0b44b94b90, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#8 0x0000000000590e64 in init_mod_child (m=0x7f0b44b95108, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#9 0x0000000000590e64 in init_mod_child (m=0x7f0b44b95418, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#10 0x0000000000590e64 in init_mod_child (m=0x7f0b44b95808, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#11 0x0000000000590e64 in init_mod_child (m=0x7f0b44b95b18, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#12 0x0000000000590e64 in init_mod_child (m=0x7f0b44b96090, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#13 0x0000000000590e64 in init_mod_child (m=0x7f0b44b963d8, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947
No locals.
#15 0x00000000004a64c4 in main_loop () at main.c:1706
i = 8
pid = 27087
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260\264\326, \v\177\000\000\000\000\000\020\004\000\000\000\260\264\326,\v\177\000\000\06 0SA\000\000\000\000\000\260i\316\333\001\000\000\000@g\316\333\377\177\000\0 00\032dN\000\000\000\000\000h\021\266Dz\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#16 0x00000000004ab8bf in main (argc=7, argv=0x7fffdbce69b8) at main.c:2578
cfg_stream = 0x1f28010
c = -1
r = 0
tmp = 0x7fffdbce6f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1996811280
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fffdbce688e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda@gmail.com] Envoyé : vendredi 11 septembre 2015 16:11 À : Igor Potjevlesch igor.potjevlesch@gmail.com; 'Kamailio (SER) - Users Mailing List' sr-users@lists.sip-router.org Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
this one is also from the cleanup process at shut down -- if you look at the backtrace, if frames are executing shutdown_children(), cleanup(), destory_modules()... then it is the shutdown. There should be another core that is the reason for the runtime problem -- if there was something wrong with the memory structures at runtime, then shutdown can end in it as well.
Do you have other core files from where to extract the bt full?
Just to rule it out, are you having any private modules or patches?
Cheers, Daniel
On 11/09/15 16:03, Igor Potjevlesch wrote:
Hello,
I got two more crashes but they seem to be different.
The logs mentioned:
kernel: kamailio[27096] general protection ip:7f0b3e0bae55 sp:7fffdbce5fd0 error:0 in dialog.so[7f0b3e077000+8f000]
/usr/local/sbin/kamailio[27056]: ALERT: <core> [main.c:784]: handle_sigs(): child process 27096 exited by a signal 11
Both gdb are related to dialog module:
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"
#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0
var = 0x7fb6976154b0
__FUNCTION__ = "destroy_dlg"
#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001
l_dlg = 0x7fb6978e9008
i = 2087
__FUNCTION__ = "destroy_dlg_table"
#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670
foo = 0x7fb6af43d440
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = -1755228944
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [ mailto:igor.potjevlesch@gmail.com mailto:igor.potjevlesch@gmail.com] Envoyé : jeudi 10 septembre 2015 16:43 À : mailto:miconda@gmail.com miconda@gmail.com; 'Kamailio (SER) - Users Mailing List' mailto:sr-users@lists.sip-router.org sr-users@lists.sip-router.org Objet : RE: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello Daniel,
There is no particular reason. It's just that our others Kamailio are running 4.2.3. So, we try to have the same version on all nodes.
Regarding the backtrace, I'm surprised, because there was no manual shutdown.
Unfortunately, for this occurrence of the crash, it's the only coredump available. There was another crash later this day with two coredumps and the following backtrace:
[ ]
Regards,
Igor.
De : sr-users [ mailto:sr-users-bounces@lists.sip-router.org mailto:sr-users-bounces@lists.sip-router.org] De la part de Daniel-Constantin Mierla Envoyé : jeudi 10 septembre 2015 12:57 À : Kamailio (SER) - Users Mailing List < mailto:sr-users@lists.sip-router.org sr-users@lists.sip-router.org> Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
there are newer releases in branch 4.2, was there any reason to stop at 4.2.3?
Back to the backtrace, this one is from shutdown, happening during the cleanup, not showing the reason of the crash at runtime, unless there was a manual shut down triggered at that moment. Do you have another core file?
Cheers, Daniel
On 10/09/15 12:36, Igor Potjevlesch wrote:
Hello,
We got some others crashes even after updated to 4.2.3.
From the last one, I got the following into "bt full":
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000, frag=0x7f7d422ea368) at mem/f_malloc.c:245
f = 0x7f7d4206d188
hash = 4
#1 0x000000000061a2fe in fm_free (qm=0x7f7d4206d000, p=0x7f7d422ea3a0, file=0x7f7d53408ff0 "dialog: dlg_timer.c", func=0x7f7d53409d00 "destroy_dlg_timer", line=95)
at mem/f_malloc.c:614
f = 0x7f7d422ea368
__FUNCTION__ = "fm_free"
#2 0x00007f7d533e441f in destroy_dlg_timer () at dlg_timer.c:95
__FUNCTION__ = "destroy_dlg_timer"
#3 0x00007f7d53393268 in mod_destroy () at dialog.c:784
No locals.
#4 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d59e9d670
foo = 0x7f7d59e9d440
__FUNCTION__ = "destroy_modules"
#5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#7 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 1107776752
__FUNCTION__ = "handle_sigs"
#8 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4147
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260T\aB}\177 \000\000\000\000\000\020\004\000\000\000\260T\aB}\177\000\000\060SA\000\000\ 000\000\000\260:\240\343\001\000\000\000@8\240\343\377\177\000\000\032dN\000 \000\000\000\000h\261\346Yz\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x00000000004ab8bf in main (argc=7, argv=0x7fffe3a03ab8) at main.c:2578
cfg_stream = 0x1a04010
c = -1
r = 0
tmp = 0x7fffe3a04f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3320376299
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fffe3a0398e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [ mailto:igor.potjevlesch@gmail.com mailto:igor.potjevlesch@gmail.com] Envoyé : lundi 20 juillet 2015 16:22 À : mailto:sr-users@lists.sip-router.org sr-users@lists.sip-router.org Objet : Multiple crashes of Kamailio 4.2.1
Hello,
3 crashes occurred today (we never seen crashes on this server before) during a period of 20 minutes.
I had a look to the coredumps. Here is a quick overview of the first entries of "bt full":
core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0) at h_table.c:175
core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal (cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0) at t_hooks.c:286
core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0) at h_table.c:157
core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0, p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762
core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000, frag=0x7f19864d53e8) at mem/f_malloc.c:245
core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000, frag=0x7f19864f4cc8) at mem/f_malloc.c:208
I don't know how to proceed now. Can you help me to highlight the issue here? Thank you.
Regards,
Igor.
_______________________________________________ SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list mailto:sr-users@lists.sip-router.org sr-users@lists.sip-router.org http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
Hello,
from the second trace, can you get output for:
frame 0 list info locals p *dlg
Cheers, Daniel
On 11/09/15 18:23, Igor Potjevlesch wrote:
Hello Daniel,
From the two crashes occurred today, I got 2 coredump. So I copy/past the result from these 4 backtraces:
No privates modules or patches. It's a regular 4.2.3.
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0 var = 0x7fb6976154b0 __FUNCTION__ = "destroy_dlg"#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001 l_dlg = 0x7fb6978e9008 i = 2087 __FUNCTION__ = "destroy_dlg_table"#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670 foo = 0x7fb6af43d440 __FUNCTION__ = "destroy_modules"#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0 __FUNCTION__ = "cleanup"#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0 chld_status = 139 memlog = -1755228944 __FUNCTION__ = "handle_sigs"#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8 pid = 4424 si = 0x0 si_desc = "udp receiver child=7sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8 __FUNCTION__ = "main_loop"#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010 c = -1 r = 0 tmp = 0x7fff08879f70 "" tmp_len = 0 port = 0 proto = 32767 options = 0x6fcc00":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1 seed = 2249241156 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xc2 p = 0x7fff08877e7e "" __FUNCTION__ = "main"(gdb) bt full
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
i = 2087 tm = 1441978496 dlg = 0xb02030a01201001 tdlg = 0xb02030a01201001 __FUNCTION__ = "dlg_clean_run"#1 0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0) at dialog.c:1260
No locals.
#2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1 "Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>, param=0x0,
interval=90000) at timer_proc.c:235 pid = 0 ts1 = 373077626 ts2 = 90000#3 0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740
__FUNCTION__ = "child_init"#4 0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at sr_module.c:921
__FUNCTION__ = "init_mod_child"#5 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#6 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#7 0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#8 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#9 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947
No locals.
#15 0x00000000004a64c4 in main_loop () at main.c:1706
i = 8 pid = 4424 si = 0x0 si_desc = "udp receiver child=7sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8 __FUNCTION__ = "main_loop"#16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010 c = -1 r = 0 tmp = 0x7fff08879f70 "" tmp_len = 0 port = 0 proto = 32767 options = 0x6fcc00":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1 seed = 2249241156 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xc2 p = 0x7fff08877e7e "" __FUNCTION__ = "main"
Hello Daniel,
Here is the output:
(gdb) frame 0
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
244 dlg = dlg->next;
(gdb) list
239 {
240 lock_set_get(d_table->locks, d_table->entries[i].lock_idx);
241 dlg = d_table->entries[i].first;
242 while (dlg) {
243 tdlg = dlg;
244 dlg = dlg->next;
245 if(tdlg->state==DLG_STATE_UNCONFIRMED && tdlg->init_ts<tm-300) {
246 /* dialog in early state older than 5min */
247 LM_NOTICE("dialog in early state is too old (%p ref %d)\n",
248 tdlg, tdlg->ref);
(gdb) info locals
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
(gdb) p *dlg
Cannot access memory at address 0xb02030a01201001
(gdb)
I hope this will help.
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda@gmail.com] Envoyé : jeudi 17 septembre 2015 11:40 À : Igor Potjevlesch igor.potjevlesch@gmail.com; 'Kamailio (SER) - Users Mailing List' sr-users@lists.sip-router.org Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
from the second trace, can you get output for:
frame 0 list info locals p *dlg
Cheers, Daniel
On 11/09/15 18:23, Igor Potjevlesch wrote:
Hello Daniel,
From the two crashes occurred today, I got 2 coredump. So I copy/past the
result from these 4 backtraces:
No privates modules or patches. It's a regular 4.2.3.
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"
#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0
var = 0x7fb6976154b0
__FUNCTION__ = "destroy_dlg"
#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001
l_dlg = 0x7fb6978e9008
i = 2087
__FUNCTION__ = "destroy_dlg_table"
#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670
foo = 0x7fb6af43d440
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = -1755228944
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
(gdb) bt full
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
#1 0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0) at dialog.c:1260
No locals.
#2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1 "Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>, param=0x0,
interval=90000) at timer_proc.c:235
pid = 0
ts1 = 373077626
ts2 = 90000
#3 0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740
__FUNCTION__ = "child_init"
#4 0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at sr_module.c:921
__FUNCTION__ = "init_mod_child"
#5 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#6 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#7 0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#8 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#9 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947
No locals.
#15 0x00000000004a64c4 in main_loop () at main.c:1706
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
Hello,
can you test with latest version branch 4.2? I backported several patches related to dialog module, among them some related to a race for deleted dialogs detected as spiral, which may be the reason for this crash.
Cheers, Daniel
On 17/09/15 12:25, Igor Potjevlesch wrote:
Hello Daniel,
Here is the output:
*(gdb) frame 0*
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
244 dlg = dlg->next;
*(gdb) list*
239 {
240 lock_set_get(d_table->locks, d_table->entries[i].lock_idx);
241 dlg = d_table->entries[i].first;
242 while (dlg) {
243 tdlg = dlg;
244 dlg = dlg->next;
245 if(tdlg->state==DLG_STATE_UNCONFIRMED && tdlg->init_ts<tm-300) {
246 /* dialog in early state older than 5min */
247 LM_NOTICE("dialog in early state is too old (%p ref %d)\n",
248 tdlg, tdlg->ref);
*(gdb) info locals*
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
*(gdb) p *dlg*
Cannot access memory at address 0xb02030a01201001
(gdb)
I hope this will help.
Regards,
Igor.
*De :*Daniel-Constantin Mierla [mailto:miconda@gmail.com] *Envoyé :* jeudi 17 septembre 2015 11:40 *À :* Igor Potjevlesch igor.potjevlesch@gmail.com; 'Kamailio (SER) - Users Mailing List' sr-users@lists.sip-router.org *Objet :* Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
from the second trace, can you get output for:
frame 0 list info locals p *dlg
Cheers, Daniel
On 11/09/15 18:23, Igor Potjevlesch wrote:
Hello Daniel, From the two crashes occurred today, I got 2 coredump. So I copy/past the result from these 4 backtraces: No privates modules or patches. It's a regular 4.2.3. (gdb) bt full #0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156 No locals. #1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182 __FUNCTION__ = "remove_dialog_timer" #2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357 ret = 0 var = 0x7fb6976154b0 __FUNCTION__ = "destroy_dlg" #3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438 dlg = 0xb02030a01201001 l_dlg = 0x7fb6978e9008 i = 2087 __FUNCTION__ = "destroy_dlg_table" #4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783 No locals. #5 0x0000000000590d79 in destroy_modules () at sr_module.c:811 t = 0x7fb6af43d670 foo = 0x7fb6af43d440 __FUNCTION__ = "destroy_modules" #6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569 memlog = 0 __FUNCTION__ = "cleanup" #7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711 __FUNCTION__ = "shutdown_children" #8 0x000000000049f6e1 in handle_sigs () at main.c:802 chld = 0 chld_status = 139 memlog = -1755228944 __FUNCTION__ = "handle_sigs" #9 0x00000000004a6fbf in main_loop () at main.c:1757 i = 8 pid = 4424 si = 0x0 si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000" nrprocs = 8 __FUNCTION__ = "main_loop" #10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578 cfg_stream = 0x18b4010 c = -1 r = 0 tmp = 0x7fff08879f70 "" tmp_len = 0 port = 0 proto = 32767 options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:" ret = -1 seed = 2249241156 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xc2 p = 0x7fff08877e7e "" __FUNCTION__ = "main" (gdb) bt full #0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244 i = 2087 tm = 1441978496 dlg = 0xb02030a01201001 tdlg = 0xb02030a01201001 __FUNCTION__ = "dlg_clean_run" #1 0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0) at dialog.c:1260 No locals. #2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1 "Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>, param=0x0, interval=90000) at timer_proc.c:235 pid = 0 ts1 = 373077626 ts2 = 90000 #3 0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740 __FUNCTION__ = "child_init" #4 0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at sr_module.c:921 __FUNCTION__ = "init_mod_child" #5 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #6 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #7 0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #8 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #9 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at sr_module.c:918 __FUNCTION__ = "init_mod_child" #14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947 No locals. #15 0x00000000004a64c4 in main_loop () at main.c:1706 i = 8 pid = 4424 si = 0x0 si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000" nrprocs = 8 __FUNCTION__ = "main_loop" #16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578 cfg_stream = 0x18b4010 c = -1 r = 0 tmp = 0x7fff08879f70 "" tmp_len = 0 port = 0 proto = 32767 options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:" ret = -1 seed = 2249241156 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xc2 p = 0x7fff08877e7e "" __FUNCTION__ = "main"-- Daniel-Constantin Mierla http://twitter.com/#!/miconda http://twitter.com/#%21/miconda - http://www.linkedin.com/in/miconda Book: SIP Routing With Kamailio - http://www.asipto.com Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
Hello,
Yes ok. I will schedule this update and I will let you know.
Thank you.
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda@gmail.com] Envoyé : jeudi 17 septembre 2015 17:43 À : Igor Potjevlesch igor.potjevlesch@gmail.com; 'Kamailio (SER) - Users Mailing List' sr-users@lists.sip-router.org Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
can you test with latest version branch 4.2? I backported several patches related to dialog module, among them some related to a race for deleted dialogs detected as spiral, which may be the reason for this crash.
Cheers, Daniel
On 17/09/15 12:25, Igor Potjevlesch wrote:
Hello Daniel,
Here is the output:
(gdb) frame 0
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
244 dlg = dlg->next;
(gdb) list
239 {
240 lock_set_get(d_table->locks, d_table->entries[i].lock_idx);
241 dlg = d_table->entries[i].first;
242 while (dlg) {
243 tdlg = dlg;
244 dlg = dlg->next;
245 if(tdlg->state==DLG_STATE_UNCONFIRMED && tdlg->init_ts<tm-300) {
246 /* dialog in early state older than 5min */
247 LM_NOTICE("dialog in early state is too old (%p ref %d)\n",
248 tdlg, tdlg->ref);
(gdb) info locals
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
(gdb) p *dlg
Cannot access memory at address 0xb02030a01201001
(gdb)
I hope this will help.
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda@gmail.com] Envoyé : jeudi 17 septembre 2015 11:40 À : Igor Potjevlesch mailto:igor.potjevlesch@gmail.com igor.potjevlesch@gmail.com; 'Kamailio (SER) - Users Mailing List' mailto:sr-users@lists.sip-router.org sr-users@lists.sip-router.org Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
from the second trace, can you get output for:
frame 0 list info locals p *dlg
Cheers, Daniel
On 11/09/15 18:23, Igor Potjevlesch wrote:
Hello Daniel,
From the two crashes occurred today, I got 2 coredump. So I copy/past the
result from these 4 backtraces:
No privates modules or patches. It's a regular 4.2.3.
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"
#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0
var = 0x7fb6976154b0
__FUNCTION__ = "destroy_dlg"
#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001
l_dlg = 0x7fb6978e9008
i = 2087
__FUNCTION__ = "destroy_dlg_table"
#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670
foo = 0x7fb6af43d440
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = -1755228944
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
(gdb) bt full
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
#1 0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0) at dialog.c:1260
No locals.
#2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1 "Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>, param=0x0,
interval=90000) at timer_proc.c:235
pid = 0
ts1 = 373077626
ts2 = 90000
#3 0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740
__FUNCTION__ = "child_init"
#4 0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at sr_module.c:921
__FUNCTION__ = "init_mod_child"
#5 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#6 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#7 0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#8 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#9 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947
No locals.
#15 0x00000000004a64c4 in main_loop () at main.c:1706
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
Hello,
The upgrade to 4.2.6 has been done 2 weeks ago.
We got a new crash today but I'm not sure that it's the same issue:
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000619694 in fm_extract_free (qm=0x7f6c97620000, frag=0x7f6c97904468) at mem/f_malloc.c:206
206 *pf=frag->u.nxt_free;
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000619694 in fm_extract_free (qm=0x7f6c97620000, frag=0x7f6c97904468) at mem/f_malloc.c:206
pf = 0x69442d746e65746e
hash = 2097
#1 0x000000000061ad68 in fm_malloc (qm=0x7f6c97620000, size=1216, file=0x7577a0 "<core>: mem/shm_mem.c", func=0x75855c "sh_realloc", line=89) at mem/f_malloc.c:490
f = 0x7f6c97620b48
frag = 0x7f6c97904468
hash = 160
__FUNCTION__ = "fm_malloc"
#2 0x0000000000620b53 in sh_realloc (p=0x7f6c978e8a48, size=1213) at mem/shm_mem.c:89
r = 0x1ac47418bf0
__FUNCTION__ = "sh_realloc"
#3 0x0000000000620e0b in _shm_resize (p=0x7f6c978e8a48, s=1213, file=0x7f6cadf27673 "tm: t_reply.c", func=0x7f6cadf2c391 "relay_reply", line=1961) at mem/shm_mem.c:114
__FUNCTION__ = "_shm_resize"
#4 0x00007f6caded8fdb in relay_reply (t=0x7f6c9792d4a0, p_msg=0x7f6caf5a2358, branch=0, msg_status=183, cancel_data=0x7fff474183a0, do_put_on_wait=1) at t_reply.c:1960
relay = 0
save_clone = 0
buf = 0x7f6caf483790 "SIP/2.0 183 Session Progress\r\nf: sip:++33123456789@A.B.C.D:5060;tag=gK0823f4a1\r\nt: sip:+33987654321@D.C.B.A;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"...
res_len = 1053
relayed_code = 183
relayed_msg = 0x7f6caf5a2358
reply_bak = 0x4000000
bm = {to_tag_val = {s = 0x7fff47418180 "`Õ\227l\177", len = -1377010389}}
totag_retr = 0
reply_status = RPS_PROVISIONAL
uas_rb = 0x7f6c9792d560
to_tag = 0x7f6cadec8c8f
reason = {s = 0x474183c8 <Address 0x474183c8 out of bounds>, len = 1024}
onsend_params = {req = 0x7fff474181a0, rpl = 0x7f6cade93bec, param = 0x415440, code = 1195478000, flags = 3, branch = 0, t_rbuf = 0x0, dst = 0x1, send_buf = {
s = 0x7f6c9792ea38 "\001", len = 6781848}}
__FUNCTION__ = "relay_reply"
#5 0x00007f6cadedc899 in reply_received (p_msg=0x7f6caf5a2358) at t_reply.c:2511
msg_status = 183
last_uac_status = 100
ack = 0x7f6caf428010 "\001"
ack_len = 0
branch = 0
reply_status = -1354595880
onreply_route = 1
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 10955836}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 10955836}}}}
uac = 0x7f6c9792d608
t = 0x7f6c9792d4a0
lack_dst = {send_sock = 0x4000000, to = {s = {sa_family = 10604, sa_data = "\247\000\000\000\000\000\r)\247\000\000\000\000"}, sin = {sin_family = 10604,
sin_port = 167, sin_addr = {s_addr = 0}, sin_zero = "\r)\247\000\000\000\000"}, sin6 = {sin6_family = 10604, sin6_port = 167, sin6_flowinfo = 0, sin6_addr = {
__in6_u = {__u6_addr8 = "\r)\247\000\000\000\000\000p\225Z\257l\177\000", __u6_addr16 = {10509, 167, 0, 0, 38256, 44890, 32620, 0}, __u6_addr32 = {10955021,
0, 2941949296, 32620}}}, sin6_scope_id = 2940756480}}, id = 32620, proto = 40 '(', send_flags = {f = 122 'z', blst_imask = 72 'H'}}
backup_user_from = 0xa827f0
backup_user_to = 0xa827f8
backup_domain_from = 0xa82800
backup_domain_to = 0xa82808
backup_uri_from = 0xa827e0
backup_uri_to = 0xa827e8
backup_xavps = 0xa82920
replies_locked = 1
branch_ret = 0
prev_branch = -1353047176
blst_503_timeout = 32620
hf = 0x47c47418470
onsend_params = {req = 0x7fff47418360, rpl = 0x47deb8, param = 0x0, code = -1354201032, flags = 32620, branch = 0, t_rbuf = 0xa72c3c, dst = 0xa7290d, send_buf = {
s = 0x7fff47418420 "\350'\250", len = 6402299}}
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {140104775116112, -3429479277312539647, 4281408, 140734388866032, 0, 0,
-3429479272707193855, 3429074136233477121}, __mask_was_saved = 0, __saved_mask = {__val = {0, 140734388864032, 6439748, 140734388863792, 140104760342234,
140734388864064, 0, 67108864, 65539104, 1286592, 1569760, 1576600, 8, 94, 140104760342234, 1474369258384}}}}}
__FUNCTION__ = "reply_received"
#6 0x000000000048cc3a in do_forward_reply (msg=0x7f6caf5a2358, mode=0) at forward.c:783
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000',
send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
new_len = 1
r = 0
ip = {af = 1195476416, len = 32767, u = {addrl = {6466393, 280}, addr32 = {6466393, 0, 280, 0}, addr16 = {43865, 98, 0, 0, 280, 0, 0, 0},
addr = "Y\253b\000\000\000\000\000\030\001\000\000\000\000\000"}}
s = 0x4 <Address 0x4 out of bounds>
len = 0
__FUNCTION__ = "do_forward_reply"
#7 0x000000000048e27d in forward_reply (msg=0x7f6caf5a2358) at forward.c:885
No locals.
#8 0x0000000000509c9c in receive_msg (
buf=0xa727c0 "SIP/2.0 183 Session Progress\r\nf: sip:++33123456789@A.B.C.D:5060;tag=gK0823f4a1\r\nt: sip:+33987654321@D.C.B.A;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"..., len=1148, rcv_info=0x7fff474187c0) at receive.c:275
---Type <return> to continue, or q <return> to quit---
msg = 0x7f6caf5a2358
ctx = {rec_lev = 10237056, run_flags = 0, last_retcode = 0, jmp_env = {{__jmpbuf = {0, 0, 0, 272136986608, 1812476198913, 0, 272145363728, 272145384176},
__mask_was_saved = 0, __saved_mask = {__val = {140104773706736, 140734388864864, 1, 140104373011696, 272137013029, 50195, 1024, 5490444048, 140104373011696,
140734388864784, 6299381, 140734388865072, 140104373011696, 81, 6299509, 140734388865152}}}}}
ret = 1195476832
inb = {
s = 0xa727c0 "SIP/2.0 183 Session Progress\r\nf: sip:++33123456789@A.B.C.D:5060;tag=gK0823f4a1\r\nt: sip:+33987654321@D.C.B.A;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"..., len = 1148}
__FUNCTION__ = "receive_msg"
#9 0x0000000000608f02 in udp_rcv_loop () at udp_server.c:521
len = 1148
buf = "SIP/2.0 183 Session Progress\r\nf: sip:++33123456789@A.B.C.D:5060;tag=gK0823f4a1\r\nt: sip:+33987654321@D.C.B.A;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"...
tmp = 0x3f30d2b2f2 <Address 0x3f30d2b2f2 out of bounds>
from = 0x7f6caf488590
fromlen = 16
ri = {src_ip = {af = 2, len = 4, u = {addrl = {151524537, 0}, addr32 = {151524537, 0, 0, 0}, addr16 = {5305, 2312, 0, 0, 0, 0, 0, 0},
addr = "\271\024\b\t", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u = {addrl = {1016190299, 0}, addr32 = {1016190299, 0, 0, 0}, addr16 = {54619,
15505, 0, 0, 0, 0, 0, 0}, addr = "[Õ<", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {
s = {sa_family = 2, sa_data = "\023Ĺ\024\b\t\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 151524537},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 151524537, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},
bind_address = 0x7f6caf44a2b0, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#10 0x00000000004a6d9b in main_loop () at main.c:1629
i = 4
pid = 0
si = 0x7f6caf44a2b0
si_desc = "udp receiver child=4 sock=D.C.B.A:5060\000\177\000\000\060\211AG\377\177\000\000\003zN\000\000\00 0\000\000\016\b\000\000\377\177\000\000\260\204b\227l\177\000\000\000\000\00 0\020\004\000\000\000\260\204b\227l\177\000\000@TA\000\000\000\000\000\360\2 13AG\001\000\000\000\200\211AG\377\177\000\000\246zN\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#11 0x00000000004acedf in main (argc=7, argv=0x7fff47418bf8) at main.c:2581
cfg_stream = 0x21fb010
c = -1
r = 0
tmp = 0x7fff47419f70 ""
tmp_len = 32767
port = 1195477710
proto = 0
options = 0x6ff8f8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2329478669
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x40d11c
p = 0xc2 <Address 0xc2 out of bounds>
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com] Envoyé : vendredi 18 septembre 2015 15:37 À : miconda@gmail.com; 'Kamailio (SER) - Users Mailing List' sr-users@lists.sip-router.org Objet : RE: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
Yes ok. I will schedule this update and I will let you know.
Thank you.
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda@gmail.com] Envoyé : jeudi 17 septembre 2015 17:43 À : Igor Potjevlesch <igor.potjevlesch@gmail.com mailto:igor.potjevlesch@gmail.com >; 'Kamailio (SER) - Users Mailing List' <sr-users@lists.sip-router.org mailto:sr-users@lists.sip-router.org > Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
can you test with latest version branch 4.2? I backported several patches related to dialog module, among them some related to a race for deleted dialogs detected as spiral, which may be the reason for this crash.
Cheers, Daniel
On 17/09/15 12:25, Igor Potjevlesch wrote:
Hello Daniel,
Here is the output:
(gdb) frame 0
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
244 dlg = dlg->next;
(gdb) list
239 {
240 lock_set_get(d_table->locks, d_table->entries[i].lock_idx);
241 dlg = d_table->entries[i].first;
242 while (dlg) {
243 tdlg = dlg;
244 dlg = dlg->next;
245 if(tdlg->state==DLG_STATE_UNCONFIRMED && tdlg->init_ts<tm-300) {
246 /* dialog in early state older than 5min */
247 LM_NOTICE("dialog in early state is too old (%p ref %d)\n",
248 tdlg, tdlg->ref);
(gdb) info locals
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
(gdb) p *dlg
Cannot access memory at address 0xb02030a01201001
(gdb)
I hope this will help.
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda@gmail.com] Envoyé : jeudi 17 septembre 2015 11:40 À : Igor Potjevlesch mailto:igor.potjevlesch@gmail.com igor.potjevlesch@gmail.com; 'Kamailio (SER) - Users Mailing List' mailto:sr-users@lists.sip-router.org sr-users@lists.sip-router.org Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
from the second trace, can you get output for:
frame 0 list info locals p *dlg
Cheers, Daniel
On 11/09/15 18:23, Igor Potjevlesch wrote:
Hello Daniel,
From the two crashes occurred today, I got 2 coredump. So I copy/past the
result from these 4 backtraces:
No privates modules or patches. It's a regular 4.2.3.
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"
#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0
var = 0x7fb6976154b0
__FUNCTION__ = "destroy_dlg"
#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001
l_dlg = 0x7fb6978e9008
i = 2087
__FUNCTION__ = "destroy_dlg_table"
#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670
foo = 0x7fb6af43d440
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = -1755228944
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
(gdb) bt full
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
#1 0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0) at dialog.c:1260
No locals.
#2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1 "Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>, param=0x0,
interval=90000) at timer_proc.c:235
pid = 0
ts1 = 373077626
ts2 = 90000
#3 0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740
__FUNCTION__ = "child_init"
#4 0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at sr_module.c:921
__FUNCTION__ = "init_mod_child"
#5 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#6 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#7 0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#8 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#9 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at sr_module.c:918
__FUNCTION__ = "init_mod_child"
#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947
No locals.
#15 0x00000000004a64c4 in main_loop () at main.c:1706
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26 6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0 00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
-
Daniel-Constantin Mierla -
Gary Wallis -
Igor Potjevlesch