[SR-Users] Multiple crashes of Kamailio 4.2.1
Igor Potjevlesch
igor.potjevlesch at gmail.com
Fri Oct 16 18:43:58 CEST 2015
Hello,
The upgrade to 4.2.6 has been done 2 weeks ago.
We got a new crash today but I'm not sure that it's the same issue:
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m
256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000619694 in fm_extract_free (qm=0x7f6c97620000,
frag=0x7f6c97904468) at mem/f_malloc.c:206
206 *pf=frag->u.nxt_free;
Missing separate debuginfos, use: debuginfo-install
bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64
elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64
keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64
libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64
libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64
libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64
lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64
mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64
nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64
nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64
openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64
perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64
rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64
xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000619694 in fm_extract_free (qm=0x7f6c97620000,
frag=0x7f6c97904468) at mem/f_malloc.c:206
pf = 0x69442d746e65746e
hash = 2097
#1 0x000000000061ad68 in fm_malloc (qm=0x7f6c97620000, size=1216,
file=0x7577a0 "<core>: mem/shm_mem.c", func=0x75855c "sh_realloc", line=89)
at mem/f_malloc.c:490
f = 0x7f6c97620b48
frag = 0x7f6c97904468
hash = 160
__FUNCTION__ = "fm_malloc"
#2 0x0000000000620b53 in sh_realloc (p=0x7f6c978e8a48, size=1213) at
mem/shm_mem.c:89
r = 0x1ac47418bf0
__FUNCTION__ = "sh_realloc"
#3 0x0000000000620e0b in _shm_resize (p=0x7f6c978e8a48, s=1213,
file=0x7f6cadf27673 "tm: t_reply.c", func=0x7f6cadf2c391 "relay_reply",
line=1961) at mem/shm_mem.c:114
__FUNCTION__ = "_shm_resize"
#4 0x00007f6caded8fdb in relay_reply (t=0x7f6c9792d4a0,
p_msg=0x7f6caf5a2358, branch=0, msg_status=183, cancel_data=0x7fff474183a0,
do_put_on_wait=1) at t_reply.c:1960
relay = 0
save_clone = 0
buf = 0x7f6caf483790 "SIP/2.0 183 Session Progress\r\nf:
<sip:++33123456789 at A.B.C.D:5060>;tag=gK0823f4a1\r\nt:
<sip:+33987654321 at D.C.B.A>;tag=1a5678369670920151016103449\r\ni:
185131394_133144958 at A.B.C.D\r\nCSeq:"...
res_len = 1053
relayed_code = 183
relayed_msg = 0x7f6caf5a2358
reply_bak = 0x4000000
bm = {to_tag_val = {s = 0x7fff47418180 "`Õ\227l\177", len =
-1377010389}}
totag_retr = 0
reply_status = RPS_PROVISIONAL
uas_rb = 0x7f6c9792d560
to_tag = 0x7f6cadec8c8f
reason = {s = 0x474183c8 <Address 0x474183c8 out of bounds>, len =
1024}
onsend_params = {req = 0x7fff474181a0, rpl = 0x7f6cade93bec, param =
0x415440, code = 1195478000, flags = 3, branch = 0, t_rbuf = 0x0, dst = 0x1,
send_buf = {
s = 0x7f6c9792ea38 "\001", len = 6781848}}
__FUNCTION__ = "relay_reply"
#5 0x00007f6cadedc899 in reply_received (p_msg=0x7f6caf5a2358) at
t_reply.c:2511
msg_status = 183
last_uac_status = 100
ack = 0x7f6caf428010 "\001"
ack_len = 0
branch = 0
reply_status = -1354595880
onreply_route = 1
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text =
{s = 0x0, len = 10955836}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len =
10955836}}}}
uac = 0x7f6c9792d608
t = 0x7f6c9792d4a0
lack_dst = {send_sock = 0x4000000, to = {s = {sa_family = 10604,
sa_data = "\247\000\000\000\000\000\r)\247\000\000\000\000"}, sin =
{sin_family = 10604,
sin_port = 167, sin_addr = {s_addr = 0}, sin_zero =
"\r)\247\000\000\000\000"}, sin6 = {sin6_family = 10604, sin6_port = 167,
sin6_flowinfo = 0, sin6_addr = {
__in6_u = {__u6_addr8 =
"\r)\247\000\000\000\000\000p\225Z\257l\177\000", __u6_addr16 = {10509, 167,
0, 0, 38256, 44890, 32620, 0}, __u6_addr32 = {10955021,
0, 2941949296, 32620}}}, sin6_scope_id = 2940756480}},
id = 32620, proto = 40 '(', send_flags = {f = 122 'z', blst_imask = 72 'H'}}
backup_user_from = 0xa827f0
backup_user_to = 0xa827f8
backup_domain_from = 0xa82800
backup_domain_to = 0xa82808
backup_uri_from = 0xa827e0
backup_uri_to = 0xa827e8
backup_xavps = 0xa82920
replies_locked = 1
branch_ret = 0
prev_branch = -1353047176
blst_503_timeout = 32620
hf = 0x47c47418470
onsend_params = {req = 0x7fff47418360, rpl = 0x47deb8, param = 0x0,
code = -1354201032, flags = 32620, branch = 0, t_rbuf = 0xa72c3c, dst =
0xa7290d, send_buf = {
s = 0x7fff47418420 "\350'\250", len = 6402299}}
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env =
{{__jmpbuf = {140104775116112, -3429479277312539647, 4281408,
140734388866032, 0, 0,
-3429479272707193855, 3429074136233477121}, __mask_was_saved
= 0, __saved_mask = {__val = {0, 140734388864032, 6439748, 140734388863792,
140104760342234,
140734388864064, 0, 67108864, 65539104, 1286592, 1569760,
1576600, 8, 94, 140104760342234, 1474369258384}}}}}
__FUNCTION__ = "reply_received"
#6 0x000000000048cc3a in do_forward_reply (msg=0x7f6caf5a2358, mode=0) at
forward.c:783
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000'
<repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr
= 0},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 =
{sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0,
0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id
= 0, proto = 0 '\000',
send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
new_len = 1
r = 0
ip = {af = 1195476416, len = 32767, u = {addrl = {6466393, 280},
addr32 = {6466393, 0, 280, 0}, addr16 = {43865, 98, 0, 0, 280, 0, 0, 0},
addr =
"Y\253b\000\000\000\000\000\030\001\000\000\000\000\000"}}
s = 0x4 <Address 0x4 out of bounds>
len = 0
__FUNCTION__ = "do_forward_reply"
#7 0x000000000048e27d in forward_reply (msg=0x7f6caf5a2358) at
forward.c:885
No locals.
#8 0x0000000000509c9c in receive_msg (
buf=0xa727c0 "SIP/2.0 183 Session Progress\r\nf:
<sip:++33123456789 at A.B.C.D:5060>;tag=gK0823f4a1\r\nt:
<sip:+33987654321 at D.C.B.A>;tag=1a5678369670920151016103449\r\ni:
185131394_133144958 at A.B.C.D\r\nCSeq:"..., len=1148, rcv_info=0x7fff474187c0)
at receive.c:275
---Type <return> to continue, or q <return> to quit---
msg = 0x7f6caf5a2358
ctx = {rec_lev = 10237056, run_flags = 0, last_retcode = 0, jmp_env
= {{__jmpbuf = {0, 0, 0, 272136986608, 1812476198913, 0, 272145363728,
272145384176},
__mask_was_saved = 0, __saved_mask = {__val =
{140104773706736, 140734388864864, 1, 140104373011696, 272137013029, 50195,
1024, 5490444048, 140104373011696,
140734388864784, 6299381, 140734388865072,
140104373011696, 81, 6299509, 140734388865152}}}}}
ret = 1195476832
inb = {
s = 0xa727c0 "SIP/2.0 183 Session Progress\r\nf:
<sip:++33123456789 at A.B.C.D:5060>;tag=gK0823f4a1\r\nt:
<sip:+33987654321 at D.C.B.A>;tag=1a5678369670920151016103449\r\ni:
185131394_133144958 at A.B.C.D\r\nCSeq:"..., len = 1148}
__FUNCTION__ = "receive_msg"
#9 0x0000000000608f02 in udp_rcv_loop () at udp_server.c:521
len = 1148
buf = "SIP/2.0 183 Session Progress\r\nf:
<sip:++33123456789 at A.B.C.D:5060>;tag=gK0823f4a1\r\nt:
<sip:+33987654321 at D.C.B.A>;tag=1a5678369670920151016103449\r\ni:
185131394_133144958 at A.B.C.D\r\nCSeq:"...
tmp = 0x3f30d2b2f2 <Address 0x3f30d2b2f2 out of bounds>
from = 0x7f6caf488590
fromlen = 16
ri = {src_ip = {af = 2, len = 4, u = {addrl = {151524537, 0}, addr32
= {151524537, 0, 0, 0}, addr16 = {5305, 2312, 0, 0, 0, 0, 0, 0},
addr = "\271\024\b\t", '\000' <repeats 11 times>}}, dst_ip =
{af = 2, len = 4, u = {addrl = {1016190299, 0}, addr32 = {1016190299, 0, 0,
0}, addr16 = {54619,
15505, 0, 0, 0, 0, 0, 0}, addr = "[Õ<", '\000' <repeats 11
times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0,
proto_reserved2 = 0, src_su = {
s = {sa_family = 2, sa_data =
"\023Ĺ\024\b\t\000\000\000\000\000\000\000"}, sin = {sin_family = 2,
sin_port = 50195, sin_addr = {s_addr = 151524537},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 =
{sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 151524537, sin6_addr =
{__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0,
0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},
bind_address = 0x7f6caf44a2b0, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#10 0x00000000004a6d9b in main_loop () at main.c:1629
i = 4
pid = 0
si = 0x7f6caf44a2b0
si_desc = "udp receiver child=4
sock=D.C.B.A:5060\000\177\000\000\060\211AG\377\177\000\000\003zN\000\000\00
0\000\000\016\b\000\000\377\177\000\000\260\204b\227l\177\000\000\000\000\00
0\020\004\000\000\000\260\204b\227l\177\000\000 at TA\000\000\000\000\000\360\2
13AG\001\000\000\000\200\211AG\377\177\000\000\246zN\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#11 0x00000000004acedf in main (argc=7, argv=0x7fff47418bf8) at main.c:2581
cfg_stream = 0x21fb010
c = -1
r = 0
tmp = 0x7fff47419f70 ""
tmp_len = 32767
port = 1195477710
proto = 0
options = 0x6ff8f8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2329478669
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x40d11c
p = 0xc2 <Address 0xc2 out of bounds>
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [mailto:igor.potjevlesch at gmail.com]
Envoyé : vendredi 18 septembre 2015 15:37
À : miconda at gmail.com; 'Kamailio (SER) - Users Mailing List'
<sr-users at lists.sip-router.org>
Objet : RE: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
Yes ok. I will schedule this update and I will let you know.
Thank you.
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda at gmail.com]
Envoyé : jeudi 17 septembre 2015 17:43
À : Igor Potjevlesch <igor.potjevlesch at gmail.com
<mailto:igor.potjevlesch at gmail.com> >; 'Kamailio (SER) - Users Mailing List'
<sr-users at lists.sip-router.org <mailto:sr-users at lists.sip-router.org> >
Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
can you test with latest version branch 4.2? I backported several patches
related to dialog module, among them some related to a race for deleted
dialogs detected as spiral, which may be the reason for this crash.
Cheers,
Daniel
On 17/09/15 12:25, Igor Potjevlesch wrote:
Hello Daniel,
Here is the output:
(gdb) frame 0
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
244 dlg = dlg->next;
(gdb) list
239 {
240 lock_set_get(d_table->locks,
d_table->entries[i].lock_idx);
241 dlg = d_table->entries[i].first;
242 while (dlg) {
243 tdlg = dlg;
244 dlg = dlg->next;
245 if(tdlg->state==DLG_STATE_UNCONFIRMED &&
tdlg->init_ts<tm-300) {
246 /* dialog in early state older than
5min */
247 LM_NOTICE("dialog in early state is
too old (%p ref %d)\n",
248 tdlg, tdlg->ref);
(gdb) info locals
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
(gdb) p *dlg
Cannot access memory at address 0xb02030a01201001
(gdb)
I hope this will help.
Regards,
Igor.
De : Daniel-Constantin Mierla [mailto:miconda at gmail.com]
Envoyé : jeudi 17 septembre 2015 11:40
À : Igor Potjevlesch <mailto:igor.potjevlesch at gmail.com>
<igor.potjevlesch at gmail.com>; 'Kamailio (SER) - Users Mailing List'
<mailto:sr-users at lists.sip-router.org> <sr-users at lists.sip-router.org>
Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
from the second trace, can you get output for:
frame 0
list
info locals
p *dlg
Cheers,
Daniel
On 11/09/15 18:23, Igor Potjevlesch wrote:
Hello Daniel,
>From the two crashes occurred today, I got 2 coredump. So I copy/past the
result from these 4 backtraces:
No privates modules or patches. It's a regular 4.2.3.
(gdb) bt full
#0 0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at
dlg_timer.c:156
No locals.
#1 0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at
dlg_timer.c:182
__FUNCTION__ = "remove_dialog_timer"
#2 0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357
ret = 0
var = 0x7fb6976154b0
__FUNCTION__ = "destroy_dlg"
#3 0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438
dlg = 0xb02030a01201001
l_dlg = 0x7fb6978e9008
i = 2087
__FUNCTION__ = "destroy_dlg_table"
#4 0x00007fb6a8933263 in mod_destroy () at dialog.c:783
No locals.
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7fb6af43d670
foo = 0x7fb6af43d440
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at
main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = -1755228944
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7
sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26
6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S
A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0
00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
(gdb) bt full
#0 0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244
i = 2087
tm = 1441978496
dlg = 0xb02030a01201001
tdlg = 0xb02030a01201001
__FUNCTION__ = "dlg_clean_run"
#1 0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0)
at dialog.c:1260
No locals.
#2 0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1
"Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>,
param=0x0,
interval=90000) at timer_proc.c:235
pid = 0
ts1 = 373077626
ts2 = 90000
#3 0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740
__FUNCTION__ = "child_init"
#4 0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at
sr_module.c:921
__FUNCTION__ = "init_mod_child"
#5 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#6 0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#7 0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#8 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#9 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at
sr_module.c:918
__FUNCTION__ = "init_mod_child"
#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947
No locals.
#15 0x00000000004a64c4 in main_loop () at main.c:1706
i = 8
pid = 4424
si = 0x0
si_desc = "udp receiver child=7
sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\26
6\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060S
A\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\0
00\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578
cfg_stream = 0x18b4010
c = -1
r = 0
tmp = 0x7fff08879f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 2249241156
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fff08877e7e ""
__FUNCTION__ = "main"
--
Daniel-Constantin Mierla
http://twitter.com/#!/miconda <http://twitter.com/#%21/miconda> -
http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin -
http://asipto.com/u/kat
--
Daniel-Constantin Mierla
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin -
http://asipto.com/u/kat
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.sip-router.org/pipermail/sr-users/attachments/20151016/9f04a757/attachment.html>
More information about the sr-users
mailing list