Hello,

 

The upgrade to 4.2.6 has been done 2 weeks ago.

 

We got a new crash today but I'm not sure that it's the same issue:

 

Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m 256 -M 64'.

Program terminated with signal 11, Segmentation fault.

#0  0x0000000000619694 in fm_extract_free (qm=0x7f6c97620000, frag=0x7f6c97904468) at mem/f_malloc.c:206

206             *pf=frag->u.nxt_free;

Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64 elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64 libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64 libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64 nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64 nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64 openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64 perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64 xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64

(gdb) bt full

#0  0x0000000000619694 in fm_extract_free (qm=0x7f6c97620000, frag=0x7f6c97904468) at mem/f_malloc.c:206

        pf = 0x69442d746e65746e

        hash = 2097

#1  0x000000000061ad68 in fm_malloc (qm=0x7f6c97620000, size=1216, file=0x7577a0 "<core>: mem/shm_mem.c", func=0x75855c "sh_realloc", line=89) at mem/f_malloc.c:490

        f = 0x7f6c97620b48

        frag = 0x7f6c97904468

        hash = 160

        __FUNCTION__ = "fm_malloc"

#2  0x0000000000620b53 in sh_realloc (p=0x7f6c978e8a48, size=1213) at mem/shm_mem.c:89

        r = 0x1ac47418bf0

        __FUNCTION__ = "sh_realloc"

#3  0x0000000000620e0b in _shm_resize (p=0x7f6c978e8a48, s=1213, file=0x7f6cadf27673 "tm: t_reply.c", func=0x7f6cadf2c391 "relay_reply", line=1961) at mem/shm_mem.c:114

        __FUNCTION__ = "_shm_resize"

#4  0x00007f6caded8fdb in relay_reply (t=0x7f6c9792d4a0, p_msg=0x7f6caf5a2358, branch=0, msg_status=183, cancel_data=0x7fff474183a0, do_put_on_wait=1) at t_reply.c:1960

        relay = 0

        save_clone = 0

        buf = 0x7f6caf483790 "SIP/2.0 183 Session Progress\r\nf: <sip:++33123456789@A.B.C.D:5060>;tag=gK0823f4a1\r\nt: <sip:+33987654321@D.C.B.A>;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"...

        res_len = 1053

        relayed_code = 183

        relayed_msg = 0x7f6caf5a2358

        reply_bak = 0x4000000

        bm = {to_tag_val = {s = 0x7fff47418180 "`Õ\227l\177", len = -1377010389}}

        totag_retr = 0

        reply_status = RPS_PROVISIONAL

        uas_rb = 0x7f6c9792d560

        to_tag = 0x7f6cadec8c8f

        reason = {s = 0x474183c8 <Address 0x474183c8 out of bounds>, len = 1024}

        onsend_params = {req = 0x7fff474181a0, rpl = 0x7f6cade93bec, param = 0x415440, code = 1195478000, flags = 3, branch = 0, t_rbuf = 0x0, dst = 0x1, send_buf = {

            s = 0x7f6c9792ea38 "\001", len = 6781848}}

        __FUNCTION__ = "relay_reply"

#5  0x00007f6cadedc899 in reply_received (p_msg=0x7f6caf5a2358) at t_reply.c:2511

        msg_status = 183

        last_uac_status = 100

        ack = 0x7f6caf428010 "\001"

        ack_len = 0

        branch = 0

        reply_status = -1354595880

        onreply_route = 1

        cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 10955836}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 10955836}}}}

        uac = 0x7f6c9792d608

        t = 0x7f6c9792d4a0

        lack_dst = {send_sock = 0x4000000, to = {s = {sa_family = 10604, sa_data = "\247\000\000\000\000\000\r)\247\000\000\000\000"}, sin = {sin_family = 10604,

              sin_port = 167, sin_addr = {s_addr = 0}, sin_zero = "\r)\247\000\000\000\000"}, sin6 = {sin6_family = 10604, sin6_port = 167, sin6_flowinfo = 0, sin6_addr = {

                __in6_u = {__u6_addr8 = "\r)\247\000\000\000\000\000p\225Z\257l\177\000", __u6_addr16 = {10509, 167, 0, 0, 38256, 44890, 32620, 0}, __u6_addr32 = {10955021,

                    0, 2941949296, 32620}}}, sin6_scope_id = 2940756480}}, id = 32620, proto = 40 '(', send_flags = {f = 122 'z', blst_imask = 72 'H'}}

        backup_user_from = 0xa827f0

        backup_user_to = 0xa827f8

        backup_domain_from = 0xa82800

        backup_domain_to = 0xa82808

        backup_uri_from = 0xa827e0

        backup_uri_to = 0xa827e8

        backup_xavps = 0xa82920

        replies_locked = 1

        branch_ret = 0

        prev_branch = -1353047176

        blst_503_timeout = 32620

        hf = 0x47c47418470

        onsend_params = {req = 0x7fff47418360, rpl = 0x47deb8, param = 0x0, code = -1354201032, flags = 32620, branch = 0, t_rbuf = 0xa72c3c, dst = 0xa7290d, send_buf = {

            s = 0x7fff47418420 "\350'\250", len = 6402299}}

        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {140104775116112, -3429479277312539647, 4281408, 140734388866032, 0, 0,

                -3429479272707193855, 3429074136233477121}, __mask_was_saved = 0, __saved_mask = {__val = {0, 140734388864032, 6439748, 140734388863792, 140104760342234,

                  140734388864064, 0, 67108864, 65539104, 1286592, 1569760, 1576600, 8, 94, 140104760342234, 1474369258384}}}}}

        __FUNCTION__ = "reply_received"

#6  0x000000000048cc3a in do_forward_reply (msg=0x7f6caf5a2358, mode=0) at forward.c:783

        new_buf = 0x0

        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0},

              sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {

                  __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000',

          send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}

        new_len = 1

        r = 0

        ip = {af = 1195476416, len = 32767, u = {addrl = {6466393, 280}, addr32 = {6466393, 0, 280, 0}, addr16 = {43865, 98, 0, 0, 280, 0, 0, 0},

            addr = "Y\253b\000\000\000\000\000\030\001\000\000\000\000\000"}}

        s = 0x4 <Address 0x4 out of bounds>

        len = 0

        __FUNCTION__ = "do_forward_reply"

#7  0x000000000048e27d in forward_reply (msg=0x7f6caf5a2358) at forward.c:885

No locals.

#8  0x0000000000509c9c in receive_msg (

    buf=0xa727c0 "SIP/2.0 183 Session Progress\r\nf: <sip:++33123456789@A.B.C.D:5060>;tag=gK0823f4a1\r\nt: <sip:+33987654321@D.C.B.A>;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"..., len=1148, rcv_info=0x7fff474187c0) at receive.c:275

---Type <return> to continue, or q <return> to quit---

        msg = 0x7f6caf5a2358

        ctx = {rec_lev = 10237056, run_flags = 0, last_retcode = 0, jmp_env = {{__jmpbuf = {0, 0, 0, 272136986608, 1812476198913, 0, 272145363728, 272145384176},

              __mask_was_saved = 0, __saved_mask = {__val = {140104773706736, 140734388864864, 1, 140104373011696, 272137013029, 50195, 1024, 5490444048, 140104373011696,

                  140734388864784, 6299381, 140734388865072, 140104373011696, 81, 6299509, 140734388865152}}}}}

        ret = 1195476832

        inb = {

          s = 0xa727c0 "SIP/2.0 183 Session Progress\r\nf: <sip:++33123456789@A.B.C.D:5060>;tag=gK0823f4a1\r\nt: <sip:+33987654321@D.C.B.A>;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"..., len = 1148}

        __FUNCTION__ = "receive_msg"

#9  0x0000000000608f02 in udp_rcv_loop () at udp_server.c:521

        len = 1148

        buf = "SIP/2.0 183 Session Progress\r\nf: <sip:++33123456789@A.B.C.D:5060>;tag=gK0823f4a1\r\nt: <sip:+33987654321@D.C.B.A>;tag=1a5678369670920151016103449\r\ni: 185131394_133144958@A.B.C.D\r\nCSeq:"...

        tmp = 0x3f30d2b2f2 <Address 0x3f30d2b2f2 out of bounds>

        from = 0x7f6caf488590

        fromlen = 16

        ri = {src_ip = {af = 2, len = 4, u = {addrl = {151524537, 0}, addr32 = {151524537, 0, 0, 0}, addr16 = {5305, 2312, 0, 0, 0, 0, 0, 0},

              addr = "\271\024\b\t", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u = {addrl = {1016190299, 0}, addr32 = {1016190299, 0, 0, 0}, addr16 = {54619,

                15505, 0, 0, 0, 0, 0, 0}, addr = "[Õ<", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {

            s = {sa_family = 2, sa_data = "\023Ĺ\024\b\t\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 151524537},

              sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 151524537, sin6_addr = {__in6_u = {

                  __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},

          bind_address = 0x7f6caf44a2b0, proto = 1 '\001'}

        __FUNCTION__ = "udp_rcv_loop"

#10 0x00000000004a6d9b in main_loop () at main.c:1629

        i = 4

        pid = 0

        si = 0x7f6caf44a2b0

        si_desc = "udp receiver child=4 sock=D.C.B.A:5060\000\177\000\000\060\211AG\377\177\000\000\003zN\000\000\000\000\000\016\b\000\000\377\177\000\000\260\204b\227l\177\000\000\000\000\000\020\004\000\000\000\260\204b\227l\177\000\000@TA\000\000\000\000\000\360\213AG\001\000\000\000\200\211AG\377\177\000\000\246zN\000\000\000\000"

        nrprocs = 8

        __FUNCTION__ = "main_loop"

#11 0x00000000004acedf in main (argc=7, argv=0x7fff47418bf8) at main.c:2581

        cfg_stream = 0x21fb010

        c = -1

        r = 0

        tmp = 0x7fff47419f70 ""

        tmp_len = 32767

        port = 1195477710

        proto = 0

        options = 0x6ff8f8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"

        ret = -1

        seed = 2329478669

        rfd = 4

        debug_save = 0

        debug_flag = 0

        dont_fork_cnt = 0

        n_lst = 0x40d11c

        p = 0xc2 <Address 0xc2 out of bounds>

        __FUNCTION__ = "main"

 

Regards,

 

Igor.

 

De : Igor Potjevlesch [mailto:igor.potjevlesch@gmail.com]
Envoyé : vendredi 18 septembre 2015 15:37
À : miconda@gmail.com; 'Kamailio (SER) - Users Mailing List' <sr-users@lists.sip-router.org>
Objet : RE: [SR-Users] Multiple crashes of Kamailio 4.2.1

 

Hello,

 

Yes ok. I will schedule this update and I will let you know.


Thank you.

Regards,


Igor.

 

De : Daniel-Constantin Mierla [mailto:miconda@gmail.com]
Envoyé : jeudi 17 septembre 2015 17:43
À : Igor Potjevlesch <igor.potjevlesch@gmail.com>; 'Kamailio (SER) - Users Mailing List' <sr-users@lists.sip-router.org>
Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1

 

Hello,

can you test with latest version branch 4.2? I backported several patches related to dialog module, among them some related to a race for deleted dialogs detected as spiral, which may be the reason for this crash.

Cheers,
Daniel

On 17/09/15 12:25, Igor Potjevlesch wrote:

Hello Daniel,

 

Here is the output:

 

(gdb) frame 0

#0  0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244

244                             dlg = dlg->next;

(gdb) list

239             {

240                     lock_set_get(d_table->locks, d_table->entries[i].lock_idx);

241                     dlg = d_table->entries[i].first;

242                     while (dlg) {

243                             tdlg = dlg;

244                             dlg = dlg->next;

245                             if(tdlg->state==DLG_STATE_UNCONFIRMED && tdlg->init_ts<tm-300) {

246                                     /* dialog in early state older than 5min */

247                                     LM_NOTICE("dialog in early state is too old (%p ref %d)\n",

248                                                     tdlg, tdlg->ref);

(gdb) info locals

i = 2087

tm = 1441978496

dlg = 0xb02030a01201001

tdlg = 0xb02030a01201001

__FUNCTION__ = "dlg_clean_run"

(gdb) p *dlg

Cannot access memory at address 0xb02030a01201001

(gdb)

 

I hope this will help.

 

Regards,

 

Igor.

 

 

De : Daniel-Constantin Mierla [mailto:miconda@gmail.com]
Envoyé : jeudi 17 septembre 2015 11:40
À : Igor Potjevlesch <igor.potjevlesch@gmail.com>; 'Kamailio (SER) - Users Mailing List' <sr-users@lists.sip-router.org>
Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1

 

Hello,

from the second trace, can you get output for:

frame 0
list
info locals
p *dlg

Cheers,
Daniel

On 11/09/15 18:23, Igor Potjevlesch wrote:

Hello Daniel,

 

>From the two crashes occurred today, I got 2 coredump. So I copy/past the result from these 4 backtraces:

 

No privates modules or patches. It's a regular 4.2.3.

 

(gdb) bt full

#0  0x00007fb6a8984c0e in remove_dialog_timer_unsafe (tl=0x7fb6978e9060) at dlg_timer.c:156

No locals.

#1  0x00007fb6a8985001 in remove_dialog_timer (tl=0x7fb6978e9060) at dlg_timer.c:182

        __FUNCTION__ = "remove_dialog_timer"

#2  0x00007fb6a8966bb7 in destroy_dlg (dlg=0x7fb6978e9008) at dlg_hash.c:357

        ret = 0

        var = 0x7fb6976154b0

        __FUNCTION__ = "destroy_dlg"

#3  0x00007fb6a8967b35 in destroy_dlg_table () at dlg_hash.c:438

        dlg = 0xb02030a01201001

        l_dlg = 0x7fb6978e9008

        i = 2087

        __FUNCTION__ = "destroy_dlg_table"

#4  0x00007fb6a8933263 in mod_destroy () at dialog.c:783

No locals.

#5  0x0000000000590d79 in destroy_modules () at sr_module.c:811

        t = 0x7fb6af43d670

        foo = 0x7fb6af43d440

        __FUNCTION__ = "destroy_modules"

#6  0x000000000049bb43 in cleanup (show_status=1) at main.c:569

        memlog = 0

        __FUNCTION__ = "cleanup"

#7  0x000000000049d10b in shutdown_children (sig=15, show_status=1) at main.c:711

        __FUNCTION__ = "shutdown_children"

#8  0x000000000049f6e1 in handle_sigs () at main.c:802

        chld = 0

        chld_status = 139

        memlog = -1755228944

        __FUNCTION__ = "handle_sigs"

#9  0x00000000004a6fbf in main_loop () at main.c:1757

        i = 8

        pid = 4424

        si = 0x0

        si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"

        nrprocs = 8

        __FUNCTION__ = "main_loop"

#10 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578

        cfg_stream = 0x18b4010

        c = -1

        r = 0

        tmp = 0x7fff08879f70 ""

        tmp_len = 0

        port = 0

        proto = 32767

        options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"

        ret = -1

        seed = 2249241156

        rfd = 4

        debug_save = 0

        debug_flag = 0

        dont_fork_cnt = 0

        n_lst = 0xc2

        p = 0x7fff08877e7e ""

        __FUNCTION__ = "main"

 

(gdb) bt full

#0  0x00007fb6a8964e55 in dlg_clean_run (ti=23317351) at dlg_hash.c:244

        i = 2087

        tm = 1441978496

        dlg = 0xb02030a01201001

        tdlg = 0xb02030a01201001

        __FUNCTION__ = "dlg_clean_run"

#1  0x00007fb6a8938dd6 in dlg_clean_timer_exec (ticks=23317351, param=0x0) at dialog.c:1260

No locals.

#2  0x00000000005fd540 in fork_sync_timer (child_id=-1, desc=0x7fb6a89970f1 "Dialog Clean Timer", make_sock=1, f=0x7fb6a8938dbd <dlg_clean_timer_exec>, param=0x0,

    interval=90000) at timer_proc.c:235

        pid = 0

        ts1 = 373077626

        ts2 = 90000

#3  0x00007fb6a8932b50 in child_init (rank=0) at dialog.c:740

        __FUNCTION__ = "child_init"

#4  0x0000000000591129 in init_mod_child (m=0x7fb6af43d670, rank=0) at sr_module.c:921

        __FUNCTION__ = "init_mod_child"

#5  0x0000000000590e64 in init_mod_child (m=0x7fb6af43e1b0, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#6  0x0000000000590e64 in init_mod_child (m=0x7fb6af43e728, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#7  0x0000000000590e64 in init_mod_child (m=0x7fb6af43eb90, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#8  0x0000000000590e64 in init_mod_child (m=0x7fb6af43f108, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#9  0x0000000000590e64 in init_mod_child (m=0x7fb6af43f418, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#10 0x0000000000590e64 in init_mod_child (m=0x7fb6af43f808, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#11 0x0000000000590e64 in init_mod_child (m=0x7fb6af43fb18, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#12 0x0000000000590e64 in init_mod_child (m=0x7fb6af440090, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#13 0x0000000000590e64 in init_mod_child (m=0x7fb6af4403d8, rank=0) at sr_module.c:918

        __FUNCTION__ = "init_mod_child"

#14 0x0000000000591433 in init_child (rank=0) at sr_module.c:947

No locals.

#15 0x00000000004a64c4 in main_loop () at main.c:1706

        i = 8

        pid = 4424

        si = 0x0

        si_desc = "udp receiver child=7 sock=A.B.C.D:5060\000\000\000\000\016\b\000\000\377\177\000\000\260Ta\227\266\177\000\000\000\000\000\020\004\000\000\000\260Ta\227\266\177\000\000\060SA\000\000\000\000\000\240\177\207\b\001\000\000\000\060}\207\b\377\177\000\000\032dN\000\000\000\000\000h\261@\257z\000\000\000\276}p\000\000\000\000"

        nrprocs = 8

        __FUNCTION__ = "main_loop"

#16 0x00000000004ab8bf in main (argc=7, argv=0x7fff08877fa8) at main.c:2578

        cfg_stream = 0x18b4010

        c = -1

        r = 0

        tmp = 0x7fff08879f70 ""

        tmp_len = 0

        port = 0

        proto = 32767

        options = 0x6fcc00 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"

        ret = -1

        seed = 2249241156

        rfd = 4

        debug_save = 0

        debug_flag = 0

        dont_fork_cnt = 0

        n_lst = 0xc2

        p = 0x7fff08877e7e ""

        __FUNCTION__ = "main"

 



-- 
Daniel-Constantin Mierla
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat

 

-- 
Daniel-Constantin Mierla
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat