Hello,

it is still related to tm and the other related crashes. This one can be avoided via setting:

mem_safety=1

in kamailio.cfg

I saw that other developers will start soon to investigate, I will look over the reports soon as well, currently being out of office.

Cheers,
Daniel

On 28/02/14 13:31, Tuan Viet Nguyen wrote:
Hello,

I have another kamailio crash with 4.1.1. The scenario is simple:
+ call a number, fail
+ retry on another peer, fails

In this case, I use t_reply("404", "Not found") then exit. It seems that we have a double qm_free.

Log file
Feb 28 13:15:56 kamailio23 /usr/local/sbin/kamailio[29595]: WARNING: tm [t_lookup.c:1536]: t_unref(): WARNING: script writer didn't release transaction
Feb 28 13:15:56 kamailio23 /usr/local/sbin/kamailio[29583]: : <core> [mem/q_malloc.c:468]: qm_free(): BUG: qm_free: freeing already freed pointer (0x7f9dedb3e110), called from tm: h_table.c: free_cell(157), first free tm: h_table.c: free_cell(157) - aborting
Feb 28 13:15:56 kamailio23 /usr/local/sbin/kamailio[29627]: : <core> [pass_fd.c:293]: receive_fd(): ERROR: receive_fd: EOF on 19
Feb 28 13:15:56 kamailio23 /usr/local/sbin/kamailio[29576]: ALERT: <core> [main.c:775]: handle_sigs(): child process 29583 exited by a signal 6
Feb 28 13:15:56 kamailio23 /usr/local/sbin/kamailio[29576]: ALERT: <core> [main.c:778]: handle_sigs(): core was generated

core.kamailio.29583
(gdb) bt full
#0  0x00007f9df6afb475 in raise () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#1  0x00007f9df6afe6f0 in abort () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#2  0x0000000000548253 in qm_free (qm=0x7f9ded878000, p=0x7f9dedb3e110, file=0x7f9df515b62d "tm: h_table.c", func=0x7f9df515b7d8 "free_cell", line=157) at mem/q_malloc.c:470
        f = 0x7f9dedb3e0e0
        size = 140316274568736
        next = 0x0
        prev = 0x0
        __FUNCTION__ = "qm_free"
#3  0x00007f9df50ee2b3 in free_cell (dead_cell=0x7f9dedb39490) at h_table.c:157
        b = 0x0
        i = 2
        rpl = 0x0
        tt = 0x177dd596e30
        foo = 0x7fffdd597400
        cbs = 0x7f9dedb39490
        cbs_tmp = 0x7fffdd596f10
        __FUNCTION__ = "free_cell"
#4  0x00007f9df511c662 in t_unref (p_msg=0x7f9df68bfdb0) at t_lookup.c:1546
        kr = 12
        __FUNCTION__ = "t_unref"
#5  0x00007f9df5146f4d in w_t_unref (foo=0x7f9df68bfdb0, flags=2147483649, bar=0x0) at tm.c:765
No locals.
#6  0x00000000004d6f27 in exec_post_script_cb (msg=0x7f9df68bfdb0, type=REQUEST_CB_TYPE) at script_cb.c:195
        cb = 0x7f9df68a97d0
        flags = 2147483649
#7  0x00000000004a6e24 in receive_msg (buf=0x921620 "ACK sip:0123456789@10.100.8.7 SIP/2.0\r\nVia: SIP/2.0/UDP 10.100.8.94;branch=z9hG4bKbcd8.9b0d4b2682a07fefaa22406761659624.0\r\nMax-Forwards: 15\r\nFrom: \"via_test\" <sip:0310193301@10.100.8.12>;tag=as4d03043"...,
    len=375, rcv_info=0x7fffdd5970d0) at receive.c:228
        msg = 0x7f9df68bfdb0
        ctx = {rec_lev = -581341280, run_flags = 32767, last_retcode = 5, jmp_env = {{__jmpbuf = {8857080, 0, 0, 0, 0, 140316433164786, 17179869189, 0}, __mask_was_saved = 8856992, __saved_mask = {__val = {1, 140316271892032, 1656210553, 3713626240, 1024, 8008593472,
                  140316271892032, 140736907014208, 5472467, 1577608202, 140316271892032, 50195, 140316271892032, 140316426284544, 8282899704, 140736907014272}}}}}
        ret = 0
        inb = {s = 0x921620 "ACK sip:0123456789@10.100.8.7 SIP/2.0\r\nVia: SIP/2.0/UDP 10.100.8.94;branch=z9hG4bKbcd8.9b0d4b2682a07fefaa22406761659624.0\r\nMax-Forwards: 15\r\nFrom: \"via_test\" <sip:0310193301@10.100.8.12>;tag=as4d03043"..., len = 375}
        __FUNCTION__ = "receive_msg"
#8  0x000000000053c0d8 in udp_rcv_loop () at udp_server.c:536
        len = 375
        buf = "ACK sip:0123456789@10.100.8.7 SIP/2.0\r\nVia: SIP/2.0/UDP 10.100.8.94;branch=z9hG4bKbcd8.9b0d4b2682a07fefaa22406761659624.0\r\nMax-Forwards: 15\r\nFrom: \"via_test\" <sip:0310193301@10.100.8.12>;tag=as4d03043"...
        tmp = 0x13e0751a0eb8aa4f <Address 0x13e0751a0eb8aa4f out of bounds>
        from = 0x7f9df68a5408
        fromlen = 16
        ri = {src_ip = {af = 2, len = 4, u = {addrl = {1577608202, 4290224}, addr32 = {1577608202, 0, 4290224, 0}, addr16 = {25610, 24072, 0, 0, 30384, 65, 0, 0}, addr = "\nd\b^\000\000\000\000\260vA\000\000\000\000"}}, dst_ip = {af = 2, len = 4, u = {addrl = {
                117990410, 0}, addr32 = {117990410, 0, 0, 0}, addr16 = {25610, 1800, 0, 0, 0, 0, 0, 0}, addr = "\nd\b\a", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2,
              sa_data = "\023\304\nd\b^\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 1577608202}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 1577608202,
              sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, bind_address = 0x7f9df6704da8, proto = 1 '\001'}
        __FUNCTION__ = "udp_rcv_loop"
#9  0x000000000046ec98 in main_loop () at main.c:1617
        i = 5
        pid = 0
        si = 0x7f9df6704da8
        si_desc = "udp receiver child=5 sock=10.100.8.7:5060\000\000\000\001", '\000' <repeats 19 times>, "\020\000\000\000\000\000\000\000y\304\267b\000\000\000\000\260vA\000\000\000\000\000\000tY\335\377\177", '\000' <repeats 18 times>, "@rY\335\377\177\000\000\002\266K\000\000\000\000"
        nrprocs = 8
        __FUNCTION__ = "main_loop"
#10 0x0000000000471c38 in main (argc=5, argv=0x7fffdd597408) at main.c:2533
        cfg_stream = 0x164c010
        c = -1
        r = 0
        tmp = 0x7fffdd597438 "\211~Y\335\377\177"
        tmp_len = 0
        port = 5
        proto = 0
        options = 0x5de800 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 4192398120
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0xbf
        p = 0x416bd9 "H\203\304\b\303" <Address 0x416bde out of bounds>
        __FUNCTION__ = "main"


core.kamailio.29576
(gdb) bt full
#0  0x00007f9df6afb475 in raise () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#1  0x00007f9df6afe6f0 in abort () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#2  0x0000000000548253 in qm_free (qm=0x7f9ded878000, p=0x7f9dedb3e110, file=0x7f9df515b62d "tm: h_table.c", func=0x7f9df515b7d8 "free_cell", line=157) at mem/q_malloc.c:470
        f = 0x7f9dedb3e0e0
        size = 140316274568736
        next = 0x0
        prev = 0x0
        __FUNCTION__ = "qm_free"
#3  0x00007f9df50ee2b3 in free_cell (dead_cell=0x7f9dedb39490) at h_table.c:157
        b = 0x7f9df464b480 "dialog: dlg_cb.c"
        i = 0
        rpl = 0x7f9df464b8e0
        tt = 0x7f9df4a8cc75
        foo = 0x5000548a29
        cbs = 0xd000000001
        cbs_tmp = 0x1edb086c8
        __FUNCTION__ = "free_cell"
#4  0x00007f9df50ef480 in free_hash_table () at h_table.c:441
        p_cell = 0x7f9dedb39490
        tmp_cell = 0x7f9deda23d90
        i = 36299
        __FUNCTION__ = "free_hash_table"
#5  0x00007f9df5102d35 in tm_shutdown () at t_funcs.c:122
        __FUNCTION__ = "tm_shutdown"
#6  0x00000000004f8101 in destroy_modules () at sr_module.c:817
        t = 0x7f9df670fd50
        foo = 0x7f9df670f588
        __FUNCTION__ = "destroy_modules"
#7  0x00000000004689b2 in cleanup (show_status=1) at main.c:560
        memlog = 32669
        __FUNCTION__ = "cleanup"
#8  0x0000000000469aab in shutdown_children (sig=15, show_status=1) at main.c:702
        __FUNCTION__ = "shutdown_children"
#9  0x000000000046b146 in handle_sigs () at main.c:793
        chld = 0
        chld_status = 134
        memlog = 0
        __FUNCTION__ = "handle_sigs"
#10 0x000000000046f549 in main_loop () at main.c:1746
        i = 8
        pid = 29627
        si = 0x0
        si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000\001", '\000' <repeats 19 times>, "\020\000\000\000\000\000\000\000y\304\267b\000\000\000\000\260vA\000\000\000\000\000\000tY\335\377\177", '\000' <repeats 18 times>, "@rY\335\377\177\000\000\002\266K\000\000\000\000"
        nrprocs = 8
        __FUNCTION__ = "main_loop"
#11 0x0000000000471c38 in main (argc=5, argv=0x7fffdd597408) at main.c:2533
        cfg_stream = 0x164c010
        c = -1
        r = 0
        tmp = 0x7fffdd597438 "\211~Y\335\377\177"
        tmp_len = 0
        port = 5
        proto = 0
        options = 0x5de800 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 4192398120
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0xbf
        p = 0x416bd9 "H\203\304\b\303" <Address 0x416bde out of bounds>
        __FUNCTION__ = "main"


Thanks for your help


_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users

-- 
Daniel-Constantin Mierla - http://www.asipto.com
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda