Hello Again,

I was able to reproduce again. It crashed from the same point and it seems that frequency of this issue has reduced but it is not fixed yet.
As per my observation, it happens when entry of destination from KA list is removed during an ongoing KA transaction. I am removing destination from KA list when endpoint de registers so this case is possible. I have also observed same on pcaps. Below are the screenshots :

OPTIONS :
Screenshot from 2020-09-04 23-02-14

REGISTER :
Screenshot from 2020-09-04 23-01-25

Kamailio has dispatched 200 OK for register at 06:33:58.187520 and it received 200 OK for OPTIONS at 06:33:58.411501. Kamailio crashed exactly at a same time.

Core Dump :

GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-110.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/local/sbin/kamailio...done.
[New LWP 12701]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/usr/local/sbin/kamailio -m 5000 -M 500 -P /run/kamailio/kamailio.pid'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007f09c138dc7c in ka_options_callback (t=0x7f0896cc1530, type=1024, ps=0x7ffdb22e9d20) at keepalive_core.c:126
126                             ka_dest->statechanged_clb(&ka_dest->uri, state, ka_dest->user_attr);
Missing separate debuginfos, use: debuginfo-install cyrus-sasl-lib-2.1.26-23.el7.x86_64 glibc-2.17-222.el7.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-19.el7.x86_64 libcom_err-1.42.9-12.el7_5.x86_64 libcurl-7.29.0-51.el7.x86_64 libevent-2.0.21-4.el7.x86_64 libgcc-4.8.5-36.el7_6.2.x86_64 libidn-1.28-4.el7.x86_64 libselinux-2.5-12.el7.x86_64 libssh2-1.4.3-10.el7_2.1.x86_64 libstdc++-4.8.5-36.el7_6.2.x86_64 libuuid-2.23.2-52.el7_5.1.x86_64 mariadb-libs-5.5.60-1.el7_5.x86_64 nspr-4.19.0-1.el7_5.x86_64 nss-3.36.0-5.el7_5.x86_64 nss-softokn-freebl-3.36.0-5.el7_5.x86_64 nss-util-3.36.0-1.el7_5.x86_64 openldap-2.4.44-15.el7_5.x86_64 openssl-libs-1.0.2k-16.el7_6.1.x86_64 pcre-8.32-17.el7.x86_64 zlib-1.2.7-18.el7.x86_64
(gdb) bt full
#0  0x00007f09c138dc7c in ka_options_callback (t=0x7f0896cc1530, type=1024, ps=0x7ffdb22e9d20) at keepalive_core.c:126
        uri = {
          s = 0x7f0896cc39b7 "sip:7459@10.50.8.6:9090;alias=10.50.8.1~5060~1;pb-ip=174.198.221.83;pb-pt=3939;tp=tcp>\r\nFrom: <sip:keepalive@fromsbc>;tag=ee3b67b160621ee354865fd8162b5b29-1b81a45f\r\nCSeq: 10 OPTIONS\r\nCall-ID: 2557bb53"..., len = 85}
        msg = 0x0
        state = 1
        state_routes = {0x7f09c1393844 "", 0x7f09c139393d "keepalive:dst-up", 0x7f09c139394e "keepalive:dst-down"}
        ka_dest = 0x7f088ac83048
        __FUNCTION__ = "ka_options_callback"
#1  0x00007f09c8321dd8 in run_trans_callbacks_internal (cb_lst=0x7f0896cc15a8, type=1024, trans=0x7f0896cc1530, params=0x7ffdb22e9d20)
    at t_hooks.c:258
        cbp = 0x7f0896cc3b60
        backup_from = 0xb1f170 <def_list+16>
        backup_to = 0xb1f178 <def_list+24>
        backup_dom_from = 0xb1f180 <def_list+32>
        backup_dom_to = 0xb1f188 <def_list+40>
        backup_uri_from = 0xb1f160 <def_list>
        backup_uri_to = 0xb1f168 <def_list+8>
        backup_xavps = 0xb1ea60 <_xavp_list_head>
        backup_xavus = 0xb1ea68 <_xavu_list_head>
        backup_xavis = 0xb1ea70 <_xavi_list_head>
        __FUNCTION__ = "run_trans_callbacks_internal"
#2  0x00007f09c8321f0a in run_trans_callbacks (type=1024, trans=0x7f0896cc1530, req=0x0, rpl=0x7f09cb832030, code=200) at t_hooks.c:285
        params = {req = 0x0, rpl = 0x7f09cb832030, param = 0x7f0896cc3b70, code = 200, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {
            s = 0x0, len = 0}}
#3  0x00007f09c82b2bef in local_reply (t=0x7f0896cc1530, p_msg=0x7f09cb832030, branch=0, msg_status=200, cancel_data=0x7ffdb22ea110)
    at t_reply.c:2265
        local_store = 0
        local_winner = 0
        reply_status = RPS_COMPLETED
        winning_msg = 0x7f09cb832030
        winning_code = 200
        totag_retr = 0
        __FUNCTION__ = "local_reply"
#4  0x00007f09c82b5a8b in reply_received (p_msg=0x7f09cb832030) at t_reply.c:2648
---Type <return> to continue, or q <return> to quit---
        msg_status = 200
        last_uac_status = 0
        ack = 0x5a62bc <sr_event_exec+415> "\211E\374\213E\374\351)\003"
        ack_len = 2989400432
        branch = 0
        reply_status = -881216520
        onreply_route = 0
        cancel_data = {cancel_bitmap = 0, reason = {cause = 200, u = {text = {s = 0x0, len = 11245581}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, 
                len = 11245581}}}}
        uac = 0x7f0896cc17c0
        t = 0x7f0896cc1530
        lack_dst = {send_sock = 0xab9917 <buf.7136+791>, to = {s = {sa_family = 38925, sa_data = "\253\000\000\000\000\000p\241.\262\375\177\000"}, 
            sin = {sin_family = 38925, sin_port = 171, sin_addr = {s_addr = 0}, sin_zero = "p\241.\262\375\177\000"}, sin6 = {sin6_family = 38925, 
              sin6_port = 171, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = "p\241.\262\375\177\000\000\001\344o\000\000\000\000", 
                  __u6_addr16 = {41328, 45614, 32765, 0, 58369, 111, 0, 0}, __u6_addr32 = {2989400432, 32765, 7332865, 0}}}, sin6_scope_id = 0}, 
            sas = {ss_family = 38925, 
              __ss_padding = "\253\000\000\000\000\000p\241.\262\375\177\000\000\001\344o", '\000' <repeats 13 times>, "\370\263y\313\t\177\000\000\000)\205\000\000\000\000\000\375\227\253\000\207\000\000\000\324L\205\000\000\000\000\000(\325w\313\t\177\000\000\027\231\253\000\000\000\000\000\003\230\253\000\000\000\000\000\060\241.\262\375\177\000\000P\262y\313\t\177\000\000\027\231\253\000\000\000\000\000\020\226\253\000\000\000\000", 
              __ss_align = 139680045183032}}, id = 0, send_flags = {f = 0, blst_imask = 0}, proto = -123 '\205', proto_pad0 = -105 '\227', 
          proto_pad1 = 171}
        backup_user_from = 0x53c7a0 <register_module+4320>
        backup_user_to = 0x521ec0 <check_self_port+680>
        backup_domain_from = 0x3c6b40
        backup_domain_to = 0x7365ed <get_hdr_field+6416>
        backup_uri_from = 0x19
        backup_uri_to = 0x8
        backup_xavps = 0x7ffdb22ea320
        backup_xavus = 0x0
        backup_xavis = 0x0
        replies_locked = 1
        branch_ret = -881246328
        prev_branch = 0
        blst_503_timeout = 7579963
        hf = 0x852900
---Type <return> to continue, or q <return> to quit---
        onsend_params = {req = 0x7ffdb22ea170, rpl = 0x6c65c9 <qm_malloc+2479>, param = 0x852900, code = 7560264, flags = 0, branch = 0, 
          t_rbuf = 0x3c6298, dst = 0x3192, send_buf = {s = 0x41b980 <_start> "1\355I\211\321^H\211\342H\203\344\360PTI\307\300`\313|", 
            len = -1305562432}}
        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {12690, 139680045183568, 8726784, 7141200, 140727592853440, 
                139680040542224, 139680045061360, 139680040542224}, __mask_was_saved = -1305567232, __saved_mask = {__val = {139679877548472, 
                  11245445, 0, 524288000, 7315946, 3959664, 5381872, 5490592, 8, 139680045061456, 139680045061440, 140727592853552, 5923516, 0, 
                  21483449424, 0}}}}}
        bctx = 0x7f09cb832030
        keng = 0x0
        ret = 0
        evname = {s = 0x7f09c8354ffc "on_sl_reply", len = 11}
        __FUNCTION__ = "reply_received"
#5  0x0000000000526903 in do_forward_reply (msg=0x7f09cb832030, mode=0) at core/forward.c:757
        new_buf = 0x0
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {
                s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {
                __in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, 
              sin6_scope_id = 0}, sas = {ss_family = 0, __ss_padding = '\000' <repeats 117 times>, __ss_align = 0}}, id = 0, send_flags = {f = 0, 
            blst_imask = 0}, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0}
        new_len = 1
        r = 1
        ip = {af = 4307328, len = 0, u = {addrl = {12690, 4259919}, addr32 = {12690, 0, 4259919, 0}, addr16 = {12690, 0, 0, 0, 79, 65, 0, 0}, 
            addr = "\222\061\000\000\000\000\000\000O\000A\000\000\000\000"}}
        s = 0x7f09cb832860 ""
        len = 0
        __FUNCTION__ = "do_forward_reply"
#6  0x00000000005285d5 in forward_reply (msg=0x7f09cb832030) at core/forward.c:858
No locals.
#7  0x00000000005bc8c2 in receive_msg (
    buf=0xab9600 <buf.7136> "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 198.136.226.6:5060;received=10.50.8.6;branch=z9hG4bK56d5.462830a6", '0' <repeats 24 times>, ".0\r\nRecord-Route: <sip:198.136.226.1:5060;lr>\r\nContact: <sip:100.103.40.117:38"..., len=791, rcv_info=0x7ffdb22eaa40)
    at core/receive.c:509
        msg = 0x7f09cb832030
        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 0, jmp_env = {{__jmpbuf = {0, 0, 0, 0, 0, 0, 0, 0}, __mask_was_saved = 0, __saved_mask = {
                __val = {0 <repeats 13 times>, 12690, 139680578527568, 126}}}}}
---Type <return> to continue, or q <return> to quit---
        bctx = 0x0
        ret = -1
        tvb = {tv_sec = 0, tv_usec = 0}
        tve = {tv_sec = 0, tv_usec = 0}
        diff = 0
        inb = {
          s = 0xab9600 <buf.7136> "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 198.136.226.6:5060;received=10.50.8.6;branch=z9hG4bK56d5.462830a6", '0' <repeats 24 times>, ".0\r\nRecord-Route: <sip:198.136.226.1:5060;lr>\r\nContact: <sip:100.103.40.117:38"..., len = 791}
        netinfo = {data = {s = 0x0, len = 0}, rcv = 0x0, dst = 0x0}
        keng = 0x0
        evp = {data = 0x7ffdb22ea570, obuf = {s = 0x0, len = 0}, rcv = 0x7ffdb22eaa40, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0}
        cidlockidx = 0
        cidlockset = 0
        errsipmsg = 0
        exectime = 0
        __FUNCTION__ = "receive_msg"
#8  0x00000000004bb55c in udp_rcv_loop () at core/udp_server.c:543
        len = 791
        buf = "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 198.136.226.6:5060;received=10.50.8.6;branch=z9hG4bK56d5.462830a6", '0' <repeats 24 times>, ".0\r\nRecord-Route: <sip:198.136.226.1:5060;lr>\r\nContact: <sip:100.103.40.117:38"...
        tmp = 0x7f0888bbe520 ""
        fromaddr = 0x7f09cb79d880
        fromaddrlen = 16
        rcvi = {src_ip = {af = 2, len = 4, u = {addrl = {101200394, 0}, addr32 = {101200394, 0, 0, 0}, addr16 = {12810, 1544, 0, 0, 0, 0, 0, 0}, 
              addr = "\n2\b\006", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u = {addrl = {101200394, 0}, addr32 = {101200394, 0, 0, 
                0}, addr16 = {12810, 1544, 0, 0, 0, 0, 0, 0}, addr = "\n2\b\006", '\000' <repeats 11 times>}}, src_port = 9090, dst_port = 5060, 
          proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "#\202\n2\b\006\000\000\000\000\000\000\000"}, sin = {
              sin_family = 2, sin_port = 33315, sin_addr = {s_addr = 101200394}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {
              sin6_family = 2, sin6_port = 33315, sin6_flowinfo = 101200394, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, 
                  __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 2, 
              __ss_padding = "#\202\n2\b\006", '\000' <repeats 111 times>, __ss_align = 0}}, bind_address = 0x7f09cb3c2d98, proto = 1 '\001', 
          proto_pad0 = 0 '\000', proto_pad1 = 0}
        evp = {data = 0x0, obuf = {s = 0x0, len = 0}, rcv = 0x0, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0}
        printbuf = "\000\000\000\000\a\000\000\000\240\356=\313\t\177", '\000' <repeats 90 times>, "Tm\032\211\b\177\000\000\200\271A\000\000\000\000\000\300\262.\262\375\177", '\000' <repeats 18 times>, "\220\251.\262\375\177\000\000\006=T", '\000' <repeats 93 times>...
---Type <return> to continue, or q <return> to quit---
        i = -1
        j = 112640
        l = 1
        __FUNCTION__ = "udp_rcv_loop"
#9  0x0000000000429de7 in main_loop () at main.c:1683
        i = 6
        pid = 0
        si = 0x7f09cb3c2d98
        si_desc = "udp receiver child=6 sock=10.50.8.6:5060 (198.136.226.6:5060)\000\000\000\004\000\000\000\000\000\000\000\000@\270\210\b\177", '\000' <repeats 11 times>, "\235\032\211\b\177\000\000`\255.\262\375\177\000\000mCL\000\000\000\000\000\200\271A\000\000\000\000\000PH\201\313\t\177\000"
        nrprocs = 16
        woneinit = 1
        __FUNCTION__ = "main_loop"
#10 0x0000000000433b40 in main (argc=7, argv=0x7ffdb22eb2c8) at main.c:2859
        cfg_stream = 0x1cfb010
        c = -1
        r = 0
        tmp = 0x7ffdb22ebf26 ""
        tmp_len = 0
        port = 0
        proto = 0
        ahost = 0x0
        aport = 0
        options = 0x7cf8a0 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
        ret = -1
        seed = 1260081917
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x7f09ea8b2a00 <intel_02_known>
        p = 0x0
        st = {st_dev = 23, st_ino = 36410, st_nlink = 2, st_mode = 16877, st_uid = 0, st_gid = 5001, __pad0 = 0, st_rdev = 0, st_size = 40, 
          st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1599228627, tv_nsec = 605193865}, st_mtim = {tv_sec = 1599228597, 
            tv_nsec = 485081596}, st_ctim = {tv_sec = 1599228627, tv_nsec = 605193865}, __unused = {0, 0, 0}}
---Type <return> to continue, or q <return> to quit---
        tbuf = "\340\363s\352\t\177\000\000ȔB\353\t\177", '\000' <repeats 90 times>, "p\262\252\000\000\000\000\000\200\271A\000\000\000\000\000\300\262.\262\375\177", '\000' <repeats 26 times>, "\036\237\"\353\t\177\000\000\001", '\000' <repeats 23 times>, "\340\363s\352\t\177\000\000\220\262.\262\375\177\000\000*"...
        option_index = 0
        long_options = {{name = 0x7d1a8f "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7cd0a1 "version", has_arg = 0, flag = 0x0, 
            val = 118}, {name = 0x7d1a94 "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x7d1a9a "subst", has_arg = 1, flag = 0x0, 
            val = 1025}, {name = 0x7d1aa0 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7d1aa9 "substdefs", has_arg = 1, flag = 0x0, 
            val = 1027}, {name = 0x7d1ab3 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7d1abd "loadmodule", has_arg = 1, flag = 0x0, 
            val = 1029}, {name = 0x7d1ac8 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7d1ad1 "log-engine", has_arg = 1, flag = 0x0, 
            val = 1031}, {name = 0x7d1adc "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
        __FUNCTION__ = "main"

Thanks


You are receiving this because you are subscribed to this thread.
Reply to this email directly, view it on GitHub, or unsubscribe.