Hello Again,
I was able to reproduce again. It crashed from the same point and it seems that frequency of this issue has reduced but it is not fixed yet. As per my observation, it happens when entry of destination from KA list is removed during an ongoing KA transaction. I am removing destination from KA list when endpoint de registers so this case is possible. I have also observed same on pcaps. Below are the screenshots :
OPTIONS : 
REGISTER : 
Kamailio has dispatched 200 OK for register at 06:33:58.187520 and it received 200 OK for OPTIONS at 06:33:58.411501. Kamailio crashed exactly at a same time.
Core Dump : ``` GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-110.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: http://www.gnu.org/software/gdb/bugs/... Reading symbols from /usr/local/sbin/kamailio...done. [New LWP 12701] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/local/sbin/kamailio -m 5000 -M 500 -P /run/kamailio/kamailio.pid'. Program terminated with signal 11, Segmentation fault. #0 0x00007f09c138dc7c in ka_options_callback (t=0x7f0896cc1530, type=1024, ps=0x7ffdb22e9d20) at keepalive_core.c:126 126 ka_dest->statechanged_clb(&ka_dest->uri, state, ka_dest->user_attr); Missing separate debuginfos, use: debuginfo-install cyrus-sasl-lib-2.1.26-23.el7.x86_64 glibc-2.17-222.el7.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-19.el7.x86_64 libcom_err-1.42.9-12.el7_5.x86_64 libcurl-7.29.0-51.el7.x86_64 libevent-2.0.21-4.el7.x86_64 libgcc-4.8.5-36.el7_6.2.x86_64 libidn-1.28-4.el7.x86_64 libselinux-2.5-12.el7.x86_64 libssh2-1.4.3-10.el7_2.1.x86_64 libstdc++-4.8.5-36.el7_6.2.x86_64 libuuid-2.23.2-52.el7_5.1.x86_64 mariadb-libs-5.5.60-1.el7_5.x86_64 nspr-4.19.0-1.el7_5.x86_64 nss-3.36.0-5.el7_5.x86_64 nss-softokn-freebl-3.36.0-5.el7_5.x86_64 nss-util-3.36.0-1.el7_5.x86_64 openldap-2.4.44-15.el7_5.x86_64 openssl-libs-1.0.2k-16.el7_6.1.x86_64 pcre-8.32-17.el7.x86_64 zlib-1.2.7-18.el7.x86_64 (gdb) bt full #0 0x00007f09c138dc7c in ka_options_callback (t=0x7f0896cc1530, type=1024, ps=0x7ffdb22e9d20) at keepalive_core.c:126 uri = { s = 0x7f0896cc39b7 "sip:7459@10.50.8.6:9090;alias=10.50.8.1~5060~1;pb-ip=174.198.221.83;pb-pt=3939;tp=tcp>\r\nFrom: sip:keepalive@fromsbc;tag=ee3b67b160621ee354865fd8162b5b29-1b81a45f\r\nCSeq: 10 OPTIONS\r\nCall-ID: 2557bb53"..., len = 85} msg = 0x0 state = 1 state_routes = {0x7f09c1393844 "", 0x7f09c139393d "keepalive:dst-up", 0x7f09c139394e "keepalive:dst-down"} ka_dest = 0x7f088ac83048 __FUNCTION__ = "ka_options_callback" #1 0x00007f09c8321dd8 in run_trans_callbacks_internal (cb_lst=0x7f0896cc15a8, type=1024, trans=0x7f0896cc1530, params=0x7ffdb22e9d20) at t_hooks.c:258 cbp = 0x7f0896cc3b60 backup_from = 0xb1f170 <def_list+16> backup_to = 0xb1f178 <def_list+24> backup_dom_from = 0xb1f180 <def_list+32> backup_dom_to = 0xb1f188 <def_list+40> backup_uri_from = 0xb1f160 <def_list> backup_uri_to = 0xb1f168 <def_list+8> backup_xavps = 0xb1ea60 <_xavp_list_head> backup_xavus = 0xb1ea68 <_xavu_list_head> backup_xavis = 0xb1ea70 <_xavi_list_head> __FUNCTION__ = "run_trans_callbacks_internal" #2 0x00007f09c8321f0a in run_trans_callbacks (type=1024, trans=0x7f0896cc1530, req=0x0, rpl=0x7f09cb832030, code=200) at t_hooks.c:285 params = {req = 0x0, rpl = 0x7f09cb832030, param = 0x7f0896cc3b70, code = 200, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = { s = 0x0, len = 0}} #3 0x00007f09c82b2bef in local_reply (t=0x7f0896cc1530, p_msg=0x7f09cb832030, branch=0, msg_status=200, cancel_data=0x7ffdb22ea110) at t_reply.c:2265 local_store = 0 local_winner = 0 reply_status = RPS_COMPLETED winning_msg = 0x7f09cb832030 winning_code = 200 totag_retr = 0 __FUNCTION__ = "local_reply" #4 0x00007f09c82b5a8b in reply_received (p_msg=0x7f09cb832030) at t_reply.c:2648 ---Type <return> to continue, or q <return> to quit--- msg_status = 200 last_uac_status = 0 ack = 0x5a62bc <sr_event_exec+415> "\211E\374\213E\374\351)\003" ack_len = 2989400432 branch = 0 reply_status = -881216520 onreply_route = 0 cancel_data = {cancel_bitmap = 0, reason = {cause = 200, u = {text = {s = 0x0, len = 11245581}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 11245581}}}} uac = 0x7f0896cc17c0 t = 0x7f0896cc1530 lack_dst = {send_sock = 0xab9917 <buf.7136+791>, to = {s = {sa_family = 38925, sa_data = "\253\000\000\000\000\000p\241.\262\375\177\000"}, sin = {sin_family = 38925, sin_port = 171, sin_addr = {s_addr = 0}, sin_zero = "p\241.\262\375\177\000"}, sin6 = {sin6_family = 38925, sin6_port = 171, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = "p\241.\262\375\177\000\000\001\344o\000\000\000\000", __u6_addr16 = {41328, 45614, 32765, 0, 58369, 111, 0, 0}, __u6_addr32 = {2989400432, 32765, 7332865, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 38925, __ss_padding = "\253\000\000\000\000\000p\241.\262\375\177\000\000\001\344o", '\000' <repeats 13 times>, "\370\263y\313\t\177\000\000\000)\205\000\000\000\000\000\375\227\253\000\207\000\000\000\324L\205\000\000\000\000\000(\325w\313\t\177\000\000\027\231\253\000\000\000\000\000\003\230\253\000\000\000\000\000\060\241.\262\375\177\000\000P\262y\313\t\177\000\000\027\231\253\000\000\000\000\000\020\226\253\000\000\000\000", __ss_align = 139680045183032}}, id = 0, send_flags = {f = 0, blst_imask = 0}, proto = -123 '\205', proto_pad0 = -105 '\227', proto_pad1 = 171} backup_user_from = 0x53c7a0 <register_module+4320> backup_user_to = 0x521ec0 <check_self_port+680> backup_domain_from = 0x3c6b40 backup_domain_to = 0x7365ed <get_hdr_field+6416> backup_uri_from = 0x19 backup_uri_to = 0x8 backup_xavps = 0x7ffdb22ea320 backup_xavus = 0x0 backup_xavis = 0x0 replies_locked = 1 branch_ret = -881246328 prev_branch = 0 blst_503_timeout = 7579963 hf = 0x852900 ---Type <return> to continue, or q <return> to quit--- onsend_params = {req = 0x7ffdb22ea170, rpl = 0x6c65c9 <qm_malloc+2479>, param = 0x852900, code = 7560264, flags = 0, branch = 0, t_rbuf = 0x3c6298, dst = 0x3192, send_buf = {s = 0x41b980 <_start> "1\355I\211\321^H\211\342H\203\344\360PTI\307\300`\313|", len = -1305562432}} ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {12690, 139680045183568, 8726784, 7141200, 140727592853440, 139680040542224, 139680045061360, 139680040542224}, __mask_was_saved = -1305567232, __saved_mask = {__val = {139679877548472, 11245445, 0, 524288000, 7315946, 3959664, 5381872, 5490592, 8, 139680045061456, 139680045061440, 140727592853552, 5923516, 0, 21483449424, 0}}}}} bctx = 0x7f09cb832030 keng = 0x0 ret = 0 evname = {s = 0x7f09c8354ffc "on_sl_reply", len = 11} __FUNCTION__ = "reply_received" #5 0x0000000000526903 in do_forward_reply (msg=0x7f09cb832030, mode=0) at core/forward.c:757 new_buf = 0x0 dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = { s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = { __in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 0, __ss_padding = '\000' <repeats 117 times>, __ss_align = 0}}, id = 0, send_flags = {f = 0, blst_imask = 0}, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0} new_len = 1 r = 1 ip = {af = 4307328, len = 0, u = {addrl = {12690, 4259919}, addr32 = {12690, 0, 4259919, 0}, addr16 = {12690, 0, 0, 0, 79, 65, 0, 0}, addr = "\222\061\000\000\000\000\000\000O\000A\000\000\000\000"}} s = 0x7f09cb832860 "" len = 0 __FUNCTION__ = "do_forward_reply" #6 0x00000000005285d5 in forward_reply (msg=0x7f09cb832030) at core/forward.c:858 No locals. #7 0x00000000005bc8c2 in receive_msg ( buf=0xab9600 <buf.7136> "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 198.136.226.6:5060;received=10.50.8.6;branch=z9hG4bK56d5.462830a6", '0' <repeats 24 times>, ".0\r\nRecord-Route: sip:198.136.226.1:5060;lr\r\nContact: <sip:100.103.40.117:38"..., len=791, rcv_info=0x7ffdb22eaa40) at core/receive.c:509 msg = 0x7f09cb832030 ctx = {rec_lev = 0, run_flags = 0, last_retcode = 0, jmp_env = {{__jmpbuf = {0, 0, 0, 0, 0, 0, 0, 0}, __mask_was_saved = 0, __saved_mask = { __val = {0 <repeats 13 times>, 12690, 139680578527568, 126}}}}} ---Type <return> to continue, or q <return> to quit--- bctx = 0x0 ret = -1 tvb = {tv_sec = 0, tv_usec = 0} tve = {tv_sec = 0, tv_usec = 0} diff = 0 inb = { s = 0xab9600 <buf.7136> "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 198.136.226.6:5060;received=10.50.8.6;branch=z9hG4bK56d5.462830a6", '0' <repeats 24 times>, ".0\r\nRecord-Route: sip:198.136.226.1:5060;lr\r\nContact: <sip:100.103.40.117:38"..., len = 791} netinfo = {data = {s = 0x0, len = 0}, rcv = 0x0, dst = 0x0} keng = 0x0 evp = {data = 0x7ffdb22ea570, obuf = {s = 0x0, len = 0}, rcv = 0x7ffdb22eaa40, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0} cidlockidx = 0 cidlockset = 0 errsipmsg = 0 exectime = 0 __FUNCTION__ = "receive_msg" #8 0x00000000004bb55c in udp_rcv_loop () at core/udp_server.c:543 len = 791 buf = "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 198.136.226.6:5060;received=10.50.8.6;branch=z9hG4bK56d5.462830a6", '0' <repeats 24 times>, ".0\r\nRecord-Route: sip:198.136.226.1:5060;lr\r\nContact: <sip:100.103.40.117:38"... tmp = 0x7f0888bbe520 "" fromaddr = 0x7f09cb79d880 fromaddrlen = 16 rcvi = {src_ip = {af = 2, len = 4, u = {addrl = {101200394, 0}, addr32 = {101200394, 0, 0, 0}, addr16 = {12810, 1544, 0, 0, 0, 0, 0, 0}, addr = "\n2\b\006", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u = {addrl = {101200394, 0}, addr32 = {101200394, 0, 0, 0}, addr16 = {12810, 1544, 0, 0, 0, 0, 0, 0}, addr = "\n2\b\006", '\000' <repeats 11 times>}}, src_port = 9090, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "#\202\n2\b\006\000\000\000\000\000\000\000"}, sin = { sin_family = 2, sin_port = 33315, sin_addr = {s_addr = 101200394}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = { sin6_family = 2, sin6_port = 33315, sin6_flowinfo = 101200394, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 2, __ss_padding = "#\202\n2\b\006", '\000' <repeats 111 times>, __ss_align = 0}}, bind_address = 0x7f09cb3c2d98, proto = 1 '\001', proto_pad0 = 0 '\000', proto_pad1 = 0} evp = {data = 0x0, obuf = {s = 0x0, len = 0}, rcv = 0x0, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0} printbuf = "\000\000\000\000\a\000\000\000\240\356=\313\t\177", '\000' <repeats 90 times>, "Tm\032\211\b\177\000\000\200\271A\000\000\000\000\000\300\262.\262\375\177", '\000' <repeats 18 times>, "\220\251.\262\375\177\000\000\006=T", '\000' <repeats 93 times>... ---Type <return> to continue, or q <return> to quit--- i = -1 j = 112640 l = 1 __FUNCTION__ = "udp_rcv_loop" #9 0x0000000000429de7 in main_loop () at main.c:1683 i = 6 pid = 0 si = 0x7f09cb3c2d98 si_desc = "udp receiver child=6 sock=10.50.8.6:5060 (198.136.226.6:5060)\000\000\000\004\000\000\000\000\000\000\000\000@\270\210\b\177", '\000' <repeats 11 times>, "\235\032\211\b\177\000\000`\255.\262\375\177\000\000mCL\000\000\000\000\000\200\271A\000\000\000\000\000PH\201\313\t\177\000" nrprocs = 16 woneinit = 1 __FUNCTION__ = "main_loop" #10 0x0000000000433b40 in main (argc=7, argv=0x7ffdb22eb2c8) at main.c:2859 cfg_stream = 0x1cfb010 c = -1 r = 0 tmp = 0x7ffdb22ebf26 "" tmp_len = 0 port = 0 proto = 0 ahost = 0x0 aport = 0 options = 0x7cf8a0 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 1260081917 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x7f09ea8b2a00 <intel_02_known> p = 0x0 st = {st_dev = 23, st_ino = 36410, st_nlink = 2, st_mode = 16877, st_uid = 0, st_gid = 5001, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1599228627, tv_nsec = 605193865}, st_mtim = {tv_sec = 1599228597, tv_nsec = 485081596}, st_ctim = {tv_sec = 1599228627, tv_nsec = 605193865}, __unused = {0, 0, 0}} ---Type <return> to continue, or q <return> to quit--- tbuf = "\340\363s\352\t\177\000\000ȔB\353\t\177", '\000' <repeats 90 times>, "p\262\252\000\000\000\000\000\200\271A\000\000\000\000\000\300\262.\262\375\177", '\000' <repeats 26 times>, "\036\237"\353\t\177\000\000\001", '\000' <repeats 23 times>, "\340\363s\352\t\177\000\000\220\262.\262\375\177\000\000*"... option_index = 0 long_options = {{name = 0x7d1a8f "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7cd0a1 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7d1a94 "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x7d1a9a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7d1aa0 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7d1aa9 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7d1ab3 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7d1abd "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7d1ac8 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7d1ad1 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7d1adc "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" ```
Thanks