Here is core dump info:

[Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/sbin/kamailio --atexit=no -DD -P /run/kamailio/kamailio.pid -f /etc/kamail'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f3245d49acf in raise () from /lib64/libc.so.6 Missing separate debuginfos, use: yum debuginfo-install glibc-2.28-225.el8_8.6.x86_64 libblkid-2.32.1-42.el8_8.x86_64 libgcc-8.5.0-18.el8.x86_64 libmount-2.32.1-42.el8_8.x86_64 libselinux-2.9-8.el8.x86_64 libxcrypt-4.1.1-6.el8.x86_64 mariadb-connector-c-3.1.11-2.el8_3.x86_64 openssl-libs-1.1.1k-9.el8_7.x86_64 pcre2-10.32-3.el8_6.x86_64 perl-libs-5.26.3-422.el8.x86_64 systemd-libs-239-74.el8_8.5.x86_64 zlib-1.2.11-21.el8_7.x86_64 (gdb) bt full #0 0x00007f3245d49acf in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f3245d1cea5 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x0000000000722804 in qm_debug_check_frag () No symbol table info available. #3 0x000000000072648a in qm_free () No symbol table info available. #4 0x0000000000731bc9 in qm_shm_free () No symbol table info available. #5 0x000000000043d719 in xavp_free () No symbol table info available. #6 0x000000000043ee7b in xavp_destroy_list () No symbol table info available. #7 0x000000000043eef2 in xavp_reset_list () No symbol table info available. #8 0x0000000000616156 in ksr_msg_env_reset () No symbol table info available. #9 0x0000000000615a66 in receive_msg () No symbol table info available. #10 0x00000000004d6ef0 in udp_rcv_loop () No symbol table info available. #11 0x000000000042d844 in main_loop () No symbol table info available. #12 0x0000000000439ba7 in main () No symbol table info available. (gdb) info locals No symbol table info available. (gdb) list 1 dl-call-libc-early-init.c: No such file or directory. (gdb)

Here is the GDB analyse with kamailio-debuginfo package added:

(gdb) bt full #0 0x00007f3245d49acf in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f3245d1cea5 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x0000000000722804 in qm_debug_check_frag (qm=0x7f323c77c000, f=0x7f323ca16560, file=0x846305 "", line=81, efile=0x8b4ca9 "", eline=546) at core/mem/q_malloc.c:143 p = 0x8b5d48 <__func__.6986> __func__ = '\000' <repeats 19 times> #3 0x000000000072648a in qm_free (qmp=0x7f323c77c000, p=0x7f323ca165a0, file=0x846305 "", func=0x848348 <__func__.4658> "", line=81, mname=0x846300 "") at core/mem/q_malloc.c:546 qm = 0x7f323c77c000 f = 0x7f323ca16560 size = 140728364251424 next = 0x0 prev = 0x43ee05 <xavp_destroy_list+717> __func__ = "\000\000\000\000\000\000\000" #4 0x0000000000731bc9 in qm_shm_free (qmp=0x7f323c77c000, p=0x7f323ca165a0, file=0x846305 "", func=0x848348 <__func__.4658> "", line=81, mname=0x846300 "") at core/mem/q_malloc.c:1532 No locals. #5 0x000000000043d719 in xavp_free (xa=0x7f323ca165a0) at core/xavp.c:81 __func__ = "\000\000\000\000\000\000\000\000\000" #6 0x000000000043ee7b in xavp_destroy_list (head=0xb3bb40 <_xavp_list_head>) at core/xavp.c:635 avp = 0x7f32439e8f40 foo = 0x7f323ca165a0 __func__ = '\000' <repeats 17 times> #7 0x000000000043eef2 in xavp_reset_list () at core/xavp.c:647 __PRETTY_FUNCTION__ = '\000' <repeats 15 times> #8 0x0000000000616156 in ksr_msg_env_reset () at core/receive.c:669 No locals. #9 0x0000000000615a66 in receive_msg ( buf=0xb4cf60 <buf> "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.227.83.49;branch=z9hG4bKf6ff.6818bd21", '0' <repeats 24 times>, ".0\r\nTo: sip:cs1-tool-misc.orange-voicemail.net:5060;tag=4151cbdcbc42485256cba2778b769caa.3f3d\r\nFrom: "..., len=399, rcv_info=0x7ffde0293ff0) at core/receive.c:634 msg = 0x7f3245691cc0 ctx = {rec_lev = -534168368, run_flags = 32765, last_retcode = 7532591, jmp_env = {{__jmpbuf = {-1, 0, 139853889502976, 139853885763600, 140728898420735, 139853861857172, 120259084312, 139853900247200}, __mask_was_saved = 0, __saved_mask = {__val = {7590947731968763392, 3076992, 0, 140728364253504, 139853897549040, 9011416, 8762712, 399, 139853897549200, 206158430256, 140728364252472, 140728364252224, 7590947731968763392, 139853897549040, 9011411, 9011416}}}}} bctx = 0x0 ret = 0 tvb = {tv_sec = 1733506143, tv_usec = 409012} tve = {tv_sec = 1733506143, tv_usec = 409233} diff = 221 inb = { s = 0xb4cf60 <buf> "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.227.83.49;branch=z9hG4bKf6ff.6818bd21", '0' <repeats 24 times>, ".0\r\nTo: sip:cs1-tool-misc.orange-voicemail.net:5060;tag=4151cbdcbc42485256cba2778b769caa.3f3d\r\nFrom: "..., len = 399} netinfo = {data = {s = 0x0, len = 0}, bufsize = 0, rcv = 0x0, dst = 0x0} keng = 0x0 evp = {data = 0x7ffde0293b50, obuf = {s = 0x0, len = 0}, rcv = 0x7ffde0293ff0, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0} cidlockidx = 0 cidlockset = 0 errsipmsg = 0 exectime = 1 __func__ = '\000' <repeats 11 times> #10 0x00000000004d6ef0 in udp_rcv_loop () at core/udp_server.c:760 len = 399 buf = "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.227.83.49;branch=z9hG4bKf6ff.6818bd21", '0' <repeats 24 times>, ".0\r\nTo: sip:cs1-tool-misc.orange-voicemail.net:5060;tag=4151cbdcbc42485256cba2778b769caa.3f3d\r\nFrom: "... tmp = 0xffffffff3c7be010 <error: Cannot access memory at address 0xffffffff3c7be010> fromaddr = 0x7f3245677e50 --Type <RET> for more, q to quit, c to continue without paging-- fromaddrlen = 16 rcvi = {src_ip = {af = 2, len = 4, u = {addrl = {156492554, 0}, addr32 = {156492554, 0, 0, 0}, addr16 = {58122, 2387, 0, 0, 0, 0, 0, 0}, addr = "\n\343S\t", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u = {addrl = {827581194, 0}, addr32 = {827581194, 0, 0, 0}, addr16 = {58122, 12627, 0, 0, 0, 0, 0, 0}, addr = "\n\343S1", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "\023\304\n\343S\t\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 156492554}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 156492554, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 2, __ss_padding = "\023\304\n\343S\t", '\000' <repeats 111 times>, __ss_align = 0}}, bind_address = 0x7f3245471d40, rflags = (unknown: 0), proto = 1 '\001', proto_pad0 = 0 '\000', proto_pad1 = 0} evp = {data = 0x0, obuf = {s = 0x0, len = 0}, rcv = 0x0, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0} printbuf = "SIP/2.0 200 OK 0D 0A Via: SIP/2.0/UDP 10.227.83.49;branch=z9hG4bKf6ff.6818bd21", '0' <repeats 24 times>, ".0 0D \000\000\000e͇\000\000\000\000\000\a\000\000\000\000\000\000\000\360>)\340\375\177\000\000l\335Z", '\000' <repeats 13 times>, "\300F\206\000\000\000\000\000\000\000\000\000\024\000\000\000\031D\205\000\000\000\000\000\340M4\001\001\000\000\000-D\205", '\000' <repeats 13 times>, "\360"... i = 100 j = 109 l = 4 __func__ = '\000' <repeats 12 times> #11 0x000000000042d844 in main_loop () at main.c:1815 i = 0 pid = 0 si = 0x7f3245471d40 si_desc = "udp receiver child=0 sock=10.227.83.49:5060\000\062\177\000\000\020\000\000\000\003\000\000\000xKHE2\177\000\000pG)\340\375\177\000\000\020)gE2\177\000\000Ӏ\211\000\000\000\000\000\250\362\203\000\000\000\000\000\000\000\000\004\000\000\000\000\220E\342E2\177\000\000\060\000\000\000\060\000\000\000(C)\340\375\177\000" nrprocs = 4 woneinit = 0 __func__ = "\000\000\000\000\000\000\000\000\000" #12 0x0000000000439ba7 in main (argc=11, argv=0x7ffde0294848) at main.c:3256 cfg_stream = 0x10a22a0 c = -1 r = 0 tmp = 0x7ffde0294ed5 "" tmp_len = 32562 port = 5060 proto = 0 aproto = 0 ahost = 0x0 aport = 0 options = 0x842608 "" ret = -1 seed = 3917384781 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 2 n_lst = 0x7f3246cae7c0 p = 0x7f3245e645e8 <do_sym+424> "" st = {st_dev = 24, st_ino = 22591, st_nlink = 2, st_mode = 16832, st_uid = 248, st_gid = 248, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = { tv_sec = 1733503573, tv_nsec = 707823169}, st_mtim = {tv_sec = 1733506027, tv_nsec = 976781619}, st_ctim = {tv_sec = 1733506027, tv_nsec = 976781619}, __glibc_reserved = {0, 0, 0}} l1 = 64 tbuf = "@\357\247F2\177\000\000\000\000\000\000\000\000\000\000\370\066\312F2\177\000\000@\357\247F2\177\000\000`\347\247F2\177\000\000x\345\312F2\177\000\000`D)\340\375\177\000\000\000\340\247F2\177\000\000\274\351\247F2\177\000\000\223ΨF2\177\000\000\350\337\317E2\177\000\000\254\202\226\006\000\000\000\000\036\060\252F2\177\000\000\254\202\226\006\000\000\000\000\205\317c\t\000\000\000\000YϨF2\177\000\000\020\060\252F2\177\000\000\354\312\317E2\177\000\000\000\000\000\000\000\000\000\000д\312F2\177\000\000\001\000\000\000\000\000\000\000ve6h\342\004\000\000\020\342\312F2\177\000\000\200B\312F2\177\000\000\020\342\312F2\177\000\000\320K="... option_index = 12 long_options = {{name = 0x844a46 "", has_arg = 0, flag = 0x0, val = 104}, {name = 0x83f754 "", has_arg = 0, flag = 0x0, val = 118}, {name = 0x844a4b "", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x844a51 "", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x844a57 "", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x844a60 "", has_arg = 1, flag = 0x0, --Type <RET> for more, q to quit, c to continue without paging-- val = 1027}, {name = 0x844a6a "", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x844a74 "", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x844a7f "", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x844a88 "", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x844a93 "", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x844a99 "", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x844aa3 "", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x844aaa "", has_arg = 0, flag = 0x0, val = 1035}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __func__ = "\000\000\000\000"

(gdb) bt #0 0x00007f3245d49acf in raise () from /lib64/libc.so.6 #1 0x00007f3245d1cea5 in abort () from /lib64/libc.so.6 #2 0x0000000000722804 in qm_debug_check_frag (qm=0x7f323c77c000, f=0x7f323ca16560, file=0x846305 "", line=81, efile=0x8b4ca9 "", eline=546) at core/mem/q_malloc.c:143 #3 0x000000000072648a in qm_free (qmp=0x7f323c77c000, p=0x7f323ca165a0, file=0x846305 "", func=0x848348 <__func__.4658> "", line=81, mname=0x846300 "") at core/mem/q_malloc.c:546 #4 0x0000000000731bc9 in qm_shm_free (qmp=0x7f323c77c000, p=0x7f323ca165a0, file=0x846305 "", func=0x848348 <__func__.4658> "", line=81, mname=0x846300 "") at core/mem/q_malloc.c:1532 #5 0x000000000043d719 in xavp_free (xa=0x7f323ca165a0) at core/xavp.c:81 #6 0x000000000043ee7b in xavp_destroy_list (head=0xb3bb40 <_xavp_list_head>) at core/xavp.c:635 #7 0x000000000043eef2 in xavp_reset_list () at core/xavp.c:647 #8 0x0000000000616156 in ksr_msg_env_reset () at core/receive.c:669 #9 0x0000000000615a66 in receive_msg ( buf=0xb4cf60 <buf> "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.227.83.49;branch=z9hG4bKf6ff.6818bd21", '0' <repeats 24 times>, ".0\r\nTo: sip:cs1-tool-misc.orange-voicemail.net:5060;tag=4151cbdcbc42485256cba2778b769caa.3f3d\r\nFrom: "..., len=399, rcv_info=0x7ffde0293ff0) at core/receive.c:634 #10 0x00000000004d6ef0 in udp_rcv_loop () at core/udp_server.c:760 #11 0x000000000042d844 in main_loop () at main.c:1815 #12 0x0000000000439ba7 in main (argc=11, argv=0x7ffde0294848) at main.c:3256

(gdb) list 2066 & now if we don't need it */ 2067 #ifdef USE_SLOW_TIMER 2068 + 1 /* slow timer process */ 2069 #endif 2070 #ifdef USE_TCP 2071 + ((!tcp_disable) ? (1 /* tcp main */ + tcp_listeners) : 0) 2072 #endif 2073 #ifdef USE_SCTP 2074 + ((!sctp_disable) ? sctp_listeners : 0) 2075 #endif

No private module. The current kamailio.cfg is running under rhel 7/kamailio 5.0 with million requests per day and without any errors. It seems that if migration process (=> rhel 8 / kamailio 5.8) fails it is related to dispatcher module changes (attrs_avp, dst_avp...no longer available). I am not able to respond to the question "Do you process the SIP OPTIONS keepalives via event routes or the replies via reply routes in any way?"

Here is my dispatcher.list file: # # dispatcher destination sets # # gateways

############# Call Server

99 sip:cs1-tool-misc.voicemail.net:5060 0 0

############# Fixed sfl

1 sip:frnappcoe3sfl05.voicemail.net:5060 0 0

############# Sip tool

##### 2 sip:vm-tool-misc.voicemail.net:5060 0 0

dispatcher module parameters modparam("dispatcher", "list_file", "/etc/kamailio/dispatcher.list") modparam("dispatcher", "ds_probing_mode", 1) modparam("dispatcher", "ds_probing_threshold",3) modparam("dispatcher", "flags", 3) #modparam("dispatcher", "attrs_avp", "$avp(dsattrs)") #modparam("dispatcher", "dst_avp", "$avp(dsdst)") #modparam("dispatcher", "grp_avp", "$avp(dsgrp)") #modparam("dispatcher", "cnt_avp", "$avp(dscnt)") modparam("dispatcher", "ds_hash_size", 8) modparam("dispatcher", "ds_ping_interval", 10) modparam("dispatcher", "ds_ping_reply_codes", "class=2;class=3") #modparam("dispatcher", "sock_avp", "$avp(dssocket)")

Some lines commented out for 5.8 version.

I will try to build a simplified kamailio.cfg to reproduce the issue.

Do you have tls enabled? If yes, be sure that you set `tls_threads_mode = 2`, see more:

- https://www.kamailio.org/docs/modules/devel/modules/tls.html#tls.notes - https://www.kamailio.org/wikidocs/cookbooks/devel/core/#tls_threads_mode

github-actions[bot] left a comment (kamailio/kamailio#4061)

This issue is stale because it has been open 6 weeks with no activity. Remove stale label or comment or this will be closed in 2 weeks.