[SR-Users] crash when sending reply

Kamailio 5.5 crashed when sending reply (bt full below). Before the crash there had been several mysql related error messages ("Lost connection to MySQL server during query" and "Too many connections") in syslog when async worker tried to insert accounting data. So perhaps this core dump has something to do with that. -- Juha Reading symbols from /usr/bin/sip-proxy...done. [New LWP 21324] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/sip-proxy -f /etc/sip-proxy/sip-proxy.cfg -P /run/sip-proxy/sip-proxy.'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x00007fd274ce3b6e in run_trans_callbacks_internal (cb_lst=0x7fd2532503a8, type=512, trans=0x7fd253250330, params=0x7fff9b8eb2b0) at t_hooks.c:254 254 t_hooks.c: No such file or directory. (gdb) bt full #0 0x00007fd274ce3b6e in run_trans_callbacks_internal (cb_lst=0x7fd2532503a8, type=512, trans=0x7fd253250330, params=0x7fff9b8eb2b0) at t_hooks.c:254 cbp = 0x10 backup_from = 0x5613ccfb37f0 <def_list+16> backup_to = 0x5613ccfb37f8 <def_list+24> backup_dom_from = 0x5613ccfb3800 <def_list+32> backup_dom_to = 0x5613ccfb3808 <def_list+40> backup_uri_from = 0x5613ccfb37e0 <def_list> backup_uri_to = 0x5613ccfb37e8 <def_list+8> backup_xavps = 0x5613ccf56270 <_xavp_list_head> backup_xavus = 0x5613ccf56278 <_xavu_list_head> backup_xavis = 0x5613ccf56280 <_xavi_list_head> __func__ = "run_trans_callbacks_internal" #1 0x00007fd274ce41f7 in run_trans_callbacks_with_buf (type=512, rbuf=0x7fd253250400, req=0x7fd252d6dd20, repl=0x7fd2758bf0b0, flags=0) at t_hooks.c:303 params = {req = 0x7fd252d6dd20, rpl = 0x7fd2758bf0b0, param = 0x7fd254760f60, code = 486, flags = 0, branch = 0, t_rbuf = 0x7fd253250400, dst = 0x7fd253250450, send_buf = { s = 0x7fd25325b9f0 "@", len = 579}} trans = 0x7fd253250330 #2 0x00007fd274c88c15 in relay_reply (t=0x7fd253250330, p_msg=0x7fd2758bf0b0, branch=0, msg_status=486, cancel_data=0x7fff9b8eb580, do_put_on_wait=1) at t_reply.c:2133 relay = 0 save_clone = 0 buf = 0x7fd275ca0ff0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX.com;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"... res_len = 579 relayed_code = 486 relayed_msg = 0x7fd2758bf0b0 reply_bak = 0x7fd2510f1000 bm = {to_tag_val = {s = 0x7fd25216e4c0 "@\003", len = 1377234752}} totag_retr = 0 reply_status = RPS_COMPLETED uas_rb = 0x7fd253250400 to_tag = 0x7fd25216e820 reason = {s = 0x7fd25216e830 "0", len = 1394935248} onsend_params = {req = 0x9b8eb420, rpl = 0x7fd274c6b10d <futex_get+40>, param = 0x1, code = 1359984720, flags = 32722, branch = 0, t_rbuf = 0x7fff9b8eb460, dst = 0xa0679800, send_buf = { s = 0x19b8eb450 <error: Cannot access memory at address 0x19b8eb450>, len = 1359984720}} ip = {af = 2609820752, len = 32767, u = {addrl = {94643040895493, 0}, addr32 = {3436528133, 22035, 0, 0}, addr16 = {16901, 52437, 22035, 0, 0, 0, 0, 0}, addr = "\005B\325\314\023V\000\000\000\000\000\000\000\000\000"}} __func__ = "relay_reply" #3 0x00007fd274c8de6b in reply_received (p_msg=0x7fd2758bf0b0) at t_reply.c:2680 msg_status = 486 last_uac_status = 180 ack = 0x7fd25216e830 "0" ack_len = 459 branch = 0 reply_status = 32722 onreply_route = 2 cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 1977055608}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 1977055608}}}} uac = 0x7fd2532505c0 t = 0x7fd253250330 lack_dst = {send_sock = 0x40, to = {s = {sa_family = 49168, sa_data = "\207u\322\177\000\000\220\266\216\233\377\177\000"}, sin = {sin_family = 49168, sin_port = 30087, sin_addr = { s_addr = 32722}, sin_zero = "\220\266\216\233\377\177\000"}, sin6 = {sin6_family = 49168, sin6_port = 30087, sin6_flowinfo = 32722, sin6_addr = {__in6_u = { __u6_addr8 = "\220\266\216\233\377\177\000\000L=\215s\322\177\000", __u6_addr16 = {46736, 39822, 32767, 0, 15692, 29581, 32722, 0}, __u6_addr32 = {2609821328, 32767, 1938636108, 32722}}}, sin6_scope_id = 1971831024}, sas = {ss_family = 49168, __ss_padding = "\207u\322\177\000\000\220\266\216\233\377\177\000\000L=\215s\322\177\000\000\360\300\207u\322\177", '\000' <repeats 13 times>, "\b\000\000\000\000\060Z\264u\322\177\000\000\213\t\342\316\023V\000\000;\b\342\316\023V\000\000\020\313\320\000\000\000\000\000\020\000\000\000\000\000\000\000I\001\000\000\000\000\000\000\000\230g\240\210\347A\357\300\266\216\233\377\177\000\000\000\230g\240\210\347", <incomplete sequence \357>, __ss_align = 0}}, id = 0, send_flags = {f = 0, blst_imask = 0}, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0} backup_user_from = 0x5613ccfb37f0 <def_list+16> backup_user_to = 0x5613ccfb37f8 <def_list+24> backup_domain_from = 0x5613ccfb3800 <def_list+32> backup_domain_to = 0x5613ccfb3808 <def_list+40> backup_uri_from = 0x5613ccfb37e0 <def_list> backup_uri_to = 0x5613ccfb37e8 <def_list+8> backup_xavps = 0x5613ccf56270 <_xavp_list_head> backup_xavus = 0x5613ccf56278 <_xavu_list_head> backup_xavis = 0x5613ccf56280 <_xavi_list_head> replies_locked = 1 branch_ret = 0 prev_branch = 1971830800 blst_503_timeout = 32722 hf = 0xb5c750 onsend_params = {req = 0x7fff9b8eb5e0, rpl = 0xef41e788a0679800, param = 0x30, code = 0, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x7fd27d972cb0 <__syslog>, send_buf = { s = 0x5613ccec69b3 "INFO", len = -1603823616}} ctx = {rec_lev = 0, run_flags = 2, last_retcode = -1, jmp_env = {{__jmpbuf = {0, -5133089567146157739, 0, 140542026919088, 94643042412979, 536870912, -5133089567232140971, -1505551737941960363}, __mask_was_saved = 0, __saved_mask = {__val = {140735803210560, 13683472, 7765887371, 140735803209568, 94643041089583, 0, 17240315422543681536, 0, 0, 0, 0, 94643042412979, 17240315422543681536, 140735803209872, 94643041372964, 2866715017336}}}}} bctx = 0x7fd27587c010 keng = 0x0 ret = 0 evname = {s = 0x7fd274d3696f "on_sl_reply", len = 11} __func__ = "reply_received" #4 0x00005613ccbba0d4 in do_forward_reply (msg=0x7fd2758bf0b0, mode=0) at core/forward.c:764 new_buf = 0x0 dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 0, __ss_padding = '\000' <repeats 117 times>, __ss_align = 0}}, id = 0, send_flags = {f = 0, blst_imask = 0}, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0} new_len = 0 r = 2 ip = {af = 1, len = 6356993, u = {addrl = {140735803210560, 140541891965104}, addr32 = {2609822528, 32767, 1972105392, 32722}, addr16 = {47936, 39822, 32767, 0, 61616, 30091, 32722, 0}, addr = "@\273\216\233\377\177\000\000\260\360\213u\322\177\000"}} s = 0xef41e788a0679800 <error: Cannot access memory at address 0xef41e788a0679800> len = -1685145520 __func__ = "do_forward_reply" #5 0x00005613ccbbc23c in forward_reply (msg=0x7fd2758bf0b0) at core/forward.c:865 No locals. #6 0x00005613ccc5eea9 in receive_msg ( buf=0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"..., len=667, rcv_info=0x7fd253b7cad8) at core/receive.c:587 msg = 0x7fd2758bf0b0 ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {0, -5133089566915471019, 0, 140542026919088, 94643042412979, 536870912, -5133089567001454251, -1505551737941960363}, __mask_was_saved = 0, __saved_mask = {__val = {94643073835016, 65535, 17240315422543681536, 18446744073709551536, 2, 0, 140542026919088, 94643042412979, 536870912, 140735803211024, 94643040635195, 140735803211696, 65535, 65536, 4095, 140735803211580}}}}} bctx = 0x0 ret = 1 tvb = {tv_sec = 0, tv_usec = 0} tve = {tv_sec = 0, tv_usec = 0} diff = 0 inb = {s = 0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"..., len = 667} netinfo = {data = {s = 0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX;tag=as6964f29d\r\nCal"..., len = 667}, rcv = 0x7fd253b7cad8, dst = 0x0} keng = 0x0 evp = {data = 0x7fff9b8eba90, obuf = {s = 0x0, len = 0}, rcv = 0x7fd253b7cad8, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0} cidlockidx = 0 cidlockset = 0 errsipmsg = 0 exectime = 0 __func__ = "receive_msg" #7 0x00005613ccd1cf15 in receive_tcp_msg (tcpbuf=0x7fd253b7ce60 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"..., len=667, rcv_info=0x7fd253b7cad8, con=0x7fd253b7cac0) at core/tcp_read.c:1424 buf = 0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"... bsize = 65535 blen = 65535 __func__ = "receive_tcp_msg" #8 0x00005613ccd1f554 in tcp_read_req (con=0x7fd253b7cac0, bytes_read=0x7fff9b8ebf34, read_flags=0x7fff9b8ebf3c) at core/tcp_read.c:1607 bytes = 667 total_bytes = 667 resp = 1 size = 0 req = 0x7fd253b7cbe8 dst = {send_sock = 0x7fff9b8ebec0, to = {s = {sa_family = 32062, sa_data = "\303\314\023V\000\000p\276\216\233\377\177\000"}, sin = {sin_family = 32062, sin_port = 52419, sin_addr = {s_addr = 22035}, sin_zero = "p\276\216\233\377\177\000"}, sin6 = {sin6_family = 32062, sin6_port = 52419, sin6_flowinfo = 22035, sin6_addr = {__in6_u = {__u6_addr8 = "p\276\216\233\377\177\000\000\030\000\000\000\000\000\000", __u6_addr16 = {48752, 39822, 32767, 0, 24, 0, 0, 0}, __u6_addr32 = {2609823344, 32767, 24, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 32062, __ss_padding = "\303\314\023V\000\000p\276\216\233\377\177\000\000\030", '\000' <repeats 15 times>, "\225q\\\037P\000\000\000\070,\214u\322\177\000\000\000\244\374\314\023V\000\000\000\000\000\000\001 \000\000\070,\214u\322\177\000\000\001\000\000\000\000\000\000\000\300ʷS\322\177\000\000\200\276\216\233\377\177\000\000\070,\214u\322\177\000\000\200\277\216\233\377\177\000\000\b\000\000\000\000\000\000", __ss_align = 20}}, id = 1, send_flags = {f = 1, blst_imask = 0}, proto = 31 '\037', proto_pad0 = 0 '\000', proto_pad1 = 0} c = 49 '1' ret = 0 __func__ = "tcp_read_req" #9 0x00005613ccd23eb4 in handle_io (fm=0x7fd2758c2c38, events=1, idx=-1) at core/tcp_read.c:1857 ret = 0 n = 1404553920 read_flags = RD_CONN_SHORT_READ con = 0x7fd253b7cac0 s = 1404554328 resp = 1 t = 526152085 ee = 0x0 __func__ = "handle_io" #10 0x00005613ccd0dd1e in io_wait_loop_epoll (h=0x5613ccfca240 <io_w>, t=2, repeat=0) at core/io_wait.h:1070 n = 1 r = 0 fm = 0x7fd2758c2c38 revents = 1 __func__ = "io_wait_loop_epoll" #11 0x00005613ccd25981 in tcp_receive_loop (unix_sock=39) at core/tcp_read.c:1978 __func__ = "tcp_receive_loop" #12 0x00005613ccb8561b in tcp_init_children (woneinit=0x7fff9b8ec338) at core/tcp_main.c:5139 r = 2 i = 7 reader_fd_1 = 39 pid = 0 si_desc = "tcp receiver (generic)\000\000\205\220\274\314\023V\000\000\020Î\233\377\177\000\000\301\200\313\314\000\000\000\000AS\000\000\000\000\000\000۸\352\314\023V\000\000\246\000\216\233\377\177\000\000\034\352\340\314\023V\000\000\340\302\216\233\001\000\000\000\360\311\017Q\322\177\000\000\340\302\216\233\377\177\000\000\340\227:S\001\000\000\000\000Î\233\377\177\000\000\274\353\340\314\023V\000" si = 0x0 __func__ = "tcp_init_children" #13 0x00005613cca62bba in main_loop () at main.c:1857 i = 8 pid = 21313 si = 0x0 si_desc = "udp receiver child=7 sock=XXX.XXX.XXX.XXX:5060\000\000\000\003\000\000\000 \000\000\000\000\230g\240\210\347A\357\000\000\000\000\000\000\000\000\210P\347\314\023V\000\000\000\000\000\000\000\000\000\000\260,\227}\322\177\000\000\263i\354\314\023V\000\000\000\000\000 \000\000\000\000\260Ď\233\377\177\000\000\t\205\317\314\023V\000" nrprocs = 8 woneinit = 1 __func__ = "main_loop" #14 0x00005613cca6d4aa in main (argc=17, argv=0x7fff9b8ec9f8) at main.c:3053 cfg_stream = 0x5613cecb6290 c = -1 r = 0 tmp = 0x7fff9b8ecf3f "" tmp_len = 2109797928 port = 32722 proto = 2109962736 ahost = 0x0 aport = 0 options = 0x5613cce77ce0 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 3769828628 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x0 p = 0x0 st = {st_dev = 21, st_ino = 17087, st_nlink = 2, st_mode = 16832, st_uid = 109, st_gid = 115, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1620845793, tv_nsec = 887985330}, st_mtim = {tv_sec = 1625090397, tv_nsec = 157181302}, st_ctim = {tv_sec = 1625090401, tv_nsec = 197141517}, __glibc_reserved = {0, 0, 0}} tbuf = "\000\000\000\000\000\000\000\000\b\000\000\000\000\000\000\000\060\207\303}\322\177\000\000\000\000\000\000\000\000\000\000\240q\237\233\377\177\000\000\364\212\301}\322\177\000\000\b\000\000\000\000\000\000\000\230r\237\233\377\177\000\000\060\207\303}\322\177\000\000\230ǎ\233\377\177\000\000\224ǎ\233\377\177", '\000' <repeats 18 times>, "\230r\237\233\377\177\000\000\250q\237\233\377\177\000\000'\376\237}\322\177\000\000&\260be\000\000\000\000\000Ɏ\233\377\177\000\000\300\212\225\001\000\000\000\000`Ȏ\233\377\177\000\000PȎ\233\377\177\000\000\230ǎ\233\377\177\000\000X\207\303}\322\177\000\000\001", '\000' <repeats 31 times>... option_index = 0 long_options = {{name = 0x5613cce79f3f "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x5613cce754e0 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x5613cce79f44 "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x5613cce79f4a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x5613cce79f50 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x5613cce79f59 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x5613cce79f63 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x5613cce79f6d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x5613cce79f78 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x5613cce79f81 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x5613cce79f8c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x5613cce79f92 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x5613cce79f9c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __func__ = "main"