There was an unsafe list iteration - can you try with:
- https://github.com/kamailio/kamailio/commit/325a45e846faae3e5dfa333727d5fab2...
If all ok, you can backport.
Cheers, Daniel
On 04.03.20 22:18, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
If you can reproduce it, watch what the timer processes do during that time frame. Get the list of processes with 'kamctl ps', then when the issue is exposed, grab the backtraces of all processes with:
kamctl trap
A file is created with the backtraces taken with gdb. Sent it over here to analyze, if you cannot spot an issue there.
Will do that. Now it happened that K crashed when tm.clean was given at the time when the number transactions was growing. Below is backtrace of that crash.
-- Juha
(gdb) bt full #0 0x00007f5151fa3cf8 in tm_clean_lifetime () at h_table.c:617 r = 53487 tcell = 0x0 texp = 1117017528 #1 0x00007f5151fe1afd in tm_rpc_clean (rpc=0x7f51513b0a00 <func_param>, c=0x7f51513b0920 <_jsonrpc_ctx_global>) at t_stats.c:299 No locals. #2 0x00007f515138fc9f in jsonrpc_dispatch (msg=0x7fff3d1dc210, s1=0x0, s2=0x0) at jsonrpcs_mod.c:1294 rpce = 0x7f5152dcf360 ctx = 0x7f51513b0920 <_jsonrpc_ctx_global> ret = 0 nj = 0x0 val = {s = 0x557bb08fe9d0 "tm.clean", len = 8} __func__ = "jsonrpc_dispatch" #3 0x0000557bae5c28b5 in do_action (h=0x7fff3d1dc130, a=0x7f51537ebd30, msg=0x7fff3d1dc210) at core/action.c:1073 ret = -5 v = 1390957541 dst = {send_sock = 0xc00000, to = {s = {sa_family = 6, sa_data = "\000\000\000\000\000\000\220\270\035=\377\177\000"}, sin = {sin_family = 6, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\220\270\035=\377\177\000"}, sin6 = {sin6_family = 6, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = "\220\270\035=\377\177\000\000\000\b\302\tv\223\062\213", __u6_addr16 = {47248, 15645, 32767, 0, 2048, 2498, 37750, 35634}, __u6_addr32 = {1025357968, 32767, 163710976, 2335347574}}}, sin6_scope_id = 1025358088}}, id = 32767, send_flags = {f = 50272, blst_imask = 21212}, proto = 81 'Q', proto_pad0 = 127 '\177', proto_pad1 = 0} tmp = 0x7f5152ab3c1e "H\205\300\177\345ƃ\224" new_uri = 0x7f51537e9da8 "" end = 0x7fff3d1dc210 "\255\004" crt = 0x557bb08fc931 "ip_proxy_cache" cmd = 0x7f5152dd27c0 len = 4 user = 0 uri = {user = {s = 0x7fff3d1db580 "\340\265\035=\377\177", len = 1390199904}, passwd = {s = 0x557bae7f69e3 "INFO", len = -1368709420}, host = {s = 0x7fff3d1db5a0 "", len = 1389801488}, port = {s = 0x7fff3d1db640 "\320\006", len = 1389801488}, params = { s = 0x7fff3d1db5e0 "\020\266\035=\377\177", len = 1356175867}, sip_params = {s = 0x7fff3d1db7f0 "", len = 0}, headers = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 1260448}, port_no = 56128, proto = 159, type = ERROR_URI_T, flags = (unknown: 11322464), transport = {s = 0xb7fab0 <error: Cannot access memory at address 0xb7fab0>, len = 16}, ttl = {s = 0x11e <error: Cannot access memory at address 0x11e>, len = 163710976}, user_param = {s = 0x7fff3d1db610 "\220\266\035=\377\177", len = -1370752186}, maddr = {s = 0x0, len = 0}, method = {s = 0x7f5153158560 "\320\006", len = 0}, lr = {s = 0x7fff3d1db690 "\360\266\035=\377\177", len = -1368723422}, r2 = {s = 0x7f51531531f0 "\002", len = 1389801488}, gr = {s = 0x7f51537ea518 "\002", len = 1393897952}, transport_val = {s = 0x6d0 <error: Cannot access memory at address 0x6d0>, len = 1389801488}, ttl_val = {s = 0x7f51531530d0 <incomplete sequence \360>, len = 1393899728}, user_param_val = {s = 0x7f51531531f8 ">5\025SQ\177", len = 1390199904}, maddr_val = {s = 0x557bae7f69e3 "INFO", len = 536870912}, method_val = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 6}, lr_val = {s = 0x7fff3d1db6f0 "P\267\035=\377\177", len = -1371351967}, r2_val = {s = 0x7fff3d1dc210 "\255\004", len = 1400808728}, gr_val = {s = 0x7fff3d1db730 "\220\267\035=\377\177", len = 163710976}} next_hop = {user = {s = 0x0, len = 1390199904}, passwd = {s = 0x557bae7f69e3 "INFO", len = 536870912}, host = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 6}, port = {s = 0x7fff3d1db750 "", len = -1371324103}, params = { s = 0x7fff3d1db7f0 "", len = 1400806792}, sip_params = {s = 0x19000017ac <error: Cannot access memory at address 0x19000017ac>, len = 1400808728}, headers = {s = 0x7fff3d1dc210 "\255\004", len = -1369366375}, port_no = 46992, proto = 15645, type = 32767, flags = (unknown: 1390199904), transport = {s = 0x557bae7f69e3 "INFO", len = 536870912}, ttl = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 6}, user_param = {s = 0x7fff3d1db8a0 "\340\271\035=\377\177", len = -1371320250}, maddr = { s = 0x557bae7f69e3 "INFO", len = 536870912}, method = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 6}, lr = {s = 0x7fff3d1db7f0 "", len = 1387308127}, r2 = {s = 0x7f51537eac00 "\031", len = 1025358088}, gr = { s = 0x7fff3d1dc210 "\255\004", len = 1025360176}, transport_val = {s = 0x17ac3d1db7f0 <error: Cannot access memory at address 0x17ac3d1db7f0>, len = 6060}, ttl_val = {s = 0x557bae7f69e3 "INFO", len = 536870912}, user_param_val = {s = 0x7f51537e9d88 "\a", len = 1400808728}, maddr_val = {s = 0x0, len = 0}, method_val = {s = 0x0, len = 0}, lr_val = {s = 0x557bae95a842 <ut_buf_int2str+18> "976", len = 1398451376}, r2_val = {s = 0x7fff3d1db900 "\360\275\331R\001", len = 1390350000}, gr_val = {s = 0x0, len = 1390199904}} u = 0x7f5152abbc8d port = 21269 dst_host = 0x7fff3d1db7f8 i = 0 flags = 32593 avp = 0x7fff3d1db4f8 st = {flags = 1400806800, id = 32593, name = {n = 1025360400, s = {s = 0x7fff3d1dc210 "\255\004", len = 1025357344}, re = 0x7fff3d1dc210}, avp = 0x557b00000000} sct = 0x7f51511a819b <pv_get_huri+69> sjt = 0x7f5153164bb0 rve = 0x7fff3d1dc248 mct = 0x7fff3d1db7f8 rv = 0x7fff3d1db4d0 rv1 = 0x7f5152dcc4a0 c1 = {cache_type = 2927585763, val_type = 21883, c = {avp_val = {n = 536870912, s = {s = 0x20000000 <error: Cannot access memory at address 0x20000000>, len = 12582912}, re = 0x20000000}, pval = {rs = { s = 0x20000000 <error: Cannot access memory at address 0x20000000>, len = 12582912}, ri = 6, flags = 0}}, i2s = "\340\271\035=\377\177\000\000\242KC\256{U\000\000\000\000\300\000\000"} s = {s = 0x7f51537e9da8 "", len = 1025360400} srevp = {0x557bae7f69e3, 0x20000000} evp = {data = 0x0, rcv = 0x0, dst = 0x0} mod_f_params = {{type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, { type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = { number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}} __func__ = "do_action" #4 0x0000557bae5d0626 in run_actions (h=0x7fff3d1dc130, a=0x7f51537eba80, msg=0x7fff3d1dc210) at core/action.c:1578 t = 0x7f51537ebd30 ret = 1 tvb = {tv_sec = 0, tv_usec = 0} tve = {tv_sec = 0, tv_usec = 0} tz = {tz_minuteswest = 21, tz_dsttime = 0} tdiff = 2962221528 __func__ = "run_actions" #5 0x0000557bae5c2821 in do_action (h=0x7fff3d1dc130, a=0x7f51537ebf50, msg=0x7fff3d1dc210) at core/action.c:1064 --Type <RET> for more, q to quit, c to continue without paging-- ret = 1 v = 1 dst = {send_sock = 0x7f5153027c10, to = {s = {sa_family = 2048, sa_data = "\302\tv\223\062\213y\237\037QQ\177\000"}, sin = {sin_family = 2048, sin_port = 2498, sin_addr = {s_addr = 2335347574}, sin_zero = "y\237\037QQ\177\000"}, sin6 = {sin6_family = 2048, sin6_port = 2498, sin6_flowinfo = 2335347574, sin6_addr = {__in6_u = {__u6_addr8 = "y\237\037QQ\177\000\000\017\000\000\000Q\177\000", __u6_addr16 = {40825, 20767, 32593, 0, 15, 0, 32593, 0}, __u6_addr32 = {1361026937, 32593, 15, 32593}}}, sin6_scope_id = 2927585763}}, id = 4, send_flags = {f = 39616, blst_imask = 44675}, proto = 123 '{', proto_pad0 = 85 'U', proto_pad1 = 0} tmp = 0x500000000 <error: Cannot access memory at address 0x500000000> new_uri = 0x7fff3d1dc2b0 "" end = 0x7f5152d38eda "" crt = 0x0 cmd = 0x7f5153b25fa0 <_IO_strn_jumps> len = 21 user = 32593 uri = {user = {s = 0x4009e6720 <error: Cannot access memory at address 0x4009e6720>, len = -1367595136}, passwd = {s = 0x7fff3d1dbcc8 ""\027q\256{U", len = 163710976}, host = {s = 0x557bae95a830 <ut_buf_int2str> "", len = 0}, port = { s = 0xd68 <error: Cannot access memory at address 0xd68>, len = 17}, params = {s = 0x0, len = 1390199904}, sip_params = {s = 0x557bae7f69e3 "INFO", len = 536870912}, headers = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 1025359116}, port_no = 48352, proto = 15645, type = URN_URI_T, flags = (URI_USER_NORMALIZE | URI_SIP_USER_PHONE | unknown: 1390957584), transport = {s = 0x43d1dbd3c <error: Cannot access memory at address 0x43d1dbd3c>, len = -1412931264}, ttl = { s = 0x7f5152e85418 "\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 1390957592}, user_param = {s = 0x7fff3d1dbe50 "p\277\035=\377\177", len = -1368320222}, maddr = {s = 0x557bae95a842 <ut_buf_int2str+18> "976", len = 163710976}, method = {s = 0x7fff3d1dbd00 "P\275\035=\377\177", len = 1393988016}, lr = {s = 0x7f5152e8549d "", len = 1390957561}, r2 = {s = 0x7fff3d1dbd50 "\260\275\035=\377\177", len = 84243696}, gr = {s = 0x37500000000 <error: Cannot access memory at address 0x37500000000>, len = 1025359520}, transport_val = {s = 0x7f51537eccf8 "", len = -1368709420}, ttl_val = {s = 0x7fff3d1dbd70 "", len = 1389801488}, user_param_val = { s = 0x557bae95a842 <ut_buf_int2str+18> "976", len = 1389801488}, maddr_val = {s = 0x7fff3d1dbdb0 "\340\275\035=\377\177", len = 1356175867}, method_val = {s = 0x4ae7f69e3 <error: Cannot access memory at address 0x4ae7f69e3>, len = 0}, lr_val = { s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 1260448}, r2_val = {s = 0x9fdb40 <error: Cannot access memory at address 0x9fdb40>, len = 11322464}, gr_val = {s = 0xb7fab0 <error: Cannot access memory at address 0xb7fab0>, len = 16}} next_hop = {user = {s = 0x11e <error: Cannot access memory at address 0x11e>, len = 163710976}, passwd = {s = 0x7fff3d1dbde0 "P\276\035=\377\177", len = -1370752186}, host = {s = 0x0, len = 384}, port = {s = 0x7f51531692a0 <incomplete sequence \360>, len = 0}, params = {s = 0x7fff3d1dbe50 "p\277\035=\377\177", len = -1368726502}, sip_params = {s = 0xf0 <error: Cannot access memory at address 0xf0>, len = 1390957594}, headers = {s = 0x0, len = 1393988016}, port_no = 37280, proto = 21270, type = 32593, flags = (unknown: 163710976), transport = {s = 0xb7fab0 <error: Cannot access memory at address 0xb7fab0>, len = 1390199904}, ttl = {s = 0x557bae7f69e3 "INFO", len = 536870912}, user_param = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 6}, maddr = {s = 0x7fff3d1dbf70 "\020\300\035=\377\177", len = -1368302309}, method = {s = 0x7f51531d2600 "\006", len = 1389801488}, lr = {s = 0x7fff3d1dbed0 "", len = 1356175867}, r2 = {s = 0x10 <error: Cannot access memory at address 0x10>, len = 0}, gr = {s = 0xc00000 <error: Cannot access memory at address 0xc00000>, len = 1392671776}, transport_val = {s = 0x7f5152e8549d "", len = 1390957556}, ttl_val = {s = 0xb7fab0 <error: Cannot access memory at address 0xb7fab0>, len = 16}, user_param_val = { s = 0x11e <error: Cannot access memory at address 0x11e>, len = 163710976}, maddr_val = {s = 0x7fff3d1dbf00 "p\277\035=\001", len = -1370752186}, method_val = {s = 0x0, len = 0}, lr_val = {s = 0x7f5152d6b0f0 "", len = 0}, r2_val = { s = 0x400000013d1dbf70 <error: Cannot access memory at address 0x400000013d1dbf70>, len = 1025360400}, gr_val = { s = 0x7f5152e8541a "Host: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 1393988016}} u = 0x7f512ecf128c port = 21250 dst_host = 0xe i = 1 flags = 0 avp = 0x557bae7f69e3 st = {flags = 0, id = 0, name = {n = 4, s = {s = 0x4 <error: Cannot access memory at address 0x4>, len = 0}, re = 0x4}, avp = 0x10} sct = 0x7f51539be112 <_IO_vfprintf_internal+1826> sjt = 0x557bae7f69e3 rve = 0x7f51537eb2f0 mct = 0x7f5152de7980 rv = 0x0 rv1 = 0x7fff3d1dc140 c1 = {cache_type = 12582912, val_type = RV_NONE, c = {avp_val = {n = 163710976, s = {s = 0x8b32937609c20800 <error: Cannot access memory at address 0x8b32937609c20800>, len = 1025359888}, re = 0x8b32937609c20800}, pval = {rs = { s = 0x8b32937609c20800 <error: Cannot access memory at address 0x8b32937609c20800>, len = 1025359888}, ri = -1368282998, flags = 21883}}, i2s = "\022\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\002\000\000\000\000"} s = {s = 0x1352 <error: Cannot access memory at address 0x1352>, len = 0} srevp = {0xab00000004, 0x0} evp = {data = 0x0, rcv = 0x0, dst = 0x0} mod_f_params = {{type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, { type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = { number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}} __func__ = "do_action" #6 0x0000557bae5d0626 in run_actions (h=0x7fff3d1dc130, a=0x7f51537e7400, msg=0x7fff3d1dc210) at core/action.c:1578 t = 0x7f51537ebf50 ret = 1 tvb = {tv_sec = 0, tv_usec = 0} tve = {tv_sec = 0, tv_usec = 0} tz = {tz_minuteswest = 1025360400, tz_dsttime = 32767} tdiff = 0 __func__ = "run_actions" #7 0x00007f51511a943e in xhttp_process_request (orig_msg=0x7f5153876780, new_buf=0x7f5152e853e0 "POST /RPC HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", new_len=189) at xhttp_mod.c:296 ret = 0 backup_rt = 1 tmp_msg = {id = 1197, pid = 0, tval = {tv_sec = 0, tv_usec = 0}, fwd_send_flags = {f = 0, blst_imask = 0}, rpl_send_flags = {f = 3, blst_imask = 0}, first_line = {type = 1, flags = 2, len = 20, u = {request = {method = { s = 0x7f5152e853e0 "POST /RPC HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 4}, uri = { s = 0x7f5152e853e5 "/RPC HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 4}, version = { s = 0x7f5152e853ea "HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 8}, method_value = 65536}, reply = {version = { s = 0x7f5152e853e0 "POST /RPC HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 4}, status = { s = 0x7f5152e853e5 "/RPC HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 4}, reason = { s = 0x7f5152e853ea "HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 8}, statuscode = 65536}}}, via1 = 0x7f51531691b0, via2 = 0x0, headers = 0x7f5153027c20, last_header = 0x7f51531ae720, parsed_flag = 18446744073709551615, h_via1 = 0x7f5153027c20, h_via2 = 0x0, callid = 0x0, to = 0x0, cseq = 0x0, from = 0x0, contact = 0x0, --Type <RET> for more, q to quit, c to continue without paging-- maxforwards = 0x0, route = 0x0, record_route = 0x0, content_type = 0x7f51531ae6c0, content_length = 0x7f51531ae720, authorization = 0x0, expires = 0x0, proxy_auth = 0x0, supported = 0x0, require = 0x0, proxy_require = 0x0, unsupported = 0x0, allow = 0x0, event = 0x0, accept = 0x7f51531bb9f0, accept_language = 0x0, organization = 0x0, priority = 0x0, subject = 0x0, user_agent = 0x0, server = 0x0, content_disposition = 0x0, diversion = 0x0, rpid = 0x0, refer_to = 0x0, session_expires = 0x0, min_se = 0x0, sipifmatch = 0x0, subscription_state = 0x0, date = 0x0, identity = 0x0, identity_info = 0x0, pai = 0x0, ppi = 0x0, path = 0x0, privacy = 0x0, min_expires = 0x0, body = 0x0, eoh = 0x7f5152e85475 "\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", unparsed = 0x7f5152e85475 "\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", rcv = {src_ip = {af = 2, len = 4, u = {addrl = {362853934, 0}, addr32 = {362853934, 0, 0, 0}, addr16 = {46638, 5536, 0, 0, 0, 0, 0, 0}, addr = ".\266\240\025", '\000' <repeats 11 times>}}, dst_ip = {af = 2, len = 4, u = {addrl = {346076718, 0}, addr32 = {346076718, 0, 0, 0}, addr16 = {46638, 5280, 0, 0, 0, 0, 0, 0}, addr = ".\266\240\024", '\000' <repeats 11 times>}}, src_port = 43976, dst_port = 6060, proto_reserved1 = 1262, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "\253\310.\266\240\025\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 51371, sin_addr = {s_addr = 362853934}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 51371, sin6_flowinfo = 362853934, sin6_addr = {__in6_u = {__u6_addr8 = "\000\000\000\000\000\000\000\000\060\317\035=\377\177\000", __u6_addr16 = {0, 0, 0, 0, 53040, 15645, 32767, 0}, __u6_addr32 = {0, 0, 1025363760, 32767}}}, sin6_scope_id = 0}}, bind_address = 0x7f5152dcc460, proto = 2 '\002', proto_pad0 = 0 '\000', proto_pad1 = 0}, buf = 0x7f5152e853e0 "POST /RPC HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: x.x.x.x:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 189, new_uri = { s = 0x0, len = 0}, dst_uri = {s = 0x0, len = 0}, parsed_uri_ok = 0, parsed_uri = {user = {s = 0x0, len = 0}, passwd = {s = 0x0, len = 0}, host = {s = 0x0, len = 0}, port = {s = 0x0, len = 0}, params = {s = 0x0, len = 0}, sip_params = {s = 0x0, len = 0}, headers = {s = 0x0, len = 0}, port_no = 0, proto = 0, type = ERROR_URI_T, flags = (unknown: 0), transport = {s = 0x0, len = 0}, ttl = {s = 0x0, len = 0}, user_param = {s = 0x0, len = 0}, maddr = {s = 0x0, len = 0}, method = {s = 0x0, len = 0}, lr = {s = 0x0, len = 0}, r2 = {s = 0x0, len = 0}, gr = {s = 0x0, len = 0}, transport_val = {s = 0x0, len = 0}, ttl_val = {s = 0x0, len = 0}, user_param_val = {s = 0x0, len = 0}, maddr_val = {s = 0x0, len = 0}, method_val = {s = 0x0, len = 0}, lr_val = {s = 0x0, len = 0}, r2_val = {s = 0x0, len = 0}, gr_val = {s = 0x0, len = 0}}, parsed_orig_ruri_ok = 0, parsed_orig_ruri = {user = {s = 0x0, len = 0}, passwd = {s = 0x0, len = 0}, host = {s = 0x0, len = 0}, port = {s = 0x0, len = 0}, params = {s = 0x0, len = 0}, sip_params = { s = 0x0, len = 0}, headers = {s = 0x0, len = 0}, port_no = 0, proto = 0, type = ERROR_URI_T, flags = (unknown: 0), transport = {s = 0x0, len = 0}, ttl = {s = 0x0, len = 0}, user_param = {s = 0x0, len = 0}, maddr = {s = 0x0, len = 0}, method = {s = 0x0, len = 0}, lr = {s = 0x0, len = 0}, r2 = {s = 0x0, len = 0}, gr = {s = 0x0, len = 0}, transport_val = {s = 0x0, len = 0}, ttl_val = {s = 0x0, len = 0}, user_param_val = {s = 0x0, len = 0}, maddr_val = {s = 0x0, len = 0}, method_val = {s = 0x0, len = 0}, lr_val = {s = 0x0, len = 0}, r2_val = {s = 0x0, len = 0}, gr_val = {s = 0x0, len = 0}}, add_rm = 0x0, body_lumps = 0x0, reply_lump = 0x0, add_to_branch_s = '\000' <repeats 57 times>, add_to_branch_len = 0, hash_index = 0, msg_flags = 0, flags = 0, xflags = {0, 0}, set_global_address = {s = 0x0, len = 0}, set_global_port = {s = 0x0, len = 0}, force_send_socket = 0x0, path_vec = {s = 0x0, len = 0}, instance = {s = 0x0, len = 0}, reg_id = 0, ruid = {s = 0x0, len = 0}, location_ua = {s = 0x0, len = 0}, otcpid = 0, ldv = { flow = {decoded = 0, rcv = {src_ip = {af = 0, len = 0, u = {addrl = {0, 0}, addr32 = {0, 0, 0, 0}, addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, addr = '\000' <repeats 15 times>}}, dst_ip = {af = 0, len = 0, u = {addrl = {0, 0}, addr32 = {0, 0, 0, 0}, addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, addr = '\000' <repeats 15 times>}}, src_port = 0, dst_port = 0, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, bind_address = 0x0, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0}}}} msg = 0x7fff3d1dc210 ra_ctx = {rec_lev = 2, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {139987259278432, -6960915947526443003, 93989696924131, 536870912, 12582912, 6, -6960915947698409467, -3860432251912212475}, __mask_was_saved = 0, __saved_mask = {__val = { 18446744073709551615, 0, 4, 93986769338368, 16, 12582944, 140734218749576, 3617854161208475648, 0, 0, 139987273275232, 139987269897424, 139987258674898, 11, 139987258674898, 139987227742213}}}}} keng = 0x0 evrtname = {s = 0x7f51511b1598 "xhttp:request", len = 13} __func__ = "xhttp_process_request" #8 0x00007f51511aac55 in xhttp_handler (msg=0x7f5153876780) at xhttp_mod.c:383 ret = 0 fake_msg = 0x7f5152e853e0 "POST /RPC HTTP/1.1\r\nVia: SIP/2.0/TCP X.X.X.X:43976\r\nHost: X.X.X.X:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}" fake_msg_len = 189 pmatch = {rm_so = 1025362464, rm_eo = 32767} c = 83 'S' __func__ = "xhttp_handler" #9 0x0000557bae559a24 in nonsip_msg_run_hooks (msg=0x7f5153876780) at core/nonsip_hooks.c:112 r = 0 ret = 0 #10 0x0000557bae5d6720 in receive_msg (buf=0x557bb08fee80 "POST /RPC HTTP/1.1\r\nHost: X.X.X.X:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len=151, rcv_info=0x7f512fe42828) at core/receive.c:348 msg = 0x7f5153876780 ctx = {rec_lev = 4, run_flags = 0, last_retcode = 803482386, jmp_env = {{__jmpbuf = {8, 65536, 27, 0, 140734218751644, 139986672560144, 7, 70364449210396}, __mask_was_saved = 121, __saved_mask = {__val = {10030241455286650880, 65535, 139986672560144, 93989696924131, 536870912, 12582912, 6, 140734218751232, 93989695295308, 519691042940, 2, 0, 65535, 140734218751644, 139986672560144, 18446744073709551536}}}}} bctx = 0x7fff3d1dcc10 ret = 0 tvb = {tv_sec = 140734218751644, tv_usec = 139986672560904} tve = {tv_sec = 139986672560144, tv_usec = 139986672560144} tz = {tz_minuteswest = -1369026089, tz_dsttime = 21883} diff = 0 inb = {s = 0x557bb08fee80 "POST /RPC HTTP/1.1\r\nHost: X.X.X.X:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len = 151} netinfo = {data = {s = 0x1400020001 <error: Cannot access memory at address 0x1400020001>, len = 803482376}, rcv = 0x4, dst = 0x7f512fe42b0d} keng = 0x0 evp = {data = 0x7fff3d1dcb30, rcv = 0x7f512fe42828, dst = 0x0} cidlockidx = 0 cidlockset = 0 errsipmsg = 0 exectime = 0 __func__ = "receive_msg" #11 0x0000557bae670357 in receive_tcp_msg (tcpbuf=0x7f512fe42b08 "POST /RPC HTTP/1.1\r\nHost: X.X.X.X:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}", len=151, rcv_info=0x7f512fe42828, con=0x7f512fe42810) at core/tcp_read.c:1448 buf = 0x557bb08fee80 "POST /RPC HTTP/1.1\r\nHost: X.X.X.X:6060\r\nAccept: */*\r\nContent-type: application/json\r\nContent-length: 38\r\n\r\n{"jsonrpc":"2.0", "method":"tm.clean"}" bsize = 65535 blen = 65535 __func__ = "receive_tcp_msg" #12 0x0000557bae6725f3 in tcp_read_req (con=0x7f512fe42810, bytes_read=0x7fff3d1dce98, read_flags=0x7fff3d1dce9c) at core/tcp_read.c:1631 bytes = 151 total_bytes = 151 resp = 1 size = 24 req = 0x7f512fe42890 dst = {send_sock = 0x0, to = {s = {sa_family = 61136, sa_data = "\342/Q\177\000\000\330\316\035=\377\177\000"}, sin = {sin_family = 61136, sin_port = 12258, sin_addr = {s_addr = 32593}, sin_zero = "\330\316\035=\377\177\000"}, sin6 = {sin6_family = 61136, sin6_port = 12258, sin6_flowinfo = 32593, sin6_addr = {__in6_u = {__u6_addr8 = "\330\316\035=\377\177\000\000\b\000\000\000\000\000\000", __u6_addr16 = {52952, 15645, 32767, 0, 8, 0, 0, 0}, __u6_addr32 = {1025363672, 32767, 8, 0}}}, sin6_scope_id = 20}}, id = 0, send_flags = {f = 1, blst_imask = 0}, proto = 1 '\001', proto_pad0 = 0 '\000', proto_pad1 = 0} --Type <RET> for more, q to quit, c to continue without paging-- c = 49 '1' ret = 1025363424 __func__ = "tcp_read_req" #13 0x0000557bae6755a2 in handle_io (fm=0x7f5152db1498, events=1, idx=-1) at core/tcp_read.c:1804 ret = 8 n = 8 read_flags = 1 con = 0x7f512fe42810 s = 28 resp = -2 t = 1117018954 __func__ = "handle_io" #14 0x0000557bae662ba9 in io_wait_loop_epoll (h=0x557bae90ad00 <io_w>, t=2, repeat=0) at core/io_wait.h:1062 n = 1 r = 0 fm = 0x7f5152db1498 revents = 1 __func__ = "io_wait_loop_epoll" #15 0x0000557bae6775f5 in tcp_receive_loop (unix_sock=59) at core/tcp_read.c:1974 __func__ = "tcp_receive_loop" #16 0x0000557bae51044a in tcp_init_children () at core/tcp_main.c:5174 r = 8 i = 7 reader_fd_1 = 59 pid = 0 si_desc = "tcp receiver (tcp:X.X.X.X:6060)\000\000\000\062\023\000\000\001", '\000' <repeats 11 times>, "\260\034\246SQ\177\000\000\343i\177\256{U\000\000\000\000\000 \000\000\000\000\000\000\300\000\000\000\000\000\006\000\000\000\000\000\000\000 \321\035=\377\177\000\000\8V\256{U\000\000 \321\035=\377\177\000\000\254\204a\256\000\000\000" si = 0x0 __func__ = "tcp_init_children" #17 0x0000557bae4228ce in main_loop () at main.c:1761 i = 8 pid = 4910 si = 0x0 si_desc = "udp receiver child=7 sock=127.0.0.1:5070\000\060\066\060\000\000\000\000\020\322\035=\377\177\000\000\000\b\302\tv\223\062\213\004\000\000\000\000\000\000\000\260\034\246SQ\177\000\000\343i\177\256{U\000\000\000\000\000 \000\000\000\000\000\000\300\000\000\000\000\000\006\000\000\000\000\000\000\000P\322\035=\377\177\000\000>0d\256{U\000" nrprocs = 8 woneinit = 1 __func__ = "main_loop" #18 0x0000557bae42abb9 in main (argc=17, argv=0x7fff3d1dd738) at main.c:2802 cfg_stream = 0x557bb0796260 c = -1 r = 0 tmp = 0x7fff3d1def3f "" tmp_len = 17 port = 0 proto = 1025365816 ahost = 0x0 aport = 0 options = 0x557bae7a0618 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 1972091384 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x0 p = 0x0 st = {st_dev = 21, st_ino = 1930, st_nlink = 2, st_mode = 16832, st_uid = 109, st_gid = 115, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1572122046, tv_nsec = 75984014}, st_mtim = {tv_sec = 1583307292, tv_nsec = 782512911}, st_ctim = {tv_sec = 1583307633, tv_nsec = 722908545}, __glibc_reserved = {0, 0, 0}} tbuf = "\000\000\000\000\000\000\000\000\b\000\000\000\000\000\000\000\060G\272SQ\177\000\000\000\000\000\000\000\000\000\000\240q\037=\377\177\000\000\364J\270SQ\177\000\000\b\000\000\000\000\000\000\000\230r\037=\377\177\000\000\060G\272SQ\177\000\000\330\324\035=\377\177\000\000\324\324\035=\377\177", '\000' <repeats 18 times>, "\230r\037=\377\177\000\000\250q\037=\377\177\000\000'\356\256SQ\177\000\000&\260be\000\000\000\000@\326\035=\377\177\000\000\300\212\225\001\000\000\000\000\240\325\035=\377\177\000\000\220\325\035=\377\177\000\000\330\324\035=\377\177\000\000XG\272SQ\177\000\000\001", '\000' <repeats 31 times>... option_index = 0 long_options = {{name = 0x557bae7a2fca "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x557bae79d438 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x557bae7a2fcf "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x557bae7a2fd5 "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x557bae7a2fdb "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x557bae7a2fe4 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x557bae7a2fee "server-id", has_arg = 1, flag = 0x0, val = 1028}, { name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __func__ = "main"