[SR-Users] Possible memory leak?

Hello all, I've been experiencing memory exhaustion (all 8GiB of it) on hosts running kamailio for quite some time now while on the 5.4.x train. I can't really say about previous versions, because the setup was quite different back when I was using those. After about 2 days of operation, RAM is exhausted and kamailio is killed by the kernel OOM which frees up memory again. Restarting kamailio via systemd will also free up most memory. I can't seem to find what is using it up, so I'm guessing there might be a memory leak, but I'm not 100% sure so I thought I'd ask the list for opinions. I'm including the output of kamcmd pkg and shm stats and kamctl stats here. My take on those is that there's nothing to show where all this RAM is going. Kamailio version is 5.4.5+bpo10 on debian buster and is started with '-P /run/kamailio/kamailio.pid -f /etc/kamailio/kamailio.cfg -m 2048 -M 16' (51 processes total). Thanks! kamctl stats { "jsonrpc": "2.0", "result": [ "core:bad_URIs_rcvd = 6470677", "core:bad_msg_hdr = 25370", "core:drop_replies = 0", "core:drop_requests = 342064", "core:err_replies = 0", "core:err_requests = 0", "core:fwd_replies = 24367", "core:fwd_requests = 375703", "core:rcv_replies = 45021597", "core:rcv_replies_18x = 834856", "core:rcv_replies_1xx = 1505425", "core:rcv_replies_1xx_bye = 17", "core:rcv_replies_1xx_cancel = 7", "core:rcv_replies_1xx_invite = 1497313", "core:rcv_replies_1xx_message = 0", "core:rcv_replies_1xx_prack = 0", "core:rcv_replies_1xx_refer = 0", "core:rcv_replies_1xx_reg = 0", "core:rcv_replies_1xx_update = 0", "core:rcv_replies_2xx = 41764997", "core:rcv_replies_2xx_bye = 328464", "core:rcv_replies_2xx_cancel = 223572", "core:rcv_replies_2xx_invite = 371129", "core:rcv_replies_2xx_message = 190", "core:rcv_replies_2xx_prack = 0", "core:rcv_replies_2xx_refer = 3205", "core:rcv_replies_2xx_reg = 12783948", "core:rcv_replies_2xx_update = 0", "core:rcv_replies_3xx = 1052", "core:rcv_replies_3xx_bye = 0", "core:rcv_replies_3xx_cancel = 0", "core:rcv_replies_3xx_invite = 1052", "core:rcv_replies_3xx_message = 0", "core:rcv_replies_3xx_prack = 0", "core:rcv_replies_3xx_refer = 0", "core:rcv_replies_3xx_reg = 0", "core:rcv_replies_3xx_update = 0", "core:rcv_replies_401 = 17120", "core:rcv_replies_404 = 525917", "core:rcv_replies_407 = 553", "core:rcv_replies_480 = 23138", "core:rcv_replies_486 = 74201", "core:rcv_replies_4xx = 1715492", "core:rcv_replies_4xx_bye = 2146", "core:rcv_replies_4xx_cancel = 594", "core:rcv_replies_4xx_invite = 317850", "core:rcv_replies_4xx_message = 208", "core:rcv_replies_4xx_prack = 0", "core:rcv_replies_4xx_refer = 1", "core:rcv_replies_4xx_reg = 2382", "core:rcv_replies_4xx_update = 0", "core:rcv_replies_5xx = 29462", "core:rcv_replies_5xx_bye = 15", "core:rcv_replies_5xx_cancel = 10", "core:rcv_replies_5xx_invite = 3501", "core:rcv_replies_5xx_message = 0", "core:rcv_replies_5xx_prack = 0", "core:rcv_replies_5xx_refer = 0", "core:rcv_replies_5xx_reg = 25157", "core:rcv_replies_5xx_update = 0", "core:rcv_replies_6xx = 5169", "core:rcv_replies_6xx_bye = 0", "core:rcv_replies_6xx_cancel = 0", "core:rcv_replies_6xx_invite = 5169", "core:rcv_replies_6xx_message = 0", "core:rcv_replies_6xx_prack = 0", "core:rcv_replies_6xx_refer = 0", "core:rcv_replies_6xx_reg = 0", "core:rcv_replies_6xx_update = 0", "core:rcv_requests = 67620661", "core:rcv_requests_ack = 707431", "core:rcv_requests_bye = 336874", "core:rcv_requests_cancel = 226878", "core:rcv_requests_info = 70", "core:rcv_requests_invite = 1038320", "core:rcv_requests_message = 796", "core:rcv_requests_notify = 6027113", "core:rcv_requests_options = 29079952", "core:rcv_requests_prack = 0", "core:rcv_requests_publish = 40344", "core:rcv_requests_refer = 3217", "core:rcv_requests_register = 20919429", "core:rcv_requests_subscribe = 5296608", "core:rcv_requests_update = 0", "core:unsupported_methods = 0", "dialog:active_dialogs = 458", "dialog:early_dialogs = 123", "dialog:expired_dialogs = 105", "dialog:failed_dialogs = 321224", "dialog:processed_dialogs = 645092", "dns:failed_dns_request = 534", "dns:slow_dns_request = 0", "mysql:driver_errors = 18", "shmem:fragments = 8836", "shmem:free_size = 1955034032", "shmem:max_used_size = 250791392", "shmem:real_used_size = 192449616", "shmem:total_size = 2147483648", "shmem:used_size = 105997528", "siptrace:traced_replies = 0", "siptrace:traced_requests = 0", "sl:1xx_replies = 0", "sl:200_replies = 14330540", "sl:202_replies = 0", "sl:2xx_replies = 0", "sl:300_replies = 0", "sl:301_replies = 0", "sl:302_replies = 0", "sl:3xx_replies = 0", "sl:400_replies = 3", "sl:401_replies = 7979168", "sl:403_replies = 14845", "sl:404_replies = 1159237", "sl:407_replies = 2445205", "sl:408_replies = 0", "sl:483_replies = 79336", "sl:4xx_replies = 28", "sl:500_replies = 0", "sl:5xx_replies = 0", "sl:6xx_replies = 0", "sl:failures = 1", "sl:received_ACKs = 342064", "sl:sent_err_replies = 0", "sl:sent_replies = 26008362", "sl:xxx_replies = 0", "sst:expired_sst = 0", "tcp:con_reset = 0", "tcp:con_timeout = 0", "tcp:connect_failed = 0", "tcp:connect_success = 0", "tcp:current_opened_connections = 0", "tcp:current_write_queue_size = 0", "tcp:established = 24562", "tcp:local_reject = 0", "tcp:passive_open = 24562", "tcp:send_timeout = 0", "tcp:sendq_full = 0", "tmx:2xx_transactions = 41122607", "tmx:3xx_transactions = 1049", "tmx:4xx_transactions = 1904640", "tmx:5xx_transactions = 8025", "tmx:6xx_transactions = 5165", "tmx:UAC_transactions = 2602132", "tmx:UAS_transactions = 43036902", "tmx:active_transactions = 145", "tmx:inuse_transactions = 888", "tmx:rpl_absorbed = 1541413", "tmx:rpl_generated = 1111996", "tmx:rpl_received = 44996889", "tmx:rpl_relayed = 43455476", "tmx:rpl_sent = 44567472" ], "id": 15546 } kamcmd mod.stats all pkg Module: core { init_io_wait(469): 8040 init_modules(911): 16 get_hdr_field(116): 464 parse_headers(325): 128 rpc_hash_add(151): 2048 cnt_hash_add(394): 2048 register_select_table(458): 48 sr_wtimer_add(373): 80 add_callback(59): 312 sr_wtimer_init(351): 136 cfg_declare(50): 3168 fix_sock_str(500): 480 fix_hostname(1466): 96 grp_hash_add(241): 728 init_dst_set(89): 32296 cfg_script_fixup(303): 8 cfg_script_fixup(247): 288 cfg_script_fixup(241): 120 new_db_id(318): 304 fix_expr(554): 192 parse_pvname_list(1599): 480 mk_switch_jmp_table(73): 448 fix_param(1140): 672 pv_parse_index(776): 1232 subst_parser(291): 144 subst_parser(274): 64 subst_parser(301): 8 route_new_list(199): 1152 init_io_wait(516): 7256 db_do_init2(299): 280 parse_select(204): 2496 fix_param(1079): 704 mk_match_cond_table(100): 944 mk_name_lst(3754): 72 mk_listen_id(3771): 96 yyparse(2853): 19728 fix_param(1045): 11792 fixup_regexp_null(213): 368 tr_new(1730): 12352 fix_match(490): 384 fixup_pvar_all(279): 14512 mk_rval_expr2(2703): 845408 mk_elem(90): 528 fix_match_rve(3059): 1464 pv_cache_add(349): 63872 mk_rval_expr1(2643): 522656 mk_case_stm(3879): 3432 mk_rval_expr_v(2562): 2513784 mk_action(118): 493240 parse_params2(593): 6496 set_mod_param_regex(145): 1248 set_mod_param_regex(134): 2520 tr_table_add(1959): 360 register_module(253): 39928 register_module(236): 4784 load_module(437): 2624 new_cfg_script_var(134): 88 dupl_string_name(74): 56 dupl_string(49): 184 new_cfg_script_var(126): 104 pv_parse_format(1173): 30744 cfg_new_group(79): 864 new_sock_info(304): 96 new_sock_info(299): 2400 yyparse(659): 352 yyparse(1879): 216 add_alias(103): 256 add_alias(99): 320 yyparse(2388): 408 yyparse(678): 368 sr_push_yy_state(1695): 208 sr_push_yy_state(1634): 584 addstr(1448): 572072 rpc_hash_add(105): 28128 str_hash_alloc(59): 512 pv_table_add(236): 19448 pv_init_buffer(2139): 327680 pv_init_buffer(2129): 320 init_nonsip_hooks(43): 24 rval_get_str(1281): 54920 init_rlist(146): 24 fix_socket_list(1675): 56 route_add(124): 5312 str_hash_alloc(59): 768 mk_rval_expr_v(2575): 3304 pp_define(1827): 1344 init_counters(125): 128 cnt_hash_add(341): 27856 str_hash_alloc(59): 1280 Total: 5708952 } Module: permissions { get_pathname(243): 64 Total: 64 } Module: auth { generate_random_secret(240): 32 generate_random_secret(239): 32 Total: 64 } Module: dialog { dlg_bridge_init_hdrs(66): 80 Total: 80 } Module: acc { parse_acc_extra(116): 6384 Total: 6384 } Module: xlog { mod_init(212): 8200 xdbg_fixup_helper(498): 7736 xlog_fixup_helper(533): 36280 Total: 52216 } Module: dmq { make_socket_str_from_uri(143): 24 Total: 24 } Module: rr { register_rrcb(61): 96 Total: 96 } Module: jsonrpcs { jsonrpc_dgram_mod_init(182): 40 jsonrpc_init_fifo_server(123): 128 jsonrpc_fifo_mod_init(525): 40 Total: 208 } Module: sqlops { sqlops_tr_buffer_init(46): 2048 sql_init_con(83): 480 Total: 2528 } Module: db_mysql { db_mysql_new_connection(78): 3560 db_mysql_new_connection(68): 96 Total: 3656 } Module: phonenum { sr_phonenum_add_item(99): 16 sr_phonenum_add_item(93): 608 pv_parse_phonenum_name(131): 224 Total: 848 } Module: pv { tr_parse_string(2717): 96 tr_parse_uri(2806): 64 tr_parse_string(2639): 336 tr_parse_string(2628): 424 tr_parse_paramlist(2978): 288 tr_parse_paramlist(2906): 192 tr_parse_string(2569): 120 pv_parse_xavp_name(541): 2664 pv_parse_xavp_name(515): 3720 tr_parse_string(2484): 864 tr_parse_string(2473): 904 add_var(58): 4384 tr_parse_string(2519): 2144 tr_parse_string(2509): 5184 add_var(65): 1544 Total: 22928 } Module: textops { tr_txt_parse_re(212): 32 hname_fixup(3528): 1960 fixup_method(3598): 1648 Total: 3640 } Module: tmx { pv_parse_t_var_name(390): 176 Total: 176 } Module: rtpengine { build_rtpp_socks(1801): 24 child_init(1987): 240 Total: 264 } Module: htable { fixup_ht_key(352): 352 pv_parse_ht_name(158): 2240 Total: 2592 } Module: textopsx { fixup_hname_param(593): 320 Total: 320 } Module: avpops { avpops_parse_pvar(57): 64 Total: 64 } Module: ndb_redis { redisc_add_server(348): 216648 Total: 216648 } Module: sl { sl_register_callback(514): 64 Total: 64 } Module: siptrace { mod_init(341): 384 Total: 384 } Module: tm { Total: 0 } Module: kex { Total: 0 } Module: topos { Total: 0 } Module: mtree { Total: 0 } Module: cfgutils { Total: 0 } Module: dispatcher { Total: 0 } Module: outbound { Total: 0 } kamcmd mod.stats all shm Module: core { dns_cache_mk_rd_entry(1110): 1288 counters_prefork_init(212): 65280 cfg_clone_str(130): 136 cfg_shmize(217): 824 main_loop(1341): 8 init_pt(107): 8 init_pt(106): 8 init_pt(105): 7552 build_req_buf_from_sip_req(2218): 791408 xavp_new_value(116): 186472 sip_msg_shm_clone(496): 4101904 msg_lump_cloner(986): 755320 create_avp(175): 145064 register_timer(995): 216 cfg_register_ctx(47): 64 init_tcp(4962): 8192 init_tcp(4956): 32768 init_tcp(4947): 8 init_tcp(4940): 8 init_tcp(4933): 8 init_tcp(4927): 8 init_tcp(4915): 8 init_avps(90): 8 init_avps(89): 8 init_dst_blacklist(438): 16384 init_dst_blacklist(430): 8 timer_alloc(498): 96 init_dns_cache(361): 8 init_dns_cache(352): 16384 init_dns_cache(344): 16 init_dns_cache(336): 8 init_timer(267): 8 init_timer(266): 16384 init_timer(265): 8 init_timer(264): 8 init_timer(253): 8 init_timer(221): 8 init_timer(210): 278544 init_timer(209): 8 init_timer(197): 8 cfg_child_cb_new(829): 64 sr_cfg_init(361): 8 sr_cfg_init(354): 8 sr_cfg_init(347): 8 sr_cfg_init(335): 8 sr_cfg_init(323): 8 ksr_shutdown_phase_init(120): 8 qm_shm_lock_init(1202): 8 Total: 6424576 } Module: permissions { domain_name_table_insert(978): 96 addr_hash_table_insert(458): 2776 subnet_table_insert(670): 336 init_addresses(526): 8 new_domain_name_table(865): 2048 init_addresses(512): 8 new_subnet_table(635): 57456 init_addresses(498): 8 new_addr_hash_table(418): 2048 init_trusted(220): 8 new_hash_table(98): 2048 Total: 66840 } Module: auth { Total: 0 } Module: dialog { dlg_update_cseq(636): 456 dlg_get_iuid_shm_clone(308): 4688 set_dlg_profile(580): 139680 dlg_set_leg_info(553): 64608 dlg_set_leg_info(544): 16296 build_new_dlg(469): 356616 dlg_set_leg_info(536): 29176 new_dlg_var(127): 624976 new_dlg_var(108): 1675696 init_dlg_table(297): 131088 init_dlg_table(289): 8 init_dlg_table(282): 8 init_dlg_table(277): 8 init_dlg_timer(57): 8 init_dlg_timer(48): 32 new_dlg_profile(188): 2272 new_dlg_var(117): 642216 dlg_set_leg_info(562): 18208 init_dlg_callback(55): 32 register_dlgcb(124): 189728 Total: 3895800 } Module: acc { Total: 0 } Module: xlog { Total: 0 } Module: dmq { alloc_job_queue(255): 64 shm_str_dup(802): 280 build_dmq_node(164): 3616 add_peer(68): 168 mod_init(236): 8 mod_init(229): 48 init_dmq_node_list(70): 24 init_peer_list(33): 24 Total: 4232 } Module: rr { Total: 0 } Module: jsonrpcs { Total: 0 } Module: sqlops { Total: 0 } Module: db_mysql { Total: 0 } Module: phonenum { Total: 0 } Module: pv { add_shvar(177): 8 add_shvar(170): 64 lock_set_alloc(66): 80 Total: 152 } Module: textops { Total: 0 } Module: tmx { tmx_check_pretran(271): 6280 tmx_check_pretran(250): 3320 tmx_init_pretran_table(90): 512 Total: 10112 } Module: rtpengine { rtpengine_hash_table_init(111): 14336 rtpengine_hash_table_init(96): 2048 rtpengine_hash_table_init(85): 1024 rtpengine_hash_table_init(76): 2048 rtpengine_hash_table_init(67): 2048 rtpengine_hash_table_init(58): 32 add_rtpengine_socks(978): 96 add_rtpengine_socks(959): 256 get_rtpp_set(873): 8 get_rtpp_set(864): 56 mod_init(1606): 8 mod_init(1599): 24 mod_init(1586): 8 mod_init(1579): 8 mod_init(1565): 16 rtpp_function_call(2657): 1367600 shm_str_dup(802): 2205800 Total: 3595416 } Module: htable { ht_init_tables(394): 2574336 ht_cell_new(186): 68875496 ht_add_table(315): 256 ht_add_table(289): 8424 Total: 71458512 } Module: textopsx { Total: 0 } Module: avpops { Total: 0 } Module: ndb_redis { Total: 0 } Module: sl { init_sl_stats_child(125): 9792 sl_startup(90): 8 init_sl_stats(110): 8 Total: 9808 } Module: siptrace { mod_init(320): 8 Total: 8 } Module: tm { cancel_reason_pack(1224): 104 update_totag_set(321): 656 build_local_reparse(328): 28384 update_totag_set(320): 864 _reply_light(500): 21480 init_tm_stats_child(56): 13056 build_cell(334): 8250680 relay_reply(2064): 439848 build_uac_req(1597): 118672 insert_tmcb(135): 49272 init_tmcb_lists(72): 16 init_tmcb_lists(70): 16 init_hash_table(482): 2097152 tm_rpc_response_list_init(77): 16 Total: 11020216 } Module: kex { pkg_proc_stats_init(79): 2448 Total: 2448 } Module: topos { lock_set_alloc(66): 2064 Total: 2064 } Module: mtree { mt_add_to_tree(229): 2656 mod_init(247): 8 mt_add_to_tree(255): 9559984 mt_add_to_tree(291): 811224 mt_add_to_tree(301): 269848 mt_init_tree(149): 152 mt_init_tree(136): 256 mt_init_tree(125): 176 mt_init_tree(114): 2376 mt_table_spec(840): 8 Total: 10646688 } Module: cfgutils { mod_init(898): 32 mod_init(892): 8 mod_init(884): 8 Total: 48 } Module: dispatcher { mod_init(484): 80 reindex_dests(723): 7680 ds_avl_insert(3621): 11984 ds_set_attrs(301): 960 pack_dest(411): 1456 ds_init_data(262): 16 ds_init_data(254): 16 ds_parse_reply_codes(1112): 808 mod_init(339): 8 mod_init(337): 8 ds_ping_active_init(138): 8 Total: 23024 } Module: outbound { mod_init(99): 24 Total: 24 }