Re: [SR-Users] Possible memory leak?

Hello all, I've been experiencing memory exhaustion (all 8GiB of it) on hosts running kamailio for quite some time now while on the 5.4.x train. I can't really say about previous versions, because the setup was quite different back when I was using those. After about 2 days of operation, RAM is exhausted and kamailio is killed by the kernel OOM which frees up memory again. Restarting kamailio via systemd will also free up most memory. I can't seem to find what is using it up, so I'm guessing there might be a memory leak, but I'm not 100% sure so I thought I'd ask the list for opinions. I'm including the output of kamcmd pkg and shm stats and kamctl stats here. My take on those is that there's nothing to show where all this RAM is going. Kamailio version is 5.4.5+bpo10 on debian buster and is started with '-P /run/kamailio/kamailio.pid -f /etc/kamailio/kamailio.cfg -m 2048 -M 16' (51 processes total). Thanks! kamctl stats {   "jsonrpc":  "2.0",   "result": [     "core:bad_URIs_rcvd = 6470677",     "core:bad_msg_hdr = 25370",     "core:drop_replies = 0",     "core:drop_requests = 342064",     "core:err_replies = 0",     "core:err_requests = 0",     "core:fwd_replies = 24367",     "core:fwd_requests = 375703",     "core:rcv_replies = 45021597",     "core:rcv_replies_18x = 834856",     "core:rcv_replies_1xx = 1505425",     "core:rcv_replies_1xx_bye = 17",     "core:rcv_replies_1xx_cancel = 7",     "core:rcv_replies_1xx_invite = 1497313",     "core:rcv_replies_1xx_message = 0",     "core:rcv_replies_1xx_prack = 0",     "core:rcv_replies_1xx_refer = 0",     "core:rcv_replies_1xx_reg = 0",     "core:rcv_replies_1xx_update = 0",     "core:rcv_replies_2xx = 41764997",     "core:rcv_replies_2xx_bye = 328464",     "core:rcv_replies_2xx_cancel = 223572",     "core:rcv_replies_2xx_invite = 371129",     "core:rcv_replies_2xx_message = 190",     "core:rcv_replies_2xx_prack = 0",     "core:rcv_replies_2xx_refer = 3205",     "core:rcv_replies_2xx_reg = 12783948",     "core:rcv_replies_2xx_update = 0",     "core:rcv_replies_3xx = 1052",     "core:rcv_replies_3xx_bye = 0",     "core:rcv_replies_3xx_cancel = 0",     "core:rcv_replies_3xx_invite = 1052",     "core:rcv_replies_3xx_message = 0",     "core:rcv_replies_3xx_prack = 0",     "core:rcv_replies_3xx_refer = 0",     "core:rcv_replies_3xx_reg = 0",     "core:rcv_replies_3xx_update = 0",     "core:rcv_replies_401 = 17120",     "core:rcv_replies_404 = 525917",     "core:rcv_replies_407 = 553",     "core:rcv_replies_480 = 23138",     "core:rcv_replies_486 = 74201",     "core:rcv_replies_4xx = 1715492",     "core:rcv_replies_4xx_bye = 2146",     "core:rcv_replies_4xx_cancel = 594",     "core:rcv_replies_4xx_invite = 317850",     "core:rcv_replies_4xx_message = 208",     "core:rcv_replies_4xx_prack = 0",     "core:rcv_replies_4xx_refer = 1",     "core:rcv_replies_4xx_reg = 2382",     "core:rcv_replies_4xx_update = 0",     "core:rcv_replies_5xx = 29462",     "core:rcv_replies_5xx_bye = 15",     "core:rcv_replies_5xx_cancel = 10",     "core:rcv_replies_5xx_invite = 3501",     "core:rcv_replies_5xx_message = 0",     "core:rcv_replies_5xx_prack = 0",     "core:rcv_replies_5xx_refer = 0",     "core:rcv_replies_5xx_reg = 25157",     "core:rcv_replies_5xx_update = 0",     "core:rcv_replies_6xx = 5169",     "core:rcv_replies_6xx_bye = 0",     "core:rcv_replies_6xx_cancel = 0",     "core:rcv_replies_6xx_invite = 5169",     "core:rcv_replies_6xx_message = 0",     "core:rcv_replies_6xx_prack = 0",     "core:rcv_replies_6xx_refer = 0",     "core:rcv_replies_6xx_reg = 0",     "core:rcv_replies_6xx_update = 0",     "core:rcv_requests = 67620661",     "core:rcv_requests_ack = 707431",     "core:rcv_requests_bye = 336874",     "core:rcv_requests_cancel = 226878",     "core:rcv_requests_info = 70",     "core:rcv_requests_invite = 1038320",     "core:rcv_requests_message = 796",     "core:rcv_requests_notify = 6027113",     "core:rcv_requests_options = 29079952",     "core:rcv_requests_prack = 0",     "core:rcv_requests_publish = 40344",     "core:rcv_requests_refer = 3217",     "core:rcv_requests_register = 20919429",     "core:rcv_requests_subscribe = 5296608",     "core:rcv_requests_update = 0",     "core:unsupported_methods = 0",     "dialog:active_dialogs = 458",     "dialog:early_dialogs = 123",     "dialog:expired_dialogs = 105",     "dialog:failed_dialogs = 321224",     "dialog:processed_dialogs = 645092",     "dns:failed_dns_request = 534",     "dns:slow_dns_request = 0",     "mysql:driver_errors = 18",     "shmem:fragments = 8836",     "shmem:free_size = 1955034032",     "shmem:max_used_size = 250791392",     "shmem:real_used_size = 192449616",     "shmem:total_size = 2147483648",     "shmem:used_size = 105997528",     "siptrace:traced_replies = 0",     "siptrace:traced_requests = 0",     "sl:1xx_replies = 0",     "sl:200_replies = 14330540",     "sl:202_replies = 0",     "sl:2xx_replies = 0",     "sl:300_replies = 0",     "sl:301_replies = 0",     "sl:302_replies = 0",     "sl:3xx_replies = 0",     "sl:400_replies = 3",     "sl:401_replies = 7979168",     "sl:403_replies = 14845",     "sl:404_replies = 1159237",     "sl:407_replies = 2445205",     "sl:408_replies = 0",     "sl:483_replies = 79336",     "sl:4xx_replies = 28",     "sl:500_replies = 0",     "sl:5xx_replies = 0",     "sl:6xx_replies = 0",     "sl:failures = 1",     "sl:received_ACKs = 342064",     "sl:sent_err_replies = 0",     "sl:sent_replies = 26008362",     "sl:xxx_replies = 0",     "sst:expired_sst = 0",     "tcp:con_reset = 0",     "tcp:con_timeout = 0",     "tcp:connect_failed = 0",     "tcp:connect_success = 0",     "tcp:current_opened_connections = 0",     "tcp:current_write_queue_size = 0",     "tcp:established = 24562",     "tcp:local_reject = 0",     "tcp:passive_open = 24562",     "tcp:send_timeout = 0",     "tcp:sendq_full = 0",     "tmx:2xx_transactions = 41122607",     "tmx:3xx_transactions = 1049",     "tmx:4xx_transactions = 1904640",     "tmx:5xx_transactions = 8025",     "tmx:6xx_transactions = 5165",     "tmx:UAC_transactions = 2602132",     "tmx:UAS_transactions = 43036902",     "tmx:active_transactions = 145",     "tmx:inuse_transactions = 888",     "tmx:rpl_absorbed = 1541413",     "tmx:rpl_generated = 1111996",     "tmx:rpl_received = 44996889",     "tmx:rpl_relayed = 43455476",     "tmx:rpl_sent = 44567472"   ],   "id": 15546 } kamcmd mod.stats all pkg