11 Feb
2021
11 Feb
'21
11:22 p.m.
Hello,
I think I have the same problem, at least in my case the problem is with
the Shared memory.
Kamailio is configured to use 4GB of shared memory (-m 4096) and after
restart, he is getting all 4GB of memory and keep it decreasing slowly
until it reaches the low values (40-50MiB) and then giving such errors in
the log file:
ERROR: <core> [core/mem/q_malloc.c:298]: qm_find_free():
qm_find_free(0x7f5a4ce08000, 65832); Free fragment not found!
ERROR: <core> [core/mem/q_malloc.c:432]: qm_malloc():
qm_malloc(0x7f5a4ce08000, 65832) called from websocket: ws_conn.c:
wsconn_add(197), module: websocket; Free fragment not found!
ERROR: websocket [ws_conn.c:199]: wsconn_add(): allocating shared memory
In my case the issue seems to be because of the WebSocket module, we have
another server running without this module and don't have such an issue. As
a workaround, we fixed it by restarting it weekly during the maintenance
window when the call volume is not so high. But, would be nice to have it
fixed within the application.
version: kamailio 5.4.3 (x86_64/linux) b0a629
kamcmd core.shmmem
{
total: 0
free: -645796512
used: 636107136
real_used: 645796512
max_used: 670026920
fragments: 5161
}
kamcmd mod.stats all shm
Module: core
{
tcpconn_new(1134): 5277264
build_req_buf_from_sip_req(2218): 83064
sip_msg_shm_clone(496): 442968
counters_prefork_init(212): 112896
cfg_clone_str(130): 240
cfg_shmize(217): 1264
main_loop(1313): 8
init_pt(107): 8
init_pt(106): 8
init_pt(105): 6216
cfg_register_ctx(47): 128
cfg_parse_str(907): 504
register_timer(995): 432
init_tcp(4962): 8192
init_tcp(4956): 32768
init_tcp(4947): 8
init_tcp(4940): 8
init_tcp(4933): 8
init_tcp(4927): 8
init_tcp(4915): 8
init_avps(90): 8
init_avps(89): 8
init_dst_blacklist(438): 16384
init_dst_blacklist(430): 8
timer_alloc(498): 96
init_dns_cache(361): 8
init_dns_cache(352): 16384
init_dns_cache(344): 16
init_dns_cache(336): 8
init_timer(267): 8
init_timer(266): 16384
init_timer(265): 8
init_timer(264): 8
init_timer(253): 8
init_timer(221): 8
init_timer(210): 278544
init_timer(209): 8
init_timer(197): 8
cfg_child_cb_new(829): 64
sr_cfg_init(361): 8
sr_cfg_init(354): 8
sr_cfg_init(347): 8
sr_cfg_init(335): 8
sr_cfg_init(323): 8
qm_shm_lock_init(1202): 8
Total: 6294016
}
Module: snmpstats
{
initInterprocessBuffers(100): 8
initInterprocessBuffers(86): 40
initInterprocessBuffers(85): 40
register_message_code_statistics(281): 576
register_message_code_statistics(278): 576
Total: 1240
}
Module: db_mysql
{
Total: 0
}
Module: permissions
{
domain_name_table_insert(978): 736
addr_hash_table_insert(458): 4096
init_addresses(526): 8
new_domain_name_table(865): 2048
init_addresses(512): 8
new_subnet_table(635): 57456
init_addresses(498): 8
new_addr_hash_table(418): 2048
hash_table_insert(164): 48
hash_table_insert(134): 216
init_trusted(220): 8
new_hash_table(98): 2048
Total: 68728
}
Module: auth
{
Total: 0
}
Module: rr
{
Total: 0
}
Module: dialog
{
dlg_get_iuid_shm_clone(308): 16
new_dlg_var(108): 2160
new_dlg_var(117): 712
new_dlg_var(127): 584
init_dlg_table(297): 131088
init_dlg_table(289): 8
init_dlg_table(282): 8
init_dlg_table(277): 8
init_dlg_timer(57): 8
init_dlg_timer(48): 32
new_dlg_profile(188): 960
Total: 135584
}
Module: ipops
{
Total: 0
}
Module: jsonrpcs
{
Total: 0
}
Module: xlog
{
Total: 0
}
Module: sqlops
{
Total: 0
}
Module: rtpengine
{
shm_str_dup(779): 2808
rtpp_function_call(2627): 3248
rtpengine_hash_table_init(111): 14336
rtpengine_hash_table_init(96): 2048
rtpengine_hash_table_init(85): 1024
rtpengine_hash_table_init(76): 2048
rtpengine_hash_table_init(67): 2048
rtpengine_hash_table_init(58): 32
add_rtpengine_socks(975): 24
add_rtpengine_socks(956): 64
get_rtpp_set(870): 8
get_rtpp_set(861): 56
mod_init(1596): 8
mod_init(1589): 24
mod_init(1576): 8
mod_init(1569): 8
mod_init(1555): 16
Total: 27808
}
Module: registrar
{
Total: 0
}
Module: pv
{
Total: 0
}
Module: textops
{
Total: 0
}
Module: htable
{
ht_cell_new(186): 400288
ht_init_tables(394): 17920
ht_add_table(289): 1560
Total: 419768
}
Module: geoip2
{
Total: 0
}
Module: db_cluster
{
dbcl_init_con(112): 16
Total: 16
}
Module: tm
{
build_cell(334): 873064
prepare_new_uac(532): 4336
relay_reply(2058): 46704
prepare_new_uac(548): 7312
init_tm_stats_child(56): 10752
insert_tmcb(135): 1112
init_tmcb_lists(72): 16
init_tmcb_lists(70): 16
init_hash_table(482): 2097152
Total: 3040464
}
Module: usrloc
{
mem_update_ucontact(275): 360
new_urecord(58): 19848
new_ucontact(96): 76072
shm_str_dup(779): 81856
build_stat_name(57): 64
new_udomain(99): 49152
new_udomain(92): 48
new_dlist(760): 16
new_dlist(752): 32
register_ulcb(94): 32
new_urecord(65): 9160
init_ulcb_list(45): 16
Total: 236656
}
Module: websocket
{
wsconn_add(197): 588836728
wsconn_init(109): 16
wsconn_init(100): 8192
wsconn_init(89): 8
wsconn_init(79): 8
Total: 588844952
}
Module: tls
{
tls_complete_init(230): 15184
ksr_tls_fix_domain(1018): 1480
tls_new_domain(165): 832
tls_new_cfg(1514): 48
tls_ct_wq_init(49): 8
mod_init(362): 8
mod_init(346): 8
ser_realloc(288): 104272
ser_malloc(282): 36830792
lock_set_alloc(66): 184
Total: 36952816
}
Module: pike
{
new_ip_node(201): 3000
pike_init(123): 16
lock_set_alloc(66): 1040
init_ip_tree(115): 4112
pike_init(105): 8
Total: 8176
}
Module: sl
{
init_sl_stats_child(125): 8064
sl_startup(90): 8
init_sl_stats(110): 8
Total: 8080
}
Module: debugger
{
dbg_init_pid_list(589): 23520
dbg_init_bp_list(541): 32
dbg_init_mod_levels(1185): 1024
Total: 24576
}
Module: tmx
{
tmx_init_pretran_table(90): 512
Total: 512
}
Module: kex
{
pkg_proc_stats_init(79): 2184
Total: 2184
}
Module: uac
{
uac_reg_init_ht(240): 512
uac_reg_init_ht(226): 24
uac_reg_init_ht(214): 512
uac_reg_init_ht(202): 24
uac_reg_init_ht(190): 8
reg_active_init(164): 8
Total: 1088
}
Module: cfgutils
{
mod_init(898): 8
mod_init(892): 8
mod_init(884): 8
Total: 24
}
Module: dialplan
{
mod_init(195): 8
add_rule2hash(591): 192
add_rule2hash(568): 144
dpl_str_to_shm(350): 368
build_rule(502): 720
repl_exp_parse(354): 864
repl_exp_parse(332): 48
reg_ex_comp(400): 1672
init_data(207): 8
init_data(200): 16
mod_init(162): 24
Total: 4064
}
Module: domain
{
hash_table_install(147): 24
hash_table_install(138): 24
hash_table_install(131): 48
mod_init(206): 8
mod_init(196): 1032
mod_init(194): 1032
mod_init(192): 24
Total: 2192
}
Module: exec
{
initialize_kill(149): 8
Total: 8
}
On Mon, 8 Feb 2021 at 17:56, Daniel-Constantin Mierla <miconda(a)gmail.com>
wrote:
Hello,
can you provide more details about what memory leak you talk? Is it about
shared memory (shm) or private memory (pkg) managed by kamailio or by the
system memory? How do you observed there is a leak? Did you get error
messages in Kamailio logs? If yes, can you paste them here?
Cheers,
Daniel
On 08.02.21 18:34, Uttam Giri wrote:
Hi,
We have observed increased memory utilization in Kamailio, It increases
every day, even though traffic is not increased. We are using
Kamailio+RTPEngine in our setup.
Phone---->Kamailio+RTPEngine------>Asterisk
/etc/kamailio# kamcmd mod.stats all shm
Module: core
{
sip_msg_shm_clone(496): 542688
msg_lump_cloner(986): 178776
build_req_buf_from_sip_req(2218): 111968
xavp_new_value(116): 22240
counters_prefork_init(212): 39680
cfg_clone_str(130): 40
cfg_shmize(217): 712
main_loop(1313): 8
init_pt(107): 8
init_pt(106): 8
init_pt(105): 4592
register_timer(995): 192
init_tcp(4962): 8192
init_tcp(4956): 32768
init_tcp(4947): 8
init_tcp(4940): 8
init_tcp(4933): 8
init_tcp(4927): 8
init_tcp(4915): 8
init_avps(90): 8
init_avps(89): 8
timer_alloc(498): 48
init_dst_blacklist(438): 16384
init_dst_blacklist(430): 8
init_timer(267): 8
init_timer(266): 16384
init_timer(265): 8
init_timer(264): 8
init_timer(253): 8
init_timer(221): 8
init_timer(210): 278544
init_timer(209): 8
init_timer(197): 8
cfg_child_cb_new(829): 64
sr_cfg_init(361): 8
sr_cfg_init(354): 8
sr_cfg_init(347): 8
sr_cfg_init(335): 8
sr_cfg_init(323): 8
qm_shm_lock_init(1202): 8
Total: 1253464
}
Module: sl
{
init_sl_stats_child(125): 5952
sl_startup(90): 8
init_sl_stats(110): 8
Total: 5968
}
Module: siptrace
{
mod_init(319): 8
Total: 8
}
Module: permissions
{
Total: 0
}
Module: xlog
{
Total: 0
}
Module: rtpengine
{
rtpp_function_call(2627): 1076024
shm_str_dup(779): 553016
rtpengine_hash_table_init(111): 14336
rtpengine_hash_table_init(96): 2048
rtpengine_hash_table_init(85): 1024
rtpengine_hash_table_init(76): 2048
rtpengine_hash_table_init(67): 2048
rtpengine_hash_table_init(58): 32
add_rtpengine_socks(975): 24
add_rtpengine_socks(956): 64
get_rtpp_set(870): 8
get_rtpp_set(861): 56
mod_init(1596): 8
mod_init(1589): 24
mod_init(1576): 8
mod_init(1569): 8
mod_init(1555): 16
Total: 1650792
}
Module: pv
{
add_shvar(177): 8
add_shvar(170): 64
lock_set_alloc(66): 80
Total: 152
}
Module: phonenum
{
Total: 0
}
Module: textops
{
Total: 0
}
Module: rtimer
{
Total: 0
}
Module: tm
{
_reply_light(496): 1640
build_cell(334): 973848
relay_reply(2058): 69376
init_tm_stats_child(56): 7936
init_tmcb_lists(72): 16
init_tmcb_lists(70): 16
init_hash_table(482): 2097152
Total: 3149984
}
Module: kex
{
pkg_proc_stats_init(79): 1488
Total: 1488
}
Module: tmx
{
tmx_init_pretran_table(90): 256
Total: 256
}
Module: utils
{
conf_init(552): 24
init_shmlock(134): 8
Total: 32
}
Module: xhttp_prom
{
prom_metric_init(134): 8
Total: 8
}
Module: dispatcher
{
ds_ht_init(89): 12288
reindex_dests(723): 768
mod_init(484): 80
ds_avl_insert(3621): 1712
pack_dest(411): 72
ds_ht_init(79): 32
ds_init_data(262): 16
ds_init_data(254): 16
mod_init(339): 8
mod_init(337): 8
ds_ping_active_init(138): 8
Total: 15008
}
/etc/kamailio# kamcmd mod.stats all pkg
Module: core
{
init_io_wait(469): 6192
init_modules(885): 16
rpc_hash_add(151): 1024
cnt_hash_add(394): 2048
add_callback(59): 168
register_select_table(458): 48
sr_wtimer_init(351): 136
cfg_new_group(79): 624
fix_sock_str(517): 1024
fix_sock_str(500): 240
add_alias(103): 96
add_alias(99): 96
init_dst_set(89): 32296
grp_hash_add(241): 512
fix_hostname(1457): 56
fix_socket_list(1666): 24
route_new_list(199): 184
fix_param(1114): 88
fix_param(1053): 312
fix_expr(554): 192
init_io_wait(516): 3224
subst_str(514): 720
mk_rval_expr2(2681): 17360
mk_elem(90): 232
mk_new_net(52): 48
tr_new(1730): 240
mk_rval_expr_v(2553): 216
yyparse(2852): 736
mk_rval_expr1(2621): 19096
mk_rval_expr_v(2540): 109152
pv_cache_add(349): 5616
pv_parse_format(1173): 4984
fix_param(1019): 2568
mk_action(118): 44416
cfg_declare(50): 3136
set_mod_param_regex(145): 88
set_mod_param_regex(134): 104
tr_table_add(1959): 360
register_module(233): 28784
register_module(216): 2808
load_module(417): 1552
sr_push_yy_state(1627): 1008
new_sock_info(348): 16
new_sock_info(339): 32
new_sock_info(327): 32
new_sock_info(304): 48
new_sock_info(299): 1200
add_alias(103): 56
add_alias(99): 64
yyparse(677): 40
yyparse(2387): 128
yyparse(658): 144
yyparse(1878): 168
get_hdr_field(116): 232
parse_headers(325): 64
subst_parser(301): 120
subst_parser(291): 1440
subst_parser(274): 640
pp_subst_add(58): 240
sr_push_yy_state(1697): 24
addstr(1445): 69072
rpc_hash_add(105): 16280
str_hash_alloc(59): 512
pv_table_add(236): 17248
pv_init_buffer(2139): 327680
pv_init_buffer(2129): 320
init_nonsip_hooks(43): 24
init_rlist(146): 40
rval_get_str(1259): 3696
route_add(124): 968
str_hash_alloc(59): 768
sr_push_yy_state(1687): 464
pp_define(1819): 568
init_counters(125): 128
cnt_hash_add(341): 27096
str_hash_alloc(59): 1280
Total: 762656
}
Module: sl
{
sl_register_callback(514): 64
Total: 64
}
Module: siptrace
{
mod_init(340): 384
Total: 384
}
Module: permissions
{
get_pathname(243): 64
Total: 64
}
Module: xlog
{
mod_init(212): 4104
xdbg_fixup_helper(498): 616
xlog_fixup_helper(533): 3720
Total: 8440
}
Module: rtpengine
{
child_init(1964): 240
build_rtpp_socks(1791): 8
Total: 248
}
Module: pv
{
pv_parse_xavp_name(541): 112
pv_parse_xavp_name(515): 112
tr_parse_string(2639): 120
tr_parse_string(2628): 120
tr_parse_string(2519): 104
tr_parse_string(2509): 64
tr_parse_string(2484): 64
tr_parse_string(2473): 64
add_var(65): 176
add_var(58): 344
Total: 1280
}
Module: phonenum
{
pv_parse_phonenum_name(131): 32
sr_phonenum_add_item(99): 8
sr_phonenum_add_item(93): 304
Total: 344
}
Module: textops
{
fixup_method(3598): 128
hname_fixup(3528): 504
Total: 632
}
Module: rtimer
{
stm_e_param(384): 112
stm_t_param(299): 48
Total: 160
}
Module: tm
{
Total: 0
}
Module: kex
{
Total: 0
}
Module: tmx
{
Total: 0
}
Module: utils
{
Total: 0
}
Module: xhttp_prom
{
Total: 0
}
Module: dispatcher
{
Total: 0
}
/etc/kamailio# kamailio -v
version: kamailio 5.4.3 (x86_64/linux)
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS,
DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC,
F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT,
USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST,
HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024,
BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: unknown
compiled with gcc 6.3.0
Any idea which could cause a memory leak. Does any of the module cause an
issue?
Thanks in Advance.
Uttam
_______________________________________________
Kamailio (SER) - Users Mailing
Listsr-users@lists.kamailio.orghttps://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierla -- www.asipto.comwww.twitter.com/miconda --
www.linkedin.com/in/miconda
Funding: https://www.paypal.me/dcmierla
_______________________________________________
Kamailio (SER) - Users Mailing List
sr-users(a)lists.kamailio.org
https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users