Description

Kamailio runs out of memory and stops processing requests.

ERROR: <core> [core/tcp_main.c:973]: tcpconn_new(): mem. allocation failure
ERROR: <core> [core/tcp_main.c:4042]: handle_new_connect(): tcpconn_new failed, closing socket
ERROR: uac [uac_reg.c:533]: reg_ht_add(): no more shm
ERROR: uac [uac_reg.c:1347]: uac_reg_load_db(): Error adding reg to htable

Troubleshooting

Restarting the Kamailio process resolves the issue

Reproduction

The issue occurs every 48 hours or so on the Kamailio instance

Debugging Data

#kamcmd mod.stats all pkg
Module: core
{
	init_io_wait(475): 6480
	init_dst_set(83): 32208
	init_io_wait(524): 3576
	yyparse(1782): 8
	mk_new_net(52): 48
	fix_param(1196): 128
	route_new_list(200): 352
	fix_expr(541): 616
	exp_optimize_right(411): 40
	fixup_regexp_null(213): 216
	fix_param(1257): 352
	yyparse(2729): 4872
	mk_rval_expr1(2616): 68256
	mk_elem(90): 976
	mk_rval_expr2(2674): 321160
	mk_rval_expr_v(2548): 768
	mk_rval_expr_v(2537): 808136
	mk_action(118): 196384
	parse_params2(585): 560
	set_mod_param_regex(121): 848
	set_mod_param_regex(110): 264
	tr_table_add(1875): 400
	sr_cmd_exports_convert(235): 34160
	register_module(280): 6192
	load_module(496): 2072
	new_sock_info(235): 176
	new_sock_info(230): 3640
	yyparse(642): 352
	yyparse(1732): 288
	add_alias(93): 80
	add_alias(91): 128
	yyparse(1446): 8
	yyparse(2262): 296
	yyparse(661): 416
	subst_str(514): 416
	subst_parser(301): 264
	subst_parser(291): 2384
	subst_parser(274): 960
	pp_subst_add(57): 312
	pp_define(1766): 1096
	addstr(1399): 261168
	str_hash_alloc(59): 512
	pv_table_add(236): 22616
	parse_pvname_list(1515): 2496
	tr_new(1646): 1512
	fix_match_rve(3028): 2152
	pv_cache_add(347): 20664
	fix_param(1162): 18936
	init_modules(1026): 16
	cnt_hash_add(383): 2048
	db_do_init2(299): 336
	dupl_string_name(72): 32
	dupl_string(48): 144
	new_db_id(272): 288
	get_abs_pathname(262): 80
	rpc_hash_add(146): 2048
	pv_parse_format(1150): 40592
	get_hdr_field(116): 232
	parse_headers(320): 64
	register_select_table(458): 104
	rpc_hash_add(100): 26264
	add_callback(59): 288
	sr_wtimer_init(308): 136
	cfg_new_group(79): 1160
	cfg_declare(50): 4120
	fix_socket_list(1509): 96
	fix_sock_str(420): 912
	fix_hostname(1301): 160
	grp_hash_add(234): 952
	pv_init_buffer(2055): 163840
	pv_init_buffer(2045): 320
	init_nonsip_hooks(43): 24
	init_rlist(146): 40
	route_add(124): 1784
	str_hash_alloc(59): 768
	rval_get_str(1256): 33648
	init_counters(122): 128
	cnt_hash_add(332): 26352
	str_hash_alloc(59): 1280
	Total: 2138200
}

Module: ndb_redis
{
	redis_parse_index(627): 88
	pv_parse_redisc_name(704): 16496
	redisc_add_server(257): 24064
	Total: 40648
}

Module: pv
{
	tr_parse_string(2340): 64
	tr_parse_string(2310): 192
	tr_parse_string(2300): 192
	tr_parse_string(2360): 32
	tr_parse_string(2264): 544
	add_var(65): 512
	add_var(58): 2080
	tr_parse_string(2275): 496
	tr_parse_paramlist(2668): 32
	Total: 4144
}

Module: sqlops
{
	sql_get_result(200): 64
	pv_parse_dbr_name(134): 864
	sql_init_con(83): 240
	sqlops_tr_buffer_init(46): 2048
	Total: 3216
}

Module: textops
{
	tr_txt_parse_re(212): 96
	hname_fixup(2819): 616
	fixup_method(2889): 1256
	Total: 1968
}

Module: xhttp_rpc
{
	child_init(789): 832
	Total: 832
}

Module: db_mysql
{
	db_mysql_new_connection(75): 4088
	db_mysql_new_connection(65): 96
	Total: 4184
}

Module: auth
{
	generate_random_secret(239): 32
	generate_random_secret(238): 32
	Total: 64
}

Module: htable
{
	pv_parse_ht_name(158): 160
	fixup_ht_key(339): 88
	Total: 248
}

Module: xlog
{
	xlog_fixup_helper(535): 17472
	mod_init(214): 4104
	xdbg_fixup_helper(500): 3352
	Total: 24928
}

Module: cnxcc
{
	__alloc_redis(149): 24
	__alloc_redis(146): 40
	Total: 64
}

Module: rtpengine
{
	build_rtpp_socks(1611): 8
	child_init(1768): 240
	Total: 248
}

Module: mangler
{
	compile_expresions(557): 64
	compile_expresions(541): 64
	Total: 128
}

Module: acc
{
	parse_acc_extra(116): 672
	Total: 672
}

Module: rr
{
	register_rrcb(61): 32
	Total: 32
}

Module: dialog
{
	dlg_bridge_init_hdrs(66): 80
	Total: 80
}

Module: tm
{
	Total: 0
}

Module: uac
{
	Total: 0
}

Module: tls
{
	Total: 0
}

Module: sl
{
	Total: 0
}

Module: tmx
{
	Total: 0
}

Module: kex
{
	Total: 0
}

Module: usrloc
{
	Total: 0
}

Module: pike
{
	Total: 0
}

Module: websocket
{
	Total: 0
}

Module: cfgutils
{
	Total: 0
}

Module: nathelper
{
	Total: 0
}

Module: dispatcher
{
	Total: 0
}

Log Messages

kamailio[6722]: ERROR: <core> [core/tcp_main.c:973]: tcpconn_new(): mem. allocation failure
kamailio[6722]: ERROR: <core> [core/tcp_main.c:4042]: handle_new_connect(): tcpconn_new failed, closing socket
kamailio[6722]: ERROR: tm [t_msgbuilder.c:1594]: build_uac_req(): no more shared memory (544)

Possible Solutions

Restarting Kamailio

Additional Information

• Kamailio Version - output of kamailio -v

version: kamailio 5.1.7 (x86_64/linux) 567df3
flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144 MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: 567df3
compiled on 04:30:41 Mar 14 2019 with gcc 7.3.0

• Operating System:

cat /etc/lsb-release
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=18.04
DISTRIB_CODENAME=bionic
DISTRIB_DESCRIPTION="Ubuntu 18.04.2 LTS"

4.15.0-1034-aws #36-Ubuntu SMP Tue Mar 5 23:17:16 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux

 cat /proc/meminfo
MemTotal:        4038176 kB
MemFree:          130020 kB
MemAvailable:     747324 kB
Buffers:           15136 kB
Cached:          2851248 kB
SwapCached:            0 kB
Active:           930936 kB
Inactive:        2672836 kB
Active(anon):     737672 kB
Inactive(anon):  2097272 kB
Active(file):     193264 kB
Inactive(file):   575564 kB
Unevictable:           0 kB
Mlocked:               0 kB
SwapTotal:             0 kB
SwapFree:              0 kB
Dirty:               120 kB
Writeback:             0 kB
AnonPages:        734064 kB
Mapped:          2180672 kB
Shmem:           2097544 kB
Slab:             154288 kB
SReclaimable:     101108 kB
SUnreclaim:        53180 kB
KernelStack:        4176 kB
PageTables:        81940 kB
NFS_Unstable:          0 kB
Bounce:                0 kB
WritebackTmp:          0 kB
CommitLimit:     2019088 kB
Committed_AS:    4529756 kB
VmallocTotal:   34359738367 kB
VmallocUsed:           0 kB
VmallocChunk:          0 kB
HardwareCorrupted:     0 kB
AnonHugePages:         0 kB
ShmemHugePages:        0 kB
ShmemPmdMapped:        0 kB
CmaTotal:              0 kB
CmaFree:               0 kB
HugePages_Total:       0
HugePages_Free:        0
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
DirectMap4k:      221184 kB
DirectMap2M:     3973120 kB

Also, here is the output of "grep alloc qm_status.txt | awk '{ print substr( $0, 16, length($0) ) }' | sort | uniq -c | sort -k1n" (as per https://www.kamailio.org/wiki/tutorials/troubleshooting/memory):

https://www.dropbox.com/s/act3whe6gumyzvj/alloc.txt?dl=1

—
You are receiving this because you are subscribed to this thread.
Reply to this email directly, view it on GitHub, or mute the thread.