13 May
2022
13 May
'22
6:58 a.m.
### Description
Crash occurred during load test.
#### Reproduction
This cannot be reproduced at will.
It happened after about 45 days of load testing.
#### Debugging Data
```
[root@lab002207-flip-server ~]$ gdb /usr/local/src/git/kamailio-5.5/src/kamailio /core
GNU gdb (Debian 10.1-1.7) 10.1.90.20210103-git
Copyright (C) 2021 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<https://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /usr/local/src/git/kamailio-5.5/src/kamailio...
warning: Can't open file /dev/zero (deleted) during file-backed mapping note
processing
[New LWP 189001]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/src/git/kamailio-5.5/src/kamailio -m 1024 -f
/usr/local/etc/kamailio'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f909231e31e in atomic_get (v=0x7f96781153a0) at
../../core/mem/../atomic/atomic_common.h:66
66 return atomic_get_int(&(v->val));
(gdb) bt full
#0 0x00007f909231e31e in atomic_get (v=0x7f96781153a0) at
../../core/mem/../atomic/atomic_common.h:66
No locals.
#1 0x00007f909232b1a4 in dlg_ref_helper (dlg=0x7f905640bbe0, cnt=1, fname=0x7f9092368a61
"dlg_handlers.c", fline=1057) at dlg_hash.c:1074
mypid = 189001
d_entry = 0x7f9678115388
__func__ = "dlg_ref_helper"
#2 0x00007f9092311fbf in dlg_set_tm_waitack (t=0x7f9053a98af8, dlg=0x7f905640bbe0) at
dlg_handlers.c:1057
iuid = 0x7f905739a3c8
__func__ = "dlg_set_tm_waitack"
#3 0x00007f909230c412 in dlg_onreply (t=0x7f9053a98af8, type=1048576,
param=0x7ffeb3d73300) at dlg_handlers.c:578
dlg = 0x7f905640bbe0
iuid = 0x7f9055715ea0
new_state = 5
old_state = 2
unref = 1
event = 4
tag = {s = 0x80d000001ff <error: Cannot access memory at address
0x80d000001ff>, len = -1832753723}
req = 0x7f90555eff70
rpl = 0xffffffffffffffff
__func__ = "dlg_onreply"
#4 0x00007f9092b5d399 in run_trans_callbacks_internal (cb_lst=0x7f9053a98b70,
type=1048576, trans=0x7f9053a98af8, params=0x7ffeb3d73300) at t_hooks.c:258
cbp = 0x7f905362d490
backup_from = 0x555a7cfc3e90 <def_list+16>
backup_to = 0x555a7cfc3e98 <def_list+24>
backup_dom_from = 0x555a7cfc3ea0 <def_list+32>
backup_dom_to = 0x555a7cfc3ea8 <def_list+40>
backup_uri_from = 0x555a7cfc3e80 <def_list>
backup_uri_to = 0x555a7cfc3e88 <def_list+8>
backup_xavps = 0x555a7cfc3fd8 <_xavp_list_head>
backup_xavus = 0x555a7cfc3fe0 <_xavu_list_head>
backup_xavis = 0x555a7cfc3fe8 <_xavi_list_head>
__func__ = "run_trans_callbacks_internal"
#5 0x00007f9092b5d5b2 in run_trans_callbacks_with_buf (type=1048576, rbuf=0x7f9053a98bc8,
req=0x7f90555eff70, repl=0xffffffffffffffff, flags=0) at t_hooks.c:303
params = {req = 0x7f90555eff70, rpl = 0xffffffffffffffff, param = 0x7f905362d4a0,
code = 408, flags = 0, branch = 0, t_rbuf = 0x7f9053a98bc8, dst = 0x7f9053a98c18, send_buf
= {
s = 0x7f9054157d60 "SIP/2.0 408 Request Timeout\r\nVia: SIP/2.0/UDP
10.255.255.208:5015;rport=5015;branch=z9hG4bK-2093375-4498-18;received=192.168.2.253\r\nFrom:
\"the.username.for.the.user.546\" <sip:the.username.for.the.user"...,
len = 449}}
trans = 0x7f9053a98af8
#6 0x00007f9092bd422d in relay_reply (t=0x7f9053a98af8, p_msg=0xffffffffffffffff,
branch=0, msg_status=408, cancel_data=0x7ffeb3d73520, do_put_on_wait=0) at
t_reply.c:2094
relay = 0
save_clone = 0
buf = 0x7f9092f17b68 "SIP/2.0 408 Request Timeout\r\nVia: SIP/2.0/UDP
10.255.255.208:5015;rport=5015;branch=z9hG4bK-2093375-4498-18;received=192.168.2.253\r\nFrom:
\"the.username.for.the.user.546\" <sip:the.username.for.the.user"...
res_len = 449
relayed_code = 408
relayed_msg = 0xffffffffffffffff
reply_bak = 0x7ffeb3d734c0
bm = {to_tag_val = {
s = 0x7f9092f17c88 "67946a7773289cd2c8623ce9ed050d66-614e90ad\r\nCall-ID:
4498-2093375(a)10.255.255.208\r\nCSeq: 802 INVITE\r\nServer: kamailio (5.5.4
(x86_64/linux))\r\nContent-Length: 0\r\n\r\n", len = 41}}
totag_retr = 0
reply_status = RPS_COMPLETED
uas_rb = 0x7f9053a98bc8
to_tag = 0x7f9092c3db30 <tm_tag>
reason = {s = 0x555a7ce2b463 "Request Timeout", len = 15}
--Type <RET> for more, q to quit, c to continue without paging--c
onsend_params = {req = 0x28, rpl = 0x7f9052b1eac0, param = 0x157b82568, code =
1365092040, flags = 32656, branch = 0, t_rbuf = 0x7ffeb3d73450, dst = 0x0, send_buf = {s =
0x21d8 <error: Cannot access memory at address 0x21d8>, len = 1365092040}}
ip = {af = 3017225408, len = 32766, u = {addrl = {140257839409728, 1}, addr32 =
{1387391552, 32656, 1, 0}, addr16 = {59968, 21169, 32656, 0, 1, 0, 0, 0}, addr =
"@\352\261R\220\177\000\000\001\000\000\000\000\000\000"}}
__func__ = "relay_reply"
#7 0x00007f9092b61ec0 in fake_reply (t=0x7f9053a98af8, branch=0, code=408) at
timer.c:295
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len =
1403620088}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 1403620088}}}}
do_cancel_branch = 1
reply_status = 189001
#8 0x00007f9092b6232e in final_response_handler (r_buf=0x7f9053a98d98, t=0x7f9053a98af8)
at timer.c:462
silent = 0
branch_ret = 0
prev_branch = 0
now = 0
#9 0x00007f9092b623f2 in retr_buf_handler (ticks=2176292369, tl=0x7f9053a98db8, p=0x1f40)
at timer.c:518
rbuf = 0x7f9053a98d98
fr_remainder = 0
retr_remainder = 32656
retr_interval = 1365093472
new_retr_interval_ms = 4294967296
crt_retr_interval_ms = 0
t = 0x7f9053a98af8
__func__ = "retr_buf_handler"
#10 0x0000555a7ccb5a9f in timer_list_expire (t=2176292369, h=0x7f905161ed40,
slow_l=0x7f905161fab8, slow_mark=5300) at core/timer.c:857
tl = 0x7f9053a98db8
ret = 0
#11 0x0000555a7ccb5fa9 in timer_handler () at core/timer.c:922
saved_ticks = 2176292369
run_slow_timer = 0
i = 180
__func__ = "timer_handler"
#12 0x0000555a7ccb64ac in timer_main () at core/timer.c:961
No locals.
#13 0x0000555a7c9fe6cf in main_loop () at main.c:1839
i = 12
pid = 0
si = 0x0
si_desc = "udp receiver child=11
sock=192.168.2.207:9060\000\270\000\200\070׳\376\177\000\000\000\000\000\000\000\000\000\000\220\070׳\376\177\000\000);]\221\220\177\000\000\b$Β\220\177\000\000]\201]\221\220\177",
'\000' <repeats 14 times>,
"\001\000\000\000\220\070׳\376\177\000\000\342\f\320|ZU\000"
nrprocs = 12
woneinit = 1
__func__ = "main_loop"
#14 0x0000555a7ca092ab in main (argc=8, argv=0x7ffeb3d73e78) at main.c:3053
cfg_stream = 0x555a7ea552d0
c = -1
r = 0
tmp = 0x7ffeb3d74d0a ""
tmp_len = 0
port = 0
proto = 0
ahost = 0x0
aport = 0
options = 0x555a7ce150b8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 487513963
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x0
p = 0xc2 <error: Cannot access memory at address 0xc2>
st = {st_dev = 22, st_ino = 2337, st_nlink = 2, st_mode = 16877, st_uid = 0,
st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 60, st_blksize = 4096, st_blocks = 0,
st_atim = {tv_sec = 1646805011, tv_nsec = 284331240}, st_mtim = {tv_sec = 1647223778,
tv_nsec = 289750340}, st_ctim = {tv_sec = 1647223782, tv_nsec = 957818051},
__glibc_reserved = {0, 0, 0}}
tbuf =
"\020\377y\223\220\177\000\000\300\nG\223\001\000\000\000\377\377\377\377",
'\000' <repeats 12 times>,
"(\346z\223\220\177\000\000\350\231}\223\220\177\000\000\377\377\377\377",
'\000' <repeats 12 times>,
"@\305E\223\220\177\000\000\020\004z\223\220\177\000\000\350\244}\223\220\177\000\000\204\351z\223\220\177\000\000\060\344z\223\220\177\000\000Xbc\223\220\177\000\000h\231}\223\220\177\000\000`\220}\223\220\177\000\000\300>׳\376\177\000\000\200\241}\223\220\177\000\000\000\000\000\000\000\000\000\000#\326{\223\220\177\000\000\001",
'\000' <repeats 23 times>,
"(\346z\223\220\177\000\000\320<׳\376\177\000\000\003E"...
option_index = 12
long_options = {{name = 0x555a7ce17516 "help", has_arg = 0, flag = 0x0,
val = 104}, {name = 0x555a7ce12514 "version", has_arg = 0, flag = 0x0, val =
118}, {name = 0x555a7ce1751b "alias", has_arg = 1, flag = 0x0, val = 1024},
{name = 0x555a7ce17521 "subst", has_arg = 1, flag = 0x0, val = 1025}, {name =
0x555a7ce17527 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name =
0x555a7ce17530 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name =
0x555a7ce1753a "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name =
0x555a7ce17544 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name =
0x555a7ce1754f "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name =
0x555a7ce17558 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name =
0x555a7ce17563 "debug", has_arg = 1, flag = 0x0, val = 1032}, {name =
0x555a7ce17569 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name =
0x555a7ce17573 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0,
has_arg = 0, flag = 0x0, val = 0}}
__func__ = "main"
(gdb) info locals
No locals.
(gdb) list
61
62 #define atomic_set(at_var, value) (atomic_set_int(&((at_var)->val), (value)))
63
64 inline static int atomic_get(atomic_t *v)
65 {
66 return atomic_get_int(&(v->val));
67 }
68
69 /*@} */
70
(gdb)
```
### Additional Information
* **Kamailio Version** - output of `kamailio -v`
```
version: kamailio 5.5.4 (x86_64/linux) 54c9df
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST,
DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY,
USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR,
USE_DST_BLOCKLIST, HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535,
DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: 54c9df
compiled on 10:27:27 Mar 14 2022 with gcc 10.2.1
```
* **Operating System**:
```
[root@lab002207-flip-server ~]$ lsb_release -a
uNo LSB modules are available.
Distributor ID: Debian
Description: Debian GNU/Linux 11 (bullseye)
Release: 11
Codename: bullseye
[root@lab002207-flip-server ~]$ uname -a
Linux lab002207-flip-server 5.10.0-11-amd64 #1 SMP Debian 5.10.92-2 (2022-02-28) x86_64
GNU/Linux
```
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3107
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3107(a)github.com>