4 Jul
2024
4 Jul
'24
9:41 a.m.
### Description
Kamailio version 5.7.4 crashed with a core file. Maybe it's related to the dmq module.
#### Debugging Data
```
(gdb) bt full
#0 atomic_inc_int (var=0x7f5700000000) at
../../core/parser/../mem/../atomic/atomic_x86.h:236
No locals.
#1 cfg_update_local (no_cbs=<optimized out>) at ../../core/cfg/cfg_struct.h:381
group = <optimized out>
last_cb = 0x7f57c99545e0
prev_cb = <optimized out>
group = <optimized out>
last_cb = <optimized out>
prev_cb = <optimized out>
__func__ = "cfg_update_local"
#2 worker_loop (id=id@entry=2) at worker.c:57
worker = <optimized out>
current_job = <optimized out>
peer_response = {resp_code = 1, content_type = {s = 0x55ea7b42cfca
<init_mod_child+90> "\205\300\017\210\r\002", len = 15}, reason = {s =
0x0, len = 2}, body = {
s = 0x7ffcf6347c30 "\001", len = 2069833640}}
ret_value = <optimized out>
not_parsed = <optimized out>
dmq_node = 0x0
__func__ = "worker_loop"
#3 0x00007f57d9426a11 in child_init (rank=<optimized out>) at dmq.c:321
i = 2
newpid = <optimized out>
__func__ = "child_init"
#4 0x000055ea7b42cfca in init_mod_child (m=0x7f57dd005640, rank=rank@entry=0) at
core/sr_module.c:911
ret = 0
__func__ = "init_mod_child"
#5 0x000055ea7b42cfa4 in init_mod_child (m=0x7f57dd005b90, rank=rank@entry=0) at
core/sr_module.c:903
ret = <optimized out>
__func__ = "init_mod_child"
#6 0x000055ea7b42cfa4 in init_mod_child (m=0x7f57dd006910, rank=rank@entry=0) at
core/sr_module.c:903
ret = <optimized out>
__func__ = "init_mod_child"
#7 0x000055ea7b42cfa4 in init_mod_child (m=0x7f57dd006e70, rank=rank@entry=0) at
core/sr_module.c:903
ret = <optimized out>
__func__ = "init_mod_child"
#8 0x000055ea7b42cfa4 in init_mod_child (m=0x7f57dd008660, rank=rank@entry=0) at
core/sr_module.c:903
ret = <optimized out>
__func__ = "init_mod_child"
#9 0x000055ea7b42cfa4 in init_mod_child (m=0x7f57dd0089d0, rank=rank@entry=0) at
core/sr_module.c:903
ret = <optimized out>
__func__ = "init_mod_child"
#10 0x000055ea7b42cfa4 in init_mod_child (m=0x7f57dd008d70, rank=rank@entry=0) at
core/sr_module.c:903
ret = <optimized out>
__func__ = "init_mod_child"
#11 0x000055ea7b43239e in init_child (rank=rank@entry=0) at core/sr_module.c:990
ret = <optimized out>
type = <optimized out>
__func__ = "init_child"
#12 0x000055ea7b288365 in main_loop () at main.c:1929
i = <optimized out>
pid = <optimized out>
si = 0x0
si_desc = "udp receiver child=15
sock=172.20.21.4:5060\000\000\000\000\000H\211\064\366\374\177\000\000\301'C{\352U\000\000\276\000\000\000\002\000\000\000\017\247`{\352U\000\000\062\323b{\352U\000\000U\024\000\000\000\000\000\000\004\000_{\352U\000\000\000\272\030D\344\223ԃ\270Rc{\352U\000\000\004\000\000\000\000\000\000"
nrprocs = <optimized out>
woneinit = 1
__func__ = "main_loop"
error = <optimized out>
#13 0x000055ea7b27be9e in main (argc=<optimized out>, argv=<optimized out>) at
main.c:3213
cfg_stream = <optimized out>
c = <optimized out>
r = <optimized out>
tmp = 0x7ffcf6349e4e ""
tmp_len = -554008000
port = 32599
proto = 0
ahost = 0x0
aport = 0
options = 0x55ea7b5f2cc8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 1892320457
rfd = <optimized out>
debug_save = <optimized out>
debug_flag = <optimized out>
dont_fork_cnt = <optimized out>
n_lst = <optimized out>
p = <optimized out>
st = {st_dev = 0, st_ino = 0, st_nlink = 0, st_mode = 0, st_uid = 0, st_gid = 0,
__pad0 = 0, st_rdev = 0, st_size = 0, st_blksize = 0, st_blocks = 0, st_atim = {tv_sec =
0,
tv_nsec = 0}, st_mtim = {tv_sec = 0, tv_nsec = 0}, st_ctim = {tv_sec = 0,
tv_nsec = 0}, __glibc_reserved = {0, 0, 0}}
tbuf = '\000' <repeats 20 times>, " ", '\000'
<repeats 35 times>, "\006\000\000\000\000\000\000\200", '\000'
<repeats 56 times>, "\006\000\000\000\000\000\000\200", '\000'
<repeats 168 times>...
option_index = 9
long_options = {{name = 0x55ea7b5f12ea "help", has_arg = 0, flag = 0x0,
val = 104}, {name = 0x55ea7b5fa98e "version", has_arg = 0, flag = 0x0, val =
118}, {
name = 0x55ea7b60a690 "alias", has_arg = 1, flag = 0x0, val = 1024},
{name = 0x55ea7b5f12ef "subst", has_arg = 1, flag = 0x0, val = 1025}, {name =
0x55ea7b5f12f5 "substdef",
has_arg = 1, flag = 0x0, val = 1026}, {name = 0x55ea7b5f12fe
"substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x55ea7b5f1308
"server-id", has_arg = 1, flag = 0x0,
val = 1028}, {name = 0x55ea7b5f1312 "loadmodule", has_arg = 1, flag
= 0x0, val = 1029}, {name = 0x55ea7b5f131d "modparam", has_arg = 1, flag = 0x0,
val = 1030}, {
name = 0x55ea7b5f1326 "log-engine", has_arg = 1, flag = 0x0, val =
1031}, {name = 0x55ea7b5faaab "debug", has_arg = 1, flag = 0x0, val = 1032}, {
name = 0x55ea7b5f1331 "cfg-print", has_arg = 0, flag = 0x0, val =
1033}, {name = 0x55ea7b5f133b "atexit", has_arg = 1, flag = 0x0, val = 1034}, {
name = 0x55ea7b5f1342 "all-errors", has_arg = 0, flag = 0x0, val =
1035}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__func__ = "main"
(gdb) info locals
No locals.
(gdb) list
231 in ../../core/parser/../mem/../atomic/atomic_x86.h
```
#### Log Messages
```
Jul 2 15:24:44 hostname kernel: kamailio[1098]: segfault at 7f5700000000 ip
00007f57d94467a0 sp 00007ffcf6347b00 error 6 in dmq.so[7f57d9426000+23000]
```
### Additional Information
* **Kamailio Version** - output of `kamailio -v`
```
version: kamailio 5.7.4 (x86_64/linux)
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST,
DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, MEM_JOIN_FREE, Q_MALLOC, F_MALLOC, TLSF_MALLOC,
DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER,
USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535,
DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id:
compiled with gcc 10.2.1
```
* **Operating System**:
```
Debian: 11.10 - Bullseye
Linux 5.10.0-30-amd64 #1 SMP Debian 5.10.218-1 (2024-06-01) x86_64 GNU/Linux
```
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905(a)github.com>
4 Jul
4 Jul
11:46 a.m.
Do you run some rpc commands to change values for global or module parameters?
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2208437876
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2208437876(a)github.com>
3:29 p.m.
Hi,
no we don't change anything. We use dmq to populate data to another node.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2208855439
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2208855439(a)github.com>
5 Jul
5 Jul
1:21 p.m.
The rpc commands do not have to be about dmq itself, but any other core or module
parameter. Isn't there any rpc command executed to change Kamailio parameter values?
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2210616280
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2210616280(a)github.com>
3:19 p.m.
No. The Kamailio ist "just" doing it's job. But I will double check if we
have RPC calls.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2210777867
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2210777867(a)github.com>
3:26 p.m.
We use the event_route[xhttp:request] to regurarly call jsonrpc_dispatch(). Every 5
minutes to get some data for monitoring. But nothing else. From the DMQ module we use
dmq_handle_message() and dmq_is_from_node().
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2210786915
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2210786915(a)github.com>
9 Jul
9 Jul
2:28 p.m.
Are you using custom global variables (parameters)?
- https://www.kamailio.org/wikidocs/cookbooks/5.8.x/core/#custom-global-param…
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2217402447
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2217402447(a)github.com>
4:49 p.m.
We use the custom global variables. We set them in the first lines of the config (32 - 39)
right after the core paramters but before the loadmodule takes place.
They have the form
`foo.bar="1.2.3.4`
We don't set another value. Not via rpc or other options. We tried that in the past
and the kamailio crashed very often, so we don't do that.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2217800207
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2217800207(a)github.com>
16 Jul
16 Jul
12:38 p.m.
Not being the developers of the custom global variables, I have very little experience
with them, and I couldn't spot anything wrong quickly. The crash seems to be when
checking if one of these variables was changed or not.
As alternative suggestion, if you don't need to change them, then better use `#!define
...`, which can also be accessed with the variable `$def(...)`.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2230457393
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2230457393(a)github.com>
29 Jul
29 Jul
12:17 p.m.
This sounds like we have some work to do. We try to use the defines and check.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2255418065
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2255418065(a)github.com>
30 Jul
30 Jul
12:49 p.m.
We changed the global variables to defines. This is working with the `$def()` option.
But, according to [the
docs](https://www.kamailio.org/wikidocs/cookbooks/devel/core/#custom-global…,
we should be able to use the global variables while Kamailio is running and without
crash.
It seems like it's not the case and this is a failure.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2257940621
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2257940621(a)github.com>
2:39 p.m.
The developer of that part (which was developed by SER project during 2005-2008, when
Kamailio was separate project, then inherited it as part of SIP-Router project merging) is
no longer active. The design is rather complex, without much documentation, but the code
is available and you can try to understand and fix issues that you find there. Or ask for
a refund :-) ...
Besides this potential issue (which is not confirmed, btw, I just suggested alternative
based on code that I am more familiar with and offer same kind of functionality), over the
years a couple of other design flaws were reported related to this component, like
accumulating memory on value changing because of inactive processes that do not get the
chance to update the reference counters.
Independently at that time, Kamailio implemented shared memory variables ($shv(...)) for
cases when change of value at runtime via rpc is needed, or local private memory variables
($var(...)) or defines for variables/values that do not need to be changed. The $shv and
$var can be initialized via pv modparams. Personally I use these ones because I feel more
confident in the code behind them and the ability to fix in case of troubles.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2258143928
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2258143928(a)github.com>
5:28 p.m.
Thanks for the explanation. We will check the shared memory vars for our needs!
Thanks!
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2258494821
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2258494821(a)github.com>
5:28 p.m.
Closed #3905 as completed.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#event-13696981167
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issue/3905/issue_event/13696981167(a)github.com>
5:55 p.m.
The developer of that part (which was developed by SER
project during 2005-2008, when Kamailio was separate project, then inherited it as part of
SIP-Router project merging) is no longer active. The design is rather complex, without
much documentation, but the code is available and you can try to understand and fix issues
that you find there. Or ask for a refund :-) ...
If not even @miconda wants to touch or fix the code, maybe we should consider deprecating
it. At least remove it from the sample configuration as a start.
--
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/3905#issuecomment-2258555354
You are receiving this because you are subscribed to this thread.
Message ID: <kamailio/kamailio/issues/3905/2258555354(a)github.com>
145
days inactive
171
days old
14 comments
3 participants
participants (3)
-
Benjamin -
Daniel-Constantin Mierla
-
Olle E. Johansson