Description

kamailio 5.2.3 crash at free_cell_helper

Troubleshooting

Reproduction

This cannot be reproduced at will.

Debugging Data

[root@VM-110-141-centos /data/coredump]# gdb -c core_kamailio_1676550458 /usr/local/services/kamailio/sbin/kamailio
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.tl2
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/local/services/kamailio/sbin/kamailio...done.
[New LWP 4452]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Missing separate debuginfo for /usr/local/services/kamailio/lib64/kamailio/libjansson.so.4
Try: yum --enablerepo='*debug*' install /usr/lib/debug/.build-id/fc/cf02c89115bf860ddba32e8b703205ffeeabf1.debug
Core was generated by `/usr/local/services/kamailio/sbin/kamailio -f /usr/local/services/kamailio/etc/'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007f8ede8e113a in free_cell_helper (dead_cell=0x7f8eb7dcb1b8, silent=0, fname=0x7f8ede9ece02 "timer.c", fline=689) at h_table.c:182
182     h_table.c: No such file or directory.
Missing separate debuginfos, use: debuginfo-install bash-4.2.46-34.tl2.3.x86_64 cyrus-sasl-lib-2.1.26-23.el7.x86_64 glibc-2.17-323.tl2.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-51.tl2.x86_64 libcom_err-1.42.9-19.tl2.x86_64 libcurl-7.29.0-59.tl2.1.x86_64 libevent-2.0.21-4.el7.x86_64 libgcc-4.8.5-44.tl2.1.x86_64 libidn-1.28-4.el7.x86_64 libselinux-2.5-15.tl2.x86_64 libssh2-1.8.0-4.tl2.x86_64 libstdc++-4.8.5-44.tl2.1.x86_64 mariadb-libs-5.5.68-1.tl2.x86_64 nspr-4.32.0-1.tl2.x86_64 nss-3.67.0-4.tl2.x86_64 nss-softokn-freebl-3.67.0-3.tl2.x86_64 nss-util-3.67.0-1.tl2.x86_64 openldap-2.4.44-24.tl2.x86_64 openssl-libs-1.0.2k-22.tl2.1.x86_64 pcre-8.32-17.el7.x86_64 python-libs-2.7.5-90.tl2.1.x86_64 zlib-1.2.7-19.tl2.x86_64
(gdb) bt full
#0  0x00007f8ede8e113a in free_cell_helper (dead_cell=0x7f8eb7dcb1b8, silent=0, fname=0x7f8ede9ece02 "timer.c", fline=689) at h_table.c:182
        b = 0x7f8ede8decc4 <futex_release+29> "\211E\374\203}\374\002\017\224\300\017\266\300H\205\300t3H\213E\350\307\004$"
        i = 1
        rpl = 0x7fffb5166580
        tt = 0x7fffb51665a0
        foo = 0x100000000
        cbs = 0x7f8e00000001
        cbs_tmp = 0x7f8e00000001
        __FUNCTION__ = "free_cell_helper"
#1  0x00007f8ede99381e in wait_handler (ti=367281959, wait_tl=0x7f8eb7dcb240, data=0x7f8eb7dcb1b8) at timer.c:689
        p_cell = 0x7f8eb7dcb1b8
        ret = 3080640456
        unlinked = 0
        rcount = 1
        __FUNCTION__ = "wait_handler"
#2  0x0000000000487769 in timer_list_expire (t=367281959, h=0x7f8eb7a318a8, slow_l=0x7f8eb7a34d70, slow_mark=42793) at core/timer.c:874
        tl = 0x7f8eb7dcb240
        ret = 0
#3  0x0000000000487bca in timer_handler () at core/timer.c:939
        saved_ticks = 367281959
        run_slow_timer = 0
        i = 809
        __FUNCTION__ = "timer_handler"
#4  0x0000000000488074 in timer_main () at core/timer.c:978
No locals.
#5  0x0000000000425a31 in main_loop () at main.c:1727
        i = 8
        pid = 0
        si = 0x0
        si_desc = "udp receiver child=7 sock=11.177.110.141:35560\000(175.27.32.249:15060)\000\177\000\000\223U{\000\000\000\000\000-\000\000\000\000\000\000\000\000\000\000 \000\000\000\000\000\000\000\004\000\000\000\000\257*\245\346\216\177\000\000\360\260y\000\000\000\000\000\000\227\265\342\216\177\000"
        nrprocs = 8
        woneinit = 1
        __FUNCTION__ = "main_loop"
#6  0x000000000042c72d in main (argc=13, argv=0x7fffb5166bd8) at main.c:2696
        cfg_stream = 0xfdc010
        c = -1
        r = 0
        tmp = 0x7fffb5168757 ""
        tmp_len = 0
        port = 0
        proto = 2496
        options = 0x76a700 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
        ret = -1
        seed = 99772630
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x7fffb5166a90
        p = 0x7f8ee6a00d2d <intel_check_word+173> "H\205\300t\025\017\266P\003\017\266\312D9\371\017\204\316"
        st = {st_dev = 21, st_ino = 32759713, st_nlink = 2, st_mode = 16877, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1671615993, 
            tv_nsec = 68240759}, st_mtim = {tv_sec = 1671620803, tv_nsec = 306084978}, st_ctim = {tv_sec = 1671620803, tv_nsec = 306084978}, __unused = {0, 0, 0}}
        __FUNCTION__ = "main"
(gdb) info locals
cfg_stream = 0xfdc010
c = -1
r = 0
tmp = 0x7fffb5168757 ""
tmp_len = 0
port = 0
proto = 2496
options = 0x76a700 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 99772630
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7fffb5166a90
p = 0x7f8ee6a00d2d <intel_check_word+173> "H\205\300t\025\017\266P\003\017\266\312D9\371\017\204\316"
st = {st_dev = 21, st_ino = 32759713, st_nlink = 2, st_mode = 16877, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1671615993, 
    tv_nsec = 68240759}, st_mtim = {tv_sec = 1671620803, tv_nsec = 306084978}, st_ctim = {tv_sec = 1671620803, tv_nsec = 306084978}, __unused = {0, 0, 0}}
__FUNCTION__ = "main"
(gdb) list
177     in h_table.c
(gdb) 

Log Messages

kamailio.log

Feb 16 20:27:33 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4434]: INFO: {10 OPTIONS 3c7f4fb43ea4086d-4451@11.177.110.141} <script>: keepalive-host is up: OPTIONS sip:9.137.223.152:50180 (<null>)
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4425]: ALERT: <core> [main.c:756]: handle_sigs(): child process 4452 exited by a signal 11
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4425]: ALERT: <core> [main.c:759]: handle_sigs(): core was generated
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4425]: INFO: <core> [main.c:781]: handle_sigs(): terminating due to SIGCHLD
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4470]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4469]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4462]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4466]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4467]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4468]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4464]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4449]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4460]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4465]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4461]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4440]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4459]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4429]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4457]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4430]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4456]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4442]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4432]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4458]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4437]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4444]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4463]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4439]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4454]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4434]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4448]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4451]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4436]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4447]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4450]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4435]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4431]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4441]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4446]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4428]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4445]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4427]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4453]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4438]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4455]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4433]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4443]: INFO: <core> [main.c:836]: sig_usr(): signal 15 received
Feb 16 20:27:41 VM-110-141-centos /usr/local/services/kamailio/sbin/kamailio[4425]: CRITICAL: <core> [core/mem/q_malloc.c:137]: qm_debug_check_frag(): BUG: qm: fragm. 0x7f8eb7d04778 (address 0x7f8eb7d047b0) end overwritten (1, abcdefed)! Memory allocator was called from dialplan: dp_db.c:687. Fragment marked by dialplan: dp_db.c:400. Exec from core/mem/q_malloc.c:504.

messages

[Thu Feb 16 20:27:44 2023] kamailio[4452]: segfault at 7f8e00000021 ip 00007f8ede8e113a sp 00007fffb51664a0 error 4 in tm.so[7f8ede8c5000+13a000]

SIP Traffic

Possible Solutions

Additional Information

[root@VM-110-141-centos /usr/local/services/kamailio/sbin]# ./kamailio -version
version: kamailio 5.2.3 (x86_64/linux) c36229
flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144 MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: c36229 
compiled on 16:50:57 Jul 16 2019 with gcc 4.8.5
[root@VM-110-141-centos /data/coredump]# uname -a
Linux VM-110-141-centos 4.14.105-1-tlinux3-0023 #1 SMP Mon Jan 10 18:17:33 CST 2022 x86_64 x86_64 x86_64 GNU/Linux


Reply to this email directly, view it on GitHub, or unsubscribe.
You are receiving this because you are subscribed to this thread.Message ID: <kamailio/kamailio/issues/3379@github.com>