[SR-Users] Segfault on kamailio 5.0.2. Suddenly stops processing traffic, then generates a core when restarting

[Joel Serrano]

Hi all,

I'm running into a weird situation with our newly deployed Kamailio
servers. These servers are only doing SSL offloading and load balancing
between another set of servers.

Debian v9.0 + Kamailio v5.0.2 (from kamailio50 repository).

After some time of running, kamailio suddenly stops responding to any
request but the proccess is still alive. When we restart the service, a
core is generated.

I have opened an issue in GitHub:
https://github.com/kamailio/kamailio/issues/1172

I'm attaching the backtrace here too in case anyone can take a look at it,
unfortunately I'm lost right now.

In dmesg I saw:

[138953.744199] kamailio[12052]: segfault at 7efaf32f51b8 ip
00007efcf3b5df69 sp 00007ffe325dc0c0 error 4 in
libcrypto.so.1.1[7efcf39f6000+265000]


Thanks,
Joel.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.kamailio.org/pipermail/sr-users/attachments/20170628/4fc1f0c2/attachment.html>
-------------- next part --------------
[New LWP 12052]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f /etc/kamailio/sbc.cfg -'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00007efcf3b5df69 in ?? () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
#0  0x00007efcf3b5df69 in ?? () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#1  0x00007efcf3b5e234 in OPENSSL_cleanup () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#2  0x00007efcf7721910 in __run_exit_handlers (status=0, listp=0x7efcf7a855d8 <__exit_funcs>, run_list_atexit=run_list_atexit at entry=true, run_dtors=run_dtors at entry=true) at exit.c:83
        atfct = <optimized out>
        onfct = <optimized out>
        cxafct = <optimized out>
        f = <optimized out>
#3  0x00007efcf772196a in __GI_exit (status=<optimized out>) at exit.c:105
No locals.
#4  0x000055f0a9483d54 in handle_sigs () at main.c:699
        chld = 845005216
        chld_status = 845005248
        any_chld_stopped = 32766
        memlog = 0
        __func__ = "handle_sigs"
#5  0x000055f0a948ea43 in main_loop () at main.c:1758
        i = 8
        pid = 12091
        si = 0x0
        si_desc = "udp receiver child=7 sock=[2602:FF37:0:1:0:0:C601:377A]:5060\000\367v\215\200\262+\363\372~\000\000\025\067\236\003\000\000\000\000\300\017H\251\360U\000\000\340\305]2\376\177", '\000' <repeats 18 times>, "\020\303]2\376\177\000\000a\202g\251\360U\000"
        nrprocs = 8
        woneinit = 1
        __func__ = "main_loop"
#6  0x000055f0a9494fb9 in main (argc=13, argv=0x7ffe325dc5e8) at main.c:2646
        cfg_stream = 0x55f0aa7fb010
        c = -1
        r = 0
        tmp = 0x7ffe325dceb6 ""
        tmp_len = -133383536
        port = 32508
        proto = 845006016
        options = 0x55f0a97d80e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
        ret = -1
        seed = 3679304743
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x0
        p = 0xffffffff <error: Cannot access memory at address 0xffffffff>
        st = {st_dev = 19, st_ino = 14769, st_nlink = 2, st_mode = 16832, st_uid = 109, st_gid = 113, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1498523393, tv_nsec = 731915583}, st_mtim = {tv_sec = 1498533865, tv_nsec = 274210743}, st_ctim = {tv_sec = 1498533865, tv_nsec = 274210743}, __glibc_reserved = {0, 0, 0}}
        __func__ = "main"