### Description SEGV coredump on Kamailio stop or restart -- this doesn't happen every time, but about 90% of the time. ``` Stack trace of thread 12773: #0 0x00007f02de3ad286 __GI___strlen_sse2 (libc.so.6) #1 0x00007f02de385835 fputs_unlocked (libc.so.6) #2 0x00007f02de4132b1 syslog (libc.so.6) #3 0x000000000042aa60 main (kamailio) #4 0x00007f02de32400a __libc_start_main (libc.so.6) #5 0x000000000041800a _start (kamailio) ``` ### Troubleshooting I do not use the `xlog` module and use `log_systemd` module functions within the script but do not set `log_engine_type="systemd"` due to issue #799. Module loading order ``` ~]# grep loadmod /etc/kamailio/kamailio.cfg loadmodule "log_systemd.so" loadmodule "tls.so" loadmodule "db_postgres.so" loadmodule "jsonrpcs.so" loadmodule "kex.so" loadmodule "corex.so" loadmodule "tm.so" loadmodule "tmx.so" loadmodule "sl.so" loadmodule "rr.so" loadmodule "pv.so" loadmodule "maxfwd.so" loadmodule "usrloc.so" loadmodule "registrar.so" loadmodule "textops.so" loadmodule "siputils.so" #loadmodule "xlog.so" loadmodule "sanity.so" loadmodule "ctl.so" loadmodule "cfg_rpc.so" loadmodule "acc.so" loadmodule "counters.so" loadmodule "sqlops.so" loadmodule "auth.so" loadmodule "auth_db.so" loadmodule "permissions.so" loadmodule "alias_db.so" loadmodule "speeddial.so" loadmodule "domain.so" loadmodule "presence.so" loadmodule "presence_conference.so" loadmodule "presence_dialoginfo.so" loadmodule "presence_mwi.so" loadmodule "presence_reginfo.so" loadmodule "presence_xml.so" loadmodule "pua.so" loadmodule "pua_usrloc.so" loadmodule "pua_dialoginfo.so" loadmodule "nathelper.so" loadmodule "rtpengine.so" loadmodule "ipops.so" loadmodule "sdpops.so" loadmodule "htable.so" loadmodule "pike.so" loadmodule "xmlrpc.so" loadmodule "debugger.so" loadmodule "dialog.so" loadmodule "group.so" loadmodule "uac.so" loadmodule "imc.so" loadmodule "lcr.so" loadmodule "app_python.so" loadmodule "textopsx.so" loadmodule "msilo.so" loadmodule "tcpops.so" loadmodule "topos.so" loadmodule "diversion.so" loadmodule "tmrec.so" loadmodule "xhttp.so" loadmodule "websocket.so" loadmodule "auth_ephemeral.so" ``` systemd unit file ``` [Unit] Description=Kamailio SIP Server Documentation=man:kamailio(8) https://www.kamailio.org/ After=network.target [Service] Type=forking User=kamailio Environment=SHM_SIZE=64 Environment=PKG_SIZE=8 ExecStartPre=/usr/sbin/kamailio -c ExecStart=/usr/sbin/kamailio -m $SHM_SIZE -M $PKG_SIZE -P /run/kamailio/kamailio.pid ExecStopPost=/usr/bin/rm -f /run/kamailio/kamailio.pid PIDFile=/run/kamailio/kamailio.pid Restart=on-failure RestartSec=30 [Install] WantedBy=multi-user.target ``` #### Reproduction The issue usually occurs when stopping or restarting Kamailio #### Debugging Data ``` #0 __strlen_sse2 () at ../sysdeps/x86_64/multiarch/../strlen.S:120 No locals. #1 0x00007f02de385835 in __GI___fputs_unlocked (str=0x7f02ddb597c8 <error: Cannot access memory at address 0x7f02ddb597c8>, fp=fp@entry=0x131ceb0) at iofputs_u.c:34 len = <optimized out> result = <optimized out> #2 0x00007f02de4132b1 in __GI___vsyslog_chk (ap=0x7ffe7e661398, fmt=0x742598 "%s: %s%s(): error sending exit status: %s [%d]\n", flag=-1, pri=<optimized out>) at ../misc/syslog.c:205 buf = 0x0 now_tm = {tm_sec = 49, tm_min = 29, tm_hour = 12, tm_mday = 24, tm_mon = 1, tm_year = 118, tm_wday = 6, tm_yday = 54, tm_isdst = 0, tm_gmtoff = -21600, tm_zone = 0x11a2350 "CST"} fd = <optimized out> failbuf = "\000\000\000\000\005\000\000\000\216\247J\336\002\177", '\000' <repeats 14 times> now = 1519496989 f = 0x131ceb0 bufsize = 0 msgoff = 20 saved_errno = 32 #3 __syslog (pri=<optimized out>, fmt=0x742598 "%s: %s%s(): error sending exit status: %s [%d]\n") at ../misc/syslog.c:117 ap = {{gp_offset = 16, fp_offset = 48, overflow_arg_area = 0x7ffe7e661500, reg_save_area = 0x7ffe7e661410}} #4 0x000000000042aa60 in main (argc=7, argv=0x7ffe7e6617a8) at main.c:2664 __llevel = -1 cfg_stream = 0x10fb260 c = -1 r = 0 tmp = 0x7ffe7e662ee9 "" tmp_len = 0 port = 0 proto = 0 options = 0x740400 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 2750509941 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x7f02ded2d6d0 p = 0x7ffe7e661600 "\377\377\377\377" st = {st_dev = 22, st_ino = 23605, st_nlink = 2, st_mode = 16872, st_uid = 986, st_gid = 983, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1519444521, tv_nsec = 392061379}, st_mtim = {tv_sec = 1519496899, tv_nsec = 273235419}, st_ctim = {tv_sec = 1519496899, tv_nsec = 273235419}, __glibc_reserved = {0, 0, 0}} __func__ = "main" __llevel = -1 cfg_stream = 0x10fb260 c = -1 r = 0 tmp = 0x7ffe7e662ee9 "" tmp_len = 0 port = 0 proto = 0 options = 0x740400 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 2750509941 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x7f02ded2d6d0 p = 0x7ffe7e661600 "\377\377\377\377" st = {st_dev = 22, st_ino = 23605, st_nlink = 2, st_mode = 16872, st_uid = 986, st_gid = 983, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1519444521, tv_nsec = 392061379}, st_mtim = {tv_sec = 1519496899, tv_nsec = 273235419}, st_ctim = {tv_sec = 1519496899, tv_nsec = 273235419}, __glibc_reserved = {0, 0, 0}} __func__ = "main" 115 #ifdef AS_STRNLEN 116 andq $-16, %rax 117 FIND_ZERO 118 #else 119 /* Test first 16 bytes unaligned. */ 120 movdqu (%rax), %xmm4 121 PCMPEQ %xmm0, %xmm4 122 pmovmskb %xmm4, %edx 123 test %edx, %edx 124 je L(next48_bytes) ``` #### Log Messages ``` Feb 24 12:29:49 kamailio[12773]: ERROR: db_postgres [km_pg_con.c:115]: db_postgres_new_connection(): timeout expired Feb 24 12:29:49 kamailio[12773]: ERROR: db_postgres [km_pg_con.c:148]: db_postgres_new_connection(): cleaning up 0x7f02ddcd2bb8=pkg_free() Feb 24 12:29:49 kamailio[12773]: ERROR: <core> [db.c:318]: db_do_init2(): could not add connection to the pool Feb 24 12:29:49 kamailio[12773]: ERROR: usrloc [usrloc_mod.c:421]: child_init(): child(0): failed to connect to database Feb 24 12:29:49 kamailio[12784]: INFO: <core> [main.c:826]: sig_usr(): signal 15 received Feb 24 12:29:49 kamailio[12773]: ERROR: <core> [core/sr_module.c:940]: init_mod_child(): error while initializing module usrloc (/usr/lib64/kamailio/modules/usrloc.so) (idx: 0 rank: 0 desc: [main]) Feb 24 12:29:49 kamailio[12773]: ERROR: <core> [main.c:1709]: main_loop(): error in init_child Feb 24 12:29:49 kamailio[12783]: INFO: <core> [main.c:826]: sig_usr(): signal 15 received Feb 24 12:29:49 kamailio[12778]: INFO: <core> [main.c:826]: sig_usr(): signal 15 received Feb 24 12:29:49 kamailio[12781]: INFO: <core> [main.c:826]: sig_usr(): signal 15 received Feb 24 12:29:49 kamailio[12776]: INFO: <core> [main.c:826]: sig_usr(): signal 15 received Feb 24 12:29:49 kamailio[12782]: INFO: <core> [main.c:826]: sig_usr(): signal 15 received Feb 24 12:29:49 systemd[1]: Failed to start Kamailio SIP Server. Feb 24 12:29:49 audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=kamailio comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=failed' Feb 24 12:29:49 systemd[1]: kamailio.service: Unit entered failed state. Feb 24 12:29:49 systemd[1]: kamailio.service: Failed with result 'timeout'. Feb 24 12:29:49 kamailio[12773]: WARNING: <core> [core/mem/q_malloc.c:476]: qm_free(): WARNING: free(0) called from permissions: permissions.c: mod_exit(666) Feb 24 12:29:49 kamailio[12773]: WARNING: <core> [core/mem/q_malloc.c:476]: qm_free(): WARNING: free(0) called from permissions: permissions.c: mod_exit(669) Feb 24 12:29:49 kamailio[12773]: INFO: <core> [core/sctp_core.c:53]: sctp_core_destroy(): SCTP API not initialized Feb 24 12:29:49 audit[12773]: ANOM_ABEND auid=4294967295 uid=986 gid=983 ses=4294967295 subj=system_u:system_r:unconfined_service_t:s0 pid=12773 comm="kamailio" exe="/usr/sbin/kamailio" sig=11 res=1 Feb 24 12:29:49 kernel: show_signal_msg: 7 callbacks suppressed Feb 24 12:29:49 kernel: kamailio[12773]: segfault at 7f02ddb597c8 ip 00007f02de3ad286 sp 00007ffe7e661308 error 4 in libc-2.26.so[7f02de303000+1da000] Feb 24 12:29:49 systemd[1]: Created slice system-systemd\x2dcoredump.slice. Feb 24 12:29:49 systemd[1]: Started Process Core Dump (PID 12817/UID 0). Feb 24 12:29:49 audit[1]: SERVICE_START pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=systemd-coredump@0-12817-0 comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success' Feb 24 12:29:50 systemd-coredump[12818]: Process 12773 (kamailio) of user 986 dumped core. Stack trace of thread 12773: #0 0x00007f02de3ad286 __GI___strlen_sse2 (libc.so.6) #1 0x00007f02de385835 fputs_unlocked (libc.so.6) #2 0x00007f02de4132b1 syslog (libc.so.6) #3 0x000000000042aa60 main (kamailio) #4 0x00007f02de32400a __libc_start_main (libc.so.6) #5 0x000000000041800a _start (kamailio) ``` ### Additional Information * **Kamailio Version** - output of `kamailio -v` ``` version: kamailio 5.1.1 (x86_64/linux) flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144, MAX_LISTEN 16, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB poll method support: poll, epoll_lt, epoll_et, sigio_rt, select. id: unknown compiled on 21:14:17 Feb 23 2018 with gcc 7.3.1 ``` * **Operating System**: ``` Linux 4.15.4-300.fc27.x86_64 #1 SMP Mon Feb 19 23:31:15 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux ``` -- You are receiving this because you are subscribed to this thread. Reply to this email directly or view it on GitHub: https://github.com/kamailio/kamailio/issues/1454