18 Sep
2015
18 Sep
'15
6:55 a.m.
The problem of having kamailio stopping processing messages happened again:
```
[root@lab002185-flip-server kamailio-4.4]# kamctl fifo version
Server:: kamailio (4.4.0-dev4 (x86_64/linux))
Build:: mi_core.c compiled on 20:23:03 Sep 14 2015 with gcc 4.4.7
Flags:: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE,
USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, DBG_SR_MEMORY,
USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR,
USE_DST_BLACKLIST, HAVE_RESOLV_RES
GIT:: 171553
[root@lab002185-flip-server kamailio-4.4]# kamctl fifo get_statistics dialog:
dialog:active_dialogs = 18446744073709551613
dialog:early_dialogs = 1
dialog:expired_dialogs = 2319
dialog:failed_dialogs = 354478
dialog:processed_dialogs = 4891209
[root@lab002185-flip-server kamailio-4.4]# kamctl fifo ps
Process:: ID=0 PID=27872 Type=attendant
Process:: ID=1 PID=27887 Type=udp receiver child=0 sock=192.168.2.185:5060
Process:: ID=2 PID=27888 Type=udp receiver child=1 sock=192.168.2.185:5060
Process:: ID=3 PID=27889 Type=udp receiver child=2 sock=192.168.2.185:5060
Process:: ID=4 PID=27890 Type=udp receiver child=3 sock=192.168.2.185:5060
Process:: ID=5 PID=27891 Type=slow timer
Process:: ID=6 PID=27892 Type=timer
Process:: ID=7 PID=27898 Type=MI FIFO
Process:: ID=8 PID=27900 Type=Dialog Clean Timer
Process:: ID=9 PID=27901 Type=SNMP AgentX
Process:: ID=10 PID=27904 Type=MI DATAGRAM
Process:: ID=11 PID=27906 Type=ctl handler
Process:: ID=12 PID=27907 Type=tcp receiver (generic) child=0
Process:: ID=13 PID=27908 Type=tcp receiver (generic) child=1
Process:: ID=14 PID=27909 Type=tcp receiver (generic) child=2
Process:: ID=15 PID=27914 Type=tcp receiver (generic) child=3
Process:: ID=16 PID=27916 Type=tcp main process
```
I collected the bt of one of the UDP receivers:
```
[root@lab002185-flip-server kamailio-4.4]# gdb attach 27887
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-83.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
attach: No such file or directory.
Attaching to process 27887
Reading symbols from /usr/local/src/git/kamailio-4.4/kamailio...done.
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/local/lib64/kamailio/modules/tm.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/tm.so
Reading symbols from /usr/local/lib64/kamailio/modules/tmx.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/tmx.so
Reading symbols from /usr/local/lib64/kamailio/libkmi.so.1...done.
Loaded symbols for /usr/local/lib64/kamailio/libkmi.so.1
Reading symbols from /usr/local/lib64/kamailio/libkcore.so.1...done.
Loaded symbols for /usr/local/lib64/kamailio/libkcore.so.1
Reading symbols from /usr/local/lib64/kamailio/modules/db_mysql.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/db_mysql.so
Reading symbols from /usr/lib64/mysql/libmysqlclient.so.16...(no debugging symbols
found)...done.
Loaded symbols for /usr/lib64/mysql/libmysqlclient.so.16
Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /lib64/libnsl.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnsl.so.1
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /usr/lib64/libssl.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libssl.so.10
Reading symbols from /usr/lib64/libcrypto.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libcrypto.so.10
Reading symbols from /usr/local/lib64/kamailio/libsrdb2.so.1...done.
Loaded symbols for /usr/local/lib64/kamailio/libsrdb2.so.1
Reading symbols from /usr/local/lib64/kamailio/libsrdb1.so.1...done.
Loaded symbols for /usr/local/lib64/kamailio/libsrdb1.so.1
Reading symbols from /lib64/libfreebl3.so...(no debugging symbols found)...done.
Loaded symbols for /lib64/libfreebl3.so
Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgssapi_krb5.so.2
Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5.so.3
Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libk5crypto.so.3
Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5support.so.0
Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /usr/local/lib64/kamailio/modules/sl.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/sl.so
Reading symbols from /usr/local/lib64/kamailio/modules/rr.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/rr.so
Reading symbols from /usr/local/lib64/kamailio/modules/maxfwd.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/maxfwd.so
Reading symbols from /usr/local/lib64/kamailio/modules/usrloc.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/usrloc.so
Reading symbols from /usr/local/lib64/kamailio/libsrutils.so.1...done.
Loaded symbols for /usr/local/lib64/kamailio/libsrutils.so.1
Reading symbols from /usr/local/lib64/kamailio/modules/registrar.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/registrar.so
Reading symbols from /usr/local/lib64/kamailio/modules/textops.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/textops.so
Reading symbols from /usr/local/lib64/kamailio/modules/mi_fifo.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/mi_fifo.so
Reading symbols from /usr/local/lib64/kamailio/modules/uri_db.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/uri_db.so
Reading symbols from /usr/local/lib64/kamailio/modules/xlog.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/xlog.so
Reading symbols from /usr/local/lib64/kamailio/modules/acc.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/acc.so
Reading symbols from /usr/local/lib64/kamailio/modules/carrierroute.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/carrierroute.so
Reading symbols from /usr/local/lib64/kamailio/libtrie.so.1...done.
Loaded symbols for /usr/local/lib64/kamailio/libtrie.so.1
Reading symbols from /usr/local/lib64/kamailio/modules/nathelper.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/nathelper.so
Reading symbols from /usr/local/lib64/kamailio/modules/dialog.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/dialog.so
Reading symbols from /usr/local/lib64/kamailio/modules/app_perl.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/app_perl.so
Reading symbols from /usr/lib64/perl5/CORE/libperl.so...(no debugging symbols
found)...done.
Loaded symbols for /usr/lib64/perl5/CORE/libperl.so
Reading symbols from /lib64/libutil.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libutil.so.1
Reading symbols from /usr/local/lib64/kamailio/modules/snmpstats.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/snmpstats.so
Reading symbols from /usr/lib64/libnetsnmpagent.so.20...(no debugging symbols
found)...done.
Loaded symbols for /usr/lib64/libnetsnmpagent.so.20
Reading symbols from /usr/lib64/libnetsnmphelpers.so.20...(no debugging symbols
found)...done.
Loaded symbols for /usr/lib64/libnetsnmphelpers.so.20
Reading symbols from /usr/lib64/libnetsnmpmibs.so.20...(no debugging symbols
found)...done.
Loaded symbols for /usr/lib64/libnetsnmpmibs.so.20
Reading symbols from /usr/lib64/libnetsnmp.so.20...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libnetsnmp.so.20
Reading symbols from /lib64/libwrap.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libwrap.so.0
Reading symbols from /usr/lib64/libsensors.so.4...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libsensors.so.4
Reading symbols from /usr/lib64/librpm.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/librpm.so.1
Reading symbols from /usr/lib64/librpmio.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/librpmio.so.1
Reading symbols from /lib64/libpopt.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libpopt.so.0
Reading symbols from /usr/lib64/libnss3.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libnss3.so
Reading symbols from /lib64/libbz2.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libbz2.so.1
Reading symbols from /usr/lib64/libelf.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libelf.so.1
Reading symbols from /usr/lib64/liblzma.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/liblzma.so.0
Reading symbols from /usr/lib64/liblua-5.1.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/liblua-5.1.so
Reading symbols from /lib64/libcap.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcap.so.2
Reading symbols from /lib64/libacl.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libacl.so.1
Reading symbols from /lib64/libdb-4.7.so...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdb-4.7.so
Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/librt.so.1
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Reading symbols from /usr/lib64/libnssutil3.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libnssutil3.so
Reading symbols from /lib64/libplc4.so...(no debugging symbols found)...done.
Loaded symbols for /lib64/libplc4.so
Reading symbols from /lib64/libplds4.so...(no debugging symbols found)...done.
Loaded symbols for /lib64/libplds4.so
Reading symbols from /lib64/libnspr4.so...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnspr4.so
Reading symbols from /lib64/libattr.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libattr.so.1
Reading symbols from /usr/local/lib64/kamailio/modules/auth.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/auth.so
Reading symbols from /usr/local/lib64/kamailio/modules/auth_db.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/auth_db.so
Reading symbols from /usr/local/lib64/kamailio/modules/lcr.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/lcr.so
Reading symbols from /lib64/libpcre.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libpcre.so.0
Reading symbols from /usr/local/lib64/kamailio/modules/alias_db.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/alias_db.so
Reading symbols from /usr/local/lib64/kamailio/modules/domain.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/domain.so
Reading symbols from /usr/local/lib64/kamailio/modules/uac.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/uac.so
Reading symbols from /usr/local/lib64/kamailio/modules/avpops.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/avpops.so
Reading symbols from /usr/local/lib64/kamailio/modules/userblacklist.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/userblacklist.so
Reading symbols from /usr/local/lib64/kamailio/modules/mi_datagram.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/mi_datagram.so
Reading symbols from /usr/local/lib64/kamailio/modules/cfgutils.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/cfgutils.so
Reading symbols from /usr/local/lib64/kamailio/modules/rtpproxy.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/rtpproxy.so
Reading symbols from /usr/local/lib64/kamailio/modules/siputils.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/siputils.so
Reading symbols from /usr/local/lib64/kamailio/modules/pv.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/pv.so
Reading symbols from /usr/local/lib64/kamailio/modules/kex.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/kex.so
Reading symbols from /usr/local/lib64/kamailio/modules/corex.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/corex.so
Reading symbols from /usr/local/lib64/kamailio/modules/ctl.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/ctl.so
Reading symbols from /usr/local/lib64/kamailio/modules/msilo.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/msilo.so
Reading symbols from /usr/local/lib64/kamailio/modules/tls.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/tls.so
Reading symbols from /usr/local/lib64/kamailio/modules/dispatcher.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/dispatcher.so
Reading symbols from /usr/local/lib64/kamailio/modules/sqlops.so...done.
Loaded symbols for /usr/local/lib64/kamailio/modules/sqlops.so
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2
Reading symbols from /lib64/libnss_dns.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_dns.so.2
0x0000003aadce5209 in syscall () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.5-7.el6_0.x86_64
db4-4.7.25-19.el6_6.x86_64 elfutils-libelf-0.161-3.el6.x86_64
glibc-2.12-1.166.el6_7.1.x86_64 keyutils-libs-1.4-5.el6.x86_64
krb5-libs-1.10.3-42.el6.x86_64 libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64
libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-22.el6.x86_64 libgcc-4.4.7-16.el6.x86_64
libselinux-2.0.94-5.8.el6.x86_64 lm_sensors-libs-3.1.1-17.el6.x86_64
lua-5.1.4-4.1.el6.x86_64 mysql-libs-5.1.73-5.el6_6.x86_64
net-snmp-libs-5.5-54.el6_7.1.x86_64 nspr-4.10.8-1.el6_6.x86_64 nss-3.19.1-3.el6_6.x86_64
nss-softokn-freebl-3.14.3-22.el6_6.x86_64 nss-util-3.19.1-1.el6_6.x86_64
openssl-1.0.1e-42.el6.x86_64 pcre-7.8-7.el6.x86_64 perl-libs-5.10.1-141.el6.x86_64
popt-1.13-7.el6.x86_64 rpm-libs-4.8.0-47.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64
xz-libs-4.999.9-0.5.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt
#0 0x0000003aadce5209 in syscall () from /lib64/libc.so.6
#1 0x00007fd4fdc3ab2a in futex_get (lock=0x7ffdebebb714) at ../../mem/../futexlock.h:121
#2 0x00007fd4fdc46b6a in update_dialog_dbinfo (cell=0x7fd4ea434738) at
dlg_db_handler.c:873
#3 0x00007fd4fdc8a93a in pv_set_dlg_variable (msg=0x7fd500f8a468, param=0x7fd500dc3120,
op=254, val=0x7ffdebebb820) at dlg_var.c:428
#4 0x00000000004a092c in lval_pvar_assign (h=0x7ffdebebca80, msg=0x7fd500f8a468,
lv=0x7fd500f54d30, rv=0x7fd500f55008) at lvalue.c:351
#5 0x00000000004a1331 in lval_assign (h=0x7ffdebebca80, msg=0x7fd500f8a468,
lv=0x7fd500f54d30, rve=0x7fd500f55000) at lvalue.c:399
#6 0x0000000000429a7f in do_action (h=0x7ffdebebca80, a=0x7fd500f55728,
msg=0x7fd500f8a468) at action.c:1429
#7 0x000000000042b429 in run_actions (h=0x7ffdebebca80, a=0x7fd500f55728,
msg=0x7fd500f8a468) at action.c:1548
#8 0x000000000041e93b in do_action (h=0x7ffdebebca80, a=0x7fd500f5fd10,
msg=0x7fd500f8a468) at action.c:1044
#9 0x000000000042b429 in run_actions (h=0x7ffdebebca80, a=0x7fd500f5fd10,
msg=0x7fd500f8a468) at action.c:1548
#10 0x000000000042bb77 in run_top_route (a=0x7fd500f5fd10, msg=0x7fd500f8a468,
c=0x7ffdebebca80) at action.c:1634
#11 0x00007fd500a67e80 in reply_received (p_msg=0x7fd500f8a468) at t_reply.c:2317
#12 0x0000000000491c1b in do_forward_reply (msg=0x7fd500f8a468, mode=0) at forward.c:747
#13 0x0000000000493308 in forward_reply (msg=0x7fd500f8a468) at forward.c:849
#14 0x00000000005119de in receive_msg (
buf=0xa92340 "SIP/2.0 406 Not Acceptable\r\nTo:
<sip:09011112222@192.168.2.185:5060>;tag=16045SIPpTag014890951\r\nFrom: 0312341234
<sip:0312341234@test1.com>;tag=16047SIPpTag002445293\r\nCall-ID:
2445293-16047(a)192.168.0."quot;..., len=488, rcv_info=0x7ffdebebd050) at receive.c:255
#15 0x0000000000617928 in udp_rcv_loop () at udp_server.c:495
#16 0x00000000004ac857 in main_loop () at main.c:1579
#17 0x00000000004b2f11 in main (argc=9, argv=0x7ffdebebd4e8) at main.c:2566
```
---
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/301#issuecomment-141336345