[sr-dev] rtpengine crashes

Alex Balashov abalashov at evaristesys.com
Wed Jul 2 19:07:41 CEST 2014 

Interestingly enough, the polling loop of rtpengine seems to be intact:

[root at sd-rtp01 log]# strace -p 30323
Process 30323 attached - interrupt to quit
restart_syscall(<... resuming interrupted call ...>) = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, ^C <unfinished ...>
Process 30323 detached

So, I'm not sure why it wouldn't be processing the ping messages. It 
must be a programmatic issue. Maybe? When I ran this strace and waited 
for one of these:

Jul  2 17:04:14 proxyhost /usr/local/sbin/kamailio[22442]: ERROR: 
rtpproxy-ng [rtpproxy.c:1549]: send_rtpp_command(): proxy 
<udp:209.75.186.78:5050> does not respond, disable it

I saw nothing but this:

nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0
nanosleep({0, 100000000}, NULL)         = 0

...

Re: the current thread state with attached GDB:

(gdb) thread apply all bt

Thread 7 (Thread 0x7f874f280700 (LWP 30324)):
#0  0x000000304e233a1d in sigtimedwait () from /lib64/libc.so.6
#1  0x00000000004072f1 in sighandler (x=<value optimized out>) at main.c:114
#2  0x000000000040bb5f in thread_detach_func (d=<value optimized out>)
     at aux.c:160
#3  0x000000304e6079d1 in start_thread () from /lib64/libpthread.so.0
#4  0x000000304e2e8b6d in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x7f874e87f700 (LWP 30325)):
#0  0x000000304e2accdd in nanosleep () from /lib64/libc.so.6
#1  0x000000304e2e1e54 in usleep () from /lib64/libc.so.6
#2  0x000000000040af82 in poller_timers_wait_run (p=0x269a740, max=100000)
     at poller.c:500
#3  0x000000000040725d in timer_loop (d=0x269a740) at main.c:535
#4  0x000000000040bb5f in thread_detach_func (d=<value optimized out>)
     at aux.c:160
#5  0x000000304e6079d1 in start_thread () from /lib64/libpthread.so.0
#6  0x000000304e2e8b6d in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x7f874de7e700 (LWP 30326)):
#0  0x000000304e60b5bc in pthread_cond_wait@@GLIBC_2.3.2 ()
    from /lib64/libpthread.so.0
---Type <return> to continue, or q <return> to quit---
#1  0x0000000000415240 in cookie_cache_lookup (c=0x269e050, 
s=0x7f874de6d650)
     at cookie_cache.c:57
#2  0x0000000000415b20 in control_ng_incoming (obj=0x269e030,
     buf=<value optimized out>, sin=0x7f874de7d700, addr=
     0x7f874de7d6c0 "209.75.186.86:39889") at control_ng.c:86
#3  0x000000000041573e in udp_listener_incoming (fd=6, p=0x269d670,
     x=<value optimized out>) at udp_listener.c:52
#4  0x000000000040b4ce in poller_poll (p=0x269a740,
     timeout=<value optimized out>) at poller.c:354
#5  0x000000000040722d in poller_loop (d=0x269a740) at main.c:542
#6  0x000000000040bb5f in thread_detach_func (d=<value optimized out>)
     at aux.c:160
#7  0x000000304e6079d1 in start_thread () from /lib64/libpthread.so.0
#8  0x000000304e2e8b6d in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x7f874d47d700 (LWP 30327)):
#0  0x000000304e60ac50 in pthread_rwlock_wrlock () from 
/lib64/libpthread.so.0
#1  0x000000000040f29f in call_destroy (c=0x7f87400b25c0) at call.c:1981
#2  0x0000000000411c16 in call_delete_branch (m=<value optimized out>,
     callid=<value optimized out>, branch=<value optimized out>,
     fromtag=<value optimized out>, totag=0x7f874d46c540, output=
     0x7f8740062c28) at call.c:2486
#3  0x000000000041f901 in call_delete_ng (input=<value optimized out>, m=
---Type <return> to continue, or q <return> to quit---
     0x269b400, output=0x7f8740062c28) at call_interfaces.c:649
#4  0x0000000000415fc8 in control_ng_incoming (obj=0x269e030,
     buf=<value optimized out>, sin=0x7f874d47c700, addr=
     0x7f874d47c6c0 "209.75.186.86:41930") at control_ng.c:117
#5  0x000000000041573e in udp_listener_incoming (fd=6, p=0x269d670,
     x=<value optimized out>) at udp_listener.c:52
#6  0x000000000040b4ce in poller_poll (p=0x269a740,
     timeout=<value optimized out>) at poller.c:354
#7  0x000000000040722d in poller_loop (d=0x269a740) at main.c:542
#8  0x000000000040bb5f in thread_detach_func (d=<value optimized out>)
     at aux.c:160
#9  0x000000304e6079d1 in start_thread () from /lib64/libpthread.so.0
#10 0x000000304e2e8b6d in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x7f874ca7c700 (LWP 30328)):
#0  0x000000304e60ac50 in pthread_rwlock_wrlock () from 
/lib64/libpthread.so.0
#1  0x000000000040f29f in call_destroy (c=0x7f8740a2c120) at call.c:1981
#2  0x0000000000411c16 in call_delete_branch (m=<value optimized out>,
     callid=<value optimized out>, branch=<value optimized out>,
     fromtag=<value optimized out>, totag=0x7f874ca6b540, output=
     0x7f8728222828) at call.c:2486
#3  0x000000000041f901 in call_delete_ng (input=<value optimized out>, m=
     0x269b400, output=0x7f8728222828) at call_interfaces.c:649
---Type <return> to continue, or q <return> to quit---
#4  0x0000000000415fc8 in control_ng_incoming (obj=0x269e030,
     buf=<value optimized out>, sin=0x7f874ca7b700, addr=
     0x7f874ca7b6c0 "209.75.186.86:39889") at control_ng.c:117
#5  0x000000000041573e in udp_listener_incoming (fd=6, p=0x269d670,
     x=<value optimized out>) at udp_listener.c:52
#6  0x000000000040b4ce in poller_poll (p=0x269a740,
     timeout=<value optimized out>) at poller.c:354
#7  0x000000000040722d in poller_loop (d=0x269a740) at main.c:542
#8  0x000000000040bb5f in thread_detach_func (d=<value optimized out>)
     at aux.c:160
#9  0x000000304e6079d1 in start_thread () from /lib64/libpthread.so.0
#10 0x000000304e2e8b6d in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x7f8747fff700 (LWP 30329)):
#0  0x000000304e60ac50 in pthread_rwlock_wrlock () from 
/lib64/libpthread.so.0
#1  0x000000000040f29f in call_destroy (c=0x7f872c22a1d0) at call.c:1981
#2  0x0000000000411c16 in call_delete_branch (m=<value optimized out>,
     callid=<value optimized out>, branch=<value optimized out>,
     fromtag=<value optimized out>, totag=0x7f8747fee540, output=
     0x7f872c2d4828) at call.c:2486
#3  0x000000000041f901 in call_delete_ng (input=<value optimized out>, m=
     0x269b400, output=0x7f872c2d4828) at call_interfaces.c:649
#4  0x0000000000415fc8 in control_ng_incoming (obj=0x269e030,
---Type <return> to continue, or q <return> to quit---
     buf=<value optimized out>, sin=0x7f8747ffe700, addr=
     0x7f8747ffe6c0 "209.75.186.86:36511") at control_ng.c:117
#5  0x000000000041573e in udp_listener_incoming (fd=6, p=0x269d670,
     x=<value optimized out>) at udp_listener.c:52
#6  0x000000000040b4ce in poller_poll (p=0x269a740,
     timeout=<value optimized out>) at poller.c:354
#7  0x000000000040722d in poller_loop (d=0x269a740) at main.c:542
#8  0x000000000040bb5f in thread_detach_func (d=<value optimized out>)
     at aux.c:160
#9  0x000000304e6079d1 in start_thread () from /lib64/libpthread.so.0
#10 0x000000304e2e8b6d in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7f874f2817c0 (LWP 30323)):
#0  0x000000304e2accdd in nanosleep () from /lib64/libc.so.6
#1  0x000000304e2e1e54 in usleep () from /lib64/libc.so.6
#2  0x000000000040a98a in main (argc=1, argv=0x7fffc99076d8) at main.c:562


-- Alex

-- 
Alex Balashov - Principal
Evariste Systems LLC
Tel: +1-678-954-0670
Web: http://www.evaristesys.com/, http://www.alexbalashov.com/

Please be kind to the English language:

http://www.entrepreneur.com/article/232906

More information about the sr-dev mailing list