[SR-Users] Memory Leaks with Kamailio using SEAS module

ziad habchi ziad.habchi at jinnysoftware.com
Tue Dec 3 09:37:23 CET 2013


Hi , 

 

I do have other core files; 

 

Below is the trace:

 

 

 

#0  0x00ba7430 in __kernel_vsyscall ()

No symbol table info available.

#1  0x0030ab11 in raise () from /lib/libc.so.6

No symbol table info available.

#2  0x0030c3ea in abort () from /lib/libc.so.6

No symbol table info available.

#3  0x081a155a in qm_debug_frag (qm=0xf72e8008, f=<value optimized out>) at
mem/q_malloc.c:161

        __FUNCTION__ = "qm_debug_frag"

#4  0x081a18d2 in qm_malloc (qm=0xf72e8008, size=<value optimized out>,
file=0x826b443 "<core>: parser/parse_rr.c", func=0x826bc00
"do_parse_rr_body", line=72) at mem/q_malloc.c:384

        f = 0xf7336d18

        hash = 25391

        list_cntr = 1

        __FUNCTION__ = "qm_malloc"

#5  0x081c06cb in do_parse_rr_body (buf=0x20 <Address 0x20 out of bounds>,
len=33, head=0xfff46750) at parser/parse_rr.c:72

        r = <value optimized out>

        last = 0x0

        s = {

          s = 0xf7335421 "<sip:qatar:5073;transport=TCP;lr>\r\nRoute:
<sip:qatar:5073;transport=TCP;lr>\r\nFrom:
<sip:+9613010005 at peru>;tag=1672649062\r\nTo:
<sip:+9613414332 at peru>\r\nCall-ID: 883711576\r\nCSeq: 20
MESSAGE\r\nContent-Type"..., len = 33}

        hooks = {contact = {expires = 0x8264900, q = 0xfff466f8, methods =
0xfff46758, received = 0x81a1bf5, instance = 0x9f, reg_id = 0x8264900, ob =
0x823b9dc}, uri = {transport = 0x8264900, lr = 0xfff466f8, 

            r2 = 0xfff46758, maddr = 0x81a1bf5, ttl = 0x9f, dstip =
0x8264900, dstport = 0x823b9dc, ftag = 0x8263e72, ob = 0x826515a},
event_dialog = {call_id = 0x8264900, from_tag = 0xfff466f8, to_tag =
0xfff46758, 

            include_session_description = 0x81a1bf5, sla = 0x9f, ma =
0x8264900}}

        __FUNCTION__ = "do_parse_rr_body"

#6  0x081c1fe1 in parse_rr (_h=0xf732f7e0) at parser/parse_rr.c:176

        r = 0x0

        __FUNCTION__ = "parse_rr"

#7  0x00e2ef1c in ac_uac_req (the_as=0xf54cb6e0, processor_id=1 '\001',
flags=0, 

    action=0xf73353b2 "4\254^XMESSAGE sip:+9613414332 at peru SIP/2.0\r\nVia:
SIP/2.0/TCP 193.100.200.18:5070;branch=z9hG4bK649641154\r\nRoute:
<sip:qatar:5073;transport=TCP;lr>\r\nRoute:
<sip:qatar:5073;transport=TCP;lr>\r\nFrom: <sip:+"..., len=545) at
seas_action.c:1067

        cseq = <value optimized out>

        err_buf =
"\240CG\000\370\247\360\b\364/G\000\025\000\000\000\350\245\360\b\220i\364\3
77\227\354;\000\000\000\000\000`\247\360\b\230\000\000\000\000@\000\000\000$
G\000\000\000\000\000.^5\000\002\000\000\000;\000\000\000\020\000\000\000\00
2\000\000\000\v\000\000\000q\000\000\000\001\000\000\000O\001\000\000\000\00
0\000\000
\034\000\000\310T\354\b\204i\364\377\254a5\000v/5\000\350\245\360\bLj\364\37
7li\364\377\373\361\064"

        my_msg = 0xf7336378

        my_dlg = 0xf54e0624

        retval = <value optimized out>

        uac_id = 883711576

        sip_error = -759384

        ret = <value optimized out>

        err_ret = <value optimized out>

        headers = {

          s = 0xf7336ef4 "Content-Type: text/plain\r\nContent-Encoding:
sip:qatar:5073\r\nMax-Forwards: 70\r\nUser-Agent: Jinny SIP Client\r\nDate:
Mon, 2 Dec 2013 16:59:2 GMT\r\nExpires: 30\r\nP-Asserted-Identity:
sip:ip-sm-gw.ims-core:5"..., len = 205}

        body = {s = 0xf7336cf0 "Hi I'm Alice !\r\n", len = 16}

        fake_uri = {s = 0x0, len = 149989216}

        uac_r = {method = 0x8f0a5e8, headers = 0x8f0a5e8, body = 0x472ff4,
dialog = 0x352f76, cb_flags = -759420, cb = 0x8f0a760, cbp = 0x0}

        __FUNCTION__ = "ac_uac_req"

#8  0x00e32def in process_action (the_as=0xf54cb6e0) at seas_action.c:309

        ac_len = 555

        processor_id = 1 '\001'

        type = 4 '\004'

---Type <return> to continue, or q <return> to quit---

        flags = 0

        __FUNCTION__ = "process_action"

#9  0x00e33b18 in process_input (fd=11) at seas_action.c:194

        j = 136947160

        __FUNCTION__ = "process_input"

#10 0x00e3451b in dispatch_actions () at seas_action.c:161

        fd = 11

        n = <value optimized out>

        ret = <value optimized out>

        timeout = <value optimized out>

        elapsed_ms = <value optimized out>

       fds = {{fd = 11, events = 17, revents = 0}}

        last = {tv_sec = -143742160, tv_usec = 0}

        now = {tv_sec = 136726784, tv_usec = 25334}

        ktimeout = 0

        __FUNCTION__ = "dispatch_actions"

#11 0x00e18be2 in spawn_action_dispatcher (the_as=0xf54cb6d0) at
event_dispatcher.c:1122

        pid = <value optimized out>

        __FUNCTION__ = "spawn_action_dispatcher"

#12 0x00e1b617 in add_new_as (event_idx=<value optimized out>,
action_idx=<value optimized out>, as=0xf54cb6d0) at event_dispatcher.c:575

        ev = <value optimized out>

        j = 0

        the_as = 0xf54cb6e0

        tmp = 0x0

        __FUNCTION__ = "add_new_as"

#13 0x00e1d348 in handle_unc_as_data (fd=<value optimized out>) at
event_dispatcher.c:988

        i = 5

        j = 0

        k = 5

        len = 7

        name1 = 0xe4aacc "smsc_as"

        as = 0xf54cb6d0

        __FUNCTION__ = "handle_unc_as_data"

#14 0x00e1ec6c in dispatcher_main_loop () at event_dispatcher.c:295

        poll_fds = {{fd = 5, events = 1, revents = 0}, {fd = 7, events = 1,
revents = 0}, {fd = 6, events = 1, revents = 0}, {fd = 11, events = 17,
revents = 0}, {fd = 12, events = 17, revents = 0}, {fd = 3206374, 

            events = 28792, revents = -12}, {fd = 95421144, events = 0,
revents = 0}, {fd = 3876636, events = -13429, revents = 2061}}

        poll_tmp = <value optimized out>

        i = -757768

        j = <value optimized out>

        k = <value optimized out>

        fd = 0

        poll_events = 0

        socks = {5, 7}

        chld_status = 14803099

        as_nr = 0

        unc_as_nr = 1

        chld = 137409248

        as = <value optimized out>

        __FUNCTION__ = "dispatcher_main_loop"

#15 0x00e35ea5 in seas_child_init (rank=0) at seas.c:741

---Type <return> to continue, or q <return> to quit---

        pid = <value optimized out>

        __FUNCTION__ = "seas_child_init"

#16 0x0813a622 in init_mod_child (m=0xf732b550, rank=0) at sr_module.c:893

        __FUNCTION__ = "init_mod_child"

#17 0x080b7ea5 in main_loop () at main.c:1715

        i = 4

        pid = 0

        si = <value optimized out>

        si_desc = "udp receiver child=3
sock=193.100.200.18:5070\000\063\367\030\311\341\001\001\000\000\000\234\266
L\365жL\365\v\020\000\000l\302\061\365\270\266L\365\270q\364\377\000\000\000
\000X\243\360\b\000\000\000\000\203\000\000\000X\243\360\b\001", '\000'
<repeats 23 times>"\270, q\364\377"

        nrprocs = 4

        __FUNCTION__ = "main_loop"

#18 0x080bb57e in main (argc=5, argv=0xfff47364) at main.c:2566

        cfg_stream = 0x8ebf008

        c = <value optimized out>

        r = 0

        tmp = 0x80512dd "__libc_start_main"

        tmp_len = 4657636

        port = 136523065

        proto = -757064

        ret = <value optimized out>

        seed = 3203921318

        rfd = 4

        debug_save = 136559068

        debug_flag = 0

        dont_fork_cnt = 0

        n_lst = 0x823b9dc

        p = 0x804aea8 "E\005"

        __FUNCTION__ = "main"

 

From: Daniel-Constantin Mierla [mailto:miconda at gmail.com] 
Sent: Tuesday, December 3, 2013 10:00 AM
To: ziad habchi; 'Kamailio (SER) - Users Mailing List'
Subject: Re: [SR-Users] Memory Leaks with Kamailio using SEAS module

 

Hello,

that backtrace is from shutdown, if you don't have another core file, then
the initial one was overwritten. You have to enable one core per process. In
most of system it is:

echo "1" > /proc/sys/kernel/core_uses_pid

You have to reproduce the issue again and get the backtrace from all core
files.

Cheers,
Daniel

On 12/3/13 8:42 AM, ziad habchi wrote:

Hi ,
 
Yes it generated a core file; the debuginfos are missing, how can I complie
kamailio without stripping them?
 
below is the output of bt full:
 
#0  0x00359bc1 in __strlen_ia32 () from /lib/libc.so.6
No symbol table info available.
#1  0x003244df in vfprintf () from /lib/libc.so.6
No symbol table info available.
#2  0x003beea6 in __vsyslog_chk () from /lib/libc.so.6
No symbol table info available.
#3  0x003bf027 in syslog () from /lib/libc.so.6
No symbol table info available.
#4  0x0819f0a2 in qm_status (qm=0xf52e8000) at mem/q_malloc.c:761
        f = 0xf54ccfbc
        i = 56
        j = <value optimized out>
        h = <value optimized out>
        memlog = 1
        mem_summary = <value optimized out>
        __FUNCTION__ = "qm_status"
#5  0x081a1555 in qm_debug_frag (qm=0xf52e8000, f=<value optimized out>) at
mem/q_malloc.c:160
        __FUNCTION__ = "qm_debug_frag"
#6  0x081a2820 in qm_free (qm=0xf52e8000, p=0xf54ccfd4, file=0xea2ac9 "kex:
pkg_stats.c", func=0xea2b60 "pkg_proc_stats_destroy", line=111) at
mem/q_malloc.c:462
        f = 0xf54ccfbc
        size = <value optimized out>
        next = <value optimized out>
        prev = <value optimized out>
        __FUNCTION__ = "qm_free"
#7  0x00ea1ab6 in pkg_proc_stats_destroy () at pkg_stats.c:111
        __FUNCTION__ = "pkg_proc_stats_destroy"
#8  0x00e9e157 in destroy () at kex_mod.c:170
No locals.
#9  0x0813aeac in destroy_modules () at sr_module.c:790
        t = 0xf73287ec
        foo = 0xf73287ec
        __FUNCTION__ = "destroy_modules"
#10 0x080b4af0 in cleanup (show_status=1) at main.c:573
        memlog = <value optimized out>
        __FUNCTION__ = "cleanup"
#11 0x080b5af9 in shutdown_children (sig=<value optimized out>,
show_status=1) at main.c:715
        __FUNCTION__ = "shutdown_children"
#12 0x080b5ff3 in handle_sigs () at main.c:745
        chld = <value optimized out>
        chld_status = <value optimized out>
        memlog = <value optimized out>
        __FUNCTION__ = "handle_sigs"
#13 0x080b7f57 in main_loop () at main.c:1767
        i = <value optimized out>
        pid = <value optimized out>
        si = <value optimized out>
        si_desc = "udp receiver child=3
sock=193.100.200.18:5070\000\063\367\030\311\341\001\001\000\000\000\234\266
L\365жL\365\v\020\000\000l\302\061\365\270\266L\365\270q\364\377\000\000\000
\000X\243\360\b\000\000\000\000\2---Type <return> to continue, or q <return>
to quit---
03\000\000\000X\243\360\b\001", '\000' <repeats 23 times>"\270, q\364\377"
        nrprocs = 4
        __FUNCTION__ = "main_loop"
#14 0x080bb57e in main (argc=5, argv=0xfff47364) at main.c:2566
        cfg_stream = 0x8ebf008
        c = <value optimized out>
        r = 70188
        tmp = 0x80512dd "__libc_start_main"
        tmp_len = 4657636
        port = 136523065
        proto = -757064
        ret = <value optimized out>
        seed = 3203921318
        rfd = 4
        debug_save = 136559068
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x823b9dc
        p = 0x804aea8 "E\005"
        __FUNCTION__ = "main"
 
-----Original Message-----
From: sr-users-bounces at lists.sip-router.org
<mailto:sr-users-bounces at lists.sip-router.org> 
[mailto:sr-users-bounces at lists.sip-router.org] On Behalf Of
Daniel-Constantin Mierla
Sent: Monday, December 2, 2013 7:40 PM
To: Kamailio (SER) - Users Mailing List
Subject: Re: [SR-Users] Memory Leaks with Kamailio using SEAS module
 
Hello,
 
did you get a core file? If yes, send here the output for 'bt full' 
taken in gdb.
 
Cheers,
Daniel
 
On 12/2/13 5:14 PM, zhabchi wrote:

the following is the error i got when the client connection to SEAS exited

:

  tm [t_reply.c:604]: _reply_light(): ERROR: _reply_light: cannot 
allocate shmem buffer
 
i recompiled Kamailio with MEMDBG=1 and here what i got
 
<core> [mem/q_malloc.c:159]: qm_debug_frag(): BUG: qm_*: prev. fragm. 
tail overwritten(c0c0c000, abcdefed)[0xf7336d18:0xf7336d30]!
 
 
 
I am running Kamailio with -m 2048 paramter
 
 
zhabchi wrote

Dear Support,
 
  
 
I am having an "out of memory" problem while using kamailio with SEAS 
module.
 
After using Kamailio with the SEAS module for few hours with a high 
load , I am getting "out of memory" error that I believe is caused 
from a memory leak.
 
I would like your help interpreting this error.
 
  
 
The output of kamailio -v is :
 
  
 
version: kamailio 4.0.4 (i386/linux) cabe58
 
flags: STATS: Off, USE_IPV6, USE_TCP, USE_TLS, TLS_HOOKS, 
USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, 
SHM_MMAP, PKG_MALLOC, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, 
USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, 
HAVE_RESOLV_RES
 
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144, MAX_LISTEN 16, 
MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 4MB
 
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
 
id: cabe58
 
compiled on 14:23:27 Dec  2 2013 with gcc 4.1.2
 
  
 
kamctl ps :
 
  
 
Process::  ID=0 PID=31191 Type=attendant
 
Process::  ID=1 PID=31192 Type=udp receiver child=0
sock=193.100.200.18:5070
 
Process::  ID=2 PID=31193 Type=udp receiver child=1
sock=193.100.200.18:5070
 
Process::  ID=3 PID=31194 Type=udp receiver child=2
sock=193.100.200.18:5070
 
Process::  ID=4 PID=31195 Type=udp receiver child=3
sock=193.100.200.18:5070
 
Process::  ID=5 PID=31196 Type=slow timer
 
Process::  ID=6 PID=31197 Type=timer
 
Process::  ID=7 PID=31198 Type=ctl handler
 
Process::  ID=8 PID=31199 Type=MI FIFO
 
Process::  ID=9 PID=31200 Type=SEAS
 
Process::  ID=10 PID=31201 Type=tcp receiver (generic) child=0
 
Process::  ID=11 PID=31202 Type=tcp receiver (generic) child=1
 
Process::  ID=12 PID=31203 Type=tcp receiver (generic) child=2
 
Process::  ID=13 PID=31204 Type=tcp receiver (generic) child=3
 
Process::  ID=14 PID=31205 Type=tcp main process
 
  
 
  
 
kamcmd pkg.stats:
 
{
 
                 entry: 0
 
                 pid: 31191
 
                 rank: 0
 
                 used: 72480
 
                 free: 4091840
 
                 real_used: 102448
 
}
 
{
 
                 entry: 1
 
                 pid: 31192
 
                 rank: 1
 
                 used: 80624
 
                 free: 4083696
 
                 real_used: 110592
 
}
 
{
 
                 entry: 2
 
                 pid: 31193
 
                 rank: 2
 
                 used: 80624
 
                 free: 4083696
 
                 real_used: 110592
 
}
 
{
 
                 entry: 3
 
                 pid: 31194
 
                 rank: 3
 
                 used: 80624
 
                 free: 4083696
 
                 real_used: 110592
 
}
 
{
 
                 entry: 4
 
                 pid: 31195
 
                 rank: 4
 
                 used: 80624
 
                 free: 4083696
 
                 real_used: 110592
 
}
 
{
 
                 entry: 5
 
                 pid: 31196
 
                 rank: -1
 
                 used: 2276560
 
                 free: 4083744
 
                 real_used: 17770256
 
}
 
{
 
                 entry: 6
 
                 pid: 31197
 
                 rank: -1
 
                 used: 1787296
 
                 free: 4083744
 
                 real_used: 17280992
 
}
 
{
 
                 entry: 7
 
                 pid: 31198
 
                 rank: -2
 
                 used: 77760
 
                 free: 4086544
 
                 real_used: 107744
 
}
 
{
 
                 entry: 8
 
                 pid: 0
 
                 rank: 0
 
                 used: 89920
 
                 free: 4074352
 
                 real_used: 119936
 
}
 
{
 
                 entry: 9
 
                 pid: 0
 
                 rank: 0
 
                 used: 0
 
                 free: 0
 
                 real_used: 0
 
}
 
{
 
                 entry: 10
 
                 pid: 31201
 
                 rank: 5
 
                 used: 140240
 
                 free: 4023728
 
                 real_used: 170560
 
}
 
{
 
                 entry: 11
 
                 pid: 31202
 
                 rank: 6
 
                 used: 140240
 
                 free: 4023712
 
                 real_used: 170576
 
}
 
{
 
                 entry: 12
 
                 pid: 31203
 
                 rank: 7
 
                 used: 140240
 
                 free: 4023856
 
                 real_used: 170432
 
}
 
{
 
                 entry: 13
 
                 pid: 31204
 
                 rank: 8
 
                 used: 140240
 
                 free: 4023712
 
                 real_used: 170576
 
}
 
{
 
                 entry: 14
 
                 pid: 31205
 
                 rank: -4
 
                 used: 3877344
 
                 free: 4030176
 
                 real_used: 19269056
 
}
 
  
 
As you can see above , the output of the SEAS module is not showing 
(pid
31200)
 
Moreover , I can see that the real_used value in the kamcmd 
core.shmmem keep on increasing , and free decrease until I finally 
get an error "out of memory"
 
  
 
When my SEAS client exit the output of kamcmd core.shmmem :
 
{
 
                 total: 33554432
 
                 free: 16273440
 
                 used: 1787296
 
                 real_used: 17280992
 
                 max_used: 33553520
 
                 fragments: 8098
 
}
 
  
 
The traffic is a simple SIP MESSAGE from a seagull simulator, with a 
200OK reply from my SEAS client.
 
  
 
I appreciate your support,
 
Thank you in advance,
 
Ziad Habchi
 
  
 
  
 
  
 
  
 
 
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing 
list sr-users at .sip-router <mailto:sr-users at .sip-router>  
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users

 
 
 
 
--
View this message in context: 
http://sip-router.1086192.n5.nabble.com/Memory-Leaks-with-Kamailio-usi
ng-SEAS-module-tp123410p123417.html
Sent from the Users mailing list archive at Nabble.com.
 
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing 
list sr-users at lists.sip-router.org <mailto:sr-users at lists.sip-router.org>  
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users

 
--
Daniel-Constantin Mierla - http://www.asipto.com
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
 
 
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users at lists.sip-router.org <mailto:sr-users at lists.sip-router.org> 
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
 
 
-----
No virus found in this message.
Checked by AVG - www.avg.com <http://www.avg.com> 
Version: 2014.0.4259 / Virus Database: 3629/6886 - Release Date: 12/02/13
 





-- 
Daniel-Constantin Mierla - http://www.asipto.com
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda

No virus found in this message.
Checked by AVG - www.avg.com <http://www.avg.com> 
Version: 2014.0.4259 / Virus Database: 3629/6886 - Release Date: 12/02/13

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.sip-router.org/pipermail/sr-users/attachments/20131203/7d689a5c/attachment-0001.html>


More information about the sr-users mailing list