[sr-dev] High memory consumption during load tests

Andrey Deykunov deykunov at gmail.com
Wed Apr 8 12:19:03 CEST 2020


Hi,

I've noticed more than 14 Gb RSS memory consumption during load test (8000
WSS clients send registrations):
[image: pic2.png]

[image: pic1.png]


# ps aux | grep kamailio
hpbx     15948  0.0  0.1 2232328 22920 ?       S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15949  1.8  3.8 2232460 789212 ?      S    01:40   1:23
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15950  1.8  3.8 2232460 790164 ?      S    01:40   1:22
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15951  1.7  3.8 2232460 790656 ?      S    01:40   1:22
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15952  1.8  3.8 2232460 790180 ?      S    01:40   1:22
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15953  1.7  3.8 2232460 788192 ?      S    01:40   1:22
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15954  1.8  3.8 2232460 789484 ?      S    01:40   1:22
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15955  1.7  3.8 2232460 789640 ?      S    01:40   1:22
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15959  1.8  3.8 2232460 788104 ?      S    01:40   1:22
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15960  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15961  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15963  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15964  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15966  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15968  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15969  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15971  0.0  0.0 2232328 8272 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15972  0.0  0.0 2232328 8152 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15973  0.0  0.0 2232328 8152 ?        S    01:40   0:02
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15975  0.0  0.0 2232328 8156 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15977  4.8  4.0 2232428 835908 ?      S    01:40   3:43
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15979  0.0  2.5 2232452 517192 ?      S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15980  0.0  2.5 2232328 515344 ?      S    01:40   0:03
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15984  0.0  0.0 2232328 8004 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15986  0.0  0.0 2232328 8276 ?        S    01:40   0:00
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15987  2.4  3.9 2232492 817888 ?      S    01:40   1:50
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15989  2.4  3.9 2232492 816604 ?      S    01:40   1:52
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15991  2.4  3.9 2232492 816952 ?      S    01:40   1:51
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15992  2.3  3.9 2232492 817236 ?      S    01:40   1:49
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15993  2.4  3.9 2232492 818900 ?      S    01:40   1:51
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15994  2.4  3.9 2232492 817692 ?      S    01:40   1:50
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15996  2.4  3.9 2232492 816832 ?      S    01:40   1:50
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     15998  2.3  3.9 2232492 814336 ?      S    01:40   1:49
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg
hpbx     16001  2.1  3.1 2232328 655908 ?      S    01:40   1:37
/var/lib/ums/sbin/kamailio -m 2048 -M 12 -x qm -P
/var/run/kamailio/kamailio.pid -f /var/lib/ums/etc/sipp/sipp.cfg


Shared memory consumption is:
# /var/lib/ums/sbin/kamcmd -s tcp:localhost:2048 stats.get_statistics shmem:
shmem:fragments = 4
shmem:free_size = 1291909000
shmem:max_used_size = 855663448
shmem:real_used_size = 855574648
shmem:total_size = 2147483648
shmem:used_size = 818055392


Also, I don't understand why kamailio maps more than 2 GB per every child
process:
# pmap -x $(pidof kamailio) | grep total
total kB         2232332  659732  653680
total kB         2232496  816132  802908
total kB         2232496  818680  805456
total kB         2232496  818544  805320
total kB         2232496  818900  805676
total kB         2232496  817888  804664
total kB         2232496  818436  805212
total kB         2232496  818124  804900
total kB         2232496  818508  805284
total kB         2232332    8344    3648
total kB         2232332    8072    3580
total kB         2232332  517360  512444
total kB         2232456  519752  514580
total kB         2232432  835996  829552
total kB         2232332    8428    3576
total kB         2232332    8832    3852
total kB         2232332   10380    4604
total kB         2232332    8480    3564
total kB         2232332    8480    3564
total kB         2232332    8480    3564
total kB         2232332    8480    3564
total kB         2232332    8480    3564
total kB         2232332    8480    3564
total kB         2232332    8480    3564
total kB         2232332    8480    3564
total kB         2232464  789152  777596
total kB         2232464  790128  778484
total kB         2232464  790904  779004
total kB         2232464  788796  777240
total kB         2232464  790284  778384
total kB         2232464  790732  779176
total kB         2232464  790360  778716
total kB         2232464  790512  778612
total kB         2232332   22920    8992


Any ideas what is happening here?

Andrey
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.kamailio.org/pipermail/sr-dev/attachments/20200408/5698b57c/attachment-0001.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: pic1.png
Type: image/png
Size: 82883 bytes
Desc: not available
URL: <http://lists.kamailio.org/pipermail/sr-dev/attachments/20200408/5698b57c/attachment-0002.png>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: pic2.png
Type: image/png
Size: 25268 bytes
Desc: not available
URL: <http://lists.kamailio.org/pipermail/sr-dev/attachments/20200408/5698b57c/attachment-0003.png>


More information about the sr-dev mailing list