[sr-dev] [kamailio/kamailio] Kamailio 5.2.2 - Segmentation fault in libcrypto.so.1.1 (#1860)

shaunjstokes notifications at github.com
Tue Apr 16 21:09:14 CEST 2019


Unfortunately that didn't fix the problem, however the core dump has changed.

```
Reading symbols from /usr/local/kamailio/sbin/kamailio...done.
[New LWP 26088]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/kamailio/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f /usr/loc'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  aesni_ecb_encrypt () at crypto/aes/aesni-x86_64.s:624
624             movups  (%rcx),%xmm1
```

```
(gdb) disas aesni_ecb_encrypt
Dump of assembler code for function aesni_ecb_encrypt:
   0x00007ffba23869e0 <+0>:     and    $0xfffffffffffffff0,%rdx
   0x00007ffba23869e4 <+4>:     je     0x7ffba2386f34 <aesni_ecb_encrypt+1364>
   0x00007ffba23869ea <+10>:    mov    0xf0(%rcx),%eax
   0x00007ffba23869f0 <+16>:    movups (%rcx),%xmm0
   0x00007ffba23869f3 <+19>:    mov    %rcx,%r11
   0x00007ffba23869f6 <+22>:    mov    %eax,%r10d
   0x00007ffba23869f9 <+25>:    test   %r8d,%r8d
   0x00007ffba23869fc <+28>:    je     0x7ffba2386c50 <aesni_ecb_encrypt+624>
   0x00007ffba2386a02 <+34>:    cmp    $0x80,%rdx
   0x00007ffba2386a09 <+41>:    jb     0x7ffba2386af7 <aesni_ecb_encrypt+279>
   0x00007ffba2386a0f <+47>:    movdqu (%rdi),%xmm2
   0x00007ffba2386a13 <+51>:    movdqu 0x10(%rdi),%xmm3
   0x00007ffba2386a18 <+56>:    movdqu 0x20(%rdi),%xmm4
   0x00007ffba2386a1d <+61>:    movdqu 0x30(%rdi),%xmm5
   0x00007ffba2386a22 <+66>:    movdqu 0x40(%rdi),%xmm6
   0x00007ffba2386a27 <+71>:    movdqu 0x50(%rdi),%xmm7
   0x00007ffba2386a2c <+76>:    movdqu 0x60(%rdi),%xmm8
   0x00007ffba2386a32 <+82>:    movdqu 0x70(%rdi),%xmm9
   0x00007ffba2386a38 <+88>:    lea    0x80(%rdi),%rdi
   0x00007ffba2386a3f <+95>:    sub    $0x80,%rdx
   0x00007ffba2386a46 <+102>:   jmp    0x7ffba2386aae <aesni_ecb_encrypt+206>
   0x00007ffba2386a48 <+104>:   nopl   0x0(%rax,%rax,1)
   0x00007ffba2386a50 <+112>:   movups %xmm2,(%rsi)
   0x00007ffba2386a53 <+115>:   mov    %r11,%rcx
   0x00007ffba2386a56 <+118>:   movdqu (%rdi),%xmm2
   0x00007ffba2386a5a <+122>:   mov    %r10d,%eax
   0x00007ffba2386a5d <+125>:   movups %xmm3,0x10(%rsi)
   0x00007ffba2386a61 <+129>:   movdqu 0x10(%rdi),%xmm3
   0x00007ffba2386a66 <+134>:   movups %xmm4,0x20(%rsi)
   0x00007ffba2386a6a <+138>:   movdqu 0x20(%rdi),%xmm4
   0x00007ffba2386a6f <+143>:   movups %xmm5,0x30(%rsi)
   0x00007ffba2386a73 <+147>:   movdqu 0x30(%rdi),%xmm5
   0x00007ffba2386a78 <+152>:   movups %xmm6,0x40(%rsi)
   0x00007ffba2386a7c <+156>:   movdqu 0x40(%rdi),%xmm6
   0x00007ffba2386a81 <+161>:   movups %xmm7,0x50(%rsi)
   0x00007ffba2386a85 <+165>:   movdqu 0x50(%rdi),%xmm7
   0x00007ffba2386a8a <+170>:   movups %xmm8,0x60(%rsi)
   0x00007ffba2386a8f <+175>:   movdqu 0x60(%rdi),%xmm8
   0x00007ffba2386a95 <+181>:   movups %xmm9,0x70(%rsi)
   0x00007ffba2386a9a <+186>:   lea    0x80(%rsi),%rsi
   0x00007ffba2386aa1 <+193>:   movdqu 0x70(%rdi),%xmm9
   0x00007ffba2386aa7 <+199>:   lea    0x80(%rdi),%rdi
   0x00007ffba2386aae <+206>:   callq  0x7ffba23867c0 <_aesni_encrypt8>
   0x00007ffba2386ab3 <+211>:   sub    $0x80,%rdx
   0x00007ffba2386aba <+218>:   jae    0x7ffba2386a50 <aesni_ecb_encrypt+112>
   0x00007ffba2386abc <+220>:   movups %xmm2,(%rsi)
   0x00007ffba2386abf <+223>:   mov    %r11,%rcx
   0x00007ffba2386ac2 <+226>:   movups %xmm3,0x10(%rsi)
   0x00007ffba2386ac6 <+230>:   mov    %r10d,%eax
   0x00007ffba2386ac9 <+233>:   movups %xmm4,0x20(%rsi)
   0x00007ffba2386acd <+237>:   movups %xmm5,0x30(%rsi)
   0x00007ffba2386ad1 <+241>:   movups %xmm6,0x40(%rsi)
   0x00007ffba2386ad5 <+245>:   movups %xmm7,0x50(%rsi)
   0x00007ffba2386ad9 <+249>:   movups %xmm8,0x60(%rsi)
   0x00007ffba2386ade <+254>:   movups %xmm9,0x70(%rsi)
   0x00007ffba2386ae3 <+259>:   lea    0x80(%rsi),%rsi
   0x00007ffba2386aea <+266>:   add    $0x80,%rdx
   0x00007ffba2386af1 <+273>:   je     0x7ffba2386f34 <aesni_ecb_encrypt+1364>
   0x00007ffba2386af7 <+279>:   movups (%rdi),%xmm2
   0x00007ffba2386afa <+282>:   cmp    $0x20,%rdx
   0x00007ffba2386afe <+286>:   jb     0x7ffba2386b70 <aesni_ecb_encrypt+400>
   0x00007ffba2386b00 <+288>:   movups 0x10(%rdi),%xmm3
   0x00007ffba2386b04 <+292>:   je     0x7ffba2386ba0 <aesni_ecb_encrypt+448>
   0x00007ffba2386b0a <+298>:   movups 0x20(%rdi),%xmm4
   0x00007ffba2386b0e <+302>:   cmp    $0x40,%rdx
   0x00007ffba2386b12 <+306>:   jb     0x7ffba2386bc0 <aesni_ecb_encrypt+480>
   0x00007ffba2386b18 <+312>:   movups 0x30(%rdi),%xmm5
   0x00007ffba2386b1c <+316>:   je     0x7ffba2386be0 <aesni_ecb_encrypt+512>
   0x00007ffba2386b22 <+322>:   movups 0x40(%rdi),%xmm6
   0x00007ffba2386b26 <+326>:   cmp    $0x60,%rdx
   0x00007ffba2386b2a <+330>:   jb     0x7ffba2386c00 <aesni_ecb_encrypt+544>
   0x00007ffba2386b30 <+336>:   movups 0x50(%rdi),%xmm7
   0x00007ffba2386b34 <+340>:   je     0x7ffba2386c20 <aesni_ecb_encrypt+576>
   0x00007ffba2386b3a <+346>:   movdqu 0x60(%rdi),%xmm8
   0x00007ffba2386b40 <+352>:   xorps  %xmm9,%xmm9
   0x00007ffba2386b44 <+356>:   callq  0x7ffba23867c0 <_aesni_encrypt8>
   0x00007ffba2386b49 <+361>:   movups %xmm2,(%rsi)
   0x00007ffba2386b4c <+364>:   movups %xmm3,0x10(%rsi)
   0x00007ffba2386b50 <+368>:   movups %xmm4,0x20(%rsi)
   0x00007ffba2386b54 <+372>:   movups %xmm5,0x30(%rsi)
   0x00007ffba2386b58 <+376>:   movups %xmm6,0x40(%rsi)
   0x00007ffba2386b5c <+380>:   movups %xmm7,0x50(%rsi)
   0x00007ffba2386b60 <+384>:   movups %xmm8,0x60(%rsi)
   0x00007ffba2386b65 <+389>:   jmpq   0x7ffba2386f34 <aesni_ecb_encrypt+1364>
   0x00007ffba2386b6a <+394>:   nopw   0x0(%rax,%rax,1)
   0x00007ffba2386b70 <+400>:   movups (%rcx),%xmm0
   0x00007ffba2386b73 <+403>:   movups 0x10(%rcx),%xmm1
   0x00007ffba2386b77 <+407>:   lea    0x20(%rcx),%rcx
   0x00007ffba2386b7b <+411>:   xorps  %xmm0,%xmm2
   0x00007ffba2386b7e <+414>:   aesenc %xmm1,%xmm2
   0x00007ffba2386b83 <+419>:   dec    %eax
=> 0x00007ffba2386b85 <+421>:   movups (%rcx),%xmm1
   0x00007ffba2386b88 <+424>:   lea    0x10(%rcx),%rcx
   0x00007ffba2386b8c <+428>:   jne    0x7ffba2386b7e <aesni_ecb_encrypt+414>
   0x00007ffba2386b8e <+430>:   aesenclast %xmm1,%xmm2
   0x00007ffba2386b93 <+435>:   movups %xmm2,(%rsi)
   0x00007ffba2386b96 <+438>:   jmpq   0x7ffba2386f34 <aesni_ecb_encrypt+1364>
   0x00007ffba2386b9b <+443>:   nopl   0x0(%rax,%rax,1)
   0x00007ffba2386ba0 <+448>:   callq  0x7ffba2386340 <_aesni_encrypt2>
   0x00007ffba2386ba5 <+453>:   movups %xmm2,(%rsi)
   0x00007ffba2386ba8 <+456>:   movups %xmm3,0x10(%rsi)
   0x00007ffba2386bac <+460>:   jmpq   0x7ffba2386f34 <aesni_ecb_encrypt+1364>
   0x00007ffba2386bb1 <+465>:   nopl   0x0(%rax,%rax,1)
   0x00007ffba2386bb6 <+470>:   nopw   %cs:0x0(%rax,%rax,1)
   0x00007ffba2386bc0 <+480>:   callq  0x7ffba2386400 <_aesni_encrypt3>
   0x00007ffba2386bc5 <+485>:   movups %xmm2,(%rsi)
   0x00007ffba2386bc8 <+488>:   movups %xmm3,0x10(%rsi)
   0x00007ffba2386bcc <+492>:   movups %xmm4,0x20(%rsi)
   0x00007ffba2386bd0 <+496>:   jmpq   0x7ffba2386f34 <aesni_ecb_encrypt+1364>
```

```
(gdb) i r
rax            0xfc3ba328       4231766824
rbx            0x1      1
rcx            0x7ffb96805000   140718538510336
rdx            0x10     16
rsi            0x7ffb58c04108   140717502513416
rdi            0x7ffb56b0c898   140717467945112
rbp            0x7ffcff3bd080   0x7ffcff3bd080
rsp            0x7ffcff3bd038   0x7ffcff3bd038
r8             0x1      1
r9             0x0      0
r10            0x0      0
r11            0x7ffb5a3a8270   140717527302768
r12            0x55e1430f52d3   94425981080275
r13            0x40000000       1073741824
r14            0x10000000       268435456
r15            0x6      6
rip            0x7ffba2386b85   0x7ffba2386b85 <aesni_ecb_encrypt+421>
eflags         0x10287  [ CF PF SF IF RF ]
cs             0x33     51
ss             0x2b     43
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x0      0
```

I've been looking at buffers and limits to see if we're perhaps hitting some kind of limit that could be triggering this and have found that increasing tls_max_connections and tcp_max_connections to 32768 appears to increase the threshold before the segmentation fault occurs. We're now able to get >20000 presence subscriptions with-out any segmentation faults.

I have more testing to do before I can reach a conclusion but it appears there may be a correlation between tls_max_connections and the segmentation fault.

-- 
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/1860#issuecomment-483804879
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.kamailio.org/pipermail/sr-dev/attachments/20190416/109bb0d9/attachment-0001.html>


More information about the sr-dev mailing list