[SR-Users] Info: initialization of libssl 1.1.+ with PTHREAD_PROCESS_SHARED attribute for pthread mutex and rwlock

Andrew Chen achen at fuze.com
Thu Sep 19 16:52:31 CEST 2019


So I got the latest bt full output;

(gdb) bt full
#0  0x00007f160fa6c585 in ?? () from
/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#1  0x00007f160fb485a7 in ?? () from
/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#2  0x00007f160fb5412b in EVP_EncryptUpdate () from
/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#3  0x00007f160fb8691e in ?? () from
/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#4  0x00007f160fb86bfd in ?? () from
/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#5  0x00007f160fb87a1f in RAND_DRBG_generate () from
/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#6  0x00007f160fb87cbe in RAND_DRBG_bytes () from
/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1
No symbol table info available.
#7  0x00007f160fedeb65 in ?? () from /usr/lib/x86_64-linux-gnu/libssl.so.1.1
No symbol table info available.
#8  0x00007f160ff0ff36 in ?? () from /usr/lib/x86_64-linux-gnu/libssl.so.1.1
No symbol table info available.
#9  0x00007f160feff4b2 in ?? () from /usr/lib/x86_64-linux-gnu/libssl.so.1.1
No symbol table info available.
#10 0x00007f160feeb4c4 in SSL_do_handshake () from
/usr/lib/x86_64-linux-gnu/libssl.so.1.1
No symbol table info available.
#11 0x00007f161017e6b8 in tls_accept (c=0x7f1608df0398,
error=0x7ffd4dedaff0) at tls_server.c:422
        ret = 1307422640
        ssl = 0x7f160c1bec60
        cert = 0x7f160a6d1590
        tls_c = 0x7f160b44db70
        tls_log = 269998018
        __func__ = "tls_accept"
        pkey = 0x0
#12 0x00007f1610187dbd in tls_read_f (c=0x7f1608df0398,
flags=0x7ffd4defb38c) at tls_server.c:1116
        r = 0x7f1608df0418
        bytes_free = 16391
        bytes_read = 517
        read_size = 16391
        ssl_error = 0
        ssl_read = 0
        ssl = 0x7f160c1bec60
        rd_buf =
"\026\003\001\002\000\001\000\001\374\003\003\n\346\346\f\214\235\236PQX\256*\264ߟV\245<\333\034\031>&eYG@\"\347\030n\251
K\226\357.n\343\376\t\245\341\217t\201\300R6\215\217ٸ\333fIH@
\315$xV,\301\235\000\"\032\032\023\001\023\002\023\003\300+\300/\300,\300\060̨̩\300\023\300\024\000\234\000\235\000/\000\065\000\n\001\000\001\221ZZ\000\000\000\000\000#\000!\000\000\
036ashintgfnsgvip.fuzemeeting.com
\000\027\000\000\377\001\000\001\000\000\n\000\n\000\bZZ\000\035\000\027\000\030\000\v\000\002\001\000\000#\000\320>\035Z\257\345\344\373\256\355\310l\317"...
        wr_buf = "\026\003\003\000\332\004\000\000\326\000\000\034
\000\320(n\330?\374]\200\n\032\367U\217쮹2!\247\305s\207\037t\347z.&(L\351\036xD\035\343\067\005o7\271\f[\276l\246\024\231WU\020\322H\223\062uFr\020\375\256\070^\355<\347\277\v\227\021\330\027\310(\276o\375ԁBM\226\362\364\321o\035[\002
\277\357\232.\266:\301\365\315\t\350\230\334\v\334\064\003&\017؉\315\350\375d\233\027\273\034\311\020R\314\003!\210\250&&\253\306C1/\371-g\371\001\374I\353\363r\270\347s9\245\256\255&\361\276d\201\274\003\030,}\345\256\035\"5\370i\237\267\301\232\300Ѹ{c\212\344\235\375^\005\200a\375\372\241\066-3"...
        rd = {buf = 0x7ffd4dedb0c0 "\026\003\001\002", pos = 517, used =
517, size = 65536}
        wr = {buf = 0x7ffd4deeb0c0 "\026\003\003", pos = 0, used = 0, size
= 65536}
        tls_c = 0x7f160b44db70
        enc_rd_buf = 0x0
        n = 0
        flush_flags = 32765
        err_src = 0x7f16101b05ae "TLS read:"
        x = 1307554700
        tls_dbg = 0
        __func__ = "tls_read_f"
--Type <RET> for more, q to quit, c to continue without paging--
#13 0x000055c48a04e753 in tcp_read_headers (c=0x7f1608df0398,
read_flags=0x7ffd4defb38c) at core/tcp_read.c:469
        bytes = 550
        remaining = 21956
        p = 0x10500000004 <error: Cannot access memory at address
0x10500000004>
        r = 0x7f1608df0418
        mc = 32534
        body_len = 5731
        mfline = 0x0
        mtransid = {s = 0x7f16166d8216 "UTC", len = 48}
        __func__ = "tcp_read_headers"
#14 0x000055c48a0564d8 in tcp_read_req (con=0x7f1608df0398,
bytes_read=0x7ffd4defb388, read_flags=0x7ffd4defb38c) at
core/tcp_read.c:1496
        bytes = -1
        total_bytes = 0
        resp = 1
        size = 4294967298
        req = 0x7f1608df0418
        dst = {send_sock = 0x7ffd4defb2e0, to = {s = {sa_family = 15407,
sa_data = "E\004\026\177\000\000ȳ\357M\375\177\000"}, sin = {sin_family =
15407, sin_port = 1093, sin_addr = {s_addr = 32534},
              sin_zero = "ȳ\357M\375\177\000"}, sin6 = {sin6_family =
15407, sin6_port = 1093, sin6_flowinfo = 32534, sin6_addr = {__in6_u =
{__u6_addr8 = "ȳ\357M\375\177\000\000\004\263\357M\375\177\000",
                  __u6_addr16 = {46024, 19951, 32765, 0, 45828, 19951,
32765, 0}, __u6_addr32 = {1307554760, 32765, 1307554564, 32765}}},
sin6_scope_id = 20}}, id = 4, proto = -32 '\340', send_flags = {
            f = 35360, blst_imask = 21956}}
        c = 105 'i'
        ret = 117082496
        __func__ = "tcp_read_req"
#15 0x000055c48a05b5fd in handle_io (fm=0x7f1614851900, events=1, idx=-1)
at core/tcp_read.c:1804
        ret = 8
        n = 8
        read_flags = 1
        con = 0x7f1608df0398
        s = 30
        resp = 1
        t = 551514846
        __func__ = "handle_io"
#16 0x000055c48a048b5f in io_wait_loop_epoll (h=0x55c48a4ff020 <io_w>, t=2,
repeat=0) at core/io_wait.h:1065
        n = 1
        r = 0
        fm = 0x7f1614851900
        revents = 1
        __func__ = "io_wait_loop_epoll"
#17 0x000055c48a05d647 in tcp_receive_loop (unix_sock=90) at
core/tcp_read.c:1974
        __func__ = "tcp_receive_loop"
#18 0x000055c489f0ef52 in tcp_init_children () at core/tcp_main.c:4853
        r = 25
        i = 15
        reader_fd_1 = 90
        pid = 0
        si_desc = "tcp receiver
(tls:206.81.180.49:443)\000\000\000\000\003\000\000\000\001",
'\000' <repeats 11 times>,
"\004p.\a\026\177\000\000Ae\037\212\000\000\000\000H\273]\024\026\177\000\000\260j\032\212\000\000\000\000\003\000\000\000fJ\000\000#Q\037\212\304U\000\000P\315c\026\026\177\000\000#Q\037\212\304U\000\000\000\000\000\b\000\000\000"
        si = 0x0
        __func__ = "tcp_init_children"
#19 0x000055c489e0f94a in main_loop () at main.c:1745
        i = 4
        pid = 19045
        si = 0x0
        si_desc = "udp receiver child=3
sock=206.81.180.49:443\000\000\000\000\000\200w\033\212\304U\000\000\000\232\263!\326\tU\322p\220.\a\026\177\000\000P\315c\026\026\177\000\000#Q\037\212\304U\000\000\000\0--Type
<RET> for more, q to quit, c to continue without paging--
00\000\b\000\000\000\000\000\000\000\002\000\000\000\000\006\000\000\000\000\000\000\000`\267\357M\375\177\000\000\000\207\003\212\304U\000"
        nrprocs = 4
        woneinit = 1
        __func__ = "main_loop"
#20 0x000055c489e16a7b in main (argc=13, argv=0x7ffd4defba28) at main.c:2696
        cfg_stream = 0x55c48a681260
        c = -1
        r = 0
        tmp = 0x7ffd4defced1 ""
        tmp_len = 1307556016
        port = 32765
        proto = 3
        options = 0x55c48a192568
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
        ret = -1
        seed = 1834649487
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x7f16166d6787
        p = 0x9c0 <error: Cannot access memory at address 0x9c0>
        st = {st_dev = 23, st_ino = 430, st_nlink = 2, st_mode = 16872,
st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize =
4096, st_blocks = 0, st_atim = {tv_sec = 1568885337,
            tv_nsec = 27216035}, st_mtim = {tv_sec = 1568885337, tv_nsec =
27216035}, st_ctim = {tv_sec = 1568885337, tv_nsec = 27216035},
__glibc_reserved = {0, 0, 0}}
        __func__ = "main"


Since I'm seeing memory related errors, here is a list of running kam
processes with shared/private mem settings on each of them:

5 S kamailio 28521     1  0  80   0 - 79922 pause  09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28531 28521  0  80   0 - 80017 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28532 28521  0  80   0 - 79986 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28533 28521  0  80   0 - 80018 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28534 28521  0  80   0 - 79986 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28535 28521  0  80   0 - 79922 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28536 28521  0  80   0 - 79922 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28537 28521  0  80   0 - 79922 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28538 28521  0  80   0 - 79922 skb_wa 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28539 28521  0  80   0 - 79922 sigtim 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28540 28521  0  80   0 - 79922 pause  09:53 ?        00:00:05
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28541 28521  0  80   0 - 79922 poll_s 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28542 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28543 28521  0  80   0 - 79922 hrtime 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28544 28521  0  80   0 - 79922 poll_s 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28545 28521  0  80   0 - 79922 poll_s 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28546 28521  0  80   0 - 79922 poll_s 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28547 28521  0  80   0 - 79922 poll_s 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28548 28521  0  80   0 - 79922 pipe_w 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28549 28521  0  80   0 - 79922 poll_s 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28550 28521  0  80   0 - 79922 hrtime 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28551 28521  0  80   0 - 79987 ep_pol 09:53 ?        00:00:01
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28552 28521  0  80   0 - 79984 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28553 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28554 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28555 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28556 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28557 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28558 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28559 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28560 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28561 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28562 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28564 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28565 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28566 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28567 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28568 28521  0  80   0 - 80054 ep_pol 09:53 ?        00:00:01
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28569 28521  0  80   0 - 80021 ep_pol 09:53 ?        00:00:01
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28570 28521  0  80   0 - 80021 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28571 28521  0  80   0 - 80021 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28572 28521  0  80   0 - 79990 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28573 28521  0  80   0 - 79988 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28574 28521  0  80   0 - 79989 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28575 28521  0  80   0 - 79989 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28576 28521  0  80   0 - 79924 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28577 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28578 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28579 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28580 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28581 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28582 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28583 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:00
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio
1 S kamailio 28584 28521  0  80   0 - 79922 ep_pol 09:53 ?        00:00:02
/usr/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f
/etc/kamailio/kamailio.cfg -m 128 -M 32 -u kamailio -g kamailio



On Wed, Sep 18, 2019 at 4:35 PM Andrew Chen <achen at fuze.com> wrote:

> Yup understood.  I'll provide the bt when we can reproduce this.
>
> Thanks
>
> On Wed, Sep 18, 2019 at 4:32 PM Henning Westerholt <hw at skalatan.de> wrote:
>
>> Hi Andrew,
>>
>> the 5.2.4 does not contain the 5.3-pre1 workaround yet.
>>
>> So if you test the workaround you need to install the lastest 5.3 from
>> the git master debian repository.
>>
>> But nevertheless it would be good to look at the backtrace, while you
>> have it.
>>
>> Cheers,
>>
>> Henning
>> Am 18.09.19 um 22:10 schrieb Joel Serrano:
>>
>> Hi Andrew,
>>
>> Download from here:
>> http://deb.kamailio.org/kamailio52/pool/main/k/kamailio/
>>
>> You have kamailio-dbg v5.2.4 (latest 5.2) for all distros...
>>
>> Example for amd64+stretch:
>> http://deb.kamailio.org/kamailio52/pool/main/k/kamailio/kamailio-dbg_5.2.4%2Bstretch_amd64.deb
>>
>> Cheers,
>> Joel.
>>
>>
>>
>>
>>
>> On Wed, Sep 18, 2019 at 12:16 PM Andrew Chen <achen at fuze.com> wrote:
>>
>>> Hi Henning,
>>>
>>> I'm running 5.2.4.  Will this deb package work?
>>>
>>> On Wed, Sep 18, 2019 at 2:39 PM Andrew Chen <achen at fuze.com> wrote:
>>>
>>>> Thanks Henning
>>>>
>>>> On Wed, Sep 18, 2019 at 2:13 PM Henning Westerholt <hw at skalatan.de>
>>>> wrote:
>>>>
>>>>> Hello Andrew,
>>>>>
>>>>> strange - it should be there, as it is available on deb.kamailio.org.
>>>>>
>>>>> Here e.g. the one for 64bit stretch:
>>>>>
>>>>>
>>>>> https://deb.kamailio.org/kamailiodev-nightly/pool/main/k/kamailio/kamailio-dbg_5.3.0~pre1%2B0~20190918005413.1490%2Bstretch_amd64.deb
>>>>>
>>>>> Cheers,
>>>>>
>>>>> Henning
>>>>> Am 18.09.19 um 18:52 schrieb Andrew Chen:
>>>>>
>>>>> I don't see one for debug:
>>>>>
>>>>> root at ashintgkama51:~ # apt-cache search kamailio
>>>>>
>>>>> resource-agents - Cluster Resource Agents
>>>>>
>>>>> kamailio - very fast, dynamic and configurable SIP server
>>>>>
>>>>> kamailio-autheph-modules - authentication using ephemeral credentials
>>>>> for the Kamailio SIP server
>>>>>
>>>>> kamailio-berkeley-bin - Berkeley database module for Kamailio - helper
>>>>> program
>>>>>
>>>>> kamailio-berkeley-modules - Berkeley database module for the Kamailio
>>>>> SIP server
>>>>>
>>>>> kamailio-carrierroute-modules - carrierroute module for the Kamailio
>>>>> SIP server
>>>>>
>>>>> kamailio-cnxcc-modules - cnxcc modules for the Kamailio SIP server
>>>>>
>>>>> kamailio-cpl-modules - CPL module (CPL interpreter engine) for the
>>>>> Kamailio SIP server
>>>>>
>>>>> kamailio-erlang-modules - Erlang modules for the Kamailio SIP server
>>>>>
>>>>> kamailio-geoip-modules - geoip module for the Kamailio SIP Server
>>>>>
>>>>> kamailio-geoip2-modules - geoip2 module for the Kamailio SIP Server
>>>>>
>>>>> kamailio-ims-modules - IMS module for the Kamailio SIP server
>>>>>
>>>>> kamailio-json-modules - Json parser and Json-RPC modules for the
>>>>> Kamailio SIP server
>>>>>
>>>>> On Wed, Sep 18, 2019 at 11:26 AM Daniel-Constantin Mierla <
>>>>> miconda at gmail.com> wrote:
>>>>>
>>>>>> You have to install the debug symbols package, otherwise the
>>>>>> backtrace doesn't show any useful information. It just require to install
>>>>>> that package and then get again the backtraces. The package name should be
>>>>>> kamailio-dbg.
>>>>>>
>>>>>> Cheers,
>>>>>> Daniel
>>>>>> On 18.09.19 16:37, Andrew Chen wrote:
>>>>>>
>>>>>> Sorry..forgot bt full output
>>>>>>
>>>>>> (gdb) bt full
>>>>>>
>>>>>> #0  0x0000562fcc3544c0 in ?? ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> #1  0x0000562fcc354632 in ?? ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> #2  0x0000562fcc354974 in local_timer_run ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> #3  0x0000562fcc3fb256 in ?? ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> #4  0x0000562fcc3fb64c in tcp_receive_loop ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> #5  0x0000562fcc2acf52 in tcp_init_children ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> #6  0x0000562fcc1ad94a in main_loop ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> #7  0x0000562fcc1b4a7b in main ()
>>>>>>
>>>>>> No symbol table info available.
>>>>>>
>>>>>> (gdb)
>>>>>>
>>>>>> On Wed, Sep 18, 2019 at 10:35 AM Andrew Chen <achen at fuze.com> wrote:
>>>>>>
>>>>>>> backtrace output:
>>>>>>>
>>>>>>>
>>>>>>> (gdb) bt
>>>>>>>
>>>>>>> #0  0x0000562fcc3544c0 in ?? ()
>>>>>>>
>>>>>>> #1  0x0000562fcc354632 in ?? ()
>>>>>>>
>>>>>>> #2  0x0000562fcc354974 in local_timer_run ()
>>>>>>>
>>>>>>> #3  0x0000562fcc3fb256 in ?? ()
>>>>>>>
>>>>>>> #4  0x0000562fcc3fb64c in tcp_receive_loop ()
>>>>>>>
>>>>>>> #5  0x0000562fcc2acf52 in tcp_init_children ()
>>>>>>>
>>>>>>> #6  0x0000562fcc1ad94a in main_loop ()
>>>>>>>
>>>>>>> #7  0x0000562fcc1b4a7b in main ()
>>>>>>>
>>>>>>> (gdb)
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> On Wed, Sep 18, 2019 at 10:01 AM Andrew Chen <achen at fuze.com> wrote:
>>>>>>>
>>>>>>>> I'll look for the core.
>>>>>>>>
>>>>>>>> As part of our release process, I don't think we can release master
>>>>>>>> branch of Kamailio.
>>>>>>>>
>>>>>>>> On Wed, Sep 18, 2019 at 9:54 AM Daniel-Constantin Mierla <
>>>>>>>> miconda at gmail.com> wrote:
>>>>>>>>
>>>>>>>>> Hello,
>>>>>>>>>
>>>>>>>>> did you get core files? If yes, it would be good to see the gdb
>>>>>>>>> bracktraces.
>>>>>>>>>
>>>>>>>>> Have you used the pre-loaded library workaround?
>>>>>>>>>
>>>>>>>>> The code was not backported to branch 5.2 yet, I wanted to wait a
>>>>>>>>> bit and see if there are reports with good/bad results. If you want to use
>>>>>>>>> it, then you have to install nightly builds from master branch (development
>>>>>>>>> version, should be 5.3.0-pre1 at this moment).
>>>>>>>>>
>>>>>>>>> Cheers,
>>>>>>>>> Daniel
>>>>>>>>> On 18.09.19 15:26, Andrew Chen wrote:
>>>>>>>>>
>>>>>>>>> Hi Daniel,
>>>>>>>>> Which apt repository can we get this workaround?   Today I noticed
>>>>>>>>> our Kamailio 5.2.4 cored during one of our load test sessions:
>>>>>>>>>
>>>>>>>>> Wed Sep 18 09:31:14 2019] kamailio[26997]: segfault at
>>>>>>>>> 7f8501366000 ip 00007f8501e58585 sp 00007fffd5c84248 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f8501dd4000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:12:29 2019] kamailio[27110]: segfault at
>>>>>>>>> 7f1a8eea1000 ip 00007f1a8f993585 sp 00007ffddc8848c8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f1a8f90f000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:12:43 2019] kamailio[11060]: segfault at 20 ip
>>>>>>>>> 00007f0a85dac128 sp 00007fff6847aad0 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f0a85c40000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:12:44 2019] kamailio[11059]: segfault at f0 ip
>>>>>>>>> 00007f0a85cc43ea sp 00007fff6847a9e8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f0a85c40000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:13:02 2019] kamailio[11187]: segfault at
>>>>>>>>> 7f75e06c8000 ip 00007f75e11ba585 sp 00007ffe29b47c88 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f75e1136000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:13:17 2019] kamailio[11445]: segfault at
>>>>>>>>> 7f0f3743f000 ip 00007f0f37f31585 sp 00007ffde54daa38 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f0f37ead000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:13:36 2019] kamailio[11644]: segfault at
>>>>>>>>> 7f804309c000 ip 00007f8043b8e585 sp 00007ffe89fe8228 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f8043b0a000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:13:52 2019] kamailio[11799]: segfault at
>>>>>>>>> 7ff59c7e2000 ip 00007ff59d2d4585 sp 00007ffd69d76bb8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7ff59d250000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 10:13:52 2019] kamailio[11792]: segfault at
>>>>>>>>> 7ff59c7e2000 ip 00007ff59d2d4585 sp 00007ffd69d76bb8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7ff59d250000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:48:01 2019] kamailio[12066]: segfault at
>>>>>>>>> 7f22dca6d000 ip 00007f22dd55f585 sp 00007fffbc766ab8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f22dd4db000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:48:14 2019] kamailio[16344]: segfault at
>>>>>>>>> 7f7cca42d000 ip 00007f7ccaf1f585 sp 00007ffe99f16108 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f7ccae9b000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:48:32 2019] kamailio[16708]: segfault at
>>>>>>>>> 7fb309141000 ip 00007fb309c33585 sp 00007ffd07a5f3c8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7fb309baf000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:48:49 2019] kamailio[16824]: segfault at f0 ip
>>>>>>>>> 00007fc59095b3ea sp 00007ffe58a490b8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7fc5908d7000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:04 2019] kamailio[16956]: segfault at
>>>>>>>>> 7fcc22e58000 ip 00007fcc2394a585 sp 00007fffa9c52258 error 4 in
>>>>>>>>> libcrypto.so.1.1[7fcc238c6000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:04 2019] kamailio[16945]: segfault at f0 ip
>>>>>>>>> 00007fcc2394a3ea sp 00007fffa9c52128 error 4 in
>>>>>>>>> libcrypto.so.1.1[7fcc238c6000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:16 2019] kamailio[17244]: segfault at
>>>>>>>>> 7f586ed20000 ip 00007f586f812585 sp 00007ffeae1adcd8 error 4 in
>>>>>>>>> libcrypto.so.1.1[7f586f78e000+29b000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17431]: segfault at 8 ip
>>>>>>>>> 0000562fcc44c5d5 sp 00007ffe825c9dd0 error 4 in
>>>>>>>>> kamailio[562fcc17c000+4a2000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17386]: segfault at 10 ip
>>>>>>>>> 00007f96c3735c3e sp 00007ffe825ca270 error 4 in
>>>>>>>>> dialog.so[7f96c36d6000+be000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17423]: segfault at 0 ip
>>>>>>>>> 0000562fcc3544c0 sp 00007ffe825ca1f0 error 6 in
>>>>>>>>> kamailio[562fcc17c000+4a2000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17378]: segfault at 0 ip
>>>>>>>>> 00007f96c5dd099f sp 00007ffe825c98e0 error 4 in tm.so[7f96c5da5000+13f000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17417]: segfault at 0 ip
>>>>>>>>> 0000562fcc3544c0 sp 00007ffe825ca1f0 error 6 in
>>>>>>>>> kamailio[562fcc17c000+4a2000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17416]: segfault at 0 ip
>>>>>>>>> 0000562fcc3544c0 sp 00007ffe825ca1f0 error 6 in
>>>>>>>>> kamailio[562fcc17c000+4a2000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17380]: segfault at 0 ip
>>>>>>>>> 00007f96c5dd099f sp 00007ffe825c98e0 error 4 in tm.so[7f96c5da5000+13f000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:33 2019] kamailio[17381]: segfault at 0 ip
>>>>>>>>> 00007f96c5dd099f sp 00007ffe825c98e0 error 4 in tm.so[7f96c5da5000+13f000]
>>>>>>>>>
>>>>>>>>> [Wed Sep 18 12:49:34 2019] kamailio[17387]: segfault at 8 ip
>>>>>>>>> 0000562fcc25d1f3 sp 00007ffe825ca370 error 6 in
>>>>>>>>> kamailio[562fcc17c000+4a2000]
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> and I was wondering if this core is related.
>>>>>>>>>
>>>>>>>>> Thanks
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> On Fri, Sep 13, 2019 at 5:18 PM Alex Balashov <
>>>>>>>>> abalashov at evaristesys.com> wrote:
>>>>>>>>>
>>>>>>>>>> Hi Daniel,
>>>>>>>>>>
>>>>>>>>>> Thanks a lot for this work; it's definitely important that
>>>>>>>>>> Kamailio
>>>>>>>>>> build against OpenSSL v1.1, and I'll be happy to contribute some
>>>>>>>>>> testing
>>>>>>>>>> in a live environment in the coming weeks. Much appreciated.
>>>>>>>>>>
>>>>>>>>>> Have a good weekend,
>>>>>>>>>>
>>>>>>>>>> -- Alex
>>>>>>>>>>
>>>>>>>>>> On Fri, Sep 13, 2019 at 01:34:12PM +0200, Daniel-Constantin
>>>>>>>>>> Mierla wrote:
>>>>>>>>>>
>>>>>>>>>> > Hello,
>>>>>>>>>> > after couple of months waiting for feedback about the
>>>>>>>>>> workaround with the
>>>>>>>>>> > preloaded shared library to set the PTHREAD_PROCESS_SHARED
>>>>>>>>>> attribute for
>>>>>>>>>> > pthread mutex/rwlock in order to avoid deadlocks and as it
>>>>>>>>>> seems to be the
>>>>>>>>>> > solution, I just pushed a few commits to move that code in the
>>>>>>>>>> core of
>>>>>>>>>> > kamailio.
>>>>>>>>>> >
>>>>>>>>>> > It this way the initialization with the PTHREAD_PROCESS_SHARED
>>>>>>>>>> attribute is
>>>>>>>>>> > done by default when libssl v1.1 is detected at compile time,
>>>>>>>>>> without the
>>>>>>>>>> > need of using the approach with a preloaded library.
>>>>>>>>>> >
>>>>>>>>>> > The shared library is still kept in the tls module:
>>>>>>>>>> >   *
>>>>>>>>>> >
>>>>>>>>>> https://github.com/kamailio/kamailio/tree/master/src/modules/tls/utils/openssl_mutex_shared
>>>>>>>>>> >
>>>>>>>>>> > Probably is going to be removed in the near future.
>>>>>>>>>> >
>>>>>>>>>> > I am asking the community members that use Kamailio on systems
>>>>>>>>>> with libssl
>>>>>>>>>> > v1.1+ to test the master branch and give feedback. The master
>>>>>>>>>> branch is
>>>>>>>>>> > anyhow in testing phase for preparing the next major release
>>>>>>>>>> v5.3.0, so it
>>>>>>>>>> > will help for that as well.
>>>>>>>>>> >
>>>>>>>>>> > If no issue is going to be reported soon, the commits will be
>>>>>>>>>> backported to
>>>>>>>>>> > stable branches, to re-enable smooth use of Kamailio with
>>>>>>>>>> libssl v1.1+.
>>>>>>>>>> >
>>>>>>>>>> > When libssl v1.1+ is detected, the core links with pthread, but
>>>>>>>>>> that was
>>>>>>>>>> > already present in the Makefiles.defs for different compile
>>>>>>>>>> options. The
>>>>>>>>>> > pthread more or less a default library on linuxes/unixes, so
>>>>>>>>>> this should
>>>>>>>>>> > not introduce any major dependency to the Kamailio core.
>>>>>>>>>> >
>>>>>>>>>> > Cheers,
>>>>>>>>>> > Daniel
>>>>>>>>>> >
>>>>>>>>>> > --
>>>>>>>>>> > Daniel-Constantin Mierla - http://www.asipto.com
>>>>>>>>>> > http://twitter.com/#!/miconda -
>>>>>>>>>> http://www.linkedin.com/in/miconda
>>>>>>>>>>
>>>>>>>>>> > _______________________________________________
>>>>>>>>>> > Kamailio (SER) - Users Mailing List
>>>>>>>>>> > sr-users at lists.kamailio.org
>>>>>>>>>> > https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> --
>>>>>>>>>> Alex Balashov | Principal | Evariste Systems LLC
>>>>>>>>>>
>>>>>>>>>> Tel: +1-706-510-6800 / +1-800-250-5920 (toll-free)
>>>>>>>>>> Web: http://www.evaristesys.com/, http://www.csrpswitch.com/
>>>>>>>>>>
>>>>>>>>>> _______________________________________________
>>>>>>>>>> Kamailio (SER) - Users Mailing List
>>>>>>>>>> sr-users at lists.kamailio.org
>>>>>>>>>> https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> --
>>>>>>>>> Andy Chen
>>>>>>>>> Sr. Telephony Lead Engineer
>>>>>>>>> 415 516 5535 (M)
>>>>>>>>> achen@ <achen at thinkingphones.com>fuze.com
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> *Confidentiality Notice: The information contained in this e-mail
>>>>>>>>> and any
>>>>>>>>> attachments may be confidential. If you are not an intended
>>>>>>>>> recipient, you
>>>>>>>>> are hereby notified that any dissemination, distribution or
>>>>>>>>> copying of this
>>>>>>>>> e-mail is strictly prohibited. If you have received this e-mail in
>>>>>>>>> error,
>>>>>>>>> please notify the sender and permanently delete the e-mail and any
>>>>>>>>> attachments immediately. You should not retain, copy or use this
>>>>>>>>> e-mail or
>>>>>>>>> any attachment for any purpose, nor disclose all or any part of the
>>>>>>>>> contents to any other person. Thank you.*
>>>>>>>>>
>>>>>>>>> _______________________________________________
>>>>>>>>> Kamailio (SER) - Users Mailing Listsr-users at lists.kamailio.orghttps://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
>>>>>>>>>
>>>>>>>>> --
>>>>>>>>> Daniel-Constantin Mierla -- www.asipto.comwww.twitter.com/miconda -- www.linkedin.com/in/miconda
>>>>>>>>> Kamailio Advanced Training, Oct 21-23, 2019, Berlin, Germany -- https://asipto.com/u/kat
>>>>>>>>>
>>>>>>>>>
>>>>>>>>
>>>>>>>> --
>>>>>>>> Andy Chen
>>>>>>>> Sr. Telephony Lead Engineer
>>>>>>>> 415 516 5535 (M)
>>>>>>>> achen@ <achen at thinkingphones.com>fuze.com
>>>>>>>>
>>>>>>>>
>>>>>>>
>>>>>>> --
>>>>>>> Andy Chen
>>>>>>> Sr. Telephony Lead Engineer
>>>>>>> 415 516 5535 (M)
>>>>>>> achen@ <achen at thinkingphones.com>fuze.com
>>>>>>>
>>>>>>>
>>>>>>
>>>>>> --
>>>>>> Andy Chen
>>>>>> Sr. Telephony Lead Engineer
>>>>>> 415 516 5535 (M)
>>>>>> achen@ <achen at thinkingphones.com>fuze.com
>>>>>>
>>>>>>
>>>>>> *Confidentiality Notice: The information contained in this e-mail and
>>>>>> any
>>>>>> attachments may be confidential. If you are not an intended
>>>>>> recipient, you
>>>>>> are hereby notified that any dissemination, distribution or copying
>>>>>> of this
>>>>>> e-mail is strictly prohibited. If you have received this e-mail in
>>>>>> error,
>>>>>> please notify the sender and permanently delete the e-mail and any
>>>>>> attachments immediately. You should not retain, copy or use this
>>>>>> e-mail or
>>>>>> any attachment for any purpose, nor disclose all or any part of the
>>>>>> contents to any other person. Thank you.*
>>>>>>
>>>>>> --
>>>>>> Daniel-Constantin Mierla -- www.asipto.comwww.twitter.com/miconda -- www.linkedin.com/in/miconda
>>>>>> Kamailio Advanced Training, Oct 21-23, 2019, Berlin, Germany -- https://asipto.com/u/kat
>>>>>>
>>>>>>
>>>>>
>>>>> --
>>>>> Andy Chen
>>>>> Sr. Telephony Lead Engineer
>>>>> 415 516 5535 (M)
>>>>> achen@ <achen at thinkingphones.com>fuze.com
>>>>>
>>>>>
>>>>> *Confidentiality Notice: The information contained in this e-mail and
>>>>> any
>>>>> attachments may be confidential. If you are not an intended recipient,
>>>>> you
>>>>> are hereby notified that any dissemination, distribution or copying of
>>>>> this
>>>>> e-mail is strictly prohibited. If you have received this e-mail in
>>>>> error,
>>>>> please notify the sender and permanently delete the e-mail and any
>>>>> attachments immediately. You should not retain, copy or use this
>>>>> e-mail or
>>>>> any attachment for any purpose, nor disclose all or any part of the
>>>>> contents to any other person. Thank you.*
>>>>>
>>>>> _______________________________________________
>>>>> Kamailio (SER) - Users Mailing Listsr-users at lists.kamailio.orghttps://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
>>>>>
>>>>> --
>>>>> Kamailio Merchandising - https://skalatan.de/merchandising/
>>>>> Kamailio services - https://skalatan.de/services
>>>>> Henning Westerholt - https://skalatan.de/blog/
>>>>>
>>>>>
>>>>
>>>> --
>>>> Andy Chen
>>>> Sr. Telephony Lead Engineer
>>>> 415 516 5535 (M)
>>>> achen@ <achen at thinkingphones.com>fuze.com
>>>>
>>>>
>>>
>>> --
>>> Andy Chen
>>> Sr. Telephony Lead Engineer
>>> 415 516 5535 (M)
>>> achen@ <achen at thinkingphones.com>fuze.com
>>>
>>>
>>> *Confidentiality Notice: The information contained in this e-mail and any
>>> attachments may be confidential. If you are not an intended recipient,
>>> you
>>> are hereby notified that any dissemination, distribution or copying of
>>> this
>>> e-mail is strictly prohibited. If you have received this e-mail in error,
>>> please notify the sender and permanently delete the e-mail and any
>>> attachments immediately. You should not retain, copy or use this e-mail
>>> or
>>> any attachment for any purpose, nor disclose all or any part of the
>>> contents to any other person. Thank you.*
>>> _______________________________________________
>>> Kamailio (SER) - Users Mailing List
>>> sr-users at lists.kamailio.org
>>> https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
>>>
>> --
>> Kamailio Merchandising - https://skalatan.de/merchandising/
>> Kamailio services - https://skalatan.de/services
>> Henning Westerholt - https://skalatan.de/blog/
>>
>>
>
> --
> Andy Chen
> Sr. Telephony Lead Engineer
> 415 516 5535 (M)
> achen@ <achen at thinkingphones.com>fuze.com
>
>

-- 
Andy Chen
Sr. Telephony Lead Engineer
415 516 5535 (M)
achen@ <achen at thinkingphones.com>fuze.com

-- 
*Confidentiality Notice: The information contained in this e-mail and any

attachments may be confidential. If you are not an intended recipient, you

are hereby notified that any dissemination, distribution or copying of this

e-mail is strictly prohibited. If you have received this e-mail in error,

please notify the sender and permanently delete the e-mail and any

attachments immediately. You should not retain, copy or use this e-mail or

any attachment for any purpose, nor disclose all or any part of the

contents to any other person. Thank you.*
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.kamailio.org/pipermail/sr-users/attachments/20190919/cadd3f92/attachment.html>


More information about the sr-users mailing list