[SR-Users] Multiple crashes of Kamailio 4.2.1
Igor Potjevlesch
igor.potjevlesch at gmail.com
Thu Sep 10 16:43:00 CEST 2015
Hello Daniel,
There is no particular reason. It's just that our others Kamailio are
running 4.2.3. So, we try to have the same version on all nodes.
Regarding the backtrace, I'm surprised, because there was no manual
shutdown.
Unfortunately, for this occurrence of the crash, it's the only coredump
available. There was another crash later this day with two coredumps and the
following backtrace:
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m
256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d2d49f000,
frag=0x7f7d2d769830) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d2d49f000,
frag=0x7f7d2d769830) at mem/f_malloc.c:245
f = 0x7f7d2d49f758
hash = 97
#1 0x000000000061a2fe in fm_free (qm=0x7f7d2d49f000, p=0x7f7d2d769868,
file=0x7f7d43d77f4d "tm: h_table.c", func=0x7f7d43d78228 "free_cell",
line=159) at mem/f_malloc.c:614
f = 0x7f7d2d769830
__FUNCTION__ = "fm_free"
#2 0x00007f7d43cbbfa4 in free_cell (dead_cell=0x7f7d2d75e600) at
h_table.c:159
b = 0x0
i = 1
rpl = 0x0
tt = 0x0
foo = 0x0
cbs = 0x0
cbs_tmp = 0x7f7d2d75d378
__FUNCTION__ = "free_cell"
#3 0x00007f7d43cbd655 in free_hash_table () at h_table.c:448
p_cell = 0x7f7d2d75e600
tmp_cell = 0x7f7d2d5e89e0
i = 31354
__FUNCTION__ = "free_hash_table"
#4 0x00007f7d43ce1906 in tm_shutdown () at t_funcs.c:122
__FUNCTION__ = "tm_shutdown"
#5 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d452c0a80
foo = 0x7f7d452c04b0
__FUNCTION__ = "destroy_modules"
#6 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#7 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at
main.c:711
__FUNCTION__ = "shutdown_children"
#8 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 759854320
__FUNCTION__ = "handle_sigs"
#9 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 3362
si = 0x0
si_desc = "udp receiver child=7
sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260tJ-
}\177\000\000\000\000\000\020\004\000\000\000\260tJ-}\177\000\000\060SA\000\
000\000\000\000\220\351\330\364\001\000\000\000
\347\330\364\377\177\000\000\032dN\000\000\000\000\000h\321)Ez\000\000\000\2
76}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#10 0x00000000004ab8bf in main (argc=7, argv=0x7ffff4d8e998) at main.c:2578
cfg_stream = 0x2019010
c = -1
r = 0
tmp = 0x7ffff4d8ff70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3024771980
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7ffff4d8e86e ""
__FUNCTION__ = "main"
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m
256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x000000000061743c in fm_extract_free (qm=0x7f7d2d49f000,
frag=0x7f7d2d78d098) at mem/f_malloc.c:208
208 if(frag->u.nxt_free) frag->u.nxt_free->prv_free = pf;
#0 0x000000000061743c in fm_extract_free (qm=0x7f7d2d49f000,
frag=0x7f7d2d78d098) at mem/f_malloc.c:208
pf = 0x7f7d2d49f758
hash = 2097
#1 0x0000000000618af4 in fm_malloc (qm=0x7f7d2d49f000, size=776,
file=0x754600 "<core>: mem/shm_mem.c", func=0x7553bc "sh_realloc", line=89)
at mem/f_malloc.c:490
f = 0x7f7d2d49f758
frag = 0x7f7d2d78d098
hash = 97
__FUNCTION__ = "fm_malloc"
#2 0x000000000061e8df in sh_realloc (p=0x7f7d2d763ff0, size=774) at
mem/shm_mem.c:89
r = 0x149f4d8e990
__FUNCTION__ = "sh_realloc"
#3 0x000000000061eb97 in _shm_resize (p=0x7f7d2d763ff0, s=774,
file=0x7f7d43d9d263 "tm: t_reply.c", func=0x7f7d43da1ef1 "relay_reply",
line=1944) at mem/shm_mem.c:114
__FUNCTION__ = "_shm_resize"
#4 0x00007f7d43d4fdc7 in relay_reply (t=0x7f7d2d7711a8,
p_msg=0x7f7d454170b0, branch=0, msg_status=180, cancel_data=0x7ffff4d8e150,
do_put_on_wait=1) at t_reply.c:1943
relay = 0
save_clone = 0
buf = 0x7f7d4541de08 "SIP/2.0 180 Ringing\r\nCSeq: 1
INVITE\r\nCall-ID: 7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom:
\"0123456789\"<sip:0123456789 at A.B.C.D>;tag=95ffcd055e0f78f7d5d397020e89288dc
9eabbab\r\nTo: <sip:01"...
res_len = 614
relayed_code = 180
relayed_msg = 0x7f7d454170b0
reply_bak = 0x4000000
bm = {to_tag_val = {s = 0xfffffffff4d8df20 <Address
0xfffffffff4d8df20 out of bounds>, len = 1}}
totag_retr = 0
reply_status = RPS_PROVISIONAL
uas_rb = 0x7f7d2d771268
to_tag = 0x7f7d2d772740
reason = {s = 0x7ffff4d8e178 "", len = 1161916592}
onsend_params = {req = 0x7ffff4d8df50, rpl = 0x7f7d43d0b170, param =
0x415330, code = -187111024, flags = 3, branch = 0, t_rbuf = 0x0, dst =
0xa6fa1c, send_buf = {
s = 0x7ffff4d8e060 "\320\341\330\364\377\177", len = 6772964}}
__FUNCTION__ = "relay_reply"
#5 0x00007f7d43d53642 in reply_received (p_msg=0x7f7d454170b0) at
t_reply.c:2493
msg_status = 180
last_uac_status = 183
ack = 0x7f7d4529d010 "\001"
ack_len = 0
branch = 0
reply_status = 1160368600
onreply_route = 1
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text =
{s = 0x0, len = 10943004}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len =
10943004}}}}
uac = 0x7f7d2d771310
t = 0x7f7d2d7711a8
lack_dst = {send_sock = 0x4000000, to = {s = {sa_family = 63657,
sa_data = "\246\000\000\000\000\000\035\370\246\000\000\000\000"}, sin =
{sin_family = 63657,
sin_port = 166, sin_addr = {s_addr = 0}, sin_zero =
"\035\370\246\000\000\000\000"}, sin6 = {sin6_family = 63657, sin6_port =
166, sin6_flowinfo = 0,
sin6_addr = {__in6_u = {__u6_addr8 =
"\035\370\246\000\000\000\000\000\220\313/E}\177\000", __u6_addr16 = {63517,
166, 0, 0, 52112, 17711, 32637, 0},
__u6_addr32 = {10942493, 0, 1160760208, 32637}}},
sin6_scope_id = 1160759624}}, id = 32637, proto = 48 '0', send_flags = {f =
174 '\256',
blst_imask = 47 '/'}}
backup_user_from = 0xa7f790
backup_user_to = 0xa7f798
backup_domain_from = 0xa7f7a0
backup_domain_to = 0xa7f7a8
backup_uri_from = 0xa7f780
backup_uri_to = 0xa7f788
backup_xavps = 0xa7f8c0
replies_locked = 1
branch_ret = 0
prev_branch = 1161916624
blst_503_timeout = 32637
hf = 0x2bcf4d8e220
onsend_params = {req = 0x7ffff4d8e110, rpl = 0x47dd84, param = 0x0,
code = 1160754848, flags = 32637, branch = 0, t_rbuf = 0xa6fa1c, dst =
0xa6f853, send_buf = {
s = 0x7ffff4d8e1d0 "\210", <incomplete sequence \367\247>, len =
6393479}}
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env =
{{__jmpbuf = {140176009556648, 387096857427510471, 4281136, 140737301244304,
0, 0, 387096857536562375,
-387072352975780665}, __mask_was_saved = 0, __saved_mask =
{__val = {0, 140737301242320, 6430864, 140737301242080, 140175994783450,
140737301242048, 0,
67108864, 65540256, 1285944, 1568608, 1574280, 8, 88,
140175994783450, 1473240891392}}}}}
__FUNCTION__ = "reply_received"
#6 0x000000000048bf5d in do_forward_reply (msg=0x7f7d454170b0, mode=0) at
forward.c:783
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000'
<repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr
= 0},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 =
{sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0,
0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id
= 0, proto = 0 '\000',
send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
new_len = 1
r = 0
ip = {af = 4107854704, len = 32767, u = {addrl = {6457509, 280},
addr32 = {6457509, 0, 280, 0}, addr16 = {34981, 98, 0, 0, 280, 0, 0, 0},
addr =
"\245\210b\000\000\000\000\000\030\001\000\000\000\000\000"}}
s = 0x4 <Address 0x4 out of bounds>
len = 0
__FUNCTION__ = "do_forward_reply"
#7 0x000000000048d598 in forward_reply (msg=0x7f7d454170b0) at
forward.c:885
No locals.
#8 0x0000000000508610 in receive_msg (
buf=0xa6f760 "SIP/2.0 180 Ringing\r\nCSeq: 1 INVITE\r\nCall-ID:
7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom:
\"0987654321\"<sip:0987654321 at A.B.C.D>;tag=95f
fcd055e0f78f7d5d397020e89288dc9eabbab\r\nTo: <sip:09"...,
len=700, rcv_info=0x7ffff4d8e570) at receive.c:275
msg = 0x7f7d454170b0
ctx = {rec_lev = 10224768, run_flags = 0, last_retcode = 0, jmp_env
= {{__jmpbuf = {0, 0, 0, 272136986608, 1812476198913, 0, 272145363728,
272145384176},
__mask_was_saved = 0, __saved_mask = {__val =
{140176008147952, 140737301243152, 1, 140175607493872, 272137013029, 50195,
1024, 8402822336, 140175607493872,
140737301243072, 6292521, 140737301243360,
140175607493872, 81, 6292649, 140737301243440}}}}}
ret = -187112176
inb = {
s = 0xa6f760 "SIP/2.0 180 Ringing\r\nCSeq: 1 INVITE\r\nCall-ID:
7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom:
\"0987654321\"<sip:0987654321 at A.B.C.D>;tag=95ffcd055e0f78f7d5d397020e89288dc
9eabbab\r\nTo: <sip:09"..., len = 700}
__FUNCTION__ = "receive_msg"
#9 0x0000000000607436 in udp_rcv_loop () at udp_server.c:521
len = 700
buf = "SIP/2.0 180 Ringing\r\nCSeq: 1 INVITE\r\nCall-ID:
7fff-4e1-872015131549-FFGGTT-0-A.B.C.D\r\nFrom:
\"0987654321\"<sip:0987654321 at A.B.C.D>;tag=95ffcd055e0f78f7d5d397020e89288dc
9eabbab\r\nTo: <sip:09"...
tmp = 0x3f50636ee0 <Address 0x3f50636ee0 out of bounds>
from = 0x7f7d452fd4b0
fromlen = 16
ri = {src_ip = {af = 2, len = 4, u = {addrl = {546428251, 0}, addr32
= {546428251, 0, 0, 0}, addr16 = {54619, 8337, 0, 0, 0, 0, 0, 0},
addr = "[Õ ", '\000' <repeats 11 times>}}, dst_ip = {af = 2,
len = 4, u = {addrl = {1016190299, 0}, addr32 = {1016190299, 0, 0, 0},
addr16 = {54619, 15505, 0,
0, 0, 0, 0, 0}, addr = "[Õ<", '\000' <repeats 11 times>}},
src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0,
src_su = {s = {
sa_family = 2, sa_data = "\023\304[Õ
\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195,
sin_addr = {s_addr = 546428251},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 =
{sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 546428251, sin6_addr =
{__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0,
0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},
bind_address = 0x7f7d452bf2b0, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#10 0x00000000004a5f0b in main_loop () at main.c:1629
i = 3
pid = 0
si = 0x7f7d452bf2b0
si_desc = "udp receiver child=3
sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260tJ-
}\177\000\000\000\000\000\020\004\000\000\000\260tJ-}\177\000\000\060SA\000\
000\000\000\000\220\351\330\364\001\000\000\000
\347\330\364\377\177\000\000\032dN\000\000\000\000\000h\321)Ez\000\000\000\2
76}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#11 0x00000000004ab8bf in main (argc=7, argv=0x7ffff4d8e998) at main.c:2578
cfg_stream = 0x2019010
c = -1
r = 0
tmp = 0x7ffff4d8ff70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3024771980
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7ffff4d8e86e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : sr-users [mailto:sr-users-bounces at lists.sip-router.org] De la part de
Daniel-Constantin Mierla
Envoyé : jeudi 10 septembre 2015 12:57
À : Kamailio (SER) - Users Mailing List <sr-users at lists.sip-router.org>
Objet : Re: [SR-Users] Multiple crashes of Kamailio 4.2.1
Hello,
there are newer releases in branch 4.2, was there any reason to stop at
4.2.3?
Back to the backtrace, this one is from shutdown, happening during the
cleanup, not showing the reason of the crash at runtime, unless there was a
manual shut down triggered at that moment. Do you have another core file?
Cheers,
Daniel
On 10/09/15 12:36, Igor Potjevlesch wrote:
Hello,
We got some others crashes even after updated to 4.2.3.
>From the last one, I got the following into "bt full":
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio.pid -m
256 -M 64'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000,
frag=0x7f7d422ea368) at mem/f_malloc.c:245
245 if (*f) (*f)->prv_free = &(frag->u.nxt_free);
Missing separate debuginfos, use: debuginfo-install
bzip2-libs-1.0.5-7.el6_0.x86_64 db4-4.7.25-18.el6_4.x86_64
elfutils-libelf-0.152-1.el6.x86_64 glibc-2.12-1.132.el6.x86_64
keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6_4.6.x86_64
libacl-2.2.49-6.el6.x86_64 libattr-2.4.44-7.el6.x86_64
libcap-2.16-5.5.el6.x86_64 libcom_err-1.41.12-18.el6.x86_64
libgcc-4.4.7-11.el6.x86_64 libselinux-2.0.94-5.3.el6_4.1.x86_64
lm_sensors-libs-3.1.1-17.el6.x86_64 lua-5.1.4-4.1.el6.x86_64
mysql-libs-5.1.73-3.el6_5.x86_64 net-snmp-libs-5.5-50.el6_6.1.x86_64
nspr-4.10.0-1.el6.x86_64 nss-3.15.1-15.el6.x86_64
nss-softokn-freebl-3.14.3-9.el6.x86_64 nss-util-3.15.1-3.el6.x86_64
openssl-1.0.1e-30.el6_6.4.x86_64 pcre-7.8-6.el6.x86_64
perl-libs-5.10.1-136.el6.x86_64 popt-1.13-7.el6.x86_64
rpm-libs-4.8.0-37.el6.x86_64 tcp_wrappers-libs-7.6-57.el6.x86_64
xz-libs-4.999.9-0.3.beta.20091007git.el6.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0 0x0000000000617612 in fm_insert_free (qm=0x7f7d4206d000,
frag=0x7f7d422ea368) at mem/f_malloc.c:245
f = 0x7f7d4206d188
hash = 4
#1 0x000000000061a2fe in fm_free (qm=0x7f7d4206d000, p=0x7f7d422ea3a0,
file=0x7f7d53408ff0 "dialog: dlg_timer.c", func=0x7f7d53409d00
"destroy_dlg_timer", line=95)
at mem/f_malloc.c:614
f = 0x7f7d422ea368
__FUNCTION__ = "fm_free"
#2 0x00007f7d533e441f in destroy_dlg_timer () at dlg_timer.c:95
__FUNCTION__ = "destroy_dlg_timer"
#3 0x00007f7d53393268 in mod_destroy () at dialog.c:784
No locals.
#4 0x0000000000590d79 in destroy_modules () at sr_module.c:811
t = 0x7f7d59e9d670
foo = 0x7f7d59e9d440
__FUNCTION__ = "destroy_modules"
#5 0x000000000049bb43 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#6 0x000000000049d10b in shutdown_children (sig=15, show_status=1) at
main.c:711
__FUNCTION__ = "shutdown_children"
#7 0x000000000049f6e1 in handle_sigs () at main.c:802
chld = 0
chld_status = 139
memlog = 1107776752
__FUNCTION__ = "handle_sigs"
#8 0x00000000004a6fbf in main_loop () at main.c:1757
i = 8
pid = 4147
si = 0x0
si_desc = "udp receiver child=7
sock=91.213.145.60:5060\000\000\000\000\016\b\000\000\377\177\000\000\260T\a
B}\177\000\000\000\000\000\020\004\000\000\000\260T\aB}\177\000\000\060SA\00
0\000\000\000\000\260:\240\343\001\000\000\000 at 8\240\343\377\177\000\000\032
dN\000\000\000\000\000h\261\346Yz\000\000\000\276}p\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x00000000004ab8bf in main (argc=7, argv=0x7fffe3a03ab8) at main.c:2578
cfg_stream = 0x1a04010
c = -1
r = 0
tmp = 0x7fffe3a04f70 ""
tmp_len = 0
port = 0
proto = 32767
options = 0x6fcc00
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3320376299
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xc2
p = 0x7fffe3a0398e ""
__FUNCTION__ = "main"
Regards,
Igor.
De : Igor Potjevlesch [mailto:igor.potjevlesch at gmail.com]
Envoyé : lundi 20 juillet 2015 16:22
À : sr-users at lists.sip-router.org <mailto:sr-users at lists.sip-router.org>
Objet : Multiple crashes of Kamailio 4.2.1
Hello,
3 crashes occurred today (we never seen crashes on this server before)
during a period of 20 minutes.
I had a look to the coredumps. Here is a quick overview of the first entries
of "bt full":
core.27671 : #0 0x00007f5577048d65 in run_trans_callbacks_internal
(cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0)
at t_hooks.c:286
core.27656 : #0 0x00007f557700a048 in free_cell (dead_cell=0x7f5560abbce0)
at h_table.c:175
core.27685 : #0 0x00007f5577048d65 in run_trans_callbacks_internal
(cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0)
at t_hooks.c:286
core.27689 : #0 0x00007f5577048d65 in run_trans_callbacks_internal
(cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0)
at t_hooks.c:286
core.27691 : #0 0x00007f5577048d65 in run_trans_callbacks_internal
(cb_lst=0x7f5560abbd50, type=2, trans=0x7f5560abbce0, params=0x7fff3712acb0)
at t_hooks.c:286
core.31977 : #0 0x00007f038e66fdee in free_cell (dead_cell=0x7f037811ebe0)
at h_table.c:157
core.31994 : #0 0x00007f038e6ab65a in t_forward_nonack (t=0x7f037811ebe0,
p_msg=0x7f038fdca438, proxy=0x0, proto=0) at t_fwd.c:1762
core.3700 : #0 0x000000000061697a in fm_insert_free (qm=0x7f19861fa000,
frag=0x7f19864d53e8) at mem/f_malloc.c:245
core.3727 : #0 0x00000000006167a4 in fm_extract_free (qm=0x7f19861fa000,
frag=0x7f19864f4cc8) at mem/f_malloc.c:208
I don't know how to proceed now. Can you help me to highlight the issue
here? Thank you.
Regards,
Igor.
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users at lists.sip-router.org <mailto:sr-users at lists.sip-router.org>
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierla
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin -
http://asipto.com/u/kat
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.sip-router.org/pipermail/sr-users/attachments/20150910/621fd463/attachment.html>
More information about the sr-users
mailing list