[sr-dev] [kamailio/kamailio] crush: anot alliged memory access (#2798)
sergey-safarov
notifications at github.com
Sun Jul 11 21:14:26 CEST 2021
### Description
I still have core generation with not alleged memory access (Related #2788 and #2736).
### Troubleshooting
I have prepared minimal kamailio config that allow reproduce issue on master branch (tested 8762c56a1c78e4ba151ef9fb4290b3938c0c984f).
#### Reproduction
Required start Kamailio with config below:
```
################
pv_buffer_slots = 30
loadmodule "ipops.so"
loadmodule "pv.so"
loadmodule "textops.so"
loadmodule "outbound.so"
loadmodule "corex.so"
loadmodule "kex.so"
loadmodule "tm.so"
loadmodule "tmx.so"
loadmodule "sl.so"
loadmodule "rr.so"
loadmodule "uac.so"
loadmodule "usrloc.so"
loadmodule "registrar.so"
#!substdef "!DEF01!$(version(num))!g"
#!substdef "!DEF02!$HN(f)!g"
#!substdef "!DEF03!$HN(d)!g"
#!substdef "!DEF04!$HN(i)!g"
# access to not existen default value
#!substdef "!DEF05!$def(NULL)!g"
#!substdef "!DEF06!$def(NULL)!g"
### SQL substdef
#!substdef "!DEF07!insert into dispatcher (setid, destination) select \$var(SetId), \"\$var(MediaUrl)\" from DUAL where not exists(select * from dispatcher where destination = \"\$var(MediaUrl)\")!g"
#!substdef "!DEF08!select a.event, count(distinct watcher_username || \"@\" || watcher_domain) count_unique, count(*) count from event_list a, active_watchers b where b.event = a.event group by a.event!g"
#!substdef "!DEF09!select event, (select count(*) from presentity b where username = \"\$var(SetId)\" and domain = \"\$var(SetId)\" and b.event = a.event) count from event_list a!g"
#!substdef "!DEF10!select event, (select count(*) from active_watchers b where presentity_uri = \"\$var(presentity)\" and b.event = a.event) count from event_list a!g"
#!substdef "!DEF11!delete from presentity where domain=\"\$var(SetId)\" and username = \"\$var(SetId)\"!g"
#!substdef "!DEF12!delete from active_watchers where callid = \"\$ci\"!g"
#!substdef "!DEF13!delete from active_watchers where watcher_username=\"\$fU\" and presentity_uri=\"\$var(presentity_uri)\" and to_user=\"\$tU\" and watcher_domain=\"\$fd\" and event=\"\$hdr(Event)\"!g"
#!substdef "!DEF14!delete from presentity where domain=\"\$var(SetId)\"!g"
#!substdef "!DEF15!update active_watchers set expires = \$TS where watcher_domain=\"\$var(SetId)\"!g"
#!substdef "!DEF16!update active_watchers set expires = \$TS where id in (select * from (select b.id from presentity a inner join active_watchers b on a.username = b.to_user and a.domain = b.to_domain and a.event = b.event where a.sender = \"\$var(MediaUrl)\") AS presentity_temp)!g"
#!substdef "!DEF17!select * from active_watchers_log where presentity_uri = \"\$var(presentity_uri)\"!g"
#!substdef "!DEF18!select * from active_watchers where watcher_domain = \"\$var(Domain)\"!g"
#!substdef "!DEF19!select count(*) as count from presentity where username = \"\$var(SetId)\" and domain = \"\$var(SetId)\" and event = \"\$var(SetId)\"!g"
#!substdef "!DEF20!delete from presentity where sender = \"\$var(MediaUrl)\"!g"
####### Routing Logic ########
route
{
$var(tmp) = $hdr(X-hdr1);
$var(tmp) = $hdr(X-hdr2);
$var(tmp) = $hdr(X-hdr3);
$var(tmp) = $hdr(X-hdr4);
$var(tmp) = $hdr(X-hdr5);
$var(tmp) = $hdr(X-hdr6);
$var(tmp) = $hdr(X-hdr7);
$var(tmp) = $hdr(X-hdr8);
$var(tmp) = $hdr(X-hdr9);
$var(tmp) = $hdr(X-hdr10);
$var(tmp) = $hdr(X-hdr11);
$var(tmp) = $hdr(X-hdr12);
$var(tmp) = $hdr(X-hdr13);
$var(tmp) = $hdr(X-hdr14);
$var(tmp) = $hdr(X-hdr15);
if (reg_fetch_contacts("location", "$hdr(X-hdr16)", "callee")) {
return;
}
}
```
#### Debugging Data
```
Program received signal SIGSEGV, Segmentation fault.
0x00007ffff7dd56b1 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6
Missing separate debuginfos, use: dnf debuginfo-install libgcc-10.2.1-9.fc33.x86_64 openssl-libs-1.1.1k-1.fc33.x86_64 systemd-libs-246.13-1.fc33.x86_64 zlib-1.2.11-23.fc33.x86_64
(gdb) bt
#0 0x00007ffff7dd56b1 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6
#1 0x00007ffff6391fd4 in pv_parse_hdr_name (sp=0x7ffff652c0d8, in=0x7fffffffd560) at pv_core.c:3617
#2 0x0000000000568916 in pv_parse_spec2 (in=0x7ffff652c0c0, e=0x7ffff652c0d8, silent=0) at core/pvapi.c:969
#3 0x0000000000563ffd in pv_cache_add (name=0x7fffffffd750) at core/pvapi.c:359
#4 0x00000000005658c8 in pv_spec_lookup (name=0x7fffffffd810, len=0x7fffffffd80c) at core/pvapi.c:498
#5 0x000000000056c06c in pv_parse_format (in=0x7fffffffd910, el=0x7ffff652bfa8) at core/pvapi.c:1194
#6 0x0000000000631b16 in fix_param (type=256, param=0x7ffff65224f0) at core/sr_module.c:1223
#7 0x000000000063218e in fix_param_types (types=256, param=0x7ffff65224f0) at core/sr_module.c:1345
#8 0x000000000050f721 in fixup_spve_null (param=0x7ffff65224f0, param_no=1) at core/mod_fix.c:564
#9 0x00007ffff5b97cbb in fetchc_fixup (param=0x7ffff65224f0, param_no=2) at registrar.c:808
#10 0x00000000005a6aed in fix_actions (a=0x7ffff6522478) at core/route.c:932
#11 0x00000000005def48 in fix_rval (rv=0x7ffff65237d8, rve=0x7ffff65237d0) at core/rvalue.c:2992
#12 0x00000000005f0a90 in fix_rval_expr (p=0x7ffff65237d0) at core/rvalue.c:3919
#13 0x00000000005a210b in fix_actions (a=0x7ffff6517ce0) at core/route.c:719
#14 0x00000000005b3e03 in fix_rl (rt=0x91f140 <main_rt>) at core/route.c:2102
#15 0x00000000005b3e3a in fix_rls () at core/route.c:2118
#16 0x00000000004381c0 in main (argc=12, argv=0x7fffffffe5f8) at main.c:3066
(gdb) bt full
#0 0x00007ffff7dd56b1 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007ffff6391fd4 in pv_parse_hdr_name (sp=0x7ffff652c0d8, in=0x7fffffffd560) at pv_core.c:3617
s = {s = 0x7ffff1d72f60 "`.\327\361\377\177", len = 4312016}
p = 0x0
nsp = 0x0
hdr = {type = HDR_PAI_T, name = {s = 0x41cbd0 <_start> "\363\017\036\372\061\355I\211\321^H\211\342H\203\344\360PTI\307\300@\330\177", len = -11088}, body = {s = 0x566521 <pv_lookup_spec_name+910> "\205\300u<H\213EȋP\020H\213\205p\377\377\377\211\020H\213E\310H\213P\030H\213\205p\377\377\377H\211P\bH\213E\310H\213P H\213\205p\377\377\377H\211P\020H\213E\310\353\037H\213E\310H\213 at PH\211E\310H\203", <incomplete sequence \310>, len = -162348840}, len = -10928, parsed = 0x80d116, next = 0x7ffff652bf07}
__func__ = "pv_parse_hdr_name"
#2 0x0000000000568916 in pv_parse_spec2 (in=0x7ffff652c0c0, e=0x7ffff652c0d8, silent=0) at core/pvapi.c:969
p = 0x7ffff652c144 ")"
s = {s = 0x7ffff652c13d "X-hdr16)", len = 7}
pvname = {s = 0x7ffff652c139 "hdr(X-hdr16)", len = 3}
pvstate = 5
tr = 0x0
pte = 0x7ffff64fef70
n = 0
__func__ = "pv_parse_spec2"
#3 0x0000000000563ffd in pv_cache_add (name=0x7fffffffd750) at core/pvapi.c:359
pvn = 0x7ffff652c0c0
pvid = 3075582644
p = 0x0
__func__ = "pv_cache_add"
#4 0x00000000005658c8 in pv_spec_lookup (name=0x7fffffffd810, len=0x7fffffffd80c) at core/pvapi.c:498
pvs = 0x0
tname = {s = 0x7ffff65226b0 "$hdr(X-hdr16)", len = 13}
__func__ = "pv_spec_lookup"
#5 0x000000000056c06c in pv_parse_format (in=0x7fffffffd910, el=0x7ffff652bfa8) at core/pvapi.c:1194
p = 0x7ffff65226b0 "$hdr(X-hdr16)"
p0 = 0x80b00000001 <error: Cannot access memory at address 0x80b00000001>
n = 1
e = 0x7ffff652c038
e0 = 0x0
s = {s = 0x7ffff65226b0 "$hdr(X-hdr16)", len = 13}
len = 13
__func__ = "pv_parse_format"
#6 0x0000000000631b16 in fix_param (type=256, param=0x7ffff65224f0) at core/sr_module.c:1223
p = 0x7ffff652bf98
name = {s = 0x7ffff65226b0 "$hdr(X-hdr16)", len = 13}
s = {s = 0x80d116 "core: core/counters.c", len = -163113056}
num = 32767
err = 0
__func__ = "fix_param"
#7 0x000000000063218e in fix_param_types (types=256, param=0x7ffff65224f0) at core/sr_module.c:1345
ret = 0
t = 256
#8 0x000000000050f721 in fixup_spve_null (param=0x7ffff65224f0, param_no=1) at core/mod_fix.c:564
ret = 32767
fp = 0x7ffff1d72f60
__func__ = "fixup_spve_null"
#9 0x00007ffff5b97cbb in fetchc_fixup (param=0x7ffff65224f0, param_no=2) at registrar.c:808
No locals.
#10 0x00000000005a6aed in fix_actions (a=0x7ffff6522478) at core/route.c:932
t = 0x7ffff6522478
p = 0x4f0086e1b9
tmp = 0x89a29f "core: core/cfg/cfg_struct.c"
tmp_p = 0x7ffff65226b0
ret = 0
i = 1
cmd = 0x7ffff650dde0
s = {s = 0x7fffffffdc30 "", len = 7295555}
he = 0x41cbd0 <_start>
ip = {af = 0, len = 0, u = {addrl = {21483676084, 140737488346160}, addr32 = {8839604, 5, 4294958128, 32767}, addr16 = {57780, 134, 5, 0, 56368, 65535, 32767, 0}, addr = "\264\341\206\000\005\000\000\000\060\334\377\377\377\177\000"}}
si = 0x33
lval = 0x0
rve = 0x90000000c
err_rve = 0x4bc015 <sr_event_exec+415>
rve_type = 32767
err_type = 4294957888
expected_type = 32767
rv = 0x7ffff5bc5360 <default_registrar_cfg>
rve_param_no = 0
__func__ = "fix_actions"
#11 0x00000000005def48 in fix_rval (rv=0x7ffff65237d8, rve=0x7ffff65237d0) at core/rvalue.c:2992
__func__ = "fix_rval"
#12 0x00000000005f0a90 in fix_rval_expr (p=0x7ffff65237d0) at core/rvalue.c:3919
rve = 0x7ffff65237d0
ret = 0
__func__ = "fix_rval_expr"
#13 0x00000000005a210b in fix_actions (a=0x7ffff6517ce0) at core/route.c:719
t = 0x7ffff65228e8
p = 0x7ffff1adb000
tmp = 0xffffffff006f4dd9 <error: Cannot access memory at address 0xffffffff006f4dd9>
tmp_p = 0x4
ret = 0
i = 8566243
cmd = 0x41cbd0 <_start>
s = {s = 0x82b4d8 "core", len = -163131376}
he = 0x7fffffffe010
ip = {af = 4294959040, len = 32767, u = {addrl = {7294619, 8694965}, addr32 = {7294619, 0, 8694965, 0}, addr16 = {20123, 111, 0, 0, 44213, 132, 0, 0}, addr = "\233No\000\000\000\000\000\265\254\204\000\000\000\000"}}
si = 0x100000000
lval = 0x7ffff65218a8
rve = 0x7ffff65237d0
err_rve = 0x0
rve_type = RV_INT
err_type = 4294959040
expected_type = RV_NONE
rv = 0x41cbd0 <_start>
rve_param_no = -8016
__func__ = "fix_actions"
#14 0x00000000005b3e03 in fix_rl (rt=0x91f140 <main_rt>) at core/route.c:2102
i = 0
ret = 51
#15 0x00000000005b3e3a in fix_rls () at core/route.c:2118
ret = 0
#16 0x00000000004381c0 in main (argc=12, argv=0x7fffffffe5f8) at main.c:3066
cfg_stream = 0x9c32d0
c = -1
r = 0
tmp = 0x7fffffffe8b6 ""
tmp_len = 896
port = 896
proto = 896
ahost = 0x0
aport = 0
options = 0x801218 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 2632239041
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 2
n_lst = 0x0
p = 0xc2 <error: Cannot access memory at address 0xc2>
st = {st_dev = 47, st_ino = 279340502, st_nlink = 2, st_mode = 16832, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 120, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1622741422, tv_nsec = 474562221}, st_mtim = {tv_sec = 1625629861, tv_nsec = 261687069}, st_ctim = {tv_sec = 1625629861, tv_nsec = 261687069}, __glibc_reserved = {0, 0, 0}}
tbuf = "\360\341\377\377\377\177\000\000\000\000\000\000\000\000\000\000\360\341\377\377\377\177", '\000' <repeats 18 times>, "\260\027\375\367\377\177\000\000\350\317\377\367\377\177\000\000\b\345\377\367\377\177\000\000\340\031\375\367\377\177\000\000\025\217\376\367\377\177\000\000$f\307\367\377\177\000\000\354K\377\367\377\177\000\000\336K\377\367\377\177\000\000\205\317c\t\000\000\000\000\300S\374\367\377\177\000\000ߏ\376\367\377\177\000\000\000\000\000\000\254\202\226\006\334P\307\367\377\177\000\000\000\000\000\000\000\000\000\000\300S\374\367\377\177\000\000\001\000\000\000\000\000\000\000\"\247\177\336s\027\000\000\240\341\377\367\377\177\000\000\370\377\377\377\377\377\377\377\240\341\377\367\377\177\000\000R"...
option_index = 12
long_options = {{name = 0x8038f6 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7fe521 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x8038fb "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x803901 "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x803907 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x803910 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x80391a "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x803924 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x80392f "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x803938 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x803943 "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x803949 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x803953 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__func__ = "main"
(gdb) info locals
No symbol table info available.
(gdb) list
1976 +((!sctp_disable)?sctp_listeners:0)
1977 #endif
1978 ;
1979 }
1980
1981 int main(int argc, char** argv)
1982 {
1983
1984 FILE* cfg_stream;
1985 int c,r;
(gdb) f 1
#1 0x00007ffff6391fd4 in pv_parse_hdr_name (sp=0x7ffff652c0d8, in=0x7fffffffd560) at pv_core.c:3617
3617 memcpy(p, in->s, in->len);
(gdb) info locals
s = {s = 0x7ffff1d72f60 "`.\327\361\377\177", len = 4312016}
p = 0x0
nsp = 0x0
hdr = {type = HDR_PAI_T, name = {s = 0x41cbd0 <_start> "\363\017\036\372\061\355I\211\321^H\211\342H\203\344\360PTI\307\300@\330\177", len = -11088}, body = {s = 0x566521 <pv_lookup_spec_name+910> "\205\300u<H\213EȋP\020H\213\205p\377\377\377\211\020H\213E\310H\213P\030H\213\205p\377\377\377H\211P\bH\213E\310H\213P H\213\205p\377\377\377H\211P\020H\213E\310\353\037H\213E\310H\213 at PH\211E\310H\203", <incomplete sequence \310>, len = -162348840}, len = -10928, parsed = 0x80d116, next = 0x7ffff652bf07}
__func__ = "pv_parse_hdr_name"
(gdb) list
3612 {
3613 LM_ERR("name too long\n");
3614 return -1;
3615 }
3616 p = pv_get_buffer();
3617 memcpy(p, in->s, in->len);
3618 p[in->len] = ':';
3619 s.s = p;
3620 s.len = in->len+1;
3621
(gdb) f 2
#2 0x0000000000568916 in pv_parse_spec2 (in=0x7ffff652c0c0, e=0x7ffff652c0d8, silent=0) at core/pvapi.c:969
969 if(pte->parse_name(e, &s)!=0)
(gdb) info locals
p = 0x7ffff652c144 ")"
s = {s = 0x7ffff652c13d "X-hdr16)", len = 7}
pvname = {s = 0x7ffff652c139 "hdr(X-hdr16)", len = 3}
pvstate = 5
tr = 0x0
pte = 0x7ffff64fef70
n = 0
__func__ = "pv_parse_spec2"
(gdb) list
964 LM_ERR("pvar \"%.*s\" does not get empty name param\n",
965 pvname.len, pvname.s);
966 goto error;
967 }
968 s.len = p - s.s;
969 if(pte->parse_name(e, &s)!=0)
970 {
971 if (!silent)
972 LM_ERR("pvar \"%.*s\" has an invalid name param [%.*s]\n",
973 pvname.len, pvname.s, s.len, s.s);
(gdb) f 3
#3 0x0000000000563ffd in pv_cache_add (name=0x7fffffffd750) at core/pvapi.c:359
359 p = pv_parse_spec(&pvn->pvname, &pvn->spec);
(gdb) info locals
pvn = 0x7ffff652c0c0
pvid = 3075582644
p = 0x0
__func__ = "pv_cache_add"
(gdb) list
354 }
355 memset(pvn, 0, sizeof(pv_cache_t) + name->len + 1);
356 pvn->pvname.len = name->len;
357 pvn->pvname.s = (char*)pvn + sizeof(pv_cache_t);
358 memcpy(pvn->pvname.s, name->s, name->len);
359 p = pv_parse_spec(&pvn->pvname, &pvn->spec);
360
361 if(p==NULL)
362 {
363 pkg_free(pvn);
```
#### Log Messages
```
[root at safarov-dell kamailio]# gdb --args kamailio --atexit=no -DD -P /run/kamailio/kamailio.pid -f /etc/kamailio/kamailio.cfg -m 64 -M 24 -E
GNU gdb (GDB) Fedora 10.1-2.fc33
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<https://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from kamailio...
(gdb) set pagination off
(gdb) r
Starting program: /usr/local/sbin/kamailio --atexit=no -DD -P /run/kamailio/kamailio.pid -f /etc/kamailio/kamailio.cfg -m 64 -M 24 -E
```
### Additional Information
* **Kamailio Version** - output of `kamailio -v`
```
[root at safarov-dell kamailio]# kamailio -v
version: kamailio 5.6.0-dev0 (x86_64/linux) 8762c5
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: 8762c5
compiled on 17:11:19 Jul 6 2021 with gcc 10.2.1
```
* **Operating System**:
```
[root at safarov-dell kamailio]# cat /etc/os-release
NAME=Fedora
VERSION="33 (Container Image)"
ID=fedora
VERSION_ID=33
VERSION_CODENAME=""
PLATFORM_ID="platform:f33"
PRETTY_NAME="Fedora 33 (Container Image)"
ANSI_COLOR="0;38;2;60;110;180"
LOGO=fedora-logo-icon
CPE_NAME="cpe:/o:fedoraproject:fedora:33"
HOME_URL="https://fedoraproject.org/"
DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f33/system-administrators-guide/"
SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
BUG_REPORT_URL="https://bugzilla.redhat.com/"
REDHAT_BUGZILLA_PRODUCT="Fedora"
REDHAT_BUGZILLA_PRODUCT_VERSION=33
REDHAT_SUPPORT_PRODUCT="Fedora"
REDHAT_SUPPORT_PRODUCT_VERSION=33
PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
VARIANT="Container Image"
VARIANT_ID=container
```
--
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/2798
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.kamailio.org/pipermail/sr-dev/attachments/20210711/d873babb/attachment-0001.htm>
More information about the sr-dev
mailing list