[sr-dev] [kamailio/kamailio] crush: anot alliged memory access (#2798)

11 Jul 2021


      ### Description
I still have core generation with not alleged memory access (Related #2788 and #2736).
### Troubleshooting
I have prepared minimal kamailio config that allow reproduce issue on master branch (tested 8762c56a1c78e4ba151ef9fb4290b3938c0c984f).
#### Reproduction
Required start Kamailio with config below:
```
################
pv_buffer_slots = 30
loadmodule "ipops.so"
loadmodule "pv.so"
loadmodule "textops.so"
loadmodule "outbound.so"
loadmodule "corex.so"
loadmodule "kex.so"
loadmodule "tm.so"
loadmodule "tmx.so"
loadmodule "sl.so"
loadmodule "rr.so"
loadmodule "uac.so"
loadmodule "usrloc.so"
loadmodule "registrar.so"
#!substdef "!DEF01!$(version(num))!g"
#!substdef "!DEF02!$HN(f)!g"
#!substdef "!DEF03!$HN(d)!g"
#!substdef "!DEF04!$HN(i)!g"
# access to not existen default value
#!substdef "!DEF05!$def(NULL)!g"
#!substdef "!DEF06!$def(NULL)!g"
### SQL substdef
#!substdef "!DEF07!insert into dispatcher (setid, destination) select $var(SetId), "$var(MediaUrl)" from DUAL where not exists(select * from dispatcher where destination = "$var(MediaUrl)")!g"
#!substdef "!DEF08!select a.event, count(distinct watcher_username || "@" || watcher_domain) count_unique, count(*) count from event_list a, active_watchers b where b.event = a.event group by a.event!g"
#!substdef "!DEF09!select event, (select count(*) from presentity b where username = "$var(SetId)" and domain = "$var(SetId)" and b.event = a.event) count from event_list a!g"
#!substdef "!DEF10!select event, (select count(*) from active_watchers b where presentity_uri = "$var(presentity)" and b.event = a.event) count from event_list a!g"
#!substdef "!DEF11!delete from presentity where domain="$var(SetId)" and username = "$var(SetId)"!g"
#!substdef "!DEF12!delete from active_watchers where callid = "$ci"!g"
#!substdef "!DEF13!delete from active_watchers where watcher_username="$fU" and presentity_uri="$var(presentity_uri)" and to_user="$tU" and watcher_domain="$fd" and event="$hdr(Event)"!g"
#!substdef "!DEF14!delete from presentity where domain="$var(SetId)"!g"
#!substdef "!DEF15!update active_watchers set expires = $TS where watcher_domain="$var(SetId)"!g"
#!substdef "!DEF16!update active_watchers set expires = $TS where id in (select * from (select b.id from presentity a inner join active_watchers b on a.username = b.to_user and a.domain = b.to_domain and a.event = b.event where a.sender = "$var(MediaUrl)") AS presentity_temp)!g"
#!substdef "!DEF17!select * from active_watchers_log where presentity_uri = "$var(presentity_uri)"!g"
#!substdef "!DEF18!select * from active_watchers where watcher_domain = "$var(Domain)"!g"
#!substdef "!DEF19!select count(*) as count from presentity where username = "$var(SetId)" and domain = "$var(SetId)" and event = "$var(SetId)"!g"
#!substdef "!DEF20!delete from presentity where sender = "$var(MediaUrl)"!g"
####### Routing Logic ########
route
{
    $var(tmp) = $hdr(X-hdr1);
    $var(tmp) = $hdr(X-hdr2);
    $var(tmp) = $hdr(X-hdr3);
    $var(tmp) = $hdr(X-hdr4);
    $var(tmp) = $hdr(X-hdr5);
    $var(tmp) = $hdr(X-hdr6);
    $var(tmp) = $hdr(X-hdr7);
    $var(tmp) = $hdr(X-hdr8);
    $var(tmp) = $hdr(X-hdr9);
    $var(tmp) = $hdr(X-hdr10);
    $var(tmp) = $hdr(X-hdr11);
    $var(tmp) = $hdr(X-hdr12);
    $var(tmp) = $hdr(X-hdr13);
    $var(tmp) = $hdr(X-hdr14);
    $var(tmp) = $hdr(X-hdr15);
    if (reg_fetch_contacts("location", "$hdr(X-hdr16)", "callee")) {
        return;
    }
}
```
#### Debugging Data
```
Program received signal SIGSEGV, Segmentation fault.
0x00007ffff7dd56b1 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6
Missing separate debuginfos, use: dnf debuginfo-install libgcc-10.2.1-9.fc33.x86_64 openssl-libs-1.1.1k-1.fc33.x86_64 systemd-libs-246.13-1.fc33.x86_64 zlib-1.2.11-23.fc33.x86_64
(gdb) bt
#0  0x00007ffff7dd56b1 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6
#1  0x00007ffff6391fd4 in pv_parse_hdr_name (sp=0x7ffff652c0d8, in=0x7fffffffd560) at pv_core.c:3617
#2  0x0000000000568916 in pv_parse_spec2 (in=0x7ffff652c0c0, e=0x7ffff652c0d8, silent=0) at core/pvapi.c:969
#3  0x0000000000563ffd in pv_cache_add (name=0x7fffffffd750) at core/pvapi.c:359
#4  0x00000000005658c8 in pv_spec_lookup (name=0x7fffffffd810, len=0x7fffffffd80c) at core/pvapi.c:498
#5  0x000000000056c06c in pv_parse_format (in=0x7fffffffd910, el=0x7ffff652bfa8) at core/pvapi.c:1194
#6  0x0000000000631b16 in fix_param (type=256, param=0x7ffff65224f0) at core/sr_module.c:1223
#7  0x000000000063218e in fix_param_types (types=256, param=0x7ffff65224f0) at core/sr_module.c:1345
#8  0x000000000050f721 in fixup_spve_null (param=0x7ffff65224f0, param_no=1) at core/mod_fix.c:564
#9  0x00007ffff5b97cbb in fetchc_fixup (param=0x7ffff65224f0, param_no=2) at registrar.c:808
#10 0x00000000005a6aed in fix_actions (a=0x7ffff6522478) at core/route.c:932
#11 0x00000000005def48 in fix_rval (rv=0x7ffff65237d8, rve=0x7ffff65237d0) at core/rvalue.c:2992
#12 0x00000000005f0a90 in fix_rval_expr (p=0x7ffff65237d0) at core/rvalue.c:3919
#13 0x00000000005a210b in fix_actions (a=0x7ffff6517ce0) at core/route.c:719
#14 0x00000000005b3e03 in fix_rl (rt=0x91f140 <main_rt>) at core/route.c:2102
#15 0x00000000005b3e3a in fix_rls () at core/route.c:2118
#16 0x00000000004381c0 in main (argc=12, argv=0x7fffffffe5f8) at main.c:3066
(gdb) bt full
#0  0x00007ffff7dd56b1 in __memmove_avx_unaligned_erms () from /lib64/libc.so.6
No symbol table info available.
#1  0x00007ffff6391fd4 in pv_parse_hdr_name (sp=0x7ffff652c0d8, in=0x7fffffffd560) at pv_core.c:3617
        s = {s = 0x7ffff1d72f60 "`.\327\361\377\177", len = 4312016}
        p = 0x0
        nsp = 0x0
        hdr = {type = HDR_PAI_T, name = {s = 0x41cbd0 <_start> "\363\017\036\372\061\355I\211\321^H\211\342H\203\344\360PTI\307\300@\330\177", len = -11088}, body = {s = 0x566521 <pv_lookup_spec_name+910> "\205\300u<H\213EȋP\020H\213\205p\377\377\377\211\020H\213E\310H\213P\030H\213\205p\377\377\377H\211P\bH\213E\310H\213P H\213\205p\377\377\377H\211P\020H\213E\310\353\037H\213E\310H\213@PH\211E\310H\203", <incomplete sequence \310>, len = -162348840}, len = -10928, parsed = 0x80d116, next = 0x7ffff652bf07}
        __func__ = "pv_parse_hdr_name"
#2  0x0000000000568916 in pv_parse_spec2 (in=0x7ffff652c0c0, e=0x7ffff652c0d8, silent=0) at core/pvapi.c:969
        p = 0x7ffff652c144 ")"
        s = {s = 0x7ffff652c13d "X-hdr16)", len = 7}
        pvname = {s = 0x7ffff652c139 "hdr(X-hdr16)", len = 3}
        pvstate = 5
        tr = 0x0
        pte = 0x7ffff64fef70
        n = 0
        __func__ = "pv_parse_spec2"
#3  0x0000000000563ffd in pv_cache_add (name=0x7fffffffd750) at core/pvapi.c:359
        pvn = 0x7ffff652c0c0
        pvid = 3075582644
        p = 0x0
        __func__ = "pv_cache_add"
#4  0x00000000005658c8 in pv_spec_lookup (name=0x7fffffffd810, len=0x7fffffffd80c) at core/pvapi.c:498
        pvs = 0x0
        tname = {s = 0x7ffff65226b0 "$hdr(X-hdr16)", len = 13}
        __func__ = "pv_spec_lookup"
#5  0x000000000056c06c in pv_parse_format (in=0x7fffffffd910, el=0x7ffff652bfa8) at core/pvapi.c:1194
        p = 0x7ffff65226b0 "$hdr(X-hdr16)"
        p0 = 0x80b00000001 <error: Cannot access memory at address 0x80b00000001>
        n = 1
        e = 0x7ffff652c038
        e0 = 0x0
        s = {s = 0x7ffff65226b0 "$hdr(X-hdr16)", len = 13}
        len = 13
        __func__ = "pv_parse_format"
#6  0x0000000000631b16 in fix_param (type=256, param=0x7ffff65224f0) at core/sr_module.c:1223
        p = 0x7ffff652bf98
        name = {s = 0x7ffff65226b0 "$hdr(X-hdr16)", len = 13}
        s = {s = 0x80d116 "core: core/counters.c", len = -163113056}
        num = 32767
        err = 0
        __func__ = "fix_param"
#7  0x000000000063218e in fix_param_types (types=256, param=0x7ffff65224f0) at core/sr_module.c:1345
        ret = 0
        t = 256
#8  0x000000000050f721 in fixup_spve_null (param=0x7ffff65224f0, param_no=1) at core/mod_fix.c:564
        ret = 32767
        fp = 0x7ffff1d72f60
        __func__ = "fixup_spve_null"
#9  0x00007ffff5b97cbb in fetchc_fixup (param=0x7ffff65224f0, param_no=2) at registrar.c:808
No locals.
#10 0x00000000005a6aed in fix_actions (a=0x7ffff6522478) at core/route.c:932
        t = 0x7ffff6522478
        p = 0x4f0086e1b9
        tmp = 0x89a29f "core: core/cfg/cfg_struct.c"
        tmp_p = 0x7ffff65226b0
        ret = 0
        i = 1
        cmd = 0x7ffff650dde0
        s = {s = 0x7fffffffdc30 "", len = 7295555}
        he = 0x41cbd0 <_start>
        ip = {af = 0, len = 0, u = {addrl = {21483676084, 140737488346160}, addr32 = {8839604, 5, 4294958128, 32767}, addr16 = {57780, 134, 5, 0, 56368, 65535, 32767, 0}, addr = "\264\341\206\000\005\000\000\000\060\334\377\377\377\177\000"}}
        si = 0x33
        lval = 0x0
        rve = 0x90000000c
        err_rve = 0x4bc015 <sr_event_exec+415>
        rve_type = 32767
        err_type = 4294957888
        expected_type = 32767
        rv = 0x7ffff5bc5360 <default_registrar_cfg>
        rve_param_no = 0
        __func__ = "fix_actions"
#11 0x00000000005def48 in fix_rval (rv=0x7ffff65237d8, rve=0x7ffff65237d0) at core/rvalue.c:2992
        __func__ = "fix_rval"
#12 0x00000000005f0a90 in fix_rval_expr (p=0x7ffff65237d0) at core/rvalue.c:3919
        rve = 0x7ffff65237d0
        ret = 0
        __func__ = "fix_rval_expr"
#13 0x00000000005a210b in fix_actions (a=0x7ffff6517ce0) at core/route.c:719
        t = 0x7ffff65228e8
        p = 0x7ffff1adb000
        tmp = 0xffffffff006f4dd9 <error: Cannot access memory at address 0xffffffff006f4dd9>
        tmp_p = 0x4
        ret = 0
        i = 8566243
        cmd = 0x41cbd0 <_start>
        s = {s = 0x82b4d8 "core", len = -163131376}
        he = 0x7fffffffe010
        ip = {af = 4294959040, len = 32767, u = {addrl = {7294619, 8694965}, addr32 = {7294619, 0, 8694965, 0}, addr16 = {20123, 111, 0, 0, 44213, 132, 0, 0}, addr = "\233No\000\000\000\000\000\265\254\204\000\000\000\000"}}
        si = 0x100000000
        lval = 0x7ffff65218a8
        rve = 0x7ffff65237d0
        err_rve = 0x0
        rve_type = RV_INT
        err_type = 4294959040
        expected_type = RV_NONE
        rv = 0x41cbd0 <_start>
        rve_param_no = -8016
        __func__ = "fix_actions"
#14 0x00000000005b3e03 in fix_rl (rt=0x91f140 <main_rt>) at core/route.c:2102
        i = 0
        ret = 51
#15 0x00000000005b3e3a in fix_rls () at core/route.c:2118
        ret = 0
#16 0x00000000004381c0 in main (argc=12, argv=0x7fffffffe5f8) at main.c:3066
        cfg_stream = 0x9c32d0
        c = -1
        r = 0
        tmp = 0x7fffffffe8b6 ""
        tmp_len = 896
        port = 896
        proto = 896
        ahost = 0x0
        aport = 0
        options = 0x801218 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
        ret = -1
        seed = 2632239041
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 2
        n_lst = 0x0
        p = 0xc2 <error: Cannot access memory at address 0xc2>
        st = {st_dev = 47, st_ino = 279340502, st_nlink = 2, st_mode = 16832, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 120, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1622741422, tv_nsec = 474562221}, st_mtim = {tv_sec = 1625629861, tv_nsec = 261687069}, st_ctim = {tv_sec = 1625629861, tv_nsec = 261687069}, __glibc_reserved = {0, 0, 0}}
        tbuf = "\360\341\377\377\377\177\000\000\000\000\000\000\000\000\000\000\360\341\377\377\377\177", '\000' <repeats 18 times>, "\260\027\375\367\377\177\000\000\350\317\377\367\377\177\000\000\b\345\377\367\377\177\000\000\340\031\375\367\377\177\000\000\025\217\376\367\377\177\000\000$f\307\367\377\177\000\000\354K\377\367\377\177\000\000\336K\377\367\377\177\000\000\205\317c\t\000\000\000\000\300S\374\367\377\177\000\000ߏ\376\367\377\177\000\000\000\000\000\000\254\202\226\006\334P\307\367\377\177\000\000\000\000\000\000\000\000\000\000\300S\374\367\377\177\000\000\001\000\000\000\000\000\000\000"\247\177\336s\027\000\000\240\341\377\367\377\177\000\000\370\377\377\377\377\377\377\377\240\341\377\367\377\177\000\000R"...
        option_index = 12
        long_options = {{name = 0x8038f6 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7fe521 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x8038fb "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x803901 "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x803907 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x803910 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x80391a "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x803924 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x80392f "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x803938 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x803943 "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x803949 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x803953 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
        __func__ = "main"
(gdb)   info locals
No symbol table info available.
(gdb) list
1976			+((!sctp_disable)?sctp_listeners:0)
1977	#endif
1978			;
1979	}
1980	
1981	int main(int argc, char** argv)
1982	{
1983	
1984		FILE* cfg_stream;
1985		int c,r;
(gdb) f 1
#1  0x00007ffff6391fd4 in pv_parse_hdr_name (sp=0x7ffff652c0d8, in=0x7fffffffd560) at pv_core.c:3617
3617		memcpy(p, in->s, in->len);
(gdb)   info locals
s = {s = 0x7ffff1d72f60 "`.\327\361\377\177", len = 4312016}
p = 0x0
nsp = 0x0
hdr = {type = HDR_PAI_T, name = {s = 0x41cbd0 <_start> "\363\017\036\372\061\355I\211\321^H\211\342H\203\344\360PTI\307\300@\330\177", len = -11088}, body = {s = 0x566521 <pv_lookup_spec_name+910> "\205\300u<H\213EȋP\020H\213\205p\377\377\377\211\020H\213E\310H\213P\030H\213\205p\377\377\377H\211P\bH\213E\310H\213P H\213\205p\377\377\377H\211P\020H\213E\310\353\037H\213E\310H\213@PH\211E\310H\203", <incomplete sequence \310>, len = -162348840}, len = -10928, parsed = 0x80d116, next = 0x7ffff652bf07}
__func__ = "pv_parse_hdr_name"
(gdb) list
3612		{
3613			LM_ERR("name too long\n");
3614			return -1;
3615		}
3616		p = pv_get_buffer();
3617		memcpy(p, in->s, in->len);
3618		p[in->len] = ':';
3619		s.s = p;
3620		s.len = in->len+1;
3621	
(gdb) f 2
#2  0x0000000000568916 in pv_parse_spec2 (in=0x7ffff652c0c0, e=0x7ffff652c0d8, silent=0) at core/pvapi.c:969
969			if(pte->parse_name(e, &s)!=0)
(gdb)   info locals
p = 0x7ffff652c144 ")"
s = {s = 0x7ffff652c13d "X-hdr16)", len = 7}
pvname = {s = 0x7ffff652c139 "hdr(X-hdr16)", len = 3}
pvstate = 5
tr = 0x0
pte = 0x7ffff64fef70
n = 0
__func__ = "pv_parse_spec2"
(gdb) list
964					LM_ERR("pvar "%.*s" does not get empty name param\n",
965							pvname.len, pvname.s);
966				goto error;
967			}
968			s.len = p - s.s;
969			if(pte->parse_name(e, &s)!=0)
970			{
971				if (!silent)
972					LM_ERR("pvar "%.*s" has an invalid name param [%.*s]\n",
973							pvname.len, pvname.s, s.len, s.s);
(gdb) f 3
#3  0x0000000000563ffd in pv_cache_add (name=0x7fffffffd750) at core/pvapi.c:359
359		p = pv_parse_spec(&pvn->pvname, &pvn->spec);
(gdb)   info locals
pvn = 0x7ffff652c0c0
pvid = 3075582644
p = 0x0
__func__ = "pv_cache_add"
(gdb) list
354		}
355		memset(pvn, 0, sizeof(pv_cache_t) + name->len + 1);
356		pvn->pvname.len = name->len;
357		pvn->pvname.s = (char*)pvn + sizeof(pv_cache_t);
358		memcpy(pvn->pvname.s, name->s, name->len);
359		p = pv_parse_spec(&pvn->pvname, &pvn->spec);
360	
361		if(p==NULL)
362		{
363			pkg_free(pvn);
```
#### Log Messages
```
[root@safarov-dell kamailio]# gdb --args kamailio --atexit=no -DD -P /run/kamailio/kamailio.pid -f /etc/kamailio/kamailio.cfg -m 64 -M 24 -E
GNU gdb (GDB) Fedora 10.1-2.fc33
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
https://www.gnu.org/software/gdb/bugs/.
Find the GDB manual and other documentation resources online at:
    http://www.gnu.org/software/gdb/documentation/.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from kamailio...
(gdb)  set pagination off
(gdb) r
Starting program: /usr/local/sbin/kamailio --atexit=no -DD -P /run/kamailio/kamailio.pid -f /etc/kamailio/kamailio.cfg -m 64 -M 24 -E
```
### Additional Information
* **Kamailio Version** - output of `kamailio -v`
```
[root@safarov-dell kamailio]# kamailio -v
version: kamailio 5.6.0-dev0 (x86_64/linux) 8762c5
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: 8762c5 
compiled on 17:11:19 Jul  6 2021 with gcc 10.2.1
```
* **Operating System**:
```
[root@safarov-dell kamailio]# cat /etc/os-release 
NAME=Fedora
VERSION="33 (Container Image)"
ID=fedora
VERSION_ID=33
VERSION_CODENAME=""
PLATFORM_ID="platform:f33"
PRETTY_NAME="Fedora 33 (Container Image)"
ANSI_COLOR="0;38;2;60;110;180"
LOGO=fedora-logo-icon
CPE_NAME="cpe:/o:fedoraproject:fedora:33"
HOME_URL="https://fedoraproject.org/"
DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f33/system-administrators-guide/"
SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
BUG_REPORT_URL="https://bugzilla.redhat.com/"
REDHAT_BUGZILLA_PRODUCT="Fedora"
REDHAT_BUGZILLA_PRODUCT_VERSION=33
REDHAT_SUPPORT_PRODUCT="Fedora"
REDHAT_SUPPORT_PRODUCT_VERSION=33
PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
VARIANT="Container Image"
VARIANT_ID=container
```
-- 
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/2798

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

[sr-dev] [kamailio/kamailio] crush: anot alliged memory access (#2798)