5 Aug
2019
5 Aug
'19
11:57 a.m.
Hello,
this seems similar to the issue reported in the tracker:
- https://github.com/kamailio/kamailio/issues/1895#issuecomment-500895303
I was troubleshooting on that system in the past, but recently I couldn't get more there. Not sure why the mysql query is not timing out, on the net there were mentions that is cannot be ensured, it's more like a best try.
You should install debug symbols for libmysqlclient library and see if you get when in the libmysqlclient code is getting stuck.
Do you have tls enabled in kamailio?
Cheers, Daniel
On 05.08.19 17:03, David Villasmil wrote:
At least try setting db_mode for the dialog module to 0 (zero)
On Mon, 5 Aug 2019 at 16:02, David Villasmil <david.villasmil.work@gmail.com mailto:david.villasmil.work@gmail.com> wrote:
Have you tried taking out the dB support and see if it keeps happening? The by talks a lot about MySQL... no bT expert here... On Mon, 5 Aug 2019 at 14:07, M. Salman <msalman212@gmail.com <mailto:msalman212@gmail.com>> wrote: Hi Team, Facing 100% Kamilio CPU issue, here are the details, any pointers/hints would help. Thanks *Scenario:* Random Kamailio CPU shoot, restart fixes it. Happens once or twice in a day. It's a pre-production server with around 30 extensions, not much load. For now, it has 2 CPUs and 4 G memory. *[Info]* uname -a Linux ip-X-aws #46-Ubuntu SMP Thu Jul 4 13:38:28 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux kamailio -V version: kamailio 5.2.3 (x86_64/linux) c36229 flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144 MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB poll method support: poll, epoll_lt, epoll_et, sigio_rt, select. id: c36229 *(gdb) bt full* #0 0x00007f0069d97aab in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #1 0x00007f0069d7d6bd in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #2 0x00007f0069d657c7 in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #3 0x00007f0069d3ebce in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #4 0x00007f0069d3fc4c in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #5 0x00007f0069d33edc in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #6 0x00007f0069d34fa8 in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #7 0x00007f0069d36d71 in mysql_real_query () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20 No symbol table info available. #8 0x00007f006a31e4b4 in db_mysql_submit_query (h=0x7f006a6cdb20, s=0x7f0069af92d0 <sql_str>) at km_dbase.c:116 t = 1564528172 i = 0 code = 0 _func_ = "db_mysql_submit_query" #9 0x00007f00698e8fb3 in db_do_submit_query (h=0x7f006a6cdb20, query=0x7f0069af92d0 <sql_str>, submit_query=0x7f006a31dd3c <db_mysql_submit_query>) at db_query.c:58 ret = 10 tvb = {tv_sec = 0, tv_usec = 0} tve = {tv_sec = 0, tv_usec = 2077076196242373120} tz = {tz_minuteswest = 0, tz_dsttime = 0} tdiff = 0 _func_ = "db_do_submit_query" #10 0x00007f00698ec1dc in db_do_delete (h=0x7f006a6cdb20, k=0x7ffcdb440fa0, o=0x0, v=0x7ffcdb440fb0, _n=2, val2str=0x7f006a340429 <db_mysql_val2str>, submit_query=0x7f006a31dd3c <db_mysql_submit_query>) at db_query.c:299 off = 68 ret = 36 ---Type <return> to continue, or q <return> to quit--- _func_ = "db_do_delete" #11 0x00007f006a322f6a in db_mysql_delete (h=0x7f006a6cdb20, k=0x7ffcdb440fa0, o=0x0, v=0x7ffcdb440fb0, _n=2) at km_dbase.c:515 No locals. #12 0x00007f0065b6da70 in remove_dialog_from_db (cell=0x7f005c5ed3d8) at dlg_db_handler.c:722 values = {{type = DB1_INT, nul = 0, free = 477963473, val = {int_val = 1460, ll_val = 140720308487604, double_val = 6.9525070095907518e-310, time_val = 140720308487604, string_val = 0x7ffc000005b4 <error: Cannot access memory at address 0x7ffc000005b4>, str_val = { s = 0x7ffc000005b4 <error: Cannot access memory at address 0x7ffc000005b4>, len = 0}, blob_val = { s = 0x7ffc000005b4 <error: Cannot access memory at address 0x7ffc000005b4>, len = 0}, bitmap_val = 1460, uint_val = 1460, ull_val = 140720308487604}}, {type = DB1_INT, nul = 0, free = 0, val = {int_val = 8020, ll_val = 139637976735572, double_val = 6.899032715982534e-310, time_val = 139637976735572, string_val = 0x7f0000001f54 <error: Cannot access memory at address 0x7f0000001f54>, str_val = { s = 0x7f0000001f54 <error: Cannot access memory at address 0x7f0000001f54>, len = 0}, blob_val = { s = 0x7f0000001f54 <error: Cannot access memory at address 0x7f0000001f54>, len = 0}, bitmap_val = 8020, uint_val = 8020, ull_val = 139637976735572}}} match_keys = {0x7f0065dfc6c0 <h_entry_column>, 0x7f0065dfc6b0 <h_id_column>} vars_match_keys = {0x7f0065dfc7f0 <vars_h_entry_column>, 0x7f0065dfc7e0 <vars_h_id_column>} _func_ = "remove_dialog_from_db" #13 0x00007f0065b49375 in destroy_dlg (dlg=0x7f005c5ed3d8) at dlg_hash.c:371 ret = 1 var = 0x7f0065b45442 <futex_get+40> _func_ = "destroy_dlg" #14 0x00007f0065b50ca0 in dlg_unref_helper (dlg=0x7f005c5ed3d8, cnt=2, fname=0x7f0065be1eff "dlg_handlers.c", fline=1182) at dlg_hash.c:961 d_entry = 0x7f005c45b4c0 _func_ = "dlg_unref_helper" #15 0x00007f0065b7f708 in unref_dlg_from_cb (t=0x7f005c5b5dc0, type=131072, param=0x7ffcdb4412a0) at dlg_handlers.c:1182 dlg = 0x7f005c5ed3d8 iuid = 0x7f005c568c38 #16 0x00007f0068295b05 in run_trans_callbacks_internal (cb_lst=0x7f005c5b5e38, type=131072, trans=0x7f005c5b5dc0, params=0x7ffcdb4412a0) at t_hooks.c:260 cbp = 0x7f005c606118 backup_from = 0x5569cc3da8b0 <def_list+16> backup_to = 0x5569cc3da8b8 <def_list+24> ---Type <return> to continue, or q <return> to quit--- backup_dom_from = 0x5569cc3da8c0 <def_list+32> backup_dom_to = 0x5569cc3da8c8 <def_list+40> backup_uri_from = 0x5569cc3da8a0 <def_list> backup_uri_to = 0x5569cc3da8a8 <def_list+8> backup_xavps = 0x5569cc3da2c0 <_xavp_list_head> _func_ = "run_trans_callbacks_internal" #17 0x00007f0068295c2c in run_trans_callbacks (type=131072, trans=0x7f005c5b5dc0, req=0x0, rpl=0x0, code=0) at t_hooks.c:287 params = {req = 0x0, rpl = 0x0, param = 0x7f005c606128, code = 0, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = { s = 0x0, len = 0}} #18 0x00007f00681cbf45 in free_cell_helper (dead_cell=0x7f005c5b5dc0, silent=0, fname=0x7f00682de882 "timer.c", fline=689) at h_table.c:166 b = 0x7f005c5b5dc0 "" i = 1 rpl = 0x7ffcdb4413a0 tt = 0x0 foo = 0x7ffcdb441390 cbs = 0x7f005c378700 cbs_tmp = 0x1681ca80f _func_ = "free_cell_helper" #19 0x00007f0068283bf6 in wait_handler (ti=855479619, wait_tl=0x7f005c5b5e48, data=0x7f005c5b5dc0) at timer.c:689 p_cell = 0x7f005c5b5dc0 ret = 1 unlinked = 0 rcount = 1 _func_ = "wait_handler" #20 0x00005569cbdb0661 in timer_list_expire (t=855479619, h=0x7f005c238af0, slow_l=0x7f005c23ae08, slow_mark=5646) at core/timer.c:874 tl = 0x7f005c5b5e48 ret = 0 #21 0x00005569cbdb0b09 in timer_handler () at core/timer.c:939 saved_ticks = 855479619 run_slow_timer = 0 i = 526 _func_ = "timer_handler" #22 0x00005569cbdb0fd2 in timer_main () at core/timer.c:978 ---Type <return> to continue, or q <return> to quit--- No locals. #23 0x00005569cbcf1382 in main_loop () at main.c:1727 i = 8 pid = 0 si = 0x0 si_desc = "udp receiver child=7 sock=172.31.2.33:6000 <http://172.31.2.33:6000> (13.54.252.29:6000 <http://13.54.252.29:6000>)\000\034\004\000\000\000\000\000\000\000\321$}\034\000\000\000\000P0\316\313iU\000\000 \031D\333\374\177", '\000' <repeats 18 times>, "`\026D\333\374\177\000\000\360S\361\313iU\000" nrprocs = 8 woneinit = 1 _func_ = "main_loop" #24 0x00005569cbcf8a7b in main (argc=13, argv=0x7ffcdb441928) at main.c:2696 cfg_stream = 0x5569cd7c7260 c = -1 r = 0 tmp = 0x7ffcdb441f39 "" tmp_len = -616294480 port = 32764 proto = 3 options = 0x5569cc074190 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 3114863458 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x7f006af01787 p = 0x9c0 <error: Cannot access memory at address 0x9c0> st = {st_dev = 23, st_ino = 456, st_nlink = 2, st_mode = 16877, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 60, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1564229247, tv_nsec = 809557407}, st_mtim = {tv_sec = 1564229229, tv_nsec = 217250917}, st_ctim = {tv_sec = 1564229279, tv_nsec = 666090748}, __glibc_reserved = {0, 0, 0}} _func_ = "main" (gdb) -- Regards M. Salman _______________________________________________ Kamailio (SER) - Users Mailing List sr-users@lists.kamailio.org <mailto:sr-users@lists.kamailio.org> https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users -- Regards, David Villasmil email: david.villasmil.work@gmail.com <mailto:david.villasmil.work@gmail.com> phone: +34669448337-- Regards,
David Villasmil email: david.villasmil.work@gmail.com mailto:david.villasmil.work@gmail.com phone: +34669448337
Kamailio (SER) - Users Mailing List sr-users@lists.kamailio.org https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierla -- www.asipto.com
www.twitter.com/miconda -- www.linkedin.com/in/miconda