21 May
2024
21 May
'24
4:16 p.m.
Hello,
I'm in a situation where i have 5 proxies (5.5.7) deployed on AWS on
6cni.xlarge instances (4 cpu/8gb), all they're doing is taking TLS
connections and forwarding everything to the proxy-dialogs (5.5.2).
We opened traffic on these 5 proxies for about 20k clients.
CPU just goes to 100% and stays there... (actually moving from core to core
100%). We were doing tls offloading on AWS but routing was a bit of a mess
(receiving tls traffic on a non-tls, etc) and we never saw a problem
CPU-wise there.
config looks like
#!define WORKERS 32
#!define MAX_TCP_TLS_CONNECTIONS 30000
#!define OPEN_FILES 65535
#!define TCP_CHILDREN_PROCESS 16
tcp_accept_no_cl=no # Needed for HTTP (
https://www.kamailio.org/wiki/cookbooks/devel/core#tcp_accept_no_cl)
enable_tls=yes
tls_port_no=TLS_PORT
auto_aliases=no
tcp_children=32
#
socket_workers=WORKERS
listen=EXTERNAL_LISTEN_SOCKET
listen=EXTERNAL_LISTEN_SOCKET_TLS
#
socket_workers=WORKERS
listen=INTERNAL_UDP_SOCKET
I even attached gdb and all i see is:
(gdb) bt
#0 0x00001499de9e9853 in __libc_pause () at
../sysdeps/unix/sysv/linux/pause.c:29
#1 0x000055a58c1a8e6f in main_loop ()
#2 0x000055a58c1b29a1 in main ()
(gdb)
#0 0x00001499de9e9853 in __libc_pause () at
../sysdeps/unix/sysv/linux/pause.c:29
#1 0x000055a58c1a8e6f in main_loop ()
#2 0x000055a58c1b29a1 in main ()
(gdb)
#0 0x00001499de9e9853 in __libc_pause () at
../sysdeps/unix/sysv/linux/pause.c:29
#1 0x000055a58c1a8e6f in main_loop ()
#2 0x000055a58c1b29a1 in main ()
(gdb)
#0 0x00001499de9e9853 in __libc_pause () at
../sysdeps/unix/sysv/linux/pause.c:29
#1 0x000055a58c1a8e6f in main_loop ()
#2 0x000055a58c1b29a1 in main ()
which doesn't seem like it's doing much, and stats look ok:
kamcmd> stats.fetch all
{
core.bad_URIs_rcvd: 0
core.bad_msg_hdr: 0
core.drop_replies: 0
core.drop_requests: 0
core.err_replies: 0
core.err_requests: 0
core.fwd_replies: 0
core.fwd_requests: 72
core.rcv_replies: 4134
core.rcv_replies_18x: 159
core.rcv_replies_1xx: 279
core.rcv_replies_1xx_bye: 0
core.rcv_replies_1xx_cancel: 0
core.rcv_replies_1xx_invite: 279
core.rcv_replies_1xx_message: 0
core.rcv_replies_1xx_prack: 0
core.rcv_replies_1xx_refer: 0
core.rcv_replies_1xx_reg: 0
core.rcv_replies_1xx_update: 0
core.rcv_replies_2xx: 3068
core.rcv_replies_2xx_bye: 68
core.rcv_replies_2xx_cancel: 39
core.rcv_replies_2xx_invite: 72
core.rcv_replies_2xx_message: 0
core.rcv_replies_2xx_prack: 0
core.rcv_replies_2xx_refer: 0
core.rcv_replies_2xx_reg: 975
core.rcv_replies_2xx_update: 0
core.rcv_replies_3xx: 0
core.rcv_replies_3xx_bye: 0
core.rcv_replies_3xx_cancel: 0
core.rcv_replies_3xx_invite: 0
core.rcv_replies_3xx_message: 0
core.rcv_replies_3xx_prack: 0
core.rcv_replies_3xx_refer: 0
core.rcv_replies_3xx_reg: 0
core.rcv_replies_3xx_update: 0
core.rcv_replies_401: 658
core.rcv_replies_404: 0
core.rcv_replies_407: 72
core.rcv_replies_480: 4
core.rcv_replies_486: 0
core.rcv_replies_4xx: 776
core.rcv_replies_4xx_bye: 2
core.rcv_replies_4xx_cancel: 1
core.rcv_replies_4xx_invite: 115
core.rcv_replies_4xx_message: 0
core.rcv_replies_4xx_prack: 0
core.rcv_replies_4xx_refer: 0
core.rcv_replies_4xx_reg: 658
core.rcv_replies_4xx_update: 0
core.rcv_replies_5xx: 0
core.rcv_replies_5xx_bye: 0
core.rcv_replies_5xx_cancel: 0
core.rcv_replies_5xx_invite: 0
core.rcv_replies_5xx_message: 0
core.rcv_replies_5xx_prack: 0
core.rcv_replies_5xx_refer: 0
core.rcv_replies_5xx_reg: 0
core.rcv_replies_5xx_update: 0
core.rcv_replies_6xx: 11
core.rcv_replies_6xx_bye: 0
core.rcv_replies_6xx_cancel: 0
core.rcv_replies_6xx_invite: 11
core.rcv_replies_6xx_message: 0
core.rcv_replies_6xx_prack: 0
core.rcv_replies_6xx_refer: 0
core.rcv_replies_6xx_reg: 0
core.rcv_replies_6xx_update: 0
core.rcv_requests: 2170
core.rcv_requests_ack: 210
core.rcv_requests_bye: 83
core.rcv_requests_cancel: 45
core.rcv_requests_info: 0
core.rcv_requests_invite: 196
core.rcv_requests_message: 0
core.rcv_requests_notify: 0
core.rcv_requests_options: 0
core.rcv_requests_prack: 0
core.rcv_requests_publish: 0
core.rcv_requests_refer: 0
core.rcv_requests_register: 1636
core.rcv_requests_subscribe: 0
core.rcv_requests_update: 0
core.unsupported_methods: 0
dns.failed_dns_request: 0
dns.slow_dns_request: 0
shmem.fragments: 270
shmem.free_size: 4283453736
shmem.max_used_size: 13711272
shmem.real_used_size: 11513560
shmem.total_size: 4294967296
shmem.used_size: 8167352
sl.1xx_replies: 0
sl.200_replies: 0
sl.202_replies: 0
sl.2xx_replies: 0
sl.300_replies: 0
sl.301_replies: 0
sl.302_replies: 0
sl.3xx_replies: 0
sl.400_replies: 0
sl.401_replies: 0
sl.403_replies: 0
sl.404_replies: 0
sl.407_replies: 0
sl.408_replies: 0
sl.483_replies: 0
sl.4xx_replies: 0
sl.500_replies: 0
sl.5xx_replies: 0
sl.6xx_replies: 0
sl.failures: 0
sl.received_ACKs: 0
sl.sent_err_replies: 0
sl.sent_replies: 0
sl.xxx_replies: 0
tcp.con_reset: 5
tcp.con_timeout: 54
tcp.connect_failed: 3
tcp.connect_success: 0
tcp.current_opened_connections: 44
tcp.current_write_queue_size: 0
tcp.established: 295
tcp.local_reject: 0
tcp.passive_open: 295
tcp.send_timeout: 0
tcp.sendq_full: 0
tmx.2xx_transactions: 3074
tmx.3xx_transactions: 0
tmx.4xx_transactions: 780
tmx.5xx_transactions: 0
tmx.6xx_transactions: 11
tmx.UAC_transactions: 1914
tmx.UAS_transactions: 3860
tmx.active_transactions: 1
tmx.inuse_transactions: 12
tmx.rpl_absorbed: 162
tmx.rpl_generated: 247
tmx.rpl_received: 4134
tmx.rpl_relayed: 3972
tmx.rpl_sent: 4219
}
any ideas?
thanks!,
David Villasmil
email: david.villasmil.work(a)gmail.com
phone: +34669448337