hello,
I am having Suricata as IDS on passive optical TAP, where incoming traffic to my network is copied to interface enp3s0f0 and outgoing traffic from my network is copied to enp130s0f0.
Server running Suricata has two sockets each with 14cores 2 threads CPUs:
root@hadar:/usr/local/bin# cat /sys/devices/system/node/node0/cpulist
0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54
root@hadar:/usr/local/bin# cat /sys/devices/system/node/node1/cpulist
1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55
Each used NIC is connected to one of two sockets:
root@hadar:/usr/local/bin# cat /sys/class/net/enp3s0f0/device/numa_node
0
root@hadar:/usr/local/bin# cat /sys/class/net/enp130s0f0/device/numa_node
1
All CPU except both threads of the first CPU on both NUMAs (0,28,1,29) are disabled for system use by kernel params in grub.
nohz_full=2-27,30-55 isolcpus=2-27,30-55 rcu_nocbs=2-27,30-55
I am using RSS to steer incoming traffic into 26 queues per NIC, these queue are pinned by CPU SMP affinity to corresponding reserved cores. I can see in top sirq load being equally spread across the selected cores and also /proc/interrupts corresponds to the set RSS / affinity.
to set NIC I am calling following script:
suricata_interface.sh enp3s0f0 26 '2,4,6,8,10,12,14,16,18,20,22,24,26,30,32,34,36,38,40,42,44,46,48,50,52,54'
suricata_interface.sh enp130s0f0 26 '3,5,7,9,11,13,15,17,19,21,23,25,27,31,33,35,37,39,41,43,45,47,49,51,53,55'
#!/bin/bash
DEV=$1
CORE_NUM=$2
CORE_LIST=$3
#rmmod i40e && modprobe i40e
ip link set $DEV down
ethtool -L $DEV combined $CORE_NUM
ethtool -K $DEV rxhash on
ethtool -K $DEV ntuple on
#link up and sisable arp on TAP interface
ip link set $DEV promisc on arp off up
#disable autonegotiation for TAP interface
ethtool -A $DEV autoneg off
#disable pause frames
ethtool -A $DEV rx off tx off
#set RSS CPU affinity
/usr/local/bin/set_irq_affinity.sh $CORE_LIST $DEV
#set RSS symetric hashing key
ethtool -X $DEV hkey 6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A:6D:5A equal $CORE_NUM
#set RSS hash function to Toeplitz
ethtool -X $DEV hfunc toeplitz
#Manage interrupts
ethtool -C $DEV adaptive-rx off adaptive-tx off rx-usecs 125
#Lower the NIC ring descriptor size
ethtool -G $DEV rx 512
#disable NIC offloading
for offload in rx tx tso ufo gso gro lro tx-nocache-copy sg txvlan rxvlan; do
/sbin/ethtool -K $DEV $offload off 2>&1 > /dev/null;
done
#set hashing tuple - sd (src IP, dst IP), sdfn (src IP, dst IP, src port, dst port)
for proto in tcp4 udp4 ah4 esp4 sctp4 tcp6 udp6 ah6 esp6 sctp6; do
/sbin/ethtool -N $DEV rx-flow-hash $proto sd 2>&1 > /dev/null;
I configured Suricata 5.0.3 as IDS, af_packet, workers.
- interface: enp3s0f0
threads: 26
cluster-id: 99
cluster-type: cluster_qm
defrag: yes
use-mmap: yes
mmap-locked: yes
tpacket-v3: yes
ring-size: 200000
block-size: 1048576
- interface: enp130s0f0
threads: 26
cluster-id: 98
cluster-type: cluster_qm
defrag: yes
use-mmap: yes
mmap-locked: yes
tpacket-v3: yes
ring-size: 200000
block-size: 1048576
threading:
set-cpu-affinity: yes
# Tune cpu affinity of threads. Each family of threads can be bound
# on specific CPUs.
#
# These 2 apply to the all runmodes:
# management-cpu-set is used for flow timeout handling, counters
# worker-cpu-set is used for 'worker' threads
#
# Additionally, for autofp these apply:
# receive-cpu-set is used for capture threads
# verdict-cpu-set is used for IPS verdict threads
#
cpu-affinity:
- management-cpu-set:
cpu: [ 0, 1 ] # include only these CPUs in affinity settings
mode: "balanced"
prio:
default: "medium"
#- receive-cpu-set:
# cpu: [ 0 ] # include only these CPUs in affinity settings
- worker-cpu-set:
cpu: [ "2","4","6","8","10","12","14","16","18","20","22","24","26","30","32","34","36","38","40","42","44","46","48","50","52","54","3","5","7","9","11","13","15","17","19","21","23","25","27","31","33","35","37","39","41","43","45","47","49","51","53","55" ]
mode: "exclusive"
# Use explicitely 3 threads and don't compute number by using
# detect-thread-ratio variable:
# threads: 3
threads: 52
prio:
# low: [ 0 ]
# medium: [ "1-2" ]
# high: [ 3 ]
default: "high"
#- verdict-cpu-set:
# cpu: [ 0 ]
# prio:
# default: "high"
As you can see, two clusters are created (each for one NIC), each shall be given 26 worker threads, which shall correspond to available cores on one NUMA (sockets) which is connected to the same NIC.
Suricata could start and I can see all expected threads are running:
root@hadar:/etc/suricata# ps -T -p 1545
PID SPID TTY TIME CMD
1545 1545 ? 00:00:26 Suricata-Main
1545 1547 ? 01:35:06 W#01-enp3s0f0
1545 1548 ? 01:39:31 W#02-enp3s0f0
1545 1549 ? 01:35:11 W#03-enp3s0f0
1545 1550 ? 01:32:43 W#04-enp3s0f0
1545 1551 ? 01:40:19 W#05-enp3s0f0
1545 1552 ? 01:37:15 W#06-enp3s0f0
1545 1553 ? 01:31:10 W#07-enp3s0f0
1545 1554 ? 01:45:21 W#08-enp3s0f0
1545 1555 ? 01:41:50 W#09-enp3s0f0
1545 1556 ? 01:35:02 W#10-enp3s0f0
1545 1557 ? 01:37:18 W#11-enp3s0f0
1545 1558 ? 01:37:42 W#12-enp3s0f0
1545 1559 ? 01:33:30 W#13-enp3s0f0
1545 1560 ? 01:37:07 W#14-enp3s0f0
1545 1561 ? 01:39:57 W#15-enp3s0f0
1545 1562 ? 01:33:36 W#16-enp3s0f0
1545 1563 ? 01:34:50 W#17-enp3s0f0
1545 1564 ? 01:31:02 W#18-enp3s0f0
1545 1565 ? 01:30:30 W#19-enp3s0f0
1545 1566 ? 01:40:07 W#20-enp3s0f0
1545 1567 ? 01:33:19 W#21-enp3s0f0
1545 1568 ? 01:42:29 W#22-enp3s0f0
1545 1569 ? 01:38:14 W#23-enp3s0f0
1545 1570 ? 01:39:29 W#24-enp3s0f0
1545 1571 ? 01:36:19 W#25-enp3s0f0
1545 1572 ? 01:39:14 W#26-enp3s0f0
1545 1573 ? 00:04:57 W#01-enp130s0f0
1545 1574 ? 00:06:10 W#02-enp130s0f0
1545 1575 ? 00:04:35 W#03-enp130s0f0
1545 1576 ? 00:04:23 W#04-enp130s0f0
1545 1577 ? 00:04:06 W#05-enp130s0f0
1545 1578 ? 00:04:57 W#06-enp130s0f0
1545 1579 ? 00:04:47 W#07-enp130s0f0
1545 1580 ? 00:05:38 W#08-enp130s0f0
1545 1581 ? 00:05:37 W#09-enp130s0f0
1545 1582 ? 00:04:36 W#10-enp130s0f0
1545 1583 ? 00:04:38 W#11-enp130s0f0
1545 1584 ? 00:05:12 W#12-enp130s0f0
1545 1585 ? 00:04:41 W#13-enp130s0f0
1545 1586 ? 00:04:59 W#14-enp130s0f0
1545 1587 ? 00:06:56 W#15-enp130s0f0
1545 1588 ? 00:04:36 W#16-enp130s0f0
1545 1589 ? 00:05:16 W#17-enp130s0f0
1545 1590 ? 00:04:11 W#18-enp130s0f0
1545 1591 ? 00:04:56 W#19-enp130s0f0
1545 1592 ? 00:04:20 W#20-enp130s0f0
1545 1593 ? 00:04:57 W#21-enp130s0f0
1545 1594 ? 00:04:20 W#22-enp130s0f0
1545 1595 ? 00:05:46 W#23-enp130s0f0
1545 1596 ? 00:04:13 W#24-enp130s0f0
1545 1597 ? 00:05:30 W#25-enp130s0f0
1545 1598 ? 00:03:52 W#26-enp130s0f0
1545 1599 ? 00:01:33 FM#01
1545 1600 ? 00:01:53 FR#01
1545 1601 ? 00:00:00 CW
1545 1602 ? 00:00:02 CS
1545 1603 ? 00:00:00 US
The issue:
workers thread on some cores seesm to do nothing - see result of top bellow
You can see cores 0,1,28,29 not doing much as these are reserved for OS + suricata management threads, but you would expect rest of the cores to be loaded heavily by suricata workers threads. Unfortunatelly cores 30-55 are idling.
top - 21:43:04 up 2:40, 1 user, load average: 17.89, 18.27, 18.54
Tasks: 430 total, 1 running, 429 sleeping, 0 stopped, 0 zombie
%Cpu0 : 1.0 us, 0.0 sy, 0.0 ni, 99.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
%Cpu1 : 1.3 us, 0.3 sy, 0.0 ni, 98.3 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
%Cpu2 : 39.2 us, 0.0 sy, 0.0 ni, 58.2 id, 0.0 wa, 0.0 hi, 2.6 si, 0.0 st
%Cpu3 : 55.2 us, 0.0 sy, 0.0 ni, 43.0 id, 0.0 wa, 0.0 hi, 1.8 si, 0.0 st
%Cpu4 : 42.0 us, 0.0 sy, 0.0 ni, 55.5 id, 0.0 wa, 0.0 hi, 2.6 si, 0.0 st
%Cpu5 : 99.7 us, 0.0 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 0.3 si, 0.0 st
%Cpu6 : 96.7 us, 0.0 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 3.3 si, 0.0 st
%Cpu7 : 64.9 us, 0.4 sy, 0.0 ni, 33.7 id, 0.0 wa, 0.0 hi, 1.1 si, 0.0 st
%Cpu8 : 74.9 us, 0.0 sy, 0.0 ni, 22.0 id, 0.0 wa, 0.0 hi, 3.1 si, 0.0 st
%Cpu9 : 75.3 us, 0.0 sy, 0.0 ni, 24.0 id, 0.0 wa, 0.0 hi, 0.7 si, 0.0 st
%Cpu10 : 60.8 us, 0.0 sy, 0.0 ni, 34.6 id, 0.0 wa, 0.0 hi, 4.5 si, 0.0 st
%Cpu11 : 76.0 us, 0.0 sy, 0.0 ni, 23.6 id, 0.0 wa, 0.0 hi, 0.3 si, 0.0 st
%Cpu12 : 79.5 us, 0.0 sy, 0.0 ni, 17.8 id, 0.0 wa, 0.0 hi, 2.7 si, 0.0 st
%Cpu13 : 40.0 us, 0.0 sy, 0.0 ni, 59.6 id, 0.0 wa, 0.0 hi, 0.4 si, 0.0 st
%Cpu14 : 81.2 us, 0.0 sy, 0.0 ni, 17.5 id, 0.0 wa, 0.0 hi, 1.4 si, 0.0 st
%Cpu15 : 65.8 us, 0.0 sy, 0.0 ni, 33.8 id, 0.0 wa, 0.0 hi, 0.4 si, 0.0 st
%Cpu16 : 97.7 us, 0.0 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 2.3 si, 0.0 st
%Cpu17 : 87.7 us, 0.0 sy, 0.0 ni, 12.0 id, 0.0 wa, 0.0 hi, 0.3 si, 0.0 st
%Cpu18 : 78.7 us, 0.0 sy, 0.0 ni, 18.9 id, 0.0 wa, 0.0 hi, 2.4 si, 0.0 st
%Cpu19 : 93.6 us, 0.0 sy, 0.0 ni, 5.4 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
%Cpu20 : 44.7 us, 0.0 sy, 0.0 ni, 51.6 id, 0.0 wa, 0.0 hi, 3.6 si, 0.0 st
%Cpu21 : 96.3 us, 0.0 sy, 0.0 ni, 3.0 id, 0.0 wa, 0.0 hi, 0.7 si, 0.0 st
%Cpu22 : 76.7 us, 0.0 sy, 0.0 ni, 20.5 id, 0.0 wa, 0.0 hi, 2.8 si, 0.0 st
%Cpu23 : 68.1 us, 0.4 sy, 0.0 ni, 31.2 id, 0.0 wa, 0.0 hi, 0.4 si, 0.0 st
%Cpu24 : 56.4 us, 0.0 sy, 0.0 ni, 41.4 id, 0.0 wa, 0.0 hi, 2.1 si, 0.0 st
%Cpu25 : 58.6 us, 0.0 sy, 0.0 ni, 39.9 id, 0.0 wa, 0.0 hi, 1.4 si, 0.0 st
%Cpu26 : 90.8 us, 0.0 sy, 0.0 ni, 6.4 id, 0.0 wa, 0.0 hi, 2.7 si, 0.0 st
%Cpu27 : 98.7 us, 0.0 sy, 0.0 ni, 0.0 id, 0.0 wa, 0.0 hi, 1.3 si, 0.0 st
%Cpu28 : 0.0 us, 0.7 sy, 0.0 ni, 99.3 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
%Cpu29 : 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
%Cpu30 : 0.7 us, 0.3 sy, 0.0 ni, 96.0 id, 0.0 wa, 0.0 hi, 3.0 si, 0.0 st
%Cpu31 : 1.7 us, 0.0 sy, 0.0 ni, 97.2 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
%Cpu32 : 2.4 us, 0.0 sy, 0.0 ni, 95.3 id, 0.0 wa, 0.0 hi, 2.4 si, 0.0 st
%Cpu33 : 1.3 us, 0.0 sy, 0.0 ni, 98.0 id, 0.0 wa, 0.0 hi, 0.7 si, 0.0 st
%Cpu34 : 0.7 us, 0.0 sy, 0.0 ni, 96.0 id, 0.0 wa, 0.0 hi, 3.4 si, 0.0 st
%Cpu35 : 1.4 us, 0.0 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
%Cpu36 : 1.3 us, 0.0 sy, 0.0 ni, 96.3 id, 0.0 wa, 0.0 hi, 2.3 si, 0.0 st
%Cpu37 : 3.7 us, 0.0 sy, 0.0 ni, 95.3 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
%Cpu38 : 1.4 us, 0.0 sy, 0.0 ni, 95.9 id, 0.0 wa, 0.0 hi, 2.7 si, 0.0 st
%Cpu39 : 2.0 us, 0.0 sy, 0.0 ni, 96.6 id, 0.0 wa, 0.0 hi, 1.3 si, 0.0 st
%Cpu40 : 1.7 us, 0.0 sy, 0.0 ni, 94.3 id, 0.0 wa, 0.0 hi, 4.1 si, 0.0 st
%Cpu41 : 1.7 us, 0.0 sy, 0.0 ni, 97.3 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
%Cpu42 : 1.7 us, 0.0 sy, 0.0 ni, 96.3 id, 0.0 wa, 0.0 hi, 2.0 si, 0.0 st
%Cpu43 : 1.7 us, 0.0 sy, 0.0 ni, 97.9 id, 0.0 wa, 0.0 hi, 0.3 si, 0.0 st
%Cpu44 : 1.7 us, 0.0 sy, 0.0 ni, 94.5 id, 0.0 wa, 0.0 hi, 3.8 si, 0.0 st
%Cpu45 : 0.7 us, 0.0 sy, 0.0 ni, 99.0 id, 0.0 wa, 0.0 hi, 0.3 si, 0.0 st
%Cpu46 : 1.7 us, 0.0 sy, 0.0 ni, 95.9 id, 0.0 wa, 0.0 hi, 2.4 si, 0.0 st
%Cpu47 : 1.0 us, 0.3 sy, 0.0 ni, 97.3 id, 0.0 wa, 0.0 hi, 1.3 si, 0.0 st
%Cpu48 : 1.4 us, 0.0 sy, 0.0 ni, 96.6 id, 0.0 wa, 0.0 hi, 2.0 si, 0.0 st
%Cpu49 : 0.7 us, 0.0 sy, 0.0 ni, 99.0 id, 0.0 wa, 0.0 hi, 0.3 si, 0.0 st
%Cpu50 : 0.7 us, 0.0 sy, 0.0 ni, 95.9 id, 0.0 wa, 0.0 hi, 3.4 si, 0.0 st
%Cpu51 : 1.3 us, 0.0 sy, 0.0 ni, 96.6 id, 0.0 wa, 0.0 hi, 2.0 si, 0.0 st
%Cpu52 : 1.7 us, 0.0 sy, 0.0 ni, 95.8 id, 0.0 wa, 0.0 hi, 2.4 si, 0.0 st
%Cpu53 : 0.7 us, 0.0 sy, 0.0 ni, 97.9 id, 0.0 wa, 0.0 hi, 1.4 si, 0.0 st
%Cpu54 : 6.5 us, 0.0 sy, 0.0 ni, 91.5 id, 0.0 wa, 0.0 hi, 2.0 si, 0.0 st
%Cpu55 : 0.7 us, 0.0 sy, 0.0 ni, 98.3 id, 0.0 wa, 0.0 hi, 1.0 si, 0.0 st
MiB Mem : 128835.5 total, 103668.6 free, 24739.8 used, 427.1 buff/cache
MiB Swap: 3815.0 total, 3815.0 free, 0.0 used. 103244.1 avail Mem
I have no idea where is the issue with these CPU not being used by Suricata, any help is apreciated. Thank you.
L.