hello all,
i use HTTP 1.1 and around 20k ids rules on an edge router, that uses multiple psuedo (virtual) tap interfaces to mirror packets from ingress LAN to suricata (4.1.6) that runs as a process.
Case 1 : if i do 3.8 Gbps with HTTP 100KB HTTP transactions - > 4496 transactions/sec[ 224 flows/sec or 4496/224=20 transactions per flow] @ 436k packets per second there are no drops . ring size is 40K @98% CPU
Case2: if i do 3.8 Gbps with HTTP 1000KB HTTP transactions - > 476 transactions/sec[23 flows/sec or 476/23 = 20 transactions/flow] @ 446k packets per second there are drops unless i increase the ring size to 190K . @ 96% cpu
Qns:
- when both cases have the same packets per second why i am having to increase the ring size only for 1000KB transactions ?
- i am able to achieve 3.8Gbps for 1000KB downloads with 80K ring-size if i increase the number of worker-cpus to 2. each cpu doing 54-70% . why is it possible to reduce the ring size by half in this case?
- when i increase the ring-size to 190k the downside is the suricata memory pre-alloc increasing 3 times, is there a different tuning paramter other than ring-size that can help this situation, other than increasing the number of cpus. ?
- one observation is that when i run the 1000KB transactions at 5Gbps without any suricata rules there are no drops.
Thanks In Advance
------------------------------------------------------------------------------------
Date: 4/28/2022 -- 08:29:28 (uptime: 0d, 00h 10m 18s)
------------------------------------------------------------------------------------
Counter | TM Name | Value
------------------------------------------------------------------------------------
capture.kernel_packets | Total | 188699089
capture.kernel_drops | Total | 18671128
decoder.pkts | Total | 169991247
decoder.bytes | Total | 188046242047
decoder.ipv4 | Total | 169991247
decoder.ethernet | Total | 169991247
decoder.tcp | Total | 169991247
decoder.avg_pkt_size | Total | 1106
decoder.max_pkt_size | Total | 1462
flow.tcp | Total | 9941
tcp.sessions | Total | 9936
tcp.syn | Total | 9936
tcp.synack | Total | 9931
tcp.rst | Total | 8679
tcp.stream_depth_reached | Total | 9925
tcp.reassembly_gap | Total | 12
tcp.overlap | Total | 218714
app_layer.flow.http | Total | 508
app_layer.tx.http | Total | 169058
flow_mgr.closed_pruned | Total | 5085
flow_mgr.new_pruned | Total | 15
flow_mgr.est_pruned | Total | 3379
flow.spare | Total | 10027
flow_mgr.flows_checked | Total | 106
flow_mgr.flows_notimeout | Total | 57
flow_mgr.flows_timeout | Total | 49
flow_mgr.flows_timeout_inuse | Total | 22
flow_mgr.flows_removed | Total | 27
flow_mgr.rows_checked | Total | 65536
flow_mgr.rows_skipped | Total | 65405
flow_mgr.rows_empty | Total | 26
flow_mgr.rows_maxlen | Total | 2
tcp.memuse | Total | 5734400
tcp.reassembly_memuse | Total | 7560892
http.memuse | Total | 11073752
flow.memuse | Total | 7685136
CONFIG 1 below: → below config is put in a child yaml file and sourced at the end of suricata.yaml file
%YAML 1.1
---
# Suricata configuration file. In addition to the comments describing all
# options in this file, full documentation can be found at:
# https://suricata.readthedocs.io/en/latest/configuration/suricata-yaml.html
##
## Step 1: inform Suricata about your network
##
vars:
# more specific is better for alert accuracy and performance
address-groups:
#HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
#HOME_NET: "[192.168.0.0/16]"
#HOME_NET: "[10.0.0.0/8]"
#HOME_NET: "[172.16.0.0/12]"
HOME_NET: "any"
#EXTERNAL_NET: "!$HOME_NET"
EXTERNAL_NET: "any"
HTTP_SERVERS: "$HOME_NET"
SMTP_SERVERS: "$HOME_NET"
SQL_SERVERS: "$HOME_NET"
DNS_SERVERS: "$HOME_NET"
TELNET_SERVERS: "$HOME_NET"
AIM_SERVERS: "$EXTERNAL_NET"
DC_SERVERS: "$HOME_NET"
DNP3_SERVER: "$HOME_NET"
DNP3_CLIENT: "$HOME_NET"
MODBUS_CLIENT: "$HOME_NET"
MODBUS_SERVER: "$HOME_NET"
ENIP_CLIENT: "$HOME_NET"
ENIP_SERVER: "$HOME_NET"
FTP_SERVERS: "$HOME_NET"
port-groups:
HTTP_PORTS: "80"
SHELLCODE_PORTS: "!80"
ORACLE_PORTS: 1521
SSH_PORTS: 22
DNP3_PORTS: 20000
MODBUS_PORTS: 502
FILE_DATA_PORTS: "[$HTTP_PORTS,110,143]"
FTP_PORTS: 21
VXLAN_PORTS: 4789
##
## Step 2: select outputs to enable
##
# The default logging directory. Any log or output file will be
# placed here if its not specified with a full path name. This can be
# overridden with the -l command line parameter.
default-log-dir: /var/log/suricata/
# global stats configuration
stats:
enabled: yes
# The interval field (in seconds) controls at what interval
# the loggers are invoked.
interval: 30
# Add decode events as stats.
#decoder-events: true
# Decoder event prefix in stats. Has been 'decoder' before, but that leads
# to missing events in the eve.stats records. See issue #2225.
decoder-events-prefix: "decoder.event"
# Add stream events as stats.
#stream-events: false
# Configure the type of alert (and other) logging you would like.
outputs:
# a line based alerts log similar to Snort's fast.log
- fast:
enabled: no
filename: fast.log
append: yes
#filetype: regular # 'regular', 'unix_stream' or 'unix_dgram'
# Extensible Event Format (nicknamed EVE) event log in JSON format
- eve-log:
enabled: no
filetype: regular #regular|syslog|unix_dgram|unix_stream|redis
filename: eve.json
#prefix: "@cee: " # prefix to prepend to each log entry
# the following are valid when type: syslog above
#identity: "suricata"
#facility: local3
#level: Info ## possible levels: Emergency, Alert, Critical,
## Error, Warning, Notice, Info, Debug
# Stats.log contains data from various counters of the Suricata engine.
- stats:
enabled: yes
filename: stats.log
#filename: /var/log/suricata/stats/stats-%Y-%m-%d-%H:%M:%S.log
append: no # append to file (yes) or overwrite it (no)
totals: yes # stats for all threads merged together
threads: no # per thread stats
#null-values: yes # print counters that have value 0
#rotate-interval: 30s
# a line based alerts log similar to fast.log into syslog
- syslog:
enabled: yes
# reported identity to syslog. If ommited the program name (usually
# suricata) will be used.
identity: "suricata"
facility: local3
level: Notice ## possible levels: Emergency, Alert, Critical,
## Error, Warning, Notice, Info, Debug
# Number of packets preallocated per thread. The default is 1024. A higher number
# will make sure each CPU will be more easily kept busy, but may negatively
# impact caching.
max-pending-packets: 1024
# Runmode the engine should use. Please check --list-runmodes to get the available
# runmodes for each packet acquisition method. Defaults to "autofp" (auto flow pinned
# load balancing).
#runmode: autofp
runmode: workers
# Specifies the kind of flow load balancer used by the flow pinned autofp mode.
#
# Supported schedulers are:
#
# round-robin - Flows assigned to threads in a round robin fashion.
# active-packets - Flows assigned to threads that have the lowest number of
# unprocessed packets (default).
# hash - Flow allocated using the address hash. More of a random
# technique. Was the default in Suricata 1.2.1 and older.
#
#autofp-scheduler: active-packets
# Preallocated size for packet. Default is 1514 which is the classical
# size for pcap on ethernet. You should adjust this value to the highest
# packet size (MTU + hardware header) on your system.
default-packet-size: 1522
CONFIG 2 BELOW, - > below config is put in a child yaml file and sourced at the end of suricata.yaml file
%YAML 1.1
---
### DO NOT EDIT! This file is modified during runtime ###
### based on SilverPeak's CPU configuration ###
threading:
set-cpu-affinity: yes
cpu-affinity:
- management-cpu-set:
cpu: [ 4 ]
- receive-cpu-set:
cpu: [ 4 ]
- worker-cpu-set:
cpu: [ 4 ]
mode: "exclusive"
threads: 1
prio:
low: [ 0 ]
medium: [ 0 ]
high: [ 4 ]
default: "high"
detect-thread-ratio: 1.0
# Linux high speed capture support
af-packet:
- interface: default
threads: 2
cluster-type: cluster_flow
defrag: yes
use-mmap: yes
ring-size: 40000
checksum-checks: no
stream:
memcap: 64mb
checksum-validation: no
inline: no
reassembly:
memcap: 400mb
depth: 10kb
toserver-chunk-size: 2560
toclient-chunk-size: 2560
randomize-chunk-size: yes