I use Suricata to get the http data from network card with lua script,but the data is confused. For example, the HttpGetResponseLine() method should be return that like ‘GET /api/abc/path1?param=1’,but the result is ‘param=1&GET’ some times.
There is another case about the HttpGetResponseBody(), it often has to reverse a fixed string ‘{“code”:200,“msg”:null,“content”:null}’,nobody knows where this string came from,because replay the http immediately,the results are completely different.
All these problems look like thread safety problems
How are these same fields logged in eve.json when you see this? Is it the same in eve or are things correct there?
the eve log not enabled.my log output by the lua script.
I want complete http data, is there any other way besides lua script.
This is my lua script:
json = require("cjson")
function init(args)
local needs = {}
needs["protocol"] = "http"
return needs
end
function log(args)
if (not available()) then
http = http + 1
return
end
-- 定义数据包变量
http_data = {
src_ip = nil,
src_port = nil,
target_ip = nil,
target_port = nil,
request_host = HttpGetRequestHost(),
request_uri = HttpGetRequestUriRaw(),
request_line = HttpGetRequestLine(),
request_header = HttpGetRawRequestHeaders(),
response_line = HttpGetResponseLine(),
response_header = HttpGetRawResponseHeaders(),
timestamp = SCFlowTimestamps(),
record_time = os.date("%Y-%m-%d %H:%M:%S"),
request_method="",
request_body = "",
response_body = ""
}
-- network
ip_version, http_data.src_ip, http_data.target_ip, protocol, http_data.src_port, http_data.target_port = SCFlowTuple()
-- request body
a, o, e = HttpGetRequestBody();
if (a ~= nil)
then
for n, v in ipairs(a) do
http_data.request_body = http_data.request_body .. v
end
end
-- response body
a, o, e = HttpGetResponseBody();
if (a ~= nil)
then
for n, v in ipairs(a) do
http_data.response_body = http_data.response_body .. v
end
end
if http_data.request_host == nil then
http_data.request_host = HttpGetRequestHeader("Host")
end
-- 如果仍未查询到request host 则丢弃数据
if http_data.request_host == nil then
http = http + 1
SCLogInfo('已丢弃无效数据,src:' .. http_data.src_ip..':'..http_data.src_port..',target:'..http_data.target_ip..':'..http_data.target_port)
print('已丢弃无效数据,src:' .. http_data.src_ip..':'..http_data.src_port..',target:'..http_data.target_ip..':'..http_data.target_port)
return
end
if http_data.request_uri == '/libhtp::request_uri_not_seen' then
http_data.request_uri = HttpGetRequestUriNormalized()
end
-- 当前时间
local currentDate = os.date("%Y%m%d")
local file_name = "/data/suricata/httplog/" .. currentDate .. ".json"
local http_log = assert(io.open(file_name, "a"))
http_log:write(json.encode(http_data) .. "\r\n")
local record_log = "record:" ..
http_data.src_ip .. "-" .. http_data.target_ip .. "-" .. http_data.timestamp .. " ===> " .. file_name
print(record_log)
http_log:flush()
http_log:close()
http = http + 1
SCLogInfo('已处理http请求数:' .. http)
print('已处理http请求数:' .. http)
end
-- 判断是否需要记录
function available()
local request_uri = HttpGetRequestUriRaw()
for i = 1, #scope_exclude do
find_value = string.find(request_uri, scope_exclude[i])
if find_value ~= nil then
local log = '规则:' .. scope_exclude[i] .. ' 已过滤url:' .. request_uri
print(log)
SCLogInfo(log)
return false
end
end
return true
end
function setup(args)
http = 0
scope_exclude = {
".*%.gif$",
".*%.jpg$",
".*%.png$",
".*%.css$",
".*%.js$",
".*%.ico$",
".*%.ttf$",
".*%.woff2$",
".*%.woff$",
".*%.gif%?",
".*%.jpg%?",
".*%.png%?",
".*%.css%?",
".*%.js%?",
".*%.ico%?",
".*%.ttf%?",
".*%.woff2%?",
".*%.woff%?"
}
end
function deinit(args)
end
And this is my suricata config:
%YAML 1.1
---
vars:
address-groups:
HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
EXTERNAL_NET: "!$HOME_NET"
HTTP_SERVERS: "$HOME_NET"
SMTP_SERVERS: "$HOME_NET"
SQL_SERVERS: "$HOME_NET"
DNS_SERVERS: "$HOME_NET"
TELNET_SERVERS: "$HOME_NET"
AIM_SERVERS: "$EXTERNAL_NET"
DC_SERVERS: "$HOME_NET"
DNP3_SERVER: "$HOME_NET"
DNP3_CLIENT: "$HOME_NET"
MODBUS_CLIENT: "$HOME_NET"
MODBUS_SERVER: "$HOME_NET"
ENIP_CLIENT: "$HOME_NET"
ENIP_SERVER: "$HOME_NET"
port-groups:
HTTP_PORTS: "80"
SHELLCODE_PORTS: "!80"
ORACLE_PORTS: 1521
SSH_PORTS: 22
DNP3_PORTS: 20000
MODBUS_PORTS: 502
FILE_DATA_PORTS: "[$HTTP_PORTS,110,143]"
FTP_PORTS: 21
GENEVE_PORTS: 6081
VXLAN_PORTS: 4789
TEREDO_PORTS: 3544
default-log-dir: /var/log/suricata/
stats:
enabled: yes
interval: 8
outputs:
- fast:
enabled: no
filename: fast.log
append: yes
- eve-log:
enabled: no
filename: eve.json
pcap-file: false
community-id: false
community-id-seed: 0
xff:
enabled: no
mode: extra-data
deployment: reverse
header: X-Forwarded-For
types:
- alert:
tagged-packets: yes
- anomaly:
enabled: yes
types:
- http:
- dns:
- tls:
- files:
- smtp:
- ftp
- rdp
- nfs
- smb
- tftp
- ikev2
- dcerpc
- krb5
- snmp
- rfb
- sip
- dhcp:
enabled: yes
extended: no
- ssh
- mqtt:
- stats:
- flow
- http-log:
enabled: yes
filename: http-url.log
append: no
customformat: "%{%D-%H:%M:%S}t.%z ^ %m ^ %h %u ^ %s ^ %B ^ %a ^ %p ^ %A ^ %P ^ %{User-agent}i"
- tls-log:
append: yes
- tls-store:
enabled: no
- pcap-log:
enabled: no
filename: log.pcap
limit: 1000mb
max-files: 2000
compression: none
- alert-debug:
enabled: no
filename: alert-debug.log
append: yes
- alert-prelude:
enabled: no
profile: suricata
log-packet-content: no
log-packet-header: yes
- stats:
enabled: yes
filename: stats.log
- syslog:
enabled: no
facility: local5
- file-store:
version: 2
enabled: no
xff:
enabled: no
mode: extra-data
deployment: reverse
header: X-Forwarded-For
- tcp-data:
enabled: no
type: file
filename: tcp-data.log
- http-body-data:
enabled: no
type: file
filename: http-data.log
- lua:
enabled: yes
scripts:
- /data/suricata/webauth-v2.lua
logging:
default-log-level: notice
default-output-filter:
outputs:
- console:
enabled: yes
- file:
enabled: yes
level: info
filename: suricata.log
- syslog:
enabled: no
facility: local5
format: "[%i] <%d> -- "
af-packet:
- interface: eth0
cluster-id: 99
cluster-type: cluster_flow
defrag: yes
- interface: default
pcap:
- interface: eth0
- interface: default
pcap-file:
checksum-checks: auto
app-layer:
protocols:
rfb:
enabled: no
detection-ports:
dp: 5900, 5901, 5902, 5903, 5904, 5905, 5906, 5907, 5908, 5909
mqtt:
krb5:
enabled: no
snmp:
enabled: no
ikev2:
enabled: no
tls:
enabled: no
detection-ports:
dp: 443
dcerpc:
enabled: no
ftp:
enabled: no
rdp:
ssh:
enabled: no
http2:
enabled: no
http1-rules: no
smtp:
enabled: no
raw-extraction: no
mime:
decode-mime: yes
decode-base64: yes
decode-quoted-printable: yes
header-value-depth: 2000
extract-urls: no
body-md5: no
inspected-tracker:
content-limit: 100000
content-inspect-min-size: 32768
content-inspect-window: 4096
imap:
enabled: detection-only
smb:
enabled: no
detection-ports:
dp: 139, 445
nfs:
enabled: no
tftp:
enabled: no
dns:
tcp:
enabled: no
detection-ports:
dp: 53
udp:
enabled: no
detection-ports:
dp: 53
http:
enabled: yes
memcap: 50gb
libhtp:
default-config:
personality: IDS
request-body-limit: 10mb
response-body-limit: 20mb
request-body-minimal-inspect-size: 32kb
request-body-inspect-window: 4kb
response-body-minimal-inspect-size: 40kb
response-body-inspect-window: 16kb
response-body-decompress-layer-limit: 2
http-body-inline: auto
swf-decompression:
enabled: yes
type: both
compress-depth: 100kb
decompress-depth: 100kb
double-decode-path: no
double-decode-query: no
server-config:
modbus:
enabled: no
detection-ports:
dp: 502
stream-depth: 0
dnp3:
enabled: no
detection-ports:
dp: 20000
enip:
enabled: no
detection-ports:
dp: 44818
sp: 44818
ntp:
enabled: yes
dhcp:
enabled: yes
sip:
asn1-max-frames: 256
coredump:
max-dump: unlimited
host-mode: auto
unix-command:
enabled: auto
legacy:
uricontent: enabled
engine-analysis:
rules-fast-pattern: yes
rules: yes
pcre:
match-limit: 3500
match-limit-recursion: 1500
host-os-policy:
windows: [0.0.0.0/0]
bsd: []
bsd-right: []
old-linux: []
linux: []
old-solaris: []
solaris: []
hpux10: []
hpux11: []
irix: []
macos: []
vista: []
windows2k3: []
defrag:
memcap: 80gb
hash-size: 65536
prealloc: yes
timeout: 300
flow:
memcap: 80gb
hash-size: 65536
prealloc: 10000
emergency-recovery: 30
vlan:
use-for-tracking: true
flow-timeouts:
default:
new: 30
established: 300
closed: 0
bypassed: 100
emergency-new: 10
emergency-established: 100
emergency-closed: 0
emergency-bypassed: 50
tcp:
new: 60
established: 600
closed: 60
bypassed: 100
emergency-new: 5
emergency-established: 100
emergency-closed: 10
emergency-bypassed: 50
udp:
new: 30
established: 300
bypassed: 100
emergency-new: 10
emergency-established: 100
emergency-bypassed: 50
icmp:
new: 30
established: 300
bypassed: 100
emergency-new: 10
emergency-established: 100
emergency-bypassed: 50
stream:
memcap: 80gb
midstream: true
async-oneside: true
drop-invalid: yes
reassembly:
memcap: 50gb
toserver-chunk-size: 5000
toclient-chunk-size: 5000
randomize-chunk-size: yes
raw: no
host:
hash-size: 4096
prealloc: 1000
memcap: 10gb
decoder:
teredo:
enabled: true
vxlan:
enabled: true
vntag:
enabled: false
geneve:
enabled: true
detect:
profile: medium
custom-values:
toclient-groups: 3
toserver-groups: 25
sgh-mpm-context: auto
inspection-recursion-limit: 3000
prefilter:
default: mpm
grouping:
profiling:
grouping:
dump-to-disk: false
include-mpm-stats: false
mpm-algo: auto
spm-algo: auto
threading:
set-cpu-affinity: no
cpu-affinity:
- management-cpu-set:
- receive-cpu-set:
- worker-cpu-set:
cpu: [ "all" ]
mode: "exclusive"
prio:
low: [ 0 ]
medium: [ "1-2" ]
high: [ 3 ]
default: "medium"
detect-thread-ratio: 1.0
luajit:
states: 128
profiling:
rules:
enabled: no
filename: rule_perf.log
append: yes
limit: 10
json: yes
keywords:
enabled: no
filename: keyword_perf.log
append: yes
prefilter:
enabled: no
filename: prefilter_perf.log
append: yes
rulegroups:
enabled: no
filename: rule_group_perf.log
append: yes
packets:
enabled: no
filename: packet_stats.log
append: yes
csv:
enabled: no
filename: packet_stats.csv
locks:
enabled: no
filename: lock_stats.log
append: yes
pcap-log:
enabled: no
filename: pcaplog_stats.log
append: yes
nfq:
nflog:
- group: 2
buffer-size: 18432
- group: default
qthreshold: 1
qtimeout: 100
max-size: 20000
capture:
netmap:
- interface: eth2
- interface: default
pfring:
- interface: eth0
threads: auto
cluster-id: 99
cluster-type: cluster_flow
- interface: default
ipfw:
napatech:
streams: ["0-3"]
enable-stream-stats: no
auto-config: yes
hardware-bypass: yes
inline: no
ports: [0-1,2-3]
hashmode: hash5tuplesorted
default-rule-path: /var/lib/suricata/rules
rule-files:
- /root/local.rules
classification-file: /etc/suricata/classification.config
reference-config-file: /etc/suricata/reference.config
can you enable eve http as a test? If we see it be correct in eve it is likely an issue limited to the lua integration.
1 Like