<problem solved via Suricata json stats, not telegraf>
Hello,
Finnaly got Telegraf working to read some system statistics and more important Suricata stats via a unix socket. Telegraf ship the data over a tcp socket connection to an ELK server which is receiving the data via Logtash. And yes it is finally in Elasticsearch, but I need someone to push me in the right direction how to best process this Telegraf data (json formatted).
Now it pops up in Elasticsearch with a message text field containing this. Any advise or push in the right direction much appriciated, and yes I will make a blog about myt suricata 7 dpdk telegraf ELK journey.
Cheers,
André
{“fields”:{“app_layer_error_bittorrent-dht_alloc”:0,“app_layer_error_bittorrent-dht_gap”:0,“app_layer_error_bittorrent-dht_internal”:0,“app_layer_error_bittorrent-dht_parser”:0,“app_layer_error_dhcp_alloc”:0,“app_layer_error_dhcp_gap”:0,“app_layer_error_dhcp_internal”:0,“app_layer_error_dhcp_parser”:0,“app_layer_error_dnp3_alloc”:0,“app_layer_error_dnp3_gap”:0,“app_layer_error_dnp3_internal”:0,“app_layer_error_dnp3_parser”:0,“app_layer_error_dns_tcp_alloc”:0,“app_layer_error_dns_tcp_gap”:0,“app_layer_error_dns_tcp_internal”:0,“app_layer_error_dns_tcp_parser”:0,“app_layer_error_dns_udp_alloc”:0,“app_layer_error_dns_udp_internal”:0,“app_layer_error_dns_udp_parser”:0,“app_layer_error_enip_tcp_alloc”:0,“app_layer_error_enip_tcp_gap”:0,“app_layer_error_enip_tcp_internal”:0,“app_layer_error_enip_tcp_parser”:0,“app_layer_error_enip_udp_alloc”:0,“app_layer_error_enip_udp_internal”:0,“app_layer_error_enip_udp_parser”:0,“app_layer_error_failed_tcp_gap”:0,“app_layer_error_ftp-data_alloc”:0,“app_layer_error_ftp-data_gap”:5,“app_layer_error_ftp-data_internal”:0,“app_layer_error_ftp-data_parser”:0,“app_layer_error_ftp_alloc”:0,“app_layer_error_ftp_gap”:8,“app_layer_error_ftp_internal”:0,“app_layer_error_ftp_parser”:2,“app_layer_error_http2_alloc”:0,“app_layer_error_http2_gap”:1,“app_layer_error_http2_internal”:0,“app_layer_error_http2_parser”:0,“app_layer_error_http_alloc”:0,“app_layer_error_http_gap”:0,“app_layer_error_http_internal”:0,“app_layer_error_http_parser”:7,“app_layer_error_ike_alloc”:0,“app_layer_error_ike_gap”:0,“app_layer_error_ike_internal”:0,“app_layer_error_ike_parser”:0,“app_layer_error_imap_alloc”:0,“app_layer_error_imap_gap”:0,“app_layer_error_imap_internal”:0,“app_layer_error_imap_parser”:0,“app_layer_error_krb5_tcp_alloc”:0,“app_layer_error_krb5_tcp_gap”:2,“app_layer_error_krb5_tcp_internal”:0,“app_layer_error_krb5_tcp_parser”:0,“app_layer_error_krb5_udp_alloc”:0,“app_layer_error_krb5_udp_internal”:0,“app_layer_error_krb5_udp_parser”:0,“app_layer_error_modbus_alloc”:0,“app_layer_error_modbus_gap”:0,“app_layer_error_modbus_internal”:0,“app_layer_error_modbus_parser”:0,“app_layer_error_mqtt_alloc”:0,“app_layer_error_mqtt_gap”:0,“app_layer_error_mqtt_internal”:0,“app_layer_error_mqtt_parser”:0,“app_layer_error_nfs_tcp_alloc”:0,“app_layer_error_nfs_tcp_gap”:0,“app_layer_error_nfs_tcp_internal”:0,“app_layer_error_nfs_tcp_parser”:0,“app_layer_error_nfs_udp_alloc”:0,“app_layer_error_nfs_udp_internal”:0,“app_layer_error_nfs_udp_parser”:0,“app_layer_error_ntp_alloc”:0,“app_layer_error_ntp_gap”:0,“app_layer_error_ntp_internal”:0,“app_layer_error_ntp_parser”:0,“app_layer_error_pgsql_alloc”:0,“app_layer_error_pgsql_gap”:0,“app_layer_error_pgsql_internal”:0,“app_layer_error_pgsql_parser”:31,“app_layer_error_quic_alloc”:0,“app_layer_error_quic_gap”:0,“app_layer_error_quic_internal”:0,“app_layer_error_quic_parser”:16112,“app_layer_error_rdp_alloc”:0,“app_layer_error_rdp_gap”:5,“app_layer_error_rdp_internal”:0,“app_layer_error_rdp_parser”:0,“app_layer_error_rfb_alloc”:0,“app_layer_error_rfb_gap”:0,“app_layer_error_rfb_internal”:0,“app_layer_error_rfb_parser”:0,“app_layer_error_smb_alloc”:0,“app_layer_error_smb_gap”:0,“app_layer_error_smb_internal”:0,“app_layer_error_smb_parser”:0,“app_layer_error_smtp_alloc”:0,“app_layer_error_smtp_gap”:11,“app_layer_error_smtp_internal”:0,“app_layer_error_smtp_parser”:1,“app_layer_error_snmp_alloc”:0,“app_layer_error_snmp_gap”:0,“app_layer_error_snmp_internal”:0,“app_layer_error_snmp_parser”:6,“app_layer_error_ssh_alloc”:0,“app_layer_error_ssh_gap”:475,“app_layer_error_ssh_internal”:0,“app_layer_error_ssh_parser”:14,“app_layer_error_telnet_alloc”:0,“app_layer_error_telnet_gap”:0,“app_layer_error_telnet_internal”:0,“app_layer_error_telnet_parser”:0,“app_layer_error_tftp_alloc”:0,“app_layer_error_tftp_gap”:0,“app_layer_error_tftp_internal”:0,“app_layer_error_tftp_parser”:0,“app_layer_error_tls_alloc”:0,“app_layer_error_tls_gap”:9337,“app_layer_error_tls_internal”:0,“app_layer_error_tls_parser”:5689,“app_layer_expectations”:5,“app_layer_flow_bittorrent-dht”:489,“app_layer_flow_dhcp”:9144,“app_layer_flow_dnp3”:0,“app_layer_flow_dns_tcp”:12005,“app_layer_flow_dns_udp”:1556318,“app_layer_flow_enip_tcp”:0,“app_layer_flow_enip_udp”:0,“app_layer_flow_failed_tcp”:271312,“app_layer_flow_failed_udp”:837571,“app_layer_flow_ftp”:372,“app_layer_flow_ftp-data”:328,“app_layer_flow_http”:189820,“app_layer_flow_http2”:4,“app_layer_flow_ike”:58,“app_layer_flow_imap”:0,“app_layer_flow_krb5_tcp”:164901,“app_layer_flow_krb5_udp”:12,“app_layer_flow_modbus”:0,“app_layer_flow_mqtt”:0,“app_layer_flow_nfs_tcp”:0,“app_layer_flow_nfs_udp”:0,“app_layer_flow_ntp”:29854,“app_layer_flow_pgsql”:29768,“app_layer_flow_quic”:36518,“app_layer_flow_rdp”:12,“app_layer_flow_rfb”:0,“app_layer_flow_smb”:75654,“app_layer_flow_smtp”:2259,“app_layer_flow_snmp”:332724,“app_layer_flow_ssh”:8447,“app_layer_flow_telnet”:1,“app_layer_flow_tftp”:0,“app_layer_flow_tls”:966597,“app_layer_tx_bittorrent-dht”:1156,“app_layer_tx_dhcp”:177333,“app_layer_tx_dnp3”:0,“app_layer_tx_dns_tcp”:23792,“app_layer_tx_dns_udp”:3078350,“app_layer_tx_enip_tcp”:0,“app_layer_tx_enip_udp”:0,“app_layer_tx_ftp”:4686,“app_layer_tx_ftp-data”:0,“app_layer_tx_http”:639201,“app_layer_tx_http2”:21240,“app_layer_tx_ike”:503,“app_layer_tx_imap”:0,“app_layer_tx_krb5_tcp”:329542,“app_layer_tx_krb5_udp”:18,“app_layer_tx_modbus”:0,“app_layer_tx_mqtt”:0,“app_layer_tx_nfs_tcp”:0,“app_layer_tx_nfs_udp”:0,“app_layer_tx_ntp”:43989,“app_layer_tx_pgsql”:29372,“app_layer_tx_quic”:987281,“app_layer_tx_rdp”:36,“app_layer_tx_rfb”:0,“app_layer_tx_smb”:838726,“app_layer_tx_smtp”:2392,“app_layer_tx_snmp”:22200241,“app_layer_tx_ssh”:0,“app_layer_tx_telnet”:0,“app_layer_tx_tftp”:0,“app_layer_tx_tls”:0,“capture_dpdk_ierrors”:55,“capture_dpdk_imissed”:48784586,“capture_dpdk_no_mbufs”:0,“capture_packets”:1375833188,“capture_rx_errors”:48784641,“capture_tx_errors”:0,“decoder_arp”:3592,“decoder_avg_pkt_size”:902,“decoder_bytes”:1197643700416,“decoder_chdlc”:0,“decoder_erspan”:0,“decoder_esp”:10783,“decoder_ethernet”:1327381749,“decoder_event_chdlc_pkt_too_small”:0,“decoder_event_dce_pkt_too_small”:0,“decoder_event_erspan_header_too_small”:0,“decoder_event_erspan_too_many_vlan_layers”:0,“decoder_event_erspan_unsupported_version”:0,“decoder_event_esp_pkt_too_small”:0,“decoder_event_ethernet_pkt_too_small”:0,“decoder_event_geneve_unknown_payload_type”:0,“decoder_event_gre_pkt_too_small”:0,“decoder_event_gre_version0_flags”:0,“decoder_event_gre_version0_hdr_too_big”:0,“decoder_event_gre_version0_malformed_sre_hdr”:0,“decoder_event_gre_version0_recur”:0,“decoder_event_gre_version1_chksum”:0,“decoder_event_gre_version1_flags”:0,“decoder_event_gre_version1_hdr_too_big”:0,“decoder_event_gre_version1_malformed_sre_hdr”:0,“decoder_event_gre_version1_no_key”:0,“decoder_event_gre_version1_recur”:0,“decoder_event_gre_version1_route”:0,“decoder_event_gre_version1_ssr”:0,“decoder_event_gre_version1_wrong_protocol”:0,“decoder_event_gre_wrong_version”:0,“decoder_event_icmpv4_ipv4_trunc_pkt”:0,“decoder_event_icmpv4_ipv4_unknown_ver”:0,“decoder_event_icmpv4_pkt_too_small”:0,“decoder_event_icmpv4_unknown_code”:208,“decoder_event_icmpv4_unknown_type”:0,“decoder_event_icmpv6_experimentation_type”:0,“decoder_event_icmpv6_ipv6_trunc_pkt”:0,“decoder_event_icmpv6_ipv6_unknown_version”:0,“decoder_event_icmpv6_mld_message_with_invalid_hl”:0,“decoder_event_icmpv6_pkt_too_small”:0,“decoder_event_icmpv6_unassigned_type”:0,“decoder_event_icmpv6_unknown_code”:0,“decoder_event_icmpv6_unknown_type”:0,“decoder_event_ieee8021ah_header_too_small”:0,“decoder_event_ipraw_invalid_ip_version”:0,“decoder_event_ipv4_frag_ignored”:0,“decoder_event_ipv4_frag_overlap”:0,“decoder_event_ipv4_frag_pkt_too_large”:0,“decoder_event_ipv4_hlen_too_small”:0,“decoder_event_ipv4_icmpv6”:0,“decoder_event_ipv4_iplen_smaller_than_hlen”:0,“decoder_event_ipv4_opt_duplicate”:0,“decoder_event_ipv4_opt_eol_required”:0,“decoder_event_ipv4_opt_invalid”:0,“decoder_event_ipv4_opt_invalid_len”:0,“decoder_event_ipv4_opt_malformed”:0,“decoder_event_ipv4_opt_pad_required”:0,“decoder_event_ipv4_opt_unknown”:0,“decoder_event_ipv4_pkt_too_small”:0,“decoder_event_ipv4_trunc_pkt”:0,“decoder_event_ipv4_wrong_ip_version”:0,“decoder_event_ipv6_data_after_none_header”:0,“decoder_event_ipv6_dstopts_only_padding”:0,“decoder_event_ipv6_dstopts_unknown_opt”:0,“decoder_event_ipv6_exthdr_ah_res_not_null”:0,“decoder_event_ipv6_exthdr_dupl_ah”:0,“decoder_event_ipv6_exthdr_dupl_dh”:0,“decoder_event_ipv6_exthdr_dupl_eh”:0,“decoder_event_ipv6_exthdr_dupl_fh”:0,“decoder_event_ipv6_exthdr_dupl_hh”:0,“decoder_event_ipv6_exthdr_dupl_rh”:0,“decoder_event_ipv6_exthdr_invalid_optlen”:0,“decoder_event_ipv6_exthdr_useless_fh”:0,“decoder_event_ipv6_fh_non_zero_reserved_field”:0,“decoder_event_ipv6_frag_ignored”:0,“decoder_event_ipv6_frag_invalid_length”:0,“decoder_event_ipv6_frag_overlap”:0,“decoder_event_ipv6_frag_pkt_too_large”:0,“decoder_event_ipv6_hopopts_only_padding”:0,“decoder_event_ipv6_hopopts_unknown_opt”:0,“decoder_event_ipv6_icmpv4”:0,“decoder_event_ipv6_ipv4_in_ipv6_too_small”:0,“decoder_event_ipv6_ipv4_in_ipv6_wrong_version”:0,“decoder_event_ipv6_ipv6_in_ipv6_too_small”:0,“decoder_event_ipv6_ipv6_in_ipv6_wrong_version”:0,“decoder_event_ipv6_pkt_too_small”:0,“decoder_event_ipv6_rh_type_0”:0,“decoder_event_ipv6_trunc_exthdr”:0,“decoder_event_ipv6_trunc_pkt”:0,“decoder_event_ipv6_unknown_next_header”:812,“decoder_event_ipv6_wrong_ip_version”:0,“decoder_event_ipv6_zero_len_padn”:0,“decoder_event_ltnull_pkt_too_small”:0,“decoder_event_ltnull_unsupported_type”:0,“decoder_event_mpls_bad_label_implicit_null”:0,“decoder_event_mpls_bad_label_reserved”:0,“decoder_event_mpls_bad_label_router_alert”:0,“decoder_event_mpls_header_too_small”:0,“decoder_event_mpls_pkt_too_small”:0,“decoder_event_mpls_unknown_payload_type”:0,“decoder_event_nsh_bad_header_length”:0,“decoder_event_nsh_header_too_small”:0,“decoder_event_nsh_reserved_type”:0,“decoder_event_nsh_unknown_payload”:0,“decoder_event_nsh_unsupported_type”:0,“decoder_event_nsh_unsupported_version”:0,“decoder_event_ppp_ip4_pkt_too_small”:0,“decoder_event_ppp_ip6_pkt_too_small”:0,“decoder_event_ppp_pkt_too_small”:0,“decoder_event_ppp_unsup_proto”:0,“decoder_event_ppp_vju_pkt_too_small”:0,“decoder_event_ppp_wrong_type”:0,“decoder_event_pppoe_malformed_tags”:0,“decoder_event_pppoe_pkt_too_small”:0,“decoder_event_pppoe_wrong_code”:0,“decoder_event_sctp_pkt_too_small”:0,“decoder_event_sll_pkt_too_small”:0,“decoder_event_tcp_hlen_too_small”:0,“decoder_event_tcp_invalid_optlen”:0,“decoder_event_tcp_opt_duplicate”:0,“decoder_event_tcp_opt_invalid_len”:0,“decoder_event_tcp_pkt_too_small”:0,“decoder_event_udp_hlen_invalid”:0,“decoder_event_udp_hlen_too_small”:0,“decoder_event_udp_len_invalid”:0,“decoder_event_udp_pkt_too_small”:0,“decoder_event_vlan_header_too_small”:0,“decoder_event_vlan_too_many_layers”:0,“decoder_event_vlan_unknown_type”:0,“decoder_event_vntag_header_too_small”:0,“decoder_event_vntag_unknown_type”:0,“decoder_event_vxlan_unknown_payload_type”:0,“decoder_geneve”:0,“decoder_gre”:0,“decoder_icmpv4”:2844299,“decoder_icmpv6”:10061,“decoder_ieee8021ah”:0,“decoder_invalid”:0,“decoder_ipv4”:1327289088,“decoder_ipv4_in_ipv6”:0,“decoder_ipv6”:132130,“decoder_ipv6_in_ipv6”:0,“decoder_max_mac_addrs_dst”:0,“decoder_max_mac_addrs_src”:0,“decoder_max_pkt_size”:1540,“decoder_mpls”:0,“decoder_nsh”:0,“decoder_null”:0,“decoder_pkts”:1327381749,“decoder_ppp”:0,“decoder_pppoe”:0,“decoder_raw”:0,“decoder_sctp”:0,“decoder_sll”:0,“decoder_tcp”:1061525649,“decoder_teredo”:0,“decoder_too_many_layers”:0,“decoder_udp”:262872163,“decoder_unknown_ethertype”:19532,“decoder_vlan”:882402399,“decoder_vlan_qinq”:0,“decoder_vlan_qinqinq”:0,“decoder_vntag”:0,“decoder_vxlan”:0,“defrag_ipv4_fragments”:154854,“defrag_ipv4_reassembled”:62561,“defrag_ipv6_fragments”:0,“defrag_ipv6_reassembled”:0,“defrag_max_frag_hits”:0,“detect_alert”:80678,“detect_alert_queue_overflow”:0,“detect_alerts_suppressed”:6475591,“detect_engines_id”:0,“detect_engines_last_reload”:“2023-08-12T13:46:49.280197+0200”,“detect_engines_rules_failed”:3,“detect_engines_rules_loaded”:71675,“file_store_open_files”:0,“flow_active”:301973,“flow_bypassed_bytes”:0,“flow_bypassed_closed”:0,“flow_bypassed_local_bytes”:58062726627,“flow_bypassed_local_capture_bytes”:0,“flow_bypassed_local_capture_pkts”:0,“flow_bypassed_local_pkts”:66039025,“flow_bypassed_pkts”:0,“flow_emerg_mode_entered”:0,“flow_emerg_mode_over”:0,“flow_end_state_closed”:1949867,“flow_end_state_established”:1146378,“flow_end_state_local_bypassed”:163718,“flow_end_state_new”:11588093,“flow_end_tcp_liberal”:13237,“flow_end_tcp_state_close_wait”:5151,“flow_end_tcp_state_closed”:1843935,“flow_end_tcp_state_closing”:0,“flow_end_tcp_state_established”:15755,“flow_end_tcp_state_fin_wait1”:1918,“flow_end_tcp_state_fin_wait2”:2304,“flow_end_tcp_state_last_ack”:86335,“flow_end_tcp_state_none”:0,“flow_end_tcp_state_syn_recv”:6386,“flow_end_tcp_state_syn_sent”:8621298,“flow_end_tcp_state_time_wait”:19597,“flow_get_used”:0,“flow_get_used_eval”:0,“flow_get_used_eval_busy”:0,“flow_get_used_eval_reject”:0,“flow_get_used_failed”:0,“flow_icmpv4”:170835,“flow_icmpv6”:1097,“flow_memcap”:0,“flow_memuse”:493424640,“flow_mgr_flows_checked”:92921094,“flow_mgr_flows_evicted”:14664729,“flow_mgr_flows_evicted_needs_work”:9929946,“flow_mgr_flows_notimeout”:78604161,“flow_mgr_flows_timeout”:14316933,“flow_mgr_full_hash_pass”:486,“flow_mgr_rows_maxlen”:21,“flow_mgr_rows_per_sec”:115343,“flow_recycler_queue_avg”:2,“flow_recycler_queue_max”:1355,“flow_recycler_recycled”:4734782,“flow_spare”:1138027,“flow_tcp”:12175396,“flow_tcp_reuse”:2567,“flow_total”:15150029,“flow_udp”:2802688,“flow_wrk_flows_evicted”:8537422,“flow_wrk_flows_evicted_needs_work”:9942957,“flow_wrk_flows_evicted_pkt_inject”:10423945,“flow_wrk_flows_injected”:9933720,“flow_wrk_flows_injected_max”:683,“flow_wrk_spare_sync”:110976,“flow_wrk_spare_sync_avg”:99,“flow_wrk_spare_sync_empty”:0,“flow_wrk_spare_sync_incomplete”:30293,“ftp_memcap”:0,“ftp_memuse”:2674,“http_memcap”:0,“http_memuse”:9985268,“memcap_pressure”:11,“memcap_pressure_max”:52,“tcp_ack_unseen_data”:10028577,“tcp_active_sessions”:107013,“tcp_insert_data_normal_fail”:0,“tcp_insert_data_overlap_fail”:0,“tcp_invalid_checksum”:0,“tcp_memuse”:86592240,“tcp_midstream_pickups”:0,“tcp_overlap”:143946,“tcp_overlap_diff_data”:0,“tcp_pkt_on_wrong_thread”:0,“tcp_pseudo”:124546,“tcp_pseudo_failed”:0,“tcp_reassembly_gap”:32486,“tcp_reassembly_memuse”:726615760,“tcp_rst”:1653168,“tcp_segment_from_cache”:37424153,“tcp_segment_from_pool”:6332729,“tcp_segment_memcap_drop”:0,“tcp_sessions”:10709692,“tcp_ssn_from_cache”:3856111,“tcp_ssn_from_pool”:6853581,“tcp_ssn_memcap_drop”:0,“tcp_stream_depth_reached”:7915,“tcp_syn”:13149883,“tcp_synack”:3128696,“uptime”:3703},“name”:“suricata”,“tags”:{“event_type”:“stats”,“host”:“serverids”},“timestamp”:1691844467}