From b328e1891a4d0aabd2e93ba30cca676afe687df5 Mon Sep 17 00:00:00 2001 From: "Zane C. Bowers-Hadley" Date: Thu, 21 Oct 2021 22:30:03 -0500 Subject: [PATCH] update to the newest postfix stuff, aggregate default to off as that appears to be buggy, resulting in lots of lines being ignored --- .../logstash/conf.d/50-filter-postfix.conf | 14 ------- .../51-filter-postfix-aggregate.conf.off | 38 +++++++++++++++++++ .../logstash/patterns.d/postfix.grok | 12 ++++-- 3 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 Search-ESsearcher/logstash/conf.d/51-filter-postfix-aggregate.conf.off diff --git a/Search-ESsearcher/logstash/conf.d/50-filter-postfix.conf b/Search-ESsearcher/logstash/conf.d/50-filter-postfix.conf index 32c9ef6..070da86 100644 --- a/Search-ESsearcher/logstash/conf.d/50-filter-postfix.conf +++ b/Search-ESsearcher/logstash/conf.d/50-filter-postfix.conf @@ -167,10 +167,6 @@ filter { } } - - - - # process key-value data if it exists if [postfix_keyvalue_data] { kv { @@ -266,15 +262,5 @@ filter { "postfix_postscreen_violation_time", "float" ] } - - # add geoip for postfix - if [program] =~ /.*postfix.*/ { - geoip { - source => "postfix_client_ip" - } - mutate { - convert => [ "[geoip][coordinates]", "float" ] - } - } } diff --git a/Search-ESsearcher/logstash/conf.d/51-filter-postfix-aggregate.conf.off b/Search-ESsearcher/logstash/conf.d/51-filter-postfix-aggregate.conf.off new file mode 100644 index 0000000..c46b895 --- /dev/null +++ b/Search-ESsearcher/logstash/conf.d/51-filter-postfix-aggregate.conf.off @@ -0,0 +1,38 @@ +filter { + if ![postfix_queueid] { + drop {} + } else if [program] == "postfix/qmgr" and [postfix_from]{ + aggregate { + task_id => "%{postfix_queueid}" + code => " + map['postfix_from'] = event.get('postfix_from') + map['postfix_size'] = event.get('postfix_size') + map['postfix_nrcpt'] = event.get('postfix_nrcpt') + " + } + } else if [program] == "postfix/smtpd" { + aggregate { + task_id => "%{postfix_queueid}" + code => " + map['postfix_client_hostname'] = event.get('postfix_client_hostname') + map['postfix_client_ip'] = event.get('postfix_client_ip') + " + } + } else if [program] == "postfix/cleanup" { + aggregate { + task_id => "%{postfix_queueid}" + code => " + map['postfix_message-id'] = event.get('postfix_message-id') + " + } + } else if [program] == "postfix/smtp" { + aggregate { + task_id => "%{postfix_queueid}" + code => " + map.each do |key, value| + event.set(key, value) + end + " + } + } +} diff --git a/Search-ESsearcher/logstash/patterns.d/postfix.grok b/Search-ESsearcher/logstash/patterns.d/postfix.grok index 215e049..bac7f52 100644 --- a/Search-ESsearcher/logstash/patterns.d/postfix.grok +++ b/Search-ESsearcher/logstash/patterns.d/postfix.grok @@ -1,5 +1,7 @@ +# Version: 1.0.0 + # common postfix patterns -POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}) +POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}|NOQUEUE) POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[%{IP:postfix_client_ip}\](:%{INT:postfix_client_port})? POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.) @@ -26,6 +28,7 @@ POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:pos # helper patterns GREEDYDATA_NO_COLON [^:]* GREEDYDATA_NO_SEMICOLON [^;]* +GREEDYDATA_NO_BRACKET [^<>]* STATUS_WORD [\w-]* # warning patterns @@ -37,12 +40,15 @@ POSTFIX_WARNING %{POSTFIX_WARNING_WITH_KV}|%{POSTFIX_WARNING_WITHOUT_KV} POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT_INFO} POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT_INFO}( %{GREEDYDATA:postfix_command_counter_data})? POSTFIX_SMTPD_LOSTCONN %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( \(%{INT} bytes\))?)? from %{POSTFIX_CLIENT_INFO}(: %{GREEDYDATA:postfix_smtpd_lostconn_reason})? -POSTFIX_SMTPD_NOQUEUE NOQUEUE: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} +POSTFIX_SMTPD_NOQUEUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_improper_pipelining_data} POSTFIX_SMTPD_PROXY proxy-%{POSTFIX_ACTION:postfix_proxy_result}: (%{POSTFIX_SMTP_STAGE:postfix_proxy_smtp_stage}): %{POSTFIX_PROXY_MESSAGE:postfix_proxy_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} # cleanup patterns POSTFIX_CLEANUP_MILTER %{POSTFIX_QUEUEID:postfix_queueid}: milter-%{POSTFIX_ACTION:postfix_milter_result}: %{GREEDYDATA:postfix_milter_message}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}(: %{GREEDYDATA:postfix_milter_data})? +POSTFIX_CLEANUP_PREPEND_TYPE (header|body) +POSTFIX_CLEANUP_PREPEND %{POSTFIX_QUEUEID:postfix_queueid}: prepend: %{POSTFIX_CLEANUP_PREPEND_TYPE:postfix_prepend_type} %{GREEDYDATA:postfix_prepend_trigger} from %{POSTFIX_CLIENT_INFO}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}: %{GREEDYDATA:postfix_prepend_value} +POSTFIX_CLEANUP_MESSAGEID %{POSTFIX_QUEUEID:postfix_queueid}: message-id=? # qmgr patterns POSTFIX_QMGR_REMOVED %{POSTFIX_QUEUEID:postfix_queueid}: removed @@ -107,7 +113,7 @@ POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_sc # aggregate all patterns POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE} -POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE} +POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MESSAGEID}|%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_CLEANUP_PREPEND}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE} POSTFIX_QMGR %{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_EXPIRED}|%{POSTFIX_WARNING} POSTFIX_PIPE %{POSTFIX_PIPE_ANY} POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING}