vvelox
/
Search-ESsearcher
mirror of https://github.com/VVelox/Search-ESsearcher.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

update to the newest postfix stuff, aggregate default to off as that appears to be buggy, resulting in lots of lines being ignored

This commit is contained in:
Zane C. B-H 2021-10-21 22:30:03 -05:00
parent 9936df5321
commit b328e1891a
3 changed files with 47 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Search-ESsearcher/logstash/conf.d/50-filter-postfix.conf
View File

@ -167,10 +167,6 @@ filter {
}
}
# process key-value data if it exists
if [postfix_keyvalue_data] {
kv {
@ -266,15 +262,5 @@ filter {
"postfix_postscreen_violation_time", "float"
]
}
# add geoip for postfix
if [program] =~ /.*postfix.*/ {
geoip {
source => "postfix_client_ip"
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
}
}

38
Search-ESsearcher/logstash/conf.d/51-filter-postfix-aggregate.conf.off Normal file
View File

@ -0,0 +1,38 @@
filter {
if ![postfix_queueid] {
drop {}
} else if [program] == "postfix/qmgr" and [postfix_from]{
aggregate {
task_id => "%{postfix_queueid}"
code => "
map['postfix_from'] = event.get('postfix_from')
map['postfix_size'] = event.get('postfix_size')
map['postfix_nrcpt'] = event.get('postfix_nrcpt')
"
}
} else if [program] == "postfix/smtpd" {
aggregate {
task_id => "%{postfix_queueid}"
code => "
map['postfix_client_hostname'] = event.get('postfix_client_hostname')
map['postfix_client_ip'] = event.get('postfix_client_ip')
"
}
} else if [program] == "postfix/cleanup" {
aggregate {
task_id => "%{postfix_queueid}"
code => "
map['postfix_message-id'] = event.get('postfix_message-id')
"
}
} else if [program] == "postfix/smtp" {
aggregate {
task_id => "%{postfix_queueid}"
code => "
map.each do |key, value|
event.set(key, value)
end
"
}
}
}

12
Search-ESsearcher/logstash/patterns.d/postfix.grok
View File

@ -1,5 +1,7 @@
# Version: 1.0.0
# common postfix patterns
POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,})
POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}|NOQUEUE)
POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[%{IP:postfix_client_ip}\](:%{INT:postfix_client_port})?
POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service}
POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.)
@ -26,6 +28,7 @@ POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:pos
# helper patterns
GREEDYDATA_NO_COLON [^:]*
GREEDYDATA_NO_SEMICOLON [^;]*
GREEDYDATA_NO_BRACKET [^<>]*
STATUS_WORD [\w-]*
# warning patterns
@ -37,12 +40,15 @@ POSTFIX_WARNING %{POSTFIX_WARNING_WITH_KV}|%{POSTFIX_WARNING_WITHOUT_KV}
POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT_INFO}
POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT_INFO}( %{GREEDYDATA:postfix_command_counter_data})?
POSTFIX_SMTPD_LOSTCONN %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( \(%{INT} bytes\))?)? from %{POSTFIX_CLIENT_INFO}(: %{GREEDYDATA:postfix_smtpd_lostconn_reason})?
POSTFIX_SMTPD_NOQUEUE NOQUEUE: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
POSTFIX_SMTPD_NOQUEUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_improper_pipelining_data}
POSTFIX_SMTPD_PROXY proxy-%{POSTFIX_ACTION:postfix_proxy_result}: (%{POSTFIX_SMTP_STAGE:postfix_proxy_smtp_stage}): %{POSTFIX_PROXY_MESSAGE:postfix_proxy_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
# cleanup patterns
POSTFIX_CLEANUP_MILTER %{POSTFIX_QUEUEID:postfix_queueid}: milter-%{POSTFIX_ACTION:postfix_milter_result}: %{GREEDYDATA:postfix_milter_message}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}(: %{GREEDYDATA:postfix_milter_data})?
POSTFIX_CLEANUP_PREPEND_TYPE (header|body)
POSTFIX_CLEANUP_PREPEND %{POSTFIX_QUEUEID:postfix_queueid}: prepend: %{POSTFIX_CLEANUP_PREPEND_TYPE:postfix_prepend_type} %{GREEDYDATA:postfix_prepend_trigger} from %{POSTFIX_CLIENT_INFO}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}: %{GREEDYDATA:postfix_prepend_value}
POSTFIX_CLEANUP_MESSAGEID %{POSTFIX_QUEUEID:postfix_queueid}: message-id=<?%{GREEDYDATA_NO_BRACKET:postfix_message-id}>?
# qmgr patterns
POSTFIX_QMGR_REMOVED %{POSTFIX_QUEUEID:postfix_queueid}: removed
@ -107,7 +113,7 @@ POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_sc
# aggregate all patterns
POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE}
POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}
POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MESSAGEID}|%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_CLEANUP_PREPEND}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}
POSTFIX_QMGR %{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_EXPIRED}|%{POSTFIX_WARNING}
POSTFIX_PIPE %{POSTFIX_PIPE_ANY}
POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING}