most of the work for 0.3.0

Search-ESsearcher/Changes

@@ -1,5 +1,12 @@
 Revision history for Search-ESsearcher
 
+0.2.0	 2019-06-03/03:40
+		 - The bf2b template now properly processes --ip
+		 - Add the httpAccess template.
+		 - Add a missing flag to the help for bf2b.
+		 - Added the option for pretty printing -S via -p
+
+
 0.1.0	 2019-06-02/09:00
 		 - Add bf2b, beats fail2ban support.
 		 - Actually set the output template now.

Search-ESsearcher/MANIFEST

@@ -1,5 +1,6 @@
 Changes
 lib/Search/ESsearcher.pm
+lib/Search/ESsearcher/Templates/httpAccess.pm
 lib/Search/ESsearcher/Templates/syslog.pm
 lib/Search/ESsearcher/Templates/bf2b.pm
 Makefile.PL

Search-ESsearcher/bin/essearcher

@@ -12,9 +12,10 @@ use warnings;
 use Search::ESsearcher;
 use Getopt::Long qw(:config pass_through);
 use Data::Dumper;
+use JSON;
 
 sub version{
-	print "essearch: 0.0.1\n";
+	print "essearch: 0.1.0\n";
 };
 
 # disable color if asked
@@ -35,6 +36,7 @@ my $help;
 my $warn;
 my $critical;
 my $check;
+my $pretty;
 GetOptions(
 		   's=s' => \$search,
 		   'g=s' => \$options,
@@ -49,6 +51,7 @@ GetOptions(
 		   'n=s' => \$check,
 		   'w=s' => \$warn,
 		   'c=s' => \$critical,
+		   'p' => \$pretty,
 		   );
 
 # if -n is set, make sure we have -w and -c
@@ -100,6 +103,7 @@ if ( $help ){
              Any of the above being set will override this.
 -e <elastic> The elasticsearch config to use.
 -S           Print the search out after filling it in and exit.
+-p           Print the search JSON prettily.
 -R           Run the search and print it via Data::Dumper.
 -i           Invert the results.
 -n <check>   Operate as a nagios style check.
@@ -138,6 +142,17 @@ $ess->search_set( $search );
 $ess->load_search;
 my $filled_in=$ess->search_fill_in;
 if ( $print_search ){
+	#clean it up if requested
+	if ( $pretty ){
+#		eval{
+			my $json=JSON->new;
+			$json->pretty(1);
+			$json->relaxed(1);
+			$json->canonical(1);
+			my $decoded=$json->decode( $filled_in );
+			$filled_in=$json->encode($decoded);
+#		}
+	}
 	print $filled_in;
 	exit 255;
 }
@@ -256,6 +271,14 @@ Any of the above being set will override this.
 
 The elasticsearch config to use.
 
+=head2 -p
+
+If -S is given, then it will attempt to pretty print the JSON.
+
+This will involve parsing it and then turning it back into JSON.
+
+So it will spit out ugly JSON is malformed JSON is fed into it.
+
 =head2 -S
 
 Print the search out after filling it in and exit.
@@ -307,4 +330,8 @@ Print the help.
 
 If this is set, it disables color output.
 
+=head1 NOTE
+
+This script has dynamic command line options.
+
 =cut

Search-ESsearcher/lib/Search/ESsearcher.pm

@@ -17,11 +17,11 @@ Search::ESsearcher - Provides a handy system for doing templated elasticsearch s
 
 =head1 VERSION
 
-Version 0.1.0
+Version 0.2.0
 
 =cut
 
-our $VERSION = '0.1.0';
+our $VERSION = '0.2.0';
 
 
 =head1 SYNOPSIS
@@ -897,6 +897,19 @@ sub search_run{
 		$results=$self->{es}->search( $self->{search_hash} );
 	};
 
+	# @timestamp can't be handled via 
+		if (
+		( ref( $results ) eq 'HASH' ) ||
+		( defined( $results->{hits} ) )||
+		( defined( $results->{hits}{hits} ) )
+		){
+			foreach my $item ( @{ $results->{hits}{hits} } ){
+				if (!defined( $item->{'_source'}{'timestamp'})  )  {
+					$item->{'_source'}{'timestamp'}=$item->{'_source'}{'@timestamp'}
+				}
+			}
+	}
+
 	return $results;
 }
 

Search-ESsearcher/lib/Search/ESsearcher/Templates/bf2b.pm

@@ -225,7 +225,7 @@ return '
 					  }
 					   },
 					  [% END %]
-					  [% IF o.clientip %]
+					  [% IF o.ip %]
 					  {"query_string": {
 						  "default_field": "clientip",
 						  "query": [% aon( o.ip ).json %]
@@ -323,22 +323,27 @@ sub help{
 
 
 
+--status <status>    The status value of the message.
 --host <log host>    The system beats in question is running on.
---country <country>  The 2 letter country code.
 --jail <jail>        The fail2ban jail in question.
+--ip <ip>            The IP to search for.
+
+--country <country>  The 2 letter country code.
 --region <state>     The state/province/etc to search for.
 --postal <zipcode>   The postal code to search for.
 --city <cide>        The city to search for.
---ip <ip>            The IP to search for.
---dgt <date>          Date greater than.
---dgte <date>         Date greater than or equal to.
---dlt <date>          Date less than.
---dlte <date>         Date less than or equal to.
---msg <message>       Messages to match.
---field <field>       The term field to use for matching them all.
---field2 <field2>     The term field to use for what beats is setting.
---fieldv <fieldv>     The value of the term field to matching them all.
---field2v <field2v>   The value to look for in the field beats is setting.
+
+--dgt <date>         Date greater than.
+--dgte <date>        Date greater than or equal to.
+--dlt <date>         Date less than.
+--dlte <date>        Date less than or equal to.
+
+--msg <message>      Messages to match.
+
+--field <field>      The term field to use for matching them all.
+--field2 <field2>    The term field to use for what beats is setting.
+--fieldv <fieldv>    The value of the term field to matching them all.
+--field2v <field2v>  The value to look for in the field beats is setting.
 
 
 AND, OR, or NOT shortcut

Search-ESsearcher/lib/Search/ESsearcher/Templates/syslog.pm

@@ -264,11 +264,14 @@ sub help{
 --facility <facility> The syslog facility.
 --severity <severity> The severity level of the message.
 --pid <pid>           The PID that sent the message.
+
 --dgt <date>          Date greater than.
 --dgte <date>         Date greater than or equal to.
 --dlt <date>          Date less than.
 --dlte <date>         Date less than or equal to.
+
 --msg <message>       Messages to match.
+
 --field <field>       The term field to use for matching them all.
 --fieldv <fieldv>     The value of the term field to matching them all.
 

Search-ESsearcher/t/03-load.t

@@ -0,0 +1,13 @@
+#!perl -T
+use 5.006;
+use strict;
+use warnings;
+use Test::More;
+
+plan tests => 1;
+
+BEGIN {
+    use_ok( 'Search::ESsearcher::Templates::httpAccess' ) || print "Bail out!\n";
+}
+
+diag( "Testing Search::ESsearcher::Templates::httpAccess $Search::ESsearcher::Templates::httpAccess::VERSION, Perl $], $^X" );