This commit is contained in:
Zane C. B-H 2019-06-02 02:19:47 -05:00
parent a9d7fb9713
commit defc8b0dff
4 changed files with 502 additions and 37 deletions

View File

@ -25,6 +25,7 @@ my %WriteMakefileArgs = (
'Template::Plugin::JSON' => '0.08',
'Time::ParseDate' => '2015.103',
'Term::ANSIColor' => '4.06',
'Data::Dumper' => '2.173',
},
dist => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', },
clean => { FILES => 'Search-ESsearcher-*' },

View File

@ -6,6 +6,10 @@ use Search::ESsearcher;
use Getopt::Long qw(:config pass_through);
use Data::Dumper;
sub version{
print "essearch: 0.0.0\n";
};
# set all the templates the servers use to to fault
my $search;
my $options;
@ -15,6 +19,7 @@ my $module;
my $invert;
my $print_search;
my $print_results;
my $help;
GetOptions(
's=s' => \$search,
'g=s' => \$options,
@ -24,6 +29,8 @@ GetOptions(
'R' => \$print_results,
'm=s' => \$module,
'i' => \$invert,
'h' => \$help,
'help' => \$help,
);
# Use module as the base to use allowing
@ -41,6 +48,28 @@ if (defined( $module )){
}
my $ess = Search::ESsearcher->new();
# print the help if asked to
if ( $help ){
&version;
print '
-s <search> The search template to use.
-g <getopts> The getopts config to use.
-o <output> Thee output template to use.
-m <module> Set all of the above to the same.
Any of the above being set will override this.
-e <elastic> The elasticsearch config to use.
-S Print the search out after filling it in and exit.
-R Run the search and print it via Data::Dumper.
-i Invert the results.
-h Print the help.
--help Print the help.
The printed help after this line varies based on the loaded
search template. If one is found it will be printed.
'.$ess->fetch_help;
exit 255;
}
# reels in the options
$ess->options_set( $options );

View File

@ -26,19 +26,11 @@ our $VERSION = '0.0.0';
=head1 SYNOPSIS
Quick summary of what the module does.
Perhaps a little code snippet.
use Search::ESsearcher;
my $ess = Search::ESsearcher->new();
=head1 EXPORT
A list of functions that can be exported. You can delete this section
if you don't export anything, such as for a purely object-oriented module.
=head1 METHODS
=head2 new
@ -165,10 +157,81 @@ sub elastic_set{
return 1;
}
=head2 fetch_help
This fetches the help for the current search and returns it.
Failsure to find one, results in a empty message being returned.
my $help=$ess->fetch_help;
=cut
sub fetch_help{
my $self=$_[0];
if ( ! $self->errorblank ) {
return undef;
}
my $file=undef;
my $data=undef;
# ~/ -> etc -> module -> error
if (
( defined( $ENV{'HOME'} ) ) &&
( -f $ENV{'HOME'}.'/.config/essearcher/help/'.$self->{search} )
) {
$file=$ENV{'HOME'}.'/.config/essearcher/help/'.$self->{search};
} elsif (
( defined( $self->{base} ) ) &&
( -f $self->{base}.'/etc/essearcher/help/'.$self->{search} )
) {
$file=$self->{base}.'/etc/essearcher/help/'.$self->{search};
} else {
# do a quick check of making sure we have a valid name before trying a module...
# not all valid names are perl module name valid, but it will prevent arbitrary code execution
if ( $self->name_validate( $self->{options} ) ) {
my $to_eval='use Search::ESsearcher::Templates::'.$self->{search}.
'; $data=Search::ESsearcher::Templates::'.$self->{search}.'->help;';
eval( $to_eval );
}
# if undefined, it means the eval failed
if ( ! defined( $data ) ) {
$self->{error}=2;
$self->{errorString}='No help file with the name "'.$self->{search}.'" was found';
$self->warn;
return '';
}
}
if ( ! defined( $data ) ) {
my $fh;
if (! open($fh, '<', $file ) ) {
$self->{error}=1;
$self->{errorString}='Failed to open "'.$file.'"',
$self->warn;
return '';
}
# if it is larger than 2M bytes, something is wrong as the template
# it takes is literally longer than all HHGTTG books combined
if (! read($fh, $data, 200000000 )) {
$self->{error}=1;
$self->{errorString}='Failed to read "'.$file.'"',
$self->warn;
return '';
}
close($fh);
}
return $data;
}
=head2 get_options
This fetches the options.
This fetches the options for use later
when filling in the search template.
$ess->get_options;
=cut
@ -191,6 +254,10 @@ sub get_options{
=head2 load_options
This loads the currently set options.
$ess->load_options;
=cut
sub load_options{
@ -261,7 +328,12 @@ sub load_options{
return 1;
}
=head2 load_output
=head2 load_elastic
This loads the currently specified config file
containing the Elasticsearch config JSON.
$ess->load_elastic;
=cut
@ -340,6 +412,13 @@ sub load_elastic{
=head2 load_output
This loads the currently specified output template.
While this is save internally, the template is also
returned as a string.
my $outpot_template=$ess->load_output;
=cut
sub load_output{
@ -376,7 +455,7 @@ sub load_output{
$self->{error}=2;
$self->{errorString}='No options file with the name "'.$self->{output}.'" was found';
$self->warn;
return undef;
return '';
}
}
@ -386,7 +465,7 @@ sub load_output{
$self->{error}=1;
$self->{errorString}='Failed to open "'.$file.'"',
$self->warn;
return undef;
return '';
}
# if it is larger than 2M bytes, something is wrong as the template
# it takes is literally longer than all HHGTTG books combined
@ -394,7 +473,7 @@ sub load_output{
$self->{error}=1;
$self->{errorString}='Failed to read "'.$file.'"',
$self->warn;
return undef;
return '';
}
close($fh);
}
@ -402,10 +481,18 @@ sub load_output{
# we have now completed with out error, so save it
$self->{output_template}=$data;
return $data;
}
=head2 load_search
This loads the currently specified search template.
While this is save internally, the template is also
returned as a string.
my $search_template=$ess->load_search;
=cut
sub load_search{
@ -473,6 +560,17 @@ sub load_search{
=head2 name_valide
This validates a config name.
One option is taken and that is the name to valid.
The returned value is a perl boolean based on if it
it is valid or not.
if ( ! $ess->name_validate( $name ) ){
print "Name is not valid.\n";
}
=cut
sub name_validate{
@ -498,6 +596,11 @@ sub name_validate{
=head options_get
This returns the currently set options
config name.
my $options=$ess->options_get;
=cut
sub options_get{
@ -512,6 +615,13 @@ sub options_get{
=head options_set
This sets the options config name to use.
One option is taken and this is the config name.
If it is undefiend, then the default is used.
$ess->options_set( $name );
=cut
sub options_set{
@ -540,6 +650,11 @@ sub options_set{
=head output_get
This returns the currently set output
template name.
my $output=$ess->output_get;
=cut
sub output_get{
@ -555,6 +670,14 @@ sub output_get{
=head output_set
This sets the output template name to use.
One option is taken and this is the template name.
If it is undefiend, then the default is used.
$ess->output_set( $name );
=cut
sub output_set{
@ -583,6 +706,18 @@ sub output_set{
=head2 results_process
This processes the results from search_run.
One option is taken and that is the return from search_run.
The returned value from this is array of each document found
after it has been formated using the set output template.
my $results=$ess->search_run;
my @formated=$ess->results_process( $results );
@formated=reverse(@formated);
print join("\n", @formated)."\n";
=cut
sub results_process{
@ -650,6 +785,12 @@ sub results_process{
=head search_get
This returns the currently set search
template name.
my $search=$ess->search_get;
=cut
sub search_get{
@ -665,6 +806,12 @@ sub search_get{
=head2 search_fill_in
This fills in the loaded search template.
The results are saved internally as well as returned.
my $filled_in=$ess->search_fill_in;
=cut
sub search_fill_in{
@ -727,6 +874,14 @@ sub search_fill_in{
=head2 search_run
This is used to run the search after search_fill_in
has been called.
The returned value is of the type that would be returned
by L<Search::Elasticsearch>->search.
my $results=$ess->search_run;
=cut
sub search_run{
@ -747,6 +902,13 @@ sub search_run{
=head search_set
This sets the search template name to use.
One option is taken and this is the template name.
If it is undefiend, then the default is used.
$ess->search_sets( $name );
=cut
sub search_set{
@ -773,6 +935,205 @@ sub search_set{
return 1;
}
=head1 Configuration And Usage
Configs, help, and templates are looked for in the following manner and order,
with the following of the elasticsearch config.
$ENV{HOME}."/.config/essearcher/".$item."/".$name
$base.'/etc/essearcher/".$item."/".$name
Search::ESsearcher::Templates::$name->$item
ERROR
Item can be any of the following.
elastic
help
output
options
search
The basic idea is you have matching output, options
and search that you can use to perform queries and
print the results.
Each template/config is its own file under the directory
named after its purpose. So the options template fail2ban
would be 'options/fail2ban'.
=head2 elastic
This directory contains JSON formatted config files
for use with connecting to the Elasticsearch server.
This is then read in and converted to a hash and feed
to L<Search::Elasticsearch>->new.
By default it will attempt to connect to it on
"127.0.0.1:9200". The JSON equivalent would be...
{ "nodes": [ "127.0.0.1:9200" ] }
=head2 options
This is a file that will be used as a string for with
L<Getopt::Long>. They will be passed to the templates
as a hash.
=head2 help
This contains information on the options the search uses.
This is just a text file containing information and is not
required.
If you are writing a module, it should definitely be present.
=head2 search
This is a L<Template> template that will be filled in using
the data from the passed command line options and used
to run the search.
The end result should be valid JSON that can be turned
into a hash for feeding L<Search::Elasticsearch>->search.
When writing search templates, it is highly suggested
to use L<Template::Plugin::JSON> for when inserting variables
as it will automatically escape them.
=head2 output
This is a L<Template> template that will be filled in using
the data from the passed command line options and the returned
results.
It will be used for each returned document. bin/essearcher will
then join the array with "\n".
=head1 TEMPLATES
=head2 o
This is a hash that contains the parsed options.
Below is a example with the option --program and
transforming it a JSON save value.
[% USE JSON ( pretty => 1 ) %]
[% DEFAULT o.program = "*" %]
{"query_string": {
"default_field": "program",
"query": [% o.program.json %]
}
},
=head2 aon
This is AND, OR, or NOT sub that handles
the following in a string, transforming them
from the punctuation to the logic.
, OR
+ AND
! NOT
So the string "postfix,spamd" would become
"postfix OR spamd".
Can be used like below.
[% USE JSON ( pretty => 1 ) %]
[% DEFAULT o.program = "*" %]
{"query_string": {
"default_field": "program",
"query": [% aon( o.program ).json %]
}
},
This function is only available for the search template.
=head2 c
This wraps L<Term::ANSIColor>->color.
[% c("cyan") %][% f.timestamp %] [% c("bright_blue") %][% f.logsource %]
This function is only available for the output template.
=head2 pd
This is a time helper.
/^-/ appends "now" to it. So "-5m" becomes "now-5m".
/^u\:/ takes what is after ":" and uses Time::ParseDate to convert
it to a unix time value.
Any thing not matching maching any of the above will just be passed on.
[% IF o.dgt %]
{"range": {
"@timestamp": {
"gt": [% pd( o.dgt ).json %]
}
}
},
[% END %]
=head1 Modules
Additonal modules bundling help, options, search, and output
can be made. The requirement for these are as below.
They need to exist below Search::ESsearcher::Templates.
Provide the following functions that return strings.
help
options
search
output
Basic information as to what is required to make it work in logstash
or the like is also good as well.
=head1 ERROR CODES/FLAGS
All error handling is done via L<Error::Helper>.
=head2 1 / IOerror
Failed to perform some sort of file operation.
=head2 2 / NOfile
The specified template/config does not exist.
=head2 3 / nameIsInvalid
Invalid name specified.
=head2 4 / searchNotUsable
Errored while processing the template.
For more information on writing templates, see L<Template>.
=head2 5 / elasticnotloadable
The Elasticsearch config does not parse as JSON, preventing
it from being loaded.
=head2 6 / notResults
The return value passed to results_process deos not appear to
be a results return. Most likely the search errored and returned
undef or a blank value.
=head1 AUTHOR
Zane C. Bowers-Hadley, C<< <vvelox at vvelox.net> >>

View File

@ -6,7 +6,7 @@ use warnings;
=head1 NAME
Search::ESsearcher::Templates::syslog - Provides a basic syslog template
Search::ESsearcher::Templates::syslog - Provides syslog support for essearcher.
=head1 VERSION
@ -16,15 +16,6 @@ Version 0.0.0
our $VERSION = '0.0.0';
=head1 SYNOPSIS
use Search::ESsearcher::Templates::syslog;
my $options = Search::ESsearcher::Templates::syslog->options;
my $search = Search::ESsearcher::Templates::syslog->search;
my $output = Search::ESsearcher::Templates::syslog->output;
=head1 LOGSTASH
This uses a logstash configuration below.
@ -50,6 +41,89 @@ This uses a logstash configuration below.
The important bit is "type" being set to "syslog". If that is not used,
use the command line options field and fieldv.
=head1 Options
=head2 --host <log host>
The syslog server.
=head2 --src <src server>
The source server sending to the syslog server.
=head2 --program <program>
The name of the daemon/program in question.
=head2 --size <count>
The number of items to return.
=head2 --facility <facility>
The syslog facility.
=head2 --severity <severity>
The severity level of the message.
=head2 --pid <pid>
The PID that sent the message.
=head2 --dgt <date>
Date greater than.
=head2 --dgte <date>
Date greater than or equal to.
=head2 --dlt <date>
Date less than.
=head2 --dlte <date>
Date less than or equal to.
=head2 --msg <message>
Messages to match.
=head2 --field <field>
The term field to use for matching them all.
=head2 --fieldv <fieldv>
The value of the term field to matching them all.
=head1 AND, OR, or NOT shortcut
, OR
+ AND
! NOT
A list seperated by any of those will be transformed
These may be used with program, facility, pid, or host.
example: --program postfix,spamd
results: postfix OR spamd
=head1 date
date
/^-/ appends "now" to it. So "-5m" becomes "now-5m".
/^u\:/ takes what is after ":" and uses Time::ParseDate to convert it to a
unix time value.
Any thing not matching maching any of the above will just be passed on.
=cut
@ -183,20 +257,20 @@ sub output{
sub help{
return '
host <log host> The syslog server.
src <src server> The source server sending to the syslog server.
program <program> The name of the daemon/program in question.
size <count> The number of items to return.
facility <facility> The syslog facility.
severity <severity> The severity level of the message.
pid <pid> The PID that sent the message.
dgt <date> Date greater than.
dgte <date> Date greater than or equal to.
dlt <date> Date less than.
dlte <date> Date less than or equal to.
msg <message> Messages to match.
field <field> The term field to use for matching them all.
fieldv <fieldv> The value of the term field to matching them all.
--host <log host> The syslog server.
--src <src server> The source server sending to the syslog server.
--program <program> The name of the daemon/program in question.
--size <count> The number of items to return.
--facility <facility> The syslog facility.
--severity <severity> The severity level of the message.
--pid <pid> The PID that sent the message.
--dgt <date> Date greater than.
--dgte <date> Date greater than or equal to.
--dlt <date> Date less than.
--dlte <date> Date less than or equal to.
--msg <message> Messages to match.
--field <field> The term field to use for matching them all.
--fieldv <fieldv> The value of the term field to matching them all.