misc...
This commit is contained in:
parent
a9d7fb9713
commit
defc8b0dff
|
@ -25,6 +25,7 @@ my %WriteMakefileArgs = (
|
|||
'Template::Plugin::JSON' => '0.08',
|
||||
'Time::ParseDate' => '2015.103',
|
||||
'Term::ANSIColor' => '4.06',
|
||||
'Data::Dumper' => '2.173',
|
||||
},
|
||||
dist => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', },
|
||||
clean => { FILES => 'Search-ESsearcher-*' },
|
||||
|
|
|
@ -6,6 +6,10 @@ use Search::ESsearcher;
|
|||
use Getopt::Long qw(:config pass_through);
|
||||
use Data::Dumper;
|
||||
|
||||
sub version{
|
||||
print "essearch: 0.0.0\n";
|
||||
};
|
||||
|
||||
# set all the templates the servers use to to fault
|
||||
my $search;
|
||||
my $options;
|
||||
|
@ -15,6 +19,7 @@ my $module;
|
|||
my $invert;
|
||||
my $print_search;
|
||||
my $print_results;
|
||||
my $help;
|
||||
GetOptions(
|
||||
's=s' => \$search,
|
||||
'g=s' => \$options,
|
||||
|
@ -24,6 +29,8 @@ GetOptions(
|
|||
'R' => \$print_results,
|
||||
'm=s' => \$module,
|
||||
'i' => \$invert,
|
||||
'h' => \$help,
|
||||
'help' => \$help,
|
||||
);
|
||||
|
||||
# Use module as the base to use allowing
|
||||
|
@ -41,6 +48,28 @@ if (defined( $module )){
|
|||
}
|
||||
|
||||
my $ess = Search::ESsearcher->new();
|
||||
# print the help if asked to
|
||||
if ( $help ){
|
||||
&version;
|
||||
print '
|
||||
-s <search> The search template to use.
|
||||
-g <getopts> The getopts config to use.
|
||||
-o <output> Thee output template to use.
|
||||
-m <module> Set all of the above to the same.
|
||||
Any of the above being set will override this.
|
||||
-e <elastic> The elasticsearch config to use.
|
||||
-S Print the search out after filling it in and exit.
|
||||
-R Run the search and print it via Data::Dumper.
|
||||
-i Invert the results.
|
||||
-h Print the help.
|
||||
--help Print the help.
|
||||
|
||||
The printed help after this line varies based on the loaded
|
||||
search template. If one is found it will be printed.
|
||||
|
||||
'.$ess->fetch_help;
|
||||
exit 255;
|
||||
}
|
||||
|
||||
# reels in the options
|
||||
$ess->options_set( $options );
|
||||
|
|
|
@ -26,19 +26,11 @@ our $VERSION = '0.0.0';
|
|||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
Quick summary of what the module does.
|
||||
|
||||
Perhaps a little code snippet.
|
||||
|
||||
use Search::ESsearcher;
|
||||
|
||||
my $ess = Search::ESsearcher->new();
|
||||
|
||||
=head1 EXPORT
|
||||
|
||||
A list of functions that can be exported. You can delete this section
|
||||
if you don't export anything, such as for a purely object-oriented module.
|
||||
|
||||
=head1 METHODS
|
||||
|
||||
=head2 new
|
||||
|
@ -165,10 +157,81 @@ sub elastic_set{
|
|||
return 1;
|
||||
}
|
||||
|
||||
=head2 fetch_help
|
||||
|
||||
This fetches the help for the current search and returns it.
|
||||
Failsure to find one, results in a empty message being returned.
|
||||
|
||||
my $help=$ess->fetch_help;
|
||||
|
||||
=cut
|
||||
|
||||
sub fetch_help{
|
||||
my $self=$_[0];
|
||||
|
||||
if ( ! $self->errorblank ) {
|
||||
return undef;
|
||||
}
|
||||
|
||||
my $file=undef;
|
||||
my $data=undef;
|
||||
|
||||
# ~/ -> etc -> module -> error
|
||||
if (
|
||||
( defined( $ENV{'HOME'} ) ) &&
|
||||
( -f $ENV{'HOME'}.'/.config/essearcher/help/'.$self->{search} )
|
||||
) {
|
||||
$file=$ENV{'HOME'}.'/.config/essearcher/help/'.$self->{search};
|
||||
} elsif (
|
||||
( defined( $self->{base} ) ) &&
|
||||
( -f $self->{base}.'/etc/essearcher/help/'.$self->{search} )
|
||||
) {
|
||||
$file=$self->{base}.'/etc/essearcher/help/'.$self->{search};
|
||||
} else {
|
||||
# do a quick check of making sure we have a valid name before trying a module...
|
||||
# not all valid names are perl module name valid, but it will prevent arbitrary code execution
|
||||
if ( $self->name_validate( $self->{options} ) ) {
|
||||
my $to_eval='use Search::ESsearcher::Templates::'.$self->{search}.
|
||||
'; $data=Search::ESsearcher::Templates::'.$self->{search}.'->help;';
|
||||
eval( $to_eval );
|
||||
}
|
||||
# if undefined, it means the eval failed
|
||||
if ( ! defined( $data ) ) {
|
||||
$self->{error}=2;
|
||||
$self->{errorString}='No help file with the name "'.$self->{search}.'" was found';
|
||||
$self->warn;
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
if ( ! defined( $data ) ) {
|
||||
my $fh;
|
||||
if (! open($fh, '<', $file ) ) {
|
||||
$self->{error}=1;
|
||||
$self->{errorString}='Failed to open "'.$file.'"',
|
||||
$self->warn;
|
||||
return '';
|
||||
}
|
||||
# if it is larger than 2M bytes, something is wrong as the template
|
||||
# it takes is literally longer than all HHGTTG books combined
|
||||
if (! read($fh, $data, 200000000 )) {
|
||||
$self->{error}=1;
|
||||
$self->{errorString}='Failed to read "'.$file.'"',
|
||||
$self->warn;
|
||||
return '';
|
||||
}
|
||||
close($fh);
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
=head2 get_options
|
||||
|
||||
This fetches the options.
|
||||
This fetches the options for use later
|
||||
when filling in the search template.
|
||||
|
||||
$ess->get_options;
|
||||
|
||||
=cut
|
||||
|
||||
|
@ -191,6 +254,10 @@ sub get_options{
|
|||
|
||||
=head2 load_options
|
||||
|
||||
This loads the currently set options.
|
||||
|
||||
$ess->load_options;
|
||||
|
||||
=cut
|
||||
|
||||
sub load_options{
|
||||
|
@ -261,7 +328,12 @@ sub load_options{
|
|||
return 1;
|
||||
}
|
||||
|
||||
=head2 load_output
|
||||
=head2 load_elastic
|
||||
|
||||
This loads the currently specified config file
|
||||
containing the Elasticsearch config JSON.
|
||||
|
||||
$ess->load_elastic;
|
||||
|
||||
=cut
|
||||
|
||||
|
@ -340,6 +412,13 @@ sub load_elastic{
|
|||
|
||||
=head2 load_output
|
||||
|
||||
This loads the currently specified output template.
|
||||
|
||||
While this is save internally, the template is also
|
||||
returned as a string.
|
||||
|
||||
my $outpot_template=$ess->load_output;
|
||||
|
||||
=cut
|
||||
|
||||
sub load_output{
|
||||
|
@ -376,7 +455,7 @@ sub load_output{
|
|||
$self->{error}=2;
|
||||
$self->{errorString}='No options file with the name "'.$self->{output}.'" was found';
|
||||
$self->warn;
|
||||
return undef;
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -386,7 +465,7 @@ sub load_output{
|
|||
$self->{error}=1;
|
||||
$self->{errorString}='Failed to open "'.$file.'"',
|
||||
$self->warn;
|
||||
return undef;
|
||||
return '';
|
||||
}
|
||||
# if it is larger than 2M bytes, something is wrong as the template
|
||||
# it takes is literally longer than all HHGTTG books combined
|
||||
|
@ -394,7 +473,7 @@ sub load_output{
|
|||
$self->{error}=1;
|
||||
$self->{errorString}='Failed to read "'.$file.'"',
|
||||
$self->warn;
|
||||
return undef;
|
||||
return '';
|
||||
}
|
||||
close($fh);
|
||||
}
|
||||
|
@ -402,10 +481,18 @@ sub load_output{
|
|||
# we have now completed with out error, so save it
|
||||
$self->{output_template}=$data;
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
=head2 load_search
|
||||
|
||||
This loads the currently specified search template.
|
||||
|
||||
While this is save internally, the template is also
|
||||
returned as a string.
|
||||
|
||||
my $search_template=$ess->load_search;
|
||||
|
||||
=cut
|
||||
|
||||
sub load_search{
|
||||
|
@ -473,6 +560,17 @@ sub load_search{
|
|||
|
||||
=head2 name_valide
|
||||
|
||||
This validates a config name.
|
||||
|
||||
One option is taken and that is the name to valid.
|
||||
|
||||
The returned value is a perl boolean based on if it
|
||||
it is valid or not.
|
||||
|
||||
if ( ! $ess->name_validate( $name ) ){
|
||||
print "Name is not valid.\n";
|
||||
}
|
||||
|
||||
=cut
|
||||
|
||||
sub name_validate{
|
||||
|
@ -498,6 +596,11 @@ sub name_validate{
|
|||
|
||||
=head options_get
|
||||
|
||||
This returns the currently set options
|
||||
config name.
|
||||
|
||||
my $options=$ess->options_get;
|
||||
|
||||
=cut
|
||||
|
||||
sub options_get{
|
||||
|
@ -512,6 +615,13 @@ sub options_get{
|
|||
|
||||
=head options_set
|
||||
|
||||
This sets the options config name to use.
|
||||
|
||||
One option is taken and this is the config name.
|
||||
If it is undefiend, then the default is used.
|
||||
|
||||
$ess->options_set( $name );
|
||||
|
||||
=cut
|
||||
|
||||
sub options_set{
|
||||
|
@ -540,6 +650,11 @@ sub options_set{
|
|||
|
||||
=head output_get
|
||||
|
||||
This returns the currently set output
|
||||
template name.
|
||||
|
||||
my $output=$ess->output_get;
|
||||
|
||||
=cut
|
||||
|
||||
sub output_get{
|
||||
|
@ -555,6 +670,14 @@ sub output_get{
|
|||
|
||||
=head output_set
|
||||
|
||||
|
||||
This sets the output template name to use.
|
||||
|
||||
One option is taken and this is the template name.
|
||||
If it is undefiend, then the default is used.
|
||||
|
||||
$ess->output_set( $name );
|
||||
|
||||
=cut
|
||||
|
||||
sub output_set{
|
||||
|
@ -583,6 +706,18 @@ sub output_set{
|
|||
|
||||
=head2 results_process
|
||||
|
||||
This processes the results from search_run.
|
||||
|
||||
One option is taken and that is the return from search_run.
|
||||
|
||||
The returned value from this is array of each document found
|
||||
after it has been formated using the set output template.
|
||||
|
||||
my $results=$ess->search_run;
|
||||
my @formated=$ess->results_process( $results );
|
||||
@formated=reverse(@formated);
|
||||
print join("\n", @formated)."\n";
|
||||
|
||||
=cut
|
||||
|
||||
sub results_process{
|
||||
|
@ -650,6 +785,12 @@ sub results_process{
|
|||
|
||||
=head search_get
|
||||
|
||||
This returns the currently set search
|
||||
template name.
|
||||
|
||||
my $search=$ess->search_get;
|
||||
|
||||
|
||||
=cut
|
||||
|
||||
sub search_get{
|
||||
|
@ -665,6 +806,12 @@ sub search_get{
|
|||
|
||||
=head2 search_fill_in
|
||||
|
||||
This fills in the loaded search template.
|
||||
|
||||
The results are saved internally as well as returned.
|
||||
|
||||
my $filled_in=$ess->search_fill_in;
|
||||
|
||||
=cut
|
||||
|
||||
sub search_fill_in{
|
||||
|
@ -727,6 +874,14 @@ sub search_fill_in{
|
|||
|
||||
=head2 search_run
|
||||
|
||||
This is used to run the search after search_fill_in
|
||||
has been called.
|
||||
|
||||
The returned value is of the type that would be returned
|
||||
by L<Search::Elasticsearch>->search.
|
||||
|
||||
my $results=$ess->search_run;
|
||||
|
||||
=cut
|
||||
|
||||
sub search_run{
|
||||
|
@ -747,6 +902,13 @@ sub search_run{
|
|||
|
||||
=head search_set
|
||||
|
||||
This sets the search template name to use.
|
||||
|
||||
One option is taken and this is the template name.
|
||||
If it is undefiend, then the default is used.
|
||||
|
||||
$ess->search_sets( $name );
|
||||
|
||||
=cut
|
||||
|
||||
sub search_set{
|
||||
|
@ -773,6 +935,205 @@ sub search_set{
|
|||
return 1;
|
||||
}
|
||||
|
||||
=head1 Configuration And Usage
|
||||
|
||||
Configs, help, and templates are looked for in the following manner and order,
|
||||
with the following of the elasticsearch config.
|
||||
|
||||
$ENV{HOME}."/.config/essearcher/".$item."/".$name
|
||||
$base.'/etc/essearcher/".$item."/".$name
|
||||
Search::ESsearcher::Templates::$name->$item
|
||||
ERROR
|
||||
|
||||
Item can be any of the following.
|
||||
|
||||
elastic
|
||||
help
|
||||
output
|
||||
options
|
||||
search
|
||||
|
||||
The basic idea is you have matching output, options
|
||||
and search that you can use to perform queries and
|
||||
print the results.
|
||||
|
||||
Each template/config is its own file under the directory
|
||||
named after its purpose. So the options template fail2ban
|
||||
would be 'options/fail2ban'.
|
||||
|
||||
=head2 elastic
|
||||
|
||||
This directory contains JSON formatted config files
|
||||
for use with connecting to the Elasticsearch server.
|
||||
|
||||
This is then read in and converted to a hash and feed
|
||||
to L<Search::Elasticsearch>->new.
|
||||
|
||||
By default it will attempt to connect to it on
|
||||
"127.0.0.1:9200". The JSON equivalent would be...
|
||||
|
||||
{ "nodes": [ "127.0.0.1:9200" ] }
|
||||
|
||||
=head2 options
|
||||
|
||||
This is a file that will be used as a string for with
|
||||
L<Getopt::Long>. They will be passed to the templates
|
||||
as a hash.
|
||||
|
||||
=head2 help
|
||||
|
||||
This contains information on the options the search uses.
|
||||
|
||||
This is just a text file containing information and is not
|
||||
required.
|
||||
|
||||
If you are writing a module, it should definitely be present.
|
||||
|
||||
=head2 search
|
||||
|
||||
This is a L<Template> template that will be filled in using
|
||||
the data from the passed command line options and used
|
||||
to run the search.
|
||||
|
||||
The end result should be valid JSON that can be turned
|
||||
into a hash for feeding L<Search::Elasticsearch>->search.
|
||||
|
||||
When writing search templates, it is highly suggested
|
||||
to use L<Template::Plugin::JSON> for when inserting variables
|
||||
as it will automatically escape them.
|
||||
|
||||
=head2 output
|
||||
|
||||
This is a L<Template> template that will be filled in using
|
||||
the data from the passed command line options and the returned
|
||||
results.
|
||||
|
||||
It will be used for each returned document. bin/essearcher will
|
||||
then join the array with "\n".
|
||||
|
||||
=head1 TEMPLATES
|
||||
|
||||
=head2 o
|
||||
|
||||
This is a hash that contains the parsed options.
|
||||
|
||||
Below is a example with the option --program and
|
||||
transforming it a JSON save value.
|
||||
|
||||
[% USE JSON ( pretty => 1 ) %]
|
||||
[% DEFAULT o.program = "*" %]
|
||||
|
||||
{"query_string": {
|
||||
"default_field": "program",
|
||||
"query": [% o.program.json %]
|
||||
}
|
||||
},
|
||||
|
||||
=head2 aon
|
||||
|
||||
This is AND, OR, or NOT sub that handles
|
||||
the following in a string, transforming them
|
||||
from the punctuation to the logic.
|
||||
|
||||
, OR
|
||||
+ AND
|
||||
! NOT
|
||||
|
||||
So the string "postfix,spamd" would become
|
||||
"postfix OR spamd".
|
||||
|
||||
Can be used like below.
|
||||
|
||||
[% USE JSON ( pretty => 1 ) %]
|
||||
[% DEFAULT o.program = "*" %]
|
||||
|
||||
{"query_string": {
|
||||
"default_field": "program",
|
||||
"query": [% aon( o.program ).json %]
|
||||
}
|
||||
},
|
||||
|
||||
This function is only available for the search template.
|
||||
|
||||
=head2 c
|
||||
|
||||
This wraps L<Term::ANSIColor>->color.
|
||||
|
||||
[% c("cyan") %][% f.timestamp %] [% c("bright_blue") %][% f.logsource %]
|
||||
|
||||
This function is only available for the output template.
|
||||
|
||||
=head2 pd
|
||||
|
||||
This is a time helper.
|
||||
|
||||
/^-/ appends "now" to it. So "-5m" becomes "now-5m".
|
||||
|
||||
/^u\:/ takes what is after ":" and uses Time::ParseDate to convert
|
||||
it to a unix time value.
|
||||
|
||||
Any thing not matching maching any of the above will just be passed on.
|
||||
|
||||
[% IF o.dgt %]
|
||||
{"range": {
|
||||
"@timestamp": {
|
||||
"gt": [% pd( o.dgt ).json %]
|
||||
}
|
||||
}
|
||||
},
|
||||
[% END %]
|
||||
|
||||
|
||||
=head1 Modules
|
||||
|
||||
Additonal modules bundling help, options, search, and output
|
||||
can be made. The requirement for these are as below.
|
||||
|
||||
They need to exist below Search::ESsearcher::Templates.
|
||||
|
||||
Provide the following functions that return strings.
|
||||
|
||||
help
|
||||
options
|
||||
search
|
||||
output
|
||||
|
||||
Basic information as to what is required to make it work in logstash
|
||||
or the like is also good as well.
|
||||
|
||||
=head1 ERROR CODES/FLAGS
|
||||
|
||||
All error handling is done via L<Error::Helper>.
|
||||
|
||||
=head2 1 / IOerror
|
||||
|
||||
Failed to perform some sort of file operation.
|
||||
|
||||
=head2 2 / NOfile
|
||||
|
||||
The specified template/config does not exist.
|
||||
|
||||
=head2 3 / nameIsInvalid
|
||||
|
||||
Invalid name specified.
|
||||
|
||||
=head2 4 / searchNotUsable
|
||||
|
||||
Errored while processing the template.
|
||||
|
||||
For more information on writing templates, see L<Template>.
|
||||
|
||||
=head2 5 / elasticnotloadable
|
||||
|
||||
The Elasticsearch config does not parse as JSON, preventing
|
||||
it from being loaded.
|
||||
|
||||
=head2 6 / notResults
|
||||
|
||||
The return value passed to results_process deos not appear to
|
||||
be a results return. Most likely the search errored and returned
|
||||
undef or a blank value.
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
Zane C. Bowers-Hadley, C<< <vvelox at vvelox.net> >>
|
||||
|
|
|
@ -6,7 +6,7 @@ use warnings;
|
|||
|
||||
=head1 NAME
|
||||
|
||||
Search::ESsearcher::Templates::syslog - Provides a basic syslog template
|
||||
Search::ESsearcher::Templates::syslog - Provides syslog support for essearcher.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
|
@ -16,15 +16,6 @@ Version 0.0.0
|
|||
|
||||
our $VERSION = '0.0.0';
|
||||
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
use Search::ESsearcher::Templates::syslog;
|
||||
|
||||
my $options = Search::ESsearcher::Templates::syslog->options;
|
||||
my $search = Search::ESsearcher::Templates::syslog->search;
|
||||
my $output = Search::ESsearcher::Templates::syslog->output;
|
||||
|
||||
=head1 LOGSTASH
|
||||
|
||||
This uses a logstash configuration below.
|
||||
|
@ -50,6 +41,89 @@ This uses a logstash configuration below.
|
|||
The important bit is "type" being set to "syslog". If that is not used,
|
||||
use the command line options field and fieldv.
|
||||
|
||||
=head1 Options
|
||||
|
||||
=head2 --host <log host>
|
||||
|
||||
The syslog server.
|
||||
|
||||
=head2 --src <src server>
|
||||
|
||||
The source server sending to the syslog server.
|
||||
|
||||
=head2 --program <program>
|
||||
|
||||
The name of the daemon/program in question.
|
||||
|
||||
=head2 --size <count>
|
||||
|
||||
The number of items to return.
|
||||
|
||||
=head2 --facility <facility>
|
||||
|
||||
The syslog facility.
|
||||
|
||||
=head2 --severity <severity>
|
||||
|
||||
The severity level of the message.
|
||||
|
||||
=head2 --pid <pid>
|
||||
|
||||
The PID that sent the message.
|
||||
|
||||
=head2 --dgt <date>
|
||||
|
||||
Date greater than.
|
||||
|
||||
=head2 --dgte <date>
|
||||
|
||||
Date greater than or equal to.
|
||||
|
||||
=head2 --dlt <date>
|
||||
|
||||
Date less than.
|
||||
|
||||
=head2 --dlte <date>
|
||||
|
||||
Date less than or equal to.
|
||||
|
||||
=head2 --msg <message>
|
||||
|
||||
Messages to match.
|
||||
|
||||
=head2 --field <field>
|
||||
|
||||
The term field to use for matching them all.
|
||||
|
||||
=head2 --fieldv <fieldv>
|
||||
|
||||
The value of the term field to matching them all.
|
||||
|
||||
=head1 AND, OR, or NOT shortcut
|
||||
|
||||
, OR
|
||||
+ AND
|
||||
! NOT
|
||||
|
||||
A list seperated by any of those will be transformed
|
||||
|
||||
These may be used with program, facility, pid, or host.
|
||||
|
||||
example: --program postfix,spamd
|
||||
|
||||
results: postfix OR spamd
|
||||
|
||||
=head1 date
|
||||
|
||||
date
|
||||
|
||||
/^-/ appends "now" to it. So "-5m" becomes "now-5m".
|
||||
|
||||
/^u\:/ takes what is after ":" and uses Time::ParseDate to convert it to a
|
||||
unix time value.
|
||||
|
||||
Any thing not matching maching any of the above will just be passed on.
|
||||
|
||||
=cut
|
||||
|
||||
|
||||
|
@ -183,20 +257,20 @@ sub output{
|
|||
sub help{
|
||||
return '
|
||||
|
||||
host <log host> The syslog server.
|
||||
src <src server> The source server sending to the syslog server.
|
||||
program <program> The name of the daemon/program in question.
|
||||
size <count> The number of items to return.
|
||||
facility <facility> The syslog facility.
|
||||
severity <severity> The severity level of the message.
|
||||
pid <pid> The PID that sent the message.
|
||||
dgt <date> Date greater than.
|
||||
dgte <date> Date greater than or equal to.
|
||||
dlt <date> Date less than.
|
||||
dlte <date> Date less than or equal to.
|
||||
msg <message> Messages to match.
|
||||
field <field> The term field to use for matching them all.
|
||||
fieldv <fieldv> The value of the term field to matching them all.
|
||||
--host <log host> The syslog server.
|
||||
--src <src server> The source server sending to the syslog server.
|
||||
--program <program> The name of the daemon/program in question.
|
||||
--size <count> The number of items to return.
|
||||
--facility <facility> The syslog facility.
|
||||
--severity <severity> The severity level of the message.
|
||||
--pid <pid> The PID that sent the message.
|
||||
--dgt <date> Date greater than.
|
||||
--dgte <date> Date greater than or equal to.
|
||||
--dlt <date> Date less than.
|
||||
--dlte <date> Date less than or equal to.
|
||||
--msg <message> Messages to match.
|
||||
--field <field> The term field to use for matching them all.
|
||||
--fieldv <fieldv> The value of the term field to matching them all.
|
||||
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue