jchkmail_counters_

#!/usr/bin/perl
# -*- perl -*-

=pod

=encoding UTF-8

=head1 NAME

  jchkmail_counters_

=head1 APPLICABLE SYSTEMS

  MTAs running j-chkmail mail filter - http://www.j-chkmail.org

=head1 CONFIGURATION

  [jchkmail_counters_*]
    user root

The following environment variables may be defined :

    # disable some instances of the plugin
    env.disable    no

=head1 BUGS/GOTCHAS

  None known, but some improvements to do...

=head1 AUTHOR

    Jose-Marcio Martins da Cruz - mailto:Jose-Marcio.Martins@mines-paristech.fr
    Ecole Nationale Superieure des Mines de Paris

  This plugin was inspired by the work of Christophe Decor
    Christophe Decor - mailto:decor@supagro.inra.fr
    INRA - Institut National de Recherche Agronomique

=head1 COPYRIGHT
  Copyright Jose-Marcio Martins da Cruz

=head1 VERSION

  1.0 - Feb 2014

=head1 MAGIC MARKERS

 #%# family=contrib
 #%# capabilities=autoconf suggest

=cut

use strict;
use warnings;

use lib $ENV{'MUNIN_LIBDIR'};
use Munin::Plugin;

my $SMCF     = "/etc/mail/jchkmail/j-chkmail.cf";
my $FSTATS   = "/var/jchkmail/files/j-stats";
my $WORKDIR  = "/var/jchkmail/wdb";
my $CONSTDIR = "/var/jchkmail/cdb";

# ------------------------------------------------
#
#
my $debug = 0;

my $AppName = $0;
$AppName = `basename $AppName`;
chomp $AppName;

my $AppExt = undef;
if ($AppName =~ /_([a-z0-9.-]+)$/i) {
  $AppExt = $1;
}

my $disable = defined $ENV{'disable'} &&  $ENV{'disable'} =~ /yes|oui/i;

my %StaticData = ();
GetStaticData(\%StaticData);

my $AppType = "";
if (defined $AppExt && exists $StaticData{config}{"$AppExt+type"}) {
  $AppType = $StaticData{config}{"$AppExt+type"};
}

# valider results from Munin::Plugin
# statefile is used only by jchkmail_counters_ratiospamham
my $StateFile = undef;
if (exists $ENV{MUNIN_PLUGSTATE} && defined $ENV{MUNIN_PLUGSTATE}) {
  $StateFile = "$ENV{MUNIN_PLUGSTATE}/$AppName.state-joe";
}
if (exists $ENV{statefile} && defined $ENV{statefile}) {
  $StateFile = $ENV{statefile};
}

# ------------------------------------------------
#
#
if ($#ARGV >= 0 && $ARGV[0] eq "autoconf") {

  unless (-f $SMCF) {
    print "no\n";
    exit 0;
  }
  print "yes\n";
  exit 0;
}

# ------------------------------------------------
#
#
if ($#ARGV >= 0 && $ARGV[0] eq "suggest") {

  # results from /var/jchkmail/files/j-stats
  if (-f $SMCF && -f $FSTATS) {
    my %Data = ();
    my $line = GetStatsLine(\%Data);
    exit 1 unless defined $line && $line ne "";

    foreach my $k (keys %{$StaticData{config}}) {
      next unless $k =~ /(.*)[+]type$/;
      my $graph = $1;
      my $type  = $StaticData{config}{$k};
      if ($type =~ /^(counters|ratio)$/) {
        next unless exists $StaticData{config}{$graph . "+data"};
        my $data = $StaticData{config}{$graph . "+data"};
        $data =~ tr#,/# #;
        my @fields = split " ", $data;
        my $ok     = 1;
        my $bad    = "";
        for my $f (@fields) {
          unless (exists $Data{$f}) {
            $ok = 0;
            $bad .= " $f";
          }
        }
        if ($ok) {
          print "$graph\n" if $ok;
        } else {
          printf "# %-12s - Lacking fields : %s\n", $graph, $bad;
        }
      }
    }
  }

  # results from /var/jchkmail/wdb
  if (-f $SMCF && (-d $WORKDIR || -d $CONSTDIR)) {
    foreach my $k (keys %{$StaticData{config}}) {
      next unless $k =~ /(.*)[+]type$/;
      my $graph = $1;
      my $type  = $StaticData{config}{$k};
      if ($type =~ /^(dbcounters)$/) {
        next unless exists $StaticData{config}{$graph . "+data"};
        my $data = $StaticData{config}{$graph . "+data"};
        $data =~ tr#,/# #;
        my @fields = split " ", $data;
        my $ok     = 1;
        my $bad    = "";
        for my $f (@fields) {
          unless (-f "$WORKDIR/$f.db" || -f "$CONSTDIR/$f.db") {
            $ok = 0;
            $bad .= " $f";
          }
        }
        if ($ok) {
          print "$graph\n" if $ok;
        } else {
          printf "# %-12s - Lacking fields : %s\n", $graph, $bad;
        }
      }
    }
  }
  exit 0;
}

# ------------------------------------------------
#
#
if (defined $AppExt && $AppType eq "counters") {
  my %Data = ();
  my $line = GetStatsLine(\%Data);
  exit 1 unless defined $line && $line ne "";

  my $DataAggregate = $StaticData{config}{"$AppExt+data"};
  printf "# Data Aggregate : %s\n", $DataAggregate;
  my @Values = split " ", $DataAggregate;
  if ($#ARGV >= 0 && $ARGV[0] eq "config") {
    my @Keys = grep {$_ =~ /^$AppExt-/;} keys %{$StaticData{config}};
    foreach my $k (@Keys) {
      my $v = $StaticData{config}{$k};
      $k =~ s/^$AppExt-//;
      printf "%-16s %s\n", $k, $v;
    }
    printf "graph_order %s\n", (join " ", sort @Values);
    foreach my $k (@Values) {
      if (exists $Data{$k}) {
        my $label = $k;
        if (exists $StaticData{label}{$k}) {
          $label = $StaticData{label}{$k};
        }
        printf "%s.label %s\n", $k, $label;
        if (exists $StaticData{label}{"$k.info"}) {
          $label = $StaticData{label}{"$k.info"};
        }
        printf "%s.info %s\n", $k, $label;
        printf "%s.min 0\n", $k;
        printf "%s.type DERIVE\n", $k;
      }
    }
    exit 0;
  }

  if ($#ARGV < 0) {
    foreach my $k (@Values) {
      if (exists $Data{$k}) {
        printf "%s.value %s\n", $k, $Data{$k};
      }
    }
    exit 0;
  }
  exit 1;
}

# ------------------------------------------------
#
#
if (defined $AppExt && $AppType eq "ratio") {
  my %Data = ();

  my $line = GetStatsLine(\%Data);
  exit 1 unless defined $line && $line ne "";

  my $DataAggregate = $StaticData{config}{"$AppExt+data"};
  printf "# Data Aggregate : %s\n", $DataAggregate;
  my ($Ratios, $Base, undef) = split "/", $DataAggregate;
  exit 1 unless defined $Ratios && defined $Base;

  my @Values = split ",",    $Ratios;
  my @Den    = split "[ ]+", $Base;

  if ($#ARGV >= 0 && $ARGV[0] eq "config") {
    my @Keys = grep {$_ =~ /^$AppExt-/;} keys %{$StaticData{config}};
    foreach my $k (@Keys) {
      my $v = $StaticData{config}{$k};
      $k =~ s/^$AppExt-//;
      printf "%-16s %s\n", $k, $v;
    }

    my $nr = 0;
    printf "graph_order %s\n", (join " ", sort @Values);
    foreach my $num (@Values) {
      printf "# running for %s\n", $num;
      my @Num = split "[ ]+", $num;

      my $k = sprintf "%s%02d", $AppExt, $nr++;
      my $label = $k;
      if (exists $StaticData{label}{$k}) {
        $label = $StaticData{label}{$k};
      }
      printf "%s.label %s\n", $k, $label;
      if (exists $StaticData{label}{"$k.info"}) {
        $label = $StaticData{label}{"$k.info"};
      }
      printf "%s.info %s\n", $k, $label;
      printf "%s.min 0\n",   $k;
      printf "%s.max 100\n", $k;
      printf "%s.type GAUGE\n", $k;
    }
    exit 0;
  }

  if ($#ARGV < 0) {
    my %Old = ();
    my %Diff = ();
    if (EvalBoolean($StaticData{config}{"$AppExt+state"})) {
      ReadState(\%Old);
      %Diff = ();
      DiffFromState(\%Old, \%Data, \%Diff);
    } else {
      %Diff = %Data;
    }
    if ($debug) {
      foreach my $k (sort keys %Data) {
        printf "# DIFF %-16s %12d %12d %12d\n", $k, $Old{$k}, $Data{$k},
          $Diff{$k};
      }
    }
    my $nr = 0;
    foreach my $num (@Values) {
      printf "# running for %s\n", $num;
      my @Num = split "[ ]+", $num;
      my $n   = 0;
      my $d   = 0;
      foreach my $k (@Num) {
        if (exists $Diff{$k}) {
          $n += $Diff{$k};
        }
      }
      foreach my $k (@Den) {
        if (exists $Diff{$k}) {
          $d += $Diff{$k};
        }
      }
      my $k = sprintf "%s%02d", $AppExt, $nr++;
      printf "%s.value %s\n", $k, $d > 0 ? ((100. * $n) / $d) : 0;
    }
    if (EvalBoolean($StaticData{config}{"$AppExt+state"})) {
      SaveState(\%Data);
    }
    exit 0;
  }
  exit 1;
}

# ------------------------------------------------
#
#
if (defined $AppExt && $AppType eq "dbcounters") {
  my $DataAggregate = $StaticData{config}{"$AppExt+data"};
  printf "# Data Aggregate : %s\n", $DataAggregate;
  my @Values = split " ", $DataAggregate;
  if ($#ARGV >= 0 && $ARGV[0] eq "config") {
    my @Keys = grep {$_ =~ /^$AppExt-/;} keys %{$StaticData{config}};
    foreach my $k (@Keys) {
      my $v = $StaticData{config}{$k};
      $k =~ s/^$AppExt-//;
      printf "%-16s %s\n", $k, $v;
    }
    printf "graph_order %s\n", (join " ", sort @Values);
    foreach my $k (@Values) {
      if (1) {
        my $label = $k;
        if (exists $StaticData{label}{$k}) {
          $label = $StaticData{label}{$k};
        }
        printf "%s.label %s\n", $k, $label;
        if (exists $StaticData{label}{"$k.info"}) {
          $label = $StaticData{label}{"$k.info"};
        }
        printf "%s.info %s\n", $k, $label;
        printf "%s.type GAUGE\n", $k;
      }
    }
    exit 0;
  }

  if ($#ARGV < 0) {
    foreach my $k (@Values) {
      my $dbfile = "$WORKDIR/$k.db";
      $dbfile = "$CONSTDIR/$k.db" unless -f $dbfile;
      next unless -f $dbfile;
      my @DATA = `j-makemap -c -b $dbfile`;
      chomp @DATA;
      my $nrec = 0;
      foreach my $line (@DATA) {
        if ($line =~ /\s+(\d+)\s+records/) {
          $nrec = $1;
          printf "%s.value %s\n", $k, $nrec;
          last;
        }
      }
    }
    exit 0;
  }
  exit 1;
}

# ------------------------------------------------
#
#
sub GetStatsLine {
  my ($h, undef) = @_;
  return undef unless -f $FSTATS;
  my $line = `tail -1 $FSTATS`;
  return undef unless defined $line && $line ne "";

  my @Values = split / /, $line;
  my $time = time();
  if ($Values[0] =~ /^\d+$/) {
    $time = shift @Values;
  }
  $h->{timestamp} = $time;
  foreach my $lx (@Values) {
    $lx = lc $lx;
    if ($lx =~ /(\S+)=\((\d+)\)/) {
      $h->{$1} = $2;
    }
  }
  return $line;
}

# ------------------------------------------------
#
#
sub ReadState {
  my ($h, undef) = @_;
  return 0 unless defined $h;

  return 0 unless defined $StateFile && $StateFile ne "" && -f $StateFile;

  open FSTATE, "< $StateFile" || return 0;
  while (my $s = <FSTATE>) {
    chomp $s;
    next if $s =~ /^\s*$/;
    next if $s =~ /^\s*#/;
    my ($a, $b) = split " ", $s;
    $b = 0 unless defined $b;
    $h->{$a} = $b;
  }
  close FSTATE;
  {
    my %St = Munin::Plugin::restore_state();
    foreach my $k (sort keys %St) {
      printf "# %-20s %s\n", $k, $St{$k};
    }
  }
  return 1;
}

# ------------------------------------------------
#
#
sub SaveState {
  my ($h, undef) = @_;

  return 0 unless defined $StateFile && $StateFile ne "";
  my @ST = qw();
  open FSTATE, "> $StateFile" || return 0;
  foreach my $k (keys %{$h}) {
    push @ST, (sprintf "%s %s\n", $k, $h->{$k});
    printf FSTATE "%-20s %s\n", $k, $h->{$k};
  }
  close FSTATE;

  Munin::Plugin::save_state(%$h);
  return 1;
}

# ------------------------------------------------
#
#
sub DiffFromState {
  my ($old, $new, $delta, undef) = @_;
  return 0 unless defined $old && defined $new;
  $delta = $new unless defined $delta;

  return 0 unless exists $old->{timestamp};
  return 0 unless exists $new->{timestamp};
  my $dt = $new->{timestamp} - $old->{timestamp};
  return 0 unless $dt > 1;

  $delta->{timestamp} = $new->{timestamp};
  foreach my $k (sort keys %{$new}) {
    next if $k eq "timestamp";
    if (exists $old->{$k}) {
      $delta->{$k} = $new->{$k} - $old->{$k};
      $delta->{$k} *= 300. / $dt;
    } else {
      $delta->{$k} = 0.;
    }
  }
  return 1;
}

# ------------------------------------------------
#
#
sub GetStaticData {
  my ($h, undef) = @_;
  my @DATA = <DATA>;
  my @SOPT = qw();
  chomp @DATA;
  my $in  = 0;
  my $key = "";
  foreach my $s (@DATA) {
    if ($key eq "") {
      if ($s =~ /==\s+BEGIN\s+(\S+)\s+==/) {
        $key = $1;
      }
      next;
    }
    if ($s =~ /==\s+END\s+(\S*\s+)?==/) {
      $key = "";
      next;
    }
    next if $s =~ /^\s*$/;
    push @{$h->{$key}{_data_}}, $s;
    my ($a, $b) = split " ", $s, 2;
    next unless defined $a && $a ne "";
    $b = "" unless defined $b;
    $h->{$key}{$a} = $b;
  }
}

# ------------------------------------------------
#
#
sub EvalBoolean {
  my ($v, undef) = @_;

  # return defined $v && $v =~ /^(1|yes|true|oui|vrai)$/i;
  if (defined $v) {
    return 1 if $v =~ /^(1|yes|true|oui|vrai)$/i;
  }
  return 0;
}

# ------------------------------------------------
#
#
__DATA__

== BEGIN label ==
conn                      Connections
conn.info                 SMTP Connections per time unit
msgs                      Messages
msgs.info                 Messages per time unit
rcpt                      Recipients
rcpt.info                 Recipients per time unit

greymsgs                  Greylisted Messages
greymsgs.info             Greylisted Messages per time unit
greyrcpt                  Greylisted Recipients
greyrcpt.info             Greylisted Recipients per time unit

bayesham                  Legitimate (Stat Filter)
bayesham.info             Legitimate Messages - Statistical Filter
bayesspam                 Unwanted (Stat Filter)
bayesspam.info            Unwanted Messages - Statistical Filter

matching                  Pattern Matching
matching.info             Unwanted Messages - Pattern Matching Filter
urlbl                     URL Blacklist
urlbl.info                Unwanted Messages - URL Blacklist

throttle                  Connection Rate
throttle.info             Connections rejected by connection rate
badrcpt                   Bad Recipients
badrcpt.info              Connections rejected - excessive bad recipients
localuser                 Internal addresses
localuser.info            Messages sent to internal use only addresses
badmx                     Bad Sender MX
badmx.info                Messages rejected - Bad Sender MX
rcptrate                  Recipient Rate
rcptrate.info             Messages rejected by recipient rate

files                     Files
files.info                Files
xfiles                    X-Files
xfiles.info               X-Files

kbytes                    Volume (KBytes)
kbytes.info               Volume (KBytes) per time unit

j-greyvalid               Grey Validated records
j-greyvalid.info          Grey Validated records
j-greypend                Grey Waiting records
j-greypend.info           Grey Waiting records
j-greywhitelist           Grey Whitelisted records
j-greywhitelist.info      Grey Whitelisted records
j-greyblacklist           j-greyblacklist
j-greyblacklist.info      j-greyblacklist

ratiospamham              Ratio Messages
ratiospamham.info         Ratio Messages
ratiospamham00            Legitimate messages
ratiospamham00.info       Legitimate messages
ratiospamham01            Unwanted messages
ratiospamham01.info       Unwanted messages

== END label ==


== BEGIN config ==
activity-graph_title        j-chkmail - Filter activity
activity-graph_category     mail
activity-graph_vlabel       counts per second
activity-graph_scale        no
activity+data               conn msgs rcpt
activity+type               counters
activity+enable             yes

greylisting-graph_title     j-chkmail - Greylisting activity
greylisting-graph_category  mail
greylisting-graph_vlabel    counts per second
greylisting-graph_scale     no
greylisting+data            greymsgs greyrcpt
greylisting+type            counters
greylisting+enable          yes

statfilter-graph_title      j-chkmail - Statistical classification
statfilter-graph_category   mail
statfilter-graph_vlabel     counts per second
statfilter-graph_scale      no
statfilter+data             bayesham bayesspam
statfilter+type             counters
statfilter+enable           yes

contentfilter-graph_title      j-chkmail - Content Filtering
contentfilter-graph_category   mail
contentfilter-graph_vlabel     counts per second
contentfilter-graph_scale      no
contentfilter+data             bayesspam matching urlbl
contentfilter+type             counters
contentfilter+enable           yes

behaviour-graph_title       j-chkmail - Behaviour filtering - rejections
behaviour-graph_category    mail
behaviour-graph_vlabel      counts per second
behaviour-graph_scale       no
behaviour+data              throttle badrcpt localuser badmx rcptrate
behaviour+type              counters
behaviour+enable            yes

xfiles-graph_title          j-chkmail - Suspected files filtering
xfiles-graph_category       mail
xfiles-graph_vlabel         counts per second
xfiles-graph_scale          no
xfiles+data                 files xfiles
xfiles+type                 counters
xfiles+enable               yes

volume-graph_title          j-chkmail - Volume handled by the filter
volume-graph_category       mail
volume-graph_vlabel         KBytes per second
volume-graph_scale          no
volume+data                 kbytes
volume+type                 counters
volume+enable               yes

ratiospamham-graph_title       j-chkmail - Statistical classification
ratiospamham-graph_category    mail
ratiospamham-graph_vlabel      Ratio (%)
ratiospamham-graph_scale       yes
ratiospamham-graph_args        --lower-limit 0 --upper-limit 100 --rigid
ratiospamham+data              bayesham, bayesspam / bayesham bayesspam
ratiospamham+type              ratio
ratiospamham+state             yes
ratiospamham+enable            yes

greydb-graph_title             j-chkmail - Greylisting Database Size
greydb-graph_category          mail
greydb-graph_vlabel            records
greydb-graph_scale             no
greydb-graph_args              --lower-limit 0
greydb+data                    j-greypend j-greyvalid j-greywhitelist
greydb+type                    dbcounters
greydb+enable                  yes

cdb-graph_title                j-chkmail - Static databases
cdb-graph_category             mail
cdb-graph_vlabel               records
cdb-graph_scale                no
cdb-graph_args                 --lower-limit 0
cdb+data                       j-policy j-urlbl j-bayes j-rcpt
cdb+type                       dbcounters
cdb+enable                     yes

== END config ==