shorewall_log

#!/usr/bin/env python3
# -*- python -*-

"""

=head1 NAME

Plugin to monitor iptables logs configured by shorewall

=head1 CONFIGURATION

  logfile: Path to the iptables log file, or "journald" to use journald.
           When journalctl exists, the default is "journald", otherwise
           "/var/log/kern.log".
  journalctlargs: Arguments passed to journalctl to select the right logs.
                  The default is "SYSLOG_IDENTIFIER=kernel".
  taggroups: Space separated list of groups. A group contains a tag if the
             group is substring of the tag. Tags belonging to the same group
             will be combined in one graph.
  tagfilter: Space separated list of filters. When a tag is matched by a
             filter (i.e. if the filter is a substring of the tag) it is
             ignored.
  prefixformat: The format of the prefix configured in iptables, this is the
                LOGFORMAT option in shorewall.conf. When not set the entire
                prefix is used.
  include_ungrouped: when a tag is found that does not belong to a group,
                     make it it's own group

Example:

Using /var/log/kern.log as logfile:

=over 2

  [shorewall_log]
  group adm
  env.logfile /var/log/kern.log

=back

Using journald:

=over 2

  [shorewall_log]
  group systemd-journal

=back

=head1 HISTORY

2017-11-03: v1.0 Bert Van de Poel <bert@bhack.net>: created
2020-07-16: v2.0 Vincent Vanlaer <vincenttc@ulyssis.org>: rewrite
                 - read all tags from iptables config, instead of the last 24h
                   of logs
                 - add support for journald
                 - use cursors for accuracy
                 - convert to multigraph to reduce load

=head1 USAGE

Parameters understood:

  config   (required)
  autoconf (optional - used by munin-config)

=head1 MAGIC MARKERS

 #%# family=auto
 #%# capabilities=autoconf

=cut
"""


import sys
import os


def autoconf() -> str:
    if sys.version_info < (3, 5):
        return 'no (This plugin requires python 3.5 or higher)'

    if os.getenv('MUNIN_CAP_MULTIGRAPH', '0') != '1':
        return 'no (No multigraph support)'

    import shutil

    if not shutil.which('shorewall'):
        return 'no (No shorewall executable found)'

    if not shutil.which('iptables-save'):
        return 'no (No iptables-save executable found, required for tag enumeration)'

    return 'yes'


if len(sys.argv) == 2 and sys.argv[1] == "autoconf":
    print(autoconf())
    sys.exit()


from collections import defaultdict, namedtuple
from typing import Set, Iterator, TextIO, Dict, Tuple
from itertools import takewhile
from subprocess import run, PIPE
import shlex
import shutil
import pickle
import re


logfile = os.getenv('logfile', 'journald' if shutil.which('journalctl') else '/var/log/kern.log')
journalctl_args = list(shlex.split(os.getenv('journalctlargs',
                                             'SYSLOG_IDENTIFIER=kernel')))
taggroups = os.getenv('taggroups')
taggroups = taggroups.split() if taggroups else []
tagfilter = os.getenv('tagfilter')
tagfilter = tagfilter.split() if tagfilter else []
include_ungrouped = (os.getenv('includeungrouped', 'true').lower() == 'true')
prefix_format = os.getenv('prefixformat')
if prefix_format:
    if sys.version_info < (3, 7):
        prefix_format = re.escape(prefix_format).replace('\\%s', '(.+)').replace('\\%d', '\\d+')
    else:  # % is no longer escaped
        prefix_format = re.escape(prefix_format).replace('%s', '(.+)').replace('%d', '\\d+')
    prefix_format = re.compile(prefix_format)


def get_logtags() -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]]]:
    rules = (run(['iptables-save'], stdout=PIPE, universal_newlines=True).
             stdout.splitlines())
    tags = defaultdict(set)
    groups = defaultdict(set)

    # every line is an iptables rule, in the iptables command/args syntax
    # (without the 'iptables' command listed), eg.
    #                                    "-A INPUT -p tcp -s 10.3.3.7 -j DROP"
    for line in rules:
        args = iter(shlex.split(line))
        for arg in args:
            # we only want rules that log packets, not that accept/drop/...
            if arg == '-j' and next(args) != 'LOG':
                break
            # and we only need to know the logging tag, and add it to the list
            if arg == '--log-prefix':
                prefix = next(args)

                if prefix_format:
                    tag = prefix_format.match(prefix)[1]
                else:
                    tag = prefix.rstrip()

                if any(ignored in tag for ignored in tagfilter):
                    continue
                tags[tag].add(prefix)

                for group in taggroups:
                    if group in tag:
                        groups[group].add(tag)
                        break
                else:
                    if include_ungrouped:
                        groups[tag].add(tag)

                break

    return groups, tags


State = namedtuple('State', ['journal', 'file'])


def load_state() -> State:
    try:
        with open(os.getenv('MUNIN_STATEFILE'), 'rb') as f:
            return pickle.load(f)
    except OSError:
        return State(None, None)


def save_state(state: State):
    with open(os.getenv('MUNIN_STATEFILE'), 'wb') as f:
        return pickle.dump(state, f)


def get_lines_journalctl(state: State) -> Iterator[str]:
    cursor = state.journal

    def catch_cursor(line: str):
        cursor_id = '-- cursor: '
        if line.startswith(cursor_id):
            save_state(State(line[len(cursor_id):], None))
            return False
        else:
            return True

    if not cursor:  # prevent reading the entire journal on first run
        journal = run(['journalctl', '--no-pager', '--quiet', '--lines=0',
                       '--show-cursor', *journalctl_args],
                      stdout=PIPE, universal_newlines=True)
    else:
        journal = run(['journalctl', '--no-pager', '--quiet', '--show-cursor',
                       '--after-cursor', cursor, *journalctl_args],
                      stdout=PIPE, universal_newlines=True)

    yield from filter(catch_cursor, journal.stdout.splitlines())


def reverse_read(f: TextIO) -> Iterator[str]:
    BUFSIZE = 4096
    f.seek(0, 2)
    position = f.tell()
    remainder = ''
    while position > 0:
        position = max(position - BUFSIZE, 0)
        f.seek(position)
        lines = f.read(BUFSIZE).splitlines()
        lines[-1] += remainder
        remainder = lines.pop(0)
        yield from reversed(lines)
    yield remainder


def get_lines_logfile(path: str, state: State) -> Iterator[str]:
    with open(path, 'r') as f:
        cursor = state.file

        reader = reverse_read(f)

        if not cursor:
            save_state(State(None, next(reader)))
            return
        else:
            new_cursor = next(reader)
            save_state(State(None, new_cursor))
            yield new_cursor
        yield from takewhile(lambda x: x != cursor, reader)


def get_tagcount(tags: Dict[str, Set[str]]) -> Dict[str, int]:
    count = defaultdict(int)
    state = load_state()

    if logfile == 'journald':
        lines = get_lines_journalctl(state)
        offset = 5
    else:
        lines = get_lines_logfile(logfile, state)
        offset = 6

    for line in lines:
        if 'IN=' not in line:
            continue
        line = line.replace('  ', ' ').split(' ', maxsplit=offset)[-1]

        for tag, prefixes in tags.items():
            for prefix in prefixes:
                if line.startswith(prefix):
                    count[tag] += 1
                    break
            else:
                continue
            break

    return count


def fetch():
    groups, logtags = get_logtags()
    tagcount = get_tagcount(logtags)

    for group, tags in groups.items():
        print('multigraph shorewall_{}'.format(group))
        for tag in tags:
            print('{}.value {}'.format(tag.lower(), tagcount[tag]))


def config():

    for group, tags in get_logtags()[0].items():
        print('multigraph shorewall_{}'.format(group))
        print('graph_title Shorewall Logs for {}'.format(group))
        print('graph_vlabel entries per ${graph_period}')
        print('graph_category shorewall')

        for tag in sorted(tags):
            print('{}.label {}'.format(tag.lower(), tag))
            print('{}.type ABSOLUTE'.format(tag.lower()))
            print('{}.draw AREASTACK'.format(tag.lower()))


if len(sys.argv) == 2 and sys.argv[1] == "config":
    config()
else:
    fetch()

# flake8: noqa: E265,E402