#!/usr/bin/perl
# Copyright (c) Sep 2011-2013 Wolfram Schneider, http://bbbike.org
#
# munin-bbbike - munin statistics for BBBike, searches per interval

use Getopt::Long;
use Data::Dumper;
use File::stat;
use IO::File;
use File::Basename;
use CGI;

use strict;
use warnings;

# 0: quiet, 1: normal, 2: verbose
my $debug = defined $ENV{MUNIN_BBBIKE_DEBUG} ? $ENV{MUNIN_BBBIKE_DEBUG} : 0;

my $database    = '/var/tmp/munin-bbbike';
my $logfile     = '/var/log/lighttpd/bbbike.log';
my $logfile_api = '/var/log/lighttpd/api.bbbike.log';
my $type        = 'route';

# ignore requests from localhost or 10.x.x.x
my $ignore_localrequests = 1;

# standard munin interval of 5 minutes
my $interval = 1;    # 5

# run as frontend proxy, with full hostname requests: GET http://www.bbbike.org/Foo
my $frontend_proxy = 0;

# munin-bbbike config file
my $munin_config_file = "/etc/munin/munin-bbbike.conf";

# {fieldname}.min?
# should be 0 or 1
my $min_value = 0;

# ignore old offset files (> 60 minutes)
my $ignore_old_offset = 1;

# reset offset
my $reset = 0;

my $config = {
    'ajax'   => [qw/api log street-coord crossing maptype weather location/],
    'output' => [
        qw/print gpx-track gpx-route kml-track pdf png palmdoc googlemaps mapserv xml/
    ],
    'pref' => [
        qw/speed cat_np cat_N1 cat_N2 cat_H1 cat_H2 cat_N_RW cat_N_RW1
          quality_np quality_Q0 quality_Q1 specialvehicle_np specialvehicle_trailer specialvehicle_childseat
          unlit ferry/
    ],
    'maptype' => [
        qw/bing_birdview bing_hybrid bing_map cycle hike_bike hybrid mapnik mapnik_bw mapnik_de
          public_transport roadmap satellite tah terrain/
    ],
    'appid' => [
        qw/android-0 vmz ios1 ios2 android1 android2 wp0 last0 bbbikede guest other useragent/
    ],
};

sub usage {
    my $types = join " | ", &get_types;

    <<EOF;
usage: $0 [ options ] config

--debug=0..2    		default: $debug
--logfile=/path/to/logfile	default: $logfile
--logfile-api=/path/to/logfile	default: $logfile_api
--database=/path/to/logfile	default: $database
--type=<stat type>		$types, default: $type
--interval=interval		interval in minutes, default: $interval;
--frontend-proxy=[0,1]		run as frontend proxy, default: $frontend_proxy
--ignore-localrequests=[0..1]	ignore localhost or 10.x.x.x requests, default: $ignore_localrequests
--reset=[0,1]                   reset offset to zero

Munin bbbike config file: $munin_config_file
EOF
}

sub interval {
    my $value = shift;
    my $period = shift || $interval;

    $period = 1 if $period <= 0;

    return ( $value / $period );
}

sub get_types {
    my @types = qw/ajax route output pref maptype appid/;
    return sort @types;
}

# escape label names for older munin releases
sub escape_label {
    my $label = shift;

    $label =~ s/-/_/g;

    return $label;
}

sub config_routes () {
    <<EOF;
graph_title Route Searches
graph_vlabel Searches
graph_category BBBike
graph_info Number of Route Searches
graph_period minute
graph_scale no
data.label Route Searches
data.min $min_value
EOF
}

# routes by appid
sub config_appid () {
    my $data = <<EOF;
graph_title Route Searches by Apps
graph_vlabel Searches
graph_category BBBike
graph_info Number of Route Searches by Apps
graph_period minute
graph_scale no
EOF

    foreach my $label ( sort @{ $config->{'appid'} } ) {
        my $label_e = escape_label($label);
        $data .= "$label_e.label $label\n";
        $data .= "$label_e.min $min_value\n";
    }

    return $data;
}

sub config_pref () {
    my $data = <<EOF;
graph_title Route Preferences
graph_vlabel Preferences
graph_category BBBike
graph_info Prefernces
graph_period minute
graph_scale no
speed.cdef speed,10,/
EOF

    foreach my $label ( sort @{ $config->{'pref'} } ) {
        my $label_e = escape_label($label);
        $data .= "$label_e.label $label\n";
        $data .= "$label_e.min $min_value\n";
    }

    return $data;
}

sub config_ajax () {
    my $data = <<EOF;
graph_title Ajax
graph_vlabel Ajax Requests
graph_category BBBike
graph_info Number of Ajax Requests
graph_period minute
graph_scale no
EOF

    foreach my $label ( sort @{ $config->{'ajax'} } ) {
        my $label_e = escape_label($label);
        $data .= "$label_e.label $label\n";
        $data .= "$label_e.min $min_value\n";
    }

    return $data;
}

sub config_output () {
    my $data = <<EOF;
graph_title Output Formats
graph_vlabel Output Formats
graph_category BBBike
graph_info Number of Output Formats
graph_period minute
graph_scale no
EOF

    foreach my $label ( sort @{ $config->{'output'} } ) {
        my $label_e = escape_label($label);
        $data .= "$label_e.label $label\n";
        $data .= "$label_e.min $min_value\n";
    }

    return $data;
}

sub config_maptype () {
    my $data = <<EOF;
graph_title Maptype
graph_vlabel Maptype Requests
graph_category BBBike
graph_info Number of Maptype Requests
graph_period minute
graph_scale no
EOF

    foreach my $label ( sort @{ $config->{'maptype'} } ) {
        my $label_e = escape_label($label);
        $data .= "$label_e.label $label\n";
        $data .= "$label_e.min $min_value\n";
    }

    return $data;
}

sub config {
    my $type = shift;

    return
        $type eq 'ajax'    ? &config_ajax
      : $type eq 'route'   ? &config_routes
      : $type eq 'appid'   ? &config_appid
      : $type eq 'output'  ? &config_output
      : $type eq 'pref'    ? &config_pref
      : $type eq 'maptype' ? &config_maptype
      :                      die "Unknown stat type: $type\n";
}

sub display_results {
    my $config = shift;
    my $count  = shift;

    my $data = "";

    foreach my $label ( sort @$config ) {
        $data .=
            "$label.value "
          . ( exists $count->{$label} ? interval( $count->{$label} ) : 0 )
          . "\n";
    }

    return $data;
}

sub parse_log {
    my %args = @_;

    my $logfile  = $args{'logfile'};
    my $type     = $args{'type'};
    my $database = $args{'database'};
    my $interval = $args{'interval'};
    $database .= "-$<-$type.txt";

    my $st = stat($logfile) or die "stat $logfile: $!";
    my $offset = $st->size;

    # check for stale databases which are too old
    my $regnerate_database = 1;
    if ( -e $database ) {
        my $database_st = stat($database) or die "stat $database: $!";

        # ok, fresh data
        if ( $database_st->mtime >= time() - 60 * 60 ) {
            $regnerate_database = 0;
        }
        $regnerate_database = 0 if $ignore_old_offset == 0;
    }

    # first run, save file offset of logfile, do nothing
    if ($regnerate_database) {
        write_offset( $database, $offset );

        # run again to produce first results
        if ( !$args{'first_run'} ) {
            return parse_log( @_, 'first_run' => 1 );
        }
        return;
    }

    my $last_offset = get_offset($database);

    my $data = "";
    if ( $type eq 'route' ) {
        my $route_count = count_routes(
            'logfile' => $logfile,
            'offset'  => $last_offset,
            'type'    => $type
        );

        $data = "data.value " . &interval($route_count) . "\n";
    }
    elsif ( $type eq 'pref' ) {
        my $count = count_pref(
            'logfile' => $logfile,
            'offset'  => $last_offset,
            'type'    => $type
        );
        $data = display_results( $config->{'pref'}, $count );
    }

    elsif ( $type eq 'ajax' ) {
        my $count = count_ajax(
            'logfile' => $logfile,
            'offset'  => $last_offset,
            'type'    => $type,
            'scripts' => $config->{'ajax'}
        );
        $data = display_results( $config->{'ajax'}, $count );
    }
    elsif ( $type eq 'maptype' ) {
        my $count = count_maptype(
            'logfile' => $logfile,
            'offset'  => $last_offset,
            'type'    => $type,
            'scripts' => $config->{'maptype'}
        );
        $data = display_results( $config->{'maptype'}, $count );
    }
    elsif ( $type eq 'output' ) {
        my $count = count_output(
            'logfile' => $logfile,
            'offset'  => $last_offset,
            'type'    => $type,
            'output'  => $config->{'output'}
        );
        $data = display_results( $config->{'output'}, $count );
    }
    elsif ( $type eq 'appid' ) {
        my $count = count_appid(
            'logfile' => $logfile_api,
            'offset'  => $last_offset,
            'type'    => $type,
            'output'  => $config->{'appid'}
        );
        $data = display_results( $config->{'appid'}, $count );
    }
    else {
        die "Unknown type: $type\n";
    }

    # store current log file size in database for next run
    $st = stat( $type eq 'appid' ? $logfile_api : $logfile )
      or die "stat $logfile: $!";
    write_offset( $database, $st->size );

    return $data;
}

sub get_offset {
    my $file = shift;

    if ($reset) {
        warn "Reset offset to zero in $file\n" if $debug;
        return 0;
    }

    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";
    my $number = <$fh>;

    if ( defined $number ) {
        chomp($number);
    }
    else {
        $number = 0;
    }

    warn "Got offset $number from $file\n" if $debug;
    return $number;
}

sub write_offset {
    my $file   = shift;
    my $offset = shift;

    warn "Store offset $offset in $file\n" if $debug;
    my $fh = IO::File->new( $file, "w" ) or die "open $file: $!\n";
    print $fh $offset;
    $fh->close;
}

sub localrequest {
    my $host = shift;

    $host =~ /^10\.|^127\.0\.0\.1/ ? 1 : 0;
}

#
# parse the bbbike access log file and count route
# searches (parameters startc, zielc, pref_seen)
#
sub count_routes {
    my %args   = @_;
    my $file   = $args{'logfile'};
    my $offset = $args{'offset'};

    my $counter = 0;
    warn "open $file\n" if $debug;
    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";

    if ( defined $offset ) {
        warn "Starting at offset: $offset\n" if $debug;
        seek( $fh, $offset, 0 );
    }

    while (<$fh>) {
        next if !/" 200 /;    # successfull requests only

        # proxy request with hostname
        s,"GET http://.*?/,"GET /, if $frontend_proxy;

        next if $ignore_localrequests && localrequest($_);

        if (m,"GET /\S+pref_seen=[12],) {
            $counter++;
        }
    }

    return $counter;
}

#
# parse the bbbike access log file and count preferences
#
sub count_pref {
    my %args   = @_;
    my $file   = $args{'logfile'};
    my $offset = $args{'offset'};

    warn "open $file\n" if $debug;
    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";

    if ( defined $offset ) {
        warn "Starting at offset: $offset\n" if $debug;
        seek( $fh, $offset, 0 );
    }

    my $counter = 0;
    my $res = { "speed" => 0 };
    while (<$fh>) {
        next if !/" 200 /;    # successfull requests only

        # proxy request with hostname
        s,"GET http://.*?/,"GET /, if $frontend_proxy;

        next if $ignore_localrequests && localrequest($_);

        if (   m,"GET /\S+pref_seen=1,
            && /startc=[0-9\-\+]/
            && /zielc=[0-9\-\+]/ )
        {

            if (m,"GET (/\S+),) {
                my $q = CGI->new($1);
                $counter++;
                if ( my $s = $q->param("pref_speed") || 20 ) {
                    $res->{"speed"} += $s;
                }
                if ( my $s = $q->param("pref_cat") || "np" ) {
                    $res->{"cat_$s"}++;
                }
                if ( my $s = $q->param("pref_quality") || "np" ) {
                    $res->{"quality_$s"}++;
                }
                if ( my $s = $q->param("pref_specialvehicle") || "np" ) {
                    $res->{"specialvehicle_$s"}++;
                }
                $res->{"ferry"}++ if $q->param("pref_ferry");
                $res->{"unlit"}++ if $q->param("pref_unlit");
            }
        }
    }

    return $res if !$counter;

    foreach my $key ( keys %$res ) {
        $res->{$key} = $res->{$key} / $counter;
    }

    return $res;
}

#
# parse the bbbike access log file and count ajax request
#
sub count_ajax {
    my %args   = @_;
    my $file   = $args{'logfile'};
    my $offset = $args{'offset'};

    my $counter = {};
    warn "open $file\n" if $debug;
    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";

    if ( defined $offset ) {
        warn "Starting at offset: $offset\n" if $debug;
        seek( $fh, $offset, 0 );
    }

    while (<$fh>) {
        next if !/" 200 /;    # successfull requests only

        # proxy request with hostname
        s,"GET http://.*?/,"GET /, if $frontend_proxy;

        next if $ignore_localrequests && localrequest($_);

        if (m,"GET /cgi/(.*?)(\.cgi)?\?,) {
            $counter->{$1}++;
        }
    }

    return $counter;
}

#
# parse the bbbike access log file and count map type request
#
sub count_maptype {
    my %args   = @_;
    my $file   = $args{'logfile'};
    my $offset = $args{'offset'};

    my $counter = {};
    warn "open $file\n" if $debug;
    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";

    if ( defined $offset ) {
        warn "Starting at offset: $offset\n" if $debug;
        seek( $fh, $offset, 0 );
    }

    while (<$fh>) {
        next if !/" 200 /;    # successfull requests only

        # proxy request with hostname
        s,"GET http://.*?/,"GET /, if $frontend_proxy;

        next if $ignore_localrequests && localrequest($_);

        if (m,"GET /cgi/maptype.cgi.*?maptype=(\S+),) {
            $counter->{$1}++;
        }
    }

    return $counter;
}

#
# parse the bbbike access log file and count map type request
#
sub count_appid {
    my %args   = @_;
    my $file   = $args{'logfile'};
    my $offset = $args{'offset'};

    my $counter = {};
    warn "open $file\n" if $debug;
    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";

    if ( defined $offset ) {
        warn "Starting at offset: $offset\n" if $debug;
        seek( $fh, $offset, 0 );
    }

    my %hash = map { $_ => 1 } @{ $config->{"appid"} };
    my $id;

    while (<$fh>) {

        #next if !/" 200 /;    # successfull requests only

        # proxy request with hostname
        s,"GET http://.*?/,"GET /, if $frontend_proxy;

        next if $ignore_localrequests && localrequest($_);

        if (m,"GET /\S+pref_seen=[12],) {

            if (/appid=(.*?)["\s&;]/) {
                $id = exists $hash{$1} ? $1 : "other";
                $counter->{$id}++;
            }
            elsif (m, "BBBike/\d+ CFNetwork,) {
                $counter->{"useragent"}++;
            }
        }
    }

    return $counter;
}

#
# parse the bbbike access log file and count output
# requests (GPX, KML, print)
#
sub count_output {
    my %args   = @_;
    my $file   = $args{'logfile'};
    my $offset = $args{'offset'};

    my $counter = {};
    warn "open $file\n" if $debug;
    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";

    if ( defined $offset ) {
        warn "Starting at offset: $offset\n" if $debug;
        seek( $fh, $offset, 0 );
    }

# "GET /Erlangen/?startc=10.95985%2C49.58876;zielc=11.02811%2C49.57724;pref_seen=1;pref_speed=20;pref_cat=;pref_quality=;pref_specialvehicle=;scope=;output_as=print
    while (<$fh>) {
        next if !/" 200 /;    # successfull requests only

        # proxy request with hostname
        s,"GET http://.*?/,"GET /, if $frontend_proxy;

        next if $ignore_localrequests && localrequest($_);

        if (m,"GET /\S+output_as=(.*?)[;&\s],) {
            $counter->{$1}++;
        }

        elsif (m,"GET /\S+imagetype=(.*?)[;&\s],) {
            my $imagetype = $1 || "";

            # count all PDF as pdf (landscape + auto + portrait)
            $imagetype =~ s/^pdf.*/pdf/;
            $counter->{$imagetype}++ if $imagetype ne "";
        }

        # bbbike.de
        elsif (m,"POST /cgi-bin/bbbikegooglemap.cgi,) {
            $counter->{"googlemaps"}++;
        }
        elsif (m,"GET /cgi-bin/mapserv\?,) {
            $counter->{"mapserv"}++;
        }

        elsif (m,"POST ,) {

            # proxy request with hostname
            s,"POST http://.*?/,"POST /, if $frontend_proxy;

            if (   m,"POST /[A-Z][a-z]+\S+/ ,
                || m,"POST /[A-Z][a-z]+\S+/\?[a-z]+, )
            {
                $counter->{"pdf"}++;
            }
        }
    }

    return $counter;
}

#
# detect type by program path
# ./munin-bbbike-ajax => ./munin-bbbike --type=ajax
#
sub detect_type {
    my $program = basename($0);

    if ( $program =~ /-(\w+)$/ ) {
        my $t = $1;
        return $t if grep { $t eq $_ } &get_types;
    }
}

sub parse_config {
    my $file = shift;

    warn "Open config file $file\n" if $debug;

    return if !-f $file;
    warn "open $file\n" if $debug;
    my $fh = IO::File->new( $file, "r" ) or die "open $file: $!\n";

    my @param;
    while (<$fh>) {
        chomp;
        s/^\s*#.*//;
        if ($_) {
            push @param, split /\s+/, $_;
        }
    }

    warn "Read config: ", join( " ", @param ), "\n" if $debug;
    return @param;
}

######################################################################
# main
#

my $help;

# read parameters from script path name
$type = &detect_type if &detect_type;

# read parameters from munin-bbbike config file
my @config = parse_config($munin_config_file);
unshift( @ARGV, @config ) if @config;

GetOptions(
    "debug=i"                => \$debug,
    "database=s"             => \$database,
    "logfile=s"              => \$logfile,
    "logfile-api=s"          => \$logfile_api,
    "type=s"                 => \$type,
    "interval=i"             => \$interval,
    "frontend-proxy=i"       => \$frontend_proxy,
    "ignore-localrequests=i" => \$ignore_localrequests,
    "reset=i"                => \$reset,
    "help"                   => \$help,
) or die usage;

die usage if $help;
die "Unknown type '$type'\n\n" . &usage if !grep { $type eq $_ } &get_types;

if ( defined $ARGV[0] && $ARGV[0] eq 'config' ) {
    print &config($type);
}
else {
    my $text = &parse_log(
        'logfile'  => $logfile,
        'database' => $database,
        'interval' => $interval,
        'type'     => $type
    );
    print $text if $text;
}

__END__

=head1 NAME

bbbike - munin plugin for bbbike

=head1 CONFIGURATION

    BBBIKE_ROOT_DIR=/path/to/bbbike
    cd /etc/munin/plugins
    for subtype in ajax appid maptype output pref route; do ln -s $BBBIKE_ROOT_DIR/misc/munin-plugins/bbbike bbbike-$subtype; done

Also create a configuration file /etc/munin/munin-bbbike.conf with
additional commandline arguments, especially one for the logfile:

    --logfile=/var/log/apache2/bbbike.de_access.log

=cut
