#!/usr/bin/perl -w
# -*- perl -*-

#
# Author: Slaven Rezic
#
# Copyright (C) 2007,2010,2012,2013,2018,2022 Slaven Rezic. All rights reserved.
# This program is free software; you can redistribute it and/or
# modify it under the same terms as Perl itself.
#
# Mail: slaven@rezic.de
# WWW:  http://www.rezic.de/eserte/
#

use strict;
use Safe;
use Getopt::Long;
use POSIX qw(strftime);

my $do_cumulated;
my $do_cumulated_per_dir;
my $do_start_goal;
my $max;
my $do_automax;
my $do_allow_test;
my $do_ignore_bots;
my $debug;

my $ignore_test_points;

our %start; # our because of soft references
our %goal;
my %seen;

# This list build with:
#     make test on bbbike
#     cd /tmp/coordssession
#     perl -le 'for (<*>) { $x = do $_; @c = split /\!/, $x->{routestringrep}; print $c[0]; print $c[-1] }'|sort -u
#
no warnings 'qw';
my %test_start_goal = map {($_,1)}
    qw(
	  -11240,-2525
	  -11831,-70
	  -12493,-1896
	  -13556,-1140
	  -14355,-606
	  -3360,2917
	  10059,10147
	  11016,9599
	  1381,11335
	  14395,11753
	  14598,11245
	  14752,11041
	  14794,10844
	  14798,10985
	  16861,5935
	  17210,6303
	  26615,14054
	  27342,-3023
	  8982,8781
	  9203,13463
	  9222,8787
	  9509,10391
     );

GetOptions("cumulated!" => \$do_cumulated,
	   "cumulated-per-dir!" => \$do_cumulated_per_dir,
	   "start-goal!" => \$do_start_goal,
	   "ignore-test-points!" => \$ignore_test_points,
	   "max=i" => \$max,
	   "automax!" => \$do_automax,
	   "allow-test!" => \$do_allow_test,
	   "ignore-bots!" => \$do_ignore_bots,
	   "debug!" => \$debug,
	  )
    or die "usage: $0 [-cumulated | -cumulated-per-dir | -start-goal]
	[-ignore-test-points] [-automax | -max ...] [-allow-test] [-ignore-bots]
        [-debug]
	/path/to/coordssessiondir ...";

my $cpt = Safe->new;

my @files_and_dirs = @ARGV;
if (!@files_and_dirs) {
    die "Please specify a directory or file containing coordssession files";
}

for my $file_or_dir (@files_and_dirs) {
    handle_path($file_or_dir);
}
sub handle_path {
    my($file_or_dir) = @_;
    warn "Check $file_or_dir...\n" if $debug;
    if (-d $file_or_dir) {
	my $dir = $file_or_dir;
	opendir my $DIR, $dir
	    or die $!;
	my $count = 0;
	while(my $file = readdir $DIR) {
	    my $path = "$dir/$file";
	    next if !-f $path;
	    handle_path($path);
	}
    } elsif (-f $file_or_dir) {
	if ($file_or_dir =~ m{\.zip$}) {
	    require IO::Uncompress::Unzip; # support also Zip64 (unlike Archive::Zip)
	    $IO::Uncompress::Unzip::UnzipError = $IO::Uncompress::Unzip::UnzipError if 0; # cease -w
	    my $zip = IO::Uncompress::Unzip->new($file_or_dir)
		or die "Can't open $file_or_dir: $IO::Uncompress::Unzip::UnzipError";
	    my $status;
	    for ($status = 1; $status > 0; $status = $zip->nextStream) {
		my $name = $zip->getHeaderInfo->{Name};
		warn "  Check $name ...\n" if $debug;

		my $buf;
		while(($status = $zip->read(my $chunk)) > 0) {
		    $buf .= $chunk;
		}
		my $data = $cpt->reval($buf);
		handle_data($data);
		last if $status < 0;
	    }
	    die "Error processing $file_or_dir: $!"
		if $status < 0;
	} elsif ($file_or_dir =~ m{\.(tar\.gz|tgz)$}) {
	    require Archive::Tar;
	    my $next = Archive::Tar->iter($file_or_dir, 1);
	    while(my $f = $next->()) {
		warn "  Check " . $f->name . "...\n" if $debug;
		my $buf = $f->get_content;
		my $data = $cpt->reval($buf);
		handle_data($data);
	    }
	} else {
	    my $data;
	    if ($file_or_dir =~ m{\.gz$}) {
		require PerlIO::gzip;
		open my $fh, "<:gzip", $file_or_dir
		    or die "Can't open $file_or_dir: $!";
		local $/ = undef;
		my $buf = <$fh>;
		$data = $cpt->reval($buf);
	    } else {
		$data = $cpt->rdo($file_or_dir);
	    }
	    handle_data($data);
	}
    } else {
	warn "Ignore $file_or_dir...\n";
    }
}

sub handle_data {
    my($data) = @_;
    if (UNIVERSAL::isa($data, "HASH") && $data->{routestringrep}) {
	unless ($do_allow_test) {
	    if (($data->{'user_agent'}||'') =~ m{BBBike-?Test/}) { # ignore test accesses
		warn "  Ignoring test access...\n" if $debug;
		return;
	    }
	}
	if ($do_ignore_bots && $data->{'user_agent'}) {
	    # Apple iMessage, see
	    #   https://webmasters.stackexchange.com/questions/137914/spike-in-traffic-from-facebot-twitterbot-user-agent
	    #   https://medium.com/@siggi/apples-imessage-impersonates-twitter-facebook-bots-when-scraping-cef85b2cbb7d
	    # First appearance: Nov 2016. UA stable since this time (checked: Apr 2022).
	    if ($data->{'user_agent'} eq 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0') {
		warn "  Ignoring bot (Apple iMessage)...\n" if $debug;
		return;
	    }
	}
	if ($do_ignore_bots && $data->{'remote_ip'}) {
	    if ($data->{'remote_ip'} eq '194.165.16.68') {
		warn "  Ignoring bot (illegal, SQL)...\n" if $debug;
		return;
	    }
	}
	(my $coords = $data->{routestringrep}) =~ s{!}{ }g;
	if ($do_cumulated || $do_cumulated_per_dir) {
	    my @coords = split / /, $coords;
	    for my $i (1 .. $#coords) {
		if ($do_cumulated_per_dir) {
		    $seen{$coords[$i-1]}{$coords[$i]}++;
		} else {
		    if ($seen{$coords[$i-1]}{$coords[$i]}) {
			$seen{$coords[$i-1]}{$coords[$i]}++;
		    } else {
			$seen{$coords[$i]}{$coords[$i-1]}++;
		    }
		}
	    }
	} elsif ($do_start_goal) {
	    my @coords = split / /, $coords;
	    $start{$coords[0]}++;
	    $goal{$coords[-1]}++;
	} else {
	    my $date = POSIX::strftime("%FT%T", localtime $data->{time});
	    my($ip) = $data->{remote_ip} =~ m{(\S+)$}; # strip local ips, show the last ip in chain
	    my $name = "$date $ip";
	    my $prefs = $data->{prefs};
	    if ($prefs) {
		$name .= " " . join(" ", map {
		    (my $short_key = $_) =~ s{^pref_}{};
		    $short_key . ':' . $prefs->{$_};
		} grep { length $prefs->{$_} } grep { $_ ne 'pref_seen' } sort keys %$prefs);
	    }
	    print "$name\tX $coords\n";
	}
    }
}

if ($do_automax) {
    $max = 0;
    while(my($k1,$v1) = each %seen) {
	while(my($k2,$v2) = each %$v1) {
	    $max = $v2 if $v2 > $max;
	}
    }
}

if ($do_cumulated || $do_cumulated_per_dir) {
    if (defined $max) {
	print <<EOF;
#: category_color.w1: #c0c0ff
#: category_color.w2: #8080ff
#: category_color.w3: #4040c0
#: category_color.w4: #000080
#:
EOF
    }
    while(my($k1,$v1) = each %seen) {
	while(my($k2,$v2) = each %$v1) {
	    my $cat;
	    if (defined $max) {
		my $ratio = $v2/$max;
		if ($ratio <= 0.25) {
		    $cat = "w1";
		} elsif ($ratio <= 0.5) {
		    $cat = "w2";
		} elsif ($ratio <= 0.75) {
		    $cat = "w3";
		} else {
		    $cat = "w4";
		}
	    } else {
		$cat = 'X';
	    }
	    if ($do_cumulated_per_dir) {
		print "$v2\t$cat; $k1 $k2\n";
	    } else {
		print "$v2\t$cat $k1 $k2\n";
	    }
	}
    }
} elsif ($do_start_goal) {
    my $max_width = 20;
    my $logarithmic = 1.5;
    for my $width (1..$max_width) {
	$width = int(log($width)/log($logarithmic));
	$width = 1 if ($width < 1);
	my $display_width = 3+int(($max_width-3)/(13-1)*$width); # 1..13 -> 3..20
	print "#: category_width.w$width: $display_width\n";
    }
    print "#:\n";
    for my $type ('start', 'goal') {
	no strict 'refs';
	my $start_or_goal = \%{$type};
	#my $cat = $type eq 'start' ? '#008000' : '#800000';
	while(my($point,$count) = each %$start_or_goal) {
	    next if $ignore_test_points && $test_start_goal{$point};
	    my $cat = int($count/10);
	    if ($cat < 1) { $cat = 1 }
	    elsif ($cat > $max_width) { $cat = $max_width }
	    $cat = "w$cat";
	    print "$count ($type)\t$cat $point\n";
	}
    }
}

__END__
