#!/usr/bin/perl -w
# -*- perl -*-

#
# Author: Slaven Rezic
#
# Copyright (C) 2009,2011,2013,2014,2015,2017,2018,2019,2021,2023 Slaven Rezic. All rights reserved.
# This program is free software; you can redistribute it and/or
# modify it under the same terms as Perl itself.
#
# Mail: slaven@rezic.de
# WWW:  http://www.rezic.de/eserte/
#

use strict;
use FindBin;
use lib (
	 "$FindBin::RealBin/..",
	 "$FindBin::RealBin/../lib",
	 "$FindBin::RealBin/../miscsrc",
	);

use vars qw($VERSION);
$VERSION = 0.20;

use Cwd            qw(realpath);
use File::Basename qw(dirname basename);
use File::Temp     qw(tempfile);
use Getopt::Long;

use BBBikeUtil     qw(is_in_path first);

my @target_defs =
    (
     ['wasserstrassen_lowres'],
     ['largest_city_is_center'],
     ['title_for_dataset'],
     ['inaccessible_streets'],
     ['strassen_cooked', dependency => 'inaccessible_streets' ],
     ['fragezeichen_cooked'],
     #['make_long_streets']
     ['mount',           default => 0], # lohnt sich nicht, zu wenige Hhenpunkte bei osm
     ['coastline_hack',  default => 0], # it's a hack
     ['check_crossings', default => 0], # slow
     ['check_nearest',   default => 0], # slow
     ['icao_metar',      default => 0], # XXX needs unpublished scripts/modules/data, but this will change hopefully
     ['addr_sort'],
     ['zip_points', dependency => 'addr_sort', default => 0], # experimental
     ['cityparts',       default => 0], # to be decided whether it should be default
     ['addr_suburbs',    default => 0], # experimental and very slow
     ['write_meta',      default => 0], # only needed if just additional meta options need to be written
    );
my(@targets, @run_targets);
for my $target_def (@target_defs) {
    my($target_name, %opts) = @$target_def;
    my $dependency = delete $opts{dependency}; # currently just "documentation"
    my $default    = exists $opts{default} ? delete $opts{default} : 1;
    die "Unhandled target options: " . join(" ", %opts) if %opts;
    push @targets, $target_name;
    if ($default) {
	push @run_targets, $target_name;
    }
}

my %skip_target; # skip default targets
my %only_target; # run only the listed targets
my %add_target;  # add more targets to the default ones

sub usage (;$) {
    my $msg = shift;
    warn $msg, "\n" if $msg;
    die "usage: $0 [--debug=0..3] [-force]\n" . join(" ", map {
	(my $optname = $_) =~ s{_}{-}g;
	"\t[-skip-$optname | -only-$optname | -add-$optname]\n"
    } @targets) . "\tdatadir\n\nDefault targets are:\n@run_targets\n";
}

my $set_dataset_title_octets;
# XXX should be generalized, or not be necessary
my $coastline_hack_anchor;
my $addr_suburbs_with_berlin_data;
my @addr_suburbs_filter_city;

my $debug = 1;
my $force;
my %add_meta;
GetOptions(
	   (map {
	       (my $optname = $_) =~ s{_}{-}g;
	       (
		"skip-$optname" => \$skip_target{$_},
		"only-$optname" => \$only_target{$_},
		"add-$optname"  => \$add_target{$_},
	       );
	   } @targets),
	   "debug=i" => \$debug,
	   "local-language=s" => \$add_meta{local_language},
	   "city-names=s"     => \$add_meta{city_names},
	   'neighbours=s'     => \$add_meta{neighbours},
	   "other-names:s"    => \$add_meta{other_names},
	   "region=s"         => \$add_meta{region},
	   "force!" => \$force,
	   # targets with additional arguments
	   "dataset-title=s" => \$set_dataset_title_octets,
	   # XXX
	   "coastline-hack-anchor=s" => \$coastline_hack_anchor,
	   "addr-suburbs-with-berlin-data" => \$addr_suburbs_with_berlin_data,
	   'addr-suburbs-filter-city=s@' => \@addr_suburbs_filter_city,
	   "h|help" => sub { usage },
	   "v|version" => sub {
	       print basename($0) . " $VERSION\n";
	       exit 0;
	   },
	  )
    or usage;

my $miscsrcdir = realpath $FindBin::RealBin;
my $bbbikedir = dirname $miscsrcdir;
my $datadir = shift or usage "Please specify data directory";
$datadir = realpath $datadir;
chdir $datadir or die "Can't chdir to $datadir: $!";

my $set_dataset_title;
if (defined $set_dataset_title_octets) {
    require Encode;
    $set_dataset_title = Encode::decode_utf8($set_dataset_title_octets);
}

if (grep { $_ } values %only_target) {
    @run_targets = grep { $only_target{$_} } @targets;
}

my %targets_to_run = map { ($_, 1) } grep { !$skip_target{$_} } @run_targets;
for my $target (keys %add_target) {
    $targets_to_run{$_} = 1 if $add_target{$target};
}

for my $target (@run_targets) {
    if ($skip_target{$target}) {
	print STDERR "Skip target $target.\n";
	next;
    }
    delete $targets_to_run{$target};
    no strict 'refs';
    my $target_sub = "target_$target";
    if (!defined &{$target_sub}) {
	die "No subroutine for $target exists";
    }
    print STDERR "Running target $target ... " if $debug;
    &{$target_sub};
    print STDERR "\n" if $debug;
}

sub target_addr_sort {
    if (!-e "_addr") {
	print STDERR "_addr does not exist, skipping";
	return;
    }
    require Strassen::Core;
    my $glob_dir = Strassen->get_global_directives('_addr');
    if (($glob_dir->{strassen_lookup_suitable}->[0]||'') eq 'yes') {
	print STDERR "up-to-date";
	return;
    }
    require GeocoderAddr;
    my $geocoderaddr = GeocoderAddr->new('_addr');
    $geocoderaddr->convert_for_lookup('_addr_sorted');
    rename '_addr_sorted', '_addr'
	or die "Renaming _addr_sorted to _addr failed: $!";
}

sub target_addr_suburbs {
    if (!-e "_addr") {
	print STDERR "_addr does not exist, skipping";
	return;
    }
    require Strassen::Core;
    my $glob_dir = Strassen->get_global_directives('_addr');
    if (($glob_dir->{columns}->[0]||'') =~ /(^|,)suburb($|,)/ && _is_up_to_date("_addr", ["cityparts"])) {
	# don't print "up-to-date", _is_up_to_date already does this
	return;
    }

    my %street_zip_to_citypart;
    if ($addr_suburbs_with_berlin_data) {
	require PLZ;
	my $plz = PLZ->new;
	$plz->load;
	for my $rec (@{ $plz->{Data} }) {
	    my $str = $rec->[PLZ::FILE_NAME()];
	    $str =~ s{(s)tr\.}{$1trae}i; # normalize for osm
	    $street_zip_to_citypart{$str}->{$rec->[PLZ::FILE_ZIP()]}->{$rec->[PLZ::FILE_CITYPART()]} = 1;
	}
	# turn cityparts into comma-separated list
	while(my($k1,$v1) = each %street_zip_to_citypart) {
	    while(my($k2,$v2) = each %{ $street_zip_to_citypart{$k1} }) {
		$street_zip_to_citypart{$k1}->{$k2} = join(", ", sort keys %{ $street_zip_to_citypart{$k1}->{$k2} });
	    }
	}
    }

    # XXX the following should go to a separate module XXX
    # XXX quite slow, takes currently ~20min for Berlin's _addr

    # XXX check errors using this method:
    # : cat /tmp/_addr.new | fgrep '|Berlin|' | grep -P '\|\t' | less
    # -> currently 22 errors, all of them seem to be mapping errors
    # or buildings very close to the border
    require VectorUtil;

    my %citypart_to_polygons;
    {
	my $cityparts_s = Strassen->new_stream('cityparts');
	$cityparts_s->read_stream
	    (sub {
		 my($r) = @_;
		 my @c = map { [ split /,/, $_] } @{ $r->[Strassen::COORDS()] };
		 push @{ $citypart_to_polygons{$r->[Strassen::NAME()]} }, \@c;
	     });
    }

    my $addr_s = Strassen->new_stream('_addr');
    my $new_addr_s = Strassen->new;
    $addr_s->read_stream
	(sub {
	     my($r) = @_;
	     my @f = split /\|/, $r->[Strassen::NAME()];
	     my $p = [ split /,/, $r->[Strassen::COORDS()]->[0] ];
	     my($strname,$zip,$city) = @f[0,2,3];
	     if (@addr_suburbs_filter_city && !first(sub { $_ eq $city }, @addr_suburbs_filter_city)) {
		 warn "ignore @f because it's not in @addr_suburbs_filter_city...\n"
		     if $debug >= 2;
		 next;
	     }

	     my $found_citypart;
	 CITYPARTS: {
		 if ($addr_suburbs_with_berlin_data && $city eq 'Berlin') {
		     $found_citypart = $street_zip_to_citypart{$strname}->{$zip};
		     last CITYPARTS if defined $found_citypart;
		     warn "Nothing found for $r->[Strassen::NAME()], need to run slow cityparts fallback...\n"
			 if $debug >= 2;
		 }

		 for my $citypart (keys %citypart_to_polygons) {
		     for my $polygon (@{ $citypart_to_polygons{$citypart} }) {
			 if (VectorUtil::point_in_polygon($p, $polygon)) {
			     $found_citypart = $citypart;
			     last CITYPARTS;
			 }
		     }
		 }
	     }
	     if (!$found_citypart) {
		 warn "No citypart found for $r->[Strassen::NAME()].\n"
		     if $debug;
		 $found_citypart = '';
	     }
	     $r->[Strassen::NAME()] .= "|$found_citypart";
	     $new_addr_s->push_ext($r);
	 });
    $new_addr_s->set_global_directives($addr_s->get_global_directives);
    $new_addr_s->set_global_directive(columns => 'street,housenunmber,postcode,city,suburb');
    my $dest_file = "/tmp/_addr.new"; #XXX should write to final destination
    $new_addr_s->write($dest_file);
    warn "NOTE: currently the file _addr is not overwritten, but the change was written to $dest_file\n";
}

sub target_zip_points {
    return if _is_up_to_date("_zip_points", ["_addr"]);
    if (!-e "_addr") {
	print STDERR "_addr does not exist, skipping";
	return;
    }

    my @colors = (
		  '#ff0000', '#00ff00', '#0000ff', '#ffff00', '#ff00ff', '#00ffff',
		  '#800000', '#008000', '#000080', '#808000', '#800080', '#008080',
		 );
    my $color_i = -1;
    my %zip_to_color;
    my $allocate_new_color = sub {
	$color_i++;
	if ($color_i > $#colors) {
	    $color_i = 0;
	}
	$colors[$color_i];
    };
    my $get_zip_color = sub {
	my($zip) = @_;
	my $color = $zip_to_color{$zip};
	return $color if $color;
	$color = $allocate_new_color->();
	$zip_to_color{$zip} = $color;
    };

    require Strassen::Core;

    if (1 && is_in_path('cut') && is_in_path('sort')) {
	# First pass: get all postal codes, sort them and allocate colors.
	# This way the likelihood is increased that adjacent postal codes
	# get different colors.
	#
	# Theoretically could also use Strassen::Core, but
	# using unix tools like cut and sort is faster
	# (1.5s vs. 8s on a slow machine).
	open my $fh, 'cut -d"|" -f3 _addr | sort -u |'
	    or die $!;
	my %zip;
	while(<$fh>) {
	    chomp;
	    next if /^#/;
	    $zip{$_} = 1;
	}
	close $fh
	    or die $!;
	for my $zip (sort keys %zip) { # sorting nummerically or alphabetically shouldn't matter in most cases
	    $get_zip_color->($zip); # only using the side-effect of allocating a color
	}
    }

    my $s = Strassen->new_stream('_addr');
    open my $ofh, ">", "_zip_points~"
	or die "Can't write to _zip_points~: $!";

    # XXX hack: streaming interface cannot deal with global directives
    my $glob_dir = Strassen->get_global_directives('_addr');
    print $ofh Strassen::global_directives_as_string($glob_dir);
    my $encoding = $glob_dir->{encoding}->[0];
    Strassen::switch_encoding($ofh, $encoding) if $encoding;

    $s->read_stream
	(sub {
	     my($r) = @_;
	     my($str, $hnr, $zip, $place) = split /\|/, $r->[Strassen::NAME()];
	     my $color = $get_zip_color->($zip);
	     print $ofh join(" ", $str, $hnr, $zip, $place) . "\t$color " . $r->[Strassen::COORDS()]->[0] . "\n";
	 });
    close $ofh
	or die "Error while writing to _zip_points~: $!";
    rename "_zip_points~", "_zip_points"
	or die "Error while renaming _zip_points~ to _zip_points: $!";
}

sub target_wasserstrassen_lowres {
    return if _is_up_to_date("wasserstrassen-lowres", ["wasserstrassen"]);

    my(undef, $tmp1) = tempfile(UNLINK => 1, SUFFIX => "-wlores.bbd") or die $!;
    my(undef, $tmp2) = tempfile(UNLINK => 1, SUFFIX => "-wlores.bbd") or die $!;
    my $cmd = <<EOF;
set -e
cat wasserstrassen | $miscsrcdir/grepstrassen --catrx '^(F:W1)\$' > $tmp1
cat wasserstrassen | $miscsrcdir/grepstrassen --catrx '^(W1|W2|F:W|F:I)\$' --minarea 3 >> $tmp1
$^X $miscsrcdir/simplify_streets -tolerance 300 -algorithm 'douglas-peucker' $tmp1 > $tmp2
mv -f $tmp2 wasserstrassen-lowres
chmod ugo-wx,ugo+r wasserstrassen-lowres
EOF
    system $cmd;
    die "The command $cmd failed: $?" if $? != 0;
}

# Note: this is not perfect. It just picks the first of the cities
# with largest category. This relies on osm2bbd to sort the cities by
# population (which it currently does). This also sets "center_name".
sub target_largest_city_is_center {
    my $meta = eval { _load_meta() };
    if ($@) {
	print STDERR $@->{msg};
	return;
    }
    if (!$force && $meta->{center}) {
	print STDERR "center already set in meta.yml, do not overwrite, skipping" if $debug;
	return;
    }
    my $max = _get_largest_city();
    if (defined $max) {
	print STDERR "setting $max->[Strassen::NAME()] ($max->[Strassen::COORDS()][0]) as center ";
	$meta->{center} = [ map { $_+0 } split /,/, $max->[Strassen::COORDS()][0] ];
	if ($meta->{coordsys} eq 'wgs84') {
	    $meta->{center_wgs84} = [ @{ $meta->{center} } ]; # create a copy, yes; helps for the simple-minded Geography::FromMeta loader
	} elsif ($meta->{coordsys} eq 'bbbike') {
	    require Karte::Polar;
	    $Karte::Polar::obj = $Karte::Polar::obj if 0; # cease -w
	    $meta->{center_wgs84} = [ $Karte::Polar::obj->standard2map(@{ $meta->{center} }) ];
	} else {
	    die "Unhandled or invalid coordsys '$meta->{coordsys}'";
	}
	$meta->{center_name} = $max->[Strassen::NAME()];
	_write_meta($meta);
    }
}

# By default "dataset_title" is the name of the largest city, which is
# the same as "center_name". If you need another dataset_title,
# especially if the coverage is not a city but a region, then use the
# --dataset-title option
sub target_title_for_dataset {
    my $meta = eval { _load_meta() };
    if ($@) {
	print STDERR $@->{msg};
	return;
    }
    if (!$force && $meta->{dataset_title}) {
	print STDERR "dataset_title already set in meta.yml, do not overwrite, skipping" if $debug;
	return;
    }
    if (!defined $set_dataset_title) {
	my $max = _get_largest_city();
	if (defined $max) {
	    $set_dataset_title = $max->[Strassen::NAME()];
	} else {
	    print STDERR "Cannot guess dataset title, no largest city found.\n" if $debug;
	    return;
	}
    }
    print STDERR "setting $set_dataset_title as title for dataset " if $debug;
    $meta->{dataset_title} = $set_dataset_title;
    _write_meta($meta);
}

sub target_inaccessible_streets {
    return if _is_up_to_date("inaccessible_strassen", ["strassen", "gesperrt"]);

    # XXX This is a bug somewhere, maybe integer operation is used
    # somewhere?
    require Strassen::Core;
    my $glob_dir = Strassen->get_global_directives("$datadir/strassen");
    if ($glob_dir->{map} && $glob_dir->{map}[0] !~ m{^(bbbike|standard|wgs84|polar)$}) {
	print STDERR "only map=bbbike currently supported, skipping";
	return;
    }

    my(undef, $tmp) = tempfile(UNLINK => 1, SUFFIX => "-inaccessible.bbd") or die $!;
    my $cmd = <<EOF;
set -e
$miscsrcdir/search_inaccessible_points -stats -cache -shorter -street $datadir/strassen -street $datadir/faehren -blocked $datadir/gesperrt -blockedtype einbahn -blockedtype sperre > $tmp
mv -f $tmp inaccessible_strassen
chmod ugo-wx,ugo+r inaccessible_strassen
EOF
    system $cmd;
    die "The command $cmd failed: $?" if $? != 0;
}

sub target_strassen_cooked {
    return if _is_up_to_date("strassen-cooked", ["strassen", "inaccessible_strassen"]);
    die "Fatal: inaccessible_strassen not existent or empty, please make sure to run same-named postprocessing target.\n"
	if !-s "inaccessible_strassen";

    my @cmd = (
	       $^X, "$FindBin::RealBin/create_cooked_bbd",
	       "-o", "strassen-cooked",
	       "strassen", "inaccessible_strassen",
	      );
    if ($debug >= 2) {
	print STDERR "Running '@cmd'...\n";
    }
    system @cmd;
    if ($? != 0) {
	die "The command '@cmd' failed: $?";
    }
}

sub target_fragezeichen_cooked {
    return if _is_up_to_date("fragezeichen-cooked", ["strassen", "fragezeichen"]);

    my @cmd = (
	       $^X, "$FindBin::RealBin/create_cooked_bbd",
	       "-variant", "fragezeichen",
	       "-o", "fragezeichen-cooked",
	       "fragezeichen",
	       "strassen",
	      );
    if ($debug >= 2) {
	print STDERR "Running '@cmd'...\n";
    }
    system @cmd;
    if ($? != 0) {
	die "The command '@cmd' failed: $?";
    }
}

sub target_make_long_streets {
    die <<EOF;
NYI, Strassen::Combine still have bugs. Also there must be some "tag"
in the generated file to prevent useless re-calculation of the file.
EOF
}

sub target_mount {
    return if _is_up_to_date("mount", ["hoehe", "gesperrt", "strassen"]);

    my @cmd = ("$FindBin::RealBin/steigung_stat",
	       '-minmount', '1.0',
	       '-commentscompatible',
	       '-tolerant', # do not croak on linesegs with length=0
	       '-sperre', "$datadir/gesperrt",
	       '-str', "$datadir/strassen",
	       '-i', "$datadir/hoehe",
	       '-o', "$datadir/mount~"
	      );
    if ($debug >= 2) {
	print STDERR "Running '@cmd'...\n";
    }
    {
	local $ENV{PERL_HASH_SEED} = 0;
	local $ENV{PERL_PERTURB_KEYS} = 'NO';
	system @cmd;
	if ($? != 0) {
	    die "The command '@cmd' failed: $?";
	}
    }
    rename "$datadir/mount~", "$datadir/mount"
	or die "Can't rename mount~ to mount in $datadir: $!";
}

sub target_coastline_hack {
    if (!-e "$datadir/_coastline") {
	print STDERR "No _coastline file available, coastline hack not applicable...\n";
	return;
    }

    if (!$coastline_hack_anchor) {
	print STDERR "Please set -coastline-hack-anchor option!\n";
	return;
    }

    # check if coastline hack was already applied
    {
	my $do_truncate_at_pos;

	{
	    open my $fh, "$datadir/wasserstrassen"
		or do {
		    warn "'wasserstrassen' cannot be opened ($!), cannot apply coastline hack...\n";
		    return;
		};
	    my $last_pos = 0;
	    while(<$fh>) {
		if (m{^#: section coastline_hack vvv}) {
		    if (!$force) {
			print STDERR "up-to-date";
			return;
		    } else {
			$do_truncate_at_pos = $last_pos;
			last;
		    }
		}
		$last_pos = tell $fh;
	    }
	    close $fh; # explicitely close, don't rely on scope exit
	}

	# Don't use "truncate $fh" for compatibility with Win32.
	if (defined $do_truncate_at_pos) {
	    truncate "$datadir/wasserstrassen", $do_truncate_at_pos
		or die "Cannot truncate '$datadir/wasserstrassen' to position $do_truncate_at_pos: $!";
	}
    }

    my $meta = eval { _load_meta() };
    if (!$meta) {
	print STDERR "Cannot apply coastline hack without meta file.\n";
	return;
    }

    my @add_coastline;
    {
	my $bbox = $meta->{bbox} or die "Unexpected: no bbox found in meta.yml!";
	if      ($coastline_hack_anchor eq 'nw') {
	    push @add_coastline, "$bbox->[0],$bbox->[3]";
	} elsif ($coastline_hack_anchor eq 'n') {
	    push @add_coastline, "$bbox->[0],$bbox->[3]", "$bbox->[2],$bbox->[3]"; # XXX order?
	} elsif ($coastline_hack_anchor eq 'ne') {
	    push @add_coastline, "$bbox->[2],$bbox->[3]";
	} elsif ($coastline_hack_anchor eq 'w') {
	    push @add_coastline, "$bbox->[0],$bbox->[1]", "$bbox->[0],$bbox->[3]"; # XXX order?
	} elsif ($coastline_hack_anchor eq 'e') {
	    push @add_coastline, "$bbox->[2],$bbox->[1]", "$bbox->[2],$bbox->[3]"; # XXX order?
	} elsif ($coastline_hack_anchor eq 'sw') {
	    push @add_coastline, "$bbox->[0],$bbox->[1]";
	} elsif ($coastline_hack_anchor eq 's') {
	    push @add_coastline, "$bbox->[0],$bbox->[1]", "$bbox->[2],$bbox->[1]"; # XXX order?
	} elsif ($coastline_hack_anchor eq 'se') {
	    push @add_coastline, "$bbox->[2],$bbox->[1]";
	} else {
	    die "Invalid or missing value for -coastline-hack-anchor (must be a Himmelsrichtung)"; # XXX english word?
	}
    }

    my @cmd = ("$FindBin::RealBin/combine_streets.pl",
	       "-closedpolygon",
	       "-encoding", "utf-8",
	       "$datadir/_coastline",
	      );
    my @lines;
    if ($debug >= 2) {
	print STDERR "Running '@cmd'...\n";
    }
    open my $fh, "-|", @cmd
	or die "While running @cmd: $!";
    while(<$fh>) {
	next if m{^#}; # ignore comment and directive lines, for now
	push @lines, $_;
    }
    close $fh
	or die "Problem while running @cmd: $!";
    @lines = sort { length($b) <=> length($a) } @lines;
    # XXX assume first one is the sea, and others are islands
    open my $ofh, ">>", "$datadir/wasserstrassen"
	or die "Can't append to $datadir/wasserstrassen: $!";
    binmode $ofh; # combine_streets.pl output is binary
    print $ofh "#: section coastline_hack vvv\n";
    for (@lines[1..$#lines]) {
	s{\tW}{\tF:I};
	print $ofh $_;
    }
    {
	chomp $lines[0];
	$lines[0] .= " @add_coastline"; # XXX see above, possible order problem...
	$lines[0] =~ s{\tW}{\tF:W};
	$lines[0] .= "\n";
	print $ofh $lines[0];
    }
    print $ofh "#: section ^^^^^^^^^^^^^^^^^^\n";
    close $ofh
	or die "While appending coastline_hack to $datadir/wasserstrassen: $!";

    if (!$targets_to_run{'wasserstrassen_lowres'}) {
	print STDERR <<EOF;
Consider to recreate wasserstrassen-lowres by using

    $^X $0 -only-wasserstrassen-lowres $datadir

EOF
    }
}

sub target_check_crossings {
    return if _is_up_to_date("crossings.bbd", ["strassen", "strassen_bab"]);

    my @cmd = ($^X, "$FindBin::RealBin/check_crossings",
	       "-encoding", "utf-8",
	       "-included-brunnels",
	       "-o", "$datadir/crossings.bbd",
	       "$datadir/strassen", "$datadir/strassen_bab",
	       ($debug >= 3 ? "-v-if-interactive" : ()),
	      );
    if ($debug >= 2) {
	print STDERR "Running '@cmd'...\n";
    }
    system @cmd;
    die "The command @cmd failed: $?" if $? != 0;
}

sub target_check_nearest {
    return if _is_up_to_date("check_nearest.bbd", ["strassen", "strassen_bab", "fragezeichen", "faehren"]);

    my @cmd = ($^X, "$FindBin::RealBin/check_nearest",
	       "-asbbd",
	       "-strfile", "$datadir/strassen",
	       "-strfile", "$datadir/strassen_bab",
	       "-strfile", "$datadir/fragezeichen",
	       "-strfile", "$datadir/faehren",
	       "-maxdist", "20",
	      );
    if ($debug >= 2) {
	print STDERR "Running '@cmd', writing to check_nearest.bbd...\n";
    }
    open my $fh, '-|', @cmd
	or die "While running @cmd: $!";
    open my $ofh, ">", "check_nearest.bbd~"
	or die "Cannot write to check_nearest.bbd~: $!";
    while(<$fh>) {
	print $ofh $_;
    }
    close $ofh
	or die "Error while writing to check_nearest.bbd~: $!";
    rename "check_nearest.bbd~", "check_nearest.bbd"
	or die "Error while renaming to check_nearest.bbd: $!";
}

sub target_icao_metar {
    return if _is_up_to_date("icao_metar", ["icao"]);

    # Check prerequisite data
    if (!-e "icao" || -z "icao") {
	print STDERR "no icao file in generated data, skipping";
	return;
    }

    # Check prerequisite tools
    my $gweather_locations_dir = "$ENV{HOME}/devel/gweather-locations";
    my $locations_yml = "$gweather_locations_dir/Locations2.yml";
    if (!-r $locations_yml) {
	print STDERR "no Locations2.yml available, skipping";
	if ($debug >= 2) {
	    warn "File is expected in $locations_yml";
	}
	return;
    }
    if (!eval { require BBBikeYAML; 1 }) {
	print STDERR "no BBBikeYAML available, skipping";
	return;
    }

    # Process
    my $locations = BBBikeYAML::LoadFile($locations_yml); # die if problems!
    my %icao_exists = map { ($_->[0] => 1) } @$locations;
    require Strassen::Core;
    my $s = Strassen->new_stream("$datadir/icao");
    my $new_s = Strassen->new;
    $s->read_stream(sub {
			my $r = shift;
			my $name = $r->[Strassen::NAME()];
			if (my($icao) = $name =~ m{^(.{4})\s}) {
			    if ($icao_exists{$icao}) {
				$new_s->push_ext($r);
			    } elsif ($debug >= 2) {
				warn "Skipping unsupported icao location <$icao> (full name: $name).\n";
			    }
			} elsif ($debug >= 2) {
			    warn "WARN: cannot parse name <$name>\n";
			}
		    });
    $new_s->set_global_directives($s->get_global_directives);
    $new_s->write("$datadir/icao_metar~");
    rename "$datadir/icao_metar~", "$datadir/icao_metar"
	or die "Error while renaming $datadir/icao_metar~ to $datadir/icao_metar: $!";
}

# Created out of admin_level=10 boundaries --- in Berlin these are the "Ortsteile", which
# is the largest entity with unique street names.
sub target_cityparts {
    my $admin_level = 10; # maybe could be made configurable if needed

    if (!-e "_boundary") {
	print STDERR "No _boundary file in generated data --- please run osm2bbd without --ignore-underline.\n";
	return;
    }

    return if _is_up_to_date("cityparts", ["_boundary"]);

    my($tmpfh,$tmpfile) = tempfile(UNLINK => 1, DIR => $datadir) or die $!;

    my @cmd = (
	       "$FindBin::RealBin/combine_streets.pl",
	       "-closedpolygon",
	       "$datadir/_boundary",
	      );
    if ($debug >= 2) {
	print STDERR "Running '@cmd'...\n";
    }
    open my $fh, "-|", @cmd
	or die "While running @cmd: $!";
    while(<$fh>) {
	if (/^#/) {
	    # keep
	} elsif (!/\tZ${admin_level} /) {
	    # skip
	    next;
	} else {
	    s/\tZ${admin_level}/\tZ/;
	}
	print $tmpfh $_;
    }
    close $fh
	or die "Problem while running @cmd: $!";
    rename $tmpfile, "$datadir/cityparts"
	or die "Can't rename temporary file to $datadir/cityparts: $!";
    chmod((0666 &~ umask), "$datadir/cityparts");
}

sub target_write_meta {
    require Encode;

    my $meta = eval { _load_meta() };

    for my $key (keys %add_meta) {
	my $val = $add_meta{$key};
	if (defined $val) {
	    $val = Encode::decode_utf8($val);
	    if ($key eq 'neighbours') { # special handling
		$val = eval $val;
		die qq{Error while eval'ing "$val": $@} if $@;
	    }
	    $meta->{$key} = $val;
	}
    }

    _write_meta($meta);
}

######################################################################

sub _load_meta {
    # assume to be in the right directory
    if (!eval { require BBBikeYAML; BBBikeYAML->import(qw(LoadFile DumpFile)); 1 } &&
	!eval { require YAML::Syck; YAML::Syck->import(qw(LoadFile DumpFile)); 1 } &&
	!eval { require YAML;       YAML      ->import(qw(LoadFile DumpFile)); 1 }
       ) {
	die +{ msg => "no YAML::XS/YAML/YAML::Syck available, skipping",
	       code => 'NO_YAML',
	     };
    }
    my $meta = eval { LoadFile("meta.yml") };
    if (!$meta) {
	die +{ msg => "cannot load meta.yml, skipping",
	       code => 'INVALID_META',
	     };
    }
    $meta;
}

sub _write_meta {
    my $meta = shift;

    require Data::Dumper;

    DumpFile("meta.yml~", $meta);

    open my $ofh, ">", "meta.dd~"
	or die "Cannot write to meta.dd~: $!";
    print $ofh Data::Dumper->new([$meta],['meta'])->Sortkeys(1)->Useqq(1)->Dump;
    close $ofh
	or die "While closing meta.dd~: $!";
    rename "meta.dd~", "meta.dd"
	or die "While renaming meta.dd~ to meta.dd: $!";
    rename "meta.yml~", "meta.yml"
	or die "While renaming meta.yml~ to meta.yml: $!";
}

# Returns a 'Strassen' record
sub _get_largest_city {
    my $max = undef;
    require Strassen::Core;
    my $s = Strassen->new_stream("$datadir/orte");
    $s->read_stream(sub {
			my $r = shift;
			if (!defined $max || $max->[Strassen::CAT()] < $r->[Strassen::CAT()]) {
			    $max = $r;
			}
		    }, UseLocalDirectives => 0);
    $max;
}

sub _is_up_to_date {
    my($file, $deps) = @_;
    return 0 if $force;
    return 0 if !-e $file;
    for my $dep (@$deps) {
	return 0 if -M $file > -M $dep;
    }
    print STDERR "up-to-date";
    return 1;
}

__END__

=head1 NAME

osm2bbd-postprocess - do additional postprocessing of the osm2bbd result

=head1 SYNOPSIS

    osm2bbd-postprocess [--debug=0..3] [-force] [options] datadirectory

=head1 DESCRIPTION

Do additional postprocessing of the result of a L<osm2bbd> run. If no
further options are given, then some default postprocessing tasks (or
"targets") will be run. Usually C<osm2bbd-postprocess> can determine
that a postprocessing task was already run and won't do it again
unless the C<-force> option is given.

Run C<osm2bbd-postprocess --help> to see a list of all options and all
tasks.

=head2 TASKS (TARGETS)

All listed tasks are run by default except noted otherwise.

=head3 wasserstrassen-lowres

Create a lowres variant of waterway items, suitable to be shown in the
overview window of the Perl/Tk application.

=head3 largest-city-is-center

Picks the largest city in the area and writes it as "center" and
"center_name" to the meta file.

=head3 title-for-dataset

The C<title-for-dataset> task can be given an option C<--dataset-title
I<title>>, which should be an utf8-encoded name for the dataset. If
not given, then the task would choose the name of the largest city in
the area as the dataset title.

=head3 inaccessible-streets

Find which streets are not accessible and create a new file
F<inaccessible_strassen>. Will be used e.g. for geocoding purposes to
exclude coordinates which cannot be reached anyway.

=head3 strassen-cooked

Create a new file F<strassen-cooked> which is a reduced set of
F<strassen> not containing inaccessible streets (see
L</inaccessible_streets>).

=head3 fragezeichen-cooked

Create a new file F<fragezeichen-cooked> which is a reduced set of
F<fragezeichen> not containing points already found in F<strassen>.

=head3 mount

Create a new file F<mount>. Not run by default.

=head3 coastline-hack

The C<coastline-hack> task needs the additional option
C<-coastline-hack-anchor> with any cardinal direction (e.g. C<n>,
C<nw> etc.) for the direction of the "open sea". Not run by default.

=head3 check-crossings

Run the L<check_crossings> script. Not run by default.

=head3 check-nearest

Run the L<check_nearest> script. Not run by default.

=head3 icao-metar

Create the F<icao_metar> file. This file may be used to find locations
in the data area which has weather data in the internet available. Not
run by default, as it uses dependencies currently not bundled with
bbbike.

=head3 addr-sort

The C<addr-sort> task sorts the F<_addr> file to be suitable for fast
L<GeocoderAddr> lookups.

=head3 zip-points

Create a new file F<_zip_points> which is a colored representation of
zip points. This is mainly used for debugging data problems. Not run
by default.

=head3 cityparts

The C<cityparts> task creates a new file F<cityparts>. Not run by
defauult.

=head3 addr-suburbs

The C<addr-suburbs> task adds a "suburb" (citypart) column to the
F<_addr> by matching points with the F<cityparts> file. This task is
currently quite slow (~20 minutes for data of size of Berlin). Not run
by default. The following options may also be specified:

=over

=item -addr-suburbs-filter-city I<cityname>

If specified, then filter out all records not matching any of the
given city names. May be specified multiple times.

=item -addr-suburbs-with-berlin-data

Experimental. Use the file F<Berlin.coords.data> for faster matching.
This cuts down the time to less than one minute. A notable change is
that instead of a specific suburb name a list of suburbs may be
generated per record.

=back

=head3 write-meta

The C<write-meta> task has to be called (e.g with C<--add-write-meta>
or C<--only-write-meta> switch) if any of the following options is
set:

=over

=item C<-local-language>

Sets the C<local_language> field in the meta file.

=item C<-city-names>

Sets the C<city_names> field in the meta file.

=item C<-neighbours>

Sets the C<neighbours> field in the meta file. The value is expected
to be a perl data structure which will be eval'ed.

=item C<-other-names>

Sets the C<other_names> field in the meta file.

=item C<-region>

Sets the C<region> field in the meta file.

=back

All of these expect values encoded as utf-8.

=head1 AUTHOR

Slaven Rezic

=head1 SEE ALSO

L<osm2bbd>
