#!/usr/bin/perl -w
# -*- perl -*-

#
# Author: Slaven Rezic
#
# Copyright (C) 2008,2009,2010,2011,2012,2018,2025,2026 Slaven Rezic. All rights reserved.
# This program is free software; you can redistribute it and/or
# modify it under the same terms as Perl itself.
#
# WWW:  https://github.com/eserte/bbbike
#

# Download a data from openstreetmap into the specified directory
# (option -o). The dataset may be used by osm2bbd for further
# conversion.

# Call script without arguments for usage.

use strict;
use FindBin;
use lib "$FindBin::RealBin/..";

use File::Basename qw(dirname);
use File::Glob qw(bsd_glob);
use Getopt::Long;
use LWP::UserAgent;

use BBBikeUtil qw(is_in_path);

sub save_pwd (&);

our $VERSION = "0.08";

use constant TIMEOUT => 60;

use vars qw($OSM_API_URL);
$OSM_API_URL = "https://api.openstreetmap.org/api/0.6";
#$OSM_API_URL = "https://xapi.openstreetmap.org/api/0.6";
#$OSM_API_URL = "http://osmxapi.hypercube.telascience.org/api/0.6";
#$OSM_API_URL = "https://www.informationfreeway.org/api/0.6";

my $ltlnqr = qr{([-+]?\d+(?:\.\d+)?)};
my $osm_download_file_qr       = qr{/download_$ltlnqr,$ltlnqr,$ltlnqr,$ltlnqr\.osm(?:\.gz|\.bz2)?$};
my $short_osm_download_file_qr =           qr{$ltlnqr,$ltlnqr,$ltlnqr,$ltlnqr\.osm(?:\.gz|\.bz2)?$};

my $debug = 1;
my $do_reload;
my $do_reload_or_download;
my $min_age;
my $n;
my $o;
my $xstep = 0.01;
my $ystep = 0.01;
my $do_round;
my $do_report;
my $socksproxy;
my $timeout = TIMEOUT;

sub usage () {
    die <<EOF;
usage: $0 [options] -o directory -- x0 y0 x1 y1
       $0 [options] -reload existing.osm ...
       $0 [options] -reload-or-download possibly-non-existing.osm ...

First form:  specify a bbox (four parameters, x0 y0 x1 y1)
Second form: refresh existing osm files downloaded with this tool
             Alternatively specify a directory containing osm files
Third form:  like second form, but files must not exist yet

-n:        no exec
-debug i:  set debugging/verbosity level (default=$debug)
-xstep x:  set steps on the x axis (default=$xstep)
-ystep y:  set steps on the y axis (default=$ystep)
-step  xy: set steps for both the x and y axis
-round:    make sure bbox is rounded to 1/10 or 1/100 degree
           NOTE: works only with -x/ystep 0.1 or 0.01!
-report:   just report what URLs would be loaded to STDOUT

-socksproxy=socks://host:port 	set socks proxy
-timeout seconds 	set timeout, (default $timeout seconds)

With -reload and -reload-or-download:
-minage days: download only files which are older than days days
              handy if a download process aborted and should be
              restarted
-osm-api-url URL (default $OSM_API_URL )

For all of Berlin use:
    $0 -o destdir 13.01 52.33 13.77 52.68
EOF
}

GetOptions("reload!" => \$do_reload,
	   "reload-or-download!" => \$do_reload_or_download,
	   "minage=f" => \$min_age,
	   "o=s" => \$o,
	   "debug=i" => \$debug,
	   "n" => \$n,
	   "xstep=f" => \$xstep,
	   "ystep=f" => \$ystep,
	   "step=f" => sub {
	       $xstep = $ystep = $_[1];
	   },
	   "round!" => \$do_round,
	   "report!" => \$do_report,
	   "socksproxy=s" => \$socksproxy,
	   "timeout=i" => \$timeout,
	   "osm-api-url=s" => \$OSM_API_URL,
	  )
    or usage;

($do_report && $do_reload_or_download)
    and die "Can't specify -reload and -reload-or-download together.\n";

my $ua = eval {
    require LWP::UserAgent;
    my $ua = LWP::UserAgent->new;
    #$ua->agent("The Hare and the Hedgehog");

    ## the download server seems to block this user agent -> use the LWP default
    $ua->agent("downloadosm/$VERSION LWP/$LWP::VERSION [part of BBBike]");
    my @decodable = grep { $_ ne 'deflate' } HTTP::Message::decodable(); # script cannot handle deflate correctly
    $ua->default_header('Accept-Encoding' => join(", ", @decodable));
    $ua->timeout($timeout);
    $ua;
};
if ($@) {
    warn "Cannot load LWP::UserAgent, will fallback to wget or curl...\n";
}
activate_socksproxy($socksproxy) if $socksproxy;

my @reload;
if ($do_reload || $do_reload_or_download) {
    @reload = @ARGV;
    if (!@reload) {
	if ($do_reload_or_download) {
	    die "Please specify .osm files to download or reload!\n";
	} else {
	    die "Please specify .osm files to reload!\n";
	}
    }
} else {
    if (!@ARGV) {
	usage;
    }
}

if (@reload == 1 && -d $reload[0]) {
    @reload = bsd_glob("$reload[0]/*.osm");
}

if (@reload) {
    if ($min_age) {
	@reload = grep { (!-f $_ && $do_reload_or_download) || -M $_ >= $min_age || -z $_ } @reload;
    }
    if ($do_reload) {
	for my $reload (@reload) {
	    if (!-r $reload) {
		die "The file <$reload> does not exist or is not readable.\n";
	    }
	}
    }
    for my $reload (@reload) {
	my $ltlnqr = qr{([-+]?\d+(?:\.\d+)?)};

	if ($reload !~ $short_osm_download_file_qr) {
	    die "The file name <$reload> does not look correct, i.e. created by this tool.\n";
	}
	my($x,$y,$xend,$yend) = ($1,$2,$3,$4);
	save_pwd {
	    my $dir = dirname($reload);
	    chdir $dir
		or die "Can't chdir to $dir: $!";
	    download_osm_tile($x,$y,$xend,$yend, 1);
	};
    }
} else {
    my @bbox = @ARGV;
    if (@bbox == 2) { # assume x1,y1 x2,y2
	@bbox = ((split /,/, $bbox[0]),
		 (split /,/, $bbox[1])
		);
	if (grep { !defined $_ } @bbox) {
	    die "Input arguments does not seem to be in format x1,y1 x2,y2";
	}
    }
    if (@bbox != 4) {
	usage;
    }
    if (!$o) {
	warn "-o option is missing\n";
	usage;
    }
    if (!-d $o) {
	warn "Creating directory $o...\n" if $debug;
	mkdir $o;
    }
    chdir $o
	or die "Cannot change to directory $o: $!\n";

    my($x1,$y1,$x2,$y2) = @bbox;

    ($x1,$x2) = ($x2,$x1) if $x1 > $x2;
    ($y1,$y2) = ($y2,$y1) if $y1 > $y2;

    if ($do_round) {
	if (eval { require Math::Round; 1 }) {
	    $x1 = Math::Round::nlowmult($xstep, $x1);
	    $y1 = Math::Round::nlowmult($ystep, $y1);
	    $x2 = Math::Round::nhimult($xstep, $x2);
	    $y2 = Math::Round::nhimult($ystep, $y2);
	} else {
	    if ($xstep != 0.01 && $xstep != 0.1) {
		die "-round option works only with -xstep=0.01 or 0.1 (or install Math::Round)!";
	    }
	    if ($ystep != 0.01 && $ystep != 0.05 && $ystep != 0.1) {
		die "-round option works only with -ystep=0.01 or 0.1 (or install Math::Round)!";
	    }
	    if ($xstep != $ystep) {
		die "Currently -xstep has to be equal -ystep for -round option (or install Math::Round)!";
	    }
	    my $dqr = $xstep == 0.1 ? '\d' : '\d\d'; $dqr = qr{$dqr};
	    my $up = sub {
		my $dir = $_[1];
		if ($_[0] =~ m{\.$dqr.*[1-9]}) {
		    $_[0] += ($dir * $xstep);
		    $_[0] =~ s{(\.$dqr).*}{$1};
		}
	    };
	    my $down = sub {
		$_[0] =~ s{(\.$dqr).*}{$1};
	    };

	    for ($x1, $y1) {
		if ($_ >= 0) {
		    $down->($_);
		} else {
		    $up->($_, -1);
		}
	    }
	    for ($x2, $y2) {
		if ($_ >= 0) {
		    $up->($_, +1);
		} else {
		    $down->($_);
		}
	    }
	}
    }

    use constant FIXPOINTS => 6;
    my $xstep_fix = FixPoint->new($xstep, FIXPOINTS);
    my $ystep_fix = FixPoint->new($ystep, FIXPOINTS);
    my $x1_fix    = FixPoint->new($x1, FIXPOINTS);
    my $x2_fix    = FixPoint->new($x2, FIXPOINTS);
    my $y1_fix    = FixPoint->new($y1, FIXPOINTS);
    my $y2_fix    = FixPoint->new($y2, FIXPOINTS);

    if ($debug) {
	warn "Approx. number of tiles to download: " .
	    (int(($y2_fix-$y1_fix)/$ystep_fix) * int(($x2_fix-$x1_fix)/$xstep_fix)) .
		"\n";
    }

    for(my $x_fix = $x1_fix; $x_fix < $x2_fix; $x_fix+=$xstep_fix) {
	for(my $y_fix = $y1_fix; $y_fix < $y2_fix; $y_fix+=$ystep_fix) {
	    my $xend_fix = $x_fix+$xstep_fix;
	    my $yend_fix = $y_fix+$ystep_fix;
	    download_osm_tile($x_fix->numify,$y_fix->numify,$xend_fix->numify,$yend_fix->numify, 0);
	} 
    }
}

sub activate_socksproxy {
    my $socksproxy = shift;

    if (!$ua) {
	warn "LWP::UserAgent is not set, ignore socks proxy!\n";
	return;
    }

    eval {
    	require LWP::Protocol::socks;
    	$ua->proxy([qw(http https)] => $socksproxy);
    };

    if ($@) {
    	warn "Cannot load LWP::Protocol::socks, ignore socks proxy...\n";
	return;
    }
}

sub osm_round {
    my $real = shift;

    my $number = shift || 100_000;
 
    my $round = int($real * $number + 0.5) / $number;

    if ($debug >= 3) {	
    	warn "$real <-> $round ", $real - $round, "\n" if $real != $round;
    }

    return $round;
}

sub download_osm_tile {
    my($x,$y,$xend,$yend, $reload) = @_;
    my $url = "$OSM_API_URL/map?bbox=$x,$y,$xend,$yend";
    my $dest = "download_$x,$y,$xend,$yend.osm";

    if ($do_report) {
	if (-s "$dest.bz2") {
	    print "$dest.bz2\t$url\t1\n";
	} elsif (-s "$dest.gz") {
	    print "$dest.gz\t$url\t1\n";
	} elsif (-s $dest) {
	    print "$dest\t$url\t1\n";
	} else {
	    print "$dest\t$url\t0\n";
	}
	return;
    }
    if ((-s $dest || -s "$dest.gz") && !$reload) {
	warn "$dest already exists, skipping...\n" if $debug;
    } else {
	if ($ua) {
	    if ($n) {
		warn "Would mirror $url to $dest\n";
	    } else {
		my $real_dest = $dest;
		if ($debug >= 1) {
		    warn "Mirroring $url -> $dest...\n";
		}
		my $resp = $ua->mirror($url, $dest);
		if ($debug >= 3) {
		    warn "Request:\n" . $resp->request->as_string . "\nResponse:\n" . $resp->dump;
		}
		if (!$resp->is_success) {
		    die "No success while mirroring '$url' to '$dest': " . $resp->dump;
		} else {
		    no warnings 'uninitialized'; # content-encoding header may be missing
		    if ($resp->header('content-encoding') eq 'gzip' && $dest !~ m{\.gz$}) {
			if ($debug >= 1) {
			    warn "Rename $dest -> $dest.gz...\n";
			}
			rename $dest, "$dest.gz"
			    or die "Cannot rename $dest to $dest.gz: $!";
			$real_dest = "$dest.gz";
		    } elsif ($dest !~ m{\.gz$}) {
			system('gzip', '-f', $dest);
			die "gzip $dest: $@" if $? != 0;
			$real_dest = "$dest.gz";
		    }
		}
		if ($real_dest =~ /\.gz$/) {
		    _maybe_convert_zlib($real_dest);
		}
	    }
	} else {
	    my @cmd;
	    if ($ua) {
	    } elsif (is_in_path("wget")) {
		@cmd = ('wget', '--timeout=' . $timeout);
		push @cmd, '--verbose' if $debug >= 2;
		push @cmd, '-O', $dest, $url;
	    } elsif (is_in_path("curl")) {
		@cmd = ('curl', '-s', '-S', '-f', '-m', $timeout); 
		push(@cmd, '--verbose') if $debug >= 2;
		push(@cmd, ('-o', $dest, $url));
	    } else {
		die "Neither wget nor curl available, cannot proceed";
	    }

	    if ($n) {
		warn "Would: @cmd\n";
	    } else {
		system(@cmd) == 0
		    or die "Failed while running <@cmd>: $?";
	    }
	}
    }
}

{
    package FixPoint;
    use constant FIXPOINTSDOTS => 2;


    use overload
	'+'  => 'add',
	'0+' => 'numify',
	'fallback' => 1,
	;
    sub new {
	my($class, $number, $prec) = @_;
	$prec = 0 if !defined $prec;
	$number *= 10**$prec;
	bless [$number, $prec], $class;
    }
    sub add {
	my($self, $other) = @_;
	die "NYI" if $self->[1] != $other->[1];
	bless [$self->[0] + $other->[0], $self->[1]], ref $self;
    }
    sub numify {
	my($self) = @_;
	my $result = $self->[0] / 10**$self->[1];

        my $padding = 1;

        if ($padding) {
	   $result .= '.' if $result !~ /\./;
	   $result =~ /\.(\d*)$/;
           for(my $i = length($1); $i < FIXPOINTSDOTS; $i++) {
		$result .= '0';
           }
        }

	return $result;
    }
}

# openstreetmap started to generate zlib content, which is gzip
# minus some headers. gzip/gunzip can handle such files, but
# XML::LibXML cannot, so we decompress and compress such files.
# Requires IPC::Run.
sub _maybe_convert_zlib {
    my $file = shift;
    if (open my $ifh, '<', $file) {
	binmode $ifh;
	read($ifh, my $buf, 2);
	if ($buf =~ /^\x78/) {
	    if ($debug >= 1) {
		warn "INFO: zlib file detected --- need to decompress and recompress it.\n";
	    }
	} else {
	    # probably a proper gzip file
	    return;
	}
    }

    require IPC::Run;
    IPC::Run::run(['gzip', '-cd', $file], '|', ['gzip'], '>', "$file~")
	    or die "Error while decompressing and recompressing file $file to $file~...\n";
    -s "$file~"
	or die "Destination file $file~ is empty";
    rename "$file~", $file
	or die "Error renaming $file~ to $file: $!";
}

# REPO BEGIN
# REPO NAME save_pwd /home/e/eserte/work/srezic-repository 
# REPO MD5 0f7791cf8e3b62744d7d5cfbd9ddcb07
sub save_pwd (&) {
    my $code = shift;
    require Cwd;
    my $pwd = Cwd::getcwd();
    eval {
	$code->();
    };
    my $err = $@;
    chdir $pwd or die "Can't chdir back to $pwd: $!";
    die $err if $err;
}
# REPO END

__END__

=head1 NAME

downloadosm - download data from openstreetmap

=head1 SYNOPSIS

    downloadosm -o directory -- x0 y0 x1 y1
    downloadosm -reload existing.osm
    downloadosm -reload-or-download possibly-non-existing.osm

=head1 DESCRIPTION

downloadosm downloads osm tiles.

With the first form a bounding box (x=longitude, y=latitude) has to be
specified and an output directory for the downloaded tiles.

With the second form existing osm files may be refreshed. The bounding
box of the osm file is determined from the filename (which includes
the tile bbox).

Run C<downloadosm --help> for more options.

=head1 AUTHOR

Slaven Rezic

=head1 SEE ALSO

L<osm2bbd>.

=cut
