"""Pull in what's available for HFMETAR MADIS data

Run from RUN_20_AFTER.sh for current hour
Run from RUN_40_AFTER.sh for current hour and 2 hours ago
Run from RUN_59_AFTER.sh for current hour
"""

import os
import sys
import warnings
from datetime import datetime, timedelta, timezone

import click
import numpy as np
from metar import Metar
from metpy.units import masked_array, units
from netCDF4 import chartostring
from psycopg import DataError
from pyiem.database import get_dbconnc
from pyiem.observation import Observation
from pyiem.reference import TRACE_VALUE
from pyiem.util import convert_value, logger, mm2inch, ncopen, utc
from tqdm import tqdm

warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.simplefilter("ignore", RuntimeWarning)
LOG = logger()


def vsbyfmt(val):
    """Tricky formatting of vis"""
    # NB: we aren't dealing with exact round numbers here akrherz/iem#255
    if val == 0:
        return 0
    if val <= 0.07:
        return "1/16"
    if val <= 0.13:
        return "1/8"
    if val <= 0.26:
        return "1/4"
    if val <= 0.38:
        return "3/8"
    if val <= 0.51:
        return "1/2"
    if val <= 1.1:
        return "1"
    if val <= 1.6:
        return "1 1/2"
    if val <= 2.1:
        return "2"
    if val <= 2.6:
        return "2 1/2"
    return int(val)


def process_sky(data: dict, skycs: list[str], skyls: list):
    """Process the sky cover"""
    mtr = ""
    for i, (skyc, skyl) in enumerate(zip(skycs, skyls, strict=True), start=1):
        if skyc == "":
            continue
        data[f"skyc{i}"] = skyc
        if skyc != "CLR":
            data[f"skyl{i}"] = int(np.round(skyl, 0))  # GH287
            mtr += f"{skyc}{(skyl / 100.0):03.0f} "
        else:
            mtr += "CLR "
    return mtr


def safe_convert(
    data: dict, col: str, ncfn: str, stations: list[str]
) -> list[str]:
    """Be careful."""
    res = []
    try:
        return chartostring(data[col][:])
    except UnicodeDecodeError:
        LOG.warning("Encountered bad unicode in %s, %s", col, ncfn)
        # Try one at a time
        for i, station in enumerate(stations):
            try:
                res.append(str(chartostring(data[col][i])))
            except UnicodeDecodeError:

                def safe_decode(val):
                    if np.ma.is_masked(val):
                        return ""
                    return val.decode("ascii", errors="replace")

                LOG.warning(
                    "Bad recNum:%s @%s entry for %s %s",
                    i,
                    datetime(1970, 1, 1)
                    + timedelta(seconds=data["observationTime"][i]),
                    station,
                    "".join(safe_decode(x) for x in data[col][i]),
                )
                res.append("")
    # Ensure no null bytes
    return [a.replace("\x00", "") for a in res]


def process(ncfn):
    """Process this file"""
    pgconn, icursor = get_dbconnc("iem")
    xref = {}
    icursor.execute(
        "SELECT id, network from stations where "
        "(network ~* 'ASOS' and country = 'US') or id in ('PGSN')"
    )
    for row in icursor:
        xref[row["id"]] = row["network"]
    icursor.close()
    nc = ncopen(ncfn)
    data = {}
    for vname in [
        "stationId",
        "observationTime",
        "temperature",
        "dewpoint",
        "altimeter",  # Pa
        "windDir",
        "windSpeed",  # mps
        "windGust",  # mps
        "visibility",  # m
        "precipAccum",
        "presWx",
        "skyCvr",
        "skyCovLayerBase",
        "autoRemark",
        "operatorRemark",
    ]:
        data[vname] = nc.variables[vname][:]
        for qc in ["QCR", "QCD"]:
            vname2 = vname + qc
            if vname2 in nc.variables:
                data[vname2] = nc.variables[vname2][:]
    for vname in ["temperature", "dewpoint"]:
        data[vname + "C"] = convert_value(data[vname], "degK", "degC")
        data[vname] = convert_value(data[vname], "degK", "degF")
    for vname in ["windSpeed", "windGust"]:
        data[vname] = (
            masked_array(data[vname], units("meter / second"))
            .to(units("knots"))
            .magnitude
        )

    data["altimeter"] = convert_value(data["altimeter"], "pascal", "inch_Hg")
    data["skyCovLayerBase"] = convert_value(
        data["skyCovLayerBase"], "meter", "foot"
    )
    data["visibility"] = convert_value(data["visibility"], "meter", "mile")
    data["precipAccum"] = mm2inch(data["precipAccum"])
    # Assume that this is always OK, ha
    stations = chartostring(data["stationId"][:])
    # MADIS folks have refused to fix the GIGO, so downstream folks can't punt
    presentwxs = safe_convert(data, "presWx", ncfn, stations)
    skycs = safe_convert(data, "skyCvr", ncfn, stations)
    autoremarks = safe_convert(data, "autoRemark", ncfn, stations)
    opremarks = safe_convert(data, "operatorRemark", ncfn, stations)

    def decision(i, fieldname, tolerance):
        """Our decision if we are going to take a HFMETAR value or not"""
        if data[fieldname][i] is np.ma.masked:
            return None
        # Found troubles with values of 255 not getting QCd
        if (
            fieldname in ["windSpeed", "windGust"]
            and data[fieldname][i] >= 250
        ):
            return None
        if data[f"{fieldname}QCR"][i] == 0:
            return data[fieldname][i]
        # Now we have work to do
        departure = np.ma.max(np.ma.abs(data[f"{fieldname}QCD"][i, :]))
        if departure <= tolerance:
            return data[fieldname][i]
        return None

    pgconn, icursor = get_dbconnc("iem")
    updates = 0
    for i, sid in tqdm(
        enumerate(stations),
        total=len(stations),
        disable=(not sys.stdout.isatty()),
    ):
        if len(sid) < 3:
            continue
        sid3 = sid.removeprefix("K")
        ts = datetime(1970, 1, 1) + timedelta(
            seconds=data["observationTime"][i]
        )
        ts = ts.replace(tzinfo=timezone.utc)

        mtr = f"{sid} {ts:%d%H%M}Z AUTO "
        network = xref.get(sid3, "ASOS")
        iem = Observation(sid3, network, ts)

        #  06019G23KT
        val = decision(i, "windDir", 15)
        if val is not None:
            iem.data["drct"] = round(val, 0)
            mtr += f"{iem.data['drct']:03.0f}"
        else:
            mtr += "///"

        val = decision(i, "windSpeed", 10)
        if val is not None:
            iem.data["sknt"] = round(val, 0)
            mtr += f"{iem.data['sknt']:02.0f}"
        else:
            mtr += "//"

        val = decision(i, "windGust", 10)
        if val is not None and val > 0:
            iem.data["gust"] = round(val, 0)
            mtr += f"G{iem.data['gust']:02.0f}"
        mtr += "KT "

        val = decision(i, "visibility", 4)
        if val is not None:
            iem.data["vsby"] = float(val)
            mtr += f"{vsbyfmt(iem.data['vsby'])}SM "

        presentwx = presentwxs[i]
        if presentwx != "":
            # database storage is comma delimited
            iem.data["wxcodes"] = presentwx.replace("\x00", "").split(" ")
            mtr += f"{presentwx} "

        mtr += process_sky(iem.data, skycs[i], data["skyCovLayerBase"][i])

        t = ""
        tgroup = "T"
        val = decision(i, "temperature", 10)
        if val is not None:
            # Recall the pain enabling this
            tmpc = float(data["temperatureC"][i])
            tc = tmpc if tmpc > 0 else (0 - tmpc)
            tf = "M" if tmpc < 0 else ""
            t = f"{tf}{tc:02.0f}/"
            tf = "1" if tmpc < 0 else "0"
            tgroup += f"{tf}{(tc * 10.0):03.0f}"
        val = decision(i, "dewpoint", 10)
        if t != "" and val is not None:
            tmpc = float(data["dewpointC"][i])
            tc = tmpc if tmpc > 0 else (0 - tmpc)
            tf = "M" if tmpc < 0 else ""
            t = f"{t}{tf}{tc:02.0f} "
            tf = "1" if tmpc < 0 else "0"
            tgroup += f"{tf}{(tc * 10.0):03.0f}"
        if len(t) > 4:
            mtr += t
        val = decision(i, "altimeter", 20)
        if val is not None:
            iem.data["alti"] = float(round(val, 2))
            mtr += f"A{(iem.data['alti'] * 100.0):.0f} "

        mtr += "RMK "
        val = decision(i, "precipAccum", 25)
        if val is not None:
            if val > 0.009:
                iem.data["phour"] = float(round(val, 2))
                mtr += f"P{(iem.data['phour'] * 100.0):04.0f} "
            elif val > 0:
                # Trace
                mtr += "P0000 "
                iem.data["phour"] = TRACE_VALUE

        if tgroup != "T":
            mtr += f"{tgroup} "

        if autoremarks[i] != "" or opremarks[i] != "":
            mtr += f"{autoremarks[i]} {opremarks[i]} "
        mtr += "MADISHF"
        # Account for some common cruft
        mtr = mtr.replace("!", "! ").replace("\x00", "")
        # Eat our own dogfood
        try:
            metar = Metar.Metar(mtr)
            iem.data["raw"] = " ".join(mtr.strip().split())
            # Glean peak wind gust
            if (
                metar.wind_speed_peak
                and metar.wind_dir_peak
                and metar.peak_wind_time
            ):
                iem.data["peak_wind_gust"] = metar.wind_speed_peak.value("KT")
                iem.data["peak_wind_drct"] = metar.wind_dir_peak.value()
                iem.data["peak_wind_time"] = metar.peak_wind_time.replace(
                    tzinfo=timezone.utc
                )
            if not iem.save(
                icursor, force_current_log=True, skip_current=True
            ):
                LOG.warning(
                    "Unknown station? %s %s %s %s",
                    sid3,
                    network,
                    ts,
                    mtr,
                )
            updates += 1
        except DataError as exp:
            LOG.warning(
                "DataError for %s %s %s",
                ncfn,
                iem.data,
                exp,
            )
        except Exception as exp:
            LOG.exception("dogfooding %s resulted in error", mtr, exc_info=exp)

        if updates % 100 == 0:
            icursor.close()
            pgconn.commit()
            icursor = pgconn.cursor()

    icursor.close()
    pgconn.commit()
    pgconn.close()


def find_fn(hours: int) -> str | None:
    """Figure out which file to run for"""
    utcnow = utc()
    for i in range(hours, 5):
        ts = utcnow - timedelta(hours=i)
        for j in range(300, -1, -1):
            fn = ts.strftime(f"/mesonet/data/madis/hfmetar/%Y%m%d_%H00_{j}.nc")
            if os.path.isfile(fn):
                return fn
    LOG.warning("no MADIS HFMETAR file found!")
    return None


@click.command()
@click.option("--hours", type=int, required=False)
@click.option("--filename", type=str, default=None)
def main(hours: int | None, filename: str | None) -> None:
    """Do Something"""
    if filename is None:
        filename = find_fn(hours)
    if filename:
        process(filename)


if __name__ == "__main__":
    main()
