#!/usr/bin/env bash
#
# Refresh the gallery dashboards that `gen_gallery.py` cannot rebuild on its own,
# then run the generator so every other page and `gallery.html` are rebuilt with
# the same binary.
#
# WHY THIS EXISTS (issue #4312)
# -----------------------------
# `gen_gallery.py` rebuilds every figure whose source data is committed. Four are
# built from datasets not committed here (Pittsburgh 311 is 269 MB, Boston 311 is
# 146 MB, `*.tsv*` is gitignored; the Brazil LPG and Colombia air-quality inputs
# are open-data exports kept out of the tree), so their committed HTML is reused
# as-is and silently drifts behind every change to the viz chrome. In #4312 all
# six stale pages had missed the row<->map cross-link hooks added in #4306.
#
# The generator already detects the drift -- `warn_stale_data_viewers` and
# `warn_stale_parcats_pages` name the offending pages on every run. This script
# is the other half: the thing that can actually fix them.
#
# The four dashboards it builds by hand are the ONLY ones it builds by hand. The
# four LLM (`--dictionary infer`) dashboards are regenerated by the generator
# itself via QSV_VIZ_REGEN_LLM -- do not hand-run those, or they end up skipping
# the cdnify/validate/iframe post-processing the generator applies.
#
# DATA
# ----
# Set QSV_VIZ_DATA_DIR (default ~/qsv-viz-datasets). It must hold:
#
#   pittsburgh_311.tsv                 269 MB  WPRDC Pittsburgh 311, exported as TSV
#                                              https://data.wprdc.org/dataset/pittsburgh-311-data
#   pittsburgh_neighborhoods.geojson   1.1 MB  WPRDC neighborhood boundaries
#   boston311-2025.tsv                 146 MB  Boston 311 service requests, CY2025, as TSV
#                                              https://data.boston.gov/dataset/311-service-requests
#   boston_neighborhood_boundaries.json 1.3 MB Boston neighborhood boundaries
#   ultimas-4-semanas-glp.ssv          2.2 MB  Brazil LPG (GLP) prices, last 4 weeks,
#                                              semicolon-separated export
#                                              https://dados.gov.br/dados/conjuntos-dados/serie-historica-de-precos-de-combustiveis-e-de-glp
#   brazil_geo.json                     14 MB  Brazil state boundaries (feature ids are
#                                              the two-letter state siglas)
#   calidad-aire-pm-colombia.csv       1.3 MB  Colombia PM10/PM2.5 station measurements
#                                              2011-2024 (datos.gov.co export)
#
# (Colombia's department boundaries -- colombia_departments.geojson -- ARE
# committed in examples/viz, so they are not required here.)
#
# The directory is deliberately OUTSIDE the repo so a `git clean` can't take it.
# qsv writes its stats/index sidecars beside each input, so the directory needs
# to stay writable; those sidecars are rebuilt automatically if absent or stale.
#
# GEOJSON SHORTCUTS
# -----------------
# The gallery displays Pittsburgh's command as `--geojson pittsburgh-neighborhoods`,
# which is a QSV_GEOJSON_SHORTCUTS alias (see `qsv viz --help`), not a file. The
# alias also supplies `--feature-id-key properties.hood`. This script passes the
# file and the key directly instead, which is exactly equivalent and needs no env
# setup. Do NOT define QSV_GEOJSON_SHORTCUTS in a repo `.env` to make the alias
# work: qsv loads `.env` with `dotenvy::dotenv_override()`, which clobbers the
# value the shortcut tests inject and breaks 6 of them.
#
# USAGE
# -----
#   ./refresh_external_dashboards.sh
#
#   QSV_VIZ_DATA_DIR=/vol/data ./refresh_external_dashboards.sh   # data elsewhere
#   QSV_BIN=target/release/qsv ./refresh_external_dashboards.sh   # specific binary
#   QSV_VIZ_REGEN_LLM=2 ./refresh_external_dashboards.sh          # re-infer dictionaries
#
# QSV_VIZ_REGEN_LLM defaults to 1: the four `--dictionary infer` dashboards are
# rebuilt, reusing their committed `<stem>.schema.json` sidecars, so no LLM is
# contacted and the run stays byte-stable. Use 2 ONLY when the dictionaries
# themselves must be re-inferred (a model or prompt change) -- it needs a live
# local LLM (LM Studio / Ollama), makes real calls, and its output is NOT
# byte-stable, so review that diff by hand.

set -euo pipefail

VIZ_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$VIZ_DIR/../.." && pwd)"
DATA_DIR="${QSV_VIZ_DATA_DIR:-$HOME/qsv-viz-datasets}"
REGEN_LLM="${QSV_VIZ_REGEN_LLM:-1}"

die() { printf '\nERROR: %s\n' "$*" >&2; exit 1; }
step() { printf '\n== %s\n' "$*"; }

# --- preflight: fail loudly and specifically, never skip ---------------------
# Silent skipping is how these pages drifted in the first place.

if [ -n "${QSV_BIN:-}" ]; then
    QSV="$QSV_BIN"
    [ -x "$QSV" ] || die "QSV_BIN is set to '$QSV_BIN' but that is not executable."
    # Resolve to an absolute path NOW. Both viz runs below execute inside `cd "$DATA_DIR"`,
    # and gen_gallery.py re-invokes QSV_BIN from examples/viz -- so a RELATIVE value (like the
    # `QSV_BIN=target/release/qsv` this script's own header suggests) passes the -x check here
    # against the caller's cwd and then does not exist at either of those two later cwds.
    QSV="$(cd "$(dirname "$QSV")" && pwd)/$(basename "$QSV")"
elif [ -x "$REPO/target/debug/qsv" ]; then
    QSV="$REPO/target/debug/qsv"
elif [ -x "$REPO/target/release/qsv" ]; then
    QSV="$REPO/target/release/qsv"
else
    die "no qsv binary found. Build one with:
    cargo build --locked --bin qsv -F all_features
  or point QSV_BIN at an existing binary."
fi

# viz is feature-gated and is NOT in the prebuilt/distributed binaries, so a qsv
# on PATH will usually fail here. Catch it now rather than 90s into a stats run.
"$QSV" --version 2>&1 | grep -q 'viz' \
    || die "'$QSV' was built without the viz feature. Rebuild with:
    cargo build --locked --bin qsv -F all_features"

[ -d "$DATA_DIR" ] || die "data directory not found: $DATA_DIR
  Set QSV_VIZ_DATA_DIR, or populate that directory (see the header of this script
  for the required files and where each one comes from)."

for f in pittsburgh_311.tsv pittsburgh_neighborhoods.geojson \
         boston311-2025.tsv boston_neighborhood_boundaries.json \
         ultimas-4-semanas-glp.ssv brazil_geo.json \
         calidad-aire-pm-colombia.csv; do
    [ -f "$DATA_DIR/$f" ] || die "missing required input: $DATA_DIR/$f
  See the header of this script for where to download it."
done

[ -w "$DATA_DIR" ] || die "$DATA_DIR is not writable; qsv writes its stats and
  index sidecars beside each input."

printf 'qsv      : %s\n' "$QSV"
printf '         : %s\n' "$("$QSV" --version | cut -d' ' -f1-2)"
printf 'data dir : %s\n' "$DATA_DIR"
printf 'viz dir  : %s\n' "$VIZ_DIR"
printf 'regen-llm: %s\n' "$REGEN_LLM"

# --- 1. Pittsburgh 311 -------------------------------------------------------
# Run from DATA_DIR so the stats cache and .idx land beside the input.

step "Pittsburgh 311 -> pitt311data.html"
( cd "$DATA_DIR" && "$QSV" viz smart pittsburgh_311.tsv \
    --smarter \
    --dictionary "$VIZ_DIR/pitt311data.schema.json" \
    --dict-info \
    --bivariate \
    --geojson pittsburgh_neighborhoods.geojson \
    --feature-id-key properties.hood \
    --dataset-pid https://data.wprdc.org/dataset/pittsburgh-311-data \
    -o "$VIZ_DIR/pitt311data.html" )

# --- 2. Boston 311 (2025) ----------------------------------------------------
# QSV_VIZ_MAX_POINTS lifts the point cap so all ~265k mapped points are embedded,
# which is what the gallery caption describes. --photos turns closed_photo into
# dwell-to-preview map imagery.

step "Boston 311 (2025) -> smart_boston_311_2025.html"
( cd "$DATA_DIR" && QSV_VIZ_MAX_POINTS=300000 "$QSV" viz smart boston311-2025.tsv \
    --smarter \
    --dictionary "$VIZ_DIR/boston311.schema.json" \
    --dict-info \
    --bivariate \
    --photos \
    --geojson boston_neighborhood_boundaries.json \
    --feature-id-key properties.name \
    --dataset-pid https://data.boston.gov/dataset/311-service-requests/resource/9d7c2214-4709-478a-a2e8-fb2020a5bb94 \
    -o "$VIZ_DIR/smart_boston_311_2025.html" )

# --- 3. Brazil LPG prices (Portuguese) ---------------------------------------
# QSV_PREFER_DMY: the .ssv export's dates are day-first. --language pt resolves
# through the curated pt-BR locale (the bare tag is a registered alias). The
# committed schema supplies the Portuguese field labels --dict-info renders.

step "Brazil LPG (GLP) -> smart-brazil-lpg-4-semanas.html"
( cd "$DATA_DIR" && QSV_PREFER_DMY=1 "$QSV" viz smart ultimas-4-semanas-glp.ssv \
    --smarter \
    --dictionary "$VIZ_DIR/ultimas-4-semanas-glp.schema.json" \
    --dict-info \
    --bivariate \
    --geojson brazil_geo.json \
    --language pt \
    --dataset-pid https://dados.gov.br/dados/conjuntos-dados/serie-historica-de-precos-de-combustiveis-e-de-glp \
    -o "$VIZ_DIR/smart-brazil-lpg-4-semanas.html" )

# --- 4. Colombia air quality (Spanish) ----------------------------------------
# The department boundaries are the COMMITTED colombia_departments.geojson (the
# hand-tuned schema's geo.latitude/geo.longitude concepts are load-bearing --
# without them the Data Schematic has no map at all; see the gallery caption).

step "Colombia air quality (PM) -> smart-colombia-calidad-aire.html"
( cd "$DATA_DIR" && "$QSV" viz smart calidad-aire-pm-colombia.csv \
    --language es \
    --geojson "$VIZ_DIR/colombia_departments.geojson" \
    --feature-id-key properties.shapeName \
    --dictionary "$VIZ_DIR/calidad-aire-pm-colombia.schema.json" \
    --dict-info \
    --bivariate \
    -o "$VIZ_DIR/smart-colombia-calidad-aire.html" )

# --- 5. The generator --------------------------------------------------------
# LAST, deliberately. warn_stale_data_viewers picks its reference from a page it
# just built, so running it after the hand builds turns its warning into a
# real verification of them. Reverse the order and it compares against the old
# files and reports nothing useful.

step "gen_gallery.py (rebuilds every committed-data figure + gallery.html)"
QSV_VIZ_REGEN_LLM="$REGEN_LLM" QSV_BIN="$QSV" python3 "$VIZ_DIR/gen_gallery.py"

# --- 6. Verify ---------------------------------------------------------------
# The issue's own test: every page that ships a data viewer must carry all four
# `qsv-data-*` hooks. A page built by the current binary has 4 of 4; a stale
# committed artifact has fewer.

step "data-viewer hooks (want 4 of 4 on every page)"
rc=0
for page in pitt311data smart_boston_311_2025 smart-brazil-lpg-4-semanas \
            smart-colombia-calidad-aire smart_dict_sunburst \
            smart_dict_treemap smart_geospatial smart_world_choropleth; do
    # `|| true` on the grep is load-bearing: a page with ZERO hooks -- the exact failure this
    # loop exists to catch -- makes grep exit 1, which under `set -euo pipefail` kills the
    # script mid-assignment, so it would never print "0 of 4" nor reach the die() below.
    n=$({ grep -oE 'qsv-data-(dt-ready|note|selection|selinfo)' "$VIZ_DIR/$page.html" || true; } \
        | sort -u | wc -l | tr -d ' ')
    printf '  %-28s %s of 4\n' "$page" "$n"
    [ "$n" = "4" ] || rc=1
done

if [ "$rc" != "0" ]; then
    die "at least one page is still missing data-viewer hooks (see above)."
fi

printf '\nAll eight refreshed. gen_gallery.py printed no staleness warnings above,\n'
printf 'which is the other half of the check.\n'
