"""SWE-bench grader — evaluates a solver agent via the harbor CLI.

Runs `harbor run -d swebench-verified@1.0` with the agent's solve.py as a custom
harbor agent, then parses the job result JSON for the pass rate.

Each `coral eval` runs a fixed slice of instances. The size is selected by
mode:
  - tune (`coral eval --tune`): `tune_size` instances (default 5) — cheap
    smoke test, hidden from the leaderboard by default.
  - real (default): `real_size` instances (default 30) — full eval, shown
    on the leaderboard. The score is the raw pass rate on that slice.

There is no tier promotion: every real eval uses the same `real_size`, so
scores are directly comparable across attempts.
"""

from __future__ import annotations

import json
import shutil
import subprocess
import time
from pathlib import Path

from coral.grader import TaskGrader
from coral.types import ScoreBundle


class Grader(TaskGrader):
    def describe_tune(self) -> str:
        """What `--tune` does on this grader.

        Tune mode runs a fixed `tune_size` slice (default 5 instances)
        regardless of best previous score, and is excluded from the
        leaderboard — use it to smoke-test a change cheaply.
        """
        tune_size = int(self.args.get("tune_size", 5))
        return (
            f"runs a fixed {tune_size}-instance slice; hidden from the "
            "leaderboard (use as a cheap smoke test, not a gate)"
        )

    def evaluate(self) -> ScoreBundle:
        dataset = self.args.get("dataset", "swebench-verified@1.0")
        n_concurrent = int(self.args.get("n_concurrent", 5))
        tune_size = int(self.args.get("tune_size", 5))
        real_size = int(self.args.get("real_size", 30))
        agent_timeout_multiplier = float(self.args.get("agent_timeout_multiplier", 1.0))
        verifier_timeout_multiplier = float(self.args.get("verifier_timeout_multiplier", 1.0))

        # Verify solve.py exists
        solver_path = Path(self.codebase_path) / "solve.py"
        if not solver_path.exists():
            return self.fail(
                "solve.py not found in codebase",
                feedback="Your codebase must contain a solve.py with a SolverAgent class.",
            )

        # Syntax check
        try:
            compile(solver_path.read_text(), str(solver_path), "exec")
        except SyntaxError as e:
            return self.fail(
                f"solve.py has syntax error: {e}",
                feedback=f"Fix the syntax error in solve.py: {e}",
            )

        # Find harbor CLI
        harbor_cmd = self._find_harbor_cmd()
        if not harbor_cmd:
            return self.fail(
                "harbor CLI not found",
                feedback="Install harbor (`uvx harbor --version` to verify) or ensure `uvx` is available.",
            )

        # Pick the slice size for this mode. Tune always runs the small
        # `tune_size` slice; real always runs the `real_size` slice.
        if self.tune:
            n_tasks, mode = tune_size, "tune"
        else:
            n_tasks, mode = real_size, "real"

        # Persist harbor logs in the per-attempt eval_logs dir so they survive
        # the grader-checkout cleanup (coral/grader/daemon.py:_remove_worktree).
        # Symlinked into each agent worktree at `<shared_dir>/eval_logs/<hash>/harbor_logs/`.
        job_dir = self.eval_logs_dir / "harbor_logs"
        job_dir.mkdir(parents=True, exist_ok=True)
        job_name = f"eval_{mode}_{int(time.time())}"

        start = time.time()
        harbor_result = self._run_harbor(
            harbor_cmd=harbor_cmd,
            dataset=dataset,
            job_dir=job_dir,
            job_name=job_name,
            n_tasks=n_tasks,
            n_concurrent=n_concurrent,
            agent_timeout_multiplier=agent_timeout_multiplier,
            verifier_timeout_multiplier=verifier_timeout_multiplier,
            mode=mode,
        )

        if isinstance(harbor_result, ScoreBundle):
            return harbor_result

        pass_rate, feedback = harbor_result
        elapsed = time.time() - start
        explanation = f"{mode}: {pass_rate:.1%} pass rate on {n_tasks} instances in {elapsed:.0f}s"
        return self.score(
            pass_rate, explanation, feedback=feedback,
            metadata={"raw_score": pass_rate, "n_tasks": n_tasks, "mode": mode},
        )

    def _run_harbor(
        self,
        harbor_cmd: list[str],
        dataset: str,
        job_dir: Path,
        job_name: str,
        n_tasks: int,
        n_concurrent: int,
        agent_timeout_multiplier: float,
        verifier_timeout_multiplier: float,
        mode: str = "",
    ) -> tuple[float, str] | ScoreBundle:
        """Run harbor and return (pass_rate, feedback) or a ScoreBundle on error."""
        import os

        cmd = [
            *harbor_cmd,
            "run",
            "-d", dataset,
            "--agent-import-path", "solve:SolverAgent",
            "-o", str(job_dir),
            "--job-name", job_name,
            "-n", str(n_concurrent),
            "--yes",
            "--agent-timeout-multiplier", str(agent_timeout_multiplier),
            "--verifier-timeout-multiplier", str(verifier_timeout_multiplier),
        ]
        if n_tasks > 0:
            cmd.extend(["-l", str(n_tasks)])

        env = {**os.environ, "PYTHONPATH": self.codebase_path}
        timeout = self.timeout or 14400

        start = time.time()
        try:
            result = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
                timeout=timeout,
                env=env,
                cwd=self.codebase_path,
            )
        except subprocess.TimeoutExpired:
            return self.fail(
                f"Harbor run timed out after {timeout}s",
                feedback=f"Evaluation timed out after {timeout}s.",
            )

        elapsed = time.time() - start

        # Parse results
        result_path = job_dir / job_name / "result.json"
        if not result_path.exists():
            stderr_tail = result.stderr.strip()[-1000:] if result.stderr else ""
            stdout_tail = result.stdout.strip()[-1000:] if result.stdout else ""
            return self.fail(
                f"Harbor run produced no result.json (exit code {result.returncode})",
                feedback=f"Harbor failed.\nstderr: {stderr_tail}\nstdout: {stdout_tail}",
            )

        try:
            job_result = json.loads(result_path.read_text())
        except json.JSONDecodeError as e:
            return self.fail(f"Failed to parse result.json: {e}")

        return self._parse_job_result(job_result, job_dir / job_name, elapsed, mode)

    def _parse_job_result(
        self, job_result: dict, job_dir: Path, elapsed: float, mode: str = "",
    ) -> tuple[float, str]:
        """Parse harbor job result.json and return (pass_rate, feedback).

        Harbor v0.13 result.json schema:
          - top: n_total_trials, n_errors (may be None when no errors)
          - stats: n_completed_trials, n_errored_trials, n_pending_trials, ...
          - stats.evals[<key>]: n_trials (per-eval count), n_errors, pass_at_k, reward_stats
        """
        stats = job_result.get("stats", job_result)
        n_completed = stats.get("n_completed_trials", 0)
        n_errors = stats.get("n_errored_trials", 0) or 0

        if n_completed == 0:
            return (0.0, "No trials completed")

        # Aggregate pass rate from evals
        evals = stats.get("evals", {})
        total_passed = 0
        total_trials = 0
        reward_details = []

        for eval_key, eval_stats in evals.items():
            eval_n = eval_stats.get("n_trials", 0)
            eval_errors = eval_stats.get("n_errors", 0)
            pass_at_k = eval_stats.get("pass_at_k", {})

            # pass@1 is the primary metric
            pass_rate = pass_at_k.get("1", pass_at_k.get(1, 0.0))

            # Count passed from reward_stats
            reward_stats = eval_stats.get("reward_stats", {})
            for reward_key, value_map in reward_stats.items():
                for value, trial_names in value_map.items():
                    val = float(value)
                    if val > 0:
                        total_passed += len(trial_names)
                    total_trials += len(trial_names)

            reward_details.append(
                f"{eval_key}: pass@1={pass_rate:.1%}, "
                f"trials={eval_n}, errors={eval_errors}"
            )

        # Compute overall pass rate
        if total_trials > 0:
            overall_rate = total_passed / total_trials
        else:
            overall_rate = 0.0
            for eval_stats in evals.values():
                p = eval_stats.get("pass_at_k", {})
                overall_rate = p.get("1", p.get(1, 0.0))
                break

        # Build feedback
        lines = [
            f"## SWE-bench Results ({mode}): {overall_rate:.1%} pass rate",
            f"Completed {n_completed} trials in {elapsed:.0f}s "
            f"({n_errors} errors)",
            "",
        ]
        for detail in reward_details:
            lines.append(f"- {detail}")

        # Per-task results
        task_results = self._collect_task_results(job_dir)
        if task_results:
            lines.append("")
            lines.append("### Per-task results")
            for task_name, passed in task_results:
                status = "PASS" if passed else "FAIL"
                lines.append(f"- `{task_name}`: {status}")

        # Per-trial failure details
        failure_lines = self._collect_trial_failures(job_dir, max_show=10)
        if failure_lines:
            lines.append("")
            lines.append("### Failure details")
            lines.extend(failure_lines)

        # Point agent to the logs (no shared-dir prefix — runtime-agnostic).
        # eval_logs/ is symlinked into each worktree's shared state dir
        # (.claude/, .codex/, .opencode/, .kiro/) by setup_shared_state, so
        # the agent prepends their own shared-dir to access via Read.
        logs_path = self.eval_logs_worktree_path(job_dir)
        lines.append("")
        lines.append("### Logs")
        lines.append(f"Full harbor logs (agent trajectories, terminal recordings, verifier output): `{logs_path}/` (under your shared state dir)")

        feedback = "\n".join(lines)
        return (overall_rate, feedback)

    def _collect_task_results(self, job_dir: Path) -> list[tuple[str, bool]]:
        """Collect per-task pass/fail results from trial result files."""
        results = []
        for trial_dir in sorted(job_dir.iterdir()):
            if not trial_dir.is_dir():
                continue
            result_file = trial_dir / "result.json"
            if not result_file.exists():
                continue
            try:
                trial_result = json.loads(result_file.read_text())
                task_name = trial_result.get("task_name", trial_dir.name)
                exception = trial_result.get("exception_info")
                if exception:
                    results.append((task_name, False))
                    continue
                verifier = trial_result.get("verifier_result")
                if verifier and verifier.get("rewards"):
                    passed = any(float(v) > 0 for v in verifier["rewards"].values())
                    results.append((task_name, passed))
                else:
                    results.append((task_name, False))
            except (json.JSONDecodeError, OSError):
                continue
        return results

    def _collect_trial_failures(self, job_dir: Path, max_show: int = 10) -> list[str]:
        """Collect failure details from individual trial result files."""
        lines = []
        count = 0
        for trial_dir in sorted(job_dir.iterdir()):
            if not trial_dir.is_dir():
                continue
            result_file = trial_dir / "result.json"
            if not result_file.exists():
                continue
            try:
                trial_result = json.loads(result_file.read_text())
                verifier = trial_result.get("verifier_result")
                exception = trial_result.get("exception_info")

                is_failure = False
                if exception:
                    is_failure = True
                elif verifier and verifier.get("rewards"):
                    rewards = verifier["rewards"]
                    if all(float(v) == 0 for v in rewards.values()):
                        is_failure = True

                if is_failure and count < max_show:
                    task_name = trial_result.get("task_name", trial_dir.name)
                    if exception:
                        exc_type = exception.get("exception_type", "Error")
                        exc_msg = exception.get("message", "")[:100]
                        lines.append(f"- `{task_name}`: {exc_type}: {exc_msg}")
                    else:
                        lines.append(f"- `{task_name}`: tests failed")
                    count += 1
            except (json.JSONDecodeError, OSError):
                continue

        if count >= max_show:
            lines.append(f"- ... and more")
        return lines

    def _find_harbor_cmd(self) -> list[str] | None:
        """Find how to invoke the harbor CLI."""
        # try to see if docker requires sudo
        prefix = []
        try:
            result = subprocess.run(
                ["docker", "ps"],
                capture_output=True,
                timeout=60,
            )
            if result.returncode != 0:
                # docker needs sudo to run
                prefix.append("sudo")
        except Exception:
            pass
        # Prefer uvx (installs/runs from PyPI in an isolated env)
        uvx_path = shutil.which("uvx")
        if uvx_path:
            try:
                result = subprocess.run(
                    [uvx_path, "harbor", "--version"],
                    capture_output=True,
                    timeout=60,
                )
                if result.returncode == 0:
                    return [*prefix, uvx_path, "harbor"]
            except Exception:
                pass
        harbor_path = shutil.which("harbor")
        if harbor_path:
            return [*prefix, harbor_path]
        return None
