// node:test unit tests for build-eval-summary.ts (port of Build-EvalSummary.Tests.ps1).
// Run from eng/common/scripts/eval:  npm test

import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { after, before, beforeEach, describe, it } from "node:test";

import { getEvalSummary, formatEvalSummaryMarkdown } from "../build-eval-summary.ts";

// Convenience: build the summary and also write the markdown, returning both.
function summarize(resultsRoot, outFile) {
  const shards = getEvalSummary(resultsRoot);
  const markdown = formatEvalSummaryMarkdown(shards);
  fs.writeFileSync(outFile, markdown, "utf8");
  return { shards, markdown };
}

describe("build-eval-summary", () => {
  let root;
  let outFile;

  before(() => {
    root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-summary-"));

    const newJunit = (shard, xml) => {
      const dir = path.join(root, `eval-result-${shard}`);
      fs.mkdirSync(dir, { recursive: true });
      fs.writeFileSync(path.join(dir, "junit.xml"), xml);
    };

    // area_github: 2 pass.
    newJunit(
      "area_github",
      `<testsuites>
  <testsuite name="github">
    <testcase name="lists issues" time="1.2" />
    <testcase name="creates pr" time="0.8" />
  </testsuite>
</testsuites>`
    );

    // area_typespec: 1 pass, 1 fail, 1 skip.
    newJunit(
      "area_typespec",
      `<testsuites>
  <testsuite name="typespec">
    <testcase name="adds arm resource" time="2.0" />
    <testcase name="renames client" time="1.5"><failure message="assertion failed">expected tool call</failure></testcase>
    <testcase name="generation step 2" time="0.0"><skipped /></testcase>
  </testsuite>
</testsuites>`
    );

    // area_multitrial: two stimuli, each run 5 times, gated at 0.8.
    // 'flaky pass' 4/5 (>= 0.8 passes); 'flaky fail' 2/5 (< 0.8 fails).
    newJunit(
      "area_multitrial",
      `<testsuites>
  <testsuite name="multitrial">
    <properties><property name="threshold" value="0.8" /></properties>
    <testcase name="flaky pass (trial 1)" time="1.0" />
    <testcase name="flaky pass (trial 2)" time="1.0" />
    <testcase name="flaky pass (trial 3)" time="1.0" />
    <testcase name="flaky pass (trial 4)" time="1.0"><failure message="x">nope</failure></testcase>
    <testcase name="flaky pass (trial 5)" time="1.0" />
    <testcase name="flaky fail (trial 1)" time="1.0" />
    <testcase name="flaky fail (trial 2)" time="1.0" />
    <testcase name="flaky fail (trial 3)" time="1.0"><failure message="x">nope</failure></testcase>
    <testcase name="flaky fail (trial 4)" time="1.0"><failure message="x">nope</failure></testcase>
    <testcase name="flaky fail (trial 5)" time="1.0"><failure message="x">nope</failure></testcase>
  </testsuite>
</testsuites>`
    );

    // area_multifile: one shard with TWO JUnit files; total must be 4, not double-counted.
    const multiDir = path.join(root, "eval-result-area_multifile");
    fs.mkdirSync(multiDir, { recursive: true });
    fs.writeFileSync(
      path.join(multiDir, "part1.xml"),
      `<testsuites>
  <testsuite name="multifile-a">
    <testcase name="alpha" time="1.0" />
    <testcase name="beta" time="1.0" />
  </testsuite>
</testsuites>`
    );
    fs.writeFileSync(
      path.join(multiDir, "part2.xml"),
      `<testsuites>
  <testsuite name="multifile-b">
    <testcase name="gamma" time="1.0" />
    <testcase name="delta" time="1.0"><failure message="x">nope</failure></testcase>
  </testsuite>
</testsuites>`
    );
  });

  after(() => fs.rmSync(root, { recursive: true, force: true }));

  beforeEach(() => {
    outFile = path.join(root, `summary-${Math.random().toString(36).slice(2)}.md`);
  });

  it("aggregates pass/fail/skip per shard", () => {
    const { shards } = summarize(root, outFile);
    assert.equal(shards.area_github.total, 2);
    assert.equal(shards.area_github.failed, 0);
    assert.equal(shards.area_typespec.total, 3);
    assert.equal(shards.area_typespec.failed, 1);
    assert.equal(shards.area_typespec.skipped, 1);
  });

  it("captures the failing scenario name", () => {
    const { shards } = summarize(root, outFile);
    assert.ok(shards.area_typespec.failures.includes("renames client (0/1 runs passed)"));
  });

  it("collapses per-trial testcases to one stimulus and applies the threshold", () => {
    const { shards } = summarize(root, outFile);
    assert.equal(shards.area_multitrial.total, 2);
    assert.equal(shards.area_multitrial.failed, 1);
    assert.ok(shards.area_multitrial.failures.includes("flaky fail (2/5 runs passed)"));
    assert.ok(!shards.area_multitrial.failures.includes("flaky pass (4/5 runs passed)"));
  });

  it("does not double-count totals when a shard has multiple JUnit files", () => {
    const { shards } = summarize(root, outFile);
    assert.equal(shards.area_multifile.total, 4);
    assert.equal(shards.area_multifile.failed, 1);
    const deltaCount = shards.area_multifile.failures.filter((f) => f.startsWith("delta ")).length;
    assert.equal(deltaCount, 1);
  });

  it("writes a Markdown file with an overall FAILED header when any shard is red", () => {
    const { markdown } = summarize(root, outFile);
    assert.match(markdown, /## .* Vally eval results — FAILED/);
    assert.match(markdown, /\| area_github \| .* \| 2 \| 0 \| 0 \|/);
    assert.match(markdown, /failing scenarios/i);
    assert.match(markdown, /renames client/);
  });

  it("reports PASSED when no shard has failures", () => {
    const passRoot = path.join(root, "pass-only");
    const dir = path.join(passRoot, "eval-result-area_github");
    fs.mkdirSync(dir, { recursive: true });
    fs.writeFileSync(
      path.join(dir, "junit.xml"),
      `<testsuites><testsuite name="github"><testcase name="ok" time="0.1" /></testsuite></testsuites>`
    );
    const { markdown } = summarize(passRoot, outFile);
    assert.match(markdown, /## .* Vally eval results — PASSED/);
  });

  it("reports NO RESULTS (not PASSED) when an XML has zero testcases", () => {
    const emptyRoot = path.join(root, "empty-results");
    const dir = path.join(emptyRoot, "eval-result-area_empty");
    fs.mkdirSync(dir, { recursive: true });
    fs.writeFileSync(
      path.join(dir, "junit.xml"),
      `<testsuites><testsuite name="empty"></testsuite></testsuites>`
    );
    const { markdown } = summarize(emptyRoot, outFile);
    assert.match(markdown, /## .* Vally eval results — NO RESULTS/);
    assert.doesNotMatch(markdown, /results — PASSED/);
    assert.match(markdown, /No eval testcases were found/);
  });

  it("falls back to a meaningful shard name when not under eval-result-*", () => {
    const fbRoot = path.join(root, "fallback");
    const dir = path.join(fbRoot, "_unit5", "2026-06-17T23-53-02-457Z");
    fs.mkdirSync(dir, { recursive: true });
    fs.writeFileSync(
      path.join(dir, "eval-results.junit.xml"),
      `<testsuites><testsuite name="x"><testcase name="ok" time="0.1" /></testsuite></testsuites>`
    );
    const { shards } = summarize(fbRoot, outFile);
    assert.ok("_unit5" in shards);
    assert.ok(!("unknown" in shards));
  });

  it("supersedes an earlier job attempt with the highest attempt on rerun", () => {
    const rerunRoot = path.join(root, "rerun");
    // Attempt 1 failed; attempt 2 (the rerun) passed. Only attempt 2 should count, and the
    // shard name must drop the -<attempt> suffix so it reads as a single shard.
    const a1 = path.join(rerunRoot, "eval-result-area_flaky-1");
    const a2 = path.join(rerunRoot, "eval-result-area_flaky-2");
    fs.mkdirSync(a1, { recursive: true });
    fs.mkdirSync(a2, { recursive: true });
    fs.writeFileSync(
      path.join(a1, "junit.xml"),
      `<testsuites><testsuite name="flaky"><testcase name="does thing" time="1.0"><failure message="x">nope</failure></testcase></testsuite></testsuites>`
    );
    fs.writeFileSync(
      path.join(a2, "junit.xml"),
      `<testsuites><testsuite name="flaky"><testcase name="does thing" time="1.0" /></testsuite></testsuites>`
    );
    const { shards } = summarize(rerunRoot, outFile);
    assert.ok("area_flaky" in shards);
    assert.ok(!("area_flaky-1" in shards));
    assert.ok(!("area_flaky-2" in shards));
    assert.equal(shards.area_flaky.total, 1);
    assert.equal(shards.area_flaky.failed, 0);
  });
});
