import type { LLMEvaluatorTemplate } from "@phoenix/components/evaluators/templates/types";
import type { ClassificationEvaluatorAnnotationConfig } from "@phoenix/types";

const SYSTEM_PROMPT = `
First, describe the persona of your evaluator (e.g. "You are an expert at evaluating the correctness of a model's response").

<rubric>
Provide bullet points for how the LLM should score both correct and incorrect examples, e.g.:
A correct answer:
- Matches the reference answer exactly
- etc.

An incorrect answer:
- Differs from the reference answer
- etc.
</rubric>

<instructions>
Provide instructions for how the LLM should process the example, e.g.:
- Find all facts in the reference answer and provided answer
- etc.
</instructions>

<reminder>
Provide any remaining important reminders for the LLM to keep in mind while grading, e.g.:
- Different words with the same semantic meaning are the same
</reminder>
`;

const USER_PROMPT = `
<reference_answer>
{{reference}}
</reference_answer>

<provided_answer>
{{output}}
</provided_answer>
`;

const DEFAULT_OUTPUT_CONFIG: ClassificationEvaluatorAnnotationConfig = {
  name: "score",
  optimizationDirection: "MAXIMIZE",
  values: [
    { label: "true", score: 1 },
    { label: "false", score: 0 },
  ],
};

export const DEFAULT_EVALUATOR_TEMPLATE: LLMEvaluatorTemplate = {
  systemPrompt: SYSTEM_PROMPT.trim(),
  userPrompt: USER_PROMPT.trim(),
  outputConfigs: [DEFAULT_OUTPUT_CONFIG],
};
