from secrets import token_hex
from typing import Any, AsyncIterator

import pytest
import sqlalchemy as sa
from sqlalchemy import select
from sqlalchemy.orm import selectinload
from strawberry.relay.types import GlobalID

from phoenix.db import models
from phoenix.db.types.annotation_configs import (
    CategoricalAnnotationConfig,
    CategoricalAnnotationValue,
    CategoricalOutputConfig,
    FreeformAnnotationConfig,
    OptimizationDirection,
)
from phoenix.db.types.evaluators import InputMapping
from phoenix.db.types.identifier import Identifier as IdentifierModel
from phoenix.db.types.model_provider import ModelProvider
from phoenix.db.types.prompts import (
    PromptChatTemplate,
    PromptMessage,
    PromptOpenAIInvocationParameters,
    PromptOpenAIInvocationParametersContent,
    PromptStringTemplate,
    PromptTemplateFormat,
    PromptTemplateType,
    PromptToolChoiceOneOrMore,
    PromptToolChoiceSpecificFunctionTool,
    PromptToolFunction,
    PromptToolFunctionDefinition,
    PromptTools,
    TextContentPart,
)
from phoenix.server.types import DbSessionFactory
from tests.unit.graphql import AsyncGraphQLClient


def _canonical_tools(
    name: str,
    description: str,
    parameters: dict[str, Any],
    tool_choice_name: str | None = None,
) -> dict[str, Any]:
    """Build a canonical PromptToolsInput dict for a single function tool."""
    return dict(
        tools=[
            dict(
                function=dict(
                    name=name,
                    description=description,
                    parameters=parameters,
                ),
            )
        ],
        toolChoice=dict(functionName=tool_choice_name) if tool_choice_name else None,
    )


class TestDatasetLLMEvaluatorMutations:
    _MUTATION = """
      mutation($input: CreateDatasetLLMEvaluatorInput!) {
        createDatasetLlmEvaluator(input: $input) {
          evaluator {
            id
            name
            description
            outputConfigs {
              ... on CategoricalAnnotationConfig {
                name
                description
                values { label score }
              }
              ... on ContinuousAnnotationConfig {
                name
                description
                lowerBound
                upperBound
              }
            }
            evaluator {
              ... on LLMEvaluator {
                id
                name
                description
                kind
                prompt { id }
                promptVersion { id }
              }
            }
          }
          query { __typename }
        }
      }
    """

    _PROMPT_VERSION_QUERY = """
      query($id: ID!) {
        node(id: $id) {
          ... on PromptVersion {
            template {
              ... on PromptChatTemplate {
                messages { role content { ... on TextContentPart { text { text } } } }
              }
            }
          }
        }
      }
    """

    async def _create(self, gql_client: AsyncGraphQLClient, **input_fields: Any) -> Any:
        """Private helper to execute create mutation with given input fields."""
        input_fields.setdefault("inputMapping", {"literalMapping": {}, "pathMapping": {}})
        return await gql_client.execute(self._MUTATION, {"input": input_fields})

    async def _verify_prompt_version_messages(
        self, gql_client: AsyncGraphQLClient, prompt_version_id: str, expected_text: str
    ) -> None:
        """Private helper to fetch and verify promptVersion message content."""
        result = await gql_client.execute(self._PROMPT_VERSION_QUERY, {"id": prompt_version_id})
        assert result.data and not result.errors
        messages = result.data["node"]["template"]["messages"]
        actual_text = messages[0]["content"][0]["text"]["text"]
        assert actual_text == expected_text

    async def test_create_dataset_llm_evaluator_basic(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Success: OpenAI evaluator
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="test-evaluator",
            description="test description",
            promptVersion=dict(
                description="prompt version",
                templateFormat="MUSTACHE",
                template=dict(
                    messages=[dict(role="USER", content=[dict(text=dict(text="Eval {{input}}"))])]
                ),
                invocationParameters=dict(openai=dict(temperature=0.0)),
                tools=_canonical_tools(
                    name="correctness",
                    description="test description",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetLlmEvaluator"]["evaluator"]
        llm_evaluator_data = dataset_evaluator["evaluator"]
        assert dataset_evaluator["name"] == "test-evaluator"
        assert dataset_evaluator["description"] == "test description"
        assert len(dataset_evaluator["outputConfigs"]) == 1
        assert dataset_evaluator["outputConfigs"][0]["name"] == "correctness"
        assert dataset_evaluator["outputConfigs"][0]["description"] == "description"
        assert len(dataset_evaluator["outputConfigs"][0]["values"]) == 2
        assert llm_evaluator_data["kind"] == "LLM"
        await self._verify_prompt_version_messages(
            gql_client, llm_evaluator_data["promptVersion"]["id"], "Eval {{input}}"
        )

        dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
        async with db() as session:
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.evaluator_id is not None
            # user_id is None when authentication is disabled
            assert db_dataset_evaluator.user_id is None
            llm_evaluator = await session.get(
                models.LLMEvaluator, db_dataset_evaluator.evaluator_id
            )
            assert llm_evaluator and llm_evaluator.kind == "LLM"
            assert db_dataset_evaluator.input_mapping == InputMapping(
                literal_mapping={}, path_mapping={}
            )
            assert db_dataset_evaluator.description == "test description"
            assert db_dataset_evaluator.output_configs is not None
            assert len(db_dataset_evaluator.output_configs) == 1
            dataset_eval_config = db_dataset_evaluator.output_configs[0]
            assert dataset_eval_config.name == "correctness"
            assert isinstance(dataset_eval_config, CategoricalAnnotationConfig)
            assert len(dataset_eval_config.values) == 2
            assert llm_evaluator.output_configs is not None
            assert len(llm_evaluator.output_configs) == 1
            output_config = llm_evaluator.output_configs[0]
            assert output_config.name == "correctness"
            assert isinstance(output_config, CategoricalAnnotationConfig)
            assert len(output_config.values) == 2

        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="anthropic",
            description="anthropic",
            promptVersion=dict(
                description="anthropic version",
                templateFormat="MUSTACHE",
                template=dict(messages=[dict(role="USER", content=[dict(text=dict(text="Rate"))])]),
                invocationParameters=dict(anthropic=dict(temperature=0.7, maxTokens=50)),
                tools=_canonical_tools(
                    name="correctness",
                    description="anthropic",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="ANTHROPIC",
                modelName="claude-3-opus-20240229",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetLlmEvaluator"]["evaluator"]
        llm_evaluator_data = dataset_evaluator["evaluator"]
        await self._verify_prompt_version_messages(
            gql_client, llm_evaluator_data["promptVersion"]["id"], "Rate"
        )

        # Success: Multiple evaluators for same dataset
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="second",
            description="second",
            promptVersion=dict(
                templateFormat="MUSTACHE",
                template=dict(
                    messages=[dict(role="USER", content=[dict(text=dict(text="Second"))])]
                ),
                invocationParameters=dict(anthropic=dict(temperature=0.5, maxTokens=100)),
                tools=_canonical_tools(
                    name="correctness",
                    description="second",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="ANTHROPIC",
                modelName="claude-3-opus-20240229",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetLlmEvaluator"]["evaluator"]
        llm_evaluator_data = dataset_evaluator["evaluator"]
        await self._verify_prompt_version_messages(
            gql_client, llm_evaluator_data["promptVersion"]["id"], "Second"
        )
        async with db() as session:
            evaluators = await session.scalars(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id
                )
            )
            assert len(evaluators.all()) >= 2

        # Failure: Nonexistent dataset
        result = await self._create(
            gql_client,
            datasetId=str(GlobalID("Dataset", "999")),
            name="test",
            description="test",
            promptVersion=dict(
                templateFormat="MUSTACHE",
                template=dict(messages=[dict(role="USER", content=[dict(text=dict(text="Test"))])]),
                invocationParameters=dict(openai=dict(temperature=0.5)),
                tools=_canonical_tools(
                    name="correctness",
                    description="test",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.errors and "Dataset with id 999 not found" in result.errors[0].message

        # Failure: Invalid template format
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="invalid",
            promptVersion=dict(
                templateFormat="MUSTACHE",
                template=dict(messages=[dict(role="USER", content="invalid")]),
                invocationParameters=dict(openai=dict(temperature=0.5)),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.errors

    async def test_create_dataset_llm_evaluator_with_existing_prompt_version(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test creating evaluator with existing prompt_version_id."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # First, create a prompt and prompt version
        async with db() as session:
            prompt = models.Prompt(
                name=IdentifierModel.model_validate(f"test-prompt-{token_hex(4)}"),
                description="test prompt",
                prompt_versions=[
                    models.PromptVersion(
                        template_type=PromptTemplateType.CHAT,
                        template_format=PromptTemplateFormat.MUSTACHE,
                        template=PromptChatTemplate(
                            type="chat",
                            messages=[
                                PromptMessage(
                                    role="user",
                                    content=[
                                        TextContentPart(
                                            type="text",
                                            text="Original: {{input}}",
                                        )
                                    ],
                                )
                            ],
                        ),
                        invocation_parameters=PromptOpenAIInvocationParameters(
                            type="openai",
                            openai=PromptOpenAIInvocationParametersContent(temperature=0.0),
                        ),
                        tools=PromptTools(
                            type="tools",
                            tools=[
                                PromptToolFunction(
                                    type="function",
                                    function=PromptToolFunctionDefinition(
                                        name="correctness",
                                        description="original description",
                                        parameters={
                                            "type": "object",
                                            "properties": {
                                                "correctness": {
                                                    "type": "string",
                                                    "enum": ["correct", "incorrect"],
                                                }
                                            },
                                            "required": ["correctness"],
                                        },
                                    ),
                                )
                            ],
                            tool_choice=PromptToolChoiceSpecificFunctionTool(
                                type="specific_function",
                                function_name="correctness",
                            ),
                        ),
                        response_format=None,
                        model_provider=ModelProvider.OPENAI,
                        model_name="gpt-4",
                        metadata_={},
                    )
                ],
            )
            session.add(prompt)
            await session.flush()
            existing_prompt_version = prompt.prompt_versions[0]
            existing_prompt_version_id = str(
                GlobalID("PromptVersion", str(existing_prompt_version.id))
            )
            existing_prompt_id = prompt.id

        # Create evaluator with different prompt contents
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="test-evaluator-different",
            description="test description",
            promptVersionId=existing_prompt_version_id,
            promptVersion=dict(
                description=None,
                templateFormat="MUSTACHE",
                template=dict(
                    messages=[
                        dict(role="USER", content=[dict(text=dict(text="Updated: {{input}}"))])
                    ]
                ),
                invocationParameters=dict(openai=dict(temperature=0.5)),
                tools=_canonical_tools(
                    name="correctness",
                    description="test description",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetLlmEvaluator"]["evaluator"]

        # Verify the evaluator uses the existing prompt but new prompt version
        async with db() as session:
            dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            llm_evaluator = await session.get(
                models.LLMEvaluator,
                db_dataset_evaluator.evaluator_id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert llm_evaluator is not None
            # Verify prompt ID matches the existing prompt
            assert llm_evaluator.prompt_id == existing_prompt_id
            # Verify prompt version ID is different (new version created)
            assert llm_evaluator.prompt_version_tag is not None
            new_prompt_version_id = llm_evaluator.prompt_version_tag.prompt_version_id
            assert new_prompt_version_id != existing_prompt_version.id
            # Verify tag was added to the new prompt version
            assert llm_evaluator.prompt_version_tag.prompt_id == existing_prompt_id

            # Verify two prompt versions exist (original + new)
            prompt_versions = await session.scalars(
                select(models.PromptVersion).where(
                    models.PromptVersion.prompt_id == existing_prompt_id
                )
            )
            prompt_versions_list = prompt_versions.all()
            assert len(prompt_versions_list) == 2
            # Verify the new version has the updated content
            new_version = next(
                (v for v in prompt_versions_list if v.id == new_prompt_version_id), None
            )
            assert new_version is not None
            # Verify template is a chat template and has the updated content
            assert isinstance(new_version.template, PromptChatTemplate)
            assert (
                new_version.template.messages[0].content[0].text == "Updated: {{input}}"  # type: ignore[union-attr]
            )

    async def test_create_dataset_llm_evaluator_without_prompt_version_id(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test creating evaluator without prompt_version_id (existing behavior)."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Create evaluator without prompt_version_id
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="test-evaluator-new",
            description="test description",
            promptVersion=dict(
                description=None,
                templateFormat="MUSTACHE",
                template=dict(
                    messages=[dict(role="USER", content=[dict(text=dict(text="New: {{input}}"))])]
                ),
                invocationParameters=dict(openai=dict(temperature=0.0)),
                tools=_canonical_tools(
                    name="correctness",
                    description="test description",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetLlmEvaluator"]["evaluator"]

        # Verify a new prompt and prompt version were created
        async with db() as session:
            dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            llm_evaluator = await session.get(
                models.LLMEvaluator,
                db_dataset_evaluator.evaluator_id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert llm_evaluator is not None
            # Verify prompt exists
            prompt = await session.get(models.Prompt, llm_evaluator.prompt_id)
            assert prompt is not None
            # Verify prompt version exists and tag was added
            assert llm_evaluator.prompt_version_tag is not None
            prompt_version = await session.get(
                models.PromptVersion, llm_evaluator.prompt_version_tag.prompt_version_id
            )
            assert prompt_version is not None
            assert prompt_version.prompt_id == prompt.id
            # Verify only one prompt version exists for this prompt
            prompt_versions = await session.scalars(
                select(models.PromptVersion).where(models.PromptVersion.prompt_id == prompt.id)
            )
            assert len(prompt_versions.all()) == 1

    async def test_create_dataset_llm_evaluator_with_identical_prompt_content(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test creating evaluator with existing prompt_version_id and identical content doesn't create new version."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # First, create a prompt and prompt version
        async with db() as session:
            prompt = models.Prompt(
                name=IdentifierModel.model_validate(f"test-prompt-{token_hex(4)}"),
                description="test prompt",
                prompt_versions=[
                    models.PromptVersion(
                        template_type=PromptTemplateType.CHAT,
                        template_format=PromptTemplateFormat.MUSTACHE,
                        template=PromptChatTemplate(
                            type="chat",
                            messages=[
                                PromptMessage(
                                    role="user",
                                    content=[
                                        TextContentPart(
                                            type="text",
                                            text="Original: {{input}}",
                                        )
                                    ],
                                )
                            ],
                        ),
                        invocation_parameters=PromptOpenAIInvocationParameters(
                            type="openai",
                            openai=PromptOpenAIInvocationParametersContent(temperature=0.5),
                        ),
                        tools=PromptTools(
                            type="tools",
                            tools=[
                                PromptToolFunction(
                                    type="function",
                                    function=PromptToolFunctionDefinition(
                                        name="correctness",
                                        description="test description",
                                        parameters={
                                            "type": "object",
                                            "properties": {
                                                "label": {
                                                    "type": "string",
                                                    "enum": ["correct", "incorrect"],
                                                    "description": "correctness",
                                                }
                                            },
                                            "required": ["label"],
                                        },
                                    ),
                                )
                            ],
                            tool_choice=PromptToolChoiceSpecificFunctionTool(
                                type="specific_function",
                                function_name="correctness",
                            ),
                        ),
                        response_format=None,
                        model_provider=ModelProvider.OPENAI,
                        model_name="gpt-4",
                        metadata_={},
                    )
                ],
            )
            session.add(prompt)
            await session.flush()
            existing_prompt_version = prompt.prompt_versions[0]
            existing_prompt_version_id = str(
                GlobalID("PromptVersion", str(existing_prompt_version.id))
            )
            existing_prompt_id = prompt.id

        # Create evaluator with identical prompt contents
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="test-evaluator-identical",
            description="test description",
            promptVersionId=existing_prompt_version_id,
            promptVersion=dict(
                description=None,
                templateFormat="MUSTACHE",
                template=dict(
                    messages=[
                        dict(role="USER", content=[dict(text=dict(text="Original: {{input}}"))])
                    ]
                ),
                invocationParameters=dict(openai=dict(temperature=0.5)),
                tools=_canonical_tools(
                    name="correctness",
                    description="test description",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetLlmEvaluator"]["evaluator"]

        # Verify the evaluator uses the existing prompt and SAME prompt version (no new version created)
        async with db() as session:
            dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            llm_evaluator = await session.get(
                models.LLMEvaluator,
                db_dataset_evaluator.evaluator_id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert llm_evaluator is not None
            # Verify prompt ID matches the existing prompt
            assert llm_evaluator.prompt_id == existing_prompt_id
            # Verify prompt version ID is the SAME (no new version created)
            assert llm_evaluator.prompt_version_tag is not None
            assert llm_evaluator.prompt_version_tag.prompt_version_id == existing_prompt_version.id
            # Verify tag was added to the existing prompt version
            assert llm_evaluator.prompt_version_tag.prompt_id == existing_prompt_id

            # Verify only ONE prompt version exists (no new one created)
            prompt_versions = await session.scalars(
                select(models.PromptVersion).where(
                    models.PromptVersion.prompt_id == existing_prompt_id
                )
            )
            prompt_versions_list = prompt_versions.all()
            assert len(prompt_versions_list) == 1

    async def test_create_dataset_llm_evaluator_creates_evaluator_label(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test that creating an LLM evaluator labels its prompt with 'evaluator'."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="label-test",
            description="desc",
            promptVersion=dict(
                templateFormat="MUSTACHE",
                template=dict(messages=[dict(role="USER", content=[dict(text=dict(text="x"))])]),
                invocationParameters=dict(openai=dict()),
                tools=_canonical_tools(
                    name="out",
                    description="desc",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(type="string", enum=["a", "b"], description="out")
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="out",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="out",
                        optimizationDirection="MAXIMIZE",
                        values=[dict(label="a", score=1), dict(label="b", score=0)],
                    )
                )
            ],
        )
        assert result.data and not result.errors

        # Verify the prompt has the "evaluator" label
        async with db() as session:
            evaluator_id = int(
                GlobalID.from_id(
                    result.data["createDatasetLlmEvaluator"]["evaluator"]["id"]
                ).node_id
            )
            de = await session.get(models.DatasetEvaluators, evaluator_id)
            assert de is not None
            llm_eval = await session.get(models.LLMEvaluator, de.evaluator_id)
            assert llm_eval is not None

            assoc = await session.scalar(
                select(models.PromptPromptLabel).where(
                    models.PromptPromptLabel.prompt_id == llm_eval.prompt_id
                )
            )
            assert assoc is not None
            label = await session.get(models.PromptLabel, assoc.prompt_label_id)
            assert label is not None
            assert label.name == "evaluator"

    async def test_create_dataset_llm_evaluator_with_nonexistent_prompt_version_id(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test creating evaluator with nonexistent prompt_version_id raises error."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        nonexistent_prompt_version_id = str(GlobalID("PromptVersion", "999999"))

        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="test-evaluator-error",
            description="test description",
            promptVersionId=nonexistent_prompt_version_id,
            promptVersion=dict(
                description=None,
                templateFormat="MUSTACHE",
                template=dict(
                    messages=[dict(role="USER", content=[dict(text=dict(text="Test: {{input}}"))])]
                ),
                invocationParameters=dict(openai=dict(temperature=0.0)),
                tools=_canonical_tools(
                    name="correctness",
                    description="test description",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="description",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )
        assert result.errors
        assert "not found" in result.errors[0].message.lower()

    async def test_create_llm_evaluators_with_same_name_on_different_datasets(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test that creating evaluators with the same name on different datasets succeeds."""
        async with db() as session:
            second_dataset = models.Dataset(name=f"second-dataset-{token_hex(4)}", metadata_={})
            session.add(second_dataset)
            await session.flush()
            second_dataset_id = second_dataset.id

        dataset1_gid = str(GlobalID("Dataset", str(empty_dataset.id)))
        dataset2_gid = str(GlobalID("Dataset", str(second_dataset_id)))

        evaluator_input = dict(
            description="test description",
            promptVersion=dict(
                templateFormat="MUSTACHE",
                template=dict(
                    messages=[dict(role="USER", content=[dict(text=dict(text="Eval {{input}}"))])]
                ),
                invocationParameters=dict(openai=dict(temperature=0.0)),
                tools=_canonical_tools(
                    name="correctness",
                    description="test description",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="correctness",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[
                dict(
                    categorical=dict(
                        name="correctness",
                        description="correctness eval",
                        optimizationDirection="MAXIMIZE",
                        values=[
                            dict(label="correct", score=1),
                            dict(label="incorrect", score=0),
                        ],
                    )
                )
            ],
        )

        result1 = await self._create(
            gql_client,
            datasetId=dataset1_gid,
            name="my-evaluator",
            **evaluator_input,
        )
        assert result1.data and not result1.errors

        result2 = await self._create(
            gql_client,
            datasetId=dataset2_gid,
            name="my-evaluator",
            **evaluator_input,
        )
        assert result2.data and not result2.errors

        assert result1.data["createDatasetLlmEvaluator"]["evaluator"]["name"] == "my-evaluator"
        assert result2.data["createDatasetLlmEvaluator"]["evaluator"]["name"] == "my-evaluator"

        name1 = result1.data["createDatasetLlmEvaluator"]["evaluator"]["evaluator"]["name"]
        name2 = result2.data["createDatasetLlmEvaluator"]["evaluator"]["evaluator"]["name"]
        assert name1 != name2

    async def test_create_dataset_llm_evaluator_rejects_empty_output_configs(
        self,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            name="empty-configs",
            description="test",
            promptVersion=dict(
                templateFormat="MUSTACHE",
                template=dict(messages=[dict(role="USER", content=[dict(text=dict(text="Test"))])]),
                invocationParameters=dict(openai=dict(temperature=0.5)),
                tools=_canonical_tools(
                    name="test",
                    description="test",
                    parameters=dict(
                        type="object",
                        properties=dict(
                            label=dict(
                                type="string",
                                enum=["correct", "incorrect"],
                                description="correctness",
                            )
                        ),
                        required=["label"],
                    ),
                    tool_choice_name="test",
                ),
                modelProvider="OPENAI",
                modelName="gpt-4",
            ),
            outputConfigs=[],
        )
        assert result.errors
        assert "at least 1 item" in result.errors[0].message.lower()


class TestUpdateDatasetLLMEvaluatorMutation:
    _UPDATE_MUTATION = """
      mutation($input: UpdateDatasetLLMEvaluatorInput!) {
        updateDatasetLlmEvaluator(input: $input) {
          evaluator {
            id
            name
            outputConfigs {
              ... on CategoricalAnnotationConfig {
                name
                description
                optimizationDirection
                values { label score }
              }
            }
            evaluator {
              ... on LLMEvaluator {
                id
                name
                description
                kind
                promptVersion { id }
              }
            }
          }
          query { __typename }
        }
      }
    """

    async def test_update_dataset_llm_evaluator(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        llm_evaluator: models.LLMEvaluator,
    ) -> None:
        """Test updating an LLM evaluator via its DatasetEvaluator assignment."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Get the dataset_evaluator ID from the fixture
        async with db() as session:
            dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id,
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id,
                )
            )
            assert dataset_evaluator is not None
            dataset_evaluator_id = str(GlobalID("DatasetEvaluator", str(dataset_evaluator.id)))

        # Update the evaluator with new values
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "datasetId": dataset_id,
                    "name": "updated-evaluator-name",
                    "description": "updated description",
                    "promptVersion": dict(
                        description="updated prompt version",
                        templateFormat="MUSTACHE",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Updated: {{input}}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.5)),
                        tools=_canonical_tools(
                            name="result",
                            description="updated description",
                            parameters=dict(
                                type="object",
                                properties=dict(
                                    label=dict(
                                        type="string",
                                        enum=["good", "bad"],
                                        description="result",
                                    )
                                ),
                                required=["label"],
                            ),
                            tool_choice_name="result",
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="result",
                                description="updated output description",
                                optimizationDirection="MINIMIZE",
                                values=[
                                    dict(label="good", score=1),
                                    dict(label="bad", score=0),
                                ],
                            )
                        )
                    ],
                }
            },
        )
        assert result.data and not result.errors

        updated_evaluator = result.data["updateDatasetLlmEvaluator"]["evaluator"]
        assert updated_evaluator["name"] == "updated-evaluator-name"
        llm_data = updated_evaluator["evaluator"]
        assert llm_data["description"] == "updated description"
        assert llm_data["kind"] == "LLM"

        async with db() as session:
            db_evaluator = await session.get(models.LLMEvaluator, llm_evaluator.id)
            assert db_evaluator is not None
            assert db_evaluator.description == "updated description"
            # user_id is None when authentication is disabled
            assert db_evaluator.user_id is None
            db_dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id
                )
            )
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.output_configs is not None
            assert len(db_dataset_evaluator.output_configs) == 1
            assert db_dataset_evaluator.output_configs[0].name == "result"
            # user_id is None when authentication is disabled
            assert db_dataset_evaluator.user_id is None
            assert db_evaluator.output_configs[0].name == "result"

        # Second update: from 1 output config to 2 (multi-output)
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "datasetId": dataset_id,
                    "name": "multi-output-evaluator",
                    "description": "multi output description",
                    "promptVersion": dict(
                        description="multi output prompt version",
                        templateFormat="MUSTACHE",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Evaluate: {{input}}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.5)),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="quality",
                                        description="multi output description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["good", "bad"],
                                                    description="quality",
                                                )
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                ),
                                dict(
                                    function=dict(
                                        name="relevance",
                                        description="multi output description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["relevant", "irrelevant"],
                                                    description="relevance",
                                                )
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                ),
                            ],
                            toolChoice=dict(functionName="quality"),
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="quality",
                                description="quality assessment",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="good", score=1),
                                    dict(label="bad", score=0),
                                ],
                            )
                        ),
                        dict(
                            categorical=dict(
                                name="relevance",
                                description="relevance assessment",
                                optimizationDirection="MINIMIZE",
                                values=[
                                    dict(label="relevant", score=1),
                                    dict(label="irrelevant", score=0),
                                ],
                            )
                        ),
                    ],
                }
            },
        )
        assert result.data and not result.errors

        updated_evaluator = result.data["updateDatasetLlmEvaluator"]["evaluator"]
        assert updated_evaluator["name"] == "multi-output-evaluator"

        output_configs = updated_evaluator["outputConfigs"]
        assert len(output_configs) == 2
        assert output_configs[0]["name"] == "quality"
        assert output_configs[0]["description"] == "quality assessment"
        assert output_configs[0]["optimizationDirection"] == "MAXIMIZE"
        assert len(output_configs[0]["values"]) == 2
        assert output_configs[1]["name"] == "relevance"
        assert output_configs[1]["description"] == "relevance assessment"
        assert output_configs[1]["optimizationDirection"] == "MINIMIZE"
        assert len(output_configs[1]["values"]) == 2

        async with db() as session:
            db_evaluator = await session.get(models.LLMEvaluator, llm_evaluator.id)
            assert db_evaluator is not None
            assert len(db_evaluator.output_configs) == 2

            first_config = db_evaluator.output_configs[0]
            assert isinstance(first_config, CategoricalAnnotationConfig)
            assert first_config.name == "quality"
            assert first_config.description == "quality assessment"
            assert len(first_config.values) == 2

            second_config = db_evaluator.output_configs[1]
            assert isinstance(second_config, CategoricalAnnotationConfig)
            assert second_config.name == "relevance"
            assert second_config.description == "relevance assessment"
            assert len(second_config.values) == 2

    async def test_update_without_prompt_version_id_creates_new_prompt(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        llm_evaluator: models.LLMEvaluator,
    ) -> None:
        """Test that updating without prompt_version_id creates a new prompt and version."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        original_prompt_id = llm_evaluator.prompt_id

        # Get the dataset_evaluator ID
        async with db() as session:
            dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id,
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id,
                )
            )
            assert dataset_evaluator is not None
            dataset_evaluator_id = str(GlobalID("DatasetEvaluator", str(dataset_evaluator.id)))
            original_prompt = await session.get(models.Prompt, original_prompt_id)
            assert original_prompt is not None

        # Update without prompt_version_id
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "datasetId": dataset_id,
                    "name": "updated-evaluator",
                    "description": "updated description",
                    "promptVersion": dict(
                        description="new prompt version",
                        templateFormat="MUSTACHE",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="New prompt: {{input}}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.7)),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="result",
                                        description="updated description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["yes", "no"],
                                                    description="result",
                                                )
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                )
                            ],
                            toolChoice=dict(oneOrMore=True),
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="result",
                                description="description",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="yes", score=1),
                                    dict(label="no", score=0),
                                ],
                            )
                        )
                    ],
                }
            },
        )
        assert result.data and not result.errors

        # Verify a new prompt and version were created
        async with db() as session:
            db_evaluator = await session.get(
                models.LLMEvaluator,
                llm_evaluator.id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert db_evaluator is not None
            # Verify prompt_id changed (new prompt created)
            assert db_evaluator.prompt_id != original_prompt_id
            new_prompt = await session.get(models.Prompt, db_evaluator.prompt_id)
            assert new_prompt is not None
            # Verify prompt_version_tag exists and points to new prompt
            assert db_evaluator.prompt_version_tag is not None
            assert db_evaluator.prompt_version_tag.prompt_id == db_evaluator.prompt_id
            # Verify only one version exists for the new prompt
            prompt_versions = await session.scalars(
                select(models.PromptVersion).where(
                    models.PromptVersion.prompt_id == db_evaluator.prompt_id
                )
            )
            assert len(prompt_versions.all()) == 1
            # Verify the version has the new content
            new_version = await session.get(
                models.PromptVersion, db_evaluator.prompt_version_tag.prompt_version_id
            )
            assert new_version is not None
            assert isinstance(new_version.template, PromptChatTemplate)
            assert (
                new_version.template.messages[0].content[0].text == "New prompt: {{input}}"  # type: ignore[union-attr]
            )

    async def test_update_with_prompt_version_id_different_prompt(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        llm_evaluator: models.LLMEvaluator,
    ) -> None:
        """Test that updating with prompt_version_id pointing to different prompt switches prompts."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        original_prompt_id = llm_evaluator.prompt_id

        # Create a different prompt with a version
        async with db() as session:
            different_prompt = models.Prompt(
                name=IdentifierModel.model_validate(f"different-prompt-{token_hex(4)}"),
                description="different prompt",
                prompt_versions=[
                    models.PromptVersion(
                        template_type=PromptTemplateType.CHAT,
                        template_format=PromptTemplateFormat.MUSTACHE,
                        template=PromptChatTemplate(
                            type="chat",
                            messages=[
                                PromptMessage(
                                    role="user",
                                    content=[
                                        TextContentPart(
                                            type="text",
                                            text="Different: {{input}}",
                                        )
                                    ],
                                )
                            ],
                        ),
                        invocation_parameters=PromptOpenAIInvocationParameters(
                            type="openai",
                            openai=PromptOpenAIInvocationParametersContent(temperature=0.0),
                        ),
                        tools=PromptTools(
                            type="tools",
                            tools=[
                                PromptToolFunction(
                                    type="function",
                                    function=PromptToolFunctionDefinition(
                                        name="result",
                                        description="different",
                                        parameters={
                                            "type": "object",
                                            "properties": {
                                                "label": {
                                                    "type": "string",
                                                    "enum": ["yes", "no"],
                                                    "description": "result",
                                                }
                                            },
                                            "required": ["label"],
                                        },
                                    ),
                                )
                            ],
                            tool_choice=PromptToolChoiceOneOrMore(type="one_or_more"),
                        ),
                        response_format=None,
                        model_provider=ModelProvider.OPENAI,
                        model_name="gpt-4",
                        metadata_={},
                    )
                ],
            )
            session.add(different_prompt)
            await session.flush()
            different_prompt_version_id = str(
                GlobalID("PromptVersion", str(different_prompt.prompt_versions[0].id))
            )
            different_prompt_id = different_prompt.id

            dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id,
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id,
                )
            )
            assert dataset_evaluator is not None
            dataset_evaluator_id = str(GlobalID("DatasetEvaluator", str(dataset_evaluator.id)))

        # Update with prompt_version_id pointing to different prompt
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "datasetId": dataset_id,
                    "name": "updated-evaluator",
                    "description": "updated description",
                    "promptVersionId": different_prompt_version_id,
                    "promptVersion": dict(
                        description="updated prompt version",
                        templateFormat="MUSTACHE",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Updated: {{input}}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.5)),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="result",
                                        description="updated description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["yes", "no"],
                                                    description="result",
                                                )
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                )
                            ],
                            toolChoice=dict(oneOrMore=True),
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="result",
                                description="description",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="yes", score=1),
                                    dict(label="no", score=0),
                                ],
                            )
                        )
                    ],
                }
            },
        )
        assert result.data and not result.errors

        # Verify evaluator switched to different prompt
        async with db() as session:
            db_evaluator = await session.get(
                models.LLMEvaluator,
                llm_evaluator.id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert db_evaluator is not None
            # Verify prompt_id changed to different prompt
            assert db_evaluator.prompt_id == different_prompt_id
            assert db_evaluator.prompt_id != original_prompt_id
            # Verify prompt_version_tag points to different prompt
            assert db_evaluator.prompt_version_tag is not None
            assert db_evaluator.prompt_version_tag.prompt_id == different_prompt_id

    async def test_update_with_prompt_version_id_content_changed(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        llm_evaluator: models.LLMEvaluator,
    ) -> None:
        """Test that updating with prompt_version_id but changed content creates new version."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Get the current prompt version
        async with db() as session:
            current_prompt_version = await session.scalar(
                select(models.PromptVersion)
                .where(models.PromptVersion.prompt_id == llm_evaluator.prompt_id)
                .order_by(models.PromptVersion.id.desc())
                .limit(1)
            )
            assert current_prompt_version is not None
            current_prompt_version_id = str(
                GlobalID("PromptVersion", str(current_prompt_version.id))
            )
            original_prompt_id = llm_evaluator.prompt_id

            dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id,
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id,
                )
            )
            assert dataset_evaluator is not None
            dataset_evaluator_id = str(GlobalID("DatasetEvaluator", str(dataset_evaluator.id)))

        # Update with prompt_version_id but different content
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "datasetId": dataset_id,
                    "name": "updated-evaluator",
                    "description": "updated description",
                    "promptVersionId": current_prompt_version_id,
                    "promptVersion": dict(
                        description="changed content",
                        templateFormat="MUSTACHE",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Changed: {{input}}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.8)),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="result",
                                        description="updated description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["yes", "no"],
                                                    description="result",
                                                )
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                )
                            ],
                            toolChoice=dict(oneOrMore=True),
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="result",
                                description="description",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="yes", score=1),
                                    dict(label="no", score=0),
                                ],
                            )
                        )
                    ],
                }
            },
        )
        assert result.data and not result.errors

        # Verify new version was created
        async with db() as session:
            db_evaluator = await session.get(
                models.LLMEvaluator,
                llm_evaluator.id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert db_evaluator is not None
            # Verify prompt_id stayed the same
            assert db_evaluator.prompt_id == original_prompt_id
            # Verify prompt_version_tag points to new version
            assert db_evaluator.prompt_version_tag is not None
            new_version_id = db_evaluator.prompt_version_tag.prompt_version_id
            assert new_version_id != current_prompt_version.id
            # Verify new version has changed content
            new_version = await session.get(models.PromptVersion, new_version_id)
            assert new_version is not None
            assert isinstance(new_version.template, PromptChatTemplate)
            assert (
                new_version.template.messages[0].content[0].text == "Changed: {{input}}"  # type: ignore[union-attr]
            )

    async def test_update_with_prompt_version_id_content_unchanged(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        llm_evaluator: models.LLMEvaluator,
    ) -> None:
        """Test that updating with prompt_version_id and unchanged content doesn't create new version."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        original_prompt_id = llm_evaluator.prompt_id

        async with db() as session:
            current_prompt_version = await session.scalar(
                select(models.PromptVersion)
                .where(models.PromptVersion.prompt_id == original_prompt_id)
                .order_by(models.PromptVersion.id.desc())
                .limit(1)
            )
            assert current_prompt_version is not None
            current_prompt_version_id = str(
                GlobalID("PromptVersion", str(current_prompt_version.id))
            )

            dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id,
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id,
                )
            )
            assert dataset_evaluator is not None
            dataset_evaluator_id = str(GlobalID("DatasetEvaluator", str(dataset_evaluator.id)))

        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "datasetId": dataset_id,
                    "name": "updated-evaluator",
                    "description": llm_evaluator.description,
                    "promptVersionId": current_prompt_version_id,
                    "promptVersion": dict(
                        description=None,
                        templateFormat="F_STRING",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Test evaluator: {input}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict()),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="correctness",
                                        description="test llm evaluator",
                                        parameters={
                                            "type": "object",
                                            "properties": {
                                                "label": {
                                                    "type": "string",
                                                    "enum": ["correct", "incorrect"],
                                                    "description": "correctness",
                                                }
                                            },
                                            "required": ["label"],
                                        },
                                    ),
                                )
                            ],
                            toolChoice=dict(oneOrMore=True),
                        ),
                        responseFormat=None,
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="correctness",
                                description="description",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="correct", score=1),
                                    dict(label="incorrect", score=0),
                                ],
                            )
                        )
                    ],
                }
            },
        )
        assert result.data and not result.errors

        # Verify no new version was created
        async with db() as session:
            db_evaluator = await session.get(
                models.LLMEvaluator,
                llm_evaluator.id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert db_evaluator is not None
            # Verify prompt_id stayed the same
            assert db_evaluator.prompt_id == original_prompt_id
            # Verify prompt_version_tag points to the original version (no new version created)
            assert db_evaluator.prompt_version_tag is not None
            tag_version_id = db_evaluator.prompt_version_tag.prompt_version_id
            assert tag_version_id == current_prompt_version.id
            # Verify only one version exists (no new version created)
            prompt_versions = await session.scalars(
                select(models.PromptVersion).where(
                    models.PromptVersion.prompt_id == original_prompt_id
                )
            )
            assert len(prompt_versions.all()) == 1

    async def test_update_with_prompt_version_id_same_prompt(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        llm_evaluator: models.LLMEvaluator,
    ) -> None:
        """Test that updating with prompt_version_id pointing to same prompt updates tag."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        original_prompt_id = llm_evaluator.prompt_id

        # Create a second version for the same prompt
        async with db() as session:
            # Get existing prompt
            prompt = await session.get(models.Prompt, original_prompt_id)
            assert prompt is not None

            # Create a second version
            # Use exact values that will match the test input
            second_version = models.PromptVersion(
                description="updated",
                template_type=PromptTemplateType.CHAT,
                template_format=PromptTemplateFormat.F_STRING,
                template=PromptChatTemplate(
                    type="chat",
                    messages=[
                        PromptMessage(
                            role="user",
                            content=[TextContentPart(type="text", text="Second version: {input}")],
                        )
                    ],
                ),
                invocation_parameters=PromptOpenAIInvocationParameters(
                    type="openai",
                    openai=PromptOpenAIInvocationParametersContent(temperature=0.0),
                ),
                tools=PromptTools(
                    type="tools",
                    tools=[
                        PromptToolFunction(
                            type="function",
                            function=PromptToolFunctionDefinition(
                                name="correctness",
                                description="updated description",
                                parameters={
                                    "type": "object",
                                    "properties": {
                                        "label": {
                                            "type": "string",
                                            "enum": ["correct", "incorrect"],
                                            "description": "correctness",
                                        }
                                    },
                                    "required": ["label"],
                                },
                            ),
                        )
                    ],
                    tool_choice=PromptToolChoiceOneOrMore(type="one_or_more"),
                ),
                response_format=None,
                model_provider=ModelProvider.OPENAI,
                model_name="gpt-4",
                metadata_={},
                prompt_id=original_prompt_id,
            )
            session.add(second_version)
            await session.flush()
            second_version_id = str(GlobalID("PromptVersion", str(second_version.id)))

            dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id,
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id,
                )
            )
            assert dataset_evaluator is not None
            dataset_evaluator_id = str(GlobalID("DatasetEvaluator", str(dataset_evaluator.id)))

        # Update with prompt_version_id pointing to the second version (same prompt)
        # Use identical content so no new version is created
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "datasetId": dataset_id,
                    "name": "updated-evaluator",
                    "description": "updated description",
                    "promptVersionId": second_version_id,
                    "promptVersion": dict(
                        description="updated",
                        templateFormat="F_STRING",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Second version: {input}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.0)),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="correctness",
                                        description="updated description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["correct", "incorrect"],
                                                    description="correctness",
                                                )
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                )
                            ],
                            toolChoice=dict(oneOrMore=True),
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="correctness",
                                description="description",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="correct", score=1),
                                    dict(label="incorrect", score=0),
                                ],
                            )
                        )
                    ],
                }
            },
        )
        assert result.data and not result.errors

        # Verify evaluator still points to same prompt, and tag points to second version
        async with db() as session:
            db_evaluator = await session.get(
                models.LLMEvaluator,
                llm_evaluator.id,
                options=(selectinload(models.LLMEvaluator.prompt_version_tag),),
            )
            assert db_evaluator is not None
            # Verify prompt_id stayed the same
            assert db_evaluator.prompt_id == original_prompt_id
            # Verify prompt_version_tag points to the second version (no new version created)
            assert db_evaluator.prompt_version_tag is not None
            assert db_evaluator.prompt_version_tag.prompt_version_id == second_version.id
            assert db_evaluator.prompt_version_tag.prompt_id == original_prompt_id
            # Verify only 2 versions exist (original + second, no new one created)
            prompt_versions = await session.scalars(
                select(models.PromptVersion).where(
                    models.PromptVersion.prompt_id == original_prompt_id
                )
            )
            assert len(prompt_versions.all()) == 2


class TestCreateDatasetBuiltinEvaluatorMutation:
    _MUTATION = """
      mutation($input: CreateDatasetBuiltinEvaluatorInput!) {
        createDatasetBuiltinEvaluator(input: $input) {
          evaluator {
            id
            name
            evaluator {
              ... on BuiltInEvaluator {
                id
                name
                description
                kind
              }
            }
          }
          query { __typename }
        }
      }
    """

    async def _create(self, gql_client: AsyncGraphQLClient, **input_fields: Any) -> Any:
        """Private helper to execute create mutation with given input fields."""
        input_fields.setdefault("inputMapping", {"literalMapping": {}, "pathMapping": {}})
        return await gql_client.execute(self._MUTATION, {"input": input_fields})

    async def test_create_dataset_builtin_evaluator(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Get valid builtin evaluator IDs from the database
        async with db() as session:
            builtin_evaluator_ids = list(
                await session.scalars(select(models.BuiltinEvaluator.id).limit(2))
            )
        assert len(builtin_evaluator_ids) > 0, "No builtin evaluators available for testing"
        builtin_evaluator_id = builtin_evaluator_ids[0]
        builtin_evaluator_gid = str(GlobalID("BuiltInEvaluator", str(builtin_evaluator_id)))

        # Success: Create builtin evaluator with default input_mapping
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=builtin_evaluator_gid,
            name="test-builtin-evaluator",
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetBuiltinEvaluator"]["evaluator"]
        builtin_evaluator_data = dataset_evaluator["evaluator"]
        assert dataset_evaluator["name"] == "test-builtin-evaluator"
        assert builtin_evaluator_data["kind"] == "BUILTIN"

        dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
        async with db() as session:
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.evaluator_id == builtin_evaluator_id
            # user_id is None when authentication is disabled
            assert db_dataset_evaluator.user_id is None
            assert db_dataset_evaluator.input_mapping == InputMapping(
                literal_mapping={}, path_mapping={}
            )

        # Success: Create builtin evaluator with custom input_mapping
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=builtin_evaluator_gid,
            name="test-builtin-evaluator-2",
            inputMapping=dict(
                literalMapping={"key": "value"},
                pathMapping={"input": "$.context.input"},
            ),
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetBuiltinEvaluator"]["evaluator"]
        assert dataset_evaluator["name"] == "test-builtin-evaluator-2"

        dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
        async with db() as session:
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.input_mapping == InputMapping(
                literal_mapping={"key": "value"},
                path_mapping={"input": "$.context.input"},
            )

        # Success: Multiple builtin evaluators for same dataset
        if len(builtin_evaluator_ids) > 1:
            second_builtin_evaluator_id = builtin_evaluator_ids[1]
            second_builtin_evaluator_gid = str(
                GlobalID("BuiltInEvaluator", str(second_builtin_evaluator_id))
            )
            result = await self._create(
                gql_client,
                datasetId=dataset_id,
                evaluatorId=second_builtin_evaluator_gid,
                name="test-builtin-evaluator-3",
            )
            assert result.data and not result.errors

            async with db() as session:
                evaluators = await session.scalars(
                    select(models.DatasetEvaluators).where(
                        models.DatasetEvaluators.dataset_id == empty_dataset.id,
                        models.DatasetEvaluators.evaluator_id.isnot(None),
                    )
                )
                assert len(evaluators.all()) >= 2

        # Failure: Nonexistent dataset
        result = await self._create(
            gql_client,
            datasetId=str(GlobalID("Dataset", "999")),
            evaluatorId=builtin_evaluator_gid,
            name="test",
        )
        assert result.errors and "Dataset with id" in result.errors[0].message

        # Failure: Invalid evaluator type (not BuiltInEvaluator)
        invalid_evaluator_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=invalid_evaluator_id,
            name="test",
        )
        assert result.errors and "Invalid evaluator" in result.errors[0].message

        # Failure: Nonexistent builtin evaluator
        nonexistent_builtin_id = -999999
        nonexistent_builtin_gid = str(GlobalID("BuiltInEvaluator", str(nonexistent_builtin_id)))
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=nonexistent_builtin_gid,
            name="test",
        )
        assert result.errors and "not found" in result.errors[0].message.lower()

        # Failure: Nonexistent builtin evaluator ID (ID doesn't exist in database)
        nonexistent_positive_id = 99999
        nonexistent_positive_id_gid = str(
            GlobalID("BuiltInEvaluator", str(nonexistent_positive_id))
        )
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=nonexistent_positive_id_gid,
            name="test",
        )
        assert result.errors and "not found" in result.errors[0].message.lower()

        # Failure: Duplicate display name for same dataset
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=builtin_evaluator_gid,
            name="test-builtin-evaluator",  # Same as first one
        )
        assert result.errors and "already exists" in result.errors[0].message.lower()

    async def test_create_dataset_builtin_evaluator_duplicate_name_different_evaluators(
        self,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        db: DbSessionFactory,
        synced_builtin_evaluators: None,
    ) -> None:
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Get valid builtin evaluator IDs from the database
        async with db() as session:
            builtin_evaluator_ids = list(
                await session.scalars(select(models.BuiltinEvaluator.id).limit(2))
            )
        assert len(builtin_evaluator_ids) >= 2, "Need at least 2 builtin evaluators for this test"
        first_builtin_evaluator_id = builtin_evaluator_ids[0]
        second_builtin_evaluator_id = builtin_evaluator_ids[1]

        first_builtin_evaluator_gid = str(
            GlobalID("BuiltInEvaluator", str(first_builtin_evaluator_id))
        )
        second_builtin_evaluator_gid = str(
            GlobalID("BuiltInEvaluator", str(second_builtin_evaluator_id))
        )

        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=first_builtin_evaluator_gid,
            name="shared_name",
        )
        assert result.data and not result.errors
        assert result.data["createDatasetBuiltinEvaluator"]["evaluator"]["name"] == "shared_name"

        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=second_builtin_evaluator_gid,
            name="shared_name",
        )
        assert result.errors and "already exists" in result.errors[0].message.lower()

    async def test_create_dataset_builtin_evaluator_invalid_continuous_bounds(
        self,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        db: DbSessionFactory,
        synced_builtin_evaluators: None,
    ) -> None:
        """Test that continuous output config validates bounds."""
        # Look up the builtin evaluator ID by key from the database
        async with db() as session:
            levenshtein_id = await session.scalar(
                select(models.BuiltinEvaluator.id).where(
                    models.BuiltinEvaluator.key == "levenshtein_distance"
                )
            )
        assert levenshtein_id is not None

        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        levenshtein_gid = str(GlobalID("BuiltInEvaluator", str(levenshtein_id)))

        # Invalid: lower_bound >= upper_bound should fail
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=levenshtein_gid,
            name="test-levenshtein",
            outputConfigs=[
                {
                    "continuous": {
                        "name": "levenshtein_distance",
                        "optimizationDirection": "MAXIMIZE",
                        "lowerBound": 0.0,
                        "upperBound": -1.0,
                    }
                }
            ],
        )
        assert result.errors

        # Valid: lower_bound < upper_bound should succeed
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=levenshtein_gid,
            name="test-levenshtein-2",
            outputConfigs=[
                {
                    "continuous": {
                        "name": "levenshtein_distance",
                        "optimizationDirection": "MAXIMIZE",
                        "lowerBound": 5.0,
                        "upperBound": 10.0,
                    }
                }
            ],
        )
        assert result.data and not result.errors

    async def test_create_builtin_evaluator_without_output_configs_stores_none(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """
        When creating a builtin dataset evaluator without specifying output_configs,
        the DB should store None (no eager copy of base configs).
        """
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        async with db() as session:
            builtin_evaluator_id = await session.scalar(select(models.BuiltinEvaluator.id).limit(1))
        assert builtin_evaluator_id is not None
        builtin_evaluator_gid = str(GlobalID("BuiltInEvaluator", str(builtin_evaluator_id)))

        # Create without outputConfigs
        result = await self._create(
            gql_client,
            datasetId=dataset_id,
            evaluatorId=builtin_evaluator_gid,
            name="no-override-builtin",
        )
        assert result.data and not result.errors

        dataset_evaluator = result.data["createDatasetBuiltinEvaluator"]["evaluator"]
        dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)

        # DB level: output_configs should be None (no eager copy)
        async with db() as session:
            db_record = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_record is not None
            assert db_record.output_configs is None


class TestUpdateDatasetBuiltinEvaluatorMutation:
    _UPDATE_MUTATION = """
      mutation($input: UpdateDatasetBuiltinEvaluatorInput!) {
        updateDatasetBuiltinEvaluator(input: $input) {
          evaluator {
            id
            name
            evaluator {
              ... on BuiltInEvaluator {
                id
                name
                description
                kind
              }
            }
          }
          query { __typename }
        }
      }
    """

    async def test_update_dataset_builtin_evaluator(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """Test updating a builtin evaluator via its DatasetEvaluator assignment."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Get a valid builtin evaluator ID from the database
        async with db() as session:
            builtin_evaluator_id = await session.scalar(select(models.BuiltinEvaluator.id).limit(1))
        assert builtin_evaluator_id is not None, "No builtin evaluators available for testing"
        builtin_evaluator_gid = str(GlobalID("BuiltInEvaluator", str(builtin_evaluator_id)))

        # First, create a builtin evaluator to update
        create_result = await gql_client.execute(
            """
            mutation($input: CreateDatasetBuiltinEvaluatorInput!) {
              createDatasetBuiltinEvaluator(input: $input) {
                evaluator {
                  id
                  name
                }
              }
            }
            """,
            {
                "input": {
                    "datasetId": dataset_id,
                    "evaluatorId": builtin_evaluator_gid,
                    "name": "original-name",
                    "inputMapping": dict(
                        literalMapping={"key": "original"},
                        pathMapping={"input": "$.context.original"},
                    ),
                }
            },
        )
        assert create_result.data and not create_result.errors
        dataset_evaluator_id = create_result.data["createDatasetBuiltinEvaluator"]["evaluator"][
            "id"
        ]

        # Update the evaluator with new display name
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "name": "updated-name",
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                }
            },
        )
        assert result.data and not result.errors

        updated_evaluator = result.data["updateDatasetBuiltinEvaluator"]["evaluator"]
        assert updated_evaluator["name"] == "updated-name"
        builtin_data = updated_evaluator["evaluator"]
        assert builtin_data["kind"] == "BUILTIN"

        # Verify database state
        dataset_evaluator_rowid = int(GlobalID.from_id(dataset_evaluator_id).node_id)
        async with db() as session:
            db_dataset_evaluator = await session.get(
                models.DatasetEvaluators, dataset_evaluator_rowid
            )
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.name.root == "updated-name"
            assert db_dataset_evaluator.input_mapping == InputMapping(
                literal_mapping={}, path_mapping={}
            )
            # user_id is None when authentication is disabled
            assert db_dataset_evaluator.user_id is None

        # Update the evaluator with new input_mapping
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "name": "updated-name",
                    "inputMapping": dict(
                        literalMapping={"new_key": "new_value"},
                        pathMapping={"output": "$.context.output"},
                    ),
                }
            },
        )
        assert result.data and not result.errors

        async with db() as session:
            db_dataset_evaluator = await session.get(
                models.DatasetEvaluators, dataset_evaluator_rowid
            )
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.input_mapping == InputMapping(
                literal_mapping={"new_key": "new_value"},
                path_mapping={"output": "$.context.output"},
            )

        # Update both display name and input_mapping
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "name": "final-name",
                    "inputMapping": dict(
                        literalMapping={"final": "value"},
                        pathMapping={},
                    ),
                }
            },
        )
        assert result.data and not result.errors

        updated_evaluator = result.data["updateDatasetBuiltinEvaluator"]["evaluator"]
        assert updated_evaluator["name"] == "final-name"

        async with db() as session:
            db_dataset_evaluator = await session.get(
                models.DatasetEvaluators, dataset_evaluator_rowid
            )
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.name.root == "final-name"
            assert db_dataset_evaluator.input_mapping == InputMapping(
                literal_mapping={"final": "value"},
                path_mapping={},
            )

        # Failure: Nonexistent dataset evaluator
        nonexistent_id = str(GlobalID("DatasetEvaluator", "999999"))
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": nonexistent_id,
                    "name": "test",
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                }
            },
        )
        assert result.errors and "not found" in result.errors[0].message.lower()

        # Failure: Try to update a non-builtin evaluator (LLM evaluator)
        # First create an LLM evaluator
        prompt = models.Prompt(
            name=IdentifierModel.model_validate(f"test-prompt-{token_hex(4)}"),
            description="test prompt",
            prompt_versions=[
                models.PromptVersion(
                    template_type=PromptTemplateType.STRING,
                    template_format=PromptTemplateFormat.F_STRING,
                    template=PromptStringTemplate(type="string", template="Test: {input}"),
                    invocation_parameters=PromptOpenAIInvocationParameters(
                        type="openai", openai=PromptOpenAIInvocationParametersContent()
                    ),
                    tools=None,
                    response_format=None,
                    model_provider=ModelProvider.OPENAI,
                    model_name="gpt-4",
                    metadata_={},
                )
            ],
        )
        llm_evaluator_name = IdentifierModel.model_validate(f"test-llm-eval-{token_hex(4)}")
        llm_evaluator = models.LLMEvaluator(
            name=llm_evaluator_name,
            description="test llm evaluator",
            kind="LLM",
            output_configs=[
                CategoricalOutputConfig(
                    type="CATEGORICAL",
                    name="test",
                    optimization_direction=OptimizationDirection.MAXIMIZE,
                    description="test description",
                    values=[
                        CategoricalAnnotationValue(label="good", score=1.0),
                        CategoricalAnnotationValue(label="bad", score=0.0),
                    ],
                )
            ],
            prompt=prompt,
            dataset_evaluators=[
                models.DatasetEvaluators(
                    dataset_id=empty_dataset.id,
                    name=llm_evaluator_name,
                    description="test description",
                    output_configs=None,
                    input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                    project=models.Project(
                        name=f"{empty_dataset.name}/{llm_evaluator_name}",
                        description="Project for llm evaluator",
                    ),
                )
            ],
        )
        async with db() as session:
            session.add(llm_evaluator)
            await session.flush()
            llm_dataset_evaluator = await session.scalar(
                select(models.DatasetEvaluators).where(
                    models.DatasetEvaluators.dataset_id == empty_dataset.id,
                    models.DatasetEvaluators.evaluator_id == llm_evaluator.id,
                )
            )
            assert llm_dataset_evaluator is not None
            llm_dataset_evaluator_id = str(
                GlobalID("DatasetEvaluator", str(llm_dataset_evaluator.id))
            )

        # Try to update the LLM evaluator (should fail)
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": llm_dataset_evaluator_id,
                    "name": "updated-llm-name",
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                }
            },
        )
        assert result.errors and "non-built-in evaluator" in result.errors[0].message.lower()

        # Cleanup
        async with db() as session:
            await session.execute(
                sa.delete(models.LLMEvaluator).where(models.LLMEvaluator.id == llm_evaluator.id)
            )
            await session.execute(
                sa.delete(models.PromptVersion).where(models.PromptVersion.prompt_id == prompt.id)
            )
            await session.execute(sa.delete(models.Prompt).where(models.Prompt.id == prompt.id))

        # Failure: Duplicate display name on the same dataset
        # The unique constraint is on (dataset_id, name),
        # so any evaluator with a duplicate name on the same dataset should fail
        create_result2 = await gql_client.execute(
            """
            mutation($input: CreateDatasetBuiltinEvaluatorInput!) {
              createDatasetBuiltinEvaluator(input: $input) {
                evaluator {
                  id
                  name
                }
              }
            }
            """,
            {
                "input": {
                    "datasetId": dataset_id,
                    "evaluatorId": builtin_evaluator_gid,  # Same builtin evaluator
                    "name": "other-evaluator",
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                }
            },
        )
        assert create_result2.data and not create_result2.errors

        # Try to update the first evaluator to have the same name as the second
        # (both are the same builtin_evaluator_id, so this should fail)
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "name": "other-evaluator",  # Same as second assignment
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                }
            },
        )
        assert result.errors and "already exists" in result.errors[0].message.lower()

    async def test_update_dataset_builtin_evaluator_invalid_continuous_bounds(
        self,
        gql_client: AsyncGraphQLClient,
        db: DbSessionFactory,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """Test that update mutation validates bounds for continuous config."""
        # Look up the builtin evaluator ID by key from the database
        async with db() as session:
            levenshtein_id = await session.scalar(
                select(models.BuiltinEvaluator.id).where(
                    models.BuiltinEvaluator.key == "levenshtein_distance"
                )
            )
        assert levenshtein_id is not None

        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        levenshtein_gid = str(GlobalID("BuiltInEvaluator", str(levenshtein_id)))

        create_result = await gql_client.execute(
            """
            mutation($input: CreateDatasetBuiltinEvaluatorInput!) {
              createDatasetBuiltinEvaluator(input: $input) {
                evaluator { id }
              }
            }
            """,
            {
                "input": {
                    "datasetId": dataset_id,
                    "evaluatorId": levenshtein_gid,
                    "name": "test-levenshtein-update",
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                }
            },
        )
        assert create_result.data and not create_result.errors
        dataset_evaluator_id = create_result.data["createDatasetBuiltinEvaluator"]["evaluator"][
            "id"
        ]

        # Invalid: lower_bound >= upper_bound should fail
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "name": "test-levenshtein-update",
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "outputConfigs": [
                        {
                            "continuous": {
                                "name": "levenshtein_distance",
                                "optimizationDirection": "MAXIMIZE",
                                "lowerBound": 0.0,
                                "upperBound": -1.0,
                            }
                        }
                    ],
                }
            },
        )
        assert result.errors

        # Valid: lower_bound < upper_bound should succeed
        result = await gql_client.execute(
            self._UPDATE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorId": dataset_evaluator_id,
                    "name": "test-levenshtein-update",
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "outputConfigs": [
                        {
                            "continuous": {
                                "name": "levenshtein_distance",
                                "optimizationDirection": "MAXIMIZE",
                                "lowerBound": 5.0,
                                "upperBound": 10.0,
                            }
                        }
                    ],
                }
            },
        )
        assert result.data and not result.errors


@pytest.fixture
async def empty_dataset(db: DbSessionFactory) -> AsyncIterator[models.Dataset]:
    """Inserts an empty dataset."""
    dataset = models.Dataset(name=f"test dataset {token_hex(4)}", description="test", metadata_={})
    async with db() as session:
        session.add(dataset)
        await session.flush()
    yield dataset
    async with db() as session:
        await session.execute(sa.delete(models.Dataset).where(models.Dataset.id == dataset.id))


@pytest.fixture
async def llm_evaluator(
    db: DbSessionFactory, empty_dataset: models.Dataset
) -> AsyncIterator[models.LLMEvaluator]:
    """Inserts an LLM evaluator with dataset relationship."""
    evaluator_name = IdentifierModel.model_validate(f"test-llm-evaluator-{token_hex(4)}")
    evaluator_description = "test llm evaluator"
    annotation_name = "correctness"

    tools = PromptTools(
        type="tools",
        tools=[
            PromptToolFunction(
                type="function",
                function=PromptToolFunctionDefinition(
                    name=annotation_name,
                    description=evaluator_description,
                    parameters={
                        "type": "object",
                        "properties": {
                            "label": {
                                "type": "string",
                                "enum": ["correct", "incorrect"],
                                "description": annotation_name,
                            }
                        },
                        "required": ["label"],
                    },
                ),
            )
        ],
        tool_choice=PromptToolChoiceOneOrMore(type="one_or_more"),
    )

    prompt = models.Prompt(
        name=IdentifierModel.model_validate(f"test-prompt-{token_hex(4)}"),
        description="test prompt",
        prompt_versions=[
            models.PromptVersion(
                template_type=PromptTemplateType.CHAT,
                template_format=PromptTemplateFormat.F_STRING,
                template=PromptChatTemplate(
                    type="chat",
                    messages=[
                        PromptMessage(
                            role="user",
                            content=[TextContentPart(type="text", text="Test evaluator: {input}")],
                        )
                    ],
                ),
                invocation_parameters=PromptOpenAIInvocationParameters(
                    type="openai",
                    openai=PromptOpenAIInvocationParametersContent(),
                ),
                tools=tools,
                response_format=None,
                model_provider=ModelProvider.OPENAI,
                model_name="gpt-4",
                metadata_={},
            )
        ],
    )
    evaluator = models.LLMEvaluator(
        name=evaluator_name,
        description=evaluator_description,
        kind="LLM",
        output_configs=[
            CategoricalOutputConfig(
                type="CATEGORICAL",
                name=annotation_name,
                optimization_direction=OptimizationDirection.MAXIMIZE,
                description="correctness description",
                values=[
                    CategoricalAnnotationValue(label="correct", score=1.0),
                    CategoricalAnnotationValue(label="incorrect", score=0.0),
                ],
            )
        ],
        prompt=prompt,
        dataset_evaluators=[
            models.DatasetEvaluators(
                dataset_id=empty_dataset.id,
                name=evaluator_name,
                description="correctness description",
                output_configs=None,
                input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                project=models.Project(
                    name=f"{empty_dataset.name}/{evaluator_name}",
                    description="Project for evaluator",
                ),
            ),
        ],
    )
    async with db() as session:
        session.add(evaluator)
        await session.flush()
        prompt_version = prompt.prompt_versions[0]
        tag_name = IdentifierModel.model_validate(f"{evaluator_name.root}-evaluator-{token_hex(4)}")
        prompt_tag = models.PromptVersionTag(
            name=tag_name,
            prompt_id=prompt.id,
            prompt_version_id=prompt_version.id,
        )
        evaluator.prompt_version_tag = prompt_tag
        session.add(evaluator)

    yield evaluator
    async with db() as session:
        await session.execute(
            sa.delete(models.LLMEvaluator).where(models.LLMEvaluator.id == evaluator.id)
        )
        await session.execute(
            sa.delete(models.PromptVersionTag).where(models.PromptVersionTag.prompt_id == prompt.id)
        )
        await session.execute(
            sa.delete(models.PromptVersion).where(models.PromptVersion.prompt_id == prompt.id)
        )
        await session.execute(sa.delete(models.Prompt).where(models.Prompt.id == prompt.id))


class TestDeleteDatasetEvaluators:
    """Tests for the delete_dataset_evaluators mutation."""

    _DELETE_MUTATION = """
      mutation($input: DeleteDatasetEvaluatorsInput!) {
        deleteDatasetEvaluators(input: $input) {
          datasetEvaluatorIds
          query { __typename }
        }
      }
    """

    async def test_delete_builtin_evaluator(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """Test deleting a built-in evaluator removes the DatasetEvaluators row and project."""
        # Get a valid builtin evaluator ID from the database
        async with db() as session:
            builtin_evaluator_id = await session.scalar(select(models.BuiltinEvaluator.id).limit(1))
        assert builtin_evaluator_id is not None, "No builtin evaluators available for testing"

        # Create a builtin dataset evaluator
        async with db() as session:
            dataset_evaluator = models.DatasetEvaluators(
                dataset_id=empty_dataset.id,
                evaluator_id=builtin_evaluator_id,
                name=IdentifierModel.model_validate("test-builtin-evaluator"),
                input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                project=models.Project(
                    name=f"{empty_dataset.name}/test-builtin-evaluator",
                    description="Project for builtin evaluator",
                ),
            )
            session.add(dataset_evaluator)
            await session.flush()
            dataset_evaluator_id = dataset_evaluator.id
            project_id = dataset_evaluator.project_id
            dataset_evaluator_gid = str(GlobalID("DatasetEvaluator", str(dataset_evaluator_id)))

        # Delete the evaluator
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {"input": {"datasetEvaluatorIds": [dataset_evaluator_gid]}},
        )

        # Verify success
        assert not result.errors
        assert result.data is not None
        assert (
            dataset_evaluator_gid in result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"]
        )

        # Verify the DatasetEvaluators row and associated project are deleted
        # but the shared built-in evaluator definition is preserved.
        async with db() as session:
            deleted_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert deleted_evaluator is None

            deleted_project = await session.get(models.Project, project_id)
            assert deleted_project is None

            preserved_builtin = await session.get(models.BuiltinEvaluator, builtin_evaluator_id)
            assert preserved_builtin is not None

    async def test_delete_dataset_link_to_builtin_preserves_other_dataset_links(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """Regression test for #13090.

        Deleting one dataset's link to a shared built-in must not delete the built-in
        itself, nor any other dataset's link that points to the same built-in.
        """
        async with db() as session:
            builtin_evaluator_id = await session.scalar(select(models.BuiltinEvaluator.id).limit(1))
        assert builtin_evaluator_id is not None, "No builtin evaluators available for testing"

        async with db() as session:
            dataset_b = models.Dataset(
                name=f"dataset-b-{token_hex(4)}",
                description="second dataset for shared-builtin test",
                metadata_={},
            )
            session.add(dataset_b)
            await session.flush()
            dataset_b_id = dataset_b.id

            link_a = models.DatasetEvaluators(
                dataset_id=empty_dataset.id,
                evaluator_id=builtin_evaluator_id,
                name=IdentifierModel.model_validate(f"shared-builtin-a-{token_hex(4)}"),
                input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                project=models.Project(
                    name=f"{empty_dataset.name}/shared-builtin-a-{token_hex(4)}",
                    description="Project for dataset A link",
                ),
            )
            link_b = models.DatasetEvaluators(
                dataset_id=dataset_b_id,
                evaluator_id=builtin_evaluator_id,
                name=IdentifierModel.model_validate(f"shared-builtin-b-{token_hex(4)}"),
                input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                project=models.Project(
                    name=f"{dataset_b.name}/shared-builtin-b-{token_hex(4)}",
                    description="Project for dataset B link",
                ),
            )
            session.add_all([link_a, link_b])
            await session.flush()
            link_a_id = link_a.id
            link_b_id = link_b.id
            link_b_project_id = link_b.project_id
            link_a_gid = str(GlobalID("DatasetEvaluator", str(link_a_id)))

        # Delete only dataset A's link to the built-in.
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {"input": {"datasetEvaluatorIds": [link_a_gid]}},
        )

        assert not result.errors
        assert result.data is not None
        assert result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"] == [link_a_gid]

        async with db() as session:
            assert await session.get(models.DatasetEvaluators, link_a_id) is None

            preserved_builtin = await session.get(models.BuiltinEvaluator, builtin_evaluator_id)
            assert preserved_builtin is not None, (
                "shared built-in evaluator must not be deleted when one dataset's link is removed"
            )

            preserved_link_b = await session.get(models.DatasetEvaluators, link_b_id)
            assert preserved_link_b is not None, (
                "dataset B's link must not be cascade-deleted by removing dataset A's link"
            )

            preserved_link_b_project = await session.get(models.Project, link_b_project_id)
            assert preserved_link_b_project is not None

    async def test_delete_llm_evaluator(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test deleting an LLM evaluator cascades to delete the Evaluator record."""
        evaluator_name = IdentifierModel.model_validate(f"test-llm-evaluator-delete-{token_hex(4)}")

        # Create an LLM evaluator with dataset relationship
        async with db() as session:
            prompt = models.Prompt(
                name=IdentifierModel.model_validate(f"test-prompt-{token_hex(4)}"),
                description="test prompt",
                prompt_versions=[
                    models.PromptVersion(
                        template_type="CHAT",
                        template_format="MUSTACHE",
                        template=PromptChatTemplate(
                            type="chat",
                            messages=[
                                PromptMessage(
                                    role="user",
                                    content=[TextContentPart(type="text", text="Test: {{input}}")],
                                )
                            ],
                        ),
                        invocation_parameters=PromptOpenAIInvocationParameters(
                            type="openai",
                            openai=PromptOpenAIInvocationParametersContent(),
                        ),
                        tools=None,
                        response_format=None,
                        model_provider=ModelProvider.OPENAI,
                        model_name="gpt-4",
                        metadata_={},
                    )
                ],
            )
            llm_evaluator = models.LLMEvaluator(
                name=evaluator_name,
                description="test llm evaluator for deletion",
                kind="LLM",
                output_configs=[
                    CategoricalOutputConfig(
                        type="CATEGORICAL",
                        name="correctness",
                        optimization_direction=OptimizationDirection.MAXIMIZE,
                        description="correctness description",
                        values=[
                            CategoricalAnnotationValue(label="correct", score=1.0),
                            CategoricalAnnotationValue(label="incorrect", score=0.0),
                        ],
                    )
                ],
                prompt=prompt,
                dataset_evaluators=[
                    models.DatasetEvaluators(
                        dataset_id=empty_dataset.id,
                        name=evaluator_name,
                        description="test description",
                        output_configs=None,
                        input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                        project=models.Project(
                            name=f"{empty_dataset.name}/{evaluator_name}",
                            description="Project for llm evaluator deletion",
                        ),
                    )
                ],
            )
            session.add(llm_evaluator)
            await session.flush()
            evaluator_id = llm_evaluator.id
            dataset_evaluator_id = llm_evaluator.dataset_evaluators[0].id
            project_id = llm_evaluator.dataset_evaluators[0].project_id
            dataset_evaluator_gid = str(GlobalID("DatasetEvaluator", str(dataset_evaluator_id)))

        # Delete the dataset evaluator
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {"input": {"datasetEvaluatorIds": [dataset_evaluator_gid]}},
        )

        # Verify success
        assert not result.errors
        assert result.data is not None
        assert (
            dataset_evaluator_gid in result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"]
        )

        # Verify the DatasetEvaluators, Evaluator, and Project rows are deleted
        async with db() as session:
            deleted_dataset_evaluator = await session.get(
                models.DatasetEvaluators, dataset_evaluator_id
            )
            assert deleted_dataset_evaluator is None

            deleted_evaluator = await session.get(models.Evaluator, evaluator_id)
            assert deleted_evaluator is None

            deleted_project = await session.get(models.Project, project_id)
            assert deleted_project is None

    async def test_delete_multiple_evaluators_mixed(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """Test deleting a mix of built-in and LLM evaluators in a single call."""
        # Get a valid builtin evaluator ID from the database
        async with db() as session:
            builtin_evaluator_id = await session.scalar(select(models.BuiltinEvaluator.id).limit(1))
        assert builtin_evaluator_id is not None, "No builtin evaluators available for testing"

        # Create a builtin dataset evaluator
        async with db() as session:
            builtin_dataset_evaluator = models.DatasetEvaluators(
                dataset_id=empty_dataset.id,
                evaluator_id=builtin_evaluator_id,
                name=IdentifierModel.model_validate("test-builtin-for-batch-delete"),
                input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                project=models.Project(
                    name=f"{empty_dataset.name}/test-builtin-for-batch-delete",
                    description="Project for builtin batch delete",
                ),
            )
            session.add(builtin_dataset_evaluator)
            await session.flush()
            builtin_de_id = builtin_dataset_evaluator.id
            builtin_project_id = builtin_dataset_evaluator.project_id
            builtin_de_gid = str(GlobalID("DatasetEvaluator", str(builtin_de_id)))

        # Create an LLM evaluator
        evaluator_name = IdentifierModel.model_validate(f"test-llm-batch-delete-{token_hex(4)}")
        async with db() as session:
            prompt = models.Prompt(
                name=IdentifierModel.model_validate(f"test-prompt-batch-{token_hex(4)}"),
                description="test prompt",
                prompt_versions=[
                    models.PromptVersion(
                        template_type="CHAT",
                        template_format="MUSTACHE",
                        template=PromptChatTemplate(
                            type="chat",
                            messages=[
                                PromptMessage(
                                    role="user",
                                    content=[TextContentPart(type="text", text="Test: {{input}}")],
                                )
                            ],
                        ),
                        invocation_parameters=PromptOpenAIInvocationParameters(
                            type="openai",
                            openai=PromptOpenAIInvocationParametersContent(),
                        ),
                        tools=None,
                        response_format=None,
                        model_provider=ModelProvider.OPENAI,
                        model_name="gpt-4",
                        metadata_={},
                    )
                ],
            )
            llm_evaluator = models.LLMEvaluator(
                name=evaluator_name,
                description="test llm evaluator for batch deletion",
                kind="LLM",
                output_configs=[
                    CategoricalOutputConfig(
                        type="CATEGORICAL",
                        name="correctness",
                        optimization_direction=OptimizationDirection.MAXIMIZE,
                        description="correctness description",
                        values=[
                            CategoricalAnnotationValue(label="correct", score=1.0),
                            CategoricalAnnotationValue(label="incorrect", score=0.0),
                        ],
                    )
                ],
                prompt=prompt,
                dataset_evaluators=[
                    models.DatasetEvaluators(
                        dataset_id=empty_dataset.id,
                        name=evaluator_name,
                        description="test description",
                        output_configs=None,
                        input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                        project=models.Project(
                            name=f"{empty_dataset.name}/{evaluator_name}",
                            description="Project for batch delete llm evaluator",
                        ),
                    )
                ],
            )
            session.add(llm_evaluator)
            await session.flush()
            llm_evaluator_id = llm_evaluator.id
            llm_de_id = llm_evaluator.dataset_evaluators[0].id
            llm_project_id = llm_evaluator.dataset_evaluators[0].project_id
            llm_de_gid = str(GlobalID("DatasetEvaluator", str(llm_de_id)))

        # Delete both evaluators in one call
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {"input": {"datasetEvaluatorIds": [builtin_de_gid, llm_de_gid]}},
        )

        # Verify success
        assert not result.errors
        assert result.data is not None
        deleted_ids = result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"]
        assert builtin_de_gid in deleted_ids
        assert llm_de_gid in deleted_ids

        # Verify all rows including projects are deleted
        async with db() as session:
            # Builtin dataset evaluator should be deleted
            deleted_builtin_de = await session.get(models.DatasetEvaluators, builtin_de_id)
            assert deleted_builtin_de is None

            # Builtin project should be deleted
            deleted_builtin_project = await session.get(models.Project, builtin_project_id)
            assert deleted_builtin_project is None

            # LLM dataset evaluator and its parent evaluator should be deleted
            deleted_llm_de = await session.get(models.DatasetEvaluators, llm_de_id)
            assert deleted_llm_de is None

            deleted_llm_evaluator = await session.get(models.Evaluator, llm_evaluator_id)
            assert deleted_llm_evaluator is None

            # LLM project should be deleted
            deleted_llm_project = await session.get(models.Project, llm_project_id)
            assert deleted_llm_project is None

    async def test_delete_nonexistent_evaluator(
        self,
        gql_client: AsyncGraphQLClient,
    ) -> None:
        """Test deleting a nonexistent evaluator returns empty result."""
        nonexistent_gid = str(GlobalID("DatasetEvaluator", "999999"))

        # Attempt to delete nonexistent evaluator
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {"input": {"datasetEvaluatorIds": [nonexistent_gid]}},
        )

        # Should succeed but return empty list since no evaluator was found
        assert not result.errors
        assert result.data is not None
        assert result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"] == []

    async def test_delete_invalid_id_type(
        self,
        gql_client: AsyncGraphQLClient,
    ) -> None:
        """Test deleting with invalid ID type returns error."""
        invalid_gid = str(GlobalID("Dataset", "1"))  # Wrong type

        # Attempt to delete with invalid type
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {"input": {"datasetEvaluatorIds": [invalid_gid]}},
        )

        # Should return error
        assert result.errors is not None
        assert "Invalid dataset evaluator id" in result.errors[0].message

    async def test_delete_empty_list(
        self,
        gql_client: AsyncGraphQLClient,
    ) -> None:
        """Test deleting with empty list returns empty result."""
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {"input": {"datasetEvaluatorIds": []}},
        )

        # Should succeed with empty result
        assert not result.errors
        assert result.data is not None
        assert result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"] == []

    async def test_delete_llm_evaluator_with_prompt_deletion(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test deleting an LLM evaluator with deleteAssociatedPrompt=True also deletes the prompt."""
        evaluator_name = IdentifierModel.model_validate(f"test-llm-with-prompt-del-{token_hex(4)}")

        # Create an LLM evaluator with prompt
        async with db() as session:
            prompt = models.Prompt(
                name=IdentifierModel.model_validate(f"test-prompt-to-delete-{token_hex(4)}"),
                description="test prompt to be deleted",
                prompt_versions=[
                    models.PromptVersion(
                        template_type="CHAT",
                        template_format="MUSTACHE",
                        template=PromptChatTemplate(
                            type="chat",
                            messages=[
                                PromptMessage(
                                    role="user",
                                    content=[TextContentPart(type="text", text="Test: {{input}}")],
                                )
                            ],
                        ),
                        invocation_parameters=PromptOpenAIInvocationParameters(
                            type="openai",
                            openai=PromptOpenAIInvocationParametersContent(),
                        ),
                        tools=None,
                        response_format=None,
                        model_provider=ModelProvider.OPENAI,
                        model_name="gpt-4",
                        metadata_={},
                    )
                ],
            )
            llm_evaluator = models.LLMEvaluator(
                name=evaluator_name,
                description="test llm evaluator for prompt deletion",
                kind="LLM",
                output_configs=[
                    CategoricalOutputConfig(
                        type="CATEGORICAL",
                        name="correctness",
                        optimization_direction=OptimizationDirection.MAXIMIZE,
                        description="correctness description",
                        values=[
                            CategoricalAnnotationValue(label="correct", score=1.0),
                            CategoricalAnnotationValue(label="incorrect", score=0.0),
                        ],
                    )
                ],
                prompt=prompt,
                dataset_evaluators=[
                    models.DatasetEvaluators(
                        dataset_id=empty_dataset.id,
                        name=evaluator_name,
                        description="test description",
                        output_configs=None,
                        input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                        project=models.Project(
                            name=f"{empty_dataset.name}/{evaluator_name}",
                            description="Project for llm evaluator with prompt deletion",
                        ),
                    )
                ],
            )
            session.add(llm_evaluator)
            await session.flush()
            prompt_id = prompt.id
            evaluator_id = llm_evaluator.id
            dataset_evaluator_id = llm_evaluator.dataset_evaluators[0].id
            project_id = llm_evaluator.dataset_evaluators[0].project_id
            dataset_evaluator_gid = str(GlobalID("DatasetEvaluator", str(dataset_evaluator_id)))

        # Delete with deleteAssociatedPrompt=True (default)
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorIds": [dataset_evaluator_gid],
                    "deleteAssociatedPrompt": True,
                }
            },
        )

        # Verify success
        assert not result.errors
        assert result.data is not None
        assert (
            dataset_evaluator_gid in result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"]
        )

        # Verify the evaluator, prompt, and project are all deleted
        async with db() as session:
            deleted_dataset_evaluator = await session.get(
                models.DatasetEvaluators, dataset_evaluator_id
            )
            assert deleted_dataset_evaluator is None

            deleted_evaluator = await session.get(models.Evaluator, evaluator_id)
            assert deleted_evaluator is None

            deleted_prompt = await session.get(models.Prompt, prompt_id)
            assert deleted_prompt is None

            deleted_project = await session.get(models.Project, project_id)
            assert deleted_project is None

    async def test_delete_llm_evaluator_without_prompt_deletion(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Test deleting an LLM evaluator with deleteAssociatedPrompt=False preserves the prompt."""
        evaluator_name = IdentifierModel.model_validate(f"test-llm-keep-prompt-{token_hex(4)}")

        # Create an LLM evaluator with prompt
        async with db() as session:
            prompt = models.Prompt(
                name=IdentifierModel.model_validate(f"test-prompt-to-keep-{token_hex(4)}"),
                description="test prompt to be kept",
                prompt_versions=[
                    models.PromptVersion(
                        template_type="CHAT",
                        template_format="MUSTACHE",
                        template=PromptChatTemplate(
                            type="chat",
                            messages=[
                                PromptMessage(
                                    role="user",
                                    content=[TextContentPart(type="text", text="Test: {{input}}")],
                                )
                            ],
                        ),
                        invocation_parameters=PromptOpenAIInvocationParameters(
                            type="openai",
                            openai=PromptOpenAIInvocationParametersContent(),
                        ),
                        tools=None,
                        response_format=None,
                        model_provider=ModelProvider.OPENAI,
                        model_name="gpt-4",
                        metadata_={},
                    )
                ],
            )
            llm_evaluator = models.LLMEvaluator(
                name=evaluator_name,
                description="test llm evaluator to keep prompt",
                kind="LLM",
                output_configs=[
                    CategoricalOutputConfig(
                        type="CATEGORICAL",
                        name="correctness",
                        optimization_direction=OptimizationDirection.MAXIMIZE,
                        description="correctness description",
                        values=[
                            CategoricalAnnotationValue(label="correct", score=1.0),
                            CategoricalAnnotationValue(label="incorrect", score=0.0),
                        ],
                    )
                ],
                prompt=prompt,
                dataset_evaluators=[
                    models.DatasetEvaluators(
                        dataset_id=empty_dataset.id,
                        name=evaluator_name,
                        description="test description",
                        output_configs=None,
                        input_mapping=InputMapping(literal_mapping={}, path_mapping={}),
                        project=models.Project(
                            name=f"{empty_dataset.name}/{evaluator_name}",
                            description="Project for llm evaluator without prompt deletion",
                        ),
                    )
                ],
            )
            session.add(llm_evaluator)
            await session.flush()
            prompt_id = prompt.id
            evaluator_id = llm_evaluator.id
            dataset_evaluator_id = llm_evaluator.dataset_evaluators[0].id
            project_id = llm_evaluator.dataset_evaluators[0].project_id
            dataset_evaluator_gid = str(GlobalID("DatasetEvaluator", str(dataset_evaluator_id)))

        # Delete with deleteAssociatedPrompt=False
        result = await gql_client.execute(
            self._DELETE_MUTATION,
            {
                "input": {
                    "datasetEvaluatorIds": [dataset_evaluator_gid],
                    "deleteAssociatedPrompt": False,
                }
            },
        )

        # Verify success
        assert not result.errors
        assert result.data is not None
        assert (
            dataset_evaluator_gid in result.data["deleteDatasetEvaluators"]["datasetEvaluatorIds"]
        )

        # Verify the evaluator and project are deleted but prompt is preserved
        async with db() as session:
            deleted_dataset_evaluator = await session.get(
                models.DatasetEvaluators, dataset_evaluator_id
            )
            assert deleted_dataset_evaluator is None

            deleted_evaluator = await session.get(models.Evaluator, evaluator_id)
            assert deleted_evaluator is None

            # Project should be deleted
            deleted_project = await session.get(models.Project, project_id)
            assert deleted_project is None

            # Prompt should still exist
            kept_prompt = await session.get(models.Prompt, prompt_id)
            assert kept_prompt is not None

        # Clean up the prompt
        async with db() as session:
            await session.execute(sa.delete(models.Prompt).where(models.Prompt.id == prompt_id))


class TestMultiOutputEvaluators:
    """Tests for multi-output evaluator functionality."""

    _CREATE_LLM_MUTATION = """
      mutation($input: CreateDatasetLLMEvaluatorInput!) {
        createDatasetLlmEvaluator(input: $input) {
          evaluator {
            id
            name
            description
            outputConfigs {
              ... on CategoricalAnnotationConfig {
                name
                description
                optimizationDirection
                values { label score }
              }
              ... on ContinuousAnnotationConfig {
                name
                description
                lowerBound
                upperBound
                optimizationDirection
              }
            }
            evaluator {
              ... on LLMEvaluator {
                id
                name
                kind
              }
            }
          }
          query { __typename }
        }
      }
    """

    _CREATE_BUILTIN_MUTATION = """
      mutation($input: CreateDatasetBuiltinEvaluatorInput!) {
        createDatasetBuiltinEvaluator(input: $input) {
          evaluator {
            id
            name
            outputConfigs {
              ... on CategoricalAnnotationConfig {
                name
                description
                optimizationDirection
                values { label score }
              }
              ... on ContinuousAnnotationConfig {
                name
                description
                lowerBound
                upperBound
                optimizationDirection
              }
            }
            evaluator {
              ... on BuiltInEvaluator {
                id
                name
                kind
                outputConfigs {
                  ... on CategoricalAnnotationConfig {
                    name
                    description
                    values { label score }
                  }
                  ... on ContinuousAnnotationConfig {
                    name
                    description
                    lowerBound
                    upperBound
                  }
                }
              }
            }
          }
          query { __typename }
        }
      }
    """

    async def test_create_evaluator_with_multiple_configs(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Create LLM evaluator with 2+ output configs and verify all configs are stored."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        # Note: LLM evaluator validation requires a 'label' property in the tool function
        # The first output config must be categorical and:
        # - Its values must match the label enum
        # - The label description must match the first output config name
        result = await gql_client.execute(
            self._CREATE_LLM_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetId": dataset_id,
                    "name": "multi-output-evaluator",
                    "description": "multi-output evaluator description",
                    "promptVersion": dict(
                        description="multi-output prompt",
                        templateFormat="MUSTACHE",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Evaluate: {{input}}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.0)),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="quality",
                                        description="multi-output evaluator description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["good", "bad"],
                                                    description="quality",
                                                ),
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                ),
                                dict(
                                    function=dict(
                                        name="relevance",
                                        description="multi-output evaluator description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["relevant", "irrelevant"],
                                                    description="relevance",
                                                ),
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                ),
                            ],
                            toolChoice=dict(functionName="quality"),
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="quality",
                                description="Quality assessment",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="good", score=1),
                                    dict(label="bad", score=0),
                                ],
                            )
                        ),
                        dict(
                            categorical=dict(
                                name="relevance",
                                description="Relevance assessment",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="relevant", score=1),
                                    dict(label="irrelevant", score=0),
                                ],
                            )
                        ),
                    ],
                }
            },
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetLlmEvaluator"]["evaluator"]

        # Verify all output configs are returned
        assert len(dataset_evaluator["outputConfigs"]) == 2
        config_names = [c["name"] for c in dataset_evaluator["outputConfigs"]]
        assert "quality" in config_names
        assert "relevance" in config_names

        # Verify first config details
        quality_config = next(
            c for c in dataset_evaluator["outputConfigs"] if c["name"] == "quality"
        )
        assert quality_config["description"] == "Quality assessment"
        assert quality_config["optimizationDirection"] == "MAXIMIZE"
        assert len(quality_config["values"]) == 2

        # Verify second config details
        relevance_config = next(
            c for c in dataset_evaluator["outputConfigs"] if c["name"] == "relevance"
        )
        assert relevance_config["description"] == "Relevance assessment"
        assert len(relevance_config["values"]) == 2

        # Verify database storage
        dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
        async with db() as session:
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            llm_evaluator = await session.get(
                models.LLMEvaluator, db_dataset_evaluator.evaluator_id
            )
            assert llm_evaluator is not None
            assert len(llm_evaluator.output_configs) == 2
            db_config_names = [c.name for c in llm_evaluator.output_configs]
            assert "quality" in db_config_names
            assert "relevance" in db_config_names

    async def test_duplicate_config_names_rejected(
        self,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
    ) -> None:
        """Duplicate config names within evaluator are rejected."""
        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))

        result = await gql_client.execute(
            self._CREATE_LLM_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetId": dataset_id,
                    "name": "duplicate-config-evaluator",
                    "description": "duplicate config evaluator description",
                    "promptVersion": dict(
                        description="duplicate config prompt",
                        templateFormat="MUSTACHE",
                        template=dict(
                            messages=[
                                dict(
                                    role="USER",
                                    content=[dict(text=dict(text="Evaluate: {{input}}"))],
                                )
                            ]
                        ),
                        invocationParameters=dict(openai=dict(temperature=0.0)),
                        tools=dict(
                            tools=[
                                dict(
                                    function=dict(
                                        name="quality",
                                        description="duplicate config evaluator description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["good", "bad"],
                                                    description="quality",
                                                ),
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                ),
                                dict(
                                    function=dict(
                                        name="quality",
                                        description="duplicate config evaluator description",
                                        parameters=dict(
                                            type="object",
                                            properties=dict(
                                                label=dict(
                                                    type="string",
                                                    enum=["high", "low"],
                                                    description="quality",
                                                ),
                                            ),
                                            required=["label"],
                                        ),
                                    ),
                                ),
                            ],
                            toolChoice=dict(functionName="quality"),
                        ),
                        modelProvider="OPENAI",
                        modelName="gpt-4",
                    ),
                    "outputConfigs": [
                        dict(
                            categorical=dict(
                                name="quality",
                                description="First quality config",
                                optimizationDirection="MAXIMIZE",
                                values=[
                                    dict(label="good", score=1),
                                    dict(label="bad", score=0),
                                ],
                            )
                        ),
                        dict(
                            categorical=dict(
                                name="quality",  # Duplicate name
                                description="Second quality config",
                                optimizationDirection="MINIMIZE",
                                values=[
                                    dict(label="high", score=1),
                                    dict(label="low", score=0),
                                ],
                            )
                        ),
                    ],
                }
            },
        )
        # Should fail with duplicate name error
        assert result.errors is not None
        error_message = result.errors[0].message.lower()
        assert "duplicate" in error_message or "unique" in error_message

    async def test_custom_output_configs(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """Create a builtin evaluator with custom output configs and verify storage."""
        # Look up the exact_match evaluator (has categorical config)
        async with db() as session:
            exact_match_evaluator = await session.scalar(
                select(models.BuiltinEvaluator).where(models.BuiltinEvaluator.key == "exact_match")
            )
        assert exact_match_evaluator is not None, "ExactMatch builtin evaluator not found"

        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        evaluator_gid = str(GlobalID("BuiltInEvaluator", str(exact_match_evaluator.id)))

        # Get the base config name from the builtin evaluator
        assert exact_match_evaluator.output_configs is not None
        assert len(exact_match_evaluator.output_configs) > 0
        base_config = exact_match_evaluator.output_configs[0]
        base_config_name = base_config.name

        # Create with custom output configs
        result = await gql_client.execute(
            self._CREATE_BUILTIN_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetId": dataset_id,
                    "evaluatorId": evaluator_gid,
                    "name": "exact-match-with-custom-config",
                    "outputConfigs": [
                        {
                            "categorical": {
                                "name": base_config_name,
                                "optimizationDirection": "MINIMIZE",
                                "values": [
                                    {"label": "match", "score": 1.0},
                                    {"label": "no_match", "score": 0.0},
                                ],
                            }
                        }
                    ],
                }
            },
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetBuiltinEvaluator"]["evaluator"]

        # Verify the custom config was applied
        output_configs = dataset_evaluator["outputConfigs"]
        assert len(output_configs) >= 1
        matched_config = next((c for c in output_configs if c["name"] == base_config_name), None)
        assert matched_config is not None
        assert matched_config["optimizationDirection"] == "MINIMIZE"

        # Verify database storage
        dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
        async with db() as session:
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            assert db_dataset_evaluator.output_configs is not None
            assert len(db_dataset_evaluator.output_configs) == 1
            config = db_dataset_evaluator.output_configs[0]
            assert config.name == base_config_name
            assert not isinstance(config, FreeformAnnotationConfig)
            # The config can be stored as either a string or an enum
            config_direction = config.optimization_direction
            assert config_direction in (
                OptimizationDirection.MINIMIZE,
                OptimizationDirection.MINIMIZE.value,
                "MINIMIZE",
            )

    async def test_mixed_config_types(
        self,
        db: DbSessionFactory,
        gql_client: AsyncGraphQLClient,
        empty_dataset: models.Dataset,
        synced_builtin_evaluators: None,
    ) -> None:
        """Test that builtin evaluators with continuous configs work correctly.

        Note: LLM evaluators currently only support categorical configs due to validation
        constraints. This test uses a builtin evaluator with a continuous config to verify
        continuous config support in the multi-output system.
        """
        # Look up the levenshtein_distance evaluator (has continuous config)
        async with db() as session:
            levenshtein_evaluator = await session.scalar(
                select(models.BuiltinEvaluator).where(
                    models.BuiltinEvaluator.key == "levenshtein_distance"
                )
            )
        assert levenshtein_evaluator is not None, "LevenshteinDistance builtin evaluator not found"

        dataset_id = str(GlobalID("Dataset", str(empty_dataset.id)))
        evaluator_gid = str(GlobalID("BuiltInEvaluator", str(levenshtein_evaluator.id)))

        # Get the base config from the builtin evaluator
        assert levenshtein_evaluator.output_configs is not None
        assert len(levenshtein_evaluator.output_configs) > 0
        base_config = levenshtein_evaluator.output_configs[0]
        base_config_name = base_config.name

        # Create the builtin evaluator and verify continuous config is returned
        result = await gql_client.execute(
            self._CREATE_BUILTIN_MUTATION,
            {
                "input": {
                    "inputMapping": {"literalMapping": {}, "pathMapping": {}},
                    "datasetId": dataset_id,
                    "evaluatorId": evaluator_gid,
                    "name": "levenshtein-continuous-test",
                }
            },
        )
        assert result.data and not result.errors
        dataset_evaluator = result.data["createDatasetBuiltinEvaluator"]["evaluator"]

        # Verify the continuous config is returned with bounds
        output_configs = dataset_evaluator["outputConfigs"]
        assert len(output_configs) >= 1

        # Find the continuous config
        continuous_config = next(
            (c for c in output_configs if "lowerBound" in c or "upperBound" in c), None
        )
        assert continuous_config is not None, "Expected a continuous config with bounds"
        assert continuous_config["name"] == base_config_name

        # Verify database storage
        dataset_evaluator_id = int(GlobalID.from_id(dataset_evaluator["id"]).node_id)
        async with db() as session:
            db_dataset_evaluator = await session.get(models.DatasetEvaluators, dataset_evaluator_id)
            assert db_dataset_evaluator is not None
            # The base config should be a continuous type
            builtin_evaluator = await session.get(
                models.BuiltinEvaluator, db_dataset_evaluator.evaluator_id
            )
            assert builtin_evaluator is not None
            assert len(builtin_evaluator.output_configs) >= 1
            config_types = {c.name: c.type for c in builtin_evaluator.output_configs}
            assert config_types.get(base_config_name) == "CONTINUOUS"
