import io

import pytest
from werkzeug.datastructures import FileStorage, MultiDict
from wtforms import ValidationError
from wtforms.validators import StopValidation

from app import (
    DATA_SET_EXTERNAL_ID_COLUMN_HEADER,
    DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER,
    ExpressionReference,
)
from app.common.collections.types import YesNoAnswer
from app.common.data.models import Expression
from app.common.data.types import (
    CollectionStatusEnum,
    DataSourceType,
    ExpressionType,
    ManagedExpressionsEnum,
    QuestionDataType,
    QuestionPresentationOptions,
    SubmissionEventType,
    TasklistSectionStatusEnum,
)
from app.common.expressions.managed import GreaterThan, IsYes, LessThan
from app.common.helpers.collections import SubmissionHelper
from app.deliver_grant_funding.forms import (
    RequestChangesSubmissionForm,
    SelectDataSourceQuestionForm,
    UploadDataSetForm,
)
from tests.models import ALL_COLUMN_TYPE_HEADERS_STR, FactoryAnswer


def _build_file_upload_form_data(csv_content: str) -> MultiDict:
    file = FileStorage(
        stream=io.BytesIO(csv_content.encode("utf-8")),
        filename="test.csv",
        content_type="text/csv",
    )
    data = MultiDict(
        [
            ("name", "Test Data Set"),
            ("data_source_type", DataSourceType.GRANT_RECIPIENT),
            ("file", file),
        ]
    )
    return data


def _select_data_source_question_form(
    *,
    current_component,
    subject_reference=None,
    expression_type=None,
    managed_expression_name=None,
) -> SelectDataSourceQuestionForm:
    return SelectDataSourceQuestionForm(
        form=current_component.form,
        interpolate=SubmissionHelper.get_interpolator(collection=current_component.form.collection),
        subject_reference=subject_reference,
        current_component=current_component,
        expression_type=expression_type,
        managed_expression_name=managed_expression_name,
    )


class TestSelectDataSourceQuestionForm:
    def test_only_includes_earlier_questions_in_the_form_if_given_a_current_question(self, factories):
        form = factories.form.create()
        questions = factories.question.create_batch(5, form=form, data_type=QuestionDataType.NUMBER)

        data_source_form = _select_data_source_question_form(current_component=questions[2])

        assert len(data_source_form.question.choices) == 3
        # '' is the default "no answer" choice
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            ExpressionReference.from_question(questions[0]),
            ExpressionReference.from_question(questions[1]),
        }

    def test_questions_in_a_same_page_group_excluded(self, factories):
        group = factories.group.create(
            presentation_options=QuestionPresentationOptions(show_questions_on_the_same_page=False)
        )
        questions = factories.question.create_batch(5, form=group.form, parent=group, data_type=QuestionDataType.NUMBER)

        data_source_form = _select_data_source_question_form(current_component=questions[2])

        assert len(data_source_form.question.choices) == 3
        # '' is the default "no answer" choice
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            ExpressionReference.from_question(questions[0]),
            ExpressionReference.from_question(questions[1]),
        }

        group.presentation_options.show_questions_on_the_same_page = True
        group.form.clear_caches()
        group.clear_caches()

        data_source_form = _select_data_source_question_form(current_component=questions[2])

        assert data_source_form.question.choices == []

    def test_show_all_question_datatypes_if_no_expression(self, factories):
        form = factories.form.create()
        text_question = factories.question.create(form=form, data_type=QuestionDataType.TEXT_SINGLE_LINE)
        yes_no_question = factories.question.create(form=form, data_type=QuestionDataType.YES_NO)
        date_question = factories.question.create(form=form, data_type=QuestionDataType.DATE)
        integer_question = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)

        data_source_form = _select_data_source_question_form(current_component=integer_question)

        assert len(data_source_form.question.choices) == 4
        # '' is the default "no answer" choice
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            str(ExpressionReference.from_question(text_question)),
            str(ExpressionReference.from_question(yes_no_question)),
            str(ExpressionReference.from_question(date_question)),
        }

    def test_managed_expression_excludes_mismatched_data_types(self, factories):
        form = factories.form.create()
        factories.question.create(form=form, data_type=QuestionDataType.TEXT_MULTI_LINE)
        factories.question.create(form=form, data_type=QuestionDataType.YES_NO)
        integer_questions = factories.question.create_batch(4, form=form, data_type=QuestionDataType.NUMBER)

        data_source_form = _select_data_source_question_form(
            current_component=integer_questions[2],
            expression_type=ExpressionType.CONDITION,
            managed_expression_name=ManagedExpressionsEnum.GREATER_THAN,
        )

        assert len(data_source_form.question.choices) == 3
        # '' is the default "no answer" choice
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            ExpressionReference.from_question(integer_questions[0]),
            ExpressionReference.from_question(integer_questions[1]),
        }

    def test_excludes_same_page_groups_and_unregistered_data_types(self, factories):
        form = factories.form.create()
        integer_q1 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        integer_q2 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        factories.question.create(form=form, data_type=QuestionDataType.TEXT_MULTI_LINE)

        group = factories.group.create(
            form=form, presentation_options=QuestionPresentationOptions(show_questions_on_the_same_page=True)
        )
        factories.question.create(form=form, parent=group, data_type=QuestionDataType.NUMBER)
        integer_q4 = factories.question.create(form=form, parent=group, data_type=QuestionDataType.NUMBER)

        data_source_form = _select_data_source_question_form(
            current_component=integer_q4,
            expression_type=ExpressionType.CONDITION,
            managed_expression_name=ManagedExpressionsEnum.GREATER_THAN,
        )

        assert len(data_source_form.question.choices) == 3
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            ExpressionReference.from_question(integer_q1),
            ExpressionReference.from_question(integer_q2),
        }

    def test_calculated_condition_excludes_unregistered_data_types(self, factories):
        form = factories.form.create()
        integer_q1 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        integer_q2 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        factories.question.create(form=form, data_type=QuestionDataType.YES_NO)
        factories.question.create(form=form, data_type=QuestionDataType.TEXT_MULTI_LINE)

        group = factories.group.create(
            form=form, presentation_options=QuestionPresentationOptions(show_questions_on_the_same_page=True)
        )
        factories.question.create(form=form, parent=group, data_type=QuestionDataType.NUMBER)
        text_q4 = factories.question.create(form=form, parent=group, data_type=QuestionDataType.TEXT_SINGLE_LINE)

        data_source_form = _select_data_source_question_form(
            current_component=text_q4,
            expression_type=ExpressionType.CONDITION,
            managed_expression_name=None,
        )

        assert len(data_source_form.question.choices) == 3
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            ExpressionReference.from_question(integer_q1),
            ExpressionReference.from_question(integer_q2),
        }

    def test_managed_condition_with_question_subject_reference(self, factories):
        form = factories.form.create()
        integer_q1 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        integer_q2 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        factories.question.create(form=form, data_type=QuestionDataType.YES_NO)
        factories.question.create(form=form, data_type=QuestionDataType.TEXT_MULTI_LINE)

        group = factories.group.create(
            form=form, presentation_options=QuestionPresentationOptions(show_questions_on_the_same_page=True)
        )
        integer_q3 = factories.question.create(form=form, parent=group, data_type=QuestionDataType.NUMBER)
        text_q4 = factories.question.create(form=form, parent=group, data_type=QuestionDataType.TEXT_SINGLE_LINE)

        data_source_form = _select_data_source_question_form(
            current_component=text_q4,
            subject_reference=ExpressionReference.from_question(integer_q3),
            expression_type=ExpressionType.CONDITION,
            managed_expression_name=None,
        )

        assert len(data_source_form.question.choices) == 3
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            ExpressionReference.from_question(integer_q1),
            ExpressionReference.from_question(integer_q2),
        }

    def test_managed_condition_with_data_set_subject_reference(self, factories):
        form = factories.form.create()
        integer_q1 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        integer_q2 = factories.question.create(form=form, data_type=QuestionDataType.NUMBER)
        factories.question.create(form=form, data_type=QuestionDataType.YES_NO)
        text_question = factories.question.create(form=form, data_type=QuestionDataType.TEXT_MULTI_LINE)

        group = factories.group.create(
            form=form, presentation_options=QuestionPresentationOptions(show_questions_on_the_same_page=True)
        )
        factories.question.create(form=form, parent=group, data_type=QuestionDataType.NUMBER)
        factories.question.create(form=form, parent=group, data_type=QuestionDataType.TEXT_SINGLE_LINE)

        data_set = factories.data_source.create(
            grant=form.collection.grant,
            collection=form.collection,
            type=DataSourceType.GRANT_RECIPIENT,
        )

        data_source_form = _select_data_source_question_form(
            current_component=text_question,
            subject_reference=ExpressionReference.from_data_source_column(data_set, "c_allocation"),
            expression_type=ExpressionType.CONDITION,
            managed_expression_name=None,
        )

        assert len(data_source_form.question.choices) == 3
        assert {q[0] for q in data_source_form.question.choices} == {
            "",
            ExpressionReference.from_question(integer_q1),
            ExpressionReference.from_question(integer_q2),
        }


class TestUploadDataSetForm:
    def test_new_data_valid(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR + "\na,b,£100,1.2,hello,5,$10,12km" + "\na,b,£100,1.2,hello,1,$10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is True

    def test_new_data_valid_with_missing_values(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR + "\na,b,£100,,hello,5,$10,12km" + "\na,b,£100,1.2,hello,,$10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is True

    @pytest.mark.parametrize("is_existing", (True, False))
    def test_remove_referenced_column_raises_error(self, factories, is_existing):
        collection = factories.collection.create()
        if is_existing:
            data_source = factories.data_source.create(
                grant=collection.grant,
                collection=collection,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
            )
            question = factories.question.create(
                form__collection=collection, text=f"How did you spend the ((d_{data_source.id.hex}.c_allocation))?"
            )

            assert len(data_source.depended_on_by_columns) == 1

        data = _build_file_upload_form_data(
            csv_content=(
                f"{DATA_SET_EXTERNAL_ID_COLUMN_HEADER},{DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER},"
                + "Another column\na,b,1000\nc,d,3000"
            )
        )
        form = UploadDataSetForm(
            existing_data_source_names=[],
            existing_datasource=data_source if is_existing else None,
            collection=collection,
        )
        form.process(data)

        if is_existing:
            assert form.validate() is False
            assert "Column 'Allocation' is missing from the selected file" in form.data_errors[0]
            assert f"is being used in '{question.name}'" in form.data_errors[0]
        else:
            assert form.validate() is True

    def test_remove_multiple_referenced_columns_raises_all_errors(self, factories):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            grant=collection.grant,
            collection=collection,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        question_1 = factories.question.create(
            form__collection=collection,
            text=f"How did you spend the ((d_{data_source.id.hex}.c_british_pounds))?",
        )
        question_2 = factories.question.create(
            form__collection=collection,
            text=f"Why did you pick ((d_{data_source.id.hex}.c_just_text))?",
        )

        assert len(data_source.depended_on_by_columns) == 2

        data = _build_file_upload_form_data(
            csv_content=(
                f"{DATA_SET_EXTERNAL_ID_COLUMN_HEADER},{DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER},"
                + "Another column\na,b,1000\nc,d,3000"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert len(form.data_errors) == 2
        assert (
            f"Column 'British pounds' is missing from the selected file but is being used in '{question_1.name}' "
            + "Add the column to the file or remove the form reference"
        ) in form.data_errors
        assert (
            f"Column 'Just text' is missing from the selected file but is being used in '{question_2.name}' "
            + "Add the column to the file or remove the form reference"
        ) in form.data_errors

    def test_error_wording_for_different_types_of_reference_when_column_removed(self, factories):
        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        question_1 = factories.question.create(
            form__collection=collection,
            text=f"How did you approach the theme of ((d_{data_source.id.hex}.c_just_text))?",
        )
        question_2 = factories.question.create(
            form__collection=collection,
            text="How much did you spend in total?",
            expressions=[
                Expression.from_evaluatable_expression(
                    LessThan(
                        subject_reference=ExpressionReference.from_data_source_column(data_source, "c_british_pounds"),
                        maximum_value=100,
                    ),
                    ExpressionType.CONDITION,
                    factories.user.create(),
                )
            ],
        )
        question_3 = factories.question.create(
            form__collection=collection,
            text="How many staff in total worked on the project?",
            expressions=[
                Expression.from_evaluatable_expression(
                    GreaterThan(
                        subject_reference=ExpressionReference.from_data_source_column(data_source, "c_decimal_number"),
                        minimum_value=100,
                    ),
                    ExpressionType.VALIDATION,
                    factories.user.create(),
                )
            ],
        )

        data = _build_file_upload_form_data(
            csv_content=(
                f"{DATA_SET_EXTERNAL_ID_COLUMN_HEADER},{DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER},"
                + "Another column\na,b,1000\nc,d,3000"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert len(form.data_errors) == 3
        assert (
            f"Column 'Just text' is missing from the selected file but is being used in '{question_1.name}' "
            + "Add the column to the file or remove the form reference"
        ) in form.data_errors
        assert (
            "Column 'British pounds' is missing from the selected file but is being used in a condition for "
            + f"'{question_2.name}' Add the column to the file or remove the form reference"
        ) in form.data_errors
        assert (
            "Column 'Decimal number' is missing from the selected file but is being referenced in validation for "
            + f"'{question_3.name}' Add the column to the file or remove the form reference"
        ) in form.data_errors

    def test_same_column_referenced_in_multiple_places_raises_all_errors(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
        )
        question_1 = factories.question.create(
            form__collection=collection, text=f"How did you spend the ((d_{data_source.id.hex}.c_allocation))?"
        )
        question_2 = factories.question.create(
            form__collection=collection,
            text="How did you spend the money?",
            expressions=[
                Expression.from_evaluatable_expression(
                    GreaterThan(
                        subject_reference=ExpressionReference.from_data_source_column(data_source, "c_allocation"),
                        minimum_value=100,
                    ),
                    ExpressionType.CONDITION,
                    factories.user.create(),
                )
            ],
        )
        question_3 = factories.question.create(
            form__collection=collection,
            text="How did you spend the money?",
            expressions=[
                Expression.from_evaluatable_expression(
                    GreaterThan(
                        subject_reference=ExpressionReference.from_data_source_column(data_source, "c_allocation"),
                        minimum_value=100,
                    ),
                    ExpressionType.VALIDATION,
                    factories.user.create(),
                )
            ],
        )

        data = _build_file_upload_form_data(
            csv_content=(
                f"{DATA_SET_EXTERNAL_ID_COLUMN_HEADER},{DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER},"
                + "Another column\na,b,1000\nc,d,3000"
            )
        )

        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert len(form.data_errors) == 3
        assert (
            f"Column 'Allocation' is missing from the selected file but is being used in '{question_1.name}' "
            + "Add the column to the file or remove the form reference"
        ) in form.data_errors
        assert (
            "Column 'Allocation' is missing from the selected file but is being used in a condition for "
            + f"'{question_2.name}' Add the column to the file or remove the form reference"
        ) in form.data_errors
        assert (
            "Column 'Allocation' is missing from the selected file but is being referenced in validation for "
            + f"'{question_3.name}' Add the column to the file or remove the form reference"
        ) in form.data_errors

    @pytest.mark.parametrize("bad_value", ["£twelve", "abc", "$100", "£1oo"])
    def test_new_data_in_existing_columns_does_not_match_formatting_british_pounds(self, factories, bad_value):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR
                + "\na,b,£100,1.2,hello,5,$10,12km"
                + f"\na,b,{bad_value},1.2,hello,5,$10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert (
            "One or more numbers in column 'British pounds' are not formatted as British pounds to 2 decimal places "
            + "with the '£' prefix. For example, £100.00"
        ) in form.data_errors

    def test_new_data_in_existing_columns_does_not_match_prefix(self, factories):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR + "\na,b,£100,1.2,hello,5,$10,12km" + "\na,b,£100,1.2,hello,5,€10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert ("One or more numbers in column 'Whole number prefix' do not match the prefix ('$')") in form.data_errors

    def test_new_data_in_existing_columns_does_not_match_suffix(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR + "\na,b,£100,1.2,hello,5,$10,12km" + "\na,b,£100,1.2,hello,5,$10,12miles"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert (
            "One or more numbers in column 'Whole number suffix' do not match the suffix ('km')"
        ) in form.data_errors

    def test_new_data_in_existing_columns_does_not_match_integer(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR + "\na,b,£100,1.2,hello,5,$10,12km" + "\na,b,£100,1.2,hello,1.5,$10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert ("One or more numbers in column 'Whole number' are not whole numbers") in form.data_errors

    def test_new_data_in_existing_columns_is_not_a_decimal_number(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR + "\na,b,£100,1.2,hello,5,$10,12km" + "\na,b,£100,hello,hello,5,$10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert ("One or more numbers in column 'Decimal number' are not decimal numbers") in form.data_errors

    def test_new_data_in_existing_columns_has_too_many_decimal_places(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR + "\na,b,£100,1.2,hello,5,$10,12km" + "\na,b,£100,1.2345,hello,5,$10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert ("One or more numbers in column 'Decimal number' has more than 3 decimal places") in form.data_errors

    def test_multiple_format_errors_in_same_column_appear_once(
        self,
        factories,
    ):

        collection = factories.collection.create()
        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            has_column_of_each_type=True,
        )
        data = _build_file_upload_form_data(
            csv_content=(
                ALL_COLUMN_TYPE_HEADERS_STR
                + "\na,b,£100,1.2,hello,5,$10,12km"
                + "\na,b,£twelve,1.2,hello,5,$10,12km"
                + "\na,b,£1oo,1.2,hello,5,$10,12km"
                + "\na,b,£100,1.2,hello,5,$10,12km"
            )
        )
        form = UploadDataSetForm(existing_data_source_names=[], existing_datasource=data_source, collection=collection)
        form.process(data)

        assert form.validate() is False
        assert len(form.data_errors) == 1
        assert (
            "One or more numbers in column 'British pounds' are not formatted as British pounds to 2 decimal places "
            + "with the '£' prefix. For example, £100.00"
        ) in form.data_errors

    def test_new_data_valid_but_changes_submitted_submission(
        self,
        factories,
    ):

        collection = factories.collection.create()
        gr1 = factories.grant_recipient.create(grant=collection.grant)
        gr2 = factories.grant_recipient.create(grant=collection.grant)

        data_source = factories.data_source.create(
            collection=collection,
            grant=collection.grant,
            type=DataSourceType.GRANT_RECIPIENT,
            create_gr_org_items=True,
            create_gr_org_items__data=[100, 200],
        )
        factories.question.create(
            form__collection=collection,
            data_type=QuestionDataType.TEXT_SINGLE_LINE,
            text=f"How did you spend the ((d_{data_source.id.hex}.c_allocation))?",
        )

        data = _build_file_upload_form_data(
            csv_content=(
                f"{DATA_SET_EXTERNAL_ID_COLUMN_HEADER},{DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER},Allocation"
                + f"\n{gr1.organisation.external_id},{gr1.organisation.name},120"
                + f"\n{gr2.organisation.external_id},{gr2.organisation.name},200"
            )
        )
        form = UploadDataSetForm(
            existing_data_source_names=[],
            existing_datasource=data_source,
            collection=collection,
            locked_orgs=[gr1.organisation],
        )
        form.process(data)

        assert form.validate() is False
        assert form.changed_column_errors["Allocation"][0] == gr1.organisation.name

    class TestValidateDataForExistingSubmissions:
        def test_valid_none_submitted(self, factories):
            collection = factories.collection.create()
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
            )
            form = UploadDataSetForm(
                existing_data_source_names=[], existing_datasource=data_source, collection=collection, locked_orgs=[]
            )
            form._validate_data_for_existing_submissions(
                existing_datasource=data_source,
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: "E000123",
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test",
                        "Allocation": "123",
                    }
                ],
            )

        def test_valid_submitted_unchanged(self, factories):
            collection = factories.collection.create()
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=["100"],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                locked_orgs=[gr1.organisation],
            )
            form._validate_data_for_existing_submissions(
                existing_datasource=data_source,
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr1.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test",
                        "Allocation": "100",
                    }
                ],
            )

        def test_valid_with_prefix(self, factories):
            collection = factories.collection.create()
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=["100"],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                locked_orgs=[gr1.organisation],
            )
            form._validate_data_for_existing_submissions(
                existing_datasource=data_source,
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr1.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test",
                        "Allocation": "£100",  # same value, but with the optional prefix
                    }
                ],
            )

        def test_valid_with_formatting_mismatch(self, factories):
            collection = factories.collection.create()
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=["100000000"],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                locked_orgs=[gr1.organisation],
            )
            form._validate_data_for_existing_submissions(
                existing_datasource=data_source,
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr1.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test",
                        "Allocation": "£1,0000000,0     ",  # same value, but with odd formatting
                    }
                ],
            )

        def test_invalid_missing_row_for_submitted_org(self, factories):
            collection = factories.collection.create()
            gr = factories.grant_recipient.create(grant=collection.grant, organisation__external_id="E000123")
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                locked_orgs=[gr.organisation],
            )
            with pytest.raises(ValidationError) as e:
                form._validate_data_for_existing_submissions(
                    existing_datasource=data_source,
                    rows=[
                        {
                            DATA_SET_EXTERNAL_ID_COLUMN_HEADER: "E000999",
                            DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test999",
                            "Allocation": "123",
                        }
                    ],
                )
            assert str(e.value) == f"The file does not contain a row for grant recipient {gr.organisation.name}"

        def test_invalid_changes_data_for_submitted_org(self, factories):
            collection = factories.collection.create()
            gr = factories.grant_recipient.create(grant=collection.grant, organisation__external_id="E000123")
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                locked_orgs=[gr.organisation],
            )
            with pytest.raises(StopValidation) as e:
                form._validate_data_for_existing_submissions(
                    existing_datasource=data_source,
                    rows=[
                        {
                            DATA_SET_EXTERNAL_ID_COLUMN_HEADER: "E000123",
                            DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test",
                            "Allocation": "123",
                        }
                    ],
                )
            assert str(e.value) == "There is a problem"
            assert form.changed_column_errors["Allocation"][0] == gr.organisation.name

        def test_invalid_missing_data_for_submitted_org(self, factories):
            collection = factories.collection.create()
            gr = factories.grant_recipient.create(grant=collection.grant, organisation__external_id="E000123")
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                locked_orgs=[gr.organisation],
            )
            with pytest.raises(StopValidation) as e:
                form._validate_data_for_existing_submissions(
                    existing_datasource=data_source,
                    rows=[
                        {
                            DATA_SET_EXTERNAL_ID_COLUMN_HEADER: "E000123",
                            DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test",
                            "Allocation": "",
                        }
                    ],
                )
            assert str(e.value) == "There is a problem"
            assert form.changed_column_errors["Allocation"][0] == gr.organisation.name

        def test_invalid_removes_column_for_submitted_org(self, factories):
            collection = factories.collection.create()
            gr = factories.grant_recipient.create(grant=collection.grant, organisation__external_id="E000123")
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                locked_orgs=[gr.organisation],
            )
            with pytest.raises(StopValidation) as e:
                form._validate_data_for_existing_submissions(
                    existing_datasource=data_source,
                    rows=[
                        {
                            DATA_SET_EXTERNAL_ID_COLUMN_HEADER: "E000123",
                            DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: "test",
                            "New column": "123",
                        }
                    ],
                )
            assert str(e.value) == "There is a problem"
            assert form.removed_column_errors["Allocation"][0] == gr.organisation.name

    class TestValidateNoDroppedOrganisationItems:
        def test_valid_when_no_organisation_items_dropped(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.OPEN)
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            gr2 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100, 200],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                all_organisations=[gr1.organisation, gr2.organisation],
            )

            form._validate_no_dropped_organisation_items(
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr1.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: gr1.organisation.name,
                        "Allocation": "999",
                    },
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr2.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: gr2.organisation.name,
                        "Allocation": "888",
                    },
                ],
            )

        def test_valid_when_no_existing_organisation_items(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.OPEN)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
            )
            form = UploadDataSetForm(
                existing_data_source_names=[], existing_datasource=data_source, collection=collection
            )

            form._validate_no_dropped_organisation_items(rows=[])

        def test_valid_when_missing_organisation_was_removed_from_grant(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.OPEN)
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            removed_org = factories.organisation.create()
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
            )
            factories.data_source_organisation_item.create(
                data_source=data_source, external_id=gr1.organisation.external_id, _data={"c_allocation": 100}
            )
            factories.data_source_organisation_item.create(
                data_source=data_source, external_id=removed_org.external_id, _data={"c_allocation": 200}
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                all_organisations=[gr1.organisation],
            )

            form._validate_no_dropped_organisation_items(
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr1.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: gr1.organisation.name,
                        "Allocation": "999",
                    },
                ],
            )

        def test_valid_when_missing_organisation_has_no_data_in_this_data_set(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.OPEN)
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
            )
            factories.data_source_organisation_item.create(
                data_source=data_source, external_id=gr1.organisation.external_id, _data={"c_allocation": 100}
            )
            gr2 = factories.grant_recipient.create(grant=collection.grant)
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                all_organisations=[gr1.organisation, gr2.organisation],
            )

            form._validate_no_dropped_organisation_items(
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr1.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: gr1.organisation.name,
                        "Allocation": "999",
                    },
                ],
            )

        def test_not_enforced_when_collection_is_draft(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.DRAFT)
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            gr2 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100, 200],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                all_organisations=[gr1.organisation, gr2.organisation],
            )

            form._validate_no_dropped_organisation_items(
                rows=[
                    {
                        DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr2.organisation.external_id,
                        DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: gr2.organisation.name,
                        "Allocation": "888",
                    },
                ],
            )

        def test_raises_for_single_missing_organisation(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.SCHEDULED)
            gr1 = factories.grant_recipient.create(grant=collection.grant, organisation__name="Rivendell")
            gr2 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100, 200],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                all_organisations=[gr1.organisation, gr2.organisation],
            )

            with pytest.raises(ValidationError) as e:
                form._validate_no_dropped_organisation_items(
                    rows=[
                        {
                            DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr2.organisation.external_id,
                            DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: gr2.organisation.name,
                            "Allocation": "888",
                        },
                    ],
                )
            assert str(e.value) == (
                f"Missing grant recipient ‘{gr1.organisation.name}’ in the selected file. Make sure the data set "
                f"includes the grant recipient so they can complete the open {collection.type.constants.singular}"
            )

        def test_raises_for_multiple_missing_organisations(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.OPEN)
            gr1 = factories.grant_recipient.create(grant=collection.grant, organisation__name="Rivendell")
            gr2 = factories.grant_recipient.create(grant=collection.grant, organisation__name="Lothlorien")
            gr3 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100, 200, 300],
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                all_organisations=[gr1.organisation, gr2.organisation, gr3.organisation],
            )

            with pytest.raises(ValidationError) as e:
                form._validate_no_dropped_organisation_items(
                    rows=[
                        {
                            DATA_SET_EXTERNAL_ID_COLUMN_HEADER: gr3.organisation.external_id,
                            DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER: gr3.organisation.name,
                            "Allocation": "999",
                        },
                    ],
                )
            assert str(e.value) == (
                f"Missing grant recipients ‘{gr2.organisation.name}’ and ‘{gr1.organisation.name}’ in the selected file"
                ". Make sure the data set includes the grant recipients so they can complete the open "
                f"{collection.type.constants.singular}"
            )

        def test_runs_as_part_of_full_form_validation(self, factories):
            collection = factories.collection.create(status=CollectionStatusEnum.OPEN)
            gr1 = factories.grant_recipient.create(grant=collection.grant)
            gr2 = factories.grant_recipient.create(grant=collection.grant)
            data_source = factories.data_source.create(
                collection=collection,
                grant=collection.grant,
                type=DataSourceType.GRANT_RECIPIENT,
                create_gr_org_items=True,
                create_gr_org_items__data=[100, 200],
            )
            data = _build_file_upload_form_data(
                csv_content=(
                    f"{DATA_SET_EXTERNAL_ID_COLUMN_HEADER},{DATA_SET_GRANT_RECIPIENT_COLUMN_HEADER},Allocation"
                    + f"\n{gr2.organisation.external_id},{gr2.organisation.name},200"
                )
            )
            form = UploadDataSetForm(
                existing_data_source_names=[],
                existing_datasource=data_source,
                collection=collection,
                all_organisations=[gr1.organisation, gr2.organisation],
            )
            form.process(data)

            assert form.validate() is False
            assert form.file.errors == [
                f"Missing grant recipient ‘{gr1.organisation.name}’ in the selected file. Make sure the data set "
                f"includes the grant recipient so they can complete the open {collection.type.constants.singular}"
            ]


class TestRequestChangesSubmissionForm:
    def test_section_choices_exclude_not_needed_sections(self, factories):
        # Creating a Section with a Yes/No question - this will condition Section 2
        yes_no_question = factories.question.create(data_type=QuestionDataType.YES_NO)
        needed_form = yes_no_question.form

        # Creating Section 2 - this will be Not Needed when Section 1 is answered No
        not_needed_form = factories.form.create(collection=needed_form.collection)
        factories.question.create(
            form=not_needed_form,
            expressions=[
                Expression.from_evaluatable_expression(
                    IsYes(subject_reference=ExpressionReference.from_question(yes_no_question)),
                    ExpressionType.CONDITION,
                    needed_form.collection.created_by,
                )
            ],
        )

        # Create and submit submission with Section 1 answered No, which will make Section 2 Not Needed
        submission = factories.submission.create(
            collection=needed_form.collection,
            answers=[FactoryAnswer(yes_no_question, YesNoAnswer(False))],
        )
        factories.submission_event.create(
            submission=submission,
            related_entity_id=needed_form.id,
            event_type=SubmissionEventType.FORM_RUNNER_FORM_COMPLETED,
        )

        # Check Section status are as expected
        helper = SubmissionHelper(submission)
        assert helper.get_tasklist_status_for_form(needed_form) == TasklistSectionStatusEnum.COMPLETED
        assert helper.get_tasklist_status_for_form(not_needed_form) == TasklistSectionStatusEnum.NOT_NEEDED

        form = RequestChangesSubmissionForm(submission_helper=helper)
        choice_ids = [choice[0] for choice in form.section_ids.choices]

        # Not needed form is not included in the choices, but the needed form is
        assert str(needed_form.id) in choice_ids
        assert str(not_needed_form.id) not in choice_ids
