#!groovy

// trick from https://github.com/jenkinsci/workflow-cps-global-lib-plugin/pull/43
def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)

def target_backends = ['aws', 'gce', 'oci', 'docker', 'k8s-local-kind-aws', 'k8s-eks', 'azure', 'xcloud-aws', 'xcloud-gce', 'vs-docker', 'vs-aws']
def sct_runner_backends = ['aws', 'gce', 'oci', 'docker', 'k8s-local-kind-aws', 'k8s-eks', 'azure', 'xcloud-aws', 'xcloud-gce', 'vs-docker', 'vs-aws']

def createRunConfiguration(String backend) {
	def scylla_version = params.scylla_version ?: getLatestScyllaRelease('scylla')

    def configuration = [
        backend: backend,
        test_name: 'longevity_test.LongevityTest.test_custom_time',
        test_config: 'test-cases/PR-provision-test.yaml',
        scylla_version: scylla_version,
        availability_zone: '',
        region: 'eu-west-1',
    ]
    if (backend == 'aws' || backend == 'vs-aws') {
        configuration.availability_zone = 'a'
    }
    if (backend == 'gce') {
        configuration.gce_datacenter = "us-east1"
    } else if (backend == 'azure') {
        configuration.azure_region_name = 'eastus'
    } else if (backend == 'oci') {
        configuration.oci_region_name = 'us-ashburn-1'
        configuration.availability_zone = 'a,b,c'
        configuration.simulated_racks = 0
        configuration.oci_instance_type_db = 'VM.Standard3.Flex:4:32'
        configuration.data_volume_disk_num = 2
        configuration.data_volume_disk_type = 'ultra'
        configuration.data_volume_disk_size = 50
        configuration.provision_type = 'spot'
        configuration.scylla_version = 'master:latest'
    } else if (backend.contains('docker')) {
        // use latest version of nightly image for docker backend, until we get rid off rebuilding docker image for SCT
        configuration.scylla_version = 'latest'
        configuration.docker_image = 'scylladb/scylla-nightly'
        configuration.test_config = "test-cases/PR-provision-test-docker.yaml"
        configuration.availability_zone = 'a'
    } else if (backend in ['k8s-local-kind-aws', 'k8s-eks']) {
        configuration.test_config = "test-cases/scylla-operator/functional.yaml"
        configuration.test_name = "functional_tests/scylla_operator"
        configuration.functional_tests = true
        configuration.availability_zone = 'a,b'
    } else if (backend in  ['xcloud-aws', 'xcloud-gce']) {
        configuration.backend = 'xcloud'
        configuration.xcloud_env = 'staging'

        if (backend == 'xcloud-gce') {
            configuration.xcloud_provider = 'gce'
            configuration.gce_datacenter = "us-east1"
        }
        if (backend == 'xcloud-aws') {
            configuration.xcloud_provider = 'aws'
            configuration.availability_zone = 'a'
            configuration.test_config = '["test-cases/PR-provision-test.yaml"]'
        }
    }

    if (backend in ['vs-docker', 'vs-aws']) {
        configuration.test_config = '["' + configuration.test_config + '", "configurations/vector_store_provision_test.yaml"]'
        if (backend == 'vs-docker') {
            configuration.backend = 'docker'
        } else if (backend == 'vs-aws') {
            configuration.backend = 'aws'
        }
    }

    return configuration
}

def runSendEmail(){
    sh """#!/bin/bash

    set -xe
    env
    export LAST_COMMIT=`git rev-parse HEAD`
    export RECIPIENTS=`git show -s --format='%ae' \$LAST_COMMIT`,`git show -s --format='%ce' \$LAST_COMMIT`
    echo "Start send email ..."
    ./docker/env/hydra.sh send-email --logdir "`pwd`" --email-recipients "\${RECIPIENTS}"
    echo "Email sent"
    """
}

def runRestoreMonitoringStack(){
    sh """#!/bin/bash

    set -xe
    env

    echo "Restoring Monitor stack for test-id \$SCT_TEST_ID"
    ./docker/env/hydra.sh investigate show-monitor \$SCT_TEST_ID --kill true
    """
}

pipeline {
    agent {
        label {
            label "aws-sct-builders-eu-west-1-v3-CI"
        }
    }
    parameters {
        string(defaultValue: '',
               description: 'the scylla version to use for the provision tests, if empty automatically selects latest release',
               name: 'scylla_version')
        string(defaultValue: '',
               description: 'Actual user requesting job start, for automated job builds (e.g. through Argus)',
               name: 'requested_by_user')
    }
    environment {
        AWS_ACCESS_KEY_ID         = credentials('qa-aws-secret-key-id')
        AWS_SECRET_ACCESS_KEY = credentials('qa-aws-secret-access-key')
    }
    options {
        timestamps()
        disableConcurrentBuilds(abortPrevious: true)
        buildDiscarder(logRotator(numToKeepStr: '10'))
    }
    stages {
        stage("tag builder") {
        steps {
                script {
                    tagBuilder()
                }
            }
        }
        stage("precommit") {
            options {
                timeout(time: 15, unit: 'MINUTES')
            }
            steps {
                catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                    script {
                        dockerLogin(params)
                        // also check the commit-message for the rules we want
                        sh 'touch ./.git/COMMIT_EDITMSG'
                        sh '''#!/bin/bash
                        set -o pipefail
                        ./docker/env/hydra.sh pre-commit 2>&1 | tee precommit-output.log
                        '''
                    }
                }
            }
            post {
                success {
                    script {
                        pullRequestSetResult('success', 'jenkins/precommit', 'Precommit passed')
                    }
                }
                failure {
                    script {
                        pullRequestSetResult('failure', 'jenkins/precommit', 'Precommit failed')
                    }
                }
            }
        }
        stage("unittest") {
            options {
                timeout(time: 20, unit: 'MINUTES')
            }
            steps {
                catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                    withEnv(["AWS_ACCESS_KEY_ID=", "AWS_SECRET_ACCESS_KEY="]) {
                        script {
                            checkoutQaInternal(params)
                            sh './docker/env/hydra.sh unit-tests --junit-xml unit-tests-junit.xml'
                        }
                    }
                }
            }
            post {
                always {
                    junit testResults: 'unit-tests-junit.xml', allowEmptyResults: true, keepProperties: true
                }
                success {
                    script {
                        pullRequestSetResult('success', 'jenkins/unittests', 'All unit tests are passed')
                    }
                }
                failure {
                    script {
                        pullRequestSetResult('failure', 'jenkins/unittests', 'Some unit tests failed')
                    }
                }
            }
        }
        stage("collect tests") {
            options {
                timeout(time: 10, unit: 'MINUTES')
            }
            steps {
                catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                    withEnv(["AWS_ACCESS_KEY_ID=", "AWS_SECRET_ACCESS_KEY="]) {
                        script {
                            sh './docker/env/hydra.sh python -m pytest --collect-only -q -p no:cacheprovider *_test.py'
                        }
                    }
                }
            }
            post {
                success {
                    script {
                        pullRequestSetResult('success', 'jenkins/collect_tests', 'All SCT test modules are collectable')
                    }
                }
                failure {
                    script {
                        pullRequestSetResult('failure', 'jenkins/collect_tests', 'Some SCT test modules fail pytest collection')
                    }
                }
            }
        }
        stage("lint test-cases") {
            options {
                timeout(time: 10, unit: 'MINUTES')
            }
            steps {
                catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                    script {
                        checkoutQaInternal(params)
                        sh ''' ./docker/env/hydra.sh lint-pipelines --junit-xml lint-pipelines-junit.xml '''
                    }
                }
            }
            post {
                success {
                    script {
                        pullRequestSetResult('success', 'jenkins/lint_test_cases', 'All test cases are passed')
                    }
                }
                failure {
                    script {
                        pullRequestSetResult('failure', 'jenkins/lint_test_cases', 'Some test cases failed')
                    }
                }
            }
        }
        stage("integration tests") {
            when {
                expression {
                    return pullRequestContainsLabels("test-integration")
                }
            }
            options {
                timeout(time: 45, unit: 'MINUTES')
            }
            steps {
                catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                    script {
                        def curr_params = createRunConfiguration('docker')
                        def builder = getJenkinsLabels(curr_params.backend, curr_params.region, curr_params.gce_datacenter, curr_params.azure_region_name, curr_params.oci_region_name)
                        dockerLogin(params)
                        withEnv(["SCT_TEST_ID=${UUID.randomUUID().toString()}",]) {
                            dir('scylla-cluster-tests') {
                                checkout scm

                                dockerLogin(params)

                                wrap([$class: 'BuildUser']) {
                                    echo "calling createSctRunner"
                                    timeout(time: 5, unit: 'MINUTES') {
                                        createSctRunner(curr_params, 50 , builder.region)
                                    }
                                }
                                sh """#!/bin/bash
                                    set -xe
                                    echo "start integration-tests ..."
                                    RUNNER_IP=\$(cat sct_runner_ip||echo "")
                                    ./docker/env/hydra.sh --execute-on-runner \${RUNNER_IP} integration-tests --junit-xml integration-tests-junit.xml
                                    echo "end  integration-tests ..."
                                """
                                sh """#!/bin/bash
                                    set -x
                                    echo "Fetching integration test JUnit XML from runner ..."
                                    RUNNER_IP=\$(cat sct_runner_ip||echo "")
                                    if [ -n "\${RUNNER_IP}" ]; then
                                        ./docker/env/hydra.sh fetch-junit-from-runner \${RUNNER_IP} -b docker || true
                                        cp -f results/junit.xml integration-tests-junit.xml 2>/dev/null || true
                                    fi
                                """
                            }
                        }
                    }
                }
                dir('scylla-cluster-tests') {
                    sh """#!/bin/bash
                        set -x
                        echo "fetching junit report from runner ..."
                        RUNNER_IP=\$(cat sct_runner_ip||echo "")
                        if [ -n "\${RUNNER_IP}" ]; then
                            eval \$(ssh-agent)
                            ssh-add ~/.ssh/scylla-test 2>/dev/null || true
                            ssh-add ~/.ssh/scylla_test_id_ed25519 2>/dev/null || true
                            scp -o StrictHostKeyChecking=no ubuntu@\${RUNNER_IP}:/home/ubuntu/scylla-cluster-tests/integration-tests-junit.xml integration-tests-junit.xml || echo "WARNING: Failed to fetch JUnit XML report"
                            eval \$(ssh-agent -k)
                        fi
                    """
                }
            }
            post {
                always {
                    script {
                        dir('scylla-cluster-tests') {
                            junit testResults: 'integration-tests-junit.xml', allowEmptyResults: true, keepProperties: true
                        }
                    }
                }
                success {
                    script {
                        pullRequestSetResult('success', 'jenkins/integration-tests', 'All integration tests are passed')
                    }
                }
                failure {
                    script {
                        pullRequestSetResult('failure', 'jenkins/integration-tests', 'Some integration tests failed')
                    }
                }
            }
        }
        stage("provision test") {
            when {
                expression {
                    def labels = [
                        "test-provision",
                        "test-provision-docker",
                        "test-provision-aws", "test-provision-aws-reuse",
                        "test-provision-gce", "test-provision-gce-reuse",
                        "test-provision-azure", "test-provision-azure-reuse",
                        "test-provision-oci", "test-provision-oci-reuse",
                        "test-provision-k8s-local-kind-aws", "test-provision-k8s-local-kind-aws-reuse",
                        "test-provision-k8s-eks", "test-provision-k8s-eks-reuse",
                        "test-provision-xcloud-aws", "test-provision-xcloud-aws-reuse",
                        "test-provision-xcloud-gce", "test-provision-xcloud-gce-reuse",
                        "test-provision-vs-docker",
                        "test-provision-vs-aws", "test-provision-vs-aws-reuse",
                    ].join(",")
                    return pullRequestContainsLabels(labels)
                }
            }
            steps {
                script {
                    def sctParallelTests = [:]
                    target_backends.each {
                        def backend = it
                        if (pullRequestContainsLabels("test-provision,test-provision-${backend},test-provision-${backend}-reuse")) {
                            sctParallelTests["provision test on ${backend}"] = {
                                def curr_params = createRunConfiguration(backend)
                                def working_dir = "${backend}/scylla-cluster-tests"
                                def builder = getJenkinsLabels(curr_params.backend, curr_params.region, curr_params.gce_datacenter, curr_params.azure_region_name, curr_params.oci_region_name, curr_params)
                                withEnv(["SCT_TEST_ID=${UUID.randomUUID().toString()}",]) {
                                    script {
                                        def result = null
                                        dir(working_dir) {
                                            checkout scm
                                        }
                                        dockerLogin(params)
                                        if (sct_runner_backends.contains(backend)){
                                            try {
                                                wrap([$class: 'BuildUser']) {
                                                    dir(working_dir) {
                                                        timeout(time: 5, unit: 'MINUTES') {
                                                            echo "calling createSctRunner"
                                                            createSctRunner(curr_params, 90 , builder.region)
                                                        }
                                                    }
                                                }
                                            } catch(Exception err) {
                                                echo "${err}"
                                                result = 'FAILURE'
                                                pullRequestSetResult('failure', "jenkins/provision_${backend}", 'Some test cases are failed')
                                            }
                                        }
                                        try {
                                            wrap([$class: 'BuildUser']) {
                                                dir(working_dir) {
                                                    timeout(time: 30, unit: 'MINUTES') {
                                                        if (curr_params.backend == 'xcloud') {
                                                            echo "Scylla Cloud backend selected: provisioning loader nodes only on ${curr_params.xcloud_provider} cloud provider"
                                                        }
                                                        if (curr_params.backend == 'xcloud' || curr_params.backend == 'aws' || curr_params.backend == 'gce' || curr_params.backend == 'azure' || curr_params.backend == 'oci') {
                                                            provisionResources(curr_params, builder.region)
                                                        } else if (curr_params.backend.contains('docker')) {
                                                            sh """
                                                                echo 'Tests are to be executed on Docker backend in SCT-Runner. No additional resources to be provisioned.'
                                                            """
                                                        } else {
                                                            sh """
                                                                echo 'Skipping resource provisioning because this backend is not supported'
                                                            """
                                                        }
                                                    }
                                                }
                                            }
                                        } catch(Exception err) {
                                            echo "${err}"
                                            result = 'FAILURE'
                                            pullRequestSetResult('failure', "jenkins/provision_${backend}", 'Resource provisioning failed')
                                        }
                                        try {
                                            wrap([$class: 'BuildUser']) {
                                                dir(working_dir) {
                                                    timeout(time: 5, unit: 'MINUTES') {
                                                        createArgusTestRun(curr_params)
                                                    }
                                                }
                                            }
                                        } catch(Exception err) {
                                            echo "${err}"
                                        }
                                        try {
                                            wrap([$class: 'BuildUser']) {
                                                env.BUILD_USER_ID=env.CHANGE_AUTHOR
                                                timeout(time: 300, unit: 'MINUTES') {
                                                    dir(working_dir) {
                                                        echo "Run provision test"
                                                        runSctTest(curr_params, builder.region, curr_params.get('functional_tests', false))
                                                        result = 'SUCCESS'
                                                        pullRequestSetResult('success', "jenkins/provision_${backend}", 'All test cases are passed')
                                                    }
                                                }
                                            }
                                        } catch(Exception err) {
                                            echo "${err}"
                                            result = 'FAILURE'
                                            pullRequestSetResult('failure', "jenkins/provision_${backend}", 'Some test cases are failed')
                                        }
                                        try {
                                            wrap([$class: 'BuildUser']) {
                                                timeout(time: 90, unit: 'MINUTES') {
                                                    dir(working_dir) {
                                                        runCollectLogs(curr_params, builder.region)
                                                    }
                                                }
                                            }
                                        } catch(Exception err) {
                                            echo "${err}"
                                        }
                                        if (!(backend in ['k8s-local-kind-aws', 'k8s-eks'])) {
                                            try {
                                                wrap([$class: 'BuildUser']) {
                                                    timeout(time: 25, unit: 'MINUTES') {
                                                        dir(working_dir) {
                                                            runRestoreMonitoringStack()
                                                        }
                                                    }
                                                }
                                            } catch(Exception err) {
                                                echo "${err}"
                                                currentBuild.result = 'FAILURE'
                                            }
                                        }
                                        if (pullRequestContainsLabels("test-provision-${backend}-reuse")) {
                                            try {
                                                withEnv(["SCT_REUSE_CLUSTER=${env.SCT_TEST_ID}"]) {
                                                    wrap([$class: 'BuildUser']) {
                                                        env.BUILD_USER_ID=env.CHANGE_AUTHOR
                                                        timeout(time: 300, unit: 'MINUTES') {
                                                            dir(working_dir) {
                                                                echo "Reuse the cluster and re-run provision test"
                                                                runSctTest(curr_params, builder.region, curr_params.get('functional_tests', false))
                                                                result = 'SUCCESS'
                                                                pullRequestSetResult('success', "jenkins/provision_${backend}", 'All test cases are passed')
                                                            }
                                                        }
                                                    }
                                                }
                                            } catch(Exception err) {
                                                echo "${err}"
                                                result = 'FAILURE'
                                                pullRequestSetResult('failure', "jenkins/provision_${backend}", 'Some test cases are failed during cluster reuse test')
                                            }
                                        }
                                        try {
                                            wrap([$class: 'BuildUser']) {
                                                dir(working_dir) {
                                                    timeout(time: 5, unit: 'MINUTES') {
                                                        finishArgusTestRun(curr_params, currentBuild)
                                                    }
                                                }
                                            }
                                        } catch(Exception err) {
                                            echo "${err}"
                                        }
                                        try {
                                            wrap([$class: 'BuildUser']) {
                                                timeout(time: 30, unit: 'MINUTES') {
                                                    dir(working_dir) {
                                                        runCleanupResource(curr_params, builder.region)
                                                    }
                                                }
                                            }
                                        } catch(Exception err) {
                                            echo "${err}"
                                        }
                                        if (result == 'FAILURE'){
                                            currentBuild.result = 'FAILURE'
                                            sh "exit 1"
                                        }
                                    }
                                }
                            }
                        }
                    }
                    parallel sctParallelTests
                }
            }
        }
    }
    post {
        always {
            script {
                catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                    postTestSummaryComment(
                        junitXmlPaths: ['unit-tests-junit.xml', 'scylla-cluster-tests/integration-tests-junit.xml', 'lint-pipelines-junit.xml'],
                        precommitLog: 'precommit-output.log',
                        stageName: 'Test Summary',
                    )
                }
            }
        }
    }
}
