/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"context"
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/rest"
	kopsapi "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/cloudinstances"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
)

type MockCloud struct {
	awsup.MockAWSCloud
	Groups map[string]*cloudinstances.CloudInstanceGroup

	t                      *testing.T
	expectedCluster        *kopsapi.Cluster
	expectedInstanceGroups []kopsapi.InstanceGroup
}

var _ fi.Cloud = (*MockCloud)(nil)

func BuildMockCloud(t *testing.T, groups map[string]*cloudinstances.CloudInstanceGroup, expectedCluster *kopsapi.Cluster, expectedInstanceGroups []kopsapi.InstanceGroup) *MockCloud {
	m := MockCloud{
		MockAWSCloud:           *awsup.BuildMockAWSCloud("us-east-1", "abc"),
		Groups:                 groups,
		t:                      t,
		expectedCluster:        expectedCluster,
		expectedInstanceGroups: expectedInstanceGroups,
	}
	return &m
}

func (c *MockCloud) GetCloudGroups(cluster *kopsapi.Cluster, instancegroups []*kopsapi.InstanceGroup, warnUnmatched bool, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) {
	assert.Equal(c.t, c.expectedCluster, cluster, "cluster")

	igs := make([]kopsapi.InstanceGroup, 0, len(instancegroups))
	for _, ig := range instancegroups {
		igs = append(igs, *ig)
	}
	assert.ElementsMatch(c.t, c.expectedInstanceGroups, igs)

	// TODO assert nodes contains all the nodes in the mock kubernetes.Interface?

	return c.Groups, nil
}

func testValidate(t *testing.T, groups map[string]*cloudinstances.CloudInstanceGroup, objects []runtime.Object) (*ValidationCluster, error) {
	return testValidateWithTolerance(t, groups, objects, 0)
}

func testValidateWithTolerance(t *testing.T, groups map[string]*cloudinstances.CloudInstanceGroup, objects []runtime.Object, maxUnreadyNodes int) (*ValidationCluster, error) {
	ctx := context.TODO()

	cluster := &kopsapi.Cluster{
		ObjectMeta: metav1.ObjectMeta{Name: "testcluster.k8s.local"},
		Spec: kopsapi.ClusterSpec{
			Networking: kopsapi.NetworkingSpec{
				Topology: &kopsapi.TopologySpec{
					DNS: kopsapi.DNSTypeNone,
				},
			},
		},
	}

	if len(groups) == 0 {
		groups = make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["master-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{
					Name: "master-1",
				},
				Spec: kopsapi.InstanceGroupSpec{
					Role: kopsapi.InstanceGroupRoleControlPlane,
				},
			},
			MinSize: 1,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "master-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
		}
	}

	instanceGroups := make([]kopsapi.InstanceGroup, 0, len(groups))
	objects = append([]runtime.Object(nil), objects...)
	for _, g := range groups {
		instanceGroups = append(instanceGroups, *g.InstanceGroup)
		for _, member := range g.Ready {
			node := member.Node
			if node != nil {
				objects = append(objects, node)
			}
		}
		for _, member := range g.NeedUpdate {
			node := member.Node
			if node != nil {
				objects = append(objects, node)
			}
		}
	}

	mockcloud := BuildMockCloud(t, groups, cluster, instanceGroups)

	restConfig := &rest.Config{
		Host: "https://api.testcluster.k8s.local",
	}
	validator, err := NewClusterValidator(cluster, mockcloud, &kopsapi.InstanceGroupList{Items: instanceGroups}, nil, nil, maxUnreadyNodes, restConfig, fake.NewClientset(objects...))
	if err != nil {
		return nil, err
	}
	return validator.Validate(ctx)
}

func Test_ValidateCloudGroupMissing(t *testing.T) {
	ctx := context.TODO()

	cluster := &kopsapi.Cluster{
		ObjectMeta: metav1.ObjectMeta{Name: "testcluster.k8s.local"},
		Spec: kopsapi.ClusterSpec{
			Networking: kopsapi.NetworkingSpec{
				Topology: &kopsapi.TopologySpec{
					DNS: kopsapi.DNSTypeNone,
				},
			},
		},
	}

	instanceGroups := []kopsapi.InstanceGroup{
		{
			ObjectMeta: metav1.ObjectMeta{
				Name: "node-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleNode,
			},
		},
	}

	mockcloud := BuildMockCloud(t, nil, cluster, instanceGroups)

	restConfig := &rest.Config{
		Host: "https://api.testcluster.k8s.local",
	}
	validator, err := NewClusterValidator(cluster, mockcloud, &kopsapi.InstanceGroupList{Items: instanceGroups}, nil, nil, 0, restConfig, fake.NewClientset())
	require.NoError(t, err)
	v, err := validator.Validate(ctx)
	require.NoError(t, err)
	if !assert.Len(t, v.Failures, 1) ||
		!assert.Equal(t, &ValidationError{
			Kind:          "InstanceGroup",
			Name:          "node-1",
			Message:       "InstanceGroup \"node-1\" is missing from the cloud provider",
			InstanceGroup: &instanceGroups[0],
		}, v.Failures[0]) {
		printDebug(t, v)
	}
}

func Test_ValidateNodesNotEnough(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "node-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleNode,
			},
		},
		MinSize:    2,
		TargetSize: 3,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
		NeedUpdate: []*cloudinstances.CloudInstance{
			{
				ID: "i-00002",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
	}

	v, err := testValidate(t, groups, nil)
	require.NoError(t, err)
	if !assert.Len(t, v.Failures, 1) ||
		!assert.Equal(t, &ValidationError{
			Kind:          "InstanceGroup",
			Name:          "node-1",
			Message:       "InstanceGroup \"node-1\" did not have enough nodes 2 vs 3",
			InstanceGroup: groups["node-1"].InstanceGroup,
		}, v.Failures[0]) {
		printDebug(t, v)
	}
}

func Test_ValidateDetachedNodesDontCount(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "node-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleNode,
			},
		},
		MinSize:    2,
		TargetSize: 2,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
		NeedUpdate: []*cloudinstances.CloudInstance{
			{
				ID: "i-00002",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
				Status: cloudinstances.CloudInstanceStatusDetached,
			},
		},
	}

	v, err := testValidate(t, groups, nil)
	require.NoError(t, err)
	if !assert.Len(t, v.Failures, 1) ||
		!assert.Equal(t, &ValidationError{
			Kind:          "InstanceGroup",
			Name:          "node-1",
			Message:       "InstanceGroup \"node-1\" did not have enough nodes 1 vs 2",
			InstanceGroup: groups["node-1"].InstanceGroup,
		}, v.Failures[0]) {
		printDebug(t, v)
	}
}

func Test_ValidateNodeNotReady(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "node-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleNode,
			},
		},
		MinSize:    2,
		TargetSize: 2,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
		NeedUpdate: []*cloudinstances.CloudInstance{
			{
				ID: "i-00002",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionFalse},
						},
					},
				},
			},
		},
	}

	v, err := testValidate(t, groups, nil)
	require.NoError(t, err)
	if !assert.Len(t, v.Failures, 1) ||
		!assert.Equal(t, &ValidationError{
			Kind:          "Node",
			Name:          "node-1b",
			Message:       "node \"node-1b\" of role \"node\" is not ready",
			InstanceGroup: groups["node-1"].InstanceGroup,
		}, v.Failures[0]) {
		printDebug(t, v)
	}
}

func Test_ValidateMastersNotEnough(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "master-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleControlPlane,
			},
		},
		MinSize:    2,
		TargetSize: 3,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "master-1a"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
		NeedUpdate: []*cloudinstances.CloudInstance{
			{
				ID: "i-00002",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "master-1b"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
	}

	v, err := testValidate(t, groups, nil)
	require.NoError(t, err)
	if !assert.Len(t, v.Failures, 1) ||
		!assert.Equal(t, &ValidationError{
			Kind:          "InstanceGroup",
			Name:          "master-1",
			Message:       "InstanceGroup \"master-1\" did not have enough nodes 2 vs 3",
			InstanceGroup: groups["node-1"].InstanceGroup,
		}, v.Failures[0]) {
		printDebug(t, v)
	}
}

func Test_ValidateMasterNotReady(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "master-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleControlPlane,
			},
		},
		MinSize:    2,
		TargetSize: 2,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "master-1a"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
		NeedUpdate: []*cloudinstances.CloudInstance{
			{
				ID: "i-00002",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "master-1b"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionFalse},
						},
					},
				},
			},
		},
	}

	v, err := testValidate(t, groups, nil)
	require.NoError(t, err)
	if !assert.Len(t, v.Failures, 1) ||
		!assert.Equal(t, &ValidationError{
			Kind:          "Node",
			Name:          "master-1b",
			Message:       "node \"master-1b\" of role \"control-plane\" is not ready",
			InstanceGroup: groups["node-1"].InstanceGroup,
		}, v.Failures[0]) {
		printDebug(t, v)
	}
}

func Test_ValidateMasterStaticPods(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "master-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleControlPlane,
			},
		},
		MinSize:    1,
		TargetSize: 1,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{
						Name:   "master-1a",
						Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
					},
					Status: v1.NodeStatus{
						Addresses: []v1.NodeAddress{
							{
								Address: "1.2.3.4",
							},
						},
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
		NeedUpdate: []*cloudinstances.CloudInstance{
			{
				ID: "i-00002",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{
						Name:   "master-1b",
						Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
					},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
						Addresses: []v1.NodeAddress{
							{
								Address: "5.6.7.8",
							},
						},
					},
				},
			},
			{
				ID: "i-00003",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{
						Name:   "master-1c",
						Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
					},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionFalse},
						},
						Addresses: []v1.NodeAddress{
							{
								Address: "9.10.11.12",
							},
						},
					},
				},
			},
		},
	}

	var podList []map[string]string
	expectedFailures := []*ValidationError{
		{
			Kind:          "Node",
			Name:          "master-1c",
			Message:       "node \"master-1c\" of role \"control-plane\" is not ready",
			InstanceGroup: groups["node-1"].InstanceGroup,
		},
	}

	for i, pod := range []string{
		"kube-apiserver",
		"kube-controller-manager",
		"kube-scheduler",
	} {
		podList = append(podList, []map[string]string{
			{
				"name":              fmt.Sprintf("pod-a-%d", i),
				"ready":             "true",
				"k8s-app":           pod,
				"phase":             string(v1.PodRunning),
				"priorityClassName": "system-cluster-critical",
				"hostip":            "1.2.3.4",
				"nodeName":          "master-1a",
			},
			{
				"name":              fmt.Sprintf("pod-b-%d", i),
				"namespace":         "other",
				"ready":             "true",
				"k8s-app":           pod,
				"phase":             string(v1.PodRunning),
				"priorityClassName": "system-cluster-critical",
				"hostip":            "5.6.7.8",
				"nodeName":          "master-1b",
			},
		}...)
		expectedFailures = append(expectedFailures, &ValidationError{
			Kind:          "Node",
			Name:          "master-1b",
			Message:       "control-plane node \"master-1b\" is missing " + pod + " pod",
			InstanceGroup: groups["node-1"].InstanceGroup,
		})
	}

	v, err := testValidate(t, groups, makePodList(podList))
	require.NoError(t, err)
	if !assert.ElementsMatch(t, v.Failures, expectedFailures) {
		printDebug(t, v)
	}
}

func Test_ValidateNoPodFailures(t *testing.T) {
	testpods := []map[string]string{}

	for _, phase := range []v1.PodPhase{
		v1.PodPending,
		v1.PodRunning,
		v1.PodSucceeded,
		v1.PodFailed,
		v1.PodUnknown,
	} {
		for _, priority := range []string{"", "otherPriority"} {
			testpods = append(testpods, []map[string]string{
				{
					"name":              fmt.Sprintf("ready-%s-%s", priority, string(phase)),
					"namespace":         "kube-system",
					"priorityClassName": priority,
					"ready":             "true",
					"phase":             string(phase),
				},
				{
					"name":              fmt.Sprintf("notready-%s-%s", priority, string(phase)),
					"namespace":         "kube-system",
					"priorityClassName": priority,
					"ready":             "false",
					"phase":             string(phase),
				},
			}...)
		}
	}

	for _, namespace := range []string{"kube-system", "otherNamespace"} {
		for _, priority := range []string{"node", "cluster"} {
			testpods = append(testpods, []map[string]string{
				{
					"name":              fmt.Sprintf("ready-%s-%s", priority, namespace),
					"namespace":         namespace,
					"priorityClassName": fmt.Sprintf("system-%s-critical", priority),
					"ready":             "true",
					"phase":             string(v1.PodRunning),
				},
				{
					"name":              fmt.Sprintf("notready-%s-%s", priority, namespace),
					"namespace":         namespace,
					"priorityClassName": fmt.Sprintf("system-%s-critical", priority),
					"ready":             "false",
					"phase":             string(v1.PodSucceeded),
				},
			}...)
		}
	}

	v, err := testValidate(t, nil, makePodList(testpods))

	require.NoError(t, err)
	if !assert.Empty(t, v.Failures) {
		printDebug(t, v)
	}
}

func Test_ValidatePodFailure(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "node-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleNode,
			},
		},
		MinSize:    1,
		TargetSize: 1,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-1a",
					},
					Status: v1.NodeStatus{
						Addresses: []v1.NodeAddress{
							{
								Address: "1.2.3.4",
							},
						},
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-1b",
					},
					Status: v1.NodeStatus{
						Addresses: []v1.NodeAddress{
							{
								Address: "5.6.7.8",
							},
						},
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-1c",
					},
					Status: v1.NodeStatus{
						Addresses: []v1.NodeAddress{
							{
								Address: "9.10.11.12",
							},
						},
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
	}

	for _, tc := range []struct {
		name     string
		phase    v1.PodPhase
		expected string
	}{
		{
			name:     "pending",
			phase:    v1.PodPending,
			expected: "pending",
		},
		{
			name:     "notready",
			phase:    v1.PodRunning,
			expected: "not ready (container1,container2)",
		},
		{
			name:     "unknown",
			phase:    v1.PodUnknown,
			expected: "unknown phase",
		},
	} {
		for _, priority := range []string{"node", "cluster"} {
			for _, namespace := range []string{"kube-system", "otherNamespace"} {
				for hostIp, nodeName := range map[string]string{"1.2.3.4": "node-1a", "5.6.7.8": "node-1b", "9.10.11.12": "node-1c"} {
					t.Run(fmt.Sprintf("%s-%s-%s", tc.name, priority, namespace), func(t *testing.T) {
						v, err := testValidate(t, groups, makePodList(
							[]map[string]string{
								{
									"name":              "pod1",
									"namespace":         namespace,
									"priorityClassName": fmt.Sprintf("system-%s-critical", priority),
									"ready":             "false",
									"phase":             string(tc.phase),
									"hostip":            hostIp,
									"nodeName":          nodeName,
								},
							},
						))

						var podInstanceGroup *kopsapi.InstanceGroup
						if priority == "node" {
							podInstanceGroup = groups["node-1"].InstanceGroup
						}

						expected := ValidationError{
							Kind:          "Pod",
							Name:          fmt.Sprintf("%s/pod1", namespace),
							Message:       fmt.Sprintf("system-%s-critical pod \"pod1\" is %s", priority, tc.expected),
							InstanceGroup: podInstanceGroup,
						}

						require.NoError(t, err)
						if !assert.Len(t, v.Failures, 1) ||
							!assert.Equal(t, &expected, v.Failures[0]) {
							printDebug(t, v)
						}
					})
				}
			}
		}
	}
}

func printDebug(t *testing.T, v *ValidationCluster) {
	t.Logf("cluster - %d failures", len(v.Failures))
	for _, fail := range v.Failures {
		t.Logf("  failure: %+v", fail)
	}
}

func dummyPod(podMap map[string]string) v1.Pod {
	var labels map[string]string
	if podMap["k8s-app"] != "" {
		labels = map[string]string{"k8s-app": podMap["k8s-app"]}
	}
	namespace := podMap["namespace"]
	if namespace == "" {
		namespace = "kube-system"
	}
	return v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      podMap["name"],
			Namespace: namespace,
			Labels:    labels,
		},
		Spec: v1.PodSpec{
			PriorityClassName: podMap["priorityClassName"],
			NodeName:          podMap["nodeName"],
		},
		Status: v1.PodStatus{
			Phase: v1.PodPhase(podMap["phase"]),
			ContainerStatuses: []v1.ContainerStatus{
				{
					Name:  "container1",
					Ready: podMap["ready"] == "true",
				},
				{
					Name:  "container2",
					Ready: podMap["ready"] == "true",
				},
			},
			HostIP: podMap["hostip"],
		},
	}
}

// MakePodList constructs api.PodList from a list of pod attributes
func makePodList(pods []map[string]string) []runtime.Object {
	var list []runtime.Object
	for _, pod := range pods {
		p := dummyPod(pod)
		list = append(list, &p)
	}
	return list
}

func Test_ValidateBastionNodes(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["ig1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "ig1",
			},
		},
		Ready: []*cloudinstances.CloudInstance{
			{
				ID:   "i-00001",
				Node: nil,
			},
		},
	}

	// When an instancegroup's nodes are not ready, that is an error
	t.Run("instancegroup's nodes not ready", func(t *testing.T) {
		groups["ig1"].InstanceGroup.Spec.Role = kopsapi.InstanceGroupRoleNode
		v, err := testValidate(t, groups, nil)
		require.NoError(t, err)
		if !assert.Len(t, v.Failures, 1) {
			printDebug(t, v)
		} else if !assert.Equal(t, "machine \"i-00001\" has not yet joined cluster", v.Failures[0].Message) {
			printDebug(t, v)
		}
	})

	// Except for a bastion instancegroup - those are not expected to join as nodes
	t.Run("bastion instancegroup nodes not ready", func(t *testing.T) {
		groups["ig1"].InstanceGroup.Spec.Role = kopsapi.InstanceGroupRoleBastion
		v, err := testValidate(t, groups, nil)
		require.NoError(t, err)
		if !assert.Empty(t, v.Failures, "Bastion nodes are not expected to join cluster") {
			printDebug(t, v)
		}
	})
}

func Test_ValidateDetachedNodesNotValidated(t *testing.T) {
	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
		InstanceGroup: &kopsapi.InstanceGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name: "node-1",
			},
			Spec: kopsapi.InstanceGroupSpec{
				Role: kopsapi.InstanceGroupRoleNode,
			},
		},
		MinSize:    2,
		TargetSize: 2,
		Ready: []*cloudinstances.CloudInstance{
			{
				ID: "i-00001",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
			{
				ID: "i-00002",
				Node: &v1.Node{
					ObjectMeta: metav1.ObjectMeta{Name: "node-2a"},
					Status: v1.NodeStatus{
						Conditions: []v1.NodeCondition{
							{Type: "Ready", Status: v1.ConditionTrue},
						},
					},
				},
			},
		},
		NeedUpdate: []*cloudinstances.CloudInstance{
			{
				ID:     "i-00003",
				Status: cloudinstances.CloudInstanceStatusDetached,
			},
		},
	}

	v, err := testValidate(t, groups, nil)
	require.NoError(t, err)
	if !assert.Empty(t, v.Failures) {
		printDebug(t, v)
	}
}

func Test_ValidateAllowedNotReadyNodes(t *testing.T) {
	t.Run("tolerates single worker node failure when within limit", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["node-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleNode},
			},
			MinSize:    2,
			TargetSize: 2,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
			NeedUpdate: []*cloudinstances.CloudInstance{
				{
					ID: "i-00002",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionFalse},
							},
						},
					},
				},
			},
		}

		v, err := testValidateWithTolerance(t, groups, nil, 1)
		require.NoError(t, err)
		assert.Empty(t, v.Failures)
	})

	t.Run("tolerates worker node failures when within max limit", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["node-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleNode},
			},
			MinSize:    2,
			TargetSize: 2, // 2 worker nodes total
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
			NeedUpdate: []*cloudinstances.CloudInstance{
				{
					ID: "i-00002",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionFalse},
							},
						},
					},
				},
			},
		}

		// 1 allowed unready node.
		v, err := testValidateWithTolerance(t, groups, nil, 1)
		require.NoError(t, err)
		assert.Empty(t, v.Failures)
	})

	t.Run("tolerates single machine not joined when within limit", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["node-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleNode},
			},
			MinSize:    2,
			TargetSize: 2,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
			NeedUpdate: []*cloudinstances.CloudInstance{
				{
					ID:   "i-00002",
					Node: nil,
				},
			},
		}

		v, err := testValidateWithTolerance(t, groups, nil, 1)
		require.NoError(t, err)
		assert.Empty(t, v.Failures)
	})

	t.Run("tolerates multiple failures on the same worker node", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["node-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleNode},
			},
			MinSize:    2,
			TargetSize: 2,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{
							Name:   "node-1a",
							Labels: map[string]string{v1.LabelHostname: "node-1a"},
						},
						Status: v1.NodeStatus{
							Addresses: []v1.NodeAddress{
								{Address: "1.2.3.4"},
							},
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
			NeedUpdate: []*cloudinstances.CloudInstance{
				{
					ID: "i-00002",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{
							Name:   "node-1b",
							Labels: map[string]string{v1.LabelHostname: "node-1b"},
						},
						Status: v1.NodeStatus{
							Addresses: []v1.NodeAddress{
								{Address: "5.6.7.8"},
							},
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionFalse},
							},
						},
					},
				},
			},
		}

		pods := makePodList([]map[string]string{
			{
				"name":              "pod1",
				"namespace":         "kube-system",
				"priorityClassName": "system-node-critical",
				"ready":             "false",
				"phase":             string(v1.PodRunning),
				"hostip":            "5.6.7.8",
				"nodeName":          "node-1b",
			},
		})

		v, err := testValidateWithTolerance(t, groups, pods, 1)
		require.NoError(t, err)
		assert.Empty(t, v.Failures)
	})

	t.Run("does not tolerate when count of failing nodes exceeds limit", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["node-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleNode},
			},
			MinSize:    3,
			TargetSize: 3,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
			NeedUpdate: []*cloudinstances.CloudInstance{
				{
					ID: "i-00002",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionFalse},
							},
						},
					},
				},
				{
					ID: "i-00003",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1c"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionFalse},
							},
						},
					},
				},
			},
		}

		v, err := testValidateWithTolerance(t, groups, nil, 1)
		require.NoError(t, err)
		assert.Len(t, v.Failures, 2)
	})

	t.Run("does not tolerate worker node failures when exceeding max limit", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["node-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleNode},
			},
			MinSize:    2,
			TargetSize: 2,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
			NeedUpdate: []*cloudinstances.CloudInstance{
				{
					ID: "i-00002",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionFalse},
							},
						},
					},
				},
			},
		}

		// 0 allowed unready nodes.
		v, err := testValidateWithTolerance(t, groups, nil, 0)
		require.NoError(t, err)
		assert.Len(t, v.Failures, 1)
	})

	t.Run("does not tolerate control-plane node failure", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["master-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "master-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleControlPlane},
			},
			MinSize:    2,
			TargetSize: 2,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "master-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
			NeedUpdate: []*cloudinstances.CloudInstance{
				{
					ID: "i-00002",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "master-1b"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionFalse},
							},
						},
					},
				},
			},
		}

		v, err := testValidateWithTolerance(t, groups, nil, 1)
		require.NoError(t, err)
		assert.Len(t, v.Failures, 1)
	})

	t.Run("does not tolerate InstanceGroup sizing mismatches", func(t *testing.T) {
		groups := make(map[string]*cloudinstances.CloudInstanceGroup)
		groups["node-1"] = &cloudinstances.CloudInstanceGroup{
			InstanceGroup: &kopsapi.InstanceGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
				Spec:       kopsapi.InstanceGroupSpec{Role: kopsapi.InstanceGroupRoleNode},
			},
			MinSize:    2,
			TargetSize: 3,
			Ready: []*cloudinstances.CloudInstance{
				{
					ID: "i-00001",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1a"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
				{
					ID: "i-00002",
					Node: &v1.Node{
						ObjectMeta: metav1.ObjectMeta{Name: "node-1b"},
						Status: v1.NodeStatus{
							Conditions: []v1.NodeCondition{
								{Type: "Ready", Status: v1.ConditionTrue},
							},
						},
					},
				},
			},
		}

		v, err := testValidateWithTolerance(t, groups, nil, 1)
		require.NoError(t, err)
		assert.Len(t, v.Failures, 1)
		assert.Equal(t, "InstanceGroup", v.Failures[0].Kind)
	})
}
