// Licensed to Apache Software Foundation (ASF) under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Apache Software Foundation (ASF) licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// Package data is used to test the trace service.
package data

import (
	"bytes"
	"context"
	"embed"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"slices"
	"strings"
	"time"

	"github.com/google/go-cmp/cmp"
	g "github.com/onsi/ginkgo/v2"
	gm "github.com/onsi/gomega"
	"go.uber.org/mock/gomock"
	grpclib "google.golang.org/grpc"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/testing/protocmp"
	"google.golang.org/protobuf/types/known/timestamppb"
	"sigs.k8s.io/yaml"

	bydbqlv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/bydbql/v1"
	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
	modelv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1"
	tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
	metadata "github.com/apache/skywalking-banyandb/banyand/metadata"
	"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
	"github.com/apache/skywalking-banyandb/pkg/bydbql"
	"github.com/apache/skywalking-banyandb/pkg/test/flags"
	"github.com/apache/skywalking-banyandb/pkg/test/helpers"
)

//go:embed input/*.yml
var inputFS embed.FS

//go:embed want/*.yml
var wantFS embed.FS

//go:embed input/*.ql
var qlFS embed.FS

//go:embed testdata/*.json
var dataFS embed.FS

// VerifyFn verify whether the query response matches the wanted result.
var VerifyFn = func(innerGm gm.Gomega, sharedContext helpers.SharedContext, args helpers.Args) {
	i, err := inputFS.ReadFile("input/" + args.Input + ".yml")
	innerGm.Expect(err).NotTo(gm.HaveOccurred())
	query := &tracev1.QueryRequest{}
	helpers.UnmarshalYAML(i, query)
	if !args.WantErr {
		verifyQLWithRequest(innerGm, args, query, sharedContext.Connection)
	}
	query.TimeRange = helpers.TimeRange(args, sharedContext)
	query.Stages = args.Stages
	c := tracev1.NewTraceServiceClient(sharedContext.Connection)
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	resp, err := c.Query(ctx, query)
	if args.WantErr {
		if err == nil {
			g.Fail("expect error")
		}
		return
	}
	innerGm.Expect(err).NotTo(gm.HaveOccurred(), query.String())
	if args.WantEmpty {
		innerGm.Expect(resp.Traces).To(gm.BeEmpty(), func() string {
			var j []byte
			j, err = MarshalToJSONWithStringBytes(resp)
			if err != nil {
				return err.Error()
			}
			var y []byte
			y, err = yaml.JSONToYAML(j)
			if err != nil {
				return err.Error()
			}
			return string(y)
		})
		return
	}
	if args.Want == "" {
		args.Want = args.Input
	}
	ww, err := wantFS.ReadFile("want/" + args.Want + ".yml")
	innerGm.Expect(err).NotTo(gm.HaveOccurred())
	want := &tracev1.QueryResponse{}
	unmarshalYAMLWithSpanEncoding(ww, want)
	for i := range want.Traces {
		slices.SortFunc(want.Traces[i].Spans, func(a, b *tracev1.Span) int {
			return bytes.Compare(a.Span, b.Span)
		})
	}
	for i := range resp.Traces {
		slices.SortFunc(resp.Traces[i].Spans, func(a, b *tracev1.Span) int {
			return bytes.Compare(a.Span, b.Span)
		})
	}

	if args.DisOrder {
		// Sort traces by first span's tag for consistency
		slices.SortFunc(want.Traces, func(a, b *tracev1.Trace) int {
			if len(a.Spans) > 0 && len(b.Spans) > 0 && len(a.Spans[0].Tags) > 0 && len(b.Spans[0].Tags) > 0 {
				return strings.Compare(a.Spans[0].Tags[0].Value.GetStr().GetValue(), b.Spans[0].Tags[0].Value.GetStr().GetValue())
			}
			return 0
		})
		slices.SortFunc(resp.Traces, func(a, b *tracev1.Trace) int {
			if len(a.Spans) > 0 && len(b.Spans) > 0 && len(a.Spans[0].Tags) > 0 && len(b.Spans[0].Tags) > 0 {
				return strings.Compare(a.Spans[0].Tags[0].Value.GetStr().GetValue(), b.Spans[0].Tags[0].Value.GetStr().GetValue())
			}
			return 0
		})
		// Sort spans within each trace for consistent ordering
		for _, trace := range want.Traces {
			slices.SortFunc(trace.Spans, func(a, b *tracev1.Span) int {
				if len(a.Tags) > 0 && len(b.Tags) > 0 {
					return strings.Compare(a.Tags[0].Value.GetStr().GetValue(), b.Tags[0].Value.GetStr().GetValue())
				}
				return 0
			})
		}
		for _, trace := range resp.Traces {
			slices.SortFunc(trace.Spans, func(a, b *tracev1.Span) int {
				if len(a.Tags) > 0 && len(b.Tags) > 0 {
					return strings.Compare(a.Tags[0].Value.GetStr().GetValue(), b.Tags[0].Value.GetStr().GetValue())
				}
				return 0
			})
		}
	}
	var extra []cmp.Option
	extra = append(extra, protocmp.IgnoreUnknown(),
		protocmp.Transform())
	success := innerGm.Expect(cmp.Equal(resp, want,
		extra...)).
		To(gm.BeTrue(), func() string {
			var j []byte
			j, err = MarshalToJSONWithStringBytes(resp)
			if err != nil {
				return err.Error()
			}
			var y []byte
			y, err = yaml.JSONToYAML(j)
			if err != nil {
				return err.Error()
			}
			return string(y)
		})
	if !success {
		return
	}
	query.Trace = true
	resp, err = c.Query(ctx, query)
	innerGm.Expect(err).NotTo(gm.HaveOccurred())
	innerGm.Expect(resp.TraceQueryResult).NotTo(gm.BeNil())
	innerGm.Expect(resp.TraceQueryResult.GetSpans()).NotTo(gm.BeEmpty())
}

func verifyQLWithRequest(innerGm gm.Gomega, args helpers.Args, yamlQuery *tracev1.QueryRequest, conn *grpclib.ClientConn) {
	qlContent, err := qlFS.ReadFile("input/" + args.Input + ".ql")
	innerGm.Expect(err).NotTo(gm.HaveOccurred())
	qlQueryStr, qlParams, err := helpers.ExtractQL(string(qlContent))
	innerGm.Expect(err).NotTo(gm.HaveOccurred())

	// Auto-inject stages clause if args.Stages is not empty
	if len(args.Stages) > 0 {
		stageClause := " ON " + strings.Join(args.Stages, ", ") + " STAGES"
		// Use regex to find and replace IN clause with groups
		// Pattern: IN followed by groups (with or without parentheses)
		re := regexp.MustCompile(`(?i)\s+IN\s+(\([^)]+\)|[a-zA-Z0-9_-]+(?:\s*,\s*[a-zA-Z0-9_-]+)*)`)
		qlQueryStr = re.ReplaceAllString(qlQueryStr, " IN $1"+stageClause)
	}

	ctrl := gomock.NewController(g.GinkgoT())
	defer ctrl.Finish()

	mockRepo := metadata.NewMockRepo(ctrl)
	trace := schema.NewMockTrace(ctrl)
	trace.EXPECT().GetTrace(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, metadata *commonv1.Metadata) (*databasev1.Trace, error) {
		client := databasev1.NewTraceRegistryServiceClient(conn)
		resp, getErr := client.Get(ctx, &databasev1.TraceRegistryServiceGetRequest{Metadata: metadata})
		if getErr != nil {
			return nil, getErr
		}
		return resp.GetTrace(), nil
	}).AnyTimes()
	mockRepo.EXPECT().TraceRegistry().AnyTimes().Return(trace)

	parsed, errStrs := bydbql.ParseQuery(qlQueryStr)
	innerGm.Expect(errStrs).To(gm.BeNil())
	innerGm.Expect(bydbql.BindParams(parsed, qlParams)).To(gm.Succeed())

	transformer := bydbql.NewTransformer(mockRepo)
	transformCtx, transformCancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer transformCancel()
	result, err := transformer.Transform(transformCtx, parsed)
	innerGm.Expect(err).NotTo(gm.HaveOccurred())

	qlQuery, ok := result.QueryRequest.(*tracev1.QueryRequest)
	innerGm.Expect(ok).To(gm.BeTrue())

	equal := cmp.Equal(qlQuery, yamlQuery,
		protocmp.IgnoreUnknown(),
		protocmp.IgnoreFields(&tracev1.QueryRequest{}, "time_range", "stages"),
		protocmp.Transform())
	if !equal {
		qlQuery.TimeRange = nil
	}
	innerGm.Expect(equal).To(gm.BeTrue(), "QL:\n%s\nYAML:\n%s", qlQuery.String(), yamlQuery.String())

	bydbqlClient := bydbqlv1.NewBydbQLServiceClient(conn)
	qlCtx, qlCancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer qlCancel()
	bydbqlResp, err := bydbqlClient.Query(qlCtx, &bydbqlv1.QueryRequest{
		Query:  qlQueryStr,
		Params: qlParams,
	})
	innerGm.Expect(err).NotTo(gm.HaveOccurred())
	innerGm.Expect(bydbqlResp).NotTo(gm.BeNil())
	_, ok = bydbqlResp.Result.(*bydbqlv1.QueryResponse_TraceResult)
	innerGm.Expect(ok).To(gm.BeTrue())
}

func loadData(stream tracev1.TraceService_WriteClient, metadata *commonv1.Metadata, dataFile string, baseTime time.Time, interval time.Duration, version *uint64) {
	var templates []interface{}
	content, err := dataFS.ReadFile("testdata/" + dataFile)
	gm.Expect(err).ShouldNot(gm.HaveOccurred())
	gm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
	gm.Expect(version).NotTo(gm.BeNil(), "version must be provided")

	for _, template := range templates {
		templateMap, ok := template.(map[string]interface{})
		gm.Expect(ok).To(gm.BeTrue())
		spanData, ok := templateMap["span"].(string)
		gm.Expect(ok).To(gm.BeTrue())
		tagsData, ok := templateMap["tags"].([]interface{})
		gm.Expect(ok).To(gm.BeTrue())

		var tagValues []*modelv1.TagValue
		for _, tag := range tagsData {
			tagBytes, marshalErr := json.Marshal(tag)
			gm.Expect(marshalErr).ShouldNot(gm.HaveOccurred())
			tagValue := &modelv1.TagValue{}
			gm.Expect(protojson.Unmarshal(tagBytes, tagValue)).ShouldNot(gm.HaveOccurred())
			tagValues = append(tagValues, tagValue)
		}
		timestamp := baseTime.Add(interval * time.Duration(*version))
		timestampTag := &modelv1.TagValue{
			Value: &modelv1.TagValue_Timestamp{
				Timestamp: timestamppb.New(timestamp),
			},
		}
		tagValues = append(tagValues, timestampTag)

		errInner := stream.Send(&tracev1.WriteRequest{
			Metadata: metadata,
			Tags:     tagValues,
			Span:     []byte(spanData),
			Version:  *version,
		})
		*version++
		gm.Expect(errInner).ShouldNot(gm.HaveOccurred())
	}
}

func loadDataWithSpec(stream tracev1.TraceService_WriteClient, md *commonv1.Metadata, dataFile string,
	baseTime time.Time, interval time.Duration, spec *tracev1.TagSpec, version *uint64,
) {
	var templates []interface{}
	content, err := dataFS.ReadFile("testdata/" + dataFile)
	gm.Expect(err).ShouldNot(gm.HaveOccurred())
	gm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())
	gm.Expect(version).NotTo(gm.BeNil(), "version must be provided")

	isFirst := true
	for i, template := range templates {
		templateMap, ok := template.(map[string]interface{})
		gm.Expect(ok).To(gm.BeTrue())
		spanData, ok := templateMap["span"].(string)
		gm.Expect(ok).To(gm.BeTrue())
		tagsData, ok := templateMap["tags"].([]interface{})
		gm.Expect(ok).To(gm.BeTrue())

		var tagValues []*modelv1.TagValue
		for _, tag := range tagsData {
			tagBytes, marshalErr := json.Marshal(tag)
			gm.Expect(marshalErr).ShouldNot(gm.HaveOccurred())
			tagValue := &modelv1.TagValue{}
			gm.Expect(protojson.Unmarshal(tagBytes, tagValue)).ShouldNot(gm.HaveOccurred())
			tagValues = append(tagValues, tagValue)
		}
		tagValues = append(tagValues, &modelv1.TagValue{
			Value: &modelv1.TagValue_Timestamp{Timestamp: timestamppb.New(baseTime.Add(time.Duration(i) * interval))},
		})

		req := &tracev1.WriteRequest{Tags: tagValues, Span: []byte(spanData), Version: *version}
		*version++
		if isFirst {
			req.Metadata = md
			req.TagSpec = spec
			isFirst = false
		}
		gm.Expect(stream.Send(req)).Should(gm.Succeed())
	}
}

// Write writes trace data to the database.
func Write(conn *grpclib.ClientConn, name string, baseTime time.Time, interval time.Duration) {
	WriteToGroup(conn, name, "test-trace-group", name, baseTime, interval)
}

// SeedAll writes all trace fixtures used by the integration suite.
func SeedAll(conn *grpclib.ClientConn, baseTime time.Time, interval time.Duration) {
	WriteToGroup(conn, "sw", "test-trace-group", "sw", baseTime, interval)
	// Seed trace data in a fully expired segment, well past the
	// "test-trace-group" group's 3-day TTL. It must never surface in query
	// results: it backs the "excludes data expired beyond TTL" case, which fails
	// without the retention filter that drops fully expired segments.
	WriteToGroup(conn, "sw", "test-trace-group", "sw", baseTime.AddDate(0, 0, -6), interval)
	WriteToGroup(conn, "zipkin", "zipkinTrace", "zipkin", baseTime, interval)
	WriteToGroup(conn, "sw", "test-trace-updated", "sw_updated", baseTime.Add(time.Minute), interval)
	time.Sleep(2 * time.Second)
	WriteToGroup(conn, "sw", "test-trace-group", "sw_mixed_traces", baseTime.Add(time.Minute), interval)
	WriteMixed(conn, baseTime.Add(2*time.Minute), interval,
		WriteSpec{
			Metadata: &commonv1.Metadata{Name: "sw", Group: "test-trace-spec"},
			DataFile: "sw_schema_order.json",
		},
		WriteSpec{
			Spec: &tracev1.TagSpec{
				TagNames: []string{"trace_id", "state", "service_id", "service_instance_id", "endpoint_id", "duration", "span_id", "timestamp"},
			},
			DataFile: "sw_spec_order.json",
		},
		WriteSpec{
			Metadata: &commonv1.Metadata{Name: "sw", Group: "test-trace-spec2"},
			Spec: &tracev1.TagSpec{
				TagNames: []string{"span_id", "duration", "endpoint_id", "service_instance_id", "service_id", "state", "trace_id", "timestamp"},
			},
			DataFile: "sw_spec_order2.json",
		})
}

// WriteToGroup writes trace data to a specific group.
func WriteToGroup(conn *grpclib.ClientConn, name, group, fileName string, baseTime time.Time, interval time.Duration) {
	metadata := &commonv1.Metadata{
		Name:  name,
		Group: group,
	}
	schema := databasev1.NewTraceRegistryServiceClient(conn)
	resp, err := schema.Get(context.Background(), &databasev1.TraceRegistryServiceGetRequest{Metadata: metadata})
	if err != nil {
		return
	}
	metadata = resp.GetTrace().GetMetadata()

	c := tracev1.NewTraceServiceClient(conn)
	ctx := context.Background()
	writeClient, err := c.Write(ctx)
	gm.Expect(err).NotTo(gm.HaveOccurred())
	version := uint64(1)
	loadData(writeClient, metadata, fmt.Sprintf("%s.json", fileName), baseTime, interval, &version)
	gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
	gm.Eventually(func() error {
		_, err := writeClient.Recv()
		return err
	}, flags.EventuallyTimeout).Should(gm.Equal(io.EOF))
}

// WriteSpec defines the specification for writing trace data.
type WriteSpec struct {
	Metadata *commonv1.Metadata
	Spec     *tracev1.TagSpec
	DataFile string
}

// WriteMixed writes trace data in schema order first, and then in spec order.
func WriteMixed(conn *grpclib.ClientConn, baseTime time.Time, interval time.Duration, writeSpecs ...WriteSpec) {
	ctx := context.Background()
	schemaClient := databasev1.NewTraceRegistryServiceClient(conn)
	c := tracev1.NewTraceServiceClient(conn)
	writeClient, err := c.Write(ctx)
	gm.Expect(err).NotTo(gm.HaveOccurred())

	var currentMd *commonv1.Metadata
	version := uint64(1)
	currentTime := baseTime
	for idx, ws := range writeSpecs {
		if ws.Metadata != nil {
			resp, getErr := schemaClient.Get(ctx, &databasev1.TraceRegistryServiceGetRequest{Metadata: ws.Metadata})
			gm.Expect(getErr).NotTo(gm.HaveOccurred())
			currentMd = resp.GetTrace().GetMetadata()
		}
		gm.Expect(currentMd).NotTo(gm.BeNil(), "first WriteSpec must have Metadata")

		var templates []interface{}
		content, readErr := dataFS.ReadFile("testdata/" + ws.DataFile)
		gm.Expect(readErr).ShouldNot(gm.HaveOccurred())
		gm.Expect(json.Unmarshal(content, &templates)).ShouldNot(gm.HaveOccurred())

		if idx == 0 {
			loadData(writeClient, currentMd, ws.DataFile, currentTime, interval, &version)
		} else {
			var mdToSend *commonv1.Metadata
			if ws.Metadata != nil {
				mdToSend = currentMd
			}
			loadDataWithSpec(writeClient, mdToSend, ws.DataFile, currentTime, interval, ws.Spec, &version)
		}
		currentTime = currentTime.Add(time.Duration(len(templates)) * interval)
	}

	gm.Expect(writeClient.CloseSend()).To(gm.Succeed())
	gm.Eventually(func() error {
		_, recvErr := writeClient.Recv()
		return recvErr
	}, flags.EventuallyTimeout).Should(gm.Equal(io.EOF))
}

// unmarshalYAMLWithSpanEncoding decodes YAML with special handling for span data.
// It converts plain strings in the YAML to base64 encoded strings before protobuf unmarshaling.
func unmarshalYAMLWithSpanEncoding(yamlData []byte, response *tracev1.QueryResponse) {
	// First convert YAML to JSON
	j, err := yaml.YAMLToJSON(yamlData)
	gm.Expect(err).NotTo(gm.HaveOccurred())

	// Parse JSON to modify span fields
	var jsonData map[string]interface{}
	err = json.Unmarshal(j, &jsonData)
	gm.Expect(err).NotTo(gm.HaveOccurred())

	// Convert span strings to base64 in traces structure
	if traces, ok := jsonData["traces"].([]interface{}); ok {
		for _, traceInterface := range traces {
			if trace, ok := traceInterface.(map[string]interface{}); ok {
				if spans, ok := trace["spans"].([]interface{}); ok {
					for _, spanInterface := range spans {
						if span, ok := spanInterface.(map[string]interface{}); ok {
							if spanValue, ok := span["span"].(string); ok {
								// Encode the plain string as base64
								span["span"] = base64.StdEncoding.EncodeToString([]byte(spanValue))
							}
						}
					}
				}
			}
		}
	}

	// Convert back to JSON
	modifiedJSON, err := json.Marshal(jsonData)
	gm.Expect(err).NotTo(gm.HaveOccurred())

	// Finally unmarshal to protobuf
	gm.Expect(protojson.Unmarshal(modifiedJSON, response)).To(gm.Succeed())
}

// MarshalToJSONWithStringBytes marshals the QueryResponse to JSON with []byte fields as strings instead of base64.
func MarshalToJSONWithStringBytes(resp *tracev1.QueryResponse) ([]byte, error) {
	// First marshal to JSON using protojson
	j, err := protojson.Marshal(resp)
	if err != nil {
		return nil, err
	}

	// Parse the JSON to modify byte fields
	var jsonData map[string]interface{}
	err = json.Unmarshal(j, &jsonData)
	if err != nil {
		return nil, err
	}

	// Convert base64 encoded span fields back to strings in traces structure
	if traces, ok := jsonData["traces"].([]interface{}); ok {
		for _, traceInterface := range traces {
			if trace, ok := traceInterface.(map[string]interface{}); ok {
				if spans, ok := trace["spans"].([]interface{}); ok {
					for _, spanInterface := range spans {
						if span, ok := spanInterface.(map[string]interface{}); ok {
							if spanB64, ok := span["span"].(string); ok {
								// Decode base64 back to original string
								spanBytes, err := base64.StdEncoding.DecodeString(spanB64)
								if err != nil {
									// If it's not valid base64, keep the original value
									continue
								}
								span["span"] = string(spanBytes)
							}
						}
					}
				}
			}
		}
	}

	// Convert back to JSON
	return json.Marshal(jsonData)
}

// GetSpanDataAsString extracts the span data as a string from a Span.
// This converts the raw bytes back to the original string value like "trace_001_span_1".
func GetSpanDataAsString(span *tracev1.Span) string {
	if span == nil || len(span.Span) == 0 {
		return ""
	}
	return string(span.Span)
}
