//
// Copyright (c) 2017-2026, Manticore Software LTD (https://manticoresearch.com)
// Copyright (c) 2001-2016, Andrew Aksyonoff
// Copyright (c) 2008-2016, Sphinx Technologies Inc
// All rights reserved
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License. You should have
// received a copy of the GPL license along with this program; if you
// did not, you can find it at http://www.gnu.org/
//

#include "frontendschema.h"

#include "indexsettings.h"
#include "queuecreator.h"

/// returns internal magic names for expressions like COUNT(*) that have a corresponding one
/// returns expression itself otherwise
static const char * GetMagicSchemaName ( const CSphString & s )
{
	if ( s=="count(*)" )
		return "@count";
	if ( s=="weight()" )
		return "@weight";
	if ( s=="groupby()" )
		return "@groupby";
	return s.IsEmpty() ? "" : s.cstr();
}


static int GetUuidIdIndex ( const ISphSchema & tSchema, const CSphString & sAttr, bool bProjectUuidId )
{
	if ( !bProjectUuidId )
		return -1;

	if ( sAttr.IsEmpty() || strcmp ( sAttr.cstr(), sphGetDocidName() )!=0 )
		return -1;

	int iUuid = tSchema.GetAttrIndex ( sphGetUuidDocidName() );
	assert ( iUuid<0 || tSchema.GetAttr(iUuid).m_eAttrType==SPH_ATTR_STRING || tSchema.GetAttr(iUuid).m_eAttrType==SPH_ATTR_STRINGPTR );
	return iUuid;
}


static bool IsFrontendAttrMatch ( const ISphSchema & tSchema, const CSphString & sSchemaAttr, const CSphString & sQueryAttr, bool bProjectUuidId )
{
	if ( GetUuidIdIndex ( tSchema, sQueryAttr, bProjectUuidId )>=0 )
		return !sSchemaAttr.IsEmpty() && strcmp ( sSchemaAttr.cstr(), sphGetUuidDocidName() )==0;

	return strcmp ( sSchemaAttr.cstr(), GetMagicSchemaName ( sQueryAttr ) )==0;
}


static int GetFrontendAttrIndex ( const ISphSchema & tSchema, const CSphString & sAttr, bool bProjectUuidId )
{
	int iUuidAttr = GetUuidIdIndex ( tSchema, sAttr, bProjectUuidId );
	if ( iUuidAttr>=0 )
		return iUuidAttr;

	return tSchema.GetAttrIndex ( GetMagicSchemaName ( sAttr ) );
}

/// a functor to sort columns by (is_aggregate ASC, column_index ASC)
struct AggregateColumnSort_fn
{
	bool IsAggr ( const CSphColumnInfo & c ) const
	{
		return c.m_eAggrFunc!=SPH_AGGR_NONE
			|| c.m_sName=="@groupby"
			|| c.m_sName=="@count"
			|| c.m_sName=="@distinct"
			|| IsSortJsonInternal ( c.m_sName );
	}

	bool IsLess ( const CSphColumnInfo & a, const CSphColumnInfo & b ) const
	{
		bool aa = IsAggr(a);
		bool bb = IsAggr(b);
		if ( aa!=bb )
			return aa < bb;
		return a.m_iIndex < b.m_iIndex;
	}
};


static void MoveDocidFirst ( CSphVector<CSphColumnInfo> & dAttrs )
{
	int iDocid = 0;
	while ( iDocid<dAttrs.GetLength() && dAttrs[iDocid].m_sName!=sphGetDocidName() )
		++iDocid;

	if ( iDocid==dAttrs.GetLength() )
	{
		assert ( 0 && "agent schema must contain numeric id" );
		return;
	}

	for ( ; iDocid>0; --iDocid )
		Swap ( dAttrs[iDocid], dAttrs[iDocid-1] );
}

///////////////////////////////////////////////////////////////////////////////

FrontendSchemaBuilder_c::FrontendSchemaBuilder_c ( const AggrResult_t & tRes, const CSphQuery & tQuery, const CSphVector<CSphQueryItem> & dItems, const CSphVector<CSphQueryItem> & dQueryItems, const sph::StringSet & hExtraColumns, bool bQueryFromAPI, bool bHaveLocals )
	: m_tRes ( tRes )
	, m_tQuery ( tQuery )
	, m_dItems ( dItems )
	, m_dQueryItems ( dQueryItems )
	, m_hExtraColumns ( hExtraColumns )
	, m_bQueryFromAPI ( bQueryFromAPI )
	, m_bHaveLocals ( bHaveLocals )
	, m_bAgent ( tQuery.m_bAgent )
{
	m_dFrontend.Resize(dItems.GetLength());
}


bool FrontendSchemaBuilder_c::Build ( bool bMaster, CSphString & sError )
{
	CollectKnownItems();
	AddAttrs();
	AddNullMask();
	if ( !CheckUnmapped(sError) )
		return false;

	Finalize();

	RemapGroupBy();
	// agent should provide raw attributes into master without any remapping
	if ( bMaster )
		RemapFacets();

	return true;
}


void FrontendSchemaBuilder_c::CollectKnownItems()
{
	// Agent wire schemas keep numeric id as a normal attr; the master applies
	// the public UUID id alias after it receives and merges agent results.
	const bool bProjectUuidId = !m_bAgent;

	ARRAY_CONSTFOREACH ( i, m_dItems )
	{
		const CSphQueryItem & tItem = m_dItems[i];

		int iCol = -1;
		if ( !m_bQueryFromAPI && tItem.m_sAlias.IsEmpty() )
			iCol = GetFrontendAttrIndex ( m_tRes.m_tSchema, tItem.m_sExpr, bProjectUuidId );

		if ( iCol>=0 )
		{
			m_dKnownAttrs.Add(i);
			m_dFrontend[i].m_sName = tItem.m_sExpr;
			m_dFrontend[i].m_iIndex = iCol;
		}
		else
			m_dUnmappedAttrs.Add(i);
	}
}


void FrontendSchemaBuilder_c::AddAttrs()
{
	bool bUsualApi = !m_bAgent && m_bQueryFromAPI;
	const bool bProjectUuidId = !m_bAgent;

	for ( int iCol=0; iCol<m_tRes.m_tSchema.GetAttrsCount(); ++iCol )
	{
		const CSphColumnInfo & tCol = m_tRes.m_tSchema.GetAttr(iCol);

		assert ( !tCol.m_sName.IsEmpty() );
		bool bMagic = IsGroupbyMagic ( tCol.m_sName ) || IsSortStringInternal ( tCol.m_sName );

		if ( !bMagic && tCol.m_pExpr )
		{
			ARRAY_FOREACH ( j, m_dUnmappedAttrs )
				if ( m_dItems[ m_dUnmappedAttrs[j] ].m_sAlias==tCol.m_sName
					|| ( IsGroupConcatValueAttr ( m_dItems[ m_dUnmappedAttrs[j] ].m_sExpr )
						&& m_dItems[ m_dUnmappedAttrs[j] ].m_sExpr==tCol.m_sName ) )
				{
					int k = m_dUnmappedAttrs[j];
					m_dFrontend[k].m_iIndex = iCol;
					m_dFrontend[k].m_sName = m_dItems[k].m_sAlias;
					m_dKnownAttrs.Add(k);
					m_dUnmappedAttrs.Remove ( j-- ); // do not skip an element next to removed one!
				}

			// FIXME?
			// really not sure if this is the right thing to do
			// but it fixes a couple queries in test_163 in compaitbility mode
			if ( m_bAgent && !m_dFrontend.Contains ( bind ( &CSphColumnInfo::m_sName ), tCol.m_sName ) )
			{
				CSphColumnInfo & t = m_dFrontend.Add();
				t.m_iIndex = iCol;
				t.m_sName = tCol.m_sName;
			}
		} else if ( bMagic && ( tCol.m_pExpr || bUsualApi ) )
		{
			ARRAY_FOREACH ( j, m_dUnmappedAttrs )
				if ( IsFrontendAttrMatch ( m_tRes.m_tSchema, tCol.m_sName, m_dItems[ m_dUnmappedAttrs[j] ].m_sExpr, bProjectUuidId ) )
				{
					int k = m_dUnmappedAttrs[j];
					m_dFrontend[k].m_iIndex = iCol;
					m_dFrontend[k].m_sName = m_dItems[k].m_sAlias;
					m_dKnownAttrs.Add(k);
					m_dUnmappedAttrs.Remove ( j-- ); // do not skip an element next to removed one!
				}

			if ( !m_dFrontend.Contains ( bind ( &CSphColumnInfo::m_sName ), tCol.m_sName ) )
			{
				CSphColumnInfo & t = m_dFrontend.Add();
				t.m_iIndex = iCol;
				t.m_sName = tCol.m_sName;
			}
		} else
		{
			bool bAdded = false;
			ARRAY_FOREACH ( j, m_dUnmappedAttrs )
			{
				int k = m_dUnmappedAttrs[j];
				const CSphQueryItem & t = m_dItems[k];
				if ( ( IsFrontendAttrMatch ( m_tRes.m_tSchema, tCol.m_sName, t.m_sExpr, bProjectUuidId ) && t.m_eAggrFunc==SPH_AGGR_NONE )
					|| ( t.m_sAlias==tCol.m_sName &&
						( GetFrontendAttrIndex ( m_tRes.m_tSchema, t.m_sExpr, bProjectUuidId )==-1 || t.m_eAggrFunc!=SPH_AGGR_NONE ) ) )
				{
					// tricky bit about naming
					//
					// in master mode, we can just use the alias or expression or whatever
					// the data will be fetched using the locator anyway, column name does not matter anymore
					//
					// in agent mode, however, we need to keep the original column names in our response
					// otherwise, queries like SELECT col1 c, count(*) c FROM dist will fail on master
					// because it won't be able to identify the count(*) aggregate by its name
					m_dFrontend[k].m_iIndex = iCol;
					m_dFrontend[k].m_sName = m_bAgent
						? tCol.m_sName
						: ( m_dItems[k].m_sAlias.IsEmpty()
							? m_dItems[k].m_sExpr
							: m_dItems[k].m_sAlias );
					m_dKnownAttrs.Add(k);
					bAdded = true;
					m_dUnmappedAttrs.Remove ( j-- ); // do not skip an element next to removed one!
				}
			}

			// column was not found in the select list directly
			// however we might need it anyway because of a non-NULL extra-schema
			// (extra-schema is additional set of columns came from right side of query
			// when you perform 'select a from index order by b', the 'b' is not displayed, but need for sorting,
			// so extra-schema in the case will contain 'b').
			// bMagic condition added for @groupbystr in the agent mode
			if ( !bAdded && m_bAgent && ( m_hExtraColumns[tCol.m_sName] || !m_bHaveLocals || bMagic ) )
			{
				CSphColumnInfo & t = m_dFrontend.Add();
				t.m_iIndex = iCol;
				t.m_sName = tCol.m_sName;
			}
		}
	}

	m_dKnownAttrs.Sort();
}


void FrontendSchemaBuilder_c::AddNullMask()
{
	int iNullMask = m_tRes.m_tSchema.GetAttrIndex ( GetNullMaskAttrName() );
	if ( iNullMask < 0 )
		return;

	for ( auto & i : m_dFrontend )
		if ( i.m_sName==GetNullMaskAttrName() )
			return;

	CSphColumnInfo & tAttr = m_dFrontend.Add();
	tAttr.m_iIndex = iNullMask;
	tAttr.m_sName = GetNullMaskAttrName();
}


bool FrontendSchemaBuilder_c::CheckUnmapped ( CSphString & sError ) const
{
	// sanity check
	// verify that we actually have all the queried select items
	assert ( m_dUnmappedAttrs.IsEmpty() || ( m_dUnmappedAttrs.GetLength()==1 && m_dItems [ m_dUnmappedAttrs[0] ].m_sExpr=="id" ) );
	ARRAY_CONSTFOREACH ( i, m_dItems )
	{
		const CSphQueryItem & tItem = m_dItems[i];
		if ( !m_dKnownAttrs.BinarySearch(i) && tItem.m_sExpr!="id" )
		{
			sError.SetSprintf ( "internal error: column '%s/%s' not found in result set schema", tItem.m_sExpr.cstr(), tItem.m_sAlias.cstr() );
			return false;
		}
	}

	return true;
}


void FrontendSchemaBuilder_c::Finalize()
{
	// finalize the frontend schema columns
	// we kept indexes into internal schema there, now use them to lookup and copy column data
	ARRAY_CONSTFOREACH ( i, m_dFrontend )
	{
		CSphColumnInfo & tFrontend = m_dFrontend[i];
		const CSphColumnInfo & s = m_tRes.m_tSchema.GetAttr ( tFrontend.m_iIndex );
		tFrontend.m_tLocator = s.m_tLocator;
		tFrontend.m_eAttrType = s.m_eAttrType;
		tFrontend.m_eAggrFunc = s.m_eAggrFunc; // for a sort loop just below
		tFrontend.m_fTdigestCompression = s.m_fTdigestCompression;
		tFrontend.m_tAggrSettings = s.m_tAggrSettings;
		tFrontend.m_iIndex = i; // to make the aggr sort loop just below stable

		tFrontend.m_uFieldFlags = s.m_uFieldFlags;
	}

	// tricky bit
	// in agents only, push aggregated columns, if any, to the end
	// for that, sort the schema by (is_aggregate ASC, column_index ASC)
	if ( m_bAgent )
	{
		m_dFrontend.Sort ( AggregateColumnSort_fn() );
		// Parsed agent attributes get fresh locators in wire order, while reducers
		// read the internal numeric docid directly from dynamic row item zero.
		MoveDocidFirst ( m_dFrontend );
	}
}


void FrontendSchemaBuilder_c::RemapGroupBy()
{
	// remap groupby() and aliased groupby() to @groupbystr or string attribute
	const CSphColumnInfo * p = nullptr;
	CSphString sJsonGroupBy;
	if ( sphJsonNameSplit ( m_tQuery.m_sGroupBy.cstr() ) )
	{
		sJsonGroupBy = SortJsonInternalSet ( m_tQuery.m_sGroupBy );
		p = m_tRes.m_tSchema.GetAttr ( sJsonGroupBy.cstr() );
	}

	if ( !p )
	{
		// try string attribute (multiple group-by still displays hashes)
		if ( !m_tQuery.m_sGroupBy.IsEmpty() )
		{
			p = m_tRes.m_tSchema.GetAttr ( m_tQuery.m_sGroupBy.cstr() );
			if ( p )
			{
				if ( p->m_eAttrType==SPH_ATTR_JSON_PTR )
				{
					sJsonGroupBy = SortJsonInternalSet ( m_tQuery.m_sGroupBy );
					p = m_tRes.m_tSchema.GetAttr ( sJsonGroupBy.cstr() );
				} else if ( p->m_eAttrType!=SPH_ATTR_STRINGPTR )
				{
					p = nullptr;
				}
			}
		}

		if ( !p )
			return;
	}

	for ( auto & tFrontend : m_dFrontend )
		if ( tFrontend.m_sName=="groupby()" )
		{
			tFrontend.m_tLocator = p->m_tLocator;
			tFrontend.m_eAttrType = p->m_eAttrType;
			tFrontend.m_eAggrFunc = p->m_eAggrFunc;
			tFrontend.m_fTdigestCompression = p->m_fTdigestCompression;
			tFrontend.m_tAggrSettings = p->m_tAggrSettings;
		}

	// check aliases too
	for ( const auto & tQueryItem : m_dQueryItems )
	{
		if ( tQueryItem.m_sExpr!="groupby()" )
			continue;

		for ( auto & tFrontend : m_dFrontend )
			if ( tFrontend.m_sName==tQueryItem.m_sAlias )
			{
				tFrontend.m_tLocator = p->m_tLocator;
				tFrontend.m_eAttrType = p->m_eAttrType;
				tFrontend.m_eAggrFunc = p->m_eAggrFunc;
				tFrontend.m_fTdigestCompression = p->m_fTdigestCompression;
				tFrontend.m_tAggrSettings = p->m_tAggrSettings;
			}
	}
}

static void CollectAliases ( const VecTraits_T<CSphQueryItem> & dItems, const CSphString & sGroupBy, const CSphString & sJsonGroupBy, sph::StringSet & hFacet )
{
	ARRAY_CONSTFOREACH ( i, dItems )
	{
		const CSphQueryItem & tItem = dItems[i];
		if ( tItem.m_sExpr.IsEmpty() ) 
			continue;

		bool bMatch = false;

		// 1st check - exact string match (raw or json)
		if ( tItem.m_sExpr == sGroupBy )
		{
			bMatch = true;
		} else if ( !sJsonGroupBy.IsEmpty() )
		{
			// 2nd - normalize JSON path
			if ( tItem.m_sExpr==sJsonGroupBy )
				bMatch = true;
			else if ( sphJsonNameSplit ( tItem.m_sExpr.cstr() ) )
				bMatch = ( SortJsonInternalSet ( tItem.m_sExpr )==sJsonGroupBy );
		}

		if ( bMatch )
		{
			hFacet.Add ( tItem.m_sExpr );
			hFacet.Add ( tItem.m_sAlias );
		}
	}
}

void FrontendSchemaBuilder_c::RemapFacets()
{
	// facets
	if ( !m_tQuery.m_bFacet && !m_tQuery.m_bFacetHead )
		return;

	if ( m_tQuery.m_sGroupBy.IsEmpty() )
		return;

	CSphString sGroupBy = m_tQuery.m_sGroupBy;
	CSphString sJsonGroupBy;
	const CSphColumnInfo * pGroupByCol = nullptr;

	if ( sphJsonNameSplit ( sGroupBy.cstr() ) )
	{
		sJsonGroupBy = SortJsonInternalSet ( sGroupBy );
		pGroupByCol = m_tRes.m_tSchema.GetAttr ( sJsonGroupBy.cstr() );
	}

	if ( !pGroupByCol )
		pGroupByCol = m_tRes.m_tSchema.GetAttr ( "@groupby" );
	if ( !pGroupByCol )
		return;

	// JSON group by \ facet - 1st @groupbystr then @groupby
	// @groupbystr has JSON items these grouped
	// @groupby is the group key hash
	const CSphColumnInfo * pGroupByStrCol = nullptr;
	
	// JSON path first - @groupbystr_j.field / @groupbystr_j['*']
	if ( !sJsonGroupBy.IsEmpty() )
	{
		CSphString sGroupByStr;
		sGroupByStr.SetSprintf ( "%s%s", GetInternalJsonPrefix(), sJsonGroupBy.cstr() );
		pGroupByStrCol = m_tRes.m_tSchema.GetAttr ( sGroupByStr.cstr() );
	}
	
	// if not found - plain attribute name - @groupbystr_j
	if ( !pGroupByStrCol )
	{
		CSphString sGroupByStr;
		sGroupByStr.SetSprintf ( "%s%s", GetInternalJsonPrefix(), sGroupBy.cstr() );
		pGroupByStrCol = m_tRes.m_tSchema.GetAttr ( sGroupByStr.cstr() );
	}

	const CSphColumnInfo * pRemapCol = pGroupByStrCol ? pGroupByStrCol : pGroupByCol;

	sph::StringSet hFacet;
	hFacet.Add ( sGroupBy );
	if ( !sJsonGroupBy.IsEmpty() )
		hFacet.Add ( sJsonGroupBy );

	CollectAliases ( m_dItems, sGroupBy, sJsonGroupBy, hFacet );
	CollectAliases ( m_tQuery.m_dItems, sGroupBy, sJsonGroupBy, hFacet );
	CollectAliases ( m_tQuery.m_dRefItems, sGroupBy, sJsonGroupBy, hFacet );

	for ( auto & tFrontend : m_dFrontend )
	{
		ESphAttr eAttr = tFrontend.m_eAttrType;
		// checking _PTR attrs only because we should not have and non-ptr attr at this point
		if ( hFacet[tFrontend.m_sName] && ( eAttr==SPH_ATTR_UINT32SET_PTR || eAttr==SPH_ATTR_INT64SET_PTR || eAttr==SPH_ATTR_FLOAT_VECTOR_PTR || eAttr==SPH_ATTR_JSON_PTR || eAttr==SPH_ATTR_JSON_FIELD_PTR ) )
		{
			tFrontend.m_tLocator = pRemapCol->m_tLocator;
			tFrontend.m_eAttrType = pRemapCol->m_eAttrType;
			tFrontend.m_eAggrFunc = pRemapCol->m_eAggrFunc;
			tFrontend.m_fTdigestCompression = pRemapCol->m_fTdigestCompression;
			tFrontend.m_tAggrSettings = pRemapCol->m_tAggrSettings;
		}
	}
}
