//
// Copyright (c) 2017-2026, Manticore Software LTD (https://manticoresearch.com)
// Copyright (c) 2001-2016, Andrew Aksyonoff
// Copyright (c) 2008-2016, Sphinx Technologies Inc
// All rights reserved
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License. You should have
// received a copy of the GPL license along with this program; if you
// did not, you can find it at http://www.gnu.org/
//

#include "sphinxint.h"
#include "sphinxrt.h"
#include "sphinxpq.h"
#include "sphinxsearch.h"
#include "sphinxsort.h"
#include "sphinxutils.h"
#include "sphinxquery/sphinxquery.h"
#include "sphinxquery/xqparser.h"
#include "fileutils.h"
#include "sphinxplugin.h"
#include "icu.h"
#include "sphinxqcache.h"
#include "attribute.h"
#include "killlist.h"
#include "histogram.h"
#include "accumulator.h"
#include "indexcheck.h"
#include "indexsettings.h"
#include "indexformat.h"
#include "coroutine.h"
#include "mini_timer.h"
#include "binlog.h"
#include "secondaryindex.h"
#include "docidlookup.h"
#include "columnarrt.h"
#include "columnarmisc.h"
#include "sphinx_alter.h"
#include "chunksearchctx.h"
#include "indexfiles.h"
#include "jieba.h"
#include "task_dispatcher.h"
#include "tracer.h"
#include "pseudosharding.h"
#include "knnmisc.h"
#include "knnlib.h"
#include "jsonsi.h"
#include "attr_embedding.h"
#include "embeddingutils.h"
#include "std/sys.h"
#include "dict/infix/infix_builder.h"

#include <sys/stat.h>
#include <fcntl.h>
#include <atomic>

#if _WIN32
#include <errno.h>
#else
#include <unistd.h>
#include <sys/time.h>

#endif

#include "attrindex_builder.h"
#include "tokenizer/tokenizer.h"
#include "queryfilter.h"

#include "secondarylib.h"

#if WITH_RE2
#include <string>
#include <re2/re2.h>
#endif

using namespace Threads;

//////////////////////////////////////////////////////////////////////////

#define RTDICT_CHECKPOINT_V5			48

// rt-segments tuning

// rate limit for flushing RAM chunk as disk chunk. Rate applied to rt_mem_limit, calculated value used as limit, when we start flushing
constexpr int MAX_SEGMENTS						= 32;
constexpr int MAX_PROGRESSION_SEGMENT			= 8;
constexpr int64_t MAX_SEGMENT_VECTOR_LEN		= INT_MAX;
constexpr double INITIAL_SAVE_RATE_LIMIT		= 0.5;		///< we start rate limiting from this value.
constexpr double SAVE_RATE_LIMIT_EMERGENCY_STEP	= 0.05;		///< emergency back-off.

int SIMULTANEOUS_SAVE_LIMIT			= 2;		///< how many save ops we allow a time
int MAX_TOLERATE_LOAD_SEGMENTS		= MAX_SEGMENTS * ( SIMULTANEOUS_SAVE_LIMIT + 1 );	///< if on load N of segments exceedes this value - perform safe loading
double MIN_SAVE_RATE_LIMIT			= 1.0/(SIMULTANEOUS_SAVE_LIMIT+1.0);	///< minimal rate limit. Calculated value will never be less that that bound
double MAX_SAVE_RATE_LIMIT			= 0.95;		///< maximal rate limit. It most probably may be reached with very low insertion rate


// in test mode - forcibly invoke high-concurrency pattern to reveal possibly races (like #4207)
static void SetTightConcurrencyLimits()
{
	sphInfo("Test mode: set SIMULTANEOUS_SAVE_LIMIT = 4");
	SIMULTANEOUS_SAVE_LIMIT			= 4;
	MAX_TOLERATE_LOAD_SEGMENTS		= MAX_SEGMENTS * ( SIMULTANEOUS_SAVE_LIMIT + 1 );
	MIN_SAVE_RATE_LIMIT				= 1.0/(SIMULTANEOUS_SAVE_LIMIT+1.0);
	MAX_SAVE_RATE_LIMIT				= MIN_SAVE_RATE_LIMIT;
}

//////////////////////////////////////////////////////////////////////////

#ifndef NDEBUG
#define Verify(_expr) assert(_expr)
#else
#define Verify(_expr) _expr
#endif

#define LOG_LEVEL_RTRDIAG false
#define LOG_LEVEL_RTDDIAG false
#define LOG_LEVEL_RTDIAGV false
#define LOG_LEVEL_RTDIAGVV false
#define LOG_LEVEL_DEBUGV false
#define LOG_COMPONENT_RTSEG __LINE__ << " " << (Coro::CurrentScheduler()?Coro::CurrentScheduler()->Name():"") << " "

// used in start/merge RAM segments
#define RTRLOG LOGINFO ( RTRDIAG, RTSEG )

// used when logging disk save/optimize
#define RTDLOG LOGINFO ( RTDDIAG, RTSEG )

// ops for save RAM segments as disk chunk
static const bool LOG_LEVEL_RTSAVEDIAG = env_exists ( "MANTICORE_LOG_RTSAVEDIAG" );
#define RTSAVELOG LOGINFO ( RTSAVEDIAG, RTSEG )
#define RTLOGV LOGINFO ( RTDIAGV, RTSEG )

static const bool LOG_LEVEL_RTSPLIT_QUERY = env_exists ( "MANTICORE_LOG_RTSPLIT_QUERY" ); // verbose logging split query events, ruled by this env variable
#define LOG_COMPONENT_RTQUERYINFO __LINE__ << " "
#define RTQUERYINFO LOGINFO ( RTSPLIT_QUERY, RTQUERYINFO )


//////////////////////////////////////////////////////////////////////////
// GLOBALS
//////////////////////////////////////////////////////////////////////////

// optimize mode for disk chunks merge fixme! retire?
static bool g_bProgressiveMerge = true;

//////////////////////////////////////////////////////////////////////////
volatile bool &RTChangesAllowed () noexcept
{
	static volatile bool bRTChangesAllowed = false;
	return bRTChangesAllowed;
}

volatile int &AutoOptimizeCutoffMultiplier() noexcept
{
	static int iAutoOptimizeCutoffMultiplier = 1;
	return iAutoOptimizeCutoffMultiplier;
}

volatile bool &OptimizeCutoffExplicit() noexcept
{
	static bool bOptimizeCutoffExplicit = false;
	return bOptimizeCutoffExplicit;
}

volatile int &ParallelChunkMergesLimit() noexcept
{
	static int iParallelChunkMerges = 1;
	return iParallelChunkMerges;
}

volatile int &MergeChunksPerJob() noexcept
{
	static int iMergeChunksPerJob = 2;
	return iMergeChunksPerJob;
}

volatile int &KNNParallelBuild() noexcept
{
	static int iKNNParallelBuild = 1;	// 1=serial (default); larger value = fixed worker count
	return iKNNParallelBuild;
}

volatile int &EmbeddingsThreads() noexcept
{
	static int iEmbeddingsThreads = 4;	// max threads the embeddings lib may use; clamped by free workers at call time
	return iEmbeddingsThreads;
}

int GetEmbeddingsThreadsToUse ( int iMaxOverride )
{
	auto * pPool = GlobalWorkPool();
	int iWorkers = pPool ? pPool->WorkingThreads() : 1;
	int iBusy = pPool ? Max ( pPool->CurTasks() - 1, 0 ) : 0; // ignore current task
	int iFree = Max ( iWorkers - iBusy, 1 );

	int iMax = ( iMaxOverride >= 0 ) ? iMaxOverride : EmbeddingsThreads();
	if ( iMax > 0 )
		iFree = Min ( iFree, iMax );
	return iFree;
}

volatile int AutoOptimizeCutoff() noexcept
{
	static int iAutoOptimizeCutoff = GetNumLogicalCPUs() * 2;
	return iAutoOptimizeCutoff;
}

volatile int AutoOptimizeCutoffKNN() noexcept
{
	static int iAutoOptimizeCutoffKNN = Max ( GetNumPhysicalCPUs() / 2, 1 );
	return iAutoOptimizeCutoffKNN;
}

static int64_t g_iFlushWriteUs = -1LL; // 111 at the end sighs uninitialized value
static int64_t g_iFlushSearchUs = 30'000'111LL;

void SetRtFlushDiskPeriod ( int iFlushWrite, int iFlushSearch )
{
	g_iFlushWriteUs = (iFlushWrite==-1)? -1LL : 1'000'000LL * iFlushWrite;
	g_iFlushSearchUs = 1'000'000LL * iFlushSearch;
}

//////////////////////////////////////////////////////////////////////////

// Variable Length Byte (VLB) encoding
// store int variable in as much bytes as actually needed to represent it

#define SPH_MAX_KEYWORD_LEN (3*SPH_MAX_WORD_LEN+4)
static_assert ( SPH_MAX_KEYWORD_LEN<255, "SPH_MAX_KEYWORD_LEN should fit in a byte" );


// Variable Length Byte (VLB) skipping (BE/LE agnostic)
static inline void SkipZipped ( const BYTE *& pIn ) noexcept
{
	while ( *pIn & 0x80U )
		++pIn;
	++pIn; // jump over last one
}

template<typename P>
static inline void ZipDword ( CSphVector<BYTE, P> & dOut, DWORD uValue ) noexcept
{
	ZipValueLE ( [&dOut] ( BYTE b ) { dOut.Add ( b ); }, uValue );
}

template<typename P>
static inline void ZipQword ( CSphVector<BYTE, P> & dOut, uint64_t uValue ) noexcept
{
	ZipValueLE ( [&dOut] ( BYTE b ) { dOut.Add ( b ); }, uValue );
}

static inline DWORD UnzipDword ( const BYTE *& pIn ) noexcept
{
	return UnzipValueLE<DWORD> ( [&pIn] () mutable { return *pIn++; } );
}

static inline void UnzipDword ( DWORD * pValue, const BYTE *& pIn ) noexcept
{
	*pValue = UnzipValueLE<DWORD> ( [&pIn] () mutable { return *pIn++; } );
}

static inline uint64_t UnzipQword ( const BYTE *& pIn ) noexcept
{
	return UnzipValueLE<uint64_t> ( [&pIn] () mutable { return *pIn++; } );
}

[[maybe_unused]] static inline void UnzipQword ( uint64_t * pValue, const BYTE *& pIn ) noexcept
{
	*pValue = UnzipValueLE<uint64_t> ( [&pIn] () mutable { return *pIn++; } );
}

template<typename P>
static inline void ZipWordid ( CSphVector<BYTE, P> & dOut, uint64_t uValue ) noexcept
{
	ZipQword ( dOut, uValue );
}

static inline uint64_t UnzipWordid ( const BYTE *& pIn ) noexcept
{
	return UnzipQword ( pIn );
}

template <DictFormat_e DICT_FORMAT>
static ByteBlob_t GetPackedKeywordByFormat ( const BYTE * pPacked )
{
	static_assert ( DICT_FORMAT!=DictFormat_e::CRC );

	if constexpr ( DICT_FORMAT==DictFormat_e::KEYWORDS_V2 )
		return GetPackedKeywordV2 ( pPacked );
	else
		return GetPackedKeywordLegacy ( pPacked );
}


static int AppendPackedKeywordLegacy ( CSphVector<BYTE> & dOut, const BYTE * pWord, int iLen )
{
	assert ( pWord );
	assert ( iLen>0 && iLen<=std::numeric_limits<BYTE>::max() );

	int iOff = dOut.GetLength();
	dOut.Add ( (BYTE)iLen );
	dOut.Append ( pWord, iLen );

	return iOff;
}


static int AppendPackedKeywordV2 ( CSphVector<BYTE> & dOut, const BYTE * pWord, int iLen )
{
	assert ( pWord );
	assert ( iLen>0 && iLen<=GetKeywordMaxStoredBytes ( DictFormat_e::KEYWORDS_V2 ) );

	int iOff = dOut.GetLength();
	BYTE dPacked[10];
	dOut.Append ( dPacked, ZipToPtrLE ( dPacked, (DWORD)iLen ) );
	dOut.Append ( pWord, iLen );

	return iOff;
}


//////////////////////////////////////////////////////////////////////////

InsertDocData_c::InsertDocData_c ( const ISphSchema & tSchema )
{
	m_tDoc.Reset ( tSchema.GetRowSize() );
	m_dFields.Resize ( tSchema.GetFieldsCount() );

	const CSphColumnInfo * pId = tSchema.GetAttr ( sphGetDocidName() );
	assert(pId);

	if ( pId->IsColumnar() )
	{
		int iColumnar = 0;
		for ( int i = 0; i < tSchema.GetAttrsCount(); i++ )
		{
			if ( tSchema.GetAttr(i).m_sName == sphGetDocidName() )
				m_iColumnarID = iColumnar;

			iColumnar++;
		}

		m_dColumnarAttrs.Resize(iColumnar);
	}
	else
	{
		m_tDocIDLocator = pId->m_tLocator;
		m_tDocIDLocator.m_bDynamic = true;
	}
}

void InsertDocData_c::SetID ( SphAttr_t tDocID )
{
	if ( m_iColumnarID>=0 )
	{
		m_dColumnarAttrs[m_iColumnarID] = tDocID;
		return;
	}
	
	m_tDoc.SetAttr ( m_tDocIDLocator, tDocID );
}


SphAttr_t InsertDocData_c::GetID() const
{
	if ( m_iColumnarID>=0 )
		return m_dColumnarAttrs[m_iColumnarID];

	return m_tDoc.GetAttr(m_tDocIDLocator);
}


void InsertDocData_c::ResetPrimaryIdState()
{
	m_eDocidMode = InsertDocidMode_e::NUMERIC;
	m_iUuidString = -1;
	m_sOwnedUuidDocid = "";
}


void InsertDocData_c::SetUuidDocidString ( CSphString && sUuid )
{
	assert ( m_iUuidString>=0 && m_iUuidString<m_dStrings.GetLength() );
	assert ( sphIsNormalizedUuidDocid ( { sUuid.cstr(), sUuid.Length() } ) );
	m_sOwnedUuidDocid.Swap ( sUuid );
	if ( m_iUuidString>=0 && m_iUuidString<m_dStrings.GetLength() )
		m_dStrings[m_iUuidString] = m_sOwnedUuidDocid.cstr();
}


const char * InsertDocData_c::GetUuidDocidString() const
{
	if ( m_iUuidString<0 || m_iUuidString>=m_dStrings.GetLength() )
		return nullptr;

	return m_dStrings[m_iUuidString];
}


static CSphString sphGenerateUuidDocid ( DocID_t tDocID )
{
	uint64_t uDocID = (uint64_t)tDocID;
	unsigned uHi32 = (unsigned)( uDocID>>32 );
	unsigned uMid16 = (unsigned)( ( uDocID>>16 ) & 0xffff );
	unsigned uVersion = 0x8000 | (unsigned)( ( uDocID>>4 ) & 0x0fff );
	unsigned uVariant = 0x8000 | (unsigned)( uDocID & 0x000f );

	CSphString sUuid;
	sUuid.SetSprintf ( "%08x-%04x-%04x-%04x-000000000000", uHi32, uMid16, uVersion, uVariant );
	return sUuid;
}


void InsertDocData_c::AddMVALength ( int iLength, bool bDefault )
{
	m_dMvas.Add ( int64_t(iLength) | ( bDefault ? DEFAULT_FLAG : 0 ) );
}


std::pair<int, bool> InsertDocData_c::ReadMVALength ( const int64_t * & pMVA )
{
	bool bDefault = !!(*pMVA & DEFAULT_FLAG);
	int iLength = int(*pMVA & ~DEFAULT_FLAG);
	pMVA++;
	return { iLength, bDefault };
}


void InsertDocData_c::FixParsedMVAs ( const CSphVector<int64_t> & dParsed, int iCount )
{
	if ( !iCount )
		return;

	// dParsed:
	// 0 - iCount elements: offset to MVA values with leading MVA element count
	// Could be not in right order

	ResetMVAs();
	for ( int i=0; i<iCount; ++i )
	{
		int iOff = dParsed[i];
		if ( !iOff )
		{
			AddMVALength(0);
			continue;
		}

		DWORD uMvaCount = dParsed[iOff];
		int64_t * pMva = m_dMvas.AddN ( uMvaCount + 1 );
		*pMva++ = uMvaCount;
		memcpy ( pMva, dParsed.Begin() + iOff + 1, sizeof(m_dMvas[0]) * uMvaCount );
	}
}

//////////////////////////////////////////////////////////////////////////


RtSegment_t::RtSegment_t ( DWORD uDocs, const ISphSchema& tSchema )
	: m_uRows ( uDocs )
	, m_tAliveRows { uDocs }
	, m_tDeadRowMap ( uDocs )
	, m_tSchema { tSchema }
{
}

RtSegment_t::~RtSegment_t ()
{
	if ( m_pRAMCounter )
		FixupRAMCounter ( -GetUsedRam() );
}


void RtSegment_t::UpdateUsedRam() const NO_THREAD_SAFETY_ANALYSIS
{
	int64_t iUsedRam = 0;
	iUsedRam += m_dWords.AllocatedBytes();
	iUsedRam += m_dDocs.AllocatedBytes();
	iUsedRam += m_dHits.AllocatedBytes();
	iUsedRam += m_dBlobs.AllocatedBytes();
	iUsedRam += m_dKeywordCheckpoints.AllocatedBytes();
	iUsedRam += m_dRows.AllocatedBytes();
	iUsedRam += m_dInfixFilterCP.AllocatedBytes();
	iUsedRam += m_tDocIDtoRowID.GetLengthBytes();
	iUsedRam += m_tUuidDocID.GetLengthBytes();
	iUsedRam += m_pDocstore ? m_pDocstore->AllocatedBytes() : 0;
	iUsedRam += m_pColumnar ? m_pColumnar->AllocatedBytes() : 0;
	FixupRAMCounter ( iUsedRam - std::exchange ( m_iUsedRam, iUsedRam ) );
}


int64_t RtSegment_t::GetUsedRam() const
{
	if ( !m_iUsedRam )
		UpdateUsedRam();

	return m_iUsedRam;
}


void RtSegment_t::FixupRAMCounter ( int64_t iDelta ) const
{
	if ( !m_pRAMCounter )
		return;

	m_pRAMCounter->fetch_add ( iDelta, std::memory_order_relaxed );
	TRACE_COUNTER ( "mem", perfetto::CounterTrack ( "RAMCounter", "bytes" ).set_unit_multiplier ( 1024 ), m_pRAMCounter->load ( std::memory_order_relaxed ) >> 10 );
}


DWORD RtSegment_t::GetMergeFactor() const
{
	return m_uRows;
}


int RtSegment_t::GetStride () const NO_THREAD_SAFETY_ANALYSIS
{
	return int ( m_dRows.GetLength() / m_uRows );
}


bool RtSegment_t::IsAlive ( DocID_t tDocid ) const
{
	RowID_t tRowID = GetRowidByDocid(tDocid);
	return tRowID!=INVALID_ROWID && !m_tDeadRowMap.IsSet(tRowID);
}


const CSphRowitem * RtSegment_t::GetDocinfoByRowID ( RowID_t tRowID ) const NO_THREAD_SAFETY_ANALYSIS
{
	return m_dRows.GetLength() ? &m_dRows[tRowID*GetStride()] : nullptr;
}


RowID_t RtSegment_t::GetAliveRowidByDocid ( DocID_t tDocID ) const
{
	RowID_t* pRowID = m_tDocIDtoRowID.Find ( tDocID );
	if ( !pRowID || m_tDeadRowMap.IsSet ( *pRowID ) )
		return INVALID_ROWID;
	return *pRowID;
}


RowID_t RtSegment_t::GetRowidByDocid ( DocID_t tDocID ) const
{
	RowID_t * pRowID = m_tDocIDtoRowID.Find(tDocID);
	return pRowID ? *pRowID : INVALID_ROWID;
}


int RtSegment_t::Kill ( DocID_t tDocID )
{
	if ( m_tDeadRowMap.Set ( GetRowidByDocid ( tDocID ) ) )
	{
		assert ( m_tAliveRows>0 );
		m_tAliveRows.fetch_sub ( 1, std::memory_order_relaxed );

		// on runtime killing expected only from single fiber, so no mutex here required as no race for pKillHook
		KillHook ( tDocID );
		return 1;
	}

	return 0;
}


int	RtSegment_t::KillMulti ( const VecTraits_T<DocID_t> & dKlist )
{
	int iTotalKilled = 0;

	// fixme: implement more efficient batch killer
	for ( auto iDocID : dKlist )
		iTotalKilled += Kill ( iDocID );

	return iTotalKilled;
}


void RtSegment_t::SetupDocstore ( const CSphSchema * pSchema )
{
	assert ( !m_pDocstore );
	m_pDocstore = CreateDocstoreRT();
	assert ( m_pDocstore );

	if ( pSchema )
		SetupDocstoreFields ( *m_pDocstore, *pSchema );
}


static void AddUuidDocid ( RtSegment_t & tSegment, const BYTE * pUuid, int iUuidLen, DocID_t tDocID )
{
	if ( !pUuid || iUuidLen<=0 )
		return;

	[[maybe_unused]] bool bAdded = tSegment.m_tUuidDocID.Add ( sphGetUuidDocidKey ( { (const char *)pUuid, iUuidLen } ), tDocID );
	assert ( bAdded );
}


void RtSegment_t::BuildDocID2RowIDMap ( const CSphSchema & tSchema )
{
	const CSphColumnInfo & tDocid = tSchema.GetAttr(0);
	const bool bColumnarId = tDocid.IsColumnar();
	const bool bUuidLinked = tDocid.IsUuidLinkedDocid();
	const CSphColumnInfo * pUuidAttr = bUuidLinked ? tSchema.GetAttr ( sphGetUuidDocidName() ) : nullptr;
	assert ( !bUuidLinked || ( pUuidAttr && pUuidAttr->m_eAttrType==SPH_ATTR_STRING && pUuidAttr->IsColumnar()==bColumnarId ) );

	m_tDocIDtoRowID.Reset(m_uRows);
	m_tUuidDocID.Reset ( bUuidLinked ? m_uRows : 0 );

	if ( !bColumnarId )
	{
		int iStride = GetStride();
		RowID_t tRowID = 0;
		FakeRL_t _ {m_tLock}; // no need true lock as the func is in game during build/merge when segment is not yet published

		for ( int i=0; i<m_dRows.GetLength(); i+=iStride )
		{
			const CSphRowitem * pRow = &m_dRows[i];
			DocID_t tDocID = sphGetDocID ( pRow );
			m_tDocIDtoRowID.Add ( tDocID, tRowID );

			if ( bUuidLinked )
			{
				int iUuidLen = 0;
				const BYTE * pUuid = sphGetBlobAttr ( pRow, pUuidAttr->m_tLocator, m_dBlobs.Begin(), iUuidLen );
				AddUuidDocid ( *this, pUuid, iUuidLen, tDocID );
			}

			tRowID++;
		}
	}
	else
	{
		std::string sError;
		auto pUuidIt = bUuidLinked ? CreateColumnarIterator ( m_pColumnar.get(), sphGetUuidDocidName(), sError ) : nullptr;
		assert ( !bUuidLinked || pUuidIt );
		auto pIt = CreateColumnarIterator ( m_pColumnar.get(), sphGetDocidName(), sError );
		assert ( pIt );
		for ( RowID_t tRowID = 0; tRowID<m_uRows; tRowID++ )
		{
			DocID_t tDocID = pIt->Get(tRowID);
			m_tDocIDtoRowID.Add ( tDocID, tRowID );

			if ( bUuidLinked )
			{
				const BYTE * pUuid = nullptr;
				int iUuidLen = pUuidIt->Get ( tRowID, pUuid );
				AddUuidDocid ( *this, pUuid, iUuidLen, tDocID );
			}
		}
	}
}

//////////////////////////////////////////////////////////////////////////

class RtDocWriter_c
{
	CSphTightVector<BYTE> &		m_dDocs;
	RowID_t						m_tLastRowID {INVALID_ROWID};

public:
	explicit RtDocWriter_c ( CSphTightVector<BYTE> & dDocs )
		: m_dDocs ( dDocs )
	{}

	inline void operator<< ( const RtDoc_t & tDoc)
	{
		m_dDocs.ReserveGap ( 5 + 5 * sizeof ( DWORD ) );
		ZipDword ( m_dDocs, tDoc.m_tRowID - std::exchange ( m_tLastRowID, tDoc.m_tRowID ) );
		ZipDword ( m_dDocs, tDoc.m_uDocFields );
		ZipDword ( m_dDocs, tDoc.m_uHits );
		if ( tDoc.m_uHits == 1 )
		{
			ZipDword ( m_dDocs, tDoc.m_uHit & 0xffffffUL );
			ZipDword ( m_dDocs, tDoc.m_uHit >> 24 );
		} else
			ZipDword ( m_dDocs, tDoc.m_uHit );
	}

	DWORD WriterPos () const
	{
		return m_dDocs.GetLength();
	}

	void ZipRestart ()
	{
		m_tLastRowID = INVALID_ROWID;
	}
};


RtDocReader_c::RtDocReader_c ( const RtSegment_t * pSeg, const RtWord_t & tWord )
{
	Init ( pSeg, tWord );
}

void RtDocReader_c::Init ( const RtSegment_t* pSeg, const RtWord_t& tWord )
{
	m_pDocs = ( pSeg->m_dDocs.begin() ? pSeg->m_dDocs.begin() + tWord.m_uDoc : nullptr );
	m_iLeft = tWord.m_uDocs;
	m_tDoc.m_tRowID = INVALID_ROWID;
}

bool RtDocReader_c::UnzipDoc ()
{
	if ( !m_iLeft || !m_pDocs )
		return false;
	const BYTE* pIn = m_pDocs;
	m_tDoc.m_tRowID += UnzipDword ( pIn );
	UnzipDword ( &m_tDoc.m_uDocFields, pIn );
	UnzipDword ( &m_tDoc.m_uHits, pIn );
	if ( m_tDoc.m_uHits == 1 )
	{
		auto a = UnzipDword ( pIn );
		auto b = UnzipDword ( pIn );
		m_tDoc.m_uHit = a + ( b << 24 );
	} else
		UnzipDword ( &m_tDoc.m_uHit, pIn );
	m_pDocs = pIn;
	--m_iLeft;
	return true;
}

class RtWordWriter_c
{
	CSphTightVector<BYTE> &				m_dWords;
	CSphVector<RtWordCheckpoint_t> &	m_dCheckpoints;
	CSphVector<BYTE> &					m_dKeywordCheckpoints;

	CSphKeywordDeltaWriter				m_tLastKeyword;
	CSphKeywordDeltaWriterV2			m_tLastKeywordV2;
	SphWordID_t							m_uLastWordID;
	DWORD								m_uLastDoc;
	int									m_iWords;

	DictFormat_e						m_eDictFormat;
	int									m_iWordsCheckpoint;
	const ESphHitless 					m_eHitlessMode = SPH_HITLESS_NONE;

public:
	RtWordWriter_c ( CSphTightVector<BYTE> & dWords, CSphVector<RtWordCheckpoint_t> & dCheckpoints,
				  CSphVector<BYTE> & dKeywordCheckpoints, DictFormat_e eDictFormat, int iWordsCheckpoint, ESphHitless eHitlessMode )
		: m_dWords ( dWords )
		, m_dCheckpoints ( dCheckpoints )
		, m_dKeywordCheckpoints ( dKeywordCheckpoints )
		, m_uLastWordID ( 0 )
		, m_uLastDoc ( 0 )
		, m_iWords ( 0 )
		, m_eDictFormat ( eDictFormat )
		, m_iWordsCheckpoint ( iWordsCheckpoint )
		, m_eHitlessMode ( eHitlessMode )
	{
		assert ( dWords.IsEmpty() );
		assert ( dCheckpoints.IsEmpty() );
		assert ( dKeywordCheckpoints.IsEmpty() );
	}

	void operator<< ( const RtWord_t& tWord ) // we never chain calls, so void as return type is ok.
	{
		if ( ++m_iWords==m_iWordsCheckpoint )
		{
			RtWordCheckpoint_t & tCheckpoint = m_dCheckpoints.Add();
			switch ( m_eDictFormat )
			{
			case DictFormat_e::CRC:
				tCheckpoint.m_uWordID = tWord.m_uWordID;
				break;

			case DictFormat_e::KEYWORDS:
				assert ( tWord.m_sWord && tWord.m_iWordLen>0 && tWord.m_iWordLen<SPH_MAX_KEYWORD_LEN );
				tCheckpoint.m_uWordID = sphPutBytes ( &m_dKeywordCheckpoints, tWord.m_sWord, tWord.m_iWordLen );
				m_dKeywordCheckpoints.Add ( '\0' ); // checkpoint is NULL terminating string
				m_tLastKeyword.Reset();
				break;

			case DictFormat_e::KEYWORDS_V2:
			{
				const BYTE * pKeyword = tWord.m_sWord;
				int iWordLen = tWord.m_iWordLen;
				assert ( pKeyword && iWordLen>0 );
				assert ( iWordLen<=GetKeywordMaxStoredBytes ( DictFormat_e::KEYWORDS_V2 ) );
				tCheckpoint.m_uWordID = sphPutBytes ( &m_dKeywordCheckpoints, pKeyword, iWordLen );
				m_dKeywordCheckpoints.Add ( '\0' );
				m_tLastKeywordV2.Reset();
			}
			break;

			}
			tCheckpoint.m_iOffset = m_dWords.GetLength();

			m_uLastWordID = 0;
			m_uLastDoc = 0;
			m_iWords = 1;
		}

		switch ( m_eDictFormat )
		{
		case DictFormat_e::CRC:
			ZipWordid ( m_dWords, tWord.m_uWordID - m_uLastWordID );
			break;

		case DictFormat_e::KEYWORDS:
			m_tLastKeyword.PutDelta ( *this, tWord.m_sWord, tWord.m_iWordLen );
			break;

		case DictFormat_e::KEYWORDS_V2:
			m_tLastKeywordV2.PutDelta ( *this, tWord.m_sWord, tWord.m_iWordLen );
			break;
		}

		m_dWords.ReserveGap ( 3+3*sizeof(DWORD) );

		DWORD uDocs = tWord.m_uDocs;
		if ( !tWord.m_bHasHitlist && m_eHitlessMode==SPH_HITLESS_SOME )
			uDocs |= HITLESS_DOC_FLAG;

		ZipDword ( m_dWords, uDocs );
		ZipDword ( m_dWords, tWord.m_uHits );
		ZipDword ( m_dWords, tWord.m_uDoc - m_uLastDoc );

		m_uLastWordID = tWord.m_uWordID;
		m_uLastDoc = tWord.m_uDoc;
	}

	void PutBytes ( const BYTE * pData, int iLen ) const
	{
		sphPutBytes ( &m_dWords, pData, iLen );
	}
};

RtWordReader_c::RtWordReader_c ( const RtSegment_t * pSeg, DictFormat_e eDictFormat, int iWordsCheckpoint, ESphHitless eHitlessMode )
	: m_dKeyword ( eDictFormat==DictFormat_e::KEYWORDS_V2 ? GetKeywordBufSize ( eDictFormat ) : 0 )
	, m_eDictFormat ( eDictFormat )
	, m_iWordsCheckpoint ( iWordsCheckpoint )
	, m_eHitlessMode ( eHitlessMode )
{
	m_tWord.m_uWordID = 0;
	Reset ( pSeg );
}

void RtWordReader_c::Reset ( const RtSegment_t * pSeg )
{
	m_pCur = pSeg->m_dWords.Begin();
	m_pMax = m_pCur + pSeg->m_dWords.GetLength();

	m_tWord.m_uDoc = 0;
	m_tWord.m_iWordLen = 0;
	m_iWords = 0;
}


const RtWord_t* RtWordReader_c::UnzipWord ()
{
	if (m_pCur >= m_pMax)
		return nullptr;

	if ( ++m_iWords==m_iWordsCheckpoint )
	{
		m_tWord.m_uDoc = 0;
		m_iWords = 1;
		++m_iCheckpoint;
		if ( m_eDictFormat==DictFormat_e::CRC )
			m_tWord.m_uWordID = 0;
	}

	const BYTE * pIn = m_pCur;
	if ( m_eDictFormat!=DictFormat_e::CRC )
	{
		// 1dddmmmm -> delta, match (packed into 1 byte)
		// 0ddddddd mmmmmmmm -> delta, match (occupy 2 bytes)
		// then delta bytes of symbols
		// overwrites previous token: delta symbols starting from match. At the end (delta+match) put 0-terminator
		int iMatch, iDelta;
		BYTE uPacked = *pIn++;
		if ( uPacked & 0x80 )
		{
			iDelta = ( ( uPacked>>4 ) & 7 ) + 1;
			iMatch = uPacked & 15;
		} else if ( m_eDictFormat==DictFormat_e::KEYWORDS_V2 && uPacked==0x7f )
		{
			iDelta = (int)UnzipIntBE ( pIn );
			iMatch = (int)UnzipIntBE ( pIn );
		} else
		{
			iDelta = uPacked & 127;
			iMatch = *pIn++;
		}
		int iLen = iMatch+iDelta;
		if ( m_eDictFormat==DictFormat_e::KEYWORDS_V2 )
		{
			assert ( iLen<=GetKeywordMaxStoredBytes ( m_eDictFormat ) );
			memcpy ( m_dKeyword.Begin()+iMatch, pIn, iDelta );
			m_dKeyword[iLen] = '\0';
			m_tWord.m_sWord = m_dKeyword.Begin();
			m_tWord.m_iWordLen = iLen;
		} else
		{
			assert ( iLen<SPH_MAX_KEYWORD_LEN );
			m_tPackedWord[0] = (BYTE)iLen;
			memcpy ( m_tPackedWord+1+iMatch, pIn, iDelta );
			m_tPackedWord[1+m_tPackedWord[0]] = 0;
			m_tWord.m_sWord = m_tPackedWord+1;
			m_tWord.m_iWordLen = iLen;
		}
		pIn += iDelta;
	} else
		m_tWord.m_uWordID += UnzipWordid ( pIn );

	UnzipDword ( &m_tWord.m_uDocs, pIn );
	UnzipDword ( &m_tWord.m_uHits, pIn );
	m_tWord.m_uDoc +=  UnzipDword ( pIn );
	m_tWord.m_bHasHitlist = ( m_eHitlessMode==SPH_HITLESS_NONE || ( m_eHitlessMode==SPH_HITLESS_SOME && !( m_tWord.m_uDocs & HITLESS_DOC_FLAG ) ) );
	m_tWord.m_uDocs = ( m_eHitlessMode==SPH_HITLESS_NONE ? m_tWord.m_uDocs : ( m_tWord.m_uDocs & HITLESS_DOC_MASK ) );
	m_pCur = pIn;
	return &m_tWord;
}


class RtHitWriter_c
{
	CSphTightVector<BYTE>& m_dHits;
	DWORD m_uLastHit = 0;

public:
	explicit RtHitWriter_c ( CSphTightVector<BYTE>& dHits )
		: m_dHits ( dHits )
	{}

	inline void operator<< ( DWORD uValue )
	{
		ZipDword ( m_dHits, uValue - std::exchange ( m_uLastHit, uValue ) );
	}

	void ZipRestart()
	{
		m_uLastHit = 0;
	}

	DWORD WriterPos() const
	{
		return m_dHits.GetLength();
	}
};

RtHitReader_c::RtHitReader_c ( const RtSegment_t& dSeg, const RtDoc_t& dDoc )
	: m_pCur ( dSeg.m_dHits.begin() + dDoc.m_uHit)
	, m_uLeft ( dDoc.m_uHits )
	, m_uValue ( EMPTY_HIT )
{}

void RtHitReader_c::Seek ( const RtSegment_t& dSeg, const RtDoc_t& dDoc )
{
	Seek ( dSeg.m_dHits.begin() + dDoc.m_uHit, dDoc.m_uHits );
}

void RtHitReader_c::Seek ( const BYTE* pHits, DWORD uHits )
{
	m_pCur = pHits;
	m_uLeft = uHits;
	m_uValue = EMPTY_HIT;
}

DWORD RtHitReader_c::UnzipHit ()
{
	if ( !m_pCur || !m_uLeft )
		return EMPTY_HIT;
	m_uValue += UnzipDword ( m_pCur );
	--m_uLeft;
	return m_uValue;
}


ByteBlob_t GetHitsBlob ( const RtSegment_t& tSeg, const RtDoc_t& tDoc )
{
	const BYTE* pHits = &tSeg.m_dHits[tDoc.m_uHit];
	const BYTE* pEnd = pHits;
	for ( auto i = 0U; i < tDoc.m_uHits; ++i )
		SkipZipped ( pEnd );
	return { pHits, pEnd - pHits };
}

//////////////////////////////////////////////////////////////////////////

/// forward ref
class RtIndex_c;

struct ChunkStats_t
{
	CSphSourceStats				m_Stats;
	CSphFixedVector<int64_t>	m_dFieldLens { SPH_MAX_FIELDS };

	void Init ( const CSphSourceStats & s, const CSphFixedVector<int64_t> & dLens )
	{
		assert ( m_dFieldLens.GetLength() == dLens.GetLength() );
		m_Stats = s;
		ARRAY_FOREACH ( i, dLens )
			m_dFieldLens[i] = dLens[i];
	}

	ChunkStats_t () = default;
	ChunkStats_t ( const CSphSourceStats & s, const CSphFixedVector<int64_t> & dLens )
	{
		Init ( s, dLens );
	}
};

struct RtAttrMergeContext_t
{
	int								m_iNumBlobs;
	RowID_t &						m_tResultRowID;
	columnar::Builder_i *			m_pColumnarBuilder;

	RtAttrMergeContext_t ( int iNumBlobs, RowID_t & tResultRowID, columnar::Builder_i * pColumnarBuilder )
		: m_iNumBlobs ( iNumBlobs )
		, m_tResultRowID ( tResultRowID )
		, m_pColumnarBuilder ( pColumnarBuilder )
	{}
};


struct RtQword_t;
struct SaveDiskDataContext_t;
struct SaveDiskDataTimings_t;

// kind of 'mini served_desc' inside index - manages state of one disk chunk
class DiskChunk_c final : public ISphRefcountedMT
{
public:
	mutable std::atomic<bool>			m_bOptimizing { false };	// to protect from simultaneous optimizing one and same chunk
	mutable bool						m_bFinallyUnlink = false;	// unlink index files on destroy
	mutable Threads::Coro::RWLock_c 	m_tLock;					// fine-grain lock between update and merge (optimize)
	mutable std::atomic<int>			m_iPendingUpdates {0};		// if right now some updates pending

	inline static CSphRefcountedPtr<const DiskChunk_c> make ( CSphIndex* pIndex ) { return CSphRefcountedPtr<const DiskChunk_c> { pIndex ? new DiskChunk_c(pIndex) : nullptr }; }
	inline static CSphRefcountedPtr<const DiskChunk_c> make ( std::unique_ptr<CSphIndex> pIndex ) { return CSphRefcountedPtr<const DiskChunk_c> { pIndex ? new DiskChunk_c(std::move(pIndex)) : nullptr }; }
	explicit operator CSphIndex* () const		{ return m_pIndex; }
	CSphIndex & Idx() 					{ return *m_pIndex; }
	CSphIndex & CastIdx () const		{ return *const_cast<CSphIndex *>(m_pIndex); } // const breakage!
	const CSphIndex & Cidx() const		{ return *m_pIndex; }

protected:
	~DiskChunk_c() final
	{
		if ( !m_pIndex )
			return;

		CSphString sDeleted = m_pIndex->GetFilebase ();
		SafeDelete( m_pIndex );
		if ( m_bFinallyUnlink )
			sphUnlinkIndex ( sDeleted.cstr (), true );
	}

private:
	CSphIndex *		m_pIndex;

	DiskChunk_c ( CSphIndex * pIndex ) : m_pIndex ( pIndex ) {}
	DiskChunk_c ( std::unique_ptr<CSphIndex> pIndex ) : m_pIndex ( pIndex.release() ) {}
};

using DiskChunkRefPtr_t = CSphRefcountedPtr<DiskChunk_c>;
using ConstDiskChunkRefPtr_t = CSphRefcountedPtr<const DiskChunk_c>;

using ConstRtSegmentSlice_t = VecTraits_T<ConstRtSegmentRefPtf_t>;
using DiskChunkSlice_t = VecTraits_T<DiskChunkRefPtr_t>;

template <typename CHUNK>
class RefCountedVec_T final : public ISphRefcountedMT, public LazyVector_T<CHUNK>
{
protected:
	~RefCountedVec_T () final = default;

public:
	RefCountedVec_T() = default;
	using BASE = LazyVector_T<CHUNK>;
};

using DiskChunkVec_c = RefCountedVec_T<ConstDiskChunkRefPtr_t>;
using RtSegVec_c = RefCountedVec_T<ConstRtSegmentRefPtf_t>;

using DiskChunkVecRefPtr_t		= CSphRefcountedPtr<DiskChunkVec_c>;
using RtSegVecRefPtr_t			= CSphRefcountedPtr<RtSegVec_c>;
using ConstDiskChunkVecRefPtr_t	= CSphRefcountedPtr<const DiskChunkVec_c>;
using ConstRtSegVecRefPtr_t		= CSphRefcountedPtr<const RtSegVec_c>;

template<typename CHUNK>
CSphRefcountedPtr<RefCountedVec_T<CHUNK>> SliceToVec ( const VecTraits_T<CHUNK> & dSegments )
{
	CSphRefcountedPtr<RefCountedVec_T<CHUNK>> dResult { new RefCountedVec_T<CHUNK> };
	dResult->Reserve ( dSegments.GetLength () );
	for ( const auto & dSeg : dSegments )
		dResult->Add ( dSeg );
	return dResult;
}

struct ConstRtData {
	ConstDiskChunkVecRefPtr_t m_pChunks;
	ConstRtSegVecRefPtr_t m_pSegs;
};
//using MutableRtData = std::pair<DiskChunkVecRefPtr_t, RtSegVecRefPtr_t>;
/*
class FiberPool_c
{
	struct HelperFiber_t
	{
		CSphString							m_sName;
		Threads::RoledSchedulerSharedPtr_t	m_tScheduler;
	};

	CSphFixedVector<HelperFiber_t>	m_dFibers;				// helpers to serialize update/change tasks
	std::atomic<DWORD>				m_iFiber;

public:

	explicit FiberPool_c (int iHelpers)
			: m_dFibers { iHelpers }
			, m_iFiber {0}
	{}

	Threads::RoledSchedulerSharedPtr_t GetFiber ()
	{
		auto iIdx = m_iFiber.fetch_add ( 1, std::memory_order_relaxed ) % m_dFibers.GetLength();
		auto& tSched = m_dFibers[iIdx].m_tScheduler;
		if ( !tSched )
		{
			if ( m_dFibers[iIdx].m_sName.IsEmpty () )
				m_dFibers[iIdx].m_sName.SetSprintf ( "FP_%d", iIdx );

			tSched = MakeAloneScheduler ( Coro::CurrentScheduler (), m_dFibers[iIdx].m_sName.cstr () );
		}
		return tSched;
	}
};
*/
// main dataset of RT index. Keeps disk chunks and RAM segments
// * provides read-only snapshots for searches
// * provides serialized access for changing chunks and segments
// * provides fiber workers for undependent processing
class RtData_c
{
	mutable RwLock_t			m_tLock;	// very short-term
	ConstDiskChunkVecRefPtr_t	m_pChunks GUARDED_BY ( m_tLock );
	ConstRtSegVecRefPtr_t		m_pSegments GUARDED_BY ( m_tLock );

	friend class RtWriter_c;

public:
	RtData_c ()
	{
		m_pChunks = new DiskChunkVec_c;
		m_pSegments = new RtSegVec_c;
	}

	~RtData_c () = default;

	ConstDiskChunkRefPtr_t DiskChunkByID ( int iChunkID ) const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		for ( auto& pChunk : *m_pChunks )
			if ( pChunk->Cidx().m_iChunk == iChunkID )
				return pChunk;
		return { nullptr };
	}

	ConstDiskChunkRefPtr_t DiskChunkByIdx ( int iChunk ) const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		if ( iChunk < 0 || iChunk >= m_pChunks->GetLength() )
			return { nullptr };
		return ( *m_pChunks )[iChunk];
	}

	ConstDiskChunkVecRefPtr_t DiskChunks () const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		return m_pChunks;
	}

	ConstRtSegVecRefPtr_t RamSegs () const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		return m_pSegments;
	}

	ConstRtData RtData () const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		return { m_pChunks, m_pSegments };
	}

	bool IsEmpty() const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		return m_pChunks->IsEmpty() && m_pSegments->IsEmpty();
	}

	int GetRamSegmentsCount() const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		return m_pSegments->GetLength();
	}

	int GetDiskChunksCount () const EXCLUDES ( m_tLock )
	{
		ScRL_t rLock ( m_tLock );
		return m_pChunks->GetLength ();
	}
};

// helper for easier access to ConstRtData members
// note: that is pointer to CONST vector of CONST chunks everywhere, keep this constage from casts!
struct RtGuard_t
{
	ConstRtData				m_tSegmentsAndChunks;
	const DiskChunkVec_c &	m_dDiskChunks;
	const RtSegVec_c &		m_dRamSegs;

	RtGuard_t ( RtGuard_t&& ) noexcept = default;
	explicit RtGuard_t ( ConstRtData tData )
			: m_tSegmentsAndChunks { std::move ( tData ) }
			, m_dDiskChunks { *m_tSegmentsAndChunks.m_pChunks }
			, m_dRamSegs { *m_tSegmentsAndChunks.m_pSegs }
	{}
};

CSphVector<int> GetChunkIds ( const VecTraits_T<DiskChunkRefPtr_t> & dChunks )
{
	CSphVector<int> dIds;
	dChunks.for_each ( [&dIds] ( const DiskChunkRefPtr_t & pIdx )
	{
		if ( !pIdx->m_bFinallyUnlink )
			dIds.Add ( pIdx->Idx ().m_iChunk );
	});
	return dIds;
}

// created with null set of ram segments and disk chunks
// on d-tr any not-null set will replace chunks and segments from the owner
// Note, if you want to modify existing set, you NEED to guard some way period between reading old / writing modified
class RtWriter_c
{
	RtSegVecRefPtr_t m_pOldRamSegs;
	DiskChunkVecRefPtr_t m_pOldDiskChunks;

	void CheckDiskConsistency() const noexcept
	{
		if ( !m_pNewDiskChunks )
			return;

		const auto pOwnerDiskChunks = m_tOwner.DiskChunks();
		const auto iOldLen = m_pOldDiskChunks->GetLength();
		const auto iCurLen = pOwnerDiskChunks->GetLength();
		if ( iOldLen != iCurLen )
			sphDie ("Amount of disk chunks changed from %d to %d during table modification. Table damaged", iOldLen, iCurLen);

		auto dOldChunks = GetChunkIds ( *m_pOldDiskChunks );
		auto dCurChunks = GetChunkIds ( *pOwnerDiskChunks );

		ARRAY_CONSTFOREACH( i, dOldChunks )
		{
			if ( dOldChunks[i] == dCurChunks[i] )
				continue;

			sphDie ("Disk chunks changed from [%s] to [%s] during table modification. Table damaged", Vec2Str( dOldChunks ).cstr(), Vec2Str( dCurChunks ).cstr());
		}
	}


	void CheckRAMConsistency() const noexcept
	{
		if ( !m_pNewRamSegs )
			return;

		const auto pOwnerRamSegs = m_tOwner.RamSegs();
		const auto iOldLen = m_pOldRamSegs->GetLength();
		const auto iCurLen = pOwnerRamSegs->GetLength();
		if ( iOldLen != iCurLen )
			sphDie ( "Amount of ram segments changed from %d to %d during table modification. Table damaged", iOldLen, iCurLen );


		ARRAY_CONSTFOREACH( i, (*m_pOldRamSegs) )
		{
			if ( (*m_pOldRamSegs)[i] == (*pOwnerRamSegs)[i] )
				continue;

			sphDie ( "RAM segments changed during table modification. Table damaged" );
		}
	}

public:
	RtData_c&					m_tOwner;
	DiskChunkVecRefPtr_t		m_pNewDiskChunks;
	RtSegVecRefPtr_t			m_pNewRamSegs;
	Handler						m_fnOnRamSegsChanged;

	RtWriter_c ( RtWriter_c&& rhs ) noexcept = default;
	RtWriter_c ( RtData_c & tOwner, Handler&& fnOnRamSegsChanged )
		: m_pOldRamSegs{ new RtSegVec_c }
		, m_pOldDiskChunks{ new DiskChunkVec_c }
		, m_tOwner( tOwner )
		, m_fnOnRamSegsChanged { std::move ( fnOnRamSegsChanged ) }
	{
		for ( const auto & pSeg: *m_tOwner.RamSegs() )
			m_pOldRamSegs->Add( pSeg );
		for ( const auto & pChunk: *m_tOwner.DiskChunks() )
			m_pOldDiskChunks->Add( pChunk );
	}


	~RtWriter_c() EXCLUDES ( m_tOwner.m_tLock )
	{
		CheckDiskConsistency();
		CheckRAMConsistency();

		if ( !m_pNewDiskChunks && !m_pNewRamSegs )
			return;

		{
			ScWL_t wLock ( m_tOwner.m_tLock );
			// use leak since we convert 'data*' to 'const data*' here.
			if ( m_pNewDiskChunks )
				m_tOwner.m_pChunks = m_pNewDiskChunks.Leak();

			if ( !m_pNewRamSegs )
				return;

			m_tOwner.m_pSegments = m_pNewRamSegs.Leak();
		}
		m_fnOnRamSegsChanged();
	}
	enum Copy_e { copy };
	enum Empty_e { empty };

	void InitRamSegs ( Empty_e ) { m_pNewRamSegs = new RtSegVec_c (); }
	void InitDiskChunks ( Empty_e ) { m_pNewDiskChunks = new DiskChunkVec_c (); }

	void InitRamSegs ( Copy_e ) EXCLUDES ( m_tOwner.m_tLock )
	{
		InitRamSegs ( empty );
		for ( const auto & pSeg : *m_tOwner.RamSegs() )
			m_pNewRamSegs->Add ( pSeg );
	}

	void InitDiskChunks ( Copy_e ) EXCLUDES ( m_tOwner.m_tLock )
	{
		InitDiskChunks ( empty );
		auto pChunks = m_tOwner.DiskChunks();
		for ( const auto & pChunk : *pChunks )
			m_pNewDiskChunks->Add ( pChunk );
	}

};

class ChunkID_c
{
	int m_iCh = -1;

public:
	int MakeChunkId ( const RtData_c& tData )
	{
		if ( m_iCh < 0 )
			tData.DiskChunks()->for_each ( [this] ( auto& pIdx ) { m_iCh = Max ( m_iCh, pIdx->Cidx().m_iChunk ); } );
		++m_iCh;
		return m_iCh;
	}

	int GetChunkId ( const RtData_c & tData ) const
	{
		int iChunkId = 0;
		for ( const auto & tChunk : *tData.DiskChunks() )
		{
			iChunkId = Max ( iChunkId, tChunk->Cidx().m_iChunk );
		};

		return iChunkId;
	}
};

class WorkerSchedulers_c
{
	RoledSchedulerSharedPtr_t	m_tSerialChunkAccess;	  // serialize changing chunks and segs vec
	RoledSchedulerSharedPtr_t	m_tChunkSaver;  // scheduler for disk manipulations.
	std::atomic<int>			m_iNextOp { 1 };

public:
	void InitWorkers()
	{
		if ( !m_tSerialChunkAccess )
		{
			m_tSerialChunkAccess = MakeAloneScheduler ( GlobalWorkPool(), "serial" );

#ifdef PERFETTO
			// set name for tracing
			auto tTrack = perfetto::Track::FromPointer ( &m_tSerialChunkAccess );
			auto tDesc = tTrack.Serialize();
			tDesc.set_name( SphSprintf ( "serial_%p", &m_tSerialChunkAccess ).cstr() );
			perfetto::TrackEvent::SetTrackDescriptor ( tTrack, tDesc );
#endif
		}

		if ( !m_tChunkSaver )
		{
			m_tChunkSaver = WrapRawScheduler ( GlobalWorkPool(), "saver" );

#ifdef PERFETTO
			// set name for tracing
			auto tTrack = perfetto::Track::FromPointer ( &m_tChunkSaver );
			auto tDesc = tTrack.Serialize();
			tDesc.set_name ( SphSprintf ( "saver_%p", &m_tChunkSaver ).cstr() );
			perfetto::TrackEvent::SetTrackDescriptor ( tTrack, tDesc );
#endif
		}
	}

#ifdef PERFETTO
	~WorkerSchedulers_c()
	{
		if ( m_tSerialChunkAccess )
			perfetto::TrackEvent::EraseTrackDescriptor ( perfetto::Track::FromPointer ( &m_tSerialChunkAccess ) );

		if ( m_tChunkSaver )
			perfetto::TrackEvent::EraseTrackDescriptor ( perfetto::Track::FromPointer ( &m_tChunkSaver ) );
	}
#endif

	Threads::SchedRole SerialChunkAccess() const RETURN_CAPABILITY ( m_tSerialChunkAccess )
	{
		return m_tSerialChunkAccess;
	}

	Threads::SchedRole SaveSegmentsWorker() const RETURN_CAPABILITY ( m_tChunkSaver )
	{
		return m_tChunkSaver;
	}

	inline int GetNextOpTicket()
	{
		auto iRes = m_iNextOp.fetch_add ( 1, std::memory_order_relaxed );
		if ( !iRes ) // zero tag has special meaning, skip it
			iRes = m_iNextOp.fetch_add ( 1, std::memory_order_relaxed );
		return iRes;
	}
};

class SaveState_c
{
public:
	enum States_e : BYTE {
		ENABLED,	// normal, saving possible
		DISCARD,	// disabled, current result will not be necessary (can escape to don't waste resources)
		DISABLED,	// disabled, current stage must be completed first
	};

	void SetState ( States_e eState )
	{
		if ( !Threads::IsInsideCoroutine() )
		{ // call from naked worker, typically indextool
			auto& t = const_cast<Value_t&> ( m_tValue.GetValueRef() );
			t.m_eValue = eState;
			return;
		}

		if ( eState!=States_e::ENABLED )
			WaitUnlockedSetState ();

		assert ( Threads::IsInsideCoroutine() );
		m_tValue.ModifyValueAndNotifyAll ( [eState] ( Value_t& t )
		{
			t.m_eValue = eState;
			if ( eState == States_e::ENABLED )
			{
				if ( t.m_iDisabledCounter > 0 )
					--t.m_iDisabledCounter;
			} else
				++t.m_iDisabledCounter;
		});
	}
	void SetShutdownFlag ()
	{
		m_tValue.ModifyValueAndNotifyAll ( [] ( Value_t& t ) { t.m_bShutdown = true; } );
	}

	bool ActiveStateIs ( States_e eValue ) const
	{
		if ( !Threads::IsInsideCoroutine() ) // call from naked worker, typically indextool
			return m_tValue.GetValueRef().m_eValue == eValue;

		if ( eValue == States_e::ENABLED )
			return m_tValue.GetValueRef().m_iDisabledCounter == 0;
		return m_tValue.GetValueRef().m_eValue == eValue;
	}

	// sleep and return true when state is enabled.
	// sleep and return false if index's shutdown happened.
	bool WaitEnabledOrShutdown () const noexcept
	{
		while (true) {
			auto tVal = m_tValue.WaitForMs ( [&] ( const Value_t& tVal ) {
				if ( tVal.m_bShutdown )
					return true;
				if ( tVal.m_eValue!=States_e::ENABLED )
					return false;
				return tVal.m_iDisabledCounter==0;
			}, 10000 ); // time doesn't matter, as shutdown abandons all timers
			if ( tVal.m_bShutdown || sphInterrupted() || tVal.m_iDisabledCounter==0 )
				return !tVal.m_bShutdown;
		}
	}

	bool IsInterrupted() const noexcept
	{
		return m_tValue.GetValue().m_bShutdown || sphInterrupted();
	}

	int GetNumOfLocks() const noexcept
	{
		return m_tValue.GetValueRef().m_iDisabledCounter;
	}

	// need to be run from serialized executor in order to avoid race between wait and modify
	void WaitLockEnabledState() noexcept
	{
		if ( !Threads::IsInsideCoroutine () )
			return;

		if ( WaitEnabledOrShutdown () )
			m_iSetWaiter.ModifyValue ( [] ( int & iVal ) { ++iVal; } );
	}

	void UnlockEnabledState () noexcept
	{
		if ( !Threads::IsInsideCoroutine () )
			return;
		m_iSetWaiter.ModifyValueAndNotifyAll ( [] ( int & iVal ) { --iVal; } );
	}

private:
	void WaitUnlockedSetState ()
	{
		if ( !Threads::IsInsideCoroutine () )
			return;
		m_iSetWaiter.Wait ( [] ( int i ) { return i<=0; } );
	}

private:
	struct Value_t
	{
		States_e m_eValue = SaveState_c::ENABLED;
		int m_iDisabledCounter = 0;
		bool m_bShutdown = false;
	};
	Coro::Waitable_T<Value_t> m_tValue;
	Coro::Waitable_T<int> m_iSetWaiter {0};
};

enum class MergeSeg_e : BYTE
{
	NONE	= 0,  	// idle
	KILLED	= 1,  	// kill happened
	NEWSEG	= 2, 	// insertion happened
	EXIT 	= 4,	// shutdown and exit
};

using AlterOp_fn = std::function < bool( CSphIndex & , CSphString & ) >;

struct ExternalFilesCacheEntry_t
{
	uint64_t	m_uHash = 0;
	CSphString	m_sStopwords;
	CSphString	m_sWordforms;
	CSphString	m_sExceptions;
	CSphString	m_sHitless;
	CSphString	m_sJieba;
};

struct RtMergeTimings_t;

using ExtFiles_h = OpenHashTable_T<uint64_t, ExternalFilesCacheEntry_t, IdentityHash_fn>;

class RtIndex_c final : public RtIndex_i, public ISphNoncopyable, public ISphWordlist, public ISphWordlistSuggest, public IndexAlterHelper_c
{
public:
						RtIndex_c ( CSphString sIndexName, CSphString sPath, CSphSchema tSchema, int64_t iRamSize );
						~RtIndex_c () final;

	bool				AddDocument ( InsertDocData_c & tDoc, bool bReplace, const CSphString & sTokenFilterOptions, CSphString & sError, CSphString & sWarning, RtAccum_t * pAccExt ) override;
	virtual bool		AddDocument ( ISphHits * pHits, const InsertDocData_c & tDoc, bool bReplace, const DocstoreBuilder_i::Doc_t * pStoredDoc, CSphString & sError, CSphString & sWarning, RtAccum_t * pAccExt );
	bool				DeleteDocument ( const VecTraits_T<DocID_t> & dDocs, CSphString & sError, RtAccum_t * pAccExt ) final;
	bool				Commit ( int * pDeleted, RtAccum_t * pAccExt, CSphString* pError = nullptr ) final;
	bool				PreCommit ( RtAccum_t * pAccExt, CSphString & sError ) final;
	void				RollBack ( RtAccum_t * pAccExt ) final;
	bool				CommitReplayable ( RtSegment_t * pNewSeg, const VecTraits_T<DocID_t> & dAccKlist, int64_t iAddTotalBytes, int & iTotalKilled, CSphString & sError );
	void				ForceRamFlush ( const char * szReason ) final;
	bool				IsFlushNeed() const final;
	bool				ForceDiskChunk() final;
	bool				AttachDiskIndex ( CSphIndex * pIndex, bool bTruncate, bool & bFatal, CSphString & sError ) final;
	bool				AttachRtIndex ( AttachArgs_t & tArgs, CSphString & sError ) final;
	bool				Truncate ( CSphString & sError, Truncate_e eAction ) final;
	bool				CheckValidateOptimizeParams ( OptimizeTask_t& tTask ) const;
	bool				CheckValidateChunk ( int& iChunk, int iChunks, bool bByOrder ) const;
	bool				StartOptimize ( OptimizeTask_t tTask ) final;
	int					OptimizesRunning() const noexcept final;
	int					GetNumOfLocks() const noexcept final;
	void				Optimize ( OptimizeTask_t tTask ) final;
	void				CheckStartAutoOptimize ();
	int					ClassicOptimize ();
	int					ProgressiveOptimize ( int iCutoff );
	int					CommonOptimize ( OptimizeTask_t tTask );
	void				DropDiskChunk ( int iChunk, int* pAffected=nullptr );
	void				DropDiskChunks ( IntVec_t& dChunks, int* pAffected=nullptr ) REQUIRES ( m_tWorkers.SerialChunkAccess() );
	bool				CompressOneChunk ( int iChunk, int* pAffected = nullptr );
	bool				DedupOneChunk ( int iChunk, int* pAffected = nullptr );
	bool				MergeTwoChunks ( int iA, int iB, int* pAffected, StringBuilder_c* sLog = nullptr );
	bool				MergeNChunks ( const char * szParentAction, VecTraits_T<ConstDiskChunkRefPtr_t> dChunks, int* pAffected, StringBuilder_c* sLog, RtMergeTimings_t & tTimings );
	bool				MergeCanRun () const;
	bool				SplitOneChunk ( int iChunkID, const char* szUvarFilter, int* pAffected = nullptr );
	bool				SplitOneChunkFast ( int iChunkID, const char * szUvarFilter, bool& bResult, int* pAffected = nullptr );
	int					ChunkIDByChunkIdx (int iChunkIdx) const;

	int64_t				GetCountDistinct ( const CSphString & sAttr, CSphString & sModifiedAttr ) const override;
	int64_t				GetCountFilter ( const CSphFilterSettings & tFilter, CSphString & sModifiedAttr ) const override;
	int64_t				GetCount() const override;
	std::pair<int64_t,int> GetPseudoShardingMetric ( const VecTraits_T<const CSphQuery> & dQueries, const VecTraits_T<int64_t> & dMaxCountDistinct, int iThreads, bool & bForceSingleThread ) const override;

	// helpers
	ConstDiskChunkRefPtr_t	ChangeDiskChunk (  const char* szParentAction, const ConstDiskChunkRefPtr_t& pChunk, CSphIndexProgress& tProgress, VecTraits_T<CSphFilterSettings> dFilters );
	ConstDiskChunkRefPtr_t	MergeDiskChunksN ( const char * szParentAction, VecTraits_T<ConstDiskChunkRefPtr_t> dChunks, CSphIndexProgress & tProgress, RtMergeTimings_t & tTimings );
	ConstDiskChunkRefPtr_t MergeChunkPrealloc ( const CSphIndex & tChunkA, const char * szParentAction );

	bool				PublishMergedChunks ( const char * szParentAction,std::function<bool ( int, DiskChunkVec_c & )> && fnPusher) REQUIRES ( m_tWorkers.SerialChunkAccess() );
	bool 				RenameOptimizedChunk ( const ConstDiskChunkRefPtr_t& pChunk, const char * szParentAction );
	bool				SkipOrDrop ( int iChunk, const CSphIndex& dChunk, bool bCheckAlive, int* pAffected = nullptr );
	void				ProcessDiskChunk ( int iChunk, VisitChunk_fn&& fnVisitor ) const final;
	void				ProcessDiskChunkEx ( int iChunk, VisitChunkEx_fn&& fnVisitor ) const final;
	template <typename VISITOR>
	void				ProcessDiskChunkByID ( int iChunkID, VISITOR&& fnVisitor ) const;
	template <typename VISITOR>
	void				ProcessDiskChunkByID ( VecTraits_T<int> dChunkIDs, VISITOR&& fnVisitor ) const;
	TokenizerRefPtr_c	CloneIndexingTokenizer() const final { return m_pTokenizerIndexing->Clone ( SPH_CLONE_INDEX ); }

	void				SetKillHookFor ( IndexSegment_c* pAccum, int iDiskChunkID ) const;
	void				SetKillHookFor ( IndexSegment_c* pAccum, VecTraits_T<int> dDiskChunkIDs ) const;

	Binlog::CheckTnxResult_t ReplayTxn ( CSphReader& tReader, CSphString & sError, BYTE uOp, Binlog::CheckTxn_fn&& fnCanContinue ) override; // cb from binlog
	Binlog::CheckTnxResult_t ReplayCommit ( CSphReader & tReader, CSphString & sError, Binlog::CheckTxn_fn && fnCanContinue );

public:
#if _WIN32
#pragma warning(push,1)
#pragma warning(disable:4100)
#endif

	int					Kill ( DocID_t tDocID ) final;
	int					KillMulti ( const VecTraits_T<DocID_t> & dKlist ) final;
	bool				IsAlive ( DocID_t tDocID ) const final;

	bool				Prealloc ( bool bStripPath, FilenameBuilder_i * pFilenameBuilder, StrVec_t & dWarnings ) final;
	void				Preread () final;
	void				PostSetup() final;
	bool				IsRT() const final { return true; }

	int					CheckThenUpdateAttributes ( AttrUpdateInc_t & tUpd, bool & bCritical, CSphString & sError, CSphString & sWarning ) final;
	bool				SaveAttributes ( CSphString & sError ) const final;
	DWORD				GetAttributeStatus () const final { return m_uDiskAttrStatus; }

	bool				AddRemoveAttribute ( bool bAdd, const AttrAddRemoveCtx_t & tCtx, CSphString & sError ) final;
	bool				AddRemoveField ( bool bAdd, const CSphString & sFieldName, DWORD uFieldFlags, CSphString & sError ) final;

	int					DebugCheck ( DebugCheckError_i & , FilenameBuilder_i * pFilenameBuilder ) final;
#if _WIN32
#pragma warning(pop)
#endif

	bool				EarlyReject ( CSphQueryContext * pCtx, CSphMatch & ) const final;
	const CSphSourceStats &		GetStats () const final { return m_tStats; }
	int64_t *			GetFieldLens() const final { return m_tSettings.m_bIndexFieldLens ? m_dFieldLens.Begin() : nullptr; }
	void				GetStatus ( CSphIndexStatus* ) const final;

	bool				MultiQuery ( CSphQueryResult& tResult, const CSphQuery& tQuery, const VecTraits_T<ISphMatchSorter*>& dAllSorters, const CSphMultiQueryArgs& tArgs ) const final;
	bool				DoGetKeywords ( CSphVector<CSphKeywordInfo>& dKeywords, const char* sQuery, const GetKeywordsSettings_t& tSettings, bool bFillOnly, CSphString* pError, const RtGuard_t& tGuard ) const;
	bool				GetKeywords ( CSphVector<CSphKeywordInfo>& dKeywords, const char* sQuery, const GetKeywordsSettings_t& tSettings, CSphString* pError ) const final;
	bool				FillKeywords ( CSphVector <CSphKeywordInfo> & dKeywords ) const final;
	void				AddKeywordStats ( BYTE* sWord, const BYTE* sTokenized, const DictRefPtr_c& pDict, bool bGetStats, int iQpos, RtQword_t* pQueryWord, CSphVector<CSphKeywordInfo>& dKeywords, const RtSegVec_c& dRamSegs ) const;

	bool				RtQwordSetup ( RtQword_t * pQword, int iSeg, const RtGuard_t& tGuard ) const;
	bool				RtQwordSetupSegment ( RtQword_t* pQword, const RtSegment_t* pCurSeg, bool bSetup ) const;

	bool				IsWordDict () const { return GetDictFormat()!=DictFormat_e::CRC; }
	DictFormat_e		GetDictFormat () const { assert ( m_pDict ); return m_pDict->GetSettings().GetDictFormat(); }
	int					GetWordCheckoint() const { return m_iWordsCheckpoint; }
	int					GetMaxCodepointLength() const { return m_iMaxCodepointLength; }

	bool				IsSameSettings ( CSphReconfigureSettings & tSettings, CSphReconfigureSetup & tSetup, StrVec_t & dWarnings, CSphString & sError ) const final;
	bool				Reconfigure ( CSphReconfigureSetup & tSetup ) final;
	int64_t				GetLastFlushTimestamp() const final;
	void				ProhibitSave() final;
	void				EnableSave() final;
	void				LockFileState ( CSphVector<CSphString> & dFiles ) final;

	void				WaitLockEnabledState () noexcept final;
	void				UnlockEnabledState () noexcept final;

	void				SetDebugCheck ( bool bCheckIdDups, int iCheckChunk ) final;

	void				CreateReader ( int64_t iSessionId ) const final;
	bool				GetDoc ( DocstoreDoc_t & tDoc, DocID_t tDocID, const VecTraits_T<int> * pFieldIds, int64_t iSessionId, bool bPack ) const final;
	int					GetFieldId ( const CSphString & sName, DocstoreDataType_e eType ) const final;
	Bson_t				ExplainQuery ( const CSphString & sQuery ) const final;
	uint64_t			GetSchemaHash () const final { return m_uSchemaHash; }

protected:
	CSphSourceStats		m_tStats;
	bool				m_bDebugCheck = false;
	bool				m_bCheckIdDups = false;
	int					m_iCheckChunk = -1;
	CSphFixedVector<int>	m_dChunkNames { 0 };

private:
	static const DWORD			META_HEADER_MAGIC	= 0x54525053;	///< my magic 'SPRT' header
	// NOTICE! meta version 21 was introduced in 2a6ea8f7 and rolled back to 20 in e1709760.
	// v22: keywords_v2 dictionary layout versioning.
	// v23: UUID primary ID linked-schema semantics.
	static constexpr DWORD		META_VERSION		= 23; // next should be 24
	// Since v20, RT meta is JSON. indextool.cpp's separate v18 constant only handles legacy binary meta.

	int							m_iStride;
	uint64_t					m_uSchemaHash = 0;
	UuidDocidRegistryPtr_t		m_pUuidRegistry;
	std::atomic<int64_t>		m_iRamChunksAllocatedRAM { 0 };

	std::atomic<bool>			m_bOptimizeStop { false };
	std::atomic<int>			m_iManualOptimizeCutoff { 0 };
	Coro::Waitable_T<int>		m_tOptimizeRuns {0};
	friend class OptimizeGuard_c;

	int64_t						m_iRtMemLimit;
	int64_t						m_iSoftRamLimit;
	double						m_fSaveRateLimit { INITIAL_SAVE_RATE_LIMIT };
	bool						m_bPathStripped = false;
	int							m_iLockFD = -1;

	ChunkID_c					m_tChunkID;
	RtData_c					m_tRtChunks;				// that is main set of disk chunks and RAM segments
	WorkerSchedulers_c			m_tWorkers;
	Coro::Waitable_T<int>		m_tUnLockedSegments { 0 }; // how many segments are not participating in any locked ops (like merge, save to disk).
	Coro::Waitable_T<MergeSeg_e> m_eSegMergeQueued { MergeSeg_e::NEWSEG };
	Coro::Waitable_T<CSphVector<int64_t>> m_tSaveTIDS { 0 }; // save operations performing now, and their TIDs
	int							m_iSavedGeneration = 0;		// SaveDiskChunk() increases generation on finish
	Coro::Waitable_T<int>		m_tNSavesNow { 0 };			// N of merge segment routines running right now
	CSphVector<int>				m_dSavingTickets GUARDED_BY ( m_tWorkers.SerialChunkAccess() );			// segments which are currently in saving to disk chunk(s)
	mutable MiniTimer_c			m_dSavingTimer;

	Coro::Waitable_T<int>		m_tBackgroundRoutines { 0 };

	int64_t						m_iSavedTID = 0;
	int64_t						m_tmSaved;
	mutable DWORD				m_uDiskAttrStatus = 0;
	std::atomic<int64_t>		m_tmDataWriten { 0 };
	std::atomic<int64_t>		m_tmDataSearched { 0 };

	int							m_iWordsCheckpoint = RTDICT_CHECKPOINT_V5;
	int							m_iMaxCodepointLength = 0;
	TokenizerRefPtr_c			m_pTokenizerIndexing;
	bool						m_bPreallocPassedOk = true;
	SaveState_c					m_tSaving;
	bool						m_bHasFiles = false;

	// fixme! make this *Lens atomic together with disk/ram data, to avoid any kind of race among them
	CSphFixedVector<int64_t>	m_dFieldLens { SPH_MAX_FIELDS };	///< total field lengths over entire index
	CSphFixedVector<int64_t>	m_dFieldLensRam { SPH_MAX_FIELDS };	///< field lengths summed over current RAM chunk
	CSphFixedVector<int64_t>	m_dFieldLensDisk { SPH_MAX_FIELDS };	///< field lengths summed over all disk chunks
	CSphVector<SphWordID_t>		m_dHitlessWords;

	std::unique_ptr<DocstoreFields_i> m_pDocstoreFields;	// rt index doesn't have its own docstore, but it must keep all fields to get their ids for GetDoc
	mutable int					m_iTrackFailedRamActions;
	int							m_iAlterGeneration = 0;		// increased every time index altered

	std::unique_ptr<TableEmbeddings_c> m_pEmbeddings;
	CSphVector<AttrWithModel_t> m_dAttrsWithModels;

	bool						BindAccum ( RtAccum_t * pAccExt, CSphString* pError = nullptr ) final;

	int							CompareWords ( const RtWord_t * pWord1, const RtWord_t * pWord2, bool bWordDict ) const;

	void						StartRoutine();
	void						StopRoutine();
	void						WaitAllRoutinesFinished() const noexcept;

	CSphFixedVector<RowID_t>	CopyAttributesFromAliveDocs ( RtSegment_t& tDstSeg, const RtSegment_t & tSrcSeg, RtAttrMergeContext_t & tCtx ) const REQUIRES ( tDstSeg.m_tLock ) REQUIRES (m_tWorkers.SerialChunkAccess());
	void						MergeKeywords ( RtSegment_t & tSeg, const RtSegment_t & tSeg1, const RtSegment_t & tSeg2, const VecTraits_T<RowID_t> & dRowMap1, const VecTraits_T<RowID_t> & dRowMap2 ) const;
	RtSegment_t *				MergeTwoSegments ( const RtSegment_t * pA, const RtSegment_t * pB ) const REQUIRES (m_tWorkers.SerialChunkAccess());
	static void					CopyWord ( RtSegment_t& tDstSeg, RtWord_t& tDstWord, RtDocWriter_c& tDstDoc, const RtSegment_t& tSrcSeg, const RtWord_t* pSrcWord, const VecTraits_T<RowID_t>& dRowMap );

	void						DeleteFieldFromDict ( RtSegment_t * pSeg, int iKillField );
	void						AddFieldToRamchunk ( const CSphString & sFieldName, DWORD uFieldFlags, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema );
	void						RemoveFieldFromRamchunk ( const CSphString & sFieldName, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema );
	void						AddRemoveFromRamDocstore ( const CSphSchema & tOldSchema, const CSphSchema & tNewSchema );

	bool						LoadMeta ( FilenameBuilder_i * pFilenameBuilder, bool bStripPath, DWORD & uVersion, bool & bRebuildInfixes, StrVec_t & dWarnings );
	bool						LoadMetaImpl ( FilenameBuilder_i * pFilenameBuilder, bool bStripPath, DWORD & uVersion, bool & bRebuildInfixes, StrVec_t & dWarnings );

	enum class LOAD_E { ParseError_e, GeneralError_e, Ok_e };
	LOAD_E						LoadMetaJson ( FilenameBuilder_i * pFilenameBuilder, bool bStripPath, DWORD & uVersion, bool & bRebuildInfixes, StrVec_t & dWarnings );
	LOAD_E						LoadMetaLegacy ( FilenameBuilder_i * pFilenameBuilder, bool bStripPath, DWORD & uVersion, bool & bRebuildInfixes, StrVec_t & dWarnings );
	bool						PreallocDiskChunks ( FilenameBuilder_i * pFilenameBuilder, StrVec_t & dWarnings );
	void						SaveMeta ( int64_t iTID, VecTraits_T<int> dChunkNames );
	void						SaveMeta ();
	bool						SaveDiskHeader ( SaveDiskDataContext_t & tCtx, const ChunkStats_t & tStats, CSphString & sError ) const;
	bool						SaveDiskData ( const char * szFilename, const ConstRtSegmentSlice_t & tSegs, const ChunkStats_t & tStats, CSphString & sError, SaveDiskDataTimings_t * pTimings = nullptr ) const;
	bool						SaveDiskChunk ( bool bForced, bool bEmergent=false ) REQUIRES ( m_tWorkers.SerialChunkAccess() );
	void						ConditionalDiskChunk ();

	std::unique_ptr<CSphIndex>	PreallocDiskChunk ( const CSphString& sChunk, int iChunk, FilenameBuilder_i * pFilenameBuilder, StrVec_t & dWarnings, CSphString & sError, const char * szName=nullptr ) const;
	bool						LoadRamChunk ( DWORD uVersion, bool bRebuildInfixes, bool bFixup = true );
	bool						SaveRamChunk ();

	bool						WriteAttributes ( SaveDiskDataContext_t & tCtx, CSphString & sError ) const;
	bool						WriteDocs ( SaveDiskDataContext_t & tCtx, CSphWriter & tWriterDict, CSphString & sError ) const;
	void						WriteCheckpoints ( SaveDiskDataContext_t & tCtx, CSphWriter & tWriterDict, SaveDiskDataTimings_t * pTimings = nullptr ) const;
	static bool					WriteDeadRowMap ( SaveDiskDataContext_t & tCtx, CSphString & sError );
	bool						StoreKNNParallel ( SaveDiskDataContext_t & tCtx, knn::Builder_i & tBuilder, const VecTraits_T<PlainOrColumnar_t> & dAttrsForKNN, int iStride, CSphString & sError ) const;

	void						GetPrefixedWords ( const char * sSubstring, int iSubLen, const char * sWildcard, Args_t & tArgs ) const final;
	void						GetInfixedWords ( const char * sSubstring, int iSubLen, const char * sWildcard, Args_t & tArgs ) const final;
	void						ScanRegexWords ( const VecTraits_T<RegexTerm_t> & dTerms, const ISphWordlist::Args_t & tArgs, const VecExpandConv_t & dConverters ) const final;
	void						GetSuggest ( const SuggestArgs_t & tArgs, SuggestResult_t & tRes ) const final;

	void						SuffixGetChekpoints ( const SuggestResult_t & tRes, const char * sSuffix, int iLen, CSphVector<DWORD> & dCheckpoints ) const final;
	void						SetCheckpoint ( SuggestResult_t & tRes, DWORD iCP ) const final;
	bool						ReadNextWord ( SuggestResult_t & tRes, DictWord_t & tWord ) const final;

	ConstRtSegmentRefPtf_t		AdoptSegment ( RtSegment_t * pNewSeg );
	int							ApplyKillList ( const VecTraits_T<DocID_t> & dAccKlist ) REQUIRES ( m_tWorkers.SerialChunkAccess() );

	bool						AddRemoveColumnarAttr ( RtGuard_t & tGuard, bool bAdd, const CSphString & sAttrName, ESphAttr eAttrType, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema, CSphString & sError );
	void						AddRemoveRowwiseAttr ( RtGuard_t & tGuard, bool bAdd, const CSphString & sAttrName, ESphAttr eAttrType, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema, CSphString & sError );

	bool						Update_DiskChunks ( AttrUpdateInc_t & tUpd, const DiskChunkSlice_t & dDiskChunks, CSphString & sError, CSphString & sWarning ) REQUIRES ( m_tWorkers.SerialChunkAccess() );

	StrVec_t					CollectOwnedExternalFiles () const REQUIRES ( m_tWorkers.SerialChunkAccess() );
	void						RemoveOutdatedOwnedExternalFiles ( StrVec_t & dOldFiles ) const REQUIRES ( m_tWorkers.SerialChunkAccess() );
	void						GetIndexFiles ( StrVec_t& dFiles, StrVec_t& dExt, const FilenameBuilder_i* = nullptr ) const override;
	DocstoreBuilder_i::Doc_t *	FetchDocFields ( DocstoreBuilder_i::Doc_t & tStoredDoc, const InsertDocData_c & tDoc, CSphSource_StringVector & tSrc, CSphVector<CSphVector<BYTE>> & dTmpAttrStorage ) const;

	void						UnlinkRAMChunk ( const char * szInfo=nullptr );
	void						WaitRAMSegmentsUnlocked ( bool bAllowOne = false ) const REQUIRES ( m_tWorkers.SerialChunkAccess() );
	bool						MergeSegmentsStep( MergeSeg_e eVal ) REQUIRES ( m_tWorkers.SerialChunkAccess() );
	void						RunMergeSegmentsWorker();
	void						StartMergeSegments ( MergeSeg_e eMergeWhat, bool bNotify=true ) REQUIRES ( m_tWorkers.SerialChunkAccess() );
	void						StopMergeSegmentsWorker();
	bool						NeedStoreWordID () const override;
	int64_t						GetMemLimit() const final { return m_iRtMemLimit; }

	bool						LoadEmbeddingModels ( CSphString & sError );
	const CSphQuery *			SetupAutoEmbeddings ( const CSphQuery & tQuery, CSphQuery & tUpdatedQuery, const ISphSchema & tMatchSchema, CSphString & sError ) const;
	bool						VerifyKNN ( InsertDocData_c & tDoc, CSphString & sError ) const;
	DocID_t						LookupCommittedUuidDocid ( const UuidDocidKey_t & tKey ) const;
	bool						ResolvePrimaryIdForAdd ( InsertDocData_c & tDoc, bool & bReplace, UuidDocidLease_c & tUuidLease, CSphString & sError ) const;
	bool						ResolveUuidDocidForAdd ( InsertDocData_c & tDoc, bool & bReplace, UuidDocidLease_c & tUuidLease, CSphString & sError ) const;

	template<typename PRED>
	int64_t						GetMemCount(PRED&& fnPred) const;

	void						DebugCheckRam ( DebugCheckError_i & tReporter );
	int							DebugCheckDisk ( DebugCheckError_i & tReporter );

	void						SetSchema ( CSphSchema tSchema );

	void						SetMemLimit ( int64_t iMemLimit );
	void						RecalculateRateLimit ( int64_t iSaved, int64_t iInserted, bool bEmergent );
	void						AlterSave ( bool bSaveRam );
	bool 						BinlogCommit ( RtSegment_t * pSeg, const VecTraits_T<DocID_t> & dKlist, int64_t iAddTotalBytes, CSphString & sError );
	bool						StopOptimize();
	void						UpdateUnlockedCount();
	bool						CheckSegmentConsistency ( const RtSegment_t* pNewSeg, bool bSilent=true ) const;

	// internal helpers/hooks
	inline RtWriter_c			RtWriter() { return { m_tRtChunks, [this] { UpdateUnlockedCount(); } }; }
	inline void					CopyChunksTo ( RtWriter_c& tWriter ) REQUIRES ( m_tWorkers.SerialChunkAccess() ) { tWriter.InitDiskChunks ( RtWriter_c::copy ); }

	// set of my rt; suitable for any usage
	inline RtGuard_t			RtGuard() const { return RtGuard_t { RtData() }; }

	// my own, or external data, if any present
	inline ConstRtData			RtData() const { return m_tRtChunks.RtData(); }

	void						DebugCheckRamSegment ( const RtSegment_t & tSegment, int iSegment, DebugCheckError_i & tReporter ) const;
	void						SaveRamFieldLengths ( CSphWriter& wrChunk ) const;
	void						SaveRamSegment ( const RtSegment_t* pSeg, CSphWriter& wrChunk ) const REQUIRES_SHARED ( pSeg->m_tLock );
	void						WriteMeta ( int64_t iTID, const VecTraits_T<int>& dChunkNames, CSphWriter& wrMeta ) const;

	CSphString					MakeDamagedName () const;
	void						DumpSegments ( VecTraits_T<const RtSegment_t*> dSegments, const CSphString& sFile ) const;
	void						DumpSegment ( const RtSegment_t* pSeg, const CSphString& sFile ) const;
	void						DumpMeta ( const CSphString& sFile ) const;
	void						DumpInsert ( const RtSegment_t* pNewSeg ) const;
	void						DumpMerge ( const RtSegment_t* pA, const RtSegment_t* pB, const RtSegment_t* pNew ) const;

	// Manage alter state
	void						RaiseAlterGeneration();
	int							GetAlterGeneration() const override;
	bool						AlterSI ( CSphString & sError ) override;
	bool						AlterKNN ( CSphString & sError ) override;
	bool						AlterApiKey ( const CSphString & sAttr, const CSphString & sKey, CSphString & sError ) override;
	bool						AlterApiUrl ( const CSphString & sAttr, const CSphString & sUrl, CSphString & sError ) override;
	bool						AlterApiTimeout ( const CSphString & sAttr, int iTimeout, CSphString & sError ) override;
	bool						AlterRebuild ( AlterOp_fn && operation, CSphString & sError, const char * sTrace );

	bool						CanAttach ( const CSphIndex * pIndex, bool bCheckFT, CSphString & sError ) const;
	bool						AttachDiskChunkMove ( CSphIndex * pIndex, bool & bFatal, CSphString & sError ) REQUIRES ( m_tWorkers.SerialChunkAccess() );
	void						AttachSetSettings ( CSphIndex * pIndex );
	bool						AttachSaveDiskChunk ();
	ConstDiskChunkRefPtr_t		PopDiskChunk();
	int							GetChunkId () const override { return m_tChunkID.GetChunkId ( m_tRtChunks ); }
	void						SetGlobalIDFPath ( const CSphString & sPath ) override;
	void						DebugDumpDict ( FILE * fp, bool bDumpOnly ) final;
	bool						AttachRtChunksExtCopy ( RtIndex_c * pSrcRtIndex, bool & bFatal, ExtFiles_h & hExtCache, CSphString & sError );
	bool						AttachRtChunksNoExtCopy ( RtIndex_c * pSrcRtIndex, bool & bFatal, CSphString & sError );
	bool						InitExtCache ( const FilenameBuilder_i * pDstFilenameBuilder, ExtFiles_h & hExtCache, CSphString & sError ) const;

	bool						ReserveEmbeddingSpace ( int64_t iDocsToFill, int iDims, CSphString & sError ) override;
	bool						InitUpdateEmbeddingState ( const CSphString & sAttr, EmbeddingPopulate_e eMode, ExtUpdState_t & tState, CSphString & sError ) override;
	bool						GetUpdateEmbedding ( ExtUpdState_t & tState, AttrUpdateSharedPtr_t & pUpdate, CSphString & sError ) const override;
	bool						ValidateUpdateEmbedding ( const ExtUpdState_t & tState, AttrUpdateInc_t & tUpd, CSphString & sError ) override;
	void						FinishUpdateEmbeddingState ( ExtUpdState_t & tState ) override;

	void						ManualOptimizeCutoff ( int iCutoff ) override;
};


RtIndex_c::RtIndex_c ( CSphString sIndexName, CSphString sPath, CSphSchema tSchema, int64_t iRamSize )
	: RtIndex_i { std::move ( sIndexName ), std::move ( sPath ) }
	, m_tmSaved ( sphMicroTimer() )
	, m_iTrackFailedRamActions {0}
{
	MEMORY ( MEM_INDEX_RT );

	SetSchema ( std::move ( tSchema ) );
	SetMemLimit ( iRamSize );

	auto iTrack = env_long ( "MANTICORE_TRACK_RT_ERRORS" );
	if ( iTrack.has_value() )
	{
		m_iTrackFailedRamActions = iTrack.value();
		sphInfo ( "MANTICORE_TRACK_RT_ERRORS env provided; up to %d insert/merge errors will be reported", m_iTrackFailedRamActions );
	}

	m_dSavingTimer.SetHandler ( [this]() { ConditionalDiskChunk(); } );
}

class OptimizeGuard_c final
{
	RtIndex_c& m_tIndex;
	const bool m_bPreviousOptimizeState;

public:
	NONCOPYMOVABLE ( OptimizeGuard_c );

	explicit OptimizeGuard_c (  RtIndex_c& tIndex )
		: m_tIndex { tIndex }
		, m_bPreviousOptimizeState { tIndex.StopOptimize() }
	{}

	~OptimizeGuard_c()
	{
		m_tIndex.m_bOptimizeStop.store ( m_bPreviousOptimizeState, std::memory_order_relaxed );
	}
};

RtIndex_c::~RtIndex_c ()
{
	if ( Threads::IsInsideCoroutine())
	{
		// From serial worker resuming on Wait() will happen after whole merger coroutine finished.
		ScopedScheduler_c tSerialFiber { m_tWorkers.SerialChunkAccess() };
		TRACE_SCHED ( "rt", "~RtIndex_c" );

		m_dSavingTimer.UnEngage();

		m_tSaving.SetShutdownFlag();
		StopMergeSegmentsWorker();
		m_tNSavesNow.Wait ( [] ( int iVal ) { return iVal == 0; } );
		OptimizeGuard_c tStopOptimize ( *this );
		Threads::Coro::Reschedule();
		WaitAllRoutinesFinished();
	}

	int64_t tmSave = sphMicroTimer();
	bool bValid = m_pTokenizer && m_pDict && m_bPreallocPassedOk;

	if ( bValid )
		bValid &= SaveRamChunk();

	if ( bValid )
		SaveMeta();

	if ( m_iLockFD>=0 )
		::close ( m_iLockFD );

	if ( bValid )
		Binlog::NotifyIndexFlush ( m_iTID, GetName(), (Binlog::Shutdown_e)sphInterrupted(), (Binlog::ForceSave_e)m_bIndexDeleted );

	if ( m_bIndexDeleted )
	{
		CSphString sFile = GetFilename ( "meta" );
		::unlink ( sFile.cstr() );
		sFile = GetFilename ( "ram" );
		::unlink ( sFile.cstr() );
		sFile = GetFilename ( SPH_EXT_SETTINGS );
		::unlink ( sFile.cstr() );
	}
	if ( !bValid )
		return;

	tmSave = sphMicroTimer() - tmSave;
	if ( tmSave>=1000 )
		sphInfo ( "rt: table %s: ramchunk saved in %d.%03d sec", GetName(), (int)(tmSave/1000000), (int)((tmSave/1000)%1000) );

	if ( !sphInterrupted() )
		sphLogDebug ( "closed table %s, valid %d, deleted %d, time %d.%03d sec", GetName(), (int)bValid, (int)m_bIndexDeleted, (int)(tmSave/1000000), (int)((tmSave/1000)%1000) );
}

void RtIndex_c::RaiseAlterGeneration()
{
	++m_iAlterGeneration;
}

int RtIndex_c::GetAlterGeneration() const
{
	return m_iAlterGeneration;
}

void RtIndex_c::UpdateUnlockedCount()
{
	if ( m_bDebugCheck )
		return;

	m_tUnLockedSegments.UpdateValueAndNotifyAll ( (int)m_tRtChunks.RamSegs()->count_of ( [] ( auto& dSeg ) { return !dSeg->m_iLocked; } ) );
}


static DocID_t LookupRamUuidDocid ( const RtGuard_t & tGuard, const UuidDocidKey_t & tKey )
{
	for ( const auto & pSeg : tGuard.m_dRamSegs )
	{
		const DocID_t * pDocID = pSeg->m_tUuidDocID.Find ( tKey );
		if ( !pDocID )
			continue;

		assert ( *pDocID );
		DocID_t tDocID = *pDocID;
		if ( !pSeg->IsAlive ( tDocID ) )
			continue;

		return tDocID;
	}

	return 0;
}


DocID_t RtIndex_c::LookupCommittedUuidDocid ( const UuidDocidKey_t & tKey ) const
{
	assert ( sphHasUuidDocid ( m_tSchema ) );

	RtGuard_t tGuard ( RtData() );
	DocID_t tDocID = LookupRamUuidDocid ( tGuard, tKey );
	if ( tDocID || tGuard.m_dDiskChunks.IsEmpty() )
		return tDocID;

	for ( const ConstDiskChunkRefPtr_t & pChunk : tGuard.m_dDiskChunks )
	{
		tDocID = pChunk->Cidx().FindAliveUuidDocid ( tKey );
		if ( tDocID )
			return tDocID;
	}

	return 0;
}


static bool IsAliveInRam ( const RtGuard_t & tGuard, DocID_t tDocID )
{
	return tGuard.m_dRamSegs.any_of ( [tDocID] ( const ConstRtSegmentRefPtf_t & p ) { return p->IsAlive ( tDocID ); } );
}


static bool IsAliveInCommittedData ( const RtGuard_t & tGuard, DocID_t tDocID )
{
	return IsAliveInRam ( tGuard, tDocID )
		|| tGuard.m_dDiskChunks.any_of ( [tDocID] ( const ConstDiskChunkRefPtr_t & p ) { return p->Cidx().IsAlive ( tDocID ); } );
}


static DocID_t GenerateAutoDocid ( const RtGuard_t & tGuard )
{
	DocID_t tDocID = 0;
	do
		tDocID = UidShort();
	while ( IsAliveInRam ( tGuard, tDocID ) );

	return tDocID;
}


bool RtIndex_c::ResolveUuidDocidForAdd ( InsertDocData_c & tDoc, bool & bReplace, UuidDocidLease_c & tUuidLease, CSphString & sError ) const
{
	assert ( tDoc.m_eDocidMode==InsertDocidMode_e::UUID_AUTO || tDoc.m_eDocidMode==InsertDocidMode_e::UUID_EXPLICIT );
	assert ( sphHasUuidDocid ( m_tSchema ) );
	assert ( m_pUuidRegistry );
	assert ( tUuidLease.IsEmpty() );
	UuidDocidRegistry_i * pUuidRegistry = m_pUuidRegistry.Ptr();

	if ( tDoc.m_iUuidString<0 )
	{
		sError = "internal error: missing uuid id value";
		return false;
	}

	if ( tDoc.m_eDocidMode==InsertDocidMode_e::UUID_AUTO )
	{
		bReplace = false;
		for ( ;; )
		{
			DocID_t tDocID = GenerateAutoDocid ( RtGuard() );
			CSphString sUuid = sphGenerateUuidDocid ( tDocID );
			UuidDocidKey_t tKey = sphGetUuidDocidKey ( { sUuid.cstr(), sUuid.Length() } );

			UuidDocidLease_c tCandidateLease;
			if ( !tCandidateLease.TryAcquire ( pUuidRegistry, tKey ) )
				continue;

			if ( LookupRamUuidDocid ( RtGuard(), tKey ) )
				continue;

			if ( pUuidRegistry->SetDocidIfEmpty ( tKey, tDocID )!=tDocID )
				continue;

			tDoc.SetID ( tDocID );
			tDoc.SetUuidDocidString ( std::move ( sUuid ) );
			tUuidLease = std::move ( tCandidateLease );
			return true;
		}
	}

	const char * szUuid = tDoc.GetUuidDocidString();
	if ( !szUuid || !*szUuid )
	{
		sError = "internal error: missing uuid id value";
		return false;
	}

	UuidDocidKey_t tKey = sphGetUuidDocidKey ( FromSz ( szUuid ) );
	tUuidLease.Acquire ( pUuidRegistry, tKey );

	DocID_t tCommittedDocID = LookupCommittedUuidDocid ( tKey );
	if ( tCommittedDocID )
	{
		if ( !bReplace )
		{
			DocID_t tMappedDocID = pUuidRegistry->GetDocid ( tKey );
			if ( tMappedDocID && tMappedDocID!=tCommittedDocID )
			{
				sError.SetSprintf ( "duplicate uuid id '%s'", szUuid );
				return false;
			}

			sError.SetSprintf ( "duplicate id '%s'", szUuid );
			return false;
		}

		DocID_t tEffectiveDocID = pUuidRegistry->SetDocidIfEmpty ( tKey, tCommittedDocID );
		if ( tEffectiveDocID!=tCommittedDocID )
		{
			sError.SetSprintf ( "duplicate uuid id '%s'", szUuid );
			return false;
		}

		tDoc.SetID ( tEffectiveDocID );
	}
	else
	{
		DocID_t tMappedDocID = pUuidRegistry->GetDocid ( tKey );
		if ( !tMappedDocID )
		{
			DocID_t tCandidateDocID = GenerateAutoDocid ( RtGuard() );
			tMappedDocID = pUuidRegistry->SetDocidIfEmpty ( tKey, tCandidateDocID );
		}

		tDoc.SetID ( tMappedDocID );
	}

	return true;
}


bool RtIndex_c::ResolvePrimaryIdForAdd ( InsertDocData_c & tDoc, bool & bReplace, UuidDocidLease_c & tUuidLease, CSphString & sError ) const
{
	if ( tDoc.m_eDocidMode==InsertDocidMode_e::UUID_AUTO || tDoc.m_eDocidMode==InsertDocidMode_e::UUID_EXPLICIT )
		return ResolveUuidDocidForAdd ( tDoc, bReplace, tUuidLease, sError );

	assert ( tDoc.m_eDocidMode==InsertDocidMode_e::NUMERIC );
	assert ( tUuidLease.IsEmpty() );
	DocID_t tDocID = tDoc.GetID();

	if ( tDocID && bReplace )
		return true;

	auto tGuard = RtGuard();
	if ( !tDocID )
	{
		bReplace = false;
		tDoc.SetID ( GenerateAutoDocid ( tGuard ) );
		return true;
	}

	assert ( !bReplace && tDocID!=0 );
	if ( IsAliveInCommittedData ( tGuard, tDocID ) )
	{
		sError.SetSprintf ( "duplicate id '" UINT64_FMT "'", tDocID );
		return false;
	}

	return true;
}


void RtIndex_c::ProcessDiskChunk ( int iChunk, VisitChunk_fn&& fnVisitor ) const
{
	auto pDiskChunks = m_tRtChunks.DiskChunks();
	if ( iChunk < 0 || iChunk >= pDiskChunks->GetLength() )
		fnVisitor ( nullptr );
	else
		fnVisitor ( &( *pDiskChunks )[iChunk]->Cidx() );
}

void RtIndex_c::ProcessDiskChunkEx ( int iChunk, VisitChunkEx_fn&& fnVisitor ) const
{
	auto pDiskChunks = m_tRtChunks.DiskChunks();
	if ( iChunk < 0 || iChunk >= pDiskChunks->GetLength() )
		fnVisitor ( nullptr, false );
	else
	{
		const auto & tChunk = ( *pDiskChunks )[iChunk];
		fnVisitor ( &tChunk->Cidx(), tChunk->m_bOptimizing.load ( std::memory_order_relaxed ) );
	}
}

template<typename VISITOR>
void RtIndex_c::ProcessDiskChunkByID ( int iChunkID, VISITOR&& fnVisitor ) const
{
	auto pChunks = m_tRtChunks.DiskChunks();
	for ( const auto& pChunk : *pChunks )
		if ( iChunkID == pChunk->Cidx().m_iChunk )
		{
			fnVisitor ( pChunk );
			break;
		}
}

template<typename VISITOR>
void RtIndex_c::ProcessDiskChunkByID ( VecTraits_T<int> dChunkIDs, VISITOR&& fnVisitor ) const
{
	auto pChunks = m_tRtChunks.DiskChunks();
	for ( int iDiskChunkID : dChunkIDs )
		for ( const auto& pChunk : *pChunks )
			if ( iDiskChunkID == pChunk->Cidx().m_iChunk )
			{
				fnVisitor ( pChunk );
				break;
			}
}

bool RtIndex_c::IsFlushNeed() const
{
	// m_iTID get managed by binlog that is why wo binlog there is no need to compare it 
	if ( Binlog::IsActive () && m_iTID>=0 && m_iTID<=m_iSavedTID )
		return false;

	return m_tSaving.ActiveStateIs ( SaveState_c::ENABLED );
}

static int64_t SegmentsGetUsedRam ( const ConstRtSegmentSlice_t& dSegments )
{
	int64_t iTotal = 0;
	for ( const RtSegment_t * pSeg : dSegments )
		iTotal += pSeg->GetUsedRam();

	return iTotal;
}

static int64_t SegmentsGetDeadRows ( const VecTraits_T<RtSegmentRefPtf_t> & dSegments )
{
	int64_t iTotal = 0;
	for ( RtSegment_t * pSeg : dSegments )
		iTotal += pSeg->m_tDeadRowMap.GetNumDeads();

	return iTotal;
}

// save RAM chunk to .ram, flush disk chunks dead-row-maps.
void RtIndex_c::ForceRamFlush ( const char* szReason )
{
	if ( !IsFlushNeed() )
		return;

	int64_t tmSave = sphMicroTimer();

	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "ForceRamFlush" );

	int64_t iUsedRam = SegmentsGetUsedRam ( *m_tRtChunks.RamSegs() );
	if ( !SaveRamChunk () )
	{
		sphWarning ( "rt: table %s: ramchunk save FAILED! (error=%s)", GetName(), m_sLastError.cstr() );
		return;
	}

	SaveMeta();
	auto pChunks = m_tRtChunks.DiskChunks();
	pChunks->for_each ( [] ( ConstDiskChunkRefPtr_t & pIdx ) { pIdx->Cidx().FlushDeadRowMap ( true ); } );
	Binlog::NotifyIndexFlush ( m_iTID, GetName(), Binlog::NoShutdown, Binlog::NoSave );

	int64_t iWasTID = std::exchange ( m_iSavedTID, m_iTID );
	auto tmNow = sphMicroTimer ();
	int64_t tmDelta = tmNow-std::exchange ( m_tmSaved, tmNow );
	tmSave = tmNow-tmSave;
	sphInfo ( "rt: table %s: ramchunk saved ok (mode=%s, last TID=" INT64_FMT ", current TID=" INT64_FMT ", "
		"ram=%d.%03d Mb, time delta=%d sec, took=%d.%03d sec)",
		GetName(), szReason, iWasTID, m_iTID, (int)(iUsedRam/1024/1024), (int)((iUsedRam/1024)%1000)
		, (int) (tmDelta/1000000), (int)(tmSave/1000000), (int)((tmSave/1000)%1000) );
}


int64_t RtIndex_c::GetLastFlushTimestamp() const
{
	return m_tmSaved;
}


//////////////////////////////////////////////////////////////////////////
// INDEXING
//////////////////////////////////////////////////////////////////////////
CSphSource_StringVector::CSphSource_StringVector ( const VecTraits_T<VecTraits_T<const char >> &dFields, const CSphSchema & tSchema )
	: CSphSource ( "$blobvector" )
{
	m_tSchema = tSchema;
	m_dFieldLengths.Reserve ( dFields.GetLength () );
	m_dFields.Reserve ( dFields.GetLength() + 1 );
	for ( const auto& dField : dFields )
	{
		m_dFields.Add ( (BYTE*)const_cast<char*> ( dField.begin() ) );
		m_dFieldLengths.Add ( dField.GetLength () );
		assert ( dField.begin() || dField.IsEmpty () );
	}
	m_dFields.Add (nullptr);
	m_iMaxHits = 0; // force all hits build
}

bool CSphSource_StringVector::Connect ( CSphString & )
{
	// no AddAutoAttrs() here; they should already be in the schema
	m_tHits.Reserve ( 1024 );
	return true;
}

void CSphSource_StringVector::Disconnect ()
{
	m_tHits.Reset();
}


template <typename T>
static void StoreAttrValue ( const InsertDocData_c & tDoc, const CSphColumnInfo & tAttr, int iColumnarAttr, int iStoredAttr, VecTraits_T<BYTE> * pAddedAttrs, CSphVector<BYTE> & dTmpStorage )
{
	T tValue = 0;
	if ( tAttr.IsColumnar() )
		tValue = (T)tDoc.m_dColumnarAttrs[iColumnarAttr];
	else
		tValue = (T)sphGetRowAttr ( tDoc.m_tDoc.m_pDynamic, tAttr.m_tLocator );

	int iBits = tAttr.m_tLocator.m_iBitCount;
	if ( tAttr.m_eAttrType==SPH_ATTR_BOOL )
		tValue = tValue ? 1 : 0;
	else
	{
		T uMask = iBits==64 ? (T)0xFFFFFFFFFFFFFFFFULL : (T)( (1ULL<<iBits)-1 );
		tValue &= uMask;
	}

	dTmpStorage.Resize ( sizeof(tValue) );
	memcpy ( dTmpStorage.Begin(), &tValue, dTmpStorage.GetLength() );
	pAddedAttrs[iStoredAttr] = dTmpStorage;
}


static void ProcessStoredAttrs ( DocstoreBuilder_i::Doc_t & tStoredDoc, const InsertDocData_c & tDoc, const CSphSchema & tSchema, CSphVector<CSphVector<BYTE>> & dTmpAttrStorage )
{
	int iNumStoredAttrs = 0;
	for ( int i = 0; i < tSchema.GetAttrsCount(); i++ )
		if ( tSchema.IsAttrStored(i) )
			iNumStoredAttrs++;

	if ( !iNumStoredAttrs )
		return;

	dTmpAttrStorage.Resize ( tSchema.GetAttrsCount() );

	VecTraits_T<BYTE> * pAddedAttrs = tStoredDoc.m_dFields.AddN ( iNumStoredAttrs );

	const char ** ppStr = tDoc.m_dStrings.Begin();
	int iStrAttr = 0;
	int iMva = 0;
	int iStoredAttr = 0;
	int iColumnarAttr = 0;

	for ( int i=0; i<tSchema.GetAttrsCount(); ++i )
	{
		const CSphColumnInfo & tAttr = tSchema.GetAttr(i);
		bool bStored = tSchema.IsAttrStored(i);

		switch ( tAttr.m_eAttrType )
		{
		case SPH_ATTR_STRING:
			{
				if ( bStored )
				{
					BYTE * pStr = ppStr ? (BYTE *) ppStr[iStrAttr] : nullptr;
					pAddedAttrs[iStoredAttr] = { pStr, pStr ? (int) strlen ((const char *) pStr ) : 0 };
				}

				iStrAttr++;
			}
			break;

		case SPH_ATTR_JSON:
			iStrAttr++;
			if ( !bStored )
				break;

			assert ( 0 && "Internal error: stored json" );
			break;

		case SPH_ATTR_UINT32SET:
		case SPH_ATTR_INT64SET:
		case SPH_ATTR_FLOAT_VECTOR:
			{
				int iNumValues = 0;
				bool bDefault = false;
				const int64_t * pMva = tDoc.GetMVA(iMva);
				std::tie ( iNumValues, bDefault ) = tDoc.ReadMVALength(pMva);
				iMva += iNumValues+1;

				if ( !bStored )
					break;

				if ( tAttr.m_eAttrType == SPH_ATTR_INT64SET )
					pAddedAttrs[iStoredAttr] = { (BYTE*)pMva, int(iNumValues*sizeof(int64_t)) };
				else
				{
					dTmpAttrStorage[i].Resize ( iNumValues*sizeof(DWORD) );
					DWORD * pAttrs = (DWORD*)dTmpAttrStorage[i].Begin();
					for ( int iValue = 0; iValue < iNumValues; iValue++ )
						pAttrs[iValue] = (DWORD)pMva[iValue];

					pAddedAttrs[iStoredAttr] = dTmpAttrStorage[i];
				}
			}
			break;

		case SPH_ATTR_BIGINT:
			if ( bStored )
				StoreAttrValue<int64_t> ( tDoc, tAttr, iColumnarAttr, iStoredAttr, pAddedAttrs, dTmpAttrStorage[i] );
			break;

		default:
			if ( bStored )
				StoreAttrValue<DWORD> ( tDoc, tAttr, iColumnarAttr, iStoredAttr, pAddedAttrs, dTmpAttrStorage[i] );
			break;
		}

		if ( tAttr.IsColumnar() )
			iColumnarAttr++;

		if ( bStored )
			iStoredAttr++;
	}
}


DocstoreBuilder_i::Doc_t * RtIndex_c::FetchDocFields ( DocstoreBuilder_i::Doc_t & tStoredDoc, const InsertDocData_c & tDoc, CSphSource_StringVector & tSrc, CSphVector<CSphVector<BYTE>> & dTmpAttrStorage ) const
{
	if ( !m_tSchema.HasStoredFields() && !m_tSchema.HasStoredAttrs() )
		return nullptr;

	tSrc.GetDocFields ( tStoredDoc.m_dFields );
	assert ( tStoredDoc.m_dFields.GetLength()==m_tSchema.GetFieldsCount() );

	// filter out non-hl fields (should already be null)
	int iField = 0;
	for ( int i = 0; i < m_tSchema.GetFieldsCount(); i++ )
	{
		if ( !m_tSchema.IsFieldStored(i) )
			tStoredDoc.m_dFields.Remove(iField);
		else
			iField++;
	}

	ProcessStoredAttrs ( tStoredDoc, tDoc, m_tSchema, dTmpAttrStorage );

	return &tStoredDoc;
}


bool RtIndex_c::VerifyKNN ( InsertDocData_c & tDoc, CSphString & sError ) const
{
	int iMva = 0;
	for ( int i = 0; i < m_tSchema.GetAttrsCount(); i++ )
	{
		const CSphColumnInfo & tAttr = m_tSchema.GetAttr(i);
		if ( !IsMvaAttr ( tAttr.m_eAttrType ) )
			continue;

		int iNumValues = 0;
		bool bDefault = false;
		const int64_t * pMva = tDoc.GetMVA(iMva);
		std::tie ( iNumValues, bDefault ) = tDoc.ReadMVALength(pMva);
		iMva += iNumValues + 1;

		if ( tAttr.m_eAttrType!=SPH_ATTR_FLOAT_VECTOR || !tAttr.IsIndexedKNN() )
			continue;

		if ( m_dAttrsWithModels.GetLength() && m_dAttrsWithModels[i].m_pModel )
		{
			if ( !bDefault && iNumValues && iNumValues!=tAttr.m_tKNN.m_iDims )
			{
				sError.SetSprintf ( "attribute '%s' has model_name=%s specified, but vector contents with %d values is provided", tAttr.m_sName.cstr(), tAttr.m_tKNNModel.m_sModelName.c_str(), iNumValues );
				return false;
			}
		}
		else if ( !bDefault && iNumValues && iNumValues!=tAttr.m_tKNN.m_iDims )
		{
			sError.SetSprintf ( "KNN error: data has %d values, index '%s' needs %d values", iNumValues, tAttr.m_sName.cstr(), tAttr.m_tKNN.m_iDims );
			return false;
		}
	}

	return true;
}


bool RtIndex_c::AddDocument ( InsertDocData_c & tDoc, bool bReplace, const CSphString & sTokenFilterOptions, CSphString & sError, CSphString & sWarning, RtAccum_t * pAcc )
{
	assert ( RTChangesAllowed () );
	assert ( m_tSchema.GetAttrIndex ( sphGetDocidName() )==0 );
	assert ( m_tSchema.GetAttr ( sphGetDocidName() )->m_eAttrType==SPH_ATTR_BIGINT );

	MEMORY ( MEM_INDEX_RT );

	if ( !BindAccum ( pAcc, &sError ) )
		return false;

	UuidDocidLease_c tUuidLease;
	if ( !ResolvePrimaryIdForAdd ( tDoc, bReplace, tUuidLease, sError ) )
		return false;
	assert ( ( tDoc.m_eDocidMode==InsertDocidMode_e::NUMERIC )==tUuidLease.IsEmpty() );

	TokenizerRefPtr_c tTokenizer = CloneIndexingTokenizer();

	if (!tTokenizer)
	{
		sError.SetSprintf ( "internal error: no indexing tokenizer available" );
		return false;
	}

	tDoc.m_tDoc.m_tRowID = pAcc->GenerateRowID();

	// OPTIMIZE? do not create filter on each(!) INSERT
	if ( !m_tSettings.m_sIndexTokenFilter.IsEmpty() )
	{
		Tokenizer::AddPluginFilterTo ( tTokenizer, m_tSettings.m_sIndexTokenFilter, sError );
		if ( !sError.IsEmpty() )
			return false;
		if ( !tTokenizer->SetFilterSchema ( m_tSchema, sError ) )
			return false;
		if ( !sTokenFilterOptions.IsEmpty() )
			if ( !tTokenizer->SetFilterOptions ( sTokenFilterOptions.cstr(), sError ) )
				return false;
	}

	// OPTIMIZE? do not create filter on each(!) INSERT
	if ( m_tSettings.m_uAotFilterMask )
		sphAotTransformFilter ( tTokenizer, m_pDict, m_tSettings.m_bIndexExactWords, m_tSettings.m_uAotFilterMask );

	CSphSource_StringVector tSrc ( tDoc.m_dFields, m_tSchema );

	// SPZ setup
	if ( m_tSettings.m_bIndexSP && !tTokenizer->EnableSentenceIndexing ( sError ) )
		return false;

	if ( !m_tSettings.m_sZones.IsEmpty() && !tTokenizer->EnableZoneIndexing ( sError ) )
		return false;

	if ( m_tSettings.m_bHtmlStrip && !tSrc.SetStripHTML ( m_tSettings.m_sHtmlIndexAttrs.cstr(), m_tSettings.m_sHtmlRemoveElements.cstr(), m_tSettings.m_bIndexSP, m_tSettings.m_sZones.cstr(), sError ) )
		return false;

	tSrc.Setup ( m_tSettings, nullptr );
	tSrc.SetTokenizer ( std::move ( tTokenizer ) );
	tSrc.SetDict ( pAcc->m_pDict );
	// OPTIMIZE? do not clone filters on each INSERT
	if ( m_pFieldFilter )
		tSrc.SetFieldFilter ( m_pFieldFilter->Clone() );
	tSrc.SetMorphFields ( m_tMorphFields );
	if ( !tSrc.Connect ( m_sLastError ) )
		return false;

	m_tSchema.CloneWholeMatch ( tSrc.m_tDocInfo, tDoc.m_tDoc );

	bool bEOF = false;
	if ( !tSrc.IterateStart ( sError ) || !tSrc.IterateDocument ( bEOF, sError ) )
		return false;

	ISphHits * pHits = tSrc.IterateHits ( sError );
	pAcc->GrabLastWarning ( sWarning );
	if ( !tSrc.GetLastWarning().IsEmpty() )
		sWarning = tSrc.GetLastWarning();

	if ( m_pEmbeddings )
		pAcc->FetchEmbeddingsSrc ( tDoc, m_dAttrsWithModels );

	if ( !VerifyKNN ( tDoc, sError ) )
		return false;

	CSphVector<CSphVector<BYTE>> dTmpAttrStorage;
	DocstoreBuilder_i::Doc_t tStoredDoc;
	DocstoreBuilder_i::Doc_t * pStoredDoc = FetchDocFields ( tStoredDoc, tDoc, tSrc, dTmpAttrStorage );
	tDoc.m_iTotalBytes = tSrc.GetStats().m_iTotalBytes;
	if ( !AddDocument ( pHits, tDoc, bReplace, pStoredDoc, sError, sWarning, pAcc ) )
		return false;

	tUuidLease.Adopt ( *pAcc );
	return true;
}


bool RtIndex_i::PrepareAccum ( RtAccum_t* pAcc, DictFormat_e eDictFormat, CSphString* pError )
{
	assert ( pAcc );
	if ( pAcc->GetIndex() && pAcc->GetIndex()!=this )
	{
		if ( pError )
			pError->SetSprintf ( "current txn is working with another table ('%s')", pAcc->GetIndex()->GetName() );
		return false;
	}

	if ( pAcc->GetIndex() && pAcc->GetSchemaHash()!=GetSchemaHash() )
	{
		if ( pError )
			pError->SetSprintf ( "current txn is working with table's another schema ('%s'), restart session", pAcc->GetIndex()->GetName() );
		return false;
	}

	assert ( pAcc->GetIndex()==nullptr || pAcc->GetIndex()==this );
	if ( !pAcc->GetIndex() )
	{
		pAcc->SetIndex ( this );
		pAcc->SetupDict ( this, m_pDict, eDictFormat );
	}
	return true;
}

bool RtIndex_c::BindAccum ( RtAccum_t * pAccExt, CSphString * pError )
{
	const bool bFirstBind = !pAccExt->GetIndex();
	if ( !PrepareAccum ( pAccExt, GetDictFormat(), pError ) )
		return false;

	assert ( ( m_tSchema.GetAttrsCount() && m_tSchema.GetAttr(0).IsUuidLinkedDocid() )==(bool)m_pUuidRegistry );
	if ( bFirstBind && m_pUuidRegistry )
		pAccExt->BindUuidRegistry ( m_pUuidRegistry );

	assert ( pAccExt->IsUuidRegistry ( m_pUuidRegistry.Ptr() ) );
	return true;
}


bool RtIndex_c::AddDocument ( ISphHits * pHits, const InsertDocData_c & tDoc, bool bReplace, const DocstoreBuilder_i::Doc_t * pStoredDoc, CSphString & sError, CSphString & sWarning, RtAccum_t * pAccExt )
{
	assert ( RTChangesAllowed () );

	auto * pAcc = (RtAccum_t *)pAccExt;

	if ( pAcc )
		pAcc->AddDocument ( pHits, tDoc, bReplace, m_tSchema.GetRowSize(), pStoredDoc );

	return !!pAcc;
}


// cook checkpoints - make NULL terminating strings from offsets
static void FixupSegmentCheckpoints ( RtSegment_t * pSeg )
{
	assert ( pSeg &&
		( !pSeg->m_dWordCheckpoints.GetLength() || pSeg->m_dKeywordCheckpoints.GetLength() ) );
	if ( !pSeg->m_dWordCheckpoints.GetLength() )
		return;

	const char * pBase = (const char *)pSeg->m_dKeywordCheckpoints.Begin();
	assert ( pBase );

	for ( auto & dCheckpoint : pSeg->m_dWordCheckpoints )
		dCheckpoint.m_szWord = pBase + dCheckpoint.m_uWordID;
}

static bool RtDebugCheckUnzipIntBE ( const BYTE * & pIn, const BYTE * pMaxWord, int & iValue )
{
	DWORD uValue = 0;
	for ( int i=0; i<5; i++ )
	{
		if ( pIn>=pMaxWord )
			return false;

		BYTE uByte = *pIn++;
		uValue = ( uValue << 7 ) | ( uByte & 0x7f );
		if ( !( uByte & 0x80 ) )
		{
			if ( uValue>INT_MAX )
				return false;

			iValue = (int)uValue;
			return true;
		}
	}

	return false;
}


static bool RtDebugCheckUnpackKeyword ( const BYTE * & pIn, const BYTE * pMaxWord, DictFormat_e eDictFormat,
	CSphVector<BYTE> & dWord, int & iWordLen, const CSphVector<BYTE> & dLastWord, int iLastWordLen,
	int iSegment, int nWordsRead, DebugCheckError_i & tReporter )
{
	if ( pIn>=pMaxWord )
	{
		tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );
		return false;
	}

	BYTE uPacked = *pIn++;
	int iMatch = 0;
	int iDelta = 0;

	if ( uPacked & 0x80 )
	{
		iDelta = ( ( uPacked>>4 ) & 7 ) + 1;
		iMatch = uPacked & 15;
	} else if ( eDictFormat==DictFormat_e::KEYWORDS_V2 && uPacked==0x7f )
	{
		if ( !RtDebugCheckUnzipIntBE ( pIn, pMaxWord, iDelta ) || !RtDebugCheckUnzipIntBE ( pIn, pMaxWord, iMatch ) )
		{
			tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );
			return false;
		}
	} else
	{
		iDelta = eDictFormat==DictFormat_e::KEYWORDS_V2 ? uPacked : ( uPacked & 127 );
		if ( pIn>=pMaxWord )
		{
			tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );
			return false;
		}

		iMatch = *pIn++;
		if ( eDictFormat!=DictFormat_e::KEYWORDS_V2 && iDelta<=8 && iMatch<=15 )
		{
			tReporter.Fail ( "wrong word-delta (segment=%d, word=%d, last_word=%s, last_len=%d, match=%d, delta=%d)",
				iSegment, nWordsRead, (const char*)dLastWord.Begin(), iLastWordLen, iMatch, iDelta );
		}
	}

	const int iMaxWordLen = GetKeywordMaxStoredBytes ( eDictFormat );
	if ( iDelta<=0 || iMatch<0 || iMatch>iLastWordLen || iMatch+iDelta>iMaxWordLen )
	{
		tReporter.Fail ( "wrong word-delta (segment=%d, word=%d, last_word=%s, last_len=%d, match=%d, delta=%d)",
			iSegment, nWordsRead, (const char*)dLastWord.Begin(), iLastWordLen, iMatch, iDelta );

		if ( iDelta>0 )
		{
			const int64_t iLeft = pMaxWord-pIn;
			if ( iDelta>=iLeft )
			{
				pIn = pMaxWord;
				tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );
			} else
				pIn += iDelta;
		}

		if ( pIn>=pMaxWord )
			tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );

		return false;
	}

	if ( iDelta>=pMaxWord-pIn )
	{
		tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );
		return false;
	}

	iWordLen = iMatch+iDelta;
	memcpy ( dWord.Begin()+iMatch, pIn, iDelta );
	dWord[iWordLen] = '\0';
	pIn += iDelta;
	return true;
}


static const RtWordCheckpoint_t * RtSearchCheckpointWrd ( const char * sWord, int iWordLen, bool bStarMode, VecTraits_T<RtWordCheckpoint_t> dCheckpoints )
{
	assert ( iWordLen>0 );

	const RtWordCheckpoint_t * pStart = dCheckpoints.begin();
	const RtWordCheckpoint_t * pEnd = &dCheckpoints.Last();

	const int iStartLen = (int)strlen ( pStart->m_szWord );
	if ( bStarMode && sphDictCmp ( sWord, iWordLen, pStart->m_szWord, iStartLen )<0 )
		return nullptr;
	if ( !bStarMode && sphDictCmpStrictly ( sWord, iWordLen, pStart->m_szWord, iStartLen )<0 )
		return nullptr;

	const int iEndLen = (int)strlen ( pEnd->m_szWord );
	if ( sphDictCmpStrictly ( sWord, iWordLen, pEnd->m_szWord, iEndLen )>=0 )
		return pEnd;

	while ( pEnd-pStart>1 )
	{
		const RtWordCheckpoint_t * pMid = pStart + (pEnd-pStart)/2;
		const int iMidLen = (int)strlen ( pMid->m_szWord );
		const int iCmpRes = sphDictCmpStrictly ( sWord, iWordLen, pMid->m_szWord, iMidLen );

		if ( !iCmpRes )
			return pMid;
		else if ( iCmpRes<0 )
			pEnd = pMid;
		else
			pStart = pMid;
	}

	assert ( pStart >= dCheckpoints.begin() );
	assert ( pStart <= &dCheckpoints.Last() );
	assert ( sphDictCmp ( sWord, iWordLen, pStart->m_szWord, (int)strlen ( pStart->m_szWord ) )>=0
		&& sphDictCmpStrictly ( sWord, iWordLen, pEnd->m_szWord, (int)strlen ( pEnd->m_szWord ) )<0 );

	return pStart;
}


template <DictFormat_e DICT_FORMAT>
static void CreateSegmentHits_T ( RtAccum_t& tAcc, RtSegment_t * pSeg, int iWordsCheckpoint, ESphHitless eHitless, const VecTraits_T<SphWordID_t> & dHitlessWords )
{
	constexpr bool bKeywordDict = DICT_FORMAT!=DictFormat_e::CRC;
	assert(pSeg);

	auto& dAccum = tAcc.m_dAccum;
	assert ( tAcc.IsKeywordDict()==bKeywordDict );
	assert ( tAcc.m_eDictFormat==DICT_FORMAT );

	CSphWordHit& tClosingHit = dAccum.Add();
	tClosingHit.m_uWordID = WORDID_MAX;
	tClosingHit.m_tRowID = INVALID_ROWID;
	tClosingHit.m_uWordPos = EMPTY_HIT;

	RtDoc_t tDoc;
	RtWord_t tWord;
	RtDocWriter_c tOutDoc ( pSeg->m_dDocs );
	RtWordWriter_c tOutWord ( pSeg->m_dWords, pSeg->m_dWordCheckpoints, pSeg->m_dKeywordCheckpoints, DICT_FORMAT, iWordsCheckpoint, eHitless );
	RtHitWriter_c tOutHit ( pSeg->m_dHits );

	[[maybe_unused]] const BYTE * pPacketBase = bKeywordDict ? tAcc.GetPackedKeywords() : nullptr;
	[[maybe_unused]] const BYTE * pPacketMax = bKeywordDict ? pPacketBase + tAcc.GetPackedLen() : nullptr;

	Hitpos_t uEmbeddedHit = EMPTY_HIT;
	Hitpos_t uPrevHit = EMPTY_HIT;

	for ( const CSphWordHit & tHit : dAccum )
	{
		// new keyword or doc; flush current doc
		if ( tHit.m_uWordID!=tWord.m_uWordID || tHit.m_tRowID!=tDoc.m_tRowID )
		{
			if ( tDoc.m_tRowID!=INVALID_ROWID )
			{
				++tWord.m_uDocs;
				tWord.m_uHits += tDoc.m_uHits;

				if ( uEmbeddedHit )
				{
					assert ( tDoc.m_uHits==1 );
					tDoc.m_uHit = uEmbeddedHit;
				}

				tOutDoc << tDoc;
				tDoc.m_uDocFields = 0;
				tDoc.m_uHits = 0;
				tDoc.m_uHit = tOutHit.WriterPos();
			}

			tDoc.m_tRowID = tHit.m_tRowID;
			tOutHit.ZipRestart ();
			uEmbeddedHit = EMPTY_HIT;
			uPrevHit = EMPTY_HIT;
		}

		// new keyword; flush current keyword
		if ( tHit.m_uWordID!=tWord.m_uWordID )
		{
			tOutDoc.ZipRestart ();
			if ( tWord.m_uWordID )
			{
				if constexpr ( bKeywordDict )
				{
					const BYTE * pPackedWord = pPacketBase + tWord.m_uWordID;
					ByteBlob_t tKeyword = GetPackedKeywordByFormat<DICT_FORMAT> ( pPackedWord );
					assert ( tKeyword.first + tKeyword.second <= pPacketMax );
					tWord.m_sWord = tKeyword.first;
					tWord.m_iWordLen = tKeyword.second;
				}
				tOutWord << tWord;
			}

			tWord.m_uWordID = tHit.m_uWordID;
			tWord.m_uDocs = 0;
			tWord.m_uHits = 0;
			tWord.m_uDoc = tOutDoc.WriterPos();
			uPrevHit = EMPTY_HIT;
			if ( eHitless==SPH_HITLESS_NONE || eHitless==SPH_HITLESS_ALL || !tWord.m_uWordID || tHit.m_uWordPos==EMPTY_HIT )
			{
				tWord.m_bHasHitlist = ( eHitless==SPH_HITLESS_NONE || !tWord.m_uWordID );
			} else
			{
				SphWordID_t tWordID = tWord.m_uWordID;
				if constexpr ( bKeywordDict )
				{
					if ( !dHitlessWords.IsEmpty() )
					{
						const BYTE * pPackedWord = pPacketBase + tWord.m_uWordID;
						assert ( pPackedWord < pPacketMax );
						ByteBlob_t tKeyword = GetPackedKeywordByFormat<DICT_FORMAT> ( pPackedWord );
						memcpy ( &tWordID, tKeyword.first + tKeyword.second, sizeof(tWordID) );
					}
				}
				tWord.m_bHasHitlist = ( dHitlessWords.BinarySearch ( tWordID )==nullptr );
			}
		}

		// might be a duplicate
		if ( uPrevHit==tHit.m_uWordPos )
			continue;

		// just a new hit
		if ( !tWord.m_bHasHitlist )
		{
			if ( !tDoc.m_uHits )
				uEmbeddedHit = tHit.m_uWordPos;
			tDoc.m_uHits = 1; // FIXME!!! hitless hit-count always 1
		} else if ( !tDoc.m_uHits )
		{
			uEmbeddedHit = tHit.m_uWordPos;
			++tDoc.m_uHits;
		} else
		{
			if ( uEmbeddedHit )
				tOutHit << std::exchange ( uEmbeddedHit, 0 );

			tOutHit << tHit.m_uWordPos;
			++tDoc.m_uHits;
		}
		uPrevHit = tHit.m_uWordPos;

		const int iField = HITMAN::GetField ( tHit.m_uWordPos );
		if ( iField<32 )
			tDoc.m_uDocFields |= ( 1UL<<iField );
	}
}


static void CreateSegmentHits ( RtAccum_t& tAcc, RtSegment_t * pSeg, int iWordsCheckpoint, ESphHitless eHitless, const VecTraits_T<SphWordID_t> & dHitlessWords )
{
	switch ( tAcc.m_eDictFormat )
	{
	case DictFormat_e::CRC:
		CreateSegmentHits_T<DictFormat_e::CRC> ( tAcc, pSeg, iWordsCheckpoint, eHitless, dHitlessWords );
		return;

	case DictFormat_e::KEYWORDS_V2:
		CreateSegmentHits_T<DictFormat_e::KEYWORDS_V2> ( tAcc, pSeg, iWordsCheckpoint, eHitless, dHitlessWords );
		return;

	case DictFormat_e::KEYWORDS:
		CreateSegmentHits_T<DictFormat_e::KEYWORDS> ( tAcc, pSeg, iWordsCheckpoint, eHitless, dHitlessWords );
		return;
	}

	assert ( 0 && "unknown dict format" );
}

RtSegment_t * CreateSegment ( RtAccum_t* pAcc, int iWordsCheckpoint, ESphHitless eHitless, const VecTraits_T<SphWordID_t> & dHitlessWords, CSphString & sError )
{
	TRACE_CONN ( "conn", "CreateSegment" );

	assert ( pAcc );
	if ( !pAcc->m_uAccumDocs )
		return nullptr;

	MEMORY ( MEM_RT_ACCUM );
	auto * pSeg = new RtSegment_t ( pAcc->m_uAccumDocs, pAcc->GetIndex()->GetInternalSchema() );
	FakeWL_t tFakeLock {pSeg->m_tLock};
	CreateSegmentHits ( *pAcc, pSeg, iWordsCheckpoint, eHitless, dHitlessWords );

	if ( pAcc->IsKeywordDict() )
		FixupSegmentCheckpoints ( pSeg );

	pSeg->m_dRows.SwapData ( pAcc->m_dAccumRows );
	pSeg->m_dBlobs.SwapData ( pAcc->m_dBlobs) ;
	std::swap ( pSeg->m_pDocstore, pAcc->m_pDocstore );

	if ( pAcc->m_pColumnarBuilder )
	{
		assert( pAcc->m_pIndex);
		pSeg->m_pColumnar = CreateColumnarRT ( pAcc->m_pIndex->GetInternalSchema(), pAcc->m_pColumnarBuilder.get() );
	}

	pSeg->BuildDocID2RowIDMap ( pAcc->m_pIndex->GetInternalSchema() );
	pAcc->m_tNextRowID = 0;

	return pSeg;
}


void RtIndex_c::CopyWord ( RtSegment_t& tDstSeg, RtWord_t& tDstWord, RtDocWriter_c& tDstDoc, const RtSegment_t& tSrcSeg, const RtWord_t* pSrcWord, const VecTraits_T<RowID_t>& dRowMap )
{
	RtDocReader_c tSrcDocs ( &tSrcSeg, *pSrcWord );
	// copy docs
	while ( tSrcDocs.UnzipDoc() )
	{
		auto tNewRowID = dRowMap[tSrcDocs->m_tRowID];
		if ( tNewRowID==INVALID_ROWID )
			continue;

		RtDoc_t tDoc = *tSrcDocs;
		tDoc.m_tRowID = tNewRowID;

		++tDstWord.m_uDocs;
		tDstWord.m_uHits += tSrcDocs->m_uHits;

		if ( tSrcDocs->m_uHits!=1 )
		{
			tDoc.m_uHit = tDstSeg.m_dHits.GetLength ();
			tDstSeg.m_dHits.Append ( GetHitsBlob ( tSrcSeg, *tSrcDocs ) );

			// this is reference of what append (hitsblob) above does.
//			RtHitWriter_c tOutHit ( &tDst );
//			tDoc.m_uHit = tOutHit.ZipHitPos();
//			for ( DWORD uValue=tInHit.UnzipHit(); uValue; uValue=tInHit.UnzipHit() )
//				tOutHit.ZipHit ( uValue );
		}

		// copy doc
		tDstDoc << tDoc;
	}
}

template<typename FN>
inline void ProcessField ( RtDoc_t & tOutDoc, DWORD uHit, int iKillField, FN&& fnProcessor )
{
	assert ( iKillField >=0 );

	int iField = HITMAN::GetField ( uHit );
	if ( iKillField==iField )
		return;

	if ( iField>iKillField )
	{
		--iField;
		HITMAN::DecrementField ( uHit );
	}

	++tOutDoc.m_uHits;
	if ( iField<32 )
		tOutDoc.m_uDocFields |= ( 1UL << iField );

	fnProcessor ( uHit );
}


static void CopyWordWithoutField ( CSphTightVector<BYTE> * pOutHits, RtDocWriter_c & tOutDocs, RtWord_t & tOutWord, const RtSegment_t & tSrc, RtDocReader_c & tInDocs, int iKillField  )
{
	assert ( iKillField>=0 );

	while ( tInDocs.UnzipDoc() )
	{
		RtDoc_t tOutDoc = *tInDocs;
		tOutDoc.m_uHits = tOutDoc.m_uDocFields = 0;

		if ( tInDocs->m_uHits!=1 )
		{
			RtHitReader_c tInHits ( tSrc, *tInDocs );
			RtHitWriter_c tOutHits ( *pOutHits );
			tOutDoc.m_uHit = tOutHits.WriterPos();
			while ( tInHits.UnzipHit() )
				ProcessField ( tOutDoc, *tInHits, iKillField, [&tOutHits] ( Hitpos_t x ) { tOutHits << x; } );
		} else
			ProcessField ( tOutDoc, tOutDoc.m_uHit, iKillField, [&tOutDoc] ( Hitpos_t x ) { tOutDoc.m_uHit = x; } );

		if ( !tOutDoc.m_uHits )
			continue;

		tOutDocs << tOutDoc;

		++tOutWord.m_uDocs;
		tOutWord.m_uHits += tOutDoc.m_uHits;
	}
}


void RtIndex_c::DeleteFieldFromDict ( RtSegment_t * pSeg, int iKillField )
{
	assert ( iKillField>=0 );

	CSphTightVector<BYTE> dWords;
	CSphVector<RtWordCheckpoint_t> dWordCheckpoints;
	CSphTightVector<BYTE> dDocs;
	CSphTightVector<BYTE> dHits;
	CSphVector<BYTE> dKeywordCheckpoints;

	const RtSegment_t & tInSeg = *pSeg;
	dWords.Reserve ( tInSeg.m_dWords.GetLength () );
	dDocs.Reserve ( tInSeg.m_dDocs.GetLength () );
	dHits.Reserve ( tInSeg.m_dHits.GetLength () );

	RtWordWriter_c tOutWords ( dWords, dWordCheckpoints, dKeywordCheckpoints, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );
	RtWordReader_c tInWord ( &tInSeg, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );

	RtWord_t tOutWord;
	while ( tInWord.UnzipWord() )
	{
		tOutWord = *tInWord;
		tOutWord.m_uDocs = tOutWord.m_uHits = 0;

		RtDocWriter_c tOutDocs ( dDocs );
		tOutWord.m_uDoc = tOutDocs.WriterPos();

		RtDocReader_c tInDocs ( &tInSeg, *tInWord );
		CopyWordWithoutField ( &dHits, tOutDocs, tOutWord, tInSeg, tInDocs, iKillField );

		// append word to the dictionary
		if ( tOutWord.m_uDocs )
			tOutWords << tOutWord;
	}

	// swap data to refreshed
	RtSegment_t & tOutSeg = *pSeg;
	tOutSeg.m_dWords.SwapData ( dWords );
	tOutSeg.m_dWordCheckpoints.SwapData ( dWordCheckpoints );
	tOutSeg.m_dDocs.SwapData ( dDocs );
	tOutSeg.m_dHits.SwapData ( dHits );
	tOutSeg.m_dKeywordCheckpoints.SwapData ( dKeywordCheckpoints );

	if ( IsWordDict() )
		FixupSegmentCheckpoints ( pSeg );

	BuildSegmentInfixes ( &tOutSeg, m_pDict->HasMorphology (), GetDictFormat(), m_tSettings.m_iMinInfixLen, m_iWordsCheckpoint, ( m_iMaxCodepointLength>1 ), m_tSettings.m_eHitless );
}


// iterate over alive rows range
class RtLiveRows_c
{
public:
	class Iterator_c
	{
	public:
		Iterator_c ( const RtLiveRows_c & tOwner, bool bBegin )
			: m_tOwner { tOwner }
			, m_tRowID { bBegin ? tOwner.FirstAliveRow () : tOwner.EndRow() }
		{}

		RowID_t operator*()	const { return m_tRowID; };
		bool operator!= ( const Iterator_c & rhs ) const { return m_tRowID!=rhs.m_tRowID; }

		Iterator_c & operator++ ()
		{
			m_tRowID = m_tOwner.NextAliveRow(m_tRowID);
			return *this;
		}
		
	private:
		const RtLiveRows_c & m_tOwner;
		RowID_t m_tRowID = 0;
	};

	explicit RtLiveRows_c ( const RtSegment_t & tSeg )
		: m_tRowIDMax ( tSeg.m_uRows )
		, m_tDeadRowMap ( tSeg.m_tDeadRowMap )
	{}

	// c++11 style iteration
	Iterator_c begin () const { return { *this, true }; }
	Iterator_c end() const { return { *this, false }; }

private:
	RowID_t		m_tRowID = 0;
	RowID_t		m_tRowIDMax = 0;
	const DeadRowMap_Ram_c & m_tDeadRowMap;

	RowID_t SkipDeadRows ( RowID_t tRowID ) const
	{
		while ( tRowID<m_tRowIDMax && m_tDeadRowMap.IsSet(tRowID) )
			++tRowID;

		return tRowID;
	}

	RowID_t FirstAliveRow() const			{ return SkipDeadRows(m_tRowID); }
	RowID_t EndRow() const					{ return m_tRowIDMax; }
	RowID_t NextAliveRow ( RowID_t tRowID ) const { return SkipDeadRows ( tRowID+1 ); }
};

template <typename BLOOM_TRAITS, BYTE KEY_VAL_COUNT>
static void SetBloom_T ( const BYTE * sWord, int iFrom, int iTo, BLOOM_TRAITS & tBloom )
{
	constexpr int uKEYMASK = KEY_VAL_COUNT * 64 - 1; // KEY_VAL_COUNT is 8 or 32
	uint64_t uHash64 = sphFNV64 ( sWord+iFrom, iTo-iFrom );
	uHash64 = ( uHash64>>32 ) ^ ( uHash64 & 0xFFFFFFFF );
	const int iByte = static_cast<int> ( uHash64 & uKEYMASK );
	const int iPos = iByte >> 6;
	const uint64_t uVal = U64C(1) << ( iByte & 63 );
	tBloom.Set ( iPos, uVal );
}


template <typename BLOOM_TRAITS, BYTE INFIX_CODEPOINT_COUNT, BYTE KEY_VAL_COUNT, bool UTF8>
bool BuildBloom_T ( const BYTE * sWord, int iLen, BLOOM_TRAITS & tBloom )
{
	if ( iLen<INFIX_CODEPOINT_COUNT )
		return false;

	if constexpr ( UTF8 )
	{
		int dOffsets[INFIX_CODEPOINT_COUNT+1] = { 0 };

		// Track the active ngram window as byte offsets into the string.
		const BYTE * s = sWord;
		const BYTE * sEnd = sWord + iLen;
		int iCodes = 0;
		while ( s<sEnd )
		{
			const BYTE uCodepoints = sphUtf8CharBytes ( *s );
			assert ( uCodepoints>=1 && uCodepoints<=4 );
			assert ( s+uCodepoints<=sEnd );
			if ( s+uCodepoints>sEnd )
				break;

			s += uCodepoints;
			if ( iCodes<INFIX_CODEPOINT_COUNT )
				dOffsets[++iCodes] = (int)( s-sWord );
			else
			{
				for ( int i=0; i<INFIX_CODEPOINT_COUNT; i++ )
					dOffsets[i] = dOffsets[i+1];
				dOffsets[INFIX_CODEPOINT_COUNT] = (int)( s-sWord );
				++iCodes;
			}

			if ( iCodes<INFIX_CODEPOINT_COUNT )
				continue;

			if ( !tBloom.IterateNext() )
				break;

			SetBloom_T<BLOOM_TRAITS, KEY_VAL_COUNT> ( sWord, dOffsets[0], dOffsets[INFIX_CODEPOINT_COUNT], tBloom );
		}

		if ( iCodes<INFIX_CODEPOINT_COUNT )
			return false;
	} else // if constexpr ( !UTF8 )
	{
		for ( int i=0; i<=iLen-INFIX_CODEPOINT_COUNT && tBloom.IterateNext(); ++i )
		{
			const int iFrom = i;
			const int iTo = i+INFIX_CODEPOINT_COUNT;
			SetBloom_T<BLOOM_TRAITS, KEY_VAL_COUNT> ( sWord, iFrom, iTo, tBloom );
		}
	}
	return true;
}

template <typename TRAITS>
bool BuildBloom_T ( const BYTE * sWord, int iLen, int iInfixCodepointCount, bool bUtf8, int iKeyValCount, TRAITS &tBloom )
{
	assert ( (iInfixCodepointCount == BLOOM_NGRAM_0) || (iInfixCodepointCount == BLOOM_NGRAM_1) );
	assert ( (iKeyValCount == BLOOM_PER_ENTRY_VALS_COUNT) || (iKeyValCount == PERCOLATE_BLOOM_WILD_COUNT) );
	switch ( 4*(iInfixCodepointCount == BLOOM_NGRAM_1) + 2*(iKeyValCount == PERCOLATE_BLOOM_WILD_COUNT) + !!bUtf8 )
	{
	case 0: return BuildBloom_T<TRAITS, BLOOM_NGRAM_0, BLOOM_PER_ENTRY_VALS_COUNT, false> (sWord, iLen, tBloom);
	case 1: return BuildBloom_T<TRAITS, BLOOM_NGRAM_0, BLOOM_PER_ENTRY_VALS_COUNT, true>  (sWord, iLen, tBloom);
	case 2: return BuildBloom_T<TRAITS, BLOOM_NGRAM_0, PERCOLATE_BLOOM_WILD_COUNT, false> (sWord, iLen, tBloom);
	case 3: return BuildBloom_T<TRAITS, BLOOM_NGRAM_0, PERCOLATE_BLOOM_WILD_COUNT, true>  (sWord, iLen, tBloom);
	case 4: return BuildBloom_T<TRAITS, BLOOM_NGRAM_1, BLOOM_PER_ENTRY_VALS_COUNT, false> (sWord, iLen, tBloom);
	case 5: return BuildBloom_T<TRAITS, BLOOM_NGRAM_1, BLOOM_PER_ENTRY_VALS_COUNT, true>  (sWord, iLen, tBloom);
	case 6: return BuildBloom_T<TRAITS, BLOOM_NGRAM_1, PERCOLATE_BLOOM_WILD_COUNT, false> (sWord, iLen, tBloom);
	case 7: return BuildBloom_T<TRAITS, BLOOM_NGRAM_1, PERCOLATE_BLOOM_WILD_COUNT, true>  (sWord, iLen, tBloom);
	}
	assert(0);
	return false;
}

// explicit instantiations
bool BuildBloom ( const BYTE * sWord, int iLen, int iInfixCodepointCount, bool bUtf8, int iKeyValCount
				  , BloomCheckTraits_t &tBloom )
{
	return BuildBloom_T ( sWord, iLen, iInfixCodepointCount, bUtf8, iKeyValCount, tBloom );
}

bool BuildBloom ( const BYTE * sWord, int iLen, int iInfixCodepointCount, bool bUtf8, int iKeyValCount
				  , BloomGenTraits_t &tBloom )
{
	return BuildBloom_T ( sWord, iLen, iInfixCodepointCount, bUtf8, iKeyValCount, tBloom );
}

void BuildSegmentInfixes ( RtSegment_t * pSeg, bool bHasMorphology, DictFormat_e eDictFormat, int iMinInfixLen, int iWordsCheckpoint, bool bUtf8, ESphHitless eHitlessMode )
{
	bool bKeywordDict = eDictFormat!=DictFormat_e::CRC;
	if ( !pSeg || !bKeywordDict || !iMinInfixLen )
		return;

	int iBloomSize = ( pSeg->m_dWordCheckpoints.GetLength()+1 ) * BLOOM_PER_ENTRY_VALS_COUNT * BLOOM_HASHES_COUNT;
	pSeg->m_dInfixFilterCP.Resize ( iBloomSize );
	// reset filters
	pSeg->m_dInfixFilterCP.Fill ( 0 );

	uint64_t * pRough = pSeg->m_dInfixFilterCP.Begin();
	RtWordReader_c rdDictRough ( pSeg, eDictFormat, iWordsCheckpoint, eHitlessMode );
	while ( rdDictRough.UnzipWord() )
	{
		const BYTE * pDictWord = rdDictRough->m_sWord;
		int iLen = rdDictRough->m_iWordLen;
		if ( bHasMorphology && *pDictWord!=MAGIC_WORD_HEAD_NONSTEMMED )
			continue;

		if ( *pDictWord<0x20 ) // anyway skip heading magic chars in the prefix, like NONSTEMMED maker
		{
			pDictWord++;
			iLen--;
		}
		if ( iLen<=0 )
			continue;

		uint64_t * pVal = pRough + rdDictRough.Checkpoint() * BLOOM_PER_ENTRY_VALS_COUNT * BLOOM_HASHES_COUNT;
		BloomGenTraits_t tBloom0 ( pVal );
		BloomGenTraits_t tBloom1 ( pVal+BLOOM_PER_ENTRY_VALS_COUNT );
		BuildBloom ( pDictWord, iLen, BLOOM_NGRAM_0, bUtf8, BLOOM_PER_ENTRY_VALS_COUNT, tBloom0 );
		BuildBloom ( pDictWord, iLen, BLOOM_NGRAM_1, bUtf8, BLOOM_PER_ENTRY_VALS_COUNT, tBloom1 );
	}
}


CSphFixedVector<RowID_t> RtIndex_c::CopyAttributesFromAliveDocs ( RtSegment_t & tDstSeg, const RtSegment_t & tSrcSeg, RtAttrMergeContext_t & tCtx ) const REQUIRES ( tDstSeg.m_tLock ) REQUIRES (m_tWorkers.SerialChunkAccess())
{
	CSphFixedVector<RowID_t> dRowMap { tSrcSeg.m_uRows };
	dRowMap.Fill ( INVALID_ROWID );

	// mark us busy - updates to this seg will be collected.
	tSrcSeg.m_bAttrsBusy.store ( true, std::memory_order_release );

	// perform merging attrs in single fiber - that eliminates concurrency with optimize
//	ScopedScheduler_c tSerialFiber { m_tWorkers.SerialChunkAccess() };
	TRACE_SCHED ( "rt", "CopyAttrs" );

	auto dColumnarIterators = CreateAllColumnarIterators ( tSrcSeg.m_pColumnar.get(), m_tSchema );
	CSphVector<int64_t> dTmp;

	const CSphColumnInfo * pBlobRowLocator = m_tSchema.GetAttr ( sphGetBlobLocatorName() );

	// ensure no update will break (resize) blob when we're merging
	SccRL_t rLock ( tSrcSeg.m_tLock );

	for ( auto tRowID : RtLiveRows_c(tSrcSeg) )
	{
		CSphRowitem * pRow = tSrcSeg.m_dRows.Begin() + (int64_t)tRowID * m_iStride;
		auto pNewRow = tDstSeg.m_dRows.AddN ( m_iStride );
		memcpy ( pNewRow, pRow, m_iStride*sizeof(CSphRowitem) );

		if ( tCtx.m_iNumBlobs )
		{
			assert ( pBlobRowLocator ) ;
			int64_t iOldOffset = sphGetRowAttr ( pRow , pBlobRowLocator->m_tLocator );
			int64_t iNewOffset = sphCopyBlobRow ( tDstSeg.m_dBlobs, tSrcSeg.m_dBlobs, iOldOffset, tCtx.m_iNumBlobs );
			sphSetRowAttr ( pNewRow, pBlobRowLocator->m_tLocator, iNewOffset );
		}

		ARRAY_FOREACH ( i, dColumnarIterators )
		{
			auto & tIt = dColumnarIterators[i];
			SetColumnarAttr ( i, tIt.second, tCtx.m_pColumnarBuilder, tIt.first, tRowID, dTmp );
		}

		if ( tDstSeg.m_pDocstore )
			tDstSeg.m_pDocstore->AddPackedDoc ( tCtx.m_tResultRowID, tSrcSeg.m_pDocstore.get(), tRowID );

		dRowMap[tRowID] = tCtx.m_tResultRowID++;
	}
	return dRowMap;
}


int RtIndex_c::CompareWords ( const RtWord_t * pWord1, const RtWord_t * pWord2, bool bWordDict ) const
{
	if ( !pWord1 )
		return pWord2 ? 1 : 0;

	if ( !pWord2 )
		return -1;

	if ( bWordDict )
		return sphDictCmpStrictly ( (const char *)pWord1->m_sWord, pWord1->m_iWordLen, (const char *)pWord2->m_sWord, pWord2->m_iWordLen );

	if ( pWord1->m_uWordID<pWord2->m_uWordID )
		return -1;

	if ( pWord1->m_uWordID>pWord2->m_uWordID )
		return 1;

	return 0;
}


void RtIndex_c::MergeKeywords ( RtSegment_t & tSeg, const RtSegment_t & tSeg1, const RtSegment_t & tSeg2,
		const VecTraits_T<RowID_t> & dRowMap1, const VecTraits_T<RowID_t> & dRowMap2 ) const
{
	tSeg.m_dWords.Reserve ( Max ( tSeg1.m_dWords.GetLength(), tSeg2.m_dWords.GetLength() ) );
	tSeg.m_dDocs.Reserve ( Max ( tSeg1.m_dDocs.GetLength(), tSeg2.m_dDocs.GetLength() ) );
	tSeg.m_dHits.Reserve ( Max ( tSeg1.m_dHits.GetLength(), tSeg2.m_dHits.GetLength() ) );

	RtDocWriter_c tOutDoc ( tSeg.m_dDocs );
	RtWordWriter_c tOut ( tSeg.m_dWords, tSeg.m_dWordCheckpoints, tSeg.m_dKeywordCheckpoints, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );
	RtWordReader_c tIn1 ( &tSeg1, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );
	RtWordReader_c tIn2 ( &tSeg2, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );
	const bool bWordDict = IsWordDict();

	const RtWord_t* pWords1 = tIn1.UnzipWord();
	const RtWord_t* pWords2 = tIn2.UnzipWord();

	while ( pWords1 || pWords2 )
	{
		int iCmp = CompareWords ( pWords1, pWords2, bWordDict );
		RtWord_t tWord = iCmp<=0 ? *pWords1 : *pWords2;
		tWord.m_uDocs = 0;
		tWord.m_uHits = 0;
		tWord.m_uDoc = tOutDoc.WriterPos();

		// if words are equal, copy both
		if ( iCmp<=0 )
			CopyWord ( tSeg, tWord, tOutDoc, tSeg1, pWords1, dRowMap1 );

		if ( iCmp>=0 )
			CopyWord ( tSeg, tWord, tOutDoc, tSeg2, pWords2, dRowMap2 );

		// append non-empty word to the dictionary
		if ( tWord.m_uDocs )
			tOut << tWord;

		// move forward. Beware, tWord refer to static buffer inside tIn1 or tIn2, so call UnzipWord() before storing the word is NOT safe!
		if ( iCmp <= 0 )
			pWords1 = tIn1.UnzipWord(); // move forward

		if ( iCmp >= 0 )
			pWords2 = tIn2.UnzipWord(); // move forward

		tOutDoc.ZipRestart();
	}
}


// it seems safe to kill documents directly during merging.
// already killed will not come to the merged.
// killed after pass of merge attributes will survive, and need to be killed finally by separate killmulti
RtSegment_t* RtIndex_c::MergeTwoSegments ( const RtSegment_t* pA, const RtSegment_t* pB ) const REQUIRES (m_tWorkers.SerialChunkAccess())
{
	////////////////////
	// merge attributes
	////////////////////
	RTRLOG << "MergeTwoSegments invoked";

	int nBlobAttrs = 0;
	for ( int i = 0; i < m_tSchema.GetAttrsCount(); ++i )
		if ( sphIsBlobAttr ( m_tSchema.GetAttr(i) ) )
			++nBlobAttrs;

	RowID_t tNextRowID = 0;

	bool bBothConsistent = CheckSegmentConsistency ( pA ) && CheckSegmentConsistency ( pB );

	auto pColumnarBuilder = CreateColumnarBuilderRT(m_tSchema);
	RtAttrMergeContext_t tCtx ( nBlobAttrs, tNextRowID, pColumnarBuilder.get() );

	auto * pSeg = new RtSegment_t (0, m_tSchema);
	FakeWL_t _ { pSeg->m_tLock }; // as pSeg is just created - we don't need real guarding and use fake lock to mute thread safety warnings

	assert ( !!pA->m_pDocstore==!!pB->m_pDocstore );
	if ( ( m_tSchema.HasStoredFields() || m_tSchema.HasStoredAttrs() ) && pA->m_pDocstore && pB->m_pDocstore )
		pSeg->SetupDocstore ( &m_tSchema );

	// we might need less because of killed, but we can not know yet. Reserving more than necessary is strictly not desirable!
	pSeg->m_dRows.Reserve ( m_iStride * Max ( pA->m_tAliveRows.load ( std::memory_order_relaxed ), pB->m_tAliveRows.load (std::memory_order_relaxed ) ) );
	pSeg->m_dBlobs.Reserve ( [pA, pB]() NO_THREAD_SAFETY_ANALYSIS { return Max ( pA->m_dBlobs.GetLength(), pB->m_dBlobs.GetLength() ); }() );

	CSphFixedVector<RowID_t> dRowMapA = CopyAttributesFromAliveDocs ( *pSeg, *pA, tCtx );
	CSphFixedVector<RowID_t> dRowMapB = CopyAttributesFromAliveDocs ( *pSeg, *pB, tCtx );

	assert ( tNextRowID<=INT_MAX );
	pSeg->m_uRows = tNextRowID;
	pSeg->m_tAliveRows.store ( pSeg->m_uRows, std::memory_order_relaxed );
	pSeg->m_tDeadRowMap.Reset ( pSeg->m_uRows );
	pSeg->m_pColumnar = CreateColumnarRT ( m_tSchema, pColumnarBuilder.get() );

	RTRLOG << "MergeTwoSegments: new seg has " << pSeg->m_uRows << " rows";

	// merged segment might be completely killed by committed data
	if ( !pSeg->m_uRows )
	{
		SafeRelease ( pSeg );
		return nullptr;
	}

	assert ( pSeg->GetStride() == m_iStride );
	pSeg->BuildDocID2RowIDMap ( m_tSchema );
	MergeKeywords ( *pSeg, *pA, *pB, dRowMapA, dRowMapB );

	if ( IsWordDict() )
		FixupSegmentCheckpoints ( pSeg );

	BuildSegmentInfixes ( pSeg, m_pDict->HasMorphology(), GetDictFormat(), m_tSettings.m_iMinInfixLen, m_iWordsCheckpoint, ( m_iMaxCodepointLength>1 ), m_tSettings.m_eHitless );

	assert ( pSeg->m_uRows );
	assert ( pSeg->m_tAliveRows==pSeg->m_uRows );

	if ( bBothConsistent && !CheckSegmentConsistency ( pSeg, false ) )
		DumpMerge ( pA, pB, pSeg );

	return pSeg;
}

namespace GatherUpdates {

	struct UpdHashFn { static inline uintptr_t Hash ( const CSphAttrUpdate* pK ) { return (uintptr_t) pK; } };
	using HashedUpd_t = CSphOrderedHash<PostponedUpdate_t*, AttrUpdateSharedPtr_t, UpdHashFn, 256>;

	inline bool CanBePeeked (int i, const CSphVector<HashedUpd_t>& dUpdates, const CSphVector<HashedUpd_t::Iterator_c>& dHeads) noexcept
	{
		ARRAY_CONSTFOREACH ( j, dHeads )
			if ( dHeads[i]->first!=dHeads[j]->first && dUpdates[j].Exists ( dHeads[i]->first ) )
				return false;
		return true;
	};

	inline void Peek ( int i, const CSphVector<HashedUpd_t> & dUpdates, CSphVector<HashedUpd_t::Iterator_c> & dHeads, CSphVector<PostponedUpdate_t> & dSerializedUpdates )
	{
		auto a = dHeads[i]->first;
		auto& tNewUpdate = dSerializedUpdates.Add();
		tNewUpdate.m_pUpdate = a;
		ARRAY_FOREACH ( j, dHeads )
			if ( a==dHeads[j]->first )
			{
				tNewUpdate.m_dRowsToUpdate.Append ( dHeads[j]->second->m_dRowsToUpdate );
				dHeads[j]->second->m_pUpdate = nullptr;
				dHeads[j]->second->m_dRowsToUpdate.Reset();
				++dHeads[j];
				if ( dHeads[j]==dUpdates[j].end() )
					dHeads.RemoveFast (j--);
			}
	};

	const VecTraits_T<PostponedUpdate_t>& AccessPostponedUpdates ( const ConstRtSegmentRefPtf_t& pSeg ) { return pSeg->m_dPostponedUpdates; }
	const VecTraits_T<PostponedUpdate_t>& AccessPostponedUpdates ( const ConstDiskChunkRefPtr_t& pChunk ) { return pChunk->Cidx ().m_dPostponedUpdates; }

	template<typename CHUNK_OR_SEG>
	CSphVector<PostponedUpdate_t> FromChunksOrSegments ( VecTraits_T<CHUNK_OR_SEG> dChunksOrSegments )
	{
		CSphVector<PostponedUpdate_t> dResult;
		CSphVector<HashedUpd_t> dUpdates;

		for ( const auto& dSeg : dChunksOrSegments )
		{
			const VecTraits_T<PostponedUpdate_t>& dPostponedUpdates = AccessPostponedUpdates (dSeg);
			if ( dPostponedUpdates.IsEmpty () )
				continue;
			auto& dHash = dUpdates.Add();
			for ( auto& tPostponedUpdate : dPostponedUpdates )
				dHash.Add ( &tPostponedUpdate, tPostponedUpdate.m_pUpdate );
		}

		if ( dUpdates.IsEmpty() )
			return dResult;

		CSphVector<HashedUpd_t::Iterator_c> dHeads ( dUpdates.GetLength() );
		ARRAY_CONSTFOREACH ( i, dUpdates )
			dHeads[i] = dUpdates[i].begin();

		while (!dHeads.IsEmpty())
			ARRAY_FOREACH ( i, dHeads )
				if ( CanBePeeked ( i, dUpdates, dHeads ) )
				{
					Peek ( i, dUpdates, dHeads, dResult );
					break;
				}

		return dResult;
	}

}; // namespace


// that is 2-nd part of postponed updates. We may have one or several update set, stored from old segments.
void RtSegment_t::UpdateAttributesOffline ( VecTraits_T<PostponedUpdate_t>& dPostUpdates ) NO_THREAD_SAFETY_ANALYSIS
{
	if ( dPostUpdates.IsEmpty() )
		return;

	CSphString sError;
	bool bCritical;

	assert ( GetStride() == m_tSchema.GetRowSize() );
	for ( auto & tPostUpdate : dPostUpdates )
	{
		AttrUpdateInc_t tUpdInc { std::move ( tPostUpdate.m_pUpdate ) };
		UpdateContext_t tCtx ( tUpdInc, m_tSchema );
		tCtx.PrepareListOfUpdatedAttributes ( sError );

		// actualize list of updates in context of new segment
		const auto & dDocids = tUpdInc.m_pUpdate->m_dDocids;
		ARRAY_FOREACH ( i, tPostUpdate.m_dRowsToUpdate )
		{
			auto& tRow = tPostUpdate.m_dRowsToUpdate[i];
			auto tRowID = GetAliveRowidByDocid ( dDocids[tRow.m_iIdx] );
			if ( tRowID==INVALID_ROWID )
				tPostUpdate.m_dRowsToUpdate.RemoveFast ( i-- );
			else
				tRow.m_tRow = tRowID;
		}

		tCtx.m_pAttrPool = m_dRows.begin();
		tCtx.m_pBlobPool = m_dBlobs.begin();
		Update_UpdateAttributes ( tPostUpdate.m_dRowsToUpdate, tCtx, bCritical, sError );
	}
}

static void CleanupHitDuplicates ( CSphTightVector<CSphWordHit> & dHits )
{
	TRACE_CONN ( "conn", "CleanupHitDuplicates" );

	if ( dHits.GetLength()<2 )
		return;

	int iSrc = 1, iDst = 1;
	while ( iSrc<dHits.GetLength() )
	{
		CSphWordHit & tDst = dHits[iDst-1];
		const CSphWordHit & tSrc = dHits[iSrc];

		DWORD uDstPos = HITMAN::GetPosWithField ( tDst.m_uWordPos );
		DWORD uSrcPos = HITMAN::GetPosWithField ( tSrc.m_uWordPos );
		DWORD uDstField = HITMAN::GetField ( tDst.m_uWordPos );
		DWORD uSrcField = HITMAN::GetField ( tSrc.m_uWordPos );
		bool bDstEnd = HITMAN::IsEnd ( tDst.m_uWordPos );

		// check for pure duplicate and multiple tail hits
		if ( tDst.m_tRowID==tSrc.m_tRowID && tDst.m_uWordID==tSrc.m_uWordID && ( uDstPos==uSrcPos || ( uDstField==uSrcField && bDstEnd ) ) )
		{
			if ( uDstPos==uSrcPos )
			{
				dHits[iDst] = dHits[iSrc];
			} else if ( bDstEnd )
			{
				tDst.m_uWordPos = HITMAN::CreateSum ( tDst.m_uWordPos, 0 ); // reset field end flag
				dHits[iDst] = dHits[iSrc];
				iDst++;
			}
			iSrc++;

		} else
		{
			dHits[iDst++] = dHits[iSrc++];
		}
	}

	dHits.Resize ( iDst );
}

static void PreCommitAcc ( RtAccum_t & tAcc, int iRowSize )
{
	tAcc.CleanupDuplicates ( iRowSize );
	tAcc.Sort();
	CleanupHitDuplicates ( tAcc.m_dAccum );
}

bool RtIndex_c::Commit ( int * pDeleted, RtAccum_t * pAcc, CSphString * pError )
{
	TRACE_CONN ( "conn", "RtIndex_c::Commit" );

	assert ( RTChangesAllowed () );
	MEMORY ( MEM_INDEX_RT );

	if ( !BindAccum ( pAcc ) )
		return true;

	// empty txn, just ignore
	if ( !pAcc->m_uAccumDocs && pAcc->m_dAccumKlist.IsEmpty() )
	{
		pAcc->Cleanup();
		return true;
	}

	if ( pAcc->GetIndexGeneration()!=m_iAlterGeneration )
	{
		if ( pError )
			pError->SetSprintf( "Can't commit to table '%s', table was altered during txn!", GetName() );
		return false;
	}


	// phase 0, build a new segment
	// accum and segment are thread local; so no locking needed yet
	// segment might be NULL if we're only killing rows this txn
	if ( !pAcc->IsPreparedForCommit() )
		PreCommitAcc ( *pAcc, m_tSchema.GetRowSize() );

	CSphString sCreateError;
	RtSegmentRefPtf_t pNewSeg { CreateSegment ( pAcc, m_iWordsCheckpoint, m_tSettings.m_eHitless, m_dHitlessWords, sCreateError ) };
	if ( !pNewSeg && !sCreateError.IsEmpty() )
	{
		if ( pError )
			*pError = sCreateError;

		return false;
	}

	assert ( !pNewSeg || pNewSeg->m_uRows>0 );
	assert ( !pNewSeg || pNewSeg->m_tAliveRows>0 );

	BuildSegmentInfixes ( pNewSeg, m_pDict->HasMorphology(), GetDictFormat(), m_tSettings.m_iMinInfixLen, m_iWordsCheckpoint, ( m_iMaxCodepointLength>1 ), m_tSettings.m_eHitless );

	// clean up parts we no longer need
	pAcc->CleanupPart();

	// sort accum klist, too
	pAcc->m_dAccumKlist.Sort ( Lesser ( [] ( DocID_t a, DocID_t b ) { return (uint64_t)a < (uint64_t)b; } ) );
	pAcc->m_dAccumKlist.Uniq (false);

	// now on to the stuff that needs locking and recovery
	CSphString sError;
	int iKilled = 0;
	if ( !CommitReplayable ( pNewSeg, pAcc->m_dAccumKlist, pAcc->m_iAccumBytes, iKilled, sError ) )
	{
		if ( pError )
			*pError = sError;
		return false;
	}

	if ( pDeleted )
		*pDeleted = iKilled;

	// done; cleanup accum
	pAcc->Cleanup();

	// reset accumulated warnings
	CSphString sWarning;
	pAcc->GrabLastWarning ( sWarning );
	return true;
}

bool RtIndex_c::PreCommit ( RtAccum_t * pAccExt, CSphString & sError )
{
	if ( !pAccExt || ( !pAccExt->m_uAccumDocs && pAccExt->m_dAccumKlist.IsEmpty() ) )
		return true;

	PreCommitAcc ( *pAccExt, m_tSchema.GetRowSize() );

	if ( !pAccExt->FetchEmbeddings ( m_pEmbeddings.get(), m_dAttrsWithModels, sError ) )
		return false;

	pAccExt->MarkPreparedForCommit();
	return true;
}

ConstRtSegmentRefPtf_t RtIndex_c::AdoptSegment ( RtSegment_t * pNewSeg )
{
	ConstRtSegmentRefPtf_t tResult {pNewSeg};
	if ( pNewSeg )
	{
		pNewSeg->AddRef ();
		pNewSeg->m_pRAMCounter = &m_iRamChunksAllocatedRAM;
		pNewSeg->UpdateUsedRam();
	}
	return tResult;
}

// CommitReplayable -> ApplyKillList
// AttachDiskIndex -> ApplyKillList
int RtIndex_c::ApplyKillList ( const VecTraits_T<DocID_t> & dAccKlist )
{
	if ( dAccKlist.IsEmpty() )
		return 0;

	assert ( Coro::CurrentScheduler() == m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "ApplyKillList" );

	int iKilled = 0;
	auto pChunks = m_tRtChunks.DiskChunks();

	if ( !Threads::IsInsideCoroutine() || m_tSaving.ActiveStateIs ( SaveState_c::ENABLED ) )
		for ( auto& pChunk : *pChunks )
			iKilled += pChunk->CastIdx().KillMulti ( dAccKlist );
	else
	{
		// if saving is disabled, and we NEED to actually mark a doc in disk chunk as deleted,
		// we'll pause that action, waiting until index is unlocked.
		bool bNeedWait = true;
		bool bEnabled = false;
		for ( auto& pChunk : *pChunks )
			iKilled += pChunk->CastIdx().CheckThenKillMulti ( dAccKlist, [this,&bNeedWait, &bEnabled]()
			{
				if ( bNeedWait )
				{
					bNeedWait = false;
					bEnabled = m_tSaving.WaitEnabledOrShutdown();
				}
				return bEnabled;
			});
	}

	auto pSegs = m_tRtChunks.RamSegs();
	for ( auto& pSeg : *pSegs )
		iKilled += const_cast<RtSegment_t*> ( pSeg.Ptr() )->KillMulti ( dAccKlist );

	return iKilled;
}

inline std::pair<int,int> Find2Minimums ( const VecTraits_T<ConstRtSegmentRefPtf_t>& dSegments )
{
	assert ( dSegments.GetLength() > 1 );
	auto fnLess = [&dSegments] ( int a, int b ) { return dSegments[a]->GetMergeFactor() < dSegments[b]->GetMergeFactor(); };

	int a, b;
	if ( fnLess ( 0, 1 ) )
		a = 0, b = 1;
	else
		a = 1, b = 0;

	for ( int i = 2; i < dSegments.GetLength(); ++i )
		if ( fnLess ( i, a ) )
		{
			b = a;
			a = i;
		} else if ( fnLess ( i, b ) )
			b = i;
	return { a, b };
}

enum class CheckMerge_e { MERGE, NOMERGE, FLUSH, FLUSH_EM };
inline CheckMerge_e CheckSegmentsPair ( std::pair<const RtSegment_t*, const RtSegment_t*> tPair, int64_t iRamLeft=INT64_MAX ) NO_THREAD_SAFETY_ANALYSIS
{
	const auto* pA = tPair.first;
	const auto* pB = tPair.second;

	int64_t iAlive = pA->m_tAliveRows.load ( std::memory_order_relaxed ) + pB->m_tAliveRows.load ( std::memory_order_relaxed );
	DWORD uRows = pA->m_uRows + pB->m_uRows;

	int64_t iEstimatedMergedSize=0;
	int64_t iMaxFutureVecLen=0;

	// check whether we have enough RAM
#define ESTIMATE( _v ) pA->_v.Relimit ( 0, ( (int64_t)pA->_v.GetLength() + pB->_v.GetLength() ) * iAlive / uRows )
#define LOC_ESTIMATE( _v ) do { auto _t=ESTIMATE(_v); iEstimatedMergedSize+=_t; if (iMaxFutureVecLen<_t) iMaxFutureVecLen=_t; } while (0)

	LOC_ESTIMATE ( m_dWords );
	LOC_ESTIMATE ( m_dDocs );
	LOC_ESTIMATE ( m_dHits );
	LOC_ESTIMATE ( m_dBlobs );
	LOC_ESTIMATE ( m_dKeywordCheckpoints );
	LOC_ESTIMATE ( m_dRows );

	// Both source maps are alive while the merged maps are built. Their
	// combined allocations are conservative bounds for the new maps.
	iEstimatedMergedSize += pA->m_tDocIDtoRowID.GetLengthBytes() + pB->m_tDocIDtoRowID.GetLengthBytes();
	iEstimatedMergedSize += pA->m_tUuidDocID.GetLengthBytes() + pB->m_tUuidDocID.GetLengthBytes();

#undef LOC_ESTIMATE
#undef ESTIMATE

	if ( iEstimatedMergedSize > iRamLeft )
		return CheckMerge_e::NOMERGE;
	if ( iMaxFutureVecLen > MAX_SEGMENT_VECTOR_LEN )
		return CheckMerge_e::FLUSH;
	return CheckMerge_e::MERGE;
}

inline CheckMerge_e CheckWeCanMerge ( std::pair<int, int>& tSmallest, const VecTraits_T<ConstRtSegmentRefPtf_t>& dSegments, int64_t iHardRamLeft, int64_t iSoftRamLeft, bool bNewAdded ) NO_THREAD_SAFETY_ANALYSIS
{
	const int iSegs = dSegments.GetLength ();

	RTLOGV << "CheckWeCanMerge(" << dSegments.GetLength() << " segs, ram soft limit " << iSoftRamLeft << " bytes, ram hard limit " << iHardRamLeft << " bytes)";

	if ( !bNewAdded && iSegs < MAX_SEGMENTS )
		return CheckMerge_e::NOMERGE;

	auto eFLUSH = CheckMerge_e::FLUSH;
	if ( iHardRamLeft<iSoftRamLeft )
	{
		iSoftRamLeft = iHardRamLeft;
		eFLUSH = CheckMerge_e::FLUSH_EM; // emergency flush. I.e. hard limit reached
	}

	// skip merging if no memory left
	if ( iSoftRamLeft <= 0 )
		return eFLUSH;

	// if N of segments is not so big - no merge need
	if ( iSegs < ( MAX_SEGMENTS - MAX_PROGRESSION_SEGMENT ) )
		return CheckMerge_e::NOMERGE;

	// take 2 smallest segments
	assert ( iSegs > 1 );
	tSmallest = Find2Minimums ( dSegments );
	ConstRtSegmentRefPtf_t & pA = dSegments[tSmallest.first];
	ConstRtSegmentRefPtf_t & pB = dSegments[tSmallest.second];

	// exit if progression is kept AND lesser MAX_SEGMENTS limit
	if ( pB->GetMergeFactor() > pA->GetMergeFactor() * 2 && iSegs<MAX_SEGMENTS )
		return CheckMerge_e::NOMERGE;

	auto eDecision = CheckSegmentsPair ( {pA, pB}, iSoftRamLeft );
	switch ( eDecision )
	{
	case CheckMerge_e::NOMERGE:
		return ( iSegs >= MAX_SEGMENTS ) ? eFLUSH : CheckMerge_e::NOMERGE;
	case CheckMerge_e::FLUSH:
		return eFLUSH;
	case CheckMerge_e::MERGE:
	default:
		return CheckMerge_e::MERGE;
	}
}

static StringBuilder_c & operator<< ( StringBuilder_c & dOut, CheckMerge_e eVal )
{
	switch ( eVal )
	{
	case CheckMerge_e::MERGE: return dOut << "MERGE";
	case CheckMerge_e::NOMERGE: return dOut << "NOMERGE";
	case CheckMerge_e::FLUSH: return dOut << "FLUSH";
	case CheckMerge_e::FLUSH_EM: return dOut << "FLUSH_EM";
	default: dOut.Sprintf ( "UNKNWN(%d)", (int)eVal );
	}
	return dOut;
}

void RtIndex_c::StartMergeSegments ( MergeSeg_e eMergeWhat, bool bNotify ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	TRACE_SCHED ( "rt", "StartMergeSegments" );
	m_eSegMergeQueued.ModifyValue ( [&eMergeWhat, &bNotify] ( MergeSeg_e& ePrevVal )
	{
		if ( ePrevVal == MergeSeg_e::EXIT )
			bNotify = false;
		else
			ePrevVal = eMergeWhat;
	});

	if ( bNotify )
		m_eSegMergeQueued.NotifyOne();
}

void RtIndex_c::StopMergeSegmentsWorker()
{
	TRACE_SCHED ( "rt", "StopMergeSegmentsWorker" );
	m_eSegMergeQueued.SetValueAndNotifyOne ( MergeSeg_e::EXIT );
}

bool RtIndex_c::MergeSegmentsStep ( MergeSeg_e eVal ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	TRACE_SCHED ( "rt", "MergeSegmentsStep" );
	// collect all RAM segments not occupied by any op (only ops we know is 'merge segments' and 'save ram chunk')
	int64_t iHardRamLeft { m_iRtMemLimit };
	int64_t iSoftRamLeft { m_iSoftRamLimit };
	LazyVector_T<ConstRtSegmentRefPtf_t> dSegments;
	for ( const auto& dSeg : *m_tRtChunks.RamSegs() )
	{
		if ( !dSeg->m_iLocked )
		{
			dSegments.Add ( dSeg );
			iSoftRamLeft -= dSeg->GetUsedRam();
		}
		iHardRamLeft -= dSeg->GetUsedRam();
	}

	TRACE_COUNTER ( "mem", perfetto::CounterTrack ( "SoftRamLeft", "bytes" ).set_unit_multiplier ( 1024 ), iSoftRamLeft >> 10 );
	TRACE_COUNTER ( "mem", perfetto::CounterTrack ( "HardRamLeft", "bytes" ).set_unit_multiplier ( 1024 ), iHardRamLeft >> 10 );

	RTLOGV << "Totally we have " << m_tRtChunks.GetRamSegmentsCount() << " segments onboard.";

	std::pair<int, int> tSmallest;
	auto eMergeAction = CheckWeCanMerge ( tSmallest, dSegments, iHardRamLeft, iSoftRamLeft, eVal == MergeSeg_e::NEWSEG );
	RTLOGV << "CheckWeCanMerge returned " << eMergeAction;

	if ( eMergeAction == CheckMerge_e::FLUSH || eMergeAction == CheckMerge_e::FLUSH_EM )
	{
		// here it might be no race, as we're in serial worker.
		TRACE_SCHED ( "wait", "MergeSegmentsStep-wait-save" );
		auto iOldGen = m_iSavedGeneration;
		m_tNSavesNow.Wait ( [] ( int iVal ) { return iVal < SIMULTANEOUS_SAVE_LIMIT; } );

		// if a save finished during wait - limits and conditions may be changed, will restart to check whether save is still necessary
		if ( m_iSavedGeneration != iOldGen )
		{
			RTLOGV << "Recheck due to just finished SaveDiskChunk";
			return true;
		}

		StartRoutine();
		Coro::Go ( [this, eMergeAction]() REQUIRES ( m_tWorkers.SerialChunkAccess() ) {
			SaveDiskChunk ( false, eMergeAction == CheckMerge_e::FLUSH_EM );
			StopRoutine();
		}, m_tWorkers.SerialChunkAccess() );
		return false; // exit into idle
	}

	int iMergeOp = 0;

	RtSegmentRefPtf_t pMerged { nullptr };

	if ( eMergeAction == CheckMerge_e::MERGE )
	{
		assert ( dSegments.GetLength() >= 2 );

		ConstRtSegmentRefPtf_t pA = dSegments[tSmallest.first];
		ConstRtSegmentRefPtf_t pB = dSegments[tSmallest.second];
		iMergeOp = m_tWorkers.GetNextOpTicket();
		pA->m_iLocked = pB->m_iLocked = iMergeOp; // mark them as retiring.

		pMerged = MergeTwoSegments ( pA, pB );

		if ( pMerged && pMerged->m_tAliveRows.load ( std::memory_order_relaxed ) )
		{
			// some updates might be applied to pA and pB during the merge. Now it is time to apply them also
			// to the merged segment.
			LazyVector_T<ConstRtSegmentRefPtf_t> dOld;
			dOld.Add ( pA );
			dOld.Add ( pB );
			auto dUpdates = GatherUpdates::FromChunksOrSegments ( dOld );
			pMerged->UpdateAttributesOffline ( dUpdates );
		}
	}

	// merged might be killed during merge op
	// as we run in serial fiber, there is no concurrency, and so, killing hooks of retired sources will not fire.
	assert ( Coro::CurrentScheduler() == m_tWorkers.SerialChunkAccess() );

	if ( pMerged && !pMerged->m_tAliveRows.load ( std::memory_order_relaxed ) )
		pMerged = nullptr;

	// we collect after merge, as some data might be changed during the merge
	for ( auto& pSeg : dSegments )
		if ( !pSeg->m_tAliveRows.load ( std::memory_order_relaxed ) )
		{
			if ( !iMergeOp )
				iMergeOp = m_tWorkers.GetNextOpTicket();
			pSeg->m_iLocked = iMergeOp;
		}
	dSegments.Reset();

	// nothing merged, and also nothing killed - nothing to do, exit into idle
	if ( !pMerged && !iMergeOp )
		return false;

	auto tNewSet = RtWriter();
	tNewSet.InitRamSegs ( RtWriter_c::empty );
	for ( const auto& pSeg : *m_tRtChunks.RamSegs() )
		if ( pSeg->m_iLocked != iMergeOp )
			tNewSet.m_pNewRamSegs->Add ( pSeg );

	if ( pMerged )
		tNewSet.m_pNewRamSegs->Add ( AdoptSegment ( pMerged ) );

	RTRLOG << "after merge " << tNewSet.m_pNewRamSegs->GetLength() << " segments on-board";
	TRACE_COUNTER ( "rt", perfetto::CounterTrack ( "Segments" ), tNewSet.m_pNewRamSegs->GetLength() );


	// chain next step
	return true;
}

void RtIndex_c::RunMergeSegmentsWorker()
{
	StartRoutine();
	Coro::Go ( [this]() REQUIRES ( m_tWorkers.SerialChunkAccess() )
	{
		TRACE_SCHED ( "rt", "RunMergeSegmentsWorker-lambda" );
		auto tResetSegMergeWorking = AtScopeExit ( [this] { StopRoutine(); } );

		while (true)
		{
			m_eSegMergeQueued.Wait ( [] ( MergeSeg_e eVal ) { return eVal != MergeSeg_e::NONE; } );
			auto eVal = m_eSegMergeQueued.ExchangeValue ( MergeSeg_e::NONE );

			assert ( eVal != MergeSeg_e::NONE );
			if ( eVal==MergeSeg_e::EXIT )
				return;

			if ( MergeSegmentsStep ( eVal ) )
				StartMergeSegments ( MergeSeg_e::NEWSEG, false );
		}
	}, m_tWorkers.SerialChunkAccess() );
}

namespace {
int CommitID() {
	static std::atomic<int> iWorker { 0 };
	return iWorker.fetch_add ( 1, std::memory_order_relaxed );
}
} // namespace


static void CalcFieldLengths ( const RtSegment_t * pSeg, const CSphSchema & tSchema, CSphVector<int64_t> & dLengths )
{
	int iFirstFieldLenAttr = tSchema.GetAttrId_FirstFieldLen();
	if ( !pSeg || iFirstFieldLenAttr<0 )
		return;

	int iFields = tSchema.GetFieldsCount(); // shortcut
	dLengths.Resize(iFields);
	dLengths.ZeroVec();

	CSphVector<PlainOrColumnar_t> dFieldLenAttrs;
	int iColumnar = 0;
	for ( int i = 0; i < tSchema.GetAttrsCount(); i++ )
	{
		const CSphColumnInfo & tAttr = tSchema.GetAttr(i);
		std::unique_ptr<Histogram_i> pHistogram = CreateHistogram ( tAttr.m_sName, tAttr.m_eAttrType );
		if ( i>=iFirstFieldLenAttr && i<=tSchema.GetAttrId_LastFieldLen() )
			dFieldLenAttrs.Add ( { tAttr, iColumnar } );

		if ( tAttr.IsColumnar() )
			iColumnar++;
	}

	auto dColumnarIterators = CreateAllColumnarIterators ( pSeg->m_pColumnar.get(), tSchema );
	for ( DWORD uRow = 0; uRow < pSeg->m_uRows; uRow++ )
		for ( int iField = 0; iField < iFields; iField++ )
			dLengths[iField] += dFieldLenAttrs[iField].Get ( uRow, pSeg->GetDocinfoByRowID(uRow), dColumnarIterators );
}


bool RtIndex_c::CommitReplayable ( RtSegment_t * pNewSeg, const VecTraits_T<DocID_t> & dAccKlist, int64_t iAddTotalBytes, int & iTotalKilled, CSphString & sError ) REQUIRES_SHARED ( pNewSeg->m_tLock )
{
	// store statistics, because pNewSeg just might get merged
	[[maybe_unused]] const int iId = CommitID();
	TRACE_VARID ( "rt", "CommitReplayable", iId );
	int iNewDocs = pNewSeg ? (int)pNewSeg->m_uRows : 0;

	// helpers for SPH_ATTR_TOKENCOUNT attributes
	CSphVector<int64_t> dLengths;
	CalcFieldLengths ( pNewSeg, m_tSchema, dLengths );

	if ( pNewSeg && !CheckSegmentConsistency ( pNewSeg, false ) )
		DumpInsert ( pNewSeg );

	// We're going to modify segments, so fall into serial fiber. From here no concurrent changes may happen
	ScopedScheduler_c tSerialFiber { m_tWorkers.SerialChunkAccess() };

	// for pure kills it is not necessary to wait, as it can't increase N of segments.
	if ( pNewSeg )
	{
		TRACE_VARID ( "rt", "wait_segments", iId );
		m_tUnLockedSegments.Wait ( [] ( int iVals ) { return iVals < MAX_SEGMENTS; } );
	}

	TRACE_VARID ( "rt", "CommitReplayable.serial", iId );

	RTLOGV << "CommitReplayable";

	// first of all, binlog txn data for recovery
	if ( !BinlogCommit ( pNewSeg, dAccKlist, iAddTotalBytes, sError ) )
		return false;

	// 1. Apply kill-list to existing chunks/segments
	iTotalKilled = ApplyKillList ( dAccKlist );

	// 2. Add new RAM-segment (if any). As we 1-st kill, then add - whole change is *not* atomic, ACID is broken here.
	if ( pNewSeg )
	{
		auto tNewState = RtWriter();
		tNewState.InitRamSegs ( RtWriter_c::copy );
		tNewState.m_pNewRamSegs->Add ( AdoptSegment ( pNewSeg ) );
	}

	// update stats
	m_tStats.m_iTotalDocuments += iNewDocs - iTotalKilled;
	m_tStats.m_iTotalBytes += iAddTotalBytes;

	if ( dLengths.GetLength() )
		for ( int i = 0; i < m_tSchema.GetFieldsCount(); ++i )
		{
			m_dFieldLensRam[i] += dLengths[i];
			m_dFieldLens[i] = m_dFieldLensRam[i] + m_dFieldLensDisk[i];
		}

	// backoff segments merging and m.b. saving disk chunk (that is not our deal, other worker will do it).
	StartMergeSegments ( pNewSeg ? MergeSeg_e::NEWSEG : MergeSeg_e::KILLED );
	m_tmDataWriten.store ( sphMicroTimer(), std::memory_order_relaxed );
	ConditionalDiskChunk();

	return true;
}

void RtIndex_c::RollBack ( RtAccum_t * pAcc )
{
	assert ( RTChangesAllowed () );

	if ( BindAccum ( pAcc ) )
		pAcc->Cleanup ();
}

bool RtIndex_c::DeleteDocument ( const VecTraits_T<DocID_t> & dDocs, CSphString & sError, RtAccum_t * pAcc )
{
	assert ( RTChangesAllowed () );
	MEMORY ( MEM_RT_ACCUM );

	if ( !BindAccum ( pAcc, &sError ) )
		return false;

	// !COMMIT should handle case when uDoc what inserted in current txn here
	pAcc->ResetPreparedForCommit();
	pAcc->m_dAccumKlist.Append ( dDocs );

	return true;
}

//////////////////////////////////////////////////////////////////////////
// LOAD/SAVE
//////////////////////////////////////////////////////////////////////////

static int GetCutOff ( const MutableIndexSettings_c & tSettings, bool bKNN );

struct Checkpoint_t
{
	uint64_t m_uWord;
	uint64_t m_uOffset;
};


bool RtIndex_c::ForceDiskChunk()
{
	MEMORY ( MEM_INDEX_RT );

	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	return SaveDiskChunk ( true );
}

// could be called from naked context (i.e. without coroutine)
void RtIndex_c::ConditionalDiskChunk ( )
{
	const int64_t tmFlushWriteUs = ( m_tMutableSettings.IsSet ( MutableName_e::DISKCHUNK_FLUSH_WRITE_TIMEOUT ) ? m_tMutableSettings.m_iFlushWrite * 1'000'000 : g_iFlushWriteUs );
	if ( tmFlushWriteUs<0 )
	{
		RTSAVELOG << "not flush because FlushSearch disabled";
		return;
	}

	// skip table without either writes or searches
	const int64_t tmDataWriten = m_tmDataWriten.load ( std::memory_order_relaxed );
	const int64_t tmDataSearched = m_tmDataSearched.load ( std::memory_order_relaxed );
	if (!tmDataWriten || !tmDataSearched)
	{
		RTSAVELOG << "not flush because no data written or searched";
		return;
	}

	const int64_t tmFlushSearchUs = ( m_tMutableSettings.IsSet ( MutableName_e::DISKCHUNK_FLUSH_SEARCH_TIMEOUT ) ? m_tMutableSettings.m_iFlushSearch * 1'000'000 : g_iFlushSearchUs );

	// recent write disables auto-flush
	const int64_t tmNextFlushByWriteUs = tmDataWriten + tmFlushWriteUs;
	const int64_t tmStopFlushBySearchUs = tmDataSearched + tmFlushSearchUs;

	const int64_t tmNow = sph::MicroTimer();

	// no recent search disables auto-flush
	// means post only or archived table
	if ( tmNow > tmStopFlushBySearchUs )
	{
		RTSAVELOG << "not flush because FlushSearch exceeded";
		return;
	}

	// no need to interrupt write stream with auto-flush
	if ( tmNow < tmNextFlushByWriteUs )
	{
		RTSAVELOG << "not flush because WriteSearch not exceeded; engage to be run after " << tmNextFlushByWriteUs - tmNow;
		m_dSavingTimer.EngageAt ( tmNextFlushByWriteUs );
		return;
	}

	// no need to auto-flush during optimize
	if ( OptimizesRunning() )
	{
		int iCutoff = GetCutOff ( m_tMutableSettings, m_tSchema.HasKNNAttrs() );
		int iFinalChunks = ( m_tRtChunks.DiskChunks()->GetLength() - 1 );
		if ( iFinalChunks>=iCutoff )
		{
			RTSAVELOG << "not flush because optimize running; engage to be run after " << tmFlushWriteUs;
			m_dSavingTimer.EngageAt ( tmNow + tmFlushWriteUs );
			return;
		}
	}

	Coro::Go ( [this]() REQUIRES ( m_tWorkers.SerialChunkAccess() )
	{
		StartRoutine();
		auto tResetSegMergeWorking = AtScopeExit ( [this] { StopRoutine(); } );
		// count how many docs are in segments now not saved
		int64_t iAliveNonSavingDocs = 0;
		for (const auto& pSeg : *m_tRtChunks.RamSegs())
		{
			if (m_dSavingTickets.none_of ( [&pSeg]( int i ) { return i == pSeg->m_iLocked; } ))
				iAliveNonSavingDocs += pSeg->m_tAliveRows.load ( std::memory_order_relaxed );
		}

		if (iAliveNonSavingDocs < SMALL_INDEX_THRESH)
		{
			RTSAVELOG << "not flush because not enough docs " << iAliveNonSavingDocs;
			return;
		}

		RTSAVELOG << "finaly FLUSH";
		m_tmDataWriten.store ( 0, std::memory_order_relaxed );
		SaveDiskChunk (false);
	}, m_tWorkers.SerialChunkAccess() );
}


struct SaveDiskDataTimings_t
{
	int64_t m_tmAttributes = 0;
	int64_t m_tmDeadRowMap = 0;
	int64_t m_tmDocs = 0;
	int64_t m_tmInfix = 0;  // Time spent in infix builder SaveEntries
	int64_t m_tmCheckpoints = 0;  // Time for checkpoint writing (excluding infix)
	int64_t m_tmHeader = 0;
};

struct SaveDiskDataContext_t : public BuildHeader_t
{
	SphOffset_t						m_tDocsOffset {0};
	SphOffset_t						m_tLastDocPos {0};
	std::unique_ptr<ISphInfixBuilder>	m_pInfixer;
	CSphVector<Checkpoint_t>		m_dCheckpoints;
	CSphVector<BYTE>				m_dKeywordCheckpoints;
	CSphVector<CSphVector<RowID_t>>	m_dRowMaps;
	IndexFileBase_c					m_tFilebase;
	const ConstRtSegmentSlice_t&	m_tRamSegments;

	SaveDiskDataContext_t ( const char * szFilename, const ConstRtSegmentSlice_t& tSegs )
		: m_tFilebase ( szFilename )
		, m_tRamSegments ( tSegs )
	{
		m_dRowMaps.Reserve ( m_tRamSegments.GetLength() );
		for ( const auto & pSeg : m_tRamSegments )
		{
			auto& dRowMap = m_dRowMaps.Add();
			dRowMap.Reserve ( pSeg->m_uRows );
			dRowMap.Resize ( pSeg->m_uRows ); // need space for ALL rows, incl. dead (they will be INVALID_ROWID)
			dRowMap.Fill ( INVALID_ROWID );
		}
		assert ( m_dRowMaps.GetLength() == m_tRamSegments.GetLength() );
	}
};


bool RtIndex_c::WriteAttributes ( SaveDiskDataContext_t & tCtx, CSphString & sError ) const
{
	auto sSPA	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPA );
	auto sSPB	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPB );
	auto sSPT	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPT );
	auto sSPHI	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPHI );
	auto sSPDS	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPDS );
	auto sSPC	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPC );
	auto sSIdx	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPIDX );
	auto sJsonSIdx = tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPJIDX );
	auto sSKNN	= tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPKNN );

	CSphWriterNonThrottled tWriterSPA;
	if ( !tWriterSPA.OpenFile ( sSPA, sError ) )
		return false;

	const CSphColumnInfo * pBlobLocatorAttr = m_tSchema.GetAttr ( sphGetBlobLocatorName() );
	AttrIndexBuilder_c tMinMaxBuilder(m_tSchema);

	BuildBufferSettings_t tSettings; // use default buffer settings

	std::unique_ptr<BlobRowBuilder_i> pBlobRowBuilder;
	if ( pBlobLocatorAttr )
	{
		pBlobRowBuilder = sphCreateBlobRowBuilder ( m_tSchema, sSPB, m_tSettings.m_tBlobUpdateSpace, tSettings.m_iBufferAttributes, sError );
		if ( !pBlobRowBuilder )
			return false;
	}

	std::unique_ptr<DocstoreBuilder_i> pDocstoreBuilder;
	if ( m_tSchema.HasStoredFields() || m_tSchema.HasStoredAttrs() )
	{
		pDocstoreBuilder = CreateDocstoreBuilder ( sSPDS, m_tSettings, tSettings.m_iBufferStorage, sError );
		if ( !pDocstoreBuilder )
			return false;

		SetupDocstoreFields ( *pDocstoreBuilder, m_tSchema );
	}

	std::unique_ptr<columnar::Builder_i> pColumnarBuilder;
	if ( m_tSchema.HasColumnarAttrs() )
	{
		pColumnarBuilder = CreateColumnarBuilder ( m_tSchema, sSPC, tSettings.m_iBufferColumnar, sError );
		if ( !pColumnarBuilder )
			return false;
	}

	HistogramContainer_c tHistograms;
	CSphVector<PlainOrColumnar_t> dAttrsForHistogram;
	BuildCreateHistograms ( tHistograms, dAttrsForHistogram, m_tSchema );

	CSphVector<PlainOrColumnar_t> dSiAttrs;
	std::unique_ptr<SI::Builder_i> pSIdxBuilder;
	std::unique_ptr<JsonSIBuilder_i> pJsonSIBuilder;
	if ( IsSecondaryLibLoaded() )
	{
		pSIdxBuilder = CreateIndexBuilder ( m_iRtMemLimit, m_tSchema, sSIdx, dSiAttrs, tSettings.m_iBufferStorage, sError );
		if ( !pSIdxBuilder )
			return false;

		if ( m_tSchema.HasJsonSIAttrs() )
		{
			pJsonSIBuilder = CreateJsonSIBuilder ( m_tSchema, sSPB, sJsonSIdx, sError );
			if ( !pJsonSIBuilder )
				return false;
		}
	}

	tCtx.m_iTotalDocuments = 0;
	for ( const auto & i : tCtx.m_tRamSegments )
		tCtx.m_iTotalDocuments += i->m_tAliveRows.load ( std::memory_order_relaxed );

	CSphVector<std::pair<PlainOrColumnar_t,int>> dAllAttrsForKNN;
	CSphVector<PlainOrColumnar_t> dAttrsForKNN;
	std::unique_ptr<knn::Builder_i> pKNNBuilder;
	if ( m_tSchema.HasKNNAttrs() )
	{
		CSphString sTmpFilename = tCtx.m_tFilebase.GetTmpFilename(SPH_EXT_SPKNN);
		sTmpFilename.SetSprintf ( "%s.4bit", sTmpFilename.cstr() );
		pKNNBuilder = BuildCreateKNN ( m_tSchema, tCtx.m_iTotalDocuments, dAllAttrsForKNN, sTmpFilename, sError );
		if ( !pKNNBuilder )
			return false;

		for ( const auto & i : dAllAttrsForKNN )
			dAttrsForKNN.Add ( i.first );
	}

	CSphFixedVector<DocidRowidPair_t> dRawLookup ( tCtx.m_iTotalDocuments );
	const CSphColumnInfo & tDocidAttr = m_tSchema.GetAttr(0);
	const bool bUuidLinked = tDocidAttr.IsUuidLinkedDocid();
	const CSphColumnInfo * pUuidAttr = bUuidLinked ? m_tSchema.GetAttr ( sphGetUuidDocidName() ) : nullptr;
	assert ( !bUuidLinked || ( pUuidAttr && pUuidAttr->m_eAttrType==SPH_ATTR_STRING && pUuidAttr->IsColumnar()==tDocidAttr.IsColumnar() ) );
	PlainOrColumnar_t tUuidAttr = bUuidLinked ? CreatePlainOrColumnar ( m_tSchema, *pUuidAttr ) : PlainOrColumnar_t();
	CSphFixedVector<UuidDocidLookupPair_t> dRawUuidLookup ( bUuidLinked ? tCtx.m_iTotalDocuments : 0 );

	int iColumnarIdLoc = -1;
	if ( m_tSchema.GetAttr(0).IsColumnar() )
		iColumnarIdLoc = 0;

	CSphVector<int64_t> dTmp;
	RowID_t tNextRowID = 0;
	int iStride = m_tSchema.GetRowSize();
	auto iStrideBytes = sizeof ( CSphRowitem ) * iStride;
	CSphFixedVector<CSphRowitem> dNewRow { iStride };
	CSphRowitem * pNewRow = dNewRow.Begin();
	ARRAY_FOREACH ( i, tCtx.m_tRamSegments )
	{
		const auto & tSeg = *tCtx.m_tRamSegments[i];

		SccRL_t rLock ( tSeg.m_tLock );
		tSeg.m_bAttrsBusy.store ( true, std::memory_order_release );

		auto dColumnarIterators = CreateAllColumnarIterators ( tSeg.m_pColumnar.get(), m_tSchema );

		for ( auto tRowID : RtLiveRows_c(tSeg) )
		{
			const CSphRowitem * pRow = tSeg.m_dRows.Begin() + (int64_t)tRowID*iStride;
			tMinMaxBuilder.Collect(pRow);
			if ( pBlobLocatorAttr )
			{
				auto tSrcOffset = sphGetRowAttr ( pRow, pBlobLocatorAttr->m_tLocator );
				auto tTargetOffsetSize = pBlobRowBuilder->Flush ( tSeg.m_dBlobs.Begin() + tSrcOffset );

				memcpy ( pNewRow, pRow, iStrideBytes );
				sphSetRowAttr ( pNewRow, pBlobLocatorAttr->m_tLocator, tTargetOffsetSize.first );
				tWriterSPA.PutBytes ( pNewRow, (int64_t)iStrideBytes );

				if ( pJsonSIBuilder )
					pJsonSIBuilder->AddRowOffsetSize ( tTargetOffsetSize );
			}
			else
				tWriterSPA.PutBytes ( pRow, (int64_t)iStrideBytes );

			DocID_t tDocID;

			ARRAY_FOREACH ( iIterator, dColumnarIterators )
			{
				auto & tIterator = dColumnarIterators[iIterator];
				SphAttr_t tAttr = SetColumnarAttr ( iIterator, tIterator.second, pColumnarBuilder.get(), tIterator.first, tRowID, dTmp );
				if ( iIterator==iColumnarIdLoc )
					tDocID = tAttr;
			}

			if ( iColumnarIdLoc<0 )
				tDocID = sphGetDocID(pRow);

			if ( bUuidLinked )
			{
				const uint8_t * pUuid = nullptr;
				int iUuidLen = tUuidAttr.Get ( tRowID, pRow, tSeg.m_dBlobs.Begin(), dColumnarIterators, pUuid );
				Str_t tUuid { (const char *)pUuid, iUuidLen };
				dRawUuidLookup[tNextRowID] = { sphGetUuidDocidKey ( tUuid ), tDocID };
			}

			BuildStoreHistograms ( tRowID, pRow, tSeg.m_dBlobs.Begin(), dColumnarIterators, dAttrsForHistogram, tHistograms );

			if ( pSIdxBuilder.get() )
			{
				pSIdxBuilder->SetRowID ( tNextRowID );
				BuildStoreSI ( tRowID, pRow, tSeg.m_dBlobs.Begin(), dColumnarIterators, dSiAttrs, pSIdxBuilder.get(), dTmp );
			}

			dRawLookup[tNextRowID] = { tDocID, tNextRowID };
			if ( pDocstoreBuilder )
			{
				assert ( tSeg.m_pDocstore );
				pDocstoreBuilder->AddDoc ( tNextRowID, tSeg.m_pDocstore->GetDoc ( tRowID, nullptr, -1, false ) );
			}

			if ( pKNNBuilder )
				BuildTrainKNN ( tRowID, tNextRowID, pRow, tSeg.m_dBlobs.Begin(), dColumnarIterators, dAttrsForKNN, *pKNNBuilder );

			tCtx.m_dRowMaps[i][tRowID] = tNextRowID++;
		}
	}

	if ( pKNNBuilder && !StoreKNNParallel ( tCtx, *pKNNBuilder, dAttrsForKNN, iStride, sError ) )
		return false;

	// rows could be killed during index save and tNextRowID could be less than tCtx.m_iTotalDocuments \ initial count
	assert ( tNextRowID<=(RowID_t)dRawLookup.GetLength() );
	VecTraits_T<DocidRowidPair_t> dLookup ( dRawLookup.Begin(), tNextRowID );
	VecTraits_T<UuidDocidLookupPair_t> dUuidLookup ( dRawUuidLookup.Begin(), bUuidLinked ? tNextRowID : 0 );
	assert ( !bUuidLinked || dUuidLookup.GetLength()==dLookup.GetLength() );

	std::string sErrorSTL;
	if ( pColumnarBuilder && !pColumnarBuilder->Done(sErrorSTL) )
	{
		sError = sErrorSTL.c_str();
		return false;
	}

	if ( pBlobRowBuilder && !pBlobRowBuilder->Done ( sError ) )
		return false;

	if ( pDocstoreBuilder )
		pDocstoreBuilder->Finalize();

	if ( pKNNBuilder && !pKNNBuilder->Save ( sSKNN.cstr(), tSettings.m_iBufferStorage, sErrorSTL ) )
	{
		sError = sErrorSTL.c_str();
		return false;
	}

	dLookup.Sort ( CmpDocidLookup_fn() );

	if ( bUuidLinked )
		dUuidLookup.Sort ( CmpUuidDocidLookup_fn() );

	bool bWritten = false;
	if ( bUuidLinked )
		bWritten = WriteDocidLookup ( sSPT, dLookup, dUuidLookup, sError );
	else
		bWritten = WriteDocidLookup ( sSPT, dLookup, sError );

	if ( !bWritten )
		return false;

	dRawLookup.Reset(0);
	dRawUuidLookup.Reset(0);

	if ( !tHistograms.Save ( sSPHI, sError ) )
		return false;

	tMinMaxBuilder.FinishCollect();

	tCtx.m_iDocinfo = tNextRowID;
	if ( tCtx.m_iDocinfo && m_tSchema.HasNonColumnarAttrs() )
	{
		const CSphTightVector<CSphRowitem> & dMinMaxRows = tMinMaxBuilder.GetCollected();

		tCtx.m_iMinMaxIndex = tWriterSPA.GetPos () / sizeof ( CSphRowitem );
		tCtx.m_iDocinfoIndex = ( dMinMaxRows.GetLength() / m_tSchema.GetRowSize() / 2 ) - 1;
		tWriterSPA.PutBytes ( dMinMaxRows.Begin(), dMinMaxRows.GetLength()*sizeof(CSphRowitem) );
	}

	tWriterSPA.CloseFile();
	if ( tWriterSPA.IsError() )
		return false;

	std::string sSidxError;
	if ( pSIdxBuilder.get() && !pSIdxBuilder->Done ( sSidxError ) )
	{
		sError = sSidxError.c_str();
		return false;
	}

	if ( pJsonSIBuilder && !pJsonSIBuilder->Done(sError) )
		return false;

	return true;
}


bool RtIndex_c::StoreKNNParallel ( SaveDiskDataContext_t & tCtx, knn::Builder_i & tBuilder, const VecTraits_T<PlainOrColumnar_t> & dAttrsForKNN, int iStride, CSphString & sError ) const
{
	{
		std::string sErrSTL;
		if ( !tBuilder.FinalizeTraining ( sErrSTL ) )
		{
			sError = sErrSTL.c_str();
			return false;
		}
	}

	// per-segment HNSW store loop
	const int iNumSegs = tCtx.m_tRamSegments.GetLength();
	const int iConcurrency = Max ( 1, Min ( KNNParallelBuild(), iNumSegs ) );

	RTSAVELOG << "parallel KNN build: concurrency=" << iConcurrency
			  << " (knn_parallel_build=" << KNNParallelBuild()
			  << " segments=" << iNumSegs << ")";

	auto pDispatcher = Dispatcher::MakeTrivial ( iNumSegs, iConcurrency );
	std::atomic<bool> bError { false };
	CSphFixedVector<CSphString> dErrors(iNumSegs);

	Coro::ExecuteN ( iConcurrency, [&]
	{
		Coro::SetThrottlingPeriodMS ( session::GetThrottlingPeriodMS() );

		auto pSource = pDispatcher->MakeSource();
		knn::BuildContext_t tBuildCtx;
		int iSeg = -1;
		while ( !bError.load ( std::memory_order_relaxed ) && pSource->FetchTask(iSeg) )
		{
			const auto & tSeg = *tCtx.m_tRamSegments[iSeg];

			SccRL_t rLock ( tSeg.m_tLock );
			tSeg.m_bAttrsBusy.store ( true, std::memory_order_release );

			auto dColumnarIterators = CreateAllColumnarIterators ( tSeg.m_pColumnar.get(), m_tSchema );

			for ( auto tRowID : RtLiveRows_c(tSeg) )
			{
				RowID_t tRowOut = tCtx.m_dRowMaps[iSeg][tRowID];
				assert ( tRowOut!=INVALID_ROWID );

				const CSphRowitem * pRow = tSeg.m_dRows.Begin() + (int64_t)tRowID * iStride;
				if ( !BuildStoreKNN ( tRowID, tRowOut, pRow, tSeg.m_dBlobs.Begin(), dColumnarIterators, dAttrsForKNN, tBuilder, tBuildCtx ) )
				{
					RecordKNNBuildError ( dErrors[iSeg], tBuildCtx, "HNSW save: BuildStoreKNN failed for segment %d row %u (no detail)", iSeg, tRowID );
					bError.store ( true, std::memory_order_relaxed );
					break;
				}
			}

			iSeg = -1; // mark consumed so FetchTask advances
		}
	});

	if ( bError.load() )
	{
		sError = ConcatKNNBuildErrors ( dErrors );
		return false;
	}

	return true;
}


bool RtIndex_c::WriteDocs ( SaveDiskDataContext_t & tCtx, CSphWriter & tWriterDict, CSphString & sError ) const
{
	CSphWriterNonThrottled tWriterHits, tWriterDocs, tWriterSkips;

	if ( !tWriterHits.OpenFile ( tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPP ), sError ) )
		return false;

	if ( !tWriterDocs.OpenFile ( tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPD ), sError ) )
		return false;

	if ( !tWriterSkips.OpenFile ( tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPE ), sError ) )
		return false;

	tWriterHits.PutByte(1);
	tWriterDocs.PutByte(1);
	tWriterSkips.PutByte(1);

	int iSegments = tCtx.m_tRamSegments.GetLength();

	RawVector_T<RtWordReader_c> dWordReaders;
	dWordReaders.Reserve_static ( iSegments );
	CSphVector<const RtWord_t*> dWords(iSegments);
	for ( int i = 0; i < iSegments; ++i )
	{
		dWordReaders.Emplace_back ( tCtx.m_tRamSegments[i], GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );
		dWords[i] = dWordReaders.Last().UnzipWord();
	}

	// loop keywords
	int iWords = 0;
	CSphKeywordDeltaWriter tLastWord;
	CSphKeywordDeltaWriterV2 tLastWordV2;
	SphWordID_t uLastWordID = 0;
	CSphVector<SkiplistEntry_t> dSkiplist;
	const DictFormat_e eDictFormat = GetDictFormat();
	const bool bKeywordsV2 = m_pDict->GetSettings().IsKeywordsV2();
	const bool bWordDict = IsWordDict();
	assert ( !bKeywordsV2 || bWordDict );

	tCtx.m_tLastDocPos = 0;

	bool bHasMorphology = m_pDict->HasMorphology();
	int iSkiplistBlockSize = m_tSettings.m_iSkiplistBlockSize;
	assert ( iSkiplistBlockSize>0 );

	while (true)
	{
		// find keyword with min id
		const RtWord_t * pWord = nullptr;
		for ( auto & i : dWords )
			if ( CompareWords ( i, pWord, bWordDict ) < 0 )
				pWord = i;

		if ( !pWord )
			break;

		SphOffset_t uDocpos = tWriterDocs.GetPos();
		SphOffset_t uLastHitpos = 0;
		RowID_t tLastRowID = INVALID_ROWID;
		RowID_t tSkiplistRowID = INVALID_ROWID;
		int iDocs = 0;
		int iHits = 0;
		dSkiplist.Resize(0);

		// loop all segments that have this keyword
		CSphBitvec tSegsWithWord ( iSegments );

		ARRAY_FOREACH ( iSegment, dWords )
		{
			if ( !CompareWords ( dWords[iSegment], pWord, bWordDict ) )
				tSegsWithWord.BitSet(iSegment);
			else
				continue;

			RtDocReader_c tDocReader ( tCtx.m_tRamSegments[iSegment], *dWords[iSegment] );
			while ( tDocReader.UnzipDoc() )
			{
				const auto* pDoc = (const RtDoc_t*)tDocReader;
				RowID_t tRowID = tCtx.m_dRowMaps[iSegment][pDoc->m_tRowID];
				if ( tRowID==INVALID_ROWID )
					continue;

				// build skiplist, aka save decoder state as needed
				if ( ( iDocs & ( iSkiplistBlockSize-1 ) )==0 )
				{
					SkiplistEntry_t & t = dSkiplist.Add();
					t.m_tBaseRowIDPlus1 = tSkiplistRowID+1;
					t.m_iOffset = tWriterDocs.GetPos();
					t.m_iBaseHitlistPos = uLastHitpos;
				}

				++iDocs;
				iHits += pDoc->m_uHits;
				tSkiplistRowID = tRowID;

				tWriterDocs.ZipOffset ( tRowID - std::exchange ( tLastRowID, tRowID ) );
				tWriterDocs.ZipInt ( pDoc->m_uHits );
				if ( pDoc->m_uHits==1 && pWord->m_bHasHitlist )
				{
					tWriterDocs.ZipInt ( pDoc->m_uHit & 0x7FFFFFUL );
					tWriterDocs.ZipInt ( pDoc->m_uHit >> 23 );
				} else
				{
					tWriterDocs.ZipInt ( pDoc->m_uDocFields );
					tWriterDocs.ZipOffset ( tWriterHits.GetPos() - std::exchange ( uLastHitpos, tWriterHits.GetPos() ) );
				}

				// loop hits from current segment
				if ( pDoc->m_uHits>1 )
				{
					DWORD uLastHit = 0;
					RtHitReader_c tInHits ( *tCtx.m_tRamSegments[iSegment], *pDoc );
					while ( DWORD uValue = tInHits.UnzipHit() )
						tWriterHits.ZipInt ( uValue - std::exchange ( uLastHit, uValue ) );
					tWriterHits.ZipInt(0);
				}
			}
		}

		// write skiplist
		int64_t iSkiplistOff = tWriterSkips.GetPos();
		for ( int i=1; i<dSkiplist.GetLength(); ++i )
		{
			const SkiplistEntry_t & tPrev = dSkiplist[i-1];
			const SkiplistEntry_t & tCur = dSkiplist[i];
			assert ( tCur.m_tBaseRowIDPlus1 - tPrev.m_tBaseRowIDPlus1>=(DWORD)iSkiplistBlockSize );
			assert ( tCur.m_iOffset - tPrev.m_iOffset>=4*iSkiplistBlockSize );
			tWriterSkips.ZipInt ( tCur.m_tBaseRowIDPlus1 - tPrev.m_tBaseRowIDPlus1 - iSkiplistBlockSize );
			tWriterSkips.ZipOffset ( tCur.m_iOffset - tPrev.m_iOffset - 4*iSkiplistBlockSize );
			tWriterSkips.ZipOffset ( tCur.m_iBaseHitlistPos - tPrev.m_iBaseHitlistPos );
		}

		// write dict entry if necessary
		if ( tWriterDocs.GetPos()!=uDocpos )
		{
			tWriterDocs.ZipInt ( 0 ); // docs over

			if ( ( iWords%SPH_WORDLIST_CHECKPOINT )==0 )
			{
				if ( iWords )
				{
					SphOffset_t uOff = bWordDict ? 0 : uDocpos - tCtx.m_tLastDocPos;
					tWriterDict.ZipInt ( 0 );
					tWriterDict.ZipOffset ( uOff ); // store last doclist length for CRC, zero for keyword dicts
				}

				// restart delta coding, once per SPH_WORDLIST_CHECKPOINT entries
				tCtx.m_tLastDocPos = 0;
				uLastWordID = 0;
				tLastWord.Reset();
				tLastWordV2.Reset();

				// begin new wordlist entry
				Checkpoint_t & tChk = tCtx.m_dCheckpoints.Add();
				tChk.m_uOffset = tWriterDict.GetPos();
				if ( eDictFormat==DictFormat_e::KEYWORDS_V2 )
					tChk.m_uWord = AppendPackedKeywordV2 ( tCtx.m_dKeywordCheckpoints, pWord->m_sWord, pWord->m_iWordLen );
				else if ( eDictFormat==DictFormat_e::KEYWORDS )
					tChk.m_uWord = AppendPackedKeywordLegacy ( tCtx.m_dKeywordCheckpoints, pWord->m_sWord, pWord->m_iWordLen );
				else
					tChk.m_uWord = pWord->m_uWordID;
			}

			++iWords;

			if ( bWordDict )
			{
				if ( bKeywordsV2 )
					tLastWordV2.PutDelta ( tWriterDict, pWord->m_sWord, pWord->m_iWordLen );
				else
					tLastWord.PutDelta ( tWriterDict, pWord->m_sWord, pWord->m_iWordLen );
				tWriterDict.ZipOffset ( uDocpos );
			} else
			{
				assert ( pWord->m_uWordID!=uLastWordID );
				tWriterDict.ZipOffset ( pWord->m_uWordID - uLastWordID );
				uLastWordID = pWord->m_uWordID;
				assert ( uDocpos>tCtx.m_tLastDocPos );
				tWriterDict.ZipOffset ( uDocpos - tCtx.m_tLastDocPos );
			}

			DWORD iDocsCount = iDocs;
			if ( !pWord->m_bHasHitlist && m_tSettings.m_eHitless==SPH_HITLESS_SOME )
				iDocsCount |= HITLESS_DOC_FLAG;

			tWriterDict.ZipInt ( iDocsCount );
			tWriterDict.ZipInt ( iHits );

			if ( bWordDict )
			{
				BYTE uHint = sphDoclistHintPack ( iDocs, tWriterDocs.GetPos()-tCtx.m_tLastDocPos );
				if ( uHint )
					tWriterDict.PutByte ( uHint );

				// build infixes
				if ( tCtx.m_pInfixer )
				{
					int64_t iCheckpoint = tCtx.m_dCheckpoints.GetLength64();
					if ( bKeywordsV2 && iCheckpoint>INFIX_CHECKPOINT_ID_MAX )
					{
						sError.SetSprintf ( "keywords_32k infix checkpoint id " INT64_FMT " exceeds supported limit %u; checkpoint-id widening is outside 4 GB .spi offset support",
							iCheckpoint, INFIX_CHECKPOINT_ID_MAX );
						return false;
					}
					tCtx.m_pInfixer->AddWord ( pWord->m_sWord, pWord->m_iWordLen, (int)iCheckpoint, bHasMorphology );
				}
			}

			// emit skiplist pointer
			if ( iDocs>iSkiplistBlockSize )
				tWriterDict.ZipOffset ( iSkiplistOff );

			tCtx.m_tLastDocPos = uDocpos;
		}

		// read next words
		for ( DWORD i = 0; i < tSegsWithWord.GetSize(); ++i )
			if ( tSegsWithWord.BitGet(i) )
				dWords[i] = dWordReaders[i].UnzipWord();
	}

	tCtx.m_tDocsOffset = tWriterDocs.GetPos();
	tWriterHits.CloseFile();
	tWriterDocs.CloseFile();
	tWriterSkips.CloseFile();
	return true;
}


void RtIndex_c::WriteCheckpoints ( SaveDiskDataContext_t & tCtx, CSphWriter & tWriterDict, SaveDiskDataTimings_t * pTimings ) const
{
	// write checkpoints
	const bool bWordDict = IsWordDict();
	SphOffset_t uOff = bWordDict ? 0 : tCtx.m_tDocsOffset - tCtx.m_tLastDocPos;
	const bool bKeywordsV2 = m_pDict->GetSettings().IsKeywordsV2();
	assert ( !bKeywordsV2 || bWordDict );

	tWriterDict.ZipInt ( 0 ); // indicate checkpoint
	tWriterDict.ZipOffset ( uOff ); // store last doclist length for CRC, zero for keyword dicts

	// flush infix hash entries, if any
	int64_t tmInfixTotal = 0;
	if ( tCtx.m_pInfixer )
	{
		int64_t tmInfix = sphMicroTimer();
		tCtx.m_pInfixer->SaveEntries ( tWriterDict );
		tmInfixTotal = sphMicroTimer() - tmInfix;
	}

	tCtx.m_iDictCheckpointsOffset = tWriterDict.GetPos();
	if ( bWordDict )
	{
		const BYTE * pCheckpoints = tCtx.m_dKeywordCheckpoints.Begin();
		if ( bKeywordsV2 )
		{
			for ( const auto & i : tCtx.m_dCheckpoints )
			{
				const BYTE * pPacked = pCheckpoints + i.m_uWord;
				ByteBlob_t tWord = GetPackedKeywordV2 ( pPacked );
				int iLen = tWord.second;
				assert ( iLen && tWord.first+iLen<=tCtx.m_dKeywordCheckpoints.Begin()+tCtx.m_dKeywordCheckpoints.GetLength() );
				tWriterDict.ZipInt ( iLen );
				tWriterDict.PutBytes ( tWord.first, iLen );
				tWriterDict.PutOffset ( i.m_uOffset );
			}
		} else
		{
			for ( const auto & i : tCtx.m_dCheckpoints )
			{
				const BYTE * pPacked = pCheckpoints + i.m_uWord;
				ByteBlob_t tWord = GetPackedKeywordLegacy ( pPacked );
				int iLen = tWord.second;
				assert ( iLen && tWord.first+iLen<=tCtx.m_dKeywordCheckpoints.Begin()+tCtx.m_dKeywordCheckpoints.GetLength() );
				tWriterDict.PutDword ( iLen );
				tWriterDict.PutBytes ( tWord.first, iLen );
				tWriterDict.PutOffset ( i.m_uOffset );
			}
		}
	} else
	{
		for ( const auto & i : tCtx.m_dCheckpoints )
		{
			tWriterDict.PutOffset ( i.m_uWord );
			tWriterDict.PutOffset ( i.m_uOffset );
		}
	}

	// flush infix hash blocks
	if ( tCtx.m_pInfixer )
	{
		int64_t tmInfixBlocks = sphMicroTimer();
		tCtx.m_iInfixBlocksOffset = tCtx.m_pInfixer->SaveEntryBlocks ( tWriterDict );
		tCtx.m_iInfixBlocksWordsSize = tCtx.m_pInfixer->GetBlocksWordsSize();
		tmInfixTotal += sphMicroTimer() - tmInfixBlocks;

		if ( !bKeywordsV2 && tCtx.m_iInfixBlocksOffset>UINT_MAX )
			sphWarning ( "INTERNAL ERROR: dictionary size " INT64_FMT " overflow at infix save", tCtx.m_iInfixBlocksOffset );
	}

	// Store total infix time (SaveEntries + SaveEntryBlocks)
	if ( pTimings )
		pTimings->m_tmInfix = tmInfixTotal;

	// flush header
	// mostly for debugging convenience
	// primary storage is in the index wide header
	if ( bKeywordsV2 )
	{
		tWriterDict.PutBlob ( g_sTagDictHeaderV2 );
		tWriterDict.ZipOffset ( tCtx.m_dCheckpoints.GetLength64() );
		tWriterDict.ZipOffset ( tCtx.m_iDictCheckpointsOffset );
		tWriterDict.ZipInt ( m_pTokenizer->GetMaxCodepointLength() );
		tWriterDict.ZipOffset ( tCtx.m_iInfixBlocksOffset );
		tWriterDict.ZipOffset ( tCtx.m_iInfixBlocksWordsSize );
	} else
	{
		tWriterDict.PutBlob ( g_sTagDictHeader );
		tWriterDict.ZipInt ( tCtx.m_dCheckpoints.GetLength() );
		tWriterDict.ZipOffset ( tCtx.m_iDictCheckpointsOffset );
		tWriterDict.ZipInt ( m_pTokenizer->GetMaxCodepointLength() );
		tWriterDict.ZipInt ( (DWORD)tCtx.m_iInfixBlocksOffset );
	}
}


bool RtIndex_c::WriteDeadRowMap ( SaveDiskDataContext_t & tCtx, CSphString & sError ) // static
{
	CSphString sName = tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPM );
	return ::WriteDeadRowMap ( sName, tCtx.m_iDocinfo, sError );
}

struct FilesCleanup_t
{
	explicit FilesCleanup_t ( const char * sFilename )
		: m_tFiles ( sFilename )
	{
	}

	~FilesCleanup_t()
	{
		if ( m_bRemoveFiles )
			m_tFiles.UnlinkExisted();
	}

	IndexFiles_c m_tFiles;
	bool m_bRemoveFiles = true;
};

// SaveDiskChunk -> SaveDiskData
// RO save RAM chunks from tSegs into new disk chunk (nothing added/released, just disk files created)
bool RtIndex_c::SaveDiskData ( const char * szFilename, const ConstRtSegmentSlice_t & tSegs, const ChunkStats_t & tStats, CSphString & sError, SaveDiskDataTimings_t * pTimings ) const
{
	RTSAVELOG << "SaveDiskData to " << szFilename << ", " << tSegs.GetLength() << " segments";

	FilesCleanup_t tFiles ( szFilename );
	sError = "";

	SaveDiskDataContext_t tCtx ( szFilename, tSegs ); // only RAM segments here in game.
	if ( m_tSettings.m_iMinInfixLen && m_pDict->GetSettings().IsWordDict() )
		tCtx.m_pInfixer = sphCreateInfixBuilder ( m_pTokenizer->GetMaxCodepointLength(), &sError, GetDictFormat() );
	if ( !sError.IsEmpty() )
		return false;

//	PauseCheck("savepause"); // catch if something happened between RtGuard() and actual updates

	// fixme: handle errors
	int64_t tmStart = sphMicroTimer();
	if ( !WriteAttributes ( tCtx, sError ) )
		return false;
	if ( pTimings )
		pTimings->m_tmAttributes = sphMicroTimer() - tmStart;

	tmStart = sphMicroTimer();
	if ( !WriteDeadRowMap ( tCtx, sError ) )
		return false;
	if ( pTimings )
		pTimings->m_tmDeadRowMap = sphMicroTimer() - tmStart;

	CSphWriterNonThrottled tWriterDict;
	CSphString sSPI = IndexFileBase_c { szFilename }.GetFilename ( SPH_EXT_SPI );

	if ( !tWriterDict.OpenFile ( sSPI.cstr(), sError ) )
		return false;

	tWriterDict.PutByte ( 1 );

	tmStart = sphMicroTimer();
	if ( !WriteDocs ( tCtx, tWriterDict, sError ) )
		return false;
	if ( pTimings )
		pTimings->m_tmDocs = sphMicroTimer() - tmStart;

	tmStart = sphMicroTimer();
	WriteCheckpoints ( tCtx, tWriterDict, pTimings );
	if ( pTimings )
	{
		int64_t tmCheckpointsTotal = sphMicroTimer() - tmStart;
		pTimings->m_tmCheckpoints = tmCheckpointsTotal - pTimings->m_tmInfix;
	}
		
	tWriterDict.CloseFile();
	if ( tWriterDict.IsError() )
		return false;

	tmStart = sphMicroTimer();
	if ( !SaveDiskHeader ( tCtx, tStats, sError ) )
		return false;
	if ( pTimings )
		pTimings->m_tmHeader = sphMicroTimer() - tmStart;

	tFiles.m_bRemoveFiles = false;
	return true;
}


static void FixupIndexSettings ( CSphIndexSettings & tSettings )
{
	tSettings.m_eHitFormat = SPH_HIT_FORMAT_INLINE;
	tSettings.m_iBoundaryStep = 0;
	tSettings.m_iStopwordStep = 1;
	tSettings.m_iOvershortStep = 1;
}


// SaveDiskChunk -> SaveDiskData -> SaveDiskHeader
bool RtIndex_c::SaveDiskHeader ( SaveDiskDataContext_t & tCtx, const ChunkStats_t & tStats, CSphString & sError ) const
{
	tCtx.m_iDictCheckpoints = tCtx.m_dCheckpoints.GetLength64 ();
	tCtx.m_iInfixCodepointBytes = ( m_tSettings.m_iMinInfixLen && m_pDict->GetSettings().IsWordDict() )
			? m_pTokenizer->GetMaxCodepointLength ()
			: 0;

	tCtx.m_iTotalBytes = tStats.m_Stats.m_iTotalBytes;

	CSphIndexSettings tSettings = m_tSettings;
	FixupIndexSettings ( tSettings );

	WriteHeader_t tWriteHeader;
	tWriteHeader.m_pSettings = &tSettings;
	tWriteHeader.m_pSchema = &m_tSchema;
	tWriteHeader.m_pTokenizer = m_pTokenizer;
	tWriteHeader.m_pDict = m_pDict;
	tWriteHeader.m_pFieldFilter = m_pFieldFilter.get();
	tWriteHeader.m_pFieldLens = m_dFieldLens.Begin();

	CSphString sName;
	JsonEscapedBuilder sJson;
	IndexWriteHeader ( tCtx, tWriteHeader, sJson, IsWordDict(), true );

	sName = tCtx.m_tFilebase.GetFilename ( SPH_EXT_SPH );
	CSphWriterNonThrottled wrHeaderJson;
	if ( !wrHeaderJson.OpenFile ( sName, sError ) )
		return false;

	wrHeaderJson.PutString ( (Str_t)sJson );
	wrHeaderJson.CloseFile();
	if ( wrHeaderJson.IsError() )
		return false;

	assert ( bson::ValidateJson ( sJson.cstr(), &sError ) );
	return true;
}

void RtIndex_c::SaveMeta ( int64_t iTID, VecTraits_T<int> dChunkNames )
{
	if ( !m_tSaving.ActiveStateIs ( SaveState_c::ENABLED ) )
		return;

	// sanity check
	if ( m_iLockFD<0 )
		return;

	// write new meta
	auto sMeta = GetFilename ( "meta" );
	auto sMetaNew = GetFilename ( "meta.new" );

	CSphString sError;

	CSphWriterNonThrottled wrMeta;
	if ( !wrMeta.OpenFile ( sMetaNew, sError ) )
		sphDie ( "failed to open file for meta serialization: %s", sError.cstr() ); // !COMMIT handle this gracefully

	WriteMeta ( iTID, dChunkNames, wrMeta );
	wrMeta.CloseFile();

	// no need to remove old but good meta in case new meta failed to save
	if ( wrMeta.IsError() )
	{
		sphWarning ( "%s", sError.cstr() );
		return;
	}

	// rename
	if ( sph::rename ( sMetaNew.cstr(), sMeta.cstr() ) )
		sphDie ( "failed to rename meta (src=%s, dst=%s, errno=%d, error=%s)",
			sMetaNew.cstr(), sMeta.cstr(), errno, strerrorm(errno) ); // !COMMIT handle this gracefully

	SaveMutableSettings ( m_tMutableSettings, GetFilename ( SPH_EXT_SETTINGS ) );
}

void RtIndex_c::WriteMeta ( int64_t iTID, const VecTraits_T<int>& dChunkNames, CSphWriter& wrMeta ) const
{
	JsonEscapedBuilder sNewMeta;
	sNewMeta.ObjectWBlock();

	// human-readable sugar
	sNewMeta.NamedString ( "meta_created_time_utc", sphCurrentUtcTime() );

	sNewMeta.NamedVal ( "meta_version", META_VERSION );
	//	sNewMeta.NamedVal ( "index_format_version", INDEX_FORMAT_VERSION );
	sNewMeta.NamedVal ( "total_documents", m_tStats.m_iTotalDocuments );
	sNewMeta.NamedVal ( "total_bytes", m_tStats.m_iTotalBytes );
	sNewMeta.NamedVal ( "tid", iTID );

	// meta v.4, save disk index format and settings, too
	sNewMeta.NamedVal ( "schema", m_tSchema );

	sNewMeta.NamedVal ( "index_settings", m_tSettings );

	sNewMeta.Named ( "tokenizer_settings" );
	SaveTokenizerSettings ( sNewMeta, m_pTokenizer, m_tSettings.m_iEmbeddedLimit );

	sNewMeta.Named ( "dictionary_settings" );
	SaveDictionarySettings ( sNewMeta, m_pDict, IsWordDict(), m_tSettings.m_iEmbeddedLimit );

	// meta v.5
	sNewMeta.NamedVal ( "words_checkpoint", m_iWordsCheckpoint);

	// meta v.7
	sNewMeta.NamedValNonDefault ( "max_codepoint_length", m_iMaxCodepointLength );
	sNewMeta.NamedValNonDefault ( "bloom_per_entry_vals_count", (int)BLOOM_PER_ENTRY_VALS_COUNT, 8 );
	sNewMeta.NamedValNonDefault ( "bloom_hashes_count",  (int)BLOOM_HASHES_COUNT, 2 );

	// meta v.11
	CSphFieldFilterSettings tFieldFilterSettings;
	if ( m_pFieldFilter )
	{
		m_pFieldFilter->GetSettings(tFieldFilterSettings);
		sNewMeta.NamedVal ( "field_filter_settings", tFieldFilterSettings );
	}

	{
		sNewMeta.Named ( "chunk_names" );
		auto _ = sNewMeta.Array();
		for ( int i : dChunkNames)
			sNewMeta << i;
	}

	// meta v.17+
	sNewMeta.NamedVal ( "soft_ram_limit", m_iRtMemLimit );
	sNewMeta.FinishBlocks();

	wrMeta.PutString ( (Str_t)sNewMeta );
	assert ( bson::ValidateJson ( sNewMeta.cstr() ) );
}

void RtIndex_c::SaveMeta()
{
	SaveMeta ( m_iTID, GetChunkIds ( *m_tRtChunks.DiskChunks() ) );
}

// looks like spinlock, but actually we switch to parallel strand and back on every tick, so it should not burn CPU
void RtIndex_c::WaitRAMSegmentsUnlocked ( bool bAllowOne ) const
{
	TRACE_SCHED ( "rt", "WaitRAMSegmentsUnlocked" );
	m_tNSavesNow.Wait ( [bAllowOne] ( int iVal ) { return iVal == ( bAllowOne ? 1 : 0 ); } );
	m_tUnLockedSegments.WaitVoid ( [this] { return m_tRtChunks.RamSegs()->none_of ( [] ( const ConstRtSegmentRefPtf_t& a ) { return a->m_iLocked; } ); });
}

template<typename PRED>
int64_t RtIndex_c::GetMemCount ( PRED&& fnPred ) const
{
	int64_t iTotal = 0;
	for ( const RtSegment_t* pSeg : *m_tRtChunks.RamSegs() )
		if ( fnPred ( pSeg ) )
			iTotal += pSeg->GetUsedRam();

	return iTotal;
}

// i.e. create new disk chunk from ram segments
bool RtIndex_c::SaveDiskChunk ( bool bForced, bool bEmergent ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	if ( !m_tSaving.WaitEnabledOrShutdown() )
		return true;

	assert ( Coro::CurrentScheduler() == m_tWorkers.SerialChunkAccess() );

	RTSAVELOG << "SaveDiskChunk (" << ( bForced ? "forced, " : "not forced, " ) << ( bEmergent ? "emergent, " : "not emergent " ) << ")";

	m_tNSavesNow.Wait ( [] ( int iVal ) { return iVal < SIMULTANEOUS_SAVE_LIMIT; } );

	// we're in serial worker - no concurrency, no race between wait() and modify()
	m_tNSavesNow.ModifyValue ( [] ( int& iVal ) { ++iVal; } );
	auto tFinallySetSaveUnactive = AtScopeExit ( [this] {
		++m_iSavedGeneration;
		m_tNSavesNow.ModifyValueAndNotifyAll ( [] ( int& iVal ) { --iVal; } );
	} );

	const int iSaveOp = m_tWorkers.GetNextOpTicket();
	TRACE_SCHED_VARID ( "rt", "SaveDiskChunk", iSaveOp );

	// if forced - wait all segments. Otherwise, can continue with subset of currently available segments
	// note that segments may be locked by currently executing MergeSegments or SaveDiskChunk. If so, we wait them finished and continue.
	// that will cause another disk chunk written right after just finished, since op is forced it is ok.
	if ( bForced )
		WaitRAMSegmentsUnlocked ( true ); // true means to wait 1, not 0 active saves (as we already increased the counter)

	// collect all non-occupied non-empty segments and lock them
	int64_t iNotMyOpRAM {0};
	int64_t iMyOpRAM {0};
	KillAccum_t dKillOnSave;
	LazyVector_T<ConstRtSegmentRefPtf_t> dSegments;
	for ( const auto & pSeg : *m_tRtChunks.RamSegs () )
	{
		if ( !pSeg->m_iLocked )
		{
			pSeg->m_iLocked = iSaveOp;
			if ( pSeg->m_tAliveRows.load ( std::memory_order_relaxed )!=0 )
			{
				pSeg->SetKillHook ( &dKillOnSave );
				dSegments.Add ( pSeg );
				iMyOpRAM += pSeg->GetUsedRam();
			}
		} else
			iNotMyOpRAM += pSeg->GetUsedRam();
	}
	AT_SCOPE_EXIT ( [&dSegments] { dSegments.for_each ( [] ( const auto& dSeg ) { dSeg->SetKillHook ( nullptr ); } ); } );
	if ( !dSegments.IsEmpty() )
		m_dSavingTickets.Add ( iSaveOp );
	UpdateUnlockedCount();

	RTSAVELOG << "SaveDiskChunk process " << dSegments.GetLength() << " segments. Active jobs " << m_tNSavesNow.GetValue() << ", op " << iSaveOp
		<< " RAM visible+retired/locked/acquired " << iNotMyOpRAM + iMyOpRAM << "+" << m_iRamChunksAllocatedRAM.load ( std::memory_order_relaxed )- iNotMyOpRAM - iMyOpRAM << "/" << iNotMyOpRAM << "/" << iMyOpRAM;
	if ( dSegments.IsEmpty() )
		return true;

	auto iTID = m_iTID;
	m_tSaveTIDS.ModifyValue ( [iTID] ( CSphVector<int64_t>& dSaves ) { dSaves.Add ( iTID ); } );
	auto tFinallyRemoveTID = AtScopeExit ( [this, iTID] { m_tSaveTIDS.ModifyValueAndNotifyAll ( [iTID] ( CSphVector<int64_t>& dSaves ) { dSaves.RemoveValue ( iTID ); } ); } );

	int64_t tmSaveWall = -sphMicroTimer(); // all time including waiting
	int64_t tmSave; // only active time
	MEMORY ( MEM_INDEX_RT );

	int iChunkID = m_tChunkID.MakeChunkId ( m_tRtChunks );
	auto sChunk = GetFilename ( iChunkID );

	// as we're going to switch fiber, we need to freeze reliable stat and m_iTID (as they could change)
	ChunkStats_t tStats ( m_tStats, m_dFieldLensRam );

	std::unique_ptr<CSphIndex> pNewChunk;
	SaveDiskDataTimings_t tTimings;
	while ( true )
	{
		// as separate subtask we 1-st flush segments to disk, and then load just flushed segment
		// if forced, continue to work in the same fiber (which is serialworker); otherwise go to global workpool, wrapped into SaveSegmentsWorker
		ScopedScheduler_c tSaveFiber { bForced ? Coro::CurrentScheduler () : m_tWorkers.SaveSegmentsWorker() };
		TRACE_SCHED_VARID ( "rt", "SaveDiskChunk-routine", iSaveOp );

		tmSave = -sphMicroTimer();
		if ( !SaveDiskData ( sChunk.cstr(), dSegments, tStats, m_sLastError, &tTimings ) )
		{
			sphWarning ( "rt: table %s failed to save disk chunk %s: %s", GetName(), sChunk.cstr(), m_sLastError.cstr() );
			tmSave += sphMicroTimer();
			break;
		}

		// bring new disk chunk online
		auto fnFnameBuilder = GetIndexFilenameBuilder ();
		StrVec_t dWarnings;
		std::unique_ptr<FilenameBuilder_i> pFilenameBuilder;
		if ( fnFnameBuilder )
			pFilenameBuilder = fnFnameBuilder ( GetName () );
		pNewChunk = PreallocDiskChunk ( sChunk, iChunkID, pFilenameBuilder.get (), dWarnings, m_sLastError );

		if ( !dWarnings.IsEmpty() )
		{
			StringBuilder_c sWarningLine ( "; " );
			for ( const auto & sItem : dWarnings )
				sWarningLine << sItem;

			sphWarning ( "rt: table %s save disk chunk %s warnings: %s", GetName(), sChunk.cstr(), sWarningLine.cstr() );
		}

		tmSave += sphMicroTimer();
		break;
	}

	// here is pickpoint: if we save some chunks in parallel, here we *NEED* to be sure, that later is not published before older
	// That is about binlog consistency: if we save trx 1-1000 and at the same time 1000-1010, last might finish faster, but it can't be committed immediately,
	// as last highest trx will be 1010, and nobody knows, that actually 1-1000 are not yet safe.
	BEGIN_SCHED ( "rt", "SaveDiskChunk-wait" ); // iSaveOp as id
	m_tSaveTIDS.WaitVoid ( [this, iTID] { return m_tSaveTIDS.GetValueRef().First() == iTID; } );
	END_SCHED( "rt" );

	assert ( Coro::CurrentScheduler() == m_tWorkers.SerialChunkAccess() );

	// here we back into serial fiber. As we're switched, we can't rely on m_iTID and index stats anymore
	if ( !pNewChunk )
	{
		sphWarning ( "rt: table %s failed to load disk chunk after RAM save: %s", GetName(), m_sLastError.cstr () );
		return false;
	}

	// applying postponed kills is ok now, since no other kills would happen as we're in serial fiber.
	if ( !dKillOnSave.m_dDocids.IsEmpty() )
	{
		RTLOGV << "SaveDiskChunk: apply postponed kills";
		tmSave -= sphMicroTimer ();
		dKillOnSave.m_dDocids.Uniq ();
		pNewChunk->KillMulti ( dKillOnSave.m_dDocids );
		dKillOnSave.m_dDocids.Reset();
		tmSave += sphMicroTimer ();
	}

	auto dUpdates = GatherUpdates::FromChunksOrSegments ( dSegments );
	if ( !dUpdates.IsEmpty () )
	{
		RTLOGV << "SaveDiskChunk: apply postponed updates";
		pNewChunk->UpdateAttributesOffline ( dUpdates );
		dUpdates.Reset();
	}

	int iSegments = dSegments.GetLength ();
	dSegments.Reset (); // we don't need them anymore

	// update field lengths (offline, will swap under lock)
	CSphFixedVector<int64_t> dNewFieldLensRam { 0 };
	CSphFixedVector<int64_t> dNewFieldLensDisk { 0 };
	if ( m_tSchema.GetAttrId_FirstFieldLen ()>=0 )
	{
		dNewFieldLensRam.Reset ( SPH_MAX_FIELDS );
		dNewFieldLensDisk.Reset ( SPH_MAX_FIELDS );
		ARRAY_FOREACH ( i, tStats.m_dFieldLens )
		{
			dNewFieldLensRam[i] = m_dFieldLensRam[i] - tStats.m_dFieldLens[i];
			dNewFieldLensDisk[i] = m_dFieldLensDisk[i] + tStats.m_dFieldLens[i];
		}
	}

	IntVec_t dChunks;
	// now new disk chunk is loaded, kills and updates applied - we ready to change global index state now.
	{
		auto tNewSet = RtWriter();
		CopyChunksTo ( tNewSet );
		tNewSet.m_pNewDiskChunks->Add ( DiskChunk_c::make ( std::move ( pNewChunk ) ) );
		tNewSet.InitRamSegs ( RtWriter_c::empty );

		for ( const auto & pSeg : *m_tRtChunks.RamSegs() )
			if ( pSeg->m_iLocked!=iSaveOp )
				tNewSet.m_pNewRamSegs->Add ( pSeg );

		dChunks = GetChunkIds ( *tNewSet.m_pNewDiskChunks );
	}

	// update field lengths
	if (m_tSchema.GetAttrId_FirstFieldLen() >= 0) {
		m_dFieldLensRam.SwapData ( dNewFieldLensRam );
		m_dFieldLensDisk.SwapData ( dNewFieldLensDisk );
	}

	SaveMeta ( iTID, dChunks );
	m_iSavedTID = iTID;

	// from this point all readers will see new state of the index.
	m_dSavingTickets.RemoveValue ( iSaveOp );
	Binlog::NotifyIndexFlush ( iTID, GetName(), Binlog::NoShutdown, Binlog::NoSave );

	// abandon .ram file
	UnlinkRAMChunk ( "SaveDiskChunk" );

	m_tmSaved = sphMicroTimer ();
	tmSaveWall += m_tmSaved;

	StringBuilder_c sInfo;
	sInfo.Sprintf ( "rt: table %s: diskchunk %d(%d), segments %d %s saved in %.6D (%.6D) sec", GetName (), iChunkID
					, dChunks.GetLength(), iSegments, bForced ? "forcibly" : "", tmSave, tmSaveWall );

	// Add timing breakdown
	if ( tTimings.m_tmAttributes || tTimings.m_tmDeadRowMap || tTimings.m_tmDocs || tTimings.m_tmInfix || tTimings.m_tmCheckpoints || tTimings.m_tmHeader )
	{
		sInfo << " [attrs=" << ( tTimings.m_tmAttributes / 1000 ) << "ms"
			  << " deadmap=" << ( tTimings.m_tmDeadRowMap / 1000 ) << "ms"
			  << " docs=" << ( tTimings.m_tmDocs / 1000 ) << "ms"
			  << " infix=" << ( tTimings.m_tmInfix / 1000 ) << "ms"
			  << " checkpoints=" << ( tTimings.m_tmCheckpoints / 1000 ) << "ms"
			  << " header=" << ( tTimings.m_tmHeader / 1000 ) << "ms"
			  << "]";
	}

	// calculate DoubleBuf percent using current save/insert rate
	auto iInserted = GetMemCount ( [iSaveOp] ( const auto* pSeg ) { return !pSeg->m_iLocked || pSeg->m_iLocked > iSaveOp; } );
	RecalculateRateLimit ( iMyOpRAM, iInserted, bEmergent );

	RTSAVELOG << sInfo.cstr() << ", op " << iSaveOp << " RAM saved/new " << iMyOpRAM << "/" << iInserted
			<< " Insert ratio is " << m_fSaveRateLimit << " (soft ram limit " << m_iSoftRamLimit << ", rt mem limit " << m_iRtMemLimit << ")";

	sInfo << ", RAM saved/new " << iMyOpRAM << "/" << iInserted << " ratio " << m_fSaveRateLimit << " (soft limit " << m_iSoftRamLimit << ", conf limit " << m_iRtMemLimit << ")";

	sphInfo ( "%s", sInfo.cstr() );

	Preread();
	CheckStartAutoOptimize();
	return true;
}

std::unique_ptr<CSphIndex> RtIndex_c::PreallocDiskChunk ( const CSphString& sChunk, int iChunk, FilenameBuilder_i * pFilenameBuilder, StrVec_t & dWarnings, CSphString & sError, const char * szName ) const
{
	MEMORY ( MEM_INDEX_DISK );

	// !COMMIT handle errors gracefully instead of dying
	auto pDiskChunk = sphCreateIndexPhrase ( ( szName ? szName : sChunk.cstr() ), sChunk );
	if ( !pDiskChunk )
	{
		sError.SetSprintf ( "disk chunk %s: alloc failed", sChunk.cstr() );
		return pDiskChunk;
	}

	pDiskChunk->m_iExpansionLimit = m_iExpansionLimit;
	pDiskChunk->SetMutableSettings ( m_tMutableSettings );
	pDiskChunk->SetGlobalIDFPath ( m_sGlobalIDFPath );
	pDiskChunk->SetBinlog ( false );
	pDiskChunk->m_iChunk = iChunk;

	if ( m_bDebugCheck )
		pDiskChunk->SetDebugCheck ( m_bCheckIdDups, -1 );

	if ( !pDiskChunk->Prealloc ( m_bPathStripped, pFilenameBuilder, dWarnings ) )
	{
		sError.SetSprintf ( "disk chunk %s: prealloc failed: %s", sChunk.cstr(), pDiskChunk->GetLastError().cstr() );
		pDiskChunk = nullptr;
	}

	return pDiskChunk;
}

RtIndex_c::LOAD_E RtIndex_c::LoadMetaLegacy ( FilenameBuilder_i * pFilenameBuilder, bool bStripPath, DWORD & uVersion, bool & bRebuildInfixes, StrVec_t & dWarnings )
{
	CSphString sMeta = GetFilename ( "meta" );

	m_sLastError = "";

	// opened and locked, lets read
	CSphAutoreader rdMeta;
	if ( !rdMeta.Open ( sMeta, m_sLastError ) )
		return LOAD_E::GeneralError_e;

	if ( rdMeta.GetDword()!=META_HEADER_MAGIC )
	{
		m_sLastError.SetSprintf ( "invalid meta file %s", sMeta.cstr() );
		return LOAD_E::ParseError_e;
	}

	uVersion = rdMeta.GetDword();
	if ( uVersion==0 || uVersion>META_VERSION )
	{
		m_sLastError.SetSprintf ( "%s is v.%u, binary is v.%u", sMeta.cstr(), uVersion, META_VERSION );
		return LOAD_E::GeneralError_e;
	}

	DWORD uMinFormatVer = 14;
	if ( uVersion<uMinFormatVer )
	{
		m_sLastError.SetSprintf ( "tables with meta prior to v.%u are no longer supported (use index_converter tool); %s is v.%u", uMinFormatVer, sMeta.cstr(), uVersion );
		return LOAD_E::GeneralError_e;
	}

	m_tStats.m_iTotalDocuments = rdMeta.GetDword();
	m_tStats.m_iTotalBytes = rdMeta.GetOffset();
	m_iTID = rdMeta.GetOffset();

	// tricky bit
	// we started saving settings into .meta from v.4 and up only
	// and those reuse disk format version, aka INDEX_FORMAT_VERSION
	// anyway, starting v.4, serialized settings take precedence over config
	// so different chunks can't have different settings any more
	CSphTokenizerSettings tTokenizerSettings;
	CSphDictSettings tDictSettings;
	CSphEmbeddedFiles tEmbeddedFiles;

	// load them settings
	DWORD uSettingsVer = rdMeta.GetDword();
	CSphSchema tSchema;
	ReadSchema ( rdMeta, tSchema, uSettingsVer );
	SetSchema ( std::move ( tSchema ) );
	LoadIndexSettings ( m_tSettings, rdMeta, uSettingsVer );
	if ( !tTokenizerSettings.Load ( pFilenameBuilder, rdMeta, tEmbeddedFiles, m_sLastError ) )
		return LOAD_E::GeneralError_e;

	{
		CSphString sWarning;
		tDictSettings.Load ( rdMeta, tEmbeddedFiles, pFilenameBuilder, sWarning );
		if ( !sWarning.IsEmpty() )
			dWarnings.Add(sWarning);
	}

	// initialize AOT if needed
	DWORD uPrevAot = m_tSettings.m_uAotFilterMask;
	m_tSettings.m_uAotFilterMask = sphParseMorphAot ( tDictSettings.m_sMorphology.cstr() );
	if ( m_tSettings.m_uAotFilterMask!=uPrevAot )
	{
		CSphString sWarning;
		sWarning.SetSprintf ( "table '%s': morphology option changed from config has no effect, ignoring", GetName() );
		dWarnings.Add(sWarning);
	}

	if ( bStripPath )
	{
		StripPath ( tTokenizerSettings.m_sSynonymsFile );
		ARRAY_FOREACH ( i, tDictSettings.m_dWordforms )
			StripPath ( tDictSettings.m_dWordforms[i] );
	}

	// recreate tokenizer
	m_pTokenizer = Tokenizer::Create ( tTokenizerSettings, &tEmbeddedFiles, pFilenameBuilder, dWarnings, m_sLastError, GetMaxTokenBytes ( tDictSettings.GetDictFormat() ) );
	if ( !m_pTokenizer )
		return LOAD_E::GeneralError_e;

	// recreate dictionary
	m_pDict = tDictSettings.IsWordDict()
		? sphCreateDictionaryKeywords ( tDictSettings, &tEmbeddedFiles, m_pTokenizer, GetName(), bStripPath, m_tSettings.m_iSkiplistBlockSize, pFilenameBuilder, m_sLastError )
		: sphCreateDictionaryCRC ( tDictSettings, &tEmbeddedFiles, m_pTokenizer, GetName(), bStripPath, m_tSettings.m_iSkiplistBlockSize, pFilenameBuilder, m_sLastError );
	if ( !m_sLastError.IsEmpty() )
		m_sLastError.SetSprintf ( "table '%s': %s", GetName(), m_sLastError.cstr() );

	if ( !m_pDict )
		return LOAD_E::GeneralError_e;

	if ( !m_sLastError.IsEmpty() )
		dWarnings.Add(m_sLastError);

	Tokenizer::AddToMultiformFilterTo ( m_pTokenizer, m_pDict->GetMultiWordforms () );

	m_iWordsCheckpoint = rdMeta.GetDword();

	// check that infixes definition changed - going to rebuild infixes
	m_iMaxCodepointLength = rdMeta.GetDword();
	int iBloomKeyLen = rdMeta.GetByte();
	int iBloomHashesCount = rdMeta.GetByte();
	bRebuildInfixes = ( iBloomKeyLen!=BLOOM_PER_ENTRY_VALS_COUNT || iBloomHashesCount!=BLOOM_HASHES_COUNT );

	if ( bRebuildInfixes )
	{
		CSphString sWarning;
		sWarning.SetSprintf ( "infix definition changed (from len=%d, hashes=%d to len=%d, hashes=%d) - rebuilding...",
			(int)BLOOM_PER_ENTRY_VALS_COUNT, (int)BLOOM_HASHES_COUNT, iBloomKeyLen, iBloomHashesCount );
		dWarnings.Add(sWarning);
	}

	std::unique_ptr<ISphFieldFilter> pFieldFilter;
	CSphFieldFilterSettings tFieldFilterSettings;
	tFieldFilterSettings.Load(rdMeta);
	if ( tFieldFilterSettings.m_dRegexps.GetLength() )
		pFieldFilter = sphCreateRegexpFilter ( tFieldFilterSettings, m_sLastError );

	if ( !sphSpawnFilterICU ( pFieldFilter, m_tSettings, tTokenizerSettings, sMeta.cstr(), m_sLastError ) )
		return LOAD_E::GeneralError_e;

	SetFieldFilter ( std::move ( pFieldFilter ) );

	int iLen = (int)rdMeta.GetDword();
	m_dChunkNames.Reset ( iLen );
	rdMeta.GetBytes ( m_dChunkNames.Begin(), iLen*sizeof(int) );

	if ( uVersion>=17 )
		SetMemLimit ( rdMeta.GetOffset() );

	return LOAD_E::Ok_e;
}


RtIndex_c::LOAD_E RtIndex_c::LoadMetaJson ( FilenameBuilder_i * pFilenameBuilder, bool bStripPath, DWORD & uVersion, bool & bRebuildInfixes, StrVec_t & dWarnings )
{

	using namespace bson;

	CSphString sMeta = GetFilename ( "meta" );

	CSphVector<BYTE> dData;
	if ( !sphJsonParse ( dData, sMeta, m_sLastError ) )
		return LOAD_E::ParseError_e;

	Bson_c tBson ( dData );
	if ( tBson.IsEmpty() || !tBson.IsAssoc() )
	{
		m_sLastError = "Something wrong read from json meta - it is either empty, either not root object.";
		return LOAD_E::ParseError_e;
	}

	// version
	uVersion = (DWORD)Int ( tBson.ChildByName ( "meta_version" ) );
	if ( uVersion==21 ) uVersion = 20; // fixme! a little hack, m.b. deal another way? v21 is minor of v20
	if ( uVersion == 0 || uVersion > META_VERSION )
	{
		m_sLastError.SetSprintf ( "%s is v.%u, binary is v.%u", sMeta.cstr(), uVersion, META_VERSION );
		return LOAD_E::GeneralError_e;
	}

	DWORD uMinFormatVer = 20;
	if ( uVersion<uMinFormatVer )
	{
		m_sLastError.SetSprintf ( "tables with meta prior to v.%u are no longer supported (use index_converter tool); %s is v.%u", uMinFormatVer, sMeta.cstr(), uVersion );
		return LOAD_E::GeneralError_e;
	}

	m_tStats.m_iTotalDocuments = Int ( tBson.ChildByName ( "total_documents" ) );
	m_tStats.m_iTotalBytes = Int ( tBson.ChildByName ( "total_bytes" ) );
	m_iTID = Int ( tBson.ChildByName ( "tid" ) );
	if ( m_iTID<0 )
		m_tSettings.m_bBinlog = false;

	// tricky bit
	// we started saving settings into .meta from v.4 and up only
	// and those reuse disk format version, aka INDEX_FORMAT_VERSION
	// anyway, starting v.4, serialized settings take precedence over config
	// so different chunks can't have different settings any more
	CSphTokenizerSettings tTokenizerSettings;
	CSphDictSettings tDictSettings;
	CSphEmbeddedFiles tEmbeddedFiles;

	// load them settings
//	DWORD uSettingsVer = Int ( tBson.ChildByName ( "index_format_version" ) );
	CSphSchema tSchema;
	ReadSchemaJson ( tBson.ChildByName ( "schema" ), tSchema );
	SetSchema ( std::move ( tSchema ) );
	LoadIndexSettingsJson ( tBson.ChildByName ( "index_settings" ), m_tSettings );
	if ( !tTokenizerSettings.Load ( pFilenameBuilder, tBson.ChildByName ( "tokenizer_settings" ), tEmbeddedFiles, m_sLastError ) )
		return LOAD_E::GeneralError_e;

	{
		CSphString sWarning;
		tDictSettings.Load ( tBson.ChildByName ( "dictionary_settings" ), tEmbeddedFiles, pFilenameBuilder, sWarning );
		if ( !sWarning.IsEmpty() )
			dWarnings.Add(sWarning);
	}

	// initialize AOT if needed
	DWORD uPrevAot = m_tSettings.m_uAotFilterMask;
	m_tSettings.m_uAotFilterMask = sphParseMorphAot ( tDictSettings.m_sMorphology.cstr() );
	if ( m_tSettings.m_uAotFilterMask!=uPrevAot )
	{
		CSphString sWarning;
		sWarning.SetSprintf ( "table '%s': morphology option changed from config has no effect, ignoring", GetName() );
		dWarnings.Add(sWarning);
	}

	if ( bStripPath )
	{
		StripPath ( tTokenizerSettings.m_sSynonymsFile );
		ARRAY_FOREACH ( i, tDictSettings.m_dWordforms )
			StripPath ( tDictSettings.m_dWordforms[i] );
	}

	// recreate tokenizer
	m_pTokenizer = Tokenizer::Create ( tTokenizerSettings, &tEmbeddedFiles, pFilenameBuilder, dWarnings, m_sLastError, GetMaxTokenBytes ( tDictSettings.GetDictFormat() ) );
	if ( !m_pTokenizer )
		return LOAD_E::GeneralError_e;

	// recreate dictionary
	m_pDict = tDictSettings.IsWordDict()
		? sphCreateDictionaryKeywords ( tDictSettings, &tEmbeddedFiles, m_pTokenizer, GetName(), bStripPath, m_tSettings.m_iSkiplistBlockSize, pFilenameBuilder, m_sLastError )
		: sphCreateDictionaryCRC ( tDictSettings, &tEmbeddedFiles, m_pTokenizer, GetName(), bStripPath, m_tSettings.m_iSkiplistBlockSize, pFilenameBuilder, m_sLastError );
	if ( !m_sLastError.IsEmpty() )
		m_sLastError.SetSprintf ( "table '%s': %s", GetName(), m_sLastError.cstr() );

	if ( !m_pDict )
		return LOAD_E::GeneralError_e;

	if ( !m_sLastError.IsEmpty() )
		dWarnings.Add(m_sLastError);

	Tokenizer::AddToMultiformFilterTo ( m_pTokenizer, m_pDict->GetMultiWordforms () );

	m_iWordsCheckpoint = (int)Int ( tBson.ChildByName ( "words_checkpoint" ) );

	// check that infixes definition changed - going to rebuild infixes
	m_iMaxCodepointLength = (int)Int ( tBson.ChildByName ( "max_codepoint_length" ) );
	int iBloomKeyLen = (int)Int ( tBson.ChildByName ( "bloom_per_entry_vals_count" ), 8 );
	int iBloomHashesCount = (int)Int ( tBson.ChildByName ( "bloom_hashes_count" ), 2 );
	bRebuildInfixes = ( iBloomKeyLen!=BLOOM_PER_ENTRY_VALS_COUNT || iBloomHashesCount!=BLOOM_HASHES_COUNT );

	if ( bRebuildInfixes )
	{
		CSphString sWarning;
		sWarning.SetSprintf ( "infix definition changed (from len=%d, hashes=%d to len=%d, hashes=%d) - rebuilding...",
			(int)BLOOM_PER_ENTRY_VALS_COUNT, (int)BLOOM_HASHES_COUNT, iBloomKeyLen, iBloomHashesCount );
		dWarnings.Add(sWarning);
	}

	std::unique_ptr<ISphFieldFilter> pFieldFilter;
	auto tFieldFilterSettingsNode = tBson.ChildByName ( "field_filter_settings" );
	if ( !IsNullNode ( tFieldFilterSettingsNode ) )
	{
		CSphFieldFilterSettings tFieldFilterSettings;
		Bson_c ( tFieldFilterSettingsNode ).ForEach ( [&tFieldFilterSettings] ( const NodeHandle_t& tNode ) {
			tFieldFilterSettings.m_dRegexps.Add ( String ( tNode ) );
		} );

		if ( !tFieldFilterSettings.m_dRegexps.IsEmpty() )
			pFieldFilter = sphCreateRegexpFilter ( tFieldFilterSettings, m_sLastError );
	}

	if ( !sphSpawnFilterICU ( pFieldFilter, m_tSettings, tTokenizerSettings, sMeta.cstr(), m_sLastError ) )
		return LOAD_E::GeneralError_e;

	SetFieldFilter ( std::move ( pFieldFilter ) );

	Bson_c tNamesVec { tBson.ChildByName ( "chunk_names" ) };
	m_dChunkNames.Reset ( tNamesVec.CountValues() );
	int iLastQ = 0;
	tNamesVec.ForEach ( [&iLastQ, this] ( const NodeHandle_t& tNode ) { m_dChunkNames[iLastQ++] = (int)Int ( tNode ); });

	SetMemLimit ( Int ( tBson.ChildByName ( "soft_ram_limit" ) ) );

	auto tIndexId = tBson.ChildByName ( "index_id" );
	if ( !IsNullNode ( tIndexId ) )
		m_iIndexId = Int ( tIndexId );

	return LOAD_E::Ok_e;
}


bool RtIndex_c::LoadMetaImpl ( FilenameBuilder_i * pFilenameBuilder, bool bStripPath, DWORD & uVersion, bool & bRebuildInfixes, StrVec_t & dWarnings )
{
	// check if we have a meta file (kinda-header)
	CSphString sMeta = GetFilename ( "meta" );

	// no readable meta? no disk part yet
	if ( !sphIsReadable ( sMeta.cstr() ) )
		return true;

	auto eRes = LoadMetaJson ( pFilenameBuilder, bStripPath, uVersion, bRebuildInfixes, dWarnings );
	if ( eRes == LOAD_E::ParseError_e )
	{
		sphInfo ( "Index meta format is not json, will try it as binary..." );
		eRes = LoadMetaLegacy ( pFilenameBuilder, bStripPath, uVersion, bRebuildInfixes, dWarnings );
		if ( eRes == LOAD_E::ParseError_e )
		{
			sphWarning ( "Unable to parse header... Error %s", m_sLastError.cstr() );
			return false;
		}
	}
	if ( eRes == LOAD_E::GeneralError_e )
	{
		sphWarning ( "Unable to load header... Error %s", m_sLastError.cstr() );
		return false;
	}

	assert ( eRes == LOAD_E::Ok_e );
	return true;
}

bool RtIndex_c::LoadMeta ( FilenameBuilder_i* pFilenameBuilder, bool bStripPath, DWORD& uVersion, bool& bRebuildInfixes, StrVec_t& dWarnings )
{
	if ( LoadMetaImpl ( pFilenameBuilder, bStripPath, uVersion, bRebuildInfixes, dWarnings ) )
		return true;

	const char* szDumpPath = getenv ( "dump_corrupt_meta" );
	if ( !szDumpPath )
		return false;

	CSphString sMeta = GetFilename ( "meta" );
	CSphString sDestPath = SphSprintf ( "%s%s", szDumpPath, "index.meta" );
	CSphString sError;
	if ( !CopyFile ( sMeta, sDestPath, sError ) )
		sphWarning ( "%s", sError.cstr() );
	return false;
}

// called once on index start lifetime
bool RtIndex_c::PreallocDiskChunks ( FilenameBuilder_i * pFilenameBuilder, StrVec_t & dWarnings ) NO_THREAD_SAFETY_ANALYSIS
{
	// load disk chunks, if any
	auto tWriter = RtWriter();
	tWriter.InitDiskChunks ( RtWriter_c::empty );
	ARRAY_FOREACH ( iName, m_dChunkNames )
	{
		int iChunkIndex = m_dChunkNames[iName];
		CSphString sChunk = GetFilename ( iChunkIndex );
		auto pChunk = DiskChunk_c::make ( PreallocDiskChunk ( sChunk, iChunkIndex, pFilenameBuilder, dWarnings, m_sLastError ) );
		if ( !pChunk )
			return false;

		auto* pIndex = (const CSphIndex*)*pChunk;

		// tricky bit
		// outgoing match schema on disk chunk should be identical to our internal (!) schema
		if ( !m_tSchema.CompareTo ( pIndex->GetMatchSchema(), m_sLastError, true, true ) )
			return false;

		tWriter.m_pNewDiskChunks->Add ( pChunk );

		// update field lengths
		if ( m_tSchema.GetAttrId_FirstFieldLen()>=0 )
		{
			int64_t * pLens = pIndex->GetFieldLens();
			if ( pLens )
				for ( int i=0; i < pIndex->GetMatchSchema().GetFieldsCount(); ++i )
					m_dFieldLensDisk[i] += pLens[i];
		}
	}

	m_dChunkNames.Reset(0);
	return true;
}


static bool PrepareEmbeddingModelsForSchema ( CSphSchema & tSchema, std::unique_ptr<TableEmbeddings_c> & pEmbeddings, CSphVector<AttrWithModel_t> & dAttrsWithModels, CSphString & sError )
{
	pEmbeddings.reset();
	dAttrsWithModels.Reset();

	bool bHaveModels = false;
	for ( int i = 0 ; i < tSchema.GetAttrsCount(); i++ )
		bHaveModels |= !tSchema.GetAttr(i).m_tKNNModel.m_sModelName.empty();

	if ( !bHaveModels )
		return true;

	dAttrsWithModels.Resize ( tSchema.GetAttrsCount() );
	pEmbeddings = std::make_unique<TableEmbeddings_c>();
	for ( int i = 0; i < tSchema.GetAttrsCount(); i++ )
	{
		const auto & tAttr = tSchema.GetAttr(i);
		dAttrsWithModels[i].m_pModel = nullptr;

		if ( tAttr.m_tKNNModel.m_sModelName.empty() )
			continue;

		if ( !ParseEmbeddingSources ( dAttrsWithModels[i].m_dFrom, tAttr.m_sKNNFrom, tSchema, sError ) )
			return false;

		if ( !pEmbeddings->Load ( tAttr.m_sName, tAttr.m_tKNNModel, sError ) )
			return false;

		auto pModel = pEmbeddings->GetModel ( tAttr.m_sName );
		assert(pModel);

		dAttrsWithModels[i].m_pModel = pModel;

		// fixme! modifying the schema
		const_cast<CSphColumnInfo&>(tAttr).m_tKNN.m_iDims = pModel->GetDims();
	}

	return true;
}

bool RtIndex_c::LoadEmbeddingModels ( CSphString & sError )
{
	std::unique_ptr<TableEmbeddings_c> pPreparedEmbeddings;
	CSphVector<AttrWithModel_t> dPreparedAttrsWithModels;
	if ( !PrepareEmbeddingModelsForSchema ( m_tSchema, pPreparedEmbeddings, dPreparedAttrsWithModels, sError ) )
		return false;

	m_pEmbeddings = std::move ( pPreparedEmbeddings );
	m_dAttrsWithModels.SwapData ( dPreparedAttrsWithModels );
	return true;
}


bool RtIndex_c::Prealloc ( bool bStripPath, FilenameBuilder_i * pFilenameBuilder, StrVec_t & dWarnings )
{
	MEMORY ( MEM_INDEX_RT );

	// locking uber alles
	// in RT backend case, we just must be multi-threaded
	// so we simply lock here, and ignore Lock/Unlock hassle caused by forks
	assert ( m_iLockFD<0 );

	m_bPreallocPassedOk = false;

	CSphString sLock = GetFilename ( "lock" );
	m_iLockFD = ::open ( sLock.cstr(), SPH_O_NEW, 0644 );
	if ( m_iLockFD<0 )
	{
		m_sLastError.SetSprintf ( "failed to open %s: %s", sLock.cstr(), strerrorm(errno) );
		return false;
	}

	if ( !sphLockEx ( m_iLockFD, false ) )
	{
		SafeClose ( m_iLockFD );
		if ( !m_bDebugCheck )
		{
			m_sLastError.SetSprintf ( "failed to lock %s: %s", sLock.cstr(), strerrorm(errno) );
			return false;
		}
	}

	DWORD uVersion = 0;
	bool bRebuildInfixes = false;
	if ( !LoadMeta ( pFilenameBuilder, bStripPath, uVersion, bRebuildInfixes, dWarnings ) )
		return false;

	CSphString sMutableFile = GetFilename ( SPH_EXT_SETTINGS );
	m_tMutableSettings.m_iMemLimit = m_iRtMemLimit; // to avoid overriding value from meta by default value, if no settings provided
	if ( !m_tMutableSettings.Load ( sMutableFile.cstr(), GetName() ) )
		return false;
	SetMemLimit ( m_tMutableSettings.m_iMemLimit );
	if ( m_tMutableSettings.IsSet ( MutableName_e::GLOBAL_IDF ) )
		m_sGlobalIDFPath = m_tMutableSettings.m_sGlobalIDFPath;

	m_bPathStripped = bStripPath;

	if ( m_tSchema.HasColumnarAttrs() && !IsColumnarLibLoaded() )
	{
		m_sLastError.SetSprintf ( "failed to load table with columnar attributes without columnar library" );
		return false;
	}

	if ( m_tSchema.HasKNNAttrs() && !IsKNNLibLoaded() )
	{
		m_sLastError.SetSprintf ( "failed to load table with knn attributes without knn library" );
		return false;
	}

	if ( m_bDebugCheck )
	{
		// load ram chunk
		m_bPreallocPassedOk = LoadRamChunk ( uVersion, bRebuildInfixes, false );
		return m_bPreallocPassedOk;
	}

	m_tWorkers.InitWorkers();
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "Prealloc" );

	if ( !PreallocDiskChunks ( pFilenameBuilder, dWarnings ) )
		return false;

	// load ram chunk
	m_bPreallocPassedOk = LoadRamChunk ( uVersion, bRebuildInfixes );

	// field lengths
	ARRAY_FOREACH ( i, m_dFieldLens )
		m_dFieldLens[i] = m_dFieldLensDisk[i] + m_dFieldLensRam[i];

	if ( !LoadEmbeddingModels(m_sLastError) )
		return false;

	// set up values for on timer save
	m_iSavedTID = m_iTID;
	m_tmSaved = sphMicroTimer();

	// maybe fixup document's counter. It may be damaged, say, if you delete some documents and then shut down daemon without saving the index.
	// in this case meta will show previous number of indexed documents, but actually part of them already killed.
	// (binlog also will not help, as kill-map is mmapped, and is not transactional, i.e. on replay it will detect the documents as already killed)
	const auto iCount = GetCount();
	if ( m_tStats.m_iTotalDocuments!=iCount) {
		sphWarning ( "Detected wrong count of indexed documents: " INT64_FMT " in meta vs " INT64_FMT " actually", m_tStats.m_iTotalDocuments, iCount );
		m_tStats.m_iTotalDocuments = iCount;
	}

	// neet to set m_iSoftRamLimit more than iUsedRam to prevent flush of disk chunk right after index load
	int64_t iUsedRam = SegmentsGetUsedRam ( *m_tRtChunks.RamSegs() );
	RecalculateRateLimit ( iUsedRam, 1, false );

	RunMergeSegmentsWorker();
	return m_bPreallocPassedOk;
}


void RtIndex_c::Preread ()
{
	auto pChunks = m_tRtChunks.DiskChunks();
	for ( auto& pChunk : *pChunks )
	{
		if (pChunk)
			pChunk->CastIdx().Preread();

		if ( sphInterrupted() )
			break;
	}
}

template<typename P>
static bool CheckVectorLength ( int iLen, int64_t iMinLen, const char * sAt, CSphString & sError )
{
	auto iSaneLen = Min ( iMinLen, P::SANE_SIZE );
	if ( iLen>=0 && iLen<iSaneLen )
		return true;

	sError.SetSprintf ( "broken table, %s length overflow (len=%d, max=" INT64_FMT ")", sAt, iLen, iSaneLen );
	return false;
}

template < typename T >
static void SaveVector ( CSphWriter & tWriter, const VecTraits_T < T > & tVector )
{
	static_assert ( IS_TRIVIALLY_COPYABLE(T), "non trivial vectors are unserializable" );
	tWriter.PutDword ( tVector.GetLength() );
	if ( tVector.GetLength() )
		tWriter.PutBytes ( tVector.Begin(), tVector.GetLengthBytes() );
}


template < typename T, typename P >
static bool LoadVector ( CSphReader & tReader, CSphVector < T, P > & tVector,
	int64_t iMinLen, const char * sAt, CSphString & sError )
{
	static_assert ( IS_TRIVIALLY_COPYABLE(T), "non trivial vectors are unserializable" );
	int iSize = tReader.GetDword();
	if ( !CheckVectorLength<P> ( iSize, iMinLen, sAt, sError ) )
		return false;

	tVector.Resize ( iSize );
	if ( tVector.GetLength() )
		tReader.GetBytes ( tVector.Begin(), (int) tVector.GetLengthBytes() );

	return true;
}

void RtIndex_c::SaveRamSegment ( const RtSegment_t* pSeg, CSphWriter& wrChunk ) const REQUIRES_SHARED ( pSeg->m_tLock )
{
	const bool bWordDict = IsWordDict();
	wrChunk.PutDword ( pSeg->m_uRows );
	wrChunk.PutDword ( (DWORD)pSeg->m_tAliveRows.load ( std::memory_order_relaxed ) );
	wrChunk.PutDword ( 0 );
	SaveVector ( wrChunk, pSeg->m_dWords );
	if ( bWordDict )
		SaveVector ( wrChunk, pSeg->m_dKeywordCheckpoints );

	wrChunk.PutDword ( pSeg->m_dWordCheckpoints.GetLength() );
	for ( const auto& dWordCheckpoint : pSeg->m_dWordCheckpoints )
	{
		wrChunk.PutOffset ( dWordCheckpoint.m_iOffset );
		if ( bWordDict )
			wrChunk.PutOffset ( dWordCheckpoint.m_szWord - (const char *)pSeg->m_dKeywordCheckpoints.Begin() );
		else
			wrChunk.PutOffset ( dWordCheckpoint.m_uWordID );
	}

	SaveVector ( wrChunk, pSeg->m_dDocs );
	SaveVector ( wrChunk, pSeg->m_dHits );
	SaveVector ( wrChunk, pSeg->m_dRows );
	pSeg->m_tDeadRowMap.Save ( wrChunk );
	SaveVector ( wrChunk, pSeg->m_dBlobs );

	if ( pSeg->m_pDocstore )
		pSeg->m_pDocstore->Save ( wrChunk );

	wrChunk.PutByte ( pSeg->m_pColumnar ? 1 : 0 );
	if ( pSeg->m_pColumnar )
		pSeg->m_pColumnar->Save ( wrChunk );

	// infixes
	SaveVector ( wrChunk, pSeg->m_dInfixFilterCP );
}

void RtIndex_c::SaveRamFieldLengths ( CSphWriter& wrChunk ) const
{
	// field lengths
	wrChunk.PutDword ( m_tSchema.GetFieldsCount() );
	for ( int i = 0; i < m_tSchema.GetFieldsCount(); ++i )
		wrChunk.PutOffset ( m_dFieldLensRam[i] );
}

bool RtIndex_c::SaveRamChunk ()
{
	if ( !m_tSaving.ActiveStateIs ( SaveState_c::ENABLED ) )
		return false;

	MEMORY ( MEM_INDEX_RT );

	CSphString sChunk = GetFilename ( "ram" );
	CSphString sNewChunk = GetFilename ( "ram.new" );

	CSphWriterNonThrottled wrChunk;
	if ( !wrChunk.OpenFile ( sNewChunk, m_sLastError ) )
		return false;

	auto pSegments = m_tRtChunks.RamSegs();
	auto& dSegments = *pSegments;
	wrChunk.PutDword ( 0 );
	wrChunk.PutDword ( dSegments.GetLength() );

	// no locks here, because it's only intended to be called from dtor
	for ( const RtSegment_t * pSeg : dSegments )
	{
		SccRL_t rLock ( pSeg->m_tLock );
		SaveRamSegment ( pSeg, wrChunk );
	}

	SaveRamFieldLengths ( wrChunk );

	wrChunk.CloseFile();
	if ( wrChunk.IsError() )
		return false;

	// rename
	if ( sph::rename ( sNewChunk.cstr(), sChunk.cstr() ) )
		sphDie ( "failed to rename ram chunk (src=%s, dst=%s, errno=%d, error=%s)",
			sNewChunk.cstr(), sChunk.cstr(), errno, strerrorm(errno) ); // !COMMIT handle this gracefully

	return true;
}


bool RtIndex_c::LoadRamChunk ( DWORD uVersion, bool bRebuildInfixes, bool bFixup ) NO_THREAD_SAFETY_ANALYSIS
{
	MEMORY ( MEM_INDEX_RT );

	CSphString sChunk = GetFilename ( "ram" );

	if ( !sphIsReadable ( sChunk.cstr(), &m_sLastError ) )
		return true;

	m_bHasFiles = true;

	CSphAutoreader rdChunk;
	if ( !rdChunk.Open ( sChunk, m_sLastError ) )
		return false;

	int64_t iFileSize = rdChunk.GetFilesize();

	assert ( m_pDict );
	const bool bWordDict = IsWordDict();
	bool bHasMorphology = m_pDict->HasMorphology();
	rdChunk.GetDword();

	auto iSegmentCount = (int) rdChunk.GetDword();
	if ( !CheckVectorLength<RtSegVec_c::BASE> ( iSegmentCount, iFileSize, "ram-chunks", m_sLastError ) )
		return false;

	CSphVector<RtSegmentRefPtf_t> dRawSegments;
	bool bSafeLoad = bFixup && iSegmentCount > MAX_TOLERATE_LOAD_SEGMENTS;

	if ( bSafeLoad )
		sphWarning ( "RAM chunk has %d segments, need to be repaired...", iSegmentCount );
	int iEmpty=0;
	DWORD uAlive = 0;

	auto tWriter = RtWriter();
	tWriter.InitRamSegs ( RtWriter_c::empty );
	for ( int i = 0; i < iSegmentCount; ++i )
	{
		DWORD uRows = rdChunk.GetDword();

		RtSegmentRefPtf_t pSeg { new RtSegment_t ( uRows, m_tSchema ) };
		pSeg->m_tAliveRows.store ( rdChunk.GetDword (), std::memory_order_relaxed );

		rdChunk.GetDword ();
		if ( !LoadVector ( rdChunk, pSeg->m_dWords, iFileSize, "ram-words", m_sLastError ) )
			return false;

		if ( bWordDict && !LoadVector ( rdChunk, pSeg->m_dKeywordCheckpoints, iFileSize, "ram-checkpoints", m_sLastError ) )
			return false;

		auto iCheckpointCount = (int) rdChunk.GetDword();
		if ( !CheckVectorLength<decltype( pSeg->m_dWordCheckpoints)> ( iCheckpointCount, iFileSize, "ram-checkpoints", m_sLastError ) )
			return false;

		pSeg->m_dWordCheckpoints.Resize ( iCheckpointCount );
		for ( auto& tWordCheckpoint : pSeg->m_dWordCheckpoints )
		{
			tWordCheckpoint.m_iOffset = (int)rdChunk.GetOffset();
			SphOffset_t uOff = rdChunk.GetOffset();
			tWordCheckpoint.m_uWordID = (SphWordID_t)uOff;
		}

		if ( bWordDict )
			FixupSegmentCheckpoints ( pSeg );

		if ( !LoadVector ( rdChunk, pSeg->m_dDocs, iFileSize, "ram-doclist", m_sLastError ) )
			return false;

		if ( !LoadVector ( rdChunk, pSeg->m_dHits, iFileSize, "ram-hitlist", m_sLastError ) )
			return false;

		if ( !LoadVector ( rdChunk, pSeg->m_dRows, iFileSize, "ram-attributes", m_sLastError ) )
			return false;

		pSeg->m_tDeadRowMap.Load ( uRows, rdChunk, m_sLastError );

		if ( !LoadVector ( rdChunk, pSeg->m_dBlobs, iFileSize, "ram-blobs", m_sLastError ) )
			return false;

		if ( uVersion>=15 && ( m_tSchema.HasStoredFields() || m_tSchema.HasStoredAttrs() ) )
		{
			pSeg->m_pDocstore = CreateDocstoreRT();
			SetupDocstoreFields ( *pSeg->m_pDocstore, m_tSchema );
			assert ( pSeg->m_pDocstore );
			if ( !pSeg->m_pDocstore->Load ( rdChunk ) )
				return false;
		}

		if ( uVersion>=19 && rdChunk.GetByte() )
		{
			pSeg->m_pColumnar = CreateColumnarRT ( m_tSchema, rdChunk, m_sLastError );
			if ( !pSeg->m_pColumnar )
				return false;
		}

		// infixes
		if ( !LoadVector ( rdChunk, pSeg->m_dInfixFilterCP, iFileSize, "ram-infixes", m_sLastError ) )
			return false;

		if ( bRebuildInfixes )
			BuildSegmentInfixes ( pSeg, bHasMorphology, GetDictFormat(), m_tSettings.m_iMinInfixLen, m_iWordsCheckpoint, ( m_iMaxCodepointLength>1 ), m_tSettings.m_eHitless );

		pSeg->BuildDocID2RowIDMap(m_tSchema);

		CheckSegmentConsistency ( pSeg );

		if ( bSafeLoad )
		{
			int64_t iAlive = pSeg->m_tAliveRows.load ( std::memory_order_relaxed );
			if ( iAlive )
			{
				uAlive += iAlive;
 				dRawSegments.Add ( pSeg );
			} else // skip that dead guy...
				++iEmpty;
		} else
			tWriter.m_pNewRamSegs->Add ( AdoptSegment ( pSeg ) );
	}

	if ( bSafeLoad )
	{
		iSegmentCount = dRawSegments.GetLength();
		sphInfo ( "RAM chunk repairing %d segments (%d totally killed segments dropped)", iSegmentCount, iEmpty );

		// ideal rt-chunk structure (model) - rows per segment, sorted desc.
		// 1024 1024 1024 ... 1024 512 256 128  64
		// |______ MAX_SEGMENTS _________________|
		// ..                      |_PROGRESSION_|
		//
		// All 'progression' segments together have about same N of rows as one non-progression,
		// we can consider that for same-sized segments we should distribute all rows over (MAX_SEGMENTS-1) segments
		// Minimal segment, in turn, is 2^PROGRESSION times smaller than typical for distribution.
		// For simplicity, let's run merge with stop on any of 2 criterias: enough rows in segments, and small enough total N of segments.
		// In 'ideal' case we will end with exactly 24 (that is MAX_SEGMENTS - MAX_PROGRESSION_SEGMENT) of maximal size, and no progression.
		// In 'real' case we most probably end with about 96 segments with different sizes, and they'll be finished to ideal 24..32 by usual route.

		auto uTargetRows = uAlive / ( MAX_SEGMENTS - MAX_PROGRESSION_SEGMENT );

		sphInfo ( "RAM chunk repairing of %d segments with %d rows. Achieve min segment with %d rows", iSegmentCount, uAlive, uTargetRows );
		auto iAliveSegments = iSegmentCount;
		CSphVector<int> dSegNums;
		dSegNums.Reserve ( iAliveSegments );
		dSegNums.Add ( 0 ); // that is for very first step
		int iPass = 0;

		while ( dRawSegments[dSegNums.First()]->m_tAliveRows.load ( std::memory_order_relaxed ) < uTargetRows && iAliveSegments >= MAX_TOLERATE_LOAD_SEGMENTS )
		{
			dSegNums.Resize ( iAliveSegments );
			for ( int i = 0, j = 0; i < iSegmentCount; ++i )
			{
				if ( dRawSegments[i] )
					dSegNums[j++] = i;
			}
			dSegNums.Sort ( Lesser ( [&dRawSegments] ( int a, int b ) { return dRawSegments[a]->m_tAliveRows.load ( std::memory_order_relaxed ) < dRawSegments[b]->m_tAliveRows.load ( std::memory_order_relaxed ); } ) );
			bool bMergeHappened = false;
			sphInfo ( "RAM chunk repairing pass %d (%d segments)", iPass+1, iAliveSegments );
			for ( int i = 0, iMax = iAliveSegments - 1; i < iMax; i += 2 )
			{
				auto& pA = dRawSegments[dSegNums[i]];
				auto& pB = dRawSegments[dSegNums[i+1]];
				if ( pB->m_tAliveRows.load ( std::memory_order_relaxed ) >= uTargetRows )
					break;

				auto eDecision = CheckSegmentsPair ( { pA, pB } );
				if ( eDecision == CheckMerge_e::MERGE )
				{
					RtSegmentRefPtf_t pMerged { MergeTwoSegments ( pA, pB ) };
					assert ( pMerged );
					pA = pMerged;
					pB = nullptr;
					--iAliveSegments;
					bMergeHappened = true;
				} // else Fixme! Case FLUSH
			}

			if ( !bMergeHappened ) // that could happen if it was no positive merge decision. For example, if all segments are huge and so can't be merged at all
				break;
			++iPass;
		}
		sphInfo ( "RAM chunk repairing: min segment %d achieved in %d passes; now %d segments left", uTargetRows, iPass, iAliveSegments );
		for ( auto& pSeg : dRawSegments )
			if ( pSeg )
				tWriter.m_pNewRamSegs->Add ( AdoptSegment ( pSeg ) );
		dRawSegments.Reset();
		sphWarning ( "RAM chunk has %d segments after repairing. You need to flush this table, otherwise result could be LOST, and repairing will start again on daemon's restart", tWriter.m_pNewRamSegs->GetLength() );
	}

	// field lengths
	auto iFields = (int) rdChunk.GetDword();
	assert ( iFields==m_tSchema.GetFieldsCount() );

	for ( int i=0; i<iFields; ++i )
		m_dFieldLensRam[i] = rdChunk.GetOffset();

	// all done
	return !rdChunk.GetErrorFlag();
}


void RtIndex_c::PostSetup()
{
	RtIndex_i::PostSetup();

	std::unique_ptr<DocstoreFields_i> pDocstoreFields;
	if ( m_tSchema.HasStoredFields() || m_tSchema.HasStoredAttrs() )
	{
		pDocstoreFields = CreateDocstoreFields();
		SetupDocstoreFields ( *pDocstoreFields, m_tSchema );
	}
	m_pDocstoreFields = std::move ( pDocstoreFields );
	m_iMaxCodepointLength = m_pTokenizer->GetMaxCodepointLength();

	// bigram filter
	if ( m_tSettings.m_eBigramIndex!=SPH_BIGRAM_NONE && BigramNeedsFreq ( m_tSettings.m_eBigramIndex ) )
	{
		m_pTokenizer->SetBuffer ( (BYTE*)const_cast<char*> ( m_tSettings.m_sBigramWords.cstr() ), m_tSettings.m_sBigramWords.Length() );

		BYTE * pTok = nullptr;
		while ( ( pTok = m_pTokenizer->GetToken() )!=nullptr )
			m_tSettings.m_dBigramWords.Add() = (const char*)pTok;

		m_tSettings.m_dBigramWords.Sort();
	}

	// FIXME!!! handle error
	m_pTokenizerIndexing = m_pTokenizer->Clone ( SPH_CLONE_INDEX );
	Tokenizer::AddBigramFilterTo ( m_pTokenizerIndexing, m_tSettings.m_eBigramIndex, m_tSettings.m_eBigramDelimiter, m_tSettings.m_sBigramWords, m_sLastError );

	// hitless
	CSphString sHitlessFiles = m_tSettings.m_sHitlessFiles;
	if ( GetIndexFilenameBuilder() )
	{
		std::unique_ptr<FilenameBuilder_i> pFilenameBuilder = GetIndexFilenameBuilder() ( GetName() );
		if ( pFilenameBuilder )
			sHitlessFiles = pFilenameBuilder->GetFullPath ( sHitlessFiles );
	}

	if ( !LoadHitlessWords ( sHitlessFiles, m_pTokenizerIndexing, m_pDict, m_dHitlessWords, m_sLastError ) )
		sphWarning ( "table '%s': %s", GetName(), m_sLastError.cstr() );

	// still need index files for index just created from config
	if ( !m_bHasFiles )
	{
		SaveRamChunk ();
		SaveMeta ();
	}
}

struct MemoryDebugCheckReader_c : public DebugCheckReader_i
{
	MemoryDebugCheckReader_c ( const BYTE * pData, const BYTE * pDataEnd )
		: m_pData ( pData )
		, m_pDataEnd ( pDataEnd )
		, m_pCur ( pData )
	{}

	int64_t GetLengthBytes () override
	{
		return ( m_pDataEnd - m_pData );
	}
	bool GetBytes ( void * pData, int iSize ) override
	{
		if ( m_pCur && m_pCur+iSize<=m_pDataEnd )
		{
			memcpy ( pData, m_pCur, iSize );
			m_pCur += iSize;
			return true;
		} else
		{
			return false;
		}
	}

	bool SeekTo ( int64_t iOff, int iHint ) final
	{
		if ( m_pData && m_pData+iOff<m_pDataEnd )
		{
			m_pCur = m_pData + iOff;
			return true;
		} else
		{
			return false;
		}
	}

	const BYTE * m_pData = nullptr;
	const BYTE * m_pDataEnd = nullptr;
	const BYTE * m_pCur = nullptr;
};

int RtIndex_c::DebugCheck ( DebugCheckError_i& tReporter, FilenameBuilder_i * )
{
	// FIXME! remove copypasted code from CSphIndex_VLN::DebugCheck

	if ( m_iLockFD<0 && m_iCheckChunk==-1 )
		sphWarning ( "failed to load RAM chunks, checking only %d disk chunks", m_dChunkNames.GetLength() );

	if ( m_iStride!=m_tSchema.GetRowSize() )
		tReporter.Fail ( "wrong attribute stride (current=%d, should_be=%d)", m_iStride, m_tSchema.GetRowSize() );

	if ( m_iRtMemLimit<=0 )
		tReporter.Fail ( "wrong RAM limit (current=" INT64_FMT ")", m_iRtMemLimit );

	if ( m_iTID<-1 ) // -1 is valid value for 'no binlog'
	{
		tReporter.Fail ( "table TID < -1 (current=" INT64_FMT ")", m_iTID );

		if ( m_iSavedTID<-1 )
			tReporter.Fail ( "table saved TID < -1 (current=" INT64_FMT ")", m_iSavedTID );

		if ( m_iTID<m_iSavedTID )
			tReporter.Fail ( "table TID < table saved TID (current=" INT64_FMT ", saved=" INT64_FMT ")", m_iTID, m_iSavedTID );
	}

	if ( m_iWordsCheckpoint!=RTDICT_CHECKPOINT_V5 )
		tReporter.Fail ( "unexpected number of words per checkpoint (expected 48, got %d)", m_iWordsCheckpoint );

	tReporter.Msg ( "checking schema..." );
	DebugCheckSchema ( m_tSchema, tReporter );

	if ( m_iCheckChunk==-1 )
		DebugCheckRam ( tReporter );

	int iFailsPlain = DebugCheckDisk ( tReporter );

	tReporter.Done();

	return int ( tReporter.GetNumFails() + iFailsPlain );
}

void RtIndex_c::DebugCheckRamSegment ( const RtSegment_t & tSegment, int iSegment, DebugCheckError_i & tReporter ) const NO_THREAD_SAFETY_ANALYSIS
{
	if ( !tSegment.m_uRows )
	{
		tReporter.Fail ( "empty RT segment (segment=%d)", iSegment );
		return;
	}

	const BYTE * pCurWord = tSegment.m_dWords.Begin();
	const BYTE * pMaxWord = pCurWord+tSegment.m_dWords.GetLength();
	const BYTE * pCurDoc = tSegment.m_dDocs.Begin();
	const BYTE * pMaxDoc = pCurDoc+tSegment.m_dDocs.GetLength();
	const BYTE * pCurHit = tSegment.m_dHits.Begin();
	const BYTE * pMaxHit = pCurHit+tSegment.m_dHits.GetLength();

	CSphVector<RtWordCheckpoint_t> dRefCheckpoints;
	int nWordsRead = 0;
	int nCheckpointWords = 0;
	int iCheckpointOffset = 0;
	SphWordID_t uPrevWordID = 0;
	DWORD uPrevDocOffset = 0;
	DWORD uPrevHitOffset = 0;

	RtWord_t tWord;
	tWord.m_bHasHitlist = false;

	const DictFormat_e eDictFormat = GetDictFormat();
	const bool bWordDict = eDictFormat!=DictFormat_e::CRC;
	const int iKeywordBufferSize = GetKeywordBufSize ( eDictFormat );
	CSphVector<BYTE> dWord ( iKeywordBufferSize );
	CSphVector<BYTE> dLastWord ( iKeywordBufferSize );
	memset ( dWord.Begin(), 0, dWord.GetLength() );
	memset ( dLastWord.Begin(), 0, dLastWord.GetLength() );
	auto szWord = (const char*)dWord.Begin();

	int iLastWordLen = 0, iDeltaWordLen = 0, iWordLen = 0;

	while ( pCurWord && pCurWord<pMaxWord )
	{
		bool bCheckpoint = ++nCheckpointWords==m_iWordsCheckpoint;
		if ( bCheckpoint )
		{
			nCheckpointWords = 1;
			iCheckpointOffset = int ( pCurWord - tSegment.m_dWords.Begin() );
			tWord.m_uDoc = 0;
			if ( !bWordDict )
				tWord.m_uWordID = 0;
			else
			{
				iDeltaWordLen = 0;
				dWord[0] = '\0';
			}
		}

		const BYTE * pIn = pCurWord;
		if ( bWordDict )
		{
			if ( !RtDebugCheckUnpackKeyword ( pIn, pMaxWord, eDictFormat, dWord, iWordLen, dWord, iDeltaWordLen,
				iSegment, nWordsRead, tReporter ) )
				break;

			auto iCalcWordLen = (int) strlen ( (const char *)dWord.Begin() );
			if ( iWordLen!=iCalcWordLen )
			{
				tReporter.Fail ( "word length mismatch (segment=%d, word=%d, read_word=%s, read_len=%d, calc_len=%d)", iSegment, nWordsRead, dWord.Begin(), iWordLen, iCalcWordLen );
			}

			if ( !iWordLen )
				tReporter.Fail ( "empty word in word list (segment=%d, word=%d)", iSegment, nWordsRead );

			const BYTE * pStr = dWord.Begin();
			const BYTE * pStringStart = pStr;
			while ( pStringStart-pStr < iWordLen )
			{
				if ( !*pStringStart )
				{
					CSphString sErrorStr;
					sErrorStr.SetBinary ( (const char*)pStr, iWordLen );
					tReporter.Fail ( "embedded zero in a word list string (segment=%d, offset=%u, string=%s)", iSegment, (DWORD)(pStringStart-pStr), sErrorStr.cstr() );
				}

				pStringStart++;
			}

			if ( iLastWordLen && iWordLen )
			{
				if ( sphDictCmpStrictly ( (const char *)dWord.Begin(), iWordLen, (const char *)dLastWord.Begin(), iLastWordLen )<=0 )
				{
					tReporter.Fail ( "word order decreased (segment=%d, word=%d, read_word=%s, last_word=%s)", iSegment, nWordsRead, dWord.Begin(), dLastWord.Begin() );
				}
			}

			memcpy ( dLastWord.Begin(), dWord.Begin(), iWordLen+1 );
			iLastWordLen = iWordLen;
			iDeltaWordLen = iWordLen;
		} else
		{
			tWord.m_uWordID += UnzipWordid ( pIn );
			if ( pIn>=pMaxWord )
				tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );

			if ( tWord.m_uWordID<=uPrevWordID )
			{
				tReporter.Fail ( "wordid decreased (segment=%d, word=%d, wordid=" UINT64_FMT ", previd=" UINT64_FMT ")", iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, (uint64_t)uPrevWordID );
			}

			uPrevWordID = tWord.m_uWordID;
		}

		UnzipDword ( &tWord.m_uDocs, pIn );
		if ( pIn>=pMaxWord )
		{
			tReporter.Fail ( "invalid docs/hits (segment=%d, word=%d, read_word=%s, docs=%u, hits=%u)", iSegment, nWordsRead, szWord, tWord.m_uDocs, tWord.m_uHits );
		}

		UnzipDword ( &tWord.m_uHits, pIn );
		if ( pIn>=pMaxWord )
			tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );

		tWord.m_uDoc += UnzipDword ( pIn );
		if ( pIn>pMaxWord )
			tReporter.Fail ( "reading past wordlist end (segment=%d, word=%d)", iSegment, nWordsRead );

		pCurWord = pIn;

		if ( !tWord.m_uDocs || !tWord.m_uHits || tWord.m_uHits<tWord.m_uDocs )
		{
			tReporter.Fail ( "invalid docs/hits (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, docs=%u, hits=%u)",
				iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tWord.m_uDocs, tWord.m_uHits );
		}

		if ( bCheckpoint )
		{
			RtWordCheckpoint_t & tCP = dRefCheckpoints.Add();
			tCP.m_iOffset = iCheckpointOffset;

			if ( bWordDict )
			{
				tCP.m_szWord = new char [iWordLen+1];
				memcpy ( (void *)tCP.m_szWord, dWord.Begin(), iWordLen+1 );
			} else
				tCP.m_uWordID = tWord.m_uWordID;
		}

		if ( uPrevDocOffset && tWord.m_uDoc<=uPrevDocOffset )
			tReporter.Fail ( "doclist offset decreased (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, doclist_offset=%u, prev_doclist_offset=%u)",
				iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tWord.m_uDoc, uPrevDocOffset );

		// read doclist
		auto uDocOffset = DWORD ( pCurDoc-tSegment.m_dDocs.Begin() );
		if ( tWord.m_uDoc!=uDocOffset )
		{
			tReporter.Fail ( "unexpected doclist offset (wordid=" UINT64_FMT "(%s)(%d), doclist_offset=%u, expected_offset=%u)",
				(uint64_t)tWord.m_uWordID, szWord, nWordsRead, tWord.m_uDoc, uDocOffset );

			if ( uDocOffset>=(DWORD)tSegment.m_dDocs.GetLength() )
			{
				tReporter.Fail ( "doclist offset pointing past doclist (segment=%d, word=%d, read_word=%s, doclist_offset=%u, doclist_size=%d)",
					iSegment, nWordsRead, szWord, uDocOffset, tSegment.m_dDocs.GetLength() );

				nWordsRead++;
				continue;
			} else
				pCurDoc = tSegment.m_dDocs.Begin()+uDocOffset;
		}

		// read all docs from doclist
		RtDoc_t tDoc;
		RowID_t tPrevRowID = INVALID_ROWID;

		for ( DWORD uDoc=0; uDoc<tWord.m_uDocs && pCurDoc<pMaxDoc; uDoc++ )
		{
			bool bEmbeddedHit = false;
			pIn = pCurDoc;

			tDoc.m_tRowID += UnzipDword ( pIn );
			if ( pIn>=pMaxDoc )
			{
				tReporter.Fail ( "reading past doclist end (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, doclist_offset=%u, doclist_size=%d)",
					iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, uDocOffset, tSegment.m_dDocs.GetLength() );
				break;
			}

			UnzipDword ( &tDoc.m_uDocFields, pIn );
			if ( pIn>=pMaxDoc )
			{
				tReporter.Fail ( "reading past doclist end (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, doclist_offset=%u, doclist_size=%d)",
					iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, uDocOffset, tSegment.m_dDocs.GetLength() );
				break;
			}

			UnzipDword ( &tDoc.m_uHits, pIn );
			if ( pIn>=pMaxDoc )
			{
				tReporter.Fail ( "reading past doclist end (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, doclist_offset=%u, doclist_size=%d)",
					iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, uDocOffset, tSegment.m_dDocs.GetLength() );
				break;
			}

			if ( tDoc.m_uHits==1 )
			{
				bEmbeddedHit = true;

				auto a = UnzipDword ( pIn );
				if ( pIn>=pMaxDoc )
				{
					tReporter.Fail ( "reading past doclist end (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, doclist_offset=%u, doclist_size=%d)",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, uDocOffset, tSegment.m_dDocs.GetLength() );
					break;
				}

				auto b = UnzipDword ( pIn );
				if ( pIn>pMaxDoc )
				{
					tReporter.Fail ( "reading past doclist end (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, doclist_offset=%u, doclist_size=%d)",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, uDocOffset, tSegment.m_dDocs.GetLength() );
					break;
				}

				tDoc.m_uHit = HITMAN::Create ( b, a );
			} else
			{
				UnzipDword ( &tDoc.m_uHit, pIn );
				if ( pIn>pMaxDoc )
				{
					tReporter.Fail ( "reading past doclist end (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, doclist_offset=%u, doclist_size=%d)",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, uDocOffset, tSegment.m_dDocs.GetLength() );
					break;
				}
			}

			pCurDoc = pIn;

			if ( uDoc && tDoc.m_tRowID<=tPrevRowID )
			{
				tReporter.Fail ( "rowid decreased (segment=%d, word=%d, read_wordid=" UINT64_FMT ", read_word=%s, rowid=%u, prev_rowid=%u)",
					iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, tPrevRowID );
			}

			if ( tDoc.m_tRowID>=tSegment.m_uRows )
				tReporter.Fail ( "invalid rowid (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u(%u))", iSegment, nWordsRead, tWord.m_uWordID, szWord, tDoc.m_tRowID, tSegment.m_uRows );

			if ( bEmbeddedHit )
			{
				DWORD uFieldId = HITMAN::GetField ( tDoc.m_uHit );
				DWORD uFieldMask = tDoc.m_uDocFields;
				int iCounter = 0;
				for ( ; uFieldMask; iCounter++ )
					uFieldMask &= uFieldMask - 1;

				if ( iCounter!=1 || tDoc.m_uHits!=1 )
				{
					tReporter.Fail ( "embedded hit with multiple occurences in a document found (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u)",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID );
				}

				if ( (int)uFieldId>m_tSchema.GetFieldsCount() || uFieldId>SPH_MAX_FIELDS )
				{
					tReporter.Fail ( "invalid field id in an embedded hit (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, field_id=%u, total_fields=%d)",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, uFieldId, m_tSchema.GetFieldsCount() );
				}

				if ( !( tDoc.m_uDocFields & ( 1 << uFieldId ) ) )
				{
					tReporter.Fail ( "invalid field id: not in doclist mask (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, field_id=%u, field_mask=%u)",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, uFieldId, tDoc.m_uDocFields );
				}
			} else
			{
				auto uExpectedHitOffset = DWORD ( pCurHit-tSegment.m_dHits.Begin() );
				if ( tDoc.m_uHit!=uExpectedHitOffset )
				{
					tReporter.Fail ( "unexpected hitlist offset (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, offset=%u, expected_offset=%u",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, tDoc.m_uHit, uExpectedHitOffset );
				}

				if ( tDoc.m_uHit && tDoc.m_uHit<=uPrevHitOffset )
				{
					tReporter.Fail ( "hitlist offset decreased (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, offset=%u, prev_offset=%u",
						iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, tDoc.m_uHit, uPrevHitOffset );
				}

				// check hitlist
				DWORD uHitlistEntry = 0;
				DWORD uLastPosInField = 0;
				DWORD uLastFieldId = 0;
				bool bLastInFieldFound = false;

				for ( DWORD uHit = 0; uHit < tDoc.m_uHits && pCurHit; uHit++ )
				{
					uHitlistEntry += UnzipDword ( pCurHit );
					if ( pCurHit>pMaxHit )
					{
						tReporter.Fail ( "reading past hitlist end (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u)", iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID );
						break;
					}

					DWORD uPosInField = HITMAN::GetPos ( uHitlistEntry );
					bool bLastInField = HITMAN::IsEnd ( uHitlistEntry );
					DWORD uFieldId = HITMAN::GetField ( uHitlistEntry );

					if ( (int)uFieldId>m_tSchema.GetFieldsCount() || uFieldId>SPH_MAX_FIELDS )
					{
						tReporter.Fail ( "invalid field id in a hitlist (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, field_id=%u, total_fields=%d)",
							iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, uFieldId, m_tSchema.GetFieldsCount() );
					}

					if ( !( tDoc.m_uDocFields & ( 1 << uFieldId ) ) )
					{
						tReporter.Fail ( "invalid field id: not in doclist mask (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, field_id=%u, field_mask=%u)",
							iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, uFieldId, tDoc.m_uDocFields );
					}

					if ( uLastFieldId!=uFieldId )
					{
						bLastInFieldFound = false;
						uLastPosInField = 0;
					}

					if ( uLastPosInField && uPosInField<=uLastPosInField )
					{
						tReporter.Fail ( "hit position in field decreased (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, pos=%u, last_pos=%u)",
							iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, uPosInField, uLastPosInField );
					}

					if ( bLastInField && bLastInFieldFound )
						tReporter.Fail ( "duplicate last-in-field hit found (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u)", iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID );

					uLastPosInField = uPosInField;
					uLastFieldId = uFieldId;
					bLastInFieldFound |= bLastInField;
				}

				uPrevHitOffset = tDoc.m_uHit;
			}

			DWORD uAvailFieldMask = ( 1 << m_tSchema.GetFieldsCount() ) - 1;
			if ( tDoc.m_uDocFields & ~uAvailFieldMask )
			{
				tReporter.Fail ( "wrong document field mask (segment=%d, word=%d, wordid=" UINT64_FMT "(%s), rowid=%u, mask=%u, total_fields=%d",
					iSegment, nWordsRead, (uint64_t)tWord.m_uWordID, szWord, tDoc.m_tRowID, tDoc.m_uDocFields, m_tSchema.GetFieldsCount() );
			}

			tPrevRowID = tDoc.m_tRowID;
		}

		uPrevDocOffset = tWord.m_uDoc;
		nWordsRead++;
	}

	if ( pCurDoc!=pMaxDoc )
		tReporter.Fail ( "unused doclist entries found (segment=%d, doclist_size=%d)", iSegment, tSegment.m_dDocs.GetLength() );

	if ( pCurHit!=pMaxHit )
		tReporter.Fail ( "unused hitlist entries found (segment=%d, hitlist_size=%d)", iSegment, tSegment.m_dHits.GetLength() );

	if ( dRefCheckpoints.GetLength()!=tSegment.m_dWordCheckpoints.GetLength() )
		tReporter.Fail ( "word checkpoint count mismatch (read=%d, calc=%d)", tSegment.m_dWordCheckpoints.GetLength(), dRefCheckpoints.GetLength() );

	for ( int i=0; i < Min ( dRefCheckpoints.GetLength(), tSegment.m_dWordCheckpoints.GetLength() ); i++ )
	{
		const RtWordCheckpoint_t & tRefCP = dRefCheckpoints[i];
		const RtWordCheckpoint_t & tCP = tSegment.m_dWordCheckpoints[i];
		const int iLen = bWordDict && tCP.m_szWord ? (int)strlen ( tCP.m_szWord ) : 0;
		const int iRefLen = bWordDict && tRefCP.m_szWord ? (int)strlen ( tRefCP.m_szWord ) : 0;
		if ( bWordDict && ( !tCP.m_szWord || !iRefLen || !iLen ) )
		{
			CSphString sCPWord, sRefCPWord;
			if ( iLen )
				sCPWord.SetBinary ( tCP.m_szWord, iLen );
			if ( iRefLen )
				sRefCPWord.SetBinary ( tRefCP.m_szWord, iRefLen );
			tReporter.Fail ( "empty word checkpoint %d ((segment=%d, read_word=%s, read_len=%u, readpos=%d, calc_word=%s, calc_len=%u, calcpos=%d)",
				i, iSegment, sCPWord.cstr(), (DWORD)iLen, tCP.m_iOffset,
				sRefCPWord.cstr(), (DWORD)iRefLen, tRefCP.m_iOffset );
		} else if ( ( bWordDict
			? sphDictCmpStrictly ( tCP.m_szWord, iLen, tRefCP.m_szWord, iRefLen )
			: sphCheckpointCmpStrictly ( tCP.m_szWord, iLen, tCP.m_uWordID, bWordDict, tRefCP ) )
			|| tRefCP.m_iOffset!=tCP.m_iOffset )
		{
			if ( bWordDict )
			{
				CSphString sCPWord, sRefCPWord;
				sCPWord.SetBinary ( tCP.m_szWord, iLen );
				sRefCPWord.SetBinary ( tRefCP.m_szWord, iRefLen );
				tReporter.Fail ( "word checkpoint %d differs (segment=%d, read_word=%s, readpos=%d, calc_word=%s, calcpos=%d)",
					i, iSegment, sCPWord.cstr(), tCP.m_iOffset, sRefCPWord.cstr(), tRefCP.m_iOffset );
			} else
			{
				tReporter.Fail ( "word checkpoint %d differs (segment=%d, readid=" UINT64_FMT ", readpos=%d, calcid=" UINT64_FMT ", calcpos=%d)",
					i, iSegment, (uint64_t)tCP.m_uWordID, tCP.m_iOffset, (int64_t)tRefCP.m_uWordID, tRefCP.m_iOffset );
			}
		}
	}

	if ( bWordDict )
		ARRAY_FOREACH ( i, dRefCheckpoints )
			SafeDeleteArray ( dRefCheckpoints[i].m_szWord );

	dRefCheckpoints.Reset ();

	MemoryDebugCheckReader_c tAttrs ( (const BYTE *)tSegment.m_dRows.begin(), (const BYTE *)tSegment.m_dRows.end() );
	MemoryDebugCheckReader_c tBlobs ( tSegment.m_dBlobs.begin(), tSegment.m_dBlobs.end() );
	DebugCheck_Attributes ( tAttrs, tBlobs, tSegment.m_uRows, 0, m_tSchema, tReporter );
	DebugCheck_DeadRowMap ( tSegment.m_tDeadRowMap.GetLengthBytes(), tSegment.m_uRows, tReporter );

	DWORD uCalcAliveRows = tSegment.m_tDeadRowMap.GetNumAlive();
	if ( tSegment.m_tAliveRows.load(std::memory_order_relaxed)!=uCalcAliveRows )
		tReporter.Fail ( "alive row count mismatch (segment=%d, expected=%u, current=%u)", iSegment, uCalcAliveRows,
				tSegment.m_tAliveRows.load ( std::memory_order_relaxed ) );

} // NOLINT function length

void RtIndex_c::DebugCheckRam ( DebugCheckError_i & tReporter ) NO_THREAD_SAFETY_ANALYSIS
{
	auto pRamSegs = m_tRtChunks.RamSegs();
	auto& dRamSegs = *pRamSegs;
	ARRAY_FOREACH ( iSegment, dRamSegs )
	{
		tReporter.Msg ( "checking RT segment %d(%d)...", iSegment, dRamSegs.GetLength() );

		if ( !dRamSegs[iSegment] )
		{
			tReporter.Fail ( "missing RT segment (segment=%d)", iSegment );
			continue;
		}

		const RtSegment_t & tSegment = *dRamSegs[iSegment];
		DebugCheckRamSegment ( tSegment, iSegment, tReporter );
	}
}

constexpr int FAILS_THRESHRT = 100;

class DebugCheckInternal : public DebugCheckError_i
{
	int64_t m_iFails { 0 };
	StringBuilder_c m_sMsg { "\n" };
public:
	bool Fail ( const char* szFmt, ... ) override;
	void Msg ( const char* szFmt, ... ) override {};
	void Progress ( const char* szFmt, ... ) override {};
	void Done() override {};
	int64_t GetNumFails() const override;
	void CheckDocidDup ( DocID_t tDocid, DWORD uRowid ) override {};
	void FinishDiskChunk ( int64_t iNumRows ) override {};

	inline const char* cstr() const { return m_sMsg.cstr(); }
};

bool DebugCheckInternal::Fail ( const char* szFmt, ... )
{
	if ( ++m_iFails >=FAILS_THRESHRT )
		return false;

	va_list ap;
	va_start ( ap, szFmt );
	m_sMsg.vSprintf ( szFmt, ap );
	va_end ( ap );
	return false;
}

int64_t DebugCheckInternal::GetNumFails() const
{
	return m_iFails;
}

CSphString RtIndex_c::MakeDamagedName () const
{
	CSphString sChunk;
	sChunk.SetSprintf ( "%s/damaged.%s.%d.%d", Binlog::GetPath().cstr(), GetName(), getpid(), m_iTrackFailedRamActions );
	return sChunk;
}

void RtIndex_c::DumpSegments ( VecTraits_T<const RtSegment_t*> dSegments, const CSphString& sFile ) const
{
	if ( dSegments.IsEmpty() )
		return;

	CSphString sLastError;
	CSphWriterNonThrottled wrChunk;
	if ( !wrChunk.OpenFile ( sFile, sLastError ) )
	{
		sphWarning ("Unable to open %s, error %s", sFile.cstr(), sLastError.cstr() );
		return;
	}

	wrChunk.PutDword ( 0 );
	wrChunk.PutDword ( dSegments.GetLength() );

	// no locks here, because it's only intended to be called from dtor
	for ( const RtSegment_t* pSeg : dSegments )
	{
		SccRL_t rLock ( pSeg->m_tLock );
		SaveRamSegment ( pSeg, wrChunk );
	}

	SaveRamFieldLengths ( wrChunk );
	wrChunk.CloseFile();
}

void RtIndex_c::DumpSegment ( const RtSegment_t* pSeg, const CSphString& sFile ) const
{
	assert ( pSeg );
	LazyVector_T<const RtSegment_t*> dSegments;
	dSegments.Add ( pSeg );
	DumpSegments ( dSegments, sFile );
}

void RtIndex_c::DumpMeta ( const CSphString& sFile ) const
{
	CSphString sLastError;

	CSphWriter wrMeta;
	if ( !wrMeta.OpenFile ( sFile, sLastError ) )
	{
		sphWarning ( "Unable to open %s, error %s", sFile.cstr(), sLastError.cstr() );
		return;
	}

	WriteMeta ( m_iTID, { nullptr, 0 }, wrMeta );
	wrMeta.CloseFile();

	// write new meta
	if ( wrMeta.IsError() )
		sphWarning ( "%s", sLastError.cstr() );
}

void RtIndex_c::DumpInsert ( const RtSegment_t* pNewSeg ) const
{
	if ( !pNewSeg || m_iTrackFailedRamActions <= 0 )
		return;

	auto tDescription = myinfo::UnsafeDescription();
	if ( tDescription.second > 6 && !memcmp ( tDescription.first, "SYSTEM ", 7 ) )
		return;

	CSphString sLastError;
	CSphString sBase = MakeDamagedName ();
	CSphString sFile;
	sFile.SetSprintf ( "%s.stmt", sBase.cstr() );

	CSphWriterNonThrottled wrContent;
	if ( !wrContent.OpenFile ( sFile, sLastError ) )
	{
		sphWarning ( "Unable to open %s, error %s", sFile.cstr(), sLastError.cstr() );
		return;
	}
	wrContent.PutBytes ( tDescription.first, tDescription.second );
	wrContent.CloseFile();

	// write new meta
	if ( wrContent.IsError() )
		sphWarning ( "%s", sLastError.cstr() );

	sFile.SetSprintf ( "%s.ram", sBase.cstr() );
	DumpSegment ( pNewSeg, sFile );
	sFile.SetSprintf ( "%s.meta", sBase.cstr() );
	DumpMeta( sFile );
	sphWarning ( "Damaged Insert saved as %s, files .stmt, .ram and .meta", sBase.cstr() );
	--m_iTrackFailedRamActions;
}

void RtIndex_c::DumpMerge ( const RtSegment_t* pA, const RtSegment_t* pB, const RtSegment_t* pNew ) const
{
	if ( m_iTrackFailedRamActions <= 0 )
		return;

	LazyVector_T<const RtSegment_t*> dSegments;
	if ( pA )
		dSegments.Add ( pA );
	if ( pB )
		dSegments.Add ( pB );

	CSphString sBase = MakeDamagedName();
	CSphString sFile;
	sFile.SetSprintf ( "%s.origin.ram", sBase.cstr() );

	DumpSegments ( dSegments, sFile );
	dSegments.Reset();
	if ( pNew )
		dSegments.Add ( pNew );
	sFile.SetSprintf ( "%s.ram", sBase.cstr() );
	DumpSegments ( dSegments, sFile );
	sFile.SetSprintf ( "%s.meta", sBase.cstr() );
	DumpMeta(sFile);
	sphWarning ( "Damaged Merge saved as %s, files .origin.ram, .ram and .meta", sBase.cstr() );
	--m_iTrackFailedRamActions;
}

bool RtIndex_c::CheckSegmentConsistency ( const RtSegment_t* pNewSeg, bool bSilent ) const
{
	assert ( pNewSeg );
	if ( m_iTrackFailedRamActions<=0 || pNewSeg->m_bConsistent )
		return true;

	DebugCheckInternal tChecker;
	DebugCheckRamSegment ( *pNewSeg, 0, tChecker );
	if ( !tChecker.GetNumFails() )
	{
		pNewSeg->m_bConsistent = true;
		return true;
	}

	if ( !bSilent )
	{
		sphWarning ( "CheckSegmentConsistency returned %d errors", (int)tChecker.GetNumFails() );
		sphWarning ( "%s", tChecker.cstr() );
	}
	return false;
}

int RtIndex_c::DebugCheckDisk ( DebugCheckError_i & tReporter )
{
	CreateFilenameBuilder_fn fnCreateFilenameBuilder = GetIndexFilenameBuilder();
	std::unique_ptr<FilenameBuilder_i> pFilenameBuilder;
	if ( fnCreateFilenameBuilder )
		pFilenameBuilder = fnCreateFilenameBuilder ( GetName() );

	VecTraits_T<int> dChunks = m_dChunkNames.Slice();
	if ( m_iCheckChunk!=-1 )
	{
		int iChunk = dChunks.GetFirst (  [&] ( int & v ) { return m_iCheckChunk==v; }  );
		if ( iChunk==-1 )
		{
			tReporter.Fail ( "failed to find disk chunk %s.%d, disk chunks total %d", GetFilebase(), m_iCheckChunk, m_dChunkNames.GetLength() );
			return 1;
		}

		dChunks = m_dChunkNames.Slice ( iChunk, 1 );
	}

	int iFailsPlain = 0;
	StrVec_t dWarnings;
	ARRAY_FOREACH ( i, dChunks )
	{
		int iChunk = dChunks[i];
		CSphString sChunk = GetFilename ( iChunk );
		tReporter.Msg ( "checking disk chunk, extension %d, %d(%d)...", dChunks[i], i, m_dChunkNames.GetLength() );

		auto pIndex = PreallocDiskChunk ( sChunk.cstr(), iChunk, pFilenameBuilder.get(), dWarnings, m_sLastError );
		if ( pIndex )
		{
			iFailsPlain += pIndex->DebugCheck ( tReporter, pFilenameBuilder.get() );
		} else
		{
			tReporter.Fail ( "%s", m_sLastError.cstr() );
			m_sLastError = "";
			iFailsPlain++;
		}
	}

	for ( const auto & i : dWarnings )
		tReporter.Msg ( "warning: %s", i.cstr() );

	return iFailsPlain;
}


void RtIndex_c::SetDebugCheck ( bool bCheckIdDups, int iCheckChunk )
{
	m_bDebugCheck = true;
	m_bCheckIdDups = bCheckIdDups;
	m_iCheckChunk = iCheckChunk;
	ProhibitSave();
}

//////////////////////////////////////////////////////////////////////////
// SEARCHING
//////////////////////////////////////////////////////////////////////////

struct RtQword_t final : public ISphQword
{
public:
	RtQword_t () = default;

	const CSphMatch & GetNextDoc() final
	{
		while (true)
		{
			if ( !m_tDocReader.UnzipDoc() )
			{
				m_tMatch.m_tRowID = INVALID_ROWID;
				return m_tMatch;
			}

			if ( !m_pSeg->m_tDeadRowMap.IsSet ( m_tDocReader->m_tRowID ) )
				break;
		}

		const auto* pDoc = (const RtDoc_t*)m_tDocReader;
		m_tMatch.m_tRowID = pDoc->m_tRowID;
		m_dQwordFields.Assign32 ( pDoc->m_uDocFields );
		m_uMatchHits = pDoc->m_uHits;
		m_iHitlistPos = (uint64_t(pDoc->m_uHits)<<32) + pDoc->m_uHit;
		m_bAllFieldsKnown = false;
		return m_tMatch;
	}

	void SeekHitlist ( SphOffset_t uOff ) final
	{
		auto iHits = (int)(uOff>>32);
		if ( iHits==1 )
		{
			m_uNextHit = DWORD(uOff);
		} else
		{
			m_uNextHit = 0;
			m_tHitReader.Seek ( m_pHits + DWORD(uOff), iHits );
		}
	}

	Hitpos_t GetNextHit () final
	{
		if ( !m_uNextHit )
			return Hitpos_t ( m_tHitReader.UnzipHit() );
		else if ( m_uNextHit==0xffffffffUL )
			return EMPTY_HIT;
		else
			return Hitpos_t ( std::exchange ( m_uNextHit, 0xffffffffUL ) );
	}

	bool SetupScan ( const RtIndex_c * pIndex, int iSegment, const RtGuard_t& tGuard ) final
	{
		return pIndex->RtQwordSetup ( this, iSegment, tGuard );
	}

	void SetupReader ( const RtSegment_t * pSeg, const RtWord_t & tWord )
	{
		m_pSeg = pSeg;
		m_tDocReader.Init ( pSeg, tWord );
		m_pHits = pSeg->m_dHits.begin();
	}

private:
	RtDocReader_c		m_tDocReader;
	RtHitReader_c		m_tHitReader;

	CSphMatch			m_tMatch;
	DWORD				m_uNextHit {0};

	const BYTE* m_pHits {nullptr};
	const RtSegment_t * m_pSeg {nullptr};
};


//////////////////////////////////////////////////////////////////////////


struct RtSubstringPayload_t : public ISphSubstringPayload
{
	RtSubstringPayload_t ( int iSegmentCount, int iDoclists )
		: m_dSegment2Doclists ( iSegmentCount )
		, m_dDoclist ( iDoclists )
	{}
	CSphFixedVector<Slice_t>	m_dSegment2Doclists;
	CSphFixedVector<Slice_t>	m_dDoclist;
};


struct RtQwordPayload_t final : public ISphQword
{
public:
	explicit RtQwordPayload_t ( const RtSubstringPayload_t * pPayload )
		: m_pPayload ( pPayload )
	{
		m_tMatch.Reset ( 0 );
		m_iDocs = m_pPayload->m_iTotalDocs;
		m_iHits = m_pPayload->m_iTotalHits;

		m_uDoclist = 0;
		m_uDoclistLeft = 0;
		m_pSegment = nullptr;
		m_uHitEmbeded = EMPTY_HIT;
	}

	const CSphMatch & GetNextDoc() final
	{
		m_iHits = 0;
		while ( true )
			if ( !m_tDocReader.UnzipDoc() )
			{
				if ( !m_uDoclistLeft )
				{
					m_tMatch.m_tRowID = INVALID_ROWID;
					return m_tMatch;
				}
				SetupReader();
			} else if ( !m_pSegment->m_tDeadRowMap.IsSet ( m_tDocReader->m_tRowID ) )
				break;

		const auto* pDoc = (const RtDoc_t*) m_tDocReader;
		m_tMatch.m_tRowID = pDoc->m_tRowID;
		m_dQwordFields.Assign32 ( pDoc->m_uDocFields );
		m_bAllFieldsKnown = false;
		m_iHits = pDoc->m_uHits;
		m_uHitEmbeded = pDoc->m_uHit;
		m_tHitReader.Seek ( *m_pSegment, *pDoc );
		return m_tMatch;
	}

	void SeekHitlist ( SphOffset_t ) final
	{}

	Hitpos_t GetNextHit () final
	{
		if ( m_iHits>1 )
			return Hitpos_t ( m_tHitReader.UnzipHit() );
		return ( m_iHits==1 ) ? Hitpos_t ( std::exchange ( m_uHitEmbeded, EMPTY_HIT ) ) : EMPTY_HIT;
	}

	bool SetupScan ( const RtIndex_c *, int iSegment, const RtGuard_t& tGuard ) final
	{
		m_tDocReader.Reset();

		if ( iSegment<0 )
		{
			m_pSegment = nullptr;
			m_uDoclist = 0;
			m_uDoclistLeft = 0;
			return false;
		}

		m_pSegment = tGuard.m_dRamSegs[iSegment];
		m_uDoclist = m_pPayload->m_dSegment2Doclists[iSegment].m_uOff;
		m_uDoclistLeft = m_pPayload->m_dSegment2Doclists[iSegment].m_uLen;

		if ( !m_uDoclistLeft )
			return false;

		SetupReader();
		return true;
	}

private:
	void SetupReader ()
	{
		assert ( m_uDoclistLeft );
		RtWord_t tWord;
		tWord.m_uDoc = m_pPayload->m_dDoclist[m_uDoclist].m_uOff;
		tWord.m_uDocs = m_pPayload->m_dDoclist[m_uDoclist].m_uLen;
		m_tDocReader.Init ( m_pSegment, tWord );
		++m_uDoclist;
		--m_uDoclistLeft;
	}

	const RtSubstringPayload_t *	m_pPayload;
	CSphMatch					m_tMatch;
	RtDocReader_c				m_tDocReader;
	RtHitReader_c				m_tHitReader;
	const RtSegment_t *			m_pSegment;

	DWORD						m_uDoclist;
	DWORD						m_uDoclistLeft;
	DWORD						m_uHitEmbeded;
};

class RtScanQword_t final : public QwordScan_c
{
public:
	explicit RtScanQword_t ( int iRowsTotal, int iAtomPos, int iFieldsCount )
		: QwordScan_c ( iRowsTotal, iAtomPos, iFieldsCount )
	{
	}

	bool SetupScan ( const RtIndex_c * pIndex, int iSegment, const RtGuard_t& tGuard ) final
	{
		m_tDoc.Reset ( 0 );

		if ( iSegment<0 )
			return false;

		const auto& pSegment = tGuard.m_dRamSegs[iSegment];

		m_iRowsCount = pSegment->m_uRows;
		m_iDocs = m_iRowsCount;
		m_bDone = ( m_iRowsCount==0 );
		m_pKilled = &pSegment->m_tDeadRowMap;

		return ( pSegment->m_tAliveRows.load(std::memory_order_relaxed)>0 );
	}

	bool IsAliveRow ( RowID_t tRowID ) const override
	{
		return !m_pKilled->IsSet ( tRowID );
	}
private:
	const DeadRowMap_Ram_c * m_pKilled { nullptr };
};


//////////////////////////////////////////////////////////////////////////

class RtQwordSetup_t final : public ISphQwordSetup
{
public:
	explicit RtQwordSetup_t ( const RtGuard_t& tGuard );
	ISphQword *	QwordSpawn ( const XQKeyword_t & ) const final;
	bool		QwordSetup ( ISphQword * pQword ) const final;
	void				SetSegment ( int iSegment ) { m_iSeg = iSegment; }
	ISphQword *			ScanSpawn ( int iAtomPos ) const final;

private:
	const RtGuard_t&	m_tGuard;
	int					m_iSeg;
};


RtQwordSetup_t::RtQwordSetup_t ( const RtGuard_t& tGuard )
	: m_tGuard ( tGuard )
	, m_iSeg ( -1 )
{ }


ISphQword * RtQwordSetup_t::QwordSpawn ( const XQKeyword_t & tWord ) const
{
	if ( !tWord.m_pPayload )
		return new RtQword_t ();
	else
		return new RtQwordPayload_t ( (const RtSubstringPayload_t *)tWord.m_pPayload );
}


bool RtQwordSetup_t::QwordSetup ( ISphQword * pQword ) const
{
	// there was two dynamic_casts here once but they're not necessary
	// maybe it's worth to rewrite class hierarchy to avoid c-casts here?
	const auto * pIndex = (const RtIndex_c *)m_pIndex;
	return pQword->SetupScan ( pIndex, m_iSeg, m_tGuard );
}


bool RtIndex_c::EarlyReject ( CSphQueryContext * pCtx, CSphMatch & tMatch ) const
{
	auto pSegment = (RtSegment_t*)const_cast<void*>(pCtx->m_pIndexData);
	tMatch.m_pStatic = pSegment->GetDocinfoByRowID ( tMatch.m_tRowID );

	pCtx->CalcFilter ( tMatch );
	if ( !pCtx->m_pFilter )
		return false;

	if ( !pCtx->m_pFilter->Eval ( tMatch ) )
	{
		pCtx->FreeDataFilter ( tMatch );
		return true;
	}
	return false;
}

ISphQword * RtQwordSetup_t::ScanSpawn ( int iAtomPos ) const
{
	return new RtScanQword_t ( (int)m_pIndex->GetStats().m_iTotalDocuments, iAtomPos, m_pIndex->GetMatchSchema().GetFieldsCount() );
}


// WARNING, setup is pretty tricky
// for RT queries, we setup qwords several times
// first pass (with NULL segment arg) should sum all stats over all segments
// others passes (with non-NULL segments) should setup specific segment (including local stats)
bool RtIndex_c::RtQwordSetupSegment ( RtQword_t * pQword, const RtSegment_t * pCurSeg, bool bSetup ) const
{
	if ( !pCurSeg )
		return false;

	SphWordID_t uWordID = pQword->m_uWordID;
	const char * sWord = pQword->m_sDictWord.cstr();
	int iWordLen = pQword->m_sDictWord.Length();
	const bool bWordDict = IsWordDict();
	bool bPrefix = false;
	if ( bWordDict && iWordLen && sWord[iWordLen-1]=='*' ) // crc star search emulation
	{
		iWordLen = iWordLen-1;
		bPrefix = true;
	}

	if ( !iWordLen )
		return false;

	// prevent prefix matching for explicitly setting prohibited by config, to be on pair with plain index (or CRC kind of index)
	if ( bPrefix && ( ( m_tSettings.GetMinPrefixLen ( bWordDict ) && iWordLen< m_tSettings.GetMinPrefixLen ( bWordDict ) )
		|| ( m_tSettings.m_iMinInfixLen && iWordLen< m_tSettings.m_iMinInfixLen ) ) )
		return false;

	// no checkpoints - check all words
	// no checkpoints matched - check only words prior to 1st checkpoint
	// checkpoint found - check words at that checkpoint
	RtWordReader_c tReader ( pCurSeg, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );

	if ( pCurSeg->m_dWordCheckpoints.GetLength() )
	{
		const RtWordCheckpoint_t * pCp = bWordDict
			  ? RtSearchCheckpointWrd ( sWord, iWordLen, false, pCurSeg->m_dWordCheckpoints )
			  : sphSearchCheckpointCrc ( uWordID, pCurSeg->m_dWordCheckpoints );

		const BYTE * pWords = pCurSeg->m_dWords.Begin();

		if ( !pCp )
		{
			tReader.m_pMax = pWords + pCurSeg->m_dWordCheckpoints.Begin()->m_iOffset;
		} else
		{
			tReader.m_pCur = pWords + pCp->m_iOffset;
			// if next checkpoint exists setup reader range
			if ( ( pCp+1 )<= ( &pCurSeg->m_dWordCheckpoints.Last() ) )
				tReader.m_pMax = pWords + pCp[1].m_iOffset;
		}
	}

	// find the word between checkpoints
	if ( bWordDict )
	{
		int iCmp = 0;
		do {
			if ( !tReader.UnzipWord() )
				return false;
			iCmp = sphDictCmpStrictly ( (const char*)tReader->m_sWord, tReader->m_iWordLen, sWord, iWordLen );
		} while ( iCmp < 0 );

		if ( !!iCmp )
			return false;

		assert ( !iCmp );
		const auto* pWord = (const RtWord_t*)tReader;
		pQword->m_iDocs += pWord->m_uDocs;
		pQword->m_iHits += pWord->m_uHits;
		pQword->m_bHasHitlist &= pWord->m_bHasHitlist;
		if ( bSetup )
			pQword->SetupReader ( pCurSeg, *pWord );

		return true;
	}

	assert ( !bWordDict );
	do
		if ( !tReader.UnzipWord() )
			return false;
	while ( tReader->m_uWordID < uWordID );

	if ( tReader->m_uWordID>uWordID )
		return false;

	const auto* pWord = (const RtWord_t*)tReader;
	pQword->m_iDocs += pWord->m_uDocs;
	pQword->m_iHits += pWord->m_uHits;
	pQword->m_bHasHitlist &= pWord->m_bHasHitlist;
	if ( bSetup )
		pQword->SetupReader ( pCurSeg, *pWord );

	return true;
}

struct RtExpandedEntry_t
{
	DWORD	m_uHash;
	int		m_iNameOff;
	int		m_iNameLen;
	int		m_iDocs;
	int		m_iHits;
};

struct RtExpandedPayload_t
{
	int		m_iDocs;
	int		m_iHits;
	DWORD	m_uDoclistOff;
};

static DWORD CalcRtExpansionHash ( const BYTE * pKeyword, int iKeywordLen, DictFormat_e eDictFormat )
{
	if ( eDictFormat==DictFormat_e::KEYWORDS_V2 )
		return sphCRC32 ( pKeyword, iKeywordLen );

	assert ( iKeywordLen<SPH_MAX_KEYWORD_LEN );
	BYTE uLegacyLen = (BYTE)iKeywordLen;
	// Keep legacy KEYWORDS ordering: old RT words were hashed as a packed
	// buffer with a length byte followed by the word bytes.
	DWORD uHash = sphCRC32 ( &uLegacyLen, 1 );
	return sphCRC32 ( pKeyword, iKeywordLen, uHash );
}

struct RtExpandedTraits_fn
{
	inline bool IsLess ( const RtExpandedEntry_t & a, const RtExpandedEntry_t & b ) const
	{
		assert ( m_sBase );
		if ( a.m_uHash!=b.m_uHash )
		{
			return a.m_uHash<b.m_uHash;
		} else
		{
			const BYTE * pA = m_sBase + a.m_iNameOff;
			const BYTE * pB = m_sBase + b.m_iNameOff;
			if ( a.m_iNameLen!=b.m_iNameLen )
				return a.m_iNameLen<b.m_iNameLen;

			return ( sphDictCmp ( (const char *)pA, a.m_iNameLen, (const char *)pB, b.m_iNameLen )<0 );
		}
	}

	inline bool IsEqual ( const RtExpandedEntry_t * a, const RtExpandedEntry_t * b ) const
	{
		assert ( m_sBase );
		if ( a->m_uHash!=b->m_uHash )
			return false;

		const BYTE * pA = m_sBase + a->m_iNameOff;
		const BYTE * pB = m_sBase + b->m_iNameOff;
		if ( a->m_iNameLen!=b->m_iNameLen )
			return false;

		return ( sphDictCmp ( (const char *)pA, a->m_iNameLen, (const char *)pB, b->m_iNameLen )==0 );
	}

	explicit RtExpandedTraits_fn ( const BYTE * sBase )
		: m_sBase ( sBase )
	{ }
	const BYTE * m_sBase;
};


struct DictEntryRtPayload_t : public DictTerm2Expanded_i
{
	DictEntryRtPayload_t ( bool bPayload, int iSegments, DictFormat_e eDictFormat )
		: m_eDictFormat ( eDictFormat )
	{
		m_bPayload = bPayload;
		m_iSegExpansionLimit = iSegments;
		if ( bPayload )
		{
			m_dWordPayload.Reserve ( 1000 );
			m_dSeg.Resize ( iSegments );
			ARRAY_FOREACH ( i, m_dSeg )
			{
				m_dSeg[i].m_uOff = 0;
				m_dSeg[i].m_uLen = 0;
			}
		}

		m_dWordExpand.Reserve ( 1000 );
		m_dWordBuf.Reserve ( 8096 );
	}

	void Add ( const RtWord_t * pWord, int iSegment )
	{
		if ( !m_bPayload || !sphIsExpandedPayload ( pWord->m_uDocs, pWord->m_uHits ) )
		{
			RtExpandedEntry_t & tExpand = m_dWordExpand.Add();

			int iOff = m_dWordBuf.GetLength();
			tExpand.m_uHash = CalcRtExpansionHash ( pWord->m_sWord, pWord->m_iWordLen, m_eDictFormat );
			tExpand.m_iNameOff = iOff;
			tExpand.m_iNameLen = pWord->m_iWordLen;
			tExpand.m_iDocs = pWord->m_uDocs;
			tExpand.m_iHits = pWord->m_uHits;
			m_dWordBuf.Append ( pWord->m_sWord, pWord->m_iWordLen );
		} else
		{
			RtExpandedPayload_t & tExpand = m_dWordPayload.Add();
			tExpand.m_iDocs = pWord->m_uDocs;
			tExpand.m_iHits = pWord->m_uHits;
			tExpand.m_uDoclistOff = pWord->m_uDoc;

			m_dSeg[iSegment].m_uOff = m_dWordPayload.GetLength();
			m_dSeg[iSegment].m_uLen++;
		}
	}

	void Convert ( ISphWordlist::Args_t & tArgs )
	{
		if ( !m_dWordExpand.GetLength() && !m_dWordPayload.GetLength() )
			return;

		int iTotalDocs = 0;
		int iTotalHits = 0;
		if ( !m_dWordExpand.IsEmpty() )
		{
			int iRtExpansionLimit = tArgs.m_iExpansionLimit * m_iSegExpansionLimit;
			if ( tArgs.m_iExpansionLimit && m_dWordExpand.GetLength()>iRtExpansionLimit )
			{
				// sort expansions by frequency desc
				// clip the less frequent ones if needed, as they are likely misspellings
				sphSort ( m_dWordExpand.Begin(), m_dWordExpand.GetLength(), ExpandedOrderDesc_T<RtExpandedEntry_t>() );
				m_dWordExpand.Resize ( iRtExpansionLimit );
			}

			// lets merge statistics for same words from different segments
			// as hash produce a lot tiny allocations here
			const BYTE * sBase = m_dWordBuf.Begin();
			RtExpandedTraits_fn fnCmp ( sBase );
			sphSort ( m_dWordExpand.Begin(), m_dWordExpand.GetLength(), fnCmp );

			const RtExpandedEntry_t * pLast = m_dWordExpand.Begin();
			tArgs.AddExpanded ( sBase+pLast->m_iNameOff, pLast->m_iNameLen, pLast->m_iDocs, pLast->m_iHits );
			iTotalDocs += pLast->m_iDocs;
			iTotalHits += pLast->m_iHits;
			for ( int i=1; i<m_dWordExpand.GetLength(); ++i )
			{
				const RtExpandedEntry_t * pCur = m_dWordExpand.Begin() + i;

				if ( fnCmp.IsEqual ( pLast, pCur ) )
				{
					tArgs.m_dExpanded.Last().m_iDocs += pCur->m_iDocs;
					tArgs.m_dExpanded.Last().m_iHits += pCur->m_iHits;
				} else
				{
					tArgs.AddExpanded ( sBase + pCur->m_iNameOff, pCur->m_iNameLen, pCur->m_iDocs, pCur->m_iHits );
					pLast = pCur;
				}
				iTotalDocs += pCur->m_iDocs;
				iTotalHits += pCur->m_iHits;
			}
			tArgs.m_tExpansionStats.m_iTerms += m_dWordExpand.GetLength();
		}

		if ( m_dWordPayload.GetLength() )
		{
			DWORD uExpansionLimit = tArgs.m_iExpansionLimit;
			int iPayloads = 0;
			for ( auto& tSeg: m_dSeg )
			{
				// reverse per segment offset to payload doc-list as offset was the end instead of start
				assert ( tSeg.m_uOff>=tSeg.m_uLen );
				tSeg.m_uOff = tSeg.m_uOff - tSeg.m_uLen;

				// per segment expansion limit clip
				if ( uExpansionLimit && tSeg.m_uLen>uExpansionLimit )
				{
					// sort expansions by frequency desc
					// per segment clip the less frequent ones if needed, as they are likely misspellings
					sphSort ( m_dWordPayload.Begin()+tSeg.m_uOff, tSeg.m_uLen,
							ExpandedOrderDesc_T<RtExpandedPayload_t>() );
					tSeg.m_uLen = uExpansionLimit;
				}

				iPayloads += tSeg.m_uLen;
				// sort by ascending doc-list offset
				sphSort ( m_dWordPayload.Begin()+tSeg.m_uOff, tSeg.m_uLen,
						bind ( &RtExpandedPayload_t::m_uDoclistOff ) );
			}

			std::unique_ptr<RtSubstringPayload_t> pPayload ( new RtSubstringPayload_t ( m_dSeg.GetLength(), iPayloads ) );

			Slice_t * pDst = pPayload->m_dDoclist.Begin();
			ARRAY_FOREACH ( i, m_dSeg )
			{
				const Slice_t & tSeg = m_dSeg[i];
				const RtExpandedPayload_t * pSrc = m_dWordPayload.Begin() + tSeg.m_uOff;
				const RtExpandedPayload_t * pEnd = pSrc + tSeg.m_uLen;
				pPayload->m_dSegment2Doclists[i].m_uOff = pPayload->m_dDoclist.Idx(pDst);
				pPayload->m_dSegment2Doclists[i].m_uLen = tSeg.m_uLen;
				while ( pSrc!=pEnd )
				{
					pDst->m_uOff = pSrc->m_uDoclistOff;
					pDst->m_uLen = pSrc->m_iDocs;
					iTotalDocs += pSrc->m_iDocs;
					iTotalHits += pSrc->m_iHits;
					++pDst;
					++pSrc;
				}
			}
			pPayload->m_iTotalDocs = iTotalDocs;
			pPayload->m_iTotalHits = iTotalHits;
			tArgs.m_pPayload = std::move ( pPayload );
			tArgs.m_tExpansionStats.m_iMerged += iPayloads;
		}

		tArgs.m_iTotalDocs = iTotalDocs;
		tArgs.m_iTotalHits = iTotalHits;
	}

	bool							m_bPayload;
	CSphVector<RtExpandedEntry_t>	m_dWordExpand;
	CSphVector<RtExpandedPayload_t>	m_dWordPayload;
	CSphVector<BYTE>				m_dWordBuf;
	CSphVector<Slice_t>				m_dSeg;
	int								m_iSegExpansionLimit = 0;
	DictFormat_e					m_eDictFormat = DictFormat_e::KEYWORDS;
};


void RtIndex_c::GetPrefixedWords ( const char * sSubstring, int iSubLen, const char * sWildcard, Args_t & tArgs ) const
{
	const DictFormat_e eDictFormat = GetDictFormat();
	WildcardBuf_t tWildcardBuf ( eDictFormat==DictFormat_e::KEYWORDS_V2 ? WildcardBufMode_e::Extended : WildcardBufMode_e::Legacy );
	const int * pWildcard = tWildcardBuf.DecodePattern ( sWildcard );

	const auto& dSegments = *(const RtSegVec_c*)tArgs.m_pIndexData.Ptr();
	DictEntryRtPayload_t tDict2Payload ( tArgs.m_bPayload, dSegments.GetLength(), eDictFormat );
	const int iSkipMagic = ( BYTE(*sSubstring)<0x20 ); // whether to skip heading magic chars in the prefix, like NONSTEMMED maker
	ARRAY_FOREACH ( iSeg, dSegments )
	{
		const RtSegment_t * pCurSeg = dSegments[iSeg];
		RtWordReader_c tReader ( pCurSeg, eDictFormat, m_iWordsCheckpoint, m_tSettings.m_eHitless );

		// find initial checkpoint or check words prior to 1st checkpoint
		if ( pCurSeg->m_dWordCheckpoints.GetLength() )
		{
			const RtWordCheckpoint_t * pCurCheckpoint = RtSearchCheckpointWrd ( sSubstring, iSubLen, true, pCurSeg->m_dWordCheckpoints );
			if ( pCurCheckpoint )
			{
				// there could be valid data prior 1st checkpoint that should be unpacked and checked
				const int iCheckpointNameLen = (int)strlen ( pCurCheckpoint->m_szWord );
				if ( pCurCheckpoint!=pCurSeg->m_dWordCheckpoints.Begin()
					|| ( sphDictCmp ( sSubstring, iSubLen, pCurCheckpoint->m_szWord, iCheckpointNameLen )==0 && iSubLen==iCheckpointNameLen ) )
				{
					tReader.m_pCur = pCurSeg->m_dWords.Begin() + pCurCheckpoint->m_iOffset;
				}
			}
		}

		// find the word between checkpoints
		while ( tReader.UnzipWord() )
		{
			const auto* pWord = (const RtWord_t*)tReader;
			int iCmp = sphDictCmp ( sSubstring, iSubLen, (const char *)pWord->m_sWord, pWord->m_iWordLen );
			if ( iCmp<0 )
				break;
			else if ( iCmp==0 && iSubLen<=pWord->m_iWordLen && sphWildcardMatch ( (const char *)pWord->m_sWord+iSkipMagic, sWildcard, pWildcard, &tWildcardBuf ) )
				tDict2Payload.Add ( pWord, iSeg );
			// FIXME!!! same case 'boxi*' matches 'box' document at plain index
			// but masked by a checkpoint search
		}
	}

	tDict2Payload.Convert ( tArgs );
}


bool ExtractInfixCheckpoints ( const char * sInfix, int iBytes, int iMaxCodepointLength, int iDictCpCount, const CSphTightVector<uint64_t> & dFilter, CSphVector<DWORD> & dCheckpoints )
{
	if ( !dFilter.GetLength() )
		return false;

	int iStart = dCheckpoints.GetLength();

	uint64_t dVals[ BLOOM_PER_ENTRY_VALS_COUNT * BLOOM_HASHES_COUNT ];
	memset ( dVals, 0, sizeof(dVals) );

	BloomGenTraits_t tBloom0 ( dVals );
	BloomGenTraits_t tBloom1 ( dVals+BLOOM_PER_ENTRY_VALS_COUNT );
	if ( !BuildBloom ( (const BYTE *)sInfix, iBytes, BLOOM_NGRAM_0, ( iMaxCodepointLength>1 ), BLOOM_PER_ENTRY_VALS_COUNT, tBloom0 ) )
		return false;
	BuildBloom ( (const BYTE *)sInfix, iBytes, BLOOM_NGRAM_1, ( iMaxCodepointLength>1 ), BLOOM_PER_ENTRY_VALS_COUNT, tBloom1 );

	for ( int iDictCp=0; iDictCp<iDictCpCount+1; iDictCp++ )
	{
		const uint64_t * pCP = dFilter.Begin() + iDictCp * BLOOM_PER_ENTRY_VALS_COUNT * BLOOM_HASHES_COUNT;
		const uint64_t * pSuffix = dVals;

		bool bMatched = true;
		for ( int iElem=0; iElem<BLOOM_PER_ENTRY_VALS_COUNT*BLOOM_HASHES_COUNT; iElem++ )
		{
			uint64_t uFilter = *pCP++;
			uint64_t uSuffix = *pSuffix++;
			if ( ( uFilter & uSuffix )!=uSuffix )
			{
				bMatched = false;
				break;
			}
		}

		if ( bMatched )
			dCheckpoints.Add ( (DWORD)iDictCp );
	}

	return ( dCheckpoints.GetLength()!=iStart );
}


void RtIndex_c::GetInfixedWords ( const char * sSubstring, int iSubLen, const char * sWildcard, Args_t & tArgs ) const
{
	// sanity checks
	if ( !sSubstring || iSubLen<=0 )
		return;

	// find those prefixes
	CSphVector<DWORD> dPoints;
	const int iSkipMagic = ( tArgs.m_bHasExactForms ? 1 : 0 ); // whether to skip heading magic chars in the prefix, like NONSTEMMED maker
	const auto& dSegments = *(const RtSegVec_c*)tArgs.m_pIndexData.Ptr();

	const DictFormat_e eDictFormat = GetDictFormat();
	DictEntryRtPayload_t tDict2Payload ( tArgs.m_bPayload, dSegments.GetLength(), eDictFormat );
	WildcardBuf_t tWildcardBuf ( eDictFormat==DictFormat_e::KEYWORDS_V2 ? WildcardBufMode_e::Extended : WildcardBufMode_e::Legacy );
	const int * pWildcard = tWildcardBuf.DecodePattern ( sWildcard );

	ARRAY_FOREACH ( iSeg, dSegments )
	{
		const RtSegment_t * pSeg = dSegments[iSeg];
		if ( !pSeg->m_dWords.GetLength() )
			continue;

		dPoints.Resize ( 0 );
		if ( !ExtractInfixCheckpoints ( sSubstring, iSubLen, m_iMaxCodepointLength, pSeg->m_dWordCheckpoints.GetLength(), pSeg->m_dInfixFilterCP, dPoints ) )
			continue;

		// walk those checkpoints, check all their words
		for ( DWORD uPoint : dPoints )
		{
			auto iNext = (int)uPoint;
			auto iCur = iNext-1;
			RtWordReader_c tReader ( pSeg, eDictFormat, m_iWordsCheckpoint, m_tSettings.m_eHitless );
			if ( iCur>=0 )
				tReader.m_pCur = pSeg->m_dWords.Begin() + pSeg->m_dWordCheckpoints[iCur].m_iOffset;
			if ( iNext<pSeg->m_dWordCheckpoints.GetLength() )
				tReader.m_pMax = pSeg->m_dWords.Begin() + pSeg->m_dWordCheckpoints[iNext].m_iOffset;

			while ( tReader.UnzipWord() )
			{
				if ( tArgs.m_bHasExactForms && *tReader->m_sWord!=MAGIC_WORD_HEAD_NONSTEMMED )
					continue;

				// check it
				if ( !sphWildcardMatch ( (const char*)tReader->m_sWord+iSkipMagic, sWildcard, pWildcard, &tWildcardBuf ) )
					continue;

				// matched, lets add
				tDict2Payload.Add ( (const RtWord_t*)tReader, iSeg );
			}
		}
	}

	tDict2Payload.Convert ( tArgs );
}

#if WITH_RE2
struct RtRegexMatch_t
{
	std::unique_ptr<RE2> m_pRe { nullptr };
	std::unique_ptr<DictEntryRtPayload_t> m_pPayload { nullptr };
};
#endif

void RtIndex_c::ScanRegexWords ( const VecTraits_T<RegexTerm_t> & dTerms, const ISphWordlist::Args_t & tArgs, const VecExpandConv_t & dConverters ) const
{
	if ( GetDictFormat()==DictFormat_e::KEYWORDS_V2 )
		return;

	assert ( dTerms.GetLength() && dTerms.GetLength()==dConverters.GetLength() );

#if WITH_RE2

	const auto & dSegments = *(const RtSegVec_c*)tArgs.m_pIndexData.Ptr();

	CSphFixedVector<RtRegexMatch_t> dRegex ( dTerms.GetLength() );
	RE2::Options tOptions;
	tOptions.set_encoding ( RE2::Options::Encoding::EncodingUTF8 );
	ARRAY_FOREACH ( i, dRegex )
	{
		dRegex[i].m_pRe = std::make_unique<RE2> ( dTerms[i].first.cstr(), tOptions );
		dRegex[i].m_pPayload = std::make_unique<DictEntryRtPayload_t> ( tArgs.m_bPayload, dSegments.GetLength(), GetDictFormat() );
		assert ( dRegex[i].m_pRe && dRegex[i].m_pPayload );
	}

	ARRAY_FOREACH ( iSeg, dSegments )
	{
		const RtSegment_t * pCurSeg = dSegments[iSeg];
		RtWordReader_c tReader ( pCurSeg, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );

		while ( tReader.UnzipWord() )
		{
			const BYTE * pDictWord = tReader->m_sWord;

			// stemmed terms should not match suffixes
			if ( tArgs.m_bHasExactForms && *pDictWord!=MAGIC_WORD_HEAD_NONSTEMMED )
				continue;

			int iLen = tReader->m_iWordLen;
			if ( *pDictWord<0x20 ) // anyway skip heading magic chars in the prefix, like NONSTEMMED maker
			{
				pDictWord++;
				iLen--;
			}

			re2::StringPiece sDictToken ( (const char *)pDictWord, iLen );

			ARRAY_FOREACH ( i, dRegex )
			{
				if ( RE2::FullMatchN ( sDictToken, *dRegex[i].m_pRe, nullptr, 0 ) )
					dRegex[i].m_pPayload->Add ( (const RtWord_t *)tReader, iSeg );
			}
		}
	}

	ARRAY_FOREACH ( i, dRegex )
		dConverters[i] = std::move( dRegex[i].m_pPayload );
#endif
}

void RtIndex_c::GetSuggest ( const SuggestArgs_t & tArgs, SuggestResult_t & tRes ) const
{
	if ( GetDictFormat()==DictFormat_e::KEYWORDS_V2 )
		return;

	auto tGuard = RtGuard();
	const auto& dSegments = tGuard.m_dRamSegs;

	// segments and disk chunks dictionaries produce duplicated entries
	tRes.m_bMergeWords = true;

	if ( dSegments.GetLength() )
	{
		assert ( !tRes.m_pWordReader && !tRes.m_pSegments );
		auto pReader = std::make_unique<RtWordReader_c> ( dSegments[0], GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );
		tRes.m_pWordReader = pReader.get();
		tRes.m_pSegments = tGuard.m_tSegmentsAndChunks.m_pSegs;
		tRes.m_bHasExactDict = m_tSettings.m_bIndexExactWords;

		// FIXME!!! cache InfixCodepointBytes as it is slow - GetMaxCodepointLength is charset_table traverse
		sphGetSuggest ( this, m_pTokenizer->GetMaxCodepointLength(), tArgs, tRes );

		tRes.m_pWordReader = nullptr;
		tRes.m_pSegments = nullptr;
	}

	int iWorstCount = 0;
	auto& dDiskChunks = tGuard.m_dDiskChunks;
	// check disk chunks from recent to oldest
	for ( int i = dDiskChunks.GetLength() - 1; i >= 0; --i )
	{
		int iWorstDist = 0;
		int iWorstDocs = 0;
		if ( tRes.m_dMatched.GetLength() )
		{
			iWorstDist = tRes.m_dMatched.Last().m_iDistance;
			iWorstDocs = tRes.m_dMatched.Last().m_iDocs;
		}

		dDiskChunks[i]->Cidx().GetSuggest ( tArgs, tRes );

		// stop checking in case worst element is same several times during loop
		if ( tRes.m_dMatched.GetLength() && iWorstDist==tRes.m_dMatched.Last().m_iDistance && iWorstDocs==tRes.m_dMatched.Last().m_iDocs )
		{
			iWorstCount++;
			if ( iWorstCount>2 )
				break;
		} else
		{
			iWorstCount = 0;
		}
	}
}

void RtIndex_c::SuffixGetChekpoints ( const SuggestResult_t & tRes, const char * sSuffix, int iLen, CSphVector<DWORD> & dCheckpoints ) const
{
	const auto& dSegments = *(const RtSegVec_c*)tRes.m_pSegments.Ptr();
	assert ( dSegments.GetLength()<0xFF );

	ARRAY_FOREACH ( iSeg, dSegments )
	{
		const RtSegment_t * pSeg = dSegments[iSeg];
		if ( !pSeg->m_dWords.GetLength () )
			continue;

		int iStart = dCheckpoints.GetLength();
		if ( !ExtractInfixCheckpoints ( sSuffix, iLen, m_iMaxCodepointLength, pSeg->m_dWordCheckpoints.GetLength(), pSeg->m_dInfixFilterCP, dCheckpoints ) )
			continue;

		DWORD iSegPacked = (DWORD)iSeg<<24;
		for ( int i=iStart; i<dCheckpoints.GetLength(); i++ )
		{
			assert ( ( dCheckpoints[i] & 0xFFFFFF )==dCheckpoints[i] );
			dCheckpoints[i] |= iSegPacked;
		}
	}
}

void RtIndex_c::SetCheckpoint ( SuggestResult_t & tRes, DWORD iCP ) const
{
	assert ( tRes.m_pWordReader && tRes.m_pSegments );
	const auto& dSegments = *(const RtSegVec_c*)tRes.m_pSegments.Ptr();
	auto* pReader = (RtWordReader_c*)tRes.m_pWordReader;

	int iSeg = iCP>>24;
	assert ( iSeg>=0 && iSeg<dSegments.GetLength() );
	const RtSegment_t * pSeg = dSegments[iSeg];
	pReader->Reset ( pSeg );

	int iNext = (int)( iCP & 0xFFFFFF );
	int iCur = iNext-1;

	if ( iCur>0 )
		pReader->m_pCur = pSeg->m_dWords.Begin() + pSeg->m_dWordCheckpoints[iCur].m_iOffset;
	if ( iNext<pSeg->m_dWordCheckpoints.GetLength() )
		pReader->m_pMax = pSeg->m_dWords.Begin() + pSeg->m_dWordCheckpoints[iNext].m_iOffset;
}

bool RtIndex_c::ReadNextWord ( SuggestResult_t & tRes, DictWord_t & tWord ) const
{
	assert ( tRes.m_pWordReader );
	auto& tReader = *(RtWordReader_c*)tRes.m_pWordReader;

	if ( !tReader.UnzipWord() )
		return false;

	const auto* pWord = (const RtWord_t*)tReader;
	tWord.m_sWord = (const char *)pWord->m_sWord;
	tWord.m_iLen = pWord->m_iWordLen;
	tWord.m_iDocs = pWord->m_uDocs;
	return true;
}


bool RtIndex_c::RtQwordSetup ( RtQword_t * pQword, int iSeg, const RtGuard_t& tGuard ) const
{
	// segment-specific setup pass
	if ( iSeg>=0 )
		return RtQwordSetupSegment ( pQword, tGuard.m_dRamSegs[iSeg], true );

	// stat-only pass
	// loop all segments, gather stats, do not setup anything
	pQword->m_iDocs = 0;
	pQword->m_iHits = 0;
	if ( tGuard.m_dRamSegs.IsEmpty() )
		return true;

	// we care about the results anyway though
	// because if all (!) segments miss this word, we must notify the caller, right?
	bool bFound = false;
	for ( const auto& pSeg : tGuard.m_dRamSegs )
		bFound |= RtQwordSetupSegment ( pQword, pSeg, false );

	// sanity check
	assert (!( bFound==true && pQword->m_iDocs==0 ) );
	return bFound;
}

void SetupExactTokenizer ( const TokenizerRefPtr_c & pTokenizer )
{
	assert ( pTokenizer );
	pTokenizer->AddSpecials ( "=" );
}

void SetupStarTokenizer ( const TokenizerRefPtr_c & pTokenizer )
{
	assert ( pTokenizer );
	pTokenizer->AddPlainChars ( "*" );
}

class SphRtFinalMatchCalc_c : public MatchProcessor_i, ISphNoncopyable
{
public:
	SphRtFinalMatchCalc_c ( int iSegments, const CSphQueryContext & tCtx )
		: m_tCtx ( tCtx )
		, m_iSegments ( iSegments )
	{
		m_dSegments.Init ( iSegments );
	}

	bool NextSegment ( int iSeg )
	{
		m_iSeg = iSeg;

		bool bSegmentGotRows = m_dSegments.BitGet ( iSeg );

		// clear current row
		m_dSegments.BitClear ( iSeg );

		// also clear 0 segment as it got forced to process
		m_dSegments.BitClear ( 0 );

		// also force to process 0 segment to mark all used segments
		return ( iSeg==0 || bSegmentGotRows );
	}

	bool HasSegments () const							{ return ( m_iSeg==0 || m_dSegments.BitCount()>0 );	}
	void Process ( CSphMatch * pMatch ) final			{ ProcessMatch ( pMatch ); }
	void Process ( VecTraits_T<CSphMatch *> & dMatches ) final { dMatches.for_each ( [this]( CSphMatch * pMatch ){ ProcessMatch ( pMatch ); } ); }
	bool ProcessInRowIdOrder() const final				{ return m_tCtx.m_dCalcFinal.any_of ( []( const ContextCalcItem_t & i ){ return i.m_pExpr && ( i.m_pExpr->IsColumnar() || i.m_pExpr->PrefersRowIdOrder() ); } );	}

private:
	const CSphQueryContext &	m_tCtx;
	int							m_iSeg = 0;
	int							m_iSegments;
	// count per segments matches
	// to skip iteration of matches at sorter and pool setup for segment without matches at sorter
	CSphBitvec					m_dSegments;

	inline void ProcessMatch ( CSphMatch * pMatch )
	{
		int iMatchSegment = pMatch->m_iTag-1;
		if ( iMatchSegment==m_iSeg )
			m_tCtx.CalcFinal ( *pMatch );

		// count all used segments at 0 pass
		if ( m_iSeg==0 && iMatchSegment<m_iSegments )
			m_dSegments.BitSet ( iMatchSegment );
	}
};


class SorterSchemaTransform_c
{
public:
			SorterSchemaTransform_c ( int iNumChunks, bool bFinalizeSorters );

	void	Set ( int iChunk, const CSphQueryResult & tChunkResult ) { m_dDiskChunkData[iChunk].Set(tChunkResult); }
	void	Transform ( ISphMatchSorter * pSorter, const RtGuard_t& tGuard );

private:
	struct DiskChunkData_t
	{
		const BYTE *			m_pBlobPool;
		columnar::Columnar_i *	m_pColumnar;

		void Set ( const CSphQueryResult & tChunkResult )
		{
			m_pBlobPool = tChunkResult.m_pBlobPool;
			m_pColumnar = tChunkResult.m_pColumnar;
		}
	};

	CSphVector<DiskChunkData_t>	m_dDiskChunkData;
	bool						m_bFinalizeSorters = false;
};


SorterSchemaTransform_c::SorterSchemaTransform_c ( int iNumChunks, bool bFinalizeSorters )
	: m_bFinalizeSorters ( bFinalizeSorters )
{
	m_dDiskChunkData.Resize(iNumChunks);
}


void SorterSchemaTransform_c::Transform ( ISphMatchSorter * pSorter, const RtGuard_t& tGuard )
{
	assert(pSorter);

	CSphBitvec tLockedSegs ( tGuard.m_dRamSegs.GetLength() );
	auto tUnlockChunks = AtScopeExit ( [&tGuard, &tLockedSegs]() NO_THREAD_SAFETY_ANALYSIS {
		ARRAY_FOREACH ( i, tGuard.m_dRamSegs )
			if ( tLockedSegs.BitGet ( i ) )
				tGuard.m_dRamSegs[i]->m_tLock.Unlock();
	});

	auto fnGetBlobPoolFromMatch = [&tGuard,&tLockedSegs,this] ( const CSphMatch * pMatch ) NO_THREAD_SAFETY_ANALYSIS
	{
		int nRamChunks = tGuard.m_dRamSegs.GetLength ();
		int iChunkId = pMatch->m_iTag-1;
		if ( iChunkId>=nRamChunks )
			return m_dDiskChunkData[iChunkId - nRamChunks].m_pBlobPool;

		if ( !tLockedSegs.BitGet ( iChunkId ) )
		{
			tLockedSegs.BitSet ( iChunkId );
			tGuard.m_dRamSegs[iChunkId]->m_tLock.ReadLock();
		}
		return (const BYTE *) tGuard.m_dRamSegs[iChunkId]->m_dBlobs.Begin();
	};

	auto fnGetColumnarFromMatch = [&tGuard,this] ( const CSphMatch * pMatch ) -> columnar::Columnar_i *
	{
		int nRamChunks = tGuard.m_dRamSegs.GetLength ();
		int iChunkId = pMatch->m_iTag-1;
		if ( iChunkId<nRamChunks )
			return tGuard.m_dRamSegs[iChunkId]->m_pColumnar.get();

		return m_dDiskChunkData[iChunkId-nRamChunks].m_pColumnar;
	};

	pSorter->TransformPooled2StandalonePtrs ( fnGetBlobPoolFromMatch, fnGetColumnarFromMatch, m_bFinalizeSorters );
}


static int64_t CalcMaxCountDistinct ( const CSphQuery & tQuery, const RtGuard_t & tGuard )
{
	if ( tQuery.m_sGroupBy.IsEmpty() )
		return -1;

	int64_t iMaxCountDistinct = -1;
	for ( auto & i : tGuard.m_dDiskChunks )
	{
		int iGroupby = GetAliasedAttrIndex ( tQuery.m_sGroupBy, tQuery, i->Cidx().GetMatchSchema() );
		if ( iGroupby>=0 )
		{
			CSphString sModifiedAttr;
			int64_t iCountDistinct = i->Cidx().GetCountDistinct ( i->Cidx().GetMatchSchema().GetAttr(iGroupby).m_sName, sModifiedAttr );
			if ( iCountDistinct==-1 )
				return -1;	// if one of the chunks doesn't have that info, we can't calculate max

			iMaxCountDistinct = Max ( iCountDistinct, iMaxCountDistinct );
		}
	}

	return iMaxCountDistinct;
}


static bool CalcDiskChunkSplits ( IntVec_t & dThreads, int iJobs, const CSphQuery & tQuery, const CSphMultiQueryArgs & tArgs, const RtGuard_t & tGuard )
{
	// less threads than jobs; no need to calculate anything. just use 1 thread per job.
	if ( tArgs.m_iThreads<=iJobs )
		return true;

	CSphVector<SplitData_t> dSplitData ( iJobs );

	int iMaxThreadsPerIndex = CalcMaxThreadsPerIndex ( tArgs.m_iThreads, iJobs );
	int64_t iMaxCountDistinct = CalcMaxCountDistinct ( tQuery, tGuard );
	ARRAY_FOREACH ( i, dSplitData )
	{
		bool bForceSingleThread = false;
		auto tMetric = tGuard.m_dDiskChunks[i]->Cidx().GetPseudoShardingMetric ( { &tQuery, 1 }, { &iMaxCountDistinct, 1 }, iMaxThreadsPerIndex, bForceSingleThread );
		if ( bForceSingleThread )
			return false;

		assert ( tMetric.first>=0 );
		auto & tSplitData = dSplitData[i];
		tSplitData.m_iMetric = tMetric.first;
		tSplitData.m_iThreadCap = tQuery.m_iConcurrency ? 0 : tMetric.second;	// ignore thread cap if concurrency is explicitly specified
	}

	// we have more free threads than disk chunks; makes sense to apply pseudo_sharding
	if ( tArgs.m_iThreads>iJobs )
		DistributeThreadsOverIndexes ( dThreads, dSplitData, tArgs.m_iThreads );

	return true;
}


static bool QueryDiskChunks ( const CSphQuery & tQuery, CSphQueryResultMeta & tResult, const CSphMultiQueryArgs & tArgs, const RtGuard_t & tGuard, VecTraits_T<ISphMatchSorter *> & dSorters, QueryProfile_c * pProfiler, bool bGotLocalDF, const SmallStringHash_T<int64_t> * pLocalDocs, int64_t iTotalDocs, const char * szIndexName, SorterSchemaTransform_c & tSSTransform, int64_t tmMaxTimer )
{
	// counter of tasks we will issue now
	int iJobs = tGuard.m_dDiskChunks.GetLength();
	if ( !iJobs )
		return true;

	assert ( !dSorters.IsEmpty () );

	auto tDispatch = GetEffectiveBaseDispatcherTemplate();
	Dispatcher::Unify ( tDispatch, tQuery.m_tMainDispatcher );

	// the context
	ClonableCtx_T<DiskChunkSearcherCtx_t, DiskChunkSearcherCloneCtx_t, Threads::ECONTEXT::ORDERED> tClonableCtx { dSorters, tResult };

	// because disk chunk search within the loop will switch the profiler state
	SwitchProfile ( pProfiler, SPH_QSTATE_INIT );

	// uninitilized dSplits (due to tClonableCtx.IsSingle()) could still cause the SplitQuery code path at the CSphIndex_VLN::MultiQuery as
	// dSplits[iChunk] -> tMultiArgs.m_iThreads
	// then at the m_dDiskChunks
	// if ( tArgs.m_iThreads>1 ) return SplitQuery
	IntVec_t dSplits {iJobs};
	dSplits.Fill(1); // moved here from CalcDiskChunkSplits to prevent short-cuts

	auto pDispatcher = Dispatcher::Make ( iJobs, tArgs.m_iThreads, tDispatch, tClonableCtx.IsSingle() || !CalcDiskChunkSplits ( dSplits, iJobs, tQuery, tArgs, tGuard ));

	const int iThreads = pDispatcher->GetConcurrency();
	tClonableCtx.LimitConcurrency ( iThreads );

	auto iStart = sphMicroTimer();
	RTQUERYINFO << "Started: " << ( sphMicroTimer()-iStart );

	std::atomic<bool> bInterrupt { false };
	std::atomic<int> bSucceed { 1 };
	auto CheckInterrupt = [&bInterrupt]() { return bInterrupt.load ( std::memory_order_relaxed ); };

	Coro::ExecuteN ( tClonableCtx.Concurrency ( iJobs ), [&]
	{
		auto pSource = pDispatcher->MakeSource();
		int iJob = -1; // make it consumed

		if ( !pSource->FetchTask ( iJob ) || CheckInterrupt() )
		{
			RTQUERYINFO << "Early finish parallel QueryDiskChunks because of empty queue";
			return; // already nothing to do, early finish.
		}

		auto tJobContext = tClonableCtx.CloneNewContext ( !iJob );
		auto& tCtx = tJobContext.first;
		auto Interrupt = [&bInterrupt, &tCtx] ( const char* szReason ) {
			tCtx.m_tMeta.m_sWarning = szReason;
			bInterrupt.store ( true, std::memory_order_relaxed );
		};
		RTQUERYINFO << "QueryDiskChunks cloned context " << tJobContext.second << " (job " << iJob << ")";
		tClonableCtx.SetJobOrder ( tJobContext.second, -iJob ); // fixme! Same as in single search, but here we walk in reverse order. Need to fix?
		Threads::Coro::SetThrottlingPeriodMS ( session::GetThrottlingPeriodMS() );
		while ( !CheckInterrupt() ) // some earlier job met error; abort.
		{
			// jobs come in ascending order from 0 up to iJobs-1.
			// We walk over disk chunk in reverse order, from last to 0-th.
			auto iChunk = iJobs - iJob - 1;
			RTQUERYINFO << "QueryDiskChunks " << tJobContext.second << ", Jb/Chunk: " << iJob << "/" << iChunk;
			iJob = -1; // mark it consumed
			myinfo::SetTaskInfo ( "%d ch %d:", Threads::Coro::NumOfRestarts(), iChunk );
			auto & dLocalSorters = tCtx.m_dSorters;
			CSphQueryResultMeta tChunkMeta;
			CSphQueryResult tChunkResult;
			tChunkResult.m_pMeta = &tChunkMeta;
			CSphQueryResultMeta & tThMeta = tCtx.m_tMeta;
			tChunkMeta.m_pProfile = tThMeta.m_pProfile;

			CSphMultiQueryArgs tMultiArgs ( tArgs.m_iIndexWeight );
			// storing index in matches tag for finding strings attrs offset later, biased against default zero and segments
			tMultiArgs.m_iTag = tGuard.m_dRamSegs.GetLength ()+iChunk+1;
			tMultiArgs.m_uPackedFactorFlags = tArgs.m_uPackedFactorFlags;
			tMultiArgs.m_bLocalDF = bGotLocalDF;
			tMultiArgs.m_pLocalDocs = pLocalDocs;
			tMultiArgs.m_iTotalDocs = iTotalDocs;
			tMultiArgs.m_iThreads = dSplits[iChunk];
			tMultiArgs.m_iTotalThreads = iThreads;
			tMultiArgs.m_bUseSICache = tArgs.m_bUseSICache;

			// we use sorters in both disk chunks and ram chunks,
			// that's why we don't want to move to a new schema before we searched ram chunks
			tMultiArgs.m_bModifySorterSchemas = false;

			bool bChunkSucceed = tGuard.m_dDiskChunks[iChunk]->Cidx().MultiQuery ( tChunkResult, tQuery, dLocalSorters, tMultiArgs ) ;
			bSucceed.fetch_and ( bChunkSucceed );
			if ( !bChunkSucceed )
				Interrupt ( "" );

			// check terms inconsistency among disk chunks
			tThMeta.MergeWordStats ( tChunkMeta );
			tSSTransform.Set ( iChunk, tChunkResult );

			if ( iChunk && sph::TimeExceeded ( tmMaxTimer ) )
				Interrupt ( "query time exceeded max_query_time" );

			if ( tThMeta.m_sWarning.IsEmpty() && !tChunkMeta.m_sWarning.IsEmpty() )
				tThMeta.m_sWarning = tChunkMeta.m_sWarning;

			tThMeta.m_bTotalMatchesApprox |= tChunkMeta.m_bTotalMatchesApprox;
			tThMeta.m_tIteratorStats.Merge ( tChunkMeta.m_tIteratorStats );

			if ( CheckInterrupt() && !tChunkMeta.m_sError.IsEmpty() )
				// FIXME? maybe handle this more gracefully (convert to a warning)?
				tThMeta.m_sError = tChunkMeta.m_sError;

			if ( !pSource->FetchTask ( iJob ) || CheckInterrupt() )
				return; // all is done

			// yield and reschedule every quant of time. It gives work to other tasks
			if ( Threads::Coro::RuntimeExceeded() )
			{
				if ( session::GetKilled() )
					Interrupt ( "query was killed" );
				else
					Threads::Coro::RescheduleAndKeepCrashQuery();
			}
		}
	});
	RTQUERYINFO "QueryDiskChunks processed in " << tClonableCtx.NumWorked() << " thread(s)";
	tClonableCtx.Finalize();

	return bSucceed;
}



bool FinalExpressionCalculation ( CSphQueryContext & tCtx, const VecTraits_T<RtSegmentRefPtf_t> & dRamChunks, VecTraits_T<ISphMatchSorter *> & dSorters, bool bFinalizeSorters, CSphQueryResultMeta & tMeta )
{
	if ( dSorters.any_of ( [&] ( ISphMatchSorter * p ) { return !p->FinalizeJoin ( tMeta.m_sError, tMeta.m_sWarning ); } ) )
		return false;

	if ( tCtx.m_dCalcFinal.IsEmpty() )
		return true;

	const int iSegmentsTotal = dRamChunks.GetLength ();

	// at 0 pass processor also fills bitmask of segments these has matches at sorter
	// then skip sorter processing for these 'empty' segments
	SphRtFinalMatchCalc_c tFinal ( iSegmentsTotal, tCtx );

	ARRAY_FOREACH_COND ( iSeg, dRamChunks, tFinal.HasSegments() )
	{
		if ( !tFinal.NextSegment ( iSeg ) )
			continue;

		// set blob pool for string on_sort expression fix up
		SccRL_t rLock ( dRamChunks[iSeg]->m_tLock );
		tCtx.SetBlobPool ( dRamChunks[iSeg]->m_dBlobs.Begin() );
		tCtx.SetColumnar ( dRamChunks[iSeg]->m_pColumnar.get() );
		tCtx.SetDocstore ( dRamChunks[iSeg]->m_pDocstore.get(), -1 );	// no need to create session/readers for RT segments

		dSorters.Apply ( [&] ( ISphMatchSorter * pTop ) { pTop->Finalize ( tFinal, false, bFinalizeSorters ); } );
	}

	return true;
}

// perform initial query transformations and expansion.
static int PrepareFTSearch ( const RtIndex_c * pThis, bool bIsStarDict, bool bKeywordDict, int iExpandKeywords, int iExpansionLimit, const CSphIndexSettings & tSettings, const CSphQuery & tQuery, cRefCountedRefPtrGeneric_t pIndexData, const DictRefPtr_c& pDict, CSphQueryResultMeta & tMeta, QueryProfile_c * pProfiler, CSphScopedPayload * pPayloads, XQQuery_t & tParsed )
{
	if ( !tParsed.m_sParseWarning.IsEmpty () )
		tMeta.m_sWarning = tParsed.m_sParseWarning;

	// transform query if needed (quorum transform, etc.)
	SwitchProfile ( pProfiler, SPH_QSTATE_TRANSFORMS );

	// FIXME!!! provide segments list instead index
	TransformExtendedQueryArgs_t tTranformArgs { GetBooleanSimplify ( tQuery ), tParsed.m_bNeedPhraseTransform, pThis };
	if ( !sphTransformExtendedQuery ( &tParsed.m_pRoot, tSettings, tMeta.m_sError, tTranformArgs, tMeta.m_sWarning ) )
		return 0;

	int iExpandKw = ExpandKeywords ( iExpandKeywords, tQuery.m_eExpandKeywords, tSettings, bKeywordDict );
	if ( iExpandKw!=KWE_DISABLED )
	{
		sphQueryExpandKeywords ( &tParsed.m_pRoot, tSettings, iExpandKw, bKeywordDict );
		tParsed.m_pRoot->Check ( true );
	}

	// this should be after keyword expansion
	TransformAotFilter ( tParsed.m_pRoot, pDict->GetWordforms (), tSettings );

	// expanding prefix in word dictionary case
	if ( bKeywordDict && bIsStarDict )
	{
		ExpansionContext_t tExpCtx;
		tExpCtx.m_pWordlist = pThis;
		tExpCtx.m_pBuf = nullptr;
		tExpCtx.m_pResult = &tMeta;
		tExpCtx.m_iMinPrefixLen = tSettings.GetMinPrefixLen ( bKeywordDict );
		tExpCtx.m_iMinInfixLen = tSettings.m_iMinInfixLen;
		tExpCtx.m_iExpansionLimit = GetExpansionLimit ( tQuery.m_iExpansionLimit, iExpansionLimit );
		tExpCtx.m_bHasExactForms = ( pDict->HasMorphology () || tSettings.m_bIndexExactWords );
		tExpCtx.m_bMergeSingles = ( tQuery.m_uDebugFlags & QUERY_DEBUG_NO_PAYLOAD )==0;
		tExpCtx.m_pPayloads = pPayloads;
		tExpCtx.m_pIndexData = std::move ( pIndexData );

		tParsed.m_pRoot = sphExpandXQNode ( tParsed.m_pRoot, tExpCtx ); // here magics happens
		if ( !ExpandRegex ( tExpCtx, tMeta.m_sError ) )
			return 0;

		tExpCtx.AggregateStats();
	}

	return ConsiderStack ( tParsed.m_pRoot, tMeta.m_sError );
}


static bool SetupFilters ( const CSphQuery & tQuery, const ISphSchema & tMatchSchema, const ISphSchema & tIndexSchema, bool bFullscan, CSphQueryContext & tCtx, CSphVector<CSphFilterSettings> & dTransformedFilters, CSphVector<FilterTreeItem_t> & dTransformedFilterTree, const CSphVector<const ISphSchema *> & dSorterSchemas, CSphQueryResultMeta & tMeta )
{
	CreateFilterContext_t tFlx;
	tFlx.m_pFilters = &tQuery.m_dFilters;
	tFlx.m_pFilterTree = &tQuery.m_dFilterTree;
	tFlx.m_pMatchSchema = &tMatchSchema;
	tFlx.m_pIndexSchema = &tIndexSchema;
	tFlx.m_eCollation = tQuery.m_eCollation;
	tFlx.m_bScan = bFullscan;
	tFlx.m_sJoinIdx = tQuery.m_sJoinIdx;
	tFlx.m_eJoinType = tQuery.m_eJoinType;
	tFlx.m_bAddKNNDistFilter = true;

	std::unique_ptr<ISphSchema> pModifiedMatchSchema;
	if ( !TransformFilters ( tFlx, dTransformedFilters, dTransformedFilterTree, pModifiedMatchSchema, tQuery.m_dItems, tMeta.m_sError ) )
		return false;

	if ( pModifiedMatchSchema )
		tFlx.m_pMatchSchema = pModifiedMatchSchema.get();

	tFlx.m_pFilters = &dTransformedFilters;
	tFlx.m_pFilterTree = dTransformedFilterTree.GetLength() ? &dTransformedFilterTree : nullptr;

	if ( !tCtx.SetupCalc ( tMeta, *tFlx.m_pMatchSchema, tIndexSchema, nullptr, nullptr, dSorterSchemas ) )
		return false;

	return tCtx.CreateFilters ( tFlx, tMeta.m_sError, tMeta.m_sWarning );
}


static bool PerformFullscan ( const VecTraits_T<RtSegmentRefPtf_t> & dRamChunks, int iMaxDynamicSize, int iIndexWeight, int iStride, int iCutoff, int64_t tmMaxTimer, QueryProfile_c * pProfiler, CSphQueryContext & tCtx, VecTraits_T<ISphMatchSorter*> & dSorters, CSphString & sWarning )
{
	if ( !iCutoff )
		return true;

	bool bRandomize = dSorters[0]->IsRandom();

	SwitchProfile ( pProfiler, SPH_QSTATE_FULLSCAN );

	// full scan
	// FIXME? OPTIMIZE? add shortcuts here too?
	CSphMatch tMatch;
	tMatch.Reset ( iMaxDynamicSize );
	tMatch.m_iWeight = iIndexWeight;

	ARRAY_FOREACH ( iSeg, dRamChunks )
	{
		RtSegment_t & tSeg = *dRamChunks[iSeg];
		SccRL_t rLock ( tSeg.m_tLock );
		auto pBlobs = tSeg.m_dBlobs.Begin();
		tCtx.SetBlobPool(pBlobs);
		for ( auto * pSorter : dSorters )
			pSorter->SetBlobPool(pBlobs);

		auto pColumnar = tSeg.m_pColumnar.get();
		tCtx.SetColumnar(pColumnar);
		for ( auto * pSorter : dSorters )
			pSorter->SetColumnar(pColumnar);

		if ( tCtx.m_pFilter )
			tCtx.m_pFilter->SetColumnar(pColumnar);

		session::Info().m_pSessionOpaque2 = (void*)tSeg.m_pDocstore.get();

		for ( auto tRowID : RtLiveRows_c(tSeg) )
		{
			tMatch.m_tRowID = tRowID;
			tMatch.m_pStatic = tSeg.m_dRows.Begin() + (int64_t)tRowID*iStride;

			tCtx.CalcFilter ( tMatch );
			if ( tCtx.m_pFilter && !tCtx.m_pFilter->Eval ( tMatch ) )
			{
				tCtx.FreeDataFilter ( tMatch );
				continue;
			}

			if ( bRandomize )
				tMatch.m_iWeight = ( sphRand() & 0xffff ) * iIndexWeight;

			tCtx.CalcSort ( tMatch );

			// storing segment in matches tag for finding strings attrs offset later, biased against default zero
			tMatch.m_iTag = iSeg+1;

			bool bNewMatch = false;
			for ( auto * pSorter: dSorters )
				bNewMatch |= pSorter->Push ( tMatch );

			// stringptr expressions should be duplicated (or taken over) at this point
			tCtx.FreeDataFilter ( tMatch );
			tCtx.FreeDataSort ( tMatch );

			// handle cutoff
			if ( bNewMatch )
				if ( --iCutoff==0 )
					return true;

			// handle timer
			if ( sph::TimeExceeded ( tmMaxTimer ) )
			{
				sWarning = "query time exceeded max_query_time";
				return true;
			}

			if ( Threads::Coro::RuntimeExceeded() )
			{
				if ( session::GetKilled() )
				{
					sWarning = "query was killed";
					return true;
				}
				Threads::Coro::RescheduleAndKeepCrashQuery();
			}
		}
	}

	return false;
}


static bool DoFullScanQuery ( const RtSegVec_c & dRamChunks, const ISphSchema & tMaxSorterSchema, const CSphQuery & tQuery, const CSphMultiQueryArgs & tArgs, int iStride, int64_t tmMaxTimer, QueryProfile_c * pProfiler, CSphQueryContext & tCtx, VecTraits_T<ISphMatchSorter*> & dSorters, CSphQueryResultMeta & tMeta )
{
	// probably redundant, but just in case
	SwitchProfile ( pProfiler, SPH_QSTATE_INIT );

	// search segments no looking to max_query_time
	// FIXME!!! move searching at segments before disk chunks as result set is safe with kill-lists
	if ( !dRamChunks.IsEmpty () )
	{
		// FIXME! OPTIMIZE! check if we can early reject the whole index

		int iCutoff = ApplyImplicitCutoff ( tQuery, dSorters, false );
		tMeta.m_bTotalMatchesApprox |= PerformFullscan ( dRamChunks, tMaxSorterSchema.GetDynamicSize(), tArgs.m_iIndexWeight, iStride, iCutoff, tmMaxTimer, pProfiler, tCtx, dSorters, tMeta.m_sWarning );
	}

	return FinalExpressionCalculation ( tCtx, dRamChunks, dSorters, tArgs.m_bFinalizeSorters, tMeta );
}


static void PerformFullTextSearch ( const RtSegVec_c & dRamChunks, RtQwordSetup_t & tTermSetup, ISphRanker * pRanker, int iIndexWeight, int iCutoff, QueryProfile_c * pProfiler, CSphQueryContext & tCtx, VecTraits_T<ISphMatchSorter*> & dSorters )
{
	if ( !iCutoff )
		return;

	bool bRandomize = dSorters[0]->IsRandom();
	// query matching
	ARRAY_FOREACH ( iSeg, dRamChunks )
	{
		const RtSegment_t * pSeg = dRamChunks[iSeg];
		SccRL_t rLock ( pSeg->m_tLock );
		SwitchProfile ( pProfiler, SPH_QSTATE_INIT_SEGMENT );

		tTermSetup.SetSegment ( iSeg );
		pRanker->Reset ( tTermSetup );

		// for lookups to work
		tCtx.m_pIndexData = pSeg;

		// set blob pool for string on_sort expression fix up
		const BYTE * pBlobPool = pSeg->m_dBlobs.Begin ();
		tCtx.SetBlobPool ( pBlobPool );
		for ( auto * pSorter : dSorters )
			pSorter->SetBlobPool ( pBlobPool );

		auto pColumnar = pSeg->m_pColumnar.get();
		tCtx.SetColumnar(pColumnar);
		for ( auto * pSorter : dSorters )
			pSorter->SetColumnar(pColumnar);

		if ( tCtx.m_pFilter )
			tCtx.m_pFilter->SetColumnar(pColumnar);

		// storing segment in matches tag for finding strings attrs offset later, biased against default zero
		int iTag = iSeg+1;
		if ( tCtx.GetPackedFactor() & SPH_FACTOR_ENABLE )
			pRanker->ExtraData ( EXTRA_SET_MATCHTAG, (void**)&iTag );

		pRanker->ExtraData ( EXTRA_SET_BLOBPOOL, (void**)&pBlobPool );
		pRanker->ExtraData ( EXTRA_SET_COLUMNAR, (void**)&pColumnar );

		CSphMatch * pMatch = pRanker->GetMatchesBuffer();
		while (true)
		{
			// ranker does profile switches internally in GetMatches()
			int iMatches = pRanker->GetMatches();
			if ( iMatches<=0 )
				break;

			SwitchProfile ( pProfiler, SPH_QSTATE_SORT );

			for ( int i=0; i<iMatches; i++ )
			{
				CSphMatch & tMatch = pMatch[i];

				tMatch.m_pStatic = pSeg->GetDocinfoByRowID ( tMatch.m_tRowID );
				tMatch.m_iWeight *= iIndexWeight;
				if ( bRandomize )
					tMatch.m_iWeight = ( sphRand() & 0xffff ) * iIndexWeight;

				tCtx.CalcSort ( tMatch );

				if ( tCtx.m_pWeightFilter && !tCtx.m_pWeightFilter->Eval ( tMatch ) )
				{
					tCtx.FreeDataSort ( tMatch );
					continue;
				}

				// storing segment in matches tag for finding strings attrs offset later, biased against default zero
				tMatch.m_iTag = iTag;

				bool bNewMatch = false;
				for ( auto* pSorter : dSorters )
				{
					bNewMatch |= pSorter->Push ( tMatch );

					if ( tCtx.GetPackedFactor() & SPH_FACTOR_ENABLE )
					{
						RowTagged_t tJustPushed = pSorter->GetJustPushed();
						VecTraits_T<RowTagged_t> dJustPopped = pSorter->GetJustPopped();
						pRanker->ExtraData ( EXTRA_SET_MATCHPUSHED, (void**)&tJustPushed );
						pRanker->ExtraData ( EXTRA_SET_MATCHPOPPED, (void**)&dJustPopped );
					}
				}

				// stringptr expressions should be duplicated (or taken over) at this point
				tCtx.FreeDataFilter ( tMatch );
				tCtx.FreeDataSort ( tMatch );

				if ( bNewMatch )
					if ( --iCutoff==0 )
						break;
			}

			if ( iCutoff==0 )
			{
				iSeg = dRamChunks.GetLength();
				break;
			}
		}
	}
}

static bool DoFullTextSearch ( const RtSegVec_c & dRamChunks, const ISphSchema & tMaxSorterSchema, const CSphQuery & tQuery, const QueryExecutionSettings_t & tQuerySettings, const CSphMultiQueryArgs & tArgs, int iMatchPoolSize, int iStackNeed, RtQwordSetup_t & tTermSetup, QueryProfile_c * pProfiler, CSphQueryContext & tCtx, VecTraits_T<ISphMatchSorter*> & dSorters, XQQuery_t & tParsed, CSphQueryResultMeta & tMeta, ISphMatchSorter * pSorter )
{
	// set zonespanlist settings
	tParsed.m_bNeedSZlist = tQuery.m_bZSlist;
	if ( IsIMocked() )
		iStackNeed = -1;

	return Threads::Coro::ContinueBool ( iStackNeed, [&] {

	// setup query
	// must happen before index-level reject, in order to build proper keyword stats
	std::unique_ptr<ISphRanker> pRanker = sphCreateRanker ( tParsed, tQuery, tQuerySettings, tMeta, tTermSetup, tCtx, tMaxSorterSchema );
	if ( !pRanker )
		return false;

	tCtx.SetupExtraData ( pRanker.get (), pSorter );

	pRanker->ExtraData ( EXTRA_SET_POOL_CAPACITY, (void **) &iMatchPoolSize );

	// check for the possible integer overflow in m_dPool.Resize
	int64_t iPoolSize = 0;
	if ( pRanker->ExtraData ( EXTRA_GET_POOL_SIZE, (void **) &iPoolSize ) && iPoolSize>INT_MAX )
	{
		tMeta.m_sError.SetSprintf ( "ranking factors pool too big (%d Mb), reduce max_matches",
				(int) ( iPoolSize / 1024 / 1024 ) );
		return false;
	}

	// probably redundant, but just in case
	SwitchProfile ( pProfiler, SPH_QSTATE_INIT );

	// search segments no looking to max_query_time
	// FIXME!!! move searching at segments before disk chunks as result set is safe with kill-lists
	if ( !dRamChunks.IsEmpty () )
	{
		// FIXME! OPTIMIZE! check if we can early reject the whole index

		// do searching
		int iCutoff = ApplyImplicitCutoff ( tQuery, dSorters, true );
		PerformFullTextSearch ( dRamChunks, tTermSetup, pRanker.get (), tArgs.m_iIndexWeight, iCutoff, pProfiler, tCtx, dSorters );
	}

	if ( !FinalExpressionCalculation ( tCtx, dRamChunks, dSorters, tArgs.m_bFinalizeSorters, tMeta ) )
		return false;

	//////////////////////
	// copying match's attributes to external storage in result set
	//////////////////////

	SwitchProfile ( pProfiler, SPH_QSTATE_FINALIZE );
	pRanker->FinalizeCache ( tMaxSorterSchema );
	return true;
	});
}

// Extract subset of chunks/segments for ops with list of chunk, i.e. for 'select .. from rt.0.2.4'
// if dChunks is empty (i.e. no subset required) - returns whole tOrigin, i.e. all chunks/segments
// if tOrigin has 3 disk chunks + 2 ram chunks, as {0d,1d,2d;0r,1r} and dChunks is {0,2,4} -
// result will have 2 disk chunks + 1 ram chunk, as {0d,2d;1r}
ConstRtData FilterReaderChunks ( ConstRtData tOrigin, const VecTraits_T<int64_t>& dChunks )
{
	if ( dChunks.IsEmpty() )
		return tOrigin;

	CSphVector<int> dOrderedChunks;
	dChunks.for_each ( [&dOrderedChunks] ( int64_t iVal ) { dOrderedChunks.Add ( (int) iVal ); } );
	dOrderedChunks.Uniq(); // implies also Sort()

	auto iDiskBound = tOrigin.m_pChunks->GetLength();
	auto iAllBound = iDiskBound + tOrigin.m_pSegs->GetLength();
	int iDiskSelected = 0;
	int iRamSelected = 0;
	dOrderedChunks.any_of ( [&] ( int iVal )
	{
		if ( iVal<iDiskBound )
		{
			++iDiskSelected;
			return false;
		}
		if ( iVal<iAllBound )
		{
			++iRamSelected;
			return false;
		}
		return true;
	});

	auto pChunks = new DiskChunkVec_c;
	pChunks->Resize ( iDiskSelected );
	for ( int i = 0; i<iDiskSelected; ++i )
		( *pChunks )[i] = ( *tOrigin.m_pChunks )[dOrderedChunks[i]];

	auto pSegments = new RtSegVec_c;
	pSegments->Resize ( iRamSelected );
	for ( int i = 0; i<iRamSelected; ++i )
		( *pSegments )[i] = ( *tOrigin.m_pSegs )[dOrderedChunks[iDiskSelected+i]-iDiskBound];

	ConstDiskChunkVecRefPtr_t pConstChunks;
	ConstRtSegVecRefPtr_t pConstSegments;
	pConstChunks = pChunks;
	pConstSegments = pSegments;
	return { pConstChunks, pConstSegments };
}


const CSphQuery * RtIndex_c::SetupAutoEmbeddings ( const CSphQuery & tQuery, CSphQuery & tUpdatedQuery, const ISphSchema & tMatchSchema, CSphString & sError ) const
{
	// check if any KNN entry needs auto-embeddings
	bool bNeedsEmb = tQuery.m_dKnnSettings.any_of ( [] ( const auto & t ) { return !!t.m_sEmbStr; } );
	if ( !m_pEmbeddings && bNeedsEmb )
	{
		sError = "Embeddings generation string specified, but embeddings are not loaded";
		return nullptr;
	}

	if ( !m_pEmbeddings || !bNeedsEmb )
		return &tQuery;

	tUpdatedQuery = tQuery;

	for ( auto & tKNN : tUpdatedQuery.m_dKnnSettings )
	{
		if ( !tKNN.m_sEmbStr || tKNN.m_sAttr.IsEmpty() )
			continue;

		auto pAttr = m_tSchema.GetAttr ( tKNN.m_sAttr.cstr() );
		if ( !pAttr )
		{
			sError.SetSprintf ( "KNN search attribute '%s' not found", tKNN.m_sAttr.cstr() );
			return nullptr;
		}

		knn::TextToEmbeddings_i * pModel = m_pEmbeddings->GetModel ( tKNN.m_sAttr );
		if ( !pModel )
		{
			sError.SetSprintf ( "No model loaded for auto embeddings attribute '%s'", tKNN.m_sAttr.cstr() );
			return nullptr;
		}

		std::vector<std::vector<float>> dEmbeddings;
		std::vector<std::string_view> dTexts;
		dTexts.push_back( tKNN.m_sEmbStr->cstr() );

		std::string sConvertError;
		if ( !pModel->Convert ( dTexts, dEmbeddings, sConvertError, GetEmbeddingsThreadsToUse ( tQuery.m_iEmbeddingsThreads ) ) )
		{
			sError.SetSprintf ( "Error generating embeddings for attribute '%s' : %s", tKNN.m_sAttr.cstr(), sConvertError.c_str() );
			return nullptr;
		}

		if ( dEmbeddings.size()!=1 )
		{
			sError.SetSprintf ( "Error generating embeddings for attribute '%s'", tKNN.m_sAttr.cstr() );
			return nullptr;
		}

		int iEmbDim = dEmbeddings[0].size();
		if ( iEmbDim!=pAttr->m_tKNN.m_iDims )
		{
			sError.SetSprintf ( "Auto generated embedding dimension mismatch: expected %d, got %d", pAttr->m_tKNN.m_iDims, iEmbDim );
			return nullptr;
		}

		tKNN.m_dVec.Resize(iEmbDim);
		memcpy ( tKNN.m_dVec.Begin(), dEmbeddings[0].data(), iEmbDim*sizeof(float) );
	}

	// update expressions that depend on KNN vec (use first KNN entry's vec for backward compat)
	if ( tUpdatedQuery.HasKnn() )
	{
		for ( int i = 0; i < tMatchSchema.GetAttrsCount(); i++ )
			if ( tMatchSchema.GetAttr(i).m_pExpr )
				tMatchSchema.GetAttr(i).m_pExpr->Command ( SPH_EXPR_SET_KNN_VEC, &tUpdatedQuery.SingleKnnSettings().m_dVec );
	}

	return &tUpdatedQuery;
}

// FIXME! missing MVA, index_exact_words support
// FIXME? any chance to factor out common backend agnostic code?
// FIXME? do we need to support pExtraFilters?
bool RtIndex_c::MultiQuery ( CSphQueryResult & tResult, const CSphQuery & tQuery, const VecTraits_T<ISphMatchSorter *> & dAllSorters, const CSphMultiQueryArgs & tArgs ) const
{
	// to avoid the checking of a ppSorters's element for NULL on every next step,
	// just filter out all nulls right here
	CSphVector<ISphMatchSorter*> dSorters;
	dSorters.Reserve ( dAllSorters.GetLength() );
	dAllSorters.Apply ([&dSorters] ( ISphMatchSorter* p ) { if ( p ) dSorters.Add(p); });
	auto& tMeta = *tResult.m_pMeta;

	auto* non_const_cthis = const_cast<RtIndex_c*>(this);
	AT_SCOPE_EXIT ( [non_const_cthis]() { non_const_cthis->m_tmDataSearched.store ( sphMicroTimer(), std::memory_order_relaxed ); non_const_cthis->ConditionalDiskChunk(); } );

	// if we have anything to work with
	if ( dSorters.IsEmpty() )
	{
		tMeta.SetQueryTimeUs ( 0 );
		return false;
	}

	assert ( tArgs.m_iTag==0 );

	MEMORY ( MEM_RT_QUERY );

	// start counting
	tMeta.SetQueryTimeUs ( 0 );
	int64_t tmQueryStart = sphMicroTimer();
	auto tmCpuQueryStart = sphTaskCpuTimer();
	QueryProfile_c * pProfiler = tMeta.m_pProfile;
	CSphScopedProfile tProf ( pProfiler, SPH_QSTATE_DICT_SETUP );

	QueryExecutionSettings_t tQuerySettings = BuildQueryExecutionSettings ( tQuery, m_tMutableSettings );

	// force ext2 mode for them
	// FIXME! eliminate this const breakage
	const_cast<CSphQuery*> ( &tQuery )->m_eMode = SPH_MATCH_EXTENDED2;

	auto tRtData = RtData();

	// debug hack (don't use ram chunk in debug modeling mode)
	if_const( MODELING )
		tRtData.m_pSegs = new RtSegVec_c;

	tRtData = FilterReaderChunks ( tRtData, tQuery.m_dIntSubkeys );
	RtGuard_t tGuard ( std::move ( tRtData ) );
	auto& dDiskChunks = tGuard.m_dDiskChunks;

	// wrappers
	DictRefPtr_c pDict = GetStatelessDict ( m_pDict );
	const bool bWordDict = IsWordDict();

	if ( bWordDict && IsStarDict ( bWordDict ) )
		SetupStarDict ( pDict );

	if ( m_tSettings.m_bIndexExactWords )
		SetupExactDict ( pDict );


	const QueryParser_i * pQueryParser = tQuery.m_pQueryParser;
	assert ( pQueryParser );
	const bool bFullscan = pQueryParser->IsFullscan ( tQuery );

	// calculate local idf for RT with disk chunks
	// in case of local_idf set but no external hash no full-scan query and RT has disk chunks
	const SmallStringHash_T<int64_t> * pLocalDocs = tArgs.m_pLocalDocs;
	SmallStringHash_T<int64_t> hLocalDocs;
	int64_t iTotalDocs = ( tArgs.m_iTotalDocs ? tArgs.m_iTotalDocs : m_tStats.m_iTotalDocuments );
	// already might local df calculated and set by distributed index
	bool bGotLocalDF = ( tArgs.m_bLocalDF && tArgs.m_pLocalDocs );
	// if not explicitly disbled lets calculate local_idf per disk chunks if it not was already calculated per distributed index
	if ( !bGotLocalDF && !bFullscan && tQuerySettings.m_eRanker!=SPH_RANK_NONE && tQuery.m_bLocalDF.value_or ( true ) && dDiskChunks.GetLength()>1 )
	{
		SwitchProfile ( pProfiler, SPH_QSTATE_LOCAL_DF );
		GetKeywordsSettings_t tSettings;
		tSettings.m_bStats = true;
		tSettings.m_bFoldStatsToUnique = true;
		// do not want to expand keywords and fold back its statistics as it could take too much time
		tSettings.m_bAllowExpansion = false;

		CSphVector < CSphKeywordInfo > dKeywords;
		DoGetKeywords ( dKeywords, tQuery.m_sQuery.cstr(), tSettings, false, nullptr, tGuard );
		for ( auto & tKw : dKeywords )
			if ( !hLocalDocs.Exists ( tKw.m_sNormalized ) ) // skip dupes
				hLocalDocs.Add ( tKw.m_iDocs, tKw.m_sNormalized );

		pLocalDocs = &hLocalDocs;
		iTotalDocs = GetStats().m_iTotalDocuments;
		bGotLocalDF = true;
	}

	SwitchProfile ( pProfiler, SPH_QSTATE_INIT );

	// select the sorter with max schema
	// uses GetAttrsCount to get working facets (was GetRowSize)
	int iMaxSchemaIndex, iMatchPoolSize;
	std::tie ( iMaxSchemaIndex, iMatchPoolSize ) = GetMaxSchemaIndexAndMatchCapacity ( dSorters );

	if ( iMaxSchemaIndex==-1 )
		return false;

	const ISphSchema & tMaxSorterSchema = *( dSorters[iMaxSchemaIndex]->GetSchema ());
	auto dSorterSchemas = SorterSchemas ( dSorters, iMaxSchemaIndex );

	CSphQuery tUpdatedQuery;
	const CSphQuery * pQueryToRun = SetupAutoEmbeddings ( tQuery, tUpdatedQuery, tMaxSorterSchema, tMeta.m_sError );
	if ( !pQueryToRun )
		return false;

	auto & tQueryToRun = *pQueryToRun;

	// FIXME! each result will point to its own MVA and string pools

	//////////////////////
	// search disk chunks
	//////////////////////

	MiniTimer_c dTimerGuard;
	int64_t tmMaxTimer = dTimerGuard.Engage ( tQueryToRun.m_uMaxQueryMsec ); // max_query_time

	SorterSchemaTransform_c tSSTransform ( dDiskChunks.GetLength(), tArgs.m_bFinalizeSorters );

	if ( !dDiskChunks.IsEmpty() )
	{
		if ( !QueryDiskChunks ( tQueryToRun, tMeta, tArgs, tGuard, dSorters, pProfiler, bGotLocalDF, pLocalDocs, iTotalDocs, GetName(), tSSTransform, tmMaxTimer ) )
			return false;
	}

	////////////////////
	// search RAM chunk
	////////////////////

	SwitchProfile ( pProfiler, SPH_QSTATE_INIT );

	// setup calculations and result schema
	CSphQueryContext tCtx ( tQueryToRun, tQuerySettings );
	tCtx.m_pProfile = pProfiler;
	tCtx.m_pLocalDocs = pLocalDocs;
	tCtx.m_iTotalDocs = iTotalDocs;
	tCtx.SetPackedFactor ( tArgs.m_uPackedFactorFlags );

	// setup search terms
	RtQwordSetup_t tTermSetup ( tGuard );
	tTermSetup.SetDict ( pDict );
	tTermSetup.m_pIndex = this;
	tTermSetup.m_iDynamicRowitems = tMaxSorterSchema.GetDynamicSize();
	tTermSetup.m_iMaxTimer = dTimerGuard.Engage ( tQueryToRun.m_uMaxQueryMsec ); // max_query_time
	tTermSetup.m_pWarning = &tMeta.m_sWarning;
	tTermSetup.SetSegment ( -1 );
	tTermSetup.m_pCtx = &tCtx;
	tTermSetup.m_bHasWideFields = ( m_tSchema.GetFieldsCount()>32 );

	// bind weights
	tCtx.BindWeights ( tQueryToRun, m_tSchema, tMeta.m_sWarning );

	CSphVector<BYTE> dFiltered;
	const BYTE * sModifiedQuery = (const BYTE *)tQueryToRun.m_sQuery.cstr();
	FieldFilterOptions_t tFFOptions { tQueryToRun.m_eJiebaMode };

	if ( m_pFieldFilter && sModifiedQuery && m_pFieldFilter->Clone ( &tFFOptions )->Apply ( sModifiedQuery, dFiltered, true ) )
		sModifiedQuery = dFiltered.Begin();

	// parse query
	SwitchProfile ( pProfiler, SPH_QSTATE_PARSE );

	XQQuery_t tParsed;
	// FIXME!!! provide segments list instead index to tTermSetup.m_pIndex

	CSphScopedPayload tPayloads;

	// FIXME!!! add proper
	// - qcache invalidation after INSERT \ DELETE \ UPDATE and for plain index afte UPDATE #256
	// - qcache duplicates removal from killed document at segment #263
	tCtx.m_bSkipQCache = true;

	int iStackNeed = -1;
	// no need to create ranker, etc if there's no query
	if ( !bFullscan )
	{
		assert ( m_pQueryTokenizer.Ptr() && m_pQueryTokenizerJson.Ptr() );
		if ( !pQueryParser->ParseQuery ( tParsed, (const char *)sModifiedQuery, &tQueryToRun, tQuerySettings, m_pQueryTokenizer, m_pQueryTokenizerJson, &m_tSchema, pDict, m_tSettings, &m_tMorphFields ) )
		{
			tMeta.m_sError = tParsed.m_sParseError;
			iStackNeed = 0;
		} else
		{
			const bool bCanExpandRamWildcards = IsStarDict ( bWordDict );
			iStackNeed = PrepareFTSearch ( this, bCanExpandRamWildcards, bWordDict, m_tMutableSettings.m_iExpandKeywords, m_iExpansionLimit, m_tSettings, tQueryToRun,(cRefCountedRefPtrGeneric_t) tGuard.m_tSegmentsAndChunks.m_pSegs, pDict, tMeta, pProfiler, &tPayloads, tParsed );
		}
	}

	// empty index, empty result. Must be AFTER PrepareFTSearch, since it prepares list of words
	if ( tGuard.m_dRamSegs.IsEmpty() )
	{
		for ( auto i : dSorters )
			tSSTransform.Transform ( i, tGuard );

		tResult.m_pDocstore = m_tSchema.HasStoredFields () || m_tSchema.HasStoredAttrs() ? this : nullptr;
		tMeta.SetQueryTimeUs ( 0 );
		return true;
	}

	if ( !iStackNeed )
		return false;

	auto& tSess = session::Info();
	tSess.m_pSessionOpaque1 = (void*)(const DocstoreReader_i*)this;
	tSess.m_pSessionOpaque2 = nullptr;

	bool bParsedFullscan = bFullscan ||  pQueryParser->IsFullscan(tParsed);

	CSphVector<CSphFilterSettings> dTransformedFilters; // holds filter settings if they were modified. filters hold pointers to those settings
	CSphVector<FilterTreeItem_t> dTransformedFilterTree;
	if ( !SetupFilters ( tQueryToRun, tMaxSorterSchema, m_tSchema, bParsedFullscan, tCtx, dTransformedFilters, dTransformedFilterTree, dSorterSchemas, tMeta ) )
		return false;

	bool bResult;
	if ( bParsedFullscan )
		bResult = DoFullScanQuery ( tGuard.m_dRamSegs, tMaxSorterSchema, tQueryToRun, tArgs, m_iStride, tmMaxTimer, pProfiler, tCtx, dSorters, tMeta );
	else
	{
		CSphMultiQueryArgs tFTArgs ( tArgs.m_iIndexWeight );
		tFTArgs.m_bFinalizeSorters = tArgs.m_bFinalizeSorters;
		tMeta.m_bBigram = ( m_tSettings.m_eBigramIndex!=SPH_BIGRAM_NONE );

		bResult = DoFullTextSearch ( tGuard.m_dRamSegs, tMaxSorterSchema, tQueryToRun, tQuerySettings, tFTArgs, iMatchPoolSize, iStackNeed, tTermSetup, pProfiler, tCtx, dSorters, tParsed, tMeta, dSorters.GetLength()==1 ? dSorters[0] : nullptr );
	}

	if (!bResult)
		return false;

	MEMORY ( MEM_RT_RES_STRINGS );

	SwitchProfile ( pProfiler, SPH_QSTATE_DYNAMIC );

	// create new standalone schema for sorters (independent of any external indexes/pools/storages)
	// modify matches inside the sorters to work with the new schema
	for ( auto i : dSorters )
		tSSTransform.Transform ( i, tGuard );

	tResult.m_pDocstore = m_tSchema.HasStoredFields() || m_tSchema.HasStoredAttrs() ? this : nullptr;
	tMeta.SetQueryTimeUs ( sphMicroTimer() - tmQueryStart );
	tMeta.m_iCpuTime += sphTaskCpuTimer ()-tmCpuQueryStart;
	return true;
}


void RtIndex_c::AddKeywordStats ( BYTE * sWord, const BYTE * sTokenized, const DictRefPtr_c& pDict, bool bGetStats, int iQpos, RtQword_t * pQueryWord, CSphVector <CSphKeywordInfo> & dKeywords, const RtSegVec_c& dRamSegs ) const
{
	assert ( !bGetStats || pQueryWord );

	SphWordID_t iWord = pDict->GetWordID ( sWord );
	if ( !iWord )
		return;

	if ( bGetStats )
	{
		pQueryWord->Reset();
		pQueryWord->m_uWordID = iWord;
		pQueryWord->m_sWord = (const char *)sTokenized;
		pQueryWord->m_sDictWord = (const char *)sWord;
		for ( const auto& pSeg : dRamSegs )
			RtQwordSetupSegment ( pQueryWord, pSeg, false );
	}

	CSphKeywordInfo & tInfo = dKeywords.Add();
	tInfo.m_sTokenized = (const char *)sTokenized;
	tInfo.m_sNormalized = (const char*)sWord;
	tInfo.m_iDocs = bGetStats ? pQueryWord->m_iDocs : 0;
	tInfo.m_iHits = bGetStats ? pQueryWord->m_iHits : 0;
	tInfo.m_iQpos = iQpos;

	RemoveDictSpecials ( tInfo.m_sNormalized, ( m_tSettings.m_eBigramIndex!=SPH_BIGRAM_NONE ) );
}


struct CSphRtQueryFilter : public ISphQueryFilter, public ISphNoncopyable
{
	const RtIndex_c *	m_pIndex;
	RtQword_t *			m_pQword;
	bool				m_bGetStats = false;
	const RtSegVec_c&	m_tGuard;

	CSphRtQueryFilter ( const RtIndex_c * pIndex, RtQword_t * pQword, const RtSegVec_c& tGuard )
		: m_pIndex ( pIndex )
		, m_pQword ( pQword )
		, m_tGuard ( tGuard )
	{}

	void AddKeywordStats ( BYTE * sWord, const BYTE * sTokenized, int iQpos, CSphVector <CSphKeywordInfo> & dKeywords ) final
	{
		assert ( m_pIndex && m_pQword );
		m_pIndex->AddKeywordStats ( sWord, sTokenized, m_pDict, m_tFoldSettings.m_bStats, iQpos, m_pQword, dKeywords, m_tGuard );
	}
};


bool RtIndex_c::DoGetKeywords ( CSphVector<CSphKeywordInfo> & dKeywords, const char * sQuery, const GetKeywordsSettings_t & tSettings, bool bFillOnly, CSphString * pError, const RtGuard_t& tGuard ) const
{
	if ( !bFillOnly )
		dKeywords.Resize ( 0 );

	if ( ( bFillOnly && !dKeywords.GetLength() ) || ( !bFillOnly && ( !sQuery || !sQuery[0] ) ) )
		return true;

	RtQword_t tQword;

	TokenizerRefPtr_c pTokenizer = m_pQueryTokenizer->Clone ( SPH_CLONE );
	pTokenizer->EnableTokenizedMultiformTracking ();

	// need to support '*' and '=' but not the other specials
	// so m_pQueryTokenizer does not work for us, gotta clone and setup one manually
	DictRefPtr_c pDict = GetStatelessDict ( m_pDict );
	const bool bWordDict = IsWordDict();

	if ( IsStarDict ( bWordDict ) )
	{
		SetupStarTokenizer ( pTokenizer );
		if ( bWordDict )
			SetupStarDict ( pDict );
	}

	if ( m_tSettings.m_bIndexExactWords )
	{
		SetupExactTokenizer ( pTokenizer );
		SetupExactDict ( pDict );
	}

	if ( !m_tSettings.m_sIndexTokenFilter.IsEmpty() )
	{
		CSphString sError;
		Tokenizer::AddPluginFilterTo ( pTokenizer, m_tSettings.m_sIndexTokenFilter, sError );
		if ( pError && !pError->IsEmpty() )
		{
			*pError = sError;
			return false;
		}
		if ( !pTokenizer->SetFilterSchema ( m_tSchema, sError ) )
		{
			if ( pError )
				*pError = sError;
			return false;
		}
	}

	const CSphString & sBlendMode = pTokenizer->GetSettings().m_sBlendMode;
	if ( !sBlendMode.IsEmpty() )
	{
		CSphString sError;
		if ( !pTokenizer->SetBlendMode ( sBlendMode.cstr(), sError ) )
		{
			if ( pError )
				*pError = sError;
			return false;
		}
	}

	CSphVector<BYTE> dFiltered;
	const BYTE * sModifiedQuery = (const BYTE *)sQuery;
	FieldFilterOptions_t tFFOptions { tSettings.m_eJiebaMode };

	if ( m_pFieldFilter && sQuery && m_pFieldFilter->Clone ( &tFFOptions )->Apply ( sModifiedQuery, dFiltered, true ) )
		sModifiedQuery = dFiltered.Begin();

	// FIXME!!! missing bigram
	bool bHasWildcards = false;

	if ( !bFillOnly )
	{
		ExpansionContext_t tExpCtx;

		// query defined options
		tExpCtx.m_iExpansionLimit = tSettings.m_iExpansionLimit ? tSettings.m_iExpansionLimit : m_iExpansionLimit;
		tExpCtx.m_bAllowExpansion = ( tSettings.m_bAllowExpansion && bWordDict && IsStarDict ( bWordDict ) );
		bool bExpandWildcards = ( tExpCtx.m_bAllowExpansion && !tSettings.m_bFoldWildcards );
		pTokenizer->SetBuffer ( sModifiedQuery, (int)strlen ( (const char*)sModifiedQuery ) );

		CSphRtQueryFilter tAotFilter ( this, &tQword, tGuard.m_dRamSegs );
		tAotFilter.m_pTokenizer = std::move ( pTokenizer );
		tAotFilter.m_pDict = std::move ( pDict );
		tAotFilter.m_pSettings = &m_tSettings;
		tAotFilter.m_tFoldSettings = tSettings;
		tAotFilter.m_tFoldSettings.m_bFoldWildcards = !bExpandWildcards;

		tExpCtx.m_pWordlist = this;
		tExpCtx.m_iMinPrefixLen = m_tSettings.GetMinPrefixLen ( bWordDict );
		tExpCtx.m_iMinInfixLen = m_tSettings.m_iMinInfixLen;
		tExpCtx.m_bHasExactForms = ( m_pDict->HasMorphology() || m_tSettings.m_bIndexExactWords );
		tExpCtx.m_bMergeSingles = false;
		tExpCtx.m_pIndexData = tGuard.m_tSegmentsAndChunks.m_pSegs;

		tAotFilter.GetKeywords ( dKeywords, tExpCtx );
		bHasWildcards = tExpCtx.m_bHasWildcards;
	} else
	{
		KeywordBuf_t dWord ( GetDictFormat() );

		ARRAY_FOREACH ( i, dKeywords )
		{
			CSphKeywordInfo & tInfo = dKeywords[i];
			int iLen = tInfo.m_sTokenized.Length();
			if ( iLen>=dWord.GetLength() )
				continue;

			memcpy ( dWord.Begin(), tInfo.m_sTokenized.cstr(), iLen );
			dWord.Begin()[iLen] = '\0';

			SphWordID_t iWord = pDict->GetWordID ( dWord.Begin() );
			if ( iWord )
			{
				tQword.Reset();
				tQword.m_uWordID = iWord;
				tQword.m_sWord = tInfo.m_sTokenized;
				tQword.m_sDictWord = (const char *)dWord.Begin();
				for ( const auto& pSeg : tGuard.m_dRamSegs )
					RtQwordSetupSegment ( &tQword, pSeg, false );

				tInfo.m_iDocs += tQword.m_iDocs;
				tInfo.m_iHits += tQword.m_iHits;
			}
		}
	}

	// process disk chunks too but only if:
	// - need term stats
	// - has terms with wildcards as these are expanded differently
	if ( !tSettings.m_bStats && !bHasWildcards )
		return true;

	if ( bFillOnly || tSettings.m_bFoldStatsToUnique )
	{
		for ( auto& pChunk : tGuard.m_dDiskChunks )
			pChunk->Cidx().FillKeywords ( dKeywords );
	} else
	{
		// bigram and expanded might differs need to merge infos
		int iWasKeywords = dKeywords.GetLength();
		CSphVector<CSphKeywordInfo> dChunkKeywords;
		for ( auto& pChunk: tGuard.m_dDiskChunks )
		{
			pChunk->Cidx().GetKeywords ( dChunkKeywords, (const char*)sModifiedQuery, tSettings, pError );
			dKeywords.Append ( dChunkKeywords );
			dChunkKeywords.Resize ( 0 );
		}

		// merge keywords from RAM parts with disk keywords
		if ( iWasKeywords!=dKeywords.GetLength() )
			UniqKeywords ( dKeywords, KeywordUniq_e::BY_NORMALIZED_AND_QPOS );
	}

	return true;
}


bool RtIndex_c::GetKeywords ( CSphVector<CSphKeywordInfo> & dKeywords, const char * sQuery, const GetKeywordsSettings_t & tSettings, CSphString * pError ) const
{
	auto tGuard = RtGuard();
	return DoGetKeywords ( dKeywords, sQuery, tSettings, false, pError, tGuard );
}


bool RtIndex_c::FillKeywords ( CSphVector<CSphKeywordInfo> & dKeywords ) const
{
	auto tGuard = RtGuard();
	return DoGetKeywords ( dKeywords, nullptr, GetKeywordsSettings_t(), true, nullptr, tGuard );
}


// for each RamSegment collect list of rows and indexes in update ctx
RowsToUpdateData_t CollectUpdatableRows ( UpdateContext_t & tCtx, const ConstRtSegmentRefPtf_t & tSegment ) noexcept
{
	RowsToUpdateData_t dUpdateSet;

	// collect idxes of alive (not-yet-updated) rows
	const auto & dDocids = tCtx.m_tUpd.m_pUpdate->m_dDocids;
	ARRAY_CONSTFOREACH ( i, dDocids )
	{
		if ( tCtx.m_tUpd.m_dUpdated.BitGet ( i ) )
			continue;

		auto tRowID = tSegment->GetAliveRowidByDocid ( dDocids[i] );
		if ( tRowID==INVALID_ROWID )
			continue;

		auto & dUpd = dUpdateSet.Add ();
		dUpd.m_tRow = tRowID;
		dUpd.m_iIdx = i;
	}
	return dUpdateSet;
}

// when RAM segment is used in merge or saving to disk - it's data finally became merged into new segment or chunk,
// and source segment then unlinked. If during that operation attributes was updated by aside 'update attributes' call,
// the changes will not participate in final result and so, will be discarded with source segment unlink.
// to deal with this, we set 'dirty' bit on segment to show that updates to it must be also collected and then applied
// to final resulting chunk/segment. That bit set before merging attributes and exists till the end of segment's lifetime.
// Here is first part of postponed merge - after update we collect docs updated in segment and store them into vec of
// updates (as it might happen be more than one update during the operation)
void RtSegment_t::MaybeAddPostponedUpdate ( const RowsToUpdate_t& dRows, const UpdateContext_t& tCtx )
{
	TRACE_CORO ( "rt", "IndexSegment_c::Update_UpdateAttributes");
	if ( !m_bAttrsBusy.load ( std::memory_order_acquire ) )
		return;

	// segment is now saving/merging - add postponed update.
	auto& tUpd = tCtx.m_tUpd;
	// count exact N of affected rows (no need to waste space for reserve in this route at all)
	auto iRows = dRows.count_of ( [&tUpd] ( auto& i ) { return tUpd.m_dUpdated.BitGet ( i.m_iIdx ); } );

	if ( !iRows )
		return;

	auto& tNewPostponedUpdate = m_dPostponedUpdates.Add();
	tNewPostponedUpdate.m_pUpdate = MakeReusableUpdate ( tUpd.m_pUpdate );
	tNewPostponedUpdate.m_dRowsToUpdate.Reserve ( iRows );

	// collect indexes of actually updated rows and docids
	dRows.for_each ( [&tUpd, &tNewPostponedUpdate] ( const auto& i ) {
		if ( tUpd.m_dUpdated.BitGet ( i.m_iIdx ) )
			tNewPostponedUpdate.m_dRowsToUpdate.Add().m_iIdx = i.m_iIdx;
	});
}

bool RtIndex_c::Update_DiskChunks ( AttrUpdateInc_t & tUpd, const DiskChunkSlice_t & dDiskChunks, CSphString & sError, CSphString & sWarning ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	TRACE_CORO ( "rt", "RtIndex_c::Update_DiskChunks" );
	assert ( Coro::CurrentScheduler() == m_tWorkers.SerialChunkAccess() );
	bool bCritical = false;

	bool bEnabled = m_tSaving.ActiveStateIs ( SaveState_c::ENABLED );
	bool bNeedWait = !bEnabled;

	for ( auto& pDiskChunk : dDiskChunks )
	{
		if ( tUpd.AllApplied () )
			break;

		pDiskChunk->m_iPendingUpdates.fetch_add ( 1, std::memory_order_relaxed );
		AT_SCOPE_EXIT ( [pDiskChunk] { pDiskChunk->m_iPendingUpdates.fetch_sub ( 1, std::memory_order_relaxed ); } );
		// acquire fine-grain lock
		BEGIN_CORO ( "wait", "disk-chunk w-lock");
		SccWL_t wLock ( pDiskChunk->m_tLock );
		END_CORO ( "wait" );

		// if saving is disabled, and we NEED to actually update a disk chunk,
		// we'll pause that action, waiting until index is unlocked.
		if ( bNeedWait )
		{
			bNeedWait = false;
			BEGIN_CORO ( "wait", "WaitEnabledOrShutdown" );
			bEnabled = m_tSaving.WaitEnabledOrShutdown ();
			END_CORO ( "wait" );
		}

		int iRes = bEnabled ? pDiskChunk->CastIdx().CheckThenUpdateAttributes ( tUpd, bCritical, sError, sWarning ) : -1;

		// FIXME! need to handle critical failures here (chunk is unusable at this point)
		assert ( !bCritical );
		if ( iRes<0 )
			return false;

		// update stats
		m_uDiskAttrStatus |= pDiskChunk->Cidx().GetAttributeStatus();
	}

	return true;
}

// thread-safe, as segment is locked up level before calling RAM segment update
bool RtSegment_t::Update_WriteBlobRow ( UpdateContext_t & tCtx, RowID_t tRowID, ByteBlob_t tBlob,
		int nBlobAttrs, const CSphAttrLocator & tBlobRowLoc, bool & bCritical, CSphString & sError ) NO_THREAD_SAFETY_ANALYSIS
{
	// fixme! Ensure pSegment->m_tLock acquired exclusively...
	auto pDocinfo = tCtx.GetDocinfo ( tRowID );
	BYTE* pExistingBlob = m_dBlobs.begin() + sphGetRowAttr ( pDocinfo, tBlobRowLoc );
	DWORD uExistingBlobLen = sphGetBlobTotalLen ( pExistingBlob, nBlobAttrs );

	bCritical = false;

	// overwrite old record
	if ( (DWORD)tBlob.second<=uExistingBlobLen )
	{
		memcpy ( pExistingBlob, tBlob.first, tBlob.second );
		return true;
	}

	int iPoolSize = m_dBlobs.GetLength();
	m_dBlobs.Append ( tBlob );

	sphSetRowAttr ( pDocinfo, tBlobRowLoc, iPoolSize );

	// update blob pool ptrs since they could have changed after the resize
	tCtx.m_pBlobPool = m_dBlobs.begin();
	return true;
}


// FIXME! might be inconsistent in case disk chunk update fails
int RtIndex_c::CheckThenUpdateAttributes ( AttrUpdateInc_t& tUpd, bool& bCritical, CSphString& sError, CSphString& sWarning )
{
	TRACE_CORO ( "rt", "CheckThenUpdateAttributes" );
	const auto& tUpdc = *tUpd.m_pUpdate;
	assert ( tUpdc.m_dRowOffset.IsEmpty() || tUpdc.m_dDocids.GetLength()==tUpdc.m_dRowOffset.GetLength() );

	if ( tUpdc.m_dDocids.IsEmpty()  || tUpd.AllApplied () )
		return 0;

	// FIXME!!! grab Writer lock to prevent segments retirement during commit(merge)
	if ( m_tRtChunks.IsEmpty() )
		return 0;

	int iUpdated = tUpd.m_uAffected;
	if ( !Update_CheckAttributes ( *tUpd.m_pUpdate, m_tSchema, sError, sWarning ) )
		return -1;

	UpdateContext_t tCtx ( tUpd, m_tSchema );
	tCtx.PrepareListOfUpdatedAttributes ( sError );

	// do update in serial fiber. That ensures no concurrency with set of chunks changing, however need to dispatch
	// with changers themselves (merge segments, merge chunks, save disk chunks).
	// fixme! Find another way (dedicated fiber?), as long op in serial fiber may pause another ops.
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	auto tGuard = RtGuard ();
//	PauseCheck("rtpause"); // catch if something happened between RtGuard() and actual updates
	TRACE_CORO ( "rt", "UpdateAttributes_serial" );
	for ( auto& dRamSegment : tGuard.m_dRamSegs )
	{
		auto dRamUpdateSet = CollectUpdatableRows ( tCtx, dRamSegment );
		if ( dRamUpdateSet.IsEmpty() )
			continue;

		auto* pSeg = const_cast<RtSegment_t*> ( (const RtSegment_t*) dRamSegment );
		BEGIN_CORO ( "wait", "ram-seg-wlock");
		SccWL_t wLock ( pSeg->m_tLock );
		END_CORO ( "wait" );

		assert ( pSeg->GetStride() == m_tSchema.GetRowSize() );

		// point context to target segment
		tCtx.m_pAttrPool = pSeg->m_dRows.begin();
		tCtx.m_pBlobPool = pSeg->m_dBlobs.begin();
		if ( !pSeg->Update_UpdateAttributes ( dRamUpdateSet, tCtx, bCritical, sError ) )
			return -1;

		pSeg->MaybeAddPostponedUpdate( dRamUpdateSet, tCtx );

		if ( tUpd.AllApplied () )
			break;
	}

	if ( !Update_DiskChunks ( tUpd, tGuard.m_dDiskChunks, sError, sWarning ) ) // fixme!
		sphWarn ( "INTERNAL ERROR: table %s update failure: %s", GetName(), sError.cstr() );

	// bump the counter, binlog the update!
	CommitUpdateAttributes ( &m_iTID, GetName(), tUpdc );

	iUpdated = tUpd.m_uAffected - iUpdated;

	if ( !tCtx.HandleWarnings ( iUpdated, sWarning, sError ) )
		return -1;

	// all done
	return iUpdated;
}


bool RtIndex_c::SaveAttributes ( CSphString & sError ) const
{
	DWORD uStatus = m_uDiskAttrStatus;
	bool bAllSaved = true;

	const auto& pDiskChunks = m_tRtChunks.DiskChunks();

	if ( pDiskChunks->IsEmpty() || m_tSaving.ActiveStateIs ( SaveState_c::DISCARD ) )
		return true;

	for ( auto& pChunk : *pDiskChunks )
		bAllSaved &= pChunk->Cidx().SaveAttributes ( sError );

	if ( uStatus==m_uDiskAttrStatus )
		m_uDiskAttrStatus = 0;

	return bAllSaved;
}

// fixme! is NOT lru-safe, ensure index locked exclusively!
// (last is true as it w-locked before 'alter' called, however optimize call need special glance!)
bool RtIndex_c::AddRemoveColumnarAttr ( RtGuard_t & tGuard, bool bAdd, const CSphString & sAttrName, ESphAttr eAttrType, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema, CSphString & sError )
{
	for ( auto& pConstSeg : tGuard.m_dRamSegs )
	{
		auto* pSeg = const_cast<RtSegment_t*> ( pConstSeg.Ptr() );
		assert ( pSeg );
		auto pBuilder = CreateColumnarBuilderRT ( tNewSchema );

		if ( !Alter_AddRemoveColumnar ( bAdd, tOldSchema, tNewSchema, pSeg->m_pColumnar.get(), pBuilder.get(), pSeg->m_uRows, GetFilebase(), sError ) )
			return false;

		if ( m_pDocstoreFields && m_pDocstoreFields->GetFieldId ( sAttrName, DOCSTORE_ATTR )!=-1 )
			m_pDocstoreFields->RemoveField ( sAttrName, DOCSTORE_ATTR );

		pSeg->m_pColumnar = CreateColumnarRT ( tNewSchema, pBuilder.get() );
		pSeg->UpdateUsedRam();
	}

	return true;
}

// fixme! is NOT lru-safe, ensure index locked exclusively!
// (last is true as it w-locked before 'alter' called, however optimize call need special glance!)
void RtIndex_c::AddRemoveRowwiseAttr ( RtGuard_t & tGuard, bool bAdd, const CSphString & sAttrName, ESphAttr eAttrType, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema, CSphString & sError )
{
	bool bHadBlobs = false;
	for ( int i = 0; i < tOldSchema.GetAttrsCount() && !bHadBlobs; ++i )
		bHadBlobs |= sphIsBlobAttr ( tOldSchema.GetAttr(i) );

	bool bHaveBlobs = false;
	for ( int i = 0; i < tNewSchema.GetAttrsCount() && !bHaveBlobs; ++i )
		bHaveBlobs |= sphIsBlobAttr ( tNewSchema.GetAttr(i) );

	bool bBlob = sphIsBlobAttr(eAttrType);
	bool bBlobsModified = bBlob && ( bAdd || bHaveBlobs==bHadBlobs );

	// now modify the ramchunk
	for ( auto& pRSeg : tGuard.m_dRamSegs )
	{
		assert ( pRSeg );

		CSphTightVector<CSphRowitem> dSPA;
		CSphTightVector<BYTE> dSPB;
		std::unique_ptr<WriteWrapper_c> pSPAWriteWrapper = CreateWriteWrapperMem ( dSPA );
		std::unique_ptr<WriteWrapper_c> pSPBWriteWrapper = CreateWriteWrapperMem ( dSPB );
		dSPA.Reserve ( (int64_t)pRSeg->m_uRows * m_iStride );

		auto * pWSeg = const_cast<RtSegment_t*> ( pRSeg.Ptr() );
		SccWL_t _ (pWSeg->m_tLock);

		const CSphRowitem* pDocinfo = pWSeg->m_dRows.begin();
		dSPB.Reserve ( pWSeg->m_dBlobs.GetLength() / 2 );		// reserve half of our current blobs, just in case
		if ( !Alter_AddRemoveRowwiseAttr ( tOldSchema, tNewSchema, pDocinfo, pRSeg->m_uRows, pWSeg->m_dBlobs.begin(), *pSPAWriteWrapper, *pSPBWriteWrapper, bAdd, sAttrName ) )
			sphWarning ( "%s attribute to %s: %s", bAdd ? "adding" : "removing", GetFilebase(), sError.cstr() );
		pWSeg->m_dRows.SwapData(dSPA);
		if ( bBlob || bBlobsModified )
			pWSeg->m_dBlobs.SwapData(dSPB);

		pRSeg->UpdateUsedRam();
	}
}

// fixme! Need fine-grain locking, not const_cast!
void RtIndex_c::AddFieldToRamchunk ( const CSphString & sFieldName, DWORD uFieldFlags, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema )
{
	if ( !(uFieldFlags & CSphColumnInfo::FIELD_STORED) )
		return;

	if ( !m_pDocstoreFields )
		m_pDocstoreFields = CreateDocstoreFields();

	assert ( m_pDocstoreFields );
	m_pDocstoreFields->AddField ( sFieldName, DOCSTORE_TEXT );
	AddRemoveFromRamDocstore ( tOldSchema, tNewSchema );
}


static int GetNumStored ( const CSphSchema & tSchema )
{
	int iStored = 0;
	for ( int i = 0; i < tSchema.GetFieldsCount(); i++ )
		if ( tSchema.IsFieldStored(i) )
			iStored++;

	for ( int i = 0; i < tSchema.GetAttrsCount(); i++ )
		if ( tSchema.IsAttrStored(i) )
			iStored++;

	return iStored;
}


void RtIndex_c::AddRemoveFromRamDocstore ( const CSphSchema & tOldSchema, const CSphSchema & tNewSchema )
{
	int iOldStored = GetNumStored(tOldSchema);
	int iNewStored = GetNumStored(tNewSchema);

	if ( iOldStored==iNewStored )
		return;

	bool bLastStored = iNewStored<iOldStored && ( !tNewSchema.HasStoredFields() && !tNewSchema.HasStoredAttrs() );

	auto pSegs = m_tRtChunks.RamSegs();
	for ( auto & pConstSeg : *pSegs )
	{
		auto * pSeg = const_cast<RtSegment_t*> ( pConstSeg.Ptr() );

		if ( bLastStored )
			pSeg->m_pDocstore.reset();
		else
		{
			auto pNewDocstore = CreateDocstoreRT();
			Alter_AddRemoveFromDocstore ( *pNewDocstore, pSeg->m_pDocstore.get(), pSeg->m_uRows, tNewSchema );
			pSeg->m_pDocstore = std::move ( pNewDocstore );
		}

		pSeg->UpdateUsedRam();
	}
}


void RtIndex_c::RemoveFieldFromRamchunk ( const CSphString & sFieldName, const CSphSchema & tOldSchema, const CSphSchema & tNewSchema )
{
	if ( m_pDocstoreFields && m_pDocstoreFields->GetFieldId ( sFieldName, DOCSTORE_TEXT )!=-1 )
		m_pDocstoreFields->RemoveField ( sFieldName, DOCSTORE_TEXT );

	int iFieldId = tOldSchema.GetFieldIndex ( sFieldName.cstr () );
	auto pSegs = m_tRtChunks.RamSegs();
	for ( auto & pConstSeg : *pSegs )
	{
		auto* pSeg = const_cast<RtSegment_t*> ( pConstSeg.Ptr() );
		assert ( pSeg );
		DeleteFieldFromDict ( pSeg, iFieldId );
	}

	AddRemoveFromRamDocstore ( tOldSchema, tNewSchema );
}


bool RtIndex_c::AddRemoveField ( bool bAdd, const CSphString & sFieldName, DWORD uFieldFlags, CSphString & sError )
{
	OptimizeGuard_c tStopOptimize ( *this );

	// go to serial fiber.
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "AddRemoveField" );

	CSphSchema tOldSchema = m_tSchema;
	CSphSchema tNewSchema = m_tSchema;

	if ( !Alter_AddRemoveFieldFromSchema ( bAdd, tNewSchema, sFieldName, uFieldFlags, sError ) )
		return false;

	m_tSchema = tNewSchema;

	auto tGuard = RtGuard();

	// modify the in-memory data of disk chunks
	// fixme: we can't rollback in-memory changes, so we just show errors here for now
	for ( auto& pChunk : tGuard.m_dDiskChunks )
		if ( !pChunk->CastIdx().AddRemoveField ( bAdd, sFieldName, uFieldFlags, sError ) )
			sphWarning ( "%s attribute to %s.%d: %s", bAdd ? "adding" : "removing", GetFilebase(), pChunk->Cidx().m_iChunk, sError.cstr() );

	if ( bAdd )
		AddFieldToRamchunk ( sFieldName, uFieldFlags, tOldSchema, tNewSchema );
	else
		RemoveFieldFromRamchunk ( sFieldName, tOldSchema, tNewSchema );

	// fixme: we can't rollback at this point
	RaiseAlterGeneration();
	AlterSave ( true );

	return true;
}

void RtIndex_c::AlterSave ( bool bSaveRam )
{
	if ( bSaveRam )
	{
		Verify ( SaveRamChunk () );
	}

	SaveMeta ();

	// fixme: notify that it was ALTER that caused the flush
	Binlog::NotifyIndexFlush ( m_iTID, GetName(), Binlog::NoShutdown, Binlog::NoSave );

	QcacheClearByIndexId ( GetIndexId() );
}

static bool IsModelBackedEmbedding ( const AttrAddRemoveCtx_t & tCtx );

bool RtIndex_c::AddRemoveAttribute ( bool bAdd, const AttrAddRemoveCtx_t & tCtx, CSphString & sError )
{
	if ( !m_tRtChunks.DiskChunks()->IsEmpty() && !m_tSchema.GetAttrsCount() )
	{
		sError = "table must already have attributes";
		return false;
	}

	// here must be exclusively LOCKED access, we don't rely upon the topmost lock and go isolated ourselves

	// stop all optimize tasks
	OptimizeGuard_c tStopOptimize ( *this );

	// go to serial fiber.
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "AddRemoveAttribute" );

	// wait all secondary service tasks (merge segments, save disk chunk) to finish and release all the segments.
	WaitRAMSegmentsUnlocked();

	// as we're in serial, here all index data exclusively belongs to us. No new commits, merges, flushes, etc. until
	// we're finished.

	AttrAddRemoveCtx_t tNewCtx = tCtx;
	AttrEngine_e eAttrEngine = CombineEngines ( m_tSettings.m_eEngine, tCtx.m_eEngine );
	if ( eAttrEngine==AttrEngine_e::COLUMNAR )
		tNewCtx.m_uFlags |= CSphColumnInfo::ATTR_COLUMNAR;
	else
		tNewCtx.m_uFlags &= ~( CSphColumnInfo::ATTR_COLUMNAR_HASHES | CSphColumnInfo::ATTR_STORED );

	CSphSchema tOldSchema = m_tSchema;
	CSphSchema tNewSchema = m_tSchema;
	if ( !Alter_AddRemoveFromSchema ( tNewSchema, tNewCtx, bAdd, sError ) )
		return false;

	auto tGuard = RtGuard();

	std::unique_ptr<TableEmbeddings_c> pPreparedEmbeddings;
	CSphVector<AttrWithModel_t> dPreparedAttrsWithModels;

	const bool bModelBackedEmbedding = ( bAdd && IsModelBackedEmbedding ( tNewCtx ) );
	if ( bModelBackedEmbedding )
	{
		if ( !PrepareEmbeddingModelsForSchema ( tNewSchema, pPreparedEmbeddings, dPreparedAttrsWithModels, sError ) )
			return false;

		const CSphColumnInfo * pAttr = tNewSchema.GetAttr ( tNewCtx.m_sName.cstr() );
		assert ( pAttr );
		if ( !pAttr )
		{
			sError.SetSprintf ( "attribute '%s' not found", tNewCtx.m_sName.cstr() );
			return false;
		}

		tNewCtx.m_tKNN = pAttr->m_tKNN;
		tNewCtx.m_tKNNModel = pAttr->m_tKNNModel;
		tNewCtx.m_sKNNFrom = pAttr->m_sKNNFrom;
	}

	m_tSchema = tNewSchema;
	m_iStride = m_tSchema.GetRowSize();

	// modify the in-memory data of disk chunks
	// fixme: we can't rollback in-memory changes, so we just show errors here for now
	for ( auto& pChunk : tGuard.m_dDiskChunks )
		if ( !pChunk->CastIdx().AddRemoveAttribute ( bAdd, tNewCtx, sError ) )
			sphWarning ( "%s attribute to %s.%d: %s", bAdd ? "adding" : "removing", GetFilebase(), pChunk->Cidx().m_iChunk, sError.cstr() );

	bool bColumnar = bAdd ? tNewSchema.GetAttr ( tNewCtx.m_sName.cstr() )->IsColumnar() : tOldSchema.GetAttr ( tNewCtx.m_sName.cstr() )->IsColumnar();
	if ( bColumnar )
	{
		if ( !AddRemoveColumnarAttr ( tGuard, bAdd, tNewCtx.m_sName, tNewCtx.m_eType, tOldSchema, tNewSchema, sError ) )
			return false;
	} else
		AddRemoveRowwiseAttr ( tGuard, bAdd, tNewCtx.m_sName, tNewCtx.m_eType, tOldSchema, tNewSchema, sError );

	AddRemoveFromRamDocstore ( tOldSchema, tNewSchema );

	if ( bModelBackedEmbedding )
	{
		m_pEmbeddings = std::move ( pPreparedEmbeddings );
		m_dAttrsWithModels.SwapData ( dPreparedAttrsWithModels );
	}

	// fixme: we can't rollback at this point
	RaiseAlterGeneration();
	AlterSave ( true );
	return true;
}


//////////////////////////////////////////////////////////////////////////
// MAGIC CONVERSIONS
//////////////////////////////////////////////////////////////////////////

// ClientSession_c::Execute->HandleMysqlAttach->AttachDiskIndex
bool RtIndex_c::AttachDiskIndex ( CSphIndex * pIndex, bool bTruncate, bool & bFatal, CSphString & sError )
{
	// from the next line we work in index simple scheduler. That made everything much simpler
	// (no need to care about locks and order of access to ram segments and disk chunks)
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "AttachDiskIndex" );

	bFatal = false;

	if ( bTruncate && !Truncate ( sError, TRUNCATE ) )
		return false;

	// safeguards
	// we do not support some disk index features in RT just yet
	if ( !CanAttach ( pIndex, true, sError ) )
		return false;

	// note: that is important. Active readers prohibited by topmost w-lock, but internal processes not!
	// fixme! Looks like a dupe, look at few lines above
	if ( bTruncate && !Truncate ( sError, TRUNCATE ) )
		return false;

	// attach to non-empty RT: first flush RAM segments to disk chunk, then apply upcoming index'es docs as k-list.
	if ( !m_tRtChunks.RamSegs()->IsEmpty() && !SaveDiskChunk ( true ) )
		return false;

	if ( !AttachDiskChunkMove ( pIndex, bFatal, sError ) )
		return false;

	// FIXME? what about copying m_TID etc?

	{	// update disk chunk list
		auto tNewSet = RtWriter();
		CopyChunksTo ( tNewSet );
		tNewSet.m_pNewDiskChunks->Add ( DiskChunk_c::make ( pIndex ) );
	}

	AttachSetSettings ( pIndex );
	PostSetup();

	// resave header file
	SaveMeta();

	// FIXME? do something about binlog too?
	// Binlog::NotifyIndexFlush ( GetName(), m_iTID, false );

	// all done, reset cache
	QcacheClearByIndexId ( GetIndexId() );
	QcacheClearByIndexId ( pIndex->GetIndexId() );
	return true;
}

bool RtIndex_c::CanAttach ( const CSphIndex * pIndex, bool bCheckFT, CSphString & sError ) const
{
	// ATTACH to exist index require these checks
	if ( !m_tRtChunks.IsEmpty() )
	{
		if ( bCheckFT && m_pTokenizer->GetSettingsFNV()!=pIndex->GetTokenizer()->GetSettingsFNV() )
		{
			sError = "ATTACH currently requires same tokenizer settings (RT-side support not implemented yet)";
			return false;
		}

		if ( bCheckFT && m_pDict->GetSettingsFNV()!=pIndex->GetDictionary()->GetSettingsFNV() )
		{
			sError = "ATTACH currently requires same dictionary settings (RT-side support not implemented yet)";
			return false;
		}

		if ( !GetMatchSchema().CompareTo ( pIndex->GetMatchSchema(), sError, true ) )
		{
			sError = "ATTACH currently requires same attributes declaration (RT-side support not implemented yet)";
			return false;
		}
	}

	return true;
}

bool RtIndex_c::AttachDiskChunkMove ( CSphIndex * pIndex, bool & bFatal, CSphString & sError ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	int iTotalKilled = 0;
	DWORD uAliveDocuments = 0;

	// attach to non-empty RT: apply upcoming index'es docs as k-list.
	if ( !m_tRtChunks.IsEmpty() )
	{
		auto dIndexDocs = pIndex->BuildDocList();
		if ( TlsMsg::HasErr () )
		{
			sError.SetSprintf ( "ATTACH failed, %s", TlsMsg::szError () );
			return false;
		}

		uAliveDocuments = dIndexDocs.GetLength();
		dIndexDocs.Uniq();
		iTotalKilled = ApplyKillList ( dIndexDocs );
	} else
		uAliveDocuments = pIndex->GetStats().m_iTotalDocuments;

	// rename that source index to our last chunk
	int iChunk = m_tChunkID.MakeChunkId ( m_tRtChunks );
	auto eRenamed = pIndex->RenameEx ( GetFilename ( iChunk ) );
	switch (eRenamed)
	{
	case RE_FATAL: // not just failed, but also rollback wasn't success. Source index is damaged!
		bFatal = true;
		// no break;
	case RE_FAIL:
		sError.SetSprintf ( "ATTACH failed, %s", pIndex->GetLastError().cstr() );
		return false;
	default: break;
	}

	m_tStats.m_iTotalBytes += pIndex->GetStats().m_iTotalBytes;
	m_tStats.m_iTotalDocuments += uAliveDocuments-iTotalKilled;

	pIndex->SetName ( SphSprintf ( "%s_%d", GetName(), iChunk ) ); // idx name is cosmetic thing
	pIndex->SetBinlog ( false );
	pIndex->m_iChunk = iChunk;

	return true;
}

void RtIndex_c::AttachSetSettings ( CSphIndex * pIndex )
{
	// copy schema from new index
	SetSchema ( pIndex->GetMatchSchema() );

	// copy tokenizer, dict etc settings from new index
	m_tSettings = pIndex->GetSettings();
	m_tSettings.m_dBigramWords.Reset();

	m_pTokenizer = pIndex->GetTokenizer()->Clone ( SPH_CLONE_INDEX );
	m_pDict = pIndex->GetDictionary()->Clone ();
}

static bool AttachCopyExt ( const CSphIndex & tSrcIndex, RtIndex_i & tDstIndex, ExtFiles_h & hExtCache, CSphString & sError );

bool RtIndex_c::AttachRtIndex ( AttachArgs_t & tArgs, CSphString & sError )
{
	// from the next line we work in index simple scheduler. That made everything much simpler
	// (no need to care about locks and order of access to ram segments and disk chunks)
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "AttachDiskIndex" );

	RtIndex_i * pSrcIndex = tArgs.m_pSrcIndex;
	bool bTruncate = tArgs.m_bTruncate;
	bool bConfigless = tArgs.m_bConfigless;

	assert ( pSrcIndex );
	tArgs.m_bFatal = false;

	if ( bTruncate && !Truncate ( sError, TRUNCATE ) )
		return false;

	// safeguards
	// we do not support some disk index features in RT just yet
	if ( !CanAttach ( pSrcIndex, !bConfigless, sError ) )
		return false;

	// note: that is important. Active readers prohibited by topmost w-lock, but internal processes not!
	// fixme! Looks like a dupe, look at few lines above
	if ( bTruncate && !Truncate ( sError, TRUNCATE ) )
		return false;

	// attach to non-empty RT: first flush RAM segments to disk chunk, then apply upcoming index'es docs as k-list.
	if ( !m_tRtChunks.RamSegs()->IsEmpty() && !SaveDiskChunk ( true ) )
		return false;

	auto * pSrcRtIndex = static_cast<RtIndex_c *>( pSrcIndex );
	if ( !pSrcRtIndex->AttachSaveDiskChunk() )
		return false;

	StrVec_t dOldSrcFiles, dNewSrcFiles, dOldDstFiles, dNewDstFiles;
	ExtFiles_h hExtCache { 32 };

	if ( bConfigless )
	{
		pSrcIndex->GetIndexFiles ( dOldSrcFiles, dOldSrcFiles );
		GetIndexFiles ( dOldDstFiles, dOldDstFiles );

		if ( !AttachRtChunksExtCopy ( pSrcRtIndex, tArgs.m_bFatal, hExtCache, sError ) )
			return false;
	} else
	{
		if ( !AttachRtChunksNoExtCopy ( pSrcRtIndex, tArgs.m_bFatal, sError ) )
			return false;
	}

	AttachSetSettings ( pSrcIndex );
	if ( bConfigless )
	{
		if ( !AttachCopyExt ( *pSrcIndex, *this, hExtCache, sError ) )
			return false;

		pSrcIndex->GetIndexFiles ( dNewSrcFiles, dNewSrcFiles );
		GetIndexFiles ( dNewDstFiles, dNewDstFiles );
		RemoveOutdatedFiles ( dNewSrcFiles, dOldSrcFiles );
		RemoveOutdatedFiles ( dNewDstFiles, dOldDstFiles );
	}

	PostSetup();

	// FIXME? what about copying m_TID etc?
	// resave header file
	m_iTID = Max ( m_iTID, pSrcRtIndex->m_iTID );
	pSrcRtIndex->SaveMeta();
	SaveMeta();

	// FIXME? do something about binlog too?
	// Binlog::NotifyIndexFlush ( GetName(), m_iTID, false );

	// all done, reset cache
	QcacheClearByIndexId ( GetIndexId() );
	QcacheClearByIndexId ( pSrcIndex->GetIndexId() );
	return true;
}

bool RtIndex_c::AttachSaveDiskChunk()
{
	if ( m_tRtChunks.RamSegs()->IsEmpty() )
		return true;

	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	return SaveDiskChunk ( true );
}

ConstDiskChunkRefPtr_t RtIndex_c::PopDiskChunk()
{
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );

	auto tNewSet = RtWriter();
	CopyChunksTo ( tNewSet );
	if ( tNewSet.m_pNewDiskChunks->IsEmpty() )
		return {};

	auto pHeadChunk = tNewSet.m_pNewDiskChunks->First();
	tNewSet.m_pNewDiskChunks->Remove(0);

	assert ( pHeadChunk );
	const auto iCount = pHeadChunk->Cidx().GetCount();
	m_tStats.m_iTotalDocuments -= iCount;
	return pHeadChunk;
}

//////////////////////////////////////////////////////////////////////////
// TRUNCATE
//////////////////////////////////////////////////////////////////////////

// kill RAM chunk file
void RtIndex_c::UnlinkRAMChunk ( const char* szInfo )
{
	CSphString sFile = GetFilename ( "ram" );
	if ( ::unlink ( sFile.cstr() ) && errno != ENOENT && szInfo )
		sphWarning ( "rt: %s failed to unlink %s: (errno=%d, error=%s)", szInfo, sFile.cstr(), errno, strerrorm ( errno ) );
}

bool RtIndex_c::Truncate ( CSphString&, Truncate_e eAction )
{
	// TRUNCATE will drop everything; so all 'optimizing' should be discarded as useless
	OptimizeGuard_c tStopOptimize ( *this );

	// do truncate in serial fiber. As it is re-enterable, don't care if we already there.
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "Truncate" );

	StrVec_t dOldExternalFiles;
	if ( eAction==TRUNCATE )
		dOldExternalFiles = CollectOwnedExternalFiles();

	// update and save meta
	// indicate 0 disk chunks, we are about to kill them anyway
	// current TID will be saved, so replay will properly skip preceding txns
	m_tStats.Reset();
	SaveMeta ( m_iTID, { nullptr, 0 } );

	// allow binlog to unlink now-redundant data files
	Binlog::NotifyIndexFlush ( m_iTID, GetName(), Binlog::NoShutdown, eAction==TRUNCATE ? Binlog::ForceSave : Binlog::DropTable );

	// kill RAM chunk file
	UnlinkRAMChunk ( "truncate" );

	// kill all disk chunks files
	auto pChunks = m_tRtChunks.DiskChunks();
	pChunks->for_each ( [] ( ConstDiskChunkRefPtr_t& t ) { t->m_bFinallyUnlink = true; } );

	{
		// remove all chunks and segments
		auto tChangeset = RtWriter();
		tChangeset.InitRamSegs ( RtWriter_c::empty );
		tChangeset.InitDiskChunks ( RtWriter_c::empty );
	}

	if ( eAction==TRUNCATE )
		RemoveOutdatedOwnedExternalFiles ( dOldExternalFiles );

	// reset cache
	QcacheClearByIndexId ( GetIndexId() );
	return true;
}

void RtIndex_c::SetKillHookFor ( IndexSegment_c* pAccum, int iDiskChunkID ) const
{
	// that ensures no concurrency
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	ProcessDiskChunkByID ( iDiskChunkID, [pAccum] ( const DiskChunk_c* p ) { p->Cidx().SetKillHook ( pAccum ); } );
}

void RtIndex_c::SetKillHookFor ( IndexSegment_c* pAccum, VecTraits_T<int> dDiskChunkIDs ) const
{
	// that ensures no concurrency
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	ProcessDiskChunkByID ( dDiskChunkIDs, [pAccum] ( const DiskChunk_c* p ) { p->Cidx().SetKillHook ( pAccum ); } );
}

// track internals of merge to catch critical points necessary for concurrent kill and update
class RTMergeCb_c final: public MergeCb_c
{
	KillAccum_t m_tKilledWhileMerge;
	RtIndex_c* m_pOwner;
	CSphVector<int> m_dTrackedChunks;

	const DiskChunk_c* m_pChunk = nullptr;
	int64_t m_iLastPayload = -1;

public:
	NONCOPYMOVABLE ( RTMergeCb_c );
	RTMergeCb_c ( std::atomic<bool>* pStop, RtIndex_c* pOwner )
		: MergeCb_c ( pStop )
		, m_pOwner ( pOwner )
	{}

	void SetEvent ( Event_e eEvent, int64_t iPayload ) final NO_THREAD_SAFETY_ANALYSIS
	{
		assert ( m_pOwner );
		RTLOGV << "SetEvent (" << eEvent << ", " << iPayload << ")";
		switch ( eEvent )
		{
		case E_COLLECT_START:
			m_dTrackedChunks.Add ( (int)iPayload );
			m_pOwner->SetKillHookFor ( &m_tKilledWhileMerge, (int)iPayload );
			break;
		case E_MERGEATTRS_START: // enter serial state/rlock
			m_iLastPayload = iPayload;
			m_pOwner->ProcessDiskChunkByID ( (int)iPayload, [this] ( const DiskChunk_c* p ) { m_pChunk = p; } );
			m_pChunk->m_tLock.ReadLock ();
			m_pChunk->CastIdx().m_bAttrsBusy.store ( true, std::memory_order_release );
			break;
		case E_MERGEATTRS_PULSE: // inside serial state/rlock
#ifndef NDEBUG
			assert ( m_iLastPayload==iPayload );
			m_pOwner->ProcessDiskChunkByID ( (int)iPayload, [this] ( const DiskChunk_c* p ) { assert ( m_pChunk == p); } );
#endif
			if ( m_pChunk && m_pChunk->m_iPendingUpdates.load ( std::memory_order_relaxed )>0 && m_pChunk->m_tLock.TestNextWlock() )
			{
				m_pChunk->m_tLock.Unlock (); // pulse lock that update can catch
				Threads::Coro::Reschedule();
				m_pChunk->m_tLock.ReadLock ();
			}
			break;
		case E_MERGEATTRS_FINISHED: // leave serial state/rlock
#ifndef NDEBUG
			assert ( m_iLastPayload==iPayload );
			m_pOwner->ProcessDiskChunkByID ( (int)iPayload, [this] ( const DiskChunk_c* p ) { assert ( m_pChunk == p); } );
#endif
			m_pChunk->m_tLock.Unlock ();
			m_iLastPayload = -1;
			m_pChunk = nullptr;
			break;
		default:
			break;
		}
	}

	~RTMergeCb_c() final
	{
		assert ( m_pOwner );
		m_pOwner->SetKillHookFor ( nullptr, m_dTrackedChunks );
	}

	CSphVector<DocID_t> GetKilled()
	{
		CSphVector<DocID_t> dDocs;
		m_tKilledWhileMerge.m_dDocids.SwapData ( dDocs );
		dDocs.Uniq();
		return dDocs;
	}

	bool HasKilled() const
	{
		return !m_tKilledWhileMerge.m_dDocids.IsEmpty();
	}
};

static StringBuilder_c & operator<< ( StringBuilder_c & dOut, MergeCb_c::Event_e eVal )
{
	switch ( eVal )
	{
	case MergeCb_c::E_IDLE: return dOut << "E_IDLE";
	case MergeCb_c::E_COLLECT_START: return dOut << "E_COLLECT_START";
	case MergeCb_c::E_COLLECT_FINISHED: return dOut << "E_COLLECT_FINISHED";
	case MergeCb_c::E_MERGEATTRS_START: return dOut << "E_MERGEATTRS_START";
	case MergeCb_c::E_MERGEATTRS_FINISHED: return dOut << "E_MERGEATTRS_FINISHED";
	case MergeCb_c::E_KEYWORDS: return dOut << "E_KEYWORDS";
	case MergeCb_c::E_FINISHED: return dOut << "E_FINISHED";
	default: dOut.Sprintf ( "UNKNWN_", (int)eVal );
	}
	return dOut;
}

static StringBuilder_c& operator<< ( StringBuilder_c& dOut, OptimizeTask_t::OptimizeVerb_e eVerb )
{
	switch ( eVerb )
	{
	case OptimizeTask_t::eManualOptimize: return dOut << "eManualOptimize";
	case OptimizeTask_t::eDrop: return dOut << "eDrop";
	case OptimizeTask_t::eCompress: return dOut << "eCompress";
	case OptimizeTask_t::eSplit: return dOut << "eSplit";
	case OptimizeTask_t::eMerge: return dOut << "eMerge";
	case OptimizeTask_t::eAutoOptimize: return dOut << "eAutoOptimize";
	default: dOut.Sprintf ( "eUnknown_", (int)eVerb );
	}
	return dOut;
}

static StringBuilder_c& operator<< ( StringBuilder_c& dOut, const OptimizeTask_t& tTask )
{

	switch ( tTask.m_eVerb )
	{
	case OptimizeTask_t::eManualOptimize: return dOut << "eManualOptimize, cutoff = " << tTask.m_iCutoff;
	case OptimizeTask_t::eDrop: return dOut << "eDrop " << tTask.m_iFrom;
	case OptimizeTask_t::eCompress: return dOut << "eCompress " << tTask.m_iFrom;
	case OptimizeTask_t::eSplit: return dOut << "eSplit " << tTask.m_iFrom << " filtering with '" << tTask.m_sUvarFilter << "'";
	case OptimizeTask_t::eMerge: return dOut << "eMerge from " << tTask.m_iFrom << " to " << tTask.m_iTo;
	case OptimizeTask_t::eAutoOptimize: return dOut << "eAutoOptimize, cutoff = " << tTask.m_iCutoff;
	default: return dOut << tTask.m_eVerb << " cutoff=" << tTask.m_iCutoff << " From=" << tTask.m_iFrom << " To=" << tTask.m_iTo << " VarFilter= '" << tTask.m_sUvarFilter << "' bByOrder=" << tTask.m_bByOrder;
	}
}

//////////////////////////////////////////////////////////////////////////
// OPTIMIZE
//////////////////////////////////////////////////////////////////////////

static int64_t GetEffectiveSize (const CSphIndexStatus& tStatus, int64_t iTotalDocs )
{
	if ( !tStatus.m_iDead ) // no docs killed
		return tStatus.m_iDiskUse;

	if ( tStatus.m_iDead==iTotalDocs ) // all docs killed
		return 0;

	// has killed docs: use naive formulae, simple part of used disk proportional to partial of alive docs
	auto fPart = double ( tStatus.m_iDead ) / double ( iTotalDocs );
	return int64_t ( tStatus.m_iDiskUse * ( 1.0f-fPart ) );
}

static int64_t GetChunkSize ( const CSphIndex& tIndex )
{
	CSphIndexStatus tDisk;
	tIndex.GetStatus ( &tDisk );
	return GetEffectiveSize ( tDisk, tIndex.GetStats().m_iTotalDocuments );
}

struct ChunkAndSize_t
{
	int m_iId;
	int64_t m_iSize;
};

static CSphVector<ChunkAndSize_t> GetSmallestChunksByID ( const DiskChunkVec_c& dDiskChunks )
{
	CSphVector<ChunkAndSize_t> dResult;
	dResult.Reserve ( dDiskChunks.GetLength() );
	for ( const auto& pDiskChunk : dDiskChunks )
	{
		if ( pDiskChunk->m_bOptimizing.load(std::memory_order_relaxed) )
			continue;
		const CSphIndex& dDiskChunk = pDiskChunk->Cidx();
		const int iChunk = dDiskChunk.m_iChunk;
		const int64_t iSize = GetChunkSize ( dDiskChunk );
		dResult.Add ( { iChunk, iSize } );
	}
	dResult.Sort ( Lesser ( [] ( const auto& a, const auto& b ) { return a.m_iSize < b.m_iSize; } ) );
	return dResult;
}

static int GetNumOfOptimizingNow ( const DiskChunkVec_c& dDiskChunks )
{
	return (int)dDiskChunks.count_of ( [] ( auto& i ) { return i->m_bOptimizing.load ( std::memory_order_relaxed ); } );
}

int RtIndex_c::ChunkIDByChunkIdx ( int iChunkIdx ) const
{
	if ( iChunkIdx < 0 )
		return iChunkIdx;

	return m_tRtChunks.DiskChunks()->operator[] ( iChunkIdx )->Cidx().m_iChunk;
}


int64_t	RtIndex_c::GetCountDistinct ( const CSphString & sAttr, CSphString & sModifiedAttr ) const
{
	// fixme! add code to calculate distinct values in RAM segments
	if ( m_tRtChunks.GetRamSegmentsCount() )
		return -1;

	auto pDiskChunks = m_tRtChunks.DiskChunks();
	if ( !pDiskChunks || pDiskChunks->GetLength()!=1 )
		return -1;

	return (*pDiskChunks)[0]->Cidx().GetCountDistinct ( sAttr, sModifiedAttr );
}


int64_t RtIndex_c::GetCountFilter ( const CSphFilterSettings & tFilter, CSphString & sModifiedAttr ) const
{
	// fixme! add code to calculate count(*) in RAM segments
	if ( m_tRtChunks.GetRamSegmentsCount() )
		return -1;

	auto pDiskChunks = m_tRtChunks.DiskChunks();
	if ( !pDiskChunks || !pDiskChunks->GetLength() )
		return 0;

	int64_t iSumCount = 0;
	for ( const auto & i : *pDiskChunks )
	{
		int64_t iCount = i->Cidx().GetCountFilter ( tFilter, sModifiedAttr );
		if ( iCount==-1 )
			return -1;

		iSumCount += iCount;
	}

	return iSumCount;
}


int64_t RtIndex_c::GetCount() const
{
	int64_t iCount = 0;
	auto pSegs = m_tRtChunks.RamSegs();
	if ( pSegs )
		for ( auto & pSeg : *pSegs )
			iCount += const_cast<RtSegment_t*> ( pSeg.Ptr() )->m_tAliveRows;

	auto pDiskChunks = m_tRtChunks.DiskChunks();
	if ( pDiskChunks )
		for ( const auto & i : *pDiskChunks )
		{
			int64_t iChunkCount = i->Cidx().GetCount();
			if ( iChunkCount==-1 )
				return -1;

			iCount += iChunkCount;
		}

	return iCount;
}


 std::pair<int64_t,int> RtIndex_c::GetPseudoShardingMetric ( const VecTraits_T<const CSphQuery> & dQueries, const VecTraits_T<int64_t> & dMaxCountDistinct, int iThreads, bool & bForceSingleThread ) const
{
	if ( MustRunInSingleThread ( dQueries, false, dMaxCountDistinct, bForceSingleThread ) )
		return { 0, 1 };

	bool bHaveKNN = dQueries.any_of ( []( auto & tQuery ){ return tQuery.HasKnn(); } );	 

	auto tGuard = RtGuard();
	int iDiskChunks = tGuard.m_dDiskChunks.GetLength();
	int iThreadCap = GetPseudoSharding() ? ( bHaveKNN ? iDiskChunks : 0 ) : iDiskChunks;
	return { GetStats().m_iTotalDocuments, iThreadCap };
}


void RtIndex_c::DropDiskChunk ( int iChunkID, int* pAffected )
{
	TRACE_SCHED ( "rt", "RtIndex_c::DropDiskChunk" );

	sphLogDebug( "rt optimize: table %s: drop disk chunk %d", GetName(), iChunkID );

	// work in serial fiber
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "DropDiskChunk" );
	auto dOldExternalFiles = CollectOwnedExternalFiles();
	{
		auto tChangeset = RtWriter();
		tChangeset.InitDiskChunks ( RtWriter_c::empty );
		auto pChunks = m_tRtChunks.RtData().m_pChunks;
		for ( auto& pChunk : *pChunks )
			if ( iChunkID == pChunk->Cidx().m_iChunk )
				pChunk->m_bFinallyUnlink = true;
			else
				tChangeset.m_pNewDiskChunks->Add ( pChunk );
	}
	SaveMeta();
	RemoveOutdatedOwnedExternalFiles ( dOldExternalFiles );
	if ( pAffected )
		++*pAffected;
}

void RtIndex_c::DropDiskChunks ( IntVec_t& dChunks, int* pAffected ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	if ( dChunks.IsEmpty() )
		return;

	dChunks.Uniq();

	RTDLOG << "Optimize: drop chunks (" << dChunks.GetLength() << ")";
	TRACE_SCHED ( "rt", "RtIndex_c::DropDiskChunks" );
	sphLogDebug( "rt optimize: table %s: drop disk chunks (%d)", GetName(), dChunks.GetLength() );

	auto dOldExternalFiles = CollectOwnedExternalFiles();
	{
		auto tChangeset = RtWriter();
		tChangeset.InitDiskChunks ( RtWriter_c::empty );
		auto pChunks = m_tRtChunks.RtData().m_pChunks;
		for ( auto& pChunk : *pChunks )
			if ( dChunks.BinarySearch ( pChunk->Cidx().m_iChunk ) )
				pChunk->m_bFinallyUnlink = true;
			else
				tChangeset.m_pNewDiskChunks->Add ( pChunk );
	}
	SaveMeta();
	RemoveOutdatedOwnedExternalFiles ( dOldExternalFiles );
	if ( pAffected )
		*pAffected+=dChunks.GetLength();
}

// perform merge, preload result, rename to final chunk and return preallocated result scheduled to dispose
ConstDiskChunkRefPtr_t RtIndex_c::ChangeDiskChunk ( const char* szParentAction, const ConstDiskChunkRefPtr_t& pChunk, CSphIndexProgress& tProgress, VecTraits_T<CSphFilterSettings> dFilters )
{
	TRACE_CORO ( "rt", "RtIndex_c::ChangeDiskChunk" );
	CSphString sError;

	const CSphIndex& tChunk = pChunk->Cidx();

	// note: klist for merged chunk will be attached during merge at the moment of copying alive rows.
	if ( !sphMerge ( &tChunk, &tChunk, dFilters, tProgress, sError ) )
	{
		if ( sError.IsEmpty() && tProgress.GetMergeCb().NeedStop() )
			sError = "interrupted because of shutdown";
		sphWarning ( "rt %s: table %s: failed to %s %s (%s)", szParentAction, GetName(), dFilters.IsEmpty() ? "merge" : "split", tChunk.GetFilebase(), sError.cstr() );
		return {};
	}

//	PauseCheck ( "postmerge" );
	return MergeChunkPrealloc ( tChunk, szParentAction );
}

ConstDiskChunkRefPtr_t RtIndex_c::MergeChunkPrealloc ( const CSphIndex & tChunk, const char * szParentAction )
{
	ConstDiskChunkRefPtr_t pChunk;

	auto fnFnameBuilder = GetIndexFilenameBuilder();
	std::unique_ptr<FilenameBuilder_i> pFilenameBuilder;
	if ( fnFnameBuilder )
		pFilenameBuilder = fnFnameBuilder ( GetName() );

	// prealloc new (optimized) chunk
	CSphString sChunk = tChunk.GetFilename ( "tmp" );

	CSphString sError;
	StrVec_t dWarnings;
	pChunk = DiskChunk_c::make ( PreallocDiskChunk ( sChunk, tChunk.m_iChunk, pFilenameBuilder.get(), dWarnings, sError, tChunk.GetName() ) );
	dWarnings.for_each ( [] ( const auto& sWarning ) { sphWarning ( "PreallocDiskChunk warning: %s", sWarning.cstr() ); } );

	if ( pChunk )
		pChunk->m_bFinallyUnlink = true; // on destroy files will be deleted. Caller must explicitly reset this flag if chunk is usable
	else
		sphWarning ( "rt %s: table %s: failed to prealloc: %s", szParentAction, GetName(), sError.cstr() );

	return pChunk;
}

struct RtMergeTimings_t : MergeTimings_t
{
	int64_t m_tmBuild = 0;
	int64_t m_tmPublish = 0;
};

ConstDiskChunkRefPtr_t RtIndex_c::MergeDiskChunksN ( const char* szParentAction, VecTraits_T<ConstDiskChunkRefPtr_t> dChunks, CSphIndexProgress& tProgress, RtMergeTimings_t& tTimings )
{
	TRACE_CORO ( "rt", "RtIndex_c::MergeDiskChunksN" );
	CSphString sError;

	if ( dChunks.IsEmpty() )
		return {};

	CSphFixedVector<const CSphIndex*> dIndexes ( dChunks.GetLength() );
	ARRAY_FOREACH ( i, dChunks )
		dIndexes[i] = &dChunks[i]->Cidx();

	ConstDiskChunkRefPtr_t pChunk;
	if ( !sphMergeN ( dIndexes, tProgress, sError, &tTimings ) )
	{
		if ( sError.IsEmpty() && tProgress.GetMergeCb().NeedStop() )
			sError = "interrupted because of shutdown";
		sphWarning ( "rt %s: table %s: failed to merge %s (%s)", szParentAction, GetName(), dChunks[0]->Cidx().GetFilebase(), sError.cstr() );
		return pChunk;
	}

	// Use the last chunk for prealloc since DoMergeN writes files using pDstIndex (last chunk)
	const CSphIndex & tChunkDst = dChunks.Last()->Cidx();
	return MergeChunkPrealloc ( tChunkDst, szParentAction );
}

bool RtIndex_c::RenameOptimizedChunk ( const ConstDiskChunkRefPtr_t& pChunk, const char* szParentAction )
{
	if ( !pChunk )
		return false;

	CSphIndex& tChunk = pChunk->CastIdx(); // const breakage is ok since we don't yet published the index
	// prepare new chunk to publish
	int iResID = m_tChunkID.MakeChunkId ( m_tRtChunks );
	auto sNewchunk = GetFilename ( iResID );
	tChunk.m_iChunk = iResID;

	// rename merged disk chunk to valid chunk name
	if ( tChunk.Rename ( sNewchunk ) )
		return true;

	sphWarning ( "rt %s: table %s: processed to cur rename failed (%s)", szParentAction, GetName(), tChunk.GetLastError().cstr() );
	return false;
}

bool RtIndex_c::PublishMergedChunks ( const char* szParentAction, std::function<bool ( int, DiskChunkVec_c& )>&& fnPusher ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	TRACE_CORO ( "rt", "PublishMergedChunks" );
	bool bReplaced = false;
	auto tChangeset = RtWriter();
	tChangeset.InitDiskChunks ( RtWriter_c::empty );
	auto pChunks = m_tRtChunks.DiskChunks();
	for ( auto& pChunk : *pChunks )
	{
		if ( fnPusher ( pChunk->Cidx().m_iChunk, *tChangeset.m_pNewDiskChunks ) )
			bReplaced = true;
		else
			tChangeset.m_pNewDiskChunks->Add ( pChunk );
	}

	if ( bReplaced )
		return true;

	sphWarning ( "rt %s: table %s: unable to locate victim chunk after merge, leave everything unchanged", szParentAction, GetName() );
	tChangeset.m_pNewDiskChunks = nullptr; // discard changes, i.e. disk chunk set will NOT be modified
	return false;
}

static int64_t NumAliveDocs ( const CSphIndex& dChunk )
{
	CSphIndexStatus tStatus;
	dChunk.GetStatus ( &tStatus );
	return dChunk.GetStats().m_iTotalDocuments - tStatus.m_iDead;
}

bool RtIndex_c::BinlogCommit ( RtSegment_t * pSeg, const VecTraits_T<DocID_t> & dKlist, int64_t iAddTotalBytes, CSphString & sError ) REQUIRES ( pSeg->m_tLock )
{
//	Tracer::AsyncOp tTracer ( "rt", "RtIndex_c::BinlogCommit" );
	return Binlog::Commit ( &m_iTID, GetName(), sError, [pSeg,&dKlist,iAddTotalBytes,bKeywordDict=IsWordDict()] (Writer_i & tWriter) REQUIRES ( pSeg->m_tLock )
	{
		tWriter.PutByte ( Binlog::COMMIT );
		if ( !pSeg || !pSeg->m_uRows )
		{
			tWriter.ZipOffset ( 0 );
			Binlog::SaveVector ( tWriter, dKlist );
			return;
		}

		tWriter.ZipOffset ( pSeg->m_uRows );
		tWriter.ZipOffset ( iAddTotalBytes );
		Binlog::SaveVector ( tWriter, pSeg->m_dWords );
		tWriter.ZipOffset ( pSeg->m_dWordCheckpoints.GetLength() );
		if ( !bKeywordDict )
		{
			for ( const auto& dWordCheckpoint : pSeg->m_dWordCheckpoints )
			{
				tWriter.ZipOffset ( dWordCheckpoint.m_iOffset );
				tWriter.ZipOffset ( dWordCheckpoint.m_uWordID );
			}
		} else
		{
			for ( const auto & dWordCheckpoint : pSeg->m_dWordCheckpoints )
			{
				tWriter.ZipOffset ( dWordCheckpoint.m_iOffset );
				tWriter.ZipOffset ( dWordCheckpoint.m_szWord - (const char *)pSeg->m_dKeywordCheckpoints.Begin() );
			}
		}

		Binlog::SaveVector ( tWriter, pSeg->m_dDocs );
		Binlog::SaveVector ( tWriter, pSeg->m_dHits );
		Binlog::SaveVector ( tWriter, pSeg->m_dRows );
		Binlog::SaveVector ( tWriter, pSeg->m_dBlobs );
		Binlog::SaveVector ( tWriter, pSeg->m_dKeywordCheckpoints );

		tWriter.PutByte ( pSeg->m_pDocstore ? 1 : 0 );
		if ( pSeg->m_pDocstore )
			pSeg->m_pDocstore->Save ( tWriter );

		tWriter.PutByte ( pSeg->m_pColumnar ? 1 : 0 );
		if ( pSeg->m_pColumnar )
			pSeg->m_pColumnar->Save ( tWriter );

		Binlog::SaveVector ( tWriter, dKlist );
	});
}


static Binlog::CheckTnxResult_t Warn ( CSphString & sError, const CSphReader & tReader )
{
	sError = tReader.GetErrorMessage();
	return {};
}


Binlog::CheckTnxResult_t RtIndex_c::ReplayCommit ( CSphReader & tReader, CSphString & sError, Binlog::CheckTxn_fn && fnCanContinue )
{
	CSphRefcountedPtr<RtSegment_t> pSeg;
	CSphVector<DocID_t> dKlist;

	int64_t iAddTotalBytes = 0;
	DWORD uRows = tReader.UnzipOffset();
	if ( uRows )
	{
		iAddTotalBytes = (int64_t)tReader.UnzipOffset();
		pSeg = new RtSegment_t(uRows, m_tSchema);
		FakeWL_t _ ( pSeg->m_tLock );

		if ( !Binlog::LoadVector ( tReader, pSeg->m_dWords ) ) return Warn ( sError, tReader );
		pSeg->m_dWordCheckpoints.Resize ( (int) tReader.UnzipOffset() ); // FIXME! sanity check
		ARRAY_FOREACH ( i, pSeg->m_dWordCheckpoints )
		{
			pSeg->m_dWordCheckpoints[i].m_iOffset = (int) tReader.UnzipOffset();
			pSeg->m_dWordCheckpoints[i].m_uWordID = (SphWordID_t )tReader.UnzipOffset();
		}

		if ( tReader.GetErrorFlag() ) return Warn ( sError, tReader );

		if ( !Binlog::LoadVector ( tReader, pSeg->m_dDocs ) ) return Warn ( sError, tReader );
		if ( !Binlog::LoadVector ( tReader, pSeg->m_dHits ) ) return Warn ( sError, tReader );
		if ( !Binlog::LoadVector ( tReader, pSeg->m_dRows )  ) return Warn ( sError, tReader );
		if ( !Binlog::LoadVector ( tReader, pSeg->m_dBlobs )  ) return Warn ( sError, tReader );
		if ( !Binlog::LoadVector ( tReader, pSeg->m_dKeywordCheckpoints ) ) return Warn ( sError, tReader );

		bool bHaveDocstore = !!tReader.GetByte();
		if ( tReader.GetErrorFlag() ) return Warn ( sError, tReader );

		if ( bHaveDocstore )
		{
			pSeg->SetupDocstore ( &(GetInternalSchema()) );
			assert ( pSeg->m_pDocstore );
			Verify ( pSeg->m_pDocstore->Load(tReader) );
		}

		bool bHaveColumnar = !!tReader.GetByte();
		if ( tReader.GetErrorFlag() ) return Warn ( sError, tReader );

		if ( bHaveColumnar )
		{
			CSphString sError;
			pSeg->m_pColumnar = CreateColumnarRT ( GetInternalSchema(), tReader, sError );
			if ( !pSeg->m_pColumnar )
				return {};
		}

		pSeg->BuildDocID2RowIDMap ( GetInternalSchema() );
	}

	if ( !Binlog::LoadVector ( tReader, dKlist ) ) return Warn ( sError, tReader );

	Binlog::CheckTnxResult_t tRes = fnCanContinue ();
	if ( tRes.m_bValid && tRes.m_bApply )
	{
		// in case dict=keywords
		// + cook checkpoint
		// + build infixes
		if ( IsWordDict() && pSeg )
		{
			FixupSegmentCheckpoints ( pSeg );
			BuildSegmentInfixes ( pSeg, GetDictionary()->HasMorphology(), GetDictFormat(), GetSettings().m_iMinInfixLen,
				GetWordCheckoint(), ( GetMaxCodepointLength()>1 ), GetSettings().m_eHitless );
		}

		// actually replay
		FakeRL_t _ ( pSeg.operator RtSegment_t*()->m_tLock);
		int iKilled = 0;

//		dKlist.Sort ( Lesser ( [] ( DocID_t a, DocID_t b ) { return (uint64_t)a < (uint64_t)b; } ) ); // should already be sorted
		CommitReplayable ( pSeg, dKlist, iAddTotalBytes, iKilled, sError );
		tRes.m_bApply = true;
	}
	return tRes;
}

Binlog::CheckTnxResult_t RtIndex_c::ReplayTxn ( CSphReader & tReader, CSphString & sError, BYTE uOp, Binlog::CheckTxn_fn && fnCanContinue )
{
	switch ( uOp )
	{
	case Binlog::UPDATE_ATTRS: return ReplayUpdate ( tReader, sError, std::move ( fnCanContinue ) );
	case Binlog::COMMIT: return ReplayCommit ( tReader, sError, std::move ( fnCanContinue ) );
	default: assert (false && "unknown op provided to replay");
	}
	return {};
}


bool RtIndex_c::SkipOrDrop ( int iChunkID, const CSphIndex& dChunk, bool bCheckAlive, int * pAffected )
{
	auto iTotalDocs = dChunk.GetStats().m_iTotalDocuments;
	auto iAliveDocs = NumAliveDocs ( dChunk );

	// all docs killed
	if ( !iAliveDocs )
	{
		sphLogDebug ( "common merge - drop %d, all (" INT64_FMT ") killed", iChunkID, iTotalDocs );
		DropDiskChunk ( iChunkID, pAffected );
		return true;
	}

	// no docs killed
	if ( bCheckAlive && iTotalDocs == iAliveDocs )
	{
		sphLogDebug ( "common merge - skip compressing %d, no killed", iChunkID );
		return true;
	}

	if ( pAffected )
		++*pAffected;
	return false;
}

class MergeGuardMulti_c
{
	CSphVector<ConstDiskChunkRefPtr_t> m_dUsedChunks;

	void SetOptimize() const noexcept
	{
		for ( const auto & pChunk : m_dUsedChunks )
			if ( pChunk )
				pChunk->m_bOptimizing.store ( true, std::memory_order_relaxed );
	}

public:
	MergeGuardMulti_c () = default;
	explicit MergeGuardMulti_c ( const VecTraits_T<ConstDiskChunkRefPtr_t> & dChunks )
	{
		m_dUsedChunks.Reserve ( dChunks.GetLength() );
		for ( const auto & pChunk : dChunks )
			m_dUsedChunks.Add ( pChunk );

		SetOptimize();
	}

	void Add ( const ConstDiskChunkRefPtr_t & pChunk ) noexcept
	{
		m_dUsedChunks.Add ( pChunk );
		SetOptimize();
	}

	virtual ~MergeGuardMulti_c()
	{
		for ( const auto & pChunk : m_dUsedChunks )
			if ( pChunk )
				pChunk->m_bOptimizing.store ( false, std::memory_order_relaxed );
	}
};

bool RtIndex_c::CompressOneChunk ( int iChunkID, int* pAffected )
{
	TRACE_CORO ( "rt", "RtIndex_c::CompressOneChunk" );
	auto pVictim = m_tRtChunks.DiskChunkByID ( iChunkID );
	if ( !pVictim )
	{
		sphWarning ( "rt optimize: table %s: compress of chunk %d failed, no chunk with such ID!", GetName(), iChunkID );
		return false;
	}
	const CSphIndex& tVictim = pVictim->Cidx();
	if ( SkipOrDrop ( iChunkID, tVictim, true, pAffected ) )
		return true;

	sphLogDebug ( "compress %d (%d kb)", iChunkID, (int)( GetChunkSize ( tVictim ) / 1024 ) );

	MergeGuardMulti_c tOptimizingGuard;
	tOptimizingGuard.Add (pVictim);

	// merge data to disk ( data is constant during that phase )
	RTMergeCb_c tMonitor ( &m_bOptimizeStop, this );
	CSphIndexProgress tProgress ( &tMonitor );
	auto pCompressed = ChangeDiskChunk ( "compress", pVictim, tProgress, { nullptr, 0 } );

	auto tFinallyStopCollectingUpdates = AtScopeExit ( [pVictim] { pVictim->CastIdx().ResetPostponedUpdates(); } );

	if ( !pCompressed )
		return false;

	if ( !RenameOptimizedChunk ( pCompressed, "compress" ) )
		return false;

	if ( tMonitor.HasKilled() && tMonitor.NeedStop() ) // if we should interrupt now, but if no kills, can continue.
		return false;

	CSphIndex& tCompressed = pCompressed->CastIdx(); // const breakage is ok since we don't yet published the index

	// going to modify list of chunks; so fall into serial fiber
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_CORO ( "rt", "CompressOneChunk.serial" );

	// reset kill hook explicitly to override default order of destruction
	SetKillHookFor ( nullptr, iChunkID );

	// and also apply already collected kills
	int iKilled = tCompressed.KillMulti ( tMonitor.GetKilled() );

	// and also apply collected updates
	auto& dUpdates = pVictim->CastIdx().m_dPostponedUpdates;
	if ( !dUpdates.IsEmpty() )
	{
		tCompressed.UpdateAttributesOffline ( dUpdates );
		dUpdates.Reset();
	}

	auto dOldExternalFiles = CollectOwnedExternalFiles();
	if ( !PublishMergedChunks ( "compress", [iChunkID, pCompressed] ( int iChunk, DiskChunkVec_c& tRes ) {
			if ( iChunk==iChunkID )
				tRes.Add ( pCompressed );
			return iChunk==iChunkID; } ) )
		return false;

	sphLogDebug ( "compressed a=%s, new=%s, killed=%d", tVictim.GetFilebase(), tCompressed.GetFilebase(), iKilled );
	pVictim->m_bFinallyUnlink = true;
	pCompressed->m_bFinallyUnlink = false;
	SaveMeta();
	Preread();
	RemoveOutdatedOwnedExternalFiles ( dOldExternalFiles );
	if ( pAffected )
		++*pAffected;
	return true;
}

bool RtIndex_c::DedupOneChunk ( int iChunkID, int* pAffected )
{
	TRACE_CORO ( "rt", "RtIndex_c::DedupOneChunk" );
	auto pVictim = m_tRtChunks.DiskChunkByID ( iChunkID );
	if ( !pVictim )
	{
		sphWarning ( "rt optimize: table %s: dedup of chunk %d failed, no chunk with such ID!", GetName(), iChunkID );
		return false;
	}
	const CSphIndex& tVictim = pVictim->Cidx();
	if ( SkipOrDrop ( iChunkID, tVictim, false, pAffected ) )
		return true;

	sphLogDebug ( "dedup %d (%d docs)", iChunkID, (int)tVictim.GetCount() );

	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_CORO ( "rt", "DedupOneChunk.serial" );

	// and also apply already collected kills
	int iKilled = pVictim->CastIdx().KillDupes();
	m_tStats.m_iTotalDocuments -= iKilled;

	if ( pAffected && iKilled>0 )
		*pAffected += 1;
	return true;
}

// catch common cases where we can't work at all (erroneous settings, etc.), or can redirect to another action
bool RtIndex_c::SplitOneChunkFast ( int iChunkID, const char* szUvarFilter, bool& bResult, int* pAffected )
{
	if ( !szUvarFilter )
	{
		sphLogDebug ( "zero filter provided to split chunk %d, perform simple compress", iChunkID );
		bResult = CompressOneChunk ( iChunkID, pAffected );
		return true;
	}

	auto pVictim = m_tRtChunks.DiskChunkByID ( iChunkID );
	if ( !pVictim )
	{
		sphWarning ( "rt optimize: table %s: split of chunk %d failed, no chunk with such ID!", GetName(), iChunkID );
		bResult = false;
		return true;
	}
	const CSphIndex& tVictim = pVictim->Cidx();
	if ( SkipOrDrop ( iChunkID, tVictim, false, pAffected ) )
	{
		bResult = true;
		return true;
	}

	if ( !NumAliveDocs ( tVictim ) )
	{
		sphLogDebug ( "chunk empty, nothing to split" );
		bResult = false;
		return true;
	}

	// create filter by @uservar - break if it is not available.
	if ( !UservarsAvailable() )
	{
		sphLogDebug ( "no global variables found" );
		bResult = false;
		return true;
	}

	const UservarIntSet_c pUservar = Uservars ( szUvarFilter );
	if ( !pUservar )
	{
		sphLogDebug ( "undefined global variable '%s'", szUvarFilter );
		bResult = false;
		return true;
	}
	return false;
}

// compress iChunk with filter id by uservar include and exclude,
// then replace original chunk with one of pieces, and insert second piece just after the first.
bool RtIndex_c::SplitOneChunk ( int iChunkID, const char* szUvarFilter, int* pAffected )
{
	TRACE_CORO ( "rt", "RtIndex_c::SplitOneChunk" );
	bool bResult;
	if ( SplitOneChunkFast ( iChunkID, szUvarFilter, bResult, pAffected ) )
		return bResult;

	auto pVictim = m_tRtChunks.DiskChunkByID ( iChunkID );
	assert ( pVictim && "non-existent chunks should be already rejected by SplitOneChunkFast" );
	CSphIndex& tVictim = pVictim->CastIdx(); // non-const need to invoke 'merge'
	sphLogDebug ( "split %d (%d kb) with %s", iChunkID, (int)( GetChunkSize ( tVictim ) / 1024 ), szUvarFilter );

	MergeGuardMulti_c tOptimizingGuard;
	tOptimizingGuard.Add (pVictim);

	const UservarIntSet_c pUservar = Uservars ( szUvarFilter );
	assert ( pUservar ); // detailed check already performed in splitOneChunkFast

	// create negative (exclusion) filter
	CSphVector<CSphFilterSettings> dFilters;
	auto & dFilter = dFilters.Add ();

	dFilter.m_sAttrName = "id";
	dFilter.m_eType = SPH_FILTER_VALUES;
	dFilter.SetExternalValues ( *pUservar );
	dFilter.m_bExclude = true;

	// prepare for real split (merge)
	RTMergeCb_c tMonitor ( &m_bOptimizeStop, this );
	CSphIndexProgress tProgress ( &tMonitor );

	auto iOriginallyAlive = NumAliveDocs ( tVictim );

	// get 1-st chunk - one which doesn't match filter the filter
	auto pChunkE = ChangeDiskChunk ( "1-st part of split", pVictim, tProgress, dFilters );

	auto tFinallyStopCollectingUpdatesE = AtScopeExit ( [pVictim] { pVictim->CastIdx().ResetPostponedUpdates();	} );

	// check forced exit after long operation (that is - after merge)
	if ( !pChunkE || tMonitor.NeedStop() )
		return false;

	CSphIndex& tIndexE = pChunkE->CastIdx(); // const breakage is ok since we don't yet published the index

	// if nothing is alive after filter applied - fast break, nothing to do.
	auto iExcludedAlive = NumAliveDocs ( tIndexE );
	if ( !iExcludedAlive )
	{
		// fool protect - either nothing, either all is filtered. No need to continue.
		sphLogDebug ( "filter selected nothing, no point to split" );
		return false;
	} else if ( iExcludedAlive == iOriginallyAlive )
	{
		// fool protect - either nothing, either all is filtered. No need to continue.
		sphLogDebug ( "filter selected everything, no point to split" );
		return false;
	}

	if ( !RenameOptimizedChunk ( pChunkE, "1-st part of split" ) )
		return false;

	// prepare <I>ncluded chunk - one with included docs, it will be placed instead of original one
	dFilter.m_bExclude = false;
	auto pChunkI = ChangeDiskChunk ( "2-nd part of split", pVictim, tProgress, dFilters );

	// check forced exit after long operation (that is - after merge)
	if ( !pChunkI || tMonitor.NeedStop() )
		return false;

	CSphIndex& tIndexI = pChunkI->CastIdx(); // const breakage is ok since we don't yet published the index

	if ( tMonitor.HasKilled() && tMonitor.NeedStop() ) // if we should interrupt now, but if no kills, can continue.
		return false;

	if ( !RenameOptimizedChunk ( pChunkI, "2-nd part of split" ) )
		return false;

	// going to modify list of chunks; so fall into serial fiber
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_CORO ( "rt", "SplitOneChunk" );

	// reset kill hook explicitly to override default order of destruction
	SetKillHookFor ( nullptr, iChunkID );

	// apply collected kill-list before including chunks to the set
	// as we are in serial worker, that is safe here; no new kills may arrive.
	int iKilled = 0;
	if ( tMonitor.HasKilled() )
	{
		auto dKilled = tMonitor.GetKilled();
		iKilled += tIndexE.KillMulti ( dKilled );
		iKilled += tIndexI.KillMulti ( dKilled );
	}

	// and also apply collected updates
	auto& dUpdates = pVictim->CastIdx().m_dPostponedUpdates;
	if ( !dUpdates.IsEmpty() )
	{
		tIndexI.UpdateAttributesOffline ( dUpdates );
		tIndexE.UpdateAttributesOffline ( dUpdates );
		dUpdates.Reset();
	}

	auto dOldExternalFiles = CollectOwnedExternalFiles();
	if ( !PublishMergedChunks ( "split",
				 [iChunkID, pChunkI, pChunkE] ( int iChunk, DiskChunkVec_c& tRes ) {
		if ( iChunk==iChunkID )
		{
			tRes.Add ( pChunkI );
			tRes.Add ( pChunkE );
			return true;
		}
		return false; } ) )
		return false;

	sphLogDebug ( "split a=%s, b=%s, killed=%d", tIndexE.GetFilebase(), tIndexI.GetFilebase(), iKilled );
	pVictim->m_bFinallyUnlink = true;
	pChunkI->m_bFinallyUnlink = false;
	pChunkE->m_bFinallyUnlink = false;
	SaveMeta();
	Preread();
	RemoveOutdatedOwnedExternalFiles ( dOldExternalFiles );
	if ( pAffected )
		++*pAffected;
	return true;
}

bool RtIndex_c::MergeTwoChunks ( int iAID, int iBID, int* pAffected, StringBuilder_c* sLog )
{
	TRACE_CORO ( "rt", "RtIndex_c::MergeTwoChunks" );

	auto pA = m_tRtChunks.DiskChunkByID ( iAID );
	if ( !pA )
	{
		sphWarning ( "rt optimize: table %s: merge chunks %d and %d failed, chunk ID %d is not valid!", GetName(), iAID, iBID, iAID );
		return false;
	}

	auto pB = m_tRtChunks.DiskChunkByID ( iBID );
	if ( !pB )
	{
		sphWarning ( "rt optimize: table %s: merge chunks %d and %d failed, chunk ID %d is not valid!", GetName(), iAID, iBID, iBID );
		return false;
	}

	MergeGuardMulti_c tOptimizingGuard;
	tOptimizingGuard.Add (pA);
	tOptimizingGuard.Add (pB);

	const char * szActionName = "common merge";

	sphLogDebug ( "%s - merging %d (%d kb) with %d (%d kb)", szActionName,
			iAID,
			(int)( GetChunkSize ( pA->Cidx() ) / 1024 ),
			iBID,
			(int)( GetChunkSize ( pB->Cidx() ) / 1024 ) );

	CSphFixedVector<ConstDiskChunkRefPtr_t> dChunks ( 2 );
	dChunks[0] = pA;
	dChunks[1] = pB;

	RtMergeTimings_t tFoo;
	return MergeNChunks ( szActionName, dChunks, pAffected, sLog, tFoo );
}

bool RtIndex_c::MergeNChunks ( const char * szParentAction, VecTraits_T<ConstDiskChunkRefPtr_t> dChunks, int* pAffected, StringBuilder_c* pLog, RtMergeTimings_t& tTimings )
{
	// merge data to disk ( data is constant during that phase )
	RTMergeCb_c tMonitor ( &m_bOptimizeStop, this );
	CSphIndexProgress tProgress ( &tMonitor );

	const auto tmBuild = sphMicroTimer();
	auto pMerged = MergeDiskChunksN ( szParentAction, dChunks, tProgress, tTimings );
	tTimings.m_tmBuild = sphMicroTimer() - tmBuild;

// finalize
	auto fnFinalize = AtScopeExit ( [&tTimings, tmFinalize = sphMicroTimer()] {tTimings.m_tmPublish = sphMicroTimer()-tmFinalize;});
	auto tFinallyStopCollectingUpdates = AtScopeExit ( [&dChunks] {
		for ( const auto & pChunk : dChunks )
			pChunk->CastIdx().ResetPostponedUpdates();
	} );

	// check forced exit after long operation (that is - after merge)
	if ( !pMerged || tMonitor.NeedStop() )
		return false;

	if ( !RenameOptimizedChunk ( pMerged, szParentAction ) )
		return false;

	CSphIndex& tMerged = pMerged->CastIdx(); // const breakage is ok since we don't yet published the index

	// going to modify list of chunks; so fall into serial fiber
	TRACE_CORO ( "rt", "RtIndex_c::MergeTwoChunks_workserial" );
	BEGIN_CORO ( "wait", "RtIndex_c::acquire serial fiber" );
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	END_CORO ("wait" );

	// reset kill hook explicitly to override default order of destruction
	for ( const auto & pChunk : dChunks )
		SetKillHookFor ( nullptr, pChunk->Cidx().m_iChunk );

	// apply collected kill-list before including chunks to the set
	// as we are in serial worker, that is safe here; no new kills may arrive.
	int iKilled = 0;
	if ( tMonitor.HasKilled() )
		iKilled = tMerged.KillMulti ( tMonitor.GetKilled() );

	// and also apply collected updates
	auto dUpdates = GatherUpdates::FromChunksOrSegments ( dChunks );
	if ( !dUpdates.IsEmpty() )
	{
		tMerged.UpdateAttributesOffline ( dUpdates );
		dUpdates.Reset();
	}

	auto fnFilter = [&dChunks, pMerged] ( int iChunk, DiskChunkVec_c& tRes ) {
		if ( iChunk==dChunks.Last()->Cidx().m_iChunk )
			tRes.Add ( pMerged );
		return dChunks.any_of ( [iChunk] ( const auto & tChunk ) { return tChunk->Cidx().m_iChunk==iChunk; } );
	};

	auto dOldExternalFiles = CollectOwnedExternalFiles();
	if ( !PublishMergedChunks ( szParentAction, fnFilter ) )
		return false;

	if ( pLog )
	{
		auto& sLog = *pLog;
		{
			ScopedComma_c comma {sLog, ", "};
			dChunks.for_each ( [&sLog] (const auto& tChunk) { sLog << tChunk->Cidx().GetFilebase();} );
		}
		sLog << " to " << tMerged.GetFilebase();
		sphLogDebug ( "optimized %s chunks to %s, killed=%d", sLog.cstr(), tMerged.GetFilebase(), iKilled );
	} else
	{
		sphLogDebug ( "optimized %d chunks to %s, killed=%d", dChunks.GetLength(), tMerged.GetFilebase(), iKilled );
	}


	for ( const auto & pChunk : dChunks )
		pChunk->m_bFinallyUnlink = true;
	pMerged->m_bFinallyUnlink = false;
	SaveMeta();
	Preread();
	RemoveOutdatedOwnedExternalFiles ( dOldExternalFiles );
	if ( pAffected )
		++*pAffected;
	return true;
}

bool RtIndex_c::StopOptimize()
{
	auto bPrevOptimizeValue = m_bOptimizeStop.exchange ( true, std::memory_order_relaxed );
	std::atomic_thread_fence ( std::memory_order_release ); // to be sure we go to Wait() _after_ m_bOptimizeStop is set to true.
	if ( Threads::IsInsideCoroutine() )
		m_tOptimizeRuns.Wait ( [] ( int i ) { return i <= 0; } );
	return bPrevOptimizeValue;
}

void RtIndex_c::StartRoutine()
{
	m_tBackgroundRoutines.ModifyValue ( [] ( int& i ) { ++i; } );
}

void RtIndex_c::StopRoutine()
{
	if ( Threads::IsInsideCoroutine() )
		m_tBackgroundRoutines.ModifyValueAndNotifyAll ( [] ( int& i ) { --i; } );
	else
		m_tBackgroundRoutines.ModifyValue ( [] ( int& i ) { --i; } );
}

void RtIndex_c::WaitAllRoutinesFinished() const noexcept
{
	if ( Threads::IsInsideCoroutine() )
		m_tBackgroundRoutines.Wait ( [] ( int iVal ) { return iVal==0; } );
}

bool RtIndex_c::CheckValidateChunk ( int& iChunk, int iChunks, bool bByOrder ) const
{
	if ( bByOrder )
	{
		if ( iChunk >= iChunks || iChunk < 0 )
		{
			sphWarning ( "rt: table %s: Optimize step: provided chunk %d is out of range [0..%d]", GetName(), iChunk, iChunks - 1 );
			return false;
		}
		iChunk = ChunkIDByChunkIdx ( iChunk );
	}
	auto pChunk = m_tRtChunks.DiskChunkByID ( iChunk );
	if ( !pChunk )
	{
		sphWarning ( "rt: table %s: Optimize step: provided chunk ID %d is invalid", GetName(), iChunk );
		return false;
	}

	if ( pChunk->m_bOptimizing.load ( std::memory_order_relaxed ) )
	{
		sphWarning ( "rt: table %s: Optimize step: provided chunk %d is now occupied in another optimize operation", GetName(), iChunk );
		return false;
	}
	return true;
}

bool RtIndex_c::CheckValidateOptimizeParams ( OptimizeTask_t& tTask ) const
{
	auto iChunks = m_tRtChunks.GetDiskChunksCount();
	switch ( tTask.m_eVerb )
	{
	case OptimizeTask_t::eMerge:
		if ( !CheckValidateChunk ( tTask.m_iTo, iChunks, tTask.m_bByOrder ) )
			return false;

		// no break; check also m_iFrom param then
	case OptimizeTask_t::eDrop:
	case OptimizeTask_t::eCompress:
	case OptimizeTask_t::eSplit:
	case OptimizeTask_t::eDedup:
		if ( !CheckValidateChunk ( tTask.m_iFrom, iChunks, tTask.m_bByOrder ) )
			return false;
	default: break;
	}
	tTask.m_bByOrder=false;
	return true;
}


bool RtIndex_c::StartOptimize ( OptimizeTask_t tTask )
{
	if ( GetNumOfLocks ()>0 )
		return false;

	StartRoutine();
	Threads::StartJob ( [tTask = std::move ( tTask ), this] () {
		// want to track optimize only at work
		auto pDesc = PublishSystemInfo ( "OPTIMIZE" );
		Optimize ( std::move ( tTask ) );
		StopRoutine();
	} );
	return true;
}


int RtIndex_c::OptimizesRunning() const noexcept
{
	return m_tOptimizeRuns.GetValue ();
}


int RtIndex_c::GetNumOfLocks () const noexcept
{
	return m_tSaving.GetNumOfLocks();
}

void RtIndex_c::Optimize ( OptimizeTask_t tTask )
{
	TRACE_CORO ( "rt", "RtIndex_c::Optimize" );
	RTDLOG << "Optimize invoked with " << tTask;
	if ( !CheckValidateOptimizeParams ( tTask ) )
		return;

	RTDLOG << "Optimize checked with " << tTask;

	const int iParallelMerges = Max ( 1, ParallelChunkMergesLimit() );
	if ( OptimizesRunning() >= iParallelMerges )
	{
		RTDLOG << "Escape optimize as " << OptimizesRunning() << " tasks is already running";
		sphLogDebug ( "Escape optimize as %d tasks is already running", OptimizesRunning() );
		return;
	}

	if ( !MergeCanRun() )
		return;

	sphLogDebug ( "rt optimize: table %s: optimization started", GetName() );

	int64_t tmStart = sphMicroTimer();
	m_tOptimizeRuns.ModifyValue ( [] ( int& i ) { ++i; } );
	auto iChunks = CommonOptimize ( std::move ( tTask ) );
	m_tOptimizeRuns.ModifyValueAndNotifyAll ( [] ( int& i ) { --i; } );

	int64_t tmPass = sphMicroTimer() - tmStart;
	int iDiskChunks = m_tRtChunks.GetDiskChunksCount();

	if ( m_tSaving.IsInterrupted() )
		LogWarning ( "rt: table %s: optimization terminated chunk(s) %d ( left %d ) in %.3t", GetName(), iChunks, iDiskChunks, tmPass );
	else if ( iChunks > 0 )
		LogInfo ( "rt: table %s: optimized %s chunk(s) %d ( left %d ) in %.3t", GetName(), g_bProgressiveMerge ? "progressive" : "regular", iChunks, iDiskChunks, tmPass );
}

bool RtIndex_c::MergeCanRun() const
{
	return !m_tSaving.IsInterrupted() && !m_bOptimizeStop.load(std::memory_order_relaxed);
}

int RtIndex_c::ClassicOptimize ()
{
	TRACE_SCHED ( "rt", "RtIndex_c::ClassicOptimize" );
	RTDLOG << "Start ClassicOptimize()";
	int iAffected = 0;
	bool bWork = true;
	while ( bWork && m_tRtChunks.GetDiskChunksCount() >= 2 )
		bWork &= MergeCanRun() && MergeTwoChunks ( ChunkIDByChunkIdx ( 0 ), ChunkIDByChunkIdx ( 1 ), &iAffected );
	return iAffected;
}

void RtIndex_c::ManualOptimizeCutoff ( int iCutoff )
{
	m_iManualOptimizeCutoff.store ( iCutoff, std::memory_order_relaxed );
}

int GetCutOff ( const MutableIndexSettings_c & tSettings, bool bKNN )
{
	if ( tSettings.IsSet ( MutableName_e::OPTIMIZE_CUTOFF ) )
		return tSettings.m_iOptimizeCutoff;

	if ( bKNN )
		return MutableIndexSettings_c::GetDefaults().m_iOptimizeCutoffKNN;

	return MutableIndexSettings_c::GetDefaults().m_iOptimizeCutoff;
}

int RtIndex_c::ProgressiveOptimize ( int iCutoff )
{
	TRACE_CORO ( "rt", "RtIndex_c::ProgressiveOptimize" );

	int iAffected = 0;
	if ( !iCutoff )
		iCutoff = GetCutOff ( m_tMutableSettings, m_tSchema.HasKNNAttrs() );

	const int iMergeChunksPerJob = Max ( 2, MergeChunksPerJob() );
	const char * szActionName = "progressive optimize";

	bool bWork = true;
	while ( bWork &= MergeCanRun() )
	{
		int iRequestedCutoff = m_iManualOptimizeCutoff.exchange ( 0, std::memory_order_acq_rel );
		if ( iRequestedCutoff>0 )
			iCutoff = Min ( iCutoff, iRequestedCutoff );

		auto pChunks = m_tRtChunks.DiskChunks();
		const int iFinallyRemainigChunks = pChunks->GetLength() - GetNumOfOptimizingNow ( *pChunks );
		if ( iFinallyRemainigChunks <= iCutoff )
			break;

		const int iJobSize = Min ( iMergeChunksPerJob, iFinallyRemainigChunks - iCutoff + 1 );

		CSphVector<ConstDiskChunkRefPtr_t> dChosenChunks;
		dChosenChunks.Reserve ( iJobSize );

		CSphVector<ChunkAndSize_t> dSmallest;
		dSmallest.Reserve ( pChunks->GetLength() );
		{
			ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
			dSmallest = GetSmallestChunksByID ( *pChunks );
			if ( dSmallest.IsEmpty() )
				break;

			IntVec_t dEmptyChunks;
			ARRAY_FOREACH ( i, dSmallest )
			if ( !dSmallest[i].m_iSize )
			{
				dEmptyChunks.Add (dSmallest[i].m_iId);
				dSmallest.RemoveFast ( i-- );
			}

			// need to resort because of RemoveFast() above
			dSmallest.Sort ( Lesser ( [] ( const auto& a, const auto& b ) { return a.m_iSize < b.m_iSize; } ) );

			if (dSmallest.GetLength()>iJobSize)
				dSmallest.Resize ( iJobSize );

			// we need to make sure that A is the oldest one
			// indexes go from oldest to newest so A must-go before B (A is always older than B)
			// this is not required by bitmap killlists, but by some other stuff (like ALTER RECONFIGURE)
			dSmallest.Sort ( Lesser ( [] ( const auto& a, const auto& b ) { return a.m_iId < b.m_iId; } ) );

			for ( const auto& tNext : dSmallest ) {
				auto pChunk = m_tRtChunks.DiskChunkByID ( tNext.m_iId );
				if ( !pChunk )
				{
					bWork = false;
					break;
				}
				dChosenChunks.Add ( pChunk );
			}

			if ( !bWork || dChosenChunks.GetLength()<2 )
				break;

			// mark them 'optimizing' from serial worker, so no concurrency with another optimizing jobs possible.
			for ( const auto & pChunk : dChosenChunks )
				pChunk->m_bOptimizing.store ( true, std::memory_order_relaxed );

			DropDiskChunks ( dEmptyChunks, &iAffected );
		} // serial fiber scope

		RTDLOG << "Optimize: merge chunks " << dChosenChunks.GetLength();

		auto tmStart = sphMicroTimer();

		MergeGuardMulti_c tGuard { dChosenChunks };

		StringBuilder_c sLog;
		RtMergeTimings_t tTimings;
		bWork &= MergeNChunks ( szActionName, dChosenChunks, &iAffected, &sLog, tTimings );

		auto tmPass = sphMicroTimer() - tmStart;
		LogInfo ( "rt: table %s: merged chunks %s in %t (build %t: attrs %t, words %t, finalize %t; publish %t). Remaining chunk count: %d",
			GetName(), sLog.cstr(), tmPass, tTimings.m_tmBuild, tTimings.m_tmAttrs, tTimings.m_tmWords, tTimings.m_tmFinalize, tTimings.m_tmPublish, m_tRtChunks.GetDiskChunksCount() );
	}

	RTDLOG << "Optimize: start compressing pass for the rest of " << m_tRtChunks.GetDiskChunksCount() << " chunks.";
	// light optimize (drop totally killed chunks) in the rest of the chunks
	for ( int i = 0; bWork && i < m_tRtChunks.GetDiskChunksCount(); ++i )
	{
		auto pVictim = m_tRtChunks.DiskChunkByIdx ( i );
		const CSphIndex& tVictim = pVictim->Cidx();
		SkipOrDrop ( tVictim.m_iChunk, tVictim, false, &iAffected );
	}

	// A late async request might arrive after the last loop iteration and should not leak into a future optimize run.
	m_iManualOptimizeCutoff.store ( 0, std::memory_order_relaxed );
	return iAffected;
}

int RtIndex_c::CommonOptimize ( OptimizeTask_t tTask )
{
	TRACE_CORO ( "rt", "RtIndex_c::CommonOptimize" );
	bool bProgressive = g_bProgressiveMerge;
	int iChunks = 0;

	switch ( tTask.m_eVerb ) // process all 'single' manual commands
	{
	case OptimizeTask_t::eMerge: MergeTwoChunks ( tTask.m_iFrom, tTask.m_iTo, &iChunks ); return iChunks;
	case OptimizeTask_t::eDrop: DropDiskChunk ( tTask.m_iFrom, &iChunks ); return iChunks;
	case OptimizeTask_t::eCompress: CompressOneChunk ( tTask.m_iFrom, &iChunks ); return iChunks;
	case OptimizeTask_t::eDedup: DedupOneChunk ( tTask.m_iFrom, &iChunks ); return iChunks;
	case OptimizeTask_t::eSplit: SplitOneChunk ( tTask.m_iFrom, tTask.m_sUvarFilter.cstr(), &iChunks ); return iChunks;
	case OptimizeTask_t::eAutoOptimize:
		bProgressive = true;
	default:
		break;
	}

	return bProgressive ? ProgressiveOptimize( tTask.m_iCutoff ) : ClassicOptimize();
}

void RtIndex_c::CheckStartAutoOptimize()
{
	TRACE_SCHED ( "rt", "CheckStartAutoOptimize" );
	int iCutoff = AutoOptimizeCutoffMultiplier();
	RTLOGV << "CheckStartAutoOptimize with cutoff=" << iCutoff;
	if ( !iCutoff )
		return;

	iCutoff *= GetCutOff ( m_tMutableSettings, m_tSchema.HasKNNAttrs() );
	if ( !m_tMutableSettings.IsSet ( MutableName_e::OPTIMIZE_CUTOFF ) && !OptimizeCutoffExplicit() )
		iCutoff = Max ( iCutoff, 2 );

	if ( m_tRtChunks.GetDiskChunksCount()<=iCutoff )
		return;

	OptimizeTask_t tTask;
	tTask.m_eVerb = OptimizeTask_t::eAutoOptimize;
	tTask.m_iCutoff = iCutoff;

	RTDLOG << "StartOptimize for " << GetName () << ", auto-optimize with cutoff " << iCutoff;

	StartOptimize ( std::move ( tTask ) );
}


//////////////////////////////////////////////////////////////////////////
// STATUS
//////////////////////////////////////////////////////////////////////////

void RtIndex_c::GetStatus ( CSphIndexStatus * pRes ) const
{
	assert ( pRes );
	if ( !pRes )
		return;

	auto tGuard = RtGuard();

	int64_t iUsedRam = SegmentsGetUsedRam ( tGuard.m_dRamSegs );
	pRes->m_iDead = SegmentsGetDeadRows ( tGuard.m_dRamSegs );

	pRes->m_iRamChunkSize = iUsedRam + tGuard.m_dRamSegs.GetLength()*int(sizeof(RtSegment_t));
	pRes->m_iRamUse = sizeof( RtIndex_c ) + pRes->m_iRamChunkSize + ( m_pUuidRegistry ? m_pUuidRegistry->AllocatedBytes() : 0 );
	pRes->m_iRamRetired = m_iRamChunksAllocatedRAM.load(std::memory_order_relaxed) - iUsedRam;

	pRes->m_iMemLimit = m_iRtMemLimit;
	pRes->m_fSaveRateLimit = m_fSaveRateLimit;

	CSphString sError;
	for ( const char * szExt : { "meta", "ram" } )
	{
		CSphAutofile fdRT ( GetFilename ( szExt ), SPH_O_READ, sError );
		int64_t iFileSize = fdRT.GetSize();
		if ( iFileSize>0 )
			pRes->m_iDiskUse += iFileSize; // that uses disk, but not occupies
	}
	CSphIndexStatus tDisk;
	for ( const auto& pChunk : tGuard.m_dDiskChunks )
	{
		pChunk->Cidx().GetStatus ( &tDisk );
		pRes->m_iRamUse += tDisk.m_iRamUse;
		pRes->m_iDiskUse += tDisk.m_iDiskUse;
		pRes->m_iMapped += tDisk.m_iMapped;
		pRes->m_iMappedResident += tDisk.m_iMappedResident;
		pRes->m_iMappedDocs += tDisk.m_iMappedDocs;
		pRes->m_iMappedResidentDocs += tDisk.m_iMappedResidentDocs;
		pRes->m_iMappedHits += tDisk.m_iMappedHits;
		pRes->m_iMappedResidentHits += tDisk.m_iMappedResidentHits;
		pRes->m_iDead += tDisk.m_iDead;
	}

	pRes->m_iNumRamChunks = tGuard.m_dRamSegs.GetLength();
	pRes->m_iNumChunks = tGuard.m_dDiskChunks.GetLength();

	pRes->m_iTID = m_iTID;
	pRes->m_iSavedTID = m_iSavedTID;
	pRes->m_iLockCount = GetNumOfLocks();
	pRes->m_iOptimizesCount = OptimizesRunning();
//	sphWarning ( "Chunks: %d, RAM: %d, DISK: %d", pRes->m_iNumChunks, (int) pRes->m_iRamUse, (int) pRes->m_iDiskUse );
}

//////////////////////////////////////////////////////////////////////////
// RECONFIGURE
//////////////////////////////////////////////////////////////////////////

bool CreateReconfigure ( const CSphString & sIndexName, bool bIsStarDict, DictFormat_e eCurrentDictFormat, const ISphFieldFilter * pFieldFilter,
	const CSphIndexSettings & tIndexSettings, uint64_t uTokHash, uint64_t uDictHash, int iMaxCodepointLength, int64_t iMemLimit,
	bool bSame, CSphReconfigureSettings & tSettings, CSphReconfigureSetup & tSetup, StrVec_t & dWarnings, CSphString & sError )
{
	CreateFilenameBuilder_fn fnCreateFilenameBuilder = GetIndexFilenameBuilder();
	std::unique_ptr<FilenameBuilder_i> pFilenameBuilder;
	if ( fnCreateFilenameBuilder )
		pFilenameBuilder = fnCreateFilenameBuilder ( sIndexName.cstr() );

	DictFormat_e eNewDictFormat = tSettings.m_tDict.GetDictFormat();
	const bool bDictFormatChanged = eNewDictFormat!=eCurrentDictFormat;
	const bool bUpgradeToKeywordsV2 = eCurrentDictFormat==DictFormat_e::KEYWORDS && eNewDictFormat==DictFormat_e::KEYWORDS_V2;
	if ( bDictFormatChanged && !bUpgradeToKeywordsV2 )
	{
		sError.SetSprintf ( "'%s': changing dict from %s to %s is not supported by RECONFIGURE",
			sIndexName.cstr(), DictFormatName ( eCurrentDictFormat ), DictFormatName ( eNewDictFormat ) );
		return true;
	}

	// FIXME!!! check missed embedded files
	TokenizerRefPtr_c pTokenizer = Tokenizer::Create ( tSettings.m_tTokenizer, nullptr, pFilenameBuilder.get(), dWarnings, sError, GetMaxTokenBytes ( tSettings.m_tDict.GetDictFormat() ) );
	if ( !pTokenizer )
	{
		sError.SetSprintf ( "'%s' failed to create tokenizer, error '%s'", sIndexName.cstr(), sError.cstr() );
		return true;
	}

	// dict setup second
	DictRefPtr_c tDict { tSettings.m_tDict.IsWordDict()
		? sphCreateDictionaryKeywords ( tSettings.m_tDict, nullptr, pTokenizer, sIndexName.cstr(), false, tIndexSettings.m_iSkiplistBlockSize, pFilenameBuilder.get(), sError )
		: sphCreateDictionaryCRC ( tSettings.m_tDict, nullptr, pTokenizer, sIndexName.cstr(), false, tIndexSettings.m_iSkiplistBlockSize, pFilenameBuilder.get(), sError ) };
	if ( !tDict )
	{
		sError.SetSprintf ( "'%s' failed to create dictionary, error '%s'", sIndexName.cstr(), sError.cstr() );
		return true;
	}

	// multiforms right after dict
	Tokenizer::AddToMultiformFilterTo ( pTokenizer, tDict->GetMultiWordforms() );

	// bigram filter
	if ( tSettings.m_tIndex.m_eBigramIndex!=SPH_BIGRAM_NONE && BigramNeedsFreq ( tSettings.m_tIndex.m_eBigramIndex ) )
	{
		pTokenizer->SetBuffer ( (BYTE*)const_cast<char*> ( tSettings.m_tIndex.m_sBigramWords.cstr() ), tSettings.m_tIndex.m_sBigramWords.Length() );

		BYTE * pTok = nullptr;
		while ( ( pTok = pTokenizer->GetToken() )!=nullptr )
			tSettings.m_tIndex.m_dBigramWords.Add() = (const char*)pTok;

		tSettings.m_tIndex.m_dBigramWords.Sort();
	}

	bool bNeedExact = ( tDict->HasMorphology() || tDict->GetWordformsFileInfos().GetLength() );
	if ( tSettings.m_tIndex.m_bIndexExactWords && !bNeedExact )
		tSettings.m_tIndex.m_bIndexExactWords = false;

	if ( tDict->GetSettings().IsWordDict() && tDict->HasMorphology() && bIsStarDict && !tSettings.m_tIndex.m_bIndexExactWords )
		tSettings.m_tIndex.m_bIndexExactWords = true;

	const uint64_t pTokenizer_GetSettingsFNV = pTokenizer->GetSettingsFNV();
	const uint64_t tDict_GetSettingsFNV = tDict->GetSettingsFNV();
	const int pTokenizer_GetMaxCodepointLength = pTokenizer->GetMaxCodepointLength();
	const uint64_t sphGetSettingsFNV_tIndexSettings = sphGetSettingsFNV ( tIndexSettings );
	const uint64_t sphGetSettingsFNV_tSettings_m_tIndex = sphGetSettingsFNV ( tSettings.m_tIndex );
	const bool tSettings_m_tMutableSettings_HasSettings = tSettings.m_tMutableSettings.HasSettings();

	// re filter
	bool bReFilterSame = true;
	CSphFieldFilterSettings tFieldFilterSettings;
	if ( pFieldFilter )
		pFieldFilter->GetSettings ( tFieldFilterSettings );
	if ( tFieldFilterSettings.m_dRegexps.GetLength()!=tSettings.m_tFieldFilter.m_dRegexps.GetLength() )
	{
		bReFilterSame = false;
	} else
	{
		CSphVector<uint64_t> dFieldFilter;
		ARRAY_FOREACH ( i, tFieldFilterSettings.m_dRegexps )
			dFieldFilter.Add ( sphFNV64 ( tFieldFilterSettings.m_dRegexps[i].cstr() ) );
		dFieldFilter.Uniq();
		uint64_t uMyFF = sphFNV64 ( dFieldFilter.Begin(), sizeof(dFieldFilter[0]) * dFieldFilter.GetLength() );

		dFieldFilter.Resize ( 0 );
		ARRAY_FOREACH ( i, tSettings.m_tFieldFilter.m_dRegexps )
			dFieldFilter.Add ( sphFNV64 ( tSettings.m_tFieldFilter.m_dRegexps[i].cstr() ) );
		dFieldFilter.Uniq();
		uint64_t uNewFF = sphFNV64 ( dFieldFilter.Begin(), sizeof(dFieldFilter[0]) * dFieldFilter.GetLength() );

		bReFilterSame = ( uMyFF==uNewFF );
	}

	// field filter
	std::unique_ptr<ISphFieldFilter> tFieldFilter;
	bool bPreprocessorSame = ( tIndexSettings.m_ePreprocessor==tSettings.m_tIndex.m_ePreprocessor );
	if ( tSettings.m_tIndex.m_ePreprocessor==Preprocessor_e::JIEBA )
	{
		// The data-dir ALTER path rejects changes to the effective Jieba
		// settings. Preserve its field-filter chain for unrelated changes,
		// rebuilding the full chain only when another filter input changed.
		if ( bReFilterSame && bPreprocessorSame && uTokHash==pTokenizer_GetSettingsFNV && pFieldFilter )
			tFieldFilter = pFieldFilter->Clone();
		else
		{
			if ( tSettings.m_tFieldFilter.m_dRegexps.GetLength() )
			{
				tFieldFilter = sphCreateRegexpFilter ( tSettings.m_tFieldFilter, sError );
				if ( !tFieldFilter )
				{
					sError.SetSprintf ( "'%s' failed to create field filter, error '%s'", sIndexName.cstr (), sError.cstr () );
					return true;
				}
			}

			if ( !SpawnFilterJieba ( tFieldFilter, tSettings.m_tIndex, tSettings.m_tTokenizer, sIndexName.cstr(), pFilenameBuilder.get(), sError ) )
			{
				sError.SetSprintf ( "'%s' failed to create field filter, error '%s'", sIndexName.cstr (), sError.cstr () );
				return true;
			}
		}
	}
	else
	{
		if ( !bReFilterSame && tSettings.m_tFieldFilter.m_dRegexps.GetLength() )
		{
			tFieldFilter = sphCreateRegexpFilter ( tSettings.m_tFieldFilter, sError );
			if ( !tFieldFilter )
			{
				sError.SetSprintf ( "'%s' failed to create field filter, error '%s'", sIndexName.cstr (), sError.cstr () );
				return true;
			}
		}

		// icu filter
		if ( !bPreprocessorSame )
		{
			if ( !sphSpawnFilterICU ( tFieldFilter, tSettings.m_tIndex, tSettings.m_tTokenizer, sIndexName.cstr(), sError ) )
			{
				sError.SetSprintf ( "'%s' failed to create field filter, error '%s'", sIndexName.cstr (), sError.cstr () );
				return true;
			}
		}
	}

	// compare options
	if ( !bSame
		|| bDictFormatChanged
		|| uTokHash!=pTokenizer_GetSettingsFNV
		|| uDictHash!=tDict_GetSettingsFNV
		|| iMaxCodepointLength!=pTokenizer_GetMaxCodepointLength
		|| sphGetSettingsFNV_tIndexSettings!=sphGetSettingsFNV_tSettings_m_tIndex
		|| !bReFilterSame
		|| !bPreprocessorSame
		|| tSettings_m_tMutableSettings_HasSettings )
	{
		tSetup.m_pTokenizer = pTokenizer.Leak();
		tSetup.m_pDict = tDict.Leak();
		tSetup.m_tIndex = tSettings.m_tIndex;
		tSetup.m_pFieldFilter = std::move ( tFieldFilter );
		tSetup.m_tMutableSettings = tSettings.m_tMutableSettings;
		return false;
	}

	return true;
}


bool RtIndex_c::IsSameSettings ( CSphReconfigureSettings & tSettings, CSphReconfigureSetup & tSetup, StrVec_t & dWarnings, CSphString & sError ) const
{
	bool bSame = true;
	if ( tSettings.m_bChangeSchema && m_uSchemaHash!=SchemaFNV ( tSettings.m_tSchema ) )
	{
		tSetup.m_tSchema = tSettings.m_tSchema;
		tSetup.m_bChangeSchema = true;
		bSame = false;
	}

	const bool bWordDict = IsWordDict();
	return CreateReconfigure ( GetName(), IsStarDict ( bWordDict ), GetDictFormat(), m_pFieldFilter.get(), m_tSettings, m_pTokenizer->GetSettingsFNV(), m_pDict->GetSettingsFNV(), m_pTokenizer->GetMaxCodepointLength(),
		GetMemLimit(), bSame, tSettings, tSetup, dWarnings, sError );
}

bool RtIndex_c::Reconfigure ( CSphReconfigureSetup & tSetup )
{
	// strength single-fiber access (don't rely upon to upstream w-lock)
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "Reconfigure" );

	if ( !ForceDiskChunk() )
		return false;

	if ( tSetup.m_bChangeSchema )
		SetSchema ( tSetup.m_tSchema );

	if ( tSetup.m_tMutableSettings.HasSettings() )
	{
		m_tMutableSettings.Combine ( tSetup.m_tMutableSettings );
		SetMemLimit ( m_tMutableSettings.m_iMemLimit );
	}

	bool bWasBinlog = m_tSettings.m_bBinlog;
	Setup ( tSetup.m_tIndex );
	bool bNewBinlog = m_tSettings.m_bBinlog;
	SetTokenizer ( tSetup.m_pTokenizer );
	SetDictionary ( tSetup.m_pDict );
	SetFieldFilter ( std::move ( tSetup.m_pFieldFilter ) );

	m_iMaxCodepointLength = m_pTokenizer->GetMaxCodepointLength();
	SetupQueryTokenizer();

	// FIXME!!! handle error
	m_pTokenizerIndexing = m_pTokenizer->Clone ( SPH_CLONE_INDEX );
	Tokenizer::AddBigramFilterTo ( m_pTokenizerIndexing, m_tSettings.m_eBigramIndex, m_tSettings.m_eBigramDelimiter, m_tSettings.m_sBigramWords, m_sLastError );

	AlterSave ( false );
	RaiseAlterGeneration();

	if ( bWasBinlog != bNewBinlog )
	{
		// AlterSave just performed flush
		m_iSavedTID = m_iTID = bNewBinlog ? Binlog::LastTidFor ( GetName () ) : -1;
		SaveMeta ();
	}

	if ( m_sGlobalIDFPath != m_tMutableSettings.m_sGlobalIDFPath )
		SetGlobalIDFPath ( m_tMutableSettings.m_sGlobalIDFPath );

	return true;
}


int	RtIndex_c::Kill ( DocID_t tDocID )
{
	assert ( 0 && "No external kills for RT");
	return 0;
}


int RtIndex_c::KillMulti ( const VecTraits_T<DocID_t> & /*dKlist*/ )
{
	assert ( 0 && "No external kills for RT");
	return 0;
}

bool RtIndex_c::IsAlive ( DocID_t tDocID ) const
{
	assert ( 0 && "No external kills for RT");
	return false;
}

uint64_t sphGetSettingsFNV ( const CSphIndexSettings & tSettings )
{
	uint64_t uHash = 0;

	DWORD uFlags = 0;
	if ( tSettings.m_bHtmlStrip )
		uFlags |= 1<<1;
	if ( tSettings.m_bIndexExactWords )
		uFlags |= 1<<2;
	if ( tSettings.m_bIndexFieldLens )
		uFlags |= 1<<3;
	if ( tSettings.m_bIndexSP )
		uFlags |= 1<<4;
	if ( tSettings.m_bBinlog )
		uFlags |= 1<<5;
	uHash = sphFNV64 ( &uFlags, sizeof(uFlags), uHash );

	int iMinPrefixLen = tSettings.RawMinPrefixLen();

	uHash = sphFNV64 ( &tSettings.m_eHitFormat, sizeof(tSettings.m_eHitFormat), uHash );
	uHash = sphFNV64 ( tSettings.m_sHtmlIndexAttrs.cstr(), tSettings.m_sHtmlIndexAttrs.Length(), uHash );
	uHash = sphFNV64 ( tSettings.m_sHtmlRemoveElements.cstr(), tSettings.m_sHtmlRemoveElements.Length(), uHash );
	uHash = sphFNV64 ( tSettings.m_sZones.cstr(), tSettings.m_sZones.Length(), uHash );
	uHash = sphFNV64 ( &tSettings.m_eHitless, sizeof(tSettings.m_eHitless), uHash );
	uHash = sphFNV64 ( tSettings.m_sHitlessFiles.cstr(), tSettings.m_sHitlessFiles.Length(), uHash );
	uHash = sphFNV64 ( &tSettings.m_eBigramIndex, sizeof(tSettings.m_eBigramIndex), uHash );
	uHash = sphFNV64 ( &tSettings.m_eBigramDelimiter, sizeof(tSettings.m_eBigramDelimiter), uHash );
	uHash = sphFNV64 ( tSettings.m_sBigramWords.cstr(), tSettings.m_sBigramWords.Length(), uHash );
	uHash = sphFNV64 ( tSettings.m_uAotFilterMask, uHash );
	uHash = sphFNV64 ( &tSettings.m_ePreprocessor, sizeof(tSettings.m_ePreprocessor), uHash );
	uHash = sphFNV64 ( tSettings.m_sIndexTokenFilter.cstr(), tSettings.m_sIndexTokenFilter.Length(), uHash );
	uHash = sphFNV64 ( iMinPrefixLen, uHash );
	uHash = sphFNV64 ( tSettings.m_iMinInfixLen, uHash );
	uHash = sphFNV64 ( tSettings.m_iMaxSubstringLen, uHash );
	uHash = sphFNV64 ( tSettings.m_iBoundaryStep, uHash );
	uHash = sphFNV64 ( tSettings.m_iOvershortStep, uHash );
	uHash = sphFNV64 ( tSettings.m_iStopwordStep, uHash );

	return uHash;
}

StrVec_t RtIndex_c::CollectOwnedExternalFiles () const
{
	// Only data_dir mode installs a filename builder. Config-defined tables
	// merely reference shared external files and must never unlink them.
	auto fnCreateFilenameBuilder = GetIndexFilenameBuilder();
	if ( !fnCreateFilenameBuilder )
		return {};

	auto pFilenameBuilder = fnCreateFilenameBuilder ( GetName() );
	if ( !pFilenameBuilder )
		return {};

	StrVec_t dFiles;
	StrVec_t dExternalFiles;
	GetIndexFiles ( dFiles, dExternalFiles, pFilenameBuilder.get() );
	return dExternalFiles;
}

void RtIndex_c::RemoveOutdatedOwnedExternalFiles ( StrVec_t & dOldFiles ) const
{
	auto dNewFiles = CollectOwnedExternalFiles();
	RemoveOutdatedFiles ( dNewFiles, dOldFiles );
}

void RtIndex_c::GetIndexFiles ( StrVec_t& dFiles, StrVec_t& dExt, const FilenameBuilder_i* pParentFilenameBuilder ) const
{
	auto fnAddFile = [this, &dFiles] ( const auto tExt ) {
		auto sFile = GetFilename ( tExt );
		if ( sphIsReadable ( sFile ) )
			dFiles.Add ( std::move ( sFile ) );
	};

	fnAddFile ( "meta" );
	fnAddFile ( "ram" );

	if ( m_tMutableSettings.NeedSave() ) // should be file already after post-setup
		fnAddFile ( SPH_EXT_SETTINGS );

	std::unique_ptr<FilenameBuilder_i> pFilenameBuilder { nullptr };
	if ( !pParentFilenameBuilder && GetIndexFilenameBuilder() )
	{
		pFilenameBuilder = GetIndexFilenameBuilder() ( GetName() );
		pParentFilenameBuilder = pFilenameBuilder.get();
	}
	GetSettingsFiles ( m_pTokenizer, m_pDict, GetSettings(), pParentFilenameBuilder, dExt );

	RtGuard().m_dDiskChunks.for_each ( [&] ( ConstDiskChunkRefPtr_t& p ) { p->Cidx().GetIndexFiles ( dFiles, dExt, pParentFilenameBuilder ); } );
	dExt.Uniq(); // might be duplicates of tok \ dict files from disk chunks
}

void RtIndex_c::ProhibitSave()
{
	StopOptimize();
	m_tSaving.SetState ( SaveState_c::DISCARD );
	std::atomic_thread_fence ( std::memory_order_release );
	// need also to wait while all disk chunk writers get out of the m_tSaving scope to make sure there will be no new disk chunks after this function
	// might be called on daemon start from the system mock functions - no need to wait there
	if ( Threads::IsInsideCoroutine() )
		m_tNSavesNow.Wait ( [] ( int iVal ) { return iVal == 0; } );
}

void RtIndex_c::EnableSave()
{
	m_tSaving.SetState ( SaveState_c::ENABLED );
	m_bOptimizeStop.store ( false, std::memory_order_relaxed );
	std::atomic_thread_fence ( std::memory_order_release );
}

// fixme! Review, if it still necessary, as SST locks everything itself.
void RtIndex_c::LockFileState ( CSphVector<CSphString>& dFiles )
{
	StopOptimize();
	ForceRamFlush ( "forced" );
	CSphString sError;
	SaveAttributes ( sError ); // fixme! report error, better discard whole locking
	// that will ensure, if current txn is applying, it will be finished (especially kill pass) before we continue.
	assert ( Threads::IsInsideCoroutine());
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	m_tSaving.SetState ( SaveState_c::DISABLED );
	std::atomic_thread_fence ( std::memory_order_release );
	// need also to wait while all disk chunk writers get out of the m_tSaving scope to make sure there will be no new disk chunks after this function
	m_tNSavesNow.Wait ( [] ( int iVal ) { return iVal == 0; } );

	GetIndexFiles ( dFiles, dFiles );
}

void RtIndex_c::WaitLockEnabledState () noexcept
{
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess () );
	m_tSaving.WaitLockEnabledState ();
}

void RtIndex_c::UnlockEnabledState () noexcept
{
	m_tSaving.UnlockEnabledState ();
}

void RtIndex_c::CreateReader ( int64_t iSessionId ) const
{
	// rt docstore doesn't need buffered readers
	// but disk chunks need them
	auto tGuard = RtGuard();

	for ( const auto & i : tGuard.m_dDiskChunks )
	{
		assert(i);
		i->Cidx().CreateReader ( iSessionId );
	}
}


bool RtIndex_c::GetDoc ( DocstoreDoc_t & tDoc, DocID_t tDocID, const VecTraits_T<int> * pFieldIds, int64_t iSessionId, bool bPack ) const
{
	auto tGuard = RtGuard();

	for ( const auto & i : tGuard.m_dRamSegs )
	{
		assert ( i && i->m_pDocstore );
		RowID_t tRowID = i->GetRowidByDocid(tDocID);
		if ( tRowID==INVALID_ROWID || i->m_tDeadRowMap.IsSet(tRowID) )
			continue;

		tDoc = i->m_pDocstore->GetDoc ( tRowID, pFieldIds, iSessionId, bPack );
		return true;
	}

	for ( const auto & i : tGuard.m_dDiskChunks )
	{
		assert(i);
		if ( i->Cidx().GetDoc ( tDoc, tDocID, pFieldIds, iSessionId, bPack ) )
			return true;
	}

	return false;
}


int RtIndex_c::GetFieldId ( const CSphString & sName, DocstoreDataType_e eType ) const
{
	return m_pDocstoreFields ? m_pDocstoreFields->GetFieldId ( sName, eType ) : -1;
}

Bson_t RtIndex_c::ExplainQuery ( const CSphString & sQuery ) const
{
	ExplainQueryArgs_t tArgs;
	tArgs.m_szQuery = sQuery.cstr();
	tArgs.m_pSchema = &GetMatchSchema();
	const bool bWordDict = IsWordDict();

	TokenizerRefPtr_c pQueryTokenizer { sphCloneAndSetupQueryTokenizer ( m_pTokenizer, IsStarDict ( bWordDict ), m_tSettings.m_bIndexExactWords, false ) };
	SetupStarTokenizer ( pQueryTokenizer );
	SetupExactTokenizer ( pQueryTokenizer );

	tArgs.m_pDict = GetStatelessDict ( m_pDict );
	SetupStarDict ( tArgs.m_pDict );
	SetupExactDict ( tArgs.m_pDict );
	if ( m_pFieldFilter )
		tArgs.m_pFieldFilter = m_pFieldFilter->Clone();
	tArgs.m_pSettings = &m_tSettings;
	tArgs.m_pWordlist = this;
	tArgs.m_pQueryTokenizer = pQueryTokenizer;
	tArgs.m_iExpandKeywords = m_tMutableSettings.m_iExpandKeywords;
	tArgs.m_iExpansionLimit = m_iExpansionLimit;
	tArgs.m_bExpandPrefix = ( bWordDict && IsStarDict ( bWordDict ) );
	tArgs.m_pMorphFields = &m_tMorphFields;

	auto tGuard = RtGuard();
	tArgs.m_pIndexData = tGuard.m_tSegmentsAndChunks.m_pSegs;

	return Explain ( tArgs );
}

bool RtIndex_c::NeedStoreWordID () const
{
	return ( m_tSettings.m_eHitless==SPH_HITLESS_SOME && m_dHitlessWords.GetLength() );
}

//////////////////////////////////////////////////////////////////////////


std::unique_ptr<RtIndex_i> sphCreateIndexRT ( CSphString sIndexName, CSphString sPath, CSphSchema tSchema, int64_t iRamSize )
{
	MEMORY ( MEM_INDEX_RT );
	return std::make_unique<RtIndex_c> ( std::move ( sIndexName ), std::move ( sPath ), std::move ( tSchema ), iRamSize );
}


void sphRTInit ( CSphString sBinlogPath, bool bCommonBinlog, const CSphConfigSection * pCommon )
{
	Binlog::Init ( std::move ( sBinlogPath ) );
	Binlog::SetCommon ( bCommonBinlog );
	if ( pCommon )
		g_bProgressiveMerge = pCommon->GetBool ( "progressive_merge", true );
}


static bool g_bTestMode = false;

void sphRTSetTestMode ()
{
	g_bTestMode = true;
	SetTightConcurrencyLimits();
}


static void SetColumnarFlag ( CSphColumnInfo & tCol, const CSphIndexSettings & tSettings )
{
	bool bAllColumnar = false;
	for ( const auto & i : tSettings.m_dColumnarAttrs )
		bAllColumnar |= i=="*";	

	for ( const auto & i : tSettings.m_dColumnarAttrs )
		if ( ( i==tCol.m_sName || bAllColumnar ) && tCol.m_eAttrType!=SPH_ATTR_JSON )
			tCol.m_uAttrFlags |= CSphColumnInfo::ATTR_COLUMNAR;
}


static void SetKNNFlag ( CSphColumnInfo & tCol, const CSphIndexSettings & tSettings )
{
	for ( const auto & i : tSettings.m_dKNN )
		if ( i.m_sName==tCol.m_sName )
			tCol.m_uAttrFlags |= CSphColumnInfo::ATTR_INDEXED_KNN;
}


bool sphRTSchemaConfigure ( const CSphConfigSection & hIndex, CSphSchema & tSchema, const CSphIndexSettings & tSettings, StrVec_t * pWarnings, CSphString & sError, bool bSkipValidation, bool bPQ )
{
	// fields
	SmallStringHash_T<BYTE> hFields;
	for ( CSphVariant * v=hIndex("rt_field"); v; v=v->m_pNext )
	{
		CSphString sFieldName = v->cstr();
		sFieldName.ToLower();
		tSchema.AddField ( sFieldName.cstr() );
		hFields.Add ( 1, sFieldName );
	}

	if ( tSchema.GetFieldsCount()>SPH_MAX_FIELDS )
	{
		sError.SetSprintf ( "too many fields (fields=%d, max=%d)", tSchema.GetFieldsCount(), SPH_MAX_FIELDS );
		return false;
	}

	// add id column
	CSphColumnInfo tDocIdCol ( sphGetDocidName() );
	tDocIdCol.m_eAttrType = SPH_ATTR_BIGINT;
	for ( CSphVariant * v = hIndex ( "rt_attr_string" ); v; v = v->m_pNext )
	{
		StrVec_t dNameParts;
		sphSplit ( dNameParts, v->cstr(), ":" );
		CSphString sAttrName = dNameParts[0];
		sAttrName.ToLower();
		if ( sAttrName==sphGetUuidDocidName() )
		{
			tDocIdCol.m_uAttrFlags |= CSphColumnInfo::ATTR_UUID_LINK;
			break;
		}
	}

	if ( !bPQ )
		SetColumnarFlag ( tDocIdCol, tSettings );

	tSchema.AddAttr ( tDocIdCol, false );

	// attrs
	constexpr int iNumTypes = 10;
	const char * sTypes[iNumTypes] = { "rt_attr_uint", "rt_attr_bigint", "rt_attr_timestamp", "rt_attr_bool", "rt_attr_float", "rt_attr_string", "rt_attr_json", "rt_attr_multi", "rt_attr_multi_64", "rt_attr_float_vector" };
	const ESphAttr iTypes[iNumTypes] = { SPH_ATTR_INTEGER, SPH_ATTR_BIGINT, SPH_ATTR_TIMESTAMP, SPH_ATTR_BOOL, SPH_ATTR_FLOAT, SPH_ATTR_STRING, SPH_ATTR_JSON, SPH_ATTR_UINT32SET, SPH_ATTR_INT64SET, SPH_ATTR_FLOAT_VECTOR };

	CSphVector<std::pair<int, CSphColumnInfo>> dOrderedColumns;

	for ( int iType = 0; iType < iNumTypes; ++iType )
	{
		for ( CSphVariant * v = hIndex ( sTypes[iType] ); v; v = v->m_pNext )
		{
			StrVec_t dNameParts;
			sphSplit ( dNameParts, v->cstr(), ":");
			CSphColumnInfo tCol ( dNameParts[0].cstr(), iTypes[iType]);
			tCol.m_sName.ToLower();

			// ignore doc id, it was added via create table to pass id attribute settings
			if ( tCol.m_sName==sphGetDocidName() )
				continue;

			// bitcount
			tCol.m_tLocator = CSphAttrLocator();
			if ( dNameParts.GetLength ()>1 )
			{
				if ( tCol.m_eAttrType==SPH_ATTR_INTEGER )
				{
					auto iBits = strtol ( dNameParts[1].cstr(), nullptr, 10 );
					if ( iBits>0 && iBits<=ROWITEM_BITS )
						tCol.m_tLocator.m_iBitCount = (int)iBits;
					else
						sError.SetSprintf ( "attribute '%s': invalid bitcount=%d (bitcount ignored)", tCol.m_sName.cstr(), (int)iBits );

				} else
					sError.SetSprintf ( "attribute '%s': bitcount is only supported for integer types (bitcount ignored)", tCol.m_sName.cstr() );
			}

			dOrderedColumns.Add ( { v->m_iTag, tCol } );
		}
	}

	dOrderedColumns.Sort ( Lesser ( [] ( const auto& a, const auto& b ) { return a.first  <  b.first; } ) );

	for ( auto& tOrderedCol : dOrderedColumns )
	{
		auto& tCol = tOrderedCol.second;
		if ( !SchemaConfigureCheckAttribute ( tSchema, tCol, sError ) )
			return false;

		if ( !bPQ )
		{
			SetColumnarFlag ( tCol, tSettings );
			SetKNNFlag ( tCol, tSettings );
		}

		tSchema.AddAttr ( tCol, false );

		if ( tCol.m_eAttrType != SPH_ATTR_STRING && hFields.Exists ( tCol.m_sName ) && !bSkipValidation )
		{
			sError.SetSprintf ( "can not add attribute that shadows '%s' field", tCol.m_sName.cstr() );
			return false;
		}
	}

	// add blob attr locator
	if ( tSchema.HasBlobAttrs() )
	{
		CSphColumnInfo tBlobLocCol ( sphGetBlobLocatorName() );
		tBlobLocCol.m_eAttrType = SPH_ATTR_BIGINT;

		// should be right after docid
		tSchema.InsertAttr ( 1, tBlobLocCol, false );

		// rebuild locators in the schema
		const char * szTmpColName = "$_tmp";
		CSphColumnInfo tTmpCol ( szTmpColName, SPH_ATTR_BIGINT );
		tSchema.AddAttr ( tTmpCol, false );
		tSchema.RemoveAttr ( szTmpColName, false );
	}

	if ( !tSchema.GetAttrsCount() && !g_bTestMode && !bSkipValidation )
	{
		sError.SetSprintf ( "no attribute configured (use rt_attr directive)" );
		return false;
	}

	return true;
}



void RtIndex_c::SetSchema ( CSphSchema tSchema )
{
	m_tSchema = std::move ( tSchema );
	m_iStride = m_tSchema.GetRowSize();
	m_uSchemaHash = SchemaFNV ( m_tSchema );

	const bool bUuidDocid = m_tSchema.GetAttrsCount() && m_tSchema.GetAttr(0).IsUuidLinkedDocid();
	if ( bUuidDocid && !m_pUuidRegistry )
		m_pUuidRegistry = CreateUuidDocidRegistry();
	else if ( !bUuidDocid && m_pUuidRegistry )
		m_pUuidRegistry = nullptr;

	if ( m_tSchema.HasStoredFields() || m_tSchema.HasStoredAttrs() )
	{
		m_pDocstoreFields = CreateDocstoreFields();
		SetupDocstoreFields ( *m_pDocstoreFields.get(), m_tSchema );
	}

	ARRAY_FOREACH ( i, m_dFieldLens )
	{
		m_dFieldLens[i] = 0;
		m_dFieldLensRam[i] = 0;
		m_dFieldLensDisk[i] = 0;
	}
}

uint64_t SchemaFNV ( const ISphSchema & tSchema )
{
	uint64_t uHash = SPH_FNV64_SEED;

	// attrs
	int iAttrsCount = tSchema.GetAttrsCount();
	for ( int i=0; i<iAttrsCount; i++ )
	{
		const CSphColumnInfo & tAttr = tSchema.GetAttr ( i );
		uHash = sphFNV64cont ( tAttr.m_sName.cstr(), uHash );
		uHash = sphFNV64 ( &tAttr.m_eAttrType, sizeof( tAttr.m_eAttrType ), uHash );
		uHash = tAttr.m_tLocator.FNV ( uHash );
	}

	// fulltext fields
	int iFieldsCount = tSchema.GetFieldsCount();
	for ( int i=0; i<iFieldsCount; i++ )
	{
		const CSphColumnInfo & tField = tSchema.GetField ( i );
		uHash = sphFNV64cont ( tField.m_sName.cstr(), uHash );
		uHash = sphFNV64 ( &tField.m_uFieldFlags, sizeof( tField.m_uFieldFlags ), uHash );
	}

	return uHash;
}

void RtIndex_c::SetMemLimit ( int64_t iMemLimit )
{
	m_iRtMemLimit = iMemLimit;
	m_iSoftRamLimit = m_iRtMemLimit * m_fSaveRateLimit;
}

void RtIndex_c::RecalculateRateLimit ( int64_t iSaved, int64_t iInserted, bool bEmergent )
{
	if ( ( iSaved + iInserted ) > 0 )
	{
		auto fRate = (double)iSaved / ( iSaved + iInserted );
		if ( bEmergent ) // emergent save happened
		{
			m_fSaveRateLimit -= SAVE_RATE_LIMIT_EMERGENCY_STEP;
			m_fSaveRateLimit = Min ( m_fSaveRateLimit, fRate );
		} else
			m_fSaveRateLimit = fRate;
	}

	m_fSaveRateLimit = Min ( MAX_SAVE_RATE_LIMIT, m_fSaveRateLimit );
	m_fSaveRateLimit = Max ( MIN_SAVE_RATE_LIMIT, m_fSaveRateLimit );
	m_iSoftRamLimit = m_iRtMemLimit * m_fSaveRateLimit;

	TRACE_COUNTER ( "mem", perfetto::CounterTrack ( "Ratio", "%" ), m_fSaveRateLimit );
}

bool RtIndex_c::AlterRebuild ( AlterOp_fn && fnOp, CSphString & sError, const char * sTrace )
{
	// strength single-fiber access (don't rely upon to upstream w-lock)
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", perfetto::StaticString{sTrace} );

	auto pChunks = m_tRtChunks.DiskChunks();
	for ( auto & tChunk : *pChunks )
	{
		if ( !fnOp ( tChunk->CastIdx(), sError ) )
			return false;
	}

	RaiseAlterGeneration();
	return true;
}

bool RtIndex_c::AlterSI ( CSphString & sError )
{
	return AlterRebuild ( []( CSphIndex & tIndex, CSphString & sError ) { return tIndex.AlterSI ( sError ); }, sError, "alter-si" );
}

bool RtIndex_c::AlterKNN ( CSphString & sError )
{
	return AlterRebuild ( []( CSphIndex & tIndex, CSphString & sError ) { return tIndex.AlterKNN ( sError ); }, sError, "alter-knn" );
}

bool RtIndex_c::AlterApiKey ( const CSphString & sAttr, const CSphString & sKey, CSphString & sError )
{
	// strength single-fiber access (don't rely upon to upstream w-lock)
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
//	TRACE_SCHED ( "rt", perfetto::StaticString{sTrace} );

	auto pAttr = m_tSchema.GetAttr ( sAttr.cstr() );
	if ( !pAttr )
	{
		sError.SetSprintf ( "attribute '%s' not found", sAttr.cstr() );
		return false;
	}

	if ( pAttr->m_tKNNModel.m_sModelName.empty() )
	{
		sError.SetSprintf ( "no embeddings model specified for attribute '%s'", sAttr.cstr() );
		return false;
	}

	std::string sOldKey = pAttr->m_tKNNModel.m_sAPIKey;
	const_cast<CSphColumnInfo *>(pAttr)->m_tKNNModel.m_sAPIKey = sKey.cstr();

	m_pEmbeddings.reset();
	if ( !LoadEmbeddingModels(sError) )
	{
		// attempt to revert
		const_cast<CSphColumnInfo *>(pAttr)->m_tKNNModel.m_sAPIKey = sOldKey;
		m_pEmbeddings.reset();
		CSphString sRevertError;
		LoadEmbeddingModels(sRevertError);
		return false;
	}

	RaiseAlterGeneration();
	AlterSave(false);

	return true;
}


bool RtIndex_c::AlterApiUrl ( const CSphString & sAttr, const CSphString & sUrl, CSphString & sError )
{
	// strength single-fiber access (don't rely upon to upstream w-lock)
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );

	auto pAttr = m_tSchema.GetAttr ( sAttr.cstr() );
	if ( !pAttr )
	{
		sError.SetSprintf ( "attribute '%s' not found", sAttr.cstr() );
		return false;
	}

	if ( pAttr->m_tKNNModel.m_sModelName.empty() )
	{
		sError.SetSprintf ( "no embeddings model specified for attribute '%s'", sAttr.cstr() );
		return false;
	}

	std::string sOldUrl = pAttr->m_tKNNModel.m_sAPIUrl;
	bool bWasCustomUrl = !sOldUrl.empty();
	bool bIsRemovingCustomUrl = bWasCustomUrl && sUrl.IsEmpty();
	
	// Allow empty string to remove/reset API_URL (use default URL)
	const_cast<CSphColumnInfo *>(pAttr)->m_tKNNModel.m_sAPIUrl = sUrl.cstr();

	m_pEmbeddings.reset();
	if ( !LoadEmbeddingModels(sError) )
	{
		const_cast<CSphColumnInfo *>(pAttr)->m_tKNNModel.m_sAPIUrl = sOldUrl;
		m_pEmbeddings.reset();
		CSphString sRevertError;
		LoadEmbeddingModels(sRevertError);

		const char * szError = sError.cstr();
		if ( bIsRemovingCustomUrl && szError && ( strstr ( szError, "Invalid API key" ) || strstr ( szError, "API key" ) ) )
			sError.SetSprintf ( "cannot remove API_URL: API key validation failed for the default endpoint. The API key may be invalid, expired, or not authorized for the default provider endpoint. To remove API_URL, first update the API key to a valid key that works with the default endpoint, or keep using a custom API_URL" );

		return false;
	}

	RaiseAlterGeneration();
	AlterSave(false);

	return true;
}


bool RtIndex_c::AlterApiTimeout ( const CSphString & sAttr, int iTimeout, CSphString & sError )
{
	// strength single-fiber access (don't rely upon to upstream w-lock)
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );

	auto pAttr = m_tSchema.GetAttr ( sAttr.cstr() );
	if ( !pAttr )
	{
		sError.SetSprintf ( "attribute '%s' not found", sAttr.cstr() );
		return false;
	}

	if ( pAttr->m_tKNNModel.m_sModelName.empty() )
	{
		sError.SetSprintf ( "no embeddings model specified for attribute '%s'", sAttr.cstr() );
		return false;
	}

	if ( iTimeout < 0 )
	{
		sError = GetAPITimeoutErrorMsg();
		return false;
	}

	// 0 means use default timeout (10 seconds), positive value is timeout in seconds
	int iOldTimeout = pAttr->m_tKNNModel.m_iAPITimeout;
	const_cast<CSphColumnInfo *>(pAttr)->m_tKNNModel.m_iAPITimeout = iTimeout;

	m_pEmbeddings.reset();
	if ( !LoadEmbeddingModels(sError) )
	{
		const_cast<CSphColumnInfo *>(pAttr)->m_tKNNModel.m_iAPITimeout = iOldTimeout;
		m_pEmbeddings.reset();
		CSphString sRevertError;
		LoadEmbeddingModels(sRevertError);
		return false;
	}

	RaiseAlterGeneration();
	AlterSave(false);

	return true;
}

void RtIndex_c::SetGlobalIDFPath ( const CSphString & sPath )
{
	m_sGlobalIDFPath = sPath;

	auto pChunks = m_tRtChunks.DiskChunks();
	for ( auto & pChunk : *pChunks )
		pChunk->CastIdx().SetGlobalIDFPath ( m_sGlobalIDFPath );
}

void RtIndex_c::DebugDumpDict ( FILE * fp, bool bDumpOnly )
{
	if ( !IsWordDict() )
		sphDie ( "DebugDumpDict() only supports dict=keywords for now" );
	if ( GetDictFormat()==DictFormat_e::KEYWORDS_V2 )
		sphDie ( "DebugDumpDict() does not support dict=keywords_32k yet" );

	if ( !bDumpOnly )
		fprintf ( fp, "keyword,docs,hits,offset\n" );

	auto tGuard = RtGuard();

	for ( const auto & pSeg : tGuard.m_dRamSegs )
	{
		RtWordReader_c tRdWord ( pSeg, GetDictFormat(), m_iWordsCheckpoint, m_tSettings.m_eHitless );
		while ( tRdWord.UnzipWord() )
			fprintf ( fp, "%s,%u,%u,0\n", tRdWord->m_sWord, tRdWord->m_uDocs, tRdWord->m_uHits );

	}

	for ( auto & tDiskChunk : tGuard.m_dDiskChunks )
		tDiskChunk->CastIdx().DebugDumpDict ( fp, true );
}

//////////////////////////////////////////////////////////////////////////

static bool SettingsAddExt ( IndexSettingsContainer_i & tContainer, const char * szName, const CSphString & sValue, const FilenameBuilder_i * pFileBuilder, CSphString & sError )
{
	if ( sValue.IsEmpty() )
		return true;

	StringBuilder_c sPaths ( " " );
	sphSplitApply ( sValue.cstr(), sValue.Length(), " \t,", [&sPaths, pFileBuilder] ( const char * sToken, int iLen )
	{
		if ( !iLen )
			return;

		CSphString sFile;
		sFile.SetBinary ( sToken, iLen );
		CSphString sPath = FormatPath ( sFile, pFileBuilder );
		if ( !sPath.IsEmpty() )
			sPaths << sPath;
	} );

	if ( sPaths.IsEmpty() )
		return true;

	CSphString sPath = (CSphString)sPaths;
	if ( !tContainer.AddOption ( szName, sPath, true ) )
	{
		sError = tContainer.GetError();
		return false;
	}

	return true;
}

static bool CollectExtSettings ( IndexSettingsContainer_i & tContainer, const CSphIndex & tSrcIndex, const FilenameBuilder_i * pSrcFileBuilder, CSphString & sError )
{
	const CSphIndexSettings & tSettings = tSrcIndex.GetSettings();
	const CSphTokenizerSettings & tTokSettings = tSrcIndex.GetTokenizer()->GetSettings();
	const CSphDictSettings & tDictSettings = tSrcIndex.GetDictionary()->GetSettings();

	if ( !SettingsAddExt ( tContainer, "stopwords", tDictSettings.m_sStopwords, pSrcFileBuilder, sError ) )
		return false;

	if ( !SettingsAddExt ( tContainer, "wordforms", Vec2Str ( tDictSettings.m_dWordforms, " " ), pSrcFileBuilder, sError ) )
		return false;

	if ( !SettingsAddExt ( tContainer, "exceptions", tTokSettings.m_sSynonymsFile, pSrcFileBuilder, sError ) )
		return false;

	if ( !SettingsAddExt ( tContainer, "hitless_words", tSettings.m_sHitlessFiles, pSrcFileBuilder, sError ) )
		return false;

	if ( !SettingsAddExt ( tContainer, "jieba_user_dict_path", tSettings.m_sJiebaUserDictPath, pSrcFileBuilder, sError ) )
		return false;

	return true;
}

static uint64_t HashFileInfo ( const CSphSavedFile & tInfo )
{
	uint64_t uHash = SPH_FNV64_SEED;
	uHash = sphFNV64 ( &tInfo.m_uSize, sizeof ( tInfo.m_uSize ), uHash );
	uHash = sphFNV64 ( &tInfo.m_uCRC32, sizeof ( tInfo.m_uCRC32 ), uHash );
	return uHash;
}

static bool HashExtFile ( const CSphString & sFile, const FilenameBuilder_i * pFileBuilder, uint64_t & uHash, CSphString & sError )
{
	uHash = 0;

	if ( sFile.IsEmpty() )
		return true;

	CSphString sPath = FormatPath ( sFile, pFileBuilder );
	CSphSavedFile tInfo;
	if ( !tInfo.Collect ( sPath.cstr(), &sError ) )
		return false;

	uHash = HashFileInfo ( tInfo );
	return true;
}

static bool HashExtFileList ( const StrVec_t & dFiles, const FilenameBuilder_i * pFileBuilder, uint64_t & uHash, CSphString & sError )
{
	uHash = 0;

	if ( dFiles.IsEmpty() )
		return true;

	CSphVector<uint64_t> dHashes;
	dHashes.Reserve ( dFiles.GetLength() );
	for ( const auto & sFile : dFiles )
	{
		uint64_t uVal = 0;
		if ( !HashExtFile ( sFile, pFileBuilder, uVal, sError ) )
			return false;
		if ( uVal )
			dHashes.Add ( uVal );
	}

	// need order‑independent file name list
	dHashes.Sort();
	uHash = SPH_FNV64_SEED;
	for ( auto uVal : dHashes )
		uHash = sphFNV64 ( &uVal, sizeof ( uVal ), uHash );

	return true;
}

static bool BuildExtCacheEntry ( const CSphIndex & tIndex, const FilenameBuilder_i * pFileBuilder, ExternalFilesCacheEntry_t & tEntry, CSphString & sError )
{
	const CSphIndexSettings & tSettings = tIndex.GetSettings();
	const CSphTokenizerSettings & tTokSettings = tIndex.GetTokenizer()->GetSettings();
	const CSphDictSettings & tDictSettings = tIndex.GetDictionary()->GetSettings();

	StrVec_t dStopwords = sphSplit ( tDictSettings.m_sStopwords.cstr(), " \t," );
	StrVec_t dHitless = sphSplit ( tSettings.m_sHitlessFiles.cstr(), " \t,"  );

	uint64_t uStopwords = 0;
	uint64_t uWordforms = 0;
	uint64_t uExceptions = 0;
	uint64_t uHitless = 0;
	uint64_t uJieba = 0;

	if ( !HashExtFileList ( dStopwords, pFileBuilder, uStopwords, sError ) )
		return false;
	if ( !HashExtFileList ( tDictSettings.m_dWordforms, pFileBuilder, uWordforms, sError ) )
		return false;
	if ( !HashExtFile ( tTokSettings.m_sSynonymsFile, pFileBuilder, uExceptions, sError ) )
		return false;
	if ( !HashExtFileList ( dHitless, pFileBuilder, uHitless, sError ) )
		return false;
	if ( !HashExtFile ( tSettings.m_sJiebaUserDictPath, pFileBuilder, uJieba, sError ) )
		return false;

	uint64_t uHash = SPH_FNV64_SEED;
	uHash = sphFNV64 ( "stopwords", 9, uHash );
	uHash = sphFNV64 ( uStopwords, uHash );
	uHash = sphFNV64 ( "wordforms", 9, uHash );
	uHash = sphFNV64 ( uWordforms, uHash );
	uHash = sphFNV64 ( "exceptions", 10, uHash );
	uHash = sphFNV64 ( uExceptions, uHash );
	uHash = sphFNV64 ( "hitless", 7, uHash );
	uHash = sphFNV64 ( uHitless, uHash );
	uHash = sphFNV64 ( "jieba", 5, uHash );
	uHash = sphFNV64 ( uJieba, uHash );

	tEntry.m_uHash = uHash;
	tEntry.m_sStopwords = tDictSettings.m_sStopwords;
	tEntry.m_sWordforms = Vec2Str ( tDictSettings.m_dWordforms, " " );
	tEntry.m_sExceptions = tTokSettings.m_sSynonymsFile;
	tEntry.m_sHitless = tSettings.m_sHitlessFiles;
	tEntry.m_sJieba = tSettings.m_sJiebaUserDictPath;

	return true;
}

static bool StoreExtCacheEntry ( const CSphIndex & tIndex, const FilenameBuilder_i * pDstFileBuilder, ExtFiles_h & hExtCache, CSphString & sError )
{
	ExternalFilesCacheEntry_t tEntry;
	if ( !BuildExtCacheEntry ( tIndex, pDstFileBuilder, tEntry, sError ) )
		return false;

	hExtCache.Add ( tEntry.m_uHash, tEntry );
	return true;
}

static bool ApplyExtSettings ( FilenameBuilder_i * pDstFileBuilder, CSphIndex & tDstIndex, IndexSettingsContainer_i & tContainer, CSphString sError )
{
	const bool bHasAny = ( tContainer.Contains ( "hitless_words" ) || tContainer.Contains ( "jieba_user_dict_path" ) || tContainer.Contains ( "exceptions" ) || tContainer.Contains ( "stopwords" ) || tContainer.Contains ( "wordforms" ) );
	if ( !bHasAny )
		return true;

	CSphIndexSettings tSettings = tDstIndex.GetSettings();
	CSphTokenizerSettings tTokSettings = tDstIndex.GetTokenizer()->GetSettings();
	CSphDictSettings tDictSettings = tDstIndex.GetDictionary()->GetSettings();

	if ( tContainer.Contains ( "hitless_words" ) )
		tSettings.m_sHitlessFiles = tContainer.GetList ( "hitless_words" );

	if ( tContainer.Contains ( "jieba_user_dict_path" ) )
		tSettings.m_sJiebaUserDictPath = tContainer.Get ( "jieba_user_dict_path" );

	if ( tContainer.Contains ( "exceptions" ) )
		tTokSettings.m_sSynonymsFile = tContainer.Get ( "exceptions" );

	if ( tContainer.Contains ( "stopwords" ) )
		tDictSettings.m_sStopwords = tContainer.GetList ( "stopwords" );

	if ( tContainer.Contains ( "wordforms" ) )
	{
		CSphString sWordforms = tContainer.GetList ( "wordforms" );
		StrVec_t dWordforms = sphSplit ( sWordforms.cstr(), " \t," );
		tDictSettings.m_dWordforms = std::move ( dWordforms );
	}

	StrVec_t dWarnings;
	TokenizerRefPtr_c pTokenizer = Tokenizer::Create ( tTokSettings, nullptr, pDstFileBuilder, dWarnings, sError, GetMaxTokenBytes ( tDictSettings.GetDictFormat() ) );
	if ( !pTokenizer )
		return false;

	DictRefPtr_c pDict { tDictSettings.IsWordDict()
		? sphCreateDictionaryKeywords ( tDictSettings, nullptr, pTokenizer, tDstIndex.GetName(), false, tSettings.m_iSkiplistBlockSize, pDstFileBuilder, sError )
		: sphCreateDictionaryCRC ( tDictSettings, nullptr, pTokenizer, tDstIndex.GetName(), false, tSettings.m_iSkiplistBlockSize, pDstFileBuilder, sError )};
	if ( !pDict )
		return false;

	tDstIndex.SetDictionary ( pDict );

	Tokenizer::AddToMultiformFilterTo ( pTokenizer, pDict->GetMultiWordforms() );
	tDstIndex.SetTokenizer ( pTokenizer );

	tDstIndex.Setup ( tSettings );
	tDstIndex.PostSetup();
	return true;
}

static bool ApplyCachedExtSettings ( FilenameBuilder_i * pDstFileBuilder, CSphIndex & tDstIndex, const ExternalFilesCacheEntry_t & tEntry, CSphString sError )
{
	std::unique_ptr<IndexSettingsContainer_i> pContainer { CreateIndexSettingsContainer() };
	if ( !tEntry.m_sStopwords.IsEmpty() )
		pContainer->Add ( "stopwords", tEntry.m_sStopwords );
	if ( !tEntry.m_sWordforms.IsEmpty() )
		pContainer->Add ( "wordforms", tEntry.m_sWordforms );
	if ( !tEntry.m_sExceptions.IsEmpty() )
		pContainer->Add ( "exceptions", tEntry.m_sExceptions );
	if ( !tEntry.m_sHitless.IsEmpty() )
		pContainer->Add ( "hitless_words", tEntry.m_sHitless );
	if ( !tEntry.m_sJieba.IsEmpty() )
		pContainer->Add ( "jieba_user_dict_path", tEntry.m_sJieba );

	return ApplyExtSettings ( pDstFileBuilder, tDstIndex, *pContainer, sError );
}

static bool AttachRtChunkExtCopy ( const CSphIndex & tSrcIndex, CSphIndex & tDstIndex, int iChunk, const FilenameBuilder_i * pSrcFileBuilder, FilenameBuilder_i * pDstFileBuilder,
	ExtFiles_h & hExtCache, const CSphString & sDstPath, CSphString & sError )
{
	ExternalFilesCacheEntry_t tSrcEntry;
	if ( !BuildExtCacheEntry ( tSrcIndex, pSrcFileBuilder, tSrcEntry, sError ) )
		return false;

	auto * pCached = hExtCache.Find ( tSrcEntry.m_uHash );
	if ( pCached )
	{
		if ( !ApplyCachedExtSettings ( pDstFileBuilder, tDstIndex, *pCached, sError ) )
			return false;

	} else
	{
		auto pChunkContainer = std::unique_ptr<IndexSettingsContainer_i> { CreateIndexSettingsContainer() };
		if ( !CollectExtSettings ( *pChunkContainer, tSrcIndex, pSrcFileBuilder, sError ) )
			return false;

		if ( !pChunkContainer->CopyExternalFiles ( sDstPath, iChunk ) )
		{
			sError = pChunkContainer->GetError();
			return false;
		}

		if ( !ApplyExtSettings ( pDstFileBuilder, tDstIndex, *pChunkContainer, sError ) )
			return false;

		pChunkContainer->ResetCleanup();

		if ( !StoreExtCacheEntry ( tDstIndex, pDstFileBuilder, hExtCache, sError ) )
			return false;
	}

	return true;
}

bool RtIndex_c::AttachRtChunksExtCopy ( RtIndex_c * pSrcRtIndex, bool & bFatal, ExtFiles_h & hExtCache, CSphString & sError ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	// prevent optimize to start during the disk chunks stealing
	OptimizeGuard_c tSrcStopOptimize ( *pSrcRtIndex );
	OptimizeGuard_c tDstStopOptimize ( *this );

	std::unique_ptr<FilenameBuilder_i> pSrcFileBuilder { GetIndexFilenameBuilder() ( pSrcRtIndex->GetName() ) };
	std::unique_ptr<FilenameBuilder_i> pDstFileBuilder { GetIndexFilenameBuilder() ( GetName() ) };

	if ( !InitExtCache ( pDstFileBuilder.get(), hExtCache, sError ) )
		return false;

	CSphString sDstPath = GetPathOnly ( GetFilebase() );
	LazyVector_T<ConstDiskChunkRefPtr_t> dOthers;

	for ( ;; )
	{
		ConstDiskChunkRefPtr_t tChunk = pSrcRtIndex->PopDiskChunk();
		if ( !tChunk )
			break;

		auto * pChunkIndex = static_cast<CSphIndex *>( *tChunk );
		tChunk->m_bFinallyUnlink = true; // destroy the disk chunks on failure
		if ( !AttachDiskChunkMove ( pChunkIndex, bFatal, sError ) )
		{
			bFatal = true; // need to destroy source index in case of failure as it does not have right amount of disk chunks anymore
			return false;
		}

		int iChunk = pChunkIndex->m_iChunk;
		if ( !AttachRtChunkExtCopy ( *pChunkIndex, *pChunkIndex, iChunk, pSrcFileBuilder.get(), pDstFileBuilder.get(), hExtCache, sDstPath, sError ) )
			return false;

		if ( !tChunk->Cidx().RewriteHeader ( sError ) )
			return false;

		dOthers.Add ( tChunk );
	}

	// update disk chunk list
	auto tNewSet = RtWriter();
	CopyChunksTo ( tNewSet );
	dOthers.for_each ( [&tNewSet] (auto& tChunk) { tChunk->m_bFinallyUnlink = false; tNewSet.m_pNewDiskChunks->Add ( tChunk ); } );

	return true;
}

bool RtIndex_c::AttachRtChunksNoExtCopy ( RtIndex_c * pSrcRtIndex, bool & bFatal, CSphString & sError ) REQUIRES ( m_tWorkers.SerialChunkAccess() )
{
	// prevent optimize to start during the disk chunks stealing
	OptimizeGuard_c tSrcStopOptimize ( *pSrcRtIndex );
	OptimizeGuard_c tDstStopOptimize ( *this );
	LazyVector_T<ConstDiskChunkRefPtr_t> dOthers;

	for ( ;; )
	{
		ConstDiskChunkRefPtr_t tChunk = pSrcRtIndex->PopDiskChunk();
		if ( !tChunk )
			break;

		auto * pChunkIndex = static_cast<CSphIndex *>( *tChunk );
		tChunk->m_bFinallyUnlink = true; // destroy the disk chunks on failure
		if ( !AttachDiskChunkMove ( pChunkIndex, bFatal, sError ) )
		{
			bFatal = true; // need to destroy source index in case of failure as it does not have right amount of disk chunks anymore
			return false;
		}

		dOthers.Add ( tChunk );
	}

	// update disk chunk list
	auto tNewSet = RtWriter();
	CopyChunksTo ( tNewSet );
	dOthers.for_each ( [&tNewSet] (auto& tChunk) { tChunk->m_bFinallyUnlink = false; tNewSet.m_pNewDiskChunks->Add ( tChunk ); } );

	return true;
}

bool RtIndex_c::InitExtCache ( const FilenameBuilder_i * pDstFileBuilder, ExtFiles_h & hExtCache, CSphString & sError ) const
{
	auto pChunks = m_tRtChunks.DiskChunks();
	for ( const auto & tChunk : *pChunks )
	{
		if ( !StoreExtCacheEntry ( tChunk->Cidx(), pDstFileBuilder, hExtCache, sError ) )
			return false;
	}

	return true;
}

bool AttachCopyExt ( const CSphIndex & tSrcIndex, RtIndex_i & tDstIndex, ExtFiles_h & hExtCache, CSphString & sError )
{
	std::unique_ptr<FilenameBuilder_i> pSrcFileBuilder { GetIndexFilenameBuilder() ( tSrcIndex.GetName() ) };
	std::unique_ptr<FilenameBuilder_i> pDstFileBuilder { GetIndexFilenameBuilder() ( tDstIndex.GetName() ) };

	const CSphString sDstPath = GetPathOnly ( tDstIndex.GetFilebase() );
	const int iSuffix = tDstIndex.GetChunkId();
	if ( !AttachRtChunkExtCopy ( tSrcIndex, tDstIndex, iSuffix, pSrcFileBuilder.get(), pDstFileBuilder.get(), hExtCache, sDstPath, sError ) )
		return false;

	assert ( tDstIndex.GetDictionary() );
	assert ( tDstIndex.GetDictionary()->GetSettings().m_dWordforms.GetLength()==tDstIndex.GetDictionary()->GetWordformsFileInfos().GetLength() );
	return true;
}

//////////////////////////////////////////////////////////////////////////
// Embedding 
//////////////////////////////////////////////////////////////////////////

bool RtIndex_c::ReserveEmbeddingSpace ( int64_t /*iDocsToFill*/, int iDims, CSphString & sError )
{
	// Reserve is a write-path preparation step, so serialize and block optimize races.
	OptimizeGuard_c tStopOptimize ( *this );
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );
	TRACE_SCHED ( "rt", "reserve-embedding-space" );
	WaitRAMSegmentsUnlocked();

	auto pChunks = m_tRtChunks.DiskChunks();
	for ( auto & pChunk : *pChunks )
	{
		const int64_t iDocsToFill = pChunk->Cidx().GetCount();
		if ( iDocsToFill<=0 )
			continue;

		CSphString sChunkError;
		if ( !pChunk->CastIdx().ReserveEmbeddingSpace ( iDocsToFill, iDims, sChunkError ) )
		{
			sError.SetSprintf ( "failed to reserve .spb for chunk %d: %s", pChunk->Cidx().m_iChunk, sChunkError.cstr() );
			return false;
		}
	}

	return true;
}

bool RtIndex_c::InitUpdateEmbeddingState ( const CSphString & sAttr, EmbeddingPopulate_e eMode, ExtUpdState_t & tState, CSphString & sError )
{
	FinishUpdateEmbeddingState ( tState );

	TRACE_SCHED ( "rt", "init-update-embedding-state" );

	if ( sAttr.IsEmpty() )
	{
		sError = "attribute for embedding update must not be empty";
		return false;
	}

	if ( !LoadEmbeddingModels ( sError ) )
		return false;

	const int iAttrIdx = m_tSchema.GetAttrIndex ( sAttr.cstr() );
	if ( iAttrIdx<0 )
	{
		sError.SetSprintf ( "attribute '%s' not found", sAttr.cstr() );
		return false;
	}

	const CSphColumnInfo & tAttr = m_tSchema.GetAttr ( iAttrIdx );
	if ( tAttr.m_eAttrType!=SPH_ATTR_FLOAT_VECTOR || !tAttr.IsIndexedKNN() )
	{
		sError.SetSprintf ( "attribute '%s' is not indexed float_vector", sAttr.cstr() );
		return false;
	}

	if ( tAttr.IsColumnar() )
	{
		sError.SetSprintf ( "attribute '%s' is columnar, embedding populate is supported for blob-stored float_vector only", sAttr.cstr() );
		return false;
	}

	if ( iAttrIdx>=m_dAttrsWithModels.GetLength() || !m_dAttrsWithModels[iAttrIdx].m_pModel )
	{
		sError.SetSprintf ( "no embeddings model specified for attribute '%s'", sAttr.cstr() );
		return false;
	}

	const AttrWithModel_t & tAttrWithModel = m_dAttrsWithModels[iAttrIdx];

	const int iDims = tAttrWithModel.m_pModel->GetDims();
	if ( iDims<=0 )
	{
		sError.SetSprintf ( "attribute '%s' has invalid embedding dims=%d", sAttr.cstr(), iDims );
		return false;
	}

	tState.m_tPreviousOptimizeStopState = StopOptimize();
	ScopedScheduler_c tSerialFiber ( m_tWorkers.SerialChunkAccess() );

	// Force current RAM segments to disk so the populate loop runs against disk chunks only.
	if ( !SaveDiskChunk ( true ) )
	{
		sError = m_sLastError;
		if ( sError.IsEmpty() )
			sError = "failed to flush RAM segments to disk before embedding populate";
		return false;
	}

	WaitRAMSegmentsUnlocked();

	auto pChunks = m_tRtChunks.DiskChunks();
	const int iNumDiskChunks = pChunks->GetLength();

	tState.m_sAttr = sAttr;
	tState.m_iAttrIdx = iAttrIdx;
	tState.m_iDims = iDims;
	tState.m_eMode = eMode;
	tState.m_dDiskChunks.Reset ( iNumDiskChunks );
	for ( int i = 0; i < iNumDiskChunks; i++ )
	{
		tState.m_dDiskChunks[i] = { ( *pChunks )[i]->Cidx().m_iChunk, 0 };
	}
	tState.m_iUpdatedRows = 0;
	tState.m_iSkippedRows = 0;
	tState.m_iAlterGenerationAtInit = GetAlterGeneration();

	return true;
}

bool RtIndex_c::GetUpdateEmbedding ( ExtUpdState_t & tState, AttrUpdateSharedPtr_t & pUpdate, CSphString & sError ) const
{
	static constexpr int EMBEDDING_COLLECT_CAP = 1000;

	pUpdate = nullptr;
	if ( sphInterrupted() )
	{
		sError = "operation interrupted";
		return false;
	}

	if ( tState.m_iAlterGenerationAtInit<0 )
	{
		sError = "embedding update state is not initialized";
		return false;
	}

	if ( GetAlterGeneration()!=tState.m_iAlterGenerationAtInit )
	{
		sError = "table altered during embedding populate; please retry";
		return false;
	}

	if ( tState.m_iAttrIdx<0 || tState.m_iAttrIdx>=m_tSchema.GetAttrsCount() )
	{
		sError = "internal error: invalid embedding attribute id";
		return false;
	}

	const CSphColumnInfo & tAttr = m_tSchema.GetAttr ( tState.m_iAttrIdx );
	if ( tAttr.m_sName!=tState.m_sAttr )
	{
		sError = "table schema changed during embedding populate; please retry";
		return false;
	}

	if ( tState.m_iAttrIdx>=m_dAttrsWithModels.GetLength() || !m_dAttrsWithModels[tState.m_iAttrIdx].m_pModel )
	{
		sError.SetSprintf ( "no embeddings model specified for attribute '%s'", tState.m_sAttr.cstr() );
		return false;
	}

	const AttrWithModel_t & tAttrWithModel = m_dAttrsWithModels[tState.m_iAttrIdx];

	const int iCurrentDims = tAttrWithModel.m_pModel->GetDims();
	if ( iCurrentDims<=0 || iCurrentDims!=tState.m_iDims )
	{
		sError.SetSprintf ( "embedding dims changed for attribute '%s'; please retry", tState.m_sAttr.cstr() );
		return false;
	}

	const int iNumDiskChunks = tState.m_dDiskChunks.GetLength();
	if ( iNumDiskChunks<=0 )
		return true;

	CSphVector<DocID_t> dDocids;
	CSphVector<CSphString> dFromTexts;
	for ( int iChunk = 0; iChunk < iNumDiskChunks && dDocids.GetLength() < EMBEDDING_COLLECT_CAP; iChunk++ )
	{
		auto & tChunkState = tState.m_dDiskChunks[iChunk];
		const int iChunkID = tChunkState.first;
		auto pChunk = m_tRtChunks.DiskChunkByID ( iChunkID );
		if ( !pChunk )
		{
			sError.SetSprintf ( "disk chunk %d is no longer available during embedding populate; please retry", iChunkID );
			return false;
		}

		SccRL_t rLock ( pChunk->m_tLock );
		if ( !pChunk->CastIdx().CollectDocsForEmbedding ( tState, tAttrWithModel.m_dFrom, dDocids, dFromTexts, sError, EMBEDDING_COLLECT_CAP, tChunkState.second ) )
			return false;
	}

	if ( dDocids.IsEmpty() )
		return true;

	std::vector<std::vector<float>> dEmbeddings;
	if ( !ConvertEmbeddings ( tAttrWithModel.m_pModel, tState.m_sAttr, dFromTexts, dEmbeddings, sError ) )
		return false;

	pUpdate = CreateFloatVectorAttrUpdate ( tState.m_sAttr, dDocids, dEmbeddings, tState.m_iDims );
	return true;
}

bool RtIndex_c::ValidateUpdateEmbedding ( const ExtUpdState_t & tState, AttrUpdateInc_t & tUpd, CSphString & sError )
{
	if ( tState.m_eMode!=EmbeddingPopulate_e::Empty || tUpd.AllApplied() )
		return true;

	if ( tState.m_iAlterGenerationAtInit<0 )
	{
		sError = "embedding update state is not initialized";
		return false;
	}

	if ( GetAlterGeneration()!=tState.m_iAlterGenerationAtInit )
	{
		sError = "table altered during embedding populate; please retry";
		return false;
	}

	const CSphColumnInfo * pBlobLoc = m_tSchema.GetAttr ( sphGetBlobLocatorName() );
	if ( !pBlobLoc )
		return true;

	if ( tState.m_iAttrIdx<0 || tState.m_iAttrIdx>=m_tSchema.GetAttrsCount() )
	{
		sError = "internal error: invalid embedding attribute id";
		return false;
	}

	const CSphColumnInfo & tAttr = m_tSchema.GetAttr ( tState.m_iAttrIdx );
	UpdateContext_t tCtx ( tUpd, m_tSchema );
	auto tGuard = RtGuard();

	for ( const auto & pRamSeg : tGuard.m_dRamSegs )
	{
		auto dRamRows = CollectUpdatableRows ( tCtx, pRamSeg );
		if ( dRamRows.IsEmpty() )
			continue;

		SccRL_t rLock ( pRamSeg->m_tLock );
		const BYTE * pBlobBase = pRamSeg->m_dBlobs.Begin();
		for ( const auto & tRow : dRamRows )
		{
			const CSphRowitem * pRow = pRamSeg->GetDocinfoByRowID ( tRow.m_tRow );
			if ( !pRow )
				continue;

			const SphAttr_t tBlobOff = sphGetRowAttr ( pRow, pBlobLoc->m_tLocator );
			const BYTE * pBlobRow = pBlobBase + tBlobOff;
			ByteBlob_t tVecBlob = sphGetBlobAttr ( pBlobRow, tAttr.m_tLocator );
			if ( IsBlobAttrZero ( tVecBlob, tState.m_iDims ) )
				continue;

			tUpd.MarkUpdated ( tRow.m_iIdx );
		}
	}

	for ( const auto & pChunk : tGuard.m_dDiskChunks )
	{
		if ( tUpd.AllApplied() )
			break;

		if ( !pChunk->CastIdx().ValidateUpdateEmbedding ( tState, tUpd, sError ) )
			return false;
	}

	return true;
}

void RtIndex_c::FinishUpdateEmbeddingState ( ExtUpdState_t & tState )
{
	if ( tState.m_tPreviousOptimizeStopState.has_value() )
		m_bOptimizeStop.store ( *tState.m_tPreviousOptimizeStopState, std::memory_order_relaxed );

	tState.m_sAttr = "";
	tState.m_iAttrIdx = -1;
	tState.m_iDims = 0;
	tState.m_eMode = EmbeddingPopulate_e::Empty;
	tState.m_dDiskChunks.Reset ( 0 );
	tState.m_iUpdatedRows = 0;
	tState.m_iSkippedRows = 0;
	tState.m_iAlterGenerationAtInit = -1;
	tState.m_tPreviousOptimizeStopState.reset();
}

bool IsModelBackedEmbedding ( const AttrAddRemoveCtx_t & tCtx )
{
	return ( tCtx.m_eType==SPH_ATTR_FLOAT_VECTOR && !!( tCtx.m_uFlags & CSphColumnInfo::ATTR_INDEXED_KNN ) && !tCtx.m_tKNNModel.m_sModelName.empty() );
}
