// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

// Protobuf headers for serialized index cache entries.
//
// These messages describe the *cache* serialization format, not the on-disk
// Lance format spec, so they live with the library (lance-index) rather than in
// the top-level `protos/` spec folder.
//
// Field numbers and enum values are append-only across all messages here: never
// renumber or reuse them. A change the proto cannot express transparently
// (adding/removing/reordering the IPC/raw sections that follow a header) must
// bump the relevant codec's `CURRENT_VERSION` instead.

syntax = "proto3";

package lance.index.cache;

// ---------------------------------------------------------------------------
// Full-text search (FTS) posting lists
// ---------------------------------------------------------------------------

// Header for a serialized `CompressedPostingList` cache entry.
message CompressedPostingHeader {
  float max_score = 1;
  uint32 length = 2;
  PostingTailCodec posting_tail_codec = 3;
  PositionStorage position_storage = 4;
  // Only meaningful when position_storage == POSITION_STORAGE_SHARED.
  PositionStreamCodec position_stream_codec = 5;
  // Number of documents in each compressed posting block. Older cache entries
  // omit this field and decode as the legacy 128-doc block size.
  uint32 block_size = 6;
  // Whether an impact IPC section follows the posting/position sections.
  bool has_impacts = 7;
}

// Header for a serialized `PlainPostingList` cache entry. Followed by an Arrow
// IPC section of (row_ids: UInt64, frequencies: Float32), then — when
// position_storage == POSITION_STORAGE_LEGACY — an IPC section of the per-doc
// position list. Plain postings never carry a shared position stream.
message PlainPostingHeader {
  // Absent when the posting has no precomputed block-max score (the in-memory
  // `max_score` is `None`); present otherwise.
  optional float max_score = 1;
  // POSITION_STORAGE_NONE or POSITION_STORAGE_LEGACY only.
  PositionStorage position_storage = 2;
}

// Header for a serialized standalone `Positions` cache entry. Followed by the
// position sections framed per `position_storage`, which is never
// POSITION_STORAGE_NONE for a standalone entry.
message PositionsHeader {
  PositionStorage position_storage = 1;
  // Only meaningful when position_storage == POSITION_STORAGE_SHARED.
  PositionStreamCodec position_stream_codec = 2;
}

// Header for a serialized `PostingListGroup`: a member count followed by that
// many `PostingList` bodies written inline. Each member body is
// self-delimiting, so members need no length prefixes, and writing them inline
// keeps their Arrow IPC sections 64-byte aligned within the group entry.
message PostingListGroupHeader {
  uint32 count = 1;
}

// Tail-block encoding of a compressed posting list.
enum PostingTailCodec {
  POSTING_TAIL_CODEC_FIXED32 = 0;
  POSTING_TAIL_CODEC_VARINT_DELTA = 1;
}

// Encoding of a shared position stream's byte buffer.
enum PositionStreamCodec {
  POSITION_STREAM_CODEC_VARINT_DOC_DELTA = 0;
  POSITION_STREAM_CODEC_PACKED_DELTA = 1;
}

// Which (if any) positions accompany the posting list, and how they are framed
// in the sections after the header.
enum PositionStorage {
  POSITION_STORAGE_NONE = 0;
  // Legacy per-doc positions as a single Arrow IPC section.
  POSITION_STORAGE_LEGACY = 1;
  // Shared stream: an Arrow IPC section of block offsets, then a raw blob of
  // the (codec-encoded) position bytes.
  POSITION_STORAGE_SHARED = 2;
}

// ---------------------------------------------------------------------------
// Scalar indices
// ---------------------------------------------------------------------------

// Header for a serialized `BTreeIndexState` cache entry, followed by a single
// Arrow IPC section holding the page-lookup batch.
message BTreeIndexHeader {
  uint64 batch_size = 1;
  // Whether an explicit page-range -> file mapping is present. Distinguishes a
  // non-range-partitioned index (false) from a range-partitioned one whose map
  // happens to be empty (true with no entries).
  bool has_ranges_to_files = 2;
  repeated RangeToFile ranges_to_files = 3;
}

// One entry of a `BTreeIndexState` page-range -> file mapping. The range is
// inclusive on both ends (a `RangeInclusive<u32>`).
message RangeToFile {
  uint32 start = 1;
  uint32 end = 2;
  uint32 page_offset = 3;
  string path = 4;
}

// ---------------------------------------------------------------------------
// Vector indices (IVF partitions)
// ---------------------------------------------------------------------------

// Headers for serialized IVF partition cache entries (`PartitionEntry<S, Q>`).
//
// Each header is followed by 64-byte-aligned Arrow IPC sections in a fixed,
// version-keyed order (sub-index, then any quantizer-specific arrays, then the
// quantizer storage batches).

// Distance metric a quantizer's storage was built for.
enum DistanceType {
  DISTANCE_TYPE_L2 = 0;
  DISTANCE_TYPE_COSINE = 1;
  DISTANCE_TYPE_DOT = 2;
  DISTANCE_TYPE_HAMMING = 3;
}

// Rotation applied by a RabitQ quantizer.
enum RotationType {
  ROTATION_TYPE_MATRIX = 0;
  ROTATION_TYPE_FAST = 1;
}

// Estimator a RabitQ quantizer uses at query time.
enum RabitQueryEstimator {
  RABIT_QUERY_ESTIMATOR_RESIDUAL_QUERY = 0;
  RABIT_QUERY_ESTIMATOR_RAW_QUERY = 1;
}

// Product quantizer. Sections: sub-index IPC, codebook IPC, storage IPC.
message PqPartitionHeader {
  DistanceType distance_type = 1;
  uint32 nbits = 2;
  uint64 num_sub_vectors = 3;
  uint64 dimension = 4;
  bool transposed = 5;
}

// Flat (float) and flat-binary quantizers. Sections: sub-index IPC, storage IPC.
message FlatPartitionHeader {
  DistanceType distance_type = 1;
  uint64 dim = 2;
}

// Scalar quantizer. Sections: sub-index IPC, storage IPC (possibly multi-batch).
message SqPartitionHeader {
  DistanceType distance_type = 1;
  uint32 num_bits = 2;
  uint64 dim = 3;
  double bounds_start = 4;
  double bounds_end = 5;
}

// Header for a serialized IVF index state (`IvfIndexState<Q>`), followed by
// three raw blobs: the IVF model protobuf, the quantizer's extra-metadata
// buffer (may be empty), and the auxiliary IVF model protobuf.
message IvfStateHeader {
  reserved 8;
  reserved "cache_key_prefix";

  string index_file_path = 1;
  string uuid = 2;
  string distance_type = 3;
  repeated string sub_index_metadata = 4;
  string sub_index_type = 5;
  string quantization_type = 6;
  // Per-quantizer `Q::Metadata` as JSON. Kept as a string because the metadata
  // type is generic over the quantizer; the proto envelope still provides
  // additive evolution for the surrounding fields.
  string quantizer_metadata_json = 7;
  uint64 index_file_size = 9;
  uint64 aux_file_size = 10;
}

// RabitQ quantizer. Sections: sub-index IPC, rotate-matrix IPC (Matrix rotation
// only), storage IPC.
message RabitPartitionHeader {
  DistanceType distance_type = 1;
  uint32 num_bits = 2;
  uint32 code_dim = 3;
  RotationType rotation_type = 4;
  // Fast-rotation sign vector; present only when rotation_type ==
  // ROTATION_TYPE_FAST (the Matrix case stores its rotation as an IPC section).
  optional bytes fast_rotation_signs = 5;
  // Estimator the RabitQ storage uses at query time (residual vs raw query).
  RabitQueryEstimator query_estimator = 6;
}
