//! Container format for split-FST log indexes.
//!
//! An SFST file holds one **primary** FST of
//! low-cardinality `key=value` pairs, zero or more **secondary** chunks
//! (mid-cardinality per-field FSTs, high-cardinality per-field sorted lists),
//! and one stream-log-entries chunk, all keyed by a [`chunk_file`] TOC for
//! O(1) random access via mmap. Each SFST covers exactly one content
//! stream — the producer's partition, opaque to this crate.
//!
//! The complete on-disk specification — chunk layout, ids, encoding,
//! version compatibility, and reader access patterns — lives in
//! `FORMAT.md` alongside this crate. Treat it as the source of truth;
//! the rustdoc here covers only the public API.
//!
//! # Crate boundaries
//!
//! This crate owns the format **and** its build: a producer fills a
//! [`RowIndex`] with interned `(timestamp, key=value)` rows and writes
//! through [`IndexWriter`]; reading goes through [`IndexReader`] (the
//! query API) or [`read_summary`] (the cheap scan that also serves
//! summary-only files), with the query vocabulary in [`query`] and the
//! per-directory file registry in [`registry`]. Decoding WAL frames
//! into rows is the producer's concern (`ng-index` is the in-tree
//! producer), so this crate never compiles the WAL/Arrow stack. The
//! chunk-level writer and reader are internal — test fixtures can
//! reach them via the `test-util` feature.
//!
//! # Example
//!
//! ```
//! use bumpalo::Bump;
//! use sfst::{IndexReader, IndexWriter, RowIndex};
//!
//! // Phase 1: a producer interns rows into a RowIndex.
//! let arena = Bump::new();
//! let mut rows = RowIndex::new(&arena, sfst::DEFAULT_CARDINALITY_THRESHOLD);
//! let token = rows.intern(None, "level=info");
//! rows.row(1_700_000_000_000_000_000, &[token]);
//!
//! // Phase 2: write the SFST (content_meta is the producer's opaque
//! // identity blob; the partition key lives in the file name, not here).
//! let (buf, summary, _metadata) =
//!     IndexWriter::write_into(&rows, std::io::Cursor::new(Vec::new()), Vec::new()).unwrap();
//! assert_eq!(summary.record_count, 1);
//!
//! // Read back through the query API.
//! let idx = IndexReader::open(buf.get_ref()).unwrap();
//! assert!(idx.primary_lookup(b"level=info").is_some());
//! ```

mod bitset;
mod build;
mod error;
mod index_reader;
mod index_writer;
mod kv_interner;
mod prefix_map;
pub mod query;
mod reader;
mod row_index;
mod schema;
mod trace_index;
mod writer;

pub mod registry;

pub use error::Error;
pub use index_reader::{BitmapFilter, IndexReader, Trace, TraceSpan};
pub use index_writer::IndexWriter;
pub use kv_interner::KvSlot;
pub(crate) use prefix_map::{BuildError, PrefixMap};
pub use query::{
    Bucket, FacetResult, Filter, Grid, Matcher, MaterializedRow, Timeline, Timestamps,
    compile_pattern, compile_query,
};
pub use reader::read_summary;
pub use registry::{File, Registry, RetentionPolicy};
pub use row_index::RowIndex;
pub use trace_index::TraceIdIndex;

/// Deterministic opaque partition key for tests. SFST treats `part_key` as an
/// opaque `u64` and never decodes it, so tests fabricate distinct keys per
/// logical stream without depending on the content-plane identity codec —
/// same label → same key, different label → (almost surely) different key.
#[cfg(test)]
pub(crate) fn opaque_part_key(namespace: &str, name: &str) -> u64 {
    use std::hash::{Hash, Hasher};
    let mut h = std::collections::hash_map::DefaultHasher::new();
    namespace.hash(&mut h);
    name.hash(&mut h);
    h.finish()
}

/// Highest SFST sequence on disk across every tenant subdir of
/// `base`. Returns `0` when `base` is missing or empty.
pub fn scan_max_sequence_recursive(base: &std::path::Path) -> std::io::Result<u64> {
    file_registry::scan_max_sequence_recursive(base, registry::SFST_EXT)
}
pub use schema::{
    BitmapValue, ColumnEntry, ColumnType, ColumnsTable, DEFAULT_CARDINALITY_THRESHOLD,
    DroppedAttributeCounts, Durations, FieldEntry, FieldTable, FieldTier, Flags, HighField,
    Histogram, IdRanges, KvId, LeafStats, Metadata, NodeId, ObservedTimestamps, ParentSpanIds,
    SchemaEdge, SchemaNode, SchemaTree, SpanId, SpanIds, Step, StreamBatch, Summary, TraceId,
    TraceIds, ValueKind,
};
// Writer-input-only vocabulary stays crate-internal (the build owns the writer).
pub(crate) use schema::{ALL_COLUMNS, ColumnSpec};

// The chunk-level reader/writer surface, exposed only for test fixtures and
// raw-format introspection (the `inspect` example).
#[cfg(feature = "test-util")]
pub use reader::ChunkReader;
#[cfg(feature = "test-util")]
pub use writer::{ChunkCounts, ChunkWriter, ColumnsPresent};

// ── Format constants ─────────────────────────────────────────────
//
// The container's magic, version, and chunk ids never leave this
// crate: producers write through [`ChunkWriter`]'s typed methods, so
// the canonical chunk order and the id encoding exist in exactly one
// place. `FORMAT.md` is the source of truth for what each id carries.

const MAGIC: &[u8; 4] = b"SFST";

// Current on-disk version. Readers accept exactly this version; any other
// is rejected on open. FORMAT.md documents the format and the bump policy.
const VERSION: u32 = 1;

const CHUNK_SUMMARY: chunk_file::ChunkId = *b"SUMR";
const CHUNK_META: chunk_file::ChunkId = *b"META";
const CHUNK_PRIMARY: chunk_file::ChunkId = *b"PRIM";
const CHUNK_TIMS: chunk_file::ChunkId = *b"TIMS";
// Optional per-row column chunks (one per column), written in the COLD region
// (after PRIM) so a query decodes a column only on demand. Each present column is
// listed in the META `ColumnsTable`; a file with no per-row columns carries none of
// these.
const CHUNK_OBSERVED_TS: chunk_file::ChunkId = *b"OBTS";
const CHUNK_TRACE_IDS: chunk_file::ChunkId = *b"TRCE";
const CHUNK_SPAN_IDS: chunk_file::ChunkId = *b"SPAN";
const CHUNK_FLAGS: chunk_file::ChunkId = *b"FLAG";
const CHUNK_DROPPED_ATTRS: chunk_file::ChunkId = *b"DRAC";
// Span-only per-row columns (traces signal): the parent span id and the span
// duration. Same cold-region placement and independent-optionality as the
// columns above; logs files carry neither.
const CHUNK_PARENT_SPAN_IDS: chunk_file::ChunkId = *b"PSPN";
const CHUNK_DURATION: chunk_file::ChunkId = *b"DURN";
// Optional `trace_id` index (cold region, after the per-row columns): a
// first-byte fanout + a position permutation sorted by `trace_id`, for O(log)
// trace-by-id lookup over the chronological `TRCE` column (see `trace_index`).
// Additive and TOC-indexed — a file without it simply lacks the chunk, and a
// reader that ignores it reads the rest unchanged, so its presence needs no
// version bump.
const CHUNK_TRACE_INDEX: chunk_file::ChunkId = *b"TIDX";

/// Minimum number of logs in each stream batch. Files with fewer than
/// `MIN_LOGS_PER_BATCH` total logs use a single batch; otherwise the
/// batch count grows up to [`MAX_STREAM_BATCHES`] so that no batch ever
/// holds fewer than ~`MIN_LOGS_PER_BATCH` entries.
pub const MIN_LOGS_PER_BATCH: u32 = 1024;

/// Hard cap on the number of stream-batch chunks per SFST. Chosen so the
/// per-value batch-membership mask fits in a `u8` (one bit per batch).
pub const MAX_STREAM_BATCHES: u8 = 8;

/// Default zstd compression level used for most chunk payloads —
/// high-card values, stream batches, timestamps, summary, metadata.
/// These payloads either carry random data (string columns, KvId
/// sequences) or are small enough that higher zstd levels don't recoup
/// their CPU cost. Private: [`ChunkWriter`] owns the level-to-chunk
/// pairing.
pub(crate) const ZSTD_LEVEL_DEFAULT: i32 = 1;

/// Elevated zstd compression level for FST chunks (primary +
/// mid-card). FSTs share prefix structure across many `key=value`
/// strings; the higher level lets zstd's longer-range match search
/// find that redundancy and pay off the extra CPU with a noticeably
/// smaller payload. Private: [`ChunkWriter`] owns the pairing.
pub(crate) const ZSTD_LEVEL_FST: i32 = 3;

/// Number of stream-batch (`SB{i}`) chunks in a file with `record_count`
/// log entries. Both writer and reader call this; the rule is the
/// format invariant, not stored in the file.
pub fn num_stream_batches(record_count: u32) -> u8 {
    (record_count / MIN_LOGS_PER_BATCH).clamp(1, MAX_STREAM_BATCHES as u32) as u8
}

/// Logical batch size for a file with `record_count` log entries. Used by
/// the writer to partition log positions into batches and by the reader
/// to decide which batch a given position belongs to.
///
/// Returns `1` for an empty file (`record_count == 0`) — there are no
/// positions to partition, but a non-zero divisor lets callers compose
/// the result with integer division without a separate `record_count == 0`
/// branch.
pub fn stream_batch_size(record_count: u32) -> u32 {
    if record_count == 0 {
        return 1;
    }
    record_count.div_ceil(num_stream_batches(record_count) as u32)
}

/// Chunk id for the mid-card field FST at `index`. The id encodes the
/// index in its trailing two bytes, big-endian, so each mid-card chunk
/// has a unique 4-byte id of the form `b"MF{hi}{lo}"`.
fn mid_field_id(index: u16) -> chunk_file::ChunkId {
    [b'M', b'F', (index >> 8) as u8, (index & 0xff) as u8]
}

/// Chunk id for the high-card field sorted list at `index`. Same shape
/// as [`mid_field_id`] but with prefix `b"HF"`.
fn high_field_id(index: u16) -> chunk_file::ChunkId {
    [b'H', b'F', (index >> 8) as u8, (index & 0xff) as u8]
}

/// Chunk id for the stream-batch chunk at `index` (0..[`MAX_STREAM_BATCHES`]).
/// Encodes the index as a single ASCII digit in the trailing byte, e.g.
/// `b"SB00"` through `b"SB07"`.
fn stream_batch_id(index: u8) -> chunk_file::ChunkId {
    // An out-of-range index would silently produce a non-digit trailing
    // byte (`b'0' + 8` is `b'8'`, but `b'0' + 10` is `b':'`) — an id no
    // reader looks for. The cap is a format invariant, so catch misuse
    // in debug rather than at runtime cost.
    debug_assert!(index < MAX_STREAM_BATCHES, "batch index out of range");
    [b'S', b'B', b'0', b'0' + index]
}

#[cfg(test)]
mod tests;
