// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//! Global index scanner: evaluates predicates against sorted global indexes
//! to produce row ID ranges for data evolution tables.
//!
//! Reference: [org.apache.paimon.index.GlobalIndexScanner](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/index/GlobalIndexScanner.java)

use super::bitmap_global_index_reader::{
    is_bitmap_floating_residual_sensitive_op, make_bitmap_key_comparator, serialize_bitmap_datum,
    BitmapGlobalIndexReader,
};
use super::global_index_types::{
    normalize_sorted_global_index_type, BITMAP_GLOBAL_INDEX_TYPE, BTREE_GLOBAL_INDEX_TYPE,
};
use crate::btree::query::{extract_between, BetweenInfo, IndexQuery};
use crate::btree::{make_key_comparator, serialize_datum, BTreeIndexMeta, BTreeIndexReader};
use crate::deletion_vector::DeletionVectorFactory;
use crate::io::FileIO;
use crate::spec::{
    DataField, DataType, Datum, FileKind, GlobalIndexSearchMode, IndexFileMeta, IndexManifestEntry,
    Predicate, PredicateOperator,
};
use crate::table::{DeletionFile, RowRange, Table};
use crate::{Error, Result};
use futures::{StreamExt, TryStreamExt};
use roaring::RoaringTreemap;
use std::cmp::Ordering;
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::sync::Mutex;

type BoxedCmp = Box<dyn Fn(&[u8], &[u8]) -> Ordering + Send + Sync>;

type EvaluateFuture<'a> = std::pin::Pin<
    Box<dyn std::future::Future<Output = Result<Option<GlobalIndexScanResult>>> + Send + 'a>,
>;

type PredicateTuple<'a> = (PredicateOperator, &'a [Datum], &'a DataType);

const DELETION_VECTORS_INDEX_TYPE: &str = "DELETION_VECTORS";
const INDEX_DIR: &str = "index";

async fn try_fold_bounded<T, Fut, Acc, Fold>(
    futures: impl IntoIterator<Item = Fut>,
    max_concurrency: usize,
    mut accumulator: Acc,
    mut fold: Fold,
) -> Result<Acc>
where
    Fut: Future<Output = Result<T>>,
    Fold: FnMut(&mut Acc, T),
{
    debug_assert!(max_concurrency > 0);
    let stream = futures::stream::iter(futures).buffer_unordered(max_concurrency);
    futures::pin_mut!(stream);
    while let Some(value) = stream.try_next().await? {
        fold(&mut accumulator, value);
    }
    Ok(accumulator)
}

struct GlobalIndexScanResult {
    row_ranges: Vec<RowRange>,
    evaluated_field_ids: HashSet<i32>,
}

/// Evaluates global index predicates and returns matching row ranges.
///
/// The scanner filters index manifest entries for global index files,
/// uses BTreeIndexMeta for file-level pruning, then reads matching
/// BTree or bitmap files to evaluate predicates and collect row IDs.
/// Opened BTreeIndexReaders are cached for reuse across evaluations.
pub(crate) struct GlobalIndexScanner {
    file_io: FileIO,
    table_path: String,
    global_index_thread_num: usize,
    btree_fallback_scan_max_size: i64,
    bitmap_fallback_scan_max_size: i64,
    /// Global index entries grouped by field_id.
    entries_by_field: Vec<(i32, Vec<GlobalIndexEntry>)>,
    /// Indexed row-id coverage grouped by field_id.
    coverage_by_field: HashMap<i32, Vec<RowRange>>,
    /// Schema fields for field_id lookup.
    schema_fields: Vec<DataField>,
    /// Cache of opened BTree readers, keyed by file name.
    reader_cache: Mutex<HashMap<String, BTreeIndexReader<BoxedCmp>>>,
}

/// A resolved global index entry with parsed metadata.
struct GlobalIndexEntry {
    file_name: String,
    index_type: GlobalIndexFileKind,
    file_size: i64,
    row_range_start: i64,
    meta: BTreeIndexMeta,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum GlobalIndexFileKind {
    BTree,
    Bitmap,
}

fn is_floating_point(data_type: &DataType) -> bool {
    matches!(data_type, DataType::Float(_) | DataType::Double(_))
}

fn bitmap_meta_may_match(
    meta: &BTreeIndexMeta,
    op: PredicateOperator,
    data_type: &DataType,
    serialized_literals: &[Vec<u8>],
    cmp: &dyn Fn(&[u8], &[u8]) -> Ordering,
) -> bool {
    if is_floating_point(data_type) && is_bitmap_floating_residual_sensitive_op(op) {
        !meta.only_nulls()
    } else {
        meta.may_match(op, serialized_literals, cmp)
    }
}

fn bitmap_meta_may_match_between(
    meta: &BTreeIndexMeta,
    data_type: &DataType,
    from_key: &[u8],
    to_key: &[u8],
    cmp: &dyn Fn(&[u8], &[u8]) -> Ordering,
) -> bool {
    if is_floating_point(data_type)
        && is_bitmap_floating_residual_sensitive_op(PredicateOperator::Between)
    {
        !meta.only_nulls()
    } else {
        meta.may_match_between(from_key, to_key, cmp)
    }
}

impl GlobalIndexFileKind {
    fn name(self) -> &'static str {
        match self {
            Self::BTree => "BTree",
            Self::Bitmap => "bitmap",
        }
    }
}

enum OpenedGlobalIndexReader {
    BTree(BTreeIndexReader<BoxedCmp>),
    Bitmap(BitmapGlobalIndexReader),
}

#[derive(Clone, Copy, Default)]
struct FallbackScanPlan {
    selected_btree: usize,
    selected_bitmap: usize,
    allow_btree: bool,
    allow_bitmap: bool,
}

struct EntryQueryPlan {
    entry_idx: usize,
    between_matches: bool,
    between_evaluated: bool,
    matching_predicates: Vec<usize>,
}

impl FallbackScanPlan {
    fn allowed(self, kind: GlobalIndexFileKind) -> bool {
        match kind {
            GlobalIndexFileKind::BTree => self.allow_btree,
            GlobalIndexFileKind::Bitmap => self.allow_bitmap,
        }
    }
}

impl OpenedGlobalIndexReader {
    async fn query(
        &self,
        op: PredicateOperator,
        literals: &[Datum],
        data_type: &DataType,
    ) -> std::io::Result<RoaringTreemap> {
        match self {
            Self::BTree(reader) => reader.query(op, literals, data_type).await,
            Self::Bitmap(reader) => reader.query(op, literals, data_type).await,
        }
    }

    async fn range_query(
        &self,
        from: &[u8],
        to: &[u8],
        data_type: &DataType,
        from_inclusive: bool,
        to_inclusive: bool,
    ) -> std::io::Result<RoaringTreemap> {
        match self {
            Self::BTree(reader) => {
                reader
                    .range_query(from, to, from_inclusive, to_inclusive)
                    .await
            }
            Self::Bitmap(reader) => {
                reader
                    .range_query(from, to, data_type, from_inclusive, to_inclusive)
                    .await
            }
        }
    }
}

impl GlobalIndexScanner {
    /// Create a scanner from index manifest entries.
    /// Returns `Ok(None)` if there are no global index entries.
    pub(crate) fn create(
        file_io: &FileIO,
        table_path: &str,
        global_index_thread_num: usize,
        btree_fallback_scan_max_size: i64,
        bitmap_fallback_scan_max_size: i64,
        index_entries: &[IndexManifestEntry],
        schema_fields: &[DataField],
    ) -> Result<Option<Self>> {
        if global_index_thread_num == 0 {
            return Err(Error::DataInvalid {
                message: "Global index thread count must be greater than 0".to_string(),
                source: None,
            });
        }
        let mut entries_by_field: std::collections::HashMap<i32, Vec<GlobalIndexEntry>> =
            std::collections::HashMap::new();
        let mut coverage_by_field: HashMap<i32, Vec<RowRange>> = HashMap::new();

        for entry in index_entries {
            if entry.kind != FileKind::Add {
                continue;
            }
            let Some(index_type) = normalize_sorted_global_index_type(&entry.index_file.index_type)
            else {
                continue;
            };
            let global_meta =
                entry
                    .index_file
                    .global_index_meta
                    .as_ref()
                    .ok_or_else(|| Error::DataInvalid {
                        message: format!(
                            "Missing global index metadata for sorted index file '{}'",
                            entry.index_file.file_name
                        ),
                        source: None,
                    })?;

            let index_meta = global_meta
                .index_meta
                .as_ref()
                .ok_or_else(|| Error::DataInvalid {
                    message: format!(
                        "Missing sorted global index metadata for file '{}'",
                        entry.index_file.file_name
                    ),
                    source: None,
                })?;
            let sorted_meta =
                BTreeIndexMeta::deserialize(index_meta).map_err(|error| Error::DataInvalid {
                    message: format!(
                        "Invalid sorted global index metadata for file '{}'",
                        entry.index_file.file_name
                    ),
                    source: Some(Box::new(error)),
                })?;

            let resolved = GlobalIndexEntry {
                file_name: entry.index_file.file_name.clone(),
                index_type: match index_type {
                    BTREE_GLOBAL_INDEX_TYPE => GlobalIndexFileKind::BTree,
                    BITMAP_GLOBAL_INDEX_TYPE => GlobalIndexFileKind::Bitmap,
                    _ => unreachable!("normalized sorted global index type"),
                },
                file_size: entry.index_file.file_size,
                row_range_start: global_meta.row_range_start,
                meta: sorted_meta,
            };

            let row_range = RowRange::new(global_meta.row_range_start, global_meta.row_range_end);
            coverage_by_field
                .entry(global_meta.index_field_id)
                .or_default()
                .push(row_range.clone());
            if let Some(extra_field_ids) = global_meta.extra_field_ids.as_ref() {
                for extra_field_id in extra_field_ids {
                    coverage_by_field
                        .entry(*extra_field_id)
                        .or_default()
                        .push(row_range.clone());
                }
            }

            entries_by_field
                .entry(global_meta.index_field_id)
                .or_default()
                .push(resolved);
        }

        if entries_by_field.is_empty() {
            return Ok(None);
        }

        Ok(Some(Self {
            file_io: file_io.clone(),
            table_path: table_path.trim_end_matches('/').to_string(),
            global_index_thread_num,
            btree_fallback_scan_max_size,
            bitmap_fallback_scan_max_size,
            entries_by_field: entries_by_field.into_iter().collect(),
            coverage_by_field,
            schema_fields: schema_fields.to_vec(),
            reader_cache: Mutex::new(HashMap::new()),
        }))
    }

    /// Evaluate a predicate against the global indexes and return matching row ranges.
    /// Returns `None` if the predicate cannot be evaluated by the global index.
    fn evaluate<'a>(&'a self, predicate: &'a Predicate) -> EvaluateFuture<'a> {
        Box::pin(async move {
            match predicate {
                Predicate::Leaf {
                    column,
                    op,
                    literals,
                    data_type,
                    ..
                } => {
                    if !is_sorted_global_index_supported_op(*op) {
                        return Ok(None);
                    }
                    let field_id = self.find_field_id_by_name(column)?;
                    let field_id = match field_id {
                        Some(id) => id,
                        None => return Ok(None),
                    };
                    let entries = match self.entries_for_field(field_id) {
                        Some(e) => e,
                        None => return Ok(None),
                    };
                    self.evaluate_leaf(entries, &[(*op, literals.as_slice(), data_type)])
                        .await
                        .map(|ranges| {
                            ranges.map(|row_ranges| GlobalIndexScanResult {
                                row_ranges,
                                evaluated_field_ids: HashSet::from([field_id]),
                            })
                        })
                }
                Predicate::And(children) => {
                    // Group leaf predicates by field_id to reuse readers
                    let mut leaf_groups: std::collections::HashMap<i32, Vec<PredicateTuple<'_>>> =
                        std::collections::HashMap::new();
                    let mut non_leaf_children = Vec::new();

                    for child in children {
                        if let Predicate::Leaf {
                            column,
                            op,
                            literals,
                            data_type,
                            ..
                        } = child
                        {
                            if is_sorted_global_index_supported_op(*op) {
                                if let Some(field_id) = self.find_field_id_by_name(column)? {
                                    if self.entries_for_field(field_id).is_some() {
                                        leaf_groups.entry(field_id).or_default().push((
                                            *op,
                                            literals.as_slice(),
                                            data_type,
                                        ));
                                        continue;
                                    }
                                }
                            }
                        }
                        non_leaf_children.push(child);
                    }

                    let mut row_ranges: Option<Vec<RowRange>> = None;
                    let mut evaluated_field_ids = HashSet::new();

                    // Evaluate grouped leaves (one open per file)
                    for (field_id, predicates) in &leaf_groups {
                        if let Some(entries) = self.entries_for_field(*field_id) {
                            if let Some(ranges) = self.evaluate_leaf(entries, predicates).await? {
                                row_ranges = Some(match row_ranges {
                                    None => ranges,
                                    Some(existing) => intersect_sorted_ranges(&existing, &ranges),
                                });
                                evaluated_field_ids.insert(*field_id);
                            }
                        }
                    }

                    // Evaluate non-leaf children recursively
                    for child in non_leaf_children {
                        if let Some(child_result) = self.evaluate(child).await? {
                            row_ranges = Some(match row_ranges {
                                None => child_result.row_ranges,
                                Some(existing) => {
                                    intersect_sorted_ranges(&existing, &child_result.row_ranges)
                                }
                            });
                            evaluated_field_ids.extend(child_result.evaluated_field_ids);
                        }
                    }

                    Ok(row_ranges.map(|row_ranges| GlobalIndexScanResult {
                        row_ranges,
                        evaluated_field_ids,
                    }))
                }
                Predicate::Or(children) => {
                    let mut all_ranges: Vec<RowRange> = Vec::new();
                    let mut evaluated_field_ids = HashSet::new();
                    for child in children {
                        match self.evaluate(child).await? {
                            Some(child_result) => {
                                all_ranges.extend(child_result.row_ranges);
                                evaluated_field_ids.extend(child_result.evaluated_field_ids);
                            }
                            None => return Ok(None),
                        }
                    }
                    let row_ranges = if all_ranges.is_empty() {
                        Vec::new()
                    } else {
                        super::merge_row_ranges(all_ranges)
                    };
                    Ok(Some(GlobalIndexScanResult {
                        row_ranges,
                        evaluated_field_ids,
                    }))
                }
                _ => Ok(None),
            }
        })
    }

    /// Evaluate multiple predicates against the same set of index entries.
    /// Opens each file once and evaluates all predicates, intersecting results.
    /// Detects between patterns (GtEq/Gt + LtEq/Lt) and merges them into a single range query.
    async fn evaluate_leaf(
        &self,
        entries: &[GlobalIndexEntry],
        predicates: &[(PredicateOperator, &[Datum], &DataType)],
    ) -> Result<Option<Vec<RowRange>>> {
        // Try to detect between pattern and split into (between, remaining)
        let (between, remaining) = extract_between(predicates);

        let effective_predicates = if between.is_some() {
            &remaining
        } else {
            predicates
        };

        // Pre-compute comparators and serialized keys for file-level pruning per predicate
        let pruning_info: Vec<_> = effective_predicates
            .iter()
            .map(|(op, literals, data_type)| {
                let btree_cmp = make_key_comparator(data_type);
                let btree_serialized = literals
                    .iter()
                    .map(|l| serialize_datum(l, data_type))
                    .collect::<Vec<_>>();
                let bitmap_cmp = make_bitmap_key_comparator(data_type);
                let bitmap_serialized = literals
                    .iter()
                    .map(|l| serialize_bitmap_datum(l, data_type))
                    .collect::<Vec<_>>();
                (
                    *op,
                    *data_type,
                    btree_cmp,
                    btree_serialized,
                    bitmap_cmp,
                    bitmap_serialized,
                )
            })
            .collect();

        let predicate_matches: Vec<Vec<bool>> = pruning_info
            .iter()
            .map(
                |(op, data_type, btree_cmp, btree_serialized, bitmap_cmp, bitmap_serialized)| {
                    entries
                        .iter()
                        .map(|entry| match entry.index_type {
                            GlobalIndexFileKind::BTree => {
                                entry.meta.may_match(*op, btree_serialized, btree_cmp)
                            }
                            GlobalIndexFileKind::Bitmap => bitmap_meta_may_match(
                                &entry.meta,
                                *op,
                                data_type,
                                bitmap_serialized,
                                bitmap_cmp.as_ref(),
                            ),
                        })
                        .collect()
                },
            )
            .collect();
        let predicate_fallback_plans: Vec<Option<FallbackScanPlan>> = effective_predicates
            .iter()
            .enumerate()
            .map(|(i, (op, _, _))| {
                requires_fallback_scan(*op)
                    .then(|| self.fallback_scan_plan(entries, &predicate_matches[i]))
            })
            .collect();

        let between_matches_by_entry: Vec<bool> = match between.as_ref() {
            Some(b) => {
                let btree_cmp = make_key_comparator(b.data_type);
                let btree_from = serialize_datum(b.from, b.data_type);
                let btree_to = serialize_datum(b.to, b.data_type);
                let bitmap_cmp = make_bitmap_key_comparator(b.data_type);
                let bitmap_from = serialize_bitmap_datum(b.from, b.data_type);
                let bitmap_to = serialize_bitmap_datum(b.to, b.data_type);
                entries
                    .iter()
                    .map(|entry| match entry.index_type {
                        GlobalIndexFileKind::BTree => {
                            entry
                                .meta
                                .may_match_between(&btree_from, &btree_to, &btree_cmp)
                        }
                        GlobalIndexFileKind::Bitmap => bitmap_meta_may_match_between(
                            &entry.meta,
                            b.data_type,
                            &bitmap_from,
                            &bitmap_to,
                            bitmap_cmp.as_ref(),
                        ),
                    })
                    .collect()
            }
            None => Vec::new(),
        };
        let between_fallback_plan = between
            .as_ref()
            .map(|_| self.fallback_scan_plan(entries, &between_matches_by_entry));

        let mut query_plans = Vec::with_capacity(entries.len());
        for (entry_idx, entry) in entries.iter().enumerate() {
            // Also check if between range may match
            let between_matches = between
                .as_ref()
                .is_some_and(|_| between_matches_by_entry[entry_idx]);
            let between_evaluated_for_entry = between_fallback_plan.is_some_and(|plan| {
                fallback_plan_evaluates_entry(plan, entry.index_type, between_matches)
            });

            // When a Between conjunct exists but the file does not overlap its
            // range, the whole AND cannot match — drop the file regardless of
            // how the remaining predicates evaluate. Without this guard, a file
            // outside the Between range but matched by some remaining predicate
            // (e.g. `BETWEEN 10 AND 20 AND id >= 0` on a file [30, 40]) would
            // be retained because `file_result` is initialized from the
            // remaining bitmap, silently dropping the Between conjunct.
            if between_evaluated_for_entry && !between_matches {
                continue;
            }

            let mut file_evaluated = between_evaluated_for_entry;
            let mut file_cannot_match = false;
            let mut file_has_unsupported_match =
                between_matches && !between_evaluated_for_entry && between_fallback_plan.is_some();
            let matching_predicates: Vec<usize> = (0..effective_predicates.len())
                .filter(|&i| {
                    let predicate_matches_entry = predicate_matches[i][entry_idx];
                    let predicate_evaluated_for_entry =
                        predicate_fallback_plans[i].is_none_or(|plan| {
                            fallback_plan_evaluates_entry(
                                plan,
                                entry.index_type,
                                predicate_matches_entry,
                            )
                        });
                    if !predicate_evaluated_for_entry {
                        file_has_unsupported_match |= predicate_matches_entry;
                        return false;
                    }
                    file_evaluated = true;
                    if !predicate_matches_entry {
                        file_cannot_match = true;
                        return false;
                    }
                    true
                })
                .collect();
            if file_cannot_match {
                continue;
            }
            if !file_evaluated {
                if file_has_unsupported_match {
                    return Ok(None);
                }
                continue;
            }

            query_plans.push(EntryQueryPlan {
                entry_idx,
                between_matches,
                between_evaluated: between_evaluated_for_entry,
                matching_predicates,
            });
        }

        // Complete all pruning and fallback decisions before starting shard I/O.
        // A later unsupported shard must fall back to the normal scan without an
        // earlier shard racing it with an I/O or query error.
        let data_type = between
            .as_ref()
            .map(|b| b.data_type)
            .or_else(|| effective_predicates.first().map(|p| p.2))
            .unwrap_or(predicates[0].2);
        let between = between.as_ref();
        let futures = query_plans.into_iter().map(|plan| async move {
            let entry = &entries[plan.entry_idx];
            let result = self
                .query_entry(entry, data_type, between, &plan, effective_predicates)
                .await?;
            Ok((entry.row_range_start, result))
        });
        let all_row_ids = try_fold_bounded(
            futures,
            self.global_index_thread_num,
            RoaringTreemap::new(),
            |all_row_ids, (row_range_start, file_result)| {
                if let Some(bitmap) = file_result {
                    for row_id in bitmap.iter() {
                        all_row_ids.insert(row_id + row_range_start as u64);
                    }
                }
            },
        )
        .await?;

        Ok(Some(bitmap_to_ranges(&all_row_ids)))
    }

    async fn query_entry(
        &self,
        entry: &GlobalIndexEntry,
        data_type: &DataType,
        between: Option<&BetweenInfo<'_>>,
        plan: &EntryQueryPlan,
        effective_predicates: &[(PredicateOperator, &[Datum], &DataType)],
    ) -> Result<Option<RoaringTreemap>> {
        let mut reader = if (plan.between_matches && plan.between_evaluated)
            || !plan.matching_predicates.is_empty()
        {
            Some(
                self.open_reader_for_entry(entry, &entry.meta, data_type)
                    .await?,
            )
        } else {
            None
        };
        let mut file_result = None;

        if plan.between_matches && plan.between_evaluated {
            let between = between.expect("evaluated between query is present");
            let serialize_key = match entry.index_type {
                GlobalIndexFileKind::BTree => serialize_datum,
                GlobalIndexFileKind::Bitmap => serialize_bitmap_datum,
            };
            let from_key = serialize_key(between.from, between.data_type);
            let to_key = serialize_key(between.to, between.data_type);
            let bitmap = reader
                .as_ref()
                .expect("reader is opened when between matches")
                .range_query(
                    &from_key,
                    &to_key,
                    between.data_type,
                    between.from_inclusive,
                    between.to_inclusive,
                )
                .await
                .map_err(|error| Self::query_error(entry, error))?;
            file_result = Some(bitmap);
        }

        for &idx in &plan.matching_predicates {
            let (op, literals, data_type) = &effective_predicates[idx];
            let bitmap = reader
                .as_ref()
                .expect("reader is opened when predicates match")
                .query(*op, literals, data_type)
                .await
                .map_err(|error| Self::query_error(entry, error))?;
            file_result = Some(match file_result {
                None => bitmap,
                Some(mut existing) => {
                    existing &= bitmap;
                    existing
                }
            });
        }

        // Each concurrent task owns its reader. Only return it to the shared
        // cache after all predicates for this shard have completed.
        if let Some(OpenedGlobalIndexReader::BTree(reader)) = reader.take() {
            self.return_reader(entry.file_name.clone(), reader);
        }
        Ok(file_result)
    }

    fn query_error(entry: &GlobalIndexEntry, error: std::io::Error) -> Error {
        Error::DataInvalid {
            message: format!(
                "Global index query failed for {} file '{}'",
                entry.index_type.name(),
                entry.file_name
            ),
            source: Some(Box::new(error)),
        }
    }

    /// Get a cached reader or open a new one for the given file.
    async fn get_or_open_reader(
        &self,
        file_name: &str,
        meta: &BTreeIndexMeta,
        data_type: &DataType,
    ) -> Result<OpenedGlobalIndexReader> {
        // Try to take from cache
        {
            let mut cache = self.reader_cache.lock().unwrap();
            if let Some(reader) = cache.remove(file_name) {
                return Ok(OpenedGlobalIndexReader::BTree(reader));
            }
        }

        // Open new reader
        let path = format!("{}/{INDEX_DIR}/{}", self.table_path, file_name);
        let input = self.file_io.new_input(&path)?;
        let file_size = input.metadata().await?.size;
        let file_reader = input.reader().await?;

        let cmp = make_key_comparator(data_type);
        BTreeIndexReader::open(Box::new(file_reader), file_size, meta, cmp)
            .await
            .map(OpenedGlobalIndexReader::BTree)
            .map_err(|e| crate::Error::DataInvalid {
                message: format!("Failed to open BTree index file: {file_name}"),
                source: Some(Box::new(e)),
            })
    }

    async fn open_reader_for_entry(
        &self,
        entry: &GlobalIndexEntry,
        meta: &BTreeIndexMeta,
        data_type: &DataType,
    ) -> Result<OpenedGlobalIndexReader> {
        match entry.index_type {
            GlobalIndexFileKind::BTree => {
                self.get_or_open_reader(&entry.file_name, meta, data_type)
                    .await
            }
            GlobalIndexFileKind::Bitmap => self
                .open_bitmap_reader(&entry.file_name)
                .await
                .map(OpenedGlobalIndexReader::Bitmap)
                .map_err(|e| crate::Error::DataInvalid {
                    message: format!(
                        "Failed to open bitmap global index file: {}",
                        entry.file_name
                    ),
                    source: Some(Box::new(e)),
                }),
        }
    }

    async fn open_bitmap_reader(
        &self,
        file_name: &str,
    ) -> std::io::Result<BitmapGlobalIndexReader> {
        let path = format!("{}/{INDEX_DIR}/{}", self.table_path, file_name);
        let input = self
            .file_io
            .new_input(&path)
            .map_err(|e| std::io::Error::other(e.to_string()))?;
        let file_size = input
            .metadata()
            .await
            .map_err(|e| std::io::Error::other(e.to_string()))?
            .size;
        let file_reader = input
            .reader()
            .await
            .map_err(|e| std::io::Error::other(e.to_string()))?;
        BitmapGlobalIndexReader::open(Box::new(file_reader), file_size).await
    }

    fn fallback_scan_plan(
        &self,
        entries: &[GlobalIndexEntry],
        selected: &[bool],
    ) -> FallbackScanPlan {
        let mut plan = FallbackScanPlan::default();
        let mut btree_total = 0i64;
        let mut bitmap_total = 0i64;
        let mut btree_valid = true;
        let mut bitmap_valid = true;

        for (entry, selected) in entries.iter().zip(selected) {
            if !selected {
                continue;
            }
            match entry.index_type {
                GlobalIndexFileKind::BTree => {
                    plan.selected_btree += 1;
                    btree_valid &= add_file_size(&mut btree_total, entry.file_size);
                }
                GlobalIndexFileKind::Bitmap => {
                    plan.selected_bitmap += 1;
                    bitmap_valid &= add_file_size(&mut bitmap_total, entry.file_size);
                }
            }
        }

        plan.allow_btree = plan.selected_btree > 0
            && btree_valid
            && self.btree_fallback_scan_max_size > 0
            && btree_total <= self.btree_fallback_scan_max_size;
        plan.allow_bitmap = plan.selected_bitmap > 0
            && bitmap_valid
            && self.bitmap_fallback_scan_max_size > 0
            && bitmap_total <= self.bitmap_fallback_scan_max_size;
        plan
    }

    /// Return a reader to the cache for future reuse.
    fn return_reader(&self, file_name: String, reader: BTreeIndexReader<BoxedCmp>) {
        let mut cache = self.reader_cache.lock().unwrap();
        cache.insert(file_name, reader);
    }

    fn find_field_id_by_name(&self, column: &str) -> Result<Option<i32>> {
        Ok(crate::table::find_field_id_by_name(
            &self.schema_fields,
            column,
        ))
    }

    fn entries_for_field(&self, field_id: i32) -> Option<&[GlobalIndexEntry]> {
        self.entries_by_field
            .iter()
            .find(|(id, _)| *id == field_id)
            .map(|(_, entries)| entries.as_slice())
    }

    /// Return row ranges not covered by global indexes for this predicate.
    ///
    /// `full` uses `[0, snapshot.next_row_id - 1]`; `detail` uses actual
    /// data-file row ranges collected by the scan. The caller unions these
    /// ranges with indexed matches, and the normal read filter evaluates the
    /// predicate on the raw rows.
    #[cfg(test)]
    fn unindexed_ranges(
        &self,
        predicate: &Predicate,
        search_mode: GlobalIndexSearchMode,
        next_row_id: Option<i64>,
        data_ranges: &[RowRange],
    ) -> Result<Vec<RowRange>> {
        let field_ids = self.collect_field_ids(predicate)?;
        Ok(self.unindexed_ranges_for_field_ids(&field_ids, search_mode, next_row_id, data_ranges))
    }

    fn unindexed_ranges_for_field_ids(
        &self,
        field_ids: &HashSet<i32>,
        search_mode: GlobalIndexSearchMode,
        next_row_id: Option<i64>,
        data_ranges: &[RowRange],
    ) -> Vec<RowRange> {
        unindexed_ranges_for_coverage(
            &self.coverage_by_field,
            field_ids,
            search_mode,
            next_row_id,
            data_ranges,
        )
    }

    #[cfg(test)]
    fn collect_field_ids(&self, predicate: &Predicate) -> Result<HashSet<i32>> {
        let mut field_ids = HashSet::new();
        self.collect_field_ids_inner(predicate, &mut field_ids)?;
        Ok(field_ids)
    }

    #[cfg(test)]
    fn collect_field_ids_inner(
        &self,
        predicate: &Predicate,
        field_ids: &mut HashSet<i32>,
    ) -> Result<()> {
        match predicate {
            Predicate::Leaf { column, .. } => {
                if let Some(field_id) = self.find_field_id_by_name(column)? {
                    field_ids.insert(field_id);
                }
            }
            Predicate::And(children) | Predicate::Or(children) => {
                for child in children {
                    self.collect_field_ids_inner(child, field_ids)?;
                }
            }
            Predicate::Not(inner) => self.collect_field_ids_inner(inner, field_ids)?,
            Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
        }
        Ok(())
    }
}

/// Whether the sorted global index can evaluate this operator directly.
/// Operators that fall outside this set bypass the index and are evaluated
/// later in the read pipeline (stats prune + parquet row filter).
fn is_sorted_global_index_supported_op(op: PredicateOperator) -> bool {
    matches!(
        op,
        PredicateOperator::Eq
            | PredicateOperator::NotEq
            | PredicateOperator::Lt
            | PredicateOperator::LtEq
            | PredicateOperator::Gt
            | PredicateOperator::GtEq
            | PredicateOperator::In
            | PredicateOperator::NotIn
            | PredicateOperator::IsNull
            | PredicateOperator::IsNotNull
            | PredicateOperator::Between
            | PredicateOperator::NotBetween
            | PredicateOperator::StartsWith
            | PredicateOperator::EndsWith
            | PredicateOperator::Contains
            | PredicateOperator::Like
    )
}

fn requires_fallback_scan(op: PredicateOperator) -> bool {
    matches!(
        op,
        PredicateOperator::Lt
            | PredicateOperator::LtEq
            | PredicateOperator::Gt
            | PredicateOperator::GtEq
            | PredicateOperator::Between
            | PredicateOperator::NotBetween
            | PredicateOperator::EndsWith
            | PredicateOperator::Contains
            | PredicateOperator::Like
    )
}

fn fallback_plan_evaluates_entry(
    plan: FallbackScanPlan,
    kind: GlobalIndexFileKind,
    selected: bool,
) -> bool {
    !selected || plan.allowed(kind)
}

fn add_file_size(total: &mut i64, file_size: i64) -> bool {
    if file_size < 0 {
        return false;
    }
    match total.checked_add(file_size) {
        Some(next) => {
            *total = next;
            true
        }
        None => false,
    }
}

/// Convert a RoaringTreemap to merged RowRanges (already sorted and deduplicated).
fn bitmap_to_ranges(bitmap: &RoaringTreemap) -> Vec<RowRange> {
    if bitmap.is_empty() {
        return Vec::new();
    }
    let mut ranges = Vec::new();
    let mut iter = bitmap.iter();
    let first = iter.next().unwrap();
    let mut start = first as i64;
    let mut end = start;

    for id in iter {
        let id = id as i64;
        if id == end + 1 {
            end = id;
        } else {
            ranges.push(RowRange::new(start, end));
            start = id;
            end = id;
        }
    }
    ranges.push(RowRange::new(start, end));
    ranges
}

/// Intersect two sorted range lists using RowRangeIndex for efficient binary search.
fn intersect_sorted_ranges(a: &[RowRange], b: &[RowRange]) -> Vec<RowRange> {
    let idx = RowRangeIndex::create(a.to_vec());
    let mut result = Vec::new();
    for r in b {
        result.extend(idx.intersected_ranges(r.from(), r.to()));
    }
    result
}

fn data_ranges_for_search_mode(
    search_mode: GlobalIndexSearchMode,
    next_row_id: Option<i64>,
    data_ranges: &[RowRange],
) -> Option<Vec<RowRange>> {
    match search_mode {
        GlobalIndexSearchMode::Fast => None,
        GlobalIndexSearchMode::Full => match next_row_id {
            Some(next_row_id) if next_row_id > 0 => Some(vec![RowRange::new(0, next_row_id - 1)]),
            _ => None,
        },
        GlobalIndexSearchMode::Detail => {
            if data_ranges.is_empty() {
                None
            } else {
                Some(data_ranges.to_vec())
            }
        }
    }
}

fn indexed_ranges_from_coverage(
    coverage_by_field: &HashMap<i32, Vec<RowRange>>,
    field_ids: &HashSet<i32>,
) -> Vec<RowRange> {
    let mut ranges: Option<Vec<RowRange>> = None;
    for field_id in field_ids {
        let Some(field_ranges) = coverage_by_field.get(field_id) else {
            return Vec::new();
        };
        if field_ranges.is_empty() {
            return Vec::new();
        }
        let field_ranges = super::merge_row_ranges(field_ranges.clone());
        ranges = Some(match ranges {
            None => field_ranges,
            Some(existing) => intersect_sorted_ranges(&existing, &field_ranges),
        });
    }
    ranges.map(super::merge_row_ranges).unwrap_or_default()
}

fn unindexed_ranges_for_coverage(
    coverage_by_field: &HashMap<i32, Vec<RowRange>>,
    field_ids: &HashSet<i32>,
    search_mode: GlobalIndexSearchMode,
    next_row_id: Option<i64>,
    data_ranges: &[RowRange],
) -> Vec<RowRange> {
    let Some(data_ranges) = data_ranges_for_search_mode(search_mode, next_row_id, data_ranges)
    else {
        return Vec::new();
    };
    let indexed_ranges = indexed_ranges_from_coverage(coverage_by_field, field_ids);
    super::source::exclude_row_ranges(&data_ranges, &indexed_ranges)
}

/// Compute row ranges not covered by a family of global index files.
///
/// This mirrors Java `GlobalIndexCoverage`: `full` compares index coverage
/// against `[0, snapshot.next_row_id - 1]`, while `detail` compares against
/// exact data-file row ranges supplied by the caller.
pub(crate) fn unindexed_ranges_for_global_index_entries(
    index_entries: &[IndexManifestEntry],
    field_ids: &HashSet<i32>,
    search_mode: GlobalIndexSearchMode,
    next_row_id: Option<i64>,
    data_ranges: &[RowRange],
    index_file_filter: impl Fn(&IndexFileMeta) -> bool,
) -> Vec<RowRange> {
    let mut coverage_by_field: HashMap<i32, Vec<RowRange>> = HashMap::new();
    for entry in index_entries {
        if entry.kind != FileKind::Add || !index_file_filter(&entry.index_file) {
            continue;
        }
        let Some(global_meta) = entry.index_file.global_index_meta.as_ref() else {
            continue;
        };
        let row_range = RowRange::new(global_meta.row_range_start, global_meta.row_range_end);
        coverage_by_field
            .entry(global_meta.index_field_id)
            .or_default()
            .push(row_range.clone());
        if let Some(extra_field_ids) = global_meta.extra_field_ids.as_ref() {
            for extra_field_id in extra_field_ids {
                coverage_by_field
                    .entry(*extra_field_id)
                    .or_default()
                    .push(row_range.clone());
            }
        }
    }
    unindexed_ranges_for_coverage(
        &coverage_by_field,
        field_ids,
        search_mode,
        next_row_id,
        data_ranges,
    )
}

/// Resolve live deletion-vector index entries into global row-id ranges.
///
/// Data-evolution DV entries are keyed by the normal anchor data file. The DV
/// bitmap positions are local to that anchor file's `first_row_id`, so this
/// helper joins index metadata with live data-file metadata before converting
/// deleted positions to global row IDs.
pub(crate) async fn deleted_row_ranges_for_data_evolution_dvs(
    table: &Table,
    index_entries: &[IndexManifestEntry],
) -> Result<Vec<RowRange>> {
    if !index_entries.iter().any(|entry| {
        entry.kind == FileKind::Add && entry.index_file.index_type == DELETION_VECTORS_INDEX_TYPE
    }) {
        return Ok(Vec::new());
    }

    let plan = table
        .new_read_builder()
        .new_scan()
        .with_scan_all_files()
        .plan()
        .await?;

    let mut first_row_ids: HashMap<(Vec<u8>, i32, String), i64> = HashMap::new();
    for split in plan.splits() {
        let partition = split.partition().to_serialized_bytes();
        let bucket = split.bucket();
        for file in split.data_files() {
            if let Some(first_row_id) = file.first_row_id {
                first_row_ids.insert(
                    (partition.clone(), bucket, file.file_name.clone()),
                    first_row_id,
                );
            }
        }
    }

    let mut ranges = Vec::new();
    let table_path = table.location().trim_end_matches('/');
    for entry in index_entries {
        if entry.kind != FileKind::Add || entry.index_file.index_type != DELETION_VECTORS_INDEX_TYPE
        {
            continue;
        }
        let Some(dv_ranges) = entry.index_file.deletion_vectors_ranges.as_ref() else {
            continue;
        };
        let index_path = format!("{table_path}/{INDEX_DIR}/{}", entry.index_file.file_name);
        for (data_file_name, meta) in dv_ranges {
            let key = (
                entry.partition.clone(),
                entry.bucket,
                data_file_name.clone(),
            );
            let first_row_id = first_row_ids.get(&key).copied().ok_or_else(|| {
                crate::Error::DataInvalid {
                    message: format!(
                        "Deletion vector references data file '{}' but no live row-tracked file was found",
                        data_file_name
                    ),
                    source: None,
                }
            })?;
            let deletion_file = DeletionFile::new(
                index_path.clone(),
                meta.offset as i64,
                meta.length as i64,
                meta.cardinality,
            );
            let deletion_vector =
                DeletionVectorFactory::read(table.file_io(), &deletion_file).await?;
            for deleted in deletion_vector.iter() {
                let deleted = i64::try_from(deleted).map_err(|_| crate::Error::DataInvalid {
                    message: format!(
                        "Deleted position {deleted} for data file '{}' exceeds i64::MAX",
                        data_file_name
                    ),
                    source: None,
                })?;
                let row_id =
                    first_row_id
                        .checked_add(deleted)
                        .ok_or_else(|| crate::Error::DataInvalid {
                            message: format!(
                                "Deleted row id overflows i64 for data file '{}'",
                                data_file_name
                            ),
                            source: None,
                        })?;
                ranges.push(RowRange::new(row_id, row_id));
            }
        }
    }

    Ok(super::merge_row_ranges(ranges))
}

/// Index for row ranges. Stores sorted, non-overlapping ranges and supports
/// efficient intersection queries via binary search.
///
/// Reference: [org.apache.paimon.utils.RowRangeIndex](https://github.com/apache/paimon/blob/master/paimon-common/src/main/java/org/apache/paimon/utils/RowRangeIndex.java)
#[derive(Debug, Clone)]
pub(crate) struct RowRangeIndex {
    ranges: Vec<RowRange>,
    starts: Vec<i64>,
    ends: Vec<i64>,
}

impl RowRangeIndex {
    /// Create a new `RowRangeIndex` from a list of ranges.
    /// Ranges are sorted and merged to eliminate overlaps.
    pub fn create(ranges: Vec<RowRange>) -> Self {
        let ranges = super::merge_row_ranges(ranges);
        let starts: Vec<i64> = ranges.iter().map(|r| r.from()).collect();
        let ends: Vec<i64> = ranges.iter().map(|r| r.to()).collect();
        Self {
            ranges,
            starts,
            ends,
        }
    }

    /// Returns the underlying ranges.
    #[cfg(test)]
    pub fn ranges(&self) -> &[RowRange] {
        &self.ranges
    }

    /// Returns true if the index has any range that intersects `[start, end]`.
    pub fn intersects(&self, start: i64, end: i64) -> bool {
        let candidate = lower_bound(&self.ends, start);
        candidate < self.starts.len() && self.starts[candidate] <= end
    }

    /// Counts rows in this index that intersect `[start, end]`.
    pub fn intersection_row_count(&self, start: i64, end: i64) -> usize {
        if start > end {
            return 0;
        }
        self.intersected_ranges(start, end)
            .into_iter()
            .fold(0usize, |total, range| {
                let len = range.to().saturating_sub(range.from()).saturating_add(1);
                total.saturating_add(usize::try_from(len).unwrap_or(usize::MAX))
            })
    }

    /// Returns the sub-ranges of this index that intersect `[start, end]`,
    /// clipped to the intersection boundaries.
    pub fn intersected_ranges(&self, start: i64, end: i64) -> Vec<RowRange> {
        let left = lower_bound(&self.ends, start);
        if left >= self.ranges.len() || self.starts[left] > end {
            return Vec::new();
        }

        let mut right = lower_bound(&self.ends, end);
        if right >= self.ranges.len() {
            right = self.ranges.len() - 1;
        }

        let mut result = Vec::new();

        // First range: clip from the left
        let first = &self.ranges[left];
        result.push(RowRange::new(start.max(first.from()), end.min(first.to())));

        // Middle ranges: fully included
        if right > left + 1 {
            for r in &self.ranges[left + 1..right] {
                result.push(r.clone());
            }
        }

        // Last range (if different from first): clip from the right
        if right != left {
            let last = &self.ranges[right];
            if last.from() <= end {
                result.push(RowRange::new(start.max(last.from()), end.min(last.to())));
            }
        }

        result
    }
}

pub(crate) fn search_limit_with_deleted_rows(
    limit: usize,
    row_range_start: i64,
    row_range_end: i64,
    deleted_rows: Option<&RowRangeIndex>,
) -> usize {
    let Some(range_len) = row_range_end
        .checked_sub(row_range_start)
        .and_then(|len| len.checked_add(1))
        .and_then(|len| usize::try_from(len).ok())
    else {
        return limit;
    };

    let deleted_count = deleted_rows
        .map(|index| index.intersection_row_count(row_range_start, row_range_end))
        .unwrap_or(0)
        .min(range_len);
    limit.saturating_add(deleted_count).min(range_len)
}

/// Binary search: find the first index where `sorted[index] >= target`.
fn lower_bound(sorted: &[i64], target: i64) -> usize {
    let mut left = 0;
    let mut right = sorted.len();
    while left < right {
        let mid = left + (right - left) / 2;
        if sorted[mid] < target {
            left = mid + 1;
        } else {
            right = mid;
        }
    }
    left
}

/// Create a GlobalIndexScanner and evaluate predicates, returning row ranges.
/// This is the main entry point for the table scan integration.
///
/// Returns `None` if global index is not available or predicates can't be evaluated.
pub(crate) struct GlobalIndexEvaluation<'a> {
    pub(crate) file_io: &'a FileIO,
    pub(crate) table_path: &'a str,
    pub(crate) index_entries: &'a [IndexManifestEntry],
    pub(crate) predicates: &'a [Predicate],
    pub(crate) schema_fields: &'a [DataField],
    pub(crate) search_mode: GlobalIndexSearchMode,
    pub(crate) global_index_thread_num: usize,
    pub(crate) btree_fallback_scan_max_size: i64,
    pub(crate) bitmap_fallback_scan_max_size: i64,
    pub(crate) next_row_id: Option<i64>,
    pub(crate) data_ranges: &'a [RowRange],
}

pub(crate) async fn evaluate_global_index(
    evaluation: GlobalIndexEvaluation<'_>,
) -> Result<Option<Vec<RowRange>>> {
    let scanner = match GlobalIndexScanner::create(
        evaluation.file_io,
        evaluation.table_path,
        evaluation.global_index_thread_num,
        evaluation.btree_fallback_scan_max_size,
        evaluation.bitmap_fallback_scan_max_size,
        evaluation.index_entries,
        evaluation.schema_fields,
    )? {
        Some(s) => s,
        None => return Ok(None),
    };

    let combined = Predicate::and(evaluation.predicates.to_vec());

    let scan_result = match scanner.evaluate(&combined).await? {
        Some(scan_result) => scan_result,
        None => return Ok(None),
    };
    let mut row_ranges = scan_result.row_ranges;
    row_ranges.extend(scanner.unindexed_ranges_for_field_ids(
        &scan_result.evaluated_field_ids,
        evaluation.search_mode,
        evaluation.next_row_id,
        evaluation.data_ranges,
    ));
    Ok(Some(super::merge_row_ranges(row_ranges)))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::btree::test_util::VecFileWrite;
    use crate::btree::{BTreeIndexWriter, BlockCompressionType};
    use crate::table::bitmap_global_index_reader::BitmapGlobalIndexWriter;
    use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
    use std::sync::Arc;

    #[tokio::test]
    async fn test_try_fold_bounded_respects_concurrency_limit() {
        for limit in [1, 3] {
            let active = Arc::new(AtomicUsize::new(0));
            let peak = Arc::new(AtomicUsize::new(0));
            let futures = (0..9usize).map(|value| {
                let active = Arc::clone(&active);
                let peak = Arc::clone(&peak);
                async move {
                    let current = active.fetch_add(1, AtomicOrdering::SeqCst) + 1;
                    peak.fetch_max(current, AtomicOrdering::SeqCst);
                    tokio::task::yield_now().await;
                    active.fetch_sub(1, AtomicOrdering::SeqCst);
                    Ok::<_, crate::Error>(value)
                }
            });

            let mut values = try_fold_bounded(futures, limit, Vec::new(), |values, value| {
                values.push(value)
            })
            .await
            .unwrap();
            values.sort_unstable();

            assert_eq!(values, (0..9).collect::<Vec<_>>());
            assert_eq!(peak.load(AtomicOrdering::SeqCst), limit);
        }
    }

    #[test]
    fn test_bitmap_to_ranges() {
        assert_eq!(
            bitmap_to_ranges(&RoaringTreemap::new()),
            Vec::<RowRange>::new()
        );

        let mut bm = RoaringTreemap::new();
        bm.insert(5);
        assert_eq!(bitmap_to_ranges(&bm), vec![RowRange::new(5, 5)]);

        let mut bm = RoaringTreemap::new();
        for id in [1, 2, 3, 5, 6, 10] {
            bm.insert(id);
        }
        assert_eq!(
            bitmap_to_ranges(&bm),
            vec![
                RowRange::new(1, 3),
                RowRange::new(5, 6),
                RowRange::new(10, 10),
            ]
        );
    }

    #[test]
    fn test_intersect_sorted_ranges() {
        let a = vec![RowRange::new(0, 10), RowRange::new(20, 30)];
        let b = vec![RowRange::new(5, 25)];
        let result = intersect_sorted_ranges(&a, &b);
        assert_eq!(result, vec![RowRange::new(5, 10), RowRange::new(20, 25)]);
    }

    #[test]
    fn test_intersect_no_overlap() {
        let a = vec![RowRange::new(0, 5)];
        let b = vec![RowRange::new(10, 20)];
        assert!(intersect_sorted_ranges(&a, &b).is_empty());
    }

    #[test]
    fn test_serialize_datum_int() {
        let key = serialize_datum(&Datum::Int(42), &DataType::Int(crate::spec::IntType::new()));
        assert_eq!(key, 42i32.to_le_bytes().to_vec());
    }

    #[test]
    fn test_serialize_datum_string() {
        let key = serialize_datum(
            &Datum::String("hello".to_string()),
            &DataType::VarChar(crate::spec::VarCharType::new(100).unwrap()),
        );
        assert_eq!(key, b"hello".to_vec());
    }

    fn assert_bitmap_floating_meta_policy(
        data_type: DataType,
        min: Datum,
        max: Datum,
        outside: Datum,
        nan: Datum,
    ) {
        let cmp = make_bitmap_key_comparator(&data_type);
        let min_key = serialize_bitmap_datum(&min, &data_type);
        let max_key = serialize_bitmap_datum(&max, &data_type);
        let outside_key = serialize_bitmap_datum(&outside, &data_type);
        let nan_key = serialize_bitmap_datum(&nan, &data_type);
        let meta = BTreeIndexMeta::new(Some(min_key.clone()), Some(max_key), false);

        assert!(!bitmap_meta_may_match(
            &meta,
            PredicateOperator::Eq,
            &data_type,
            std::slice::from_ref(&outside_key),
            cmp.as_ref(),
        ));
        assert!(!bitmap_meta_may_match(
            &meta,
            PredicateOperator::In,
            &data_type,
            std::slice::from_ref(&outside_key),
            cmp.as_ref(),
        ));
        assert!(!bitmap_meta_may_match(
            &meta,
            PredicateOperator::IsNull,
            &data_type,
            &[],
            cmp.as_ref(),
        ));
        assert!(bitmap_meta_may_match(
            &meta,
            PredicateOperator::IsNotNull,
            &data_type,
            &[],
            cmp.as_ref(),
        ));

        let nan_meta = BTreeIndexMeta::new(Some(min_key), Some(nan_key.clone()), false);
        assert!(bitmap_meta_may_match(
            &nan_meta,
            PredicateOperator::Eq,
            &data_type,
            std::slice::from_ref(&nan_key),
            cmp.as_ref(),
        ));
        assert!(bitmap_meta_may_match(
            &nan_meta,
            PredicateOperator::In,
            &data_type,
            std::slice::from_ref(&nan_key),
            cmp.as_ref(),
        ));

        assert!(bitmap_meta_may_match(
            &meta,
            PredicateOperator::Gt,
            &data_type,
            std::slice::from_ref(&outside_key),
            cmp.as_ref(),
        ));
        assert!(bitmap_meta_may_match_between(
            &meta,
            &data_type,
            &outside_key,
            &outside_key,
            cmp.as_ref(),
        ));

        let only_nulls = BTreeIndexMeta::new(None, None, true);
        assert!(bitmap_meta_may_match(
            &only_nulls,
            PredicateOperator::IsNull,
            &data_type,
            &[],
            cmp.as_ref(),
        ));
        assert!(!bitmap_meta_may_match(
            &only_nulls,
            PredicateOperator::IsNotNull,
            &data_type,
            &[],
            cmp.as_ref(),
        ));
        assert!(!bitmap_meta_may_match(
            &only_nulls,
            PredicateOperator::NotEq,
            &data_type,
            std::slice::from_ref(&outside_key),
            cmp.as_ref(),
        ));
        assert!(!bitmap_meta_may_match_between(
            &only_nulls,
            &data_type,
            &outside_key,
            &outside_key,
            cmp.as_ref(),
        ));
    }

    #[test]
    fn test_bitmap_floating_meta_prunes_equality_and_fails_open_for_ranges() {
        assert_bitmap_floating_meta_policy(
            DataType::Float(crate::spec::FloatType::new()),
            Datum::Float(-1.0),
            Datum::Float(1.0),
            Datum::Float(2.0),
            Datum::Float(f32::NAN),
        );
        assert_bitmap_floating_meta_policy(
            DataType::Double(crate::spec::DoubleType::new()),
            Datum::Double(-1.0),
            Datum::Double(1.0),
            Datum::Double(2.0),
            Datum::Double(f64::NAN),
        );
    }

    #[test]
    fn test_row_range_index_merges_overlapping() {
        let idx = RowRangeIndex::create(vec![
            RowRange::new(0, 5),
            RowRange::new(3, 10),
            RowRange::new(20, 30),
        ]);
        assert_eq!(idx.ranges().len(), 2);
        assert_eq!(idx.ranges()[0], RowRange::new(0, 10));
        assert_eq!(idx.ranges()[1], RowRange::new(20, 30));
    }

    #[test]
    fn test_row_range_index_merges_adjacent() {
        let idx = RowRangeIndex::create(vec![RowRange::new(0, 5), RowRange::new(6, 10)]);
        assert_eq!(idx.ranges().len(), 1);
        assert_eq!(idx.ranges()[0], RowRange::new(0, 10));
    }

    #[test]
    fn test_row_range_index_intersects() {
        let idx = RowRangeIndex::create(vec![RowRange::new(10, 20), RowRange::new(30, 40)]);
        assert!(idx.intersects(15, 25));
        assert!(idx.intersects(5, 10));
        assert!(idx.intersects(20, 30));
        assert!(!idx.intersects(0, 9));
        assert!(!idx.intersects(21, 29));
        assert!(!idx.intersects(41, 50));
    }

    #[test]
    fn test_row_range_index_intersected_ranges() {
        let idx = RowRangeIndex::create(vec![
            RowRange::new(10, 20),
            RowRange::new(30, 40),
            RowRange::new(50, 60),
        ]);
        let result = idx.intersected_ranges(15, 55);
        assert_eq!(
            result,
            vec![
                RowRange::new(15, 20),
                RowRange::new(30, 40),
                RowRange::new(50, 55),
            ]
        );
    }

    #[test]
    fn test_row_range_index_intersection_row_count() {
        let idx = RowRangeIndex::create(vec![
            RowRange::new(10, 20),
            RowRange::new(30, 40),
            RowRange::new(50, 60),
        ]);

        assert_eq!(idx.intersection_row_count(15, 55), 23);
        assert_eq!(idx.intersection_row_count(21, 29), 0);
        assert_eq!(idx.intersection_row_count(55, 15), 0);
    }

    #[test]
    fn test_search_limit_with_deleted_rows_expands_and_caps() {
        let idx = RowRangeIndex::create(vec![RowRange::new(2, 4), RowRange::new(8, 10)]);

        assert_eq!(search_limit_with_deleted_rows(5, 0, 19, Some(&idx)), 11);
        assert_eq!(search_limit_with_deleted_rows(18, 0, 19, Some(&idx)), 20);
        assert_eq!(search_limit_with_deleted_rows(5, 0, 19, None), 5);
    }

    #[test]
    fn test_row_range_index_empty() {
        let idx = RowRangeIndex::create(Vec::new());
        assert!(!idx.intersects(0, 100));
        assert!(idx.intersected_ranges(0, 100).is_empty());
    }

    fn le_int_key(v: i32) -> Vec<u8> {
        v.to_le_bytes().to_vec()
    }

    /// Set up a temp dir with `index/{file_name}` containing the btree testdata file,
    /// and return (FileIO, table_path, file_name, _tmp_dir).
    fn setup_testdata_table(testdata_name: &str) -> (FileIO, String, String, tempfile::TempDir) {
        let src = format!(
            "{}/testdata/btree/{testdata_name}",
            env!("CARGO_MANIFEST_DIR")
        );
        let tmp = tempfile::tempdir().unwrap();
        let index_dir = tmp.path().join("index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::copy(&src, index_dir.join(testdata_name)).unwrap();

        let table_path = format!("file://{}", tmp.path().display());
        let file_io = crate::io::FileIOBuilder::new("file").build().unwrap();
        (file_io, table_path, testdata_name.to_string(), tmp)
    }

    type BitmapTestdataTable = (FileIO, String, String, BTreeIndexMeta, tempfile::TempDir);

    fn setup_bitmap_testdata_table(file_name: &str) -> BitmapTestdataTable {
        let src = format!("{}/testdata/bitmap/{file_name}", env!("CARGO_MANIFEST_DIR"));
        let meta_src = format!(
            "{}/testdata/bitmap/{file_name}.meta",
            env!("CARGO_MANIFEST_DIR")
        );
        let tmp = tempfile::tempdir().unwrap();
        let index_dir = tmp.path().join("index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::copy(&src, index_dir.join(file_name)).unwrap();
        let meta = BTreeIndexMeta::deserialize(&std::fs::read(meta_src).unwrap()).unwrap();

        let table_path = format!("file://{}", tmp.path().display());
        let file_io = crate::io::FileIOBuilder::new("file").build().unwrap();
        (file_io, table_path, file_name.to_string(), meta, tmp)
    }

    fn setup_java_bitmap_testdata_table() -> BitmapTestdataTable {
        setup_bitmap_testdata_table("bitmap_varchar_java.index")
    }

    fn make_global_index_entry(
        file_name: &str,
        field_id: i32,
        row_range_start: i64,
        row_range_end: i64,
        meta: &BTreeIndexMeta,
    ) -> crate::spec::IndexManifestEntry {
        make_global_index_entry_with_type(
            BTREE_GLOBAL_INDEX_TYPE,
            file_name,
            field_id,
            row_range_start,
            row_range_end,
            meta,
        )
    }

    fn make_global_index_entry_with_type(
        index_type: &str,
        file_name: &str,
        field_id: i32,
        row_range_start: i64,
        row_range_end: i64,
        meta: &BTreeIndexMeta,
    ) -> crate::spec::IndexManifestEntry {
        use crate::spec::{GlobalIndexMeta, IndexFileMeta};
        IndexManifestEntry {
            version: 1,
            kind: FileKind::Add,
            partition: vec![],
            bucket: 0,
            index_file: IndexFileMeta {
                index_type: index_type.to_string(),
                file_name: file_name.to_string(),
                file_size: 0,
                row_count: 0,
                deletion_vectors_ranges: None,
                global_index_meta: Some(GlobalIndexMeta {
                    row_range_start,
                    row_range_end,
                    index_field_id: field_id,
                    extra_field_ids: None,
                    source_meta: None,
                    index_meta: Some(meta.serialize()),
                }),
            },
        }
    }

    fn int_schema_fields() -> Vec<DataField> {
        vec![DataField::new(
            1,
            "id".to_string(),
            DataType::Int(crate::spec::IntType::new()),
        )]
    }

    fn string_schema_fields() -> Vec<DataField> {
        vec![DataField::new(
            1,
            "name".to_string(),
            DataType::VarChar(crate::spec::VarCharType::string_type()),
        )]
    }

    async fn evaluate_global_index_fast(
        file_io: &FileIO,
        table_path: &str,
        entries: &[IndexManifestEntry],
        predicates: &[Predicate],
        fields: &[DataField],
    ) -> Result<Option<Vec<RowRange>>> {
        evaluate_global_index_fast_with_fallback_size(
            file_io,
            table_path,
            entries,
            predicates,
            fields,
            i64::MAX,
            i64::MAX,
        )
        .await
    }

    async fn evaluate_global_index_fast_with_fallback_size(
        file_io: &FileIO,
        table_path: &str,
        entries: &[IndexManifestEntry],
        predicates: &[Predicate],
        fields: &[DataField],
        btree_fallback_scan_max_size: i64,
        bitmap_fallback_scan_max_size: i64,
    ) -> Result<Option<Vec<RowRange>>> {
        super::evaluate_global_index(super::GlobalIndexEvaluation {
            file_io,
            table_path,
            index_entries: entries,
            predicates,
            schema_fields: fields,
            search_mode: GlobalIndexSearchMode::Fast,
            global_index_thread_num: 32,
            btree_fallback_scan_max_size,
            bitmap_fallback_scan_max_size,
            next_row_id: None,
            data_ranges: &[],
        })
        .await
    }

    fn two_field_schema_fields() -> Vec<DataField> {
        vec![
            DataField::new(
                1,
                "id".to_string(),
                DataType::Int(crate::spec::IntType::new()),
            ),
            DataField::new(
                2,
                "value".to_string(),
                DataType::Int(crate::spec::IntType::new()),
            ),
        ]
    }

    fn int_eq(column: &str, index: usize, value: i32) -> Predicate {
        Predicate::Leaf {
            column: column.to_string(),
            index,
            data_type: DataType::Int(crate::spec::IntType::new()),
            op: PredicateOperator::Eq,
            literals: vec![Datum::Int(value)],
        }
    }

    #[test]
    fn test_unindexed_ranges_fast_mode_empty() {
        let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap();
        let meta = BTreeIndexMeta::new(None, None, false);
        let entries = vec![make_global_index_entry("idx", 1, 0, 49, &meta)];
        let fields = int_schema_fields();
        let scanner = GlobalIndexScanner::create(
            &file_io,
            "memory:/t",
            32,
            i64::MAX,
            i64::MAX,
            &entries,
            &fields,
        )
        .expect("create scanner")
        .expect("scanner");

        let ranges = scanner
            .unindexed_ranges(
                &int_eq("id", 0, 7),
                GlobalIndexSearchMode::Fast,
                Some(100),
                &[RowRange::new(50, 99)],
            )
            .unwrap();
        assert!(ranges.is_empty());
    }

    #[test]
    fn test_unindexed_ranges_full_uses_snapshot_next_row_id() {
        let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap();
        let meta = BTreeIndexMeta::new(None, None, false);
        let entries = vec![make_global_index_entry("idx", 1, 0, 49, &meta)];
        let fields = int_schema_fields();
        let scanner = GlobalIndexScanner::create(
            &file_io,
            "memory:/t",
            32,
            i64::MAX,
            i64::MAX,
            &entries,
            &fields,
        )
        .expect("create scanner")
        .expect("scanner");

        let ranges = scanner
            .unindexed_ranges(
                &int_eq("id", 0, 7),
                GlobalIndexSearchMode::Full,
                Some(100),
                &[],
            )
            .unwrap();
        assert_eq!(ranges, vec![RowRange::new(50, 99)]);
    }

    #[test]
    fn test_unindexed_ranges_detail_uses_data_file_ranges() {
        let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap();
        let meta = BTreeIndexMeta::new(None, None, false);
        let entries = vec![make_global_index_entry("idx", 1, 0, 49, &meta)];
        let fields = int_schema_fields();
        let scanner = GlobalIndexScanner::create(
            &file_io,
            "memory:/t",
            32,
            i64::MAX,
            i64::MAX,
            &entries,
            &fields,
        )
        .expect("create scanner")
        .expect("scanner");

        let ranges = scanner
            .unindexed_ranges(
                &int_eq("id", 0, 7),
                GlobalIndexSearchMode::Detail,
                Some(100),
                &[
                    RowRange::new(0, 10),
                    RowRange::new(40, 60),
                    RowRange::new(80, 90),
                ],
            )
            .unwrap();
        assert_eq!(ranges, vec![RowRange::new(50, 60), RowRange::new(80, 90)]);
    }

    #[test]
    fn test_unindexed_ranges_uses_all_predicate_field_coverage() {
        let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap();
        let meta = BTreeIndexMeta::new(None, None, false);
        let entries = vec![
            make_global_index_entry("idx_id", 1, 0, 49, &meta),
            make_global_index_entry("idx_value", 2, 0, 99, &meta),
        ];
        let fields = two_field_schema_fields();
        let scanner = GlobalIndexScanner::create(
            &file_io,
            "memory:/t",
            32,
            i64::MAX,
            i64::MAX,
            &entries,
            &fields,
        )
        .expect("create scanner")
        .expect("scanner");
        let predicate = Predicate::and(vec![int_eq("id", 0, 7), int_eq("value", 1, 8)]);

        let ranges = scanner
            .unindexed_ranges(&predicate, GlobalIndexSearchMode::Full, Some(100), &[])
            .unwrap();
        assert_eq!(ranges, vec![RowRange::new(50, 99)]);
    }

    #[test]
    fn test_unindexed_ranges_missing_field_coverage_reads_all_data_ranges() {
        let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap();
        let meta = BTreeIndexMeta::new(None, None, false);
        let entries = vec![make_global_index_entry("idx_id", 1, 0, 49, &meta)];
        let fields = two_field_schema_fields();
        let scanner = GlobalIndexScanner::create(
            &file_io,
            "memory:/t",
            32,
            i64::MAX,
            i64::MAX,
            &entries,
            &fields,
        )
        .expect("create scanner")
        .expect("scanner");
        let predicate = Predicate::and(vec![int_eq("id", 0, 7), int_eq("value", 1, 8)]);

        let ranges = scanner
            .unindexed_ranges(&predicate, GlobalIndexSearchMode::Full, Some(100), &[])
            .unwrap();
        assert_eq!(ranges, vec![RowRange::new(0, 99)]);
    }

    #[test]
    fn test_unindexed_ranges_counts_extra_field_coverage() {
        let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap();
        let meta = BTreeIndexMeta::new(None, None, false);
        let mut entry = make_global_index_entry("idx_id_value", 1, 0, 99, &meta);
        entry
            .index_file
            .global_index_meta
            .as_mut()
            .unwrap()
            .extra_field_ids = Some(vec![2]);
        let fields = two_field_schema_fields();
        let scanner = GlobalIndexScanner::create(
            &file_io,
            "memory:/t",
            32,
            i64::MAX,
            i64::MAX,
            &[entry],
            &fields,
        )
        .expect("create scanner")
        .expect("scanner");

        let ranges = scanner
            .unindexed_ranges(
                &int_eq("value", 1, 8),
                GlobalIndexSearchMode::Full,
                Some(100),
                &[],
            )
            .unwrap();
        assert!(ranges.is_empty());
    }

    #[tokio::test]
    async fn test_evaluate_extra_field_only_without_composite_reader_falls_back() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let mut entry = make_global_index_entry(&file_name, 1, 0, 99, &meta);
        entry
            .index_file
            .global_index_meta
            .as_mut()
            .unwrap()
            .extra_field_ids = Some(vec![2]);
        let fields = two_field_schema_fields();
        let predicates = vec![int_eq("value", 1, 50)];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &[entry], &predicates, &fields)
                .await
                .unwrap();
        assert!(
            result.is_none(),
            "extra-field-only predicates must fall back until composite-key btree reads are supported"
        );
    }

    #[tokio::test]
    async fn test_evaluate_global_index_eq() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();

        // key=50 -> row_id=25, offset by row_range_start=0 -> global row_id=25
        let predicates = vec![Predicate::Leaf {
            column: "id".to_string(),
            index: 0,
            data_type: DataType::Int(crate::spec::IntType::new()),
            op: PredicateOperator::Eq,
            literals: vec![Datum::Int(50)],
        }];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        let ranges = result.unwrap();
        assert_eq!(ranges, vec![RowRange::new(25, 25)]);
    }

    #[tokio::test]
    async fn test_missing_index_meta_returns_error() {
        let (file_io, table_path, file_name, tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let second_file_name = "btree_int_100_no_compress_2.bin";
        std::fs::copy(
            tmp.path().join("index").join(&file_name),
            tmp.path().join("index").join(second_file_name),
        )
        .unwrap();
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let valid_entry = make_global_index_entry(&file_name, 1, 0, 99, &meta);
        let mut invalid_entry = make_global_index_entry(second_file_name, 1, 100, 199, &meta);
        invalid_entry
            .index_file
            .global_index_meta
            .as_mut()
            .unwrap()
            .index_meta = None;

        let error = evaluate_global_index_fast(
            &file_io,
            &table_path,
            &[valid_entry, invalid_entry],
            &[int_eq("id", 0, 50)],
            &int_schema_fields(),
        )
        .await
        .expect_err("missing sorted index metadata must fail the scan");

        assert!(matches!(
            error,
            crate::Error::DataInvalid { message, .. }
                if message.contains(second_file_name)
        ));
    }

    #[tokio::test]
    async fn test_invalid_index_meta_returns_error() {
        let (file_io, table_path, file_name, tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let second_file_name = "btree_int_100_no_compress_2.bin";
        std::fs::copy(
            tmp.path().join("index").join(&file_name),
            tmp.path().join("index").join(second_file_name),
        )
        .unwrap();
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let valid_entry = make_global_index_entry(&file_name, 1, 0, 99, &meta);
        let mut invalid_entry = make_global_index_entry(second_file_name, 1, 100, 199, &meta);
        let mut invalid_meta = vec![0; 9];
        invalid_meta[..4].copy_from_slice(&10i32.to_le_bytes());
        invalid_entry
            .index_file
            .global_index_meta
            .as_mut()
            .unwrap()
            .index_meta = Some(invalid_meta);

        let error = evaluate_global_index_fast(
            &file_io,
            &table_path,
            &[valid_entry, invalid_entry],
            &[int_eq("id", 0, 50)],
            &int_schema_fields(),
        )
        .await
        .expect_err("invalid sorted index metadata must fail the scan");

        assert!(matches!(
            error,
            crate::Error::DataInvalid {
                message,
                source: Some(_),
            } if message.contains(second_file_name)
        ));
    }

    #[tokio::test]
    async fn test_missing_global_index_meta_returns_error() {
        let (file_io, table_path, file_name, tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let second_file_name = "btree_int_100_no_compress_2.bin";
        std::fs::copy(
            tmp.path().join("index").join(&file_name),
            tmp.path().join("index").join(second_file_name),
        )
        .unwrap();
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let valid_entry = make_global_index_entry(&file_name, 1, 0, 99, &meta);
        let mut invalid_entry = make_global_index_entry(second_file_name, 1, 100, 199, &meta);
        invalid_entry.index_file.global_index_meta = None;

        let error = evaluate_global_index_fast(
            &file_io,
            &table_path,
            &[valid_entry, invalid_entry],
            &[int_eq("id", 0, 50)],
            &int_schema_fields(),
        )
        .await
        .expect_err("missing global index metadata must fail the scan");

        assert!(matches!(
            error,
            crate::Error::DataInvalid { message, .. }
                if message.contains(second_file_name)
        ));
    }

    #[tokio::test]
    async fn test_evaluate_java_bitmap_golden_index_eq_and_null() {
        let data_type = DataType::VarChar(crate::spec::VarCharType::string_type());
        let (file_io, table_path, file_name, meta, _tmp) = setup_java_bitmap_testdata_table();
        let entries = vec![make_global_index_entry_with_type(
            BITMAP_GLOBAL_INDEX_TYPE,
            &file_name,
            1,
            100,
            109,
            &meta,
        )];
        let fields = string_schema_fields();
        assert_eq!(meta.first_key, Some(b"alpha".to_vec()));
        assert_eq!(meta.last_key, Some(b"office".to_vec()));
        assert!(meta.has_nulls);

        let eq_predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: data_type.clone(),
            op: PredicateOperator::Eq,
            literals: vec![Datum::String("k2".to_string())],
        }];
        let eq_result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &eq_predicates, &fields)
                .await
                .unwrap();
        assert_eq!(eq_result.unwrap(), vec![RowRange::new(105, 106)]);

        let null_predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type,
            op: PredicateOperator::IsNull,
            literals: vec![],
        }];
        let null_result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &null_predicates, &fields)
                .await
                .unwrap();
        assert_eq!(null_result.unwrap(), vec![RowRange::new(104, 104)]);
    }

    async fn assert_bitmap_int_fixture(file_name: &str) {
        let data_type = DataType::Int(crate::spec::IntType::new());
        let (file_io, table_path, file_name, meta, _tmp) = setup_bitmap_testdata_table(file_name);
        let entries = vec![make_global_index_entry_with_type(
            BITMAP_GLOBAL_INDEX_TYPE,
            &file_name,
            1,
            100,
            105,
            &meta,
        )];
        let fields = int_schema_fields();
        assert_eq!(meta.first_key, Some(le_int_key(-1)));
        assert_eq!(meta.last_key, Some(le_int_key(256)));
        assert!(meta.has_nulls);

        let cases = [
            (
                PredicateOperator::Eq,
                vec![Datum::Int(0)],
                vec![RowRange::new(101, 102)],
            ),
            (
                PredicateOperator::Eq,
                vec![Datum::Int(256)],
                vec![RowRange::new(104, 104)],
            ),
            (
                PredicateOperator::In,
                vec![Datum::Int(-1), Datum::Int(1), Datum::Int(256)],
                vec![RowRange::new(100, 100), RowRange::new(103, 104)],
            ),
            (
                PredicateOperator::NotEq,
                vec![Datum::Int(0)],
                vec![RowRange::new(100, 100), RowRange::new(103, 104)],
            ),
            (
                PredicateOperator::NotIn,
                vec![Datum::Int(-1), Datum::Int(1), Datum::Int(256)],
                vec![RowRange::new(101, 102)],
            ),
            (
                PredicateOperator::IsNull,
                vec![],
                vec![RowRange::new(105, 105)],
            ),
        ];

        for (op, literals, expected) in cases {
            let predicates = vec![Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: data_type.clone(),
                op,
                literals,
            }];
            let result =
                evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                    .await
                    .unwrap()
                    .unwrap();
            assert_eq!(result, expected, "{file_name}: {op}");
        }
    }

    #[tokio::test]
    async fn test_evaluate_java_logical_order_bitmap_int_fixture() {
        assert_bitmap_int_fixture("bitmap_int_logical_java.index").await;
    }

    async fn assert_bitmap_nan_equality_uses_direct_lookup(
        data_type: DataType,
        nan_literals: [Datum; 3],
        zero: Datum,
    ) {
        let output = VecFileWrite::new();
        let captured = output.clone();
        let mut writer = BitmapGlobalIndexWriter::new(
            Box::new(output),
            1,
            BlockCompressionType::None,
            make_bitmap_key_comparator(&data_type),
        );
        for (row_id, literal) in nan_literals.iter().enumerate() {
            let key = serialize_bitmap_datum(literal, &data_type);
            writer.write(Some(&key), row_id as i64).unwrap();
        }
        let zero_key = serialize_bitmap_datum(&zero, &data_type);
        writer.write(Some(&zero_key), 3).unwrap();
        let write_result = writer.finish().await.unwrap();
        let bytes = captured.to_vec();

        let tmp = tempfile::tempdir().unwrap();
        let index_dir = tmp.path().join("index");
        std::fs::create_dir_all(&index_dir).unwrap();
        let file_name = "bitmap-current.index";
        std::fs::write(index_dir.join(file_name), &bytes).unwrap();
        let table_path = format!("file://{}", tmp.path().display());
        let file_io = crate::io::FileIOBuilder::new("file").build().unwrap();

        let mut entry = make_global_index_entry_with_type(
            BITMAP_GLOBAL_INDEX_TYPE,
            file_name,
            1,
            100,
            103,
            &write_result.meta,
        );
        entry.index_file.file_size = bytes.len() as i64;
        let entries = vec![entry];
        let fields = vec![DataField::new(1, "id".to_string(), data_type.clone())];
        let cases = [
            (PredicateOperator::Eq, vec![nan_literals[0].clone()]),
            (
                PredicateOperator::In,
                vec![nan_literals[1].clone(), nan_literals[2].clone()],
            ),
        ];

        for (op, literals) in cases {
            let predicates = vec![Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: data_type.clone(),
                op,
                literals,
            }];
            let result = evaluate_global_index_fast_with_fallback_size(
                &file_io,
                &table_path,
                &entries,
                &predicates,
                &fields,
                i64::MAX,
                0,
            )
            .await
            .unwrap()
            .unwrap();
            assert_eq!(result, vec![RowRange::new(100, 102)], "{data_type:?}: {op}");
        }
    }

    #[tokio::test]
    async fn test_bitmap_nan_equality_uses_direct_lookup_with_fallback_scan_disabled() {
        assert_bitmap_nan_equality_uses_direct_lookup(
            DataType::Float(crate::spec::FloatType::new()),
            [
                Datum::Float(f32::from_bits(0xffc0_0001)),
                Datum::Float(f32::from_bits(0x7fc0_0010)),
                Datum::Float(f32::NAN),
            ],
            Datum::Float(0.0),
        )
        .await;
        assert_bitmap_nan_equality_uses_direct_lookup(
            DataType::Double(crate::spec::DoubleType::new()),
            [
                Datum::Double(f64::from_bits(0xfff8_0000_0000_0001)),
                Datum::Double(f64::from_bits(0x7ff8_0000_0000_0010)),
                Datum::Double(f64::NAN),
            ],
            Datum::Double(0.0),
        )
        .await;
    }

    fn legacy_floating_comparator(data_type: &DataType) -> BoxedCmp {
        match data_type {
            DataType::Float(_) => Box::new(|left, right| {
                let left = f32::from_le_bytes(left.try_into().unwrap());
                let right = f32::from_le_bytes(right.try_into().unwrap());
                left.total_cmp(&right)
            }),
            DataType::Double(_) => Box::new(|left, right| {
                let left = f64::from_le_bytes(left.try_into().unwrap());
                let right = f64::from_le_bytes(right.try_into().unwrap());
                left.total_cmp(&right)
            }),
            _ => unreachable!("legacy floating comparator requires Float or Double"),
        }
    }

    async fn assert_legacy_floating_btree(
        file_name: &str,
        data_type: DataType,
        nan_keys: Vec<Vec<u8>>,
        nan_literals: Vec<Datum>,
        zero_key: Vec<u8>,
        zero_literal: Datum,
    ) {
        let mut rows = nan_keys
            .into_iter()
            .enumerate()
            .map(|(row_id, key)| (key, row_id as i64))
            .collect::<Vec<_>>();
        rows.push((zero_key, 3));
        let cmp = legacy_floating_comparator(&data_type);
        rows.sort_by(|left, right| cmp(&left.0, &right.0));
        let expected_first_key = rows.first().unwrap().0.clone();
        let expected_last_key = rows.last().unwrap().0.clone();

        let output = VecFileWrite::new();
        let captured = output.clone();
        let mut writer =
            BTreeIndexWriter::with_comparator(Box::new(output), 1, BlockCompressionType::None, cmp);
        for (key, row_id) in rows {
            writer.write(Some(&key), row_id).await.unwrap();
        }
        let write_result = writer.finish().await.unwrap();
        assert_eq!(write_result.meta.first_key, Some(expected_first_key));
        assert_eq!(write_result.meta.last_key, Some(expected_last_key));

        let tmp = tempfile::tempdir().unwrap();
        let index_dir = tmp.path().join("index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::write(index_dir.join(file_name), captured.to_vec()).unwrap();
        let table_path = format!("file://{}", tmp.path().display());
        let file_io = crate::io::FileIOBuilder::new("file").build().unwrap();
        let entries = vec![make_global_index_entry(
            file_name,
            1,
            100,
            103,
            &write_result.meta,
        )];
        let fields = vec![DataField::new(1, "id".to_string(), data_type.clone())];
        let cases = [
            (
                PredicateOperator::Eq,
                vec![zero_literal.clone()],
                vec![RowRange::new(103, 103)],
            ),
            (
                PredicateOperator::Eq,
                vec![nan_literals[0].clone()],
                vec![RowRange::new(100, 100)],
            ),
            (
                PredicateOperator::In,
                vec![
                    nan_literals[0].clone(),
                    nan_literals[1].clone(),
                    zero_literal,
                ],
                vec![RowRange::new(100, 101), RowRange::new(103, 103)],
            ),
        ];

        for (op, literals, expected) in cases {
            let predicates = vec![Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: data_type.clone(),
                op,
                literals,
            }];
            let result =
                evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                    .await
                    .unwrap()
                    .unwrap();
            assert_eq!(result, expected, "{file_name}: {op}");
        }
    }

    #[tokio::test]
    async fn test_evaluate_legacy_float_btree() {
        let nan_bits = [0xffc0_0001u32, 0xffc0_0010, 0xffff_1234];
        assert_legacy_floating_btree(
            "btree_float_legacy_rust.index",
            DataType::Float(crate::spec::FloatType::new()),
            nan_bits
                .iter()
                .map(|bits| bits.to_le_bytes().to_vec())
                .collect(),
            nan_bits
                .iter()
                .map(|bits| Datum::Float(f32::from_bits(*bits)))
                .collect(),
            0.0f32.to_le_bytes().to_vec(),
            Datum::Float(0.0),
        )
        .await;
    }

    #[tokio::test]
    async fn test_evaluate_legacy_double_btree() {
        let nan_bits = [
            0xfff8_0000_0000_0001u64,
            0xfff8_0000_0000_0010,
            0xffff_1234_5678_9abc,
        ];
        assert_legacy_floating_btree(
            "btree_double_legacy_rust.index",
            DataType::Double(crate::spec::DoubleType::new()),
            nan_bits
                .iter()
                .map(|bits| bits.to_le_bytes().to_vec())
                .collect(),
            nan_bits
                .iter()
                .map(|bits| Datum::Double(f64::from_bits(*bits)))
                .collect(),
            0.0f64.to_le_bytes().to_vec(),
            Datum::Double(0.0),
        )
        .await;
    }

    #[tokio::test]
    async fn test_evaluate_java_bitmap_golden_index_string_fallback_scan() {
        let data_type = DataType::VarChar(crate::spec::VarCharType::string_type());
        let (file_io, table_path, file_name, meta, _tmp) = setup_java_bitmap_testdata_table();
        let entries = vec![make_global_index_entry_with_type(
            BITMAP_GLOBAL_INDEX_TYPE,
            &file_name,
            1,
            100,
            109,
            &meta,
        )];
        let fields = string_schema_fields();

        let ends_with_predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: data_type.clone(),
            op: PredicateOperator::EndsWith,
            literals: vec![Datum::String("ta".to_string())],
        }];
        let ends_with_result = evaluate_global_index_fast(
            &file_io,
            &table_path,
            &entries,
            &ends_with_predicates,
            &fields,
        )
        .await
        .unwrap();
        assert_eq!(
            ends_with_result.unwrap(),
            vec![RowRange::new(101, 101), RowRange::new(103, 103)]
        );

        let contains_predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: data_type.clone(),
            op: PredicateOperator::Contains,
            literals: vec![Datum::String("ph".to_string())],
        }];
        let contains_result = evaluate_global_index_fast(
            &file_io,
            &table_path,
            &entries,
            &contains_predicates,
            &fields,
        )
        .await
        .unwrap();
        assert_eq!(
            contains_result.unwrap(),
            vec![RowRange::new(100, 100), RowRange::new(102, 102)]
        );

        let like_predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: data_type.clone(),
            op: PredicateOperator::Like,
            literals: vec![Datum::String("%ha%".to_string())],
        }];
        let like_result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &like_predicates, &fields)
                .await
                .unwrap();
        assert_eq!(
            like_result.unwrap(),
            vec![RowRange::new(100, 100), RowRange::new(102, 102)]
        );

        let less_than_predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: data_type.clone(),
            op: PredicateOperator::Lt,
            literals: vec![Datum::String("delta".to_string())],
        }];
        let less_than_result = evaluate_global_index_fast(
            &file_io,
            &table_path,
            &entries,
            &less_than_predicates,
            &fields,
        )
        .await
        .unwrap();
        assert_eq!(less_than_result.unwrap(), vec![RowRange::new(100, 102)]);

        let mut over_limit_entries = vec![make_global_index_entry_with_type(
            BITMAP_GLOBAL_INDEX_TYPE,
            &file_name,
            1,
            100,
            109,
            &meta,
        )];
        over_limit_entries[0].index_file.file_size = 2;
        let over_limit_less_than = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &over_limit_entries,
            &less_than_predicates,
            &fields,
            i64::MAX,
            1,
        )
        .await
        .unwrap();
        assert!(
            over_limit_less_than.is_none(),
            "range predicates require fallback dictionary scans and should be unsupported over budget"
        );

        let no_match_contains = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: data_type.clone(),
            op: PredicateOperator::Contains,
            literals: vec![Datum::String("zz".to_string())],
        }];
        let over_limit_result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &over_limit_entries,
            &no_match_contains,
            &fields,
            i64::MAX,
            1,
        )
        .await
        .unwrap();
        assert!(
            over_limit_result.is_none(),
            "fallback scans over budget should be unsupported instead of returning full coverage"
        );

        let direct_with_over_limit_fallback = vec![Predicate::and(vec![
            Predicate::Leaf {
                column: "name".to_string(),
                index: 0,
                data_type: data_type.clone(),
                op: PredicateOperator::Eq,
                literals: vec![Datum::String("k2".to_string())],
            },
            Predicate::Leaf {
                column: "name".to_string(),
                index: 0,
                data_type,
                op: PredicateOperator::Contains,
                literals: vec![Datum::String("zz".to_string())],
            },
        ])];
        let direct_result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &over_limit_entries,
            &direct_with_over_limit_fallback,
            &fields,
            i64::MAX,
            1,
        )
        .await
        .unwrap();
        assert_eq!(direct_result.unwrap(), vec![RowRange::new(105, 106)]);
    }

    #[tokio::test]
    async fn test_btree_fallback_scan_over_limit_is_unsupported() {
        let (file_io, table_path, file_name, tmp) =
            setup_testdata_table("btree_varchar_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(b"a".to_vec()), Some(b"yyyy".to_vec()), false);
        let fields = string_schema_fields();
        let data_type = DataType::VarChar(crate::spec::VarCharType::string_type());
        let predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type,
            op: PredicateOperator::Contains,
            literals: vec![Datum::String("not-present".to_string())],
        }];

        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let exact_result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &entries,
            &predicates,
            &fields,
            i64::MAX,
            i64::MAX,
        )
        .await
        .unwrap();
        assert_eq!(exact_result.unwrap(), Vec::<RowRange>::new());

        let mut over_limit_entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        over_limit_entries[0].index_file.file_size = 2;
        let over_limit_result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &over_limit_entries,
            &predicates,
            &fields,
            1,
            i64::MAX,
        )
        .await
        .unwrap();
        assert!(
            over_limit_result.is_none(),
            "fallback scans over budget should be unsupported instead of returning full coverage"
        );

        let second_file_name = "btree_varchar_100_no_compress_2.bin";
        std::fs::copy(
            tmp.path().join("index").join(&file_name),
            tmp.path().join("index").join(second_file_name),
        )
        .unwrap();
        let mut first = make_global_index_entry(&file_name, 1, 0, 99, &meta);
        first.index_file.file_size = 1;
        let mut second = make_global_index_entry(second_file_name, 1, 100, 199, &meta);
        second.index_file.file_size = 1;
        let total_over_limit_result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &[first, second],
            &predicates,
            &fields,
            1,
            i64::MAX,
        )
        .await
        .unwrap();
        assert!(
            total_over_limit_result.is_none(),
            "fallback budget should use selected files' total size, not per-file size"
        );
    }

    #[tokio::test]
    async fn test_fallback_scan_over_limit_with_mixed_index_kinds_is_unsupported() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_varchar_100_no_compress.bin");
        let btree_meta = BTreeIndexMeta::new(Some(b"a".to_vec()), Some(b"yyyy".to_vec()), false);
        let bitmap_meta = BTreeIndexMeta::new(Some(b"m".to_vec()), Some(b"z".to_vec()), false);
        let fields = string_schema_fields();
        let predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: DataType::VarChar(crate::spec::VarCharType::string_type()),
            op: PredicateOperator::Lt,
            literals: vec![Datum::String("delta".to_string())],
        }];

        let mut btree = make_global_index_entry_with_type(
            BTREE_GLOBAL_INDEX_TYPE,
            &file_name,
            1,
            0,
            99,
            &btree_meta,
        );
        btree.index_file.file_size = 2;
        let mut bitmap = make_global_index_entry_with_type(
            BITMAP_GLOBAL_INDEX_TYPE,
            "bitmap-no-match.index",
            1,
            100,
            199,
            &bitmap_meta,
        );
        bitmap.index_file.file_size = 1;

        let result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &[btree, bitmap],
            &predicates,
            &fields,
            1,
            i64::MAX,
        )
        .await
        .unwrap();
        assert!(
            result.is_none(),
            "an over-budget selected BTree file must stay unsupported even if bitmap files are pruned by metadata"
        );
    }

    #[tokio::test]
    async fn test_fallback_preflight_happens_before_shard_io() {
        let file_io = crate::io::FileIOBuilder::new("memory").build().unwrap();
        let table_path = "memory:/missing-index-files";
        let meta = BTreeIndexMeta::new(Some(b"a".to_vec()), Some(b"z".to_vec()), false);
        let fields = string_schema_fields();
        let predicates = vec![Predicate::Leaf {
            column: "name".to_string(),
            index: 0,
            data_type: DataType::VarChar(crate::spec::VarCharType::string_type()),
            op: PredicateOperator::Contains,
            literals: vec![Datum::String("middle".to_string())],
        }];

        let mut btree = make_global_index_entry_with_type(
            BTREE_GLOBAL_INDEX_TYPE,
            "missing-btree.index",
            1,
            0,
            99,
            &meta,
        );
        btree.index_file.file_size = 1;
        let mut bitmap = make_global_index_entry_with_type(
            BITMAP_GLOBAL_INDEX_TYPE,
            "missing-bitmap.index",
            1,
            100,
            199,
            &meta,
        );
        bitmap.index_file.file_size = 1;

        let result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            table_path,
            &[btree, bitmap],
            &predicates,
            &fields,
            1,
            0,
        )
        .await
        .expect("fallback must be decided before opening an earlier shard");

        assert!(result.is_none());
    }

    #[tokio::test]
    async fn test_evaluate_global_index_full_mode_includes_unindexed_tail() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();
        let predicates = vec![int_eq("id", 0, 50)];

        let result = super::evaluate_global_index(super::GlobalIndexEvaluation {
            file_io: &file_io,
            table_path: &table_path,
            index_entries: &entries,
            predicates: &predicates,
            schema_fields: &fields,
            search_mode: GlobalIndexSearchMode::Full,
            global_index_thread_num: 32,
            btree_fallback_scan_max_size: i64::MAX,
            bitmap_fallback_scan_max_size: i64::MAX,
            next_row_id: Some(150),
            data_ranges: &[],
        })
        .await
        .unwrap();

        assert_eq!(
            result.unwrap(),
            vec![RowRange::new(25, 25), RowRange::new(100, 149)]
        );
    }

    #[tokio::test]
    async fn test_evaluate_global_index_and_uses_evaluated_field_coverage_for_raw_fallback() {
        let src = format!(
            "{}/testdata/btree/btree_int_100_no_compress.bin",
            env!("CARGO_MANIFEST_DIR")
        );
        let tmp = tempfile::tempdir().unwrap();
        let index_dir = tmp.path().join("index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::copy(&src, index_dir.join("index_part1.bin")).unwrap();
        std::fs::copy(&src, index_dir.join("index_part2.bin")).unwrap();

        let table_path = format!("file://{}", tmp.path().display());
        let file_io = crate::io::FileIOBuilder::new("file").build().unwrap();
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);

        let mut first = make_global_index_entry("index_part1.bin", 1, 0, 49, &meta);
        first
            .index_file
            .global_index_meta
            .as_mut()
            .unwrap()
            .extra_field_ids = Some(vec![2]);
        let second = make_global_index_entry("index_part2.bin", 1, 50, 99, &meta);
        let entries = vec![first, second];
        let fields = two_field_schema_fields();

        let predicates = vec![Predicate::and(vec![
            int_eq("id", 0, 50),
            int_eq("value", 1, 8),
        ])];
        let result = super::evaluate_global_index(super::GlobalIndexEvaluation {
            file_io: &file_io,
            table_path: &table_path,
            index_entries: &entries,
            predicates: &predicates,
            schema_fields: &fields,
            search_mode: GlobalIndexSearchMode::Full,
            global_index_thread_num: 32,
            btree_fallback_scan_max_size: i64::MAX,
            bitmap_fallback_scan_max_size: i64::MAX,
            next_row_id: Some(100),
            data_ranges: &[],
        })
        .await
        .unwrap();

        assert_eq!(
            result.unwrap(),
            vec![RowRange::new(25, 25), RowRange::new(75, 75)],
            "raw fallback should use only the id field that was actually evaluated; \
             the unevaluated extra field must not widen or narrow fallback coverage"
        );
    }

    #[tokio::test]
    async fn test_evaluate_global_index_detail_mode_uses_data_ranges() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();
        let predicates = vec![int_eq("id", 0, 50)];

        let data_ranges = [RowRange::new(90, 120), RowRange::new(140, 145)];
        let result = super::evaluate_global_index(super::GlobalIndexEvaluation {
            file_io: &file_io,
            table_path: &table_path,
            index_entries: &entries,
            predicates: &predicates,
            schema_fields: &fields,
            search_mode: GlobalIndexSearchMode::Detail,
            global_index_thread_num: 32,
            btree_fallback_scan_max_size: i64::MAX,
            bitmap_fallback_scan_max_size: i64::MAX,
            next_row_id: Some(150),
            data_ranges: &data_ranges,
        })
        .await
        .unwrap();

        assert_eq!(
            result.unwrap(),
            vec![
                RowRange::new(25, 25),
                RowRange::new(100, 120),
                RowRange::new(140, 145),
            ]
        );
    }

    #[tokio::test]
    async fn test_evaluate_global_index_range() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();

        // keys 10..=20 -> keys 10,12,14,16,18,20 -> row_ids 5,6,7,8,9,10
        let predicates = vec![
            Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::GtEq,
                literals: vec![Datum::Int(10)],
            },
            Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::LtEq,
                literals: vec![Datum::Int(20)],
            },
        ];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        let ranges = result.unwrap();
        assert_eq!(ranges, vec![RowRange::new(5, 10)]);

        let mut over_limit_entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        over_limit_entries[0].index_file.file_size = 2;
        let over_limit_result = evaluate_global_index_fast_with_fallback_size(
            &file_io,
            &table_path,
            &over_limit_entries,
            &predicates,
            &fields,
            1,
            i64::MAX,
        )
        .await
        .unwrap();
        assert!(
            over_limit_result.is_none(),
            "between/range predicates require fallback scans and should be unsupported over budget"
        );
    }

    #[tokio::test]
    async fn test_evaluate_global_index_in() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();

        // IN(0, 50, 198) -> row_ids 0, 25, 99
        let predicates = vec![Predicate::Leaf {
            column: "id".to_string(),
            index: 0,
            data_type: DataType::Int(crate::spec::IntType::new()),
            op: PredicateOperator::In,
            literals: vec![Datum::Int(0), Datum::Int(50), Datum::Int(198)],
        }];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        let ranges = result.unwrap();
        assert_eq!(
            ranges,
            vec![
                RowRange::new(0, 0),
                RowRange::new(25, 25),
                RowRange::new(99, 99)
            ]
        );
    }

    #[tokio::test]
    async fn test_evaluate_global_index_no_match() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();

        // key=999 doesn't exist
        let predicates = vec![Predicate::Leaf {
            column: "id".to_string(),
            index: 0,
            data_type: DataType::Int(crate::spec::IntType::new()),
            op: PredicateOperator::Eq,
            literals: vec![Datum::Int(999)],
        }];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        let ranges = result.unwrap();
        assert!(ranges.is_empty());
    }

    #[tokio::test]
    async fn test_evaluate_global_index_with_row_offset() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        // row_range_start=1000 simulates an offset
        let entries = vec![make_global_index_entry(&file_name, 1, 1000, 1099, &meta)];
        let fields = int_schema_fields();

        // key=50 -> local row_id=25, offset -> global row_id=1025
        let predicates = vec![Predicate::Leaf {
            column: "id".to_string(),
            index: 0,
            data_type: DataType::Int(crate::spec::IntType::new()),
            op: PredicateOperator::Eq,
            literals: vec![Datum::Int(50)],
        }];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        let ranges = result.unwrap();
        assert_eq!(ranges, vec![RowRange::new(1025, 1025)]);
    }

    #[tokio::test]
    async fn test_evaluate_global_index_unknown_column() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();

        // Column "unknown" not in schema -> None (can't evaluate)
        let predicates = vec![Predicate::Leaf {
            column: "unknown".to_string(),
            index: 0,
            data_type: DataType::Int(crate::spec::IntType::new()),
            op: PredicateOperator::Eq,
            literals: vec![Datum::Int(50)],
        }];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        assert!(result.is_none());
    }

    #[tokio::test]
    async fn test_evaluate_global_index_multi_field_and() {
        // Two fields, each with its own btree index file (same data, different field_id).
        // btree_int_100_no_compress.bin: keys 0,2,4,...,198 -> row_ids 0,1,...,99
        let src = format!(
            "{}/testdata/btree/btree_int_100_no_compress.bin",
            env!("CARGO_MANIFEST_DIR")
        );
        let tmp = tempfile::tempdir().unwrap();
        let index_dir = tmp.path().join("index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::copy(&src, index_dir.join("index_field1.bin")).unwrap();
        std::fs::copy(&src, index_dir.join("index_field2.bin")).unwrap();

        let table_path = format!("file://{}", tmp.path().display());
        let file_io = crate::io::FileIOBuilder::new("file").build().unwrap();

        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);

        let fields = vec![
            DataField::new(
                1,
                "id".to_string(),
                DataType::Int(crate::spec::IntType::new()),
            ),
            DataField::new(
                2,
                "value".to_string(),
                DataType::Int(crate::spec::IntType::new()),
            ),
        ];

        let entries = vec![
            make_global_index_entry("index_field1.bin", 1, 0, 99, &meta),
            make_global_index_entry("index_field2.bin", 2, 0, 99, &meta),
        ];

        // id >= 40 AND id <= 60 → keys 40,42,...,60 → row_ids 20..30
        // value >= 44 AND value <= 52 → keys 44,46,48,50,52 → row_ids 22..26
        // AND → intersect [20..30] and [22..26] = [22..26]
        let predicates = vec![
            Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::GtEq,
                literals: vec![Datum::Int(40)],
            },
            Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::LtEq,
                literals: vec![Datum::Int(60)],
            },
            Predicate::Leaf {
                column: "value".to_string(),
                index: 1,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::GtEq,
                literals: vec![Datum::Int(44)],
            },
            Predicate::Leaf {
                column: "value".to_string(),
                index: 1,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::LtEq,
                literals: vec![Datum::Int(52)],
            },
        ];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        let ranges = result.unwrap();
        assert_eq!(ranges, vec![RowRange::new(22, 26)]);
    }

    /// Regression for the Between+remaining bug in `evaluate_leaf`. When a
    /// native `Between` leaf is paired with another conjunct (e.g. `id >= 0`),
    /// and the file's b-tree key range falls **outside** the Between range
    /// but is still matched by the remaining predicate, the whole AND must
    /// produce zero rows. Before the fix, `file_result` was initialized from
    /// the remaining predicate's bitmap and the Between conjunct was silently
    /// dropped — the test would observe the file's full row id set instead of
    /// the empty set.
    #[tokio::test]
    async fn test_between_unmatched_file_drops_remaining_match() {
        let (file_io, table_path, file_name, _tmp) =
            setup_testdata_table("btree_int_100_no_compress.bin");
        // File covers keys [0, 198] (row_ids 0..99). Pick a Between range
        // entirely below 0 so `may_match_between` is false, and a `>= 0`
        // conjunct that would otherwise scoop up every row in the file.
        let meta = BTreeIndexMeta::new(Some(le_int_key(0)), Some(le_int_key(198)), false);
        let entries = vec![make_global_index_entry(&file_name, 1, 0, 99, &meta)];
        let fields = int_schema_fields();

        let predicates = vec![Predicate::and(vec![
            Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::Between,
                literals: vec![Datum::Int(-100), Datum::Int(-50)],
            },
            Predicate::Leaf {
                column: "id".to_string(),
                index: 0,
                data_type: DataType::Int(crate::spec::IntType::new()),
                op: PredicateOperator::GtEq,
                literals: vec![Datum::Int(0)],
            },
        ])];

        let result =
            evaluate_global_index_fast(&file_io, &table_path, &entries, &predicates, &fields)
                .await
                .unwrap();
        let ranges = result.unwrap();
        assert!(
            ranges.is_empty(),
            "Between(-100..-50) AND id>=0 must produce zero rows on a file \
             whose key range is [0, 198] — got {ranges:?}"
        );
    }
}
