% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/delta_lsi.R
\name{delta_lsi}
\alias{delta_lsi}
\title{Delta Leakage Sensitivity Index (Delta LSI)}
\usage{
delta_lsi(
  fit_leaky,
  fit_guarded,
  metric = "auc",
  exchangeability = c("iid", "by_group", "within_batch", "blocked_time"),
  learner = NULL,
  higher_is_better = NULL,
  block_size = NULL,
  M_boot = 2000L,
  M_flip = 10000L,
  strict = FALSE,
  return_details = FALSE,
  seed = 42L,
  ...
)
}
\arguments{
\item{fit_leaky}{A \code{\linkS4class{LeakFit}} object from the leaky
(unprotected) evaluation pipeline.}

\item{fit_guarded}{A \code{\linkS4class{LeakFit}} object from the guarded
(leakage-protected) evaluation pipeline.}

\item{metric}{Character. Performance metric to compare. Must appear in
\code{fit@metrics} of both fits (e.g., \code{"auc"}, \code{"rmse"}).}

\item{exchangeability}{Character. Exchangeability assumption for the
sign-flip test. One of \code{"iid"} (default), \code{"by_group"},
\code{"within_batch"}, \code{"blocked_time"}.
\code{"blocked_time"} activates a block sign-flip procedure that flips
contiguous groups of repeats together, preserving serial autocorrelation
under the null; see \code{block_size}.  \code{"by_group"} and
\code{"within_batch"} are stored and reported but inference still uses the
iid sign-flip procedure (a warning is issued; contributions welcome).
\code{"iid"} (default) applies the standard independent sign-flip test.}

\item{learner}{Optional character. Learner name to select from multi-learner
fits. If \code{NULL}, the first learner found in \code{fit@metrics} is used.}

\item{higher_is_better}{Logical or \code{NULL}. Whether a higher value of
\code{metric} indicates better performance. When \code{NULL} (default),
auto-detected from the metric name: \code{"rmse"}, \code{"mse"},
\code{"mae"}, \code{"log_loss"}, \code{"brier"}, \code{"error"},
\code{"loss"}, and \code{"deviance"} are treated as lower-is-better;
all others default to higher-is-better. Setting this correctly ensures
that a positive \code{delta_lsi} always indicates leakage inflation
(the naive pipeline is artificially more optimistic than the guarded one).}

\item{block_size}{Integer or \code{NULL}. Block length for the block
sign-flip test, used only when \code{exchangeability = "blocked_time"}.
When \code{NULL} (default), the block size is auto-estimated from the
first-order autocorrelation of \eqn{\{\Delta_r\}} and capped at
\code{floor(R/3)} to ensure at least three independent blocks.  A warning
is issued when the estimate is used with \code{R_eff < 20} because the
AR(1) estimate is noisy at small sample sizes.  Provide an explicit integer
to override auto-estimation.}

\item{M_boot}{Integer. Number of bootstrap samples for BCa CI (default 2000).}

\item{M_flip}{Integer. Maximum Monte Carlo samples for sign-flip test when
R_eff > 15 (default 10000).}

\item{strict}{Logical. If \code{TRUE}, error on insufficient R_eff instead
of a warning.}

\item{return_details}{Logical. If \code{TRUE}, include the per-repeat
\eqn{\Delta_r} vector and the original fit objects in the \code{info} slot.}

\item{seed}{Integer. Random seed for bootstrap and sign-flip test.}

\item{...}{Unused. Reserved for deprecated aliases such as
\code{fit_naive}.}
}
\value{
A \code{\linkS4class{LeakDeltaLSI}} object.
}
\description{
Compares a naive (potentially leaky) cross-validation pipeline against a
guarded (leakage-protected) pipeline and quantifies leakage-induced
performance inflation using the Leakage Sensitivity Index (LSI).
}
\details{
\subsection{Method}{
For each fit, per-fold metric values are extracted from \code{fit@metrics}
(or recomputed from \code{fit@predictions} if necessary). Fold test-set
sizes are used as weights to aggregate fold metrics into per-repeat
estimates \eqn{\mu_r}. The repeat-level delta
\eqn{\Delta_r = s \cdot (\mu_r^{\text{naive}} - \mu_r^{\text{guarded}})}
captures leakage-induced performance inflation for each CV repeat, where
\eqn{s = +1} for higher-is-better metrics (e.g., AUC) and \eqn{s = -1}
for lower-is-better metrics (e.g., RMSE), so that \eqn{\Delta_r > 0}
always indicates the naive pipeline is more optimistic than the guarded one.

The \strong{delta_lsi} point estimate is the Huber M-estimator (k = 1.345)
applied to \eqn{\{\Delta_r\}}, which is robust to occasional outlier
repeats. \strong{delta_metric} is the arithmetic mean of \eqn{\{\Delta_r\}}
for easy interpretation in the original metric's units.

Pairing requires that \code{fit_leaky} and \code{fit_guarded} share
\emph{identical fold structures} (same test-set membership per fold) in
addition to the same number of repeats.  When repeat counts match but fold
structures differ, a warning is issued and the fits are treated as unpaired.

When \eqn{R_{\text{eff}} \geq 5} (equal, paired repeats), a sign-flip
randomization test (Phipson & Smyth, 2010) is performed: under
\eqn{H_0} (no leakage) the sign of each \eqn{\Delta_r} is exchangeable.
All \eqn{2^R} sign combinations are enumerated exactly for
\eqn{R \leq 15} (no continuity correction); Monte Carlo sampling is used
for larger \eqn{R} with the Phipson & Smyth (2010) correction.

BCa bootstrap confidence intervals (Efron, 1987) require
\eqn{R_{\text{eff}} \geq 10}.
}

\subsection{Inference tiers}{
\describe{
  \item{\code{"A_full_inference"}}{R_eff >= 20: point + BCa CI + sign-flip p-value; \code{inference_ok = TRUE}}
  \item{\code{"B_signflip_ci"}}{10 <= R_eff < 20: point + sign-flip p-value + BCa CI}
  \item{\code{"C_signflip"}}{5 <= R_eff < 10: point + sign-flip p-value (no CI)}
  \item{\code{"D_insufficient"}}{R_eff < 5 or unpaired: point estimate only}
}}
}
\seealso{
\code{\link{audit_leakage}}, \code{\link{fit_resample}}
}
