% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/tune_Lambda.R
\name{tune_Lambda}
\alias{tune_Lambda}
\title{Tune Smoothing and Ridge Penalties via Leave-One-Out or Generalized Cross-Validation}
\usage{
tune_Lambda(
  y,
  X,
  X_gram,
  smoothing_spline_penalty,
  A,
  K,
  p_expansions,
  N_obs,
  opt,
  use_custom_bfgs,
  C,
  colnm_expansions,
  wiggle_penalty,
  flat_ridge_penalty,
  initial_wiggle,
  initial_flat,
  unique_penalty_per_predictor,
  unique_penalty_per_partition,
  penalty_vec,
  meta_penalty,
  family,
  unconstrained_fit_fxn,
  keep_weighted_Lambda,
  iterate,
  qp_score_function,
  quadprog,
  qp_Amat,
  qp_bvec,
  qp_meq,
  tol,
  sd_y,
  delta,
  tuning_criterion,
  gcv_gamma,
  constraint_value_vectors,
  parallel,
  parallel_eigen,
  parallel_trace,
  parallel_aga,
  parallel_matmult,
  parallel_unconstrained,
  cl,
  chunk_size,
  num_chunks,
  rem_chunks,
  shared_env,
  custom_penalty_mat,
  order_list,
  glm_weight_function,
  schur_correction_function,
  need_dispersion_for_estimation,
  dispersion_function,
  observation_weights,
  homogenous_weights,
  blockfit,
  just_linear_without_interactions,
  Vhalf,
  VhalfInv,
  verbose,
  include_warnings,
  ...
)
}
\arguments{
\item{y}{List; response vectors by partition.}

\item{X}{List; design matrices by partition.}

\item{X_gram}{List; Gram matrices by partition.}

\item{smoothing_spline_penalty}{Matrix; integrated squared second derivative
penalty.}

\item{A}{Matrix; smoothness constraints at knots.}

\item{K}{Integer; number of interior knots in 1-D, number of partitions - 1
in higher dimensions.}

\item{p_expansions}{Integer; columns per partition.}

\item{N_obs}{Integer; total sample size.}

\item{opt}{Logical; TRUE to optimize penalties, FALSE to use initial values.}

\item{use_custom_bfgs}{Logical; TRUE for analytic gradient BFGS as natively
implemented, FALSE for finite differences as implemented by
\code{stats::optim()}. The analytic path is typically faster; however, for
exact LOO tuning the leverage derivative can be numerically delicate in
some problems, in which case \code{use_custom_bfgs = FALSE} remains a
conservative fallback.}

\item{C}{Matrix; polynomial expansion matrix (used for initialization).
This is the monomial expansion design used to derive starting values and
is not the inequality-constraint matrix sometimes denoted by
\eqn{\mathbf{C}} elsewhere in the package documentation.}

\item{colnm_expansions}{Character vector; column names of the expansion
matrix.}

\item{wiggle_penalty, flat_ridge_penalty}{Fixed penalty values if provided.}

\item{initial_wiggle, initial_flat}{Numeric vectors; candidate values for grid
search initialization on the raw (non-negative) scale. Converted to log
scale internally for optimization.}

\item{unique_penalty_per_predictor, unique_penalty_per_partition}{Logical;
allow predictor/partition-specific penalties.}

\item{penalty_vec}{Numeric vector; initial values for predictor/partition
penalties on the raw (non-negative) scale. Converted to log scale
internally for optimization. Use \code{c()} when no per-predictor or
per-partition penalties are needed.}

\item{meta_penalty}{The "meta" ridge penalty, a regularization for
predictor/partition penalties to pull them towards 1 on the raw scale.}

\item{family}{GLM family with optional custom tuning loss.}

\item{unconstrained_fit_fxn}{Function for unconstrained fitting.}

\item{keep_weighted_Lambda, iterate}{Logical controlling GLM fitting.}

\item{qp_score_function, quadprog, qp_Amat, qp_bvec, qp_meq}{Quadratic
programming parameters (see arguments of \code{\link[lgspline]{lgspline}}).}

\item{tol}{Numeric; convergence tolerance.}

\item{sd_y, delta}{Response standardization parameters.}

\item{tuning_criterion}{Character scalar selecting the tuning objective:
\code{"loo"} for exact leave-one-out on the transformed tuning problem or
\code{"gcv"} for generalized cross-validation. The LOO criterion itself is
still evaluated exactly on that transformed problem; the caveat is that
its fully analytic leverage derivative can be numerically sensitive in some
datasets. For very large samples, \code{"gcv"} is often the more practical
choice; as a rough guideline, it is recommended once the sample size is
above about 250,000.}

\item{gcv_gamma}{Numeric scalar, at least 1, used only when
\code{tuning_criterion = "gcv"}. Multiplies the effective degrees of
freedom in the GCV denominator.}

\item{constraint_value_vectors}{List; constraint values.}

\item{parallel}{Logical; enable parallel computation.}

\item{parallel_eigen, parallel_trace, parallel_aga}{Logical; specific parallel
flags.}

\item{parallel_matmult, parallel_unconstrained}{Logical; specific parallel
flags.}

\item{cl}{Parallel cluster object.}

\item{chunk_size, num_chunks, rem_chunks}{Integer; parallel computation
parameters.}

\item{shared_env}{Environment; shared variables exported to cluster workers.}

\item{custom_penalty_mat}{Optional custom penalty matrix.}

\item{order_list}{List; observation ordering by partition.}

\item{glm_weight_function, schur_correction_function}{Functions for GLM
weights and corrections.}

\item{need_dispersion_for_estimation, dispersion_function}{Control dispersion
estimation.}

\item{observation_weights}{Optional observation weights.}

\item{homogenous_weights}{Logical; TRUE if all weights equal.}

\item{blockfit}{Logical; when TRUE, the backfitting block decomposition
(\code{blockfit_solve}) is used in place of \code{get_B} for the
coefficient estimation step at each tuning-objective evaluation, provided
\code{just_linear_without_interactions} is non-empty and \code{K > 0}.
The dispatch condition is identical to that in \code{lgspline.fit} so
that penalties are tuned under the same estimator that will be used for
the final fit. Falls back to \code{get_B} automatically on failure.}

\item{just_linear_without_interactions}{Numeric; vector of columns for
non-spline effects without interactions.}

\item{Vhalf, VhalfInv}{Square root and inverse square root correlation
structure matrices. These are passed through to the coefficient-estimation
step used inside each tuning-objective evaluation, so any dense or
Woodbury-accelerated
correlated solve is the same one used for the final fitted model.}

\item{verbose}{Logical; print progress.}

\item{include_warnings}{Logical; print warnings/try-errors.}

\item{...}{Additional arguments passed to fitting functions.}
}
\value{
List containing:
\describe{
  \item{Lambda}{Final combined penalty matrix.}
  \item{flat_ridge_penalty}{Optimized ridge penalty.}
  \item{wiggle_penalty}{Optimized smoothing penalty.}
  \item{other_penalties}{Optimized predictor/partition penalties.}
  \item{L_predictor_list}{Predictor-specific penalty matrices.}
  \item{L_partition_list}{Partition-specific penalty matrices.}
}
}
\description{
Optimizes smoothing spline and ridge regression penalties by minimizing the
selected tuning criterion, either exact leave-one-out on the transformed
tuning problem or modified GCV. Uses BFGS optimization with analytical
gradients or finite differences. This is the top-level entry point that
orchestrates grid search initialization and quasi-Newton optimization via
internal subfunctions.
}
\details{
The tuning procedure consists of the following steps:
\enumerate{
  \item \strong{Preprocessing}: Convert raw-scale penalties to log scale,
    compute cross-products, determine pseudocount delta, ensure constraint
    matrix compatibility.
  \item \strong{Blockfit dispatch}: Pre-compute \code{flat_cols} and the
    \code{use_blockfit} flag so that every tuning-objective evaluation uses the same
    coefficient estimator as the final fit. \code{flat_cols} are identified
    by matching column names against
    \code{paste0("_", just_linear_without_interactions, "_")} in
    \code{colnm_expansions}.
  \item \strong{Grid search}: Evaluate the selected tuning criterion over a grid of
    (wiggle, ridge) penalty candidates to find a good starting point
    (see \code{.tune_grid_search}).
  \item \strong{BFGS optimization}: Minimize the selected tuning criterion via either the custom
    damped BFGS with closed-form gradients (see \code{.damped_bfgs},
    \code{.compute_gcvu_gradient}, \code{.compute_loocv_gradient}) or base R's \code{stats::optim} with
    finite-difference gradients. The GCV gradient is differentiated directly
    in the shared wiggle and flat directions; for LOO, the same direct
    differentiation is used, but the leverage derivative may be numerically
    delicate in some problems and can motivate the finite-difference
    fallback.
  \item \strong{Criterion choice}: \code{"loo"} uses exact per-observation
    leverages on the transformed tuning problem, while \code{"gcv"} uses the
    existing trace-based denominator with optional \code{gcv_gamma}
    inflation.
  \item \strong{Sample-size adjustment}: After optimization, divide the tuned
    penalties by \eqn{((N+2)/(N-2))^{2}} (equivalently multiply by
    \eqn{((N-2)/(N+2))^{2}}) for both GCV- and LOO-based tuning so that the
    final penalties are decreased at small sample sizes.
  \item \strong{Final Lambda}: Compute the final penalty matrix from
    optimized parameters via \code{compute_Lambda}.
}

Parameterization: initial penalty values are accepted on the raw
(non-negative) scale and converted to natural log-scale internally,
i.e. raw_penalty = exp(theta), so that raw penalties are always positive.
The chain rule factor d(exp(theta))/d(theta) = exp(theta) = raw_penalty.

For predictor- and partition-specific penalties, the current implementation
still uses the existing low-cost ratio approximation rather than a full
direct derivative in each extra penalty direction. Thus the most accurate
closed-form tuning gradients are presently the shared wiggle and flat
directions.

The resulting penalty matrix follows the same decomposition used throughout
the paper and package documentation:
\deqn{
  \boldsymbol{\Lambda}
  =
  \lambda_{w}\mathbf{L}_{1}
  +
  \lambda_{r}\mathbf{L}_{2}
  +
  \sum_{j}\nu_{j}\mathbf{L}^{(\mathrm{pred})}_{j}
  +
  \sum_{k}\tau_{k}\mathbf{L}^{(\mathrm{part})}_{k},
}
where the predictor- and partition-specific sums are included only when the
corresponding options are active. Internally these components are returned as
\code{L1}, \code{L2}, \code{L_predictor_list}, and \code{L_partition_list}.

If correlation structure inputs are supplied, each tuning-objective evaluation calls the
same constrained correlated solver used by the final model fit. In the
structured-correlation case, the low-rank Woodbury correction described in
\code{lgspline-details} is therefore inherited automatically through
\code{get_B}; no separate tuning-specific notation is introduced here.
}
\examples{
\dontrun{
## ## Example 1: Basic usage within lgspline ## ## ## ## ## ## ## ## ## ## ##
## tune_Lambda is called internally by lgspline; direct calls are
#  for advanced users. Here we verify that the refactored version
#  produces identical output to the original.

set.seed(42)
t <- runif(200, -5, 5)
y <- sin(t) + rnorm(200, 0, 0.5)

## Fit with automatic penalty tuning (calls tune_Lambda internally)
fit1 <- lgspline(t, y, K = 3)
cat("Wiggle penalty:", fit1$penalties$wiggle_penalty, "\n")
cat("Ridge penalty:", fit1$penalties$flat_ridge_penalty, "\n")
cat("Trace (edf):", fit1$trace_XUGX, "\n")

## ## Example 2: Fixed penalties (no tuning) ## ## ## ## ## ## ## ## ## ## ##
fit2 <- lgspline(t, y, K = 3, opt = FALSE,
                 wiggle_penalty = 1e-4,
                 flat_ridge_penalty = 0.1)
cat("Fixed wiggle:", fit2$penalties$wiggle_penalty, "\n")

## ## Example 3: blockfit path #### ## ## ## ## ## ## ## ## ## ## ## ## ## ##
## When blockfit = TRUE and just_linear_without_interactions is non-empty,
#  tune_Lambda dispatches to blockfit_solve at each GCV evaluation,
#  ensuring penalties are tuned under the same estimator used in the final
#  fit. Verify that tuned penalties are consistent across both paths.

set.seed(7)
n  <- 300
x1 <- runif(n, 0, 5)
x2 <- rnorm(n)
y2 <- sin(x1) + 0.5 * x2 + rnorm(n, 0, 0.3)
df <- data.frame(x1 = x1, x2 = x2)

## blockfit = TRUE uses blockfit_solve during tuning
fit_bf  <- lgspline(df, y2, K = 2, blockfit = TRUE,
                    just_linear_without_interactions = 2)
## blockfit = FALSE uses get_B during tuning (original path)
fit_std <- lgspline(df, y2, K = 2, blockfit = FALSE,
                    just_linear_without_interactions = 2)

cat("blockfit wiggle  :", fit_bf$penalties$wiggle_penalty, "\n")
cat("standard wiggle  :", fit_std$penalties$wiggle_penalty, "\n")
## Penalties may differ slightly; predictions should be close.
cat("Max pred diff:", max(abs(fit_bf$ytilde - fit_std$ytilde)), "\n")

## ## Example 4: Verify refactored subfunctions ## ## ## ## ## ## ## ## ## ##
## The internal .compute_meta_penalty should match hand calculation
mp <- lgspline:::.compute_meta_penalty(
  wiggle_penalty = 0.5,
  penalty_vec = c(predictor1 = 1.2, partition1 = 0.8),
  meta_penalty_coef = 1e-8,
  unique_penalty_per_predictor = TRUE,
  unique_penalty_per_partition = TRUE
)
expected <- 0.5 * 1e-8 * ((1.2 - 1)^2 + (0.8 - 1)^2) +
            0.5 * 1e-32 * (0.5 - 1)^2
stopifnot(abs(mp - expected) < 1e-20)
cat("Meta-penalty check passed.\n")

## ## Example 5: Verify gradient of meta-penalty ## ## ## ## ## ## ## ## ## #
gr <- lgspline:::.compute_meta_penalty_gradient(
  wiggle_penalty = 2.0,
  penalty_vec = c(predictor1 = 1.5),
  meta_penalty_coef = 1e-8,
  unique_penalty_per_predictor = TRUE,
  unique_penalty_per_partition = FALSE
)
# gr[1] should be 1e-32 * (2 - 1) * 2 = 2e-32
# gr[2] should be 0
# gr[3] should be 1e-8 * (1.5 - 1) * 1.5 = 7.5e-9
stopifnot(abs(gr[1] - 2e-32) < 1e-40)
stopifnot(gr[2] == 0)
stopifnot(abs(gr[3] - 7.5e-9) < 1e-17)
cat("Meta-penalty gradient check passed.\n")

## ## Example 6: Residual computation paths ## ##
## Identity link ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ##
r1 <- lgspline:::.compute_tuning_residuals(
  y = list(c(1, 2, 3)),
  preds = list(c(1.1, 1.9, 3.2)),
  delta = 0,
  family = gaussian(),
  observation_weights = list(NULL),
  K = 0,
  order_list = list(1:3)
)
stopifnot(max(abs(r1[[1]] - c(-0.1, 0.1, -0.2))) < 1e-10)
cat("Residual check passed.\n")
}

}
\seealso{
\itemize{
  \item \code{\link[stats]{optim}} for Hessian-free optimization
  \item \code{\link{compute_Lambda}} for penalty matrix construction
  \item \code{\link{compute_G_eigen}} for eigendecomposition of penalized
    Gram matrices
  \item \code{\link{get_B}} for constrained coefficient estimation
  \item \code{\link{blockfit_solve}} for the backfitting block-decomposition
    estimator used when \code{blockfit = TRUE}
}
}
\keyword{internal}
