% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/ao.R
\name{ao}
\alias{ao}
\title{Alternating Optimization}
\usage{
ao(
  f,
  initial,
  target = NULL,
  npar = NULL,
  gradient = NULL,
  hessian = NULL,
  ...,
  partition = "sequential",
  new_block_probability = 0.3,
  minimum_block_number = 1,
  minimize = TRUE,
  lower = NULL,
  upper = NULL,
  iteration_limit = Inf,
  seconds_limit = Inf,
  tolerance_value = 1e-06,
  tolerance_parameter = 1e-06,
  tolerance_parameter_norm = function(x, y) sqrt(sum((x - y)^2)),
  tolerance_history = 1,
  base_optimizer = optimizeR::Optimizer$new("stats::optim", method = "L-BFGS-B", control
    = list(maxit = 10)),
  verbose = FALSE,
  hide_warnings = TRUE,
  add_details = TRUE
)
}
\arguments{
\item{f}{[\code{function}]\cr
A \code{function} to be optimized, returning a single \code{numeric} value.

The first argument of \code{f} should be a \code{numeric} of the same length
as \code{initial}, optionally followed by any other arguments specified by
the \code{...} argument.

If \code{f} is optimized over an argument other than the first, or over more
than one argument, specify this with the \code{target} argument.}

\item{initial}{[\code{numeric()} | \code{list()}]\cr
The starting parameter values for the target argument(s).

This can also be a \code{list} of multiple starting parameter values, see details.}

\item{target}{[\code{character()} | \code{NULL}]\cr
The name(s) of the argument(s) over which \code{f} is optimized.

This can only be \code{numeric} arguments.

If \code{NULL} (default), the first argument of \code{f} is optimized.}

\item{npar}{[\code{integer()}]\cr
The length(s) of the target argument(s).

Must be specified if more than one target argument is specified via
the \code{target} argument.

Can be \code{NULL} if there is only one target argument, in which case \code{npar} is
set to \code{length(initial)}.}

\item{gradient}{[\code{function} | \code{NULL}]\cr
An optional \code{function} that returns the gradient of \code{f}.

The function signature of \code{gradient} must be identical to \code{f}.

Ignored if \code{base_optimizer} does not support custom gradient.}

\item{hessian}{[\code{function} | \code{NULL}]\cr
An optional \code{function} that returns the Hessian of \code{f}.

The function signature of \code{hessian} must be identical to \code{f}.

Ignored if \code{base_optimizer} does not support custom Hessian.}

\item{...}{Additional arguments to be passed to \code{f} (and \code{gradient}).}

\item{partition}{[\code{character(1)} | \code{list()}]\cr
Defines the parameter partition. It can be
\itemize{
\item \code{"sequential"} for treating each parameter separately,
\item \code{"random"} for a random partition in each iteration,
\item \code{"none"} for no partition (which is equivalent to joint optimization),
\item or a \code{list} of vectors of parameter indices, specifying a custom
partition for the AO process.
}

This can also be a \code{list} of multiple partition definitions, see details.}

\item{new_block_probability}{[\code{numeric(1)}]\cr
Only relevant if \code{partition = "random"}.

The probability of creating a new parameter block when creating a random
partition.

Values close to 0 result in larger parameter blocks, values close to 1
result in smaller parameter blocks.}

\item{minimum_block_number}{[\code{integer(1)}]\cr
Only relevant if \code{partition = "random"}.

The minimum number of blocks in random partitions.}

\item{minimize}{[\code{logical(1)}]\cr
Minimize during the AO process?

If \code{FALSE}, maximization is performed.}

\item{lower, upper}{[\code{numeric()} | \code{NULL}]\cr
Optional lower and upper parameter bounds.

Ignored if \code{base_optimizer} does not support parameter bounds.}

\item{iteration_limit}{[\code{integer(1)} | \code{Inf}]\cr
The maximum number of iterations through the parameter partition before
the AO process is terminated.

Can also be \code{Inf} for no iteration limit.}

\item{seconds_limit}{[\code{numeric(1)}]\cr
The time limit in seconds before the AO process is terminated.

Can also be \code{Inf} for no time limit.

Note that this stopping criterion is only checked \emph{after} a sub-problem is
solved and not \emph{within} solving a sub-problem, so the actual process
time can exceed this limit.}

\item{tolerance_value}{[\code{numeric(1)}]\cr
A non-negative tolerance value. The AO process terminates
if the absolute difference between the current function value and the value
from \code{tolerance_history} iterations earlier is smaller than
\code{tolerance_value}.

Can be \code{0} for no value threshold.}

\item{tolerance_parameter}{[\code{numeric(1)}]\cr
A non-negative tolerance value. The AO process terminates if
the distance between the current estimate and the estimate from
\code{tolerance_history} iterations earlier is smaller than
\code{tolerance_parameter}.

Can be \code{0} for no parameter threshold.

By default, the distance is measured using the Euclidean norm, but another
norm can be specified via the \code{tolerance_parameter_norm} argument.}

\item{tolerance_parameter_norm}{[\code{function}]\cr
The norm that measures the distance between two estimates. If the distance
is smaller than
\code{tolerance_parameter}, the AO process is terminated.

It must be of the form \code{function(x, y)} for two vector inputs
\code{x} and \code{y}, and return a single \code{numeric} value.
By default, the Euclidean norm \code{function(x, y) sqrt(sum((x - y)^2))}
is used.}

\item{tolerance_history}{[\code{integer(1)}]\cr
The number of iterations to look back to determine whether
\code{tolerance_value} or \code{tolerance_parameter} has been reached.}

\item{base_optimizer}{[\code{Optimizer} | \code{list()}]\cr
An \code{Optimizer} object, which can be created via
\code{\link[optimizeR]{Optimizer}}. It numerically solves the sub-problems.

By default, the \code{\link[stats]{optim}} optimizer with
\code{method = "L-BFGS-B"} is used. The default value for the number of
iterations within each block is 10.

This can also be a \code{list} of multiple base optimizers, see details.}

\item{verbose}{[\code{logical(1)}]\cr
Print tracing details during the AO process?

Not supported when using multiple processes, see details.}

\item{hide_warnings}{[\code{logical(1)}]\cr
Hide warnings during the AO process?}

\item{add_details}{[\code{logical(1)}]\cr
Add details about the AO process to the output?}
}
\value{
A \code{list} with the following elements:
\itemize{
\item \code{estimate} is the parameter vector at termination.
\item \code{estimate_split} is \code{estimate} split by \code{target} (only when applicable).
\item \code{value} is the function value at termination.
\item \code{details} is a \code{data.frame} with information about the AO process:
For each iteration (column \code{iteration}) it contains the function value
(column \code{value}), parameter values (columns starting with \code{p} followed by
the parameter index), the active parameter block (columns starting with \code{b}
followed by the parameter index, where \code{1} stands for a parameter contained
in the active parameter block and \code{0} if not), and computation times in
seconds (column \code{seconds}). Only available if \code{add_details = TRUE}.
\item \code{seconds} is the overall computation time in seconds.
\item \code{stopping_reason} is a message explaining why the AO process terminated.
}

In the case of multiple processes, the output changes slightly, see details.
}
\description{
Alternating optimization (AO) is an iterative process for optimizing a
real-valued function jointly over all its parameters by alternating
restricted optimization over parameter partitions.
}
\details{
\subsection{Multiple processes}{

AO can suffer from local optima. To increase the likelihood of finding a
better optimum, you can specify:
\itemize{
\item multiple starting parameters
\item multiple parameter partitions
\item multiple base optimizers
}

Use the \code{initial}, \code{partition}, and/or \code{base_optimizer} arguments to provide
a \code{list} of possible values for each parameter. Each combination of initial
values, parameter partitions, and base optimizers will create a separate AO
process.
\subsection{Output value}{

In the case of multiple processes, the output values refer to the best
AO process with respect to function value.

If \code{add_details = TRUE}, the following elements are added:
\itemize{
\item \code{estimates} is a \code{list} of optimal parameters in each process.
\item \code{values} is a \code{list} of optimal function values in each process.
\item \code{details} combines details of the single processes and has an
additional column \code{process} with an index for the different processes.
\item \code{seconds_each} gives the computation time in seconds for each process.
\item \code{stopping_reasons} gives the termination message for each process.
\item \code{processes} gives details on how the processes were specified.
}
}

\subsection{Parallel computation}{

By default, processes run sequentially. However, since they are independent,
they can be parallelized. To enable parallel computation, use the
\href{https://future.futureverse.org/}{\code{{future}} framework}. For example, run the
following \emph{before} the \code{ao()} call:
\preformatted{
future::plan(future::multisession, workers = 4)
}
}

\subsection{Progress updates}{

When using multiple processes, setting \code{verbose = TRUE} to print tracing
details during AO is not supported. However, you can still track the progress
using the \href{https://progressr.futureverse.org/}{\code{{progressr}} framework}.
For example, run the following \emph{before} the \code{ao()} call:
\preformatted{
progressr::handlers(global = TRUE)
progressr::handlers(
  progressr::handler_progress(":percent :eta :message")
)
}
}

}
}
\examples{
# Example 1: Minimization of Himmelblau's function --------------------------

himmelblau <- function(x) (x[1]^2 + x[2] - 11)^2 + (x[1] + x[2]^2 - 7)^2
ao(f = himmelblau, initial = c(0, 0))

# Example 2: Maximization of 2-class Gaussian mixture log-likelihood --------

normal_mixture_loglik_uc = function(mu, logsd, eta, data) {
  sd <- exp(logsd)
  e <- exp(eta[1])
  den <- 1 + e
  q1 <- e / den
  q2 <- 1 / den
  l1 <- log(q1) + dnorm(data, mu[1], sd[1], log = TRUE)
  l2 <- log(q2) + dnorm(data, mu[2], sd[2], log = TRUE)
  m <- pmax(l1, l2)
  sum(m + log(exp(l1 - m) + exp(l2 - m)))
}

set.seed(123)

data <- datasets::faithful$eruptions

fit <- ao(
  f = normal_mixture_loglik_uc,
  initial = c(mean(data) + c(-1, 1), rep(log(sd(data)), 2), 0),
  target = c("mu", "logsd", "eta"),
  npar = c(2, 2, 1),
  data = data,
  partition = "random",
  base_optimizer = optimizeR::Optimizer$new("ucminf::ucminf"),
  minimize = FALSE,
  add_details = FALSE
)

(muhat <- fit$estimate_split$mu)
(sdhat <- exp(fit$estimate_split$logsd))
e <- exp(fit$estimate_split$eta)
den <- 1 + e
(qhat <- c(e / den, 1 / den))

# Example 3: Constrained Optimization in the Setting of Example 2 -----------

# target arguments:
# - class means mu (2, unrestricted)
# - class standard deviations sd (2, must be non-negative)
# - class proportion lambda (only 1 for identification, must be in [0, 1])

normal_mixture_loglik <- function(mu, sd, lambda, data) {
  c1 <- lambda * dnorm(data, mu[1], sd[1])
  c2 <- (1 - lambda) * dnorm(data, mu[2], sd[2])
  sum(log(c1 + c2))
}

set.seed(123)

ao(
  f = normal_mixture_loglik,
  initial = runif(5),
  target = c("mu", "sd", "lambda"),
  npar = c(2, 2, 1),
  data = datasets::faithful$eruptions,
  partition = list("sequential", "random", "none"),
  minimize = FALSE,
  lower = c(-Inf, -Inf, 0, 0, 0),
  upper = c(Inf, Inf, Inf, Inf, 1),
  add_details = FALSE
)

}
