% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/policy_eval.R, R/policy_eval_functions.R,
%   R/policy_eval_online.R
\name{policy_eval}
\alias{policy_eval}
\alias{coef.policy_eval}
\alias{IC.policy_eval}
\alias{vcov.policy_eval}
\alias{print.policy_eval}
\alias{summary.policy_eval}
\alias{estimate.policy_eval}
\alias{merge.policy_eval}
\alias{+.policy_eval}
\alias{vcov.policy_eval_online}
\title{Policy Evaluation}
\usage{
policy_eval(
  policy_data,
  policy = NULL,
  policy_learn = NULL,
  g_functions = NULL,
  g_models = g_glm(),
  g_full_history = FALSE,
  save_g_functions = TRUE,
  q_functions = NULL,
  q_models = q_glm(),
  q_full_history = FALSE,
  save_q_functions = TRUE,
  c_functions = NULL,
  c_models = NULL,
  c_full_history = FALSE,
  save_c_functions = TRUE,
  m_function = NULL,
  m_model = NULL,
  m_full_history = FALSE,
  save_m_function = TRUE,
  target = "value",
  type = "dr",
  cross_fit_type = "pooled",
  variance_type = "pooled",
  M = 1,
  nrep = 1,
  min_subgroup_size = 1,
  future_args = list(future.seed = TRUE),
  name = NULL
)

\method{coef}{policy_eval}(object, ...)

\method{IC}{policy_eval}(x, ...)

\method{vcov}{policy_eval}(object, ...)

\method{print}{policy_eval}(
  x,
  digits = 4L,
  width = 35L,
  std.error = TRUE,
  level = 0.95,
  p.value = TRUE,
  ...
)

\method{summary}{policy_eval}(object, ...)

\method{estimate}{policy_eval}(
  x,
  labels = get_element(x, "name", check_name = FALSE),
  level = 0.95,
  ...
)

\method{merge}{policy_eval}(x, y, ..., paired = TRUE)

\method{+}{policy_eval}(x, ...)

\method{vcov}{policy_eval_online}(object, ...)
}
\arguments{
\item{policy_data}{Policy data object created by \code{\link[=policy_data]{policy_data()}}.}

\item{policy}{Policy object created by \code{\link[=policy_def]{policy_def()}}.}

\item{policy_learn}{Policy learner object created by \code{\link[=policy_learn]{policy_learn()}}.}

\item{g_functions}{Fitted g-model objects, see \link{nuisance_functions}.
Preferably, use \code{g_models}.}

\item{g_models}{List of action probability models/g-models for each stage
created by \code{\link[=g_empir]{g_empir()}}, \code{\link[=g_glm]{g_glm()}}, \code{\link[=g_rf]{g_rf()}}, \code{\link[=g_sl]{g_sl()}} or similar functions.
Only used for evaluation if \code{g_functions} is \code{NULL}.
If a single model is provided and \code{g_full_history} is \code{FALSE},
a single g-model is fitted across all stages. If \code{g_full_history} is
\code{TRUE} the model is reused at every stage.}

\item{g_full_history}{If TRUE, the full history is used to fit each g-model.
If FALSE, the state/Markov type history is used to fit each g-model.}

\item{save_g_functions}{If TRUE, the fitted g-functions are saved.}

\item{q_functions}{Fitted Q-model objects, see \link{nuisance_functions}.
Only valid if the Q-functions are fitted using the same policy.
Preferably, use \code{q_models}.}

\item{q_models}{Outcome regression models/Q-models created by
\code{\link[=q_glm]{q_glm()}}, \code{\link[=q_rf]{q_rf()}}, \code{\link[=q_sl]{q_sl()}} or similar functions.
Only used for evaluation if \code{q_functions} is \code{NULL}.
If a single model is provided, the model is reused at every stage.}

\item{q_full_history}{Similar to g_full_history.}

\item{save_q_functions}{Similar to save_g_functions.}

\item{c_functions}{Fitted c-model/censoring probability model objects. Preferably, use \code{c_models}.}

\item{c_models}{List of right-censoring probability models, see \link{c_model}.}

\item{c_full_history}{Similar to g_full_history.}

\item{save_c_functions}{Similar to save_g_functions.}

\item{m_function}{Fitted outcome model object for stage K+1. Preferably, use \code{m_model}.}

\item{m_model}{Outcome model for the utility at stage K+1. Only used if the final utility
contribution is missing/has been right-censored}

\item{m_full_history}{Similar to g_full_history.}

\item{save_m_function}{Similar to save_g_functions.}

\item{target}{Character string. Either "value" or "subgroup". If "value",
the target parameter is the policy value.
If "subgroup", the target parameter
is the average treatement effect among
the subgroup of subjects that would receive
treatment under the policy, see details.
"subgroup" is only implemented for \code{type = "dr"}
in the single-stage case with a dichotomous action set.}

\item{type}{Character string. Type of evaluation. Either \code{"dr"}
(doubly robust),
\code{"ipw"} (inverse propensity weighting),
or \code{"or"} (outcome regression).}

\item{cross_fit_type}{Character string.
Either "stacked", or "pooled", see details. (Only used if \code{M > 1} and target = "subgroup")}

\item{variance_type}{Character string. Either "pooled" (default),
"stacked" or "complete", see details. (Only used if \code{M > 1})}

\item{M}{Number of folds for cross-fitting.}

\item{nrep}{Number of repetitions of cross-fitting (estimates averaged over repeated cross-fittings)}

\item{min_subgroup_size}{Minimum number of observations in the evaluated subgroup (Only used if target = "subgroup").}

\item{future_args}{Arguments passed to \code{\link[future.apply:future_apply]{future.apply::future_apply()}}.}

\item{name}{Character string.}

\item{object, x, y}{Objects of class "policy_eval".}

\item{...}{Additional arguments.}

\item{digits}{Integer. Number of printed digits.}

\item{width}{Integer. Width of printed parameter name.}

\item{std.error}{Logical. Should the std.error be printed.}

\item{level}{Numeric. Level of confidence limits.}

\item{p.value}{Logical. Should the p.value for associated confidence level be printed.}

\item{labels}{Name(s) of the estimate(s).}

\item{paired}{\code{TRUE} indicates that the estimates are based on
the same data sample.}
}
\value{
\code{policy_eval()} returns an object of class "policy_eval".
The object is a list containing the following elements:
\item{\code{coef}}{Numeric vector. The estimated target parameter:
policy value or subgroup average treatment effect.}
\item{\code{IC}}{Numeric matrix. Estimated influence curve associated with
\code{coef}.}
\item{\code{type}}{Character string. The type of evaluation ("dr", "ipw",
"or").}
\item{\code{target}}{Character string. The target parameter ("value" or "subgroup")}
\item{\code{id}}{Character vector. The IDs of the observations.}
\item{\code{name}}{Character vector. Names for the each element in \code{coef}.}
\item{\code{coef_ipw}}{(only if \code{type = "dr"}) Numeric vector.
Estimate of \code{coef} based solely on inverse probability weighting.}
\item{\code{coef_or}}{(only if \code{type = "dr"}) Numeric vector.
Estimate of \code{coef} based solely on outcome regression.}
\item{\code{policy_actions}}{\link[data.table:data.table]{data.table::data.table} with keys id and stage. Actions
associated with the policy for every observation and stage.}
\item{\code{policy_object}}{(only if \code{policy = NULL} and \code{M = 1})
The policy object returned by \code{policy_learn}, see \link{policy_learn}.}
\item{\code{g_functions}}{(only if \code{M = 1}) The
fitted g-functions. Object of class "nuisance_functions".}
\item{\code{g_values}}{The fitted g-function values.}
\item{\code{q_functions}}{(only if \code{M = 1}) The
fitted Q-functions. Object of class "nuisance_functions".}
\item{\code{q_values}}{The fitted Q-function values.}
\item{\code{Z}}{(only if \code{target = "subgroup"})
Matrix with the doubly robust stage 1 scores for each action.}
\item{\code{subgroup_indicator}}{(only if \code{target = "subgroup"})
Logical matrix identifying subjects in the subgroup.
Each column represents a different subgroup threshold.}
\item{\code{cross_fits}}{(only if \code{M > 1}) List containing the
"policy_eval" object for every (validation) fold.}
\item{\code{folds}}{(only if \code{M > 1}) The (validation) folds used
for cross-fitting.}
\item{\code{cross_fit_type}}{Character string.}
\item{\code{variance_type}}{Character string.}
}
\description{
\code{policy_eval()} is used to estimate
the value of a given fixed policy
or a data adaptive policy (e.g. a policy
learned from the data). \code{policy_eval()}
is also used to estimate the average
treatment effect among the subjects who would
get the treatment under the policy.
}
\details{
Each observation has the sequential form
\deqn{O= {B, U_1, X_1, A_1, ..., U_K, X_K, A_K, U_{K+1}},}
for a possibly stochastic number of stages K.
\itemize{
\item \eqn{B} is a vector of baseline covariates.
\item \eqn{U_k} is the reward at stage k
(not influenced by the action \eqn{A_k}).
\item \eqn{X_k} is a vector of state
covariates summarizing the state at stage k.
\item \eqn{A_k} is the categorical action
within the action set \eqn{\mathcal{A}} at stage k.
}
The utility is given by the sum of the rewards, i.e.,
\eqn{U = \sum_{k = 1}^{K+1} U_k}.

A policy is a set of functions
\deqn{d = \{d_1, ..., d_K\},}
where \eqn{d_k} for \eqn{k\in \{1, ..., K\}}
maps \eqn{\{B, X_1, A_1, ..., A_{k-1}, X_k\}} into the
action set.

Recursively define the Q-models (\code{q_models}):
\deqn{Q^d_K(h_K, a_K) = E[U|H_K = h_K, A_K = a_K]}
\deqn{Q^d_k(h_k, a_k) = E[Q_{k+1}(H_{k+1},
d_{k+1}(B,X_1, A_1,...,X_{k+1}))|H_k = h_k, A_k = a_k].}
If \code{q_full_history = TRUE},
\eqn{H_k = \{B, X_1, A_1, ..., A_{k-1}, X_k\}}, and if
\code{q_full_history = FALSE}, \eqn{H_k = \{B, X_k\}}.

The g-models (\code{g_models}) are defined as
\deqn{g_k(h_k, a_k) = P(A_k = a_k|H_k = h_k).}
If \code{g_full_history = TRUE},
\eqn{H_k = \{B, X_1, A_1, ..., A_{k-1}, X_k\}}, and if
\code{g_full_history = FALSE}, \eqn{H_k = \{B, X_k\}}.
Furthermore, if \code{g_full_history = FALSE} and \code{g_models} is a
single model, it is assumed that \eqn{g_1(h_1, a_1) = ... = g_K(h_K, a_K)}.

If \code{target = "value"} and \code{type = "or"}
\code{policy_eval()} returns the empirical estimate of
the value (\code{coef}):
\deqn{E\left[Q^d_1(H_1, d_1(\cdot))\right]}

If \code{target = "value"} and \code{type = "ipw"} \code{policy_eval()}
returns the empirical estimates of
the value (\code{coef}) and influence curve (\code{IC}):
\deqn{E\left[\left(\prod_{k=1}^K I\{A_k = d_k(\cdot)\}
g_k(H_k, A_k)^{-1}\right) U\right].}
\deqn{\left(\prod_{k=1}^K I\{A_k =
d_k(\cdot)\} g_k(H_k, A_k)^{-1}\right) U -
E\left[\left(\prod_{k=1}^K
I\{A_k = d_k(\cdot)\} g_k(H_k, A_k)^{-1}\right) U\right].}

If \code{target = "value"} and
\code{type = "dr"} \code{policy_eval} returns the empirical estimates of
the value (\code{coef}) and influence curve (\code{IC}):
\deqn{E[Z_1(d,g,Q^d)(O)],}
\deqn{Z_1(d, g, Q^d)(O) - E[Z_1(d,g, Q^d)(O)],}
where
\deqn{
Z_1(d, g, Q^d)(O) = Q^d_1(H_1 , d_1(\cdot)) +
\sum_{r = 1}^K \prod_{j = 1}^{r}
\frac{I\{A_j = d_j(\cdot)\}}{g_{j}(H_j, A_j)}
\{Q_{r+1}^d(H_{r+1} , d_{r+1}(\cdot)) - Q_{r}^d(H_r , d_r(\cdot))\}.
}

If \code{target = "subgroup"}, \code{type = "dr"}, \code{K = 1},
and \eqn{\mathcal{A} = \{0,1\}}, \code{policy_eval()}
returns the empirical estimates of the subgroup average
treatment effect (\code{coef}) and influence curve (\code{IC}):
\deqn{E[Z_1(1,g,Q)(O) - Z_1(0,g,Q)(O) | d_1(\cdot) = 1],}
\deqn{\frac{1}{P(d_1(\cdot) = 1)} I\{d_1(\cdot) = 1\}
\Big\{Z_1(1,g,Q)(O) - Z_1(0,g,Q)(O) - E[Z_1(1,g,Q)(O)
- Z_1(0,g,Q)(O) | d_1(\cdot) = 1]\Big\}.}

Applying \eqn{M}-fold cross-fitting using the \{M\} argument, let
\deqn{\mathcal{Z}_{1,m}(a) = \{Z_1(a, g_m, Q_m^d)(O): O\in \mathcal{O}_m \}.}

If \code{target = "subgroup"}, \code{type = "dr"}, \code{K = 1},
\eqn{\mathcal{A} = \{0,1\}}, and \code{cross_fit_type = "pooled"},
\code{policy_eval()} returns the estimate \deqn{\frac{1}{{N^{-1} \sum_{i =
1}^N I\{d(H_i) = 1\}}} N^{-1} \sum_{m=1}^M \sum_{(Z, H) \in \mathcal{Z}_{1,m}
\times \mathcal{H}_{1,m}} I\{d_1(H) = 1\} \left\{Z(1)-Z(0)\right\}} If
\code{cross_fit_type = "stacked"} the returned estimate is \deqn{M^{-1}
\sum_{m = 1}^M \frac{1}{{n^{-1} \sum_{h \in \mathcal{H}_{1,m}} I\{d(h) =
1\}}} n^{-1} \sum_{(Z, H) \in \mathcal{Z}_{1,m} \times \mathcal{H}_{1,m}}
I\{d_1(H) = 1\} \left\{Z(1)-Z(0)\right\},} where for ease of notation we let
the integer \eqn{n} be the number of oberservations in each fold.
}
\section{S3 generics}{

The following S3 generic functions are available for an object of
class \code{policy_eval}:
\describe{
\item{\code{\link[=get_g_functions]{get_g_functions()}}}{ Extract the fitted g-functions.}
\item{\code{\link[=get_q_functions]{get_q_functions()}}}{ Extract the fitted Q-functions.}
\item{\code{\link[=get_policy]{get_policy()}}}{ Extract the fitted policy object.}
\item{\code{\link[=get_policy_functions]{get_policy_functions()}}}{Extract the fitted policy function for a given stage.}
\item{\code{\link[=get_policy_actions]{get_policy_actions()}}}{ Extract the (fitted) policy actions.}
\item{\code{\link[=plot.policy_eval]{plot.policy_eval()}}}{Plot diagnostics.}
}
}

\examples{
library("polle")
### Single stage:
d1 <- sim_single_stage(5e2, seed=1)
pd1 <- policy_data(d1,
                   action = "A",
                   covariates = list("Z", "B", "L"),
                   utility = "U")
pd1

# defining a static policy (A=1):
pl1 <- policy_def(1)

# evaluating the policy:
pe1 <- policy_eval(policy_data = pd1,
                   policy = pl1,
                   g_models = g_glm(),
                   q_models = q_glm(),
                   name = "A=1 (glm)")

# summarizing the estimated value of the policy:
# (equivalent to summary(pe1)):
pe1
coef(pe1) # value coefficient
sqrt(vcov(pe1)) # value standard error

# getting the g-function and Q-function values:
head(predict(get_g_functions(pe1), pd1))
head(predict(get_q_functions(pe1), pd1))

# getting the fitted influence curve (IC) for the value:
head(IC(pe1))

# evaluating the policy using random forest nuisance models:
set.seed(1)
pe1_rf <- policy_eval(policy_data = pd1,
                      policy = pl1,
                      g_models = g_rf(),
                      q_models = q_rf(),
                      name = "A=1 (rf)")

# merging the two estimates (equivalent to pe1 + pe1_rf):
(est1 <- merge(pe1, pe1_rf))
coef(est1)
head(IC(est1))

### Two stages:
d2 <- sim_two_stage(5e2, seed=1)
pd2 <- policy_data(d2,
                   action = c("A_1", "A_2"),
                   covariates = list(L = c("L_1", "L_2"),
                                     C = c("C_1", "C_2")),
                   utility = c("U_1", "U_2", "U_3"))
pd2

# defining a policy learner based on cross-fitted doubly robust Q-learning:
pl2 <- policy_learn(
   type = "drql",
   control = control_drql(qv_models = list(q_glm(~C_1),
                                           q_glm(~C_1+C_2))),
   full_history = TRUE,
   L = 2) # number of folds for cross-fitting

# evaluating the policy learner using 2-fold cross fitting:
pe2 <- policy_eval(type = "dr",
                   policy_data = pd2,
                   policy_learn = pl2,
                   q_models = q_glm(),
                   g_models = g_glm(),
                   M = 2, # number of folds for cross-fitting
                   name = "drql")
# summarizing the estimated value of the policy:
pe2

# getting the cross-fitted policy actions:
head(get_policy_actions(pe2))
}
\references{
van der Laan, Mark J., and Alexander R. Luedtke.
"Targeted learning of the mean outcome under an optimal dynamic treatment rule."
Journal of causal inference 3.1 (2015): 61-95.
\doi{10.1515/jci-2013-0022}
\cr \cr
Tsiatis, Anastasios A., et al. Dynamic
treatment regimes: Statistical methods for precision medicine. Chapman and
Hall/CRC, 2019. \doi{10.1201/9780429192692}.
\cr \cr
Victor Chernozhukov, Denis
Chetverikov, Mert Demirer, Esther Duflo, Christian Hansen, Whitney Newey,
James Robins, Double/debiased machine learning for treatment and structural
parameters, The Econometrics Journal, Volume 21, Issue 1, 1 February 2018,
Pages C1–C68, \doi{10.1111/ectj.12097}.
}
\seealso{
\link[lava:IC.default]{lava::IC}, \link[lava:estimate.default]{lava::estimate.default}.
}
