% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/policy_learn.R, R/policy_object_functions.R
\name{policy_learn}
\alias{policy_learn}
\alias{policy_object}
\alias{print.policy_learn}
\alias{print.policy_object}
\title{Create Policy Learner}
\usage{
policy_learn(
  type = "blip",
  control = control_blip(),
  alpha = 0,
  threshold = NULL,
  full_history = FALSE,
  L = 1,
  cross_fit_g_models = TRUE,
  cross_fit_c_models = TRUE,
  save_cross_fit_models = FALSE,
  future_args = list(future.seed = TRUE),
  name = type
)

\method{print}{policy_learn}(x, ...)

\method{print}{policy_object}(x, ...)
}
\arguments{
\item{type}{Type of policy learner method:
\itemize{
\item \code{"ql"}: Quality/Q-learning.
\item \code{"drql"}: Doubly Robust Q-learning.
\item \code{"blip"}:
Doubly Robust blip-learning (only for dichotomous actions).
\item \code{"ptl"}: Policy Tree Learning.
\item \code{"owl"}: Outcome Weighted Learning.
\item \code{"earl"}:
Efficient Augmentation and Relaxation Learning (only single stage).
\item \code{"rwl"}: Residual Weighted Learning (only single stage).
}}

\item{control}{List of control arguments.
Values (and default values) are set using
\code{control_{type}()}. Key arguments include:\cr
\code{\link[=control_drql]{control_drql()}}:\cr
\itemize{
\item \code{qv_models}:
Single element or list of V-restricted Q-models created
by \code{\link[=q_glm]{q_glm()}}, \code{\link[=q_rf]{q_rf()}}, \code{\link[=q_sl]{q_sl()}} or similar functions.
}
\code{\link[=control_blip]{control_blip()}}:\cr
\itemize{
\item \code{blip_models}:
Single element or list of V-restricted blip-models created
by \code{\link[=q_glm]{q_glm()}}, \code{\link[=q_rf]{q_rf()}}, \code{\link[=q_sl]{q_sl()}} or similar functions.
}
\code{\link[=control_ptl]{control_ptl()}}: \cr
\itemize{
\item \code{policy_vars}: Character vector/string or list of character
vectors/strings. Variable names used to construct
the V-restricted policy tree.
The names must be a subset of the history names, see get_history_names().
\item \code{hybrid}: If \code{TRUE},
\code{\link[policytree:hybrid_policy_tree]{policytree::hybrid_policy_tree()}} is used to
fit a policy tree.
\item \code{depth}:
Integer or integer vector. The depth of the fitted policy
tree for each stage.
}
\code{\link[=control_owl]{control_owl()}}: \cr
\itemize{
\item \code{policy_vars}: As in \code{control_ptl()}.
\item \code{loss}: Loss function.
The options are \code{"hinge"}, \code{"ramp"},
\code{"logit"}, \code{"logit.lasso"}, \code{"l2"}, \code{"l2.lasso"}.
\item \code{kernel}: Type of kernel
used by the support vector machine. The
options are \code{"linear"}, \code{"rbf"}.
\item \code{augment}:  If \code{TRUE} the outcomes are augmented.
}
\code{\link[=control_earl]{control_earl()}}/\code{\link[=control_rwl]{control_rwl()}}: \cr
\itemize{
\item \code{moPropen}:
Propensity model of class "ModelObj", see \link[modelObj:modelObj]{modelObj::modelObj}.
\item \code{moMain}: Main effects outcome model of class "ModelObj".
\item \code{moCont} Contrast outcome model of class "ModelObj".
\item \code{regime}:
An object of class \link{formula} specifying the design of the policy.
\item \code{surrogate}:
The surrogate 0-1 loss function. The options are
\code{"logit"}, \code{"exp"},
\code{"hinge"}, \code{"sqhinge"}, \code{"huber"}.
\item \code{kernel}: The options are
\code{"linear"}, \code{"poly"}, \code{"radial"}.
}}

\item{alpha}{Probability threshold for determining realistic actions.}

\item{threshold}{Numeric vector, thresholds for not
choosing the reference action at stage 1.}

\item{full_history}{If \code{TRUE}, the full
history is used to fit each policy function (e.g. QV-model,
policy tree). If FALSE, the single stage/
"Markov type" history is used to fit each policy function.}

\item{L}{Number of folds for cross-fitting nuisance models.}

\item{cross_fit_g_models}{If \code{TRUE}, the g-models will not be
cross-fitted even if L > 1.}

\item{cross_fit_c_models}{If \code{TRUE}, the c-models will not be
cross-fitted even if L > 1.}

\item{save_cross_fit_models}{If \code{TRUE}, the cross-fitted
models will be saved.}

\item{future_args}{Arguments passed to \code{\link[future.apply:future_apply]{future.apply::future_apply()}}.}

\item{name}{Character string.}

\item{x}{Object of class "policy_object" or "policy_learn".}

\item{...}{Additional arguments passed to print.}
}
\value{
Function of inherited class \code{"policy_learn"}.
Evaluating the function on a \link{policy_data} object returns an object of
class \link{policy_object}. A policy object is a list containing all or
some of the following elements:
\describe{
\item{\code{q_functions}}{Fitted Q-functions. Object
of class "nuisance_functions".}
\item{\code{g_functions}}{Fitted g-functions. Object
of class "nuisance_functions".}
\item{\code{action_set}}{Sorted character vector describing
the action set, i.e.,
the possible actions at each stage.}
\item{\code{alpha}}{Numeric. Probability threshold to determine
realistic actions.}
\item{\code{K}}{Integer. Maximal number of stages.}
\item{\code{qv_functions}}{(only if \code{type = "drql"}) Fitted
V-restricted Q-functions. Contains a fitted model for each stage and action.}
\item{\code{ptl_objects}}{(only if \code{type = "ptl"}) Fitted V-restricted
policy trees. Contains a \link[policytree:policy_tree]{policytree::policy_tree} for each stage.}
\item{\code{ptl_designs}}{(only if \code{type = "ptl"}) Specification of the
V-restricted design matrix for each stage}
}
}
\description{
\code{policy_learn()} is used to specify a policy
learning method (Q-learning,
doubly robust Q-learning, policy tree
learning and outcome weighted learning).
Evaluating the policy learner returns a policy object.
}
\section{S3 generics}{

The following S3 generic functions are available for an object of
class "policy_object":
\describe{
\item{\code{\link[=get_g_functions]{get_g_functions()}}}{Extract the fitted g-functions.}
\item{\code{\link[=get_q_functions]{get_q_functions()}}}{Extract the fitted Q-functions.}
\item{\code{\link[=get_policy]{get_policy()}}}{Extract the fitted policy object.}
\item{\code{\link[=get_policy_functions]{get_policy_functions()}}}{Extract the fitted policy function for
a given stage.}
\item{\code{\link[=get_policy_actions]{get_policy_actions()}}}{Extract the (fitted) policy actions.}
}
}

\examples{
library("polle")
### Two stages:
d <- sim_two_stage(5e2, seed=1)
pd <- policy_data(d,
                  action = c("A_1", "A_2"),
                  baseline = c("BB"),
                  covariates = list(L = c("L_1", "L_2"),
                                    C = c("C_1", "C_2")),
                  utility = c("U_1", "U_2", "U_3"))
pd

### V-restricted (Doubly Robust) Q-learning

# specifying the learner:
pl <- policy_learn(
  type = "drql",
  control = control_drql(qv_models = list(q_glm(formula = ~ C_1 + BB),
                                          q_glm(formula = ~ L_1 + BB))),
  full_history = TRUE
)

# evaluating the learned policy
pe <- policy_eval(policy_data = pd,
                  policy_learn = pl,
                  q_models = q_glm(),
                  g_models = g_glm())
pe
# getting the policy object:
po <- get_policy_object(pe)
# inspecting the fitted QV-model for each action strata at stage 1:
po$qv_functions$stage_1
head(get_policy(pe)(pd))
}
\references{
Doubly Robust Q-learning (\code{type = "drql"}): Luedtke, Alexander R., and
Mark J. van der Laan. "Super-learning of an optimal dynamic treatment rule."
The international journal of biostatistics 12.1 (2016): 305-332.
\doi{10.1515/ijb-2015-0052}.\cr
\cr
Policy Tree Learning (\code{type = "ptl"}): Zhou, Zhengyuan, Susan Athey,
and Stefan Wager. "Offline multi-action policy learning: Generalization and
optimization." Operations Research (2022). \doi{10.1287/opre.2022.2271}.\cr
\cr
(Augmented) Outcome Weighted Learning: Liu, Ying, et al. "Augmented
outcome‐weighted learning for estimating optimal dynamic treatment regimens."
Statistics in medicine 37.26 (2018): 3776-3788. \doi{10.1002/sim.7844}.
}
\seealso{
\code{\link[=policy_eval]{policy_eval()}}
}
