% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/scorer-model.R
\name{scorer_model}
\alias{scorer_model}
\alias{model_graded_qa}
\alias{model_graded_fact}
\title{Model-based scoring}
\usage{
model_graded_qa(
  template = NULL,
  instructions = NULL,
  grade_pattern = "(?i)GRADE\\\\s*:\\\\s*([CPI])(.*)$",
  partial_credit = FALSE,
  scorer_chat = NULL
)

model_graded_fact(
  template = NULL,
  instructions = NULL,
  grade_pattern = "(?i)GRADE\\\\s*:\\\\s*([CPI])(.*)$",
  partial_credit = FALSE,
  scorer_chat = NULL
)
}
\arguments{
\item{template}{Grading template to use--a \code{glue()} string which will take
substitutions \code{input}, \code{answer}, \code{criterion}, \code{instructions}.}

\item{instructions}{Grading instructions. If provided, this completely
replaces the default instructions, which specify e.g. how the grader should
format its output (e.g. \code{"GRADE: C"}).}

\item{grade_pattern}{A regex pattern to extract the final grade from the
judge model's response.}

\item{partial_credit}{Whether to allow partial credit.}

\item{scorer_chat}{An ellmer chat used to grade the model output, e.g.
\code{\link[ellmer:chat_anthropic]{ellmer::chat_claude()}}.}
}
\value{
A function that will grade model responses according to the given instructions.
See \link{Task}'s \code{scorer} argument for a description of the returned function.
The functions that \code{model_graded_qa()} and \code{model_graded_fact()} output
can be passed directly to \verb{$eval()}.

See the documentation for the \code{scorer} argument in \link{Task} for more
information on the return type.
}
\description{
Model-based scoring makes use of a model to score output from a solver.
\itemize{
\item \code{model_graded_qa()} scores how well a solver answers a question/answer task.
\item \code{model_graded_fact()} determines whether a solver includes a given fact
in its response.
}

The two scorers are quite similar in their implementation, but use a different
default \code{template} to evaluate correctness.
}
\examples{
# Quality assurance -----------------------------
if (!identical(Sys.getenv("ANTHROPIC_API_KEY"), "")) {
  # set the log directory to a temporary directory
  withr::local_envvar(VITALS_LOG_DIR = withr::local_tempdir())

  library(ellmer)
  library(tibble)

  simple_addition <- tibble(
    input = c("What's 2+2?", "What's 2+3?"),
    target = c("4", "5")
  )

  tsk <- Task$new(
    dataset = simple_addition,
    solver = generate(solver_chat = chat_claude(model = "claude-sonnet-4-5-20250929")),
    scorer = model_graded_qa()
  )

  tsk$eval()
}

# Factual response -------------------------------
if (!identical(Sys.getenv("ANTHROPIC_API_KEY"), "")) {
  # set the log directory to a temporary directory
  withr::local_envvar(VITALS_LOG_DIR = withr::local_tempdir())

  library(ellmer)
  library(tibble)

  r_history <- tibble(
    input = c(
      "Who created the R programming language?",
      "In what year was version 1.0 of R released?"
    ),
    target = c("Ross Ihaka and Robert Gentleman.", "2000.")
  )

  tsk <- Task$new(
    dataset = r_history,
    solver = generate(solver_chat = chat_claude(model = "claude-sonnet-4-5-20250929")),
    scorer = model_graded_fact()
  )

  tsk$eval()
}

}
\seealso{
\link{scorer_detect} for string detection-based scoring.
}
