% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/preprocessing.R
\name{lemmatize_tokens}
\alias{lemmatize_tokens}
\title{Lemmatize Tokens with Batch Processing}
\usage{
lemmatize_tokens(
  tokens,
  batch_size = 50,
  model = "en_core_web_sm",
  verbose = TRUE
)
}
\arguments{
\item{tokens}{A quanteda tokens object to lemmatize.}

\item{batch_size}{Integer; number of documents to process per batch (default: 50).}

\item{model}{Character; spaCy model to use (default: "en_core_web_sm").}

\item{verbose}{Logical; print progress messages (default: TRUE).}
}
\value{
A quanteda tokens object containing lemmatized tokens.
}
\description{
Converts tokens to their lemmatized forms using spaCy, with batch processing
to handle large document collections without timeout issues.
}
\details{
Uses spaCy for linguistic lemmatization producing proper dictionary forms
(e.g., "studies" -> "study", "better" -> "good").
Batch processing prevents timeout errors with large document collections.
}
\examples{
if (interactive()) {
tokens <- quanteda::tokens(c("The studies showed better results"))
lemmatized <- lemmatize_tokens(tokens, batch_size = 50)
}
}
\concept{preprocessing}
