% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/lexical_analysis.R
\name{lexical_diversity_analysis}
\alias{lexical_diversity_analysis}
\title{Lexical Diversity Analysis}
\usage{
lexical_diversity_analysis(x, measures = "all", texts = NULL, cache_key = NULL)
}
\arguments{
\item{x}{A quanteda DFM or tokens object. Tokens object is preferred for
accurate MTLD calculation since it preserves token order.}

\item{measures}{Character vector of measures to calculate.
Default is "all" which includes: TTR, C, R, CTTR, U, S, K, I, D, Vm, Maas, MATTR, MSTTR, MTLD, and HDD.
Most recommended: "MTLD", "MATTR", or "HDD" for length-independent measures.}

\item{texts}{Optional character vector of original texts. Required for MTLD
calculation when using DFM input (since DFM loses token order).}

\item{cache_key}{Optional cache key (e.g., from digest::digest) for caching
expensive calculations. Use the same cache_key to retrieve cached results.}
}
\value{
A list containing:
\itemize{
\item \code{lexical_diversity}: Data frame with per-document lexical diversity scores
\item \code{summary_stats}: List of summary statistics (mean, median, sd) for each measure
}
}
\description{
Calculates multiple lexical diversity metrics for a document-feature matrix (DFM)
or tokens object. Supports all quanteda.textstats measures plus MTLD
(Measure of Textual Lexical Diversity), which is the most recommended measure
according to McCarthy & Jarvis (2010) for being independent of text length.
}
\details{
MTLD (Measure of Textual Lexical Diversity) is calculated using the algorithm
from McCarthy & Jarvis (2010). It counts the number of "factors" needed to
reduce TTR below 0.72, then divides the number of tokens by the number of factors.
This provides a length-independent measure of lexical diversity.

Important notes:
\itemize{
\item For MTLD accuracy, pass a tokens object (not DFM) as input
\item If using DFM, provide the 'texts' parameter for MTLD calculation
\item MATTR and MSTTR window sizes are automatically adjusted for short documents
\item Raw TTR falls mechanically as documents lengthen; compare TTR only
across documents of similar length
\item MTLD and MATTR are most reliable at 100+ tokens per document
\item Results are cached when cache_key is provided for repeated analysis
}
}
\examples{
\donttest{
data(SpecialEduTech)
texts <- SpecialEduTech$abstract[1:10]
corp <- quanteda::corpus(texts)
toks <- quanteda::tokens(corp)
# Preferred: pass tokens object for accurate MTLD
lex_div <- lexical_diversity_analysis(toks, texts = texts)
# With caching for repeated analysis
cache_key <- digest::digest(texts)
lex_div <- lexical_diversity_analysis(toks, texts = texts, cache_key = cache_key)
# Alternative: pass DFM with texts for MTLD accuracy
dfm_obj <- quanteda::dfm(toks)
lex_div <- lexical_diversity_analysis(dfm_obj, texts = texts)
print(lex_div)
}

}
\references{
McCarthy, P. M., & Jarvis, S. (2010). MTLD, vocd-D, and HD-D: A validation study
of sophisticated approaches to lexical diversity assessment.
Behavior Research Methods, 42(2), 381-392.
}
\seealso{
\code{\link[=calculate_text_readability]{calculate_text_readability()}} for grade-level / Flesch metrics on the same input; \code{\link[=calculate_lexical_dispersion]{calculate_lexical_dispersion()}} for term spread across documents; \code{\link[=plot_lexical_diversity_distribution]{plot_lexical_diversity_distribution()}} to visualize
}
\concept{lexical}
