% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/preprocessing.R
\name{prep_texts}
\alias{prep_texts}
\title{Preprocess Text Data}
\usage{
prep_texts(
  united_tbl,
  text_field = "united_texts",
  min_char = 2,
  lowercase = TRUE,
  remove_punct = TRUE,
  remove_symbols = TRUE,
  remove_numbers = TRUE,
  remove_url = TRUE,
  remove_separators = TRUE,
  split_hyphens = TRUE,
  split_tags = TRUE,
  include_docvars = TRUE,
  keep_acronyms = FALSE,
  padding = FALSE,
  remove_stopwords = FALSE,
  stopwords_source = "snowball",
  stopwords_language = "en",
  custom_stopwords = NULL,
  custom_valuetype = "glob",
  math_mode = FALSE,
  verbose = FALSE,
  ...
)
}
\arguments{
\item{united_tbl}{A data frame that contains text data.}

\item{text_field}{The name of the column that contains the text data.}

\item{min_char}{The minimum number of characters for a token to be included (default: 2).}

\item{lowercase}{Logical; convert all tokens to lowercase (default: TRUE). Recommended for most text analysis tasks.}

\item{remove_punct}{Logical; remove punctuation from the text (default: TRUE).}

\item{remove_symbols}{Logical; remove symbols from the text (default: TRUE).}

\item{remove_numbers}{Logical; remove numbers from the text (default: TRUE).}

\item{remove_url}{Logical; remove URLs from the text (default: TRUE).}

\item{remove_separators}{Logical; remove separators from the text (default: TRUE).}

\item{split_hyphens}{Logical; split hyphenated words into separate tokens (default: TRUE).}

\item{split_tags}{Logical; split tags into separate tokens (default: TRUE).}

\item{include_docvars}{Logical; include document variables in the tokens object (default: TRUE).}

\item{keep_acronyms}{Logical; keep acronyms in the text (default: FALSE).}

\item{padding}{Logical; add padding to the tokens object (default: FALSE).}

\item{remove_stopwords}{Logical; remove stopwords from the text (default: FALSE).}

\item{stopwords_source}{Character; source for stopwords, e.g., "snowball", "stopwords-iso" (default: "snowball").}

\item{stopwords_language}{Character; language for stopwords (default: "en").}

\item{custom_stopwords}{Character vector; additional words to remove (default: NULL).}

\item{custom_valuetype}{Character; valuetype for custom_stopwords pattern matching, one of "glob", "regex", or "fixed" (default: "glob").}

\item{math_mode}{Logical; if \code{TRUE}, preserve math content (numbers, operators, symbols) by forcing \code{remove_punct}, \code{remove_symbols}, and \code{remove_numbers} all to \code{FALSE}, then strip only sentence-end punctuation such as periods, commas, question marks, exclamation marks, colons, semicolons, parentheses, brackets, braces, quotation marks, em dashes, and en dashes. The \code{min_char} default of 2 still applies, so noisy single-character tokens are dropped; pass \code{min_char = 1} to keep them. Use for math or STEM corpora where multi-character operators and numerals carry meaning (default: FALSE).}

\item{verbose}{Logical; print verbose output (default: FALSE).}

\item{...}{Additional arguments passed to \code{quanteda::tokens}.}
}
\value{
A \code{tokens} object that contains the preprocessed text data.
}
\description{
Preprocesses text data following the complete workflow implemented in the Shiny application:
\itemize{
\item Constructing a corpus from united texts
\item Tokenizing text into words with configurable options
\item Converting to lowercase with acronym preservation option
\item Applying character length filtering
\item Optional multi-word expression detection and compound term creation
\item Stopword removal and lemmatization capabilities
}

This function serves as the foundation for all subsequent text analysis workflows.
}
\examples{
\donttest{
mydata <- TextAnalysisR::SpecialEduTech

united_tbl <- TextAnalysisR::unite_cols(
  mydata,
  listed_vars = c("title", "keyword", "abstract")
)

tokens <- TextAnalysisR::prep_texts(united_tbl,
                                         text_field = "united_texts",
                                         min_char = 2,
                                         lowercase = TRUE,
                                         remove_punct = TRUE,
                                         remove_symbols = TRUE,
                                         remove_numbers = TRUE,
                                         remove_url = TRUE,
                                         remove_separators = TRUE,
                                         split_hyphens = TRUE,
                                         split_tags = TRUE,
                                         include_docvars = TRUE,
                                         keep_acronyms = FALSE,
                                         padding = FALSE,
                                         verbose = FALSE)
print(tokens)
}
}
\seealso{
\code{\link[=unite_cols]{unite_cols()}} to combine text columns first; \code{\link[=lemmatize_tokens]{lemmatize_tokens()}} to reduce words to base form (e.g., running -> run); \code{quanteda::dfm()} to build a document-feature matrix from the result
}
\concept{preprocessing}
