% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/text_mining.R
\name{textTokenizer}
\alias{textTokenizer}
\title{Tokenize Vectors into Words}
\usage{
textTokenizer(
  text,
  exclude = NULL,
  lang = NULL,
  min_word_freq = 5,
  min_word_len = 2,
  keep_spaces = FALSE,
  lowercase = TRUE,
  remove_numbers = TRUE,
  remove_punct = TRUE,
  remove_lettt = TRUE,
  laughs = TRUE,
  utf = TRUE,
  df = FALSE,
  h2o = FALSE,
  quiet = FALSE
)
}
\arguments{
\item{text}{Character vector. Sentences or texts you wish to tokenize.}

\item{exclude}{Character vector. Which words do you wish to exclude?}

\item{lang}{Character. Language in text (used for stop words). Example:
"spanish" or "english". Set to \code{NA} to ignore.}

\item{min_word_freq}{Integer. This will discard words that appear
less than <int> times. Defaults to 2. Set to \code{NA} to ignore.}

\item{min_word_len}{Integer. This will discard words that have
less than <int> characters. Defaults to 5. Set to \code{NA} to ignore.}

\item{keep_spaces}{Boolean. If you wish to keep spaces in each line
to keep unique compound words, separated with spaces, set to TRUE.
For example, 'one two' will be set as 'one_two' and treated as
a single word.}

\item{lowercase, remove_numbers, remove_punct}{Boolean.}

\item{remove_lettt}{Boolean. Repeated letters (more than 3 consecutive).}

\item{laughs}{Boolean. Try to unify all laughs texts.}

\item{utf}{Boolean. Transform all characters to UTF (no accents and crazy symbols)}

\item{df}{Boolean. Return a dataframe with a one-hot-encoding kind of
results? Each word is a column and returns if word is contained.}

\item{h2o}{Boolean. Return \code{H2OFrame}?}

\item{quiet}{Boolean. Keep quiet? If not, informative messages will be shown.}
}
\value{
data.frame. Tokenized words with counters.
}
\description{
This function transforms texts into words, calculate frequencies,
supress stop words in a given language.
}
\seealso{
Other Data Wrangling: 
\code{\link{balance_data}()},
\code{\link{categ_reducer}()},
\code{\link{cleanText}()},
\code{\link{date_cuts}()},
\code{\link{date_feats}()},
\code{\link{file_name}()},
\code{\link{formatHTML}()},
\code{\link{holidays}()},
\code{\link{impute}()},
\code{\link{left}()},
\code{\link{normalize}()},
\code{\link{num_abbr}()},
\code{\link{ohe_commas}()},
\code{\link{ohse}()},
\code{\link{quants}()},
\code{\link{removenacols}()},
\code{\link{replaceall}()},
\code{\link{replacefactor}()},
\code{\link{textFeats}()},
\code{\link{vector2text}()},
\code{\link{year_month}()},
\code{\link{zerovar}()}

Other Text Mining: 
\code{\link{cleanText}()},
\code{\link{ngrams}()},
\code{\link{remove_stopwords}()},
\code{\link{replaceall}()},
\code{\link{sentimentBreakdown}()},
\code{\link{textCloud}()},
\code{\link{textFeats}()},
\code{\link{topics_rake}()}
}
\concept{Data Wrangling}
\concept{Text Mining}
