% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/tokenize.R
\name{tokenize}
\alias{tokenize}
\title{Recompute the tokens for a document or corpus}
\usage{
tokenize(
  x,
  tokenizer,
  ...,
  hash_func = hash_string,
  minhash_func = NULL,
  keep_tokens = FALSE,
  keep_text = TRUE
)
}
\arguments{
\item{x}{A \code{\link{TextReuseTextDocument}} or
\code{\link{TextReuseCorpus}}.}

\item{tokenizer}{A function to split the text into tokens. See
\code{\link{tokenizers}}.}

\item{...}{Arguments passed on to the \code{tokenizer}.}

\item{hash_func}{A function to hash the tokens. See
\code{\link{hash_string}}.}

\item{minhash_func}{A function to create minhash signatures. See
\code{\link{minhash_generator}}.}

\item{keep_tokens}{Should the tokens be saved in the document that is
returned or discarded?}

\item{keep_text}{Should the text be saved in the document that is returned or
discarded?}
}
\value{
The modified \code{\link{TextReuseTextDocument}} or
  \code{\link{TextReuseCorpus}}.
}
\description{
Given a \code{\link{TextReuseTextDocument}} or a
\code{\link{TextReuseCorpus}}, this function recomputes the tokens and hashes
with the functions specified. Optionally, it can also recompute the minhash signatures.
}
\examples{
dir <- system.file("extdata/legal", package = "textreuse")
corpus <- TextReuseCorpus(dir = dir, tokenizer = NULL)
corpus <- tokenize(corpus, tokenize_ngrams)
head(tokens(corpus[[1]]))
}
