% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/trainers.R
\name{trainer_unigram}
\alias{trainer_unigram}
\title{Unigram tokenizer trainer}
\description{
Unigram tokenizer trainer

Unigram tokenizer trainer
}
\seealso{
Other trainer: 
\code{\link{tok_trainer}},
\code{\link{trainer_bpe}},
\code{\link{trainer_wordpiece}}
}
\concept{trainer}
\section{Super class}{
\code{\link[tok:tok_trainer]{tok::tok_trainer}} -> \code{tok_trainer_unigram}
}
\section{Methods}{
\subsection{Public methods}{
\itemize{
\item \href{#method-tok_trainer_unigram-new}{\code{trainer_unigram$new()}}
\item \href{#method-tok_trainer_unigram-clone}{\code{trainer_unigram$clone()}}
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-tok_trainer_unigram-new"></a>}}
\if{latex}{\out{\hypertarget{method-tok_trainer_unigram-new}{}}}
\subsection{Method \code{new()}}{
Constructor for the Unigram tokenizer
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{trainer_unigram$new(
  vocab_size = 8000,
  show_progress = TRUE,
  special_tokens = NULL,
  shrinking_factor = 0.75,
  unk_token = NULL,
  max_piece_length = 16,
  n_sub_iterations = 2
)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{vocab_size}}{The size of the final vocabulary, including all tokens and alphabet.}

\item{\code{show_progress}}{Whether to show progress bars while training.}

\item{\code{special_tokens}}{A list of special tokens the model should be aware of.}

\item{\code{shrinking_factor}}{The shrinking factor used at each step of training
to prune the vocabulary.}

\item{\code{unk_token}}{The token used for out-of-vocabulary tokens.}

\item{\code{max_piece_length}}{The maximum length of a given token.}

\item{\code{n_sub_iterations}}{The number of iterations of the EM algorithm to perform
before pruning the vocabulary.}

\item{\code{initial_alphabet}}{A list of characters to include in the initial alphabet,
even if not seen in the training dataset. If the strings contain more than
one character, only the first one is kept.}
}
\if{html}{\out{</div>}}
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-tok_trainer_unigram-clone"></a>}}
\if{latex}{\out{\hypertarget{method-tok_trainer_unigram-clone}{}}}
\subsection{Method \code{clone()}}{
The objects of this class are cloneable with this method.
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{trainer_unigram$clone(deep = FALSE)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{deep}}{Whether to make a deep clone.}
}
\if{html}{\out{</div>}}
}
}
}
