% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/models.R
\name{tok_model}
\alias{tok_model}
\title{Generic class for tokenization models}
\description{
Generic class for tokenization models

Generic class for tokenization models
}
\seealso{
Other model: 
\code{\link{model_bpe}},
\code{\link{model_unigram}},
\code{\link{model_wordpiece}}
}
\concept{model}
\section{Public fields}{
\if{html}{\out{<div class="r6-fields">}}
\describe{
\item{\code{.model}}{stores the pointer to the model. internal}
}
\if{html}{\out{</div>}}
}
\section{Methods}{
\subsection{Public methods}{
\itemize{
\item \href{#method-tok_model-new}{\code{tok_model$new()}}
\item \href{#method-tok_model-clone}{\code{tok_model$clone()}}
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-tok_model-new"></a>}}
\if{latex}{\out{\hypertarget{method-tok_model-new}{}}}
\subsection{Method \code{new()}}{
Initializes a genric abstract tokenizer model
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{tok_model$new(model)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{model}}{Pointer to a tokenization model}
}
\if{html}{\out{</div>}}
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-tok_model-clone"></a>}}
\if{latex}{\out{\hypertarget{method-tok_model-clone}{}}}
\subsection{Method \code{clone()}}{
The objects of this class are cloneable with this method.
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{tok_model$clone(deep = FALSE)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{deep}}{Whether to make a deep clone.}
}
\if{html}{\out{</div>}}
}
}
}
