% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/unsupervised_learning.R
\name{mec}
\alias{mec}
\title{Unsupervised Maximum Entropy Classifier for Record Linkage}
\usage{
mec(
  A,
  B,
  variables,
  comparators = NULL,
  methods = NULL,
  duplicates_in_A = FALSE,
  start_params = NULL,
  nonpar_hurdle = TRUE,
  set_construction = NULL,
  target_rate = 0.03,
  max_iter_bisection = 100,
  tol = 0.005,
  delta = 0.5,
  eps = 0.05,
  max_iter_em = 10,
  tol_em = 1,
  controls_nleqslv = list(),
  controls_kliep = control_kliep(),
  true_matches = NULL,
  verbose = FALSE
)
}
\arguments{
\item{A}{A duplicate-free \code{data.frame} or \code{data.table}.}

\item{B}{A duplicate-free \code{data.frame} or \code{data.table}.}

\item{variables}{A character vector of key variables used to create comparison vectors.}

\item{comparators}{A named list of functions for comparing pairs of records.}

\item{methods}{A named list of methods used for estimation (\code{"binary"}, \code{"continuous_parametric"}, \code{"continuous_nonparametric"} or \code{"hit_miss"}).}

\item{duplicates_in_A}{Logical indicating whether to allow \code{A} to have duplicate records.}

\item{start_params}{Start parameters for the \code{"binary"}, \code{"continuous_parametric"} and \code{"hit_miss"} methods.}

\item{nonpar_hurdle}{Logical indicating whether to use a hurdle model or not
(used only if the \code{"continuous_nonparametric"} method has been chosen for at least one variable).}

\item{set_construction}{A method for constructing the predicted set of matches (\code{"size"}, \code{"flr"} or \code{"mmr"}).}

\item{target_rate}{A target false link rate (FLR) or missing match rate
(MMR) (used only if \code{set_construction == "flr"} or \code{set_construction == "mmr"}).}

\item{max_iter_bisection}{A maximum number of iterations for the bisection procedure
(used only if \code{set_construction == "flr"} or \code{set_construction == "mmr"}).}

\item{tol}{Error tolerance in the bisection procedure
(used only if \code{set_construction == "flr"} or \code{set_construction == "mmr"}).}

\item{delta}{A numeric value specifying the tolerance for the change in the estimated number of matches between iterations.}

\item{eps}{A numeric value specifying the tolerance for the change in model parameters between iterations.}

\item{max_iter_em}{A maximum number of iterations for the EM algorithm
(used only if the \code{"hit_miss"} method has been chosen for at least one variable).}

\item{tol_em}{Error tolerance in the EM algorithm
(used only if the \code{"hit_miss"} method has been chosen for at least one variable).}

\item{controls_nleqslv}{Controls passed to the \link[nleqslv:nleqslv]{nleqslv()} function
(only if the \code{"continuous_parametric"} method has been chosen for at least one variable).}

\item{controls_kliep}{Controls passed to the \link[densityratio:kliep]{kliep()} function
(only if the \code{"continuous_nonparametric"} method has been chosen for at least one variable).}

\item{true_matches}{A \code{data.frame} or \code{data.table} indicating known matches.}

\item{verbose}{Logical indicating whether to print progress messages.}
}
\value{
Returns a list containing:\cr
\itemize{
\item{\code{M_est} -- a \code{data.table} with predicted matches,}
\item{\code{n_M_est} -- estimated classification set size,}
\item{\code{flr_est} -- estimated false link rate (FLR),}
\item{\code{mmr_est} -- estimated missing match rate (MMR),}
\item{\code{iter_bisection} -- the number of iterations in the bisection procedure,}
\item{\code{b_vars} -- a character vector of variables used for the \code{"binary"} method (with the prefix \code{"gamma_"}),}
\item{\code{cpar_vars} -- a character vector of variables used for the \code{"continuous_parametric"} method (with the prefix \code{"gamma_"}),}
\item{\code{cnonpar_vars} -- a character vector of variables used for the \code{"continuous_nonparametric"} method (with the prefix \code{"gamma_"}),}
\item{\code{hm_vars} -- a character vector of variables used for the \code{"hit_miss"} method (with the prefix \code{"gamma_"}),}
\item{\code{b_params} -- parameters estimated using the \code{"binary"} method,}
\item{\code{cpar_params} -- parameters estimated using the \code{"continuous_parametric"} method,}
\item{\code{hm_params} -- parameters estimated using the \code{"hit_miss"} method,}
\item{\code{ratio_kliep} -- a result of the \link[densityratio:kliep]{kliep()} function,}
\item{\code{variables} -- a character vector of key variables used for comparison,}
\item{\code{set_construction} -- a method for constructing the predicted set of matches,}
\item{\code{eval_metrics} -- standard metrics for quality assessment (if \code{true_matches} is provided),}
\item{\code{confusion} -- confusion matrix (if \code{true_matches} is provided).}
}
}
\description{
Implements several extensions to the maximum entropy classification (MEC) algorithm for record linkage
(see \href{https://www150.statcan.gc.ca/n1/pub/12-001-x/2022001/article/00007-eng.htm}{Lee et al. (2022)}),
iteratively estimating probability/density ratios to classify record pairs into matches and non-matches
based on comparison vectors.
}
\details{
Consider two datasets without duplicates: \eqn{A} and \eqn{B}.
Let the bipartite comparison space \eqn{\Omega = A \times B} consist of
matches \eqn{M} and non-matches \eqn{U} between the records in files
\eqn{A} and \eqn{B}. For any pair of records \eqn{(a,b) \in \Omega},
let \eqn{\pmb{\gamma}_{ab} = (\gamma_{ab}^1,\gamma_{ab}^2,
\ldots,\gamma_{ab}^K)'} be the comparison vector between
a set of key variables. The original MEC algorithm uses the binary
comparison function to evaluate record pairs across two datasets.
However, this approach may be insufficient when handling datasets
with frequent errors across multiple variables.

We propose the use of continuous comparison functions to address
the limitations of binary comparison methods. We consider every
semi-metric, i.e., a function \eqn{d: A \times B \to \mathbb{R}},
satisfying the following conditions:\cr
\enumerate{
\item{\eqn{d(x,y) \geq 0},}
\item{\eqn{d(x,y) = 0} if and only if \eqn{x = y},}
\item{\eqn{d(x,y) = d(y,x)}.}
}
For example, we can use the Jaro-Winkler distance for character variables
(which is implemented in the \code{automatedRecLin} package as \code{\link[=jarowinkler_complement]{jarowinkler_complement()}})
or the Euclidean distance for numerical variables. The \code{automatedRecLin} package allows the use of
a different comparison function for each key variable (which should be specified
as a list in the \code{comparators} argument). The default function
for each key variable is \link[reclin2:cmp_identical]{cmp_identical()}
(the binary comparison function).

The \code{\link[=mec]{mec()}} function offers different approaches to estimate the
probability/density ratio between matches and non-matches,
which should be specified as a list in the \code{methods} argument.
The available methods suitable for the binary comparison function
are \code{"binary"} and \code{"hit_miss"}. Both assume that \eqn{\gamma_{ab}^k|M}
and \eqn{\gamma_{ab}^k|U} follow Bernoulli distributions.
\code{"binary"} and \code{"hit_miss"} both estimate the parameters for the matches iteratively,
but \code{"binary"} estimates the parameters for the non-matches
only at the start, while \code{"hit_miss"} does
so iteratively using a hit-miss model (for details see
\href{https://www150.statcan.gc.ca/n1/pub/12-001-x/2022001/article/00007-eng.htm}{Lee et al. (2022)}).
\code{"binary"} is the default method for each variable.

For the continuous semi-metrics we suggest the usage
of \code{"continuous_parametric"} or \code{"continuous_nonparametric"}
method. The \code{"continuous_parametric"} method assumes that
\eqn{\gamma_{ab}^k|M} and \eqn{\gamma_{ab}^k|U} follow
hurdle Gamma distributions. The density function of a hurdle
Gamma distribution is characterized by three parameters
\eqn{p_0 \in (0,1)} and \eqn{\alpha, \beta > 0} as follows:
\deqn{
f(x;p_0,\alpha,\beta) = p_0^{\mathbb{I}(x = 0)}[(1 - p_0) v(x;\alpha,\beta)]^{\mathbb{I}(x > 0)},
}
where
\deqn{
v(x;\alpha,\beta) = \frac{\beta^{\alpha} x^{\alpha - 1} \exp(-\beta x)}
{\Gamma(\alpha)}
}
is the density function of a Gamma distribution
(for details see \href{https://ideas.repec.org/a/eee/csdana/v179y2023ics0167947322002365.html}{Vo et al. (2023)}).
At the beginning, the algorithm estimates the parameters for the non-matches
and then does it iteratively for the matches.
The \code{"continuous_nonparametric"} method does not assume anything about
the distributions of the comparison vectors. It iteratively directly
estimates the density ratio between the matches and the non-matches, using
the Kullback-Leibler Importance Estimation Procedure (KLIEP).
For details see \href{https://link.springer.com/article/10.1007/s10463-008-0197-x}{Sugiyama et al. (2008)}.

The \code{\link[=mec]{mec()}} function allows the construction of the predicted set
of matches using its estimated size or the bisection procedure,
described in \href{https://www150.statcan.gc.ca/n1/pub/12-001-x/2022001/article/00007-eng.htm}{Lee et al. (2022)},
based on a target false link rate (FLR)
or missing match rate (MMR). To use the second option, set \code{set_construction = "flr"}
or \code{set_construction = "mmr"} and
specify a target error rate using the \code{target_rate} argument.

The assumption that \eqn{A} and \eqn{B} contain no duplicate records
might be relaxed by allowing \eqn{A} to have duplicates. To do so,
set \code{duplicates_in_A = TRUE}.
}
\examples{
df_1 <- data.frame(
  name = c("Emma", "Liam", "Olivia", "Noah", "Ava",
           "Ethan", "Sophia", "Mason", "Isabella", "James"),
  surname = c("Smith", "Johnson", "Williams", "Brown", "Jones",
              "Garcia", "Miller", "Davis", "Rodriguez", "Wilson"),
  city = c("New York", "Los Angeles", "Chicago", "Houston", "Phoenix",
           "Philadelphia", "San Antonio", "San Diego", "Dallas", "San Jose")
)

df_2 <- data.frame(
  name = c(
    "Emma", "Liam", "Olivia", "Noah",
    "Ava", "Ehtan", "Sopia", "Mson",
    "Charlotte", "Benjamin", "Amelia", "Lucas"
  ),
  surname = c(
     "Smith", "Johnson", "Williams", "Brown",
    "Jnes", "Garca", "Miler", "Dvis",
    "Martinez", "Lee", "Hernandez", "Clark"
  ),
  city = c(
    "New York", "Los Angeles", "Chicago", "Houston",
    "Phonix", "Philadelpia", "San Antnio", "San Dieg",
    "Seattle", "Miami", "Boston", "Denver"
  )
)
true_matches <- data.frame(
  "a" = 1:8,
  "b" = 1:8
)

variables <- c("name", "surname", "city")
comparators <- list(
  "name" = jarowinkler_complement(),
  "surname" = jarowinkler_complement(),
  "city" = jarowinkler_complement()
)
methods <- list(
  "name" = "continuous_parametric",
  "surname" = "continuous_parametric",
  "city" = "continuous_parametric"
)

set.seed(1)
result <- mec(A = df_1, B = df_2,
              variables = variables,
              comparators = comparators,
              methods = methods,
              true_matches = true_matches)
result
}
\references{
Lee, D., Zhang, L.-C. and Kim, J. K. (2022). Maximum entropy classification for record linkage.
Survey Methodology, Statistics Canada, Catalogue No. 12-001-X, Vol. 48, No. 1.

Vo, T. H., Chauvet, G., Happe, A., Oger, E., Paquelet, S., and Garès, V. (2023).
Extending the Fellegi-Sunter record linkage model for mixed-type data with application to the French national health data system.
Computational Statistics & Data Analysis, 179, 107656.

Sugiyama, M., Suzuki, T., Nakajima, S. et al. Direct importance estimation for covariate shift adaptation.
Ann Inst Stat Math 60, 699–746 (2008). \doi{10.1007/s10463-008-0197-x}
}
\author{
Adam Struzik
}
