@article{zhu_limiting_2024,
	title = {Limiting distributions of graph-based test statistics on sparse and dense graphs},
	volume = {30},
	issn = {1350-7265},
	url = {https://projecteuclid.org/journals/bernoulli/volume-30/issue-1/Limiting-distributions-of-graph-based-test-statistics-on-sparse-and/10.3150/23-BEJ1616.full},
	doi = {10.3150/23-BEJ1616},
	abstract = {Two-sample tests utilizing a similarity graph on observations are useful for high-dimensional and non-Euclidean data due to their flexibility and good performance under a wide range of alternatives. Existing works mainly focused on sparse graphs, such as graphs with the number of edges in the order of the number of observations, and their asymptotic results imposed strong conditions on the graph that can easily be violated by commonly constructed graphs they suggested. Moreover, the graph-based tests have better performance with denser graphs under many settings. In this work, we establish the theoretical ground for graph-based tests with graphs ranging from those recommended in current literature to much denser ones.},
	number = {1},
	journal = {Bernoulli},
	author = {Zhu, Yejiong and Chen, Hao},
	month = feb,
	year = {2024},
	note = {Publisher: Bernoulli Society for Mathematical Statistics and Probability},
	keywords = {Dense graphs, graph-based methods, nonparametric two-sample tests, Stein’s method},
	pages = {770--796},
	file = {Eingereichte Version:C\:\\Users\\stolte\\Zotero\\storage\\NWNEU7X2\\Zhu und Chen - 2024 - Limiting distributions of graph-based test statist.pdf:application/pdf},
}


@Article{coin,
	title = {Implementing a class of permutation tests: The {coin}
	package},
	author = {Torsten Hothorn and Kurt Hornik and Mark A. {van de Wiel}
	and Achim Zeileis},
	journal = {Journal of Statistical Software},
	year = {2008},
	volume = {28},
	number = {8},
	pages = {1--23},
	doi = {10.18637/jss.v028.i08},
}

@Manual{GraphRankTest,
	title = {GraphRankTest: Rank in Similarity Graph Edge-Count Two-Sample Test (RISE)},
	author = {Doudou Zhou and Hao Chen},
	year = {2025},
	note = {R package version 0.1},
	url = {https://CRAN.R-project.org/package=GraphRankTest},
	doi = {10.32614/CRAN.package.GraphRankTest},
}

@Article{synthpop,
	title = {{synthpop}: Bespoke Creation of Synthetic Data in {R}},
	author = {Beata Nowok and Gillian M. Raab and Chris Dibben},
	journal = {Journal of Statistical Software},
	year = {2016},
	volume = {74},
	number = {11},
	pages = {1--26},
	doi = {10.18637/jss.v074.i11},
}

@Article{igraph,
	title = {The igraph software package for complex network research},
	author = {G{\'a}bor Cs{\'a}rdi and Tam{\'a}s Nepusz},
	journal = {InterJournal},
	volume = {Complex Systems},
	pages = {1695},
	year = {2006},
	url = {https://igraph.org},
}

@misc{stolte_empirical_num_2026,
	title={An Empirical Comparison of Methods for Quantifying the Similarity of Numeric Datasets}, 
	author={Marieke Stolte and Jörg Rahnenführer and Andrea Bommert},
	year={2026},
	archivePrefix={arXiv},
	primaryClass={stat.ME},
	url={https://arxiv.org/abs/2604.12327}, 
	doi={10.48550/arXiv.2604.12327},
}

@misc{stolte_empirical_cat_2026,
	title={An Empirical Comparison of Methods for Quantifying the Similarity of Categorical Datasets}, 
	author={Marieke Stolte and Jörg Rahnenführer and Andrea Bommert},
	year={2026},
	archivePrefix={arXiv},
	primaryClass={stat.ME},
	url={https://arxiv.org/abs/2604.11458}, 
	doi={10.48550/arXiv.2604.11458},
}

@InProceedings{zhou_new_2023,
	title = 	 {A new ranking scheme for modern data and its application to two-sample hypothesis testing},
	author =       {Zhou, Doudou and Chen, Hao},
	booktitle = 	 {Proceedings of Thirty Sixth Conference on Learning Theory},
	pages = 	 {3615--3668},
	year = 	 {2023},
	editor = 	 {Neu, Gergely and Rosasco, Lorenzo},
	volume = 	 {195},
	series = 	 {Proceedings of Machine Learning Research},
	month = 	 {12--15 Jul},
	publisher =    {PMLR},
	pdf = 	 {https://proceedings.mlr.press/v195/zhou23a/zhou23a.pdf},
	url = 	 {https://proceedings.mlr.press/v195/zhou23a.html},
	abstract = 	 {Rank-based approaches are among the most popular nonparametric methods for univariate data in tackling statistical problems such as hypothesis testing due to their robustness and effectiveness. However, they are unsatisfactory for more complex data. In the era of big data, high-dimensional and non-Euclidean data, such as networks and images, are ubiquitous and pose challenges for statistical analysis. Existing multivariate ranks such as component-wise, spatial, and depth-based ranks do not apply to non-Euclidean data and have limited performance for high-dimensional data. Instead of dealing with the ranks of observations, we propose two types of ranks applicable to complex data based on a similarity graph constructed on observations: a graph-induced rank defined by the inductive nature of the graph and an overall rank defined by the weight of edges in the graph. To illustrate their utilization, both the new ranks are used to construct test statistics for the two-sample hypothesis testing, which converge to the $\chi_2^2$ distribution under the permutation null distribution and some mild conditions of the ranks, enabling an easy type-I error control. Simulation studies show that the new method exhibits good power under a wide range of alternatives compared to existing methods. The new test is illustrated on the New York City taxi data for comparing travel patterns in consecutive months and a brain network dataset comparing male and female subjects.}
}


@misc{hagen_measuring_2020,
	title = {Measuring dataset similarity using optimal transport},
	url = {https://www.microsoft.com/en-us/research/blog/measuring-dataset-similarity-using-optimal-transport/},
	abstract = {Is FashionMNIST, a dataset of images of clothing items labeled by category, more similar to MNIST or to USPS, both of which are classification datasets of handwritten digits? This is a pretty hard question to answer, but the solution could have an impact on various aspects of machine learning. For example, it could change how […]},
	language = {en-US},
	urldate = {2025-03-14},
	journal = {Microsoft Research},
	author = {{Alvarez-Melis}, David and Fusi, Nicolo},
	month = sep,
	year = {2020},
}

@article{jeffreys_invariant_1997,
	title = {An {Invariant} {Form} for the {Prior} {Probability} in {Estimation} {Problems}},
	volume = {186},
	doi = {10.1098/rspa.1946.0056},
	abstract = {It is shown that a certain differential form depending on the values of the parameters in a law of chance is invariant for all transformations of the parameters when the law is differentiable with regard to all parameters. For laws containing a location and a scale parameter a form with a somewhat restricted type of invariance is found even when the law is not everywhere differentiable with regard to the parameters. This form has the properties required to give a general rule for stating the prior probability in a large class of estimation problems.},
	number = {1007},
	journal = {Proceedings of the Royal Society of London. Series A. Mathematical and Physical Sciences},
	author = {Jeffreys, Harold},
	month = jan,
	year = {1997},
	publisher = {Royal Society},
	pages = {453--461},
}

@article{vaserstein_markov_1969,
	title = {Markov {Processes} {Over} {Denumerable} {Products} of {Spaces}, {Describing} {Large} {Systems} of {Automata}},
	volume = {5},
	number = {3},
	journal = {Problemy Peredachi Informatsii},
	author = {Vaserstein, Leonid Nisonovich},
	year = {1969},
	publisher = {Russian Academy of Sciences, Branch of Informatics, Computer Equipment and},
	pages = {64--72},
}

@Manual{DataSimilarity,
	title = {DataSimilarity: Quantifying Similarity of Datasets and Multivariate Two- And
	k-Sample Testing},
	author = {Marieke Stolte and Luca Sauer},
	year = {2025},
	note = {R package version 0.2.0},
	url = {https://CRAN.R-project.org/package=DataSimilarity},
}


@article{chu_asymptotic_2019,
	title = {Asymptotic {Distribution}-{Free} {Change}-{Point} {Detection} for {Multivariate} and {Non}-{Euclidean} {Data}},
	volume = {47},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/18-AOS1691},
	abstract = {We consider the testing and estimation of change-points, locations where the distribution abruptly changes, in a sequence of multivariate or non-Euclidean observations. We study a nonparametric framework that utilizes similarity information among observations, which can be applied to various data types as long as an informative similarity measure on the sample space can be defined. The existing approach along this line has low power and/or biased estimates for change-points under some common scenarios. We address these problems by considering new tests based on similarity information. Simulation studies show that the new approaches exhibit substantial improvements in detecting and estimating change-points. In addition, under some mild conditions, the new test statistics are asymptotically distribution-free under the null hypothesis of no change. Analytic \$p\$-value approximations to the significance of the new test statistics for the single change-point alternative and changed interval alternative are derived, making the new approaches easy off-the-shelf tools for large datasets. The new approaches are illustrated in an analysis of New York taxi data.},
	number = {1},
	urldate = {2024-12-05},
	journal = {The Annals of Statistics},
	author = {Chu, Lynna and Chen, Hao},
	month = feb,
	year = {2019},
	publisher = {Institute of Mathematical Statistics},
	keywords = {60K35, 62G32, Change-point, graph-based tests, High-dimensional data, network data, non-Euclidean data, nonparametric, scan statistic, tail probability},
	pages = {382--414},
}


@article{marshall_bayesian_2006,
	title = {Bayesian evidence as a tool for comparing datasets},
	volume = {73},
	doi = {10.1103/PhysRevD.73.067302},
	abstract = {We introduce a new conservative test for quantifying the consistency of two or more datasets. The test is based on the Bayesian answer to the question, “How much more probable is it that all my data were generated from the same model system than if each dataset were generated from an independent set of model parameters?” We make explicit the connection between evidence ratios and the differences in peak chi-squared values, the latter of which are more widely used and more cheaply calculated. Calculating evidence ratios for three cosmological datasets [recent cosmic microwave background data (WMAP, ACBAR, CBI, VSA), SDSS galaxy redshift survey, and the most recent SNe type 1A data] we find that concordance is favored and the tightening of constraints on cosmological parameters is indeed justified.},
	number = {6},
	urldate = {2022-05-27},
	journal = {Physical Review D},
	author = {Marshall, Phil and Rajguru, Nutan and Slosar, Anže},
	month = mar,
	year = {2006},
	publisher = {American Physical Society},
	pages = {067302},
}

@inproceedings{maillet_commet_2014,
	title = {Commet: {Comparing} and combining multiple metagenomic datasets},
	shorttitle = {Commet},
	doi = {10.1109/BIBM.2014.6999135},
	abstract = {Metagenomics offers a way to analyze biotopes at the genomic level and to reach functional and taxonomical conclusions. The bio-analyzes of large metagenomic projects face critical limitations: complex metagenomes cannot be assembled and the taxonomical or functional annotations are much smaller than the real biological diversity. This motivated the development of de novo metagenomic read comparison approaches to extract information contained in metagenomic datasets. However, these new approaches do not scale up large metagenomic projects, or generate an important number of large intermediate and result files. We introduce Commet (“COmpare Multiple METagenomes”), a method that provides similarity overview between all datasets of large metagenomic projects. Directly from non-assembled reads, all against all comparisons are performed through an efficient indexing strategy. Then, results are stored as bit vectors, a compressed representation of read files, that can be used to further combine read subsets by common logical operations. Finally, Commet computes a clusterization of metagenomic datasets, which is visualized by dendrogram and heatmaps.},
	booktitle = {2014 {IEEE} {International} {Conference} on {Bioinformatics} and {Biomedicine} ({BIBM})},
	author = {Maillet, Nicolas and Collet, Guillaume and Vannier, Thomas and Lavenier, Dominique and Peterlongo, Pierre},
	month = nov,
	year = {2014},
	keywords = {Bioinformatics, Genomics, Heating, Indexes, Soil, Vectors},
	pages = {94--98},
}

@inproceedings{joshi_comparing_2011,
	address = {New York, NY, USA},
	series = {{SoCG} '11},
	title = {Comparing distributions and shapes using the kernel distance},
	isbn = {978-1-4503-0682-9},
	doi = {10.1145/1998196.1998204},
	abstract = {Starting with a similarity function between objects, it is possible to define a distance metric (the kernel distance) on pairs of objects, and more generally on probability distributions over them. These distance metrics have a deep basis in functional analysis and geometric measure theory, and have a rich structure that includes an isometric embedding into a Hilbert space. They have recently been applied to numerous problems in machine learning and shape analysis. SIn this paper, we provide the first algorithmic analysis of these distance metrics. Our main contributions are as follows: We present fast approximation algorithms for computing the kernel distance between two point sets P and Q that runs in near-linear time in the size of P ∪ Q (an explicit calculation would take quadratic time). We present polynomial-time algorithms for approximately minimizing the kernel distance under rigid transformation; they run in time O(n + poly(1/ε, log n)). We provide several general techniques for reducing complex objects to convenient sparse representations (specifically to point sets or sets of points sets) which approximately preserve the kernel distance. In particular, this allows us to reduce problems of computing the kernel distance between various types of objects such as curves, surfaces, and distributions to computing the kernel distance between point sets.},
	urldate = {2022-05-27},
	booktitle = {Proceedings of the twenty-seventh annual symposium on {Computational} geometry},
	publisher = {Association for Computing Machinery},
	author = {Joshi, Sarang and Kommaraji, Raj Varma and Phillips, Jeff M. and Venkatasubramanian, Suresh},
	month = jun,
	year = {2011},
	keywords = {distributions, kernel distance, metrics, shape matching},
	pages = {47--56},
}

@inproceedings{smola_hilbert_2007,
	address = {Berlin, Heidelberg},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {A {Hilbert} {Space} {Embedding} for {Distributions}},
	isbn = {978-3-540-75225-7},
	doi = {10.1007/978-3-540-75225-7_5},
	abstract = {We describe a technique for comparing distributions without the need for density estimation as an intermediate step. Our approach relies on mapping the distributions into a reproducing kernel Hilbert space. Applications of this technique can be found in two-sample tests, which are used for determining whether two sets of observations arise from the same distribution, covariate shift correction, local learning, measures of independence, and density estimation.},
	booktitle = {Algorithmic {Learning} {Theory}},
	publisher = {Springer},
	author = {Smola, Alex and Gretton, Arthur and Song, Le and Schölkopf, Bernhard},
	editor = {Hutter, Marcus and Servedio, Rocco A. and Takimoto, Eiji},
	year = {2007},
	keywords = {Exponential Family, Hilbert Space, Independent Component Analysis, Kernel Method, Maximal Clique},
	pages = {13--31},
}

@inproceedings{zhao_comparing_2021,
	title = {Comparing {Distributions} by {Measuring} {Differences} that {Affect} {Decision} {Making}},
	abstract = {Measuring the discrepancy between two probability distributions is a fundamental problem in machine learning and statistics. We propose a new class of discrepancies based on the optimal loss for a...},
	urldate = {2022-05-27},
	author = {Zhao, Shengjia and Sinha, Abhishek and He, Yutong and Perreault, Aidan and Song, Jiaming and Ermon, Stefano},
	month = sep,
	year = {2021},
}

@incollection{joyce_kullback-leibler_2011,
	address = {Berlin, Heidelberg},
	title = {Kullback-{Leibler} {Divergence}},
	isbn = {978-3-642-04898-2},
	urldate = {2022-05-27},
	booktitle = {International {Encyclopedia} of {Statistical} {Science}},
	publisher = {Springer},
	author = {Joyce, James M.},
	editor = {Lovric, Miodrag},
	year = {2011},
	doi = {10.1007/978-3-642-04898-2_327},
	pages = {720--722},
}

@inproceedings{ntoutsi_general_2008,
	title = {A {General} {Framework} for {Estimating} {Similarity} of {Datasets} and {Decision} {Trees}: {Exploring} {Semantic} {Similarity} of {Decision} {Trees}},
	isbn = {978-0-89871-654-2},
	shorttitle = { for estimating similarity of datasets and decision trees},
	abstract = {We study time-series classification (TSC), a fundamental task of time-series data mining. Prior work has approached TSC from two major directions: (1) similarity-based methods that classify time-series based on the nearest neighbors, and (2) deep learning models that directly learn the representations for classification in a data-driven manner. Motivated by the different working mechanisms within these two research lines, we aim to connect them in such a way as to jointly model time-series similarities and learn the representations. This is a challenging task because it is unclear how we should efficiently leverage similarity information. To tackle the challenge, we propose Similarity-Aware Time-Series Classification (SimTSC), a conceptually simple and general framework that models similarity information with graph neural networks (GNNs). Specifically, we formulate TSC as a node classification problem in graphs, where the nodes correspond to time-series, and the links correspond to pair-wise similarities. We further design a graph construction strategy and a batch training algorithm with negative sampling to improve training efficiency. We instantiate SimTSC with ResNet as the backbone and Dynamic Time Warping (DTW) as the similarity measure. Extensive experiments on the full UCR datasets and several multivariate datasets demonstrate the effectiveness of incorporating similarity information into deep learning models in both supervised and semi-supervised settings. Our code is available at https://github.com/daochenzha/SimTSC.},
	booktitle = {Proceedings of the 2008 {SIAM} {International} {Conference} on {Data} {Mining} ({SDM})},
	publisher = {Society for Industrial and Applied Mathematics},
	author = {Ntoutsi, Irene and Kalousis, Alexandros and Theodoridis, Yannis},
	month = apr,
	year = {2008},
	doi = {10.1137/1.9781611972788.73},
	pages = {810--821},
}

@inproceedings{parthasarathy_exploiting_2000,
	address = {Berlin, Heidelberg},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {Exploiting {Dataset} {Similarity} for {Distributed} {Mining}},
	isbn = {978-3-540-45591-2},
	doi = {10.1007/3-540-45591-4_52},
	abstract = {The notion of similarity is an important one in data mining. It can be used to pro vide useful structural information on data as well as enable clustering. In this paper we present an elegant method for measuring the similarity between homogeneous datasets. The algorithm presented is efficient in storage and scale, has the ability to adjust to time constraints. and can provide the user with likely causes of similarity or dis-similarity.},
	language = {en},
	booktitle = {Parallel and {Distributed} {Processing}},
	publisher = {Springer},
	author = {Parthasarathy, Srinivasan and Ogihara, Mitsunori},
	editor = {Rolim, José},
	year = {2000},
	keywords = {Association Rule, Communication Overhead, Minimum Support, Similarity Measure, Synthetic Dataset},
	pages = {399--406},
}

@inproceedings{leite_exploiting_2021,
	title = {Exploiting {Performance}-based {Similarity} between {Datasets} in {Metalearning}},
	abstract = {This paper describes an improved algorithm selection method of a previous method called active testing. This method seeks a workflow (or its particular configuration) that would lead to the highest gain in performance (e.g., accuracy). The new version uses a particular performance-based characterization of each dataset, which is in the form of a vector of performance values of different algorithms. Dataset similarity is then assessed by comparing these performance vectors. One useful measure for this comparison is Spearman’s correlation. The advantage of this measure is that it can be easily recalculated as more information is gathered. Consequently, as the tests proceed, the recommendations of the system get adjusted to the characteristics of the target dataset. We show that this new strategy leads to improved results of the active testing approach.},
	booktitle = {{AAAI} {Workshop} on {Meta}-{Learning} and {MetaDL} {Challenge}},
	publisher = {PMLR},
	author = {Leite, Rui and Brazdil, Pavel},
	month = aug,
	year = {2021},
	issn = {2640-3498},
	pages = {90--99},
}

@inproceedings{leite_selecting_2012,
	address = {Berlin, Heidelberg},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {Selecting {Classification} {Algorithms} with {Active} {Testing}},
	isbn = {978-3-642-31537-4},
	doi = {10.1007/978-3-642-31537-4_10},
	abstract = {Given the large amount of data mining algorithms, their combinations (e.g. ensembles) and possible parameter settings, finding the most adequate method to analyze a new dataset becomes an ever more challenging task. This is because in many cases testing all possibly useful alternatives quickly becomes prohibitively expensive. In this paper we propose a novel technique, called active testing, that intelligently selects the most useful cross-validation tests. It proceeds in a tournament-style fashion, in each round selecting and testing the algorithm that is most likely to outperform the best algorithm of the previous round on the new dataset. This ‘most promising’ competitor is chosen based on a history of prior duels between both algorithms on similar datasets. Each new cross-validation test will contribute information to a better estimate of dataset similarity, and thus better predict which algorithms are most promising on the new dataset. We have evaluated this approach using a set of 292 algorithm-parameter combinations on 76 UCI datasets for classification. The results show that active testing will quickly yield an algorithm whose performance is very close to the optimum, after relatively few tests. It also provides a better solution than previously proposed methods.},
	booktitle = {Machine {Learning} and {Data} {Mining} in {Pattern} {Recognition}},
	publisher = {Springer},
	author = {Leite, Rui and Brazdil, Pavel and Vanschoren, Joaquin},
	editor = {Perner, Petra},
	year = {2012},
	pages = {117--131},
}

@article{kailath_divergence_1967,
	title = {The {Divergence} and {Bhattacharyya} {Distance} {Measures} in {Signal} {Selection}},
	volume = {15},
	issn = {2162-2175},
	doi = {10.1109/TCOM.1967.1089532},
	abstract = {Minimization of the error probability to determine optimum signals is often difficult to carry out. Consequently, several suboptimum performance measures that are easier than the error probability to evaluate and manipulate have been studied. In this partly tutorial paper, we compare the properties of an often used measure, the divergence, with a new measure that we have called the Bhattacharyya distance. This new distance measure is often easier to evaluate than the divergence. In the problems we have worked, it gives results that are at least as good as, and are often better, than those given by the divergence.},
	number = {1},
	journal = {IEEE Transactions on Communication Technology},
	author = {Kailath, T.},
	month = feb,
	year = {1967},
	keywords = {Error probability, Fading, History, Information theory, Matched filters, Radar detection, Radar theory, Signal detection, Signal to noise ratio, Statistics},
	pages = {52--60},
}

@misc{noauthor_jensen-bregman_nodate,
	title = {Jensen-{Bregman} {LogDet} {Divergence} with {Application} to {Efficient} {Similarity} {Search} for {Covariance} {Matrices}},
	abstract = {Covariance matrices have found success in several computer vision applications, including activity recognition, visual surveillance, and diffusion tensor imaging. This is because they provide an easy platform for fusing multiple features compactly. An important task in all of these applications is to compare two covariance matrices using a (dis)similarity function, for which the common choice is the Riemannian metric on the manifold inhabited by these matrices. As this Riemannian manifold is not flat, the dissimilarities should take into account the curvature of the manifold. As a result, such distance computations tend to slow down, especially when the matrix dimensions are large or gradients are required. Further, suitability of the metric to enable efficient nearest neighbor retrieval is an important requirement in the contemporary times of big data analytics. To alleviate these difficulties, this paper proposes a novel dissimilarity measure for covariances, the Jensen-Bregman LogDet Divergence (JBLD). This divergence enjoys several desirable theoretical properties and at the same time is computationally less demanding (compared to standard measures). Utilizing the fact that the square root of JBLD is a metric, we address the problem of efficient nearest neighbor retrieval on large covariance datasets via a metric tree data structure. To this end, we propose a K-Means clustering algorithm on JBLD. We demonstrate the superior performance of JBLD on covariance datasets from several computer vision applications.},
}

@article{guo_novel_2021,
	title = {A novel similarity metric with application to big process data analytics},
	volume = {113},
	issn = {0967-0661},
	doi = {10.1016/j.conengprac.2021.104843},
	abstract = {Establishing a quantitative similarity between different datasets has gained prevalence and significance in many applications of process control. In industrial practice, process data are usually multi-dimensional, nonlinearly correlated, and with unknown time-varying distribution, which raise immense challenge for reasonably evaluating similarity. To address this issue, a novel similarity metric based on deep autoencoder (DAE) and the Wasserstein distance is proposed in this paper. Specifically, DAE is used to first capture nonlinear relationship embedded in multivariate process data, and the reconstruction error acts as an indicator to reveal discrepancy between two datasets. After that, the similarity is characterized by evaluating the gap between reconstruction error distributions using Wasserstein distance. The proposed similarity metric has wide applicability in a variety of data analytics tasks including pattern matching, fault diagnosis and mode classifications. Both simulated data and industrial data collected from a real iron-making process are utilized to carry out comprehensive case studies. It is shown that the proposed similarity metric not only enjoys better rationality and sensitivity than generic similarity metrics, but also effectively improves the accuracy of fault diagnosis and mode classification based on big process data.},
	journal = {Control Engineering Practice},
	author = {Guo, Zijian and Shang, Chao and Ye, Hao},
	month = aug,
	year = {2021},
	keywords = {Big process data, Deep autoencoder, Similarity metric, Unsupervised mode classification, Wasserstein distance},
	pages = {104843},
}

@article{antao_kolmogorov_2018,
	title = {Kolmogorov complexity as a data similarity metric: application in mitochondrial {DNA}},
	volume = {93},
	issn = {1573-269X},
	shorttitle = {Kolmogorov complexity as a data similarity metric},
	doi = {10.1007/s11071-018-4245-7},
	abstract = {The problem of developing a similarity index for different objects is discussed. The limitations of current metrics are evaluated and discussed. The normalized compression distance, based on the non-computable Kolmogorov complexity, is examined and compared with two alternative measures. A case study consisting of a phylogenetic tree of different mammals is constructed applying this technique with a mitochondrial DNA database.},
	number = {3},
	urldate = {2022-06-07},
	journal = {Nonlinear Dynamics},
	author = {Antão, Rómulo and Mota, Alexandre and Machado, J. A. Tenreiro},
	month = aug,
	year = {2018},
	keywords = {Kolmogorov complexity, Mitochondrial DNA, Normalized compression distance},
	pages = {1059--1071},
}

@article{shao_dynamic_2021,
	title = {A dynamic {CNN} pruning method based on matrix similarity},
	volume = {15},
	issn = {1863-1711},
	doi = {10.1007/s11760-020-01760-x},
	abstract = {Network pruning is one of the predominant approaches for deep model compression. Pruning large neural networks while maintaining their performance is often desirable because space and time complexity are reduced. Current pruning methods mainly focus on the importance of filters in the whole task. Different from previous methods, this paper focuses on the similarity between the filters or feature maps of the same layer. Firstly, cosine similarity is used as the matrix similarity measure to measure the similarity between channels, guiding the network to prune. Secondly, the proposed method is, respectively, applied to filters and feature maps pruning, and the pruning effects in different layers are summarized. Finally, we propose a method to set the pruning rate dynamically according to the situation of each layer. Our method obtains extremely sparse networks with virtually the same accuracy as the reference networks on the CIFAR-10 and ImageNet ILSVRC-12 classification tasks. On CIFAR-10, our network achieves the 52.70\% compression ratio on ResNet-56 and increases only 0.13\% on top-1 error.},
	number = {2},
	urldate = {2022-06-07},
	journal = {Signal, Image and Video Processing},
	author = {Shao, Mingwen and Dai, Junhui and Kuang, Jiandong and Meng, Deyu},
	month = mar,
	year = {2021},
	keywords = {Feature map, Filter, Model compression, Network pruning, Similarity measure},
	pages = {381--389},
}

@article{calderon_ramirez_dataset_2022,
	title = {Dataset {Similarity} to {Assess} {Semi}-supervised {Learning} {Under} {Distribution} {Mismatch} {Between} the {Labelled} and {Unlabelled} {Datasets}},
	issn = {2691-4581},
	doi = {10.1109/TAI.2022.3168804},
	abstract = {Semi-supervised deep learning (SSDL) is a popular strategy to leverage unlabelled data for machine learning when labelled data is not readily available. In real-world scenarios, different unlabelled data sources are usually available, with varying degrees of distribution mismatch regarding the labelled datasets. It begs the question which unlabelled dataset to choose for good SSDL outcomes. ftentimes, semantic heuristics are used to match unlabelled data with labelled data. However, a quantitative and systematic approach to this election problem would be preferable. In this work, we first test the SSDL MixMatch algorithm under various distribution mismatch configurations to study the impact on SSDL accuracy. Then, we propose a quantitative unlabelled dataset selection heuristic based on dataset dissimilarity measures. These are designed to systematically assess how distribution mismatch between the labelled and unlabelled datasets affects MixMatch performance. We refer to our proposed method as deep dataset dissimilarity measures (DeDiMs), designed to compare labelled and unlabelled datasets. They use the feature space of a generic Wide-ResNet, can be applied prior to learning, are quick to evaluate and model agnostic. The strong correlation in our tests between MixMatch accuracy and the proposed DeDiMs suggests that this approach can be a good fit for quantitatively ranking different unlabelled datasets prior to SSDL training.},
	journal = {IEEE Transactions on Artificial Intelligence},
	author = {Calderon Ramirez, Saul and Oala, Luis and Torrentes-Barrena, Jordina and Yang, Shengxiang and Elizondo, David and Moemeni, Armaghan and Colreavy-Donnelly, Simon and Samek, Wojciech and Molina-Cabello, Miguel and Lopez-Rubio, Ezequiel},
	year = {2022},
	keywords = {Artificial intelligence, Data models, Deep learning, Feature extraction, Semantics, Semisupervised learning, Training},
	pages = {1--1},
}

@techreport{tschopp_quantifying_2017,
	title = {Quantifying {Similarity} and {Distance} {Measures} for {Vector}-{Based} {Datasets}: {Histograms}, {Signals}, and {Probability} {Distribution} {Functions}},
	shorttitle = {Quantifying {Similarity} and {Distance} {Measures} for {Vector}-{Based} {Datasets}},
	abstract = {It is often important to characterize the similarity or dissimilarity distance between different measured or computed datasets. There are a large number of different possible similarity and distance measures that can be applied to different datasets. In this technical note, a number of different measures implemented in both MATLAB and Python as functions are used to quantify similaritydistance between 2 vector-based datasets. The scripts are attached as appendixes as is a description of their execution.},
	institution = {US Army Research Laboratory Aberdeen Proving Ground United States},
	author = {Tschopp, Mark A. and Hernandez-Rivera, Efran},
	month = feb,
	year = {2017},
	note = {Section: Technical Reports},
}

@article{krzanowski_non-parametric_2003,
	title = {Non-parametric estimation of distance between groups},
	volume = {30},
	doi = {10.1080/0266476032000076029},
	abstract = {A numerical procedure is outlined for obtaining the distance between samples from two populations. First, the probability densities in the two populations are estimated by kernel methods, and then the distance is derived by numerical integration of a suitable function of these densities. Various such functions have been proposed in the past; they are all implemented and compared with each other and with Mahalanobis D 2 on several real and simulated data sets. The results show the method to be viable, and to perform well against the Mahalanobis D 2 standard.},
	journal = {Journal of Applied Statistics},
	author = {Krzanowski, W.},
	month = aug,
	year = {2003},
	pages = {743--750},
}

@article{feurer_initializing_2015,
	title = {Initializing {Bayesian} {Hyperparameter} {Optimization} via {Meta}-{Learning}},
	volume = {29},
	issn = {2374-3468},
	doi = {10.1609/aaai.v29i1.9354},
	abstract = {Model selection and hyperparameter optimization is crucial in applying machine learning to a novel dataset. Recently, a subcommunity of machine learning has focused on solving this problem with Sequential Model-based Bayesian Optimization (SMBO), demonstrating substantial successes in many applications. However, for computationally expensive algorithms the overhead of hyperparameter optimization can still be prohibitive. In this paper we mimic a strategy human domain experts use: speed up optimization by starting from promising configurations that performed well on similar datasets. The resulting initialization technique integrates naturally into the generic SMBO framework and can be trivially applied to any SMBO method. To validate our approach, we perform extensive experiments with two established SMBO frameworks (Spearmint and SMAC) with complementary strengths; optimizing two machine learning frameworks on 57 datasets. Our initialization procedure yields mild improvements for low-dimensional hyperparameter optimization and substantially improves the state of the art for the more complex combined algorithm selection and hyperparameter optimization problem.},
	language = {en},
	number = {1},
	urldate = {2022-06-07},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Feurer, Matthias and Springenberg, Jost and Hutter, Frank},
	month = feb,
	year = {2015},
	number = {1},
	keywords = {Sequential Model-based Optimization},
}

@article{shyi-ming_chen_comparison_1995,
	title = {A comparison of similarity measures of fuzzy values},
	volume = {72},
	issn = {0165-0114},
	doi = {10.1016/0165-0114(94)00284-E},
	abstract = {This paper extends the work of Pappis and Karacapilidis (1993) to present and compare the properities of several measures of similarity of fuzzy values. The measures examined in this paper are based on the geometric model, the set-theoretic approach, and the matching function S we presented in (Chen, 1988). It is shown that several properties are common to all measures and some properties do not hold for all of them.},
	language = {en},
	number = {1},
	urldate = {2022-06-07},
	journal = {Fuzzy Sets and Systems},
	author = {{Shyi-Ming Chen} and {Ming-Shiow Yeh} and {Pei-Yung Hsiao}},
	month = may,
	year = {1995},
	keywords = {Similarity measure, Fuzzy value, Matching function},
	pages = {79--89},
}

@inproceedings{gretton_kernel_2006,
	title = {A {Kernel} {Method} for the {Two}-{Sample}-{Problem}},
	volume = {19},
	abstract = {We propose two statistical tests to determine if two samples are from different dis- tributions. Our test statistic is in both cases the distance between the means of the two samples mapped into a reproducing kernel Hilbert space (RKHS). The ﬁrst test is based on a large deviation bound for the test statistic, while the second is based on the asymptotic distribution of this statistic. The test statistic can be com- puted in O(m2) time. We apply our approach to a variety of problems, including attribute matching for databases using the Hungarian marriage method, where our test performs strongly. We also demonstrate excellent performance when compar- ing distributions over graphs, for which no alternative tests currently exist.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {MIT Press},
	author = {Gretton, Arthur and Borgwardt, Karsten and Rasch, Malte and Schölkopf, Bernhard and Smola, Alex},
	year = {2006},
}

@article{friedman_multivariate_1979,
	title = {Multivariate {Generalizations} of the {Wald}-{Wolfowitz} and {Smirnov} {Two}-{Sample} {Tests}},
	volume = {7},
	issn = {0090-5364},
	abstract = {Multivariate generalizations of the Wald-Wolfowitz runs statistic and the Smirnov maximum deviation statistic for the two-sample problem are presented. They are based on the minimal spanning tree of the pooled sample points. Some null distribution results are derived and a simulation study of power is reported.},
	number = {4},
	urldate = {2022-06-28},
	journal = {The Annals of Statistics},
	author = {Friedman, Jerome H. and Rafsky, Lawrence C.},
	year = {1979},
	publisher = {Institute of Mathematical Statistics},
	pages = {697--717},
}

@article{bickel_distribution_1969,
	title = {A {Distribution} {Free} {Version} of the {Smirnov} {Two} {Sample} {Test} in the p-{Variate} {Case}},
	volume = {40},
	issn = {0003-4851},
	number = {1},
	urldate = {2022-06-28},
	journal = {The Annals of Mathematical Statistics},
	author = {Bickel, P. J.},
	year = {1969},
	publisher = {Institute of Mathematical Statistics},
	pages = {1--23},
}

@article{henze_multivariate_1999,
	title = {On the {Multivariate} {Runs} {Test}},
	volume = {27},
	issn = {0090-5364},
	abstract = {For independent d-variate random variables X1,... ,Xm with common density f and Y1,... ,Yn with common density g, let Rm,n be the number of edges in the minimal spanning tree with vertices X1,... ,Xm, Y1,... ,Yn that connect points from different samples. Friedman and Rafsky conjectured that a test of H0: f = g that rejects H0 for small values of Rm,n should have power against general alternatives. We prove that Rm,n is asymptotically distribution-free under H0, and that the multivariate two-sample test based on Rm,n is universally consistent.},
	number = {1},
	urldate = {2022-06-28},
	journal = {The Annals of Statistics},
	author = {Henze, Norbert and Penrose, Mathew D.},
	year = {1999},
	publisher = {Institute of Mathematical Statistics},
	pages = {290--298},
}

@article{rosenbaum_exact_2005,
	title = {An {Exact} {Distribution}-{Free} {Test} {Comparing} {Two} {Multivariate} {Distributions} {Based} on {Adjacency}},
	volume = {67},
	issn = {1369-7412},
	abstract = {A new test is proposed comparing two multivariate distributions by using distances between observations. Unlike earlier tests using interpoint distances, the new test statistic has a known exact distribution and is exactly distribution free. The interpoint distances are used to construct an optimal non-bipartite matching, i.e. a matching of the observations into disjoint pairs to minimize the total distance within pairs. The cross-match statistic is the number of pairs containing one observation from the first distribution and one from the second. Distributions that are very different will exhibit few cross-matches. When comparing two discrete distributions with finite support, the test is consistent against all alternatives. The test is applied to a study of brain activation measured by functional magnetic resonance imaging during two linguistic tasks, comparing brains that are impaired by arteriovenous abnormalities with normal controls. A second exact distribution-free test is also discussed: it ranks the pairs and sums the ranks of the cross-matched pairs.},
	number = {4},
	journal = {Journal of the Royal Statistical Society B},
	author = {Rosenbaum, Paul R.},
	year = {2005},
	publisher = {Royal Statistical Society, Wiley},
	pages = {515--530},
}

@article{hall_permutation_2002,
	title = {Permutation {Tests} for {Equality} of {Distributions} in {High}-{Dimensional} {Settings}},
	volume = {89},
	issn = {0006-3444},
	abstract = {Motivated by applications in high-dimensional settings, we suggest a test of the hypothesis H0 that two sampled distributions are identical. It is assumed that two independent datasets are drawn from the respective populations, which may be very general. In particular, the distributions may be multivariate or infinite-dimensional, in the latter case representing, for example, the distributions of random functions from one Euclidean space to another. Our test uses a measure of distance between data. This measure should be symmetric but need not satisfy the triangle inequality, so it is not essential that it be a metric. The test is based on ranking the pooled dataset, with respect to the distance and relative to any fixed data value, and repeating this operation for each fixed datum. A permutation argument enables a critical point to be chosen such that the test has concisely known significance level, conditional on the set of all pairwise distances.},
	number = {2},
	journal = {Biometrika},
	author = {Hall, Peter and Tajvidi, Nader},
	year = {2002},
	publisher = {Oxford University Press, Biometrika Trust},
	pages = {359--374},
}

@article{anderson_two-sample_1994,
	title = {Two-{Sample} {Test} {Statistics} for {Measuring} {Discrepancies} {Between} {Two} {Multivariate} {Probability} {Density} {Functions} {Using} {Kernel}-{Based} {Density} {Estimates}},
	volume = {50},
	issn = {0047-259X},
	doi = {10.1006/jmva.1994.1033},
	abstract = {Test statistics are proposed for testing equality of two p-variate probability density functions. The statistics are based on the integrated square distance between two kernel-based density estimates and are two-sample versions of the statistic studied by Hall (1984, J. Multivariate Anal. 14 1-16). Particular emphasis is laid on the case where the two bandwidths are fixed and equal. Asymptotic distributional results and power calculations are supplemented by an empirical study based on univariate examples.},
	language = {en},
	number = {1},
	urldate = {2022-06-28},
	journal = {Journal of Multivariate Analysis},
	author = {Anderson, N. H. and Hall, P. and Titterington, D. M.},
	month = jul,
	year = {1994},
	pages = {41--54},
}

@article{biau_asymptotic_2005,
	title = {On the {Asymptotic} {Properties} of a {Nonparametric} {$L_1$}-{Test} {Statistic} of {Homogeneity}},
	volume = {51},
	issn = {1557-9654},
	doi = {10.1109/TIT.2005.856979},
	abstract = {We present two simple and explicit procedures for testing homogeneity of two independent multivariate samples of size n. The nonparametric tests are based on the statistic T/sub n/, which is the L/sub 1/ distance between the two empirical distributions restricted to a finite partition. Both tests reject the null hypothesis of homogeneity if T/sub n/ becomes large, i.e., if T/sub n/ exceeds a threshold. We first discuss Chernoff-type large deviation properties of T/sub n/. This results in a distribution-free strong consistent test of homogeneity. Then the asymptotic null distribution of the test statistic is obtained, leading to an asymptotically /spl alpha/-level test procedure.},
	number = {11},
	journal = {IEEE Transactions on Information Theory},
	author = {Biau, G. and Gyorfi, L.},
	month = nov,
	year = {2005},
	keywords = {Probability, Analysis of variance, Central limit theorem, consistent testing, Electrocardiography, Electroencephalography, homogeneity testing, large deviations, Parametric statistics, partitions, poissonization, Size control, Speech analysis, Statistical analysis, Statistical distributions, Testing},
	pages = {3965--3973},
}

@article{schilling_multivariate_1986,
	title = {Multivariate {Two}-{Sample} {Tests} {Based} on {Nearest} {Neighbors}},
	volume = {81},
	issn = {0162-1459},
	doi = {10.2307/2289012},
	abstract = {A new class of simple tests is proposed for the general multivariate two-sample problem based on the (possibly weighted) proportion of all k nearest neighbor comparisons in which observations and their neighbors belong to the same sample. Large values of the test statistics give evidence against the hypothesis H of equality of the two underlying distributions. Asymptotic null distributions are explicitly determined and shown to involve certain nearest neighbor interaction probabilities. Simple infinite-dimensional approximations are supplied. The unweighted version yields a distribution-free test that is consistent against all alternatives; optimally weighted statistics are also obtained and asymptotic efficiencies are calculated. Each of the tests considered is easily adapted to a permutation procedure that conditions on the pooled sample. Power performance for finite sample sizes is assessed in simulations.},
	number = {395},
	urldate = {2022-06-28},
	journal = {Journal of the American Statistical Association},
	author = {Schilling, Mark F.},
	year = {1986},
	publisher = {American Statistical Association, Taylor \& Francis, Ltd.},
	pages = {799--806},
}

@article{henze_multivariate_1988,
	title = {A {Multivariate} {Two}-{Sample} {Test} {Based} on the {Number} of {Nearest} {Neighbor} {Type} {Coincidences}},
	volume = {16},
	issn = {0090-5364},
	abstract = {For independent d-variate random samples X1, ⋯, Xn1 i.i.d. f(x), Y1, ⋯, Yn2 i.i.d. g(x), where the densities f and g are assumed to be continuous a.e., consider the number T of all k nearest neighbor comparisons in which observations and their neighbors belong to the same sample. We show that, if f = g a.e., the limiting (normal) distribution of T, as \${\textbackslash}min(n\_1, n\_2) {\textbackslash}rightarrow {\textbackslash}infty, n\_1/(n\_1 + n\_2) {\textbackslash}rightarrow {\textbackslash}tau, 0 {\textless} {\textbackslash}tau {\textless} 1\$, does not depend on f. An omnibus procedure for testing the hypothesis H0: f = g a.e. is obtained by rejecting H0 for large values of T. The result applies to a general distance (generated by a norm on Rd) for determining nearest neighbors, and it generalizes to the multisample situation.},
	number = {2},
	urldate = {2022-06-28},
	journal = {The Annals of Statistics},
	author = {Henze, Norbert},
	year = {1988},
	publisher = {Institute of Mathematical Statistics},
	pages = {772--783},
}

@article{gretton_kernel_2012,
	title = {A {Kernel} {Two}-{Sample} {Test}},
	volume = {13},
	urldate = {2022-06-28},
	journal = {Journal of Machine Learning Research},
	author = {Gretton, A. and Borgwardt, K. and Rasch, M. and Schölkopf, B. and Smola, A.},
	month = mar,
	year = {2012},
	pages = {723--773},
}

@inproceedings{gretton_optimal_2012,
	title = {Optimal kernel choice for large-scale two-sample tests},
	volume = {25},
	urldate = {2022-06-28},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Gretton, Arthur and Sejdinovic, Dino and Strathmann, Heiko and Balakrishnan, Sivaraman and Pontil, Massimiliano and Fukumizu, Kenji and Sriperumbudur, Bharath K.},
	year = {2012},
}

@article{muandet_kernel_2017,
	title = {Kernel {Mean} {Embedding} of {Distributions}: {A} {Review} and {Beyond}},
	volume = {10},
	issn = {1935-8237, 1935-8245},
	shorttitle = {Kernel {Mean} {Embedding} of {Distributions}},
	doi = {10.1561/2200000060},
	abstract = {Kernel Mean Embedding of Distributions: A Review and Beyond},
	number = {1-2},
	journal = {Foundations and Trends® in Machine Learning},
	author = {Muandet, Krikamol and Fukumizu, Kenji and Sriperumbudur, Bharath and Schölkopf, Bernhard},
	month = jun,
	year = {2017},
	publisher = {Now Publishers, Inc.},
	pages = {1--141},
}

@article{cheng_classification_2022,
	title = {Classification {Logit} {Two}-sample {Testing} by {Neural} {Networks}},
	issn = {0018-9448, 1557-9654},
	doi = {10.1109/TIT.2022.3175691},
	abstract = {The recent success of generative adversarial networks and variational learning suggests training a classifier network may work well in addressing the classical two-sample problem. Network-based tests have the computational advantage that the algorithm scales to large samples. This paper proposes a two-sample statistic which is the difference of the logit function, provided by a trained classification neural network, evaluated on the testing set split of the two datasets. Theoretically, we prove the testing power to differentiate two sub-exponential densities given that the network is sufficiently parametrized. When the two densities lie on or near to low-dimensional manifolds embedded in possibly high-dimensional space, the needed network complexity is reduced to only scale with the intrinsic dimensionality. Both the approximation and estimation error analysis are based on a new result of near-manifold integral approximation. In experiments, the proposed method demonstrates better performance than previous network-based tests using classification accuracy as the two-sample statistic, and compares favorably to certain kernel maximum mean discrepancy tests on synthetic datasets and hand-written digit datasets.},
	journal = {IEEE Transactions on Information Theory},
	author = {Cheng, Xiuyuan and Cloninger, Alexander},
	year = {2022},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Mathematics - Statistics Theory},
	pages = {1--1},
}

@inproceedings{chwialkowski_fast_2015,
	title = {Fast {Two}-{Sample} {Testing} with {Analytic} {Representations} of {Probability} {Measures}},
	volume = {28},
	abstract = {We propose a class of nonparametric two-sample tests with a cost linear in the sample size. Two tests are given, both  based on an ensemble of distances between analytic functions representing each of the distributions. The first test uses smoothed empirical characteristic functions to represent the distributions, the second uses distribution embeddings in a reproducing kernel Hilbert space. Analyticity implies that differences in the distributions may be detected almost surely at a finite number of randomly chosen locations/frequencies. The new tests are consistent against a larger class of alternatives than the previous linear-time tests based on the (non-smoothed) empirical characteristic functions, while being much faster than the current state-of-the-art quadratic-time kernel-based or energy distance-based tests. Experiments on artificial benchmarks and on challenging real-world testing problems demonstrate that our tests give a better power/time tradeoff than  competing approaches, and in some cases, better outright power than even the most expensive quadratic-time tests. This performance advantage is retained even in high dimensions, and in cases where the difference in distributions is not observable with low order statistics.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Chwialkowski, Kacper P and Ramdas, Aaditya and Sejdinovic, Dino and Gretton, Arthur},
	year = {2015},
}

@inproceedings{jitkrittum_interpretable_2016,
	title = {Interpretable {Distribution} {Features} with {Maximum} {Testing} {Power}},
	volume = {29},
	abstract = {Two semimetrics on probability distributions are proposed, given as the sum of differences of expectations of analytic functions evaluated at spatial or frequency locations (i.e, features). The features are chosen so as to maximize the distinguishability of the distributions, by optimizing a lower bound on test power for a statistical test using these features. The result is a parsimonious and interpretable indication of how and where two distributions differ locally. An empirical estimate of the test power criterion converges with increasing sample size, ensuring the quality of the returned features. In real-world benchmarks on high-dimensional text and image data, linear-time tests using the proposed semimetrics achieve comparable performance to the state-of-the-art quadratic-time maximum mean discrepancy test, while returning human-interpretable features that explain the test results.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Jitkrittum, Wittawat and Szabó, Zoltán and Chwialkowski, Kacper P and Gretton, Arthur},
	year = {2016},
}

@misc{cabitza_who_2019,
	title = {Who wants accurate models? {Arguing} for a different metrics to take classification models seriously},
	shorttitle = {Who wants accurate models?},
	doi = {10.48550/arXiv.1910.09246},
	abstract = {With the increasing availability of AI-based decision support, there is an increasing need for their certification by both AI manufacturers and notified bodies, as well as the pragmatic (real-world) validation of these systems. Therefore, there is the need for meaningful and informative ways to assess the performance of AI systems in clinical practice. Common metrics (like accuracy scores and areas under the ROC curve) have known problems and they do not take into account important information about the preferences of clinicians and the needs of their specialist practice, like the likelihood and impact of errors and the complexity of cases. In this paper, we present a new accuracy measure, the H-accuracy (Ha), which we claim is more informative in the medical domain (and others of similar needs) for the elements it encompasses. We also provide proof that the H-accuracy is a generalization of the balanced accuracy and establish a relation between the H-accuracy and the Net Benefit. Finally, we illustrate an experimentation in two user studies to show the descriptive power of the Ha score and how complementary and differently informative measures can be derived from its formulation (a Python script to compute Ha is also made available).},
	publisher = {arXiv},
	author = {Cabitza, Federico and Campagner, Andrea},
	month = oct,
	year = {2019},
	note = {arXiv:1910.09246 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@article{memoli_gromovwasserstein_2011,
	title = {Gromov–{Wasserstein} {Distances} and the {Metric} {Approach} to {Object} {Matching}},
	volume = {11},
	issn = {1615-3383},
	doi = {10.1007/s10208-011-9093-5},
	abstract = {This paper discusses certain modifications of the ideas concerning the Gromov–Hausdorff distance which have the goal of modeling and tackling the practical problems of object matching and comparison. Objects are viewed as metric measure spaces, and based on ideas from mass transportation, a Gromov–Wasserstein type of distance between objects is defined. This reformulation yields a distance between objects which is more amenable to practical computations but retains all the desirable theoretical underpinnings. The theoretical properties of this new notion of distance are studied, and it is established that it provides a strict metric on the collection of isomorphism classes of metric measure spaces. Furthermore, the topology generated by this metric is studied, and sufficient conditions for the pre-compactness of families of metric measure spaces are identified. A second goal of this paper is to establish links to several other practical methods proposed in the literature for comparing/matching shapes in precise terms. This is done by proving explicit lower bounds for the proposed distance that involve many of the invariants previously reported by researchers. These lower bounds can be computed in polynomial time. The numerical implementations of the ideas are discussed and computational examples are presented.},
	language = {en},
	number = {4},
	journal = {Foundations of Computational Mathematics},
	author = {Mémoli, Facundo},
	month = aug,
	year = {2011},
	keywords = {53C23, 54E35, 60D05, 68T10, 68U05, Data analysis, Gromov–Hausdorff distances, Gromov–Wasserstein distances, Mass transport, Metric measure spaces, Shape matching},
	pages = {417--487},
}

@inproceedings{ren_likelihood_2019,
	title = {Likelihood {Ratios} for {Out}-of-{Distribution} {Detection}},
	volume = {32},
	abstract = {Discriminative neural networks offer little or no performance guarantees when deployed on data not generated by the same process as the training distribution. On such out-of-distribution (OOD) inputs, the prediction may not only be erroneous, but confidently so, limiting the safe deployment of classifiers in real-world applications. One such challenging application is bacteria identification based on genomic sequences, which holds the promise of early detection of diseases, but requires a model that can output low confidence predictions on OOD genomic sequences from new bacteria that were not present in the training data. We introduce a genomics dataset for OOD detection that allows other researchers to benchmark progress on this important problem. We investigate deep generative model based approaches for OOD detection and observe that the likelihood score is heavily affected by population level background statistics. We propose a likelihood ratio method for deep generative models which effectively corrects for these confounding background statistics. We benchmark the OOD detection performance of the proposed method against existing approaches on the genomics dataset and show that our method achieves state-of-the-art performance. Finally, we demonstrate the generality of the proposed method by showing that it significantly improves OOD detection when applied to deep generative models of images.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Ren, Jie and Liu, Peter J. and Fertig, Emily and Snoek, Jasper and Poplin, Ryan and Depristo, Mark and Dillon, Joshua and Lakshminarayanan, Balaji},
	year = {2019},
}

@inproceedings{liu_learning_2020,
	title = {Learning {Deep} {Kernels} for {Non}-{Parametric} {Two}-{Sample} {Tests}},
	url = {https://proceedings.mlr.press/v119/liu20m.html},
	abstract = {We propose a class of kernel-based two-sample tests, which aim to determine whether two sets of samples are drawn from the same distribution. Our tests are constructed from kernels parameterized by deep neural nets, trained to maximize test power. These tests adapt to variations in distribution smoothness and shape over space, and are especially suited to high dimensions and complex data. By contrast, the simpler kernels used in prior kernel testing work are spatially homogeneous, and adaptive only in lengthscale. We explain how this scheme includes popular classifier-based two-sample tests as a special case, but improves on them in general. We provide the first proof of consistency for the proposed adaptation method, which applies both to kernels on deep features and to simpler radial basis kernels or multiple kernel learning. In experiments, we establish the superior performance of our deep kernels in hypothesis testing on benchmark and real-world data. The code of our deep-kernel-based two-sample tests is available at github.com/fengliu90/DK-for-TST.},
	booktitle = {Proceedings of the 37th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Liu, Feng and Xu, Wenkai and Lu, Jie and Zhang, Guangquan and Gretton, Arthur and Sutherland, Danica J.},
	month = nov,
	year = {2020},
	issn = {2640-3498},
	pages = {6316--6326},
}

@techreport{noauthor_multivariate_2004,
	title = {On multivariate goodness-of-fit and two-sample testing},
	institution = {Citeseer},
	year = {2004},
}

@book{noauthor_large_1973,
	title = {Large {Sample} {Theory} and {Methods}},
	isbn = {978-0-470-31643-6},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/9780470316436.ch6},
	abstract = {The prelims comprise: Some Basic Results Chi-square Tests for the Multinomial Distribution Tests Relating to Independent Samples from Multinomial Distributions Contingency Tables Some General Classes of Large Sample Tests Order Statistics Transformation of Statistics Standard Errors of Moments and Related Statistics Complements and Problems},
	language = {en},
	urldate = {2022-07-20},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1973},
	doi = {10.1002/9780470316436.ch6},
	note = {Section: 6
	\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/9780470316436.ch6},
	keywords = {analysis of categorical data, asymptotic efficiency, composite hypotheses, statistical inference, statistical methods},
}

@incollection{noauthor_multivariate_1973,
	title = {Multivariate {Analysis}},
	isbn = {978-0-470-31643-6},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/9780470316436.ch8},
	abstract = {The prelims comprise: Multivariate Normal Distribution Wishart Distribution Analysis of Dispersion Some Applications of Multivariate Tests Discriminatory Analysis (Identification) Relation between Sets of Variates Orthonormal Basis of a Random Variable Complements and Problems},
	language = {en},
	urldate = {2022-07-20},
	booktitle = {Linear {Statistical} {Inference} and its {Applications}},
	publisher = {John Wiley \& Sons, Ltd},
	year = {1973},
	doi = {10.1002/9780470316436.ch8},
	note = {Section: 8
	\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/9780470316436.ch8},
	keywords = {statistical inference, conditional distribution, multivariate analysis, multivariate distribution, random variable},
	pages = {516--604},
}

@misc{sutherland_unbiased_2021,
	title = {Unbiased estimators for the variance of {MMD} estimators},
	doi = {10.48550/arXiv.1906.02104},
	abstract = {The maximum mean discrepancy (MMD) is a kernel-based distance between probability distributions useful in many applications (Gretton et al. 2012), bearing a simple estimator with pleasing computational and statistical properties. Being able to efficiently estimate the variance of this estimator is very helpful to various problems in two-sample testing. Towards this end, Bounliphone et al. (2016) used the theory of U-statistics to derive estimators for the variance of an MMD estimator, and differences between two such estimators. Their estimator, however, drops lower-order terms, and is unnecessarily biased. We show in this note - extending and correcting work of Sutherland et al. (2017) - that we can find a truly unbiased estimator for the actual variance of both the squared MMD estimator and the difference of two correlated squared MMD estimators, at essentially no additional computational cost.},
	publisher = {arXiv},
	author = {Sutherland, Danica J.},
	month = jan,
	year = {2021},
	note = {arXiv:1906.02104 [cs, stat]},
}

@inproceedings{lopez-paz_revisiting_2017,
	title={Revisiting Classifier Two-Sample Tests},
	author={David Lopez-Paz and Maxime Oquab},
	booktitle={International Conference on Learning Representations},
	year={2017},
	url={https://openreview.net/forum?id=SJkXfE5xx}
}

@article{kanamori_f_2012,
	title = {\$f\$ -{Divergence} {Estimation} and {Two}-{Sample} {Homogeneity} {Test} {Under} {Semiparametric} {Density}-{Ratio} {Models}},
	volume = {58},
	issn = {1557-9654},
	doi = {10.1109/TIT.2011.2163380},
	abstract = {A density ratio is defined by the ratio of two probability densities. We study the inference problem of density ratios and apply a semiparametric density-ratio estimator to the two-sample homogeneity test. In the proposed test procedure, the \$f\$-divergence between two probability densities is estimated using a density-ratio estimator. The \$f\$ -divergence estimator is then exploited for the two-sample homogeneity test. We derive an optimal estimator of \$f\$-divergence in the sense of the asymptotic variance in a semiparametric setting, and provide a statistic for two-sample homogeneity test based on the optimal estimator. We prove that the proposed test dominates the existing empirical likelihood score test. Through numerical studies, we illustrate the adequacy of the asymptotic theory for finite-sample inference.},
	number = {2},
	journal = {IEEE Transactions on Information Theory},
	author = {Kanamori, Takafumi and Suzuki, Taiji and Sugiyama, Masashi},
	month = feb,
	year = {2012},
	keywords = {Asymptotic expansion, Convergence, density ratio, divergence, Estimation, Manganese, Optimized production technology, Probability distribution, Random variables, semiparametric model, two-sample test},
	pages = {708--720},
}

@article{sriperumbudur_empirical_2012,
	title = {On the empirical estimation of integral probability metrics},
	volume = {6},
	issn = {1935-7524, 1935-7524},
	doi = {10.1214/12-EJS722},
	abstract = {Given two probability measures, \${\textbackslash}mathbb\{P\}\$ and \${\textbackslash}mathbb\{Q\}\$ defined on a measurable space, \$S\$, the integral probability metric (IPM) is defined as \$\${\textbackslash}gamma\_\{{\textbackslash}mathcal\{F\}\}({\textbackslash}mathbb\{P\},{\textbackslash}mathbb\{Q\})={\textbackslash}sup{\textbackslash}left{\textbackslash}\{{\textbackslash}left{\textbackslash}vert {\textbackslash}int\_\{S\}f{\textbackslash},d{\textbackslash}mathbb\{P\}-{\textbackslash}int\_\{S\}f{\textbackslash},d{\textbackslash}mathbb\{Q\}{\textbackslash}right{\textbackslash}vert{\textbackslash},:{\textbackslash},f{\textbackslash}in{\textbackslash}mathcal\{F\}{\textbackslash}right{\textbackslash}\},\$\$ where \${\textbackslash}mathcal\{F\}\$ is a class of real-valued bounded measurable functions on \$S\$. By appropriately choosing \${\textbackslash}mathcal\{F\}\$, various popular distances between \${\textbackslash}mathbb\{P\}\$ and \${\textbackslash}mathbb\{Q\}\$, including the Kantorovich metric, Fortet-Mourier metric, dual-bounded Lipschitz distance (also called the Dudley metric), total variation distance, and kernel distance, can be obtained. In this paper, we consider the problem of estimating \${\textbackslash}gamma\_\{{\textbackslash}mathcal\{F\}\}\$ from finite random samples drawn i.i.d. from \${\textbackslash}mathbb\{P\}\$ and \${\textbackslash}mathbb\{Q\}\$. Although the above mentioned distances cannot be computed in closed form for every \${\textbackslash}mathbb\{P\}\$ and \${\textbackslash}mathbb\{Q\}\$, we show their empirical estimators to be easily computable, and strongly consistent (except for the total-variation distance). We further analyze their rates of convergence. Based on these results, we discuss the advantages of certain choices of \${\textbackslash}mathcal\{F\}\$ (and therefore the corresponding IPMs) over others—in particular, the kernel distance is shown to have three favorable properties compared with the other mentioned distances: it is computationally cheaper, the empirical estimate converges at a faster rate to the population value, and the rate of convergence is independent of the dimension \$d\$ of the space (for \$S={\textbackslash}mathbb\{R\}{\textasciicircum}\{d\}\$). We also provide a novel interpretation of IPMs and their empirical estimators by relating them to the problem of binary classification: while the IPM between class-conditional distributions is the negative of the optimal risk associated with a binary classifier, the smoothness of an appropriate binary classifier (e.g., support vector machine, Lipschitz classifier, etc.) is inversely related to the empirical estimator of the IPM between these class-conditional distributions.},
	journal = {Electronic Journal of Statistics},
	author = {Sriperumbudur, Bharath K. and Fukumizu, Kenji and Gretton, Arthur and Schölkopf, Bernhard and Lanckriet, Gert R. G.},
	month = jan,
	year = {2012},
	publisher = {Institute of Mathematical Statistics and Bernoulli Society},
	keywords = {kernel distance, 62G05, dual-bounded Lipschitz distance (Dudley metric), empirical estimation, Integral probability metrics, Kantorovich metric, Lipschitz classifier, Rademacher average, ‎reproducing kernel Hilbert ‎space, Support Vector Machine},
	pages = {1550--1599},
}

@article{wornowizki_two-sample_2016,
	title = {Two-sample homogeneity tests based on divergence measures},
	volume = {31},
	issn = {1613-9658},
	doi = {10.1007/s00180-015-0633-3},
	abstract = {The concept of f-divergences introduced by Ali and Silvey (J R Stat Soc (B) 28:131–142, 1996) provides a rich set of distance like measures between pairs of distributions. Divergences do not focus on certain moments of random variables, but rather consider discrepancies between the corresponding probability density functions. Thus, two-sample tests based on these measures can detect arbitrary alternatives when testing the equality of the distributions. We treat the problem of divergence estimation as well as the subsequent testing for the homogeneity of two-samples. In particular, we propose a nonparametric estimator for f-divergences in the case of continuous distributions, which is based on kernel density estimation and spline smoothing. As we show in extensive simulations, the new method performs stable and quite well in comparison to several existing non- and semiparametric divergence estimators. Furthermore, we tackle the two-sample homogeneity problem using permutation tests based on various divergence estimators. The methods are compared to an asymptotic divergence test as well as to several traditional parametric and nonparametric procedures under different distributional assumptions and alternatives in simulations. It turns out that divergence based methods detect discrepancies between distributions more often than traditional methods if the distributions do not differ in location only. The findings are illustrated on ion mobility spectrometry data.},
	number = {1},
	journal = {Computational Statistics},
	author = {Wornowizki, Max and Fried, Roland},
	month = mar,
	year = {2016},
	keywords = {Density ratio estimation, Hellinger distance, Kullback-Leibler divergence, Nonparametric two-sample test, Permutation test, Semiparametric two-sample test},
	pages = {291--313},
}

@inproceedings{wang_two-sample_2021,
	title = {Two-sample {Test} using {Projected} {Wasserstein} {Distance}},
	doi = {10.1109/ISIT45174.2021.9518186},
	abstract = {We develop a projected Wasserstein distance for the two-sample test, a fundamental problem in statistics and machine learning: given two sets of samples, to determine whether they are from the same distribution. In particular, we aim to circumvent the curse of dimensionality in Wasserstein distance: when the dimension is high, it has diminishing testing power, which is inherently due to the slow concentration property of Wasserstein metrics in the high dimension space. A key contribution is to couple optimal projection to find the low dimensional linear mapping to maximize the Wasserstein distance between projected probability distributions. We characterize theoretical properties of the two-sample convergence rate on IPMs and this new distance. Numerical examples validate our theoretical results.},
	booktitle = {2021 {IEEE} {International} {Symposium} on {Information} {Theory} ({ISIT})},
	author = {Wang, Jie and Gao, Rui and Xie, Yao},
	month = jul,
	year = {2021},
	keywords = {Information theory, Convergence, Probability distribution, Benchmark testing, Measurement},
	pages = {3320--3325},
}

@article{kanamori_statistical_2014,
	title = {Statistical {Analysis} of {Distance} {Estimators} with {Density} {Differences} and {Density} {Ratios}},
	volume = {16},
	issn = {1099-4300},
	doi = {10.3390/e16020921},
	abstract = {Estimating a discrepancy between two probability distributions from samples is an important task in statistics and machine learning. There are mainly two classes of discrepancy measures: distance measures based on the density difference, such as the Lp-distances, and divergence measures based on the density ratio, such as the Φ-divergences. The intersection of these two classes is the L1-distance measure, and thus, it can be estimated either based on the density difference or the density ratio. In this paper, we first show that the Bregman scores, which are widely employed for the estimation of probability densities in statistical data analysis, allows us to estimate the density difference and the density ratio directly without separately estimating each probability distribution. We then theoretically elucidate the robustness of these estimators and present numerical experiments.},
	number = {2},
	urldate = {2022-07-27},
	journal = {Entropy},
	author = {Kanamori, Takafumi and Sugiyama, Masashi},
	month = feb,
	year = {2014},
	publisher = {Multidisciplinary Digital Publishing Institute},
	keywords = {robustness, density ratio, {\textless}em{\textgreater}L$_{\textrm{1}}${\textless}/em{\textgreater}-distance, Bregman score, density difference},
	pages = {921--942},
}

@article{sugiyama_direct_2013,
	title = {Direct {Divergence} {Approximation} between {Probability} {Distributions} and {Its} {Applications} in {Machine} {Learning}},
	volume = {7},
	issn = {1976-4677},
	doi = {10.5626/JCSE.2013.7.2.99},
	abstract = {Approximating a divergence between two probability distributions from their samples is a fundamental challenge in statistics, information theory, and machine learning. A divergence approximator can be used for various purposes, such as two-sample homogeneity testing, change-point detection, and class-balance estimation. Furthermore, an approximator of a divergence between the joint distribution and the product of marginals can be used for independence testing, which has a wide range of applications, including feature selection and extraction, clustering, object matching, independent component analysis, and causal direction estimation. In this paper, we review recent advances in divergence approximation. Our emphasis is that directly approximating the divergence without estimating probability distributions is more sensible than a naive two-step approach of first estimating probability distributions and then approximating the divergence. Furthermore, despite the overwhelming popularity of the Kullback-Leibler divergence as a divergence measure, we argue that alternatives such as the Pearson divergence, the relative Pearson divergence, and the {\textless}TEX{\textgreater}\$L{\textasciicircum}2\${\textless}/TEX{\textgreater}-distance are more useful in practice because of their computationally efficient approximability, high numerical stability, and superior robustness against outliers.},
	number = {2},
	journal = {Journal of Computing Science and Engineering},
	author = {Sugiyama, Masashi and Liu, Song and du Plessis, Marthinus Christoffel and Yamanaka, Masao and Yamada, Makoto and Suzuki, Taiji and Kanamori, Takafumi},
	year = {2013},
	publisher = {Korean Institute of Information Scientists and Engineers},
	pages = {99--111},
}

@inproceedings{alvarez-melis_geometric_2020,
	title = {Geometric {Dataset} {Distances} via {Optimal} {Transport}},
	volume = {33},
	abstract = {The notion of task similarity is at the core of various machine learning paradigms, such as domain adaptation and meta-learning. Current methods to quantify it are often heuristic, make strong assumptions on the label sets across the tasks, and many are architecture-dependent, relying on task-specific optimal parameters (e.g., require training a model on each dataset). In this work we propose an alternative notion of distance between datasets that (i) is model-agnostic, (ii) does not involve training, (iii) can compare datasets even if their label sets are completely disjoint and (iv) has solid theoretical footing. This distance relies on optimal transport, which provides it with rich geometry awareness, interpretable correspondences and well-understood properties. Our results show that this novel distance provides meaningful comparison of datasets, and correlates well with transfer learning hardness across various experimental settings and datasets.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {{Alvarez-Melis}, David and Fusi, Nicolo},
	year = {2020},
	pages = {21428--21439},
}

@book{najman_modern_2017,
	title = {Modern approaches to discrete curvature},
	volume = {2184},
	publisher = {Springer},
	author = {Najman, Laurent and Romon, Pascal},
	year = {2017},
}

@inproceedings{genevay_learning_2018,
	title = {Learning {Generative} {Models} with {Sinkhorn} {Divergences}},
	abstract = {The ability to compare two degenerate probability distributions, that is two distributions supported on low-dimensional manifolds in much higher-dimensional spaces, is a crucial factor in the estimation of generative mod- els.It is therefore no surprise that optimal transport (OT) metrics and their ability to handle measures with non-overlapping sup- ports have emerged as a promising tool. Yet, training generative machines using OT raises formidable computational and statistical challenges, because of (i) the computational bur- den of evaluating OT losses, (ii) their instability and lack of smoothness, (iii) the difficulty to estimate them, as well as their gradients, in high dimension. This paper presents the first tractable method to train large scale generative models using an OT-based loss called Sinkhorn loss which tackles these three issues by relying on two key ideas: (a) entropic smoothing, which turns the original OT loss into a differentiable and more robust quantity that can be computed using Sinkhorn fixed point iterations; (b) algorithmic (automatic) differentiation of these iterations with seam- less GPU execution. Additionally, Entropic smoothing generates a family of losses interpolating between Wasserstein (OT) and Energy distance/Maximum Mean Discrepancy (MMD) losses, thus allowing to find a sweet spot leveraging the geometry of OT on the one hand, and the favorable high-dimensional sample complexity of MMD, which comes with un- biased gradient estimates. The resulting computational architecture complements nicely standard deep network generative models by a stack of extra layers implementing the loss function.},
	booktitle = {Proceedings of the {Twenty}-{First} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Genevay, Aude and Peyre, Gabriel and Cuturi, Marco},
	month = mar,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {1608--1617},
}

@article{kantorovitch_translocation_1958,
	title = {On the {Translocation} of {Masses}},
	volume = {5},
	issn = {0025-1909},
	doi = {10.1287/mnsc.5.1.1},
	abstract = {The following paper is reproduced from a Russian journal of the character of our own Proceedings of the National Academy of Sciences, Comptes Rendus (Doklady) de I'Académie des Sciences de I'URSS, 1942, Volume XXXVII, No. 7–8. The author is one of the most distinguished of Russian mathematicians. He has made very important contributions in pure mathematics in the theory of functional analysis, and has made equally important contributions to applied mathematics in numerical analysis and the theory and practice of computation. Although his exposition in this paper is quite terse and couched in mathematical language which may be difficult for some readers of Management Science to follow, it is thought that this presentation will: (1) make available to American readers generally an important work in the field of linear programming, (2) provide an indication of the type of analytic work which has been done and is being done in connection with rational planning in Russia, (3) through the specific examples mentioned indicate the types of interpretation which the Russians have made of the abstract mathematics (for example, the potential and field interpretations adduced in this country recently by W. Prager were anticipated in this paper).
	
	It is to be noted, however, that the problem of determining an effective method of actually acquiring the solution to a specific problem is not solved in this paper. In the category of development of such methods we seem to be, currently, ahead of the Russians.—A. Charnes, Northwestern Technological Institute and The Transportation Center.},
	number = {1},
	journal = {Management Science},
	author = {Kantorovitch, L.},
	month = oct,
	year = {1958},
	publisher = {INFORMS},
	pages = {1--4},
}

@inproceedings{wang_two-sample_2022,
	title = {Two-{Sample} {Test} with {Kernel} {Projected} {Wasserstein} {Distance}},
	abstract = {We develop a kernel projected Wasserstein distance for the two-sample test, an essential building block in statistics and machine learning: given two sets of samples, to determine whether they are from the same distribution. This method operates by finding the nonlinear mapping in the data space which maximizes the distance between projected distributions. In contrast to existing works about projected Wasserstein distance, the proposed method circumvents the curse of dimensionality more efficiently. We present practical algorithms for computing this distance function together with the non-asymptotic uncertainty quantification of empirical estimates. Numerical examples validate our theoretical results and demonstrate good performance of the proposed method.},
	booktitle = {Proceedings of {The} 25th {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Wang, Jie and Gao, Rui and Xie, Yao},
	month = may,
	year = {2022},
	note = {ISSN: 2640-3498},
	pages = {8022--8055},
}

@article{guo_novel_2021-1,
	title = {A novel similarity metric with application to big process data analytics},
	volume = {113},
	issn = {0967-0661},
	doi = {10.1016/j.conengprac.2021.104843},
	abstract = {Establishing a quantitative similarity between different datasets has gained prevalence and significance in many applications of process control. In industrial practice, process data are usually multi-dimensional, nonlinearly correlated, and with unknown time-varying distribution, which raise immense challenge for reasonably evaluating similarity. To address this issue, a novel similarity metric based on deep autoencoder (DAE) and the Wasserstein distance is proposed in this paper. Specifically, DAE is used to first capture nonlinear relationship embedded in multivariate process data, and the reconstruction error acts as an indicator to reveal discrepancy between two datasets. After that, the similarity is characterized by evaluating the gap between reconstruction error distributions using Wasserstein distance. The proposed similarity metric has wide applicability in a variety of data analytics tasks including pattern matching, fault diagnosis and mode classifications. Both simulated data and industrial data collected from a real iron-making process are utilized to carry out comprehensive case studies. It is shown that the proposed similarity metric not only enjoys better rationality and sensitivity than generic similarity metrics, but also effectively improves the accuracy of fault diagnosis and mode classification based on big process data.},
	journal = {Control Engineering Practice},
	author = {Guo, Zijian and Shang, Chao and Ye, Hao},
	month = aug,
	year = {2021},
	keywords = {Big process data, Deep autoencoder, Similarity metric, Unsupervised mode classification, Wasserstein distance},
	pages = {104843},
}

@article{bhattacharya_asymptotic_2020,
	title = {Asymptotic distribution and detection thresholds for two-sample tests based on geometric graphs},
	volume = {48},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/19-AOS1913},
	abstract = {In this paper, we consider the problem of testing the equality of two multivariate distributions based on geometric graphs constructed using the interpoint distances between the observations. These include the tests based on the minimum spanning tree and the \$K\$-nearest neighbor (NN) graphs, among others. These tests are asymptotically distribution-free, universally consistent and computationally efficient, making them particularly useful in modern applications. However, very little is known about the power properties of these tests. In this paper, using the theory of stabilizing geometric graphs, we derive the asymptotic distribution of these tests under general alternatives, in the Poissonized setting. Using this, the detection threshold and the limiting local power of the test based on the \$K\$-NN graph are obtained, where interesting exponents depending on dimension emerge. This provides a way to compare and justify the performance of these tests in different examples.},
	number = {5},
	journal = {The Annals of Statistics},
	author = {Bhattacharya, Bhaswar B.},
	month = oct,
	year = {2020},
	publisher = {Institute of Mathematical Statistics},
	keywords = {efficiency, 60D05, 60C05, 60F05, 62F07, 62G10, geometric probability, local power, nearest-neighbor graphs, nonparametric hypothesis testing},
	pages = {2879--2903},
}

@article{chen_new_2017,
	title = {A {New} {Graph}-{Based} {Two}-{Sample} {Test} for {Multivariate} and {Object} {Data}},
	volume = {112},
	issn = {0162-1459},
	doi = {10.1080/01621459.2016.1147356},
	abstract = {Two-sample tests for multivariate data and especially for non-Euclidean data are not well explored. This article presents a novel test statistic based on a similarity graph constructed on the pooled observations from the two samples. It can be applied to multivariate data and non-Euclidean data as long as a dissimilarity measure on the sample space can be defined, which can usually be provided by domain experts. Existing tests based on a similarity graph lack power either for location or for scale alternatives. The new test uses a common pattern that was overlooked previously, and works for both types of alternatives. The test exhibits substantial power gains in simulation studies. Its asymptotic permutation null distribution is derived and shown to work well under finite samples, facilitating its application to large datasets. The new test is illustrated on two applications: The assessment of covariate balance in a matched observational study, and the comparison of network data under different conditions.},
	number = {517},
	journal = {Journal of the American Statistical Association},
	author = {Chen, Hao and Friedman, Jerome H.},
	month = jan,
	year = {2017},
	publisher = {Taylor \& Francis},
	keywords = {General alternatives, Nonparametrics, Permutation null distribution, Similarity graph},
	pages = {397--409},
}

@article{ramdas_wasserstein_2017,
	title = {On {Wasserstein} {Two}-{Sample} {Testing} and {Related} {Families} of {Nonparametric} {Tests}},
	volume = {19},
	issn = {1099-4300},
	doi = {10.3390/e19020047},
	abstract = {Nonparametric two-sample or homogeneity testing is a decision theoretic problem that involves identifying differences between two random variables without making parametric assumptions about their underlying distributions. The literature is old and rich, with a wide variety of statistics having being designed and analyzed, both for the unidimensional and the multivariate setting. In this short survey, we focus on test statistics that involve the Wasserstein distance. Using an entropic smoothing of the Wasserstein distance, we connect these to very different tests including multivariate methods involving energy statistics and kernel based maximum mean discrepancy and univariate methods like the Kolmogorov–Smirnov test, probability or quantile (PP/QQ) plots and receiver operating characteristic or ordinal dominance (ROC/ODC) curves. Some observations are implicit in the literature, while others seem to have not been noticed thus far. Given nonparametric two-sample testing’s classical and continued importance, we aim to provide useful connections for theorists and practitioners familiar with one subset of methods but not others.},
	language = {en},
	number = {2},
	urldate = {2022-08-01},
	journal = {Entropy},
	author = {Ramdas, Aaditya and Trillos, Nicolás García and Cuturi, Marco},
	month = feb,
	year = {2017},
	publisher = {Multidisciplinary Digital Publishing Institute},
	keywords = {energy distance, entropic smoothing, maximum mean discrepancy, QQ and PP plots, ROC and ODC curves, two-sample testing, wasserstein distance},
	pages = {47},
}

@article{sejdinovic_equivalence_2013,
	title = {{Equivalence} of {Distance}-{Based} and {RKHS}-{BASED} {Statistics} in {Hypothesis} {Testing}},
	volume = {41},
	issn = {0090-5364},
	abstract = {We provide a unifying framework linking two classes of statistics used in two-sample and independence testing: on the one hand, the energy distances and distance covariances from the statistics literature; on the other, maximum mean discrepancies (MMD), that is, distances between embeddings of distributions to reproducing kernel Hilbert spaces (RKHS), as established in machine learning. In the case where the energy distance is computed with a semimetric of negative type, a positive definite kernel, termed distance kernel, may be defined such that the MMD corresponds exactly to the energy distance. Conversely, for any positive definite kernel, we can interpret the MMD as energy distance with respect to some negative-type semimetric. This equivalence readily extends to distance covariance using kernels on the product space. We determine the class of probability distributions for which the test statistics are consistent against all alternatives. Finally, we investigate the performance of the family of distance kernels in two-sample and independence tests: we show in particular that the energy distance most commonly employed in statistics is just one member of a parametric family of kernels, and that other choices from this family can yield more powerful tests.},
	number = {5},
	urldate = {2022-08-01},
	journal = {The Annals of Statistics},
	author = {Sejdinovic, Dino and Sriperumbudur, Bharath and Gretton, Arthur and Fukumizu, Kenji},
	year = {2013},
	publisher = {Institute of Mathematical Statistics},
	pages = {2263--2291},
}

@article{batu_testing_2013,
	title = {Testing {Closeness} of {Discrete} {Distributions}},
	volume = {60},
	issn = {0004-5411},
	doi = {10.1145/2432622.2432626},
	abstract = {Given samples from two distributions over an n-element set, we wish to test whether these distributions are statistically close. We present an algorithm which uses sublinear in n, specifically, O(n2/3ε−8/3 log n), independent samples from each distribution, runs in time linear in the sample size, makes no assumptions about the structure of the distributions, and distinguishes the cases when the distance between the distributions is small (less than \{ε4/3n−1/3/32, εn−1/2/4\}) or large (more than ε) in ℓ1 distance. This result can be compared to the lower bound of Ω(n2/3ε−2/3) for this problem given by Valiant [2008]. Our algorithm has applications to the problem of testing whether a given Markov process is rapidly mixing. We present sublinear algorithms for several variants of this problem as well.},
	number = {1},
	journal = {Journal of the ACM},
	author = {Batu, Tuğkan and Fortnow, Lance and Rubinfeld, Ronitt and Smith, Warren D. and White, Patrick},
	month = feb,
	year = {2013},
	keywords = {statistical distance, testing Markov chains for mixing, Testing properties of distributions},
	pages = {4:1--4:25},
}

@article{rousson_distribution-free_2002,
	title = {On {Distribution}-{Free} {Tests} for the {Multivariate} {Two}-{Sample} {Location}-{Scale} {Model}},
	volume = {80},
	issn = {0047-259X},
	doi = {10.1006/jmva.2000.1981},
	abstract = {In this paper, we propose simple exact procedures for testing both a location shift and/or a scale change between two multivariate distributions. Our tests are strictly distribution-free and can be made either scale invariant or rotation invariant. Our approach combines a generalization of the Wilcoxon test based on projections of the data onto the first principal component, a generalization of the Siegel–Tukey test based on the concept of data depth, and a bivariate test for the location problem proposed by K. V. Mardia (1967, J. Roy. Statist. Soc. Ser. B29, 320–342). In addition, we show that the limiting null distribution of a test statistic proposed by R. Y. Liu and K. Singh (1993, J. Amer. Statist. Assoc.88, 252–260) does not depend on the depth considered.},
	number = {1},
	journal = {Journal of Multivariate Analysis},
	author = {Rousson, Valentin},
	month = jan,
	year = {2002},
	keywords = {data depth, multivariate orderings, nonparametric methods, principal component analysis, rank tests},
	pages = {43--57},
}

@article{liu_quality_1993,
	title = {A {Quality} {Index} {Based} on {Data} {Depth} and {Multivariate} {Rank} {Tests}},
	volume = {88},
	issn = {0162-1459},
	doi = {10.1080/01621459.1993.10594317},
	abstract = {Let F and G be the distribution functions of two given populations on Rp, p ≥ 1. We introduce and study a Parameter Q = Q(F, G), which measures the Overall “outlyingness” of population G relative to population F. The Parameter Q can be defined using any concept of data depth. Its value ranges from 0 to 1, and is .5 when F and G are identical. We show that within the dass of elliptical distributions when G departs from F in location or G has a larger spread, or both, the value of Q dwindles down from .5. Hence Q can be used to detect the loss of accuracy or precision of a manufacturing process, and thus it should serve as an important measure in quality assurance. This in fact is the reason why we refer to Q as a quality index in this article. In addition to studying the properties of Q, we provide an exact rank test for testing Q = .5 vs. Q {\textless} .5. This can be viewed as a multivariate analog of Wilcoxon's rank sum test. The tests proposed here have power against location change and scale increase simultaneously. We introduce some estimates of Q and investigate their limiting distributions when F = G. We also consider a version of Q and its estimates, which are defined after correcting the location shift of G. In this case Q is used to measure scale increase only.},
	number = {421},
	journal = {Journal of the American Statistical Association},
	author = {Liu, Regina Y. and Singh, Kesar},
	month = mar,
	year = {1993},
	publisher = {Taylor \& Francis},
	keywords = {Data depth, Multivariate rank tests, Quality assurance, Quality index},
	pages = {252--260},
}

@article{maa_reducing_1996,
	title = {Reducing multidimensional two-sample data to one-dimensional interpoint comparisons},
	volume = {24},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/aos/1032526956},
	abstract = {The most popular technique for reducing the dimensionality in comparing two multidimensional samples of \${\textbackslash}mathbf\{X\} {\textbackslash}sim F\$ and \${\textbackslash}mathbf\{Y\}{\textbackslash}sim G\$ is to analyze distributions of interpoint comparisons based on a univariate function h (e.g. the interpoint distances). We provide a theoretical foundation for this technique, by showing that having both i) the equality of the distributions of within sample comparisons \$(h({\textbackslash}mathbf\{X\}\_1, {\textbackslash}mathbf\{X\}\_2) =\_L h({\textbackslash}mathbf\{Y\}\_1, {\textbackslash}mathbf\{Y\}\_2))\$ and ii) the equality of these with the distribution of between sample comparisons \$((h({\textbackslash}mathbf\{X\}\_1, {\textbackslash}mathbf\{X\}\_2) =\_L h({\textbackslash}mathbf\{X\}\_3, {\textbackslash}mathbf\{Y\}\_3))\$ is equivalent to the equality of the multivariate distributions \$(F = G)\$.},
	number = {3},
	journal = {The Annals of Statistics},
	author = {Maa, Jen-Fue and Pearl, Dennis K. and Bartoszyński, Robert},
	month = jun,
	year = {1996},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62H05, Characterization of distributional equality, distances, multivariate},
	pages = {1069--1074},
}

@article{biswas_distribution-free_2014,
	title = {A {Distribution-Free} {Two-Sample} {Run} {Test} {Applicable} to {High-Dimensional} {Data}},
	volume = {101},
	issn = {0006-3444},
	doi = {10.1093/biomet/asu045},
	abstract = {We propose a multivariate generalization of the univariate two-sample run test based on the shortest Hamiltonian path. The proposed test is distribution-free in finite samples. While most existing two-sample tests perform poorly or are even inapplicable to high-dimensional data, our test can be conveniently used in high-dimension, low-sample-size situations. We investigate its power when the sample size remains fixed and the dimension of the data grows to infinity. Simulated and real datasets demonstrate our method’s superiority over existing nonparametric two-sample tests.},
	number = {4},
	urldate = {2022-08-01},
	journal = {Biometrika},
	author = {Biswas, Munmun and Mukhopadhyay, Minerva and Ghosh, Anil K.},
	month = dec,
	year = {2014},
	pages = {913--926},
}

@article{aslan_new_2005,
	title = {{New} {Test} for the {Multivariate} {Two}-{Sample} {Problem} {Based} on the {Concept} of {Minimum} {Energy}},
	volume = {75},
	issn = {0094-9655},
	doi = {10.1080/00949650410001661440},
	abstract = {We introduce a new statistical quantity, the energy, to test whether two samples originate from the same distributions. The energy is a simple logarithmic function of the distances of the observations in the variate space. The distribution of the test statistic is determined by a resampling method. The power of the energy test in one dimension was studied for a variety of different test samples and compared to several nonparametric tests. In two and four dimensions, a comparison was performed with the Friedman–Rafsky and nearest neighbor tests. The two-sample energy test was shown to be especially powerful in multidimensional applications.},
	number = {2},
	journal = {Journal of Statistical Computation and Simulation},
	author = {Aslan, B. and Zech, G.},
	month = feb,
	year = {2005},
	publisher = {Taylor \& Francis},
	keywords = {Goodness-of-fit, Multidimensional, Nonparametric, Two-sample test},
	pages = {109--119},
}

@article{montero-manso_two-sample_2019,
	title = {Two-sample homogeneity testing: {A} procedure based on comparing distributions of interpoint distances},
	volume = {12},
	issn = {1932-1872},
	shorttitle = {Two-sample homogeneity testing},
	doi = {10.1002/sam.11417},
	abstract = {A new test statistic using interpoint distances is proposed to address the two-sample problem for multivariate populations. The test statistic compares univariate distributions of within and between samples pairwise distances using a Cramér-von Mises-type statistic. The critical values are approximated by means of a permutation procedure and the regularity conditions required to ensure the consistency of the test are established. Unlike other two-sample procedures, our approach compares the whole distributions of interpoint distances instead of just a few moments, thus obtaining a higher capability to detect differences in their shape or in other moments. An extensive simulation study and experiments with real data sets considered in related papers show a satisfactory performance of the proposed test under a range of alternative distributions. Compared to other two-sample tests based on interpoint distances, the experiments reveal a more robust behavior in a high-dimensional setting, being one of the most powerful tests under both location and scales changes.},
	number = {3},
	journal = {Statistical Analysis and Data Mining: The ASA Data Science Journal},
	author = {Montero-Manso, Pablo and Vilar, José A.},
	year = {2019},
	keywords = {Cramér von-Mises statistic, interpoint distance, permutation test, two-sample homogeneity test},
	pages = {234--252},
}

@article{sarkar_high-dimensional_2018,
	title = {On some high-dimensional two-sample tests based on averages of inter-point distances},
	volume = {7},
	issn = {2049-1573},
	doi = {10.1002/sta4.187},
	abstract = {Over the last two decades, several two-sample tests based on averages of inter-point distances have been proposed in the literature. Most of these tests are based on the Euclidean distance, and they can be used even when the dimension of the data is much larger than the sample size. But these tests can produce poor results in high-dimensional set-ups even when the two distributions differ widely in their scatters and shapes. To overcome these limitations, we modify some tests by replacing the Euclidean distance with a new class of distance functions. The high-dimensional consistency of these modified tests is established under appropriate regularity conditions. Numerical studies are also carried out to demonstrate the usefulness of the proposed methods. Copyright © 2018 John Wiley \& Sons, Ltd.},
	number = {1},
	journal = {Stat},
	author = {Sarkar, Soham and Ghosh, Anil K.},
	year = {2018},
	keywords = {energy distance, Cramér test, permutation test, HDLSS asymptotics},
	pages = {e187},
}

@article{li_asymptotic_2018,
	title = {Asymptotic normality of interpoint distances for high-dimensional data with applications to the two-sample problem},
	volume = {105},
	issn = {0006-3444},
	doi = {10.1093/biomet/asy020},
	abstract = {Interpoint distances have applications in many areas of probability and statistics. Thanks to their simplicity of computation, interpoint distance-based procedures are particularly appealing for analysing small samples of high-dimensional data. In this paper, we first study the asymptotic distribution of interpoint distances in the high-dimension, low-sample-size setting and show that it is normal under regularity conditions. We then construct a powerful test for the two-sample problem, which is consistent for detecting location and scale differences. Simulations show that the test compares favourably with existing distance-based tests.},
	number = {3},
	journal = {Biometrika},
	author = {Li, Jun},
	month = sep,
	year = {2018},
	pages = {529--546},
}

@article{liu_log-rank-type_2022,
	title = {Log-{Rank}-{Type} {Tests} for {Equality} of {Distributions} in {High}-{Dimensional} {Spaces}},
	issn = {1061-8600},
	doi = {10.1080/10618600.2022.2051530},
	abstract = {Motivated by applications in high-dimensional settings, we propose a novel approach for testing the equality of two or more populations by constructing a class of intensity centered score processes. The resulting tests are analogous in spirit to the well-known class of weighted log-rank statistics that are widely used in survival analysis. The test statistics are nonparametric, computationally simple, and applicable to high-dimensional data. We establish the usual large sample properties by showing that the underlying log-rank score process converges weakly to a Gaussian random field with zero mean under the null hypothesis and with a drift under the contiguous alternatives. For the Kolmogorov–Smirnov-type and the Cramer-von Mises-type statistics, we also establish the consistency result for any fixed alternative. Cutoff points for the test statistics are obtained by permutations or a simulation-based resampling method. The new approach is applied to a study of brain activation measured by functional magnetic resonance imaging when performing two linguistic tasks and also to a prostate cancer DNA microarray dataset. Supplementary materials for this article are available online.},
	journal = {Journal of Computational and Graphical Statistics},
	author = {Liu, Linxi and Meng, Yang and Wu, Xiaoru and Ying, Zhiliang and Zheng, Tian},
	month = mar,
	year = {2022},
	publisher = {Taylor \& Francis},
	keywords = {Consistency, Distribution-free method, Gaussian random field, High-dimensional data, Nonparametric tests, Random weighting, Rank-based tests},
	pages = {1--13},
}

@Manual{diproperm,
	title = {diproperm: Conduct Direction-Projection-Permutation Tests and Display Plots},
	author = {Andrew G. Allmon and J.S. Marron and Michael G. Hudgens},
	year = {2021},
	note = {R package version 0.2.0},
	url = {https://CRAN.R-project.org/package=diproperm},
}

@article{mukherjee_distribution-free_2022,
	title = {Distribution-{Free} {Multisample} {Tests} {Based} on {Optimal} {Matchings} {With} {Applications} to {Single} {Cell} {Genomics}},
	volume = {117},
	issn = {0162-1459},
	doi = {10.1080/01621459.2020.1791131},
	abstract = {In this article, we propose a nonparametric graphical test based on optimal matching, for assessing the equality of multiple unknown multivariate probability distributions. Our procedure pools the data from the different classes to create a graph based on the minimum non-bipartite matching, and then utilizes the number of edges connecting data points from different classes to examine the closeness between the distributions. The proposed test is exactly distribution-free (the null distribution does not depend on the distribution of the data) and can be efficiently applied to multivariate as well as non-Euclidean data, whenever the inter-point distances are well-defined. We show that the test is universally consistent, and prove a distributional limit theorem for the test statistic under general alternatives. Through simulation studies, we demonstrate its superior performance against other common and well-known multisample tests. The method is applied to single cell transcriptomics data obtained from the peripheral blood, cancer tissue, and tumor-adjacent normal tissue of human subjects with hepatocellular carcinoma and non-small-cell lung cancer. Our method unveils patterns in how biochemical metabolic pathways are altered across immune cells in a cancer setting, depending on the tissue location. All of the methods described herein are implemented in the R package multicross. Supplementary materials for this article are available online.},
	number = {538},
	journal = {Journal of the American Statistical Association},
	author = {Mukherjee, Somabha and Agarwal, Divyansh and Zhang, Nancy R. and Bhattacharya, Bhaswar B.},
	month = apr,
	year = {2022},
	publisher = {Taylor \& Francis},
	keywords = {Biological pathways, Distribution-free tests, Graph-based methods, Optimal matching},
	pages = {627--638},
}

@article{deb_multivariate_2021,
	title = {Multivariate {Rank}-{Based} {Distribution}-{Free} {Nonparametric} {Testing} {Using} {Measure} {Transportation}},
	volume = {118},
	number = {541},
	issn = {0162-1459},
	doi = {10.1080/01621459.2021.1923508},
	abstract = {In this article, we propose # for distribution-free nonparametric testing in multi-dimensions, based on a notion of multivariate ranks defined using the theory of measure transportation. Unlike other existing proposals in the literature, these multivariate ranks share a number of useful properties with the usual one-dimensional ranks; most importantly, these ranks are distribution-free. This crucial observation allows us to design nonparametric tests that are exactly distribution-free under the null hypothesis. We demonstrate the applicability of this approach by constructing exact distribution-free tests for two classical nonparametric problems: (I) testing for mutual independence between random vectors, and (II) testing for the equality of multivariate distributions. In particular, we propose (multivariate) rank versions of distance covariance and energy statistic for testing scenarios (I) and (II), respectively. In both these problems, we derive the asymptotic null distribution of the proposed test statistics. We further show that our tests are consistent against all fixed alternatives. Moreover, the proposed tests are computationally feasible and are well-defined under minimal assumptions on the underlying distributions (e.g., they do not need any moment assumptions). We also demonstrate the efficacy of these procedures via extensive simulations. In the process of analyzing the theoretical properties of our procedures, we end up proving some new results in the theory of measure transportation and in the limit theory of permutation statistics using Stein’s method for exchangeable pairs, which may be of independent interest.},
	journal = {Journal of the American Statistical Association},
	author = {Deb, Nabarun and Sen, Bodhisattva},
	month = may,
	year = {2021},
	publisher = {Taylor \& Francis},
	keywords = {Asymptotic null distribution, Consistency against fixed alternatives, Distance covariance, Distribution-free inference, Energy distance, Multivariate ranks, Multivariate two-sample testing, Quasi-Monte Carlo sequences, Stein’s method for exchangeable pairs, Testing for mutual independence},
	pages = {1--16},
}

@article{chen_weighted_2018,
	title = {A {Weighted} {Edge}-{Count} {Two}-{Sample} {Test} for {Multivariate} and {Object} {Data}},
	volume = {113},
	issn = {0162-1459},
	doi = {10.1080/01621459.2017.1307757},
	abstract = {Two-sample tests for multivariate data and non-Euclidean data are widely used in many fields. Parametric tests are mostly restrained to certain types of data that meets the assumptions of the parametric models. In this article, we study a nonparametric testing procedure that uses graphs representing the similarity among observations. It can be applied to any data types as long as an informative similarity measure on the sample space can be defined. The classic test based on a similarity graph has a problem when the two sample sizes are different. We solve the problem by applying appropriate weights to different components of the classic test statistic. The new test exhibits substantial power gains in simulation studies. Its asymptotic permutation null distribution is derived and shown to work well under finite samples, facilitating its application to large datasets. The new test is illustrated through an analysis on a real dataset of network data.},
	number = {523},
	journal = {Journal of the American Statistical Association},
	author = {Chen, Hao and Chen, Xu and Su, Yi},
	month = jul,
	year = {2018},
	publisher = {Taylor \& Francis},
	keywords = {Permutation null distribution, Similarity graph, Nonparametric test, Unequal sample sizes},
	pages = {1146--1155},
}

@article{chakraborty_new_2021,
	title = {A new framework for distance and kernel-based metrics in high dimensions},
	volume = {15},
	issn = {1935-7524, 1935-7524},
	abstract = {The paper presents new metrics to quantify and test for (i) the equality of distributions and (ii) the independence between two high-dimensional random vectors. We show that the energy distance based on the usual Euclidean distance cannot completely characterize the homogeneity of two high-dimensional distributions in the sense that it only detects the equality of means and the traces of covariance matrices in the high-dimensional setup. We propose a new class of metrics which inherits the desirable properties of the energy distance and maximum mean discrepancy/(generalized) distance covariance and the Hilbert-Schmidt Independence Criterion in the low-dimensional setting and is capable of detecting the homogeneity of/completely characterizing independence between the low-dimensional marginal distributions in the high dimensional setup. We further propose t-tests based on the new metrics to perform high-dimensional two-sample testing/independence testing and study their asymptotic behavior under both high dimension low sample size (HDLSS) and high dimension medium sample size (HDMSS) setups. The computational complexity of the t-tests only grows linearly with the dimension and thus is scalable to very high dimensional data. We demonstrate the superior power behavior of the proposed tests for homogeneity of distributions and independence via both simulated and real datasets.},
	number = {2},
	journal = {Electronic Journal of Statistics},
	author = {Chakraborty, Shubhadeep and Zhang, Xianyang},
	month = jan,
	year = {2021},
	publisher = {Institute of Mathematical Statistics and Bernoulli Society},
	keywords = {energy distance, maximum mean discrepancy, distance covariance, high dimensionality, Hilbert-Schmidt independence criterion, Independence test, two sample test, U-statistic},
	pages = {5455--5522},
}

@article{zhu_interpoint_2021,
	title = {Interpoint distance based two sample tests in high dimension},
	volume = {27},
	issn = {1350-7265},
	doi = {10.3150/20-BEJ1270},
	abstract = {In this paper, we study a class of two sample test statistics based on inter-point distances in the high dimensional and low/medium sample size setting. Our test statistics include the well-known energy distance and maximum mean discrepancy with Gaussian and Laplacian kernels, and the critical values are obtained via permutations. We show that all these tests are inconsistent when the two high dimensional distributions correspond to the same marginal distributions but differ in other aspects of the distributions. The tests based on energy distance and maximum mean discrepancy mainly target the differences between marginal means and variances, whereas the test based on L1-distance can capture the difference in marginal distributions. Our theory sheds new light on the limitation of inter-point distance based tests, the impact of different distance metrics, and the behavior of permutation tests in high dimension. Some simulation results and a real data illustration are also presented to corroborate our theoretical findings.},
	number = {2},
	journal = {Bernoulli},
	author = {Zhu, Changbo and Shao, Xiaofeng},
	month = may,
	year = {2021},
	publisher = {Bernoulli Society for Mathematical Statistics and Probability},
	keywords = {Permutation test, high dimensionality, two sample test, power analysis},
	pages = {1189--1211},
}

@article{li_measuring_2022,
	title = {Measuring and {Testing} {Homogeneity} of {Distributions} by {Characteristic} {Distance}},
	issn = {1613-9798},
	doi = {10.1007/s00362-022-01327-7},
	abstract = {Technological advances have enabled us to collect a lot of complex data objects, where homogeneity structure among these objects is widely used in Statistics. However, the existing metrics of homogeneity are subject to some qualifications, such as assumptions about the moment and parameters. To overcome the limitation, this paper first introduces the characteristic distance, a novel metric that entirely characterizes the homogeneity of two distributions. The proposed distance possesses some desirable statistical properties: (i) It is a distribution-free or, more commonly, nonparametric test, thus is robust to the data; (ii) It is nonnegative and equal to zero if and only if the two distributions are homogeneous; (iii) The novel measure possesses a clear and intuitive probabilistic interpretation, moreover, its empirical version is easy to calculate and can be reduced to a sum of two V-statistics. Theoretically, the asymptotic distributions, including the mixture of \$\${\textbackslash}chi {\textasciicircum}\{2\}\$\$distributions under the null hypothesis and the asymptotic normality of the alternative hypothesis are thoroughly investigated. Simulation studies and a real data application suggest that the empirical characteristic distance has a preferable power in detecting the homogeneity of distributions.},
	journal = {Statistical Papers},
	author = {Li, Xu and Hu, Wenjuan and Zhang, Baoxue},
	month = jun,
	year = {2022},
	keywords = {Two-sample test, U-statistic, Characteristic distance, Homogeneity, Permutation procedure},
}

@article{sarkar_graph-based_2020,
	title = {On some graph-based two-sample tests for high dimension, low sample size data},
	volume = {109},
	issn = {1573-0565},
	doi = {10.1007/s10994-019-05857-4},
	abstract = {Testing for equality of two high-dimensional distributions is a challenging problem, and this becomes even more challenging when the sample size is small. Over the last few decades, several graph-based two-sample tests have been proposed in the literature, which can be used for data of arbitrary dimensions. Most of these test statistics are computed using pairwise Euclidean distances among the observations. But, due to concentration of pairwise Euclidean distances, these tests have poor performance in many high-dimensional problems. Some of them can have powers even below the nominal level when the scale-difference between two distributions dominates the location-difference. To overcome these limitations, we introduce some new dissimilarity indices and use them to modify some popular graph-based tests. These modified tests use the distance concentration phenomenon to their advantage, and as a result, they outperform the corresponding tests based on the Euclidean distance in a wide variety of examples. We establish the high-dimensional consistency of these modified tests under fairly general conditions. Analyzing several simulated as well as real data sets, we demonstrate their usefulness in high dimension, low sample size situations.},
	number = {2},
	journal = {Machine Learning},
	author = {Sarkar, Soham and Biswas, Rahul and Ghosh, Anil K.},
	month = feb,
	year = {2020},
	keywords = {Permutation test, Distance concentration, High-dimensional consistency, Minimum spanning tree, Nearest neighbor, Non-bipartite matching, Shortest Hamiltonian path},
	pages = {279--306},
}

@incollection{chen_kernel_2020,
	address = {Cham},
	series = {Emerging {Topics} in {Statistics} and {Biostatistics}},
	title = {Kernel {Tests} for {One}, {Two}, and {K}-{Sample} {Goodness}-of-{Fit}: {State} of the {Art} and {Implementation} {Considerations}},
	isbn = {978-3-030-33416-1},
	shorttitle = {Kernel {Tests} for {One}, {Two}, and {K}-{Sample} {Goodness}-of-{Fit}},
	abstract = {In this article, we first discuss the fundamental role of statistical distances in the problem of goodness-of-fit and review various existing multivariate two-sample goodness-of-fit tests from both statistics and machine learning literature. The review conducted delivers the fact that there does not exist a satisfactory multivariate two-sample goodness-of-fit test. We introduce a class of one and two-sample tests constructed using the kernel-based quadratic distance, and briefly touch upon their asymptotic properties. We discuss the practical implementation of these tests, with emphasis on the kernel-based two-sample test. Finally, we use simulations and real data to illustrate the application of the kernel-based two-sample test, and compare this test with tests existing in the literature.},
	booktitle = {Statistical {Modeling} in {Biomedical} {Research}: {Contemporary} {Topics} and {Voices} in the {Field}},
	publisher = {Springer International Publishing},
	author = {Chen, Yang and Markatou, Marianthi},
	editor = {Zhao, Yichuan and Chen, Ding-Geng (Din)},
	year = {2020},
	doi = {10.1007/978-3-030-33416-1_14},
	keywords = {Goodness-of-fit, Kernel tests, Multivariate, Quadratic distance, Two-sample methods},
	pages = {309--337},
}

@article{al-labadi_bayesian_2022,
	title = {A {Bayesian} nonparametric multi-sample test in any dimension},
	volume = {106},
	issn = {1863-818X},
	doi = {10.1007/s10182-021-00419-3},
	abstract = {This paper considers a general Bayesian test for the multi-sample problem. Specifically, for M independent samples, the interest is to determine whether the M samples are generated from the same multivariate population. First, M Dirichlet processes are considered as priors for the true distributions generated the data. Then, the concentration of the distribution of the total distance between the M posterior processes is compared to the concentration of the distribution of the total distance between the M prior processes through the relative belief ratio. The total distance between processes is established based on the energy distance. Various interesting theoretical results of the approach are derived. Several examples covering the high dimensional case are considered to illustrate the approach.},
	number = {2},
	journal = {AStA Advances in Statistical Analysis},
	author = {Al-Labadi, Luai and Asl, Forough Fazeli and Saberi, Zahra},
	month = jun,
	year = {2022},
	keywords = {Simulation, 62G10, Energy distance, 62F15, 62H15, Dirichlet process prior, Multi-sample hypothesis testing, Relative belief ratio},
	pages = {217--242},
}

@inproceedings{kirchler_two-sample_2020,
	title = {Two-sample {Testing} {Using} {Deep} {Learning}},
	abstract = {We propose a two-sample testing procedure based on learned deep neural network representations. To this end, we define two test statistics that perform an asymptotic location test on data samples mapped onto a hidden layer. The tests are consistent and asymptotically control the type-1 error rate. Their test statistics can be evaluated in linear time (in the sample size). Suitable data representations are obtained in a data-driven way, by solving a supervised or unsupervised transfer-learning task on an auxiliary (potentially distinct) data set. If no auxiliary data is available, we split the data into two chunks: one for learning representations and one for computing the test statistic. In experiments on audio samples, natural images and three-dimensional neuroimaging data our tests yield significant decreases in type-2 error rate (up to 35 percentage points) compared to state-of-the-art two-sample tests such as kernel-methods and classifier two-sample tests.},
	booktitle = {Proceedings of the {Twenty} {Third} {International} {Conference} on {Artificial} {Intelligence} and {Statistics}},
	publisher = {PMLR},
	author = {Kirchler, Matthias and Khorasani, Shahryar and Kloft, Marius and Lippert, Christoph},
	month = jun,
	year = {2020},
	issn = {2640-3498},
	pages = {1387--1398},
}

@article{schott_test_2007,
	title = {A test for the equality of covariance matrices when the dimension is large relative to the sample sizes},
	volume = {51},
	issn = {0167-9473},
	doi = {10.1016/j.csda.2007.03.004},
	abstract = {A simple statistic is proposed for testing the equality of the covariance matrices of several multivariate normal populations. The asymptotic null distribution of this statistic, as both the sample sizes and the number of variables go to infinity, is shown to be normal. Consequently, this test can be used when the number of variables is not small relative to the sample sizes and, in particular, even when the number of variables exceeds the sample sizes. The finite sample size performance of the normal approximation for this method is evaluated in a simulation study.},
	number = {12},
	journal = {Computational Statistics \& Data Analysis},
	author = {Schott, James R.},
	month = aug,
	year = {2007},
	keywords = {High-dimensional data, Equal covariance matrices, Singular sample covariance matrix},
	pages = {6535--6542},
}

@article{li_two_2012,
	title = {Two sample tests for high-dimensional covariance matrices},
	volume = {40},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/12-AOS993},
	abstract = {We propose two tests for the equality of covariance matrices between two high-dimensional populations. One test is on the whole variance–covariance matrices, and the other is on off-diagonal sub-matrices, which define the covariance between two nonoverlapping segments of the high-dimensional random vectors. The tests are applicable (i) when the data dimension is much larger than the sample sizes, namely the “large \$p\$, small \$n\$” situations and (ii) without assuming parametric distributions for the two populations. These two aspects surpass the capability of the conventional likelihood ratio test. The proposed tests can be used to test on covariances associated with gene ontology terms.},
	number = {2},
	journal = {The Annals of Statistics},
	author = {Li, Jun and Chen, Song Xi},
	month = apr,
	year = {2012},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62G10, 62H15, 62G20, High-dimensional covariance, Large \$p\$ small \$n\$, likelihood ratio test, testing for gene-sets},
	pages = {908--940},
}

@article{cai_two-sample_2013,
	title = {Two-{Sample} {Covariance} {Matrix} {Testing} and {Support} {Recovery} in {High}-{Dimensional} and {Sparse} {Settings}},
	volume = {108},
	issn = {0162-1459},
	doi = {10.1080/01621459.2012.758041},
	abstract = {In the high-dimensional setting, this article considers three interrelated problems: (a) testing the equality of two covariance matrices and ; (b) recovering the support of ; and (c) testing the equality of and row by row. We propose a new test for testing the hypothesis H 0: and investigate its theoretical and numerical properties. The limiting null distribution of the test statistic is derived and the power of the test is studied. The test is shown to enjoy certain optimality and to be especially powerful against sparse alternatives. The simulation results show that the test significantly outperforms the existing methods both in terms of size and power. Analysis of a prostate cancer dataset is carried out to demonstrate the application of the testing procedures. When the null hypothesis of equal covariance matrices is rejected, it is often of significant interest to further investigate how they differ from each other. Motivated by applications in genomics, we also consider recovering the support of and testing the equality of the two covariance matrices row by row. New procedures are introduced and their properties are studied. Applications to gene selection are also discussed. Supplementary materials for this article are available online.},
	number = {501},
	urldate = {2022-08-19},
	journal = {Journal of the American Statistical Association},
	author = {Cai, Tony and Liu, Weidong and Xia, Yin},
	month = mar,
	year = {2013},
	publisher = {Taylor \& Francis},
	keywords = {Extreme value Type I distribution, Gene selection, Hypothesis testing, Sparsity},
	pages = {265--277},
}

@article{chang_comparing_2017,
	title = {Comparing large covariance matrices under weak conditions on the dependence structure and its application to gene clustering},
	volume = {73},
	issn = {1541-0420},
	doi = {10.1111/biom.12552},
	abstract = {Comparing large covariance matrices has important applications in modern genomics, where scientists are often interested in understanding whether relationships (e.g., dependencies or co-regulations) among a large number of genes vary between different biological states. We propose a computationally fast procedure for testing the equality of two large covariance matrices when the dimensions of the covariance matrices are much larger than the sample sizes. A distinguishing feature of the new procedure is that it imposes no structural assumptions on the unknown covariance matrices. Hence, the test is robust with respect to various complex dependence structures that frequently arise in genomics. We prove that the proposed procedure is asymptotically valid under weak moment conditions. As an interesting application, we derive a new gene clustering algorithm which shares the same nice property of avoiding restrictive structural assumptions for high-dimensional genomics data. Using an asthma gene expression dataset, we illustrate how the new test helps compare the covariance matrices of the genes across different gene sets/pathways between the disease group and the control group, and how the gene clustering algorithm provides new insights on the way gene clustering patterns differ between the two groups. The proposed methods have been implemented in an R-package HDtest and are available on CRAN.},
	number = {1},
	journal = {Biometrics},
	author = {Chang, Jinyuan and Zhou, Wen and Zhou, Wen-Xin and Wang, Lan},
	year = {2017},
	keywords = {Hypothesis testing, Sparsity, Differential expression analysis, Gene clustering, High dimension, Parametric bootstrap},
	pages = {31--41},
}

@article{muller_integral_1997,
	title = {Integral {Probability} {Metrics} and {Their} {Generating} {Classes} of {Functions}},
	volume = {29},
	issn = {0001-8678, 1475-6064},
	doi = {10.2307/1428011},
	abstract = {We consider probability metrics of the following type: for a class  of functions and probability measures P, Q we define  A unified study of such integral probability metrics is given. We characterize the maximal class of functions that generates such a metric. Further, we show how some interesting properties of these probability metrics arise directly from conditions on the generating class of functions. The results are illustrated by several examples, including the Kolmogorov metric, the Dudley metric and the stop-loss metric.},
	number = {2},
	journal = {Advances in Applied Probability},
	author = {Müller, Alfred},
	month = jun,
	year = {1997},
	publisher = {Cambridge University Press},
	keywords = {60B10, 60E05, INTEGRAL PROBABILITY METRICS, MAXIMAL GENERATOR, UNIFORMITY IN WEAK CONVERGENCE: STOP-LOSS METRIC},
	pages = {429--443},
}

@article{rao_diversity_1982,
	title = {Diversity and dissimilarity coefficients: {A} unified approach},
	volume = {21},
	issn = {0040-5809},
	shorttitle = {Diversity and dissimilarity coefficients},
	doi = {10.1016/0040-5809(82)90004-1},
	abstract = {Three general methods for obtaining measures of diversity within a population and dissimilarity between populations are discussed. One is based on an intrinsic notion of dissimilarity between individuals and others make use of the concepts of entropy and discrimination. The use of a diversity measure in apportionment of diversity between and within populations is discussed.},
	language = {en},
	number = {1},
	urldate = {2022-08-19},
	journal = {Theoretical Population Biology},
	author = {Rao, C. Radhakrishna},
	month = feb,
	year = {1982},
	pages = {24--43},
}

@article{burbea_differential_1984,
	title = {Differential metrics in probability spaces},
	volume = {3},
	issn = {0208-4147},
	abstract = {In this paper we discuss the construction of differential metrics in probability spaces through entropy functional and examine their relations with the information metric introduced by Rao using the Fisher information matrix in the statistical problem of classification and discrimination, and the classical Bergman metric. It is suggested that the scalar and Ricci curvatures associated with the Bergman information metric may yield results in statistical inference analogous to those of Efron using the Gaussian curvature.},
	number = {2},
	journal = {Probability and Mathematical Statistics},
	author = {Burbea, Jacob and Radhakrishna Rao, C.},
	year = {1984},
	number = {2}, 
	publisher = {Publisher: Kazimierz Urbanik Center for Probability and Mathematical Statistics},
	pages = {241--258},
}

@article{csiszar_informationstheoretische_1963,
	title = {Eine informationstheoretische {Ungleichung} und ihre {Anwendung} auf den {Beweis} der {Ergodizität} von {Markoffschen} {Ketten}},
	volume = {8},
	issn = {0541-9514},
	journal = {A Magyar Tudományos Akadémia. Matematikai Kutató Intézetének Közleményei},
	author = {Csiszár, Imre},
	year = {1963},
	mrnumber = {164374},
	pages = {85--108},
}

@misc{huang_efficient_2017,
	title = {An {Efficient} and {Distribution}-{Free} {Two}-{Sample} {Test} {Based} on {Energy} {Statistics} and {Random} {Projections}},
	doi = {10.48550/arXiv.1707.04602},
	abstract = {A common disadvantage in existing distribution-free two-sample testing approaches is that the computational complexity could be high. Specifically, if the sample size is \$N\$, the computational complexity of those two-sample tests is at least \$O(N{\textasciicircum}2)\$. In this paper, we develop an efficient algorithm with complexity \$O(N {\textbackslash}log N)\$ for computing energy statistics in univariate cases. For multivariate cases, we introduce a two-sample test based on energy statistics and random projections, which enjoys the \$O(K N {\textbackslash}log N)\$ computational complexity, where \$K\$ is the number of random projections. We name our method for multivariate cases as Randomly Projected Energy Statistics (RPES). We can show RPES achieves nearly the same test power with energy statistics both theoretically and empirically. Numerical experiments also demonstrate the efficiency of the proposed method over the competitors.},
	publisher = {arXiv},
	author = {Huang, Cheng and Huo, Xiaoming},
	month = jul,
	year = {2017},
	note = {arXiv:1707.04602 [stat]},
	keywords = {Statistics - Methodology},
}

@article{aslan_statistical_2005,
	title = {Statistical energy as a tool for binning-free, multivariate goodness-of-fit tests, two-sample comparison and unfolding},
	volume = {537},
	issn = {0168-9002},
	doi = {10.1016/j.nima.2004.08.071},
	abstract = {We introduce the novel concept of statistical energy as a statistical tool. We define statistical energy of statistical distributions in a similar way as for electric charge distributions. Charges of opposite sign are in a state of minimum energy if they are equally distributed. This property is used to check whether two samples belong to the same parent distribution, to define goodness-of-fit tests and to unfold distributions distorted by measurement. The approach is binning-free and especially powerful in multidimensional applications.},
	language = {en},
	number = {3},
	journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment},
	author = {Aslan, B. and Zech, G.},
	month = feb,
	year = {2005},
	keywords = {Goodness-of-fit, Two-sample test, Multivariate, Binning-free, Deconvolution, Unfolding},
	pages = {626--636},
}

@article{barakat_multivariate_1996,
	title = {Multivariate {Homogeneity} {Testing} {Using} an {Extended} {Concept} of {Nearest} {Neighbors}},
	volume = {38},
	issn = {1521-4036},
	doi = {10.1002/bimj.4710380509},
	abstract = {Given independent multivariate random samples Xij: j = 1, …, ni from Fi, for i = 1,2, a test is desired for H0: F1 = F2 against general alternatives. Consider the k · (n1 + n2) possible ways of choosing one observation from the combined samples and then one of its k nearest neighbors, and let Sk be the proportion of these choices in which the point and neighbor are in the same sample. Schilling (1986) proposed Sk as a test statistic, but did not indicate how to determine k. We suggest as test statistic W = N Σ kSk, which we show is equivalent to a sum of N Wilcoxon rank sums, and also to a sum of two two-sample U-statistics of degrees (1, 2) and (2, 1). Simulation with multivariate normal data suggests that our test is generally more powerful than Schilling's test using k = 1, 2, or 3. We illustrate its use with Fisher's iris data.},
	number = {5},
	journal = {Biometrical Journal},
	author = {Barakat, Ali S. and Quade, Dana and Salama, Ibrahim A.},
	year = {1996},
	keywords = {Homogeneity Testing, Nearest Neighbors},
	pages = {605--612},
}

@article{baringhaus_new_2004,
	title = {On a {New} {Multivariate} {Two}-{Sample} {Test}},
	volume = {88},
	issn = {0047-259X},
	doi = {10.1016/S0047-259X(03)00079-4},
	abstract = {In this paper we propose a new test for the multivariate two-sample problem. The test statistic is the difference of the sum of all the Euclidean interpoint distances between the random variables from the two different samples and one-half of the two corresponding sums of distances of the variables within the same sample. The asymptotic null distribution of the test statistic is derived using the projection method and shown to be the limit of the bootstrap distribution. A simulation study includes the comparison of univariate and multivariate normal distributions for location and dispersion alternatives. For normal location alternatives the new test is shown to have power similar to that of the t- and T2-Test.},
	number = {1},
	journal = {Journal of Multivariate Analysis},
	author = {Baringhaus, L. and Franz, C.},
	month = jan,
	year = {2004},
	keywords = {Bootstrapping, Cramér test, Multivariate two-sample test, Orthogonal invariance, Projection method},
	pages = {190--206},
}

@article{biswas_nonparametric_2014,
	title = {A {Nonparametric} {Two}-{Sample} {Test} {Applicable} to {High} {Dimensional} {Data}},
	volume = {123},
	issn = {0047-259X},
	doi = {10.1016/j.jmva.2013.09.004},
	abstract = {The multivariate two-sample testing problem has been well investigated in the literature, and several parametric and nonparametric methods are available for it. However, most of these two-sample tests perform poorly for high dimensional data, and many of them are not applicable when the dimension of the data exceeds the sample size. In this article, we propose a multivariate two-sample test that can be conveniently used in the high dimension low sample size setup. Asymptotic results on the power properties of our proposed test are derived when the sample size remains fixed, and the dimension of the data grows to infinity. We investigate the performance of this test on several high-dimensional simulated and real data sets, and demonstrate its superiority over several other existing two-sample tests. We also study some theoretical properties of the proposed test for situations when the dimension of the data remains fixed and the sample size tends to infinity. In such cases, it turns out to be asymptotically distribution-free and consistent under general alternatives.},
	journal = {Journal of Multivariate Analysis},
	author = {Biswas, Munmun and Ghosh, Anil K.},
	month = jan,
	year = {2014},
	keywords = {Permutation test, U-statistic, High dimensional asymptotics, Inter-point distances, Large sample distribution, Weak law of large numbers},
	pages = {160--171},
}

@article{burke_multivariate_2000,
	title = {Multivariate tests-of-fit and uniform confidence bands using a weighted bootstrap},
	volume = {46},
	issn = {0167-7152},
	doi = {10.1016/S0167-7152(99)00082-6},
	abstract = {Many tests of fit procedures use the empirical distribution function (e.d.f.) of the data and have limiting distribution dependent on the data's underlying distribution or family of distribution functions. This paper uses a weighted bootstrap method based on independent random variables instead of sampling from the uniform. A proof of convergence of the weighted bootstrap is given using strong martingales. This approach is applied to the problem of obtaining uniform confidence bands for the distribution function of multivariate data. The multivariate two-sample problem, testing whether two independent random samples come from the same multivariate distribution function, is also discussed.},
	number = {1},
	journal = {Statistics \& Probability Letters},
	author = {Burke, Murray D},
	month = jan,
	year = {2000},
	keywords = {Kolmogorov-Smirnov statistics, Multivariate confidence bands, Multivariate two-sample problem, Strong martingales, Weighted bootstrap},
	pages = {13--20},
}

@article{cao_goodness--fit_2005,
	title = {Goodness-of-fit {Tests} {Based} on the {Kernel} {Density} {Estimator}},
	volume = {32},
	issn = {1467-9469},
	doi = {10.1111/j.1467-9469.2005.00471.x},
	abstract = {Abstract. Given an i.i.d. sample drawn from a density f on the real line, the problem of testing whether f is in a given class of densities is considered. Testing procedures constructed on the basis of minimizing the L1-distance between a kernel density estimate and any density in the hypothesized class are investigated. General non-asymptotic bounds are derived for the power of the test. It is shown that the concentration of the data-dependent smoothing factor and the ‘size’ of the hypothesized class of densities play a key role in the performance of the test. Consistency and non-asymptotic performance bounds are established in several special cases, including testing simple hypotheses, translation/scale classes and symmetry. Simulations are also carried out to compare the behaviour of the method with the Kolmogorov-Smirnov test and an L2 density-based approach due to Fan [Econ. Theory10 (1994) 316].},
	number = {4},
	journal = {Scandinavian Journal of Statistics},
	author = {Cao, Ricardo and Lugosi, Gábor},
	year = {2005},
	keywords = {bandwidth, goodness-of-fit test, kernel density estimator, smoothing factor selection},
	pages = {599--616},
}

@article{chen_ensemble_2013,
	title = {Ensemble {Subsampling} for {Imbalanced} {Multivariate} {Two}-{Sample} {Tests}},
	volume = {108},
	issn = {0162-1459},
	doi = {10.1080/01621459.2013.800763},
	abstract = {Some existing nonparametric two-sample tests for equality of multivariate distributions perform unsatisfactorily when the two sample sizes are unbalanced. In particular, the power of these tests tends to diminish with increasingly unbalanced sample sizes. In this article, we propose a new testing procedure to solve this problem. The proposed test, based on the nearest neighbor method by Schilling, employs a novel ensemble subsampling scheme to remedy this issue. More specifically, the test statistic is a weighted average of a collection of statistics, each associated with a randomly selected subsample of the data. We derive the asymptotic distribution of the test statistic under the null hypothesis and show that the new test is consistent against all alternatives when the ratio of the sample sizes either goes to a finite limit or tends to infinity. Via simulated data examples we demonstrate that the new test has increasing power with increasing sample size ratio when the size of the smaller sample is fixed. The test is applied to a real-data example in the field of corporate finance. Supplementary materials for this article are available online.},
	number = {504},
	journal = {Journal of the American Statistical Association},
	author = {Chen, Lisha and Dou, Winston   Wei and Qiao, Zhihua},
	month = dec,
	year = {2013},
	publisher = {Taylor \& Francis},
	keywords = {“Ensemble Subsampling for Imbalanced Multivariate Two-Sample Tests,”, Corporate finance, Ensemble methods, Imbalanced learning, Kolmogorov–Smirnov test, Nearest neighbors methods, Nonparametric two-sample tests, Subsampling methods},
	pages = {1308--1323},
}

@article{danafar_testing_2014,
	title = {Testing {Hypotheses} by {Regularized} {Maximum} {Mean} {Discrepancy}},
	volume = {02},
	abstract = {Regularized Maximum Mean Discrepancy (RMMD), our novel measure for kernel-based hypothesis testing, excels at hypothesis tests involving multiple comparisons with power control even when sample sizes are small. We derive asymptotic distributions under the null and alternative hypotheses, and assess power control. Outstanding results are obtained on challenging benchmark datasets.},
	number = {06},
	journal = {International Journal of Computer and Information Technology},
	author = {Danafar, Somayeh and Rancoita, Paola M.V. and Glasmachers, Tobias and Whittinstal, Kevin and Schmidhuber, Jürgen},
	year = {2014},
	pages = {223--232},
}

@incollection{durbin_1_1973,
	series = {{CBMS}-{NSF} {Regional} {Conference} {Series} in {Applied} {Mathematics}},
	title = {1. {Distribution} {Theory} for {Tests} {Based} on the {Sample} {Distribution} {Function}},
	isbn = {978-0-89871-007-6},
	abstract = {In this paper we define and examine the power of the conditional-sampling oracle in the context of distribution-property testing. The conditional-sampling oracle for a discrete distribution \${\textbackslash}mu\$ takes as input a subset \$S {\textbackslash}subset [n]\$ of the domain, and outputs a random sample \$i {\textbackslash}in S\$ drawn according to \${\textbackslash}mu\$, conditioned on \$S\$ (and independently of all prior samples). The conditional-sampling oracle is a natural generalization of the ordinary sampling oracle, in which \$S\$ always equals \$[n]\$. We show that with the conditional-sampling oracle, testing uniformity, testing identity to a known distribution, and testing any label-invariant property of distributions is easier than with the ordinary sampling oracle. On the other hand, we also show that for some distribution properties the sample-complexity remains near-maximal even with conditional sampling.},
	booktitle = {Distribution {Theory} for {Tests} {Based} on the {Sample} {Distribution} {Function}},
	publisher = {Society for Industrial and Applied Mathematics},
	author = {Durbin, J.},
	month = jan,
	year = {1973},
	doi = {10.1137/1.9781611970586.ch1},
	keywords = {Markov, one-sample problem, sampling distributions, stochastic processes, test statistics, two-sample problem},
	pages = {1--64},
}

@inproceedings{eric_testing_2007,
	title = {Testing for {Homogeneity} with {Kernel} {Fisher} {Discriminant} {Analysis}},
	volume = {20},
	abstract = {We propose to test for the homogeneity of two samples by using Kernel Fisher discriminant Analysis. This provides us with a consistent nonparametric test statistic, for which we derive the asymptotic distribution under the null hypothesis. We give experimental evidence of the relevance of our method on both artificial and real datasets.},
	urldate = {2022-08-25},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Eric, Moulines and Bach, Francis and Harchaoui, Zaïd},
	year = {2007},
}

@article{alba-fernandez_test_2008,
	title = {A test for the two-sample problem based on empirical characteristic functions},
	volume = {52},
	issn = {0167-9473},
	doi = {10.1016/j.csda.2007.12.013},
	abstract = {A class of tests for the two sample problem that is based on the empirical characteristic function is investigated. They can be applied to continuous as well as to discrete data of any arbitrary fixed dimension. The tests are consistent against any fixed alternatives for adequate choices of the weight function involved in the definition of the test statistic. Both the bootstrap and the permutation procedures can be employed to estimate consistently the null distribution. The goodness of these approximations and the power of some tests in this class for finite sample sizes are investigated by simulation.},
	language = {en},
	number = {7},
	urldate = {2022-08-25},
	journal = {Computational Statistics \& Data Analysis},
	author = {Alba Fernández, V. and Jiménez Gamero, M. D. and Muñoz García, J.},
	month = mar,
	year = {2008},
	pages = {3730--3748},
}

@misc{karoui_can_2016,
	title = {Can we trust the bootstrap in high-dimension?},
	doi = {10.48550/arXiv.1608.00696},
	abstract = {We consider the performance of the bootstrap in high-dimensions for the setting of linear regression, where \$p{\textless}n\$ but \$p/n\$ is not close to zero. We consider ordinary least-squares as well as robust regression methods and adopt a minimalist performance requirement: can the bootstrap give us good confidence intervals for a single coordinate of \${\textbackslash}beta\$? (where \${\textbackslash}beta\$ is the true regression vector). We show through a mix of numerical and theoretical work that the bootstrap is fraught with problems. Both of the most commonly used methods of bootstrapping for regression -- residual bootstrap and pairs bootstrap -- give very poor inference on \${\textbackslash}beta\$ as the ratio \$p/n\$ grows. We find that the residuals bootstrap tend to give anti-conservative estimates (inflated Type I error), while the pairs bootstrap gives very conservative estimates (severe loss of power) as the ratio \$p/n\$ grows. We also show that the jackknife resampling technique for estimating the variance of \${\textbackslash}hat\{{\textbackslash}beta\}\$ severely overestimates the variance in high dimensions. We contribute alternative bootstrap procedures based on our theoretical results that mitigate these problems. However, the corrections depend on assumptions regarding the underlying data-generation model, suggesting that in high-dimensions it may be difficult to have universal, robust bootstrapping techniques.},
	publisher = {arXiv},
	author = {Karoui, Noureddine El and Purdom, Elizabeth},
	month = aug,
	year = {2016},
	keywords = {Statistics - Machine Learning, Mathematics - Statistics Theory, Statistics - Methodology, 62G09, 62J99},
}

@article{lindsay_kernels_2014,
	title = {Kernels, {Degrees} of {Freedom}, and {Power} {Properties} of {Quadratic} {Distance} {Goodness}-of-{Fit} {Tests}},
	volume = {109},
	issn = {0162-1459},
	doi = {10.1080/01621459.2013.836972},
	abstract = {In this article, we study the power properties of quadratic-distance-based goodness-of-fit tests. First, we introduce the concept of a root kernel and discuss the considerations that enter the selection of this kernel. We derive an easy to use normal approximation to the power of quadratic distance goodness-of-fit tests and base the construction of a noncentrality index, an analogue of the traditional noncentrality parameter, on it. This leads to a method akin to the Neyman-Pearson lemma for constructing optimal kernels for specific alternatives. We then introduce a midpower analysis as a device for choosing optimal degrees of freedom for a family of alternatives of interest. Finally, we introduce a new diffusion kernel, called the Pearson-normal kernel, and study the extent to which the normal approximation to the power of tests based on this kernel is valid. Supplementary materials for this article are available online.},
	number = {505},
	urldate = {2022-08-25},
	journal = {Journal of the American Statistical Association},
	author = {Lindsay, Bruce G. and Markatou, Marianthi and Ray, Surajit},
	month = jan,
	year = {2014},
	pmid = {24764609},
	publisher = {Taylor \& Francis},
	keywords = {Big data, High-dimensional testing, Midpower analysis, Optimal kernel construction, Pearson-normal kernel, Power lemma},
	pages = {395--410},
}

@article{lindsay_quadratic_2008,
	title = {Quadratic distances on probabilities: {A} unified foundation},
	volume = {36},
	issn = {0090-5364, 2168-8966},
	shorttitle = {Quadratic distances on probabilities},
	doi = {10.1214/009053607000000956},
	abstract = {This work builds a unified framework for the study of quadratic form distance measures as they are used in assessing the goodness of fit of models. Many important procedures have this structure, but the theory for these methods is dispersed and incomplete. Central to the statistical analysis of these distances is the spectral decomposition of the kernel that generates the distance. We show how this determines the limiting distribution of natural goodness-of-fit tests. Additionally, we develop a new notion, the spectral degrees of freedom of the test, based on this decomposition. The degrees of freedom are easy to compute and estimate, and can be used as a guide in the construction of useful procedures in this class.},
	number = {2},
	journal = {The Annals of Statistics},
	author = {Lindsay, Bruce G. and Markatou, Marianthi and Ray, Surajit and Yang, Ke and Chen, Shu-Chuan},
	month = apr,
	year = {2008},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62A01, 62E20, 62H10, Degrees of freedom, diffusion kernel, goodness of fit, high dimensions, model assessment, quadratic distance, spectral decomposition},
	pages = {983--1006},
}

@article{liu_triangle_2011,
	title = {A triangle test for equality of distribution functions in high dimensions},
	volume = {23},
	issn = {1048-5252},
	doi = {10.1080/10485252.2010.485644},
	abstract = {A triangle statistic is proposed for testing the equality of two multivariate continuous distributions in high-dimensional settings based on sample interpoint distances. Given two independent p-dimensional random samples, a triangle can be formed by randomly selecting one observation from one sample and two observations from the other sample. The triangle statistic estimates the probability that the distance between the two observations from the same distribution is the largest, the middle or the smallest in the triangle being formed by these three observations. We show that the test based on the triangle statistic is asymptotically distribution-free under the null hypothesis of equal, but unknown continuous distribution functions. The triangle test is compared with other nonparametric tests through a simulation study. The triangle statistic is well defined when the number of variables p is larger than the number of observations m, and its computational complexity is independent of p, making it suitable for high-dimensional settings.},
	number = {3},
	journal = {Journal of Nonparametric Statistics},
	author = {Liu, Zhenyu and Modarres, Reza},
	month = sep,
	year = {2011},
	publisher = {Taylor \& Francis},
	keywords = {interpoint distance, U-statistic, distribution function, high dimension, triangle test},
	pages = {605--615},
}

@article{maag_vnm_1968,
	title = {The {VNM} {Two}-{Sample} {Test}},
	volume = {39},
	issn = {0003-4851},
	abstract = {The statistic VNM is a two-sample statistic which may be used to test the null hypothesis H0, that two samples, sizes N and M, come from identical populations. It is an adaptation of the Kolmogorov two-sample statistic, and is defined by \$V\_\{NM\} = {\textbackslash}sup\_\{-{\textbackslash}infty{\textless}x{\textless}{\textbackslash}infty\} (F\_N(x) - G\_M(x)) - {\textbackslash}inf\_\{-{\textbackslash}infty{\textless}x{\textless}{\textbackslash}infty\} (F\_N(x) - G\_M(x))\$ where FN(x), GM(x) are the sample cumulative distribution functions. A single-sample analogue VN is defined by replacing GM(x) in the formula above by a hypothesised distribution function F(x). For large values of VNM or VN, H0 will be rejected. These statistics are particularly useful for observations recorded as points on a circle; the value obtained for VN or VNM, in contrast to that of the corresponding Kolmogorov statistic, does not depend on the choice of origin. Kuiper (1960) proved this result and suggested the use of the V statistics for the circle. He also gave series approximations to the distributions of N1/2VN and N1/2VNN, for large N, on the null hypothesis H0. Reference to a distribution will henceforth be assumed to refer to the distribution on the null hypothesis. The VNN statistic had earlier been investigated by Gnedenko and co-workers (see Gnedenko (1954)), who used it for observations on a line; the V statistics may be expected to be more powerful than the Kolmogorov statistics for certain alternatives. Gnedenko (1954) gives the exact distribution of VNN. The exact distribution of VN, in the upper and lower tails, has recently been given by Stephens (1965). In this paper we make the two-sample goodness-of-fit test available for a wide range of sample sizes by giving tables of the distribution of VNM. In the next section a formula is given with which the VNM statistic may be calculated from the ranks of the two samples, and then the goodness-of-fit tests are set out. These are called exact or approximate tests, depending on whether the probabilities used are calculated from exact or approximate formulae. In Section 3 the construction of the tables is described. To find percentage points for large N, we develop in Section 3.3 a series expansion for the distribution of N1/2VNN which differs from that given by Kuiper; the probabilities given by the two expansions are compared in Table 4, and the new series clearly gives the better results.},
	number = {3},
	urldate = {2022-08-25},
	journal = {The Annals of Mathematical Statistics},
	author = {Maag, Urs R. and Stephens, M. A.},
	year = {1968},
	publisher = {Institute of Mathematical Statistics},
	pages = {923--935},
}

@article{mondal_high_2015,
	title = {On high dimensional two-sample tests based on nearest neighbors},
	volume = {141},
	issn = {0047-259X},
	doi = {10.1016/j.jmva.2015.07.002},
	abstract = {In this article, we propose new multivariate two-sample tests based on nearest neighbor type coincidences. While several existing tests for the multivariate two-sample problem perform poorly for high dimensional data, and many of them are not applicable when the dimension exceeds the sample size, these proposed tests can be conveniently used in the high dimension low sample size (HDLSS) situations. Unlike Schilling (1986) [26] and Henze’s (1988) test based on nearest neighbors, under fairly general conditions, these new tests are found to be consistent in HDLSS asymptotic regime, where the sample size remains fixed and the dimension grows to infinity. Several high dimensional simulated and real data sets are analyzed to compare their empirical performance with some popular two-sample tests available in the literature. We further investigate the behavior of these proposed tests in classical asymptotic regime, where the dimension of the data remains fixed and the sample size tends to infinity. In such cases, they turn out to be asymptotically distribution-free and consistent under general alternatives.},
	journal = {Journal of Multivariate Analysis},
	author = {Mondal, Pronoy K. and Biswas, Munmun and Ghosh, Anil K.},
	month = oct,
	year = {2015},
	keywords = {Central limit theorem, Permutation test, HDLSS data, Large sample test, Law of large numbers, Level and power of a test},
	pages = {168--178},
}

@article{scholz_k-sample_1987,
	title = {K-{Sample} {Anderson}–{Darling} {Tests}},
	volume = {82},
	issn = {0162-1459},
	doi = {10.1080/01621459.1987.10478517},
	abstract = {Two k-sample versions of an Anderson–Darling rank statistic are proposed for testing the homogeneity of samples. Their asymptotic null distributions are derived for the continuous as well as the discrete case. In the continuous case the asymptotic distributions coincide with the (k – 1)-fold convolution of the asymptotic distribution for the Anderson–Darling one-sample statistic. The quality of this large sample approximation is investigated for small samples through Monte Carlo simulation. This is done for both versions of the statistic under various degrees of data rounding and sample size imbalances. Tables for carrying out these tests are provided, and their usage in combining independent one- or k-sample Anderson–Darling tests is pointed out. The test statistics are essentially based on a doubly weighted sum of integrated squared differences between the empirical distribution functions of the individual samples and that of the pooled sample. One weighting adjusts for the possibly different sample sizes, and the other is inside the integration placing more weight on tail differences of the compared distributions. The two versions differ mainly in the definition of the empirical distribution function. These tests are consistent against all alternatives. The use of these tests is two-fold: (a) in a one-way analysis of variance to establish differences in the sampled populations without making any restrictive parametric assumptions or (b) to justify the pooling of separate samples for increased sample size and power in further analyses. Exact finite sample mean and variance formulas for one of the two statistics are derived in the continuous case. It appears that the asymptotic standardized percentiles serve well as approximate critical points of the appropriately standardized statistics for individual sample sizes as low as 5. The application of the tests is illustrated with an example. Because of the convolution nature of the asymptotic distribution, a further use of these critical points is possible in combining independent Anderson–Darling tests by simply adding their test statistics.},
	number = {399},
	journal = {Journal of the American Statistical Association},
	author = {Scholz, F. W. and Stephens, M. A.},
	month = sep,
	year = {1987},
	publisher = {Taylor \& Francis},
	keywords = {Simulation, Consistency, Combining tests, Convolution, Empirical processes, Pearson curves},
	pages = {918--924},
}

@article{ramdas_decreasing_2015,
	title = {On the {Decreasing} {Power} of {Kernel} and {Distance} {Based} {Nonparametric} {Hypothesis} {Tests} in {High} {Dimensions}},
	volume = {29},
	copyright = {Copyright (c)},
	issn = {2374-3468},
	doi = {10.1609/aaai.v29i1.9692},
	abstract = {This paper is about two related decision theoretic problems, nonparametric two-sample testing and independence testing. There is a belief that two recently proposed solutions, based on kernels and distances between pairs of points, behave well in high-dimensional settings. We identify different sources of misconception that give rise to the above belief. Specifically, we differentiate the hardness of estimation of test statistics from the hardness of testing whether these statistics are zero or not, and explicitly discuss a notion of "fair" alternative hypotheses for these problems as dimension increases. We then demonstrate that the power of these tests actually drops polynomially with increasing dimension against fair alternatives. We end with some theoretical insights and shed light on the median heuristic for kernel bandwidth selection. Our work advances the current understanding of the power of modern nonparametric hypothesis tests in high dimensions.},
	number = {1},
	journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
	author = {Ramdas, Aaditya and Reddi, Sashank Jakkam and Poczos, Barnabas and Singh, Aarti and Wasserman, Larry},
	month = mar,
	year = {2015},
}

@article{szekely_energy_2013,
	title = {Energy statistics: {A} class of statistics based on distances},
	volume = {143},
	issn = {0378-3758},
	shorttitle = {Energy statistics},
	doi = {10.1016/j.jspi.2013.03.018},
	abstract = {Energy distance is a statistical distance between the distributions of random vectors, which characterizes equality of distributions. The name energy derives from Newton's gravitational potential energy, and there is an elegant relation to the notion of potential energy between statistical observations. Energy statistics are functions of distances between statistical observations in metric spaces. Thus even if the observations are complex objects, like functions, one can use their real valued nonnegative distances for inference. Theory and application of energy statistics are discussed and illustrated. Finally, we explore the notion of potential and kinetic energy of goodness-of-fit.},
	number = {8},
	journal = {Journal of Statistical Planning and Inference},
	author = {Székely, Gábor J. and Rizzo, Maria L.},
	month = aug,
	year = {2013},
	keywords = {Goodness-of-fit, Distance covariance, Energy distance, Distance correlation, Multivariate independence},
	pages = {1249--1272},
}

@inproceedings{zaremba_b-test_2013,
	title = {B-test: {A} {Non}-parametric, {Low} {Variance} {Kernel} {Two}-sample {Test}},
	volume = {26},
	shorttitle = {B-test},
	abstract = {We propose a family of maximum mean discrepancy (MMD) kernel two-sample tests that have low sample complexity and are consistent. The test has a hyperparameter that allows one to control the tradeoff between sample complexity and computational time. Our family of tests, which we denote as B-tests, is both computationally and statistically efficient, combining favorable properties of previously proposed MMD two-sample tests.  It does so by better leveraging samples to produce low variance estimates in the finite sample case, while avoiding a quadratic number of kernel evaluations and complex null-hypothesis approximation as would be required by tests relying on one sample U-statistics. The B-test uses a smaller than quadratic number of kernel evaluations and avoids completely the computational burden of complex null-hypothesis approximation while maintaining consistency and probabilistically conservative thresholds on Type I error. Finally, recent results of combining multiple kernels transfer seamlessly to our hypothesis test, allowing a further increase in discriminative power and decrease in sample complexity.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Zaremba, Wojciech and Gretton, Arthur and Blaschko, Matthew},
	year = {2013},
}

@book{basu_statistical_2011,
	title = {Statistical {Inference}: {The} {Minimum} {Distance} {Approach}},
	isbn = {978-1-4200-9966-9},
	shorttitle = {Statistical {Inference}},
	abstract = {In many ways, estimation by an appropriate minimum distance method is one of the most natural ideas in statistics. However, there are many different ways of constructing an appropriate distance between the data and the model: the scope of study referred to by "Minimum Distance Estimation" is literally huge. Filling a statistical resource gap, Stati},
	publisher = {CRC Press},
	author = {Basu, Ayanendranath and Shioya, Hiroyuki and Park, Chanseok},
	month = jun,
	year = {2011},
	keywords = {Mathematics / Probability \& Statistics / General, Computers / General},
}

@article{song_generalized_2021,
	title = {Generalized {Kernel} {Two}-{Sample} {Tests}},
	issn = {1464-3510},
	doi = {10.1093/biomet/asad068},
	abstract = {Kernel two-sample tests have been widely used for multivariate data to test equality of distributions. However, existing tests based on mapping distributions into a reproducing kernel Hilbert space mainly target specific alternatives and do not work well for some scenarios when the dimension of the data is moderate to high due to the curse of dimensionality. We propose a new test statistic that makes use of a common pattern under moderate and high dimensions and achieves substantial power improvements over existing kernel two-sample tests for a wide range of alternatives. We also propose alternative testing procedures that maintain high power with low computational cost, offering easy off-the-shelf tools for large datasets. The new approaches are compared to other state-of-the-art tests under various settings and show good performance. We showcase the new approaches through two applications: the comparison of musks and nonmusks using the shape of molecules, and the comparison of taxi trips starting from John F. Kennedy airport in consecutive months. All proposed methods are implemented in an R package kerTests.},
	journal = {Biometrika},
	author = {Song, Hoseung and Chen, Hao},
	month = nov,
	year = {2023},
	pages = {755--770},
}


@misc{song_new_2022,
	title = {New {Graph}-{Based} {Multi}-{Sample} {Tests} for {High}-{Dimensional} and {Non}-{Euclidean} {Data}},
	doi = {10.48550/arXiv.2205.13787},
	abstract = {Testing the equality in distributions of multiple samples is a common task in many fields. However, this problem for high-dimensional or non-Euclidean data has not been well explored. In this paper, we propose new nonparametric tests based on a similarity graph constructed on the pooled observations from multiple samples, and make use of both within-sample edges and between-sample edges, a straightforward but yet not explored idea. The new tests exhibit substantial power improvements over existing tests for a wide range of alternatives. We also study the asymptotic distributions of the test statistics, offering easy off-the-shelf tools for large datasets. The new tests are illustrated through an analysis of the age image dataset.},
	publisher = {arXiv},
	author = {Song, Hoseung and Chen, Hao},
	month = may,
	year = {2022},
	note = {arXiv:2205.13787 [stat]},
	keywords = {Statistics - Methodology},
	file = {arXiv Fulltext PDF:C\:\\Users\\stolte\\Zotero\\storage\\UY29QZNQ\\Song und Chen - 2022 - New graph-based multi-sample tests for high-dimens.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\stolte\\Zotero\\storage\\YVHPTF7W\\2205.html:text/html},
}

@misc{dowd__aut_twosamples_2022,
	title = {twosamples: {Fast} {Permutation} {Based} {Two} {Sample} {Tests}},
	shorttitle = {twosamples},
	url = {https://CRAN.R-project.org/package=twosamples},
	abstract = {Fast randomization based two sample tests. Testing the hypothesis that two samples come from the same distribution using randomization to create p-values. Included tests are: Kolmogorov-Smirnov, Kuiper, Cramer-von Mises, Anderson-Darling, Wasserstein, and DTS. The default test (two\_sample) is based on the DTS test statistic, as it is the most powerful, and thus most useful to most users. The DTS test statistic builds on the Wasserstein distance by using a weighting scheme like that of Anderson-Darling. See the companion paper at {\textless}arXiv:2007.01360{\textgreater} or {\textless}https://codowd.com/public/DTS.pdf{\textgreater} for details of that test statistic, and non-standard uses of the package (parallel for big N, weighted observations, one sample tests, etc). We also include the permutation scheme to make test building simple for others.},
	author = {Dowd  [aut, Connor and cre},
	month = jul,
	year = {2022},
}

@misc{you__aut_maotai_2022,
	title = {maotai: {Tools} for {Matrix} {Algebra}, {Optimization} and {Inference}},
	copyright = {MIT + file LICENSE},
	shorttitle = {maotai},
	url = {https://CRAN.R-project.org/package=maotai},
	abstract = {Matrix is an universal and sometimes primary object/unit in applied mathematics and statistics. We provide a number of algorithms for selected problems in optimization and statistical inference. For general exposition to the topic with focus on statistical context, see the book by Banerjee and Roy (2014, ISBN:9781420095388).},
	urldate = {2022-08-25},
	author = {You  [aut, Kisung and cre},
	month = feb,
	year = {2022},
}

@misc{scholz_ksamples_2019,
	title = {{kSamples}: {K}-{Sample} {Rank} {Tests} and their {Combinations}},
	copyright = {GPL-2 {\textbar} GPL-3 [expanded from: GPL (≥ 2)]},
	shorttitle = {{kSamples}},
	url = {https://CRAN.R-project.org/package=kSamples},
	abstract = {Compares k samples using the Anderson-Darling test, Kruskal-Wallis type tests with different rank score criteria, Steel's multiple comparison test, and the Jonckheere-Terpstra (JT) test. It computes asymptotic, simulated or (limited) exact P-values, all valid under randomization, with or without ties, or conditionally under random sampling from populations, given the observed tie pattern. Except for Steel's test and the JT test it also combines these tests across several blocks of samples. Also analyzed are 2 x t contingency tables and their blocked combinations using the Kruskal-Wallis criterion. Steel's test is inverted to provide simultaneous confidence bounds for shift parameters. A plotting function compares tail probabilities obtained under asymptotic approximation with those obtained via simulation or exact calculations.},
	urldate = {2022-08-25},
	author = {Scholz, Fritz and Zhu, Angie},
	month = may,
	year = {2019},
}

@misc{laurent_kantorovich_2022,
	title = {kantorovich: {Kantorovich} {Distance} {Between} {Probability} {Measures}},
	copyright = {GPL-3},
	shorttitle = {kantorovich},
	url = {https://CRAN.R-project.org/package=kantorovich},
	abstract = {Computes the Kantorovich distance between two probability measures on a finite set. The Kantorovich distance is also known as the Monge-Kantorovich distance or the first Wasserstein distance.},
	urldate = {2022-08-25},
	author = {Laurent, Stéphane},
	month = aug,
	year = {2022},
}

@misc{rocha_twosampletesthd_2022,
	title = {{TwoSampleTest}.{HD}: {A} {Two}-{Sample} {Test} for the {Equality} of {Distributions} for {High}-{Dimensional} {Data}},
	copyright = {GPL-2},
	shorttitle = {{TwoSampleTest}.{HD}},
	url = {https://CRAN.R-project.org/package=TwoSampleTest.HD},
	abstract = {For high-dimensional data whose main feature is a large number, p, of variables but a small sample size, the null hypothesis that the marginal distributions of p variables are the same for two groups is tested. We propose a test statistic motivated by the simple idea of comparing, for each of the p variables, the empirical characteristic functions computed from the two samples. If one rejects this global null hypothesis of no differences in distributions between the two groups, a set of permutation p-values is reported to identify which variables are not equally distributed in both groups.},
	urldate = {2022-08-25},
	author = {Rocha, Marta Cousido and González, José Carlos Soage and Álvarez, Jacobo de Uña and Hart, Jeffrey D.},
	month = apr,
	year = {2022},
}

@article{banerjee_nearest-neighbor_2020,
	title = {A nearest-neighbor based nonparametric test for viral remodeling in heterogeneous single-cell proteomic data},
	volume = {14},
	issn = {1932-6157, 1941-7330},
	doi = {10.1214/20-AOAS1362},
	abstract = {An important problem in contemporary immunology studies based on single-cell protein expression data is to determine whether cellular expressions are remodeled postinfection by a pathogen. One natural approach for detecting such changes is to use nonparametric two-sample statistical tests. However, in single-cell studies direct application of these tests is often inadequate, because single-cell level expression data from processed uninfected populations often contain attributes of several latent subpopulations with highly heterogeneous characteristics. As a result, viruses often infect these different subpopulations at different rates, in which case the traditional nonparametric two-sample tests for checking similarity in distributions are no longer conservative. In this paper, we propose a new nonparametric method for Testing Remodeling under Heterogeneity (TRUH) that can accurately detect changes in the infected samples compared to possibly heterogeneous uninfected samples. Our testing framework is based on composite nulls and is designed to allow the null model to encompass the possibility that the infected samples, though unaltered by the virus, might be dominantly arising from underrepresented subpopulations in the baseline data. The TRUH statistic, which uses nearest neighbor projections of the infected samples into the baseline uninfected population, is calibrated using a novel bootstrap algorithm. We demonstrate the nonasymptotic performance of the test via simulation experiments and also derive the large sample limit of the test statistic which provides theoretical support toward consistent asymptotic calibration of the test. We use the TRUH statistic for studying remodeling in tonsillar T cells under different types of HIV infection and find that, unlike traditional tests which do not have any heterogeneity correction, TRUH based statistical inference conforms to the biologically validated immunological theories on HIV infection.},
	number = {4},
	journal = {The Annals of Applied Statistics},
	author = {Banerjee, Trambak and Bhattacharya, Bhaswar B. and Mukherjee, Gourab},
	month = dec,
	year = {2020},
	publisher = {Institute of Mathematical Statistics},
	keywords = {HIV infection, homogeneous Poisson process, immunology, mass cytometry, nearest neighbors, Single-cell virology, two-sample tests, viral remodeling},
	pages = {1777--1805},
}

@misc{abbas_robnptests_2021,
	title = {robnptests: {Robust} {Nonparametric} {Two}-{Sample} {Tests} for {Location}/{Scale}},
	copyright = {GPL-2 {\textbar} GPL-3 [expanded from: GPL (≥ 2)]},
	shorttitle = {robnptests},
	url = {https://CRAN.R-project.org/package=robnptests},
	abstract = {Implementations of several robust nonparametric two-sample tests for location or scale differences. The test statistics are based on robust location and scale estimators, e.g. the sample median or the Hodges-Lehmann estimators as described in Fried \& Dehling (2011) {\textless}doi:10.1007/s10260-011-0164-1{\textgreater}. The p-values can be computed via the permutation principle, the randomization principle, or by using the asymptotic distributions of the test statistics under the null hypothesis, which ensures (approximate) distribution independence of the test decision. To test for a difference in scale, we apply the tests for location difference to transformed observations; see Fried (2012) {\textless}doi:10.1016/j.csda.2011.02.012{\textgreater}. Random noise on a small range can be added to the original observations in order to hold the significance level on data from discrete distributions. The location tests assume homoscedasticity and the scale tests require the location parameters to be zero.},
	urldate = {2022-08-25},
	author = {Abbas, Sermad and Brune, Barbara and Fried, Roland},
	month = nov,
	year = {2021},
}

@misc{chen_kertests_2021,
	title = {{kerTests}: {Generalized} {Kernel} {Two}-{Sample} {Tests}},
	copyright = {GPL-2 {\textbar} GPL-3 [expanded from: GPL (≥ 2)]},
	shorttitle = {{kerTests}},
	url = {https://CRAN.R-project.org/package=kerTests},
	abstract = {New kernel-based test and fast tests for testing whether two samples are from the same distribution. They work well particularly for high-dimensional data.},
	urldate = {2022-08-25},
	author = {Chen, Hoseung Song {and} Hao},
	month = sep,
	year = {2021},
}

@misc{simon_hyporf_2021,
	title = {{hypoRF}: {Random} {Forest} {Two}-{Sample} {Tests}},
	copyright = {GPL-3},
	shorttitle = {{hypoRF}},
	url = {https://CRAN.R-project.org/package=hypoRF},
	abstract = {An implementation of Random Forest-based two-sample tests as introduced in Hediger \& Michel \& Naef (2020) {\textless}arXiv:1903.06287{\textgreater}.},
	urldate = {2022-08-25},
	author = {Simon, Hediger and Michel, Loris and Naef, Jeffrey},
	month = may,
	year = {2021},
}

@misc{zhang_highdmean_2020,
	title = {{highDmean}: {Testing} {Two}-{Sample} {Mean} in {High} {Dimension}},
	copyright = {GPL-2},
	shorttitle = {{highDmean}},
	url = {https://CRAN.R-project.org/package=highDmean},
	abstract = {Implements the high-dimensional two-sample test proposed by Zhang (2019) {\textless}http://hdl.handle.net/2097/40235{\textgreater}. It also implements the test proposed by Srivastava, Katayama, and Kano (2013) {\textless}doi:10.1016/j.jmva.2012.08.014{\textgreater}. These tests are particularly suitable to high dimensional data from two populations for which the classical multivariate Hotelling's T-square test fails due to sample sizes smaller than dimensionality. In this case, the ZWL and ZWLm tests proposed by Zhang (2019) {\textless}http://hdl.handle.net/2097/40235{\textgreater}, referred to as zwl\_test() in this package, provide a reliable and powerful test.},
	urldate = {2022-08-25},
	author = {Zhang, Huaiyu and Wang, Haiyan},
	month = jun,
	year = {2020},
}

@article{zhang_more_2019,
	title = {More powerful two-sample tests for univariate and high-dimensional data},
	abstract = {Comparing the means of two populations is a common task in scientiﬁc studies. In this dissertation, we consider more powerful tests for testing the equality of means for univariate and high-dimensional settings. In the univariate case, the classical two-sample t-test is not robust to skewed population, and the large-sample test has low accuracy for ﬁnite sample sizes. The ﬁrst part of this dissertation proposes two new types of tests, the TCFU, and the TT tests, for comparing means with unequal-variance populations. The TCFU test uses Welch’s t-statistic as the test statistic and the Cornish-Fisher expansion as its critical values. The TT tests transform Welch’s t-statistic and use the normal percentiles as critical values. Four types of monotone transformations are considered for the TT tests. Power and type I error rate comparison of different tests are conducted theoretically and numerically. Analytical conditions are derived to help practitioners choose a powerful test. Two real-data examples are presented to illustrate the application of the new tests.
	
	The second part considers a more challenging situation: testing the equality of two high-dimensional means. When the sample sizes are much smaller than the dimensionality, it is not viable to construct a uniformly most powerful test. Here we propose a new test based on the average squared component-wise t-statistic. Our new test shares some similarity with the generalized component test (GCT) proposed by Gregory et al. (2015), but it differs from the latter test in the following aspects: (i) our new test constructs a different scaling parameter that can be directly estimated from the data instead of from the t-statistics sequence. (ii) it does not require the stationarity condition implicitly assumed in the GCT
	test; (iii) the new variance estimator guarantees non-negativeness as it is supposed to have; (iv) the test works well even when components of the data vector have high correlations, as long as such correlation reduces suitably fast as the separation of the component indices increases (at least with polynomial rate). The limiting distribution of the test statistic and the power function are derived. The new test is also compared with several other existing tests through Monte Carlo experiments. With acute lymphoblastic leukemia gene expression data, we demonstrated how the new test can be used to give more consistent results in detecting differently expressed Gene Ontology terms than competing tests.
	
	In the last part of the dissertation, we consider power adjustments to address a question of how to fairly compare the power of competing methods in simulation studies when they have different empirical type I error rates. After discussing some existing methods and their drawbacks, we introduce a new power adjustment method. The new power adjustment method is used to compare the simulation results in the previous two parts of the dissertation.},
	author = {Zhang, Huaiyu},
	year = {2019},
}

@misc{song_gtestsmulti_2022,
	title = {{gTestsMulti}: {New} {Graph}-{Based} {Multi}-{Sample} {Tests}},
	copyright = {GPL-2 {\textbar} GPL-3 [expanded from: GPL (≥ 2)]},
	shorttitle = {{gTestsMulti}},
	url = {https://CRAN.R-project.org/package=gTestsMulti},
	abstract = {New multi-sample tests for testing whether multiple samples are from the same distribution. They work well particularly for high-dimensional data. Song, H. and Chen, H. (2022) {\textless}arXiv:2205.13787{\textgreater}.},
	urldate = {2022-08-25},
	author = {Song, Hoseung and Chen, Hao},
	month = jun,
	year = {2022},
}


@article{chen_graph-based_2013,
	title = {{Graph}-{Based} {Tests} for {Two}-{Sample} {Comparisons} of {Categorical} {Data}},
	volume = {23},
	issn = {1017-0405},
	abstract = {We study the problem of two-sample comparison with categorical data when the contingency table is sparsely populated. In modern applications, the number of categories is often comparable to the sample size, causing existing methods to have low power. When the number of categories is large, there is often underlying structure on the sample space that can be exploited. We propose a general non-parametric approach that utilizes similarity information on the space of all categories in two sample tests. Our approach extends the graph-based tests of Friedman and Rafsky (1979) and Rosenbaum (2005), which are tests base on graphs connecting observations by similarity. Both tests require uniqueness of the underlying graph and cannot be directly applied on categorical data. We explored different ways to extend graph-based tests to the categorical setting and found two types of statistics that are both powerful and fast to compute. We showed that their permutation null distributions are asymptotically normal and that their p-value approximations under typical settings are quite accurate, facilitating the application of the new approach. The approach is illustrated through several examples.},
	number = {4},
	journal = {Statistica Sinica},
	author = {Chen, Hao and Zhang, Nancy R.},
	year = {2013},
	publisher = {Institute of Statistical Science, Academia Sinica},
	pages = {1479--1503},
}

@misc{zhang_gcat_2022,
	title = {{gCat}: {Graph}-{Based} {Two}-{Sample} {Tests} for {Categorical} {Data}},
	copyright = {GPL-2 {\textbar} GPL-3 [expanded from: GPL (≥ 2)]},
	shorttitle = {{gCat}},
	url = {https://CRAN.R-project.org/package=gCat},
	abstract = {These are two-sample tests for categorical data utilizing similarity information among the categories. They are useful when there is underlying structure on the categories.},
	urldate = {2022-08-25},
	author = {Zhang, Hao Chen {and} Nancy R.},
	month = jun,
	year = {2022},
}

@misc{puritz_fasanofranceschinitest_2022,
	title = {fasano.franceschini.test: {Fasano}-{Franceschini} {Test}: {A} {Multidimensional} {Kolmogorov}-{Smirnov} {Two}-{Sample} {Test}},
	copyright = {MIT + file LICENSE},
	shorttitle = {fasano.franceschini.test},
	url = {https://CRAN.R-project.org/package=fasano.franceschini.test},
	abstract = {An implementation of the two-sample multidimensional Kolmogorov-Smirnov test described by Fasano and Franceschini (1987) {\textless}doi:10.1093/mnras/225.1.155{\textgreater}. This test evaluates the null hypothesis that two i.i.d. random samples were drawn from the same underlying probability distribution. The data can be of any dimension, and can be of any type (continuous, discrete, or mixed).},
	urldate = {2022-08-25},
	author = {Puritz, Connor and Ness-Cohn, Elan and Braun, Rosemary and class.), Luca Weihs (Copyright holder {and} author of 'RangeTree'},
	month = aug,
	year = {2022},
}

@article{fasano_multidimensional_1987,
	title = {A multidimensional version of the {Kolmogorov}–{Smirnov} test},
	volume = {225},
	issn = {0035-8711},
	doi = {10.1093/mnras/225.1.155},
	abstract = {We discuss a generalization of the classical Kolmogorov–Smirnov test, which is suitable to analyse random samples defined in two or three dimensions. This test provides some improvements with respect to an earlier version proposed by Peacock. In particular: (i) it is faster, by a factor equal to the sample size, n, and then usable to analyse quite sizeable samples; (ii) it fully takes into account the dependence of the test statistics on the degree of correlation of data points and on the sample size; (iii) it allows for a generalization to the three-dimensional case which is still viable as regards computing time. Supported by a large number of Monte Carlo simulations, we are ensured that this test is sufficiently distribution-free for any practical purposes. We also give a simple analytic expression to make easier the calculation of the critical values of the test probability distribution. To illustrate how the test works, we use it to analyse models of the cosmological evolution of X-ray selected active galactic nuclei and we show that it is a much more sensitive goodness-of-fit test than the χ2.},
	number = {1},
	urldate = {2022-08-25},
	journal = {Monthly Notices of the Royal Astronomical Society},
	author = {Fasano, G. and Franceschini, A.},
	month = mar,
	year = {1987},
	pages = {155--170},
	file = {Full Text PDF:C\:\\Users\\stolte\\Zotero\\storage\\645M3SFE\\Fasano und Franceschini - 1987 - A multidimensional version of the Kolmogorov–Smirn.pdf:application/pdf},
}

@misc{franz_cramer_2019,
	title = {cramer: {Multivariate} {Nonparametric} {Cramer}-{Test} for the {Two}-{Sample}-{Problem}},
	copyright = {GPL-2 {\textbar} GPL-3 [expanded from: GPL (≥ 2)]},
	shorttitle = {cramer},
	url = {https://CRAN.R-project.org/package=cramer},
	abstract = {Provides R routine for the so called two-sample Cramer-Test. This nonparametric two-sample-test on equality of the underlying distributions can be applied to multivariate data as well as univariate data. It offers two possibilities to approximate the critical value both of which are included in this package.},
	urldate = {2022-08-25},
	author = {Franz, Carsten},
	month = jan,
	year = {2019},
}

@misc{mkirchler_source_2022,
	title = {Source code for the paper "{Two}-sample {Testing} {Using} {Deep} {Learning}"},
	url = {https://github.com/mkirchler/deep-2-sample-test},
	abstract = {Code for the Paper "Two-sample Testing Using Deep Learning", https://arxiv.org/abs/1910.06239},
	urldate = {2022-08-29},
	author = {mkirchler},
	month = may,
	year = {2022},
}

@inproceedings{jitkrittum_informative_2018,
	title = {Informative {Features} for {Model} {Comparison}},
	volume = {31},
	abstract = {Given two candidate models, and a set of target observations, we address the problem of measuring the relative goodness of fit of the two models. We propose two new statistical tests which are nonparametric, computationally efficient (runtime complexity is linear in the sample size), and interpretable. As a unique advantage, our tests can produce a set of examples (informative features) indicating the regions in the data domain where one model fits significantly better than the other. In a real-world problem of comparing GAN models, the test power of our new test matches that of the state-of-the-art test of relative goodness of fit, while being one order of magnitude faster.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Jitkrittum, Wittawat and Kanagawa, Heishiro and Sangkloy, Patsorn and Hays, James and Schölkopf, Bernhard and Gretton, Arthur},
	year = {2018},
}

@article{kim_classification_2021,
	title = {Classification accuracy as a proxy for two-sample testing},
	volume = {49},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/20-AOS1962},
	abstract = {When data analysts train a classifier and check if its accuracy is significantly different from chance, they are implicitly performing a two-sample test. We investigate the statistical properties of this flexible approach in the high-dimensional setting. We prove two results that hold for all classifiers in any dimensions: if its true error remains \${\textbackslash}epsilon \$-better than chance for some \${\textbackslash}epsilon {\textgreater}0\$ as \$d,n{\textbackslash}to {\textbackslash}infty \$, then (a) the permutation-based test is consistent (has power approaching to one), (b) a computationally efficient test based on a Gaussian approximation of the null distribution is also consistent. To get a finer understanding of the rates of consistency, we study a specialized setting of distinguishing Gaussians with mean-difference \${\textbackslash}delta \$ and common (known or unknown) covariance \${\textbackslash}Sigma \$, when \$d/n{\textbackslash}to c{\textbackslash}in (0,{\textbackslash}infty )\$. We study variants of Fisher’s linear discriminant analysis (LDA) such as “naive Bayes” in a nontrivial regime when \${\textbackslash}epsilon {\textbackslash}to 0\$ (the Bayes classifier has true accuracy approaching 1/2), and contrast their power with corresponding variants of Hotelling’s test. Surprisingly, the expressions for their power match exactly in terms of \$n\$, \$d\$, \${\textbackslash}delta \$, \${\textbackslash}Sigma \$, and the LDA approach is only worse by a constant factor, achieving an asymptotic relative efficiency (ARE) of \$1/{\textbackslash}sqrt\{{\textbackslash}pi \}\$ for balanced samples. We also extend our results to high-dimensional elliptical distributions with finite kurtosis. Other results of independent interest include minimax lower bounds, and the optimality of Hotelling’s test when \$d=o(n)\$. Simulation results validate our theory, and we present practical takeaway messages along with natural open problems.},
	number = {1},
	journal = {The Annals of Statistics},
	author = {Kim, Ilmun and Ramdas, Aaditya and Singh, Aarti and Wasserman, Larry},
	month = feb,
	year = {2021},
	publisher = {Institute of Mathematical Statistics},
	keywords = {Permutation test, 62H15, 62E20, Classification accuracy, high-dimensional asymptotics, Hotelling’s \$T{\textasciicircum}\{2\}\$ test, linear discriminant analysis, two sample testing},
	pages = {411--434},
}

@inproceedings{arbel_gradient_2018,
	title = {On gradient regularizers for {MMD} {GANs}},
	volume = {31},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Arbel, Michael and Sutherland, Danica J. and Bińkowski, Mikołaj and Gretton, Arthur},
	year = {2018},
}

@misc{binkowski_demystifying_2021,
	title = {Demystifying {MMD} {GANs}},
	doi = {10.48550/arXiv.1801.01401},
	abstract = {We investigate the training and performance of generative adversarial networks using the Maximum Mean Discrepancy (MMD) as critic, termed MMD GANs. As our main theoretical contribution, we clarify the situation with bias in GAN loss functions raised by recent work: we show that gradient estimators used in the optimization process for both MMD GANs and Wasserstein GANs are unbiased, but learning a discriminator based on samples leads to biased gradients for the generator parameters. We also discuss the issue of kernel choice for the MMD critic, and characterize the kernel corresponding to the energy distance used for the Cramer GAN critic. Being an integral probability metric, the MMD benefits from training strategies recently developed for Wasserstein GANs. In experiments, the MMD GAN is able to employ a smaller critic network than the Wasserstein GAN, resulting in a simpler and faster-training algorithm with matching performance. We also propose an improved measure of GAN convergence, the Kernel Inception Distance, and show how to use it to dynamically adapt learning rates during GAN training.},
	publisher = {arXiv},
	author = {Bińkowski, Mikołaj and Sutherland, Danica J. and Arbel, Michael and Gretton, Arthur},
	month = jan,
	year = {2021},
	note = {arXiv:1801.01401 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@book{shawe-taylor_kernel_2004,
	address = {Cambridge, UNITED KINGDOM},
	title = {Kernel {Methods} for {Pattern} {Analysis}},
	isbn = {978-0-511-21060-0},
	abstract = {Kernel methods provide a powerful and unified framework for pattern discovery, motivating algorithms that can act on general types of data (e.g. strings, vectors or text) and look for general types of relations (e.g. rankings, classifications, regressions, clusters). The application areas range from neural networks and pattern recognition to machine learning and data mining. This book, developed from lectures and tutorials, fulfils two major roles: firstly it provides practitioners with a large toolkit of algorithms, kernels and solutions ready to use for standard pattern discovery problems in fields such as bioinformatics, text analysis, image analysis. Secondly it provides an easy introduction for students and researchers to the growing field of kernel-based pattern analysis, demonstrating with examples how to handcraft an algorithm or a kernel for a new specific application, and covering all the necessary conceptual and mathematical tools to do so.},
	publisher = {Cambridge University Press},
	author = {Shawe-Taylor, John and Cristianini, Nello},
	year = {2004},
	keywords = {Machine learning},
}

@article{cao_empirical_2006,
	title = {Empirical likelihood tests for two-sample problems via nonparametric density estimation},
	volume = {34},
	issn = {1708-945X},
	doi = {10.1002/cjs.5550340106},
	abstract = {The authors study the problem of testing whether two populations have the same law by comparing kernel estimators of the two density functions. The proposed test statistic is based on a local empirical likelihood approach. They obtain the asymptotic distribution of the test statistic and propose a bootstrap approximation to calibrate the test. A simulation study is carried out in which the proposed method is compared with two competitors, and a procedure to select the bandwidth parameter is studied. The proposed test can be extended to more than two samples and to multivariate distributions.},
	number = {1},
	journal = {Canadian Journal of Statistics},
	author = {Cao, Ricardo and van Keilegom, Ingrid},
	year = {2006},
	keywords = {Bandwidth selection, bootstrap, comparison of two populations, empirical likelihood, hypothesis testing, kernel method, local empirical likelihood},
	pages = {61--77},
}

@article{chen_ensemble_2013-1,
	title = {Ensemble {Subsampling} for {Imbalanced} {Multivariate} {Two}-{Sample} {Tests}},
	volume = {108},
	issn = {0162-1459},
	doi = {10.1080/01621459.2013.800763},
	abstract = {Some existing nonparametric two-sample tests for equality of multivariate distributions perform unsatisfactorily when the two sample sizes are unbalanced. In particular, the power of these tests tends to diminish with increasingly unbalanced sample sizes. In this article, we propose a new testing procedure to solve this problem. The proposed test, based on the nearest neighbor method by Schilling, employs a novel ensemble subsampling scheme to remedy this issue. More specifically, the test statistic is a weighted average of a collection of statistics, each associated with a randomly selected subsample of the data. We derive the asymptotic distribution of the test statistic under the null hypothesis and show that the new test is consistent against all alternatives when the ratio of the sample sizes either goes to a finite limit or tends to infinity. Via simulated data examples we demonstrate that the new test has increasing power with increasing sample size ratio when the size of the smaller sample is fixed. The test is applied to a real-data example in the field of corporate finance. Supplementary materials for this article are available online.},
	number = {504},
	journal = {Journal of the American Statistical Association},
	author = {Chen, Lisha and Dou, Winston   Wei and Qiao, Zhihua},
	month = dec,
	year = {2013},
	publisher = {Taylor \& Francis},
	keywords = {“Ensemble Subsampling for Imbalanced Multivariate Two-Sample Tests,”, Corporate finance, Ensemble methods, Imbalanced learning, Kolmogorov–Smirnov test, Nearest neighbors methods, Nonparametric two-sample tests, Subsampling methods},
	pages = {1308--1323},
}

@article{cheng_two-sample_2020,
	title = {Two-sample statistics based on anisotropic kernels},
	volume = {9},
	issn = {2049-8772},
	doi = {10.1093/imaiai/iaz018},
	abstract = {The paper introduces a new kernel-based Maximum Mean Discrepancy (MMD) statistic for measuring the distance between two distributions given finitely many multivariate samples. When the distributions are locally low-dimensional, the proposed test can be made more powerful to distinguish certain alternatives by incorporating local covariance matrices and constructing an anisotropic kernel. The kernel matrix is asymmetric; it computes the affinity between \$n\$ data points and a set of \$n\_R\$ reference points, where \$n\_R\$ can be drastically smaller than \$n\$. While the proposed statistic can be viewed as a special class of Reproducing Kernel Hilbert Space MMD, the consistency of the test is proved, under mild assumptions of the kernel, as long as \${\textbackslash}{\textbar}p-q{\textbackslash}{\textbar} {\textbackslash}sqrt\{n\} {\textbackslash}to {\textbackslash}infty \$, and a finite-sample lower bound of the testing power is obtained. Applications to flow cytometry and diffusion MRI datasets are demonstrated, which motivate the proposed approach to compare distributions.},
	number = {3},
	journal = {Information and Inference: A Journal of the IMA},
	author = {Cheng, Xiuyuan and Cloninger, Alexander and Coifman, Ronald R},
	month = sep,
	year = {2020},
	pages = {677--719},
}

@article{harchaoui_testing_2008,
	title = {Testing for {Homogeneity} with {Kernel} {Fisher} {Discriminant} {Analysis}},
	abstract = {We propose to investigate test statistics for testing homogeneity in reproducing kernel Hilbert spaces. Asymptotic null distributions under null hypothesis are derived, and consistency against fixed and local alternatives is assessed. Finally, experimental evidence of the performance of the proposed approach on both artificial data and a speaker verification task is provided.},
	author = {Harchaoui, Zaid and Bach, Francis and Moulines, Eric},
	month = apr,
	year = {2008},
	keywords = {covariance operator, reproducing kernel Hilbert space, statistical hypothesis testing},
}

@misc{scetbon_comparing_2019,
	title = {Comparing distributions: \${\textbackslash}ell\_1\$ geometry improves kernel two-sample testing},
	shorttitle = {Comparing distributions},
	doi = {10.48550/arXiv.1909.09264},
	abstract = {Are two sets of observations drawn from the same distribution? This problem is a two-sample test. Kernel methods lead to many appealing properties. Indeed state-of-the-art approaches use the \$L{\textasciicircum}2\$ distance between kernel-based distribution representatives to derive their test statistics. Here, we show that \$L{\textasciicircum}p\$ distances (with \$p{\textbackslash}geq 1\$) between these distribution representatives give metrics on the space of distributions that are well-behaved to detect differences between distributions as they metrize the weak convergence. Moreover, for analytic kernels, we show that the \$L{\textasciicircum}1\$ geometry gives improved testing power for scalable computational procedures. Specifically, we derive a finite dimensional approximation of the metric given as the \${\textbackslash}ell\_1\$ norm of a vector which captures differences of expectations of analytic functions evaluated at spatial locations or frequencies (i.e, features). The features can be chosen to maximize the differences of the distributions and give interpretable indications of how they differs. Using an \${\textbackslash}ell\_1\$ norm gives better detection because differences between representatives are dense as we use analytic kernels (non-zero almost everywhere). The tests are consistent, while much faster than state-of-the-art quadratic-time kernel-based tests. Experiments on artificial and real-world problems demonstrate improved power/time tradeoff than the state of the art, based on \${\textbackslash}ell\_2\$ norms, and in some cases, better outright power than even the most expensive quadratic-time tests.},
	publisher = {arXiv},
	author = {Scetbon, M. and Varoquaux, G.},
	month = sep,
	year = {2019},
	note = {arXiv:1909.09264 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@article{wan_empirical_2018,
	title = {Empirical likelihood test for equality of two distributions using distance of characteristic functions},
	volume = {52},
	issn = {02331888},
	doi = {10.1080/02331888.2018.1520855},
	abstract = {In this paper, we investigate the problem of testing for the equality of two distributions. We employ a two-sample Jackknife Empirical Likelihood (JEL) approach to construct a test statistic whose limiting distribution is Chi-square distribution with degree of freedom 1, no matter what the data dimension (fixed) is. A variety of synthetic data experiments demonstrate that our JEL test statistic performs very well, with a very neat asymptotic distribution under the null hypothesis. Furthermore, we apply the test procedure to a real dataset to obtain competitive results.},
	number = {6},
	journal = {Statistics},
	author = {Wan, Yi and Liu, Zhi and Deng, Min},
	month = dec,
	year = {2018},
	publisher = {Taylor \& Francis Ltd},
	keywords = {empirical likelihood, characteristic function, CHARACTERISTIC functions, EMPIRICAL Bayes methods, Jackknife, JACKKNIFE (Statistics), NULL hypothesis, U-statistics},
	pages = {1379--1394},
}

@article{petrie_graph-theoretic_2016,
	title = {Graph-{Theoretic} {Multisample} {Tests} of {Equality} in {Distribution} for {High} {Dimensional} {Data}},
	volume = {96},
	issn = {0167-9473},
	doi = {10.1016/j.csda.2015.11.003},
	abstract = {Testing whether two or more independent samples arise from a common distribution is a classic problem in statistics. Several multivariate two-sample tests of equality are based on graphs such as the minimum spanning tree, nearest neighbor, and optimal nonbipartite perfect matching. Here, the samples are pooled and the test statistic is the number of edges in the graph that connect points with different sample identities. These tests are typically unbiased and perform well when estimates of underlying probability densities are poor. However, these tests have not been thoroughly studied when data is very high dimensional or in the multisample case. We introduce the use of orthogonal perfect matchings for testing equality in distribution. A suite of Monte Carlo simulations on artificial and real data shows that orthogonal perfect matchings and spanning trees typically have higher power than other graphs and are also more effective at discerning when samples have differences in their covariance structure compared to other nonparametric tests such as the energy and triangle tests.},
	journal = {Computational Statistics \& Data Analysis},
	author = {Petrie, Adam},
	month = apr,
	year = {2016},
	keywords = {Minimum spanning tree, Nearest neighbor, Energy, Multisample problem, Orthogonal graph, Perfect matching},
	pages = {145--158},
}

@inproceedings{li_projective_2020,
	title = {On a projective ensemble approach to two sample test for equality of distributions},
	abstract = {In this work, we propose a robust test for the multivariate two-sample problem through projective ensemble, which is a generalization of the Cramer-von Mises statistic. The proposed test statistic has a simple closed-form expression without any tuning parameters involved, it is easy to implement can be computed in quadratic time. Moreover, our test is insensitive to the dimension and consistent against all fixed alternatives, it does not require the moment assumption and is robust to the presence of outliers. We study the asymptotic behaviors of the test statistic under the null and two kinds of alternative hypotheses. We also suggest a permutation procedure to approximate critical values and employ its consistency. We demonstrate the effectiveness of our test through extensive simulation studies and a real data application.},
	booktitle = {Proceedings of the 37th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Li, Zhimei and Zhang, Yaowu},
	month = nov,
	year = {2020},
	issn = {2640-3498},
	pages = {6020--6027},
}


@article{de_la_sierra_clinical_2011,
	title = {Clinical features of 8295 patients with resistant hypertension classified on the basis of ambulatory blood pressure monitoring},
	volume = {57},
	issn = {1524-4563},
	doi = {10.1161/HYPERTENSIONAHA.110.168948},
	abstract = {We aimed to estimate the prevalence of resistant hypertension through both office and ambulatory blood pressure monitoring in a large cohort of treated hypertensive patients from the Spanish Ambulatory Blood Pressure Monitoring Registry. In addition, we also compared clinical features of patients with true or white-coat-resistant hypertension. In December 2009, we identified 68 045 treated patients with complete information for this analysis. Among them, 8295 (12.2\% of the database) had resistant hypertension (office blood pressure ≥140 and/or 90 mm Hg while being treated with ≥3 antihypertensive drugs, 1 of them being a diuretic). After ambulatory blood pressure monitoring, 62.5\% of patients were classified as true resistant hypertensives, the remaining 37.5\% having white-coat resistance. The former group was younger, more frequently men, with a longer duration of hypertension and a worse cardiovascular risk profile. The group included larger proportions of smokers, diabetics, target organ damage (including left ventricular hypertrophy, impaired renal function, and microalbuminuria), and documented cardiovascular disease. Moreover, true resistant hypertensives exhibited in a greater proportion a riser pattern (22\% versus 18\%; P{\textless}0.001). In conclusion, this study first reports the prevalence of resistant hypertension in a large cohort of patients in usual daily practice. Resistant hypertension is present in 12\% of the treated hypertensive population, but among them more than one third have normal ambulatory blood pressure. A worse risk profile is associated with true resistant hypertension, but this association is weak, thus making it necessary to assess ambulatory blood pressure monitoring for a correct diagnosis and management.},
	number = {5},
	journal = {Hypertension (Dallas, Tex.: 1979)},
	author = {de la Sierra, Alejandro and Segura, Julián and Banegas, José R. and Gorostidi, Manuel and de la Cruz, Juan J. and Armario, Pedro and Oliveras, Anna and Ruilope, Luis M.},
	month = may,
	year = {2011},
	pmid = {21444835},
	keywords = {Humans, Age Factors, Aged, Antihypertensive Agents, Blood Pressure, Blood Pressure Monitoring, Ambulatory, Chi-Square Distribution, Female, Hypertension, Male, Middle Aged, Prevalence, Registries, Risk, Sex Factors, Smoking, Spain, Statistics, Nonparametric},
	pages = {898--902},
}

@article{tsukada_high_2019,
	title = {High dimensional two-sample test based on the inter-point distance},
	volume = {34},
	abstract = {The multivariate two-sample problem has been extensively investigated, and various methods have been proposed. However, most two-sample tests perform poorly when applied to high-dimensional data, and many of them are not applicable when the dimension of the data exceeds the sample size. We reconsider two previously reported tests (Baringhaus and Franz in Stat Sin 20:1333–1361, 2010; Biswas and Ghosh in J Multivar Anal 123:160–171, 2014), and propose two new criteria. Simulations demonstrate that the power of the proposed test is stable for high-dimensional data and large samples, and the power of our test is equivalent to that of the test by Biswas and Ghosh when the covariance matrices are different. We also investigate the theoretical properties of our test when the dimension tends to infinity and the sample size is fixed, and when the dimension is fixed and the sample size tends to infinity. In these cases, the proposed test is asymptotically distribution-free and consistent.},
	number = {2},
	journal = {Computational Statistics},
	author = {Tsukada, Shin-ichi},
	year = {2019},
	publisher = {Springer},
	keywords = {High-dimensional data, Nonparametric test, Inter-point distance, Two-sample},
	pages = {599--615},
}

@incollection{chan_optimal_2013,
	series = {Proceedings},
	title = {Optimal {Algorithms} for {Testing} {Closeness} of {Discrete} {Distributions}},
	isbn = {978-1-61197-338-9},
	abstract = {We study a new framework for property testing of probability distributions, by considering distribution testing algorithms that have access to a conditional sampling oracle. This is an oracle that takes as input a subset \$S {\textbackslash}subseteq [N]\$ of the domain \$[N]\$ of the unknown probability distribution \$\{{\textbackslash}cal D\}\$ and returns a draw from the conditional probability distribution \$\{{\textbackslash}cal D\}\$ restricted to \$S\$.  This new model allows considerable flexibility in the design of distribution testing algorithms; in particular, testing algorithms in this model can be adaptive. We study a wide range of natural distribution testing problems in this new framework and some of its variants, giving both upper and lower bounds on query complexity. These problems include  testing whether \$\{{\textbackslash}cal D\}\$ is the uniform distribution \$\{{\textbackslash}cal U\}\$; testing whether \$\{{\textbackslash}cal D\} = \{{\textbackslash}cal D\}{\textasciicircum}{\textbackslash}ast\$ for an explicitly provided \$\{{\textbackslash}cal D\}{\textasciicircum}{\textbackslash}ast\$; testing whether two unknown distributions \$\{{\textbackslash}cal D\}\_1\$ and \$\{{\textbackslash}cal D\}\_2\$ are equivalent; and estimating the variation distance between \$\{{\textbackslash}cal D\}\$ and the uniform distribution. At a high level, our main finding is that the new conditional sampling framework we consider is a powerful one:  while all the problems mentioned above have \${\textbackslash}Omega({\textbackslash}sqrt\{N\})\$ sample complexity in the standard model (and in some cases the complexity must be almost linear in \$N\$), we give \$\{{\textbackslash}rm poly\}({\textbackslash}log N, 1/{\textbackslash}epsilon)\$-query algorithms (and in some cases \$\{{\textbackslash}rm poly\}(1/{\textbackslash}epsilon)\$-query algorithms independent of \$N\$) for all these problems in our conditional sampling setting.},
	booktitle = {Proceedings of the 2014  {Annual} {ACM}-{SIAM} {Symposium} on {Discrete} {Algorithms} ({SODA})},
	publisher = {Society for Industrial and Applied Mathematics},
	author = {Chan, Siu-On and Diakonikolas, Ilias and Valiant, Paul and Valiant, Gregory},
	month = dec,
	year = {2013},
	doi = {10.1137/1.9781611973402.88},
	pages = {1193--1203},
}

@article{ebner_multivariate_2018,
	title = {Multivariate goodness-of-fit on flat and curved spaces via nearest neighbor distances},
	volume = {165},
	issn = {0047-259X},
	doi = {10.1016/j.jmva.2017.12.009},
	abstract = {We present a unified approach to goodness-of-fit testing in Rd and on lower-dimensional manifolds embedded in Rd based on sums of powers of weighted volumes of kth nearest neighbor spheres. We prove asymptotic normality of a class of test statistics under the null hypothesis and under fixed alternatives. Under such alternatives, scaled versions of the test statistics converge to the α-entropy between probability distributions. A simulation study shows that the procedures are serious competitors to established goodness-of-fit tests. The tests are applied to two data sets of gamma-ray bursts in astronomy.},
	journal = {Journal of Multivariate Analysis},
	author = {Ebner, Bruno and Henze, Norbert and Yukich, Joseph E.},
	month = may,
	year = {2018},
	keywords = {-entropy, Gamma-ray burst data, Manifold, Multivariate goodness-of-fit test, Nearest neighbors, Test for uniformity on a circle or a sphere},
	pages = {231--242},
}

@article{weiss_two-sample_1960,
	title = {Two-{Sample} {Tests} for {Multivariate} {Distributions}},
	volume = {31},
	issn = {0003-4851, 2168-8990},
	abstract = {\$X(1), X(2), {\textbackslash}cdots, X(m), Y(1), Y(2), {\textbackslash}cdots, Y(n)\$ are independent \$k\$-variate random variables. The distribution of \$X(i)\$ has pdf \$f(x)\$, say, where \$x\$ denotes a \$k\$-dimensional vector throughout this paper, and the distribution of \$Y(j)\$ has pdf \$g(x)\$, say. We assume that \$f(x)\$ and \$g(x)\$ are piecewise continuous, and that each has a finite upper bound, which it is not necessary to specify. Denote by \$2R\_i\$ the distance from \$X(i)\$ to the nearest of the points \$X(1), {\textbackslash}cdots, X(i - 1), X(i + 1), {\textbackslash}cdots, X(m)\$, and denote by \$S\_i\$ the number of points \$Y(1), {\textbackslash}cdots, Y(n)\$ contained in the open sphere \${\textbackslash}\{x: {\textbar} x - X(i) {\textbar} {\textless} R\_i{\textbackslash}\}\$. Clearly, the joint distribution of \$S\_i, S\_j\$ is the same as the joint distribution of \$S\_\{i'\}, S\_\{j'\}\$, for any subscripts with \$i {\textbackslash}neq j, i' {\textbackslash}neq j'\$. Let \$r\$ be a non-negative integer, and \${\textbackslash}alpha\$ any fixed positive value. \$Q(r)\$ denotes the Lebesgue integral \${\textbackslash}int\_\{E\_k\} {\textbackslash}frac\{2{\textasciicircum}k {\textbackslash}alpha f{\textasciicircum}2 (x){\textbackslash}lbrack g(x) {\textbackslash}rbrack{\textasciicircum}r\}\{{\textbackslash}lbrack g(x) + 2{\textasciicircum}k{\textbackslash}alpha f(x) {\textbackslash}rbrack{\textasciicircum}\{r + 1\}\} dx,\$ where \$E\_k\$ denotes Euclidean \$k\$-space. We will show that \${\textbackslash}lim\_\{m {\textbackslash}rightarrow {\textbackslash}infty, m/n = {\textbackslash}alpha\} P\_\{m, n\}{\textbackslash}lbrack S\_1 = s\_1, S\_2 = s\_2{\textbackslash}rbrack = Q(s\_1)Q(s\_2),\$ for any non-negative integers \$s\_1,s\_2\$, the approach being uniform in \$s\_1,s\_2\$. Thus, in the limit \$S\_1, S\_2\$ are independently distributed, with \${\textbackslash}lim\_\{m {\textbackslash}rightarrow {\textbackslash}infty, m/n = {\textbackslash}alpha\} P\_\{m, n\}{\textbackslash}lbrack S\_1 = s\_1{\textbackslash}rbrack = Q(s\_1).\$ In [1], which discussed the univariate case, \$S\_i\$ was defined as the number of \$Y\$'s closer to \$X(i)\$ than to any other \$X\$ to their right. In the present paper, \$S\_i\$ is defined as the number of \$Y\$'s in another neighborhood of \$X(i)\$. Our present definition of \$S\_i\$ does not become for \$k = 1\$ the same as the definition of \$S\_i\$ in [1]. Rather, in the univariate case, our present definition of \$S\_i\$ is the number of \$Y\$'s lying within a distance \$R\_i\$ on either side of \$X(i)\$. However, if \${\textbackslash}lim\_\{m {\textbackslash}rightarrow {\textbackslash}infty, m/n = {\textbackslash}alpha\} P\_\{m, n\}{\textbackslash}lbrack S\_1, = s\_1, S\_2 = s\_2{\textbackslash}rbrack\$ is computed for the univariate case using the definition of \$S\_i\$ given in [1], the only way in which it differs from \$Q(s\_1)Q(s\_2)\$ is that \${\textbackslash}alpha\$ is replaced by \${\textbackslash}alpha/2\$. Thus it seems reasonable to treat the \$S\_i\$ as defined here as \$k\$-dimensional analogues of the \$S\_i\$ as defined in [1], at least for large samples. An intuitive reason for \${\textbackslash}alpha\$ being replaced by \${\textbackslash}alpha/2\$ is that in our present case, \${\textbackslash}sum{\textasciicircum}m\_\{i = 1\} S\_i\$ may be less than \$n\$, whereas in [1] this sum must always equal \$n\$. Thus in our present case, we are in a sense discarding some of the \$Y\$'s, which lowers \$n\$ relative to \$m\$ and thus raises \${\textbackslash}alpha\$ by a certain factor (2, as it happens). In our present case, \${\textbackslash}sum S\_i\$ may be less than \$n\$ because the \$R\_i\$ are chosen to make the spheres around the \$X\$'s non-overlapping, thus simplifying the analysis. The \$R\_i\$ were chosen to give the largest possible non-overlapping spheres because it would seem intuitively that the larger the spheres, the more rapid the approach of the probabilities to their limiting values.},
	number = {1},
	journal = {The Annals of Mathematical Statistics},
	author = {Weiss, Lionel},
	month = mar,
	year = {1960},
	publisher = {Institute of Mathematical Statistics},
	pages = {159--164},
}

@article{arias-castro_consistency_2016,
	title = {On the consistency of the crossmatch test},
	volume = {171},
	issn = {0378-3758},
	doi = {10.1016/j.jspi.2015.10.003},
	abstract = {Rosenbaum (2005) proposed the crossmatch test for two-sample goodness-of-fit testing in arbitrary dimensions. We prove that the test is consistent against all fixed alternatives. In the process, we develop a general consistency result based on Henze and Penrose (1999) that applies more generally.},
	journal = {Journal of Statistical Planning and Inference},
	author = {Arias-Castro, Ery and Pelletier, Bruno},
	month = apr,
	year = {2016},
	keywords = {Consistency, Goodness-of-fit testing in arbitrary dimensions, Graph-based tests, The crossmatch test},
	pages = {184--190},
}

@misc{ghosal_multivariate_2021,
	title = {Multivariate {Ranks} and {Quantiles} using {Optimal} {Transport}: {Consistency}, {Rates}, and {Nonparametric} {Testing}},
	shorttitle = {Multivariate {Ranks} and {Quantiles} using {Optimal} {Transport}},
	doi = {10.48550/arXiv.1905.05340},
	abstract = {In this paper we study multivariate ranks and quantiles, defined using the theory of optimal transport, and build on the work of Chernozhukov et al.(2017) and Hallin et al.(2021). We study the characterization, computation and properties of the multivariate rank and quantile functions and their empirical counterparts. We derive the uniform consistency of these empirical estimates to their population versions, under certain assumptions. In fact, we prove a Glivenko-Cantelli type theorem that shows the asymptotic stability of the empirical rank map in any direction. Under mild structural assumptions, we provide global and local rates of convergence of the empirical quantile and rank maps. We also provide a sub-Gaussian tail bound for the global L\_2-loss of the empirical quantile function. Further, we propose tuning parameter-free multivariate nonparametric tests -- a two-sample test and a test for mutual independence -- based on our notion of multivariate quantiles/ranks. Asymptotic consistency of these tests are shown and the rates of convergence of the associated test statistics are derived, both under the null and alternative hypotheses.},
	publisher = {arXiv},
	author = {Ghosal, Promit and Sen, Bodhisattva},
	month = may,
	year = {2021},
	note = {arXiv:1905.05340 [math, stat]},
	keywords = {Mathematics - Statistics Theory, 62G30, 62G20, 60F15, 35J96, Mathematics - Probability},
}

@article{nettleton_testing_2001,
	title = {Testing the equality of distributions of random vectors with categorical components},
	volume = {37},
	issn = {0167-9473},
	doi = {10.1016/S0167-9473(01)00015-9},
	abstract = {We develop a method for testing the equality of two or more distributions of random vectors with categorical components. We define a function that gives a distance between any two data vectors. Each observed data vector is linked with its nearest neighbor(s). The test statistic is the number of edges linking observations from different distributions. Inference is conditional on the number of observations from each distribution and the number of times each of the data vectors is observed in the pooled sample. Permutation testing and asymptotics are used to estimate the observed significance level.},
	number = {2},
	journal = {Computational Statistics \& Data Analysis},
	author = {Nettleton, Dan and Banerjee, T},
	month = aug,
	year = {2001},
	keywords = {Sparse data, Permutation test, Categorical data, Conditional inference, Minimal spanning tree, Multivariate methods, Nearest-neighbor graph, Nonparametric methods},
	pages = {195--208},
}

@article{zuo_limiting_2006,
	title = {On the limiting distributions of multivariate depth-based rank sum statistics and related tests},
	volume = {34},
	number = {6},
	journal = {The Annals of Statistics},
	author = {Zuo, Yijun and He, Xuming},
	year = {2006},
	publisher = {Institute of Mathematical Statistics},
	pages = {2879--2896},
}

@article{bai_effect_1996,
	title = {Effect of high dimension: {By} an example of a two sample problem},
	volume = {6},
	issn = {1017-0405},
	shorttitle = {Effect of high dimension},
	abstract = {With the rapid development of modern computing techniques, statisticians are dealing with data with much higher dimension. Consequently, due to their loss of accuracy or power, some classical statistical inferences are being challenged by non-exact approaches. The purpose of this paper is to point out and briefly analyze such a phenomenon and to encourage statisticians to reexamine classical statistical approaches when they are dealing with high dimensional data. As an example, we derive the asymptotic power of the classical Hotelling's T2 test and Dempster's non-exact test for a two-sample problem. Also, an asymptotically normally distributed test statistic is proposed. Our results show that both Dempster's non-exact test and the new test have higher power than Hotelling's test when the data dimension is proportionally close to the within sample degrees of freedom. Although our new test has an asymptotic power function similar to Dempster's, it does not rely on the normality assumption. Some simulation results are presented which show that the non-exact tests are more powerful than Hotelling's test even for moderately large dimension and sample sizes.},
	number = {2},
	journal = {Statistica Sinica},
	author = {Bai, Z. and Saranadasa, H.},
	year = {1996},
	keywords = {Edgeworth expansion, Hotelling T2 test, Hypothesis test, Power function, Significance test, χ2 approximation},
	pages = {311--329},
}

@article{chen_two-sample_2010,
	title = {A two-sample test for high-dimensional data with applications to gene-set testing},
	volume = {38},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/09-AOS716},
	abstract = {We propose a two-sample test for the means of high-dimensional data when the data dimension is much larger than the sample size. Hotelling’s classical T2 test does not work for this “large p, small n” situation. The proposed test does not require explicit conditions in the relationship between the data dimension and sample size. This offers much flexibility in analyzing high-dimensional data. An application of the proposed test is in testing significance for sets of genes which we demonstrate in an empirical study on a leukemia data set.},
	number = {2},
	urldate = {2022-09-01},
	journal = {The Annals of Statistics},
	author = {Chen, Song Xi and Qin, Ying-Li},
	month = apr,
	year = {2010},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62G10, 62H15, high dimension, 60K35, gene-set testing, large p small n, martingale central limit theorem, multiple comparison},
	pages = {808--835},
}

@book{puri_nonparametric_1993,
	title = {Nonparametric {Methods} in {Multivariate} {Analysis}},
	isbn = {978-0-89464-551-8},
	publisher = {Wiley},
	author = {Puri, Madan Lal and Sen, Pranab Kumar},
	year = {1993},
	keywords = {Mathematics / General},
}

@article{baringhaus_rigid_2010,
	title = {{Rigid} {Motion} {Invariant} {Two}-{Sample} {Tests}},
	volume = {20},
	issn = {1017-0405},
	abstract = {New rigid motion invariant tests for the multivariate two-sample problem are proposed. The test statistic is based on the inter-point distances between the two samples and the inter-point distances within each sample. The asymptotic null distribution of the test statistic is a weighted sum of squares of independent unit normal random variables, the weights being the eigenvalues of a certain Hilbert-Schmidt-operator depending on the unknown underlying distribution. An estimate of the limit distribution is obtained by replacing the unknown weights by the eigenvalues of a bootstrapped version of the operator. Quantiles of the estimate are chosen as critical values. The tests are shown to be consistent. Approximate Bahadur efficiencies computed for normal location alternatives, normal scale alternatives, and Lehmann's contaminated alternative are seen to coincide locally with Pitman efficiencies. The results are supported by a simulation study.},
	number = {4},
	urldate = {2022-09-01},
	journal = {Statistica Sinica},
	author = {Baringhaus, L. and Franz, C.},
	year = {2010},
	publisher = {Institute of Statistical Science, Academia Sinica},
	pages = {1333--1361},
}

@misc{boeckel_multivariate_2018,
	title = {Multivariate {Brenier} cumulative distribution functions and their application to non-parametric testing},
	doi = {10.48550/arXiv.1809.04090},
	abstract = {In this work we introduce a novel approach of construction of multivariate cumulative distribution functions, based on cyclical-monotone mapping of an original measure \${\textbackslash}mu {\textbackslash}in {\textbackslash}mathcal\{P\}{\textasciicircum}\{ac\}\_2({\textbackslash}mathbb\{R\}{\textasciicircum}d)\$ to some target measure \${\textbackslash}nu {\textbackslash}in {\textbackslash}mathcal\{P\}{\textasciicircum}\{ac\}\_2({\textbackslash}mathbb\{R\}{\textasciicircum}d)\$ , supported on a convex compact subset of \${\textbackslash}mathbb\{R\}{\textasciicircum}d\$. This map is referred to as \${\textbackslash}nu\$-Brenier distribution function (\${\textbackslash}nu\$-BDF), whose counterpart under the one-dimensional setting \$d = 1\$ is an ordinary CDF, with \${\textbackslash}nu\$ selected as \${\textbackslash}mathcal\{U\}[0, 1]\$, a uniform distribution on \$[0, 1]\$. Following one-dimensional frame-work, a multivariate analogue of Glivenko-Cantelli theorem is provided. A practical applicability of the theory is then illustrated by the development of a non-parametric pivotal two-sample test, that is rested on \$2\$-Wasserstein distance.},
	publisher = {arXiv},
	author = {Boeckel, Melf and Spokoiny, Vladimir and Suvorikova, Alexandra},
	month = sep,
	year = {2018},
	note = {arXiv:1809.04090 [math, stat]},
	keywords = {Mathematics - Statistics Theory},
}

@article{puri_class_1966,
	title = {On a {Class} of {Multivariate} {Multisample} {Rank}-{Order} {Tests}},
	volume = {28},
	issn = {0581-572X},
	abstract = {A class of rank-order tests for the multivariate several sample location and scale problems is proposed and studied here. The principle of rank-permutation tests by Chatterjee and Sen (1964, 1965b) is utilized to make these tests strictly distribution-free, and the well-known Chernoff-Savage Theorem (cf. Chernoff and Savage, 1958; Govindarajulu, LeCam and Raghavachari, 1965; Puri, 1964) is generalized here to the multivariate several sample problems. This takes care of the asymptotic power-properties of the tests.},
	number = {4},
	journal = {Sankhyā: The Indian Journal of Statistics, Series A (1961-2002)},
	author = {Puri, Madan Lal and Sen, Pranab Kumar},
	year = {1966},
	publisher = {Springer},
	pages = {353--376},
}

@misc{deb_efficiency_2021,
	title = {Efficiency {Lower} {Bounds} for {Distribution}-{Free} {Hotelling}-{Type} {Two}-{Sample} {Tests} {Based} on {Optimal} {Transport}},
	doi = {10.48550/arXiv.2104.01986},
	abstract = {The Wilcoxon rank-sum test is one of the most popular distribution-free procedures for testing the equality of two univariate probability distributions. One of the main reasons for its popularity can be attributed to the remarkable result of Hodges and Lehmann (1956), which shows that the asymptotic relative efficiency of Wilcoxon's test with respect to Student's \$t\$-test, under location alternatives, never falls below 0.864, despite the former being exactly distribution-free for all sample sizes. Even more striking is the result of Chernoff and Savage (1958), which shows that the efficiency of a Gaussian score transformed Wilcoxon's test, against the \$t\$-test, is lower bounded by 1. In this paper we study the two-sample problem in the multivariate setting and propose distribution-free analogues of the Hotelling \$T{\textasciicircum}2\$ test (the natural multidimensional counterpart of Student's \$t\$-test) based on optimal transport and obtain extensions of the above celebrated results over various natural families of multivariate distributions. Our proposed tests are consistent against a general class of alternatives and satisfy Hodges-Lehmann and Chernoff-Savage-type efficiency lower bounds, despite being entirely agnostic to the underlying data generating mechanism. In particular, a collection of our proposed tests suffer from no loss in asymptotic efficiency, when compared to Hotelling \$T{\textasciicircum}2\$. To the best of our knowledge, these are the first collection of multivariate, nonparametric, exactly distribution-free tests that provably achieve such attractive efficiency lower bounds. We also demonstrate the broader scope of our methods in optimal transport based nonparametric inference by constructing exactly distribution-free multivariate tests for mutual independence, which suffer from no loss in asymptotic efficiency against the classical Wilks' likelihood ratio test, under Konijn alternatives.},
	publisher = {arXiv},
	author = {Deb, Nabarun and Bhattacharya, Bhaswar B. and Sen, Bodhisattva},
	month = aug,
	year = {2021},
	note = {arXiv:2104.01986 [math, stat]},
	keywords = {Mathematics - Statistics Theory, Mathematics - Probability, 62G10, 62H15, 60F05},
}

@article{kim_robust_2020,
	title = {Robust multivariate nonparametric tests via projection averaging},
	volume = {48},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/19-AOS1936},
	abstract = {In this work, we generalize the Cramér–von Mises statistic via projection averaging to obtain a robust test for the multivariate two-sample problem. The proposed test is consistent against all fixed alternatives, robust to heavy-tailed data and minimax rate optimal against a certain class of alternatives. Our test statistic is completely free of tuning parameters and is computationally efficient even in high dimensions. When the dimension tends to infinity, the proposed test is shown to have comparable power to the existing high-dimensional mean tests under certain location models. As a by-product of our approach, we introduce a new metric called the angular distance which can be thought of as a robust alternative to the Euclidean distance. Using the angular distance, we connect the proposed method to the reproducing kernel Hilbert space approach. In addition to the Cramér–von Mises statistic, we demonstrate that the projection-averaging technique can be used to define robust multivariate tests in many other problems.},
	number = {6},
	journal = {The Annals of Statistics},
	author = {Kim, Ilmun and Balakrishnan, Sivaraman and Wasserman, Larry},
	month = dec,
	year = {2020},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62G10, maximum mean discrepancy, 62H15, \$U\$-statistic, 62G35, 62H20, Energy statistic, high dimension and low sample size, Independence testing, permutation tests},
	pages = {3417--3441},
}

@article{heller_sensitivity_2010,
	title = {Sensitivity {Analysis} for the {Cross}-{Match} {Test}, {With} {Applications} in {Genomics}},
	volume = {105},
	issn = {0162-1459},
	doi = {10.1198/jasa.2010.ap09260},
	abstract = {The cross-match test is an exact, distribution-free test of no treatment effect on a high-dimensional outcome in a randomized experiment. The test uses optimal nonbipartite matching to pair 2I subjects into I pairs based on similar outcomes, and the cross-match statistic A is the number of times that a treated subject was paired with a control, rejecting for small values of A. If the test is applied in an observational study in which treatments are not randomly assigned, then it may be comparing treated and control subjects who are not comparable, and thus may falsely reject a true null hypothesis of no treatment effect. We develop a sensitivity analysis for the cross-match test and apply it in an observational study of the effects of smoking on gene expression levels. In addition, we develop a sensitivity analysis for several multiple testing procedures using the cross-match test and apply it to 1627 molecular function categories in Gene Ontology.},
	number = {491},
	journal = {Journal of the American Statistical Association},
	author = {Heller, Ruth and Jensen, Shane T. and Rosenbaum, Paul R. and Small, Dylan S.},
	month = sep,
	year = {2010},
	publisher = {Taylor \& Francis},
	keywords = {Cross-match test, Multiple testing, Nonbipartite matching, Observational study, Sensitivity analysis},
	pages = {1005--1013},
}

@article{mukhopadhyay_nonparametric_2020,
	title = {A {Nonparametric} {Approach} to {High}-{Dimensional} k-{Sample} {Comparison} {Problems}},
	volume = {107},
	issn = {0006-3444},
	doi = {10.1093/biomet/asaa015},
	abstract = {High-dimensional \$k\$-sample comparison is a common task in applications. We construct a class of easy-to-implement distribution-free tests based on new nonparametric tools and unexplored connections with spectral graph theory. The test is shown to have various desirable properties and a characteristic exploratory flavour that has practical consequences for statistical modelling. Numerical examples show that the proposed method works surprisingly well across a broad range of realistic situations.},
	number = {3},
	journal = {Biometrika},
	author = {Mukhopadhyay, Subhadeep and Wang, Kaijun},
	month = sep,
	year = {2020},
	pages = {555--572},
}

@article{chen_bayesian_2014,
	title = {Bayesian nonparametric k-sample tests for censored and uncensored data},
	volume = {71},
	issn = {0167-9473},
	doi = {10.1016/j.csda.2012.11.003},
	abstract = {Polya tree priors are random probability distributions that are easily centered at standard parametric families, such as the normal. As such, they provide a convenient avenue toward creating a parametric/nonparametric test statistic “blend” for the classic problem of testing whether data distributions are the same across several subpopulations. Test-statistics that are (empirical) Bayes factors constructed from independent Polya tree priors are proposed. The Polya tree centering distributions are Gaussian with parameters estimated from the data and the p-values are obtained through the permutation of group membership indicators. Generalizations to censored and multivariate data are provided. The conceptually simple test statistic fares surprisingly well against competitors in simulations.},
	journal = {Computational Statistics \& Data Analysis},
	author = {Chen, Yuhui and Hanson, Timothy E.},
	month = mar,
	year = {2014},
	keywords = {ANOVA, Behrens–Fisher problem, Log-rank test, MANOVA, Polya tree},
	pages = {335--346},
}

@misc{zhang_bayesian_2022,
	title = {Bayesian {Kernel} {Two}-{Sample} {Testing}},
	doi = {10.48550/arXiv.2002.05550},
	abstract = {In modern data analysis, nonparametric measures of discrepancies between random variables are particularly important. The subject is well-studied in the frequentist literature, while the development in the Bayesian setting is limited where applications are often restricted to univariate cases. Here, we propose a Bayesian kernel two-sample testing procedure based on modelling the difference between kernel mean embeddings in the reproducing kernel Hilbert space utilising the framework established by Flaxman et al (2016). The use of kernel methods enables its application to random variables in generic domains beyond the multivariate Euclidean spaces. The proposed procedure results in a posterior inference scheme that allows an automatic selection of the kernel parameters relevant to the problem at hand. In a series of synthetic experiments and two real data experiments (i.e. testing network heterogeneity from high-dimensional data and six-membered monocyclic ring conformation comparison), we illustrate the advantages of our approach.},
	publisher = {arXiv},
	author = {Zhang, Qinyi and Wild, Veit and Filippi, Sarah and Flaxman, Seth and Sejdinovic, Dino},
	month = jan,
	year = {2022},
	note = {arXiv:2002.05550 [stat]},
	keywords = {Statistics - Methodology, Statistics - Computation},
}

@misc{zaremba_b_2022,
	title = {B - test},
	url = {https://github.com/wojzaremba/btest},
	urldate = {2022-09-02},
	author = {Zaremba, Wojciech},
	month = feb,
	year = {2022},
}

@misc{cloninger_two-sample-anisotropic_2021,
	title = {two-sample-anisotropic},
	url = {https://github.com/ACloninger/two-sample-anisotropic},
	abstract = {This repository contains code for the paper: "Two-sample Statistics Based on Anisotropic Kernels" by Xiuyuan Cheng, Alexander Cloninger, and Ronald R. Coifman},
	urldate = {2022-09-02},
	author = {Cloninger, Alex},
	month = sep,
	year = {2021},
}

@article{epps_omnibus_1986,
	title = {An omnibus test for the two-sample problem using the empirical characteristic function},
	volume = {26},
	issn = {0094-9655},
	doi = {10.1080/00949658608810963},
	abstract = {The empirical characteristic function (CF) is the Fourier transform of the sample distribution function. The values of its real and imaginary parts at some real number t are merely sample means of cosine and sine functions of the data, the observations being multiplied by t. Given independent samples from two populations, we develop a test for the two-sample problem which is based on a quadratic form in differences between the respective components of the empirical CFs of the two samples. The power of the CF test compares favorably with that of competing omnibus tests when the data are continuous. In the discrete case the CF procedure is also applicable and quite successful; and in this application it appears to have no competitors.},
	number = {3-4},
	journal = {Journal of Statistical Computation and Simulation},
	author = {Epps, T.W. and Singleton, Kenneth J.},
	month = dec,
	year = {1986},
	publisher = {Taylor \& Francis},
	keywords = {Anderson-Darling test, cramér-von Mises test, K-sample problem, Kolmogorov-smirnov test},
	pages = {177--203},
}

@inproceedings{gretton_fast_2009,
	title = {A {Fast}, {Consistent} {Kernel} {Two}-{Sample} {Test}},
	volume = {22},
	urldate = {2022-09-02},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Gretton, Arthur and Fukumizu, Kenji and Harchaoui, Zaïd and Sriperumbudur, Bharath K.},
	year = {2009},
}

@article{borgwardt_integrating_2006,
	title = {Integrating structured biological data by {Kernel} {Maximum} {Mean} {Discrepancy}},
	volume = {22},
	issn = {1367-4811},
	doi = {10.1093/bioinformatics/btl242},
	abstract = {MOTIVATION: Many problems in data integration in bioinformatics can be posed as one common question: Are two sets of observations generated by the same distribution? We propose a kernel-based statistical test for this problem, based on the fact that two distributions are different if and only if there exists at least one function having different expectation on the two distributions. Consequently we use the maximum discrepancy between function means as the basis of a test statistic. The Maximum Mean Discrepancy (MMD) can take advantage of the kernel trick, which allows us to apply it not only to vectors, but strings, sequences, graphs, and other common structured data types arising in molecular biology.
	RESULTS: We study the practical feasibility of an MMD-based test on three central data integration tasks: Testing cross-platform comparability of microarray data, cancer diagnosis, and data-content based schema matching for two different protein function classification schemas. In all of these experiments, including high-dimensional ones, MMD is very accurate in finding samples that were generated from the same distribution, and outperforms its best competitors.
	CONCLUSIONS: We have defined a novel statistical test of whether two samples are from the same distribution, compatible with both multivariate and structured data, that is fast, easy to implement, and works well, as confirmed by our experiments.
	AVAILABILITY: http://www.dbs.ifi.lmu.de/{\textasciitilde}borgward/MMD.},
	number = {14},
	journal = {Bioinformatics (Oxford, England)},
	author = {Borgwardt, Karsten M. and Gretton, Arthur and Rasch, Malte J. and Kriegel, Hans-Peter and Schölkopf, Bernhard and Smola, Alex J.},
	month = jul,
	year = {2006},
	pmid = {16873512},
	keywords = {Algorithms, Computational Biology, Computer Simulation, Data Interpretation, Statistical, Databases, Factual, Information Storage and Retrieval, Models, Biological, Models, Statistical, Sample Size, Statistical Distributions, Systems Integration},
	pages = {e49--57},
}

@article{sriperumbudur_hilbert_2010,
	title = {Hilbert {Space} {Embeddings} and {Metrics} on {Probability} {Measures}},
	volume = {11},
	issn = {1533-7928},
	abstract = {A Hilbert space embedding for probability measures has recently been proposed, with applications including dimensionality reduction, homogeneity testing, and independence testing. This embedding represents any probability measure as a mean element in a reproducing kernel Hilbert space (RKHS). A pseudometric on the space of probability measures can be defined as the distance between distribution embeddings: we denote this as γk, indexed by the kernel function k that defines the inner product in the RKHS.
	We present three theoretical properties of γk. First, we consider the question of determining the conditions on the kernel k for which γk is a metric: such k are denoted characteristic kernels. Unlike pseudometrics, a metric is zero only when two distributions coincide, thus ensuring the RKHS embedding maps all distributions uniquely (i.e., the embedding is injective). While previously published conditions may apply only in restricted circumstances (e.g., on compact domains), and are difficult to check, our conditions are straightforward and intuitive: integrally strictly positive definite kernels are characteristic. Alternatively, if a bounded continuous kernel is translation-invariant on ℜd, then it is characteristic if and only if the support of its Fourier transform is the entire ℜd. Second, we show that the distance between distributions under γk results from an interplay between the properties of the kernel and the distributions, by demonstrating that distributions are close in the embedding space when their differences occur at higher frequencies. Third, to understand the nature of the topology induced by γk, we relate γk to other popular metrics on probability measures, and present conditions on the kernel k under which γk metrizes the weak topology.},
	number = {50},
	journal = {Journal of Machine Learning Research},
	author = {Sriperumbudur, Bharath K. and Gretton, Arthur and Fukumizu, Kenji and Schölkopf, Bernhard and Lanckriet, Gert R. G.},
	year = {2010},
	pages = {1517--1561},
}


@article{sriperumbudur_universality_2011,
	title = {Universality, {Characteristic} {Kernels} and {RKHS} {Embedding} of {Measures}},
	volume = {12},
	issn = {1533-7928},
	abstract = {Over the last few years, two different notions of positive definite (pd) kernels---universal and characteristic---have been developing in parallel in machine learning: universal kernels are proposed in the context of achieving the Bayes risk by kernel-based classification/regression algorithms while characteristic kernels are introduced in the context of distinguishing probability measures by embedding them into a reproducing kernel Hilbert space (RKHS). However, the relation between these two notions is not well understood. The main contribution of this paper is to clarify the relation between universal and characteristic kernels by presenting a unifying study relating them to RKHS embedding of measures, in addition to clarifying their relation to other common notions of strictly pd, conditionally strictly pd and integrally strictly pd kernels. For radial kernels on ℜd, all these notions are shown to be equivalent.},
	number = {70},
	journal = {Journal of Machine Learning Research},
	author = {Sriperumbudur, Bharath K. and Fukumizu, Kenji and Lanckriet, Gert R. G.},
	year = {2011},
	pages = {2389--2410},
}

@inproceedings{fromont_kernels_2012,
	title = {Kernels {Based} {Tests} with {Non}-asymptotic {Bootstrap} {Approaches} for {Two}-sample {Problems}},
	abstract = {Considering either two independent i.i.d. samples, or two independent samples generated from a heteroscedastic regression model, or two independent Poisson processes, we address the question of testing equality of their respective distributions. We first propose single testing procedures based on a general symmetric kernel. The corresponding critical values are chosen from a wild or permutation bootstrap approach, and the obtained tests are exactly (and not just asymptotically) of level. We then introduce an aggregation method, which enables to overcome the difficulty of choosing a kernel and/or the parameters of the kernel. We derive non-asymptotic properties for the aggregated tests, proving that they may be optimal in a classical statistical sense.},
	booktitle = {Proceedings of the 25th {Annual} {Conference} on {Learning} {Theory}},
	publisher = {JMLR Workshop and Conference Proceedings},
	author = {Fromont, Magalie and Laurent, BÃ©atrice and Lerasle, Matthieu and Reynaud-Bouret, Patricia},
	month = jun,
	year = {2012},
	issn = {1938-7228},
	pages = {23.1--23.23},
}

@inproceedings{huggins_random_2018,
	title = {Random {Feature} {Stein} {Discrepancies}},
	volume = {31},
	abstract = {Computable Stein discrepancies have been deployed for a variety of applications, ranging from sampler selection in posterior inference to approximate Bayesian inference to goodness-of-fit testing. Existing convergence-determining Stein discrepancies admit strong theoretical guarantees but suffer from a computational cost that grows quadratically in the sample size. While linear-time Stein discrepancies have been proposed for goodness-of-fit testing, they exhibit avoidable degradations in testing power—even when power is explicitly optimized. To address these shortcomings, we introduce feature Stein discrepancies (ΦSDs), a new family of quality measures that can be cheaply approximated using importance sampling. We show how to construct ΦSDs that provably determine the convergence of a sample to its target and develop high-accuracy approximations—random ΦSDs (RΦSDs)—which are computable in near-linear time. In our experiments with sampler selection for approximate posterior inference and goodness-of-fit testing, RΦSDs perform as well or better than quadratic-time KSDs while being orders of magnitude faster to compute.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Huggins, Jonathan and Mackey, Lester},
	year = {2018},
}

@article{simon-gabriel_kernel_2018,
	title = {Kernel {Distribution} {Embeddings}: {Universal} {Kernels}, {Characteristic} {Kernels} and {Kernel} {Metrics} on {Distributions}},
	volume = {19},
	issn = {1533-7928},
	shorttitle = {Kernel {Distribution} {Embeddings}},
	abstract = {Kernel mean embeddings have become a popular tool in machine learning. They map probability measures to functions in a reproducing kernel Hilbert space. The distance between two mapped measures defines a semi-distance over the probability measures known as the maximum mean discrepancy (MMD). Its properties depend on the underlying kernel and have been linked to three fundamental concepts of the kernel literature: universal, characteristic and strictly positive definite kernels. The contributions of this paper are three-fold. First, by slightly extending the usual definitions of universal, characteristic and strictly positive definite kernels, we show that these three concepts are essentially equivalent. Second, we give the first complete characterization of those kernels whose associated MMD-distance metrizes the weak convergence of probability measures. Third, we show that kernel mean embeddings can be extended from probability measures to generalized measures called Schwartz-distributions and analyze a few properties of these distribution embeddings.},
	number = {44},
	journal = {Journal of Machine Learning Research},
	author = {Simon-Gabriel, Carl-Johann and Schölkopf, Bernhard},
	year = {2018},
	pages = {1--29},
}

@article{alvarez-esteban_trimmed_2008,
	title = {Trimmed {Comparison} of {Distributions}},
	volume = {103},
	issn = {0162-1459},
	doi = {10.1198/016214508000000274},
	abstract = {This article introduces an analysis of similarity of distributions based on the L2-Wasserstein distance between trimmed distributions. Our main innovation is the use of the impartial trimming methodology, already considered in robust statistics, which we adapt to this setup. Instead of simply removing data at the tails to provide some robustness to the similarity analysis, we develop a data-driven trimming method aimed at maximizing similarity between distributions. Dissimilarity is then measured in terms of the distance between the optimally trimmed distributions. We provide illustrative examples showing the improvements over previous approaches and give the relevant asymptotic results to justify the use of this methodology in applications.},
	number = {482},
	journal = {Journal of the American Statistical Association},
	author = {Álvarez-Esteban, Pedro César and del Barrio, Eustasio and Cuesta-Albertos, Juan Antonio and Matrán, Carlos},
	month = jun,
	year = {2008},
	publisher = {Taylor \& Francis},
	keywords = {Wasserstein distance, Asymptotics, Impartial trimming, Similarity, Trimmed distributions},
	pages = {697--704},
}

@article{alvarez-esteban_similarity_2012,
	title = {Similarity of samples and trimming},
	volume = {18},
	number = {2},
	journal = {Bernoulli},
	author = {Alvarez-Esteban, Pedro C. and Del Barrio, Eustasio and Cuesta-Albertos, Juan A. and Matrán, Carlos},
	year = {2012},
	publisher = {Bernoulli Society for Mathematical Statistics and Probability},
	pages = {606--634},
}

@article{bickel_sums_1983,
	title = {Sums of {Functions} of {Nearest} {Neighbor} {Distances}, {Moment} {Bounds}, {Limit} {Theorems} and a {Goodness} of {Fit} {Test}},
	volume = {11},
	issn = {0091-1798},
	abstract = {We study the limiting behavior of sums of functions of nearest neighbor distances for an m dimensional sample. We establish a central limit theorem and moment bounds for such sums and an invariance principle for the empirical process of nearest neighbor distances. As a consequence we obtain the asymptotic behavior of a practicable goodness of fit test based on nearest neighbor distances.},
	number = {1},
	urldate = {2022-09-06},
	journal = {The Annals of Probability},
	author = {Bickel, Peter J. and Breiman, Leo},
	year = {1983},
	publisher = {Institute of Mathematical Statistics},
	pages = {185--214},
}

@article{munk_nonparametric_1998,
	title = {Nonparametric {Validation} of {Similar} {Distributions} and {Assessment} of {Goodness} of {Fit}},
	volume = {60},
	issn = {1369-7412},
	abstract = {In this paper the problem of assessing the similarity of two cumulative distribution functions F and G is considered. An asymptotic test based on an α-trimmed version of Mallows distance Γ$_{\textrm{α}}$(F, G) between F and G is suggested, thus demonstrating the similarity of F and G within a preassigned Γ$_{\textrm{α}}$(F, G) neighbourhood at a controlled type I error rate. The test proposed is applied to the validation of goodness of fit and for the nonparametric assessment of bio-equivalence. It is shown that Γ$_{\textrm{α}}$(F, G) can be interpreted as average and population equivalence. Our approach is illustrated by various examples.},
	number = {1},
	journal = {Journal of the Royal Statistical Society. Series B (Statistical Methodology)},
	author = {Munk, Axel and Czado, Claudia},
	year = {1998},
	publisher = {[Royal Statistical Society, Wiley]},
	pages = {223--241},
}

@article{baumgartner_nonparametric_1998,
	title = {A {Nonparametric} {Test} for the {General} {Two}-{Sample} {Problem}},
	volume = {54},
	issn = {0006-341X},
	doi = {10.2307/2533862},
	abstract = {For two independently drawn samples of data, a novel statistical test is proposed for the null hypothesis that both samples originate from the same population. The underlying distribution function does not need to be known but must be continuous, i.e., it is a nonparametric test. It is demonstrated for suitable examples that the test is easy to apply and is at least as powerful as the commonly used nonparametric tests, i.e., the Kolmogorov-Smirnov, the Cramer-von Mises, and the Wilcoxon tests.},
	number = {3},
	journal = {Biometrics},
	author = {Baumgartner, W. and Weiß, P. and Schindler, H.},
	year = {1998},
	publisher = {Wiley, International Biometric Society},
	pages = {1129--1135},
}

@article{hettmansperger_affine-invariant_1997,
	title = {Affine-{Invariant} {Multivariate} {One}-{Sample} {Signed}-{Rank} {Tests}},
	volume = {92},
	issn = {0162-1459},
	doi = {10.1080/01621459.1997.10473681},
	abstract = {Brown and Hettmansperger introduced affine-invariant bivariate analogs of the sign, rank, and signed-rank tests based on the Oja median. In this article affine-invariant k-variate extensions of the one-sample signed-rank test and the Hodges-Lehmann estimate are considered. The necessary distribution theory is developed, and asymptotic Pitman efficiencies with respect to Hotelling's T 2 test under multivariate t distributions are tabulated. An application of the signed-rank tests to a repeated-measurement setting is presented.},
	number = {440},
	journal = {Journal of the American Statistical Association},
	author = {Hettmansperger, Thomas P. and Möttönen, Jyrki and Oja, Hannu},
	month = dec,
	year = {1997},
	publisher = {Taylor \& Francis},
	keywords = {Asymptotic efficiency, Multivariate sign test, Multivariate signed-rank test, Oja median, Randomization test, Repeated measurements.},
	pages = {1591--1600},
}

@article{pan_ball_2018,
	title = {{Ball} {Divergence}: {Nonparametric} {Two} {Sample} {Test}},
	volume = {46},
	issn = {0090-5364},
	shorttitle = {{BALL} {DIVERGENCE}},
	doi = {10.1214/17-AOS1579},
	abstract = {In this paper, we first introduce Ball Divergence, a novel measure of the
	difference between two probability measures in separable Banach spaces, and show
	that the Ball Divergence of two probability measures is zero if and only if
	these two probability measures are identical without any moment assumption.
	Using Ball Divergence, we present a metric rank test procedure to detect the
	equality of distribution measures underlying independent samples. It is
	therefore robust to outliers or heavy-tail data. We show that this multivariate
	two sample test statistic is consistent with the Ball Divergence, and it
	converges to a mixture of χ2 distributions under the null
	hypothesis and a normal distribution under the alternative hypothesis.
	Importantly, we prove its consistency against a general alternative hypothesis.
	Moreover, this result does not depend on the ratio of the two imbalanced sample
	sizes, ensuring that can be applied to imbalanced data. Numerical studies
	confirm that our test is superior to several existing tests in terms of Type I
	error and power. We conclude our paper with two applications of our method: one
	is for virtual screening in drug development process and the other is for genome
	wide expression analysis in hormone replacement therapy.},
	number = {3},
	journal = {The Annals of Statistics},
	author = {Pan, Wenliang and Tian, Yuan and Wang, Xueqin and Zhang, Heping},
	month = jun,
	year = {2018},
	pmid = {30344356},
	pmcid = {PMC6192286},
	pages = {1109--1137},
}

@misc{ramdas_adaptivity_2015,
	title = {Adaptivity and {Computation}-{Statistics} {Tradeoffs} for {Kernel} and {Distance} based {High} {Dimensional} {Two} {Sample} {Testing}},
	doi = {10.48550/arXiv.1508.00655},
	abstract = {Nonparametric two sample testing is a decision theoretic problem that involves identifying differences between two random variables without making parametric assumptions about their underlying distributions. We refer to the most common settings as mean difference alternatives (MDA), for testing differences only in first moments, and general difference alternatives (GDA), which is about testing for any difference in distributions. A large number of test statistics have been proposed for both these settings. This paper connects three classes of statistics - high dimensional variants of Hotelling's t-test, statistics based on Reproducing Kernel Hilbert Spaces, and energy statistics based on pairwise distances. We ask the question: how much statistical power do popular kernel and distance based tests for GDA have when the unknown distributions differ in their means, compared to specialized tests for MDA? We formally characterize the power of popular tests for GDA like the Maximum Mean Discrepancy with the Gaussian kernel (gMMD) and bandwidth-dependent variants of the Energy Distance with the Euclidean norm (eED) in the high-dimensional MDA regime. Some practically important properties include (a) eED and gMMD have asymptotically equal power; furthermore they enjoy a free lunch because, while they are additionally consistent for GDA, they also have the same power as specialized high-dimensional t-test variants for MDA. All these tests are asymptotically optimal (including matching constants) under MDA for spherical covariances, according to simple lower bounds, (b) The power of gMMD is independent of the kernel bandwidth, as long as it is larger than the choice made by the median heuristic, (c) There is a clear and smooth computation-statistics tradeoff for linear-time, subquadratic-time and quadratic-time versions of these tests, with more computation resulting in higher power.},
	publisher = {arXiv},
	author = {Ramdas, Aaditya and Reddi, Sashank J. and Poczos, Barnabas and Singh, Aarti and Wasserman, Larry},
	month = aug,
	year = {2015},
	note = {arXiv:1508.00655 [cs, math, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning, Mathematics - Statistics Theory, Computer Science - Artificial Intelligence, Computer Science - Information Theory},
}

@article{kim_global_2019,
	title = {Global and local two-sample tests via regression},
	volume = {13},
	issn = {1935-7524, 1935-7524},
	doi = {10.1214/19-EJS1648},
	abstract = {Two-sample testing is a fundamental problem in statistics. Despite its long history, there has been renewed interest in this problem with the advent of high-dimensional and complex data. Specifically, in the machine learning literature, there have been recent methodological developments such as classification accuracy tests. The goal of this work is to present a regression approach to comparing multivariate distributions of complex data. Depending on the chosen regression model, our framework can efficiently handle different types of variables and various structures in the data, with competitive power under many practical scenarios. Whereas previous work has been largely limited to global tests which conceal much of the local information, our approach naturally leads to a local two-sample testing framework in which we identify local differences between multivariate distributions with statistical confidence. We demonstrate the efficacy of our approach both theoretically and empirically, under some well-known parametric and nonparametric regression methods. Our proposed methods are applied to simulated data as well as a challenging astronomy data set to assess their practical usefulness.},
	number = {2},
	journal = {Electronic Journal of Statistics},
	author = {Kim, Ilmun and Lee, Ann B. and Lei, Jing},
	month = jan,
	year = {2019},
	publisher = {Institute of Mathematical Statistics and Bernoulli Society},
	keywords = {Permutation test, 62G10, 62H15, 62G20, Galaxy morphology, intrinsic dimension, kernel regression, Nearest neighbor regression, random forests},
	pages = {5253--5305},
}

@article{huskova_tests_2008,
	title = {Tests for the multivariate k-sample problem based on the empirical characteristic function},
	volume = {20},
	issn = {1048-5252},
	doi = {10.1080/10485250801948294},
	abstract = {Tests for the multivariate k-sample problem are considered. The tests are based on the weighted L2 distance between empirical characteristic functions, and afford an interesting interpretation in terms of a corresponding test statistic based on the L2 distance of pairs of non-parametric density estimators. Depending on the choice of weighting, a corresponding Dirac-type weight function reduces the test to a normalised version of the L2 distance between the sample means of the k populations. Theoretical and computational issues are considered, while the finite-sample implementation based on the permutation distribution of the test statistic shows that the new test performs well in comparison with alternative procedures of the change-point type.},
	number = {3},
	journal = {Journal of Nonparametric Statistics},
	author = {Hušková, Marie and Meintanis, Simos G.},
	month = apr,
	year = {2008},
	publisher = {Taylor \& Francis},
	keywords = {62G10, 62G20, 62G9, empirical characteristic function, k-sample problem, non-parametric test},
	pages = {263--277},
}

@article{rizzo_disco_2010,
	title = {{DISCO} {Analysis}: {A} {Nonparametric} {Extension} of {Analysis} of {Variance}},
	volume = {4},
	issn = {1932-6157, 1941-7330},
	shorttitle = {{DISCO} analysis},
	doi = {10.1214/09-AOAS245},
	abstract = {In classical analysis of variance, dispersion is measured by considering squared distances of sample elements from the sample mean. We consider a measure of dispersion for univariate or multivariate response based on all pairwise distances between-sample elements, and derive an analogous distance components (DISCO) decomposition for powers of distance in (0, 2]. The ANOVA F statistic is obtained when the index (exponent) is 2. For each index in (0, 2), this decomposition determines a nonparametric test for the multi-sample hypothesis of equal distributions that is statistically consistent against general alternatives.},
	number = {2},
	journal = {The Annals of Applied Statistics},
	author = {Rizzo, Maria L. and Székely, Gábor J.},
	month = jun,
	year = {2010},
	publisher = {Institute of Mathematical Statistics},
	keywords = {multivariate, DISCO, Distance components, multisample problem, nonparametric MANOVA extension, test equal distributions},
	pages = {1034--1055},
}

@article{ghosh_distribution-free_2016,
	title = {Distribution-free high-dimensional two-sample tests based on discriminating hyperplanes},
	volume = {25},
	issn = {1863-8260},
	doi = {10.1007/s11749-015-0467-x},
	abstract = {In this article, we propose a general procedure for multivariate generalizations of univariate distribution-free tests involving two independent samples as well as matched pair data. This proposed procedure is based on ranks of real-valued linear functions of multivariate observations. The linear function used to rank the observations is obtained by solving a classification problem between the two multivariate distributions from which the observations are generated. Our proposed tests retain the distribution-free property of their univariate analogs, and they perform well for high-dimensional data even when the dimension exceeds the sample size. Asymptotic results on their power properties are derived when the dimension grows to infinity and the sample size may or may not grow with the dimension. We analyze several high-dimensional simulated and real data sets to compare the empirical performance of our proposed tests with several other tests available in the literature.},
	number = {3},
	journal = {TEST},
	author = {Ghosh, Anil K. and Biswas, Munmun},
	month = sep,
	year = {2016},
	keywords = {62G10, 62H15, Distance-weighted discrimination, Kolmogorov–Smirnov statistic, Sign test, Signed rank test, Support vector machines, Wilcoxon–Mann–Whitney statistic},
	pages = {525--547},
}

@article{wei_direction-projection-permutation_2016,
	title = {Direction-{Projection}-{Permutation} for {High}-{Dimensional} {Hypothesis} {Tests}},
	volume = {25},
	issn = {1061-8600},
	doi = {10.1080/10618600.2015.1027773},
	abstract = {High-dimensional low sample size (HDLSS) data are becoming increasingly common in statistical applications. When the data can be partitioned into two classes, a basic task is to construct a classifier that can assign objects to the correct class. Binary linear classifiers have been shown to be especially useful in HDLSS settings and preferable to more complicated classifiers because of their ease of interpretability. We propose a computational tool called direction-projection-permutation (DiProPerm), which rigorously assesses whether a binary linear classifier is detecting statistically significant differences between two high-dimensional distributions. The basic idea behind DiProPerm involves working directly with the one-dimensional projections of the data induced by binary linear classifier. Theoretical properties of DiProPerm are studied under the HDLSS asymptotic regime whereby dimension diverges to infinity while sample size remains fixed. We show that certain variations of DiProPerm are consistent and that consistency is a nontrivial property of tests in the HDLSS asymptotic regime. The practical utility of DiProPerm is demonstrated on HDLSS gene expression microarray datasets. Finally, an empirical power study is conducted comparing DiProPerm to several alternative two-sample HDLSS tests to understand the advantages and disadvantages of each method.},
	number = {2},
	journal = {Journal of Computational and Graphical Statistics},
	author = {Wei, Susan and Lee, Chihoon and Wichers, Lindsay and Marron, J. S.},
	month = apr,
	year = {2016},
	publisher = {Taylor \& Francis},
	keywords = {Distance weighted discrimination; High-dimensional hypothesis test; High-dimensional low sample size; Linear binary classification; Permutation test; Two-sample problem},
	pages = {549--569},
}

@article{darling_kolmogorov-smirnov_1957,
	title = {The {Kolmogorov}-{Smirnov}, {Cramer}-von {Mises} {Tests}},
	volume = {28},
	issn = {0003-4851},
	number = {4},
	urldate = {2022-09-08},
	journal = {The Annals of Mathematical Statistics},
	author = {Darling, D. A.},
	year = {1957},
	publisher = {Institute of Mathematical Statistics},
	pages = {823--838},
}

@article{romano_bootstrap_1989,
	title = {Bootstrap and {Randomization} {Tests} of some {Nonparametric} {Hypotheses}},
	volume = {17},
	issn = {0090-5364},
	abstract = {In this paper, the asymptotic behavior of some nonparametric tests is studied in situations where both bootstrap tests and randomization tests are applicable. Under fairly general conditions, the tests are asymptotically equivalent in the sense that the resulting critical values and power functions are appropriately close. This implies, among other things, that the difference in the critical functions of the tests, evaluated at the observed data, tends to 0 in probability. Randomization tests may be preferable since an exact desired level of the test may be obtained for finite samples. Examples considered are: testing independence, testing for spherical symmetry, testing for exchangeability, testing for homogeneity, and testing for a change point.},
	number = {1},
	urldate = {2022-09-08},
	journal = {The Annals of Statistics},
	author = {Romano, Joseph P.},
	year = {1989},
	publisher = {Institute of Mathematical Statistics},
	pages = {141--159},
}

@article{romano_bootstrap_1988,
	title = {A {Bootstrap} {Revival} of {Some} {Nonparametric} {Distance} {Tests}},
	volume = {83},
	issn = {0162-1459},
	doi = {10.1080/01621459.1988.10478650},
	abstract = {Several tests based on the empirical measure have been proposed to test independence of variables, goodness of fit, equality of distributions, rotational invariance, and so forth. These tests have excellent power properties, but critical values are difficult, if not impossible, to obtain. Furthermore, these tests usually assume that the data are real-valued with continuous distributions. Here, critical values are determined by bootstrapping and the resulting tests are shown to have the correct asymptotic level under minimal assumptions. For example, given data Xi = (X i,1, …, Xi,d ), i = 1, …, n, it may be desired to test independence of the d components. The proposed test compares the empirical measure and the product of its marginals by taking a supremum over an appropriate Vapnik-Cervonenkis class of sets. No assumptions are made on the probability distribution of the data or on the space in which it lives; indeed, some components may be discrete, some continuous, and others categorical. Similar results are obtained for other examples. Consistency of the tests is obtained against all alternatives. A modest simulation study shows that the bootstrap works satisfactorily for moderate sample sizes.},
	number = {403},
	journal = {Journal of the American Statistical Association},
	author = {Romano, Joseph P.},
	month = sep,
	year = {1988},
	publisher = {Taylor \& Francis},
	keywords = {Nonparametric tests, Distance tests, Goodness of fit, Testing equality of distributions, Testing for rotational invariance, Testing independence, Vapnik-Cervonenkis classes},
	pages = {698--708},
}

@article{ping_bootstrap_2000,
	title = {Bootstrap tests for the equality of distributions},
	volume = {7},
	issn = {1865-2085},
	doi = {10.1007/BF03012197},
	abstract = {Testing equality of two and k distributions has long been an interesting issue in statistical inference. To overcome the sparseness of data points in high-dimensional space and deal with the general cases, we suggest several projection pursuit type statistics. Some results on the limiting distributions of the statistics are obtained. Some properties of Bootstrap approximation are investigated. Furthermore, for computational reasons an approximation for the statistics the based on Number theoretic method is applied. Several simulation experiments are performed.},
	language = {en},
	number = {2},
	journal = {Korean Journal of Computational \& Applied Mathematics},
	author = {Ping, Jing},
	month = may,
	year = {2000},
	keywords = {62G10, 62H10, 2H15, 62G09, Bootstrap approximation, Empirical process, Number Theoretic method, Projection Pursuit type statistics},
	pages = {347--362},
}

@article{praestgaard_permutation_1995,
	title = {Permutation and {Bootstrap} {Kolmogorov}-{Smirnov} {Tests} for the {Equality} of {Two} {Distributions}},
	volume = {22},
	issn = {0303-6898},
	abstract = {We consider a generalized version of the two-sample Kolmogorov-Smirnov statistic which is calculated as the supremum distance, taken over a general class of indexing functions, \${\textbackslash}scr\{F\}\_\{N\}\$, possibly depending on the joint sample size N, between the empirical measures of the two samples. The resulting class of tests is very flexible, encompassing both previous suggestions by Bickel (1969) and Romano (1989), and tests which have so far not been viewed as Kolmogorov-Smirnov tests, e.g. the supremum-based generalized log-rank tests of Fleming et al. (1987). We investigate the permutation and bootstrap versions of these tests; i.e. the procedure by which the critical values are found from the permutation or bootstrap distribution of the statistics. If the indexing class of functions \${\textbackslash}scr\{F\}\_\{N\}\$ "converges" to a limiting collection \${\textbackslash}scr\{F\}\_\{0\}\$, then we find conditions under which these procedures yield consistent tests of \$H\_\{0\}{\textbackslash}colon {\textbackslash}{\textbar}{\textbackslash}text\{P\}-{\textbackslash}text\{Q\}{\textbackslash}{\textbar}\_\{{\textbackslash}scr\{F\}\_\{0\}\}=0\$ against any alternative \${\textbackslash}{\textbar}{\textbackslash}text\{P\}-{\textbackslash}text\{Q\}{\textbackslash}{\textbar}\_\{{\textbackslash}scr\{F\}\_\{0\}\}{\textbackslash}neq 0\$. We furthermore calculate the local asymptotic power of both tests and find it to be the same for bootstrap and permutation resampling.},
	number = {3},
	journal = {Scandinavian Journal of Statistics},
	author = {Præstgaard, Jens Thomas},
	year = {1995},
	publisher = {Board of the Foundation of the Scandinavian Journal of Statistics, Wiley},
	pages = {305--322},
}

@article{hu_review_2016,
	title = {A review of 20 years of naive tests of significance for high-dimensional mean vectors and covariance matrices},
	volume = {59},
	issn = {1869-1862},
	doi = {10.1007/s11425-016-0131-0},
	abstract = {We introduce the so-called naive tests and give a brief review of the new developments. Naive testing methods are easy to understand and perform robustly, especially when the dimension is large. We focus mainly on reviewing some naive testing methods for the mean vectors and covariance matrices of high-dimensional populations, and we believe that this naive testing approach can be used widely in many other testing problems.},
	number = {12},
	journal = {Science China Mathematics},
	author = {Hu, Jiang and Bai, ZhiDong},
	month = dec,
	year = {2016},
	keywords = {62H15, hypothesis testing, 62E20, high-dimensional data, multivariate analysis of variance (MANOVA), naive testing methods},
	pages = {2281--2300},
}

@article{arias-castro_remember_2018,
	title = {Remember the curse of dimensionality: the case of goodness-of-fit testing in arbitrary dimension},
	volume = {30},
	issn = {1048-5252},
	shorttitle = {Remember the curse of dimensionality},
	doi = {10.1080/10485252.2018.1435875},
	abstract = {Despite a substantial literature on nonparametric two-sample goodness-of-fit testing in arbitrary dimensions, there is no mention there of any curse of dimensionality. In fact, in some publications, a parametric rate is derived. As we discuss below, this is because a directional alternative is considered. Indeed, even in dimension one, Ingster, Y. I. [(1987). Minimax testing of nonparametric hypotheses on a distribution density in the l\_p metrics. Theory of Probability \& Its Applications, 31(2), 333–337] has shown that the minimax rate is not parametric. In this paper, we extend his results to arbitrary dimension and confirm that the minimax rate is not only nonparametric, exhibits but also a prototypical curse of dimensionality. We further extend Ingster's work to show that the chi-squared test achieves the minimax rate. Moreover, we show that the test adapts to the intrinsic dimensionality of the data. Finally, in the spirit of Ingster, Y. I. [(2000). Adaptive chi-square tests. Journal of Mathematical Sciences, 99(2), 1110–1119], we consider a multiscale version of the chi-square test, showing that one can adapt to unknown smoothness without much loss in power.},
	number = {2},
	journal = {Journal of Nonparametric Statistics},
	author = {Arias-Castro, Ery and Pelletier, Bruno and Saligrama, Venkatesh},
	month = apr,
	year = {2018},
	publisher = {Taylor \& Francis},
	keywords = {Curse of dimensionality, goodness-of-fit testing, minimax tests},
	pages = {448--471},
}

@article{golland_permutation_2003,
	title = {Permutation tests for classification: towards statistical significance in image-based studies},
	volume = {18},
	issn = {1011-2499},
	shorttitle = {Permutation tests for classification},
	doi = {10.1007/978-3-540-45087-0_28},
	abstract = {Estimating statistical significance of detected differences between two groups of medical scans is a challenging problem due to the high dimensionality of the data and the relatively small number of training examples. In this paper, we demonstrate a non-parametric technique for estimation of statistical significance in the context of discriminative analysis (i.e., training a classifier function to label new examples into one of two groups). Our approach adopts permutation tests, first developed in classical statistics for hypothesis testing, to estimate how likely we are to obtain the observed classification performance, as measured by testing on a hold-out set or cross-validation, by chance. We demonstrate the method on examples of both structural and functional neuroimaging studies.},
	journal = {Information Processing in Medical Imaging: Proceedings of the ... Conference},
	author = {Golland, Polina and Fischl, Bruce},
	month = jul,
	year = {2003},
	pmid = {15344469},
	keywords = {Humans, Algorithms, Computer Simulation, Models, Biological, Models, Statistical, Alzheimer Disease, Brain, Brain Mapping, Image Enhancement, Image Interpretation, Computer-Assisted, Imaging, Three-Dimensional, Magnetic Resonance Imaging, Pattern Recognition, Automated, Reproducibility of Results, Sensitivity and Specificity, Subtraction Technique},
	pages = {330--341},
}

@article{hediger_use_2021,
	title = {On the {Use} of {Random} {Forest} for {Two}-{Sample} {Testing}},
	volume = {170},
	issn = {0167-9473},
	url = {https://www.sciencedirect.com/science/article/pii/S0167947322000159},
	doi = {10.1016/j.csda.2022.107435},
	abstract = {Following the line of classification-based two-sample testing, tests based on the Random Forest classifier are proposed. The developed tests are easy to use, require almost no tuning, and are applicable for any distribution on Rd. Furthermore, the built-in variable importance measure of the Random Forest gives potential insights into which variables make out the difference in distribution. An asymptotic power analysis for the proposed tests is conducted. Finally, two real-world applications illustrate the usefulness of the introduced methodology. To simplify the use of the method, the R-package “hypoRF” is provided.},
	urldate = {2024-11-25},
	journal = {Computational Statistics \& Data Analysis},
	author = {Hediger, Simon and Michel, Loris and Näf, Jeffrey},
	month = jun,
	year = {2022},
	keywords = {Classification, Distribution testing, Kernel two-sample test, MMD, Random forest, Total variation distance, U-statistics},
	pages = {107435},
}

@article{srivastava_two_2013,
	title = {A two sample test in high dimensional data},
	volume = {114},
	issn = {0047-259X},
	doi = {10.1016/j.jmva.2012.08.014},
	abstract = {In this paper we propose a test for testing the equality of the mean vectors of two groups with unequal covariance matrices based on N1 and N2 independently distributed p-dimensional observation vectors. It will be assumed that N1 observation vectors from the first group are normally distributed with mean vector μ1 and covariance matrix Σ1. Similarly, the N2 observation vectors from the second group are normally distributed with mean vector μ2 and covariance matrix Σ2. We propose a test for testing the hypothesis that μ1=μ2. This test is invariant under the group of p×p nonsingular diagonal matrices. The asymptotic distribution is obtained as (N1,N2,p)→∞ and N1/(N1+N2)→k∈(0,1) but N1/p and N2/p may go to zero or infinity. It is compared with a recently proposed non-invariant test. It is shown that the proposed test performs the best.},
	journal = {Journal of Multivariate Analysis},
	author = {Srivastava, Muni S. and Katayama, Shota and Kano, Yutaka},
	month = feb,
	year = {2013},
	keywords = {High-dimensional data, Hypothesis testing, Asymptotic theory, Behrens–Fisher problem},
	pages = {349--358},
}

@article{yu_two-sample_2007,
	title = {Two-sample {Comparison} {Based} on {Prediction} {Error}, with {Applications} to {Candidate} {Gene} {Association} {Studies}},
	volume = {71},
	issn = {1469-1809},
	doi = {10.1111/j.1469-1809.2006.00306.x},
	abstract = {To take advantage of the increasingly available high-density SNP maps across the genome, various tests that compare multilocus genotypes or estimated haplotypes between cases and controls have been developed for candidate gene association studies. Here we view this two-sample testing problem from the perspective of supervised machine learning and propose a new association test. The approach adopts the flexible and easy-to-understand classification tree model as the learning machine, and uses the estimated prediction error of the resulting prediction rule as the test statistic. This procedure not only provides an association test but also generates a prediction rule that can be useful in understanding the mechanisms underlying complex disease. Under the set-up of a haplotype-based transmission/disequilibrium test (TDT) type of analysis, we find through simulation studies that the proposed procedure has the correct type I error rates and is robust to population stratification. The power of the proposed procedure is sensitive to the chosen prediction error estimator. Among commonly used prediction error estimators, the .632+ estimator results in a test that has the best overall performance. We also find that the test using the .632+ estimator is more powerful than the standard single-point TDT analysis, the Pearson's goodness-of-fit test based on estimated haplotype frequencies, and two haplotype-based global tests implemented in the genetic analysis package FBAT. To illustrate the application of the proposed method in population-based association studies, we use the procedure to study the association between non-Hodgkin lymphoma and the IL10 gene.},
	number = {1},
	journal = {Annals of Human Genetics},
	author = {Yu, K. and Martin, R. and Rothman, N. and Zheng, T. and Lan, Q.},
	year = {2007},
	keywords = {bootstrap, candidate gene association studies, classification tree, haplotype, prediction error},
	pages = {107--118},
}

@article{cai_two-sample_2020,
	title = {Two-sample test based on classification probability},
	volume = {13},
	issn = {1932-1872},
	doi = {10.1002/sam.11438},
	abstract = {Robust classification algorithms have been developed in recent years with great success. We take advantage of this development and recast the classical two-sample test problem in the framework of classification. Based on the estimates of classification probabilities from a classifier trained from the samples, a test statistic is proposed. We explain why such a test can be a powerful test and compare its performance in terms of the power and efficiency with those of some other recently proposed tests with simulation and real-life data. The test proposed is nonparametric and can be applied to complex and high-dimensional data wherever there is a classifier that provides consistent estimate of the classification probability for such data.},
	number = {1},
	journal = {Statistical Analysis and Data Mining: The ASA Data Science Journal},
	author = {Cai, Haiyan and Goggin, Bryan and Jiang, Qingtang},
	year = {2020},
	keywords = {two-sample test, classification},
	pages = {5--13},
}

@article{rosenblatt_better-than-chance_2021,
	title = {Better-than-chance classification for signal detection},
	volume = {22},
	issn = {1465-4644},
	doi = {10.1093/biostatistics/kxz035},
	abstract = {The estimated accuracy of a classifier is a random quantity with variability. A common practice in supervised machine learning, is thus to test if the estimated accuracy is significantly better than chance level. This method of signal detection is particularly popular in neuroimaging and genetics. We provide evidence that using a classifier’s accuracy as a test statistic can be an underpowered strategy for finding differences between populations, compared to a bona fide statistical test. It is also computationally more demanding than a statistical test. Via simulation, we compare test statistics that are based on classification accuracy, to others based on multivariate test statistics. We find that the probability of detecting differences between two distributions is lower for accuracy-based statistics. We examine several candidate causes for the low power of accuracy-tests. These causes include: the discrete nature of the accuracy-test statistic, the type of signal accuracy-tests are designed to detect, their inefficient use of the data, and their suboptimal regularization. When the purpose of the analysis is the evaluation of a particular classifier, not signal detection, we suggest several improvements to increase power. In particular, to replace V-fold cross-validation with the Leave-One-Out Bootstrap.},
	number = {2},
	journal = {Biostatistics},
	author = {Rosenblatt, Jonathan D and Benjamini, Yuval and Gilron, Roee and Mukamel, Roy and Goeman, Jelle J},
	month = apr,
	year = {2021},
	pages = {365--380},
}

@article{chung_randomization_1958,
	title = {Randomization {Tests} for a {Multivariate} {Two}-{Sample} {Problem}},
	volume = {53},
	issn = {0162-1459},
	doi = {10.1080/01621459.1958.10501472},
	abstract = {With few observations involving a large number of variables the T 2 test for the multivariate two-sample problem may not exist. Some alternative tests based on randomization methods are suggested and two of these are applied to an example. Also, valid randomization tests can be obtained by using subgroups of permutations; this provides a simple method for reducing computation which is desirable when the sample sizes are not small.},
	number = {283},
	journal = {Journal of the American Statistical Association},
	author = {Chung, J. H. and Fraser, D. A. S.},
	month = sep,
	year = {1958},
	publisher = {Taylor \& Francis},
	pages = {729--735},
}

@article{liu_test_2019,
	title = {A {Test} for {Equality} of {Two} {Distributions} via {Integrating} {Characteristic} {Functions}},
	volume = {29},
	issn = {1017-0405},
	abstract = {In this study, we investigate the problem of testing the equality of two distributions by integrating the squared norm of the difference between two corresponding empirical characteristic functions. This results in a linear combination of three different U-statistics. Thus the original testing problem is reduced to testing whether this linear combination is zero. We apply the jackknife empirical likelihood (JEL) method to the new hypothesis testing problem. Under multivariate case, the log JEL statistic after scaling tends to a chi-square distribution with one degree of freedom. Simulation studies are presented to assess the finite-sample performance of our method.},
	number = {4},
	urldate = {2022-09-20},
	journal = {Statistica Sinica},
	author = {Liu, Yiming and Liu, Zhi and Zhou, Wang},
	year = {2019},
	publisher = {Institute of Statistical Science, Academia Sinica},
	pages = {1779--1801},
}

@article{liu_test_2015,
	title = {A test for equality of two distributions via jackknife empirical likelihood and characteristic functions},
	volume = {92},
	issn = {0167-9473},
	doi = {10.1016/j.csda.2015.06.004},
	abstract = {The two-sample problem: testing the equality of two distributions is investigated. A jackknife empirical likelihood (JEL) test is proposed through incorporating characteristic functions, which reduces to a two-sample U-statistic. When the dimension of data is fixed, the nonparametric Wilks’s theorem for the proposed JEL ratio statistics is established. When the dimension diverges with the sample size at a moderate rate, p=o(n1/3), it is proved that under some mild conditions the normalized JEL ratio statistic has a standard normal limit. Moreover, when the dimension exceeds the sample size, p{\textgreater}n, an alternative version of JEL test is proposed. It is verified that under the null hypothesis this alternative version of JEL test has an asymptotical chi-squared distribution with two degrees of freedom. Some numerical results via simulation study and an analysis of a microarray dataset are presented and both confirm theoretical results empirically.},
	journal = {Computational Statistics \& Data Analysis},
	author = {Liu, Zhi and Xia, Xiaochao and Zhou, Wang},
	month = dec,
	year = {2015},
	keywords = {Two-sample test, Characteristic function, Equality of distributions, Jackknife empirical likelihood, Normal limit},
	pages = {97--114},
}

@misc{zech_new_2003,
	title = {A new test for the multivariate two-sample problem based on the concept of minimum energy},
	doi = {10.48550/arXiv.math/0309164},
	abstract = {We introduce a new statistical quantity the energy to test whether two samples originate from the same distributions. The energy is a simple logarithmic function of the distances of the observations in the variate space. The distribution of the test statistic is determined by a resampling method. The power of the energy test in one dimension was studied for a variety of different test samples and compared to several nonparametric tests. In two and four dimensions a comparison was performed with the Friedman-Rafsky and nearest neighbor tests. The two-sample energy test was shown to be especially powerful in multidimensional applications.},
	publisher = {arXiv},
	author = {Zech, Guenter and Aslan, Berkan},
	month = sep,
	year = {2003},
	note = {arXiv:math/0309164
	version: 1},
	keywords = {Mathematics - Probability},
}

@article{ahmad_goodness_1993,
	title = {Goodness of fit tests based on the {L2}-norm of multivariate probability density functions},
	volume = {2},
	issn = {1048-5252},
	doi = {10.1080/10485259308832550},
	abstract = {Using the well known kernel method of estimation for multivariate probability densities due to Rosenblatt (1956), Parzen (1962) and Cacoullos (1966), testing the goodness of fit in the one sample, two sample, and testing symmetry cases are discussed. Test statistics presented here are based on the L2-norms . The estimates presented here for δ i i=l,2, 3 are modifications on those discussed by Bickel and Rosenblatt (1973), Rosenblatt (1975) and Hall (1984), and are asymptotically normal under the null and also under the alternatives with conditions much simpler than assumed in literature.},
	number = {2},
	journal = {Journal of Nonparametric Statistics},
	author = {Ahmad, Ibrahim A. and Cerrito, Patricia B.},
	month = jan,
	year = {1993},
	publisher = {Taylor \& Francis},
	keywords = {kernel method, goodness of fit, Multivariate density estimation, one and two sample problem, testing symmetry},
	pages = {169--181},
}

@article{einmahl_two-sample_2001,
	title = {The {Two}-{Sample} {Problem} in \${IR}{\textasciicircum}\{m\}\$ and {Measure}-{Valued} {Martingales}},
	volume = {36},
	issn = {0749-2170},
	abstract = {The so-called two-sample problem is one of the classical problems in mathematical statistics. It is well-known that in dimension one the two-sample Smirnov test possesses two basic properties: it is distribution free under the null hypothesis and it is sensitive to 'all' alternatives. In the multidimensional case, i.e. when the observations in the two samples are random vectors in \$IR{\textasciicircum}\{m\}\$, m ≥ 2, the Smirnov test loses its first basic property. In correspondence with the above, we define a solution of the two-sample problem to be a 'natural' stochastic process, based on the two samples, which is (α) asymptotically distribution free under the null hypothesis, and which is, intuitively speaking, (β) as sensitive as possible to all alternatives. Despite the fact that the two-sample problem has a long and very diverse history, starting with some famous papers in the thirties, the problem is essentially still open for samples in \$IR{\textasciicircum}\{m\}\$, m ≥ 2. In this paper we present an approach based on measure-valued martingales and we will show that the stochastic process obtained with this approach is a solution to the two-sample problem, i.e. it has both the properties (α) and (β), for any m ∈ N.},
	urldate = {2022-09-22},
	journal = {Lecture Notes-Monograph Series},
	author = {Einmahl, J. H. J. and Khmaladze, E. V.},
	year = {2001},
	publisher = {Institute of Mathematical Statistics},
	pages = {434--463},
}

@article{henze_almost_1992,
	title = {Almost {Sure} {Convergence} of {Certain} {Slowly} {Changing} {Symmetric} {One}- and {Multi}-{Sample} {Statistics}},
	volume = {20},
	issn = {0091-1798, 2168-894X},
	doi = {10.1214/aop/1176989819},
	abstract = {Let \$X{\textasciicircum}\{(i)\}\_j, i = 1,{\textbackslash}ldots, k; j {\textbackslash}in {\textbackslash}mathbf\{N\}\$, be independent \$d\$-dimensional random vectors which are identically distributed for each fixed \$i = 1,{\textbackslash}ldots, k\$. We give a sufficient condition for almost sure convergence of a sequence \$T\_\{n\_1,{\textbackslash}ldots, n\_k\}\$ of statistics based on \$X{\textasciicircum}\{(i)\}\_j i = 1,{\textbackslash}ldots, k; j = 1, {\textbackslash}ldots, n\_i\$, which are symmetric functions of \$X{\textasciicircum}\{(i)\}\_1,{\textbackslash}ldots, X{\textasciicircum}\{(i)\}\_\{n\_i\}\$ for each \$i\$ and do not change too much when variables are added or deleted. A key auxiliary tool for proofs is the Efron-Stein inequality. Applications include strong limits for certain nearest neighbor graph statistics, runs and empty blocks.},
	number = {2},
	journal = {The Annals of Probability},
	author = {Henze, N. and Voigt, B.},
	month = apr,
	year = {1992},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62G10, geometric probability, nearest neighbors, 60F15, Almost sure convergence, Efron-Stein inequality, empty blocks, Runs},
	pages = {1086--1098},
}


@article{oja_multivariate_2004,
	title = {Multivariate {Nonparametric} {Tests}},
	volume = {19},
	issn = {0883-4237, 2168-8745},
	doi = {10.1214/088342304000000558},
	abstract = {Multivariate nonparametric statistical tests of hypotheses are described for the one-sample location problem, the several-sample location problem and the problem of testing independence between pairs of vectors. These methods are based on affine-invariant spatial sign and spatial rank vectors. They provide affine-invariant multivariate generalizations of the univariate sign test, signed-rank test, Wilcoxon rank sum test, Kruskal–Wallis test, and the Kendall and Spearman correlation tests. While the emphasis is on tests of hypotheses, certain references to associated affine-equivariant estimators are included. Pitman asymptotic efficiencies demonstrate the excellent performance of these methods, particularly in heavy-tailed population settings. Moreover, these methods are easy to compute for data in common dimensions.},
	number = {4},
	urldate = {2022-09-23},
	journal = {Statistical Science},
	author = {Oja, Hannu and Randles, Ronald H.},
	month = nov,
	year = {2004},
	publisher = {Institute of Mathematical Statistics},
	keywords = {affine invariance, Pitman efficiency, robustness, spatial rank, spatial sign},
	pages = {598--605},
}

@article{chernozhukov_mongekantorovich_2017,
	title = {Monge–{Kantorovich} depth, quantiles, ranks and signs},
	volume = {45},
	issn = {0090-5364, 2168-8966},
	doi = {10.1214/16-AOS1450},
	abstract = {We propose new concepts of statistical depth, multivariate quantiles, vector quantiles and ranks, ranks and signs, based on canonical transportation maps between a distribution of interest on \${\textbackslash}mathbb\{R\}{\textasciicircum}\{d\}\$ and a reference distribution on the \$d\$-dimensional unit ball. The new depth concept, called Monge–Kantorovich depth, specializes to halfspace depth for \$d=1\$ and in the case of spherical distributions, but for more general distributions, differs from the latter in the ability for its contours to account for non-convex features of the distribution of interest. We propose empirical counterparts to the population versions of those Monge–Kantorovich depth contours, quantiles, ranks, signs and vector quantiles and ranks, and show their consistency by establishing a uniform convergence property for empirical (forward and reverse) transport maps, which is the main theoretical result of this paper.},
	number = {1},
	journal = {The Annals of Statistics},
	author = {Chernozhukov, Victor and Galichon, Alfred and Hallin, Marc and Henry, Marc},
	month = feb,
	year = {2017},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62G35, 62M15, empirical transport maps, multivariate signs, Statistical depth, uniform convergence of empirical transport, vector quantiles, vector ranks},
	pages = {223--256},
}

@article{preiss_measures_1991,
	title = {Measures in {Banach} spaces are determined by their values on balls},
	volume = {38},
	issn = {0025-5793},
	doi = {10.1112/S0025579300006744},
	number = {2},
	urldate = {2022-10-11},
	journal = {Mathematika. A Journal of Pure and Applied Mathematics},
	author = {Preiss, D. and Tišer, J.},
	year = {1991},
	mrnumber = {1147839},
	pages = {391--397 (1992)},
}

@article{jureckova_nonparametric_2012,
	title = {Nonparametric multivariate rank tests and their unbiasedness},
	volume = {18},
	issn = {1350-7265},
	doi = {10.3150/10-BEJ326},
	abstract = {Although unbiasedness is a basic property of a good test, many tests on vector parameters or scalar parameters against two-sided alternatives are not finite-sample unbiased. This was already noticed by Sugiura [Ann. Inst. Statist. Math. 17 (1965) 261–263]; he found an alternative against which the Wilcoxon test is not unbiased. The problem is even more serious in multivariate models. When testing the hypothesis against an alternative which fits well with the experiment, it should be verified whether the power of the test under this alternative cannot be smaller than the significance level. Surprisingly, this serious problem is not frequently considered in the literature. The present paper considers the two-sample multivariate testing problem. We construct several rank tests which are finite-sample unbiased against a broad class of location/scale alternatives and are finite-sample distribution-free under the hypothesis and alternatives. Each of them is locally most powerful against a specific alternative of the Lehmann type. Their powers against some alternatives are numerically compared with each other and with other rank and classical tests. The question of affine invariance of two-sample multivariate tests is also discussed.},
	number = {1},
	journal = {Bernoulli},
	author = {Jurečková, Jana and Kalina, Jan},
	month = feb,
	year = {2012},
	publisher = {Bernoulli Society for Mathematical Statistics and Probability},
	keywords = {Kolmogorov–Smirnov test, affine invariance, contiguity, Lehmann alternatives, Liu–Singh test, Psi test, Savage test, two-sample multivariate model, unbiasedness, Wilcoxon test},
	pages = {229--251},
}

@article{justel_multivariate_1997,
	title = {A multivariate {Kolmogorov}-{Smirnov} test of goodness of fit},
	volume = {35},
	issn = {0167-7152},
	doi = {10.1016/S0167-7152(97)00020-5},
	abstract = {This paper presents a distribution-free multivariate Kolmogorov-Smirnov goodness-of-fit test. The test uses a statistic which is built using Rosenblatt's transformation and an algorithm is developed to compute it in the bivariate case. An approximate test, that can be easily computed in any dimension, is also presented. The power of these multivariate tests is studied in a simulation study.},
	number = {3},
	journal = {Statistics \& Probability Letters},
	author = {Justel, Ana and Peña, Daniel and Zamar, Rubén},
	month = oct,
	year = {1997},
	keywords = {Kolmogorov-Smirnov statistics, Bonferroni inequality, Empirical distribution function, Rosenblatt's transformation},
	pages = {251--259},
}

@article{meintanis_permutation_2005,
	title = {Permutation tests for homogeneity based on the empirical characteristic function},
	volume = {17},
	issn = {1048-5252},
	doi = {10.1080/10485250500039494},
	abstract = {In this paper, omnibus tests are proposed for testing the homogeneity of two populations. The tests are based on weighted integrals involving the empirical characteristic function. The consistency of the tests as well as their asymptotic distribution under the null hypothesis are investigated. As the decay of the weight functions tends to infinity, the test statistics approach limit values which are related to moment differences between the two populations. The test procedure is based on resampling from the permutation distribution of the test statistic. The new tests are compared with other omnibus tests for homogeneity via a Monte Carlo procedure.},
	number = {5},
	journal = {Journal of Nonparametric Statistics},
	author = {Meintanis, Simos   G.},
	month = jul,
	year = {2005},
	publisher = {Taylor \& Francis},
	keywords = {Nonparametric test, Empirical characteristic function, Two-sample problem},
	pages = {583--592},
}

@article{meintanis_review_2016,
	title = {A review of testing procedures based on the empirical characteristic function},
	volume = {50},
	number = {1},
	journal = {South African Statistical Journal},
	author = {Meintanis, Simos G.},
	year = {2016},
	publisher = {South African Statistical Association (SASA)},
	pages = {1--14},
}

@article{zhou_two-sample_2017,
	title = {Two-sample smooth tests for the equality of distributions},
	volume = {23},
	issn = {1350-7265},
	doi = {10.3150/15-BEJ766},
	abstract = {This paper considers the problem of testing the equality of two unspecified distributions. The classical omnibus tests such as the Kolmogorov–Smirnov and Cramér–von Mises are known to suffer from low power against essentially all but location-scale alternatives. We propose a new two-sample test that modifies the Neyman’s smooth test and extend it to the multivariate case based on the idea of projection pursue. The asymptotic null property of the test and its power against local alternatives are studied. The multiplier bootstrap method is employed to compute the critical value of the multivariate test. We establish validity of the bootstrap approximation in the case where the dimension is allowed to grow with the sample size. Numerical studies show that the new testing procedures perform well even for small sample sizes and are powerful in detecting local features or high-frequency components.},
	number = {2},
	journal = {Bernoulli},
	author = {Zhou, Wen-Xin and Zheng, Chao and Zhang, Zhen},
	month = may,
	year = {2017},
	publisher = {Bernoulli Society for Mathematical Statistics and Probability},
	keywords = {Goodness-of-fit, two-sample problem, high-frequency alternations, multiplier bootstrap, Neyman’s smooth test},
	pages = {951--989},
}

@article{alba-fernandez_bootstrap_2004,
	series = {Recent {Advances} in {Computational} and {Mathematical} {Methods} for {Science} and {Engineering}},
	title = {A bootstrap algorithm for the two-sample problem using trigonometric {Hermite} spline interpolation},
	volume = {9},
	issn = {1007-5704},
	doi = {10.1016/S1007-5704(03)00117-5},
	abstract = {In this paper we study the two-sample problem using procedures based on the empirical characteristic function. We consider the L2 norm of the difference between the empirical characteristic functions and use an interpolant to obtain a numerical integration formula to approximate the test statistic. For our approximation method we have used a trigonometric Hermite interpolant, and afterwards, to estimate the p-value of the resultant statistic, a bootstrap algorithm was implemented.},
	number = {2},
	journal = {Communications in Nonlinear Science and Numerical Simulation},
	author = {Alba-Fernández, V. and Ibáñez-Pérez, M. J. and Jiménez-Gamero, M. D.},
	month = apr,
	year = {2004},
	keywords = {Bootstrap, Empirical characteristic function, Hermite spline interpolation, Homogeneity test},
	pages = {275--286},
}

@article{alba_homogeneity_2001,
	title = {A homogeneity test based on empirical characteristic functions},
	volume = {16},
	issn = {1613-9658},
	doi = {10.1007/s001800100064},
	abstract = {In this paper, a test for the homogeneity of two populations is proposed. It is based on the L2-norm of the difference of the empirical characteristic functions associated to two independent random samples of each population. A quadrature formula is used to construct the test function by using the cubic many-knot Hermite spline interpolation. In order to approximate the null distribution of the statistic, a bootstrap algorithm is used.},
	number = {2},
	journal = {Computational Statistics},
	author = {Alba, M. V. and Barrera, D. and Jiménez, M. D.},
	month = jul,
	year = {2001},
	keywords = {Bootstrap, Empirical characteristic function, Equality of distributions, Many-knot Hermite spline interpolation, Numerical quadrature},
	pages = {255--270},
}


@inproceedings{flaxman_bayesian_2016,
	address = {Arlington, Virginia, USA},
	series = {{UAI}'16},
	title = {Bayesian learning of kernel embeddings},
	isbn = {978-0-9966431-1-5},
	abstract = {Kernel methods are one of the mainstays of machine learning, but the problem of kernel learning remains challenging, with only a few heuristics and very little theory. This is of particular importance in methods based on estimation of kernel mean embeddings of probability measures. For characteristic kernels, which include most commonly used ones, the kernel mean embedding uniquely determines its probability measure, so it can be used to design a powerful statistical testing framework, which includes non-parametric two-sample and independence tests. In practice, however, the performance of these tests can be very sensitive to the choice of kernel and its lengthscale parameters. To address this central issue, we propose a new probabilistic model for kernel mean embeddings, the Bayesian Kernel Embedding model, combining a Gaussian process prior over the Reproducing Kernel Hilbert Space containing the mean embedding with a conjugate likelihood function, thus yielding a closed form posterior over the mean embedding. The posterior mean of our model is closely related to recently proposed shrinkage estimators for kernel mean embeddings, while the posterior uncertainty is a new, interesting feature with various possible applications. Critically for the purposes of kernel learning, our model gives a simple, closed form marginal pseudolikelihood of the observed data given the kernel hyperparameters. This marginal pseudolikelihood can either be optimized to inform the hyperparameter choice or fully Bayesian inference can be used.},
	booktitle = {Proceedings of the {Thirty}-{Second} {Conference} on {Uncertainty} in {Artificial} {Intelligence}},
	publisher = {AUAI Press},
	author = {Flaxman, Seth and Sejdinovic, Dino and Cunningham, John P. and Filippi, Sarah},
	month = jun,
	year = {2016},
	pages = {182--191},
}

@article{szabo_variable_2002,
	title = {Variable selection and pattern recognition with gene expression data generated by the microarray technology},
	volume = {176},
	issn = {0025-5564},
	doi = {10.1016/s0025-5564(01)00103-1},
	abstract = {Lack of adequate statistical methods for the analysis of microarray data remains the most critical deterrent to uncovering the true potential of these promising techniques in basic and translational biological studies. The popular practice of drawing important biological conclusions from just one replicate (slide) should be discouraged. In this paper, we discuss some modern trends in statistical analysis of microarray data with a special focus on statistical classification (pattern recognition) and variable selection. In addressing these issues we consider the utility of some distances between random vectors and their nonparametric estimates obtained from gene expression data. Performance of the proposed distances is tested by computer simulations and analysis of gene expression data on two different types of human leukemia. In experimental settings, the error rate is estimated by cross-validation, while a control sample is generated in computer simulation experiments aimed at testing the proposed gene selection procedures and associated classification rules.},
	number = {1},
	journal = {Mathematical Biosciences},
	author = {Szabo, A. and Boucher, K. and Carroll, W. L. and Klebanov, L. B. and Tsodikov, A. D. and Yakovlev, A. Y.},
	month = mar,
	year = {2002},
	pmid = {11867085},
	keywords = {Computer Simulation, Gene Expression Profiling, Humans, Leukemia, Myeloid, Acute, Oligonucleotide Array Sequence Analysis, Pattern Recognition, Automated, Precursor Cell Lymphoblastic Leukemia-Lymphoma},
	pages = {71--98},
}

@article{szabo_multivariate_2003,
	title = {Multivariate exploratory tools for microarray data analysis},
	volume = {4},
	issn = {1465-4644},
	doi = {10.1093/biostatistics/4.4.555},
	abstract = {The ultimate success of microarray technology in basic and applied biological sciences depends critically on the development of statistical methods for gene expression data analysis. The most widely used tests for differential expression of genes are essentially univariate. Such tests disregard the multidimensional structure of microarray data. Multivariate methods are needed to utilize the information hidden in gene interactions and hence to provide more powerful and biologically meaningful methods for finding subsets of differentially expressed genes. The objective of this paper is to develop methods of multidimensional search for biologically significant genes, considering expression signals as mutually dependent random variables. To attain these ends, we consider the utility of a pertinent distance between random vectors and its empirical counterpart constructed from gene expression data. The distance furnishes exploratory procedures aimed at finding a target subset of differentially expressed genes. To determine the size of the target subset, we resort to successive elimination of smaller subsets resulting from each step of a random search algorithm based on maximization of the proposed distance. Different stopping rules associated with this procedure are evaluated. The usefulness of the proposed approach is illustrated with an application to the analysis of two sets of gene expression data.},
	number = {4},
	journal = {Biostatistics},
	author = {Szabo, Aniko and Boucher, Kenneth and Jones, David and Tsodikov, Alexander D. and Klebanov, Lev B. and Yakovlev, Andrei Y.},
	month = oct,
	year = {2003},
	pages = {555--567},
}

@article{hall_geometric_2005,
	title = {Geometric representation of high dimension, low sample size data},
	volume = {67},
	issn = {1467-9868},
	doi = {10.1111/j.1467-9868.2005.00510.x},
	abstract = {Summary. High dimension, low sample size data are emerging in various areas of science. We find a common structure underlying many such data sets by using a non-standard type of asymptotics: the dimension tends to ∞ while the sample size is fixed. Our analysis shows a tendency for the data to lie deterministically at the vertices of a regular simplex. Essentially all the randomness in the data appears only as a random rotation of this simplex. This geometric representation is used to obtain several new statistical insights.},
	number = {3},
	journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
	author = {Hall, Peter and Marron, J. S. and Neeman, Amnon},
	year = {2005},
	keywords = {Chemometrics, Large dimensional data, Medical images, Microarrays, Multivariate analysis, Non-standard asymptotics},
	pages = {427--444},
}

@article{fleming_supremum_1987,
	title = {Supremum {Versions} of the {Log}-{Rank} and {Generalized} {Wilcoxon} {Statistics}},
	volume = {82},
	issn = {0162-1459},
	doi = {10.1080/01621459.1987.10478435},
	abstract = {Considerable progress has been made in generalizing to censored data hypothesis tests based on rank statistics. The most commonly used generalizations are the log-rank test statistic, which extends the Savage exponential scores statistic (Cox 1972; Mantel 1966) and the generalized Wilcoxon statistics (Gehan 1965; Peto and Peto 1972). Gill (1980) showed that in the two-sample problem the asymptotic efficiency properties of the Savage and Wilcoxon statistics are maintained in censored data by the log-rank and Peto and Peto Wilcoxon statistics, respectively, when censoring is the same in both samples. Heuristics and simulations have implied that some recently proposed supremum statistics, closely related to these two popular linear rank statistics, can be used to construct tests that provide good power against a much wider range of nonlocal alternatives (see, e.g., Fleming and Harrington 1981; Fleming, O'Brien, O'Fallon, and Harrington 1980; Gill 1980; Schumacher 1984). In this article we consider new and previously discussed classes of rank statistics that are related to the log-rank and generalized Wilcoxon tests. We focus on linear rank statistics and their Renyi-type supremum versions. These classes are either of the “asymptotically fully efficient” type or the “approximately distribution-free” type discussed by Leurgans (1983). The large sample distributions of these statistics are established for the general setting in which ties can be present in the data. By restricting attention to the general random censorship model, we obtain the asymptotic results by stating and applying a simplified version of Gill's (1980) limit theorems. We examine more thoroughly than in previous literature the relative operating characteristics of these newly and previously proposed censored data rank statistics. Important insights are obtained from simulations performed for small and moderate sample sizes, in equal and unequal sample sizes, and across a range of survival differences and degrees of censorship. For example, results reveal that supremum versions of the log-rank statistic are nearly as sensitive to proportional-hazards alternatives as the efficient log-rank test. In addition, these supremum versions provide greater sensitivity across a wide range of nonproportional-hazards configurations. This increase in power from the supremum statistics is less evident, however, when data are so heavily censored that one can only estimate early survival differences. Another important insight is obtained from the comparison in small samples of asymptotically fully efficient versus approximately distribution-free statistics. Although the former class has received much greater attention, the latter has the desirable property that the types of survival differences detected with highest power are less affected by degree of censorship. In certain circumstances, it would be appealing to apply simultaneously a linear rank statistic and its supremum version. Distributional results are obtained to show how this can be done appropriately and in a very straightforward manner.},
	number = {397},
	journal = {Journal of the American Statistical Association},
	author = {Fleming, Thomas R. and Harrington, David P. and O'sullivan, Margaret},
	month = mar,
	year = {1987},
	publisher = {Taylor \& Francis},
	keywords = {Proportional hazards, Rank statistics, Renyi-type test},
	pages = {312--320},
}

@article{marron_distance-weighted_2007,
	title = {Distance-{Weighted} {Discrimination}},
	volume = {102},
	issn = {0162-1459},
	abstract = {High-dimension low—sample size statistical analysis is becoming increasingly important in a wide range of applied contexts. In such situations, the popular support vector machine suffers from "data piling" at the margin, which can diminish generalizability. This leads naturally to the development of distance-weighted discrimination, which is based on second-order cone programming, a modern computationally intensive optimization method.},
	number = {480},
	journal = {Journal of the American Statistical Association},
	author = {Marron, J. S. and Todd, Michael J. and Ahn, Jeongyoun},
	year = {2007},
	publisher = {American Statistical Association, Taylor \& Francis, Ltd.},
	pages = {1267--1271},
}


@article{randles_multivariate_1990,
	title = {Multivariate rank tests for the two-sample location problem},
	volume = {19},
	issn = {0361-0926},
	doi = {10.1080/03610929008830439},
	abstract = {A multivariate affinc-invariant family of rank tests is proposed for the two sample location problem. The class of statistics introduced is built upon Randles' multivariate one-sample sign statistic based on interdirections and the multivariate one-sample signed-rank statistic of Peters and Randles. Asymptotic relative efficiencies are obtained which indicate that selected members of the class perform very well for a broad class of distributions. Further comparisons are made among several statistics using Monte Carlo results.},
	number = {11},
	journal = {Communications in Statistics - Theory and Methods},
	author = {Randles, Ronald H. and Peters, Dawn},
	month = jan,
	year = {1990},
	publisher = {Taylor \& Francis},
	keywords = {affine-invariant, location, rank, sign-test, signed-rank, two sample},
	pages = {4225--4238},
}


@article{hettmansperger_affine_1994,
	title = {Affine invariant multivariate multisample sign tests},
	volume = {56},
	number = {1},
	journal = {Journal of the Royal Statistical Society: Series B (Methodological)},
	author = {Hettmansperger, Thomas P. and Oja, Hannu},
	year = {1994},
	publisher = {Wiley Online Library},
	pages = {235--249},
}

@article{choi_approach_1997,
	title = {An {Approach} to {Multivariate} {Rank} {Tests} in {Multivariate} {Analysis} of {Variance}},
	volume = {92},
	issn = {0162-1459},
	doi = {10.1080/01621459.1997.10473680},
	abstract = {A class of multivariate rank-like quantities is defined and used to develop multivariate tests to mimic popular one-dimensional rank tests such as the Mann-Whitney/Wilcoxon two-sample test, the Jonckheere-Terpstra test for trend, and the Kruskal-Wallis one-way analysis of variance test. Tests in one-way analysis of variance are developed based on qualitative orthogonal contrasts, allowing decomposition of an overall statistic into asymptotically independent components based on the contrasts. The class of tests includes the usual normal-theory tests and the componentwise rank tests, but the main focus is on the tests based on a particular definition of multivariate rank. A study of the Pitman efficiency of the latter tests to those based on multivariate medians shows them to be superior at the normal, slightly heavy-tailed, and light-tailed distributions, whereas the median-based tests are superior for heavy tails. These results are analogous to the univariate case.},
	number = {440},
	journal = {Journal of the American Statistical Association},
	author = {Choi, K. and Marden, J.},
	month = dec,
	year = {1997},
	publisher = {Taylor \& Francis},
	keywords = {Jonckheere-Terpstra test, Kruskal-Wallis test, Mann-Whitney/Wilcoxon test, Median test, Orthogonal contrasts, Pitman efficiency.},
	pages = {1581--1590},
}

@article{hettmansperger_affine_1998,
	title = {Affine {Invariant} {Multivariate} {Rank} {Tests} for {Several} {Samples}},
	volume = {8},
	issn = {1017-0405},
	abstract = {Affine invariant analogues of the two-sample Mann-Whitney-Wilcoxon rank sum test and the c-sample Kruskal-Wallis test for the multivariate location model are introduced. The definition of a multivariate (centered) rank function in the development is based on the Oja criterion function. This work extends bivariate rank methods discussed by Brown and Hettmansperger (1987a,b) and multivariate sign methods by Hettmansperger and Oja (1994). The asymptotic distribution theory is developed to consider the Pitman asymptotic efficiencies and the theory is illustrated by an example.},
	number = {3},
	journal = {Statistica Sinica},
	author = {Hettmansperger, T. P. and Möttönen, J. and Oja, Hannu},
	year = {1998},
	publisher = {Institute of Statistical Science, Academia Sinica},
	pages = {785--800},
}

@article{hallin_distribution_2021,
	title = {Distribution and quantile functions, ranks and signs in dimension d: {A} measure transportation approach},
	volume = {49},
	issn = {0090-5364, 2168-8966},
	shorttitle = {Distribution and quantile functions, ranks and signs in dimension d},
	doi = {10.1214/20-AOS1996},
	abstract = {Unlike the real line, the real space Rd, for d≥2, is not canonically ordered. As a consequence, such fundamental univariate concepts as quantile and distribution functions and their empirical counterparts, involving ranks and signs, do not canonically extend to the multivariate context. Palliating that lack of a canonical ordering has been an open problem for more than half a century, generating an abundant literature and motivating, among others, the development of statistical depth and copula-based methods. We show that, unlike the many definitions proposed in the literature, the measure transportation-based ranks and signs introduced in Chernozhukov, Galichon, Hallin and Henry (Ann. Statist. 45 (2017) 223–256) enjoy all the properties that make univariate ranks a successful tool for semiparametric inference. Related with those ranks, we propose a new center-outward definition of multivariate distribution and quantile functions, along with their empirical counterparts, for which we establish a Glivenko–Cantelli result. Our approach is based on McCann (Duke Math. J. 80 (1995) 309–323) and our results do not require any moment assumptions. The resulting ranks and signs are shown to be strictly distribution-free and essentially maximal ancillary in the sense of Basu (Sankhyā 21 (1959) 247–256) which, in semiparametric models involving noise with unspecified density, can be interpreted as a finite-sample form of semiparametric efficiency. Although constituting a sufficient summary of the sample, empirical center-outward distribution functions are defined at observed values only. A continuous extension to the entire d-dimensional space, yielding smooth empirical quantile contours and sign curves while preserving the essential monotonicity and Glivenko–Cantelli features of the concept, is provided. A numerical study of the resulting empirical quantile contours is conducted.},
	number = {2},
	journal = {The Annals of Statistics},
	author = {Hallin, Marc and Barrio, Eustasio del and Cuesta-Albertos, Juan and Matr\'{a}n, Carlos},
	month = apr,
	year = {2021},
	publisher = {Institute of Mathematical Statistics},
	keywords = {62B05, 62G30, ancillarity, Basu theorem, cyclical monotonicity, distribution-freeness, Glivenko–Cantelli theorem, multivariate distribution function, Multivariate quantiles, multivariate ranks, multivariate signs},
	pages = {1139--1165},
}

@article{hallin_efficient_2022,
	title = {Efficient {Fully} {Distribution}-{Free} {Center}-{Outward} {Rank} {Tests} for {Multiple}-{Output} {Regression} and {MANOVA}},
	volume = {118},
	issn = {0162-1459},
	doi = {10.1080/01621459.2021.2021921},
	abstract = {Extending rank-based inference to a multivariate setting such as multiple-output regression or MANOVA with unspecified d-dimensional error density has remained an open problem for more than half a century. None of the many solutions proposed so far is enjoying the combination of distribution-freeness and efficiency that makes rank-based inference a successful tool in the univariate setting. A concept of center-outward multivariate ranks and signs based on measure transportation ideas has been introduced recently. Center-outward ranks and signs are not only distribution-free but achieve in dimension d {\textgreater} 1 the (essential) maximal ancillarity property of traditional univariate ranks. In the present case, we show that fully distribution-free testing procedures based on center-outward ranks can achieve parametric efficiency. We establish the Hájek representation and asymptotic normality results required in the construction of such tests in multiple-output regression and MANOVA models. Simulations and an empirical study demonstrate the excellent performance of the proposed procedures.},
	number = {543},
	journal = {Journal of the American Statistical Association},
	author = {Hallin, Marc and Hlubinka, Daniel and Hudecová, \v{S}\'{a}rka},
	month = jan,
	year = {2022},
	publisher = {Taylor \& Francis},
	keywords = {Distribution-free tests, Hájek representation, Multivariate ranks, Multivariate signs},
	pages = {1--17},
}


@article{holmes_two-sample_2015,
	title = {Two-sample {Bayesian} {Nonparametric} {Hypothesis} {Testing}},
	volume = {10},
	issn = {1936-0975, 1931-6690},
	doi = {10.1214/14-BA914},
	abstract = {In this article we describe Bayesian nonparametric procedures for two-sample hypothesis testing. Namely, given two sets of samples y(1){\textasciitilde}iidF(1) and y(2){\textasciitilde}iidF(2), with F(1),F(2) unknown, we wish to evaluate the evidence for the null hypothesis H0:F(1)≡F(2) versus the alternative H1:F(1)≠F(2). Our method is based upon a nonparametric Pólya tree prior centered either subjectively or using an empirical procedure. We show that the Pólya tree prior leads to an analytic expression for the marginal likelihood under the two hypotheses and hence an explicit measure of the probability of the null Pr(H0{\textbar}\{y(1),y(2)\})},
	number = {2},
	journal = {Bayesian Analysis},
	author = {Holmes, Chris C. and Caron, François and Griffin, Jim E. and Stephens, David A.},
	month = jun,
	year = {2015},
	publisher = {International Society for Bayesian Analysis},
	keywords = {Bayesian nonparametrics, Hypothesis testing, Pólya tree},
	pages = {297--320},
}

@article{ma_coupling_2011,
	title = {Coupling {Optional} {Pólya} {Trees} and the {Two} {Sample} {Problem}},
	volume = {106},
	issn = {0162-1459},
	doi = {10.1198/jasa.2011.tm10003},
	abstract = {Testing and characterizing the difference between two data samples is of fundamental interest in statistics. Existing methods such as Kolmogorov–Smirnov and Cramer–von Mises tests do not scale well as the dimensionality increases and provide no easy way to characterize the difference should it exist. In this work, we propose a theoretical framework for inference that addresses these challenges in the form of a prior for Bayesian nonparametric analysis. The new prior is constructed based on a random-partition-and-assignment procedure similar to the one that defines the standard optional Pólya tree distribution, but has the ability to generate multiple random distributions jointly. These random probability distributions are allowed to “couple,” that is, to have the same conditional distribution, on subsets of the sample space. We show that this “coupling optional Pólya tree” prior provides a convenient and effective way for both the testing of two sample difference and the learning of the underlying structure of the difference. In addition, we discuss some practical issues in the computational implementation of this prior and provide several numerical examples to demonstrate its work. Supplementary materials for this article are available online.},
	number = {496},
	journal = {Journal of the American Statistical Association},
	author = {Ma, Li and Wong, Wing Hung},
	month = dec,
	year = {2011},
	publisher = {Taylor \& Francis},
	keywords = {Bayesian inference, Case-control study, Nonparametrics, Recursive partition},
	pages = {1553--1565},
}

@article{fokianos_semiparametric_2001,
	title = {A {Semiparametric} {Approach} to the {One}-{Way} {Layout}},
	volume = {43},
	issn = {0040-1706},
	abstract = {We consider m distributions in which the first m - 1 are obtained by multiplicative exponential distortions of the mth distribution, which is a reference. The combined data from m samples, one from each distribution, are used in the semiparametric large-sample problem of estimating each distortion and the reference distribution and testing the hypothesis that the distributions are identical. The approach generalizes the classical normal-based one-way analysis of variance in the sense that it obviates the need for a completely specified parametric model. An advantage is that the probability density of the reference distribution is estimated from the combined data and not only from the mth sample. A power comparison with the t and F tests and with two nonparametric tests, obtained by means of a simulation, points to the merit of the present approach. The method is applied to rain-rate data from meterological instruments.},
	number = {1},
	journal = {Technometrics},
	author = {Fokianos, Konstantinos and Qin, Jing and Kedem, Benjamin and Short, David A.},
	year = {2001},
	publisher = {Taylor \& Francis, Ltd., American Statistical Association, American Society for Quality},
	pages = {56--65},
}

@article{fukumizu_dimensionality_2004,
	title = {Dimensionality {Reduction} for {Supervised} {Learning} with {Reproducing} {Kernel} {Hilbert} {Spaces}},
	volume = {5},
	journal = {Journal of Machine Learning Research},
	author = {Fukumizu, Kenji and Bach, Francis R. and Jordan, Michael I.},
	year = {2004},
	pages = {73--99},
}

@inproceedings{sriperumbudur_injective_2008,
	title = {Injective {Hilbert} space embeddings of probability measures},
	booktitle = {21st {Annual} {Conference} on {Learning} {Theory} ({COLT} 2008)},
	publisher = {Omnipress},
	author = {Sriperumbudur, Bharath K. and Gretton, Arthur and Fukumizu, Kenji and Lanckriet, Gert and Schölkopf, Bernhard},
	year = {2008},
	pages = {111--122},
}


@inproceedings{sutherland_generative_2019,
	title={Generative Models and Model Criticism via Optimized Maximum Mean Discrepancy},
	author={Danica J. Sutherland and Hsiao-Yu Tung and Heiko Strathmann and Soumyajit De and Aaditya Ramdas and Alex Smola and Arthur Gretton},
	booktitle={International Conference on Learning Representations},
	year={2017},
}


@inproceedings{li_mmd_2017,
	title = {{MMD} {GAN}: {Towards} {Deeper} {Understanding} of {Moment} {Matching} {Network}},
	volume = {30},
	shorttitle = {{MMD} {GAN}},
	urldate = {2022-10-27},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Li, Chun-Liang and Chang, Wei-Cheng and Cheng, Yu and Yang, Yiming and Poczos, Barnabas},
	year = {2017},
}

@inproceedings{rahimi_random_2007,
	title = {Random {Features} for {Large}-{Scale} {Kernel} {Machines}},
	volume = {20},
	abstract = {To accelerate the training of kernel machines, we propose to map the input data to a randomized low-dimensional feature space and then apply existing fast linear methods. The features are designed so that the inner products of the transformed data are approximately equal to those in the feature space of a user speciﬁed shift- invariant kernel. We explore two sets of random features, provide convergence bounds on their ability to approximate various radial basis kernels, and show that in large-scale classiﬁcation and regression tasks linear machine learning al- gorithms applied to these features outperform state-of-the-art large-scale kernel machines.},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Rahimi, Ali and Recht, Benjamin},
	year = {2007},
}

@inproceedings{le_fastfood_2013,
	title = {Fastfood - {Computing} {Hilbert} {Space} {Expansions} in loglinear time},
	abstract = {Fast nonlinear function classes are crucial for nonparametric estimation, such as in kernel methods. This paper proposes an improvement to random kitchen sinks that offers significantly faster computation in log-linear time without sacrificing accuracy. Furthermore, we show how one may adjust the regularization properties of the kernel simply by changing the spectral distribution of the projection matrix. We provide experimental results which show that even for for moderately small problems we already achieve two orders of magnitude faster computation and three orders of magnitude lower memory footprint.},
	booktitle = {Proceedings of the 30th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Le, Quoc and Sarlos, Tamas and Smola, Alexander},
	month = may,
	year = {2013},
	issn = {1938-7228},
	pages = {244--252},
}


@article{schutze_using_2012,
	title = {Using the {Averaged} {Hausdorff} {Distance} as a {Performance} {Measure} in {Evolutionary} {Multiobjective} {Optimization}},
	volume = {16},
	issn = {1941-0026},
	doi = {10.1109/TEVC.2011.2161872},
	abstract = {The Hausdorff distance dH is a widely used tool to measure the distance between different objects in several research fields. Possible reasons for this might be that it is a natural extension of the well-known and intuitive distance between points and/or the fact that dH defines in certain cases a metric in the mathematical sense. In evolutionary multiobjective optimization (EMO) the task is typically to compute the entire solution set-the so-called Pareto set-respectively its image, the Pareto front. Hence, dH should, at least at first sight, be a natural choice to measure the performance of the outcome set in particular since it is related to the terms spread and convergence as used in EMO literature. However, so far, dH does not find the general approval in the EMO community. The main reason for this is that dH penalizes single outliers of the candidate set which does not comply with the use of stochastic search algorithms such as evolutionary strategies. In this paper, we define a new performance indicator, Δp, which can be viewed as an “averaged Hausdorff distance” between the outcome set and the Pareto front and which is composed of (slight modifications of) the well-known indicators generational distance (GD) and inverted generational distance (IGD). We will discuss theoretical properties of Δp (as well as for GD and IGD) such as the metric properties and the compliance with state-of-theart multiobjective evolutionary algorithms (MOEAs), and will further on demonstrate by empirical results the potential of Δp as a new performance indicator for the evaluation of MOEAs.},
	number = {4},
	journal = {IEEE Transactions on Evolutionary Computation},
	author = {Schutze, Oliver and Esquivel, Xavier and Lara, Adriana and Coello, Carlos A. Coello},
	month = aug,
	year = {2012},
	keywords = {Approximation algorithms, Approximation methods, Averaged Hausdorff distance, Benchmark testing, Convergence, Delta modulation, generational distance, inverted generational distance, Measurement, multiobjective optimization, Optimization, performance indicator},
	pages = {504--522},
}

@book{thas_comparing_2010,
	title = {Comparing distributions},
	volume = {233},
	publisher = {Springer},
	author = {Thas, Olivier},
	year = {2010},
}

@article{barrio_central_2019,
	title = {Central limit theorems for empirical transportation cost in general dimension},
	volume = {47},
	issn = {0091-1798, 2168-894X},
	doi = {10.1214/18-AOP1275},
	abstract = {We consider the problem of optimal transportation with quadratic cost between a empirical measure and a general target probability on \${\textbackslash}mathbb\{R\}{\textasciicircum}\{d\}\$, with \$d{\textbackslash}geq1\$. We provide new results on the uniqueness and stability of the associated optimal transportation potentials, namely, the minimizers in the dual formulation of the optimal transportation problem. As a consequence, we show that a CLT holds for the empirical transportation cost under mild moment and smoothness requirements. The limiting distributions are Gaussian and admit a simple description in terms of the optimal transportation potentials.},
	number = {2},
	journal = {The Annals of Probability},
	author = {Barrio, Eustasio del and Loubes, Jean-Michel},
	month = mar,
	year = {2019},
	publisher = {Institute of Mathematical Statistics},
	keywords = {60F05, 62E20, 46N30, CLT, Efron–Stein inequality, optimal matching, Optimal transportation},
	pages = {926--951},
}

@article{zolotarev_probability_1984,
	title = {Probability {Metrics}},
	volume = {28},
	issn = {0040-585X},
	doi = {10.1137/1128025},
	number = {2},
	urldate = {2022-11-21},
	journal = {Theory of Probability \& Its Applications},
	author = {Zolotarev, V. M.},
	month = jan,
	year = {1984},
	publisher = {Society for Industrial and Applied Mathematics},
	pages = {278--302},
}


@article{rachev_approximation_1990,
	title = {Approximation of sums by compound {Poisson} distributions with respect to stop-loss distances},
	volume = {22},
	issn = {0001-8678, 1475-6064},
	doi = {10.2307/1427540},
	abstract = {The approximation of sums of independent random variables by compound Poisson distributions with respect to stop-loss distances is investigated. These distances are motivated by risk-theoretic considerations. In contrast to the usual construction of approximating compound Poisson distributions, the method suggested in this paper is to fit several moments. For two moments, this can be achieved by scale transformations. It is shown that the new approximations are more stable and improve the usual approximations by accompanying laws in examples where the probability 1 – pi that the ith summand is zero is not too large.},
	number = {2},
	journal = {Advances in Applied Probability},
	author = {Rachev, S. T. and Rüschendorf, L.},
	month = jun,
	year = {1990},
	publisher = {Cambridge University Press},
	keywords = {INSURANCE MATHEMATICS, RISK THEORY},
	pages = {350--374},
}


@book{gerber_introduction_1979,
	title = {An introduction to mathematical risk theory},
	publisher = {Huebner Foundation Monograph},
	author = {Gerber, Hans U.},
	year = {1979},
	location ={Philadelphia PA},
}

@book{dudley_real_1989,
	address = {New York},
	title = {Real {Analysis} and {Probability}},
	isbn = {978-1-351-07619-7},
	abstract = {Written by one of the best-known probabilists in the world this text offers a clear and modern presentation of modern probability theory and an exposition of the interplay between the properties of metric spaces and those of probability measures. This text is the first at this level to include discussions of the subadditive ergodic theorems, metrics for convergence in laws and the Borel isomorphism theory. The proofs for the theorems are consistently brief and clear and each chapter concludes with a set of historical notes and references. This book should be of interest to students taking degree courses in real analysis and/or probability theory.},
	publisher = {Wadsworth and Brooks},
	author = {Dudley, R. M.},
	month = dec,
	year = {1989},
	doi = {10.1201/9781351076197},
	location = {Belmont, CA},
}

@book{rachev_mass_1998,
	address = {New York},
	series = {Probability and its {Applications}},
	title = {Mass {Transportation} {Problems} {Volume} 1: {Theory}},
	isbn = {978-0-387-98350-9},
	publisher = {Springer},
	author = {Rachev, Svetlozar T. and Rüschendorf, Ludger},
	year = {1998},
	doi = {10.1007/b98893},
	keywords = {algorithms, functional analysis, Helium-Atom-Streuung, Maß, Moment, operations research, Probability theory, queueing theory, stochastic differential equation},
}

@article{sugiyama_density-difference_2013,
	title = {Density-{Difference} {Estimation}},
	volume = {25},
	issn = {0899-7667},
	doi = {10.1162/NECO_a_00492},
	abstract = {We address the problem of estimating the difference between two probability densities. A naive approach is a two-step procedure of first estimating two densities separately and then computing their difference. However, this procedure does not necessarily work well because the first step is performed without regard to the second step, and thus a small estimation error incurred in the first stage can cause a big error in the second stage. In this letter, we propose a single-shot procedure for directly estimating the density difference without separately estimating two densities. We derive a nonparametric finite-sample error bound for the proposed single-shot density-difference estimator and show that it achieves the optimal convergence rate. We then show how the proposed density-difference estimator can be used in L2-distance approximation. Finally, we experimentally demonstrate the usefulness of the proposed method in robust distribution comparison such as class-prior estimation and change-point detection.},
	number = {10},
	journal = {Neural Computation},
	author = {Sugiyama, Masashi and Kanamori, Takafumi and Suzuki, Taiji and Plessis, Marthinus Christoffel du and Liu, Song and Takeuchi, Ichiro},
	month = oct,
	year = {2013},
	pages = {2734--2775},
}

@article{ali_general_1966,
	title = {A {General} {Class} of {Coefficients} of {Divergence} of {One} {Distribution} from {Another}},
	volume = {28},
	issn = {2517-6161},
	doi = {10.1111/j.2517-6161.1966.tb00626.x},
	abstract = {Let p1 and p2 be two probability measures on the same space and let φ be the generalized Radon-Nikodym derivative of p2 with respect to p1. If C is a continuous convex function of a real variable such that the p1-expectation (generalized as in Section 3) of C(φ) provides a reasonable coefficient of the p1-dispersion of φ, then this expectation has basic properties which it is natural to demand of a coefficient of divergence of p2 from p1. A general class of coefficients of divergence is generated in this way and it is shown that various available measures of divergence, distance, discriminatory information, etc., are members of this class.},
	number = {1},
	journal = {Journal of the Royal Statistical Society: Series B (Methodological)},
	author = {Ali, S. M. and Silvey, S. D.},
	year = {1966},
	pages = {131--142},
}


@article{paul_clustering-based_2022,
	title = {Some {Clustering}-{Based} {Exact} {Distribution}-{Free} $k$-{Sample} {Tests} {Applicable} to {High} {Dimension}, {Low} {Sample} {Size} {Data}},
	volume = {190},
	issn = {0047-259X},
	url = {https://www.sciencedirect.com/science/article/pii/S0047259X21001743},
	doi = {10.1016/j.jmva.2021.104897},
	abstract = {Testing homogeneity of k(≥2) multivariate distributions is a challenging problem in statistics, especially when the dimension of the data is much larger than the sample size. Most of the existing tests often perform poorly in this high dimension, low sample size (HDLSS) regime, and many of them cannot be used at all. In this article, we propose some nonparametric tests for this purpose. These tests have the distribution-free property in finite sample situations. They are based on a high dimensional clustering algorithm that makes a partition of the data to form a contingency table. Using the cell frequencies of that table, we construct the test statistics. We can develop tests based on a k-partition of the data or estimate the number of partitions from the data and construct tests based on it. Under appropriate regularity conditions, we prove the consistency of these tests in the HDLSS asymptotic regime. We also consider a multiscale approach, where the results for different number of partitions are aggregated judiciously. Extensive simulation study and analysis of some benchmark datasets illustrate the superiority of the proposed tests over some existing methods.},
	urldate = {2024-09-25},
	journal = {Journal of Multivariate Analysis},
	author = {Paul, Biplab and De, Shyamal K. and Ghosh, Anil K.},
	month = jul,
	year = {2022},
	keywords = {Cluster analysis, Contingency tables, Dunn index, Generalized hypergeometric distribution, High dimensional asymptotics, Multiscale approach, Rand index, Tests of independence},
	pages = {104897},
	file = {ScienceDirect Snapshot:C\:\\Users\\stolte\\Zotero\\storage\\IP96AF4K\\S0047259X21001743.html:text/html},
}


@article{stolte_methods_2024,
	title = {Methods for {Quantifying} {Dataset} {Similarity}: {A} {Review}, {Taxonomy} and {Comparison}},
	volume = {18},
	issn = {1935-7516},
	shorttitle = {Methods for quantifying dataset similarity},
	doi = {10.1214/24-SS149},
	abstract = {Quantifying the similarity between datasets has widespread applications in statistics and machine learning. The performance of a predictive model on novel datasets, referred to as generalizability, depends on how similar the training and evaluation datasets are. Exploiting or transferring insights between similar datasets is a key aspect of meta-learning and transfer-learning. In simulation studies, the similarity between distributions of simulated datasets and real datasets, for which the performance of methods is assessed, is crucial. In two- or k-sample testing, it is checked, whether the underlying distributions of two or more datasets coincide. Extremely many approaches for quantifying dataset similarity have been proposed in the literature. We examine more than 100 methods and provide a taxonomy, classifying them into ten classes. In an extensive review of these methods the main underlying ideas, formal definitions, and important properties are introduced. We compare the 118 methods in terms of their applicability, interpretability, and theoretical properties, in order to provide recommendations for selecting an appropriate dataset similarity measure based on the specific goal of the dataset comparison and on the properties of the datasets at hand. An online tool facilitates the choice of the appropriate dataset similarity measure.},
	journal = {Statistics Surveys},
	author = {Stolte, Marieke and Kappenberg, Franziska and Rahnenführer, Jörg and Bommert, Andrea},
	month = jan,
	year = {2024},
	publisher = {Amer. Statist. Assoc., the Bernoulli Soc., the Inst. Math. Statist., and the Statist. Soc. Canada},
	keywords = {05C90, 62E99, 62G10, 62H15, 62H30, Binary classification, Dataset similarity, Distance measure, divergence, Energy statistic, graph-based distance, inter-point distance, kernel mean embedding, maximum mean discrepancy, meta-learning, Optimal transport, Permutation test, probability metric, similarity measure, theoretical properties, two-sample test},
	pages = {163--298},
}

@article{sarkar_perfect_2020,
	title = {On {Perfect} {Clustering} of {High} {Dimension}, {Low} {Sample} {Size} {Data}},
	volume = {42},
	issn = {1939-3539},
	url = {https://ieeexplore.ieee.org/document/8695805},
	doi = {10.1109/TPAMI.2019.2912599},
	abstract = {Popular clustering algorithms based on usual distance functions (e.g., the Euclidean distance) often suffer in high dimension, low sample size (HDLSS) situations, where concentration of pairwise distances and violation of neighborhood structure have adverse effects on their performance. In this article, we use a new data-driven dissimilarity measure, called MADD, which takes care of these problems. MADD uses the distance concentration phenomenon to its advantage, and as a result, clustering algorithms based on MADD usually perform well for high dimensional data. We establish it using theoretical as well as numerical studies. We also address the problem of estimating the number of clusters. This is a challenging problem in cluster analysis, and several algorithms are available for it. We show that many of these existing algorithms have superior performance in high dimensions when they are constructed using MADD. We also construct a new estimator based on a penalized version of the Dunn index and prove its consistency in the HDLSS asymptotic regime. Several simulated and real data sets are analyzed to demonstrate the usefulness of MADD for cluster analysis of high dimensional data.},
	number = {9},
	urldate = {2024-10-04},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	author = {Sarkar, Soham and Ghosh, Anil K.},
	month = sep,
	year = {2020},
	note = {Conference Name: IEEE Transactions on Pattern Analysis and Machine Intelligence},
	keywords = {Clustering algorithms, Dunn index, Estimation, Euclidean distance, hierarchical clustering, high dimensional consistency, Indexes, k-means clustering, pairwise distances, Rand index, Single photon emission computed tomography, Sociology, Statistics},
	pages = {2257--2272},
}


@book{rao_advanced_1952,
	title = {Advanced statistical methods in biometric research.},
	author = {Rao, C. Radhakrishna},
	year = {1952},
	publisher = {John Wiley \& Sons},
	location = {New York},
}

@Misc{densityratio,
	author = {Thom Benjamin Volker},
	title = {densityratio: Distribution {Comparison} through {Density} {Ratio} {Estimation}},
	version = {0.0.1},
	year = {2024},
	url = {https://github.com/thomvolker/densityratio},
	doi = {10.5281/zenodo.13881689},
}

@Article{dbscan,
	title = {{dbscan}: {Fast} {Density}-{Based} {Clustering} with {R}},
	author = {Michael Hahsler and Matthew Piekenbrock and Derek Doran},
	journal = {Journal of Statistical Software},
	year = {2019},
	volume = {91},
	number = {1},
	pages = {1--30},
	doi = {10.18637/jss.v091.i01},
}

@Manual{FNN,
	title = {FNN: {Fast} {Nearest} {Neighbor} {Search} {Algorithms} and {Applications}},
	author = {Alina Beygelzimer and Sham Kakadet and John Langford and Sunil Arya and David Mount and Shengqiao Li},
	year = {2024},
	note = {R package version 1.1.4},
	url = {https://CRAN.R-project.org/package=FNN},
}

@article{kullback_information_1951,
	title = {On {Information} and {Sufficiency}},
	volume = {22},
	issn = {0003-4851, 2168-8990},
	doi = {10.1214/aoms/1177729694},
	abstract = {The Annals of Mathematical Statistics},
	number = {1},
	journal = {The Annals of Mathematical Statistics},
	author = {Kullback, S. and Leibler, R. A.},
	month = mar,
	year = {1951},
	publisher = {Institute of Mathematical Statistics},
	pages = {79--86},
}

@book{rachev_probability_1991,
	address = {Chichester},
	title = {Probability metrics and the stability of stochastic models},
	publisher = {John Wiley \& Sons},
	author = {Rachev, Svetlozar T.},
	year = {1991},
}

@book{rachev_probability_2011,
	address = {New York},
	title = {A {Probability} {Metrics} {Approach} to {Financial} {Risk} {Measures}},
	isbn = {978-1-4443-9271-5},
	publisher = {John Wiley \& Sons, Ltd},
	author = {Rachev, Svetlozar T. and Stoyanov, Stoyan V. and Fabozzi, Frank J.},
	year = {2011},
}

@book{rachev_advanced_2008,
	series = {The {Frank} {J}. {Fabozzi} series},
	title = {Advanced {Stochastic} {Models}, {Risk} {Assessment}, and {Portfolio} {Optimization}: {The} {Ideal} {Risk}, {Uncertainty}, and {Performance} {Measures}},
	isbn = {978-0-470-05316-4},
	publisher = {John Wiley \& Sons},
	author = {Rachev, Svetlozar T. and Stoyanov, Stoyan and Fabozzi, Frank J.},
	year = {2008},
}

@article{sason_f_2016,
	title = {$f$ -{Divergence} {Inequalities}},
	volume = {62},
	issn = {1557-9654},
	doi = {10.1109/TIT.2016.2603151},
	abstract = {This paper develops systematic approaches to obtain f -divergence inequalities, dealing with pairs of probability measures defined on arbitrary alphabets. Functional domination is one such approach, where special emphasis is placed on finding the best possible constant upper bounding a ratio of f -divergences. Another approach used for the derivation of bounds among f -divergences relies on moment inequalities and the logarithmic-convexity property, which results in tight bounds on the relative entropy and Bhattacharyya distance in terms of χ2 divergences. A rich variety of bounds are shown to hold under boundedness assumptions on the relative information. Special attention is devoted to the total variation distance and its relation to the relative information and relative entropy, including “reverse Pinsker inequalities,” as well as on the Eγ divergence, which generalizes the total variation distance. Pinsker's inequality is extended for this type of f -divergence, a result which leads to an inequality linking the relative entropy and relative information spectrum. Integral expressions of the Rényi divergence in terms of the relative information spectrum are derived, leading to bounds on the Rényi divergence in terms of either the variational distance or relative entropy.},
	number = {11},
	journal = {IEEE Transactions on Information Theory},
	author = {Sason, Igal and Verdú, Sergio},
	month = nov,
	year = {2016},
	keywords = {Information theory, Entropy, Conferences, f-divergence, Integral equations, Joining processes, Pinsker’s inequality, Q measurement, Relative entropy, relative information, Rényi divergence, Systematics, total variation distance},
	pages = {5973--6006},
}

@article{bregman_relaxation_1967,
	title = {The relaxation method of finding the common point of convex sets and its application to the solution of problems in convex programming},
	volume = {7},
	issn = {0041-5553},
	doi = {10.1016/0041-5553(67)90040-7},
	abstract = {IN this paper we consider an iterative method of finding the common point of convex sets. This method can be regarded as a generalization of the methods discussed in [1–4]. Apart from problems which can be reduced to finding some point of the intersection of convex sets, the method considered can be applied to the approximate solution of problems in linear and convex programming.},
	number = {3},
	journal = {USSR Computational Mathematics and Mathematical Physics},
	author = {Bregman, L. M.},
	month = jan,
	year = {1967},
	pages = {200--217},
}

@article{burbea_convexity_1982,
	title = {On the convexity of some divergence measures based on entropy functions},
	volume = {28},
	issn = {1557-9654},
	doi = {10.1109/TIT.1982.1056497},
	abstract = {Three measures of divergence between vectors in a convex set of an-dimensional real vector space are defined in terms of certain types of entropy functions, and their convexity property is studied. Among other results, a classification of the entropies of degree{\textbackslash}alphais obtained by the convexity of these measures. These results have applications in information theory and biological studies.},
	number = {3},
	journal = {IEEE Transactions on Information Theory},
	author = {Burbea, J. and Rao, C.},
	month = may,
	year = {1982},
	pages = {489--495},
}

@article{pearson_criterion_1900,
	title = {On the criterion that a given system of deviations from the probable in the case of a correlated system of variables is such that it can be reasonably supposed to have arisen from random sampling},
	volume = {50},
	issn = {1941-5982},
	doi = {10.1080/14786440009463897},
	number = {302},
	urldate = {2022-12-02},
	journal = {The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science},
	author = {Pearson, Karl},
	month = jul,
	year = {1900},
	publisher = {Taylor \& Francis},
	pages = {157--175},
}

@article{yamada_relative_2013,
	title = {Relative density-ratio estimation for robust distribution comparison},
	volume = {25},
	issn = {0899-7667},
	doi = {10.1162/NECO_a_00442},
	abstract = {Divergence estimators based on direct approximation of density ratios without going through separate approximation of numerator and denominator densities have been successfully applied to machine learning tasks that involve distribution comparison such as outlier detection, transfer learning, and two-sample homogeneity test. However, since density-ratio functions often possess high fluctuation, divergence estimation is a challenging task in practice. In this letter, we use relative divergences for distribution comparison, which involves approximation of relative density ratios. Since relative density ratios are always smoother than corresponding ordinary density ratios, our proposed method is favorable in terms of nonparametric convergence speed. Furthermore, we show that the proposed divergence estimator has asymptotic variance independent of the model complexity under a parametric setup, implying that the proposed estimator hardly overfits even with complex models. Through experiments, we demonstrate the usefulness of the proposed approach.},
	number = {5},
	journal = {Neural Computation},
	author = {Yamada, Makoto and Suzuki, Taiji and Kanamori, Takafumi and Hachiya, Hirotaka and Sugiyama, Masashi},
	month = may,
	year = {2013},
	pages = {1324--1370},
}

@article{gyorfi_f-dissimilarity_1975,
	title = {f-dissimilarity: {A} general class of separation measures of several probability measures},
	volume = {16},
	shorttitle = {f-dissimilarity},
	language = {undefined},
	journal = {Topics in Information Theory. Colloq. Math. Soc. János Bolyai},
	author = {Györfi, L. and Nemetz, T.},
	year = {1975},
	pages = {309--321},
}


@article{keziou_test_2005,
	title = {Test of homogeneity in semiparametric two-sample density ratio models},
	volume = {340},
	number = {12},
	journal = {Comptes Rendus Mathématique},
	author = {Keziou, Amor and Leoni-Aubin, Samuela},
	year = {2005},
	publisher = {Elsevier},
	pages = {905--910},
}


@article{zolotarev_metric_1976,
	title = {Metric {Distances} in {Spaces} of {Random} {Variables} and their {Distributions}},
	volume = {30},
	issn = {0025-5734},
	doi = {10.1070/SM1976v030n03ABEH002280},
	language = {en},
	number = {3},
	urldate = {2022-11-24},
	journal = {Mathematics of the USSR-Sbornik},
	author = {Zolotarev, V. M.},
	month = apr,
	year = {1976},
	publisher = {IOP Publishing},
	pages = {373},
}


@article{vajda_metric_2009,
	title = {On metric divergences of probability measures},
	volume = {45},
	number = {6},
	journal = {Kybernetika},
	author = {Vajda, Igor},
	year = {2009},
	publisher = {Institute of Information Theory and Automation AS CR},
	pages = {885--900},
}

@book{cam_asymptotic_1986,
	address = {New York, NY},
	series = {Springer {Series} in {Statistics}},
	title = {Asymptotic {Methods} in {Statistical} {Decision} {Theory}},
	publisher = {Springer},
	author = {{Le Cam}, Lucien},
	year = {1986},
}

@incollection{vincze_concept_1981,
	title = {On the concept and measure of information contained in an observation},
	booktitle = {Contributions to {Probability}},
	publisher = {Elsevier},
	author = {Vincze, István},
	year = {1981},
	pages = {207--214},
}


@article{nguyen_estimating_2010,
	title = {Estimating {Divergence} {Functionals} and the {Likelihood} {Ratio} by {Convex} {Risk} {Minimization}},
	volume = {56},
	issn = {1557-9654},
	doi = {10.1109/TIT.2010.2068870},
	abstract = {We develop and analyze M-estimation methods for divergence functionals and the likelihood ratios of two probability distributions. Our method is based on a nonasymptotic variational characterization of f -divergences, which allows the problem of estimating divergences to be tackled via convex empirical risk optimization. The resulting estimators are simple to implement, requiring only the solution of standard convex programs. We present an analysis of consistency and convergence for these estimators. Given conditions only on the ratios of densities, we show that our estimators can achieve optimal minimax rates for the likelihood ratio and the divergence functionals in certain regimes. We derive an efficient optimization algorithm for computing our estimates, and illustrate their convergence behavior and practical viability by simulations.},
	number = {11},
	journal = {IEEE Transactions on Information Theory},
	author = {Nguyen, XuanLong and Wainwright, Martin J. and Jordan, Michael I.},
	month = nov,
	year = {2010},
	keywords = {Convergence, Estimation, Probability distribution, Measurement, Convex optimization, density ratio estimation, divergence estimation, Kullback-Leibler (KL) divergence, M-estimation, reproducing kernel Hilbert space (RKHS), surrogate loss functions, \$f\$-divergence, Convex functions, Entropy, Kernel},
	pages = {5847--5861},
}

@article{taneja_relative_2004,
	title = {Relative information of type s, {Csiszár}'s f-divergence, and information inequalities},
	volume = {166},
	issn = {0020-0255},
	doi = {10.1016/j.ins.2003.11.002},
	abstract = {During past years Dragomir has contributed a lot of work providing different kinds of bounds on the distance, information and divergence measures. In this paper, we have unified some of his results using the relative information of type s and relating it with the Csiszár's f-divergence.},
	number = {1},
	journal = {Information Sciences},
	author = {Taneja, Inder Jeet and Kumar, Pranesh},
	month = oct,
	year = {2004},
	keywords = {-divergence, Csiszár's -divergence, Hellinger's discrimination, Information inequalities, Relative information, Relative information of type},
	pages = {105--125},
}

@article{van_erven_renyi_2014,
	title = {R\'enyi {Divergence} and {Kullback}-{Leibler} {Divergence}},
	volume = {60},
	issn = {0018-9448, 1557-9654},
	doi = {10.1109/TIT.2014.2320500},
	abstract = {R{\textbackslash}'enyi divergence is related to R{\textbackslash}'enyi entropy much like Kullback-Leibler divergence is related to Shannon's entropy, and comes up in many settings. It was introduced by R{\textbackslash}'enyi as a measure of information that satisfies almost the same axioms as Kullback-Leibler divergence, and depends on a parameter that is called its order. In particular, the R{\textbackslash}'enyi divergence of order 1 equals the Kullback-Leibler divergence. We review and extend the most important properties of R{\textbackslash}'enyi divergence and Kullback-Leibler divergence, including convexity, continuity, limits of \${\textbackslash}sigma\$-algebras and the relation of the special order 0 to the Gaussian dichotomy and contiguity. We also show how to generalize the Pythagorean inequality to orders different from 1, and we extend the known equivalence between channel capacity and minimax redundancy to continuous channel inputs (for all orders) and present several other minimax results.},
	number = {7},
	journal = {IEEE Transactions on Information Theory},
	author = {van Erven, Tim and Harremoës, Peter},
	month = jul,
	year = {2014},
	note = {arXiv:1206.2459 [cs, math, stat]},
	keywords = {Statistics - Machine Learning, Mathematics - Statistics Theory, Computer Science - Information Theory},
	pages = {3797--3820},
}

@article{liese_divergences_2006,
	title = {On {Divergences} and {Informations} in {Statistics} and {Information} {Theory}},
	volume = {52},
	issn = {1557-9654},
	doi = {10.1109/TIT.2006.881731},
	abstract = {The paper deals with the\$f\$-divergences of CsiszÁr generalizing the discrimination information of Kullback, the total variation distance, the Hellinger divergence, and the Pearson divergence. All basic properties of\$f\$-divergences including relations to the decision errors are proved in a new manner replacing the classical Jensen inequality by a new generalized Taylor expansion of convex functions. Some new properties are proved too, e.g., relations to the statistical sufficiency and deficiency. The generalized Taylor expansion also shows very easily that all\$f\$-divergences are average statistical informations (differences between prior and posterior Bayes errors) mutually differing only in the weights imposed on various prior distributions. The statistical information introduced by De Groot and the classical information of Shannon are shown to be extremal cases corresponding to\$alpha =0\$and\$alpha =1\$in the class of the so-called Arimoto\$alpha \$-informations introduced in this paper for\$0≪ alpha ≪ 1\$by means of the Arimoto\$alpha \$-entropies. Some new examples of\$f\$-divergences are introduced as well, namely, the Shannon divergences and the Arimoto\$alpha \$-divergences leading for\$alpha uparrow 1\$to the Shannon divergences. Square roots of all these divergences are shown to be metrics satisfying the triangle inequality. The last section introduces statistical tests and estimators based on the minimal\$f\$-divergence with the empirical distribution achieved in the families of hypothetic distributions. For the Kullback divergence this leads to the classical likelihood ratio test and estimator.},
	number = {10},
	journal = {IEEE Transactions on Information Theory},
	author = {Liese, F. and Vajda, I.},
	month = oct,
	year = {2006},
	keywords = {Probability, Information theory, Statistics, Testing, Random variables, Entropy, Taylor series, sufficiency, Arimoto divergence, Arimoto entropy, Arimoto information, Automation, deficiency, discrimination information, Electronic mail, Mathematics, minimum, Shannon divergence, Shannon information, statistical information},
	pages = {4394--4412},
}

@article{topsoe_inequalities_2000,
	title = {Some inequalities for information divergence and related measures of discrimination},
	volume = {46},
	issn = {1557-9654},
	doi = {10.1109/18.850703},
	abstract = {Inequalities which connect information divergence with other measures of discrimination or distance between probability distributions are used in information theory and its applications to mathematical statistics, ergodic theory, and other scientific fields. We suggest new inequalities of this type, often based on underlying identities. As a consequence, we obtain certain improvements of the well-known Pinsker inequality. Our study depends on two measures of discrimination, called capacitory discrimination and triangular discrimination. The discussion contains references to related research and comparison with other measures of discrimination, e.g., Ali-Silvey-Csiszar (1996, 1966) divergences and, in particular, the Hellinger distance.},
	number = {4},
	journal = {IEEE Transactions on Information Theory},
	author = {Topsoe, F.},
	month = jul,
	year = {2000},
	keywords = {Information theory},
	pages = {1602--1609},
}

@inproceedings{wang_nearest-neighbor_2006,
	title = {A {Nearest}-{Neighbor} {Approach} to {Estimating} {Divergence} between {Continuous} {Random} {Vectors}},
	doi = {10.1109/ISIT.2006.261842},
	abstract = {A method for divergence estimation between multidimensional distributions based on nearest neighbor distances is proposed. Given i.i.d. samples, both the bias and the variance of this estimator are proven to vanish as sample sizes go to infinity. In experiments on high-dimensional data, the nearest neighbor approach generally exhibits faster convergence compared to previous algorithms based on partitioning},
	booktitle = {2006 {IEEE} {International} {Symposium} on {Information} {Theory}},
	author = {Wang, Qing and Kulkarni, Sanjeev R. and Verdu, Sergio},
	month = jul,
	year = {2006},
	issn = {2157-8117},
	keywords = {Convergence, Probability distribution, Random variables, Entropy, Frequency estimation, H infinity control, Multidimensional systems, Nearest neighbor searches, Neural networks, Partitioning algorithms},
	pages = {242--246},
}

@article{wang_divergence_2005,
	title = {Divergence estimation of continuous distributions based on data-dependent partitions},
	volume = {51},
	issn = {1557-9654},
	doi = {10.1109/TIT.2005.853314},
	abstract = {We present a universal estimator of the divergence D(P/spl par/Q) for two arbitrary continuous distributions P and Q satisfying certain regularity conditions. This algorithm, which observes independent and identically distributed (i.i.d.) samples from both P and Q, is based on the estimation of the Radon-Nikodym derivative dP/dQ via a data-dependent partition of the observation space. Strong convergence of this estimator is proved with an empirically equivalent segmentation of the space. This basic estimator is further improved by adaptive partitioning schemes and by bias correction. The application of the algorithms to data with memory is also investigated. In the simulations, we compare our estimators with the direct plug-in estimator and estimators based on other partitioning approaches. Experimental results show that our methods achieve the best convergence performance in most of the tested cases.},
	number = {9},
	journal = {IEEE Transactions on Information Theory},
	author = {Wang, Qing and Kulkarni, S.R. and Verdu, S.},
	month = sep,
	year = {2005},
	keywords = {Bias correction, Information theory, Testing, Convergence, divergence, Random variables, Entropy, Density measurement, Partitioning algorithms, data-dependent partition, Extraterrestrial measurements, Mutual information, Pattern recognition, Radon–Nikodym derivative, stationary and ergodic data, universal estimation of information measures},
	pages = {3064--3074},
}

@book{liese_convex_1987,
	location = {Leipzig},
	series = {Teubner-{Texte} zur {Mathematik}},
	title = {Convex statistical distances},
	volume = {95},
	publisher = {Teubner},
	author = {Liese, Friedrich and Vajda, Igor},
	year = {1987},
}


@article{renyi_measures_1961,
	title = {On {Measures} of {Entropy} and {Information}},
	volume = {4.1},
	abstract = {Berkeley Symposium on Mathematical Statistics and Probability},
	urldate = {2022-12-22},
	journal = {Proceedings of the Fourth Berkeley Symposium on Mathematical Statistics and Probability, Volume 1: Contributions to the Theory of Statistics},
	author = {Rényi, Alfréd},
	month = jan,
	year = {1961},
	publisher = {University of California Press},
	pages = {547--562},
}





@article{szekely_energy_2017,
	title = {The {Energy} of {Data}},
	volume = {4},
	doi = {10.1146/annurev-statistics-060116-054026},
	abstract = {The energy of data is the value of a real function of distances between data in metric spaces. The name energy derives from Newton's gravitational potential energy, which is also a function of distances between physical objects. One of the advantages of working with energy functions (energy statistics) is that even if the data are complex objects, such as functions or graphs, we can use their real-valued distances for inference. Other advantages are illustrated and discussed in this review. Concrete examples include energy testing for normality, energy clustering, and distance correlation. Applications include genome studies, brain studies, and astrophysics. The direct connection between energy and mind/observations/data in this review is a counterpart of the equivalence of energy and matter/mass in Einstein's E=mc2.},
	number = {1},
	urldate = {2023-01-09},
	journal = {Annual Review of Statistics and Its Application},
	author = {Székely, Gábor J. and Rizzo, Maria L.},
	year = {2017},
	keywords = {clustering, data distance, distance correlation, distance covariance, energy distance, goodness-of-fit, graph distance, multivariate independence, statistical energy, statistical tests, U-statistic, V-statistic},
	pages = {447--479},
}


@inproceedings{damodaran_deepjdot_2018,
	address = {Berlin, Heidelberg},
	title = {{DeepJDOT}: {Deep} {Joint} {Distribution} {Optimal} {Transport} for {Unsupervised} {Domain} {Adaptation}},
	isbn = {978-3-030-01224-3},
	shorttitle = {{DeepJDOT}},
	doi = {10.1007/978-3-030-01225-0_28},
	abstract = {In computer vision, one is often confronted with problems of domain shifts, which occur when one applies a classifier trained on a source dataset to target data sharing similar characteristics (e.g. same classes), but also different latent data structures (e.g. different acquisition conditions). In such a situation, the model will perform poorly on the new data, since the classifier is specialized to recognize visual cues specific to the source domain. In this work we explore a solution, named DeepJDOT, to tackle this problem: through a measure of discrepancy on joint deep representations/labels based on optimal transport, we not only learn new data representations aligned between the source and target domain, but also simultaneously preserve the discriminative information used by the classifier. We applied DeepJDOT to a series of visual recognition tasks, where it compares favorably against state-of-the-art deep domain adaptation methods.},
	urldate = {2023-01-10},
	booktitle = {Computer {Vision} – {ECCV} 2018: 15th {European} {Conference}, {Munich}, {Germany}, {September} 8-14, 2018, {Proceedings}, {Part} {IV}},
	publisher = {Springer-Verlag},
	author = {Damodaran, Bharath Bhushan and Kellenberger, Benjamin and Flamary, Rémi and Tuia, Devis and Courty, Nicolas},
	month = sep,
	year = {2018},
	keywords = {Deep domain adaptation, Optimal transport},
	pages = {467--483},
}


@inproceedings{courty_joint_2017,
	title = {Joint distribution optimal transportation for domain adaptation},
	volume = {30},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Courty, Nicolas and Flamary, Rémi and Habrard, Amaury and Rakotomamonjy, Alain},
	year = {2017},
}

@incollection{memoli_distances_2017,
	address = {Cham},
	series = {Lecture {Notes} in {Mathematics}},
	title = {Distances {Between} {Datasets}},
	isbn = {978-3-319-58002-9},
	abstract = {We overview the construction and quantitative aspects of the Gromov–Hausdorff and Gromov–Wasserstein distances.},
	booktitle = {Modern {Approaches} to {Discrete} {Curvature}},
	publisher = {Springer International Publishing},
	author = {Mémoli, Facundo},
	editor = {Najman, Laurent and Romon, Pascal},
	year = {2017},
	doi = {10.1007/978-3-319-58002-9_3},
	keywords = {Gromov Hausdorff Distance, Hardening Curve, Optimal Relative Position, Quadratic Bottleneck Assignment Problem, Space-fixed Target},
	pages = {115--132},
}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@Article{GSAR,
	title = {GSAR: Bioconductor package for gene set analysis in R},
	author = {Yasir Rahmatallah and Boris Zybailov and Frank Emmert-Streib and Galina Glazko},
	journal = {BMC Bioinformatics},
	volume = {18},
	pages = {61},
	year = {2017},
}


@Manual{cramer,
	title = {cramer: Multivariate Nonparametric Cramer-Test for the
	Two-Sample-Problem},
	author = {Carsten Franz},
	year = {2024},
	note = {R package version 0.9-4},
	url = {https://CRAN.R-project.org/package=cramer},
}

@Manual{crossmatch,
	title = {crossmatch: The Cross-Match Test},
	author = {Ruth Heller and Dylan Small and Paul Rosenbaum},
	year = {2024},
	note = {R package version 1.4-0},
	url = {https://CRAN.R-project.org/package=crossmatch},
}

@Manual{kerTests,
	title = {kerTests: Generalized Kernel Two-Sample Tests},
	author = {Hoseung Song and Hao Chen},
	year = {2023},
	note = {R package version 0.1.4},
	url = {https://CRAN.R-project.org/package=kerTests},
}

@Manual{testOTM,
	title = {testOTM: Multivariate Ranks and Quantiles by Optimal Transportation},
	author = {Peng Xu},
	year = {2019},
	note = {R package version 0.11.2},
	url = {https://github.com/Francis-Hsu/testOTM},
}

@article{puri_nonparametric_1971,
	title={Nonparametric methods in multivariate analysis},
	author={Puri, Madan Lal and Sen, Pranab Kumar and others},
	year={1971},
	publisher={Wiley}
}

@Manual{hypoRF,
	title = {hypoRF: Random Forest Two-Sample Tests},
	author = {Simon Hediger and Loris Michel and Jeffrey Naef},
	year = {2024},
	note = {R package version 1.0.1},
	url = {https://CRAN.R-project.org/package=hypoRF},
}

@Manual{multicross,
	title = {multicross: A Graph-Based Test for Comparing Multivariate Distributions in
	the Multi Sample Framework},
	author = {Somabha Mukherjee Divyansh Agarwal and Bhaswar Bhattacharya and Nancy R. Zhang},
	year = {2020},
	note = {R package version 2.1.0},
	url = {https://CRAN.R-project.org/package=multicross},
}

@Manual{LPKsample,
	title = {LPKsample: LP Nonparametric High Dimensional K-Sample Comparison},
	author = {Subhadeep Mukhopadhyay and Kaijun Wang},
	year = {2020},
	note = {R package version 2.1},
	url = {https://CRAN.R-project.org/package=LPKsample},
}

@book{rogers_convergence_1978,
	title = {{Some} {Convergence} {Properties} of {K}-{Nearest} {Neighbor} {Estimates}.},
	publisher = {Stanford University},
	author = {Rogers, WILLIAM HAYWARD},
	year = {1978},
	file = {Snapshot:C\:\\Users\\stolte\\Zotero\\storage\\MRHZ4RKI\\intermediateredirectforezproxy.html:text/html},
} 

@article{friedman_nonparametric_1973,
	title={A nonparametric procedure for comparing multivariate point sets},
	author={Friedman, Jerome H and Steppel, Sam},
	journal={Stanford Linear Accelerator Center Computation Research Group Technical Memo},
	number={153},
	year={1973}
}
@Manual{kernlabR,
	title = {kernlab: Kernel-Based Machine Learning Lab},
	author = {Alexandros Karatzoglou and Alex Smola and Kurt Hornik},
	year = {2024},
	note = {R package version 0.9-33},
	url = {https://CRAN.R-project.org/package=kernlab},
}


@Article{kernlab,
	title = {kernlab -- An {S4} Package for Kernel Methods in {R}},
	author = {Alexandros Karatzoglou and Alex Smola and Kurt Hornik and Achim Zeileis},
	journal = {Journal of Statistical Software},
	year = {2004},
	volume = {11},
	number = {9},
	pages = {1--20},
	doi = {10.18637/jss.v011.i09},
}


@inproceedings{liu_classifier_2018,
	title = {Classifier Two Sample Test for Video Anomaly Detections},
	author = {Yusha Liu and Chun-Liang Li and Barnabás Póczos},
	year = {2018},
	researchr = {https://researchr.org/publication/LiuLP18},
	cites = {0},
	citedby = {0},
	pages = {71},
	booktitle = {British Machine Vision Conference 2018, BMVC 2018, Northumbria University, Newcastle, UK, September 3-6, 2018},
	publisher = {BMVA Press},
}


@article{csiszar_1964_informationstheoretische,
	title={Eine informationstheoretische ungleichung und ihre anwendung auf beweis der ergodizitaet von markoffschen ketten},
	author={Csisz{\'a}r, Imre},
	journal={Magyer Tud. Akad. Mat. Kutato Int. Koezl.},
	volume={8},
	pages={85--108},
	year={1964}
}

@inproceedings{sutherland_2017_generative,
	title={Generative Models and Model Criticism via Optimized Maximum Mean Discrepancy},
	author={Danica J. Sutherland and Hsiao-Yu Tung and Heiko Strathmann and Soumyajit De and Aaditya Ramdas and Alex Smola and Arthur Gretton},
	booktitle={International Conference on Learning Representations},
	year={2017},
	url={https://openreview.net/forum?id=HJWHIKqgl}
}

@ARTICLE{cherian_jensen_2013,
	author={Cherian, Anoop and Sra, Suvrit and Banerjee, Arindam and Papanikolopoulos, Nikolaos},
	journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, 
	title={Jensen-Bregman LogDet Divergence with Application to Efficient Similarity Search for Covariance Matrices}, 
	year={2013},
	volume={35},
	number={9},
	pages={2161-2174},
	doi={10.1109/TPAMI.2012.259}
}


@inproceedings{sriperumbudur_kernel_2009,
	title = {Kernel Choice and Classifiability for RKHS Embeddings of Probability Distributions},
	author = {Sriperumbudur, BK. and Fukumizu, K. and Gretton, A. and Lanckriet, GRG. and Sch{\"o}lkopf, B.},
	journal = {Advances in Neural Information Processing Systems 22: 23rd Annual Conference on Neural Information Processing Systems 2009},
	booktitle = {Advances in Neural Information Processing Systems 22},
	pages = {1750-1758},
	editors = {Y Bengio and D Schuurmans and J Lafferty and C Williams and A Culotta},
	publisher = {Curran},
	organization = {Max-Planck-Gesellschaft},
	school = {Biologische Kybernetik},
	address = {Red Hook, NY, USA},
	year = {2009},
	doi = {}
}

@misc{sutherland_unbiased_2019,
	doi = {10.48550/ARXIV.1906.02104},
	author = {Sutherland, Danica J.},
	keywords = {Machine Learning (stat.ML), Machine Learning (cs.LG), FOS: Computer and information sciences, FOS: Computer and information sciences},
	title = {Unbiased estimators for the variance of MMD estimators},
	publisher = {arXiv},
	year = {2019},
}


@techreport{friedman_multivariate_2004,
	title={On Multivariate Goodness-of-Fit and Two-Sample Testing},
	author={Friedman, J},
	year={2004},
	institution={SLAC National Accelerator Lab., Menlo Park, CA (United States)}
}


@book{rao1973linear,
	title={Linear Statistical Inference and is Applications},
	author={Rao, Calyampudi Radhakrishna},
	year={1973},
	edition = {2},
	publisher={John Wiley \& Sons, Incorporated}
}

@inproceedings{johnson_comparing_1998,
	title={Comparing Massive High-Dimensional Data Sets.},
	author={Johnson, Theodore and Dasu, Tamraparni},
	booktitle={KDD},
	pages={229--233},
	year={1998}
}

@article{tatti_distances_2007,
	title={Distances between Data Sets Based on Summary Statistics.},
	author={Tatti, Nikolaj},
	journal={Journal of Machine Learning Research},
	volume={8},
	number={1},
	year={2007}
}

@article{szekely_testing_2004,
	title={Testing for equal distributions in high dimension},
	author={Sz{\'e}kely, G{\'a}bor J. and Rizzo, Maria L.},
	journal={InterStat},
	volume={5},
	number={16.10},
	pages={1249--1272},
	year={2004},
	publisher={Citeseer}
}

@Manual{energy,
	title = {energy: E-Statistics: Multivariate Inference via the Energy of Data},
	author = {Maria Rizzo and Gabor Szekely},
	year = {2024},
	note = {R package version 1.7-12},
	url = {https://CRAN.R-project.org/package=energy},
}


@Manual{R_4_1_2,
	title = {R: A Language and Environment for Statistical Computing},
	author = {{R Core Team}},
	organization = {R Foundation for Statistical Computing},
	address = {Vienna, Austria},
	year = {2024},
	url = {https://www.R-project.org/},
}

@article{roederer_probability_2001,
	title={Probability Binning Comparison: A Metric for Quantitating Multivariate Distribution Differences},
	author={Roederer, Mario and Moore, Wayne and Treister, Adam and Hardy, Richard R and Herzenberg, Leonore A},
	journal={Cytometry},
	volume={45},
	number={1},
	pages={47--55},
	year={2001},
	publisher={Wiley Online Library}
}


@inproceedings{ganti_framework_1999,
	title={A Framework for Measuring Changes in Data Characteristics},
	author={Ganti, Venkatesh and Gehrke, Johannes and Ramakrishnan, Raghu and Loh, Wei-Yin},
	booktitle={Proceedings of the 18th Symposium on Principles of Database Systems},
	pages={126--137},
	year={1999}
}

@inproceedings{wang_random_2005,
	title={A random method for quantifying changing distributions in data streams},
	author={Wang, Haixun and Pei, Jian},
	booktitle={European Conference on Principles of Data Mining and Knowledge Discovery},
	pages={684--691},
	year={2005},
	organization={Springer}
}

@article{pekerskaya2006mining,
	title={Mining changing regions from access-constrained snapshots: a cluster-embedded decision tree approach},
	author={Pekerskaya, Irene and Pei, Jian and Wang, Ke},
	journal={Journal of Intelligent Information Systems},
	volume={27},
	number={3},
	pages={215--242},
	year={2006},
	publisher={Springer}
}


@article{zhao_fastmmd_2015,
	title = {{FastMMD}: {Ensemble} of {Circular} {Discrepancy} for {Efficient} {Two}-{Sample} {Test}},
	volume = {27},
	issn = {0899-7667, 1530-888X},
	shorttitle = {{FastMMD}},
	doi = {10.1162/NECO_a_00732},
	abstract = {The maximum mean discrepancy (MMD) is a recently proposed test statistic for two-sample test. Its quadratic time complexity, however, greatly hampers its availability to large-scale applications. To accelerate the MMD calculation, in this study we propose an efficient method called FastMMD. The core idea of FastMMD is to equivalently transform the MMD with shift-invariant kernels into the amplitude expectation of a linear combination of sinusoid components based on Bochner's theorem and Fourier transform (Rahimi \& Recht, 2007). Taking advantage of sampling of Fourier transform, FastMMD decreases the time complexity for MMD calculation from \$O(N{\textasciicircum}2 d)\$ to \$O(L N d)\$, where \$N\$ and \$d\$ are the size and dimension of the sample set, respectively. Here \$L\$ is the number of basis functions for approximating kernels which determines the approximation accuracy. For kernels that are spherically invariant, the computation can be further accelerated to \$O(L N {\textbackslash}log d)\$ by using the Fastfood technique (Le et al., 2013). The uniform convergence of our method has also been theoretically proved in both unbiased and biased estimates. We have further provided a geometric explanation for our method, namely ensemble of circular discrepancy, which facilitates us to understand the insight of MMD, and is hopeful to help arouse more extensive metrics for assessing two-sample test. Experimental results substantiate that FastMMD is with similar accuracy as exact MMD, while with faster computation speed and lower variance than the existing MMD approximation methods.},
	number = {6},
	urldate = {2023-06-15},
	journal = {Neural Computation},
	author = {Zhao, Ji and Meng, Deyu},
	month = jun,
	year = {2015},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Machine Learning, Statistics - Machine Learning},
	pages = {1345--1372},
}


@Article{Ball,
	title = {{Ball}: An {R} Package for Detecting Distribution
	Difference and Association in Metric Spaces},
	author = {Jin Zhu and Wenliang Pan and Wei Zheng and Xueqin Wang},
	journal = {Journal of Statistical Software},
	year = {2021},
	volume = {97},
	number = {6},
	pages = {1--31},
	doi = {10.18637/jss.v097.i06},
}

@Manual{gTests,
	title = {gTests: Graph-Based Two-Sample Tests},
	author = {Hao Chen and Jingru Zhang},
	year = {2017},
	note = {R package version 0.2},
	url = {https://CRAN.R-project.org/package=gTests},
}

@Manual{HDLSSkST,
	title = {HDLSSkST: {Distribution}-{Free} {Exact} {High} {Dimensional} {Low} {Sample} {Size}
	$k$-{Sample} {Tests}},
	author = {Biplab Paul and Shyamal K. De and Anil K. Ghosh},
	year = {2022},
	note = {R package version 2.1.0},
	url = {https://CRAN.R-project.org/package=HDLSSkST},
}

@Manual{Ecume,
	title = {Ecume: Equality of 2 (or k) Continuous Univariate and Multivariate
	Distributions},
	author = {Hector {Roux de Bezieux}},
	year = {2024},
	note = {R package version 0.9.2},
	url = {https://CRAN.R-project.org/package=Ecume},
}

@Article{philentropy,
	title = {Philentropy: Information Theory and Distance Quantification with R},
	author = {Drost HG},
	journal = {Journal of Open Source Software},
	year = {2018},
	volume = {3},
	number = {26},
	pages = {765},
	url = {https://joss.theoj.org/papers/10.21105/joss.00765},
}


@Article{distrEx,
	title = {S4 Classes for Distributions},
	author = {P. Ruckdeschel and M. Kohl and T. Stabla and F.
	Camphausen},
	language = {English},
	year = {2006},
	journal = {R News},
	year = {2006},
	volume = {6},
	number = {2},
	pages = {2--6},
	month = {May},
	pdf = {https://CRAN.R-project.org/doc/Rnews/Rnews_2006-2.pdf},
}


@inproceedings{munoz_new_2012,
	address = {Berlin, Heidelberg},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {A {New} {Distance} for {Probability} {Measures} {Based} on the {Estimation} of {Level} {Sets}},
	isbn = {978-3-642-33266-1},
	doi = {10.1007/978-3-642-33266-1_34},
	abstract = {In this paper we propose to consider Probability Measures (PM) as generalized functions belonging to some functional space endowed with an inner product. This approach allows to introduce a new family of distances for PMs. We propose a particular (non parametric) metric for PMs belonging to this class, based on the estimation of density level sets. Some real and simulated data sets are used for a first exploration of its performance.},
	booktitle = {Artificial {Neural} {Networks} and {Machine} {Learning} – {ICANN} 2012},
	publisher = {Springer},
	author = {Muñoz, Alberto and Martos, Gabriel and Arriero, Javier and Gonzalez, Javier},
	editor = {Villa, Alessandro E. P. and Duch, Włodzisław and Érdi, Péter and Masulli, Francesco and Palm, Günther},
	year = {2012},
	keywords = {Functional Data Analysis, Hotelling Test, Probability Measure, Schwartz Distribution, Sparsity Measure},
	pages = {271--278},
}

@inproceedings{munoz_new_2013,
	address = {Berlin, Heidelberg},
	series = {Lecture {Notes} in {Computer} {Science}},
	title = {A {New} {Distance} for {Data} {Sets} in a {Reproducing} {Kernel} {Hilbert} {Space} {Context}},
	isbn = {978-3-642-41822-8},
	doi = {10.1007/978-3-642-41822-8_28},
	abstract = {In this paper we define distance functions for data sets in a reproduncing kernel Hilbert space (RKHS) context. To this aim we introduce kernels for data sets that provide a metrization of the power set. The proposed distances take into account the underlying generating probability distributions. In particular, we propose kernel distances that rely on the estimation of density level sets of the underlying data distributions, and that can be extended from data sets to probability measures. The performance of the proposed distances is tested on several simulated and real data sets.},
	booktitle = {Progress in {Pattern} {Recognition}, {Image} {Analysis}, {Computer} {Vision}, and {Applications}},
	publisher = {Springer},
	author = {Muñoz, Alberto and Martos, Gabriel and González, Javier},
	editor = {Ruiz-Shulcloper, José and Sanniti di Baja, Gabriella},
	year = {2013},
	keywords = {Bregman Divergence, Energy Distance, Kernel Distance, Size 100d, Underlie Data Distribution},
	pages = {222--229},
}

@article{munoz_estimation_2006,
	title = {Estimation of high-density regions using one-class neighbor machines},
	volume = {28},
	issn = {1939-3539},
	doi = {10.1109/TPAMI.2006.52},
	abstract = {In this paper, we investigate the problem of estimating high-density regions from univariate or multivariate data samples. We estimate minimum volume sets, whose probability is specified in advance, known in the literature as density contour clusters. This problem is strongly related to one-class support vector machines (OCSVM). We propose a new method to solve this problem, the one-class neighbor machine (OCNM) and we show its properties. In particular, the OCNM solution asymptotically converges to the exact minimum volume set prespecified. Finally, numerical results illustrating the advantage of the new method are shown.},
	number = {3},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	author = {Munoz, A. and Moguerza, J.M.},
	month = mar,
	year = {2006},
	keywords = {Clustering algorithms, Computational complexity, Concrete, Data analysis, Density functional theory, Density measurement, Index Terms- Density estimation, Kernel, kernel methods, Level set, One-Class Support Vector Machines., Support vector machines, Tin},
	pages = {476--480},
}

@online{CRAN,
	author = {{R Foundation for Statistical Computing}},
	title = {The {Comprehensive} {R} {Archive} {Network}},
	url = {https://cran.r-project.org/},
	urldate = {2025-02-10},
	year = {2025}
	
}

@online{bioconductor,
	author = {},
	title = {Bioconductor},
	url = {http://new.bioconductor.org/},
	urldate = {2023-11-02}
}


@article{lin_divergence_1991,
	title = {Divergence measures based on the {Shannon} entropy},
	volume = {37},
	issn = {1557-9654},
	doi = {10.1109/18.61115},
	abstract = {A novel class of information-theoretic divergence measures based on the Shannon entropy is introduced. Unlike the well-known Kullback divergences, the new measures do not require the condition of absolute continuity to be satisfied by the probability distributions involved. More importantly, their close relationship with the variational distance and the probability of misclassification error are established in terms of bounds. These bounds are crucial in many applications of divergence measures. The measures are also well characterized by the properties of nonnegativity, finiteness, semiboundedness, and boundedness.{\textless}{\textgreater}},
	number = {1},
	journal = {IEEE Transactions on Information Theory},
	author = {Lin, J.},
	month = jan,
	year = {1991},
	pages = {145--151},
}

@misc{stolte_review_2023,
	title = {A {Review} and {Taxonomy} of {Methods} for {Quantifying} {Dataset} {Similarity}},
	doi = {10.48550/arXiv.2312.04078},
	abstract = {In statistics and machine learning, measuring the similarity between two or more datasets is important for several purposes. The performance of a predictive model on novel datasets, referred to as generalizability, critically depends on how similar the dataset used for fitting the model is to the novel datasets. Exploiting or transferring insights between similar datasets is a key aspect of meta-learning and transfer-learning. In two-sample testing, it is checked, whether the underlying (multivariate) distributions of two datasets coincide or not. Extremely many approaches for quantifying dataset similarity have been proposed in the literature. A structured overview is a crucial first step for comparisons of approaches. We examine more than 100 methods and provide a taxonomy, classifying them into ten classes, including (i) comparisons of cumulative distribution functions, density functions, or characteristic functions, (ii) methods based on multivariate ranks, (iii) discrepancy measures for distributions, (iv) graph-based methods, (v) methods based on inter-point distances, (vi) kernel-based methods, (vii) methods based on binary classification, (viii) distance and similarity measures for datasets, (ix) comparisons based on summary statistics, and (x) different testing approaches. Here, we present an extensive review of these methods. We introduce the main underlying ideas, formal definitions, and important properties.},
	publisher = {arXiv},
	author = {Stolte, Marieke and Bommert, Andrea and Rahnenführer, Jörg},
	month = dec,
	year = {2023},
	note = {arXiv:2312.04078 [stat]},
	keywords = {62E99, 62G10, 62H15, 62H30, 05C90, Statistics - Methodology},
	
}

@article{kantorovich_mathematical_1960,
	title = {Mathematical {Methods} of {Organizing} and {Planning} {Production}},
	volume = {6},
	issn = {0025-1909},
	doi = {10.1287/mnsc.6.4.366},
	abstract = {The author of the work “Mathematical Methods of Organizing and Planning Production”, Professor L. V. Kantorovich, is an eminent authority in the field of mathematics. This work is interesting from a purely mathematical point of view since it presents an original method, going beyond the limits of classical mathematical analysis, for solving extremal problems. On the other hand, this work also provides an application of mathematical methods to questions of organizing production which merits the serious attention of workers in different branches of industry.
	
	This is the English translation of the famous 1939 article by L. V. Kantorovich, originally published in Russian.},
	number = {4},
	urldate = {2023-10-13},
	journal = {Management Science},
	author = {Kantorovich, L. V.},
	month = jul,
	year = {1960},
	publisher = {INFORMS},
	pages = {366--422},
}

@article{prokhorov_convergence_1956,
	title = {Convergence of {Random} {Processes} and {Limit} {Theorems} in {Probability} {Theory}},
	volume = {1},
	issn = {0040-585X, 1095-7219},
	doi = {10.1137/1101016},
	number = {2},
	journal = {Theory of Probability \& Its Applications},
	author = {Prokhorov, Yu. V.},
	month = jan,
	year = {1956},
	pages = {157--214},
}


@article{birnbaum_uber_1931,
	title = {Über die {Verallgemeinerung} des {Begriffes} der zueinander konjugierten {Potenzen}},
	volume = {3},
	issn = {0039-3223},
	number = {1},
	journal = {Studia Mathematica},
	author = {Birnbaum, Z. and Orlicz, W.},
	year = {1931},
	pages = {1--67},
}

@article{fan_entfernung_1943,
	title = {Entfernung zweier zufälligen {Größen} und die {Konvergenz} nach {Wahrscheinlichkeit}},
	volume = {49},
	issn = {0025-5874, 1432-1823},
	doi = {10.1007/BF01174225},
	number = {1},
	journal = {Mathematische Zeitschrift},
	author = {Fan, Ky},
	month = dec,
	year = {1943},
	pages = {681--683},
}

@phdthesis{bahr_ein_1996,
	author = {Bahr, R.},
	title = {Ein neuer Test für das mehrdimensionale Zwei-Stichproben-Problem bei allgemeiner Alternative},
	school = {Universität Hannover},
	year = {1996},
}

@Manual{KMD,
	title = {KMD: Kernel Measure of Multi-Sample Dissimilarity},
	author = {Zhen Huang},
	year = {2022},
	note = {R package version 0.1.0},
	url = {https://CRAN.R-project.org/package=KMD},
}

@article{huang_kernel_2022,
	author = {Zhen Huang and Bodhisattva Sen},
	title = {A Kernel Measure of Dissimilarity between M Distributions},
	journal = {Journal of the American Statistical Association},
	volume = {119},
	number = {548},
	pages = {3020--3032},
	year = {2024},
	doi = {10.1080/01621459.2023.2298036},
}

@inproceedings{garcia-garcia_divergences_2012,
	title = {Divergences and {Risks} for {Multiclass} {Experiments}},
	abstract = {Csiszár’s fff-divergence is a way to measure the similarity of two probability distributions. We study the extension of fff-divergence to more than two distributions to measure their joint similarity. By exploiting classical results from the comparison of experiments literature we prove the resulting divergence satisfies all the same properties as the traditional binary one. Considering the multidistribution case actually makes the proofs simpler. The key to these results is a formal bridge between these multidistribution fff-divergences and Bayes risks for multiclass classification problems.},
	language = {en},
	urldate = {2024-01-11},
	booktitle = {Proceedings of the 25th {Annual} {Conference} on {Learning} {Theory}},
	publisher = {JMLR Workshop and Conference Proceedings},
	author = {García-García, Dario and Williamson, Robert C.},
	month = jun,
	year = {2012},
	note = {ISSN: 1938-7228},
	pages = {28.1--28.20},
}

@article{zhang_graph-based_2019,
	title = {Graph-{Based} {Two}-{Sample} {Tests} for {Data} with {Repeated} {Observations}},
	volume = {32},
	issn = {1017-0405},
	url = {https://www.jstor.org/stable/27108529},
	abstract = {For two-sample comparisons, tests based on graphs constructed using the similarity information between observations are gaining attention, owing to their flexibility and good performance for high-dimensional/non-Euclidean data. However, when there are repeated observations, these graph-based tests can be problematic, because they are influenced by the choice of the similarity graph. We propose extended graph-based test statistics to resolve this problem. We also study the asymptotic properties of these extended statistics, and provide analytic formulae to approximate the p-values of the tests under finite samples, facilitating the application of the new tests in practice. The proposed tests are applied to analyze a phone-call network data set. All tests are implemented in the R package gTests.},
	number = {1},
	urldate = {2025-02-24},
	journal = {Statistica Sinica},
	author = {Zhang, Jingru and Chen, Hao},
	year = {2022},
	note = {Publisher: Institute of Statistical Science, Academia Sinica},
	pages = {391--415},
	file = {JSTOR Full Text PDF:C\:\\Users\\stolte\\Zotero\\storage\\WS99AT6F\\Zhang und Chen - 2022 - Graph-Based Two-Sample Tests for Data with Repeate.pdf:application/pdf},
}


@Manual{gCat,
	title = {gCat: Graph-Based Two-Sample Tests for Categorical Data},
	author = {Hao Chen and Nancy R. Zhang},
	year = {2022},
	note = {R package version 0.2},
	url = {https://CRAN.R-project.org/package=gCat},
}

@Manual{gTestsMulti,
	title = {gTestsMulti: New Graph-Based Multi-Sample Tests},
	author = {Hoseung Song and Hao Chen},
	year = {2023},
	note = {R package version 0.1.1},
	url = {https://CRAN.R-project.org/package=gTestsMulti},
} 

@Manual{rpart,
	title = {rpart: Recursive Partitioning and Regression Trees},
	author = {Terry Therneau and Beth Atkinson},
	year = {2025},
	note = {R package version 4.1.24},
	url = {https://CRAN.R-project.org/package=rpart},
}

@Manual{e1071,
	title = {e1071: Misc Functions of the Department of Statistics, Probability
	Theory Group (Formerly: E1071), TU Wien},
	author = {David Meyer and Evgenia Dimitriadou and Kurt Hornik and Andreas Weingessel and Friedrich Leisch},
	year = {2024},
	note = {R package version 1.7-16},
	url = {https://CRAN.R-project.org/package=e1071},
}

@article{bischl_hyperparameter_2021,
	title = {Hyperparameter {Optimization}: {Foundations}, {Algorithms}, {Best} {Practices} and {Open} {Challenges}},
	shorttitle = {Hyperparameter {Optimization}},
	abstract = {Most machine learning algorithms are configured by one or several hyperparameters that must be carefully chosen and often considerably impact performance. To avoid a time consuming and unreproducible manual trial-and-error process to find well-performing hyperparameter configurations, various automatic hyperparameter optimization (HPO) methods, e.g., based on resampling error estimation for supervised machine learning, can be employed. After introducing HPO from a general perspective, this paper reviews important HPO methods such as grid or random search, evolutionary algorithms, Bayesian optimization, Hyperband and racing. It gives practical recommendations regarding important choices to be made when conducting HPO, including the HPO algorithms themselves, performance evaluation, how to combine HPO with ML pipelines, runtime improvements, and parallelization. This work is accompanied by an appendix that contains information on specific software packages in R and Python, as well as information and recommended hyperparameter search spaces for specific learning algorithms. We also provide notebooks that demonstrate concepts from this work as supplementary files.},
	journal = {arXiv:2107.05847 [cs, stat]},
	author = {Bischl, Bernd and Binder, Martin and Lang, Michel and Pielok, Tobias and Richter, Jakob and Coors, Stefan and Thomas, Janek and Ullmann, Theresa and Becker, Marc and Boulesteix, Anne-Laure and Deng, Difan and Lindauer, Marius},
	year = {2021},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
	file = {arXiv Fulltext PDF:C\:\\Users\\stolte\\Zotero\\storage\\DNK65A2L\\Bischl et al. - 2021 - Hyperparameter Optimization Foundations, Algorith.pdf:application/pdf;arXiv.org Snapshot:C\:\\Users\\stolte\\Zotero\\storage\\TVIQ9LHK\\2107.html:text/html},
}

@article{southworth_properties_2009,
	title = {Properties of {Balanced} {Permutations}},
	volume = {16},
	issn = {1557-8666},
	doi = {10.1089/cmb.2008.0144},
	abstract = {This paper takes a close look at balanced permutations, a recently developed sample reuse method with applications in bioinformatics. It turns out that balanced permutation reference distributions do not have the correct null behavior, which can be traced to their lack of a group structure. We find that they can give p-values that are too permissive to varying degrees. In particular the observed test statistic can be larger than that of all B balanced permutations of a data set with a probability much higher than 1/(B + 1), even under the null hypothesis.},
	number = {4},
	journal = {Journal of Computational Biology: A Journal of Computational Molecular Cell Biology},
	author = {Southworth, Lucinda K. and Kim, Stuart K. and Owen, Art B.},
	year = {2009},
	keywords = {Computer Simulation, Genes, Genomics, Models, Statistical, Numerical Analysis, Computer-Assisted},
	pages = {625--638},
	file = {Volltext:C\:\\Users\\stolte\\Zotero\\storage\\2YJQQSXG\\Southworth et al. - 2009 - Properties of balanced permutations.pdf:application/pdf},
}

@Manual{randtoolbox,
	title = {randtoolbox: Generating and Testing Random Numbers},
	author = {Dutang Christophe and Savicky Petr},
	year = {2024},
	note = {R package version 2.0.5},
}

@Manual{clue,
	title = {clue: Cluster Ensembles},
	author = {Kurt Hornik},
	year = {2024},
	note = {R package version 0.3-66},
	url = {https://CRAN.R-project.org/package=clue},
}

@Article{cluePaper,
	title = {A {CLUE} for {CLUster Ensembles}},
	author = {Kurt Hornik},
	year = {2005},
	journal = {Journal of Statistical Software},
	volume = {14},
	number = {12},
	month = sep,
	doi = {10.18637/jss.v014.i12},
}

@Manual{pracma,
	title = {pracma: Practical Numerical Math Functions},
	author = {Hans W. Borchers},
	year = {2023},
	note = {R package version 2.4.4},
	url = {https://CRAN.R-project.org/package=pracma},
}

@Manual{approxOT,
	title = {{approxOT}: approximate optimal transport},
	author = {Eric A. Dunipace},
	year = {2024},
	note = {R package version 1.1},
	url = {https://github.com/ericdunipace/approxOT},
}
@Manual{expm,
	title = {expm: Matrix Exponential, Log, 'etc'},
	author = {Martin Maechler and Christophe Dutang and Vincent Goulet},
	year = {2024},
	note = {R package version 1.0-0},
	url = {https://CRAN.R-project.org/package=expm},
}

@Article{caret,
	title = {Building Predictive Models in R Using the caret Package},
	volume = {28},
	url = {https://www.jstatsoft.org/index.php/jss/article/view/v028i05},
	doi = {10.18637/jss.v028.i05},
	number = {5},
	journal = {Journal of Statistical Software},
	author = {{Kuhn} and {Max}},
	year = {2008},
	pages = {1--26},
}

@article{sutherland_three-dimensional_2004,
	title = {Three-{Dimensional} {Quantitative} {Structure}-{Activity} and {Structure}-{Selectivity} {Relationships} of {Dihydrofolate} {Reductase} {Inhibitors}},
	volume = {18},
	issn = {0920-654X},
	doi = {10.1023/b:jcam.0000047814.85293.da},
	abstract = {Three-dimensional quantitative structure-activity relationship (3D-QSAR) modelling using comparative molecular similarity indices analysis (CoMSIA) was applied to a series of 406 structurally diverse dihydrofolate reductase (DHFR) inhibitors from Pneumocystis carinii (pc) and rat liver (rl). X-ray crystal structures of three inhibitors bound to pcDHFR were used for defining the alignment rule. For pcDHFR, a QSAR model containing 6 components was selected using leave-10\%-out cross-validation (n= 240, q2 = 0.65), while a 4-component model was selected for rlDHFR (n= 237, q2 = 0.63); both include steric, electrostatic and hydrophobic contributions. The models were validated using a large test set, designed to maximise its diversity and to verify the predictive accuracy of models for extrapolation. The pcDHFR model has r2 = 0.60 and mean absolute error (MAE) = 0.57 for the test set after removing 4 outliers, and the rlDHFR model has r2 = 0.60 and MAE = 0.69 after removing 4 test set outliers. In addition, classification models predicting selectivity for pcDHFR over rlDHFR were developed using soft independent modelling by class analogy (SIMCA), with a selectivity ratio of 2 (IC50,rlDHFR/ IC50,pcDHFR) used for delimiting classes. A 5-component model including steric and electrostatic contributions has cross-validated and test set classification rates of 0.67 and 0.68 for selective inhibitors, and 0.85 and 0.72 for unselective inhibitors. The predictive accuracy of models, together with the identification of important contributions in QSAR and classification models, offer the possibility of designing potent selective inhibitors and estimating their activity prior to synthesis.},
	language = {eng},
	number = {5},
	journal = {Journal of Computer-Aided Molecular Design},
	author = {Sutherland, Jeffrey J. and Weaver, Donald F.},
	month = may,
	year = {2004},
	pmid = {15595459},
	keywords = {Animals, Crystallography, X-Ray, Folic Acid Antagonists, Liver, Models, Molecular, Pneumocystis carinii, Rats, Structure-Activity Relationship, Tetrahydrofolate Dehydrogenase},
	pages = {309--331},
}

@article{hill_impact_2007,
	title = {Impact of {Image} {Segmentation} on {High}-{Content} {Screening} {Data} {Quality} for {SK}-{BR}-3 {Cells}},
	volume = {8},
	issn = {1471-2105},
	url = {https://doi.org/10.1186/1471-2105-8-340},
	doi = {10.1186/1471-2105-8-340},
	abstract = {High content screening (HCS) is a powerful method for the exploration of cellular signalling and morphology that is rapidly being adopted in cancer research. HCS uses automated microscopy to collect images of cultured cells. The images are subjected to segmentation algorithms to identify cellular structures and quantitate their morphology, for hundreds to millions of individual cells. However, image analysis may be imperfect, especially for "HCS-unfriendly" cell lines whose morphology is not well handled by current image segmentation algorithms. We asked if segmentation errors were common for a clinically relevant cell line, if such errors had measurable effects on the data, and if HCS data could be improved by automated identification of well-segmented cells.},
	number = {1},
	urldate = {2024-11-19},
	journal = {BMC Bioinformatics},
	author = {Hill, Andrew A. and LaPan, Peter and Li, Yizheng and Haney, Steve},
	month = sep,
	year = {2007},
	keywords = {Image Segmentation Algorithm, Segmentation Algorithm, Segmented Cell, Segmented Object, Texture Feature},
	pages = {340},
	file = {Full Text PDF:C\:\\Users\\stolte\\Zotero\\storage\\ATXY3AKC\\Hill et al. - 2007 - Impact of image segmentation on high-content scree.pdf:application/pdf;Snapshot:C\:\\Users\\stolte\\Zotero\\storage\\F4XSJE92\\1471-2105-8-340.html:text/html},
}

@article{fisher_use_1936,
	title = {The {Use} of {Multiple} {Measurements} in {Taxonomic} {Problems}},
	volume = {7},
	copyright = {1936 Blackwell Publishing Ltd/University College London},
	issn = {2050-1439},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1469-1809.1936.tb02137.x},
	doi = {10.1111/j.1469-1809.1936.tb02137.x},
	abstract = {The articles published by the Annals of Eugenics (1925–1954) have been made available online as an historical archive intended for scholarly use. The work of eugenicists was often pervaded by prejudice against racial, ethnic and disabled groups. The online publication of this material for scholarly research purposes is not an endorsement of those views nor a promotion of eugenics in any way.},
	language = {en},
	number = {2},
	urldate = {2024-11-19},
	journal = {Annals of Eugenics},
	author = {Fisher, R. A.},
	year = {1936},
	pages = {179--188},
	file = {Full Text PDF:C\:\\Users\\stolte\\Zotero\\storage\\33TDDS7H\\Fisher - 1936 - The Use of Multiple Measurements in Taxonomic Prob.pdf:application/pdf;Snapshot:C\:\\Users\\stolte\\Zotero\\storage\\JZTKUM4F\\j.1469-1809.1936.tb02137.html:text/html},
}

@Manual{rlemon,
	title = {rlemon: R Access to LEMON Graph Algorithms},
	author = {Arav Agarwal and Aditya Tewari and Josh Errickson},
	year = {2023},
	note = {R package version 0.2.1},
	url = {https://CRAN.R-project.org/package=rlemon},
}


 @Article{ade4,
	title = {The {ade4} Package: Implementing the Duality Diagram for Ecologists},
	author = {St\'ephane Dray and Anne--B\'eatrice Dufour},
	journal = {Journal of Statistical Software},
	year = {2007},
	volume = {22},
	number = {4},
	pages = {1--20},
	doi = {10.18637/jss.v022.i04},
}

@article{kwak_central_2017,
	title = {Central limit theorem: the cornerstone of modern statistics},
	volume = {70},
	issn = {2005-6419},
	shorttitle = {Central limit theorem},
	url = {https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5370305/},
	doi = {10.4097/kjae.2017.70.2.144},
	abstract = {According to the central limit theorem, the means of a random sample of size, n, from a population with mean, µ, and variance, σ2, distribute normally with mean, µ, and variance, σ2n. Using the central limit theorem, a variety of parametric tests have been developed under assumptions about the parameters that determine the population probability distribution. Compared to non-parametric tests, which do not require any assumptions about the population probability distribution, parametric tests produce more accurate and precise estimates with higher statistical powers. However, many medical researchers use parametric tests to present their data without knowledge of the contribution of the central limit theorem to the development of such tests. Thus, this review presents the basic concepts of the central limit theorem and its role in binomial distributions and the Student's t-test, and provides an example of the sampling distributions of small populations. A proof of the central limit theorem is also described with the mathematical concepts required for its near-complete understanding.},
	number = {2},
	urldate = {2024-08-15},
	journal = {Korean Journal of Anesthesiology},
	author = {Kwak, Sang Gyu and Kim, Jong Hae},
	month = apr,
	year = {2017},
	pmid = {28367284},
	pmcid = {PMC5370305},
	pages = {144--156},
	file = {PubMed Central Full Text PDF:C\:\\Users\\stolte\\Zotero\\storage\\LALKKMFE\\Kwak und Kim - 2017 - Central limit theorem the cornerstone of modern s.pdf:application/pdf},
}

@Manual{densratio,
	title = {densratio: Density Ratio Estimation},
	author = {Koji Makiyama},
	year = {2019},
	note = {R package version 0.2.1},
	url = {https://CRAN.R-project.org/package=densratio},
}

@Manual{rpart.plot,
	title = {rpart.plot: Plot 'rpart' Models: An Enhanced Version of 'plot.rpart'},
	author = {Stephen Milborrow},
	year = {2024},
	note = {R package version 3.1.2},
	url = {https://CRAN.R-project.org/package=rpart.plot},
}

@Manual{nbpMatching,
	title = {nbpMatching: Functions for Optimal Non-Bipartite Matching},
	author = {Cole Beck and Bo Lu and Robert Greevy},
	year = {2024},
	note = {R package version 1.5.6},
	url = {https://CRAN.R-project.org/package=nbpMatching},
}


@Article{robert_intrinsic_1996,
  title = {Instrinsic losses},
  author = {Christian P. Robert},
  journal = {Theory and Decision},
  volume = {40},
  doi = {10.1007/BF00133173},
  year = {1996},
  pages = {191--214}
}