% DO NOT EDIT THIS FILE. It is auto-generated by "update_bib.sh".
@preamble{{\providecommand{\MaxMinAntSystem}{{$\cal MAX$--$\cal MIN$} {Ant} {System}} } # {\providecommand{\rpackage}[1]{{#1}} } # {\providecommand{\softwarepackage}[1]{{#1}} } # {\providecommand{\proglang}[1]{{#1}} } # {\providecommand{\BIBdepartment}[1]{{#1}, } }}

@article{BeuFonLopPaqVah09:tec,
  author = { Nicola Beume  and  Carlos M. Fonseca  and  Manuel L{\'o}pez-Ib{\'a}{\~n}ez  and  Lu{\'i}s Paquete  and  Jan Vahrenhold },
  title = {On the complexity of computing the hypervolume
                  indicator},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2009,
  volume = 13,
  number = 5,
  pages = {1075--1082},
  doi = {10.1109/TEVC.2009.2015575},
  abstract = {The goal of multi-objective optimization is to find
                  a set of best compromise solutions for typically
                  conflicting objectives. Due to the complex nature of
                  most real-life problems, only an approximation to
                  such an optimal set can be obtained within
                  reasonable (computing) time. To compare such
                  approximations, and thereby the performance of
                  multi-objective optimizers providing them, unary
                  quality measures are usually applied. Among these,
                  the \emph{hypervolume indicator} (or
                  \emph{S-metric}) is of particular relevance due to
                  its favorable properties. Moreover, this indicator
                  has been successfully integrated into stochastic
                  optimizers, such as evolutionary algorithms, where
                  it serves as a guidance criterion for finding good
                  approximations to the Pareto front. Recent results
                  show that computing the hypervolume indicator can be
                  seen as solving a specialized version of Klee's
                  Measure Problem.  In general, Klee's Measure Problem
                  can be solved with $\mathcal{O}(n \log n +
                  n^{d/2}\log n)$ comparisons for an input instance of
                  size $n$ in $d$ dimensions; as of this writing, it
                  is unknown whether a lower bound higher than
                  $\Omega(n \log n)$ can be proven. In this article,
                  we derive a lower bound of $\Omega(n\log n)$ for the
                  complexity of computing the hypervolume indicator in
                  any number of dimensions $d>1$ by reducing the
                  so-called \textsc{UniformGap} problem to it.  For
                  the three dimensional case, we also present a
                  matching upper bound of $\mathcal{O}(n\log n)$
                  comparisons that is obtained by extending an
                  algorithm for finding the maxima of a point set.}
}

@article{BezLopStu2017assessment,
  author = { Leonardo C. T. Bezerra  and  Manuel L{\'o}pez-Ib{\'a}{\~n}ez  and  Thomas St{\"u}tzle },
  title = {A Large-Scale Experimental Evaluation of High-Performing
                  Multi- and Many-Objective Evolutionary Algorithms},
  journal = {Evolutionary Computation},
  year = 2018,
  volume = 26,
  number = 4,
  pages = {621--656},
  doi = {10.1162/evco_a_00217},
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2015-007/},
  abstract = {Research on multi-objective evolutionary algorithms (MOEAs)
                  has produced over the past decades a large number of
                  algorithms and a rich literature on performance assessment
                  tools to evaluate and compare them. Yet, newly proposed MOEAs
                  are typically compared against very few, often a decade older
                  MOEAs. One reason for this apparent contradiction is the lack
                  of a common baseline for comparison, with each subsequent
                  study often devising its own experimental scenario, slightly
                  different from other studies. As a result, the state of the
                  art in MOEAs is a disputed topic. This article reports a
                  systematic, comprehensive evaluation of a large number of
                  MOEAs that covers a wide range of experimental scenarios. A
                  novelty of this study is the separation between the
                  higher-level algorithmic components related to
                  multi-objective optimization (MO), which characterize each
                  particular MOEA, and the underlying parameters-such as
                  evolutionary operators, population size, etc.-whose
                  configuration may be tuned for each scenario. Instead of
                  relying on a common or "default" parameter configuration that
                  may be low-performing for particular MOEAs or scenarios and
                  unintentionally biased, we tune the parameters of each MOEA
                  for each scenario using automatic algorithm configuration
                  methods. Our results confirm some of the assumed knowledge in
                  the field, while at the same time they provide new insights
                  on the relative performance of MOEAs for many-objective
                  problems. For example, under certain conditions,
                  indicator-based MOEAs are more competitive for such problems
                  than previously assumed. We also analyze problem-specific
                  features affecting performance, the agreement between
                  performance metrics, and the improvement of tuned
                  configurations over the default configurations used in the
                  literature. Finally, the data produced is made publicly
                  available to motivate further analysis and a baseline for
                  future comparisons.}
}

@article{BinGinRou2015gaupar,
  title = {Quantifying uncertainty on {Pareto} fronts with {Gaussian}
                  process conditional simulations},
  volume = 243,
  doi = {10.1016/j.ejor.2014.07.032},
  abstract = {Multi-objective optimization algorithms aim at finding
                  Pareto-optimal solutions. Recovering Pareto fronts or Pareto
                  sets from a limited number of function evaluations are
                  challenging problems. A popular approach in the case of
                  expensive-to-evaluate functions is to appeal to
                  metamodels. Kriging has been shown efficient as a base for
                  sequential multi-objective optimization, notably through
                  infill sampling criteria balancing exploitation and
                  exploration such as the Expected Hypervolume
                  Improvement. Here we consider Kriging metamodels not only for
                  selecting new points, but as a tool for estimating the whole
                  Pareto front and quantifying how much uncertainty remains on
                  it at any stage of Kriging-based multi-objective optimization
                  algorithms. Our approach relies on the Gaussian process
                  interpretation of Kriging, and bases upon conditional
                  simulations. Using concepts from random set theory, we
                  propose to adapt the Vorob'ev expectation and deviation to
                  capture the variability of the set of non-dominated
                  points. Numerical experiments illustrate the potential of the
                  proposed workflow, and it is shown on examples how Gaussian
                  process simulations and the estimated Vorob'ev deviation can
                  be used to monitor the ability of Kriging-based
                  multi-objective optimization algorithms to accurately learn
                  the Pareto front.},
  number = 2,
  journal = {European Journal of Operational Research},
  author = {Binois, Micka{\"e}l and Ginsbourger, David and Roustant, Olivier},
  year = 2015,
  keywords = {Attainment function, Expected Hypervolume Improvement,
                  Kriging, Multi-objective optimization, Vorob'ev expectation},
  pages = {386--394}
}

@article{BriFri2010approx,
  author = { Karl Bringmann  and  Tobias Friedrich },
  title = {Approximating the volume of unions and intersections of
                  high-dimensional geometric objects},
  journal = {Computational Geometry},
  year = 2010,
  volume = 43,
  number = {6--7},
  pages = {601--610},
  annote = {Proved that the hypervolume indicator is \#P-hard,},
  doi = {10.1016/j.comgeo.2010.03.004}
}

@article{Deb02nsga2,
  author = { Kalyanmoy Deb  and A. Pratap and S. Agarwal and T. Meyarivan},
  title = {A fast and elitist multi-objective genetic
                  algorithm: {NSGA-II}},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2002,
  volume = 6,
  number = 2,
  pages = {182--197},
  doi = {10.1109/4235.996017}
}

@article{DenZha2019approxhv,
  author = {Deng, Jingda and  Zhang, Qingfu },
  title = {Approximating Hypervolume and Hypervolume Contributions Using
                  Polar Coordinate},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2019,
  volume = 23,
  number = 5,
  pages = {913--918},
  month = oct,
  annote = {Proposed approximating the hypervolume using scalarizations},
  doi = {10.1109/tevc.2019.2895108}
}

@article{DiaLop2020ejor,
  author = { Juan Esteban Diaz  and  Manuel L{\'o}pez-Ib{\'a}{\~n}ez },
  title = {Incorporating Decision-Maker's Preferences into the Automatic
                  Configuration of Bi-Objective Optimisation Algorithms},
  journal = {European Journal of Operational Research},
  year = 2021,
  volume = 289,
  number = 3,
  pages = {1209--1222},
  doi = {10.1016/j.ejor.2020.07.059},
  abstract = {Automatic configuration (AC) methods are increasingly used to
                  tune and design optimisation algorithms for problems with
                  multiple objectives. Most AC methods use unary quality
                  indicators, which assign a single scalar value to an
                  approximation to the Pareto front, to compare the performance
                  of different optimisers. These quality indicators, however,
                  imply preferences beyond Pareto-optimality that may differ
                  from those of the decision maker (DM). Although it is
                  possible to incorporate DM's preferences into quality
                  indicators, e.g., by means of the weighted hypervolume
                  indicator (HV$^w$), expressing preferences in terms of weight
                  function is not always intuitive nor an easy task for a DM,
                  in particular, when comparing the stochastic outcomes of
                  several algorithm configurations. A more visual approach to
                  compare such outcomes is the visualisation of their empirical
                  attainment functions (EAFs) differences. This paper proposes
                  using such visualisations as a way of eliciting information
                  about regions of the objective space that are preferred by
                  the DM. We present a method to convert the information about
                  EAF differences into a HV$^w$ that will assign higher quality
                  values to approximation fronts that result in EAF differences
                  preferred by the DM. We show that the resulting HV$^w$ may be
                  used by an AC method to guide the configuration of
                  multi-objective optimisers according to the preferences of
                  the DM. We evaluate the proposed approach on a well-known
                  benchmark problem. Finally, we apply our approach to
                  re-configuring, according to different DM's preferences, a
                  multi-objective optimiser tackling a real-world production
                  planning problem arising in the manufacturing industry.},
  supplement = {https://doi.org/10.5281/zenodo.3749288}
}

@article{DubLopStu2011amai,
  author = { J{\'e}r{\'e}mie Dubois-Lacoste  and  Manuel L{\'o}pez-Ib{\'a}{\~n}ez  and  Thomas St{\"u}tzle },
  title = {Improving the Anytime Behavior of Two-Phase Local Search},
  journal = {Annals of Mathematics and Artificial Intelligence},
  year = 2011,
  volume = 61,
  number = 2,
  pages = {125--154},
  doi = {10.1007/s10472-011-9235-0}
}

@article{GueFon2017hv4d,
  author = { Andreia P. Guerreiro  and  Carlos M. Fonseca },
  title = {Computing and Updating Hypervolume Contributions in Up to
                  Four Dimensions},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2018,
  volume = 22,
  number = 3,
  pages = {449--463},
  month = jun,
  annote = {Proposed HV3D$^{+}$ with $O(n\log n)$ complexity and
                  HV4D$^{+}$ with $O(n^2)$ complexity},
  doi = {10.1109/tevc.2017.2729550}
}

@article{GueFonPaq2021hv,
  author = { Andreia P. Guerreiro  and  Carlos M. Fonseca  and  Lu{\'i}s Paquete },
  title = {The Hypervolume Indicator: Computational Problems and
                  Algorithms},
  journal = {{ACM} Computing Surveys},
  year = 2021,
  volume = 54,
  number = 6,
  pages = {1--42},
  doi = {10.1145/3453474}
}

@article{HerWer1987tabucol,
  author = {A. Hertz  and de Werra, D.},
  title = {Using Tabu Search Techniques for Graph Coloring},
  journal = {Computing},
  year = 1987,
  volume = 39,
  number = 4,
  pages = {345--351}
}

@article{Jen03,
  title = {Reducing the run-time complexity of multiobjective
                  {EA}s: The {NSGA-II} and other algorithms},
  author = {M. T. Jensen},
  journal = {IEEE Transactions on Evolutionary Computation},
  volume = 7,
  number = 5,
  pages = {503--515},
  year = 2003
}

@article{JohAraMcGSch1991,
  author = {David S. Johnson and Cecilia R. Aragon and  Lyle A. McGeoch  and Catherine Schevon},
  title = {Optimization by Simulated Annealing: An Experimental
                  Evaluation: Part {II}, Graph Coloring and Number Partitioning},
  journal = {Operations Research},
  year = 1991,
  volume = 39,
  number = 3,
  pages = {378--406}
}

@article{KunLucPre1975jacm,
  author = {H. T. Kung and F. Luccio and F. P. Preparata},
  title = {On Finding the Maxima of a Set of Vectors},
  journal = {Journal of the ACM},
  year = 1975,
  volume = 22,
  number = 4,
  pages = {469--476},
  doi = {10.1145/321906.321910}
}

@article{LacKlaFon2017box,
  author = {Renaud Lacour and  Kathrin Klamroth  and  Carlos M. Fonseca },
  title = {A box decomposition algorithm to compute the hypervolume
                  indicator},
  journal = {Computers \& Operations Research},
  year = 2017,
  volume = 79,
  pages = {347--360},
  doi = {10.1016/J.COR.2016.06.021}
}

@article{LopVerDreDoe2025,
  author = { Manuel L{\'o}pez-Ib{\'a}{\~n}ez  and  Diederick Vermetten  and  Johann Dreo  and  Carola Doerr },
  title = {Using the Empirical Attainment Function for Analyzing
                  Single-objective Black-box Optimization Algorithms},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2025,
  volume = 29,
  number = 5,
  pages = {1774--1782},
  annote = {Pre-print: \url{https://doi.org/10.48550/arXiv.2404.02031}},
  doi = {10.1109/TEVC.2024.3462758},
  abstract = {A widely accepted way to assess the performance of iterative
                  black-box optimizers is to analyze their empirical cumulative
                  distribution function (ECDF) of pre-defined quality targets
                  achieved not later than a given runtime. In this work, we
                  consider an alternative approach, based on the empirical
                  attainment function (EAF) and we show that the target-based
                  ECDF is an approximation of the EAF. We argue that the EAF
                  has several advantages over the target-based ECDF. In
                  particular, it does not require defining a priori quality
                  targets per function, captures performance differences more
                  precisely, and enables the use of additional summary
                  statistics that enrich the analysis. We also show that the
                  average area over the convergence curves is a
                  simpler-to-calculate, but equivalent, measure of anytime
                  performance. To facilitate the accessibility of the EAF, we
                  integrate a module to compute it into the IOHanalyzer
                  platform. Finally, we illustrate the use of the EAF via
                  synthetic examples and via the data available for the BBOB
                  suite.},
  keywords = {EAF-based ECDF}
}

@article{Muller1959sphere,
  author = {Mervin E. Muller},
  title = {A Note on a Method for Generating Points Uniformly on
                  N-Dimensional Spheres},
  journal = {Communications of the ACM},
  year = 1959,
  volume = 2,
  number = 4,
  pages = {19--20},
  doi = {10.1145/377939.377946},
  keywords = {hypersphere, sampling}
}

@article{SchEsqLarCoe2012tec,
  author = { Oliver Sch{\"u}tze  and X. Esquivel and A. Lara and  Carlos A. {Coello Coello} },
  title = {Using the Averaged {Hausdorff} Distance as a Performance
                  Measure in Evolutionary Multiobjective Optimization},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2012,
  volume = 16,
  number = 4,
  pages = {504--522},
  doi = {10.1109/TEVC.2011.2161872}
}

@article{WuAza2001metrics,
  author = {J. Wu and S. Azam},
  title = {Metrics for Quality Assessment of a Multiobjective Design
                  Optimization Solution Set},
  journal = {Journal of Mechanical Design},
  year = 2001,
  volume = 123,
  number = 1,
  pages = {18--25},
  annote = {Proposed inclusion-exclusion method for computing
                  hypervolume.},
  doi = {10.1115/1.1329875}
}

@article{ZhoZhaJin2009igdx,
  author = {Zhou, A. and  Zhang, Qingfu  and  Yaochu Jin },
  title = {Approximating the set of {Pareto}-optimal solutions in both
                  the decision and objective spaces by an estimation of
                  distribution algorithm},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2009,
  volume = 13,
  number = 5,
  pages = {1167--1189},
  doi = {10.1109/TEVC.2009.2021467},
  keywords = {multi-modal, IGDX}
}

@article{ZitThiLauFon2003:tec,
  author = { Eckart Zitzler  and  Lothar Thiele  and  Marco Laumanns  and  Carlos M. Fonseca  and  Viviane {Grunert da Fonseca} },
  title = {Performance Assessment of Multiobjective Optimizers: an
                  Analysis and Review},
  journal = {IEEE Transactions on Evolutionary Computation},
  year = 2003,
  volume = 7,
  number = 2,
  pages = {117--132},
  annote = {Proposed the combination of quality indicators; proposed
                  epsilon-indicator},
  amonth = apr,
  doi = {10.1109/TEVC.2003.810758}
}

@incollection{AugBadBroZit2009gecco,
  booktitle = {Proceedings of  the Genetic and Evolutionary Computation Conference, GECCO 2009},
  address = { New York, NY},
  year = 2009,
  publisher = {ACM Press},
  editor = { Franz Rothlauf },
  author = { Anne Auger  and  Johannes Bader  and  Dimo Brockhoff  and  Eckart Zitzler },
  title = {Articulating User Preferences in Many-Objective
                  Problems by Sampling the Weighted Hypervolume},
  pages = {555--562}
}

@incollection{AugBadBroZit2009gecco2,
  booktitle = {Proceedings of  the Genetic and Evolutionary Computation Conference, GECCO 2009},
  address = { New York, NY},
  year = 2009,
  publisher = {ACM Press},
  editor = { Franz Rothlauf },
  author = { Anne Auger  and  Johannes Bader  and  Dimo Brockhoff  and  Eckart Zitzler },
  title = {Investigating and Exploiting the Bias of the
                  Weighted Hypervolume to Articulate User Preferences},
  pages = {563--570}
}

@incollection{BezLopStu2017emo,
  editor = {Heike Trautmann and G{\"{u}}nter Rudolph and Kathrin Klamroth
                  and Oliver Sch{\"{u}}tze and Margaret M. Wiecek and Yaochu
                  Jin and Christian Grimme},
  year = 2017,
  volume = 10173,
  series = {Lecture Notes in Computer Science},
  address = { Cham, Switzerland},
  publisher = {Springer International Publishing},
  booktitle = { Evolutionary Multi-criterion Optimization, EMO 2017},
  author = { Leonardo C. T. Bezerra  and  Manuel L{\'o}pez-Ib{\'a}{\~n}ez  and  Thomas St{\"u}tzle },
  title = {An Empirical Assessment of the Properties of Inverted
                  Generational Distance Indicators on Multi- and Many-objective
                  Optimization},
  pages = {31--45},
  doi = {10.1007/978-3-319-54157-0_3}
}

@incollection{CheGinBecMol2013moda,
  address = { Heidelberg, Germany},
  publisher = {Springer International Publishing},
  booktitle = {mODa 10--Advances in Model-Oriented Design and Analysis},
  year = 2013,
  editor = {Ucinski, Dariusz and Atkinson, Anthony C.  and Patan, Maciej},
  author = {Chevalier, Cl{\'e}ment and Ginsbourger, David and Bect,
                  Julien and Molchanov, Ilya},
  title = {Estimating and Quantifying Uncertainties on Level Sets Using
                  the {Vorob}'ev Expectation and Deviation with {Gaussian}
                  Process Models},
  pages = {35--43},
  abstract = {Several methods based on Kriging have recently been proposed
                  for calculating a probability of failure involving
                  costly-to-evaluate functions. A closely related problem is to
                  estimate the set of inputs leading to a response exceeding a
                  given threshold. Now, estimating such a level set---and not
                  solely its volume---and quantifying uncertainties on it are
                  not straightforward. Here we use notions from random set
                  theory to obtain an estimate of the level set, together with
                  a quantification of estimation uncertainty. We give explicit
                  formulae in the Gaussian process set-up and provide a
                  consistency result. We then illustrate how space-filling
                  versus adaptive design strategies may sequentially reduce
                  level set estimation uncertainty.},
  doi = {10.1007/978-3-319-00218-7_5}
}

@phdthesis{ChiarandiniPhD,
  author = { Marco Chiarandini },
  title = {Stochastic Local Search Methods for Highly
                  Constrained Combinatorial Optimisation Problems},
  school = {FB Informatik, TU Darmstadt, Germany},
  year = 2005
}

@incollection{CoeSie2004igd,
  address = { Heidelberg, Germany},
  series = {Lecture Notes in Artificial Intelligence},
  volume = 2972,
  booktitle = {Proceedings of MICAI},
  publisher = {Springer},
  year = 2004,
  editor = {Monroy, Ra{\'u}l and Arroyo-Figueroa, Gustavo and Sucar, Luis
                  Enrique and Sossa, Humberto},
  author = { Carlos A. {Coello Coello}  and Reyes-Sierra, Margarita},
  title = {A Study of the Parallelization of a Coevolutionary
                  Multi-objective Evolutionary Algorithm},
  pages = {688--697},
  keywords = {IGD},
  annote = {Introduces Inverted Generational Distance (IGD)}
}

@incollection{FonGueLopPaq2011emo,
  booktitle = { Evolutionary Multi-criterion Optimization, EMO 2011},
  address = {Berlin~/ Heidelberg},
  series = {Lecture Notes in Computer Science},
  volume = 6576,
  year = 2011,
  publisher = {Springer},
  editor = { Takahashi, R. H. C.  and  Kalyanmoy Deb  and  Wanner, Elizabeth F.  and  Salvatore Greco },
  author = { Carlos M. Fonseca  and  Andreia P. Guerreiro  and  Manuel L{\'o}pez-Ib{\'a}{\~n}ez  and  Lu{\'i}s Paquete },
  title = {On the Computation of the Empirical Attainment Function},
  doi = {10.1007/978-3-642-19893-9_8},
  pages = {106--120},
  abstract = {The attainment function provides a description of the
                  location of the distribution of a random non-dominated point
                  set. This function can be estimated from experimental data
                  via its empirical counterpart, the empirical attainment
                  function (EAF). However, computation of the EAF in more than
                  two dimensions is a non-trivial task. In this article, the
                  problem of computing the empirical attainment function is
                  formalised, and upper and lower bounds on the corresponding
                  number of output points are presented. In addition, efficient
                  algorithms for the two and three-dimensional cases are
                  proposed, and their time complexities are related to lower
                  bounds derived for each case.}
}

@inproceedings{FonPaqLop06:hypervolume,
  address = {Piscataway, NJ},
  publisher = {IEEE Press},
  month = jul,
  year = 2006,
  booktitle = {Proceedings of  the 2006 Congress on Evolutionary Computation (CEC 2006)},
  key = {IEEE CEC},
  author = { Carlos M. Fonseca  and  Lu{\'i}s Paquete  and  Manuel L{\'o}pez-Ib{\'a}{\~n}ez },
  title = {An improved dimension-{}{}{}sweep
                  algorithm for the hypervolume indicator},
  pages = {1157--1163},
  doi = {10.1109/CEC.2006.1688440},
  abstract = {This paper presents a recursive, dimension-sweep
                  algorithm for computing the hypervolume indicator of
                  the quality of a set of $n$ non-dominated points in
                  $d>2$ dimensions. It improves upon the existing HSO
                  (Hypervolume by Slicing Objectives) algorithm by
                  pruning the recursion tree to avoid repeated
                  dominance checks and the recalculation of partial
                  hypervolumes. Additionally, it incorporates a recent
                  result for the three-dimensional special case.  The
                  proposed algorithm achieves $O(n^{d-2} \log n)$ time
                  and linear space complexity in the worst-case, but
                  experimental results show that the pruning
                  techniques used may reduce the time complexity
                  exponent even further.}
}

@incollection{GruFon2009:emaa,
  editor = { Thomas Bartz-Beielstein  and  Marco Chiarandini  and  Lu{\'i}s Paquete  and  Mike Preuss },
  year = 2010,
  address = {Berlin~/ Heidelberg},
  publisher = {Springer},
  booktitle = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  author = { Viviane {Grunert da Fonseca}  and  Carlos M. Fonseca },
  title = {The Attainment-Function Approach to Stochastic Multiobjective
                  Optimizer Assessment and Comparison},
  pages = {103--130},
  doi = {10.1007/978-3-642-02538-9_5}
}

@incollection{Grunert01,
  booktitle = { Evolutionary Multi-criterion Optimization, EMO 2001},
  address = {Berlin~/ Heidelberg},
  series = {Lecture Notes in Computer Science},
  volume = 1993,
  year = 2001,
  publisher = {Springer},
  editor = { Eckart Zitzler  and  Kalyanmoy Deb  and  Lothar Thiele  and  Carlos A. {Coello Coello}  and  David Corne },
  author = { Viviane {Grunert da Fonseca}  and  Carlos M. Fonseca  and  Andreia O. Hall },
  key = {Fonseca et al., 2001},
  title = {Inferential Performance Assessment of Stochastic Optimisers
                  and the Attainment Function},
  pages = {213--225},
  doi = {10.1007/3-540-44719-9_15},
  annote = {Proposed looking at anytime behavior as a multi-objective
                  problem},
  keywords = {EAF},
  abstract = {The performance of stochastic optimisers can be assessed
                  experimentally on given problems by performing multiple
                  optimisation runs, and analysing the results. Since an
                  optimiser may be viewed as an estimator for the (Pareto)
                  minimum of a (vector) function, stochastic optimiser
                  performance is discussed in the light of the criteria
                  applicable to more usual statistical
                  estimators. Multiobjective optimisers are shown to deviate
                  considerably from standard point estimators, and to require
                  special statistical methodology. The attainment function is
                  formulated, and related results from random closed-set theory
                  are presented, which cast the attainment function as a
                  mean-like measure for the outcomes of multiobjective
                  optimisers. Finally, a covariance-measure is defined, which
                  should bring additional insight into the stochastic behaviour
                  of multiobjective optimisers. Computational issues and
                  directions for further work are discussed at the end of the
                  paper.}
}

@incollection{IshMasTanNoj2015igd,
  booktitle = { Evolutionary Multi-criterion Optimization, EMO 2015 Part {I}},
  address = { Heidelberg, Germany},
  series = {Lecture Notes in Computer Science},
  volume = 9018,
  year = 2015,
  publisher = {Springer},
  editor = { Ant{\'o}nio Gaspar{-}Cunha  and Carlos Henggeler Antunes and  Carlos A. {Coello Coello} },
  author = { Ishibuchi, Hisao  and Masuda, Hiroyuki and Tanigaki, Yuki and
                  Nojima, Yusuke},
  title = {Modified Distance Calculation in Generational Distance and
                  Inverted Generational Distance},
  pages = {110--125},
  annote = {Proposed IGD+},
  keywords = {Performance metrics, multi-objective, IGD, IGD+}
}

@inproceedings{KnoCor2002cec,
  year = 2002,
  address = {Piscataway, NJ},
  publisher = {IEEE Press},
  booktitle = {Proceedings of  the 2002 Congress on Evolutionary Computation (CEC'02)},
  key = {IEEE CEC},
  author = { Joshua D. Knowles  and  David Corne },
  title = {On Metrics for Comparing Non-Dominated Sets},
  pages = {711--716}
}

@incollection{LopPaqStu09emaa,
  editor = { Thomas Bartz-Beielstein  and  Marco Chiarandini  and  Lu{\'i}s Paquete  and  Mike Preuss },
  year = 2010,
  address = {Berlin~/ Heidelberg},
  publisher = {Springer},
  booktitle = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  author = { Manuel L{\'o}pez-Ib{\'a}{\~n}ez  and  Lu{\'i}s Paquete  and  Thomas St{\"u}tzle },
  title = {Exploratory Analysis of Stochastic Local Search
                  Algorithms in Biobjective Optimization},
  pages = {209--222},
  doi = {10.1007/978-3-642-02538-9_9},
  abstract = {This chapter introduces two Perl programs that
                  implement graphical tools for exploring the
                  performance of stochastic local search algorithms
                  for biobjective optimization problems. These tools
                  are based on the concept of the empirical attainment
                  function (EAF), which describes the probabilistic
                  distribution of the outcomes obtained by a
                  stochastic algorithm in the objective space. In
                  particular, we consider the visualization of
                  attainment surfaces and differences between the
                  first-order EAFs of the outcomes of two
                  algorithms. This visualization allows us to identify
                  certain algorithmic behaviors in a graphical way.
                  We explain the use of these visualization tools and
                  illustrate them with examples arising from
                  practice.}
}

@phdthesis{LopezIbanezPhD,
  author = { Manuel L{\'o}pez-Ib{\'a}{\~n}ez },
  title = {Operational Optimisation of Water Distribution
                  Networks},
  school = {School of Engineering and the Built Environment},
  year = 2009,
  address = {Edinburgh Napier University, UK},
  url = {https://lopez-ibanez.eu/publications#LopezIbanezPhD}
}

@book{Molchanov2005theory,
  author = {Molchanov, Ilya},
  title = {Theory of Random Sets},
  publisher = {Springer},
  year = 2005,
  keywords = {Vorob'ev expectation}
}

@book{RubMel1998simulation,
  author = {R. Y. Rubinstein and B. Melamed},
  title = {Modern simulation and modeling},
  year = 1998,
  publisher = {Wiley},
  address = { New York, NY},
  notes = {Uniform sampling from the simplex.}
}

@inproceedings{VelLam1998gp,
  year = 1998,
  publisher = {Stanford University Bookstore},
  address = {Stanford University, California},
  month = jul,
  editor = {John R. Koza},
  booktitle = {Late Breaking Papers at the Genetic Programming 1998
                  Conference},
  key = {Van Veldhuizen and Lamont, 1998a},
  title = {Evolutionary Computation and Convergence to a
                  {Pareto} Front},
  author = { David A. {Van Veldhuizen}  and  Gary B. Lamont },
  pages = {221--228},
  keywords = {generational distance}
}

@incollection{ZitThi1998ppsn,
  address = { Heidelberg, Germany},
  publisher = {Springer},
  editor = { Agoston E. Eiben  and  Thomas B{\"a}ck  and  Marc Schoenauer  and  Hans-Paul Schwefel },
  volume = 1498,
  series = {Lecture Notes in Computer Science},
  year = 1998,
  booktitle = {Parallel Problem Solving from Nature -- {PPSN} {V}},
  author = { Eckart Zitzler  and  Lothar Thiele },
  title = {Multiobjective Optimization Using Evolutionary Algorithms -
                  {A} Comparative Case Study},
  pages = {292--301},
  doi = {10.1007/BFb0056872},
  annote = {Proposed hypervolume measure}
}
