2002
@inproceedings{MMP2002,
vgclass = {refpap},
author = {Henning M\"{u}ller and St\'{e}phane Marchand-Maillet and Thierry Pun},
title = {The Truth about {C}orel - Evaluation in Image Retrieval},
booktitle = {Proceedings of the International Conference on Image and
Video Retrieval (CIVR 2002)},
address = {London, UK},
number = {2383},
series = {Lecture Notes in Computer Science},
pages = {38--49},
publisher = {Springer-Verlag},
month = {July~18--19},
year = {2002},
url = {http://dx.doi.org/10.1007/3-540-45479-9_5},
url1 = {http://www.dim.hcuge.ch/medgift/publications/MMP2002.pdf},
abstract = {To demonstrate the performance of content-based image
retrieval systems (CBIRSs), there is not yet any standard data set that
is widely used. The only dataset used by a large number of research
groups are the Corel Photo CDs. There are more than 800 of those CDs,
each containing 100 pictures roughly similar in theme. Unfortunately,
basically every evaluation is done on a different subset of the image
sets thus making comparison impossible.
In this article, we compare different ways of evaluating the
performance using a subset of the Corel images with the same CBIRS and
the same set of evaluation measures. The aim is to show how easy it is
to get differing results, even when using the same image collection,
the same CBIRS and the same performance measures. This pinpoints the
fact that we need a standard database of images with a query set and
corresponding relevance judgments (RJs) to really compare systems.
The techniques used in this article to ``enhance'' the apparent
performance of a CBIRS are commonly used, sometimes described,
sometimes not. They all have a justification and seem to change the
performance of a CBIRS but they do actually not. With a larger subset
of images it is of course much easier to generate even bigger
differences in performance. The goal of this article is not to be a
guide of how to make the ``apparent'' performance of systems look good,
but rather to make readers aware of CBIRS evaluations and the
importance of standardized image databases, queries and RJ.},
}