@InProceedings{zouhar-EtAl:2026:wmt,
  author    = {Zouhar, Vilem  and  Grundkiewicz, Roman  and  Rajaee, Sara  and  Riley, Parker  and  Bawden, Rachel  and  Koehn, Philipp  and  Carpuat, Marine  and  Kocmi, Tom},
  title     = {Contrastive ESA: Human Evaluation of Multiple Translations at Once},
  booktitle      = {Proceedings of the Eleventh Conference on Machine Translation},
  month          = {October},
  year           = {2026},
  address        = {Budapest, Hungary},
  publisher      = {Association for Computational Linguistics},
  pages     = {867--879},
  abstract  = {Current human evaluation of machine translation typically assesses single outputs in isolation, a paradigm that suffers from high annotator noise and cost. We introduce Contrastive Error Span Annotation (cESA), a protocol that presents multiple translations of the source input (text, video, audio, image). In cESA, the annotator sees multiple translations of the same document, marks major and minor error spans, and then assigns a score from 0\% to 100\% on absolute scale. By allowing annotators to access the shared context across multiple outputs, cESA facilitates more consistent and efficient judgments. We validate cESA using a large-scale human evaluation of English->Japanese translations of 12 models, demonstrating reductions in annotation time and noise compared to standard pointwise evaluation. Unlike existing contrastive ranking methods, cESA yields absolute quality judgments that enable simple, interpretable non-parametric model rankings without the need for post-hoc corrections.},
  url       = {https://aclanthology.org/2026.wmt-1.47}
}