@InProceedings{vasileva:2026:wmt,
  author    = {Vasileva, Lisa},
  title     = {"I'm Sorry, I Can't Translate That": A Challenge Set for Refusal to Translate, Overgeneration and Hallucination in LLM Translation Systems},
  booktitle      = {Proceedings of the Eleventh Conference on Machine Translation},
  month          = {October},
  year           = {2026},
  address        = {Budapest, Hungary},
  publisher      = {Association for Computational Linguistics},
  pages     = {1601--1608},
  abstract  = {Large language models (LLMs) have significantly raised the bar for translation quality in the recent years. However, they are still prone to novel failure modes, including hallucinations and overgenerations, where the generated translation is partially or fully detached from the source text and contains information that is not present in the input. To facilitate the analysis of these behaviors and assess the robustness of LLM-based translation systems, we introduce a challenge set specifically designed to probe scenarios that we have observed to be particularly prone to overgeneration, hallucination, and other forms of semantic detachment from the source. We describe the construction of the challenge set and present the evaluation of the systems submitted to the WMT 2026 General Machine Translation shared task.},
  url       = {https://aclanthology.org/2026.wmt-1.92}
}