@inproceedings{e74ee3ae76ef4bf28b31aee2d8893600,
title = "MLQE-PE: A Multilingual Quality Estimation and Post-Editing Dataset",
abstract = "We present MLQE-PE, a new dataset for Machine Translation (MT) Quality Estimation (QE) and Automatic Post-Editing (APE). The dataset contains annotations for eleven language pairs, including both high- and low-resource languages. Specifically, it is annotated for translation quality with human labels for up to 10,000 translations per language pair in the following formats: sentence-level direct assessments and post-editing effort, and word-level binary good/bad labels. Apart from the quality-related scores, each source-translation sentence pair is accompanied by the corresponding post-edited sentence, as well as titles of the articles where the sentences were extracted from, and information on the neural MT models used to translate the text. We provide a thorough description of the data collection and annotation process as well as an analysis of the annotation distribution for each language pair. We also report the performance of baseline systems trained on the MLQE-PE dataset. The dataset is freely available and has already been used for several WMT shared tasks.",
keywords = "direct assessments, evaluation, Machine Translation, post-edits, quality estimation",
author = "Marina Fomicheva and Shuo Sun and Erick Fonseca and Chrysoula Zerva and Fr{\'e}d{\'e}ric Blain and Vishrav Chaudhary and Francisco Guzm{\'a}n and Nina Lopatina and Martins, \{Andr{\'e} F.T.\} and Lucia Specia",
note = "Funding Information: Marina Fomicheva, Fr{\'e}d{\'e}ric Blain and Lucia Specia were supported by funding from the Bergamot project (EU H2020 Grant No. 825303). Andr{\'e} Martins, Chrysoula Zerva and Erick Fonseca were funded by the P2020 programs Unbabel4EU (contract 042671) and MAIA (contract 045909), by the European Research Council (ERC StG DeepSPIN 758969), and by the Funda{\c c}{\~a}o para a Ci{\^e}ncia e Tecnologia through contract UIDB/50008/2020. We would like to thank Marina S{\'a}nchez-Torr{\'o}n and Camila Pohlmann for monitoring the post-editing process. We also thank Mark Fishel from the University of Tartu for providing the Estonian reference translations. Funding Information: Marina Fomicheva, Fr{\'e}d{\'e}ric Blain and Lucia Specia were supported by funding from the Bergamot project (EU H2020 Grant No. 825303). Andr{\'e} Martins, Chrysoula Zerva and Erick Fonseca were funded by the P2020 programs Unbabel4EU (contract 042671) and MAIA (contract 045909), by the European Research Council (ERC StG DeepSPIN 758969), and by the Fundac¸{\~a}o para a Ci{\^e}ncia e Tecnologia through contract UIDB/50008/2020. We would like to thank Marina S{\'a}nchez-Torr{\'o}n and Camila Pohlmann for monitoring the post-editing process. We also thank Mark Fishel from the University of Tartu for providing the Estonian reference translations. Publisher Copyright: {\textcopyright} European Language Resources Association (ELRA), licensed under CC-BY-NC-4.0.; 13th International Conference on Language Resources and Evaluation Conference, LREC 2022 ; Conference date: 20-06-2022 Through 25-06-2022",
year = "2022",
language = "English",
series = "2022 Language Resources and Evaluation Conference, LREC 2022",
publisher = "European Language Resources Association (ELRA)",
pages = "4963--4974",
editor = "Nicoletta Calzolari and Frederic Bechet and Philippe Blache and Khalid Choukri and Christopher Cieri and Thierry Declerck and Sara Goggi and Hitoshi Isahara and Bente Maegaard and Joseph Mariani and Helene Mazo and Jan Odijk and Stelios Piperidis",
booktitle = "2022 Language Resources and Evaluation Conference, LREC 2022",
}