@article{d7e145ce52934367931192384e305b11,
title = "Human evaluation of automatically generated text: Current trends and best practice guidelines",
abstract = "Currently, there is little agreement as to how Natural Language Generation (NLG) systems should be evaluated, with a particularly high degree of variation in the way that human evaluation is carried out. This paper provides an overview of how (mostly intrinsic) human evaluation is currently conducted and presents a set of best practices, grounded in the literature. These best practices are also linked to the stages that researchers go through when conducting an evaluation research (planning stage; execution and release stage), and the specific steps in these stages. With this paper, we hope to contribute to the quality and consistency of human evaluations in NLG. (C) 2020 The Authors. Published by Elsevier Ltd.",
keywords = "BIAS, DESIGN, Ethics, Human evaluation, INFORMED-CONSENT, INTERRATER RELIABILITY, LANGUAGE, Literature review, NUMBER, Natural Language Generation, Open science, POWER, RATING-SCALES, RESPONSE CATEGORIES, Recommendations, VALIDITY",
author = "Lee, {Chris van der} and Albert Gatt and Miltenburg, {Emiel van} and Emiel Krahmer",
note = "Funding Information: We received support from RAAK-PRO SIA (2014-01-51PRO) and The Netherlands Organization for Scientific Research (NWO 360-89-050), which is gratefully acknowledged. Furthermore, we want to extend our gratitude towards the anonymous reviewers and also towards Leshem Choshen, Ond?ej Du?ek, Kees van Deemter, Dimitra Gkatzia, David Howcroft, Ehud Reiter, and Sander Wubben for their valuable comments on the paper. Funding Information: We received support from RAAK-PRO SIA (2014-01-51PRO) and The Netherlands Organization for Scientific Research (NWO 360-89-050), which is gratefully acknowledged. Furthermore, we want to extend our gratitude towards the anonymous reviewers and also towards Leshem Choshen, Ond{\v r}ej Du{\v s}ek, Kees van Deemter, Dimitra Gkatzia, David Howcroft, Ehud Reiter, and Sander Wubben for their valuable comments on the paper. Publisher Copyright: {\textcopyright} 2020 The Authors",
year = "2021",
month = may,
day = "21",
doi = "10.1016/j.csl.2020.101151",
language = "English",
volume = "67",
pages = "1--24",
journal = "Computer Speech and Language: An official publication of the International Speech Communication Association (ISCA)",
issn = "0885-2308",
publisher = "Academic Press",
}