Programming exercises involving algorithms typically involve time and spatial constraints. Automated assessments for such implementations are often carried out in a black-box manner or through static analysis of the code, without considering the internal execution properties, which could lead to falsely positive evaluations of students' solutions. We present Witter, a domain-specific language for defining white-box test cases for the Java language. We evaluated programming assignment submissions from a Data Structures and Algorithms course against Witter’s test cases to determine if our approach could offer additional insight regarding incomplete algorithmic behaviour requirements. We found that a significant amount of student solutions fail to meet the desired algorithmic behavior (approx. 21%), despite passing black-box tests. Hence, we conclude that white-box tests are useful to achieve a thorough automated evaluation of this kind of exercises.
@InProceedings{b.canico_et_al:OASIcs.ICPEC.2024.2, author = {B. Cani\c{c}o, Afonso and Santos, Andr\'{e} L.}, title = {{A Domain-Specific Language for Dynamic White-Box Evaluation of Java Assignments}}, booktitle = {5th International Computer Programming Education Conference (ICPEC 2024)}, pages = {2:1--2:13}, series = {Open Access Series in Informatics (OASIcs)}, ISBN = {978-3-95977-347-8}, ISSN = {2190-6807}, year = {2024}, volume = {122}, editor = {Santos, Andr\'{e} L. and Pinto-Albuquerque, Maria}, publisher = {Schloss Dagstuhl -- Leibniz-Zentrum f{\"u}r Informatik}, address = {Dagstuhl, Germany}, URL = {https://drops.dagstuhl.de/entities/document/10.4230/OASIcs.ICPEC.2024.2}, URN = {urn:nbn:de:0030-drops-209715}, doi = {10.4230/OASIcs.ICPEC.2024.2}, annote = {Keywords: White-box assessment, student assessment, programming education} }
Feedback for Dagstuhl Publishing