autograder.cli.testing.test-submissions

 1# pylint: disable=invalid-name
 2
 3"""
 4Run a grader against multiple test assignments and ensure the output matches the expected output.
 5"""
 6
 7import argparse
 8import sys
 9import traceback
10
11import autograder.cli.parser
12import autograder.submission
13
14DEFAULT_ASSIGNMENT = 'assignment.json'
15
16def run_cli(args: argparse.Namespace) -> int:
17    """ Run the CLI. """
18
19    try:
20        test_submissions = autograder.submission.fetch_test_submissions(args.submissions)
21    except Exception as ex:
22        print(f"Failed to load submission(s) from '{args.submissions}': '{ex}'.")
23        traceback.print_exc()
24        return 101
25
26    errors = 0
27    for test_submission in sorted(test_submissions):
28        success = False
29
30        try:
31            success = autograder.submission.run_test_submission(args.assignment, test_submission)
32        except Exception as ex:
33            print(f"Failed to run submission '{test_submission}': '{ex}'.")
34            traceback.print_exc()
35
36        if (not success):
37            errors += 1
38
39    print(f"Encountered {errors} error(s) while testing {len(test_submissions)} submissions.")
40
41    if (errors > 0):
42        print("Faiure")
43    else:
44        print("Success")
45
46    return errors
47
48def main() -> int:
49    """ Get a parser, parse the args, and call run. """
50    return run_cli(_get_parser().parse_args())
51
52def _get_parser() -> argparse.ArgumentParser:
53    parser = autograder.cli.parser.get_parser(__doc__.strip())
54
55    parser.add_argument('-a', '--assignment',
56        action = 'store', type = str, required = False, default = DEFAULT_ASSIGNMENT,
57        help = 'The path to a JSON file describing an assignment (default: %(default)s).')
58
59    parser.add_argument('-s', '--submissions',
60        action = 'store', type = str, required = True,
61        help = 'The path to a dir containing one or more test submissions.')
62
63    return parser
64
65if (__name__ == '__main__'):
66    sys.exit(main())