autograder.cli.testing.test-remote-submissions

 1# pylint: disable=invalid-name
 2
 3"""
 4Submit multiple assignments to an autograder and ensure the output is as expected.
 5"""
 6
 7import argparse
 8import os
 9import sys
10import traceback
11import typing
12
13import autograder.cli.courses.assignments.submissions.common
14import autograder.api.courses.assignments.submissions.submit
15import autograder.cli.parser
16import autograder.submission
17
18def run_cli(args: argparse.Namespace) -> int:
19    """ Run the CLI. """
20
21    config = args._config_info.application_config
22
23    try:
24        test_submission_paths = autograder.submission.fetch_test_submissions(args.submissions)
25    except Exception as ex:
26        print(f"Failed to load submission(s) from '{args.submissions}': '{ex}'.")
27        traceback.print_exc()
28        return 101
29
30    errors = 0
31    for test_submission_path in sorted(test_submission_paths):
32        paths = _get_files(test_submission_path)
33
34        try:
35            api_response, grading_result = autograder.api.courses.assignments.submissions.submit.send(config,
36                    post_paths = paths, exit_on_error = True)
37        except Exception as ex:
38            print(f"Failed to run submission '{test_submission_path}': '{ex}'.")
39            traceback.print_exc()
40            errors += 1
41            continue
42
43        if (not api_response['grading-success']):
44            print("Autograder failed to grade the submission.")
45            errors += 1
46            continue
47
48        if (not autograder.submission.compare_test_submission(test_submission_path, grading_result)):
49            errors += 1
50
51    print(f"Encountered {errors} error(s) while testing {len(test_submission_paths)} submissions.")
52
53    if (errors > 0):
54        print("Faiure")
55    else:
56        print("Success")
57
58    return errors
59
60def _get_files(test_submission_path: str) -> typing.List[str]:
61    """ Collect the submission file paths. """
62
63    paths = []
64
65    test_submission_path = os.path.abspath(test_submission_path)
66
67    submission_dir = os.path.dirname(test_submission_path)
68    for dirent in os.listdir(submission_dir):
69        path = os.path.join(submission_dir, dirent)
70        if (not os.path.samefile(test_submission_path, path)):
71            paths.append(path)
72
73    return paths
74
75def main() -> int:
76    """ Get a parser, parse the args, and call run. """
77
78    return run_cli(_get_parser().parse_args())
79
80def _get_parser() -> argparse.ArgumentParser:
81    """ Get a parser for this operation. """
82
83    parser = autograder.cli.parser.get_parser(
84        __doc__.strip(),
85        autograder.api.courses.assignments.submissions.submit.BASE_API_PARAMS)
86
87    parser.add_argument('submissions', metavar = 'SUBMISSIONS_DIR',
88        action = 'store', type = str,
89        help = 'The path to a dir containing one or more test submissions.')
90
91    return parser
92
93if (__name__ == '__main__'):
94    sys.exit(main())