autograder.submission

  1import glob
  2import os
  3import subprocess
  4import sys
  5import traceback
  6import typing
  7
  8import edq.util.dirent
  9import edq.util.json
 10import edq.util.serial
 11import edq.util.time
 12
 13import autograder.assignment
 14import autograder.fileop
 15import autograder.filespec
 16
 17TEST_SUBMISSION_FILENAME: str = 'test-submission.json'
 18GRADER_FILENAME: str = 'grader.py'
 19GRADING_RESULT_FILENAME: str = 'result.json'
 20
 21CONFIG_KEY_STATIC_FILES: str = 'static-files'
 22CONFIG_KEY_PRE_STATIC_OPS: str = 'pre-static-file-ops'
 23CONFIG_KEY_POST_STATIC_OPS: str = 'post-static-file-ops'
 24CONFIG_KEY_POST_SUB_OPS: str = 'post-submission-file-ops'
 25
 26INPUT_DIRNAME: str = 'input'
 27OUTPUT_DIRNAME: str = 'output'
 28WORK_DIRNAME: str = 'work'
 29
 30def copy_assignment_files(
 31        source_dir: str,
 32        dest_dir: str,
 33        op_dir: str,
 34        files: typing.List[str],
 35        only_contents: bool = False,
 36        pre_ops: typing.Union[typing.List[autograder.fileop.FileOp], None] = None,
 37        post_ops: typing.Union[typing.List[autograder.fileop.FileOp], None] = None,
 38        ) -> None:
 39    """
 40    Copy over assignment files.
 41
 42    Full procedure::
 43    1) Do pre-copy operations.
 44    2) Copy.
 45    3) Do post-copy operations.
 46    """
 47
 48    if (pre_ops is None):
 49        pre_ops = []
 50
 51    if (post_ops is None):
 52        post_ops = []
 53
 54    # Do pre operations.
 55    autograder.fileop.exec_file_operations(pre_ops, op_dir)
 56
 57    # Copy over the assignment's files.
 58    for filespec_text in files:
 59        spec = autograder.filespec.parse(filespec_text)
 60        autograder.filespec.copy(spec, source_dir, dest_dir, only_contents)
 61
 62    # Do post operations.
 63    autograder.fileop.exec_file_operations(post_ops, op_dir)
 64
 65def fetch_test_submissions(path: str) -> typing.List[str]:
 66    """
 67    Fetch all test submission files (named TEST_SUBMISSION_FILENAME) within the given path,
 68    or the path itself if it is already pointing to a test submission.
 69    """
 70
 71    path = os.path.abspath(path)
 72    test_submissions = []
 73
 74    if (os.path.isfile(path)):
 75        if (os.path.basename(path) != TEST_SUBMISSION_FILENAME):
 76            raise ValueError(f"Passed in submission is not named like a test submission ('{TEST_SUBMISSION_FILENAME}').")
 77
 78        test_submissions.append(path)
 79    else:
 80        test_submissions += glob.glob(os.path.join(path, '**', TEST_SUBMISSION_FILENAME),
 81                recursive = True)
 82
 83    return test_submissions
 84
 85def prep_grading_dir(
 86        assignment_config_path: str,
 87        submission_dir: str,
 88        grading_dir: typing.Union[str, None] = None,
 89        skip_static: bool = False,
 90        ) -> str:
 91    """
 92    Create and return a directory for grading a submission.
 93
 94    Procedure:
 95    1) If the base out dir is None, create a temp dir.
 96    2) Create the three core directories (input/output/work) in the base dir.
 97    3) Copy over the static files (includng pre/post operations).
 98    4) Copy over the submission files (includng pre/post operations).
 99    5) Return the dirs.
100    """
101
102    if (grading_dir is None):
103        grading_dir = edq.util.dirent.get_temp_path(prefix = 'ag-py-submission-')
104
105    edq.util.dirent.mkdir(grading_dir)
106
107    input_dir, _, work_dir = make_core_dirs(grading_dir)
108
109    # Load the assignment config.
110
111    assignment_config_path = os.path.abspath(assignment_config_path)
112    assignment_base_dir = os.path.dirname(assignment_config_path)
113
114    try:
115        assignment_config = edq.util.json.load_path(assignment_config_path)
116    except Exception as ex:
117        raise ValueError("Failed to load assignment config: " + assignment_config_path) from ex
118
119    if (not skip_static):
120        # Copy static files.
121        copy_assignment_files(assignment_base_dir, work_dir, grading_dir,
122                assignment_config.get(CONFIG_KEY_STATIC_FILES, []),
123                pre_ops = assignment_config.get(CONFIG_KEY_PRE_STATIC_OPS, []),
124                post_ops = assignment_config.get(CONFIG_KEY_POST_STATIC_OPS, []))
125
126    # Copy submission files.
127    copy_assignment_files(submission_dir, input_dir, grading_dir,
128        ['.'], only_contents = True,
129        pre_ops = [],
130        post_ops = assignment_config.get(CONFIG_KEY_POST_SUB_OPS, []))
131
132    return grading_dir
133
134def make_core_dirs(base_dir: str) -> typing.Tuple[str, str, str]:
135    """
136    Create and return the three core grading directories (input, output, work).
137    """
138
139    input_dir = os.path.join(base_dir, INPUT_DIRNAME)
140    edq.util.dirent.mkdir(input_dir)
141
142    output_dir = os.path.join(base_dir, OUTPUT_DIRNAME)
143    edq.util.dirent.mkdir(output_dir)
144
145    work_dir = os.path.join(base_dir, WORK_DIRNAME)
146    edq.util.dirent.mkdir(work_dir)
147
148    return input_dir, output_dir, work_dir
149
150def run_test_submission(assignment_config_path: str, submission_config_path: str) -> bool:
151    """ Run a test submission and return if the output matches the expected submission output. """
152
153    print(f"Testing assignment '{assignment_config_path}' and submission '{submission_config_path}'.")
154
155    grading_dir = prep_grading_dir(assignment_config_path, os.path.dirname(submission_config_path))
156
157    old_module_keys = set()
158    try:
159        # Keep track of new top-level keys in sys.modules (imports) after the submission runs.
160        # This is to prevent any import of submission code that gets cached.
161        # This is in no way a complete solution, but also does not matter when run in Docker.
162        old_module_keys = set(sys.modules.keys())
163
164        actual_result = run_submission(grading_dir, assignment_config_path = assignment_config_path)
165    finally:
166        new_module_keys = set(sys.modules.keys())
167        for new_module_key in (new_module_keys - old_module_keys):
168            # Numpy and SciPy are special cases that are sensitive to reloads.
169            # Note that this would be a security concern (e.g., a submission hijacking numpy),
170            # but this is not used in docker-based grading.
171            if (new_module_key.startswith('numpy') or new_module_key.startswith('scipy')):
172                continue
173
174            if (new_module_key in sys.modules):
175                del sys.modules[new_module_key]
176
177    if (actual_result is None):
178        return False
179
180    return compare_test_submission(submission_config_path, actual_result)
181
182def compare_test_submission(
183        test_config_path: str,
184        actual_result: typing.Union[autograder.assignment.GradedAssignment, None],
185        print_result: bool = True,
186        ) -> bool:
187    """
188    Compare a grading result against the expected output of a test submission.
189    Return true if the two match.
190    """
191
192    if (actual_result is None):
193        print(f"Submission is null and cannot match expected output: '{test_config_path}'.")
194        return False
195
196    test_config = edq.util.json.load_path(test_config_path)
197
198    expected_result = autograder.assignment.GradedAssignment.from_dict(test_config['result'])
199    ignore_messages = test_config.get('ignore_messages', False)
200
201    match = actual_result.equals(expected_result, ignore_messages = ignore_messages)
202
203    if ((not match) and print_result):
204        print(f"Submission does not match expected output: '{test_config_path}'.")
205        print('Expected:')
206        print(expected_result.report(prefix = '    '))
207        print('---')
208        print('Actual:')
209        print(actual_result.report(prefix = '    '))
210        print('---')
211
212    return match
213
214def run_submission(
215        grading_dir: str,
216        assignment_config_path: typing.Union[str, None] = None,
217        grader_path: typing.Union[str, None] = None,
218        ) -> typing.Union[autograder.assignment.GradedAssignment, None]:
219    """
220    Run a submission from a pre-populated grading directory and return the result.
221    The grader path (or default grader path) will be checked first for a Python grader (GRADER_FILENAME),
222    which will be run if it exists.
223    Otherwise, the assignment config will be checked for the grader.
224    """
225
226    if (grader_path is None):
227        grader_path = os.path.join(grading_dir, WORK_DIRNAME, GRADER_FILENAME)
228
229    if (os.path.exists(grader_path)):
230        return run_python_grader(grader_path, grading_dir)
231
232    if (assignment_config_path is None):
233        raise ValueError("No assignment config path has been supplied for running a grader.")
234
235    return run_external_grader(assignment_config_path, grading_dir)
236
237def run_python_grader(grader_path: str, grading_dir: str) -> typing.Union[autograder.assignment.GradedAssignment, None]:
238    """
239    Run a standard Python-based grader.
240    Returns None on grading failure.
241    """
242
243    input_dir = os.path.join(grading_dir, INPUT_DIRNAME)
244    output_dir = os.path.join(grading_dir, OUTPUT_DIRNAME)
245    work_dir = os.path.join(grading_dir, WORK_DIRNAME)
246
247    edq.util.dirent.mkdir(work_dir)
248
249    # Ensure that the current directory is in sys.path.
250    sys.path.insert(0, '.')
251
252    # Move into the work dir for grading and back after.
253    start_dir = os.getcwd()
254
255    try:
256        os.chdir(work_dir)
257
258        assignment_class = autograder.assignment.fetch_assignment_class(grader_path)
259        if (assignment_class is None):
260            print("Failed to fetch assignment class from '{grader_path}'.")
261            return None
262
263        assignment = assignment_class(input_dir = input_dir, output_dir = output_dir,
264                work_dir = work_dir)
265        return assignment.grade()
266    except Exception:
267        print("Failed to run assignment ('{assignment_class.__name__}') on submission '{input_dir}': '{ex}'.")
268        traceback.print_exc()
269        return None
270    finally:
271        os.chdir(start_dir)
272        sys.path.pop(0)
273
274    return None
275
276def run_external_grader(assignment_config_path: str, grading_dir: str) -> autograder.assignment.GradedAssignment:
277    """ Run a grader that is not a standard Python-based grader. """
278
279    work_dir = os.path.join(grading_dir, WORK_DIRNAME)
280    output_dir = os.path.join(grading_dir, OUTPUT_DIRNAME)
281
282    edq.util.dirent.mkdir(work_dir)
283
284    # Move into the work dir for grading and back after.
285    start_dir = os.getcwd()
286
287    try:
288        os.chdir(work_dir)
289
290        assignment_config = edq.util.json.load_path(assignment_config_path)
291    except Exception as ex:
292        raise ValueError("Failed to load assignment config: " + assignment_config_path) from ex
293    finally:
294        os.chdir(start_dir)
295
296    invocation = assignment_config.get('invocation', [])
297    if (len(invocation) == 0):
298        raise ValueError(("External (any grader not using the standard Python setup)"
299            + " graders must have a non-empty 'invocation' key in their assignment config."))
300
301    subprocess.run(invocation, cwd = work_dir, check = True)
302
303    out_path = os.path.join(output_dir, GRADING_RESULT_FILENAME)
304    if (not os.path.isfile(out_path)):
305        raise ValueError(f"Could not find result after external grader ran: '{out_path}'.")
306
307    return autograder.assignment.GradedAssignment.from_path(out_path)
308
309class SubmissionSummary(edq.util.serial.DictConverter):
310    """
311    A summary of a grading submission.
312    """
313
314    def __init__(self,
315            id: str = '',
316            max_points: float = 0,
317            score: float = 0,
318            message: str = '',
319            grading_start_time: typing.Union[edq.util.time.Timestamp, int, None] = None,
320            **kwargs: typing.Any):
321        self.id: str = id
322        """ An identifier for this submission. """
323
324        self.max_points: float = max_points
325        """ The maximum number of points possible for this assignment (excluding extra credit). """
326
327        self.score: float = score
328        """ The score earned for this submission. """
329
330        self.message: str = message
331        """ A message/feedback for the student. """
332
333        if (grading_start_time is None):
334            grading_start_time = edq.util.time.Timestamp()
335
336        self.grading_start_time: typing.Any = edq.util.time.Timestamp(grading_start_time)
337        """ When grading started. """
338
339    def short_id(self) -> str:
340        """ Get the short ID for this submission. """
341
342        return self.id.split('::')[-1]
343
344    def __repr__(self) -> str:
345        message = '.'
346        if ((self.message is not None) and (self.message != '')):
347            message = f", Message: '{self.message}'."
348
349        return f"Submission ID: {self.short_id()}, Score: {self.score} / {self.max_points}, Time: {self.grading_start_time.pretty()}{message}"