autograder.assignment

  1import inspect
  2import traceback
  3import typing
  4
  5import edq.util.code
  6import edq.util.serial
  7
  8import autograder.question
  9import autograder.util.prepare_submission
 10
 11class GradedAssignment(edq.util.serial.DictConverter):
 12    """
 13    The result of an assignment being graded with a submission.
 14    """
 15
 16    def __init__(self,
 17            name: str,
 18            questions: typing.Sequence[autograder.question.GradedQuestion],
 19            message: typing.Union[str, None] = None,
 20            prologue: typing.Union[str, None] = None,
 21            epilogue: typing.Union[str, None] = None,
 22            grading_start_time: typing.Union[edq.util.time.Timestamp, int, None] = None,
 23            grading_end_time: typing.Union[edq.util.time.Timestamp, int, None] = None,
 24            proxy_start_time: typing.Union[edq.util.time.Timestamp, int, None] = None,
 25            proxy_end_time: typing.Union[edq.util.time.Timestamp, int, None] = None,
 26            **kwargs: typing.Any) -> None:
 27        self.name: str = name
 28        """ The name of the assignment. """
 29
 30        self.questions: typing.List[autograder.question.GradedQuestion] = list(questions)
 31        """ The result of grading for each question. """
 32
 33        self.message: typing.Union[str, None] = prologue
 34        """ Text to include with the grading result. """
 35
 36        self.prologue: typing.Union[str, None] = prologue
 37        """ Text to include before the grading result. """
 38
 39        self.epilogue: typing.Union[str, None] = epilogue
 40        """ Text to include after the grading result. """
 41
 42        if (grading_start_time is None):
 43            grading_start_time = edq.util.time.Timestamp()
 44
 45        self.grading_start_time: typing.Any = edq.util.time.Timestamp(grading_start_time)
 46        """ When grading started. """
 47
 48        if (grading_end_time is None):
 49            grading_end_time = edq.util.time.Timestamp()
 50
 51        self.grading_end_time: typing.Any = edq.util.time.Timestamp(grading_end_time)
 52        """ When grading ended. """
 53
 54        if (proxy_start_time is None):
 55            proxy_start_time = edq.util.time.Timestamp()
 56
 57        self.proxy_start_time: typing.Any = edq.util.time.Timestamp(proxy_start_time)
 58        """ When proxy grading started. """
 59
 60        if (proxy_end_time is None):
 61            proxy_end_time = edq.util.time.Timestamp()
 62
 63        self.proxy_end_time: typing.Any = edq.util.time.Timestamp(proxy_end_time)
 64        """ When proxy grading ended. """
 65
 66    def to_test_submission(self, options: typing.Union[typing.Dict[str, typing.Any], None] = None) -> typing.Dict[str, typing.Any]:
 67        """
 68        Output a dict that can be used as a test submission.
 69        """
 70
 71        if (options is None):
 72            options = {}
 73
 74        results = self.to_dict()
 75
 76        del results['grading_start_time']
 77        del results['grading_end_time']
 78        del results['proxy_start_time']
 79        del results['proxy_end_time']
 80
 81        questions = typing.cast(typing.List[typing.Dict[str, edq.util.serial.PODType]], results.get('questions', []))
 82
 83        for question in questions:
 84            del question['grading_start_time']
 85            del question['grading_end_time']
 86
 87        test_submission = {
 88            'result': results,
 89        }
 90
 91        test_submission.update(options)
 92
 93        return test_submission
 94
 95    def get_score(self) -> typing.Tuple[float, float]:
 96        """
 97        Return (total score, max score).
 98        """
 99
100        total_score: float = 0
101        max_points: float = 0
102
103        for question in self.questions:
104            total_score += question.score
105            max_points += question.max_points
106
107        return (total_score, max_points)
108
109    def report(self, prefix: str = '', precision: int = 2) -> str:
110        """
111        Return a string representation of the grading for this assignment.
112        """
113
114        if ((prefix != '') and (not prefix.endswith(' '))):
115            prefix += ' '
116
117        output: typing.List[str] = []
118
119        output += self._format_logue(self.prologue, prefix)
120
121        output += [
122            prefix + f"Autograder transcript for assignment: {self.name}",
123            prefix + f"Grading started at {self.grading_start_time.pretty()} and ended at {self.grading_end_time.pretty()}.",
124        ]
125
126        if ((self.message is not None) and (len(self.message) > 0)):
127            output.append(f"\n{self.message}\n")
128
129        total_score: float = 0
130        total_max_points: float = 0
131
132        for question in self.questions:
133            total_score += question.score
134            total_max_points += question.max_points
135
136            output.append(question.scoring_report(prefix = prefix, precision = precision))
137
138        total_score_str = autograder.util.math.number_to_str(total_score, precision = precision)
139        total_max_points_str = autograder.util.math.number_to_str(total_max_points, precision = precision)
140
141        output.append('')
142        output.append(f"{prefix}Total: {total_score_str} / {total_max_points_str}")
143
144        output += self._format_logue(self.epilogue, prefix)
145
146        return "\n".join(output)
147
148    def _format_logue(self, text: typing.Union[str, None], prefix: str) -> typing.List[str]:
149        """ Format the output logue (prologe/epilogue). """
150
151        if ((text is None) or (text == '')):
152            return []
153
154        lines = text.splitlines()
155        return [prefix + line.rstrip() for line in lines]
156
157    def __eq__(self, other: object) -> bool:
158        return self.equals(other)
159
160    def equals(self, other: object, **kwargs: typing.Any) -> bool:
161        """ Check two graded assignments for equality. """
162
163        if (not isinstance(other, GradedAssignment)):
164            return False
165
166        if ((self.name != other.name) or (len(self.questions) != len(other.questions))):
167            return False
168
169        for (i, question) in enumerate(self.questions):
170            if (not question.equals(other.questions[i], **kwargs)):
171                return False
172
173        return True
174
175class Assignment:
176    """
177    A collection of questions to be scored.
178    """
179
180    def __init__(self,
181            name: typing.Union[str, None] = None,
182            questions: typing.Union[typing.Sequence[autograder.question.Question], None] = None,
183            input_dir: str = '.',
184            output_dir: str = '.',
185            work_dir: str = '.',
186            prep_submission: bool = True,
187            additional_data: typing.Union[typing.Dict[str, typing.Any], None] = None,
188            **kwargs: typing.Any) -> None:
189        if (name is None):
190            name = type(self).__name__
191
192        self.name: str = name
193        """
194        The display name for this assignment.
195        Defaults to the (unqualified) name of this class.
196        """
197
198        if (questions is None):
199            questions = []
200
201        self.questions = list(questions)
202        """ The questions for this assignment. """
203
204        self.input_dir: str = input_dir
205        """ The base directory that houses student inputs for this assignment. """
206
207        self.output_dir: str = output_dir
208        """
209        The base directory where output will be written for this assignment.
210        A successful grading should result in a `result.json` created in this directory.
211        """
212
213        self.work_dir: str = work_dir
214        """ The base directory that the grader is run in. """
215
216        self.prep_submission: bool = prep_submission
217        """ Whether or not to call self.prepare_submission() to prepare the input directory before grading. """
218
219        if (additional_data is None):
220            additional_data = {}
221
222        self.additional_data: typing.Dict[str, typing.Any] = additional_data
223        """ Additional data that can be passed to the grader. """
224
225        self.result: typing.Union[GradedAssignment, None] = None
226        """ The result of grading. """
227
228    def grade(self, **kwargs: typing.Any) -> GradedAssignment:
229        """ Grade this assignment. """
230
231        try:
232            return self._grade_submission(self._prepare_submission(), **kwargs)
233        except Exception:
234            now = edq.util.time.Timestamp.now()
235
236            questions = []
237            for question in self.questions:
238                questions.append(autograder.question.GradedQuestion(
239                    name = question.name,
240                    max_points = question.max_points,
241                    score = 0,
242                    message = "Submission could not be graded.",
243                    grading_start_time = now,
244                    grading_end_time = now))
245
246            epilogue = (f"\nSubmission could not be graded because of the following error:\n---\n{traceback.format_exc()}---")
247
248            return GradedAssignment(
249                name = self.name,
250                questions = questions,
251                grading_start_time = now,
252                grading_end_time = now,
253                epilogue = epilogue)
254
255    def _grade_submission(self,
256            submission: typing.Union[object, None],
257            show_exceptions: bool = False,
258            **kwargs: typing.Any) -> GradedAssignment:
259        """
260        Grade an assignment by grading all the questions.
261
262        The submission argument is the result of _prepare_submission().
263        """
264
265        self.result = GradedAssignment(name = self.name, questions = [])
266        self.result.grading_start_time = edq.util.time.Timestamp.now()
267
268        stop_grading = False
269        for question in self.questions:
270            if (stop_grading):
271                now = edq.util.time.Timestamp.now()
272
273                self.result.questions.append(autograder.question.GradedQuestion(
274                    name = question.name,
275                    max_points = question.max_points,
276                    score = 0,
277                    message = "Grading stopped because of a hard error, skipping question...",
278                    grading_start_time = now,
279                    grading_end_time = now,
280                    skipped = True))
281            else:
282                result = question.grade(submission,
283                    additional_data = self.additional_data,
284                    show_exceptions = show_exceptions)
285
286                self.result.questions.append(result)
287
288                stop_grading = result.hard_fail
289
290        self.result.grading_end_time = edq.util.time.Timestamp.now()
291
292        return self.result
293
294    def _prepare_submission(self) -> typing.Union[object, None]:
295        """
296        Prepare the submission in the input directory for grading.
297        The result of this is what will be passed to grade().
298        Child classes may leave this default behavior or override.
299
300        This implementation will check the prep_submission argument passed in the constructor.
301        If true the input directory will be prepared, otherwise None will be returned.
302        """
303
304        if (self.prep_submission):
305            return autograder.util.prepare_submission.prepare(self.input_dir)
306
307        return None
308
309def load_assignment_classes(path: str) -> typing.List[typing.Type[Assignment]]:
310    """
311    Recursively load all the assignment classes in a path (file or dir).
312    """
313
314    module = edq.util.code.sanitize_and_import_path(path)
315    assignments = []
316
317    for name in dir(module):
318        obj = getattr(module, name)
319
320        if (not inspect.isclass(obj)):
321            continue
322
323        if (obj == autograder.assignment.Assignment):
324            continue
325
326        if (issubclass(obj, autograder.assignment.Assignment)):
327            assignments.append(obj)
328
329    return assignments
330
331def fetch_assignment_class(path: str) -> typing.Type[Assignment]:
332    """
333    Recursively fetch a single assignment class from a path.
334    If exactly one assignment is not found, then raise an error.
335    """
336
337    assignments = load_assignment_classes(path)
338
339    if (len(assignments) == 0):
340        raise ValueError(f"Assignment file ('{path}') does not contain any instances of autograder.assignment.Assignment.")
341
342    if (len(assignments) > 1):
343        raise ValueError(f"Assignment file ('{path}') contains more than one ({len(assignments)}) instances of autograder.assignment.Assignment.")
344
345    return assignments[0]