quizcomp.uploader.gradescope

  1import logging
  2import os
  3import re
  4import typing
  5
  6import bs4
  7import edq.util.dirent
  8import edq.util.json
  9import requests
 10
 11import quizcomp.model.constants
 12import quizcomp.model.quiz
 13
 14_logger = logging.getLogger(__name__)
 15
 16URL_BASE: str = 'https://www.gradescope.com'
 17URL_HOMEPAGE = URL_BASE
 18URL_LOGIN: str = f"{URL_BASE}/login"
 19URL_ASSIGNMENTS: str = f"{URL_BASE}/courses/%s/assignments"
 20URL_ASSIGNMENT: str = f"{URL_BASE}/courses/%s/assignments/%s"
 21URL_ASSIGNMENT_GROUP: str = f"{URL_BASE}/courses/%s/assignment_containers"
 22URL_ASSIGNMENT_EDIT: str = f"{URL_BASE}/courses/%s/assignments/%s/edit"
 23URL_ASSIGNMENT_RUBRIC: str = f"{URL_BASE}/courses/%s/assignments/%s/rubric/edit"
 24URL_ASSIGNMENT_ADD_RUBRIC_ITEM: str = f"{URL_BASE}/courses/%s/questions/%s/rubric_items"
 25URL_NEW_ASSIGNMENT_FORM: str = f"{URL_BASE}/courses/%s/assignments/new"
 26URL_EDIT_OUTLINE: str = f"{URL_BASE}/courses/%s/assignments/%s/outline/edit"
 27URL_PATCH_OUTLINE: str = f"{URL_BASE}/courses/%s/assignments/%s/outline"
 28
 29NAME_BOX_ID: str = 'name'
 30ID_BOX_ID: str = 'id'
 31SIGNATURE_BOX_ID: str = 'signature'
 32MANUAL_GRADING_BOX_ID: str = 'manual_grading'
 33
 34SPECIAL_QUESTION_TYPES: typing.List[str] = [
 35    NAME_BOX_ID,
 36    ID_BOX_ID,
 37    SIGNATURE_BOX_ID,
 38    MANUAL_GRADING_BOX_ID,
 39]
 40
 41EXTEND_BOX_QUESTION_TYPES: typing.List[str] = [
 42    quizcomp.model.constants.QuestionType.MA.value,
 43    quizcomp.model.constants.QuestionType.MCQ.value,
 44    quizcomp.model.constants.QuestionType.MDD.value,
 45    quizcomp.model.constants.QuestionType.TF.value,
 46]
 47
 48STANDARD_BOX_QUESTION_TYPES: typing.List[str] = [
 49    quizcomp.model.constants.QuestionType.ESSAY.value,
 50    quizcomp.model.constants.QuestionType.FIMB.value,
 51    quizcomp.model.constants.QuestionType.FITB.value,
 52    quizcomp.model.constants.QuestionType.MATCHING.value,
 53    quizcomp.model.constants.QuestionType.NUMERICAL.value,
 54    quizcomp.model.constants.QuestionType.SA.value,
 55]
 56
 57BOX_TYPES: typing.List[str] = EXTEND_BOX_QUESTION_TYPES + STANDARD_BOX_QUESTION_TYPES + SPECIAL_QUESTION_TYPES
 58
 59SP_PER_PT: int = 65536
 60
 61SESSION_ID_CREATE_ASSIGNMENT_GROUP: str = 'gradescope_create_assignment_group'
 62SESSION_ID_UPLOAD: str = 'gradescope_upload'
 63
 64class GradeScopeUploader:
 65    """
 66    Upload quizzes to GradeScope.
 67    """
 68
 69    def __init__(self,
 70            course_id: str,
 71            user: str,
 72            password: str,
 73            force: bool = False,
 74            rubric: bool = False,
 75            save_http: bool = False,
 76            **kwargs: typing.Any) -> None:
 77        super().__init__(**kwargs)
 78
 79        self.course_id: str = course_id
 80        """ The GradeScope course ID. """
 81
 82        self.user: str = user
 83        """ The GradeScope user/email to authenticate with. """
 84
 85        self.password: str = password
 86        """ The GradeScope password to authenticate with. """
 87
 88        self.force: bool = force
 89        """ Whether to remove existing quizzes before upload. """
 90
 91        self.rubric: bool = rubric
 92        """ Whether to upload the rubric items for the quiz. """
 93
 94        self.save_http = save_http
 95        """ Whether to save the HTTP exchanges to disk. """
 96
 97    def upload_quiz(self,
 98            variant: quizcomp.model.quiz.Variant,
 99            base_dir: typing.Union[str, None] = None,
100            **kwargs: typing.Any,
101            ) -> typing.Tuple[str, bool]:
102        """
103        Compile a quiz and upload it to GradeScope.
104
105        If base_dir is left None, then a temp dir will be created which will be destroyed on exit.
106        If supplied, the base_dir will be left alone when finished.
107        """
108
109        if (not isinstance(variant, quizcomp.model.quiz.Variant)):
110            raise ValueError(f"GradeScope quiz uploader requires a quizcomp.model.quiz.Variant type, found {type(variant)}.")
111
112        if (base_dir is None):
113            base_dir = edq.util.dirent.get_temp_path(prefix = 'quizcomp-gradescope-')
114
115        boxes, special_boxes = self.get_bounding_boxes(variant, base_dir)
116        return self.upload(variant, base_dir, boxes, special_boxes)
117
118    def create_assignment_group(self, title: str, gradescope_ids: typing.List[str]) -> None:
119        """ Create an assignment group with the given IDs. """
120
121        session = requests.Session()
122
123        self.login(session)
124
125        assignments_url = URL_ASSIGNMENTS % (self.course_id)
126        csrf_token = self.get_csrf_token(session, assignments_url)
127
128        headers = {
129            'x-csrf-token': csrf_token,
130        }
131
132        post_url = URL_ASSIGNMENT_GROUP % (self. course_id)
133        data = {
134            'title': title,
135            'assignment_ids[]': gradescope_ids,
136        }
137
138        response = session.post(post_url, params = data, headers = headers)  # type: ignore[arg-type]
139        response.raise_for_status()
140
141    def get_bounding_boxes(self,
142            variant: quizcomp.model.quiz.Variant,
143            base_dir: str,
144            ) -> typing.Tuple[typing.Dict[str, typing.Dict[str, typing.Any]], typing.Dict[str, typing.Dict[str, typing.Any]]]:
145        """ Get bounding boxes for answers. """
146
147        # {<quetion id>: {<part id>: box, ...}, ...}
148        boxes: typing.Dict[str, typing.Dict[str, typing.Any]] = {}
149        # {NAME_BOX_ID: box, ID_BOX_ID: box, SIGNATURE_BOX_ID: box}
150        special_boxes: typing.Dict[str, typing.Dict[str, typing.Any]] = {}
151
152        path = os.path.join(base_dir, f"{variant.get_name()}.pos")
153        if (not os.path.exists(path)):
154            raise ValueError(f"Could not find path for quiz bounding boxes: '{path}'.")
155
156        with open(path, 'r', encoding = edq.util.dirent.DEFAULT_ENCODING) as file:
157            for line in file:
158                line = line.strip()
159                if (line == ""):
160                    continue
161
162                parts = [part.strip() for part in line.split(',')]
163                if (len(parts) != 12):
164                    raise ValueError(f"Position file has row with bad number of parts. Expecting 11, found {len(parts)}.")
165
166                # "ll" == "lower-left"
167                # "ur" == "upper-right"
168                (question_id, part_id, _, raw_question_type, raw_page_number, ll_x, ll_y, ur_x, ur_y, page_width, page_height, origin) = parts
169
170                if (origin != 'bottom-left'):
171                    raise ValueError(f"Unknown bounding box origin: '{origin}'.")
172
173                # Note that the position file and GradeScope use 1-indexed pages.
174                page_number = int(raw_page_number)
175
176                if (raw_question_type not in BOX_TYPES):
177                    raise ValueError(f"Unknown content type: '{raw_question_type}'.")
178
179                extend_box_right = False
180                if (raw_question_type in EXTEND_BOX_QUESTION_TYPES):
181                    extend_box_right = True
182
183                    # There is a special case for inline MA questions.
184                    if (raw_question_type == quizcomp.model.constants.QuestionType.MA.value):
185                        extend_box_right = False
186
187                (x1, y1), (x2, y2) = self._compute_box(ll_x, ll_y, ur_x, ur_y, page_width, page_height, extend_box_right = extend_box_right)
188
189                if (raw_question_type in SPECIAL_QUESTION_TYPES):
190                    # These boxes are special.
191                    if (raw_question_type in special_boxes):
192                        raise ValueError(f"Multiple {raw_question_type} bounding boxes found.")
193
194                    special_boxes[raw_question_type] = {
195                        'page_number': page_number,
196                        'x1': x1,
197                        'y1': y1,
198                        'x2': x2,
199                        'y2': y2,
200                    }
201
202                    continue
203
204                if (question_id not in boxes):
205                    boxes[question_id] = {}
206
207                # If there is an existing box, extend it.
208                if (part_id in boxes[question_id]):
209                    old_box = boxes[question_id][part_id]
210
211                    old_x1 = old_box['x1']
212                    old_y1 = old_box['y1']
213                    old_x2 = old_box['x2']
214                    old_y2 = old_box['y2']
215
216                    old_page = old_box['page_number']
217                    if (old_page != page_number):
218                        raise ValueError(f"Question {question_id} has bounding boxes that span pages.")
219
220                    x1 = min(x1, old_x1)
221                    y1 = min(y1, old_y1)
222                    x2 = max(x2, old_x2)
223                    y2 = max(y2, old_y2)
224
225                boxes[question_id][part_id] = {
226                    'page_number': page_number,
227                    'x1': x1,
228                    'y1': y1,
229                    'x2': x2,
230                    'y2': y2,
231                }
232
233        return boxes, special_boxes
234
235    def _compute_box(self,
236            raw_ll_x: str,
237            raw_ll_y: str,
238            raw_ur_x: str,
239            raw_ur_y: str,
240            raw_page_width: str,
241            raw_page_height: str,
242            extend_box_right: bool = False,
243            ) -> typing.Tuple[typing.Tuple[float, float], typing.Tuple[float, float]]:
244        """ Compute a bounding box. """
245
246        ll_x = float(raw_ll_x.removesuffix('sp'))
247        ll_y = float(raw_ll_y.removesuffix('sp'))
248        ur_x = float(raw_ur_x.removesuffix('sp'))
249        ur_y = float(raw_ur_y.removesuffix('sp'))
250
251        page_width = float(raw_page_width.removesuffix('pt')) * SP_PER_PT
252        page_height = float(raw_page_height.removesuffix('pt')) * SP_PER_PT
253
254        # Origin is upper-left, point 1 is upper-left, point 2 is lower-right.
255        x1 = round(100.0 * (ll_x / page_width), 1)
256        y1 = round(100.0 * (1.0 - (ur_y / page_height)), 1)
257        # The lower right x should always extend (at least) to the end of the page (to capture the answers).
258        x2 = round(100.0 * (ur_x / page_width), 1)
259        y2 = round(100.0 * (1.0 - (ll_y / page_height)), 1)
260
261        if (extend_box_right):
262            # In some question types, we want to extend (at least) to the end of the page (to capture the answers).
263            x2 = max(95.0, x2)
264
265        return (x1, y1), (x2, y2)
266
267    def create_outline(self,
268            variant: quizcomp.model.quiz.Variant,
269            bounding_boxes: typing.Dict[str, typing.Dict[str, typing.Any]],
270            special_boxes: typing.Dict[str, typing.Dict[str, typing.Any]],
271            ) -> typing.Dict[str, typing.Any]:
272        """ Create GradeScope outline data. """
273
274        question_data = []
275        for (question_id, parts) in bounding_boxes.items():
276            (_, group_index, question_index) = map(int, question_id.split('.'))
277            question = variant.children[group_index].children[question_index]
278
279            if (len(parts) == 1):
280                # Single-part question.
281                question_data.append({
282                    'title': question.get_name(),
283                    'weight': question.get_points(),
284                    'crop_rect_list': list(parts.values()),
285                })
286            else:
287                # Multi-part question.
288                children = []
289                for (part_id, box) in parts.items():
290                    children.append({
291                        'title': f"{question.get_name()} - {part_id}",
292                        'weight': round(question.get_points() / len(parts), 2),
293                        'crop_rect_list': [box],
294                    })
295
296                question_data.append({
297                    'title': question.get_name(),
298                    'weight': question.get_points(),
299                    # The top-level question just needs one of the bounding boxes.
300                    'crop_rect_list': [list(parts.values())[0]],
301                    'children': children,
302                })
303
304        name_box = None
305        id_box = None
306
307        for (question_type, box) in special_boxes.items():
308            if (question_type == NAME_BOX_ID):
309                name_box = special_boxes[NAME_BOX_ID]
310            elif (question_type == ID_BOX_ID):
311                id_box = special_boxes[ID_BOX_ID]
312            else:
313                question_data.append({
314                    'title': question_type,
315                    'weight': 0,
316                    'crop_rect_list': [box],
317                })
318
319        outline = {
320            'assignment': {
321                'identification_regions': {
322                    'name': name_box,
323                    'sid': id_box
324                }
325            },
326            'question_data': question_data,
327        }
328
329        return outline
330
331    def upload(self,
332            variant: quizcomp.model.quiz.Variant,
333            base_dir: str,
334            bounding_boxes: typing.Dict[str, typing.Dict[str, typing.Any]],
335            special_boxes: typing.Dict[str, typing.Dict[str, typing.Any]],
336            ) -> typing.Tuple[str, bool]:
337        """ Upload a variant. """
338
339        path = os.path.join(base_dir, f"{variant.get_name()}.pdf")
340        if (not os.path.exists(path)):
341            raise ValueError(f"Could not find path for quiz pdf: '{path}'.")
342
343        outline = self.create_outline(variant, bounding_boxes, special_boxes)
344
345        session = requests.Session()
346
347        self.login(session)
348        _logger.debug("Logged in as '%s'.", self.user)
349
350        assignment_id = self.get_assignment_id(session, variant)
351        if (assignment_id is not None):
352            if (not self.force):
353                _logger.info("Assignment '%s' (%s) already exists. Skipping upload.", variant.get_name(), assignment_id)
354                return assignment_id, False
355
356            self.delete_assignment(session, assignment_id)
357            _logger.debug("Deleted assignment: '%s'", assignment_id)
358
359        assignment_id = self.create_assignment(session, variant, base_dir)
360        _logger.debug("Created assignment: '%s'", assignment_id)
361
362        self.submit_outline(session, assignment_id, outline)
363        _logger.debug('Submitted outline.')
364
365        if (self.rubric):
366            self.create_rubric(session, assignment_id, variant)
367            _logger.debug('Created assignment rubric.')
368
369        return assignment_id, True
370
371    def login(self, session: requests.Session) -> None:
372        """ Login to GradeScope. """
373
374        token = self.get_authenticity_token(session, URL_HOMEPAGE, action = '/login')
375
376        data = {
377            'utf8': '✓',
378            'session[email]': self.user,
379            'session[password]': self.password,
380            'session[remember_me]': 0,
381            'commit': 'Log+In',
382            'session[remember_me_sso]': 0,
383            'authenticity_token': token,
384        }
385
386        # Login.
387        response = session.post(URL_LOGIN, params = data)  # type: ignore[arg-type]
388        response.raise_for_status()
389
390    def get_authenticity_token(self, session: requests.Session, url: str, action: typing.Union[str, None] = None) -> str:
391        """ Get a token from GradeScope. """
392
393        response = session.get(url)
394        response.raise_for_status()
395
396        document = bs4.BeautifulSoup(response.text, 'html.parser')
397
398        form_selector = 'form'
399        if (action is not None):
400            form_selector = f"form[action='{action}']"
401
402        auth_input = document.select(f"{form_selector} input[name='authenticity_token']")
403        if (len(auth_input) != 1):
404            raise ValueError(f"Did not find exactly one authentication token input, found {len(auth_input)}.")
405
406        return str(auth_input[0].get('value'))
407
408    def get_csrf_token(self, session: requests.Session, url: str) -> str:
409        """ Get a CSRF token. """
410
411        # Get outline submission csrf token.
412        response = session.get(url)
413        response.raise_for_status()
414
415        document = bs4.BeautifulSoup(response.text, 'html.parser')
416        return self.parse_csrf_token(document)
417
418    def parse_csrf_token(self, document: bs4.BeautifulSoup) -> str:
419        """ Parse a CSRF token from a page. """
420
421        meta_tag = document.select('meta[name="csrf-token"]')
422        if (len(meta_tag) != 1):
423            raise ValueError(f"Did not find exactly one CSRF meta tag, found {len(meta_tag)}.")
424
425        return str(meta_tag[0].get('content'))
426
427    def get_assignment_id(self, session: requests.Session, variant: quizcomp.model.quiz.Variant) -> typing.Union[str, None]:
428        """ Get an assignment ID for the given variant, or None if there is no matching variant. """
429
430        url = URL_ASSIGNMENTS % (self.course_id)
431
432        response = session.get(url)
433        response.raise_for_status()
434
435        document = bs4.BeautifulSoup(response.text, 'html.parser')
436
437        nodes = document.select('div[data-react-class="AssignmentsTable"]')
438        if (len(nodes) != 1):
439            raise ValueError(f"Did not find exactly one assignments table, found {len(nodes)}.")
440
441        assignment_data = edq.util.json.loads(str(nodes[0].get('data-react-props')))
442
443        for row in assignment_data['table_data']:
444            if (row['type'] != 'assignment'):
445                continue
446
447            id = str(row['id']).strip().removeprefix('assignment_')
448            name = row['title'].strip()
449
450            if (name == variant.get_name()):
451                return id
452
453        return None
454
455    def delete_assignment(self, session: requests.Session, assignment_id: str) -> None:
456        """ Delete an assignment from GradeScope. """
457
458        form_url = URL_ASSIGNMENT_EDIT % (self.course_id, assignment_id)
459        token = self.get_csrf_token(session, form_url)
460
461        data = {
462            '_method': 'delete',
463            'authenticity_token': token,
464        }
465
466        delete_url = URL_ASSIGNMENT % (self.course_id, assignment_id)
467        response = session.post(delete_url, data = data)
468        response.raise_for_status()
469
470    def create_assignment(self, session: requests.Session, variant: quizcomp.model.quiz.Variant, base_dir: str) -> str:
471        """ Create an assignment in GradeScope and get its ID. """
472
473        form_url = URL_NEW_ASSIGNMENT_FORM % (self.course_id)
474        token = self.get_csrf_token(session, form_url)
475
476        data = {
477            'authenticity_token': token,
478            'assignment[title]': variant.get_name(),
479            'assignment[submissions_anonymized]': 0,
480            'assignment[student_submission]': "false",
481            'assignment[when_to_create_rubric]': 'while_grading',
482            'assignment[scoring_type]': 'negative',
483        }
484
485        path = os.path.join(base_dir, f"{variant.get_name()}.pdf")
486        files = {
487            'template_pdf': (
488                os.path.basename(path),
489                open(path, 'rb'),  # pylint: disable=consider-using-with
490            ),
491        }
492
493        create_url = URL_ASSIGNMENTS % (self.course_id)
494        response = session.post(create_url, data = data, files = files)
495        response.raise_for_status()
496
497        if (len(response.history) == 0):
498            raise ValueError(f"Failed to create assignment. Is the name ('{variant.get_name()}') unique?")
499
500        match = re.search(r'/assignments/(\d+)/outline/edit', response.history[0].text)
501        if (match is None):
502            _logger.error("--- Create Body ---\n%s\n------", response.history[0].text)
503            raise ValueError("Could not parse assignment ID from response body.")
504
505        return match.group(1)
506
507    def submit_outline(self, session: requests.Session, assignment_id: str, outline: typing.Dict[str, typing.Any]) -> None:
508        """ Submit an output to a quiz. """
509
510        edit_url = URL_EDIT_OUTLINE % (self.course_id, assignment_id)
511        csrf_token = self.get_csrf_token(session, edit_url)
512
513        headers = {
514            'Content-Type': 'application/json',
515            'x-csrf-token': csrf_token,
516        }
517
518        patch_outline_url = URL_PATCH_OUTLINE % (self.course_id, assignment_id)
519        response = session.patch(patch_outline_url,
520            data = edq.util.json.dumps(outline, separators = (',', ':')),
521            headers = headers,  # type: ignore[arg-type]
522        )
523        response.raise_for_status()
524
525    def create_rubric(self, session: requests.Session, assignment_id: str, variant: quizcomp.model.quiz.Variant) -> None:
526        """ Create rubric items for a quiz. """
527
528        questions_ids, csrf_token = self.fetch_question_ids(session, assignment_id)
529
530        for group in variant.get_groups():
531            for question in group.get_questions():
532                if (question.get_name() not in questions_ids):
533                    continue
534
535                question_ids = questions_ids[question.get_name()]
536                score = round(question.get_points() / len(question_ids), 2)
537
538                for question_id in question_ids:
539                    self.add_rubric_item(session, csrf_token, question_id, "Incorrect", score)
540
541    def add_rubric_item(self, session: requests.Session, csrf_token: str, question_id: str, description: str, score: float) -> None:
542        """ Add a single rubric item to a quiz. """
543
544        url = URL_ASSIGNMENT_ADD_RUBRIC_ITEM % (self.course_id, question_id)
545
546        headers = {
547            'Content-Type': 'application/json',
548            'x-csrf-token': csrf_token,
549        }
550
551        data = {
552            "rubric_item":{
553                "description": description,
554                "weight": str(score),
555                "group_id": None,
556            },
557        }
558
559        response = session.post(url, json = data, headers = headers)  # type: ignore[arg-type]
560        response.raise_for_status()
561
562    def fetch_question_ids(self, session: requests.Session, assignment_id: str) -> typing.Tuple[typing.Dict[str, typing.List[str]], str]:
563        """
564        Get the GradeScope id of all questions/subquestions.
565
566        Returns: {<question base name>: [question id, ...], ...}
567        """
568
569        url = URL_ASSIGNMENT_RUBRIC % (self.course_id, assignment_id)
570
571        response = session.get(url)
572        response.raise_for_status()
573
574        document = bs4.BeautifulSoup(response.text, 'html.parser')
575
576        csrf_token = self.parse_csrf_token(document)
577
578        data_tag = document.select('div[data-react-class="AssignmentRubric"]')
579        if (len(data_tag) != 1):
580            raise ValueError(f"Did not find exactly one rubric data tag, found {len(data_tag)}.")
581
582        data = edq.util.json.loads(str(data_tag[0].get('data-react-props')))
583
584        ids = {}
585        for question in data['questions']:
586            question_ids = []
587
588            children = question.get('children', None)
589            if (children is None):
590                question_ids.append(str(question['id']))
591            else:
592                question_ids = [str(child['id']) for child in children]
593
594            ids[question['title']] = question_ids
595
596        return ids, csrf_token