lms.backend.canvas.model

  1import logging
  2import os
  3import re
  4import typing
  5
  6import bs4
  7import edq.net.request
  8import edq.util.dirent
  9import html2text
 10import quizcomp.question.base
 11
 12import lms.backend.canvas.common
 13import lms.model.assignments
 14import lms.model.backend
 15import lms.model.courses
 16import lms.model.groups
 17import lms.model.groupsets
 18import lms.model.quizzes
 19import lms.model.scores
 20import lms.model.users
 21import lms.util.parse
 22
 23_logger = logging.getLogger(__name__)
 24
 25ENROLLMENT_TYPE_TO_ROLE: typing.Dict[str, lms.model.users.CourseRole] = {
 26    'ObserverEnrollment': lms.model.users.CourseRole.OTHER,
 27    'StudentEnrollment': lms.model.users.CourseRole.STUDENT,
 28    'TaEnrollment': lms.model.users.CourseRole.GRADER,
 29    'DesignerEnrollment': lms.model.users.CourseRole.ADMIN,
 30    'TeacherEnrollment': lms.model.users.CourseRole.OWNER,
 31}
 32"""
 33Canvas enrollment types mapped to roles.
 34This map is ordered by priority/power.
 35The later in the dict, the more power.
 36"""
 37
 38QUESTION_TYPE_MAPPING: typing.Dict[typing.Union[str, None], quizcomp.question.base.QuestionType] = {
 39    'essay_question': quizcomp.question.base.QuestionType.ESSAY,
 40    'fill_in_multiple_blanks_question': quizcomp.question.base.QuestionType.FIMB,
 41    'matching_question': quizcomp.question.base.QuestionType.MATCHING,
 42    'multiple_answers_question': quizcomp.question.base.QuestionType.MA,
 43    'multiple_choice_question': quizcomp.question.base.QuestionType.MCQ,
 44    'multiple_dropdowns_question': quizcomp.question.base.QuestionType.MDD,
 45    'numerical_question': quizcomp.question.base.QuestionType.NUMERICAL,
 46    'short_answer_question': quizcomp.question.base.QuestionType.FITB,
 47    'text_only_question': quizcomp.question.base.QuestionType.TEXT_ONLY,
 48    'true_false_question': quizcomp.question.base.QuestionType.TF,
 49}
 50
 51_testing_override: bool = False  # pylint: disable=invalid-name
 52""" A special override to signal testing. """
 53
 54class _ParsedText:
 55    """ The result of parsing Canvas HTML into markdown. """
 56
 57    def __init__(self,
 58            html: str,
 59            text: str,
 60            resources: typing.List[str],
 61            ) -> None:
 62        self.html: str = html
 63        """ The original HTML. """
 64
 65        self.text: str = text
 66        """ The parsed text. """
 67
 68        self.resources: typing.List[str] = resources
 69        """ The path to any resources parsed from this text. """
 70
 71def assignment(data: typing.Dict[str, typing.Any]) -> lms.model.assignments.Assignment:
 72    """
 73    Create a Canvas assignment associated with a course.
 74
 75    See: https://developerdocs.instructure.com/services/canvas/resources/assignments
 76    """
 77
 78    _parse_assignment_data(data, 'assignment')
 79    return lms.model.assignments.Assignment(**data)
 80
 81def assignment_score(data: typing.Dict[str, typing.Any]) -> lms.model.scores.AssignmentScore:
 82    """
 83    Create a Canvas assignment score.
 84
 85    See: https://developerdocs.instructure.com/services/canvas/resources/scores
 86    """
 87
 88    # Check for important fields.
 89    for field in ['id', 'assignment_id', 'user_id']:
 90        if (field not in data):
 91            raise ValueError(f"Canvas assignment score is missing '{field}' field.")
 92
 93    # Modify specific arguments before creation.
 94    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
 95    data['score'] = lms.util.parse.optional_float(data.get('score', None), 'score')
 96    data['points_possible'] = lms.util.parse.optional_float(data.get('points_possible', None), 'points_possible')
 97    data['submission_date'] = lms.backend.canvas.common.parse_timestamp(data.get('submitted_at', None))
 98    data['graded_date'] = lms.backend.canvas.common.parse_timestamp(data.get('graded_at', None))
 99
100    assignment_id = lms.util.parse.required_string(data.get('assignment_id', None), 'assignment_id')
101    data['assignment'] = lms.model.assignments.AssignmentQuery(id = assignment_id)
102
103    user_id = lms.util.parse.required_string(data.get('user_id', None), 'user_id')
104    data['user'] = lms.model.users.UserQuery(id = user_id)
105
106    return lms.model.scores.AssignmentScore(**data)
107
108def course(data: typing.Dict[str, typing.Any]) -> lms.model.courses.Course:
109    """
110    Create a Canvas course.
111
112    See: https://developerdocs.instructure.com/services/canvas/resources/courses
113    """
114
115    # Check for important fields.
116    for field in ['id']:
117        if (field not in data):
118            raise ValueError(f"Canvas course is missing '{field}' field.")
119
120    # Modify specific arguments before creation.
121    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
122
123    return lms.model.courses.Course(**data)
124
125def course_user(backend: lms.model.backend.APIBackend, data: typing.Dict[str, typing.Any]) -> lms.model.users.CourseUser:
126    """
127    Create a Canvas user associated with a course.
128
129    See: https://developerdocs.instructure.com/services/canvas/resources/users
130    """
131
132    # Check for important fields.
133    for field in ['id']:
134        if (field not in data):
135            raise ValueError(f"Canvas user is missing '{field}' field.")
136
137    # Modify specific arguments before sending them to super.
138    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
139
140    # Canvas sometimes has email under different fields.
141    if ((data.get('email', None) is None) or (len(data.get('email', '')) == 0)):
142        data['email'] = data.get('login_id', None)
143
144    enrollments = data.get('enrollments', None)
145    if (enrollments is not None):
146        data['raw_role'] = _parse_role_from_enrollments(enrollments)
147        data['role'] = ENROLLMENT_TYPE_TO_ROLE.get(data['raw_role'], None)
148
149        # Canvas has a discontinuity with its default course roles.
150        # We need to patch this during testing.
151        if ((backend.is_testing() or _testing_override) and data['email'] == 'course-admin@test.edulinq.org'):
152            data['role'] = lms.model.users.CourseRole.ADMIN
153
154    return lms.model.users.CourseUser(**data)
155
156def group(data: typing.Dict[str, typing.Any]) -> lms.model.groups.Group:
157    """
158    Create a Canvas group associated with a course.
159
160    See: https://developerdocs.instructure.com/services/canvas/resources/groups
161    """
162
163    # Check for important fields.
164    for field in ['id']:
165        if (field not in data):
166            raise ValueError(f"Canvas group is missing '{field}' field.")
167
168    # Modify specific arguments before creation.
169    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
170
171    return lms.model.groups.Group(**data)
172
173def group_set(data: typing.Dict[str, typing.Any]) -> lms.model.groupsets.GroupSet:
174    """
175    Create a Canvas group set associated with a course.
176
177    See: https://developerdocs.instructure.com/services/canvas/resources/group_categories
178    """
179
180    # Check for important fields.
181    for field in ['id']:
182        if (field not in data):
183            raise ValueError(f"Canvas group set is missing '{field}' field.")
184
185    # Modify specific arguments before creation.
186    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
187
188    return lms.model.groupsets.GroupSet(**data)
189
190def quiz(
191        backend: lms.model.backend.APIBackend,
192        data: typing.Dict[str, typing.Any],
193        fetch_resources: bool = False,
194        ) -> lms.model.quizzes.Quiz:
195    """
196    Create a Canvas quiz associated with a course.
197
198    See: https://developerdocs.instructure.com/services/canvas/resources/quizzes
199    """
200
201    _parse_assignment_data(data, 'quiz')
202
203    parsed_text = _canvas_html_to_markdown(backend, data.get('description', None), fetch_resources)
204    data['description'] = parsed_text.text
205    data['resources'] = parsed_text.resources
206
207    return lms.model.quizzes.Quiz(**data)
208
209def quiz_question(
210        backend: lms.model.backend.APIBackend,
211        data: typing.Dict[str, typing.Any],
212        fetch_resources: bool = False,
213        ) -> lms.model.quizzes.Question:
214    """
215    Create a Canvas quiz question.
216
217    See: https://developerdocs.instructure.com/services/canvas/resources/quiz_questions
218    """
219
220    # Check for important fields.
221    for field in ['id']:
222        if (field not in data):
223            raise ValueError(f"Canvas quiz question is missing '{field}' field.")
224
225    raw_question_type = data.get('question_type', None)
226    if (raw_question_type is None):
227        raise ValueError('No question type provided.')
228
229    question_type = QUESTION_TYPE_MAPPING.get(raw_question_type, None)
230    if (question_type is None):
231        raise ValueError(f"Unknown Canvas question type: '{raw_question_type}'.")
232
233    data['question_type'] = question_type
234    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
235    data['name'] = lms.util.parse.optional_string(data.get('question_name', None))
236    data['points'] = lms.util.parse.optional_float(data.get('points_possible', None), 'points')
237    data['raw_answers'] = data.get('answers', None)
238    data['group_id'] = lms.util.parse.optional_string(data.get('quiz_group_id', None))
239
240    all_resource_paths = []
241
242    parsed_text = _canvas_html_to_markdown(backend, data.get('question_text', None), fetch_resources)
243    data['prompt'] = parsed_text.text
244    all_resource_paths += parsed_text.resources
245
246    (answers, resources) = _parse_quiz_question_answers(
247        backend,
248        data.get('answers', None),
249        data.get('matching_answer_incorrect_matches', None),
250        question_type,
251        fetch_resources,
252    )
253    data['answers'] = answers
254    all_resource_paths += resources
255
256    data['resources'] = all_resource_paths
257
258    return lms.model.quizzes.Question(**data)
259
260def quiz_question_group(data: typing.Dict[str, typing.Any]) -> lms.model.quizzes.QuestionGroup:
261    """
262    Create a Canvas quiz question group.
263
264    See: https://developerdocs.instructure.com/services/canvas/resources/quiz_question_groups
265    """
266
267    # Check for important fields.
268    for field in ['id']:
269        if (field not in data):
270            raise ValueError(f"Canvas quiz question group is missing '{field}' field.")
271
272    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
273    data['name'] = lms.util.parse.optional_string(data.get('name', None))
274    data['points'] = lms.util.parse.optional_float(data.get('question_points', None), 'points')
275    data['pick_count'] = lms.util.parse.required_int(data.get('pick_count', None), 'pick_count')
276
277    return lms.model.quizzes.QuestionGroup(**data)
278
279def _canvas_html_to_markdown(
280        backend: lms.model.backend.APIBackend,
281        html: typing.Union[str, None],
282        fetch_resources: bool = False,
283        ) -> _ParsedText:
284    """
285    Parse the text from a Canvas quiz question into markdown.
286    We intend for the resulting markdown to have a little HTML as possible.
287    This is an impossible task, but we want to do our best.
288    """
289
290    if (html is None):
291        return _ParsedText('', '', [])
292
293    resources: typing.List[str] = []
294    if (fetch_resources):
295        (html, resources) = _fetch_html_resources(backend, html)
296
297    converter = html2text.HTML2Text()
298
299    converter.body_width = 0
300    converter.mark_code = True
301
302    text = converter.handle(html)
303    text = text.strip()
304
305    # Replace code tags with fences.
306    text = re.sub(r'\[/?code\]', '```', text)
307
308    # Replace placeholders (e.g., for fill in the blank questions).
309    text = re.sub(r'\[(\w+?)\]', r'<placeholder>\1</placeholder>', text)
310
311    return _ParsedText(html, text, resources)
312
313def _fetch_html_resources(backend: lms.model.backend.APIBackend, html: str) -> typing.Tuple[str, typing.List[str]]:
314    """ Fetch any resources embedded in the HTML and re-write the links for these resources. """
315
316    document = bs4.BeautifulSoup(html, 'html.parser')
317
318    resources = []
319
320    # Look for Canvas-embeded images.
321    for image in document.select('img[data-api-endpoint]'):
322        path = _fetch_file(backend, str(image.get('data-api-endpoint')))
323        if (path is None):
324            continue
325
326        image['src'] = os.path.basename(path)
327
328        resources.append(path)
329
330    return str(document), resources
331
332def _fetch_file(backend: lms.model.backend.APIBackend, info_link: str) -> typing.Union[str, None]:
333    """ Fetch a file from Canvas, write it to disk, and return the path. """
334
335    headers = backend.get_standard_headers()
336    file_info = lms.backend.canvas.common.make_get_request(info_link, headers = headers)
337
338    if (file_info is None):
339        return None
340
341    response, _ = edq.net.request.make_get(file_info['url'], headers = headers)
342
343    temp_dir = edq.util.dirent.get_temp_dir('edq-lms-canvas-')
344    path = os.path.join(temp_dir, file_info['filename'])
345
346    edq.util.dirent.write_file_bytes(path, response.content)
347
348    return path
349
350def _parse_quiz_question_answers(
351        backend: lms.model.backend.APIBackend,
352        raw_answers: typing.Union[typing.List[typing.Any], None],
353        raw_distractors: typing.Union[str, None],
354        question_type: quizcomp.question.base.QuestionType,
355        fetch_resources: bool = False,
356        ) -> typing.Tuple[typing.Union[typing.List[typing.Any], typing.Dict[str, typing.Any]], typing.List[str]]:
357    """ Parse question answers from Canvas responses. """
358
359    if (raw_answers is None):
360        return [], []
361
362    answers: typing.Union[typing.List[typing.Any], typing.Dict[str, typing.Any]] = []
363    resources: typing.List[str] = []
364
365    # Parse answers based on question type.
366    if (question_type in {quizcomp.question.base.QuestionType.ESSAY, quizcomp.question.base.QuestionType.TEXT_ONLY}):
367        pass
368    elif (question_type == quizcomp.question.base.QuestionType.FIMB):
369        answers = {}
370        for raw_answer in raw_answers:
371            key = raw_answer.get('blank_id', '')
372            if (key not in answers):
373                answers[key] = []
374
375            parsed_text = _parse_quiz_question_text(backend, raw_answer, fetch_resources)
376
377            answers[key].append(parsed_text.text)
378            resources += parsed_text.resources
379    elif (question_type == quizcomp.question.base.QuestionType.FITB):
380        answers = []
381        for raw_answer in raw_answers:
382            parsed_text = _parse_quiz_question_text(backend, raw_answer, fetch_resources)
383
384            answers.append(parsed_text.text)
385            resources += parsed_text.resources
386    elif (question_type in {quizcomp.question.base.QuestionType.MA, quizcomp.question.base.QuestionType.MCQ}):
387        (answers, choice_resources) = _parse_quiz_question_choices(backend, raw_answers, fetch_resources)
388        resources += choice_resources
389    elif (question_type == quizcomp.question.base.QuestionType.MDD):
390        # Divide up sections by blank ID (find all the possibilities for each blank).
391        # {key: [raw_answer, ...]}
392        raw_sections: typing.Dict[str, typing.List[typing.Dict[str, typing.Any]]] = {}
393        for raw_answer in raw_answers:
394            key = raw_answer.get('blank_id', '')
395            if (key not in raw_sections):
396                raw_sections[key] = []
397
398            raw_sections[key].append(raw_answer)
399
400        # Parse the choices for each section/blank.
401        answers = {}
402        for (section_key, section_raw_answers) in raw_sections.items():
403            (choices, choice_resources) = _parse_quiz_question_choices(backend, section_raw_answers, fetch_resources)
404
405            answers[section_key] = {
406                'text': section_key,
407                'values': choices
408            }
409            resources += choice_resources
410    elif (question_type == quizcomp.question.base.QuestionType.MATCHING):
411        if (raw_distractors is None):
412            raw_distractors = ''
413
414        matches = []
415        distractors = [value.strip() for value in raw_distractors.split("\n")]
416
417        for raw_answer in raw_answers:
418            matches.append([raw_answer.get('left', ''), raw_answer.get('right', '')])
419
420        answers = {
421            'matches': matches,
422            'distractors': distractors,
423        }
424    elif (question_type == quizcomp.question.base.QuestionType.NUMERICAL):
425        answers = []
426        for raw_answer in raw_answers:
427            raw_type = raw_answer.get('numerical_answer_type', None)
428
429            answer_type = str(raw_type).removesuffix('_answer')
430            answer = {'type': answer_type}
431
432            if (answer_type == quizcomp.constants.NUMERICAL_ANSWER_TYPE_EXACT):
433                answer['value'] = raw_answer.get('exact', None)
434                answer['margin'] = raw_answer.get('error_margin', 0)
435            elif (answer_type == quizcomp.constants.NUMERICAL_ANSWER_TYPE_RANGE):
436                answer['min'] = raw_answer.get('range_start', None)
437                answer['max'] = raw_answer.get('range_end', None)
438            elif (answer_type == quizcomp.constants.NUMERICAL_ANSWER_TYPE_PRECISION):
439                answer['value'] = raw_answer.get('approximate', None)
440                answer['precision'] = raw_answer.get('precision', None)
441            else:
442                raise ValueError(f"Unknown numerical answer type: '{raw_type}'.")
443
444            answers.append(answer)
445    elif (question_type == quizcomp.question.base.QuestionType.TF):
446        if (len(raw_answers) != 2):
447            raise ValueError(f"Unexpected length for T/F answers. Expected 2, found {len(raw_answers)}.")
448
449        answers = [
450            {"correct": (raw_answers[0]['weight'] > 0), "text": "True"},
451            {"correct": (raw_answers[1]['weight'] > 0), "text": "False"},
452        ]
453    else:
454        _logger.warning("Cannot form question answers, unknown question type: '%s'.", question_type)
455
456    return answers, resources
457
458def _parse_quiz_question_choices(
459        backend: lms.model.backend.APIBackend,
460        choices: list[typing.Dict[str, typing.Any]],
461        fetch_resources: bool = False,
462        ) -> typing.Tuple[typing.List[typing.Dict[str, typing.Any]], typing.List[str]]:
463    """
464    Parse the quiz question choices.
465    This works for multiple types, like MCQ and MA.
466    """
467
468    results = []
469    resources = []
470
471    for choice in choices:
472        parsed_text = _parse_quiz_question_text(backend, choice, fetch_resources)
473
474        results.append({"correct": (choice['weight'] > 0), "text": parsed_text.text})
475        resources += parsed_text.resources
476
477    return results, resources
478
479def _parse_quiz_question_text(
480        backend: lms.model.backend.APIBackend,
481        choice: typing.Dict[str, typing.Any],
482        fetch_resources: bool = False,
483        ) -> _ParsedText:
484    """ Parse text out of a Canvas question choice field. """
485
486    text = choice.get('text', '').strip()
487    if (text is None):
488        text = ''
489
490    parsed_text = _ParsedText('', text, [])
491    if (len(text) == 0):
492        parsed_text = _canvas_html_to_markdown(backend, choice.get('html', None), fetch_resources)
493
494    return parsed_text
495
496def _parse_assignment_data(data: typing.Dict[str, typing.Any], label: str) -> None:
497    """
498    Parse core assignment data.
499    """
500
501    for field in ['id']:
502        if (field not in data):
503            raise ValueError(f"Canvas {label} is missing '{field}' field.")
504
505    # Modify specific arguments before creation.
506    data['id'] = lms.util.parse.required_string(data.get('id', None), 'id')
507    data['due_date'] = lms.backend.canvas.common.parse_timestamp(data.get('due_at', None))
508    data['open_date'] = lms.backend.canvas.common.parse_timestamp(data.get('unlock_at', None))
509    data['close_date'] = lms.backend.canvas.common.parse_timestamp(data.get('lock_at', None))
510
511    # If there is no name, look for a title.
512    if (data.get('name', None) is None):
513        data['name'] = lms.util.parse.optional_string(data.get('title', None))
514
515def _parse_role_from_enrollments(enrollments: typing.Any) -> typing.Union[str, None]:
516    """
517    Try to parse the user's role from their enrollments.
518    If multiple roles are discovered, take the "highest" one.
519
520    See: https://developerdocs.instructure.com/services/canvas/resources/enrollments
521    """
522
523    if (not isinstance(enrollments, list)):
524        return None
525
526    best_role = None
527    best_index = -1
528
529    enrollment_types = list(ENROLLMENT_TYPE_TO_ROLE.keys())
530
531    for enrollment in enrollments:
532        if (not isinstance(enrollment, dict)):
533            continue
534
535        if (enrollment.get('enrollment_state', None) != 'active'):
536            continue
537
538        role = enrollment.get('role', None)
539
540        role_index = -1
541        if (role in enrollment_types):
542            role_index = enrollment_types.index(role)
543
544        if ((best_role is None) or (role_index > best_index)):
545            best_role = role
546            best_index = role_index
547
548    return best_role