lms.util.net

  1"""
  2Utilities for network and HTTP.
  3"""
  4
  5import typing
  6import urllib.parse
  7
  8import edq.net.exchange
  9import edq.util.json
 10import requests
 11
 12import lms.model.constants
 13
 14CANVAS_CLEAN_REMOVE_CONTENT_KEYS: typing.List[str] = [
 15    'created_at',
 16    'ics',
 17    'last_activity_at',
 18    'lti_context_id',
 19    'preview_url',
 20    'secure_params',
 21    'total_activity_time',
 22    'updated_at',
 23    'url',
 24    'uuid',
 25]
 26""" Keys to remove from Canvas content. """
 27
 28BLACKBOARD_CLEAN_REMOVE_CONTENT_KEYS: typing.List[str] = [
 29    'created',
 30    'modified',
 31]
 32""" Keys to remove from Blackboard content. """
 33
 34BLACKBOARD_CLEAN_REMOVE_HEADERS: typing.Set[str] = {
 35    'access-control-allow-origin',
 36    'content-encoding',
 37    'content-language',
 38    'expires',
 39    'last-modified',
 40    'p3p',
 41    'pragma',
 42    'strict-transport-security',
 43    'transfer-encoding',
 44    'vary',
 45    'x-blackboard-xsrf',
 46}
 47""" Keys to remove from Blackboard headers. """
 48
 49MOODLE_CLEAN_REMOVE_HEADERS: typing.Set[str] = {
 50    'accept-ranges',
 51    'content-encoding',
 52    'content-language',
 53    'content-script-type',
 54    'content-style-type',
 55    'expires',
 56    'keep-alive',
 57    'last-modified',
 58    'pragma',
 59    'vary',
 60}
 61""" Keys to remove from Moodle headers. """
 62
 63MOODLE_FINALIZE_REMOVE_PARAMS: typing.Set[str] = {
 64    'logintoken',
 65}
 66""" Keys to remove from Moodle headers. """
 67
 68def clean_lms_response(response: requests.Response, body: str) -> str:
 69    """
 70    A ResponseModifierFunction that attempt to identify
 71    if the requests comes from a Learning Management System (LMS),
 72    and clean the response accordingly.
 73    """
 74
 75    # Check the standard LMS Toolkit backend header.
 76    backend_type = response.headers.get(lms.model.constants.HEADER_KEY_BACKEND, '').lower()
 77
 78    if (backend_type == lms.model.constants.BACKEND_TYPE_CANVAS):
 79        return clean_canvas_response(response, body)
 80
 81    if (backend_type == lms.model.constants.BACKEND_TYPE_MOODLE):
 82        return clean_moodle_response(response, body)
 83
 84    # Try looking inside the header keys.
 85    for key in response.headers:
 86        key = key.lower().strip()
 87
 88        if ('blackboard' in key):
 89            return clean_blackboard_response(response, body)
 90
 91        if ('canvas' in key):
 92            return clean_canvas_response(response, body)
 93
 94        if ('moodle' in key):
 95            return clean_moodle_response(response, body)
 96
 97    return body
 98
 99def clean_blackboard_response(response: requests.Response, body: str) -> str:
100    """
101    See clean_lms_response(), but specifically for the Blackboard LMS.
102    This function will:
103     - Call _clean_base_response().
104     - Remove specific headers.
105    """
106
107    body = _clean_base_response(response, body)
108
109    # Work on both request and response headers.
110    for headers in [response.headers, response.request.headers]:
111        for key in list(headers.keys()):
112            if (key.strip().lower() in BLACKBOARD_CLEAN_REMOVE_HEADERS):
113                headers.pop(key, None)
114
115    # Most blackboard responses are JSON.
116    try:
117        data = edq.util.json.loads(body, strict = True)
118    except Exception:
119        # Response is not JSON.
120        return body
121
122    # Remove any content keys.
123    _recursive_remove_keys(data, set(BLACKBOARD_CLEAN_REMOVE_CONTENT_KEYS))
124
125    # Convert body back to a string.
126    body = edq.util.json.dumps(data)
127
128    return body
129
130def clean_canvas_response(response: requests.Response, body: str) -> str:
131    """
132    See clean_lms_response(), but specifically for the Canvas LMS.
133    This function will:
134     - Call _clean_base_response().
135     - Remove content keys: [last_activity_at, total_activity_time]
136    """
137
138    body = _clean_base_response(response, body)
139
140    # Most canvas responses are JSON.
141    try:
142        data = edq.util.json.loads(body, strict = True)
143    except Exception:
144        # Response is not JSON.
145        return body
146
147    # Remove any content keys.
148    _recursive_remove_keys(data, set(CANVAS_CLEAN_REMOVE_CONTENT_KEYS))
149
150    # Remove special fields.
151
152    if ('submissions/update_grades' in response.request.url):
153        data.pop('id', None)
154
155    # Convert body back to a string.
156    body = edq.util.json.dumps(data)
157
158    return body
159
160def clean_moodle_response(response: requests.Response, body: str) -> str:
161    """
162    See clean_lms_response(), but specifically for the Moodle LMS.
163    This function will:
164     - Call _clean_base_response().
165    """
166
167    body = _clean_base_response(response, body)
168
169    # Work on both request and response headers.
170    for headers in [response.headers, response.request.headers]:
171        for key in list(headers.keys()):
172            if (key.strip().lower() in MOODLE_CLEAN_REMOVE_HEADERS):
173                headers.pop(key, None)
174
175    return body
176
177def finalize_moodle_exchange(exchange: edq.net.exchange.HTTPExchange) -> edq.net.exchange.HTTPExchange:
178    """ Finalize Moodle exhanges. """
179
180    for param in MOODLE_FINALIZE_REMOVE_PARAMS:
181        exchange.parameters.pop(param, None)
182
183    return exchange
184
185def _clean_base_response(response: requests.Response, body: str,
186        keep_headers: typing.Union[typing.List[str], None] = None) -> str:
187    """
188    Do response cleaning that is common amongst all backend types.
189    This function will:
190     - Remove X- headers.
191    """
192
193    # Index requests are generally for identification, and we use headers.
194    path = urllib.parse.urlparse(response.request.url).path.strip()
195    if (path in ['', '/']):
196        body = ''
197
198    for key in list(response.headers.keys()):
199        key = key.strip().lower()
200        if ((keep_headers is not None) and (key in keep_headers)):
201            continue
202
203        if (key.startswith('x-')):
204            response.headers.pop(key, None)
205
206    return body
207
208def _recursive_remove_keys(data: typing.Any, remove_keys: typing.Set[str]) -> None:
209    """
210    Recursively descend through the given and remove any instance to the given key from any dictionaries.
211    The data should only be simple types (POD, dicts, lists, tuples).
212    """
213
214    if (isinstance(data, (list, tuple))):
215        for item in data:
216            _recursive_remove_keys(item, remove_keys)
217    elif (isinstance(data, dict)):
218        for key in list(data.keys()):
219            if (key in remove_keys):
220                del data[key]
221            else:
222                _recursive_remove_keys(data[key], remove_keys)
223
224def parse_cookies(
225        text_cookies: typing.Union[str, None],
226        strip_key_prefix: bool = True,
227        ) -> typing.Dict[str, typing.Any]:
228    """ Parse cookies out of a text string. """
229
230    cookies: typing.Dict[str, typing.Any] = {}
231
232    if (text_cookies is None):
233        return cookies
234
235    text_cookies = text_cookies.strip()
236    if (len(text_cookies) == 0):
237        return cookies
238
239    for cookie in text_cookies.split('; '):
240        parts = cookie.split('=', maxsplit = 1)
241
242        key = parts[0].lower()
243
244        if (strip_key_prefix):
245            key = key.split(', ')[-1]
246
247        if (len(parts) == 1):
248            cookies[key] = True
249        else:
250            cookies[key] = parts[1]
251
252    return cookies