autograder.testing.asserts

View Source

  1import os
  2import re
  3import typing
  4
  5import edq.testing.asserts
  6import edq.testing.unittest
  7import edq.util.json
  8
  9import autograder.testing.constants
 10
 11THIS_DIR: str = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
 12ROOT_DIR: str = os.path.join(THIS_DIR, '..', '..')
 13AUTOGRADER_TESTDATA_REPO: str = os.path.join(ROOT_DIR, 'testdata', 'autograder-testdata')
 14AUTOGRADER_SERVER_REPO: str = os.path.join(AUTOGRADER_TESTDATA_REPO, 'autograder-server')
 15API_DESCRIPTION_PATH: str = os.path.join(AUTOGRADER_SERVER_REPO, 'resources', 'api.json')
 16
 17TOKEN_CLEARTEXT_PATTERN: str = r'\w{32}'
 18TOKEN_ID_PATTERN: str = r'\w{8}-\w{4}-\w{4}-\w{4}-\w{12}'
 19
 20PRETTY_TIME_PATTERN: str = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+[\+\-]\d+:\d+'
 21PRETTY_TIME_REPLACEMENT: str = '<PRETTY TIME>'
 22
 23NORMALIZE_TIMESTAMP_KEYS: typing.Set[str] = {
 24    'analysis-timestamp',
 25    'first-timestamp',
 26    'last-timestamp',
 27    'set-time',
 28}
 29""" Keys for timestamp values to normalize. """
 30
 31_cached_api_description: typing.Union[typing.Dict[str, typing.Any], None] = None  # pylint: disable=invalid-name
 32
 33def get_expected_api_description() -> typing.Dict[str, typing.Any]:
 34    """ Get the expected API description from the submodule (or the cache). """
 35
 36    global _cached_api_description  # pylint: disable=global-statement
 37
 38    if (_cached_api_description is not None):
 39        return _cached_api_description
 40
 41    _cached_api_description = edq.util.json.load_path(API_DESCRIPTION_PATH, strict = True)
 42
 43    return _cached_api_description
 44
 45def content_equals_noramlize_json(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
 46    """
 47    A CLI test assertion function for JSON output.
 48    The output will be converted to a dict and then compared with edq.testing.unittest.BaseTest.assertJSONDictEqual().
 49    """
 50
 51    # Convert both to dicts.
 52    expected_dict = edq.util.json.loads(expected, strict = True)
 53    actual_dict = edq.util.json.loads(actual, strict = True)
 54
 55    # Normalize the actual data (the expected should already be normalized (by the tester)).
 56    actual_dict = normalize_dict(actual_dict)
 57    actual_dict = _normalize_timestamps(actual_dict)
 58
 59    test.assertJSONDictEqual(expected_dict, actual_dict)
 60
 61def contains(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
 62    """
 63    A CLI test assertion function that checks if the expected output is a contained in the actual output.
 64    """
 65
 66    test.assertIn(expected, actual)
 67
 68def content_equals_noramlize_regrade(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
 69    """ A CLI test assertion function for regrade output. """
 70
 71    # Convert both to dicts.
 72    expected_dict = edq.util.json.loads(expected, strict = True)
 73    actual_dict = edq.util.json.loads(actual, strict = True)
 74
 75    # Normalize the actual data (the expected should already be normalized (by the tester)).
 76    actual_dict = normalize_dict(actual_dict)
 77    actual_dict = _normalize_timestamps(actual_dict, keys = {'grading_start_time', 'regrade-cutoff'})
 78
 79    test.assertJSONDictEqual(expected_dict, actual_dict)
 80
 81def content_equals_stack_trace(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
 82    """
 83    A CLI test assertion function for checking if the output looks like a stack trace.
 84    """
 85
 86    # Actual output should be JSON.
 87    actual_dict = edq.util.json.loads(actual, strict = True)
 88
 89    test.assertGreater(actual_dict['count'], 5)
 90    test.assertEqual(actual_dict['count'], len(actual_dict['stacks']))
 91
 92def normalize_dict(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
 93    """ Noramlize a dict that typically comes from testing output. """
 94
 95    data = _noramlize_version(data)
 96    data = _noramlize_tokens(data)
 97    data = _normalize_timestamps(data)
 98
 99    return data
100
101def normalize_analysis(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
102    """ Noramlize a dict that comes from an analysis result. """
103
104    _normalize_timestamps(data)
105    return data
106
107def _noramlize_version(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
108    """ Normalize server version information. """
109
110    if ('server-version' not in data):
111        return data
112
113    data['server-version']['base-version'] = autograder.testing.constants.TEST_BASE_VERSION
114    data['server-version']['git-hash'] = autograder.testing.constants.TEST_GIT_HASH
115    data['server-version']['is-dirty'] = autograder.testing.constants.TEST_IS_DIRTY
116
117    return data
118
119def _noramlize_tokens(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
120    """ Normalize token identifiers. """
121
122    if ('token-id' in data):
123        data['token-id'] = autograder.testing.constants.TEST_TOKEN_ID
124
125    if ('token-cleartext' in data):
126        data['token-cleartext'] = autograder.testing.constants.TEST_TOKEN_CLEARTEXT
127
128    if ('token-info' in data):
129        _normalize_token_info(data['token-info'])
130
131    if ('tokens' in data):
132        if (data['tokens'] is not None):
133            for token_info in data['tokens']:
134                _normalize_token_info(token_info)
135
136    return data
137
138def _normalize_token_info(token_info: typing.Union[typing.Dict[str, typing.Any], None]) -> None:
139    if (token_info is None):
140        return
141
142    token_info['access-time'] = autograder.testing.constants.TEST_TIMESTAMP
143    token_info['creation-time'] = autograder.testing.constants.TEST_TIMESTAMP
144    token_info['id'] = autograder.testing.constants.TEST_TOKEN_ID
145
146def equals_clean_imageinfo(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
147    """
148    A CLI test assertion function for JSON image info.
149    """
150
151    # Convert both to dicts.
152    expected_dict = edq.util.json.loads(expected, strict = True)
153    actual_dict = edq.util.json.loads(actual, strict = True)
154
155    for content in [expected_dict, actual_dict]:
156        content['created-timestamp'] = autograder.testing.constants.TEST_TIMESTAMP
157        content['size-bytes'] = len(autograder.testing.constants.TEST_PAYLOAD_BYTES)
158
159    test.assertJSONDictEqual(expected_dict, actual_dict)
160
161def equals_api_description(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
162    """ A CLI test assertion function for the API description (read from a submodule). """
163
164    expected_dict = get_expected_api_description()
165    actual_dict = edq.util.json.loads(actual, strict = True)
166
167    test.assertJSONDictEqual(expected_dict, actual_dict)
168
169def equals_clean_tokens(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
170    """ A CLI test assertion function for text that contains tokens. """
171
172    expected = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, expected)
173    expected = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, expected)
174
175    actual = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, actual)
176    actual = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, actual)
177
178    test.assertEqual(expected, actual)
179
180def equals_clean_pretty_time(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
181    """ A CLI test assertion function for text that contains pretty timestamps. """
182
183    actual = re.sub(PRETTY_TIME_PATTERN, PRETTY_TIME_REPLACEMENT, actual)
184
185    edq.testing.asserts.content_equals_normalize(test, expected, actual)
186
187def _normalize_timestamps(data: typing.Any, keys: typing.Union[typing.Set[str], None] = None) -> typing.Any:
188    """
189    Recursively normalize specified timestamps.
190    It is assumed that the specified data comes from JSON deserialization
191    (and therefore has a limited number of types).
192    """
193
194    if (keys is None):
195        keys = NORMALIZE_TIMESTAMP_KEYS
196
197    if (isinstance(data, list)):
198        return [_normalize_timestamps(item, keys = keys) for item in data]
199
200    if (isinstance(data, dict)):
201        for (key, value) in data.items():
202            if (key in keys):
203                data[key] = autograder.testing.constants.TEST_TIMESTAMP
204            else:
205                data[key] = _normalize_timestamps(value, keys = keys)
206
207    return data

THIS_DIR: str = '/home/runner/work/autograder-py/autograder-py/autograder/testing'

ROOT_DIR: str = '/home/runner/work/autograder-py/autograder-py/autograder/testing/../..'

AUTOGRADER_TESTDATA_REPO: str = '/home/runner/work/autograder-py/autograder-py/autograder/testing/../../testdata/autograder-testdata'

AUTOGRADER_SERVER_REPO: str = '/home/runner/work/autograder-py/autograder-py/autograder/testing/../../testdata/autograder-testdata/autograder-server'

API_DESCRIPTION_PATH: str = '/home/runner/work/autograder-py/autograder-py/autograder/testing/../../testdata/autograder-testdata/autograder-server/resources/api.json'

TOKEN_CLEARTEXT_PATTERN: str = '\\w{32}'

TOKEN_ID_PATTERN: str = '\\w{8}-\\w{4}-\\w{4}-\\w{4}-\\w{12}'

PRETTY_TIME_PATTERN: str = '\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d+[\\+\\-]\\d+:\\d+'

PRETTY_TIME_REPLACEMENT: str = '<PRETTY TIME>'

NORMALIZE_TIMESTAMP_KEYS: Set[str] = {'analysis-timestamp', 'first-timestamp', 'set-time', 'last-timestamp'}

Keys for timestamp values to normalize.

def get_expected_api_description() -> Dict[str, Any]: View Source

34def get_expected_api_description() -> typing.Dict[str, typing.Any]:
35    """ Get the expected API description from the submodule (or the cache). """
36
37    global _cached_api_description  # pylint: disable=global-statement
38
39    if (_cached_api_description is not None):
40        return _cached_api_description
41
42    _cached_api_description = edq.util.json.load_path(API_DESCRIPTION_PATH, strict = True)
43
44    return _cached_api_description

Get the expected API description from the submodule (or the cache).

def content_equals_noramlize_json(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

46def content_equals_noramlize_json(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
47    """
48    A CLI test assertion function for JSON output.
49    The output will be converted to a dict and then compared with edq.testing.unittest.BaseTest.assertJSONDictEqual().
50    """
51
52    # Convert both to dicts.
53    expected_dict = edq.util.json.loads(expected, strict = True)
54    actual_dict = edq.util.json.loads(actual, strict = True)
55
56    # Normalize the actual data (the expected should already be normalized (by the tester)).
57    actual_dict = normalize_dict(actual_dict)
58    actual_dict = _normalize_timestamps(actual_dict)
59
60    test.assertJSONDictEqual(expected_dict, actual_dict)

A CLI test assertion function for JSON output. The output will be converted to a dict and then compared with edq.testing.unittest.BaseTest.assertJSONDictEqual().

def contains(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

62def contains(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
63    """
64    A CLI test assertion function that checks if the expected output is a contained in the actual output.
65    """
66
67    test.assertIn(expected, actual)

A CLI test assertion function that checks if the expected output is a contained in the actual output.

def content_equals_noramlize_regrade(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

69def content_equals_noramlize_regrade(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
70    """ A CLI test assertion function for regrade output. """
71
72    # Convert both to dicts.
73    expected_dict = edq.util.json.loads(expected, strict = True)
74    actual_dict = edq.util.json.loads(actual, strict = True)
75
76    # Normalize the actual data (the expected should already be normalized (by the tester)).
77    actual_dict = normalize_dict(actual_dict)
78    actual_dict = _normalize_timestamps(actual_dict, keys = {'grading_start_time', 'regrade-cutoff'})
79
80    test.assertJSONDictEqual(expected_dict, actual_dict)

A CLI test assertion function for regrade output.

def content_equals_stack_trace(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

82def content_equals_stack_trace(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
83    """
84    A CLI test assertion function for checking if the output looks like a stack trace.
85    """
86
87    # Actual output should be JSON.
88    actual_dict = edq.util.json.loads(actual, strict = True)
89
90    test.assertGreater(actual_dict['count'], 5)
91    test.assertEqual(actual_dict['count'], len(actual_dict['stacks']))

A CLI test assertion function for checking if the output looks like a stack trace.

def normalize_dict(data: Dict[str, Any]) -> Dict[str, Any]: View Source

 93def normalize_dict(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
 94    """ Noramlize a dict that typically comes from testing output. """
 95
 96    data = _noramlize_version(data)
 97    data = _noramlize_tokens(data)
 98    data = _normalize_timestamps(data)
 99
100    return data

Noramlize a dict that typically comes from testing output.

def normalize_analysis(data: Dict[str, Any]) -> Dict[str, Any]: View Source

102def normalize_analysis(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
103    """ Noramlize a dict that comes from an analysis result. """
104
105    _normalize_timestamps(data)
106    return data

Noramlize a dict that comes from an analysis result.

def equals_clean_imageinfo(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

147def equals_clean_imageinfo(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
148    """
149    A CLI test assertion function for JSON image info.
150    """
151
152    # Convert both to dicts.
153    expected_dict = edq.util.json.loads(expected, strict = True)
154    actual_dict = edq.util.json.loads(actual, strict = True)
155
156    for content in [expected_dict, actual_dict]:
157        content['created-timestamp'] = autograder.testing.constants.TEST_TIMESTAMP
158        content['size-bytes'] = len(autograder.testing.constants.TEST_PAYLOAD_BYTES)
159
160    test.assertJSONDictEqual(expected_dict, actual_dict)

A CLI test assertion function for JSON image info.

def equals_api_description(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

162def equals_api_description(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
163    """ A CLI test assertion function for the API description (read from a submodule). """
164
165    expected_dict = get_expected_api_description()
166    actual_dict = edq.util.json.loads(actual, strict = True)
167
168    test.assertJSONDictEqual(expected_dict, actual_dict)

A CLI test assertion function for the API description (read from a submodule).

def equals_clean_tokens(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

170def equals_clean_tokens(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
171    """ A CLI test assertion function for text that contains tokens. """
172
173    expected = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, expected)
174    expected = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, expected)
175
176    actual = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, actual)
177    actual = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, actual)
178
179    test.assertEqual(expected, actual)

A CLI test assertion function for text that contains tokens.

def equals_clean_pretty_time(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: View Source

181def equals_clean_pretty_time(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
182    """ A CLI test assertion function for text that contains pretty timestamps. """
183
184    actual = re.sub(PRETTY_TIME_PATTERN, PRETTY_TIME_REPLACEMENT, actual)
185
186    edq.testing.asserts.content_equals_normalize(test, expected, actual)

A CLI test assertion function for text that contains pretty timestamps.