autograder.testing.asserts
1import os 2import re 3import typing 4 5import edq.testing.asserts 6import edq.testing.unittest 7import edq.util.json 8 9import autograder.testing.constants 10 11THIS_DIR: str = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) 12ROOT_DIR: str = os.path.join(THIS_DIR, '..', '..') 13AUTOGRADER_TESTDATA_REPO: str = os.path.join(ROOT_DIR, 'testdata', 'autograder-testdata') 14AUTOGRADER_SERVER_REPO: str = os.path.join(AUTOGRADER_TESTDATA_REPO, 'autograder-server') 15API_DESCRIPTION_PATH: str = os.path.join(AUTOGRADER_SERVER_REPO, 'resources', 'api.json') 16 17TOKEN_CLEARTEXT_PATTERN: str = r'\w{32}' 18TOKEN_ID_PATTERN: str = r'\w{8}-\w{4}-\w{4}-\w{4}-\w{12}' 19 20PRETTY_TIME_PATTERN: str = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+[\+\-]\d+:\d+' 21PRETTY_TIME_REPLACEMENT: str = '<PRETTY TIME>' 22 23NORMALIZE_TIMESTAMP_KEYS: typing.Set[str] = { 24 'analysis-timestamp', 25 'first-timestamp', 26 'last-timestamp', 27} 28""" Keys for timestamp values to normalize. """ 29 30_cached_api_description: typing.Union[typing.Dict[str, typing.Any], None] = None # pylint: disable=invalid-name 31 32def get_expected_api_description() -> typing.Dict[str, typing.Any]: 33 """ Get the expected API description from the submodule (or the cache). """ 34 35 global _cached_api_description # pylint: disable=global-statement 36 37 if (_cached_api_description is not None): 38 return _cached_api_description 39 40 _cached_api_description = edq.util.json.load_path(API_DESCRIPTION_PATH, strict = True) 41 42 return _cached_api_description 43 44def content_equals_noramlize_json(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 45 """ 46 A CLI test assertion function for JSON output. 47 The output will be converted to a dict and then compared with edq.testing.unittest.BaseTest.assertJSONDictEqual(). 48 """ 49 50 # Convert both to dicts. 51 expected_dict = edq.util.json.loads(expected, strict = True) 52 actual_dict = edq.util.json.loads(actual, strict = True) 53 54 # Normalize the actual data (the expected should already be normalized (by the tester)). 55 actual_dict = normalize_dict(actual_dict) 56 actual_dict = _normalize_timestamps(actual_dict) 57 58 test.assertJSONDictEqual(expected_dict, actual_dict) 59 60def contains(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 61 """ 62 A CLI test assertion function that checks if the expected output is a contained in the actual output. 63 """ 64 65 test.assertIn(expected, actual) 66 67def content_equals_noramlize_regrade(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 68 """ A CLI test assertion function for regrade output. """ 69 70 # Convert both to dicts. 71 expected_dict = edq.util.json.loads(expected, strict = True) 72 actual_dict = edq.util.json.loads(actual, strict = True) 73 74 # Normalize the actual data (the expected should already be normalized (by the tester)). 75 actual_dict = normalize_dict(actual_dict) 76 actual_dict = _normalize_timestamps(actual_dict, keys = {'grading_start_time', 'regrade-cutoff'}) 77 78 test.assertJSONDictEqual(expected_dict, actual_dict) 79 80def content_equals_stack_trace(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 81 """ 82 A CLI test assertion function for checking if the output looks like a stack trace. 83 """ 84 85 # Actual output should be JSON. 86 actual_dict = edq.util.json.loads(actual, strict = True) 87 88 test.assertGreater(actual_dict['count'], 5) 89 test.assertEqual(actual_dict['count'], len(actual_dict['stacks'])) 90 91def normalize_dict(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: 92 """ Noramlize a dict that typically comes from testing output. """ 93 94 data = _noramlize_version(data) 95 data = _noramlize_tokens(data) 96 97 return data 98 99def normalize_analysis(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: 100 """ Noramlize a dict that comes from an analysis result. """ 101 102 _normalize_timestamps(data) 103 return data 104 105def _noramlize_version(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: 106 """ Normalize server version information. """ 107 108 if ('server-version' not in data): 109 return data 110 111 data['server-version']['base-version'] = autograder.testing.constants.TEST_BASE_VERSION 112 data['server-version']['git-hash'] = autograder.testing.constants.TEST_GIT_HASH 113 data['server-version']['is-dirty'] = autograder.testing.constants.TEST_IS_DIRTY 114 115 return data 116 117def _noramlize_tokens(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: 118 """ Normalize token identifiers. """ 119 120 if ('token-id' in data): 121 data['token-id'] = autograder.testing.constants.TEST_TOKEN_ID 122 123 if ('token-cleartext' in data): 124 data['token-cleartext'] = autograder.testing.constants.TEST_TOKEN_CLEARTEXT 125 126 if ('token-info' in data): 127 _normalize_token_info(data['token-info']) 128 129 if ('tokens' in data): 130 if (data['tokens'] is not None): 131 for token_info in data['tokens']: 132 _normalize_token_info(token_info) 133 134 return data 135 136def _normalize_token_info(token_info: typing.Union[typing.Dict[str, typing.Any], None]) -> None: 137 if (token_info is None): 138 return 139 140 token_info['access-time'] = autograder.testing.constants.TEST_TIMESTAMP 141 token_info['creation-time'] = autograder.testing.constants.TEST_TIMESTAMP 142 token_info['id'] = autograder.testing.constants.TEST_TOKEN_ID 143 144def equals_clean_imageinfo(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 145 """ 146 A CLI test assertion function for JSON image info. 147 """ 148 149 # Convert both to dicts. 150 expected_dict = edq.util.json.loads(expected, strict = True) 151 actual_dict = edq.util.json.loads(actual, strict = True) 152 153 for content in [expected_dict, actual_dict]: 154 content['created-timestamp'] = autograder.testing.constants.TEST_TIMESTAMP 155 content['size-bytes'] = len(autograder.testing.constants.TEST_PAYLOAD_BYTES) 156 157 test.assertJSONDictEqual(expected_dict, actual_dict) 158 159def equals_api_description(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 160 """ A CLI test assertion function for the API description (read from a submodule). """ 161 162 expected_dict = get_expected_api_description() 163 actual_dict = edq.util.json.loads(actual, strict = True) 164 165 test.assertJSONDictEqual(expected_dict, actual_dict) 166 167def equals_clean_tokens(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 168 """ A CLI test assertion function for text that contains tokens. """ 169 170 expected = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, expected) 171 expected = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, expected) 172 173 actual = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, actual) 174 actual = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, actual) 175 176 test.assertEqual(expected, actual) 177 178def equals_clean_pretty_time(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 179 """ A CLI test assertion function for text that contains pretty timestamps. """ 180 181 actual = re.sub(PRETTY_TIME_PATTERN, PRETTY_TIME_REPLACEMENT, actual) 182 183 edq.testing.asserts.content_equals_normalize(test, expected, actual) 184 185def _normalize_timestamps(data: typing.Any, keys: typing.Union[typing.Set[str], None] = None) -> typing.Any: 186 """ 187 Recursively normalize specified timestamps. 188 It is assumed that the specified data comes from JSON deserialization 189 (and therefore has a limited number of types). 190 """ 191 192 if (keys is None): 193 keys = NORMALIZE_TIMESTAMP_KEYS 194 195 if (isinstance(data, list)): 196 return [_normalize_timestamps(item, keys = keys) for item in data] 197 198 if (isinstance(data, dict)): 199 for (key, value) in data.items(): 200 if (key in keys): 201 data[key] = autograder.testing.constants.TEST_TIMESTAMP 202 else: 203 data[key] = _normalize_timestamps(value, keys = keys) 204 205 return data
THIS_DIR: str =
'/home/runner/work/autograder-py/autograder-py/autograder/testing'
ROOT_DIR: str =
'/home/runner/work/autograder-py/autograder-py/autograder/testing/../..'
AUTOGRADER_TESTDATA_REPO: str =
'/home/runner/work/autograder-py/autograder-py/autograder/testing/../../testdata/autograder-testdata'
AUTOGRADER_SERVER_REPO: str =
'/home/runner/work/autograder-py/autograder-py/autograder/testing/../../testdata/autograder-testdata/autograder-server'
API_DESCRIPTION_PATH: str =
'/home/runner/work/autograder-py/autograder-py/autograder/testing/../../testdata/autograder-testdata/autograder-server/resources/api.json'
TOKEN_CLEARTEXT_PATTERN: str =
'\\w{32}'
TOKEN_ID_PATTERN: str =
'\\w{8}-\\w{4}-\\w{4}-\\w{4}-\\w{12}'
PRETTY_TIME_PATTERN: str =
'\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d+[\\+\\-]\\d+:\\d+'
PRETTY_TIME_REPLACEMENT: str =
'<PRETTY TIME>'
NORMALIZE_TIMESTAMP_KEYS: Set[str] =
{'analysis-timestamp', 'first-timestamp', 'last-timestamp'}
Keys for timestamp values to normalize.
def
get_expected_api_description() -> Dict[str, Any]:
33def get_expected_api_description() -> typing.Dict[str, typing.Any]: 34 """ Get the expected API description from the submodule (or the cache). """ 35 36 global _cached_api_description # pylint: disable=global-statement 37 38 if (_cached_api_description is not None): 39 return _cached_api_description 40 41 _cached_api_description = edq.util.json.load_path(API_DESCRIPTION_PATH, strict = True) 42 43 return _cached_api_description
Get the expected API description from the submodule (or the cache).
def
content_equals_noramlize_json(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
45def content_equals_noramlize_json(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 46 """ 47 A CLI test assertion function for JSON output. 48 The output will be converted to a dict and then compared with edq.testing.unittest.BaseTest.assertJSONDictEqual(). 49 """ 50 51 # Convert both to dicts. 52 expected_dict = edq.util.json.loads(expected, strict = True) 53 actual_dict = edq.util.json.loads(actual, strict = True) 54 55 # Normalize the actual data (the expected should already be normalized (by the tester)). 56 actual_dict = normalize_dict(actual_dict) 57 actual_dict = _normalize_timestamps(actual_dict) 58 59 test.assertJSONDictEqual(expected_dict, actual_dict)
A CLI test assertion function for JSON output. The output will be converted to a dict and then compared with edq.testing.unittest.BaseTest.assertJSONDictEqual().
def
contains(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
61def contains(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 62 """ 63 A CLI test assertion function that checks if the expected output is a contained in the actual output. 64 """ 65 66 test.assertIn(expected, actual)
A CLI test assertion function that checks if the expected output is a contained in the actual output.
def
content_equals_noramlize_regrade(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
68def content_equals_noramlize_regrade(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 69 """ A CLI test assertion function for regrade output. """ 70 71 # Convert both to dicts. 72 expected_dict = edq.util.json.loads(expected, strict = True) 73 actual_dict = edq.util.json.loads(actual, strict = True) 74 75 # Normalize the actual data (the expected should already be normalized (by the tester)). 76 actual_dict = normalize_dict(actual_dict) 77 actual_dict = _normalize_timestamps(actual_dict, keys = {'grading_start_time', 'regrade-cutoff'}) 78 79 test.assertJSONDictEqual(expected_dict, actual_dict)
A CLI test assertion function for regrade output.
def
content_equals_stack_trace(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
81def content_equals_stack_trace(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 82 """ 83 A CLI test assertion function for checking if the output looks like a stack trace. 84 """ 85 86 # Actual output should be JSON. 87 actual_dict = edq.util.json.loads(actual, strict = True) 88 89 test.assertGreater(actual_dict['count'], 5) 90 test.assertEqual(actual_dict['count'], len(actual_dict['stacks']))
A CLI test assertion function for checking if the output looks like a stack trace.
def
normalize_dict(data: Dict[str, Any]) -> Dict[str, Any]:
92def normalize_dict(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: 93 """ Noramlize a dict that typically comes from testing output. """ 94 95 data = _noramlize_version(data) 96 data = _noramlize_tokens(data) 97 98 return data
Noramlize a dict that typically comes from testing output.
def
normalize_analysis(data: Dict[str, Any]) -> Dict[str, Any]:
100def normalize_analysis(data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: 101 """ Noramlize a dict that comes from an analysis result. """ 102 103 _normalize_timestamps(data) 104 return data
Noramlize a dict that comes from an analysis result.
def
equals_clean_imageinfo(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
145def equals_clean_imageinfo(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 146 """ 147 A CLI test assertion function for JSON image info. 148 """ 149 150 # Convert both to dicts. 151 expected_dict = edq.util.json.loads(expected, strict = True) 152 actual_dict = edq.util.json.loads(actual, strict = True) 153 154 for content in [expected_dict, actual_dict]: 155 content['created-timestamp'] = autograder.testing.constants.TEST_TIMESTAMP 156 content['size-bytes'] = len(autograder.testing.constants.TEST_PAYLOAD_BYTES) 157 158 test.assertJSONDictEqual(expected_dict, actual_dict)
A CLI test assertion function for JSON image info.
def
equals_api_description(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
160def equals_api_description(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 161 """ A CLI test assertion function for the API description (read from a submodule). """ 162 163 expected_dict = get_expected_api_description() 164 actual_dict = edq.util.json.loads(actual, strict = True) 165 166 test.assertJSONDictEqual(expected_dict, actual_dict)
A CLI test assertion function for the API description (read from a submodule).
def
equals_clean_tokens(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
168def equals_clean_tokens(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 169 """ A CLI test assertion function for text that contains tokens. """ 170 171 expected = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, expected) 172 expected = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, expected) 173 174 actual = re.sub(TOKEN_CLEARTEXT_PATTERN, autograder.testing.constants.TEST_TOKEN_CLEARTEXT, actual) 175 actual = re.sub(TOKEN_ID_PATTERN, autograder.testing.constants.TEST_TOKEN_ID, actual) 176 177 test.assertEqual(expected, actual)
A CLI test assertion function for text that contains tokens.
def
equals_clean_pretty_time(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None:
179def equals_clean_pretty_time(test: edq.testing.unittest.BaseTest, expected: str, actual: str) -> None: 180 """ A CLI test assertion function for text that contains pretty timestamps. """ 181 182 actual = re.sub(PRETTY_TIME_PATTERN, PRETTY_TIME_REPLACEMENT, actual) 183 184 edq.testing.asserts.content_equals_normalize(test, expected, actual)
A CLI test assertion function for text that contains pretty timestamps.