eegdash 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of eegdash might be problematic. Click here for more details.

Files changed (71) hide show
  1. eegdash/data_utils.py +5 -1
  2. eegdash/{aws_ingest.py → script.py} +8 -12
  3. eegdash/signalstore_data_utils.py +352 -2
  4. {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/METADATA +75 -8
  5. eegdash-0.0.2.dist-info/RECORD +10 -0
  6. {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/WHEEL +1 -1
  7. eegdash/SignalStore/__init__.py +0 -0
  8. eegdash/SignalStore/signalstore/__init__.py +0 -3
  9. eegdash/SignalStore/signalstore/adapters/read_adapters/abstract_read_adapter.py +0 -13
  10. eegdash/SignalStore/signalstore/adapters/read_adapters/domain_modeling/schema_read_adapter.py +0 -16
  11. eegdash/SignalStore/signalstore/adapters/read_adapters/domain_modeling/vocabulary_read_adapter.py +0 -19
  12. eegdash/SignalStore/signalstore/adapters/read_adapters/handmade_records/excel_study_organizer_read_adapter.py +0 -114
  13. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/axona/axona_read_adapter.py +0 -912
  14. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/ReadIntanSpikeFile.py +0 -140
  15. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/intan_read_adapter.py +0 -29
  16. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/__init__.py +0 -0
  17. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/data_to_result.py +0 -62
  18. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/get_bytes_per_data_block.py +0 -36
  19. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/notch_filter.py +0 -50
  20. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/qstring.py +0 -41
  21. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/read_header.py +0 -135
  22. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/read_one_data_block.py +0 -45
  23. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/load_intan_rhd_format.py +0 -204
  24. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/__init__.py +0 -0
  25. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/data_to_result.py +0 -60
  26. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/get_bytes_per_data_block.py +0 -37
  27. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/notch_filter.py +0 -50
  28. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/qstring.py +0 -41
  29. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/read_header.py +0 -153
  30. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/read_one_data_block.py +0 -47
  31. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/load_intan_rhs_format.py +0 -213
  32. eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/neurodata_without_borders/neurodata_without_borders_read_adapter.py +0 -14
  33. eegdash/SignalStore/signalstore/operations/__init__.py +0 -4
  34. eegdash/SignalStore/signalstore/operations/handler_executor.py +0 -22
  35. eegdash/SignalStore/signalstore/operations/handler_factory.py +0 -41
  36. eegdash/SignalStore/signalstore/operations/handlers/base_handler.py +0 -44
  37. eegdash/SignalStore/signalstore/operations/handlers/domain/property_model_handlers.py +0 -79
  38. eegdash/SignalStore/signalstore/operations/handlers/domain/schema_handlers.py +0 -3
  39. eegdash/SignalStore/signalstore/operations/helpers/abstract_helper.py +0 -17
  40. eegdash/SignalStore/signalstore/operations/helpers/neuroscikit_extractor.py +0 -33
  41. eegdash/SignalStore/signalstore/operations/helpers/neuroscikit_rawio.py +0 -165
  42. eegdash/SignalStore/signalstore/operations/helpers/spikeinterface_helper.py +0 -100
  43. eegdash/SignalStore/signalstore/operations/helpers/wrappers/neo_wrappers.py +0 -21
  44. eegdash/SignalStore/signalstore/operations/helpers/wrappers/nwb_wrappers.py +0 -27
  45. eegdash/SignalStore/signalstore/store/__init__.py +0 -8
  46. eegdash/SignalStore/signalstore/store/data_access_objects.py +0 -1181
  47. eegdash/SignalStore/signalstore/store/datafile_adapters.py +0 -131
  48. eegdash/SignalStore/signalstore/store/repositories.py +0 -928
  49. eegdash/SignalStore/signalstore/store/store_errors.py +0 -68
  50. eegdash/SignalStore/signalstore/store/unit_of_work.py +0 -97
  51. eegdash/SignalStore/signalstore/store/unit_of_work_provider.py +0 -67
  52. eegdash/SignalStore/signalstore/utilities/data_adapters/spike_interface_adapters/si_recording.py +0 -1
  53. eegdash/SignalStore/signalstore/utilities/data_adapters/spike_interface_adapters/si_sorter.py +0 -1
  54. eegdash/SignalStore/signalstore/utilities/testing/data_mocks.py +0 -513
  55. eegdash/SignalStore/signalstore/utilities/tools/dataarrays.py +0 -49
  56. eegdash/SignalStore/signalstore/utilities/tools/mongo_records.py +0 -25
  57. eegdash/SignalStore/signalstore/utilities/tools/operation_response.py +0 -78
  58. eegdash/SignalStore/signalstore/utilities/tools/purge_orchestration_response.py +0 -21
  59. eegdash/SignalStore/signalstore/utilities/tools/quantities.py +0 -15
  60. eegdash/SignalStore/signalstore/utilities/tools/strings.py +0 -38
  61. eegdash/SignalStore/signalstore/utilities/tools/time.py +0 -17
  62. eegdash/SignalStore/tests/conftest.py +0 -799
  63. eegdash/SignalStore/tests/data/valid_data/data_arrays/make_fake_data.py +0 -59
  64. eegdash/SignalStore/tests/unit/store/conftest.py +0 -0
  65. eegdash/SignalStore/tests/unit/store/test_data_access_objects.py +0 -1235
  66. eegdash/SignalStore/tests/unit/store/test_repositories.py +0 -1309
  67. eegdash/SignalStore/tests/unit/store/test_unit_of_work.py +0 -7
  68. eegdash/SignalStore/tests/unit/test_ci_cd.py +0 -8
  69. eegdash-0.0.1.dist-info/RECORD +0 -72
  70. {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/LICENSE +0 -0
  71. {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/top_level.txt +0 -0
@@ -1,1235 +0,0 @@
1
- import pytest
2
- from datetime import datetime, timezone, timedelta
3
- from signalstore.store.data_access_objects import *
4
-
5
- class TestDomainModelDAO:
6
-
7
- # Get tests (test all expected behaviors of get())
8
- # -----------------------------------------------
9
- # Category 1: Get a single item that exists; check that it has the right values; check that it has the right keys
10
- # Test 1.1: Get a single property model that exists; check that it has the right values; check that it has the right keys
11
- # Test 1.2: Get a single metamodel model that exists; check that it has the right values; check that it has the right keys
12
- # Test 1.3: Get a single data model that exists; check that it has the right values; check that it has the right keys
13
- # Category 2: Getting a single item that does not exist; check that it returns None
14
- # Test 2.1: Getting a single item that does not exist; check that it returns None
15
- # Category 3: Getting a single item with a bad schema_name argument (error)
16
- # Test 3.1: Getting a single item with a bad schema_name argument (error)
17
-
18
- @pytest.mark.parametrize('schema_name', ['dimension_of_measure', 'record_metamodel', 'spike_waveforms'])
19
- def test_get_item_that_exists(self, populated_domain_model_dao, schema_name):
20
- item = populated_domain_model_dao.get(schema_name=schema_name)
21
- assert item.get('schema_name') == schema_name, f'Expected schema_name to be "{schema_name}", got "{item.get("schema_name")}"'
22
- tos = item.get('time_of_save')
23
- assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
24
-
25
- def test_get_item_that_does_not_exist(self, populated_domain_model_dao):
26
- assert populated_domain_model_dao.get(schema_name='not_a_schema') is None
27
-
28
- def test_get_item_with_bad_schema_name_argument(self, populated_domain_model_dao):
29
- with pytest.raises(MongoDAOTypeError):
30
- populated_domain_model_dao.get(schema_name=1)
31
-
32
- # Exists tests (test all expected behaviors of exists())
33
- # ------------------------------------------------------
34
- # Category 1: Check if a single item exists that does exist
35
- # Test 1.1: Check if a single property model exists that does exist
36
- # Test 1.2: Check if a single metamodel model exists that does exist
37
- # Test 1.3: Check if a single data model exists that does exist
38
- # Category 2: Check if a single item exists that does not exist
39
- # Test 2.1: Check if a single model exists that does not exist
40
-
41
- @pytest.mark.parametrize('schema_name', ['dimension_of_measure', 'record_metamodel', 'spike_waveforms'])
42
- def test_exists__model_that_exists(self, populated_domain_model_dao, schema_name):
43
- assert populated_domain_model_dao.exists(schema_name=schema_name)
44
-
45
- def test_exists_model_that_does_not_exist(self, populated_domain_model_dao):
46
- assert not populated_domain_model_dao.exists(schema_name='not_a_schema')
47
-
48
- # Find (query) tests (test all expected behaviors of find())
49
- # ----------------------------------------------------
50
- # Category 1: Find items with no arguments
51
- # Test 1.1: Find all items with no arguments; check that it returns the right items
52
- # Category 2: Find items that exist with valid filter argument only
53
- # Test 2.1: Find all the models of a certain schema_type; check that it returns the right models
54
- # Test 2.2: Find all models with a certain metamodel_ref value; check that it returns the right models
55
- # Test 2.3: Find all the models with a property none of them have; check that it returns an empty list
56
- # Test 2.4: Find all the models with a property they have but a value none of them have; check that it returns an empty list
57
- # Category 3: Find items with a valid filter argument and a valid projection argument
58
- # Test 3.1: Find just the schema_name of all the models of a certain schema_type; check that it returns the right models
59
- # Test 3.2: Find just the schema_name and schema_description of all the models of a certain schema_type; check that it returns the right models
60
- # Category 4: Find items with bad arguments (error)
61
- # Test 4.1: Find with a bad filter argument (error)
62
- # Test 4.2: Find with a bad projection argument (error)
63
- # Test 4.3: Find with a bad filter and a bad projection argument (error)
64
-
65
- def test_find_all_models(self, populated_domain_model_dao, data_models, metamodels, property_models):
66
- all_models = data_models + metamodels + property_models
67
- response = populated_domain_model_dao.find()
68
- assert len(response) == len(all_models), f'Expected len(response) to be {len(all_models)}, got {len(response)}'
69
- for record in response:
70
- tos = record.get('time_of_save')
71
- assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
72
-
73
- @pytest.mark.parametrize('schema_type', ['property_model', 'metamodel', 'data_model'])
74
- def test_find_models_of_certain_schema_type(self, populated_domain_model_dao, schema_type):
75
- response = populated_domain_model_dao.find({'schema_type': schema_type})
76
- for record in response:
77
- assert record['schema_type'] == schema_type, f'Expected schema_type to be "{schema_type}", got "{record["schema_type"]}"'
78
- tos = record.get('time_of_save')
79
- assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
80
-
81
- @pytest.mark.parametrize('metamodel_ref', ['record_metamodel', 'xarray_dataarray_metamodel'])
82
- def test_find_models_with_certain_metamodel_ref(self, populated_domain_model_dao, metamodel_ref):
83
- response = populated_domain_model_dao.find({'metamodel_ref': metamodel_ref})
84
- for record in response:
85
- assert record['metamodel_ref'] == metamodel_ref, f'Expected metamodel_ref to be "{metamodel_ref}", got "{record["metamodel_ref"]}"'
86
- tos = record.get('time_of_save')
87
- assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
88
-
89
- def test_find_models_with_property_none_of_them_have(self, populated_domain_model_dao):
90
- response = populated_domain_model_dao.find({'not_a_property': 'not_a_value'})
91
- assert response == []
92
-
93
- def test_find_models_with_property_they_have_but_value_none_of_them_have(self, populated_domain_model_dao):
94
- response = populated_domain_model_dao.find({'schema_type': 'not_a_value'})
95
- assert response == []
96
-
97
- @pytest.mark.parametrize('projection', [{'schema_name': 1, 'schema_type': 1}, {'schema_name': 1, 'schema_type': 1, 'schema_title': 1}])
98
- @pytest.mark.parametrize('schema_type', ['property_model', 'metamodel', 'data_model'])
99
- def test_find_with_schema_type_and_projection(self, populated_domain_model_dao, schema_type, projection):
100
- response = populated_domain_model_dao.find(filter={'schema_type': schema_type}, projection=projection)
101
- for record in response:
102
- assert record['schema_type'] == schema_type, f'Expected schema_type to be "{schema_type}", got "{record["schema_type"]}"'
103
- assert set(record.keys()) == set(projection.keys()), f'Expected record to have keys {set(projection.keys())}, got {set(record.keys())}'
104
-
105
- def test_find_with_bad_filter_argument(self, populated_domain_model_dao):
106
- with pytest.raises(MongoDAOTypeError):
107
- populated_domain_model_dao.find(filter=1)
108
-
109
- def test_find_with_bad_projection_argument(self, populated_domain_model_dao):
110
- with pytest.raises(MongoDAOTypeError):
111
- populated_domain_model_dao.find(projection=1)
112
-
113
- def test_find_with_bad_filter_and_bad_projection_argument(self, populated_domain_model_dao):
114
- with pytest.raises(MongoDAOTypeError):
115
- populated_domain_model_dao.find(filter=1, projection=1)
116
-
117
- # Add tests (test all expected behaviors of add())
118
- # ------------------------------------------------
119
- # Category 1: Add a single item that does not exist; check that it exists; check that it has the right values; check that it has the right keys
120
- # Test 1.1: Add a single model that does not exist; check that it exists; check that it has the right values; check that it has the right keys
121
- # Category 2: Add a single item that does exist (error)
122
- # Test 2.1: Add a single model that does exist (error)
123
- # Category 3: Add a single item with a bad item argument (error)
124
- # Test 3.1: Add a single item with a bad document argument (error)
125
- # Test 3.2: Add a single item with a bad timestamp argument (error)
126
-
127
- def test_add_not_existing_model(self, populated_domain_model_dao):
128
- new_model = {
129
- 'schema_name': 'new_model',
130
- 'schema_type': 'data_model',
131
- 'schema_description': 'New Model',
132
- 'schema_title': 'New Model',
133
- 'json_schema': {}
134
- }
135
- populated_domain_model_dao.add(document=new_model, timestamp=datetime.now().astimezone())
136
- assert populated_domain_model_dao.exists(schema_name='new_model')
137
- # the model should have the same keys, plus a time_of_addition key
138
- # and a time_of_removal key. The time_of_removal key should be None
139
- # and the time_of_addition key should be an integer.
140
- model_with_timestamps = populated_domain_model_dao.get(schema_name='new_model')
141
- assert model_with_timestamps.keys() == {'schema_name', 'schema_type', 'schema_description', 'schema_title', 'json_schema', 'time_of_save', 'time_of_removal', 'version_timestamp'}, f"Expected model to have keys 'schema_name', 'schema_type', 'schema_description', 'schema_title', 'json_schema', 'time_of_addition', 'time_of_removal', got {model_with_timestamps.keys()}"
142
- assert isinstance(model_with_timestamps['time_of_save'], datetime)
143
- assert model_with_timestamps['time_of_removal'] is None
144
- assert model_with_timestamps['schema_name'] == 'new_model'
145
-
146
- def test_add_existing_model(self, populated_domain_model_dao):
147
- with pytest.raises(MongoDAODocumentAlreadyExistsError):
148
- populated_domain_model_dao.add(document={'schema_name': 'dimension_of_measure'}, timestamp=datetime.now().astimezone())
149
-
150
- def test_add_model_with_bad_argument(self, populated_domain_model_dao):
151
- with pytest.raises(MongoDAOTypeError):
152
- populated_domain_model_dao.add(document=1, timestamp=datetime.now().astimezone())
153
-
154
- def test_add_model_with_bad_timestamp_argument(self, populated_domain_model_dao):
155
- with pytest.raises(MongoDAOTypeError):
156
- populated_domain_model_dao.add(document={'schema_name': 'new_model'}, timestamp=1)
157
-
158
- # Mark for deletion tests (test all expected behaviors of mark_for_deletion())
159
- # ----------------------------------------------------------------------------
160
- # Category 1: Mark a single item for deletion that exists; check that it can nolonger be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
161
- # Test 1.1: Mark a single model for deletion that exists; check that it can nolonger be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
162
- # Category 2: Mark a single item for deletion that does not exist (error)
163
- # Test 2.1: Mark a single model for deletion that does not exist (error)
164
- # Category 3: Mark a single item for deletion with a bad argument (error)
165
- # Test 3.1: Mark a single model for deletion with a bad schema_name argument (error)
166
- # Test 3.2: Mark a single model for deletion with a bad timestamp argument (error)
167
-
168
- def test_mark_for_deletion_model_that_exists(self, populated_domain_model_dao):
169
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=datetime.now().astimezone())
170
- assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure'), f"Expected exists to return False, got {populated_domain_model_dao.exists(schema_name='dimension_of_measure')}"
171
- assert populated_domain_model_dao._collection.find_one({'schema_name': 'dimension_of_measure'}, {'_id':0})['time_of_removal'] is not None
172
-
173
- def test_mark_for_deletion_model_that_does_not_exist(self, populated_domain_model_dao):
174
- with pytest.raises(MongoDAODocumentNotFoundError):
175
- populated_domain_model_dao.mark_for_deletion(schema_name='not_a_schema', timestamp=datetime.now().astimezone())
176
-
177
- def test_mark_for_deletion_model_with_bad_argument(self, populated_domain_model_dao):
178
- with pytest.raises(MongoDAOTypeError):
179
- populated_domain_model_dao.mark_for_deletion(schema_name=1, timestamp=datetime.now().astimezone())
180
-
181
- def test_mark_for_deletion_model_with_bad_timestamp(self, populated_domain_model_dao):
182
- with pytest.raises(MongoDAOTypeError):
183
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=1)
184
-
185
- # List marked for deletion tests (test all expected behaviors of list_marked_for_deletion())
186
- # -----------------------------------------------------------------------------------------
187
- # Category 1: List items marked for deletion; check that it returns the right items
188
- # Test 1.1: List models marked for deletion; check that it returns the right models
189
- # Category 2: List items marked for deletion with a time threshold; check that it returns the right items
190
- # Test 2.1: List models marked for deletion before a time threshold; check that the items before the time threshold are returned and the items after the time threshold are not; try a few thresholds
191
- # Category 3: List items marked for deletion with a bad argument (error)
192
- # Test 3.1: List models marked for deletion with a bad time threshold argument (error)
193
-
194
- @pytest.mark.parametrize('to_delete', [['dimension_of_measure', 'record_metamodel', 'spike_waveforms']])
195
- def test_list_marked_for_deletion_returns_all_with_no_arg(self, populated_domain_model_dao, to_delete):
196
- for schema_name in to_delete:
197
- populated_domain_model_dao.mark_for_deletion(schema_name=schema_name, timestamp=datetime.now().astimezone())
198
- response = populated_domain_model_dao.list_marked_for_deletion()
199
- assert len(response) == len(to_delete), f'Expected len(response) to be {len(to_delete)}, got {len(response)}'
200
- for record in response:
201
- assert record['schema_name'] in to_delete, f'Expected schema_name to be in {to_delete}, got {record["schema_name"]}'
202
-
203
- @pytest.mark.parametrize('to_delete', [['dimension_of_measure', 'record_metamodel', 'spike_waveforms']])
204
- @pytest.mark.parametrize('threshold_delta', [timedelta(seconds=1.1), timedelta(seconds=2.1), timedelta(seconds=3.1)])
205
- def test_list_marked_for_deletion_returns_deletions_before_time_threshold(self, populated_domain_model_dao, timestamp, to_delete, threshold_delta):
206
- for i, schema_name in enumerate(to_delete):
207
- populated_domain_model_dao.mark_for_deletion(schema_name=schema_name, timestamp=timestamp + timedelta(seconds=i+1))
208
- response = populated_domain_model_dao.list_marked_for_deletion(time_threshold=timestamp + threshold_delta)
209
- # round threshold_delta to the nearest second to get the expected number of deletions
210
- expected_deletions = round(threshold_delta.total_seconds())
211
- assert len(response) == expected_deletions, f'Expected len(response) to be {expected_deletions}, got {len(response)}'
212
-
213
- def test_list_marked_for_deletion_with_bad_argument(self, populated_domain_model_dao):
214
- with pytest.raises(MongoDAOTypeError):
215
- populated_domain_model_dao.list_marked_for_deletion(1)
216
-
217
- # Restore tests (test all expected behaviors of restore())
218
- # --------------------------------------------------------
219
- # Category 1: Resotre the nth most recent item that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
220
- # Test 1.1: Restore the most recent model that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
221
- # Test 1.2: Restore the most recent model is idempotent; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
222
- # Test 1.3: Restore the 2nd most recent model that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
223
- # Test 1.4: Restore the nth most recent model that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
224
- # Category 2: Restore a single item that already exists (error)
225
- # Test 2.1: Restore a single model that already exists (error)
226
- # Category 3: Restore a single item that was not marked for deletion (error)
227
- # Test 3.1: Restore a single model that exists but was not marked for deletion (error)
228
- # Test 3.2: Restore a single model that does not exist and was not marked for deletion (error)
229
- # Category 4: Restore a single item with a bad schema_name argument (error)
230
- # Test 4.1: Restore a single model with a bad schema_name argument (error)
231
- # Test 4.2: Restore a single model with a bad nth_most_recent argument (error)
232
- # Test 4.3: Restore a single model with a bad timestamp argument (error)
233
- # Category 5: Restore a single item with an out of range nth_most_recent argument(error)
234
- # Test 5.1: Restore a single model with an nth_most_recent argument <= 0 (error)
235
- # Test 5.1: Restore a single model with an nth_most_recent argument > number of versions of the model (error)
236
-
237
- def test_restore_most_recent_model_that_was_marked_for_deletion(self, populated_domain_model_dao, timestamp):
238
- tod = timestamp + timedelta(seconds=1)
239
- tor = timestamp + timedelta(seconds=2)
240
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod)
241
- assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure'), f'Expected exists to return False, got {populated_domain_model_dao.exists(schema_name="dimension_of_measure")}'
242
- populated_domain_model_dao.restore(schema_name='dimension_of_measure')
243
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
244
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == 'Dimension of Measure', f'Expected schema_title to be "Dimension of Measure", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
245
-
246
- def test_restore_most_recent_model_is_idempotent(self, populated_domain_model_dao, timestamp):
247
- tod = timestamp + timedelta(seconds=1)
248
- tor = timestamp + timedelta(seconds=2)
249
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod)
250
- assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
251
- populated_domain_model_dao.restore(schema_name='dimension_of_measure')
252
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
253
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod)
254
- assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
255
- populated_domain_model_dao.restore(schema_name='dimension_of_measure')
256
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
257
-
258
- def test_restore_2nd_most_recent_model_marked_for_deletion(self, populated_domain_model_dao, timestamp):
259
- tod1 = timestamp + timedelta(seconds=1)
260
- tod2 = timestamp + timedelta(seconds=1.5)
261
- dom_record = populated_domain_model_dao.get(schema_name='dimension_of_measure')
262
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod1)
263
- assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
264
- populated_domain_model_dao.add(document=dom_record, timestamp=timestamp)
265
- assert populated_domain_model_dao.exists(schema_name='dimension_of_measure')
266
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod2)
267
- assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
268
- populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent=2)
269
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
270
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == 'Dimension of Measure', f'Expected schema_title to be "Dimension of Measure", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
271
-
272
- @pytest.mark.parametrize('nth_most_recent', [2, 3, 4, 5, 6])
273
- def test_restore_nth_most_recent_model_marked_for_deletion(self, populated_domain_model_dao, timestamp, nth_most_recent):
274
- for i in range(nth_most_recent-1):
275
- dom_doc = populated_domain_model_dao.get(schema_name='dimension_of_measure')
276
- dom_doc['schema_title'] = f'Dimension of Measure V{i+1}'
277
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp + timedelta(seconds=i))
278
- populated_domain_model_dao.add(document=dom_doc, timestamp=timestamp)
279
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp + timedelta(seconds=i+1))
280
- assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
281
- populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent=nth_most_recent)
282
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
283
- if nth_most_recent > 1:
284
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == f'Dimension of Measure V{nth_most_recent-1}', f'Expected schema_title to be "Dimension of Measure V{nth_most_recent-1}", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
285
- else:
286
- assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == 'Dimension of Measure', f'Expected schema_title to be "Dimension of Measure", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
287
-
288
- def test_restore_model_that_already_exists(self, populated_domain_model_dao, timestamp):
289
- dom_doc = populated_domain_model_dao.get(schema_name='dimension_of_measure')
290
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp)
291
- populated_domain_model_dao.add(document=dom_doc, timestamp=timestamp)
292
- with pytest.raises(MongoDAODocumentAlreadyExistsError):
293
- populated_domain_model_dao.restore(schema_name='dimension_of_measure')
294
-
295
- def test_restore_model_that_was_not_marked_for_deletion(self, populated_domain_model_dao):
296
- with pytest.raises(MongoDAORangeError):
297
- populated_domain_model_dao.restore(schema_name='dimension_of_measure')
298
-
299
- def test_restore_model_with_bad_schema_name_argument(self, populated_domain_model_dao, timestamp):
300
- with pytest.raises(MongoDAOTypeError):
301
- populated_domain_model_dao.restore(schema_name=1, timestamp=timestamp)
302
-
303
- def test_restore_model_with_bad_nth_most_recent_argument(self, populated_domain_model_dao):
304
- with pytest.raises(MongoDAOTypeError):
305
- populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent='1')
306
-
307
- def test_restore_model_with_bad_timestamp_argument(self, populated_domain_model_dao):
308
- with pytest.raises(MongoDAOTypeError):
309
- populated_domain_model_dao.restore(schema_name='dimension_of_measure', timestamp=1)
310
-
311
- @pytest.mark.parametrize('n', [-5, 0, 7])
312
- def test_restore_model_with_out_of_range_nth_most_recent_argument(self, populated_domain_model_dao, timestamp, n):
313
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp)
314
- with pytest.raises(MongoDAORangeError):
315
- populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent=n)
316
-
317
- # purge tests (test all expected behaviors of purge())
318
- # ----------------------------------------------------
319
- # Category 1: Purge with and without a time threshold; check that it deletes the appropriate items
320
- # Test 1.1: Purge with no time threshold; check that it deletes all items marked for deletion
321
- # Test 1.2: Purge with a time threshold; check that it deletes all items marked for deletion before the time threshold
322
- # Category 2: Purge with a bad argument (error)
323
- # Test 2.1: Purge with a bad time threshold argument (error)
324
-
325
- def test_purge_with_no_time_threshold(self, populated_domain_model_dao, timestamp):
326
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp)
327
- populated_domain_model_dao.mark_for_deletion(schema_name='record_metamodel', timestamp=timestamp)
328
- populated_domain_model_dao.mark_for_deletion(schema_name='spike_waveforms', timestamp=timestamp)
329
- assert len(populated_domain_model_dao.list_marked_for_deletion()) == 3
330
- populated_domain_model_dao.purge()
331
- assert populated_domain_model_dao.list_marked_for_deletion() == []
332
-
333
- @pytest.mark.parametrize('threshold_delta', [timedelta(seconds=1.1), timedelta(seconds=2.1), timedelta(seconds=3.1)])
334
- def test_purge_with_time_threshold(self, populated_domain_model_dao, timestamp, threshold_delta):
335
- populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp + timedelta(seconds=1))
336
- populated_domain_model_dao.mark_for_deletion(schema_name='record_metamodel', timestamp=timestamp + timedelta(seconds=2))
337
- populated_domain_model_dao.mark_for_deletion(schema_name='spike_waveforms', timestamp=timestamp + timedelta(seconds=3))
338
- populated_domain_model_dao.purge(time_threshold=timestamp + threshold_delta)
339
- # round threshold_delta to the nearest second to get the expected number of deletions
340
- expected_deletions = round(threshold_delta.total_seconds())
341
- assert len(populated_domain_model_dao.list_marked_for_deletion()) == 3 - expected_deletions
342
-
343
- def test_purge_with_bad_argument(self, populated_domain_model_dao):
344
- with pytest.raises(MongoDAOTypeError):
345
- populated_domain_model_dao.purge(1)
346
-
347
-
348
- class TestFileSystemDAO:
349
-
350
- # Get tests (test all expected behaviors of get())
351
- # -----------------------------------------------
352
- # Category 1A: Unversioned files that exist with default data adapter set
353
- # Test 1a.1: Get a single unversioned NetCDF dataarray that exists; check that it has the right values; check that it has the right keys
354
- # Test 1a.2: Get a single Zarr dataarray that exists; check that it has the right values; check that it has the right keys
355
- # Test 1a.3: Get a single versioned numpy mock model object that exists; check that it has the right values; check that it has the right keys
356
- # Category 1B: Versioned files that exist with default data adapter set
357
- # Test 1b.1: Get a single versioned (numpy model) file that exists with its timestamp.
358
- # Test 1b.2: Get a single versioned (numpy model) file that exists with nth_most_recent.
359
- # Test 1b.3: Get all the versions of a (numpy model) file that exist.
360
- # Category 1C: Unversioned files that exist with non-default data adapter prvided
361
- # Test 1c.1: Get a single unversioned NetCDF dataarray that exists and non-default data-adapter; check that it has the right values; check that it has the right keys
362
- # Test 1c.2: Get a single Zarr dataarray that exists and non-default data-adapter; check that it has the right values; check that it has the right keys
363
- # Test 1c.3: Get a single versioned numpy mock model object that exists and non-default data-adapter; check that it has the right values; check that it has the right keys
364
- # Category 1D: Versioned files that exist with default data adapter set
365
- # Test 1d.1: Get a single versioned (numpy model) file that exists and non-default data-adapter with its timestamp.
366
- # Test 1d.2: Get a single versioned (numpy model) file that exists and non-default data-adapter with nth_most_recent.
367
- # Test 1d.3: Get all the versions of a (numpy model) file that exist and non-default data-adapter.
368
- # Category 2: Files that do not exist
369
- # Test 2.1: Get a single NetCDF dataarray file that does not exist; check that it returns None
370
- # Test 2.2: Get a single Zarr dataarray file that does not exist; check that it returns None
371
- # Test 2.3: Get a single versioned numpy mock model object that does not exist; check that it returns None
372
- # Category 2: Bad arguments
373
- # Test 2.1: Get a single file with a bad schema_name argument (error)
374
- # Test 2.2: Get a single file with a bad data_name argument (error)
375
- # Test 2.3: Get a single file with a bad data_version argument (error)
376
-
377
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
378
- @pytest.mark.parametrize('use_data_adapter', [True, False])
379
- def test_get_file_that_exists(self, file_dao_options, data_adapter_options, use_data_adapter, file_type):
380
- file_dao = file_dao_options[file_type]
381
- if use_data_adapter:
382
- data_adapter = data_adapter_options[file_type]
383
- else:
384
- data_adapter = None
385
- data_object = file_dao.get(schema_ref='test', data_name='test', data_adapter=data_adapter)
386
- schema_ref = data_object.attrs.get('schema_ref')
387
- data_name = data_object.attrs.get('data_name')
388
- data_version = data_object.attrs.get('data_version')
389
- assert schema_ref == 'test', f'Expected schema_ref to be "spike_waveforms", got "{schema_ref}"'
390
- assert data_name == 'test', f'Expected data_name to be "spike_waveforms", got "{data_name}"'
391
- assert data_version == None, f'Expected data_version to be None, got "{data_version}"'
392
-
393
- @pytest.mark.parametrize('use_data_adapter', [True, False])
394
- def test_get_versioned_file_with_timestamp(self, populated_numpy_file_dao, model_numpy_adapter, use_data_adapter, timestamp):
395
- if use_data_adapter:
396
- data_adapter = model_numpy_adapter
397
- else:
398
- data_adapter = None
399
- ts = timestamp + timedelta(seconds=10)
400
- model = populated_numpy_file_dao.get(schema_ref='test', data_name='test', data_adapter=data_adapter)
401
- result = model.attrs["version_timestamp"]
402
- assert result == ts, f"Expected {ts} but got {result}."
403
-
404
- @pytest.mark.parametrize('n', range(11))
405
- def test_get_versioned_file_with_nth_most_recent(self, populated_numpy_file_dao, n):
406
- checkpoint = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=n)
407
- assert checkpoint is not None
408
-
409
- def test_get_all_versions_of_file_that_exists(self, populated_numpy_file_dao):
410
- checkpoint = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
411
- checkpoints = []
412
- n = 2
413
- while checkpoint is not None:
414
- checkpoints.append(checkpoint)
415
- checkpoint = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=n)
416
- n += 1
417
- assert len(checkpoints) == 10
418
- assert len(checkpoints) == populated_numpy_file_dao.n_versions(schema_ref='test', data_name='test')
419
-
420
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
421
- def test_get_file_that_does_not_exist(self, file_dao_options, file_type):
422
- file_dao = file_dao_options[file_type]
423
- assert file_dao.get(schema_ref='not_a_schema', data_name='not_a_data_name', data_adapter=None) is None
424
- assert file_dao.get(schema_ref='test', data_name='not_a_data_name', data_adapter=None) is None
425
- assert file_dao.get(schema_ref='test', data_name='test', version_timestamp=datetime.now().astimezone()+timedelta(seconds=5)) is None
426
-
427
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
428
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
429
- def test_get_file_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
430
- file_dao = file_dao_options[file_type]
431
- with pytest.raises(FileSystemDAOTypeError):
432
- data_object = file_dao.get(schema_ref=bad_arg, data_name='test', data_adapter=None)
433
-
434
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
435
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
436
- def test_get_file_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
437
- file_dao = file_dao_options[file_type]
438
- with pytest.raises(FileSystemDAOTypeError):
439
- data_object = file_dao.get(schema_ref='test', data_name=bad_arg, data_adapter=None)
440
-
441
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
442
- @pytest.mark.parametrize('bad_arg', [1.5, None, "string", {"set"}, {"hash": "map"}])
443
- def test_get_file_with_bad_nth_most_recent_arg_arg(self, file_dao_options, file_type, bad_arg):
444
- file_dao = file_dao_options[file_type]
445
- with pytest.raises(FileSystemDAOTypeError):
446
- data_object = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=bad_arg, data_adapter=None)
447
-
448
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
449
- @pytest.mark.parametrize('bad_arg', [1.5, 1, "string", {"set"}, {"hash": "map"}])
450
- def test_get_file_with_bad_version_timestamp_arg_arg(self, file_dao_options, file_type, bad_arg):
451
- file_dao = file_dao_options[file_type]
452
- with pytest.raises(FileSystemDAOTypeError):
453
- data_object = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1, version_timestamp=bad_arg, data_adapter=bad_arg)
454
-
455
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
456
- @pytest.mark.parametrize('bad_arg', [1.5, 1, datetime.now().astimezone(), "string", {"set"}, {"hash": "map"}])
457
- def test_get_file_with_bad_data_adapter_arg_arg(self, file_dao_options, file_type, bad_arg):
458
- file_dao = file_dao_options[file_type]
459
- with pytest.raises(FileSystemDAOTypeError):
460
- data_object = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1, data_adapter=bad_arg)
461
-
462
- # Exists tests (test all expected behaviors of exists())
463
- # ------------------------------------------------------
464
- # Category 1: Unversioned files that exist
465
- # Test 1.1: Check if a single unversioned NetCDF dataarray exists that exists
466
- # Test 1.2: Check if a single unversioned Zarr dataarray exists that exists
467
- # Test 1.3: Check if a single unversioned numpy mock model object exists that exists
468
- # Category 2: Versioned files that exist
469
- # Test 2.1: Check if a single versioned (numpy model) file that exists (with its timestamp).
470
- # Category 3: Files that do not exist
471
- # Test 3.1: Check if a single NetCDF dataarray file that does not exist
472
- # Test 3.2: Check if a single Zarr dataarray file that does not exist
473
- # Test 3.3: Check if a single versioned numpy mock model object that does not exist
474
- # Category 4: Bad arguments
475
- # Test 4.1: Check if a single file with a bad schema_name argument (error)
476
- # Test 4.2: Check if a single file with a bad data_name argument (error)
477
- # Test 4.3: Check if a single file with a bad data_version argument (error)
478
- # Test 4.4: Check if a single file with a bad data_adapter argument (error)
479
-
480
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
481
- def test_exists_unversioned_file_that_exists(self, file_dao_options, file_type):
482
- file_dao = file_dao_options[file_type]
483
- result = file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
484
- assert result, f'Expected exists to return True, got {result}'
485
-
486
- def test_exists_versioned_file_with_timestamp(self, populated_numpy_file_dao):
487
- x = populated_numpy_file_dao.get(schema_ref='test', data_name='test')
488
- ts = x.attrs["version_timestamp"]
489
- result = populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
490
- assert result, f'Expected exists to return True, got {result}'
491
-
492
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
493
- def test_exists_with_file_that_does_not_exist(self, file_dao_options, file_type):
494
- file_dao = file_dao_options[file_type]
495
- result = file_dao.exists(schema_ref='not_a_schema', data_name='not_a_data_name', version_timestamp=0)
496
- assert not result, f'Expected exists to return False, got {result}'
497
-
498
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
499
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
500
- def test_exists_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
501
- file_dao = file_dao_options[file_type]
502
- with pytest.raises(FileSystemDAOTypeError):
503
- file_dao.exists(schema_ref=bad_arg, data_name='test', version_timestamp=0)
504
-
505
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
506
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
507
- def test_exists_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
508
- file_dao = file_dao_options[file_type]
509
- with pytest.raises(FileSystemDAOTypeError):
510
- file_dao.exists(schema_ref='test', data_name=bad_arg, version_timestamp=0)
511
-
512
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
513
- @pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
514
- def test_exists_with_bad_version_timestamp_arg_arg(self, file_dao_options, file_type, bad_arg):
515
- file_dao = file_dao_options[file_type]
516
- with pytest.raises(FileSystemDAOTypeError):
517
- file_dao.exists(schema_ref='test', data_name='test', version_timestamp=bad_arg)
518
-
519
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
520
- @pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
521
- def test_exists_with_bad_data_adapter_arg_arg(self, file_dao_options, file_type, bad_arg):
522
- file_dao = file_dao_options[file_type]
523
- with pytest.raises(FileSystemDAOTypeError):
524
- file_dao.exists(schema_ref='test', data_name='test', data_adapter=bad_arg)
525
-
526
- # N versions tests (test all expected behaviors of n_versions())
527
- # --------------------------------------------------------------
528
- # Category 1: Versioned files that exist
529
- # Test 1.1: Get version count and assert that it is correct
530
- # Category 2: Files that do not exist
531
- # Test 2.1: Get version count and assert that it is 0
532
- # Category 3: Bad arguments
533
- # Test 3.1: Get version count with a bad schema_name argument (error)
534
- # Test 3.2: Get version count with a bad data_name argument (error)
535
-
536
- def test_n_versions_versioned_file_that_exists(self, populated_numpy_file_dao):
537
- result = populated_numpy_file_dao.n_versions(schema_ref='test', data_name='test')
538
- assert result == 10, f'Expected n_versions to return 10, got {result}'
539
-
540
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
541
- def test_n_versions_with_file_that_does_not_exist(self, file_dao_options, file_type):
542
- file_dao = file_dao_options[file_type]
543
- result = file_dao.n_versions(schema_ref='not_a_schema', data_name='not_a_data_name')
544
- assert result == 0, f'Expected n_versions to return 0, got {result}'
545
-
546
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
547
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
548
- def test_n_versions_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
549
- file_dao = file_dao_options[file_type]
550
- with pytest.raises(FileSystemDAOTypeError):
551
- file_dao.n_versions(schema_ref=bad_arg, data_name='test')
552
-
553
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
554
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
555
- def test_n_versions_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
556
- file_dao = file_dao_options[file_type]
557
- with pytest.raises(FileSystemDAOTypeError):
558
- file_dao.n_versions(schema_ref='test', data_name=bad_arg)
559
-
560
- # Add tests (test all expected behaviors of add())
561
- # ------------------------------------------------
562
- # Category 1: Unversioned files that do not exist
563
- # Test 1.1: Add a single unversioned NetCDF dataarray that does not exist; check that it exists; check that it has the right values; check that it has the right keys
564
- # Test 1.2: Add a single unversioned Zarr dataarray that does not exist; check that it exists; check that it has the right values; check that it has the right keys
565
- # Category 2: Versioned files that do not exist
566
- # Test 2.1: Add a single versioned numpy mock model object that does not exist; check that it exists; check that it has the right values; check that it has the right keys
567
- # Category 3: Unversioned that already exist
568
- # Test 3.1: Add a single unversioned NetCDF dataarray that already exists (error)
569
- # Test 3.2: Add a single unversioned Zarr dataarray that already exists (error)
570
- # Category 4: Versioned files that already exist
571
- # Test 4.1: Add a single versioned numpy mock model object that already exists and has the same timestamp (error)
572
- # Test 4.2: Add a single versioned numpy mock model object that already exists and has a different timestamp; check that it add works; check that it has the right values; check that it has the right keys
573
- # Category 5: Bad arguments
574
- # Test 5.1: Add a single file with a bad data_adapter argument (error)
575
- # Category 6: Mismatched data_object, data_adapter pairs
576
- # Test 6.1: Add a single file with a data_object and data_adapter that do not match (several combos) (error)
577
-
578
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
579
- def test_add_unversioned_file_that_does_not_exist(self, file_dao_options, new_object_options, file_type):
580
- file_dao = file_dao_options[file_type]
581
- data_object = new_object_options[file_type]
582
- file_dao.add(data_object=data_object)
583
- result = file_dao.get(schema_ref='new', data_name='new')
584
- assert result is not None
585
- schema_ref = result.attrs.get('schema_ref')
586
- data_name = result.attrs.get('data_name')
587
- assert schema_ref == 'new', f'Expected schema_ref to be "new", got "{schema_ref}"'
588
- assert data_name == 'new', f'Expected data_name to be "new", got "{data_name}"'
589
-
590
- def test_add_versioned_file_that_does_not_exist(self, populated_numpy_file_dao, new_object_options):
591
- data_object = new_object_options['numpy']
592
- populated_numpy_file_dao.add(data_object=data_object)
593
- result = populated_numpy_file_dao.get(schema_ref='new', data_name='new', nth_most_recent=1)
594
- assert result is not None
595
- schema_ref = result.attrs.get('schema_ref')
596
- data_name = result.attrs.get('data_name')
597
- assert schema_ref == 'new', f'Expected schema_ref to be "new", got "{schema_ref}"'
598
- assert data_name == 'new', f'Expected data_name to be "new", got "{data_name}"'
599
-
600
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
601
- def test_add_unversioned_file_that_already_exists(self, file_dao_options, file_type):
602
- file_dao = file_dao_options[file_type]
603
- data_object = file_dao.get(schema_ref='test', data_name='test')
604
- with pytest.raises(FileSystemDAOFileAlreadyExistsError):
605
- file_dao.add(data_object=data_object)
606
-
607
- def test_add_versioned_file_that_already_exists_with_same_timestamp(self, populated_numpy_file_dao):
608
- data_object = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent = 1)
609
- with pytest.raises(FileSystemDAOFileAlreadyExistsError):
610
- populated_numpy_file_dao.add(data_object=data_object)
611
-
612
- def test_add_versioned_file_that_already_exists_with_different_timestamp(self, populated_numpy_file_dao):
613
- data_object = populated_numpy_file_dao.get(schema_ref='test', data_name='test')
614
- new_ts = datetime.now().astimezone() + timedelta(seconds=11)
615
- data_object.attrs['version_timestamp'] = new_ts
616
- populated_numpy_file_dao.add(data_object=data_object)
617
- result = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
618
- assert result is not None
619
- schema_ref = result.attrs.get('schema_ref')
620
- data_name = result.attrs.get('data_name')
621
- version_timestamp = result.attrs.get('version_timestamp')
622
- assert schema_ref == 'test', f'Expected schema_ref to be "test", got "{schema_ref}"'
623
- assert data_name == 'test', f'Expected data_name to be "test", got "{data_name}"'
624
- assert version_timestamp == new_ts, f'Expected version_timestamp to be "{new_ts}", got "{version_timestamp}"'
625
-
626
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
627
- @pytest.mark.parametrize('bad_arg', [1, None, "string", {"set"}, {"hash": "map"}])
628
- def test_add_with_bad_data_adapter_arg(self, file_dao_options, file_type, bad_arg):
629
- file_dao = file_dao_options[file_type]
630
- with pytest.raises(FileSystemDAOTypeError):
631
- file_dao.add(data_object=bad_arg)
632
-
633
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
634
- def test_add_with_mismatched_data_object_and_data_adapter(self, file_dao_options, file_type):
635
- if file_type in ['netcdf', 'zarr']:
636
- bad_type = 'numpy'
637
- else:
638
- bad_type = 'netcdf'
639
- file_dao = file_dao_options[file_type]
640
- other_file_dao = file_dao_options[bad_type]
641
- incompatable_data_object = other_file_dao.get(schema_ref='test', data_name='test')
642
- # set attrs to new
643
- incompatable_data_object.attrs['schema_ref'] = 'new'
644
- incompatable_data_object.attrs['data_name'] = 'new'
645
- with pytest.raises(FileSystemDAOTypeError):
646
- file_dao.add(data_object=incompatable_data_object)
647
-
648
- # Mark for deletion tests (test all expected behaviors of mark_for_deletion())
649
- # -----------------------------------------------------------------------------
650
- # Category 1: Unversioned files that exist
651
- # Test 1.1: Mark a single unversioned NetCDF dataarray that exists for deletion; check that it can nolonger be accessed through get or the exists method
652
- # Test 1.2: Mark a single unversioned Zarr dataarray that exists for deletion; check that it can nolonger be accessed through get or the exists method
653
- # Category 2: Versioned files that exist
654
- # Test 2.1: Mark a single versioned numpy mock model object that exists for deletion; check that it can nolonger be accessed through get or the exists method; check that the other versions can still be accessed
655
- # Category 3: Files that do not exist
656
- # Test 3.1: Mark a single NetCDF dataarray file that does not exist for deletion (error)
657
- # Test 3.2: Mark a single Zarr dataarray file that does not exist for deletion (error)
658
- # Test 3.3: Mark a single versioned numpy mock model object that does not exist for deletion (error)
659
- # Category 4: Bad arguments
660
- # Test 4.1: Mark a single file with a bad schema_name argument (error)
661
- # Test 4.2: Mark a single file with a bad data_name argument (error)
662
- # Test 4.3: Mark a single file with a bad data_version argument (error)
663
- # Test 4.4: Mark a single file with a bad time_of_removal argument (error)
664
- # Test 4.5: Mark a single file with a bad data_adapter argument (error)
665
-
666
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
667
- def test_mark_for_deletion_unversioned_file_that_exists(self, file_dao_options, file_type):
668
- file_dao = file_dao_options[file_type]
669
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
670
- assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
671
- assert file_dao.get(schema_ref='test', data_name='test', version_timestamp=0) is None
672
-
673
- def test_mark_for_deletion_versioned_file_that_exists(self, populated_numpy_file_dao):
674
- to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
675
- ts = to_remove.attrs['version_timestamp']
676
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
677
- assert not populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
678
- assert populated_numpy_file_dao.get(schema_ref='test', data_name='test', version_timestamp=ts) is None
679
- assert populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1) is not None
680
- assert populated_numpy_file_dao.n_versions(schema_ref='test', data_name='test') == 9
681
-
682
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
683
- def test_mark_for_deletion_with_file_that_does_not_exist(self, file_dao_options, file_type):
684
- file_dao = file_dao_options[file_type]
685
- with pytest.raises(FileSystemDAOFileNotFoundError):
686
- file_dao.mark_for_deletion(schema_ref='not_a_schema', data_name='not_a_data_name', time_of_removal=datetime.now().astimezone())
687
-
688
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
689
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
690
- def test_mark_for_deletion_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
691
- file_dao = file_dao_options[file_type]
692
- with pytest.raises(FileSystemDAOTypeError):
693
- file_dao.mark_for_deletion(schema_ref=bad_arg, data_name='test', time_of_removal=datetime.now().astimezone())
694
-
695
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
696
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
697
- def test_mark_for_deletion_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
698
- file_dao = file_dao_options[file_type]
699
- with pytest.raises(FileSystemDAOTypeError):
700
- file_dao.mark_for_deletion(schema_ref='test', data_name=bad_arg, time_of_removal=datetime.now().astimezone())
701
-
702
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
703
- @pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
704
- def test_mark_for_deletion_with_bad_version_timestamp_arg_arg(self, file_dao_options, file_type, bad_arg):
705
- file_dao = file_dao_options[file_type]
706
- with pytest.raises(FileSystemDAOTypeError):
707
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=bad_arg, time_of_removal=datetime.now().astimezone())
708
-
709
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
710
- @pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
711
- def test_mark_for_deletion_with_bad_time_of_removal_arg_arg(self, file_dao_options, file_type, bad_arg):
712
- file_dao = file_dao_options[file_type]
713
- with pytest.raises(FileSystemDAOTypeError):
714
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=bad_arg)
715
-
716
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
717
- @pytest.mark.parametrize('bad_arg', [1.5, 1, datetime.now().astimezone(), "string", {"set"}, {"hash": "map"}])
718
- def test_mark_for_deletion_with_bad_data_adapter_arg_arg(self, file_dao_options, file_type, bad_arg):
719
- file_dao = file_dao_options[file_type]
720
- with pytest.raises(FileSystemDAOTypeError):
721
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone(), data_adapter=bad_arg)
722
-
723
- # List marked for deletion tests (test all expected behaviors of list_marked_for_deletion())
724
- # -----------------------------------------------------------------------------------------
725
- # Category 1: Unversioned files that exist
726
- # Test 1.1: List all unversioned NetCDF dataarrays that exist; check that it returns the right files
727
- # Test 1.2: List all unversioned Zarr dataarrays that exist; check that it returns the right files
728
- # Category 2: Versioned files that exist
729
- # Test 2.1: List all versioned numpy mock model objects that exist; check that it returns the right files
730
- # Category 3: Files that do not exist
731
- # Test 3.1: List all NetCDF dataarray files that do not exist; check that it returns an empty list
732
- # Test 3.2: List all Zarr dataarray files that do not exist; check that it returns an empty list
733
- # Test 3.3: List all versioned numpy mock model objects that do not exist; check that it returns an empty list
734
- # Category 4: Bad arguments
735
- # Test 4.1: List all files with a bad time_threshold argument (error)
736
- # Test 4.2: List all files with a bad data_adapter argument (error)
737
-
738
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
739
- def test_list_marked_for_deletion_unversioned_file_that_exists(self, file_dao_options, file_type):
740
- file_dao = file_dao_options[file_type]
741
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
742
- result = file_dao.list_marked_for_deletion()
743
- assert len(result) == 1
744
-
745
- def test_list_marked_for_deletion_versioned_file_that_exists(self, populated_numpy_file_dao):
746
- to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
747
- ts = to_remove.attrs['version_timestamp']
748
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
749
- result = populated_numpy_file_dao.list_marked_for_deletion()
750
- assert len(result) == 1
751
-
752
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
753
- def test_list_marked_for_deletion_with_file_that_does_not_exist(self, file_dao_options, file_type):
754
- file_dao = file_dao_options[file_type]
755
- result = file_dao.list_marked_for_deletion()
756
- assert len(result) == 0
757
-
758
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
759
- @pytest.mark.parametrize('bad_arg', [1.5, 1, "string", {"set"}, {"hash": "map"}])
760
- def test_list_marked_for_deletion_with_bad_time_threshold_arg(self, file_dao_options, file_type, bad_arg):
761
- file_dao = file_dao_options[file_type]
762
- with pytest.raises(FileSystemDAOTypeError):
763
- file_dao.list_marked_for_deletion(time_threshold=bad_arg)
764
-
765
- # restore tests (test all expected behaviors of restore())
766
- # ---------------------------------------------------------
767
- # Category 1: Unversioned files that exist and have been removed
768
- # Test 1.1: Restore a single unversioned NetCDF dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
769
- # Test 1.2: Restore nth_most_recent unversioned NetCDF dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
770
- # Test 1.3: Restore a single unversioned Zarr dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
771
- # Test 1.4: Restore nth_most_recent unversioned Zarr dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
772
- # Category 2: Versioned files that exist and have been removed
773
- # Test 2.1: Restore a single versioned numpy mock model object that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
774
- # Test 2.2: Restore nth_most_recent versioned numpy mock model object that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
775
- # Category 3: Unversioned files that exist and have not been removed
776
- # Test 3.1: Restore a single unversioned NetCDF dataarray that exists and has not been removed (error)
777
- # Test 3.2: Restore a single unversioned Zarr dataarray that exists and has not been removed (error)
778
- # Category 4: Versioned files that exist and have not been removed
779
- # Test 4.1: Restore a single versioned numpy mock model object that exists and has not been removed (error)
780
- # Category 5: Files that do not exist
781
- # Test 5.1: Restore a single NetCDF dataarray file that does not exist (error)
782
- # Test 5.2: Restore a single Zarr dataarray file that does not exist (error)
783
- # Test 5.3: Restore a single versioned numpy mock model object that does not exist (error)
784
- # Category 6: Bad arguments
785
- # Test 6.1: Restore a single file with a bad schema_name argument (error)
786
- # Test 6.2: Restore a single file with a bad data_name argument (error)
787
- # Test 6.3: Restore a single file with a bad data_version argument (error)
788
- # Test 6.5: Restore a single file with a bad nth_most_recent argument (error)
789
- # Test 6.4: Restore a single file with a bad time_of_removal argument (error)
790
- # Test 6.6: Restore a single file with a bad data_adapter argument (error)
791
-
792
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
793
- def test_restore_unversioned_file_that_exists_and_has_been_removed(self, file_dao_options, file_type):
794
- file_dao = file_dao_options[file_type]
795
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
796
- file_dao.restore(schema_ref='test', data_name='test')
797
- assert file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
798
- assert file_dao.get(schema_ref='test', data_name='test', version_timestamp=0) is not None
799
-
800
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
801
- @pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
802
- def test_restore_nth_most_recent_unversioned_file_that_exists_and_has_been_removed(self, file_dao_options, file_type, n):
803
- file_dao = file_dao_options[file_type]
804
- first_tod = datetime.now().astimezone()
805
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=first_tod)
806
- # restore and mark for deletion n times
807
- for i in range(n):
808
- file_dao.restore(schema_ref='test', data_name='test')
809
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=first_tod+timedelta(seconds=i+1))
810
- assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
811
- # restore one more time
812
- file_dao.restore(schema_ref='test', data_name='test')
813
- assert file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
814
- nth_most_recent = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=n)
815
- assert nth_most_recent is not None
816
-
817
- def test_restore_versioned_file_that_exists_and_has_been_removed(self, populated_numpy_file_dao):
818
- to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
819
- ts = to_remove.attrs['version_timestamp']
820
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
821
- populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=ts)
822
- assert populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
823
- assert populated_numpy_file_dao.get(schema_ref='test', data_name='test', version_timestamp=ts) is not None
824
-
825
- @pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
826
- def test_restore_nth_most_recent_versioned_file_that_exists_and_has_been_removed(self, populated_numpy_file_dao, n):
827
- first_tod = datetime.now().astimezone()
828
- to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
829
- ts = to_remove.attrs['version_timestamp']
830
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=first_tod)
831
- # restore and mark for deletion n times
832
- for i in range(n):
833
- to_remove.attrs['data_name'] = f'test'
834
- populated_numpy_file_dao.add(data_object=to_remove)
835
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name=f'test', version_timestamp=ts, time_of_removal=first_tod+timedelta(seconds=i+1))
836
- assert not populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
837
- # restore one more time
838
- populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=ts, nth_most_recent=n)
839
- assert populated_numpy_file_dao.exists(schema_ref='test', data_name=f'test', version_timestamp=ts)
840
-
841
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
842
- def test_restore_unversioned_file_that_exists_and_has_not_been_removed(self, file_dao_options, file_type):
843
- file_dao = file_dao_options[file_type]
844
- with pytest.raises(FileSystemDAORangeError):
845
- file_dao.restore(schema_ref='test', data_name='test')
846
-
847
- def test_restore_versioned_file_that_exists_and_has_not_been_removed(self, populated_numpy_file_dao):
848
- to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
849
- ts = to_remove.attrs['version_timestamp']
850
- with pytest.raises(FileSystemDAORangeError):
851
- populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=ts)
852
-
853
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
854
- def test_restore_unversioned_file_that_does_not_exist(self, file_dao_options, file_type):
855
- file_dao = file_dao_options[file_type]
856
- with pytest.raises(FileSystemDAORangeError):
857
- file_dao.restore(schema_ref='not_a_schema', data_name='not_a_data_name')
858
-
859
- def test_restore_versioned_file_that_does_not_exist(self, populated_numpy_file_dao):
860
- with pytest.raises(FileSystemDAORangeError):
861
- populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=datetime.now().astimezone())
862
-
863
- # purge tests (test all expected behaviors of purge())
864
- # ---------------------------------------------------------
865
- # Category 1: Purge all unversioned files that exist and have been removed
866
- # Test 1.1: Purge all unversioned NetCDF dataarrays that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
867
- # Test 1.2: Purge all unversioned Zarr dataarrays that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
868
- # Test 1.3: Purge all unversioned NetCDF dataarrays that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
869
- # Test 1.4: Purge all unversioned Zarr dataarrays that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
870
- # Category 2: Purge all versioned files that exist and have been removed
871
- # Test 2.1: Purge all versioned numpy mock model objects that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
872
- # Test 2.2: Purge all versioned numpy mock model objects that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method
873
- # Category 3: Purge only some unversioned files that exist and have been removed; check that the resulting purge count is correct
874
- # Test 3.1: Purge all unversioned NetCDF dataarrays that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
875
- # Test 3.2: Purge all unversioned Zarr dataarrays that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
876
- # Category 4: Purge only some versioned files that exist and have been removed
877
- # Test 4.1: Purge all versioned numpy mock model objects that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
878
- # Category 5: Unversioned files when none have been removed
879
- # Test 5.1: Purge all unversioned NetCDF dataarrays that exist and have not been removed; check that the purge count is 0
880
- # Test 5.2: Purge all unversioned Zarr dataarrays that exist and have not been removed; check that the purge count is 0
881
- # Category 6: Versioned files when none have been removed
882
- # Test 6.1: Purge all versioned numpy mock model objects that exist and have not been removed; check that the purge count is 0
883
-
884
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
885
- def test_purge_all_unversioned_files_that_exist_and_have_been_removed(self, file_dao_options, file_type):
886
- file_dao = file_dao_options[file_type]
887
- to_delete = file_dao.get(schema_ref='test', data_name='test')
888
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
889
- for i in range(9):
890
- to_delete.attrs['data_name'] = f'test{i}'
891
- to_delete.attrs['schema_ref'] = f'test{i}'
892
- file_dao.add(data_object=to_delete)
893
- assert file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
894
- file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', time_of_removal=datetime.now().astimezone(timezone.utc))
895
- count = file_dao.purge(time_threshold=None)
896
- assert count == 10
897
- assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
898
- for i in range(9):
899
- assert not file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
900
- assert len(file_dao.list_marked_for_deletion()) == 0
901
-
902
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
903
- def test_purge_all_unversioned_files_that_exist_and_have_been_removed_with_time_threshold(self, file_dao_options, file_type):
904
- file_dao = file_dao_options[file_type]
905
- to_delete = file_dao.get(schema_ref='test', data_name='test')
906
- file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
907
- for i in range(9):
908
- to_delete.attrs['data_name'] = f'test{i}'
909
- to_delete.attrs['schema_ref'] = f'test{i}'
910
- file_dao.add(data_object=to_delete)
911
- assert file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
912
- file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', time_of_removal=datetime.now().astimezone())
913
- count = file_dao.purge(time_threshold=datetime.now().astimezone(timezone.utc))
914
- assert count == 10
915
- assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
916
- for i in range(9):
917
- assert not file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
918
- assert len(file_dao.list_marked_for_deletion()) == 0
919
-
920
- def test_purge_all_versioned_files_that_exist_and_have_been_removed(self, populated_numpy_file_dao):
921
- to_delete = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
922
- ts = to_delete.attrs['version_timestamp']
923
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
924
- for i in range(9):
925
- to_delete.attrs['data_name'] = f'test{i}'
926
- to_delete.attrs['schema_ref'] = f'test{i}'
927
- populated_numpy_file_dao.add(data_object=to_delete)
928
- assert populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts)
929
- populated_numpy_file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
930
- count = populated_numpy_file_dao.purge(time_threshold=None)
931
- assert count == 10
932
- assert not populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
933
- for i in range(9):
934
- assert not populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts)
935
- assert len(populated_numpy_file_dao.list_marked_for_deletion()) == 0
936
-
937
- def test_purge_all_versioned_files_that_exist_and_have_been_removed_with_time_threshold(self, populated_numpy_file_dao):
938
- to_delete = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
939
- ts = to_delete.attrs['version_timestamp']
940
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
941
- for i in range(9):
942
- to_delete.attrs['data_name'] = f'test{i}'
943
- to_delete.attrs['schema_ref'] = f'test{i}'
944
- populated_numpy_file_dao.add(data_object=to_delete)
945
- assert populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts)
946
- populated_numpy_file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
947
- count = populated_numpy_file_dao.purge(time_threshold=datetime.now().astimezone())
948
- assert count == 10
949
-
950
- @pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
951
- @pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
952
- def test_purge_only_n_unversioned_files_that_exist_and_have_been_removed(self, file_dao_options, file_type, n):
953
- to_delete = file_dao_options[file_type].get(schema_ref='test', data_name='test')
954
- first_tod = datetime.now().astimezone()
955
- file_dao_options[file_type].mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=first_tod)
956
- for i in range(n):
957
- to_delete.attrs['data_name'] = f'test{i}'
958
- to_delete.attrs['schema_ref'] = f'test{i}'
959
- file_dao_options[file_type].add(data_object=to_delete)
960
- assert file_dao_options[file_type].exists(schema_ref=f'test{i}', data_name=f'test{i}')
961
- file_dao_options[file_type].mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', time_of_removal=first_tod + timedelta(seconds=i + 1))
962
- count = file_dao_options[file_type].purge(time_threshold=first_tod+timedelta(seconds=n - 1))
963
- assert count == n
964
-
965
- @pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
966
- def test_purge_only_n_versioned_files_that_exist_and_have_been_removed(self, populated_numpy_file_dao, n):
967
- to_delete = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
968
- ts = to_delete.attrs['version_timestamp']
969
- first_tod = datetime.now().astimezone()
970
- populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=first_tod)
971
- for i in range(n):
972
- to_delete.attrs['data_name'] = f'test{i}'
973
- to_delete.attrs['schema_ref'] = f'test{i}'
974
- to_delete.attrs['version_timestamp'] = ts
975
- populated_numpy_file_dao.add(data_object=to_delete)
976
- assert populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts), f"test{i} does not exist"
977
- populated_numpy_file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts, time_of_removal=first_tod + timedelta(seconds=i + 1))
978
- count = populated_numpy_file_dao.purge(time_threshold=first_tod+timedelta(seconds=n - 1))
979
- assert count == n, f"count is {count}"
980
-
981
-
982
- class TestInMemoryObjectDAO:
983
-
984
- # Get tests (test all expected behaviors of get())
985
- # ------------------------------------------------
986
- # Category 1: Objects that exist
987
- # Test 1.1: Get a single object that exists; check that it exists; check that it has the right values; check that it has the right keys
988
- # Category 2: Objects that do not exist
989
- # Test 2.1: Get a single object that does not exist; check that it returns None
990
- # Category 3: Bad arguments
991
- # Test 3.1: Get a single object with a bad tag argument (error)
992
-
993
- @pytest.mark.parametrize('object_type', [list, str, bool, int, float, type(np.array([])), type(xr.DataArray())])
994
- def test_get_object_that_exists(self, populated_memory_dao, object_type):
995
- result = populated_memory_dao.get(tag=f"test_{object_type.__name__.lower()}")
996
- assert type(result) == object_type
997
-
998
- def test_get_object_that_does_not_exist(self, populated_memory_dao):
999
- result = populated_memory_dao.get(tag=f"not_a_tag")
1000
- assert result is None
1001
-
1002
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
1003
- def test_get_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
1004
- with pytest.raises(InMemoryObjectDAOTypeError):
1005
- populated_memory_dao.get(tag=bad_arg)
1006
-
1007
- # Exists tests (test all expected behaviors of exists())
1008
- # ------------------------------------------------------
1009
- # Category 1: Objects that exist
1010
- # Test 1.1: Check that a single object that exists returns True
1011
- # Category 2: Objects that do not exist
1012
- # Test 2.1: Check that a single object that does not exist returns False
1013
- # Category 3: Bad arguments
1014
- # Test 3.1: Check that a single object with a bad tag argument (error)
1015
-
1016
- @pytest.mark.parametrize('object_type', [list, str, bool, int, float, type(np.array([])), type(xr.DataArray())])
1017
- def test_exists_object_that_exists(self, populated_memory_dao, object_type):
1018
- result = populated_memory_dao.exists(tag=f"test_{object_type.__name__.lower()}")
1019
- assert result
1020
-
1021
- def test_exists_object_that_does_not_exist(self, populated_memory_dao):
1022
- result = populated_memory_dao.exists(tag=f"not_a_tag")
1023
- assert not result
1024
-
1025
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
1026
- def test_exists_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
1027
- with pytest.raises(InMemoryObjectDAOTypeError):
1028
- populated_memory_dao.exists(tag=bad_arg)
1029
-
1030
- # Add tests (test all expected behaviors of add())
1031
- # ------------------------------------------------
1032
- # Category 1: Objects that do not exist
1033
- # Test 1.1: Add a single object that does not exist; check that it exists; check that it has the right values; check that it has the right keys
1034
- # Category 2: Objects that exist
1035
- # Test 2.1: Add a single object with a tag that exists (error)
1036
- # Test 2.2: Add a single object with an id(object) that exists (error)
1037
- # Category 3: Bad arguments
1038
- # Test 3.1: Add a single object with a bad tag argument (error)
1039
-
1040
- @pytest.mark.parametrize('obj', [[1,2,3], 2, False, 1.5, np.array([1,2,3]), xr.DataArray([1,2,3])])
1041
- def test_add_object_that_does_not_exist(self, populated_memory_dao, obj):
1042
- populated_memory_dao.add(tag=f"new_{type(obj).__name__.lower()}", object=obj)
1043
- result = populated_memory_dao.get(tag=f"new_{type(obj).__name__.lower()}")
1044
- # if object is iterable, use all(==) if not use ==
1045
- try:
1046
- assert result == obj
1047
- except Exception:
1048
- assert all(result == obj)
1049
-
1050
- @pytest.mark.parametrize('obj', [[1,2,3], 2, False, 1.5, np.array([1,2,3]), xr.DataArray([1,2,3])])
1051
- def test_add_object_with_tag_that_exists(self, populated_memory_dao, obj):
1052
- with pytest.raises(InMemoryObjectDAOObjectAlreadyExistsError):
1053
- populated_memory_dao.add(tag=f"test_{type(obj).__name__.lower()}", object=obj)
1054
-
1055
- def test_add_object_with_id_that_exists(self, populated_memory_dao, objects):
1056
- for obj in objects:
1057
- with pytest.raises(InMemoryObjectDAOObjectAlreadyExistsError):
1058
- populated_memory_dao.add(tag=f"new_{type(obj).__name__.lower()}", object=obj)
1059
-
1060
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
1061
- def test_add_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
1062
- with pytest.raises(InMemoryObjectDAOTypeError):
1063
- populated_memory_dao.add(tag=bad_arg, object=100)
1064
-
1065
- # mark_for_deletion tests (test all expected behaviors of mark_for_deletion())
1066
- # -----------------------------------------------------------------------------
1067
- # Category 1: Objects that exist
1068
- # Test 1.1: Mark a single object that exists for deletion; check that it can nolonger be accessed through get or the exists method; check that it is inside the deleted objects subcollection
1069
- # Category 2: Objects that do not exist
1070
- # Test 2.1: Mark a single object that does not exist for deletion (error)
1071
- # Category 3: Bad arguments
1072
- # Test 3.1: Mark a single object with a bad tag argument (error)
1073
-
1074
- @pytest.mark.parametrize('tags', ['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray'])
1075
- def test_mark_for_deletion_object_that_exists(self, populated_memory_dao, tags):
1076
- obj = populated_memory_dao.get(tag=tags)
1077
- obj_id = id(obj)
1078
- populated_memory_dao.mark_for_deletion(tag=tags, time_of_removal = datetime.now().astimezone())
1079
- assert not populated_memory_dao.exists(tag=tags)
1080
- assert populated_memory_dao.get(tag=tags) is None
1081
- del_obj = populated_memory_dao._collection['objects'][obj_id]
1082
- try:
1083
- assert all(del_obj == obj)
1084
- except Exception:
1085
- try:
1086
- assert del_obj == obj
1087
- except Exception:
1088
- assert del_obj.equals(obj)
1089
-
1090
- def test_mark_for_deletion_object_that_does_not_exist(self, populated_memory_dao):
1091
- with pytest.raises(InMemoryObjectDAOObjectNotFoundError):
1092
- populated_memory_dao.mark_for_deletion(tag='not_a_tag', time_of_removal = datetime.now().astimezone())
1093
-
1094
- @pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
1095
- def test_mark_for_deletion_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
1096
- with pytest.raises(InMemoryObjectDAOTypeError):
1097
- populated_memory_dao.mark_for_deletion(tag=bad_arg, time_of_removal = datetime.now().astimezone())
1098
-
1099
- # List marked for deletion tests (test all expected behaviors of list_marked_for_deletion())
1100
- # -----------------------------------------------------------------------------------------
1101
- # Category 1: When there are several objects that have been marked for deletion
1102
- # Test 1.1: List all objects that have been marked for deletion; check that it returns all removed objects; check that the objects are in the right order (by time_of_removal)
1103
- # Test 1.2: List only objects from before the provided time_threshold that have been marked for deletion; check that it returns the right objects; check that the objects are in the right order (by time_of_removal)
1104
- # Category 2: When no objects have been marked for deletion
1105
- # Test 2.1: List all objects that have been marked for deletion; check that it returns an empty list
1106
- # Category 3: Bad arguments
1107
- # Test 3.1: List all objects with a bad time_threshold argument (error)
1108
-
1109
- @pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
1110
- def test_list_all_objects_marked_for_deltion_with_none_time_threshold(self, populated_memory_dao, tags):
1111
- del_ids = []
1112
- for obj in tags:
1113
- del_ids.append(id(populated_memory_dao.get(tag=obj)))
1114
- populated_memory_dao.mark_for_deletion(tag=obj, time_of_removal=datetime.now().astimezone())
1115
- result = populated_memory_dao.list_marked_for_deletion()
1116
- assert len(result) == len(tags)
1117
- for i in range(len(result)):
1118
- # check the order of the objects
1119
- if i < len(result) - 1:
1120
- assert result[i]["time_of_removal"] < result[i+1]["time_of_removal"], f"The objects were not in the right order"
1121
- assert result[i]["tag"] == tags[i], f"The tag {tags[i]} was not in the right order"
1122
- # check that the objects are the same
1123
- assert result[i]["id"] == del_ids[i]
1124
-
1125
- @pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
1126
- @pytest.mark.parametrize('time_threshold_shift', list(map(lambda x: timedelta(seconds=x), range(1, 8))))
1127
- def test_list_all_objects_marked_for_deltion_with_time_threshold(self, populated_memory_dao, tags, time_threshold_shift):
1128
- del_ids = []
1129
- removal_start_time = datetime.now().astimezone()
1130
- for i, tag in enumerate(tags):
1131
- del_ids.append(id(populated_memory_dao.get(tag=tag)))
1132
- shift = timedelta(seconds=i + 0.5)
1133
- populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=removal_start_time + shift)
1134
- result = populated_memory_dao.list_marked_for_deletion(time_threshold=removal_start_time + time_threshold_shift)
1135
- assert len(result) == time_threshold_shift.seconds
1136
- for i in range(len(result)):
1137
- # check the order of the objects
1138
- if i < len(result) - 1:
1139
- assert result[i]["time_of_removal"] < result[i+1]["time_of_removal"], f"The objects were not in the right order"
1140
- # check that the objects are the same
1141
- assert result[i]["id"] == del_ids[i]
1142
-
1143
- def test_list_marked_for_deltion_when_non_have_been_marked(self, populated_memory_dao):
1144
- result = populated_memory_dao.list_marked_for_deletion()
1145
- assert len(result) == 0
1146
-
1147
- @pytest.mark.parametrize('bad_arg', [1, 12.5, {"set"}, {"hash": "map"}])
1148
- def test_list_marked_for_deletion_with_bad_time_threshold_arg(self, populated_memory_dao, bad_arg):
1149
- with pytest.raises(InMemoryObjectDAOTypeError):
1150
- populated_memory_dao.list_marked_for_deletion(time_threshold=bad_arg)
1151
-
1152
- # restore tests (test all expected behaviors of restore())
1153
- # ---------------------------------------------------------
1154
- # Category 1: Objects that exist and have been removed
1155
- # Test 1.1: Restore a single object that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
1156
- # Category 2: Objects tha have not been removed
1157
- # Test 2.1: Restore a single object that exists and has not been removed (error)
1158
- # Test 2.2: Restore a single object that does not exist (error)
1159
- # Category 3: Restore an object that has already been permanently deleted
1160
- # Test 3.1: Restore a single object that has already been permanently deleted (error)
1161
- # Category 4: Bad arguments
1162
- # Test 4.1: Restore a single object with a bad tag argument (error)
1163
-
1164
- @pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
1165
- def test_restore_object_that_exists_and_has_been_removed(self, populated_memory_dao, tags):
1166
- for tag in tags:
1167
- populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=datetime.now().astimezone())
1168
- populated_memory_dao.restore(tag=tag)
1169
- assert populated_memory_dao.exists(tag=tag)
1170
-
1171
- def test_restore_object_that_exists_and_has_not_been_removed(self, populated_memory_dao):
1172
- with pytest.raises(InMemoryObjectDAOObjectAlreadyExistsError):
1173
- populated_memory_dao.restore(tag='test_list')
1174
-
1175
- def test_restore_object_that_has_already_been_permanently_deleted(self, populated_memory_dao):
1176
- populated_memory_dao.mark_for_deletion(tag='test_list', time_of_removal=datetime.now().astimezone())
1177
- del populated_memory_dao._collection['removed']['test_list']
1178
- with pytest.raises(InMemoryObjectDAOObjectNotFoundError):
1179
- populated_memory_dao.restore(tag='test_list')
1180
-
1181
- @pytest.mark.parametrize('bad_arg', [1, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
1182
- def test_restore_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
1183
- with pytest.raises(InMemoryObjectDAOTypeError):
1184
- populated_memory_dao.restore(tag=bad_arg)
1185
-
1186
- # purge tests (test all expected behaviors of purge())
1187
- # ---------------------------------------------------------
1188
- # Category 1: Purge all objects that exist and have been removed
1189
- # Test 1.1: Purge all objects that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
1190
- # Test 1.2: Purge all objects that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
1191
- # Category 2: Purge only some objects that exist and have been removed; check that the resulting purge count is correct
1192
- # Test 2.1: Purge all objects that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
1193
- # Category 3: Objects when none have been removed
1194
- # Test 3.1: Purge all objects that exist and have not been removed; check that the purge count is 0
1195
- # Category 4: Bad arguments
1196
- # Test 4.1: Purge all objects with a bad time_threshold argument (error)
1197
-
1198
- @pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
1199
- def test_purge_all_objects_that_exist_and_have_been_removed(self, populated_memory_dao, tags):
1200
- for tag in tags:
1201
- populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=datetime.now().astimezone())
1202
- count = populated_memory_dao.purge(time_threshold=None)
1203
- assert count == len(tags)
1204
- for tag in tags:
1205
- assert not populated_memory_dao.exists(tag=tag)
1206
-
1207
- @pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
1208
- def test_purge_all_objects_that_exist_and_have_been_removed_with_time_threshold(self, populated_memory_dao, tags):
1209
- for tag in tags:
1210
- populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=datetime.now().astimezone())
1211
- count = populated_memory_dao.purge(time_threshold=datetime.now().astimezone())
1212
- assert count == len(tags)
1213
- for tag in tags:
1214
- assert not populated_memory_dao.exists(tag=tag)
1215
-
1216
- @pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
1217
- @pytest.mark.parametrize('time_threshold_shift', list(map(lambda x: timedelta(seconds=x), range(1, 8))))
1218
- def test_purge_only_some_objects_that_exist_and_have_been_removed(self, populated_memory_dao, tags, time_threshold_shift):
1219
- removal_start_time = datetime.now().astimezone()
1220
- for i, tag in enumerate(tags):
1221
- shift = timedelta(seconds=i + 0.5)
1222
- populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=removal_start_time + shift)
1223
- count = populated_memory_dao.purge(time_threshold=removal_start_time + time_threshold_shift)
1224
- assert count == time_threshold_shift.seconds
1225
- for tag in tags:
1226
- assert not populated_memory_dao.exists(tag=tag)
1227
-
1228
- def test_purge_all_objects_that_exist_and_have_not_been_removed(self, populated_memory_dao):
1229
- count = populated_memory_dao.purge(time_threshold=None)
1230
- assert count == 0
1231
-
1232
- @pytest.mark.parametrize('bad_arg', [1, 12.5, {"set"}, {"hash": "map"}])
1233
- def test_purge_with_bad_time_threshold_arg(self, populated_memory_dao, bad_arg):
1234
- with pytest.raises(InMemoryObjectDAOTypeError):
1235
- populated_memory_dao.purge(time_threshold=bad_arg)