eegdash 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of eegdash might be problematic. Click here for more details.
- eegdash/data_utils.py +5 -1
- eegdash/{aws_ingest.py → script.py} +8 -12
- eegdash/signalstore_data_utils.py +352 -2
- {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/METADATA +75 -8
- eegdash-0.0.2.dist-info/RECORD +10 -0
- {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/WHEEL +1 -1
- eegdash/SignalStore/__init__.py +0 -0
- eegdash/SignalStore/signalstore/__init__.py +0 -3
- eegdash/SignalStore/signalstore/adapters/read_adapters/abstract_read_adapter.py +0 -13
- eegdash/SignalStore/signalstore/adapters/read_adapters/domain_modeling/schema_read_adapter.py +0 -16
- eegdash/SignalStore/signalstore/adapters/read_adapters/domain_modeling/vocabulary_read_adapter.py +0 -19
- eegdash/SignalStore/signalstore/adapters/read_adapters/handmade_records/excel_study_organizer_read_adapter.py +0 -114
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/axona/axona_read_adapter.py +0 -912
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/ReadIntanSpikeFile.py +0 -140
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/intan_read_adapter.py +0 -29
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/__init__.py +0 -0
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/data_to_result.py +0 -62
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/get_bytes_per_data_block.py +0 -36
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/notch_filter.py +0 -50
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/qstring.py +0 -41
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/read_header.py +0 -135
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/intanutil/read_one_data_block.py +0 -45
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhd_format/load_intan_rhd_format.py +0 -204
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/__init__.py +0 -0
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/data_to_result.py +0 -60
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/get_bytes_per_data_block.py +0 -37
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/notch_filter.py +0 -50
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/qstring.py +0 -41
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/read_header.py +0 -153
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/intanutil/read_one_data_block.py +0 -47
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/intan/load_intan_rhs_format/load_intan_rhs_format.py +0 -213
- eegdash/SignalStore/signalstore/adapters/read_adapters/recording_acquisitions/neurodata_without_borders/neurodata_without_borders_read_adapter.py +0 -14
- eegdash/SignalStore/signalstore/operations/__init__.py +0 -4
- eegdash/SignalStore/signalstore/operations/handler_executor.py +0 -22
- eegdash/SignalStore/signalstore/operations/handler_factory.py +0 -41
- eegdash/SignalStore/signalstore/operations/handlers/base_handler.py +0 -44
- eegdash/SignalStore/signalstore/operations/handlers/domain/property_model_handlers.py +0 -79
- eegdash/SignalStore/signalstore/operations/handlers/domain/schema_handlers.py +0 -3
- eegdash/SignalStore/signalstore/operations/helpers/abstract_helper.py +0 -17
- eegdash/SignalStore/signalstore/operations/helpers/neuroscikit_extractor.py +0 -33
- eegdash/SignalStore/signalstore/operations/helpers/neuroscikit_rawio.py +0 -165
- eegdash/SignalStore/signalstore/operations/helpers/spikeinterface_helper.py +0 -100
- eegdash/SignalStore/signalstore/operations/helpers/wrappers/neo_wrappers.py +0 -21
- eegdash/SignalStore/signalstore/operations/helpers/wrappers/nwb_wrappers.py +0 -27
- eegdash/SignalStore/signalstore/store/__init__.py +0 -8
- eegdash/SignalStore/signalstore/store/data_access_objects.py +0 -1181
- eegdash/SignalStore/signalstore/store/datafile_adapters.py +0 -131
- eegdash/SignalStore/signalstore/store/repositories.py +0 -928
- eegdash/SignalStore/signalstore/store/store_errors.py +0 -68
- eegdash/SignalStore/signalstore/store/unit_of_work.py +0 -97
- eegdash/SignalStore/signalstore/store/unit_of_work_provider.py +0 -67
- eegdash/SignalStore/signalstore/utilities/data_adapters/spike_interface_adapters/si_recording.py +0 -1
- eegdash/SignalStore/signalstore/utilities/data_adapters/spike_interface_adapters/si_sorter.py +0 -1
- eegdash/SignalStore/signalstore/utilities/testing/data_mocks.py +0 -513
- eegdash/SignalStore/signalstore/utilities/tools/dataarrays.py +0 -49
- eegdash/SignalStore/signalstore/utilities/tools/mongo_records.py +0 -25
- eegdash/SignalStore/signalstore/utilities/tools/operation_response.py +0 -78
- eegdash/SignalStore/signalstore/utilities/tools/purge_orchestration_response.py +0 -21
- eegdash/SignalStore/signalstore/utilities/tools/quantities.py +0 -15
- eegdash/SignalStore/signalstore/utilities/tools/strings.py +0 -38
- eegdash/SignalStore/signalstore/utilities/tools/time.py +0 -17
- eegdash/SignalStore/tests/conftest.py +0 -799
- eegdash/SignalStore/tests/data/valid_data/data_arrays/make_fake_data.py +0 -59
- eegdash/SignalStore/tests/unit/store/conftest.py +0 -0
- eegdash/SignalStore/tests/unit/store/test_data_access_objects.py +0 -1235
- eegdash/SignalStore/tests/unit/store/test_repositories.py +0 -1309
- eegdash/SignalStore/tests/unit/store/test_unit_of_work.py +0 -7
- eegdash/SignalStore/tests/unit/test_ci_cd.py +0 -8
- eegdash-0.0.1.dist-info/RECORD +0 -72
- {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/LICENSE +0 -0
- {eegdash-0.0.1.dist-info → eegdash-0.0.2.dist-info}/top_level.txt +0 -0
|
@@ -1,1235 +0,0 @@
|
|
|
1
|
-
import pytest
|
|
2
|
-
from datetime import datetime, timezone, timedelta
|
|
3
|
-
from signalstore.store.data_access_objects import *
|
|
4
|
-
|
|
5
|
-
class TestDomainModelDAO:
|
|
6
|
-
|
|
7
|
-
# Get tests (test all expected behaviors of get())
|
|
8
|
-
# -----------------------------------------------
|
|
9
|
-
# Category 1: Get a single item that exists; check that it has the right values; check that it has the right keys
|
|
10
|
-
# Test 1.1: Get a single property model that exists; check that it has the right values; check that it has the right keys
|
|
11
|
-
# Test 1.2: Get a single metamodel model that exists; check that it has the right values; check that it has the right keys
|
|
12
|
-
# Test 1.3: Get a single data model that exists; check that it has the right values; check that it has the right keys
|
|
13
|
-
# Category 2: Getting a single item that does not exist; check that it returns None
|
|
14
|
-
# Test 2.1: Getting a single item that does not exist; check that it returns None
|
|
15
|
-
# Category 3: Getting a single item with a bad schema_name argument (error)
|
|
16
|
-
# Test 3.1: Getting a single item with a bad schema_name argument (error)
|
|
17
|
-
|
|
18
|
-
@pytest.mark.parametrize('schema_name', ['dimension_of_measure', 'record_metamodel', 'spike_waveforms'])
|
|
19
|
-
def test_get_item_that_exists(self, populated_domain_model_dao, schema_name):
|
|
20
|
-
item = populated_domain_model_dao.get(schema_name=schema_name)
|
|
21
|
-
assert item.get('schema_name') == schema_name, f'Expected schema_name to be "{schema_name}", got "{item.get("schema_name")}"'
|
|
22
|
-
tos = item.get('time_of_save')
|
|
23
|
-
assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
|
|
24
|
-
|
|
25
|
-
def test_get_item_that_does_not_exist(self, populated_domain_model_dao):
|
|
26
|
-
assert populated_domain_model_dao.get(schema_name='not_a_schema') is None
|
|
27
|
-
|
|
28
|
-
def test_get_item_with_bad_schema_name_argument(self, populated_domain_model_dao):
|
|
29
|
-
with pytest.raises(MongoDAOTypeError):
|
|
30
|
-
populated_domain_model_dao.get(schema_name=1)
|
|
31
|
-
|
|
32
|
-
# Exists tests (test all expected behaviors of exists())
|
|
33
|
-
# ------------------------------------------------------
|
|
34
|
-
# Category 1: Check if a single item exists that does exist
|
|
35
|
-
# Test 1.1: Check if a single property model exists that does exist
|
|
36
|
-
# Test 1.2: Check if a single metamodel model exists that does exist
|
|
37
|
-
# Test 1.3: Check if a single data model exists that does exist
|
|
38
|
-
# Category 2: Check if a single item exists that does not exist
|
|
39
|
-
# Test 2.1: Check if a single model exists that does not exist
|
|
40
|
-
|
|
41
|
-
@pytest.mark.parametrize('schema_name', ['dimension_of_measure', 'record_metamodel', 'spike_waveforms'])
|
|
42
|
-
def test_exists__model_that_exists(self, populated_domain_model_dao, schema_name):
|
|
43
|
-
assert populated_domain_model_dao.exists(schema_name=schema_name)
|
|
44
|
-
|
|
45
|
-
def test_exists_model_that_does_not_exist(self, populated_domain_model_dao):
|
|
46
|
-
assert not populated_domain_model_dao.exists(schema_name='not_a_schema')
|
|
47
|
-
|
|
48
|
-
# Find (query) tests (test all expected behaviors of find())
|
|
49
|
-
# ----------------------------------------------------
|
|
50
|
-
# Category 1: Find items with no arguments
|
|
51
|
-
# Test 1.1: Find all items with no arguments; check that it returns the right items
|
|
52
|
-
# Category 2: Find items that exist with valid filter argument only
|
|
53
|
-
# Test 2.1: Find all the models of a certain schema_type; check that it returns the right models
|
|
54
|
-
# Test 2.2: Find all models with a certain metamodel_ref value; check that it returns the right models
|
|
55
|
-
# Test 2.3: Find all the models with a property none of them have; check that it returns an empty list
|
|
56
|
-
# Test 2.4: Find all the models with a property they have but a value none of them have; check that it returns an empty list
|
|
57
|
-
# Category 3: Find items with a valid filter argument and a valid projection argument
|
|
58
|
-
# Test 3.1: Find just the schema_name of all the models of a certain schema_type; check that it returns the right models
|
|
59
|
-
# Test 3.2: Find just the schema_name and schema_description of all the models of a certain schema_type; check that it returns the right models
|
|
60
|
-
# Category 4: Find items with bad arguments (error)
|
|
61
|
-
# Test 4.1: Find with a bad filter argument (error)
|
|
62
|
-
# Test 4.2: Find with a bad projection argument (error)
|
|
63
|
-
# Test 4.3: Find with a bad filter and a bad projection argument (error)
|
|
64
|
-
|
|
65
|
-
def test_find_all_models(self, populated_domain_model_dao, data_models, metamodels, property_models):
|
|
66
|
-
all_models = data_models + metamodels + property_models
|
|
67
|
-
response = populated_domain_model_dao.find()
|
|
68
|
-
assert len(response) == len(all_models), f'Expected len(response) to be {len(all_models)}, got {len(response)}'
|
|
69
|
-
for record in response:
|
|
70
|
-
tos = record.get('time_of_save')
|
|
71
|
-
assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
|
|
72
|
-
|
|
73
|
-
@pytest.mark.parametrize('schema_type', ['property_model', 'metamodel', 'data_model'])
|
|
74
|
-
def test_find_models_of_certain_schema_type(self, populated_domain_model_dao, schema_type):
|
|
75
|
-
response = populated_domain_model_dao.find({'schema_type': schema_type})
|
|
76
|
-
for record in response:
|
|
77
|
-
assert record['schema_type'] == schema_type, f'Expected schema_type to be "{schema_type}", got "{record["schema_type"]}"'
|
|
78
|
-
tos = record.get('time_of_save')
|
|
79
|
-
assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
|
|
80
|
-
|
|
81
|
-
@pytest.mark.parametrize('metamodel_ref', ['record_metamodel', 'xarray_dataarray_metamodel'])
|
|
82
|
-
def test_find_models_with_certain_metamodel_ref(self, populated_domain_model_dao, metamodel_ref):
|
|
83
|
-
response = populated_domain_model_dao.find({'metamodel_ref': metamodel_ref})
|
|
84
|
-
for record in response:
|
|
85
|
-
assert record['metamodel_ref'] == metamodel_ref, f'Expected metamodel_ref to be "{metamodel_ref}", got "{record["metamodel_ref"]}"'
|
|
86
|
-
tos = record.get('time_of_save')
|
|
87
|
-
assert isinstance(tos, datetime), f'Expected time_of_save to be a datetime, got {type(tos)}'
|
|
88
|
-
|
|
89
|
-
def test_find_models_with_property_none_of_them_have(self, populated_domain_model_dao):
|
|
90
|
-
response = populated_domain_model_dao.find({'not_a_property': 'not_a_value'})
|
|
91
|
-
assert response == []
|
|
92
|
-
|
|
93
|
-
def test_find_models_with_property_they_have_but_value_none_of_them_have(self, populated_domain_model_dao):
|
|
94
|
-
response = populated_domain_model_dao.find({'schema_type': 'not_a_value'})
|
|
95
|
-
assert response == []
|
|
96
|
-
|
|
97
|
-
@pytest.mark.parametrize('projection', [{'schema_name': 1, 'schema_type': 1}, {'schema_name': 1, 'schema_type': 1, 'schema_title': 1}])
|
|
98
|
-
@pytest.mark.parametrize('schema_type', ['property_model', 'metamodel', 'data_model'])
|
|
99
|
-
def test_find_with_schema_type_and_projection(self, populated_domain_model_dao, schema_type, projection):
|
|
100
|
-
response = populated_domain_model_dao.find(filter={'schema_type': schema_type}, projection=projection)
|
|
101
|
-
for record in response:
|
|
102
|
-
assert record['schema_type'] == schema_type, f'Expected schema_type to be "{schema_type}", got "{record["schema_type"]}"'
|
|
103
|
-
assert set(record.keys()) == set(projection.keys()), f'Expected record to have keys {set(projection.keys())}, got {set(record.keys())}'
|
|
104
|
-
|
|
105
|
-
def test_find_with_bad_filter_argument(self, populated_domain_model_dao):
|
|
106
|
-
with pytest.raises(MongoDAOTypeError):
|
|
107
|
-
populated_domain_model_dao.find(filter=1)
|
|
108
|
-
|
|
109
|
-
def test_find_with_bad_projection_argument(self, populated_domain_model_dao):
|
|
110
|
-
with pytest.raises(MongoDAOTypeError):
|
|
111
|
-
populated_domain_model_dao.find(projection=1)
|
|
112
|
-
|
|
113
|
-
def test_find_with_bad_filter_and_bad_projection_argument(self, populated_domain_model_dao):
|
|
114
|
-
with pytest.raises(MongoDAOTypeError):
|
|
115
|
-
populated_domain_model_dao.find(filter=1, projection=1)
|
|
116
|
-
|
|
117
|
-
# Add tests (test all expected behaviors of add())
|
|
118
|
-
# ------------------------------------------------
|
|
119
|
-
# Category 1: Add a single item that does not exist; check that it exists; check that it has the right values; check that it has the right keys
|
|
120
|
-
# Test 1.1: Add a single model that does not exist; check that it exists; check that it has the right values; check that it has the right keys
|
|
121
|
-
# Category 2: Add a single item that does exist (error)
|
|
122
|
-
# Test 2.1: Add a single model that does exist (error)
|
|
123
|
-
# Category 3: Add a single item with a bad item argument (error)
|
|
124
|
-
# Test 3.1: Add a single item with a bad document argument (error)
|
|
125
|
-
# Test 3.2: Add a single item with a bad timestamp argument (error)
|
|
126
|
-
|
|
127
|
-
def test_add_not_existing_model(self, populated_domain_model_dao):
|
|
128
|
-
new_model = {
|
|
129
|
-
'schema_name': 'new_model',
|
|
130
|
-
'schema_type': 'data_model',
|
|
131
|
-
'schema_description': 'New Model',
|
|
132
|
-
'schema_title': 'New Model',
|
|
133
|
-
'json_schema': {}
|
|
134
|
-
}
|
|
135
|
-
populated_domain_model_dao.add(document=new_model, timestamp=datetime.now().astimezone())
|
|
136
|
-
assert populated_domain_model_dao.exists(schema_name='new_model')
|
|
137
|
-
# the model should have the same keys, plus a time_of_addition key
|
|
138
|
-
# and a time_of_removal key. The time_of_removal key should be None
|
|
139
|
-
# and the time_of_addition key should be an integer.
|
|
140
|
-
model_with_timestamps = populated_domain_model_dao.get(schema_name='new_model')
|
|
141
|
-
assert model_with_timestamps.keys() == {'schema_name', 'schema_type', 'schema_description', 'schema_title', 'json_schema', 'time_of_save', 'time_of_removal', 'version_timestamp'}, f"Expected model to have keys 'schema_name', 'schema_type', 'schema_description', 'schema_title', 'json_schema', 'time_of_addition', 'time_of_removal', got {model_with_timestamps.keys()}"
|
|
142
|
-
assert isinstance(model_with_timestamps['time_of_save'], datetime)
|
|
143
|
-
assert model_with_timestamps['time_of_removal'] is None
|
|
144
|
-
assert model_with_timestamps['schema_name'] == 'new_model'
|
|
145
|
-
|
|
146
|
-
def test_add_existing_model(self, populated_domain_model_dao):
|
|
147
|
-
with pytest.raises(MongoDAODocumentAlreadyExistsError):
|
|
148
|
-
populated_domain_model_dao.add(document={'schema_name': 'dimension_of_measure'}, timestamp=datetime.now().astimezone())
|
|
149
|
-
|
|
150
|
-
def test_add_model_with_bad_argument(self, populated_domain_model_dao):
|
|
151
|
-
with pytest.raises(MongoDAOTypeError):
|
|
152
|
-
populated_domain_model_dao.add(document=1, timestamp=datetime.now().astimezone())
|
|
153
|
-
|
|
154
|
-
def test_add_model_with_bad_timestamp_argument(self, populated_domain_model_dao):
|
|
155
|
-
with pytest.raises(MongoDAOTypeError):
|
|
156
|
-
populated_domain_model_dao.add(document={'schema_name': 'new_model'}, timestamp=1)
|
|
157
|
-
|
|
158
|
-
# Mark for deletion tests (test all expected behaviors of mark_for_deletion())
|
|
159
|
-
# ----------------------------------------------------------------------------
|
|
160
|
-
# Category 1: Mark a single item for deletion that exists; check that it can nolonger be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
|
|
161
|
-
# Test 1.1: Mark a single model for deletion that exists; check that it can nolonger be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
|
|
162
|
-
# Category 2: Mark a single item for deletion that does not exist (error)
|
|
163
|
-
# Test 2.1: Mark a single model for deletion that does not exist (error)
|
|
164
|
-
# Category 3: Mark a single item for deletion with a bad argument (error)
|
|
165
|
-
# Test 3.1: Mark a single model for deletion with a bad schema_name argument (error)
|
|
166
|
-
# Test 3.2: Mark a single model for deletion with a bad timestamp argument (error)
|
|
167
|
-
|
|
168
|
-
def test_mark_for_deletion_model_that_exists(self, populated_domain_model_dao):
|
|
169
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=datetime.now().astimezone())
|
|
170
|
-
assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure'), f"Expected exists to return False, got {populated_domain_model_dao.exists(schema_name='dimension_of_measure')}"
|
|
171
|
-
assert populated_domain_model_dao._collection.find_one({'schema_name': 'dimension_of_measure'}, {'_id':0})['time_of_removal'] is not None
|
|
172
|
-
|
|
173
|
-
def test_mark_for_deletion_model_that_does_not_exist(self, populated_domain_model_dao):
|
|
174
|
-
with pytest.raises(MongoDAODocumentNotFoundError):
|
|
175
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='not_a_schema', timestamp=datetime.now().astimezone())
|
|
176
|
-
|
|
177
|
-
def test_mark_for_deletion_model_with_bad_argument(self, populated_domain_model_dao):
|
|
178
|
-
with pytest.raises(MongoDAOTypeError):
|
|
179
|
-
populated_domain_model_dao.mark_for_deletion(schema_name=1, timestamp=datetime.now().astimezone())
|
|
180
|
-
|
|
181
|
-
def test_mark_for_deletion_model_with_bad_timestamp(self, populated_domain_model_dao):
|
|
182
|
-
with pytest.raises(MongoDAOTypeError):
|
|
183
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=1)
|
|
184
|
-
|
|
185
|
-
# List marked for deletion tests (test all expected behaviors of list_marked_for_deletion())
|
|
186
|
-
# -----------------------------------------------------------------------------------------
|
|
187
|
-
# Category 1: List items marked for deletion; check that it returns the right items
|
|
188
|
-
# Test 1.1: List models marked for deletion; check that it returns the right models
|
|
189
|
-
# Category 2: List items marked for deletion with a time threshold; check that it returns the right items
|
|
190
|
-
# Test 2.1: List models marked for deletion before a time threshold; check that the items before the time threshold are returned and the items after the time threshold are not; try a few thresholds
|
|
191
|
-
# Category 3: List items marked for deletion with a bad argument (error)
|
|
192
|
-
# Test 3.1: List models marked for deletion with a bad time threshold argument (error)
|
|
193
|
-
|
|
194
|
-
@pytest.mark.parametrize('to_delete', [['dimension_of_measure', 'record_metamodel', 'spike_waveforms']])
|
|
195
|
-
def test_list_marked_for_deletion_returns_all_with_no_arg(self, populated_domain_model_dao, to_delete):
|
|
196
|
-
for schema_name in to_delete:
|
|
197
|
-
populated_domain_model_dao.mark_for_deletion(schema_name=schema_name, timestamp=datetime.now().astimezone())
|
|
198
|
-
response = populated_domain_model_dao.list_marked_for_deletion()
|
|
199
|
-
assert len(response) == len(to_delete), f'Expected len(response) to be {len(to_delete)}, got {len(response)}'
|
|
200
|
-
for record in response:
|
|
201
|
-
assert record['schema_name'] in to_delete, f'Expected schema_name to be in {to_delete}, got {record["schema_name"]}'
|
|
202
|
-
|
|
203
|
-
@pytest.mark.parametrize('to_delete', [['dimension_of_measure', 'record_metamodel', 'spike_waveforms']])
|
|
204
|
-
@pytest.mark.parametrize('threshold_delta', [timedelta(seconds=1.1), timedelta(seconds=2.1), timedelta(seconds=3.1)])
|
|
205
|
-
def test_list_marked_for_deletion_returns_deletions_before_time_threshold(self, populated_domain_model_dao, timestamp, to_delete, threshold_delta):
|
|
206
|
-
for i, schema_name in enumerate(to_delete):
|
|
207
|
-
populated_domain_model_dao.mark_for_deletion(schema_name=schema_name, timestamp=timestamp + timedelta(seconds=i+1))
|
|
208
|
-
response = populated_domain_model_dao.list_marked_for_deletion(time_threshold=timestamp + threshold_delta)
|
|
209
|
-
# round threshold_delta to the nearest second to get the expected number of deletions
|
|
210
|
-
expected_deletions = round(threshold_delta.total_seconds())
|
|
211
|
-
assert len(response) == expected_deletions, f'Expected len(response) to be {expected_deletions}, got {len(response)}'
|
|
212
|
-
|
|
213
|
-
def test_list_marked_for_deletion_with_bad_argument(self, populated_domain_model_dao):
|
|
214
|
-
with pytest.raises(MongoDAOTypeError):
|
|
215
|
-
populated_domain_model_dao.list_marked_for_deletion(1)
|
|
216
|
-
|
|
217
|
-
# Restore tests (test all expected behaviors of restore())
|
|
218
|
-
# --------------------------------------------------------
|
|
219
|
-
# Category 1: Resotre the nth most recent item that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
|
|
220
|
-
# Test 1.1: Restore the most recent model that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
|
|
221
|
-
# Test 1.2: Restore the most recent model is idempotent; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
|
|
222
|
-
# Test 1.3: Restore the 2nd most recent model that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
|
|
223
|
-
# Test 1.4: Restore the nth most recent model that was marked for deletion; check that it can be retreived; check that it is still in the database; check that it has the right values; check that it has the right keys
|
|
224
|
-
# Category 2: Restore a single item that already exists (error)
|
|
225
|
-
# Test 2.1: Restore a single model that already exists (error)
|
|
226
|
-
# Category 3: Restore a single item that was not marked for deletion (error)
|
|
227
|
-
# Test 3.1: Restore a single model that exists but was not marked for deletion (error)
|
|
228
|
-
# Test 3.2: Restore a single model that does not exist and was not marked for deletion (error)
|
|
229
|
-
# Category 4: Restore a single item with a bad schema_name argument (error)
|
|
230
|
-
# Test 4.1: Restore a single model with a bad schema_name argument (error)
|
|
231
|
-
# Test 4.2: Restore a single model with a bad nth_most_recent argument (error)
|
|
232
|
-
# Test 4.3: Restore a single model with a bad timestamp argument (error)
|
|
233
|
-
# Category 5: Restore a single item with an out of range nth_most_recent argument(error)
|
|
234
|
-
# Test 5.1: Restore a single model with an nth_most_recent argument <= 0 (error)
|
|
235
|
-
# Test 5.1: Restore a single model with an nth_most_recent argument > number of versions of the model (error)
|
|
236
|
-
|
|
237
|
-
def test_restore_most_recent_model_that_was_marked_for_deletion(self, populated_domain_model_dao, timestamp):
|
|
238
|
-
tod = timestamp + timedelta(seconds=1)
|
|
239
|
-
tor = timestamp + timedelta(seconds=2)
|
|
240
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod)
|
|
241
|
-
assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure'), f'Expected exists to return False, got {populated_domain_model_dao.exists(schema_name="dimension_of_measure")}'
|
|
242
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure')
|
|
243
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
|
|
244
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == 'Dimension of Measure', f'Expected schema_title to be "Dimension of Measure", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
|
|
245
|
-
|
|
246
|
-
def test_restore_most_recent_model_is_idempotent(self, populated_domain_model_dao, timestamp):
|
|
247
|
-
tod = timestamp + timedelta(seconds=1)
|
|
248
|
-
tor = timestamp + timedelta(seconds=2)
|
|
249
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod)
|
|
250
|
-
assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
|
|
251
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure')
|
|
252
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
|
|
253
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod)
|
|
254
|
-
assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
|
|
255
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure')
|
|
256
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
|
|
257
|
-
|
|
258
|
-
def test_restore_2nd_most_recent_model_marked_for_deletion(self, populated_domain_model_dao, timestamp):
|
|
259
|
-
tod1 = timestamp + timedelta(seconds=1)
|
|
260
|
-
tod2 = timestamp + timedelta(seconds=1.5)
|
|
261
|
-
dom_record = populated_domain_model_dao.get(schema_name='dimension_of_measure')
|
|
262
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod1)
|
|
263
|
-
assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
|
|
264
|
-
populated_domain_model_dao.add(document=dom_record, timestamp=timestamp)
|
|
265
|
-
assert populated_domain_model_dao.exists(schema_name='dimension_of_measure')
|
|
266
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=tod2)
|
|
267
|
-
assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
|
|
268
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent=2)
|
|
269
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
|
|
270
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == 'Dimension of Measure', f'Expected schema_title to be "Dimension of Measure", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
|
|
271
|
-
|
|
272
|
-
@pytest.mark.parametrize('nth_most_recent', [2, 3, 4, 5, 6])
|
|
273
|
-
def test_restore_nth_most_recent_model_marked_for_deletion(self, populated_domain_model_dao, timestamp, nth_most_recent):
|
|
274
|
-
for i in range(nth_most_recent-1):
|
|
275
|
-
dom_doc = populated_domain_model_dao.get(schema_name='dimension_of_measure')
|
|
276
|
-
dom_doc['schema_title'] = f'Dimension of Measure V{i+1}'
|
|
277
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp + timedelta(seconds=i))
|
|
278
|
-
populated_domain_model_dao.add(document=dom_doc, timestamp=timestamp)
|
|
279
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp + timedelta(seconds=i+1))
|
|
280
|
-
assert not populated_domain_model_dao.exists(schema_name='dimension_of_measure')
|
|
281
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent=nth_most_recent)
|
|
282
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure') is not None
|
|
283
|
-
if nth_most_recent > 1:
|
|
284
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == f'Dimension of Measure V{nth_most_recent-1}', f'Expected schema_title to be "Dimension of Measure V{nth_most_recent-1}", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
|
|
285
|
-
else:
|
|
286
|
-
assert populated_domain_model_dao.get(schema_name='dimension_of_measure')['schema_title'] == 'Dimension of Measure', f'Expected schema_title to be "Dimension of Measure", got "{populated_domain_model_dao.get(schema_name="dimension_of_measure")["schema_title"]}"'
|
|
287
|
-
|
|
288
|
-
def test_restore_model_that_already_exists(self, populated_domain_model_dao, timestamp):
|
|
289
|
-
dom_doc = populated_domain_model_dao.get(schema_name='dimension_of_measure')
|
|
290
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp)
|
|
291
|
-
populated_domain_model_dao.add(document=dom_doc, timestamp=timestamp)
|
|
292
|
-
with pytest.raises(MongoDAODocumentAlreadyExistsError):
|
|
293
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure')
|
|
294
|
-
|
|
295
|
-
def test_restore_model_that_was_not_marked_for_deletion(self, populated_domain_model_dao):
|
|
296
|
-
with pytest.raises(MongoDAORangeError):
|
|
297
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure')
|
|
298
|
-
|
|
299
|
-
def test_restore_model_with_bad_schema_name_argument(self, populated_domain_model_dao, timestamp):
|
|
300
|
-
with pytest.raises(MongoDAOTypeError):
|
|
301
|
-
populated_domain_model_dao.restore(schema_name=1, timestamp=timestamp)
|
|
302
|
-
|
|
303
|
-
def test_restore_model_with_bad_nth_most_recent_argument(self, populated_domain_model_dao):
|
|
304
|
-
with pytest.raises(MongoDAOTypeError):
|
|
305
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent='1')
|
|
306
|
-
|
|
307
|
-
def test_restore_model_with_bad_timestamp_argument(self, populated_domain_model_dao):
|
|
308
|
-
with pytest.raises(MongoDAOTypeError):
|
|
309
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure', timestamp=1)
|
|
310
|
-
|
|
311
|
-
@pytest.mark.parametrize('n', [-5, 0, 7])
|
|
312
|
-
def test_restore_model_with_out_of_range_nth_most_recent_argument(self, populated_domain_model_dao, timestamp, n):
|
|
313
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp)
|
|
314
|
-
with pytest.raises(MongoDAORangeError):
|
|
315
|
-
populated_domain_model_dao.restore(schema_name='dimension_of_measure', nth_most_recent=n)
|
|
316
|
-
|
|
317
|
-
# purge tests (test all expected behaviors of purge())
|
|
318
|
-
# ----------------------------------------------------
|
|
319
|
-
# Category 1: Purge with and without a time threshold; check that it deletes the appropriate items
|
|
320
|
-
# Test 1.1: Purge with no time threshold; check that it deletes all items marked for deletion
|
|
321
|
-
# Test 1.2: Purge with a time threshold; check that it deletes all items marked for deletion before the time threshold
|
|
322
|
-
# Category 2: Purge with a bad argument (error)
|
|
323
|
-
# Test 2.1: Purge with a bad time threshold argument (error)
|
|
324
|
-
|
|
325
|
-
def test_purge_with_no_time_threshold(self, populated_domain_model_dao, timestamp):
|
|
326
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp)
|
|
327
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='record_metamodel', timestamp=timestamp)
|
|
328
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='spike_waveforms', timestamp=timestamp)
|
|
329
|
-
assert len(populated_domain_model_dao.list_marked_for_deletion()) == 3
|
|
330
|
-
populated_domain_model_dao.purge()
|
|
331
|
-
assert populated_domain_model_dao.list_marked_for_deletion() == []
|
|
332
|
-
|
|
333
|
-
@pytest.mark.parametrize('threshold_delta', [timedelta(seconds=1.1), timedelta(seconds=2.1), timedelta(seconds=3.1)])
|
|
334
|
-
def test_purge_with_time_threshold(self, populated_domain_model_dao, timestamp, threshold_delta):
|
|
335
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='dimension_of_measure', timestamp=timestamp + timedelta(seconds=1))
|
|
336
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='record_metamodel', timestamp=timestamp + timedelta(seconds=2))
|
|
337
|
-
populated_domain_model_dao.mark_for_deletion(schema_name='spike_waveforms', timestamp=timestamp + timedelta(seconds=3))
|
|
338
|
-
populated_domain_model_dao.purge(time_threshold=timestamp + threshold_delta)
|
|
339
|
-
# round threshold_delta to the nearest second to get the expected number of deletions
|
|
340
|
-
expected_deletions = round(threshold_delta.total_seconds())
|
|
341
|
-
assert len(populated_domain_model_dao.list_marked_for_deletion()) == 3 - expected_deletions
|
|
342
|
-
|
|
343
|
-
def test_purge_with_bad_argument(self, populated_domain_model_dao):
|
|
344
|
-
with pytest.raises(MongoDAOTypeError):
|
|
345
|
-
populated_domain_model_dao.purge(1)
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
class TestFileSystemDAO:
|
|
349
|
-
|
|
350
|
-
# Get tests (test all expected behaviors of get())
|
|
351
|
-
# -----------------------------------------------
|
|
352
|
-
# Category 1A: Unversioned files that exist with default data adapter set
|
|
353
|
-
# Test 1a.1: Get a single unversioned NetCDF dataarray that exists; check that it has the right values; check that it has the right keys
|
|
354
|
-
# Test 1a.2: Get a single Zarr dataarray that exists; check that it has the right values; check that it has the right keys
|
|
355
|
-
# Test 1a.3: Get a single versioned numpy mock model object that exists; check that it has the right values; check that it has the right keys
|
|
356
|
-
# Category 1B: Versioned files that exist with default data adapter set
|
|
357
|
-
# Test 1b.1: Get a single versioned (numpy model) file that exists with its timestamp.
|
|
358
|
-
# Test 1b.2: Get a single versioned (numpy model) file that exists with nth_most_recent.
|
|
359
|
-
# Test 1b.3: Get all the versions of a (numpy model) file that exist.
|
|
360
|
-
# Category 1C: Unversioned files that exist with non-default data adapter prvided
|
|
361
|
-
# Test 1c.1: Get a single unversioned NetCDF dataarray that exists and non-default data-adapter; check that it has the right values; check that it has the right keys
|
|
362
|
-
# Test 1c.2: Get a single Zarr dataarray that exists and non-default data-adapter; check that it has the right values; check that it has the right keys
|
|
363
|
-
# Test 1c.3: Get a single versioned numpy mock model object that exists and non-default data-adapter; check that it has the right values; check that it has the right keys
|
|
364
|
-
# Category 1D: Versioned files that exist with default data adapter set
|
|
365
|
-
# Test 1d.1: Get a single versioned (numpy model) file that exists and non-default data-adapter with its timestamp.
|
|
366
|
-
# Test 1d.2: Get a single versioned (numpy model) file that exists and non-default data-adapter with nth_most_recent.
|
|
367
|
-
# Test 1d.3: Get all the versions of a (numpy model) file that exist and non-default data-adapter.
|
|
368
|
-
# Category 2: Files that do not exist
|
|
369
|
-
# Test 2.1: Get a single NetCDF dataarray file that does not exist; check that it returns None
|
|
370
|
-
# Test 2.2: Get a single Zarr dataarray file that does not exist; check that it returns None
|
|
371
|
-
# Test 2.3: Get a single versioned numpy mock model object that does not exist; check that it returns None
|
|
372
|
-
# Category 2: Bad arguments
|
|
373
|
-
# Test 2.1: Get a single file with a bad schema_name argument (error)
|
|
374
|
-
# Test 2.2: Get a single file with a bad data_name argument (error)
|
|
375
|
-
# Test 2.3: Get a single file with a bad data_version argument (error)
|
|
376
|
-
|
|
377
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
378
|
-
@pytest.mark.parametrize('use_data_adapter', [True, False])
|
|
379
|
-
def test_get_file_that_exists(self, file_dao_options, data_adapter_options, use_data_adapter, file_type):
|
|
380
|
-
file_dao = file_dao_options[file_type]
|
|
381
|
-
if use_data_adapter:
|
|
382
|
-
data_adapter = data_adapter_options[file_type]
|
|
383
|
-
else:
|
|
384
|
-
data_adapter = None
|
|
385
|
-
data_object = file_dao.get(schema_ref='test', data_name='test', data_adapter=data_adapter)
|
|
386
|
-
schema_ref = data_object.attrs.get('schema_ref')
|
|
387
|
-
data_name = data_object.attrs.get('data_name')
|
|
388
|
-
data_version = data_object.attrs.get('data_version')
|
|
389
|
-
assert schema_ref == 'test', f'Expected schema_ref to be "spike_waveforms", got "{schema_ref}"'
|
|
390
|
-
assert data_name == 'test', f'Expected data_name to be "spike_waveforms", got "{data_name}"'
|
|
391
|
-
assert data_version == None, f'Expected data_version to be None, got "{data_version}"'
|
|
392
|
-
|
|
393
|
-
@pytest.mark.parametrize('use_data_adapter', [True, False])
|
|
394
|
-
def test_get_versioned_file_with_timestamp(self, populated_numpy_file_dao, model_numpy_adapter, use_data_adapter, timestamp):
|
|
395
|
-
if use_data_adapter:
|
|
396
|
-
data_adapter = model_numpy_adapter
|
|
397
|
-
else:
|
|
398
|
-
data_adapter = None
|
|
399
|
-
ts = timestamp + timedelta(seconds=10)
|
|
400
|
-
model = populated_numpy_file_dao.get(schema_ref='test', data_name='test', data_adapter=data_adapter)
|
|
401
|
-
result = model.attrs["version_timestamp"]
|
|
402
|
-
assert result == ts, f"Expected {ts} but got {result}."
|
|
403
|
-
|
|
404
|
-
@pytest.mark.parametrize('n', range(11))
|
|
405
|
-
def test_get_versioned_file_with_nth_most_recent(self, populated_numpy_file_dao, n):
|
|
406
|
-
checkpoint = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=n)
|
|
407
|
-
assert checkpoint is not None
|
|
408
|
-
|
|
409
|
-
def test_get_all_versions_of_file_that_exists(self, populated_numpy_file_dao):
|
|
410
|
-
checkpoint = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
411
|
-
checkpoints = []
|
|
412
|
-
n = 2
|
|
413
|
-
while checkpoint is not None:
|
|
414
|
-
checkpoints.append(checkpoint)
|
|
415
|
-
checkpoint = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=n)
|
|
416
|
-
n += 1
|
|
417
|
-
assert len(checkpoints) == 10
|
|
418
|
-
assert len(checkpoints) == populated_numpy_file_dao.n_versions(schema_ref='test', data_name='test')
|
|
419
|
-
|
|
420
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
421
|
-
def test_get_file_that_does_not_exist(self, file_dao_options, file_type):
|
|
422
|
-
file_dao = file_dao_options[file_type]
|
|
423
|
-
assert file_dao.get(schema_ref='not_a_schema', data_name='not_a_data_name', data_adapter=None) is None
|
|
424
|
-
assert file_dao.get(schema_ref='test', data_name='not_a_data_name', data_adapter=None) is None
|
|
425
|
-
assert file_dao.get(schema_ref='test', data_name='test', version_timestamp=datetime.now().astimezone()+timedelta(seconds=5)) is None
|
|
426
|
-
|
|
427
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
428
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
429
|
-
def test_get_file_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
|
|
430
|
-
file_dao = file_dao_options[file_type]
|
|
431
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
432
|
-
data_object = file_dao.get(schema_ref=bad_arg, data_name='test', data_adapter=None)
|
|
433
|
-
|
|
434
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
435
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
436
|
-
def test_get_file_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
|
|
437
|
-
file_dao = file_dao_options[file_type]
|
|
438
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
439
|
-
data_object = file_dao.get(schema_ref='test', data_name=bad_arg, data_adapter=None)
|
|
440
|
-
|
|
441
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
442
|
-
@pytest.mark.parametrize('bad_arg', [1.5, None, "string", {"set"}, {"hash": "map"}])
|
|
443
|
-
def test_get_file_with_bad_nth_most_recent_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
444
|
-
file_dao = file_dao_options[file_type]
|
|
445
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
446
|
-
data_object = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=bad_arg, data_adapter=None)
|
|
447
|
-
|
|
448
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
449
|
-
@pytest.mark.parametrize('bad_arg', [1.5, 1, "string", {"set"}, {"hash": "map"}])
|
|
450
|
-
def test_get_file_with_bad_version_timestamp_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
451
|
-
file_dao = file_dao_options[file_type]
|
|
452
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
453
|
-
data_object = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1, version_timestamp=bad_arg, data_adapter=bad_arg)
|
|
454
|
-
|
|
455
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
456
|
-
@pytest.mark.parametrize('bad_arg', [1.5, 1, datetime.now().astimezone(), "string", {"set"}, {"hash": "map"}])
|
|
457
|
-
def test_get_file_with_bad_data_adapter_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
458
|
-
file_dao = file_dao_options[file_type]
|
|
459
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
460
|
-
data_object = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1, data_adapter=bad_arg)
|
|
461
|
-
|
|
462
|
-
# Exists tests (test all expected behaviors of exists())
|
|
463
|
-
# ------------------------------------------------------
|
|
464
|
-
# Category 1: Unversioned files that exist
|
|
465
|
-
# Test 1.1: Check if a single unversioned NetCDF dataarray exists that exists
|
|
466
|
-
# Test 1.2: Check if a single unversioned Zarr dataarray exists that exists
|
|
467
|
-
# Test 1.3: Check if a single unversioned numpy mock model object exists that exists
|
|
468
|
-
# Category 2: Versioned files that exist
|
|
469
|
-
# Test 2.1: Check if a single versioned (numpy model) file that exists (with its timestamp).
|
|
470
|
-
# Category 3: Files that do not exist
|
|
471
|
-
# Test 3.1: Check if a single NetCDF dataarray file that does not exist
|
|
472
|
-
# Test 3.2: Check if a single Zarr dataarray file that does not exist
|
|
473
|
-
# Test 3.3: Check if a single versioned numpy mock model object that does not exist
|
|
474
|
-
# Category 4: Bad arguments
|
|
475
|
-
# Test 4.1: Check if a single file with a bad schema_name argument (error)
|
|
476
|
-
# Test 4.2: Check if a single file with a bad data_name argument (error)
|
|
477
|
-
# Test 4.3: Check if a single file with a bad data_version argument (error)
|
|
478
|
-
# Test 4.4: Check if a single file with a bad data_adapter argument (error)
|
|
479
|
-
|
|
480
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
481
|
-
def test_exists_unversioned_file_that_exists(self, file_dao_options, file_type):
|
|
482
|
-
file_dao = file_dao_options[file_type]
|
|
483
|
-
result = file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
|
|
484
|
-
assert result, f'Expected exists to return True, got {result}'
|
|
485
|
-
|
|
486
|
-
def test_exists_versioned_file_with_timestamp(self, populated_numpy_file_dao):
|
|
487
|
-
x = populated_numpy_file_dao.get(schema_ref='test', data_name='test')
|
|
488
|
-
ts = x.attrs["version_timestamp"]
|
|
489
|
-
result = populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
|
|
490
|
-
assert result, f'Expected exists to return True, got {result}'
|
|
491
|
-
|
|
492
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
493
|
-
def test_exists_with_file_that_does_not_exist(self, file_dao_options, file_type):
|
|
494
|
-
file_dao = file_dao_options[file_type]
|
|
495
|
-
result = file_dao.exists(schema_ref='not_a_schema', data_name='not_a_data_name', version_timestamp=0)
|
|
496
|
-
assert not result, f'Expected exists to return False, got {result}'
|
|
497
|
-
|
|
498
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
499
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
500
|
-
def test_exists_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
|
|
501
|
-
file_dao = file_dao_options[file_type]
|
|
502
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
503
|
-
file_dao.exists(schema_ref=bad_arg, data_name='test', version_timestamp=0)
|
|
504
|
-
|
|
505
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
506
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
507
|
-
def test_exists_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
|
|
508
|
-
file_dao = file_dao_options[file_type]
|
|
509
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
510
|
-
file_dao.exists(schema_ref='test', data_name=bad_arg, version_timestamp=0)
|
|
511
|
-
|
|
512
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
513
|
-
@pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
|
|
514
|
-
def test_exists_with_bad_version_timestamp_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
515
|
-
file_dao = file_dao_options[file_type]
|
|
516
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
517
|
-
file_dao.exists(schema_ref='test', data_name='test', version_timestamp=bad_arg)
|
|
518
|
-
|
|
519
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
520
|
-
@pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
|
|
521
|
-
def test_exists_with_bad_data_adapter_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
522
|
-
file_dao = file_dao_options[file_type]
|
|
523
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
524
|
-
file_dao.exists(schema_ref='test', data_name='test', data_adapter=bad_arg)
|
|
525
|
-
|
|
526
|
-
# N versions tests (test all expected behaviors of n_versions())
|
|
527
|
-
# --------------------------------------------------------------
|
|
528
|
-
# Category 1: Versioned files that exist
|
|
529
|
-
# Test 1.1: Get version count and assert that it is correct
|
|
530
|
-
# Category 2: Files that do not exist
|
|
531
|
-
# Test 2.1: Get version count and assert that it is 0
|
|
532
|
-
# Category 3: Bad arguments
|
|
533
|
-
# Test 3.1: Get version count with a bad schema_name argument (error)
|
|
534
|
-
# Test 3.2: Get version count with a bad data_name argument (error)
|
|
535
|
-
|
|
536
|
-
def test_n_versions_versioned_file_that_exists(self, populated_numpy_file_dao):
|
|
537
|
-
result = populated_numpy_file_dao.n_versions(schema_ref='test', data_name='test')
|
|
538
|
-
assert result == 10, f'Expected n_versions to return 10, got {result}'
|
|
539
|
-
|
|
540
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
541
|
-
def test_n_versions_with_file_that_does_not_exist(self, file_dao_options, file_type):
|
|
542
|
-
file_dao = file_dao_options[file_type]
|
|
543
|
-
result = file_dao.n_versions(schema_ref='not_a_schema', data_name='not_a_data_name')
|
|
544
|
-
assert result == 0, f'Expected n_versions to return 0, got {result}'
|
|
545
|
-
|
|
546
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
547
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
548
|
-
def test_n_versions_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
|
|
549
|
-
file_dao = file_dao_options[file_type]
|
|
550
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
551
|
-
file_dao.n_versions(schema_ref=bad_arg, data_name='test')
|
|
552
|
-
|
|
553
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
554
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
555
|
-
def test_n_versions_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
|
|
556
|
-
file_dao = file_dao_options[file_type]
|
|
557
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
558
|
-
file_dao.n_versions(schema_ref='test', data_name=bad_arg)
|
|
559
|
-
|
|
560
|
-
# Add tests (test all expected behaviors of add())
|
|
561
|
-
# ------------------------------------------------
|
|
562
|
-
# Category 1: Unversioned files that do not exist
|
|
563
|
-
# Test 1.1: Add a single unversioned NetCDF dataarray that does not exist; check that it exists; check that it has the right values; check that it has the right keys
|
|
564
|
-
# Test 1.2: Add a single unversioned Zarr dataarray that does not exist; check that it exists; check that it has the right values; check that it has the right keys
|
|
565
|
-
# Category 2: Versioned files that do not exist
|
|
566
|
-
# Test 2.1: Add a single versioned numpy mock model object that does not exist; check that it exists; check that it has the right values; check that it has the right keys
|
|
567
|
-
# Category 3: Unversioned that already exist
|
|
568
|
-
# Test 3.1: Add a single unversioned NetCDF dataarray that already exists (error)
|
|
569
|
-
# Test 3.2: Add a single unversioned Zarr dataarray that already exists (error)
|
|
570
|
-
# Category 4: Versioned files that already exist
|
|
571
|
-
# Test 4.1: Add a single versioned numpy mock model object that already exists and has the same timestamp (error)
|
|
572
|
-
# Test 4.2: Add a single versioned numpy mock model object that already exists and has a different timestamp; check that it add works; check that it has the right values; check that it has the right keys
|
|
573
|
-
# Category 5: Bad arguments
|
|
574
|
-
# Test 5.1: Add a single file with a bad data_adapter argument (error)
|
|
575
|
-
# Category 6: Mismatched data_object, data_adapter pairs
|
|
576
|
-
# Test 6.1: Add a single file with a data_object and data_adapter that do not match (several combos) (error)
|
|
577
|
-
|
|
578
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
579
|
-
def test_add_unversioned_file_that_does_not_exist(self, file_dao_options, new_object_options, file_type):
|
|
580
|
-
file_dao = file_dao_options[file_type]
|
|
581
|
-
data_object = new_object_options[file_type]
|
|
582
|
-
file_dao.add(data_object=data_object)
|
|
583
|
-
result = file_dao.get(schema_ref='new', data_name='new')
|
|
584
|
-
assert result is not None
|
|
585
|
-
schema_ref = result.attrs.get('schema_ref')
|
|
586
|
-
data_name = result.attrs.get('data_name')
|
|
587
|
-
assert schema_ref == 'new', f'Expected schema_ref to be "new", got "{schema_ref}"'
|
|
588
|
-
assert data_name == 'new', f'Expected data_name to be "new", got "{data_name}"'
|
|
589
|
-
|
|
590
|
-
def test_add_versioned_file_that_does_not_exist(self, populated_numpy_file_dao, new_object_options):
|
|
591
|
-
data_object = new_object_options['numpy']
|
|
592
|
-
populated_numpy_file_dao.add(data_object=data_object)
|
|
593
|
-
result = populated_numpy_file_dao.get(schema_ref='new', data_name='new', nth_most_recent=1)
|
|
594
|
-
assert result is not None
|
|
595
|
-
schema_ref = result.attrs.get('schema_ref')
|
|
596
|
-
data_name = result.attrs.get('data_name')
|
|
597
|
-
assert schema_ref == 'new', f'Expected schema_ref to be "new", got "{schema_ref}"'
|
|
598
|
-
assert data_name == 'new', f'Expected data_name to be "new", got "{data_name}"'
|
|
599
|
-
|
|
600
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
601
|
-
def test_add_unversioned_file_that_already_exists(self, file_dao_options, file_type):
|
|
602
|
-
file_dao = file_dao_options[file_type]
|
|
603
|
-
data_object = file_dao.get(schema_ref='test', data_name='test')
|
|
604
|
-
with pytest.raises(FileSystemDAOFileAlreadyExistsError):
|
|
605
|
-
file_dao.add(data_object=data_object)
|
|
606
|
-
|
|
607
|
-
def test_add_versioned_file_that_already_exists_with_same_timestamp(self, populated_numpy_file_dao):
|
|
608
|
-
data_object = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent = 1)
|
|
609
|
-
with pytest.raises(FileSystemDAOFileAlreadyExistsError):
|
|
610
|
-
populated_numpy_file_dao.add(data_object=data_object)
|
|
611
|
-
|
|
612
|
-
def test_add_versioned_file_that_already_exists_with_different_timestamp(self, populated_numpy_file_dao):
|
|
613
|
-
data_object = populated_numpy_file_dao.get(schema_ref='test', data_name='test')
|
|
614
|
-
new_ts = datetime.now().astimezone() + timedelta(seconds=11)
|
|
615
|
-
data_object.attrs['version_timestamp'] = new_ts
|
|
616
|
-
populated_numpy_file_dao.add(data_object=data_object)
|
|
617
|
-
result = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
618
|
-
assert result is not None
|
|
619
|
-
schema_ref = result.attrs.get('schema_ref')
|
|
620
|
-
data_name = result.attrs.get('data_name')
|
|
621
|
-
version_timestamp = result.attrs.get('version_timestamp')
|
|
622
|
-
assert schema_ref == 'test', f'Expected schema_ref to be "test", got "{schema_ref}"'
|
|
623
|
-
assert data_name == 'test', f'Expected data_name to be "test", got "{data_name}"'
|
|
624
|
-
assert version_timestamp == new_ts, f'Expected version_timestamp to be "{new_ts}", got "{version_timestamp}"'
|
|
625
|
-
|
|
626
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
627
|
-
@pytest.mark.parametrize('bad_arg', [1, None, "string", {"set"}, {"hash": "map"}])
|
|
628
|
-
def test_add_with_bad_data_adapter_arg(self, file_dao_options, file_type, bad_arg):
|
|
629
|
-
file_dao = file_dao_options[file_type]
|
|
630
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
631
|
-
file_dao.add(data_object=bad_arg)
|
|
632
|
-
|
|
633
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
634
|
-
def test_add_with_mismatched_data_object_and_data_adapter(self, file_dao_options, file_type):
|
|
635
|
-
if file_type in ['netcdf', 'zarr']:
|
|
636
|
-
bad_type = 'numpy'
|
|
637
|
-
else:
|
|
638
|
-
bad_type = 'netcdf'
|
|
639
|
-
file_dao = file_dao_options[file_type]
|
|
640
|
-
other_file_dao = file_dao_options[bad_type]
|
|
641
|
-
incompatable_data_object = other_file_dao.get(schema_ref='test', data_name='test')
|
|
642
|
-
# set attrs to new
|
|
643
|
-
incompatable_data_object.attrs['schema_ref'] = 'new'
|
|
644
|
-
incompatable_data_object.attrs['data_name'] = 'new'
|
|
645
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
646
|
-
file_dao.add(data_object=incompatable_data_object)
|
|
647
|
-
|
|
648
|
-
# Mark for deletion tests (test all expected behaviors of mark_for_deletion())
|
|
649
|
-
# -----------------------------------------------------------------------------
|
|
650
|
-
# Category 1: Unversioned files that exist
|
|
651
|
-
# Test 1.1: Mark a single unversioned NetCDF dataarray that exists for deletion; check that it can nolonger be accessed through get or the exists method
|
|
652
|
-
# Test 1.2: Mark a single unversioned Zarr dataarray that exists for deletion; check that it can nolonger be accessed through get or the exists method
|
|
653
|
-
# Category 2: Versioned files that exist
|
|
654
|
-
# Test 2.1: Mark a single versioned numpy mock model object that exists for deletion; check that it can nolonger be accessed through get or the exists method; check that the other versions can still be accessed
|
|
655
|
-
# Category 3: Files that do not exist
|
|
656
|
-
# Test 3.1: Mark a single NetCDF dataarray file that does not exist for deletion (error)
|
|
657
|
-
# Test 3.2: Mark a single Zarr dataarray file that does not exist for deletion (error)
|
|
658
|
-
# Test 3.3: Mark a single versioned numpy mock model object that does not exist for deletion (error)
|
|
659
|
-
# Category 4: Bad arguments
|
|
660
|
-
# Test 4.1: Mark a single file with a bad schema_name argument (error)
|
|
661
|
-
# Test 4.2: Mark a single file with a bad data_name argument (error)
|
|
662
|
-
# Test 4.3: Mark a single file with a bad data_version argument (error)
|
|
663
|
-
# Test 4.4: Mark a single file with a bad time_of_removal argument (error)
|
|
664
|
-
# Test 4.5: Mark a single file with a bad data_adapter argument (error)
|
|
665
|
-
|
|
666
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
667
|
-
def test_mark_for_deletion_unversioned_file_that_exists(self, file_dao_options, file_type):
|
|
668
|
-
file_dao = file_dao_options[file_type]
|
|
669
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
|
|
670
|
-
assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
|
|
671
|
-
assert file_dao.get(schema_ref='test', data_name='test', version_timestamp=0) is None
|
|
672
|
-
|
|
673
|
-
def test_mark_for_deletion_versioned_file_that_exists(self, populated_numpy_file_dao):
|
|
674
|
-
to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
675
|
-
ts = to_remove.attrs['version_timestamp']
|
|
676
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
|
|
677
|
-
assert not populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
|
|
678
|
-
assert populated_numpy_file_dao.get(schema_ref='test', data_name='test', version_timestamp=ts) is None
|
|
679
|
-
assert populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1) is not None
|
|
680
|
-
assert populated_numpy_file_dao.n_versions(schema_ref='test', data_name='test') == 9
|
|
681
|
-
|
|
682
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
683
|
-
def test_mark_for_deletion_with_file_that_does_not_exist(self, file_dao_options, file_type):
|
|
684
|
-
file_dao = file_dao_options[file_type]
|
|
685
|
-
with pytest.raises(FileSystemDAOFileNotFoundError):
|
|
686
|
-
file_dao.mark_for_deletion(schema_ref='not_a_schema', data_name='not_a_data_name', time_of_removal=datetime.now().astimezone())
|
|
687
|
-
|
|
688
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
689
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
690
|
-
def test_mark_for_deletion_with_bad_schema_ref_arg(self, file_dao_options, file_type, bad_arg):
|
|
691
|
-
file_dao = file_dao_options[file_type]
|
|
692
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
693
|
-
file_dao.mark_for_deletion(schema_ref=bad_arg, data_name='test', time_of_removal=datetime.now().astimezone())
|
|
694
|
-
|
|
695
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
696
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
697
|
-
def test_mark_for_deletion_with_bad_data_name_arg(self, file_dao_options, file_type, bad_arg):
|
|
698
|
-
file_dao = file_dao_options[file_type]
|
|
699
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
700
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name=bad_arg, time_of_removal=datetime.now().astimezone())
|
|
701
|
-
|
|
702
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
703
|
-
@pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
|
|
704
|
-
def test_mark_for_deletion_with_bad_version_timestamp_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
705
|
-
file_dao = file_dao_options[file_type]
|
|
706
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
707
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=bad_arg, time_of_removal=datetime.now().astimezone())
|
|
708
|
-
|
|
709
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
710
|
-
@pytest.mark.parametrize('bad_arg', [1.5, "string", {"set"}, {"hash": "map"}])
|
|
711
|
-
def test_mark_for_deletion_with_bad_time_of_removal_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
712
|
-
file_dao = file_dao_options[file_type]
|
|
713
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
714
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=bad_arg)
|
|
715
|
-
|
|
716
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
717
|
-
@pytest.mark.parametrize('bad_arg', [1.5, 1, datetime.now().astimezone(), "string", {"set"}, {"hash": "map"}])
|
|
718
|
-
def test_mark_for_deletion_with_bad_data_adapter_arg_arg(self, file_dao_options, file_type, bad_arg):
|
|
719
|
-
file_dao = file_dao_options[file_type]
|
|
720
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
721
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone(), data_adapter=bad_arg)
|
|
722
|
-
|
|
723
|
-
# List marked for deletion tests (test all expected behaviors of list_marked_for_deletion())
|
|
724
|
-
# -----------------------------------------------------------------------------------------
|
|
725
|
-
# Category 1: Unversioned files that exist
|
|
726
|
-
# Test 1.1: List all unversioned NetCDF dataarrays that exist; check that it returns the right files
|
|
727
|
-
# Test 1.2: List all unversioned Zarr dataarrays that exist; check that it returns the right files
|
|
728
|
-
# Category 2: Versioned files that exist
|
|
729
|
-
# Test 2.1: List all versioned numpy mock model objects that exist; check that it returns the right files
|
|
730
|
-
# Category 3: Files that do not exist
|
|
731
|
-
# Test 3.1: List all NetCDF dataarray files that do not exist; check that it returns an empty list
|
|
732
|
-
# Test 3.2: List all Zarr dataarray files that do not exist; check that it returns an empty list
|
|
733
|
-
# Test 3.3: List all versioned numpy mock model objects that do not exist; check that it returns an empty list
|
|
734
|
-
# Category 4: Bad arguments
|
|
735
|
-
# Test 4.1: List all files with a bad time_threshold argument (error)
|
|
736
|
-
# Test 4.2: List all files with a bad data_adapter argument (error)
|
|
737
|
-
|
|
738
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
739
|
-
def test_list_marked_for_deletion_unversioned_file_that_exists(self, file_dao_options, file_type):
|
|
740
|
-
file_dao = file_dao_options[file_type]
|
|
741
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
|
|
742
|
-
result = file_dao.list_marked_for_deletion()
|
|
743
|
-
assert len(result) == 1
|
|
744
|
-
|
|
745
|
-
def test_list_marked_for_deletion_versioned_file_that_exists(self, populated_numpy_file_dao):
|
|
746
|
-
to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
747
|
-
ts = to_remove.attrs['version_timestamp']
|
|
748
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
|
|
749
|
-
result = populated_numpy_file_dao.list_marked_for_deletion()
|
|
750
|
-
assert len(result) == 1
|
|
751
|
-
|
|
752
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
753
|
-
def test_list_marked_for_deletion_with_file_that_does_not_exist(self, file_dao_options, file_type):
|
|
754
|
-
file_dao = file_dao_options[file_type]
|
|
755
|
-
result = file_dao.list_marked_for_deletion()
|
|
756
|
-
assert len(result) == 0
|
|
757
|
-
|
|
758
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr', 'numpy'])
|
|
759
|
-
@pytest.mark.parametrize('bad_arg', [1.5, 1, "string", {"set"}, {"hash": "map"}])
|
|
760
|
-
def test_list_marked_for_deletion_with_bad_time_threshold_arg(self, file_dao_options, file_type, bad_arg):
|
|
761
|
-
file_dao = file_dao_options[file_type]
|
|
762
|
-
with pytest.raises(FileSystemDAOTypeError):
|
|
763
|
-
file_dao.list_marked_for_deletion(time_threshold=bad_arg)
|
|
764
|
-
|
|
765
|
-
# restore tests (test all expected behaviors of restore())
|
|
766
|
-
# ---------------------------------------------------------
|
|
767
|
-
# Category 1: Unversioned files that exist and have been removed
|
|
768
|
-
# Test 1.1: Restore a single unversioned NetCDF dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
|
|
769
|
-
# Test 1.2: Restore nth_most_recent unversioned NetCDF dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
|
|
770
|
-
# Test 1.3: Restore a single unversioned Zarr dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
|
|
771
|
-
# Test 1.4: Restore nth_most_recent unversioned Zarr dataarray that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
|
|
772
|
-
# Category 2: Versioned files that exist and have been removed
|
|
773
|
-
# Test 2.1: Restore a single versioned numpy mock model object that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
|
|
774
|
-
# Test 2.2: Restore nth_most_recent versioned numpy mock model object that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
|
|
775
|
-
# Category 3: Unversioned files that exist and have not been removed
|
|
776
|
-
# Test 3.1: Restore a single unversioned NetCDF dataarray that exists and has not been removed (error)
|
|
777
|
-
# Test 3.2: Restore a single unversioned Zarr dataarray that exists and has not been removed (error)
|
|
778
|
-
# Category 4: Versioned files that exist and have not been removed
|
|
779
|
-
# Test 4.1: Restore a single versioned numpy mock model object that exists and has not been removed (error)
|
|
780
|
-
# Category 5: Files that do not exist
|
|
781
|
-
# Test 5.1: Restore a single NetCDF dataarray file that does not exist (error)
|
|
782
|
-
# Test 5.2: Restore a single Zarr dataarray file that does not exist (error)
|
|
783
|
-
# Test 5.3: Restore a single versioned numpy mock model object that does not exist (error)
|
|
784
|
-
# Category 6: Bad arguments
|
|
785
|
-
# Test 6.1: Restore a single file with a bad schema_name argument (error)
|
|
786
|
-
# Test 6.2: Restore a single file with a bad data_name argument (error)
|
|
787
|
-
# Test 6.3: Restore a single file with a bad data_version argument (error)
|
|
788
|
-
# Test 6.5: Restore a single file with a bad nth_most_recent argument (error)
|
|
789
|
-
# Test 6.4: Restore a single file with a bad time_of_removal argument (error)
|
|
790
|
-
# Test 6.6: Restore a single file with a bad data_adapter argument (error)
|
|
791
|
-
|
|
792
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
793
|
-
def test_restore_unversioned_file_that_exists_and_has_been_removed(self, file_dao_options, file_type):
|
|
794
|
-
file_dao = file_dao_options[file_type]
|
|
795
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
|
|
796
|
-
file_dao.restore(schema_ref='test', data_name='test')
|
|
797
|
-
assert file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
|
|
798
|
-
assert file_dao.get(schema_ref='test', data_name='test', version_timestamp=0) is not None
|
|
799
|
-
|
|
800
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
801
|
-
@pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
|
|
802
|
-
def test_restore_nth_most_recent_unversioned_file_that_exists_and_has_been_removed(self, file_dao_options, file_type, n):
|
|
803
|
-
file_dao = file_dao_options[file_type]
|
|
804
|
-
first_tod = datetime.now().astimezone()
|
|
805
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=first_tod)
|
|
806
|
-
# restore and mark for deletion n times
|
|
807
|
-
for i in range(n):
|
|
808
|
-
file_dao.restore(schema_ref='test', data_name='test')
|
|
809
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=first_tod+timedelta(seconds=i+1))
|
|
810
|
-
assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
|
|
811
|
-
# restore one more time
|
|
812
|
-
file_dao.restore(schema_ref='test', data_name='test')
|
|
813
|
-
assert file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
|
|
814
|
-
nth_most_recent = file_dao.get(schema_ref='test', data_name='test', nth_most_recent=n)
|
|
815
|
-
assert nth_most_recent is not None
|
|
816
|
-
|
|
817
|
-
def test_restore_versioned_file_that_exists_and_has_been_removed(self, populated_numpy_file_dao):
|
|
818
|
-
to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
819
|
-
ts = to_remove.attrs['version_timestamp']
|
|
820
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
|
|
821
|
-
populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=ts)
|
|
822
|
-
assert populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
|
|
823
|
-
assert populated_numpy_file_dao.get(schema_ref='test', data_name='test', version_timestamp=ts) is not None
|
|
824
|
-
|
|
825
|
-
@pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
|
|
826
|
-
def test_restore_nth_most_recent_versioned_file_that_exists_and_has_been_removed(self, populated_numpy_file_dao, n):
|
|
827
|
-
first_tod = datetime.now().astimezone()
|
|
828
|
-
to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
829
|
-
ts = to_remove.attrs['version_timestamp']
|
|
830
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=first_tod)
|
|
831
|
-
# restore and mark for deletion n times
|
|
832
|
-
for i in range(n):
|
|
833
|
-
to_remove.attrs['data_name'] = f'test'
|
|
834
|
-
populated_numpy_file_dao.add(data_object=to_remove)
|
|
835
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name=f'test', version_timestamp=ts, time_of_removal=first_tod+timedelta(seconds=i+1))
|
|
836
|
-
assert not populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
|
|
837
|
-
# restore one more time
|
|
838
|
-
populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=ts, nth_most_recent=n)
|
|
839
|
-
assert populated_numpy_file_dao.exists(schema_ref='test', data_name=f'test', version_timestamp=ts)
|
|
840
|
-
|
|
841
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
842
|
-
def test_restore_unversioned_file_that_exists_and_has_not_been_removed(self, file_dao_options, file_type):
|
|
843
|
-
file_dao = file_dao_options[file_type]
|
|
844
|
-
with pytest.raises(FileSystemDAORangeError):
|
|
845
|
-
file_dao.restore(schema_ref='test', data_name='test')
|
|
846
|
-
|
|
847
|
-
def test_restore_versioned_file_that_exists_and_has_not_been_removed(self, populated_numpy_file_dao):
|
|
848
|
-
to_remove = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
849
|
-
ts = to_remove.attrs['version_timestamp']
|
|
850
|
-
with pytest.raises(FileSystemDAORangeError):
|
|
851
|
-
populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=ts)
|
|
852
|
-
|
|
853
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
854
|
-
def test_restore_unversioned_file_that_does_not_exist(self, file_dao_options, file_type):
|
|
855
|
-
file_dao = file_dao_options[file_type]
|
|
856
|
-
with pytest.raises(FileSystemDAORangeError):
|
|
857
|
-
file_dao.restore(schema_ref='not_a_schema', data_name='not_a_data_name')
|
|
858
|
-
|
|
859
|
-
def test_restore_versioned_file_that_does_not_exist(self, populated_numpy_file_dao):
|
|
860
|
-
with pytest.raises(FileSystemDAORangeError):
|
|
861
|
-
populated_numpy_file_dao.restore(schema_ref='test', data_name='test', version_timestamp=datetime.now().astimezone())
|
|
862
|
-
|
|
863
|
-
# purge tests (test all expected behaviors of purge())
|
|
864
|
-
# ---------------------------------------------------------
|
|
865
|
-
# Category 1: Purge all unversioned files that exist and have been removed
|
|
866
|
-
# Test 1.1: Purge all unversioned NetCDF dataarrays that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
867
|
-
# Test 1.2: Purge all unversioned Zarr dataarrays that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
868
|
-
# Test 1.3: Purge all unversioned NetCDF dataarrays that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
869
|
-
# Test 1.4: Purge all unversioned Zarr dataarrays that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
870
|
-
# Category 2: Purge all versioned files that exist and have been removed
|
|
871
|
-
# Test 2.1: Purge all versioned numpy mock model objects that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
872
|
-
# Test 2.2: Purge all versioned numpy mock model objects that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method
|
|
873
|
-
# Category 3: Purge only some unversioned files that exist and have been removed; check that the resulting purge count is correct
|
|
874
|
-
# Test 3.1: Purge all unversioned NetCDF dataarrays that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
875
|
-
# Test 3.2: Purge all unversioned Zarr dataarrays that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
876
|
-
# Category 4: Purge only some versioned files that exist and have been removed
|
|
877
|
-
# Test 4.1: Purge all versioned numpy mock model objects that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
878
|
-
# Category 5: Unversioned files when none have been removed
|
|
879
|
-
# Test 5.1: Purge all unversioned NetCDF dataarrays that exist and have not been removed; check that the purge count is 0
|
|
880
|
-
# Test 5.2: Purge all unversioned Zarr dataarrays that exist and have not been removed; check that the purge count is 0
|
|
881
|
-
# Category 6: Versioned files when none have been removed
|
|
882
|
-
# Test 6.1: Purge all versioned numpy mock model objects that exist and have not been removed; check that the purge count is 0
|
|
883
|
-
|
|
884
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
885
|
-
def test_purge_all_unversioned_files_that_exist_and_have_been_removed(self, file_dao_options, file_type):
|
|
886
|
-
file_dao = file_dao_options[file_type]
|
|
887
|
-
to_delete = file_dao.get(schema_ref='test', data_name='test')
|
|
888
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
|
|
889
|
-
for i in range(9):
|
|
890
|
-
to_delete.attrs['data_name'] = f'test{i}'
|
|
891
|
-
to_delete.attrs['schema_ref'] = f'test{i}'
|
|
892
|
-
file_dao.add(data_object=to_delete)
|
|
893
|
-
assert file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
|
|
894
|
-
file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', time_of_removal=datetime.now().astimezone(timezone.utc))
|
|
895
|
-
count = file_dao.purge(time_threshold=None)
|
|
896
|
-
assert count == 10
|
|
897
|
-
assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
|
|
898
|
-
for i in range(9):
|
|
899
|
-
assert not file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
|
|
900
|
-
assert len(file_dao.list_marked_for_deletion()) == 0
|
|
901
|
-
|
|
902
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
903
|
-
def test_purge_all_unversioned_files_that_exist_and_have_been_removed_with_time_threshold(self, file_dao_options, file_type):
|
|
904
|
-
file_dao = file_dao_options[file_type]
|
|
905
|
-
to_delete = file_dao.get(schema_ref='test', data_name='test')
|
|
906
|
-
file_dao.mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=datetime.now().astimezone())
|
|
907
|
-
for i in range(9):
|
|
908
|
-
to_delete.attrs['data_name'] = f'test{i}'
|
|
909
|
-
to_delete.attrs['schema_ref'] = f'test{i}'
|
|
910
|
-
file_dao.add(data_object=to_delete)
|
|
911
|
-
assert file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
|
|
912
|
-
file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', time_of_removal=datetime.now().astimezone())
|
|
913
|
-
count = file_dao.purge(time_threshold=datetime.now().astimezone(timezone.utc))
|
|
914
|
-
assert count == 10
|
|
915
|
-
assert not file_dao.exists(schema_ref='test', data_name='test', version_timestamp=0)
|
|
916
|
-
for i in range(9):
|
|
917
|
-
assert not file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=0)
|
|
918
|
-
assert len(file_dao.list_marked_for_deletion()) == 0
|
|
919
|
-
|
|
920
|
-
def test_purge_all_versioned_files_that_exist_and_have_been_removed(self, populated_numpy_file_dao):
|
|
921
|
-
to_delete = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
922
|
-
ts = to_delete.attrs['version_timestamp']
|
|
923
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
|
|
924
|
-
for i in range(9):
|
|
925
|
-
to_delete.attrs['data_name'] = f'test{i}'
|
|
926
|
-
to_delete.attrs['schema_ref'] = f'test{i}'
|
|
927
|
-
populated_numpy_file_dao.add(data_object=to_delete)
|
|
928
|
-
assert populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts)
|
|
929
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
|
|
930
|
-
count = populated_numpy_file_dao.purge(time_threshold=None)
|
|
931
|
-
assert count == 10
|
|
932
|
-
assert not populated_numpy_file_dao.exists(schema_ref='test', data_name='test', version_timestamp=ts)
|
|
933
|
-
for i in range(9):
|
|
934
|
-
assert not populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts)
|
|
935
|
-
assert len(populated_numpy_file_dao.list_marked_for_deletion()) == 0
|
|
936
|
-
|
|
937
|
-
def test_purge_all_versioned_files_that_exist_and_have_been_removed_with_time_threshold(self, populated_numpy_file_dao):
|
|
938
|
-
to_delete = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
939
|
-
ts = to_delete.attrs['version_timestamp']
|
|
940
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
|
|
941
|
-
for i in range(9):
|
|
942
|
-
to_delete.attrs['data_name'] = f'test{i}'
|
|
943
|
-
to_delete.attrs['schema_ref'] = f'test{i}'
|
|
944
|
-
populated_numpy_file_dao.add(data_object=to_delete)
|
|
945
|
-
assert populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts)
|
|
946
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts, time_of_removal=datetime.now().astimezone())
|
|
947
|
-
count = populated_numpy_file_dao.purge(time_threshold=datetime.now().astimezone())
|
|
948
|
-
assert count == 10
|
|
949
|
-
|
|
950
|
-
@pytest.mark.parametrize('file_type', ['netcdf', 'zarr'])
|
|
951
|
-
@pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
|
|
952
|
-
def test_purge_only_n_unversioned_files_that_exist_and_have_been_removed(self, file_dao_options, file_type, n):
|
|
953
|
-
to_delete = file_dao_options[file_type].get(schema_ref='test', data_name='test')
|
|
954
|
-
first_tod = datetime.now().astimezone()
|
|
955
|
-
file_dao_options[file_type].mark_for_deletion(schema_ref='test', data_name='test', time_of_removal=first_tod)
|
|
956
|
-
for i in range(n):
|
|
957
|
-
to_delete.attrs['data_name'] = f'test{i}'
|
|
958
|
-
to_delete.attrs['schema_ref'] = f'test{i}'
|
|
959
|
-
file_dao_options[file_type].add(data_object=to_delete)
|
|
960
|
-
assert file_dao_options[file_type].exists(schema_ref=f'test{i}', data_name=f'test{i}')
|
|
961
|
-
file_dao_options[file_type].mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', time_of_removal=first_tod + timedelta(seconds=i + 1))
|
|
962
|
-
count = file_dao_options[file_type].purge(time_threshold=first_tod+timedelta(seconds=n - 1))
|
|
963
|
-
assert count == n
|
|
964
|
-
|
|
965
|
-
@pytest.mark.parametrize('n', [1, 2, 3, 4, 5, 6, 7, 8, 9])
|
|
966
|
-
def test_purge_only_n_versioned_files_that_exist_and_have_been_removed(self, populated_numpy_file_dao, n):
|
|
967
|
-
to_delete = populated_numpy_file_dao.get(schema_ref='test', data_name='test', nth_most_recent=1)
|
|
968
|
-
ts = to_delete.attrs['version_timestamp']
|
|
969
|
-
first_tod = datetime.now().astimezone()
|
|
970
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref='test', data_name='test', version_timestamp=ts, time_of_removal=first_tod)
|
|
971
|
-
for i in range(n):
|
|
972
|
-
to_delete.attrs['data_name'] = f'test{i}'
|
|
973
|
-
to_delete.attrs['schema_ref'] = f'test{i}'
|
|
974
|
-
to_delete.attrs['version_timestamp'] = ts
|
|
975
|
-
populated_numpy_file_dao.add(data_object=to_delete)
|
|
976
|
-
assert populated_numpy_file_dao.exists(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts), f"test{i} does not exist"
|
|
977
|
-
populated_numpy_file_dao.mark_for_deletion(schema_ref=f'test{i}', data_name=f'test{i}', version_timestamp=ts, time_of_removal=first_tod + timedelta(seconds=i + 1))
|
|
978
|
-
count = populated_numpy_file_dao.purge(time_threshold=first_tod+timedelta(seconds=n - 1))
|
|
979
|
-
assert count == n, f"count is {count}"
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
class TestInMemoryObjectDAO:
|
|
983
|
-
|
|
984
|
-
# Get tests (test all expected behaviors of get())
|
|
985
|
-
# ------------------------------------------------
|
|
986
|
-
# Category 1: Objects that exist
|
|
987
|
-
# Test 1.1: Get a single object that exists; check that it exists; check that it has the right values; check that it has the right keys
|
|
988
|
-
# Category 2: Objects that do not exist
|
|
989
|
-
# Test 2.1: Get a single object that does not exist; check that it returns None
|
|
990
|
-
# Category 3: Bad arguments
|
|
991
|
-
# Test 3.1: Get a single object with a bad tag argument (error)
|
|
992
|
-
|
|
993
|
-
@pytest.mark.parametrize('object_type', [list, str, bool, int, float, type(np.array([])), type(xr.DataArray())])
|
|
994
|
-
def test_get_object_that_exists(self, populated_memory_dao, object_type):
|
|
995
|
-
result = populated_memory_dao.get(tag=f"test_{object_type.__name__.lower()}")
|
|
996
|
-
assert type(result) == object_type
|
|
997
|
-
|
|
998
|
-
def test_get_object_that_does_not_exist(self, populated_memory_dao):
|
|
999
|
-
result = populated_memory_dao.get(tag=f"not_a_tag")
|
|
1000
|
-
assert result is None
|
|
1001
|
-
|
|
1002
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
1003
|
-
def test_get_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
|
|
1004
|
-
with pytest.raises(InMemoryObjectDAOTypeError):
|
|
1005
|
-
populated_memory_dao.get(tag=bad_arg)
|
|
1006
|
-
|
|
1007
|
-
# Exists tests (test all expected behaviors of exists())
|
|
1008
|
-
# ------------------------------------------------------
|
|
1009
|
-
# Category 1: Objects that exist
|
|
1010
|
-
# Test 1.1: Check that a single object that exists returns True
|
|
1011
|
-
# Category 2: Objects that do not exist
|
|
1012
|
-
# Test 2.1: Check that a single object that does not exist returns False
|
|
1013
|
-
# Category 3: Bad arguments
|
|
1014
|
-
# Test 3.1: Check that a single object with a bad tag argument (error)
|
|
1015
|
-
|
|
1016
|
-
@pytest.mark.parametrize('object_type', [list, str, bool, int, float, type(np.array([])), type(xr.DataArray())])
|
|
1017
|
-
def test_exists_object_that_exists(self, populated_memory_dao, object_type):
|
|
1018
|
-
result = populated_memory_dao.exists(tag=f"test_{object_type.__name__.lower()}")
|
|
1019
|
-
assert result
|
|
1020
|
-
|
|
1021
|
-
def test_exists_object_that_does_not_exist(self, populated_memory_dao):
|
|
1022
|
-
result = populated_memory_dao.exists(tag=f"not_a_tag")
|
|
1023
|
-
assert not result
|
|
1024
|
-
|
|
1025
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
1026
|
-
def test_exists_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
|
|
1027
|
-
with pytest.raises(InMemoryObjectDAOTypeError):
|
|
1028
|
-
populated_memory_dao.exists(tag=bad_arg)
|
|
1029
|
-
|
|
1030
|
-
# Add tests (test all expected behaviors of add())
|
|
1031
|
-
# ------------------------------------------------
|
|
1032
|
-
# Category 1: Objects that do not exist
|
|
1033
|
-
# Test 1.1: Add a single object that does not exist; check that it exists; check that it has the right values; check that it has the right keys
|
|
1034
|
-
# Category 2: Objects that exist
|
|
1035
|
-
# Test 2.1: Add a single object with a tag that exists (error)
|
|
1036
|
-
# Test 2.2: Add a single object with an id(object) that exists (error)
|
|
1037
|
-
# Category 3: Bad arguments
|
|
1038
|
-
# Test 3.1: Add a single object with a bad tag argument (error)
|
|
1039
|
-
|
|
1040
|
-
@pytest.mark.parametrize('obj', [[1,2,3], 2, False, 1.5, np.array([1,2,3]), xr.DataArray([1,2,3])])
|
|
1041
|
-
def test_add_object_that_does_not_exist(self, populated_memory_dao, obj):
|
|
1042
|
-
populated_memory_dao.add(tag=f"new_{type(obj).__name__.lower()}", object=obj)
|
|
1043
|
-
result = populated_memory_dao.get(tag=f"new_{type(obj).__name__.lower()}")
|
|
1044
|
-
# if object is iterable, use all(==) if not use ==
|
|
1045
|
-
try:
|
|
1046
|
-
assert result == obj
|
|
1047
|
-
except Exception:
|
|
1048
|
-
assert all(result == obj)
|
|
1049
|
-
|
|
1050
|
-
@pytest.mark.parametrize('obj', [[1,2,3], 2, False, 1.5, np.array([1,2,3]), xr.DataArray([1,2,3])])
|
|
1051
|
-
def test_add_object_with_tag_that_exists(self, populated_memory_dao, obj):
|
|
1052
|
-
with pytest.raises(InMemoryObjectDAOObjectAlreadyExistsError):
|
|
1053
|
-
populated_memory_dao.add(tag=f"test_{type(obj).__name__.lower()}", object=obj)
|
|
1054
|
-
|
|
1055
|
-
def test_add_object_with_id_that_exists(self, populated_memory_dao, objects):
|
|
1056
|
-
for obj in objects:
|
|
1057
|
-
with pytest.raises(InMemoryObjectDAOObjectAlreadyExistsError):
|
|
1058
|
-
populated_memory_dao.add(tag=f"new_{type(obj).__name__.lower()}", object=obj)
|
|
1059
|
-
|
|
1060
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
1061
|
-
def test_add_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
|
|
1062
|
-
with pytest.raises(InMemoryObjectDAOTypeError):
|
|
1063
|
-
populated_memory_dao.add(tag=bad_arg, object=100)
|
|
1064
|
-
|
|
1065
|
-
# mark_for_deletion tests (test all expected behaviors of mark_for_deletion())
|
|
1066
|
-
# -----------------------------------------------------------------------------
|
|
1067
|
-
# Category 1: Objects that exist
|
|
1068
|
-
# Test 1.1: Mark a single object that exists for deletion; check that it can nolonger be accessed through get or the exists method; check that it is inside the deleted objects subcollection
|
|
1069
|
-
# Category 2: Objects that do not exist
|
|
1070
|
-
# Test 2.1: Mark a single object that does not exist for deletion (error)
|
|
1071
|
-
# Category 3: Bad arguments
|
|
1072
|
-
# Test 3.1: Mark a single object with a bad tag argument (error)
|
|
1073
|
-
|
|
1074
|
-
@pytest.mark.parametrize('tags', ['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray'])
|
|
1075
|
-
def test_mark_for_deletion_object_that_exists(self, populated_memory_dao, tags):
|
|
1076
|
-
obj = populated_memory_dao.get(tag=tags)
|
|
1077
|
-
obj_id = id(obj)
|
|
1078
|
-
populated_memory_dao.mark_for_deletion(tag=tags, time_of_removal = datetime.now().astimezone())
|
|
1079
|
-
assert not populated_memory_dao.exists(tag=tags)
|
|
1080
|
-
assert populated_memory_dao.get(tag=tags) is None
|
|
1081
|
-
del_obj = populated_memory_dao._collection['objects'][obj_id]
|
|
1082
|
-
try:
|
|
1083
|
-
assert all(del_obj == obj)
|
|
1084
|
-
except Exception:
|
|
1085
|
-
try:
|
|
1086
|
-
assert del_obj == obj
|
|
1087
|
-
except Exception:
|
|
1088
|
-
assert del_obj.equals(obj)
|
|
1089
|
-
|
|
1090
|
-
def test_mark_for_deletion_object_that_does_not_exist(self, populated_memory_dao):
|
|
1091
|
-
with pytest.raises(InMemoryObjectDAOObjectNotFoundError):
|
|
1092
|
-
populated_memory_dao.mark_for_deletion(tag='not_a_tag', time_of_removal = datetime.now().astimezone())
|
|
1093
|
-
|
|
1094
|
-
@pytest.mark.parametrize('bad_arg', [1, None, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
1095
|
-
def test_mark_for_deletion_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
|
|
1096
|
-
with pytest.raises(InMemoryObjectDAOTypeError):
|
|
1097
|
-
populated_memory_dao.mark_for_deletion(tag=bad_arg, time_of_removal = datetime.now().astimezone())
|
|
1098
|
-
|
|
1099
|
-
# List marked for deletion tests (test all expected behaviors of list_marked_for_deletion())
|
|
1100
|
-
# -----------------------------------------------------------------------------------------
|
|
1101
|
-
# Category 1: When there are several objects that have been marked for deletion
|
|
1102
|
-
# Test 1.1: List all objects that have been marked for deletion; check that it returns all removed objects; check that the objects are in the right order (by time_of_removal)
|
|
1103
|
-
# Test 1.2: List only objects from before the provided time_threshold that have been marked for deletion; check that it returns the right objects; check that the objects are in the right order (by time_of_removal)
|
|
1104
|
-
# Category 2: When no objects have been marked for deletion
|
|
1105
|
-
# Test 2.1: List all objects that have been marked for deletion; check that it returns an empty list
|
|
1106
|
-
# Category 3: Bad arguments
|
|
1107
|
-
# Test 3.1: List all objects with a bad time_threshold argument (error)
|
|
1108
|
-
|
|
1109
|
-
@pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
|
|
1110
|
-
def test_list_all_objects_marked_for_deltion_with_none_time_threshold(self, populated_memory_dao, tags):
|
|
1111
|
-
del_ids = []
|
|
1112
|
-
for obj in tags:
|
|
1113
|
-
del_ids.append(id(populated_memory_dao.get(tag=obj)))
|
|
1114
|
-
populated_memory_dao.mark_for_deletion(tag=obj, time_of_removal=datetime.now().astimezone())
|
|
1115
|
-
result = populated_memory_dao.list_marked_for_deletion()
|
|
1116
|
-
assert len(result) == len(tags)
|
|
1117
|
-
for i in range(len(result)):
|
|
1118
|
-
# check the order of the objects
|
|
1119
|
-
if i < len(result) - 1:
|
|
1120
|
-
assert result[i]["time_of_removal"] < result[i+1]["time_of_removal"], f"The objects were not in the right order"
|
|
1121
|
-
assert result[i]["tag"] == tags[i], f"The tag {tags[i]} was not in the right order"
|
|
1122
|
-
# check that the objects are the same
|
|
1123
|
-
assert result[i]["id"] == del_ids[i]
|
|
1124
|
-
|
|
1125
|
-
@pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
|
|
1126
|
-
@pytest.mark.parametrize('time_threshold_shift', list(map(lambda x: timedelta(seconds=x), range(1, 8))))
|
|
1127
|
-
def test_list_all_objects_marked_for_deltion_with_time_threshold(self, populated_memory_dao, tags, time_threshold_shift):
|
|
1128
|
-
del_ids = []
|
|
1129
|
-
removal_start_time = datetime.now().astimezone()
|
|
1130
|
-
for i, tag in enumerate(tags):
|
|
1131
|
-
del_ids.append(id(populated_memory_dao.get(tag=tag)))
|
|
1132
|
-
shift = timedelta(seconds=i + 0.5)
|
|
1133
|
-
populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=removal_start_time + shift)
|
|
1134
|
-
result = populated_memory_dao.list_marked_for_deletion(time_threshold=removal_start_time + time_threshold_shift)
|
|
1135
|
-
assert len(result) == time_threshold_shift.seconds
|
|
1136
|
-
for i in range(len(result)):
|
|
1137
|
-
# check the order of the objects
|
|
1138
|
-
if i < len(result) - 1:
|
|
1139
|
-
assert result[i]["time_of_removal"] < result[i+1]["time_of_removal"], f"The objects were not in the right order"
|
|
1140
|
-
# check that the objects are the same
|
|
1141
|
-
assert result[i]["id"] == del_ids[i]
|
|
1142
|
-
|
|
1143
|
-
def test_list_marked_for_deltion_when_non_have_been_marked(self, populated_memory_dao):
|
|
1144
|
-
result = populated_memory_dao.list_marked_for_deletion()
|
|
1145
|
-
assert len(result) == 0
|
|
1146
|
-
|
|
1147
|
-
@pytest.mark.parametrize('bad_arg', [1, 12.5, {"set"}, {"hash": "map"}])
|
|
1148
|
-
def test_list_marked_for_deletion_with_bad_time_threshold_arg(self, populated_memory_dao, bad_arg):
|
|
1149
|
-
with pytest.raises(InMemoryObjectDAOTypeError):
|
|
1150
|
-
populated_memory_dao.list_marked_for_deletion(time_threshold=bad_arg)
|
|
1151
|
-
|
|
1152
|
-
# restore tests (test all expected behaviors of restore())
|
|
1153
|
-
# ---------------------------------------------------------
|
|
1154
|
-
# Category 1: Objects that exist and have been removed
|
|
1155
|
-
# Test 1.1: Restore a single object that exists and has been removed; check that it exists; check that it has the right values; check that it has the right keys
|
|
1156
|
-
# Category 2: Objects tha have not been removed
|
|
1157
|
-
# Test 2.1: Restore a single object that exists and has not been removed (error)
|
|
1158
|
-
# Test 2.2: Restore a single object that does not exist (error)
|
|
1159
|
-
# Category 3: Restore an object that has already been permanently deleted
|
|
1160
|
-
# Test 3.1: Restore a single object that has already been permanently deleted (error)
|
|
1161
|
-
# Category 4: Bad arguments
|
|
1162
|
-
# Test 4.1: Restore a single object with a bad tag argument (error)
|
|
1163
|
-
|
|
1164
|
-
@pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
|
|
1165
|
-
def test_restore_object_that_exists_and_has_been_removed(self, populated_memory_dao, tags):
|
|
1166
|
-
for tag in tags:
|
|
1167
|
-
populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=datetime.now().astimezone())
|
|
1168
|
-
populated_memory_dao.restore(tag=tag)
|
|
1169
|
-
assert populated_memory_dao.exists(tag=tag)
|
|
1170
|
-
|
|
1171
|
-
def test_restore_object_that_exists_and_has_not_been_removed(self, populated_memory_dao):
|
|
1172
|
-
with pytest.raises(InMemoryObjectDAOObjectAlreadyExistsError):
|
|
1173
|
-
populated_memory_dao.restore(tag='test_list')
|
|
1174
|
-
|
|
1175
|
-
def test_restore_object_that_has_already_been_permanently_deleted(self, populated_memory_dao):
|
|
1176
|
-
populated_memory_dao.mark_for_deletion(tag='test_list', time_of_removal=datetime.now().astimezone())
|
|
1177
|
-
del populated_memory_dao._collection['removed']['test_list']
|
|
1178
|
-
with pytest.raises(InMemoryObjectDAOObjectNotFoundError):
|
|
1179
|
-
populated_memory_dao.restore(tag='test_list')
|
|
1180
|
-
|
|
1181
|
-
@pytest.mark.parametrize('bad_arg', [1, datetime.now().astimezone(), {"set"}, {"hash": "map"}])
|
|
1182
|
-
def test_restore_with_bad_tag_arg(self, populated_memory_dao, bad_arg):
|
|
1183
|
-
with pytest.raises(InMemoryObjectDAOTypeError):
|
|
1184
|
-
populated_memory_dao.restore(tag=bad_arg)
|
|
1185
|
-
|
|
1186
|
-
# purge tests (test all expected behaviors of purge())
|
|
1187
|
-
# ---------------------------------------------------------
|
|
1188
|
-
# Category 1: Purge all objects that exist and have been removed
|
|
1189
|
-
# Test 1.1: Purge all objects that exist and have been removed using time_threshold=None; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
1190
|
-
# Test 1.2: Purge all objects that exist and have been removed using time_threshold=first_tod; check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
1191
|
-
# Category 2: Purge only some objects that exist and have been removed; check that the resulting purge count is correct
|
|
1192
|
-
# Test 2.1: Purge all objects that exist and have been removed using time_threshold=first_tod+timedelta(seconds=1.2); check that they can nolonger be accessed through get or the exists method; check that the resulting purge count is correct
|
|
1193
|
-
# Category 3: Objects when none have been removed
|
|
1194
|
-
# Test 3.1: Purge all objects that exist and have not been removed; check that the purge count is 0
|
|
1195
|
-
# Category 4: Bad arguments
|
|
1196
|
-
# Test 4.1: Purge all objects with a bad time_threshold argument (error)
|
|
1197
|
-
|
|
1198
|
-
@pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
|
|
1199
|
-
def test_purge_all_objects_that_exist_and_have_been_removed(self, populated_memory_dao, tags):
|
|
1200
|
-
for tag in tags:
|
|
1201
|
-
populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=datetime.now().astimezone())
|
|
1202
|
-
count = populated_memory_dao.purge(time_threshold=None)
|
|
1203
|
-
assert count == len(tags)
|
|
1204
|
-
for tag in tags:
|
|
1205
|
-
assert not populated_memory_dao.exists(tag=tag)
|
|
1206
|
-
|
|
1207
|
-
@pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
|
|
1208
|
-
def test_purge_all_objects_that_exist_and_have_been_removed_with_time_threshold(self, populated_memory_dao, tags):
|
|
1209
|
-
for tag in tags:
|
|
1210
|
-
populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=datetime.now().astimezone())
|
|
1211
|
-
count = populated_memory_dao.purge(time_threshold=datetime.now().astimezone())
|
|
1212
|
-
assert count == len(tags)
|
|
1213
|
-
for tag in tags:
|
|
1214
|
-
assert not populated_memory_dao.exists(tag=tag)
|
|
1215
|
-
|
|
1216
|
-
@pytest.mark.parametrize('tags', [['test_list', 'test_str', 'test_bool', 'test_int', 'test_float', 'test_ndarray', 'test_dataarray']])
|
|
1217
|
-
@pytest.mark.parametrize('time_threshold_shift', list(map(lambda x: timedelta(seconds=x), range(1, 8))))
|
|
1218
|
-
def test_purge_only_some_objects_that_exist_and_have_been_removed(self, populated_memory_dao, tags, time_threshold_shift):
|
|
1219
|
-
removal_start_time = datetime.now().astimezone()
|
|
1220
|
-
for i, tag in enumerate(tags):
|
|
1221
|
-
shift = timedelta(seconds=i + 0.5)
|
|
1222
|
-
populated_memory_dao.mark_for_deletion(tag=tag, time_of_removal=removal_start_time + shift)
|
|
1223
|
-
count = populated_memory_dao.purge(time_threshold=removal_start_time + time_threshold_shift)
|
|
1224
|
-
assert count == time_threshold_shift.seconds
|
|
1225
|
-
for tag in tags:
|
|
1226
|
-
assert not populated_memory_dao.exists(tag=tag)
|
|
1227
|
-
|
|
1228
|
-
def test_purge_all_objects_that_exist_and_have_not_been_removed(self, populated_memory_dao):
|
|
1229
|
-
count = populated_memory_dao.purge(time_threshold=None)
|
|
1230
|
-
assert count == 0
|
|
1231
|
-
|
|
1232
|
-
@pytest.mark.parametrize('bad_arg', [1, 12.5, {"set"}, {"hash": "map"}])
|
|
1233
|
-
def test_purge_with_bad_time_threshold_arg(self, populated_memory_dao, bad_arg):
|
|
1234
|
-
with pytest.raises(InMemoryObjectDAOTypeError):
|
|
1235
|
-
populated_memory_dao.purge(time_threshold=bad_arg)
|