roc-film 1.13.5__py3-none-any.whl → 1.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- roc/__init__.py +2 -1
- roc/film/__init__.py +2 -2
- roc/film/commands.py +372 -323
- roc/film/config/__init__.py +0 -1
- roc/film/constants.py +101 -65
- roc/film/descriptor.json +127 -96
- roc/film/exceptions.py +28 -27
- roc/film/tasks/__init__.py +16 -16
- roc/film/tasks/cat_solo_hk.py +86 -74
- roc/film/tasks/cdf_postpro.py +438 -309
- roc/film/tasks/check_dds.py +39 -45
- roc/film/tasks/db_to_anc_bia_sweep_table.py +381 -0
- roc/film/tasks/dds_to_l0.py +232 -180
- roc/film/tasks/export_solo_coord.py +147 -0
- roc/film/tasks/file_handler.py +91 -75
- roc/film/tasks/l0_to_hk.py +117 -103
- roc/film/tasks/l0_to_l1_bia_current.py +38 -30
- roc/film/tasks/l0_to_l1_bia_sweep.py +417 -329
- roc/film/tasks/l0_to_l1_sbm.py +250 -208
- roc/film/tasks/l0_to_l1_surv.py +185 -130
- roc/film/tasks/make_daily_tm.py +40 -37
- roc/film/tasks/merge_tcreport.py +77 -71
- roc/film/tasks/merge_tmraw.py +101 -88
- roc/film/tasks/parse_dds_xml.py +21 -20
- roc/film/tasks/set_l0_utc.py +51 -49
- roc/film/tests/cdf_compare.py +565 -0
- roc/film/tests/hdf5_compare.py +84 -62
- roc/film/tests/test_dds_to_l0.py +93 -51
- roc/film/tests/test_dds_to_tc.py +8 -11
- roc/film/tests/test_dds_to_tm.py +8 -10
- roc/film/tests/test_film.py +161 -116
- roc/film/tests/test_l0_to_hk.py +64 -36
- roc/film/tests/test_l0_to_l1_bia.py +10 -14
- roc/film/tests/test_l0_to_l1_sbm.py +14 -19
- roc/film/tests/test_l0_to_l1_surv.py +68 -41
- roc/film/tests/test_metadata.py +21 -20
- roc/film/tests/tests.py +743 -396
- roc/film/tools/__init__.py +5 -5
- roc/film/tools/dataset_tasks.py +34 -2
- roc/film/tools/file_helpers.py +390 -269
- roc/film/tools/l0.py +402 -324
- roc/film/tools/metadata.py +147 -127
- roc/film/tools/skeleton.py +12 -17
- roc/film/tools/tools.py +109 -92
- roc/film/tools/xlsx2skt.py +161 -139
- {roc_film-1.13.5.dist-info → roc_film-1.14.0.dist-info}/LICENSE +127 -125
- roc_film-1.14.0.dist-info/METADATA +60 -0
- roc_film-1.14.0.dist-info/RECORD +50 -0
- {roc_film-1.13.5.dist-info → roc_film-1.14.0.dist-info}/WHEEL +1 -1
- roc/film/tasks/l0_to_anc_bia_sweep_table.py +0 -348
- roc_film-1.13.5.dist-info/METADATA +0 -120
- roc_film-1.13.5.dist-info/RECORD +0 -48
roc/film/tests/tests.py
CHANGED
@@ -9,6 +9,20 @@ import subprocess
|
|
9
9
|
|
10
10
|
from poppy.core.test import TaskTestCase
|
11
11
|
|
12
|
+
import filecmp
|
13
|
+
import os
|
14
|
+
import tempfile
|
15
|
+
from pprint import pformat
|
16
|
+
|
17
|
+
import pytest
|
18
|
+
import shutil
|
19
|
+
import unittest.mock as mock
|
20
|
+
from maser.tools.cdf.cdfcompare import cdf_compare
|
21
|
+
from poppy.core.generic.requests import download_file
|
22
|
+
from poppy.core.logger import logger
|
23
|
+
from poppy.core.test import CommandTestCase
|
24
|
+
|
25
|
+
|
12
26
|
# Tests on roc.film.tasks.metadata methods
|
13
27
|
class TestFilmMetadata(TaskTestCase):
|
14
28
|
# create the pipeline
|
@@ -25,14 +39,14 @@ class TestFilmMetadata(TaskTestCase):
|
|
25
39
|
|
26
40
|
# initialize meta dictionary with expected inputs
|
27
41
|
meta = {
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
42
|
+
"Source_name": "SOLO>Solar Orbiter",
|
43
|
+
"LEVEL": "L1>Level 1 data processing",
|
44
|
+
"Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
|
45
|
+
"Datetime": "20220103",
|
46
|
+
"Data_version": "01",
|
47
|
+
}
|
34
48
|
|
35
|
-
assert build_file_basename(meta) ==
|
49
|
+
assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
|
36
50
|
|
37
51
|
def test_build_file_basename_with_logical(self):
|
38
52
|
"""
|
@@ -45,8 +59,8 @@ class TestFilmMetadata(TaskTestCase):
|
|
45
59
|
# import method to test
|
46
60
|
from roc.film.tools.file_helpers import build_file_basename
|
47
61
|
|
48
|
-
meta = {
|
49
|
-
assert build_file_basename(meta) ==
|
62
|
+
meta = {"Logical_file_id": "solo_L1_rpw-hfr-surv_20220103_V01"}
|
63
|
+
assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
|
50
64
|
|
51
65
|
# def test_generate_filepath(self):
|
52
66
|
|
@@ -60,7 +74,7 @@ class TestFilmMetadata(TaskTestCase):
|
|
60
74
|
# import method to test
|
61
75
|
from roc.film.tools.metadata import valid_data_version
|
62
76
|
|
63
|
-
assert valid_data_version(1) ==
|
77
|
+
assert valid_data_version(1) == "01"
|
64
78
|
|
65
79
|
def test_set_logical_file_id(self):
|
66
80
|
"""
|
@@ -74,36 +88,24 @@ class TestFilmMetadata(TaskTestCase):
|
|
74
88
|
|
75
89
|
# initialize meta dictionary with expected inputs
|
76
90
|
meta = {
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
assert set_logical_file_id(meta) == 'solo_L1_rpw-hfr-surv_20220103_V01'
|
87
|
-
|
88
|
-
|
89
|
-
import filecmp
|
90
|
-
import os
|
91
|
-
import tempfile
|
92
|
-
from pprint import pformat
|
91
|
+
"File_naming_convention": "<Source_name>_<LEVEL>_<Descriptor>_"
|
92
|
+
"<Datetime>_V<Data_version>",
|
93
|
+
"Source_name": "SOLO>Solar Orbiter",
|
94
|
+
"LEVEL": "L1>Level 1 data processing",
|
95
|
+
"Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
|
96
|
+
"Datetime": "20220103",
|
97
|
+
"Data_version": "01",
|
98
|
+
}
|
93
99
|
|
94
|
-
|
95
|
-
import shutil
|
96
|
-
import unittest.mock as mock
|
97
|
-
from maser.tools.cdf.cdfcompare import cdf_compare
|
98
|
-
from poppy.core.generic.requests import download_file
|
99
|
-
from poppy.core.logger import logger
|
100
|
-
from poppy.core.test import CommandTestCase
|
100
|
+
assert set_logical_file_id(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
|
101
101
|
|
102
102
|
|
103
103
|
class TestFilmFileProd(CommandTestCase):
|
104
|
-
base_url =
|
104
|
+
base_url = (
|
105
|
+
"https://rpw.lesia.obspm.fr/roc/data/private/devtest/roc/test_data/rodp/film"
|
106
|
+
)
|
105
107
|
# test credentials
|
106
|
-
username =
|
108
|
+
username = "roctest"
|
107
109
|
password = None
|
108
110
|
|
109
111
|
@classmethod
|
@@ -113,10 +115,12 @@ class TestFilmFileProd(CommandTestCase):
|
|
113
115
|
"""
|
114
116
|
|
115
117
|
try:
|
116
|
-
cls.password = os.environ[
|
118
|
+
cls.password = os.environ["ROC_TEST_PASSWORD"]
|
117
119
|
except KeyError:
|
118
|
-
raise KeyError(
|
119
|
-
|
120
|
+
raise KeyError(
|
121
|
+
'You have to define the test user password using the "ROC_TEST_PASSWORD" environment'
|
122
|
+
"variable "
|
123
|
+
)
|
120
124
|
|
121
125
|
def setup_method(self, method):
|
122
126
|
super().setup_method(method)
|
@@ -124,27 +128,28 @@ class TestFilmFileProd(CommandTestCase):
|
|
124
128
|
self.tmp_dir_path = tempfile.mkdtemp()
|
125
129
|
|
126
130
|
def load_manifest_file(self, manifest_filepath, manifest_file_url, auth=None):
|
127
|
-
|
128
131
|
download_file(manifest_filepath, manifest_file_url, auth=auth)
|
129
132
|
|
130
133
|
with open(manifest_filepath) as manifest_file:
|
131
134
|
for line in manifest_file:
|
132
|
-
yield line.strip(
|
135
|
+
yield line.strip("\n\r")
|
133
136
|
|
134
137
|
os.remove(manifest_filepath)
|
135
138
|
|
136
139
|
def get_files(self, subdir, category=None):
|
137
|
-
categories = [
|
140
|
+
categories = ["input", "expected_output", "ancillarie"]
|
138
141
|
if category not in categories:
|
139
|
-
raise ValueError(
|
142
|
+
raise ValueError("Invalid category. Expected one of: %s" % categories)
|
140
143
|
|
141
144
|
auth = (self.username, self.password)
|
142
145
|
|
143
146
|
dir_path = os.path.join(self.tmp_dir_path, subdir, category)
|
144
147
|
os.makedirs(dir_path, exist_ok=True)
|
145
|
-
manifest_filepath = os.path.join(dir_path,
|
146
|
-
manifest_file_url = f
|
147
|
-
file_list = list(
|
148
|
+
manifest_filepath = os.path.join(dir_path, "manifest.txt")
|
149
|
+
manifest_file_url = f"{self.base_url}/{subdir}/{category}s/manifest.txt"
|
150
|
+
file_list = list(
|
151
|
+
self.load_manifest_file(manifest_filepath, manifest_file_url, auth=auth)
|
152
|
+
)
|
148
153
|
|
149
154
|
for relative_filepath in file_list:
|
150
155
|
# skip empty strings
|
@@ -154,17 +159,21 @@ class TestFilmFileProd(CommandTestCase):
|
|
154
159
|
# get the complete filepath
|
155
160
|
filepath = os.path.join(dir_path, relative_filepath)
|
156
161
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
157
|
-
download_file(
|
158
|
-
|
159
|
-
|
162
|
+
download_file(
|
163
|
+
filepath,
|
164
|
+
f"{self.base_url}/{subdir}/{category}s/{relative_filepath}",
|
165
|
+
auth=auth,
|
166
|
+
)
|
160
167
|
|
161
168
|
return dir_path, file_list
|
162
169
|
|
163
|
-
def get_diff_files(self, dirs_cmp, path=
|
170
|
+
def get_diff_files(self, dirs_cmp, path=""):
|
164
171
|
for name in dirs_cmp.diff_files:
|
165
172
|
yield os.path.join(path, name)
|
166
173
|
for parent, sub_dirs_cmp in dirs_cmp.subdirs.items():
|
167
|
-
for filepath in self.get_diff_files(
|
174
|
+
for filepath in self.get_diff_files(
|
175
|
+
sub_dirs_cmp, path=os.path.join(path, parent)
|
176
|
+
):
|
168
177
|
yield filepath
|
169
178
|
|
170
179
|
def teardown_method(self, method):
|
@@ -184,16 +193,15 @@ class TestFilmFileProd(CommandTestCase):
|
|
184
193
|
shutil.rmtree(self.tmp_dir_path)
|
185
194
|
|
186
195
|
def get_inputs(self, subdir):
|
187
|
-
return self.get_files(subdir, category=
|
196
|
+
return self.get_files(subdir, category="input")
|
188
197
|
|
189
198
|
def get_expected_outputs(self, subdir):
|
190
|
-
return self.get_files(subdir, category=
|
199
|
+
return self.get_files(subdir, category="expected_output")
|
191
200
|
|
192
201
|
def get_ancillaries(self, subdir):
|
193
|
-
return self.get_files(subdir, category=
|
202
|
+
return self.get_files(subdir, category="ancillarie")
|
194
203
|
|
195
|
-
def unzip_kernels(self, zip_path,
|
196
|
-
pipeline_dir='/pipeline'):
|
204
|
+
def unzip_kernels(self, zip_path, pipeline_dir="/pipeline"):
|
197
205
|
"""
|
198
206
|
Unzip input SPICE kernel ZIP file.
|
199
207
|
|
@@ -208,74 +216,108 @@ class TestFilmFileProd(CommandTestCase):
|
|
208
216
|
|
209
217
|
# check zip file existence
|
210
218
|
if not os.path.isfile(zip_path):
|
211
|
-
raise FileNotFoundError(
|
219
|
+
raise FileNotFoundError(
|
220
|
+
f"ZIP file containing SPICE kernel not found ({zip_path})!"
|
221
|
+
)
|
212
222
|
|
213
223
|
# Get zip directory
|
214
224
|
zip_dir = os.path.dirname(zip_path)
|
215
225
|
|
216
226
|
# Unzip ZIP file
|
217
227
|
try:
|
218
|
-
cmd = f
|
219
|
-
completed = subprocess.run(cmd,
|
220
|
-
|
221
|
-
shell=True, timeout=3600)
|
228
|
+
cmd = f"unzip -o {zip_path}"
|
229
|
+
completed = subprocess.run(cmd, cwd=zip_dir, shell=True, timeout=3600)
|
230
|
+
logger.debug(completed)
|
222
231
|
# Path of the unzipped kernels dir
|
223
|
-
kernels_dir = os.path.join(zip_dir,
|
232
|
+
kernels_dir = os.path.join(zip_dir, "kernels")
|
224
233
|
if not os.path.isdir(kernels_dir):
|
225
234
|
raise FileNotFoundError
|
226
235
|
|
227
|
-
except:
|
228
|
-
logger.exception(f
|
229
|
-
f'{completed.stderr}')
|
236
|
+
except Exception as e:
|
237
|
+
logger.exception(f"Extracting {zip_dir} has failed: \n{e}")
|
230
238
|
else:
|
231
239
|
# Move unzipped "kernels" folder into /pipeline/data/spice_kernels
|
232
240
|
if os.path.isdir(pipeline_dir):
|
233
|
-
data_dir = os.path.join(pipeline_dir,
|
241
|
+
data_dir = os.path.join(pipeline_dir, "data")
|
234
242
|
os.makedirs(data_dir, exist_ok=True)
|
235
|
-
target_dir = os.path.join(data_dir,
|
243
|
+
target_dir = os.path.join(data_dir, "spice_kernels")
|
236
244
|
shutil.move(kernels_dir, target_dir)
|
237
245
|
else:
|
238
246
|
target_dir = kernels_dir
|
239
247
|
|
240
248
|
return target_dir
|
241
249
|
|
242
|
-
@pytest.mark.parametrize(
|
243
|
-
|
244
|
-
|
250
|
+
@pytest.mark.parametrize(
|
251
|
+
"idb_source,idb_version",
|
252
|
+
[
|
253
|
+
("MIB", "20200131"),
|
254
|
+
],
|
255
|
+
)
|
245
256
|
def test_classify_tmraw(self, idb_source, idb_version):
|
246
257
|
from poppy.core.conf import Settings
|
247
258
|
|
248
|
-
input_dir_path, inputs = self.get_inputs(
|
249
|
-
expected_output_dir_path, expected_outputs = self.get_expected_outputs(
|
259
|
+
input_dir_path, inputs = self.get_inputs("classify_tmraw")
|
260
|
+
expected_output_dir_path, expected_outputs = self.get_expected_outputs(
|
261
|
+
"classify_tmraw"
|
262
|
+
)
|
250
263
|
|
251
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
264
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
252
265
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
253
266
|
|
254
267
|
# initialize the main command
|
255
|
-
main_command = [
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
268
|
+
main_command = [
|
269
|
+
"pop",
|
270
|
+
"film",
|
271
|
+
"classify_tmraw",
|
272
|
+
"--dds-files",
|
273
|
+
" ".join(
|
274
|
+
[os.path.join(input_dir_path, input_file) for input_file in inputs]
|
275
|
+
),
|
276
|
+
"--output-dir",
|
277
|
+
generated_output_dir_path,
|
278
|
+
"-ll",
|
279
|
+
"INFO",
|
280
|
+
]
|
263
281
|
|
264
282
|
# define the required plugins
|
265
|
-
plugin_list = [
|
283
|
+
plugin_list = [
|
284
|
+
"poppy.pop",
|
285
|
+
"roc.idb",
|
286
|
+
"roc.rpl",
|
287
|
+
"roc.rap",
|
288
|
+
"roc.dingo",
|
289
|
+
"roc.film",
|
290
|
+
]
|
266
291
|
|
267
292
|
# run the command
|
268
293
|
# force the value of the plugin list
|
269
|
-
with mock.patch.object(
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
self.
|
294
|
+
with mock.patch.object(
|
295
|
+
Settings,
|
296
|
+
"configure",
|
297
|
+
autospec=True,
|
298
|
+
side_effect=self.mock_configure_settings(
|
299
|
+
dictionary={"PLUGINS": plugin_list}
|
300
|
+
),
|
301
|
+
):
|
302
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
303
|
+
self.run_command(
|
304
|
+
[
|
305
|
+
"pop",
|
306
|
+
"-ll",
|
307
|
+
"INFO",
|
308
|
+
"idb",
|
309
|
+
"install",
|
310
|
+
"-s",
|
311
|
+
idb_source,
|
312
|
+
"-v",
|
313
|
+
idb_version,
|
314
|
+
"--load",
|
315
|
+
]
|
316
|
+
)
|
274
317
|
self.run_command(main_command)
|
275
318
|
|
276
319
|
# compare directory content
|
277
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
278
|
-
expected_output_dir_path)
|
320
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
279
321
|
|
280
322
|
dirs_cmp.report()
|
281
323
|
|
@@ -284,7 +326,7 @@ class TestFilmFileProd(CommandTestCase):
|
|
284
326
|
|
285
327
|
for filename in self.get_diff_files(dirs_cmp):
|
286
328
|
# compare only xml files with differences
|
287
|
-
if filename.endswith(
|
329
|
+
if filename.endswith(".xml"):
|
288
330
|
# use cdf compare to compute the differences between expected output and the command output
|
289
331
|
result = filecmp.cmpfiles(
|
290
332
|
os.path.join(generated_output_dir_path, filename),
|
@@ -292,48 +334,83 @@ class TestFilmFileProd(CommandTestCase):
|
|
292
334
|
)
|
293
335
|
# compare the difference dict with the expected one
|
294
336
|
if result:
|
295
|
-
logger.error(
|
337
|
+
logger.error(
|
338
|
+
f"Differences between expected output and the command output: {pformat(result)}"
|
339
|
+
)
|
296
340
|
|
297
341
|
assert result == {}
|
298
342
|
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
343
|
+
@pytest.mark.parametrize(
|
344
|
+
"idb_source,idb_version",
|
345
|
+
[
|
346
|
+
("MIB", "20200131"),
|
347
|
+
],
|
348
|
+
)
|
303
349
|
def test_classify_tcreport(self, idb_source, idb_version):
|
304
350
|
from poppy.core.conf import Settings
|
305
351
|
|
306
|
-
input_dir_path, inputs = self.get_inputs(
|
307
|
-
expected_output_dir_path, expected_outputs = self.get_expected_outputs(
|
352
|
+
input_dir_path, inputs = self.get_inputs("classify_tcreport")
|
353
|
+
expected_output_dir_path, expected_outputs = self.get_expected_outputs(
|
354
|
+
"classify_tcreport"
|
355
|
+
)
|
308
356
|
|
309
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
357
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
310
358
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
311
359
|
|
312
360
|
# initialize the main command
|
313
|
-
main_command = [
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
361
|
+
main_command = [
|
362
|
+
"pop",
|
363
|
+
"film",
|
364
|
+
"classify_tcreport",
|
365
|
+
"--dds-files",
|
366
|
+
" ".join(
|
367
|
+
[os.path.join(input_dir_path, input_file) for input_file in inputs]
|
368
|
+
),
|
369
|
+
"--output-dir",
|
370
|
+
generated_output_dir_path,
|
371
|
+
"-ll",
|
372
|
+
"INFO",
|
373
|
+
]
|
321
374
|
|
322
375
|
# define the required plugins
|
323
|
-
plugin_list = [
|
376
|
+
plugin_list = [
|
377
|
+
"poppy.pop",
|
378
|
+
"roc.idb",
|
379
|
+
"roc.rpl",
|
380
|
+
"roc.rap",
|
381
|
+
"roc.dingo",
|
382
|
+
"roc.film",
|
383
|
+
]
|
324
384
|
|
325
385
|
# run the command
|
326
386
|
# force the value of the plugin list
|
327
|
-
with mock.patch.object(
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
self.
|
387
|
+
with mock.patch.object(
|
388
|
+
Settings,
|
389
|
+
"configure",
|
390
|
+
autospec=True,
|
391
|
+
side_effect=self.mock_configure_settings(
|
392
|
+
dictionary={"PLUGINS": plugin_list}
|
393
|
+
),
|
394
|
+
):
|
395
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
396
|
+
self.run_command(
|
397
|
+
[
|
398
|
+
"pop",
|
399
|
+
"-ll",
|
400
|
+
"INFO",
|
401
|
+
"idb",
|
402
|
+
"install",
|
403
|
+
"-s",
|
404
|
+
idb_source,
|
405
|
+
"-v",
|
406
|
+
idb_version,
|
407
|
+
"--load",
|
408
|
+
]
|
409
|
+
)
|
332
410
|
self.run_command(main_command)
|
333
411
|
|
334
412
|
# compare directory content
|
335
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
336
|
-
expected_output_dir_path)
|
413
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
337
414
|
|
338
415
|
dirs_cmp.report()
|
339
416
|
|
@@ -342,7 +419,7 @@ class TestFilmFileProd(CommandTestCase):
|
|
342
419
|
|
343
420
|
for filename in self.get_diff_files(dirs_cmp):
|
344
421
|
# compare only xml files with differences
|
345
|
-
if filename.endswith(
|
422
|
+
if filename.endswith(".xml"):
|
346
423
|
# use cdf compare to compute the differences between expected output and the command output
|
347
424
|
result = filecmp.cmpfiles(
|
348
425
|
os.path.join(generated_output_dir_path, filename),
|
@@ -350,47 +427,83 @@ class TestFilmFileProd(CommandTestCase):
|
|
350
427
|
)
|
351
428
|
# compare the difference dict with the expected one
|
352
429
|
if result:
|
353
|
-
logger.error(
|
430
|
+
logger.error(
|
431
|
+
f"Differences between expected output and the command output: {pformat(result)}"
|
432
|
+
)
|
354
433
|
|
355
434
|
assert result == {}
|
356
435
|
|
357
|
-
@pytest.mark.parametrize(
|
358
|
-
|
359
|
-
|
436
|
+
@pytest.mark.parametrize(
|
437
|
+
"idb_source,idb_version",
|
438
|
+
[
|
439
|
+
("MIB", "20200131"),
|
440
|
+
],
|
441
|
+
)
|
360
442
|
def test_classify_process_solohk(self, idb_source, idb_version):
|
361
443
|
from poppy.core.conf import Settings
|
362
444
|
|
363
|
-
input_dir_path, inputs = self.get_inputs(
|
364
|
-
expected_output_dir_path, expected_outputs = self.get_expected_outputs(
|
445
|
+
input_dir_path, inputs = self.get_inputs("process_solohk")
|
446
|
+
expected_output_dir_path, expected_outputs = self.get_expected_outputs(
|
447
|
+
"process_solohk"
|
448
|
+
)
|
365
449
|
|
366
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
450
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
367
451
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
368
452
|
|
369
453
|
# initialize the main command
|
370
|
-
main_command = [
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
454
|
+
main_command = [
|
455
|
+
"pop",
|
456
|
+
"film",
|
457
|
+
"process_solohk",
|
458
|
+
"--dds-files",
|
459
|
+
" ".join(
|
460
|
+
[os.path.join(input_dir_path, input_file) for input_file in inputs]
|
461
|
+
),
|
462
|
+
"--output-dir",
|
463
|
+
generated_output_dir_path,
|
464
|
+
"-ll",
|
465
|
+
"INFO",
|
466
|
+
]
|
378
467
|
|
379
468
|
# define the required plugins
|
380
|
-
plugin_list = [
|
469
|
+
plugin_list = [
|
470
|
+
"poppy.pop",
|
471
|
+
"roc.idb",
|
472
|
+
"roc.rpl",
|
473
|
+
"roc.rap",
|
474
|
+
"roc.dingo",
|
475
|
+
"roc.film",
|
476
|
+
]
|
381
477
|
|
382
478
|
# run the command
|
383
479
|
# force the value of the plugin list
|
384
|
-
with mock.patch.object(
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
self.
|
480
|
+
with mock.patch.object(
|
481
|
+
Settings,
|
482
|
+
"configure",
|
483
|
+
autospec=True,
|
484
|
+
side_effect=self.mock_configure_settings(
|
485
|
+
dictionary={"PLUGINS": plugin_list}
|
486
|
+
),
|
487
|
+
):
|
488
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
489
|
+
self.run_command(
|
490
|
+
[
|
491
|
+
"pop",
|
492
|
+
"-ll",
|
493
|
+
"INFO",
|
494
|
+
"idb",
|
495
|
+
"install",
|
496
|
+
"-s",
|
497
|
+
idb_source,
|
498
|
+
"-v",
|
499
|
+
idb_version,
|
500
|
+
"--load",
|
501
|
+
]
|
502
|
+
)
|
389
503
|
self.run_command(main_command)
|
390
504
|
|
391
505
|
# compare directory content
|
392
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
393
|
-
expected_output_dir_path)
|
506
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
394
507
|
|
395
508
|
dirs_cmp.report()
|
396
509
|
|
@@ -399,7 +512,7 @@ class TestFilmFileProd(CommandTestCase):
|
|
399
512
|
|
400
513
|
for filename in self.get_diff_files(dirs_cmp):
|
401
514
|
# compare only xml files with differences
|
402
|
-
if filename.endswith(
|
515
|
+
if filename.endswith(".xml"):
|
403
516
|
# use cdf compare to compute the differences between expected output and the command output
|
404
517
|
result = filecmp.cmpfiles(
|
405
518
|
os.path.join(generated_output_dir_path, filename),
|
@@ -407,19 +520,24 @@ class TestFilmFileProd(CommandTestCase):
|
|
407
520
|
)
|
408
521
|
# compare the difference dict with the expected one
|
409
522
|
if result:
|
410
|
-
logger.error(
|
523
|
+
logger.error(
|
524
|
+
f"Differences between expected output and the command output: {pformat(result)}"
|
525
|
+
)
|
411
526
|
|
412
527
|
assert result == {}
|
413
528
|
|
414
|
-
@pytest.mark.parametrize(
|
415
|
-
|
416
|
-
|
417
|
-
|
529
|
+
@pytest.mark.parametrize(
|
530
|
+
"idb_source,idb_version",
|
531
|
+
[
|
532
|
+
("MIB", "20200131"),
|
533
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
534
|
+
],
|
535
|
+
)
|
418
536
|
def test_l0_to_l1_surv(self, idb_source, idb_version):
|
419
537
|
from poppy.core.conf import Settings
|
420
538
|
|
421
539
|
# Name of the command to test
|
422
|
-
cmd =
|
540
|
+
cmd = "l0_to_l1_surv"
|
423
541
|
|
424
542
|
input_dir_path, inputs = self.get_inputs(cmd)
|
425
543
|
expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -427,35 +545,60 @@ class TestFilmFileProd(CommandTestCase):
|
|
427
545
|
|
428
546
|
# extract spice kernels
|
429
547
|
spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
548
|
+
logger.debug(spice_kernel_dir_path)
|
430
549
|
|
431
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
550
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
432
551
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
433
552
|
|
434
553
|
# initialize the main command
|
435
|
-
main_command = [
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
554
|
+
main_command = [
|
555
|
+
"pop",
|
556
|
+
"film",
|
557
|
+
"--idb-version",
|
558
|
+
idb_version,
|
559
|
+
"--idb-source",
|
560
|
+
idb_source,
|
561
|
+
"--cdag",
|
562
|
+
cmd,
|
563
|
+
os.path.join(input_dir_path, inputs[0]),
|
564
|
+
"--output-dir",
|
565
|
+
generated_output_dir_path,
|
566
|
+
"-ll",
|
567
|
+
"INFO",
|
568
|
+
]
|
443
569
|
|
444
570
|
# define the required plugins
|
445
|
-
plugin_list = [
|
571
|
+
plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
|
446
572
|
|
447
573
|
# run the command
|
448
574
|
# force the value of the plugin list
|
449
|
-
with mock.patch.object(
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
self.
|
575
|
+
with mock.patch.object(
|
576
|
+
Settings,
|
577
|
+
"configure",
|
578
|
+
autospec=True,
|
579
|
+
side_effect=self.mock_configure_settings(
|
580
|
+
dictionary={"PLUGINS": plugin_list}
|
581
|
+
),
|
582
|
+
):
|
583
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
584
|
+
self.run_command(
|
585
|
+
[
|
586
|
+
"pop",
|
587
|
+
"-ll",
|
588
|
+
"INFO",
|
589
|
+
"idb",
|
590
|
+
"install",
|
591
|
+
"-s",
|
592
|
+
idb_source,
|
593
|
+
"-v",
|
594
|
+
idb_version,
|
595
|
+
"--load",
|
596
|
+
]
|
597
|
+
)
|
454
598
|
self.run_command(main_command)
|
455
599
|
|
456
600
|
# compare directory content
|
457
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
458
|
-
expected_output_dir_path)
|
601
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
459
602
|
|
460
603
|
dirs_cmp.report()
|
461
604
|
|
@@ -464,36 +607,41 @@ class TestFilmFileProd(CommandTestCase):
|
|
464
607
|
|
465
608
|
for filename in self.get_diff_files(dirs_cmp):
|
466
609
|
# compare only cdf files with differences
|
467
|
-
if filename.endswith(
|
610
|
+
if filename.endswith(".cdf"):
|
468
611
|
# use cdf compare to compute the differences between expected output and the command output
|
469
612
|
result = cdf_compare(
|
470
613
|
os.path.join(generated_output_dir_path, filename),
|
471
614
|
os.path.join(expected_output_dir_path, filename),
|
472
615
|
list_ignore_gatt=[
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
]
|
616
|
+
"File_ID",
|
617
|
+
"Generation_date",
|
618
|
+
"Pipeline_version",
|
619
|
+
"Pipeline_name",
|
620
|
+
"Software_version",
|
621
|
+
"IDB_version",
|
622
|
+
],
|
480
623
|
)
|
481
624
|
|
482
625
|
# compare the difference dict with the expected one
|
483
626
|
if result:
|
484
|
-
logger.error(
|
627
|
+
logger.error(
|
628
|
+
f"Differences between expected output and the command output: {pformat(result)}"
|
629
|
+
)
|
485
630
|
|
486
631
|
assert result == {}
|
487
632
|
|
488
|
-
@pytest.mark.parametrize(
|
489
|
-
|
490
|
-
|
491
|
-
|
633
|
+
@pytest.mark.parametrize(
|
634
|
+
"idb_source,idb_version",
|
635
|
+
[
|
636
|
+
("MIB", "20200131"),
|
637
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
638
|
+
],
|
639
|
+
)
|
492
640
|
def test_l0_to_hk(self, idb_source, idb_version):
|
493
641
|
from poppy.core.conf import Settings
|
494
642
|
|
495
643
|
# Name of the command to test
|
496
|
-
cmd =
|
644
|
+
cmd = "l0_to_hk"
|
497
645
|
|
498
646
|
input_dir_path, inputs = self.get_inputs(cmd)
|
499
647
|
expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -501,34 +649,66 @@ class TestFilmFileProd(CommandTestCase):
|
|
501
649
|
|
502
650
|
# extract spice kernels
|
503
651
|
spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
652
|
+
logger.debug(spice_kernel_dir_path)
|
504
653
|
|
505
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
654
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
506
655
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
507
656
|
|
508
657
|
# initialize the main command
|
509
|
-
main_command = [
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
658
|
+
main_command = [
|
659
|
+
"pop",
|
660
|
+
"film",
|
661
|
+
"--idb-version",
|
662
|
+
idb_version,
|
663
|
+
"--idb-source",
|
664
|
+
idb_source,
|
665
|
+
cmd,
|
666
|
+
os.path.join(input_dir_path, inputs[0]),
|
667
|
+
"--output-dir",
|
668
|
+
generated_output_dir_path,
|
669
|
+
"-ll",
|
670
|
+
"INFO",
|
671
|
+
]
|
516
672
|
|
517
673
|
# define the required plugins
|
518
|
-
plugin_list = [
|
674
|
+
plugin_list = [
|
675
|
+
"poppy.pop",
|
676
|
+
"roc.idb",
|
677
|
+
"roc.rpl",
|
678
|
+
"roc.rap",
|
679
|
+
"roc.dingo",
|
680
|
+
"roc.film",
|
681
|
+
]
|
519
682
|
|
520
683
|
# run the command
|
521
684
|
# force the value of the plugin list
|
522
|
-
with mock.patch.object(
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
self.
|
685
|
+
with mock.patch.object(
|
686
|
+
Settings,
|
687
|
+
"configure",
|
688
|
+
autospec=True,
|
689
|
+
side_effect=self.mock_configure_settings(
|
690
|
+
dictionary={"PLUGINS": plugin_list}
|
691
|
+
),
|
692
|
+
):
|
693
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
694
|
+
self.run_command(
|
695
|
+
[
|
696
|
+
"pop",
|
697
|
+
"-ll",
|
698
|
+
"INFO",
|
699
|
+
"idb",
|
700
|
+
"install",
|
701
|
+
"-s",
|
702
|
+
idb_source,
|
703
|
+
"-v",
|
704
|
+
idb_version,
|
705
|
+
"--load",
|
706
|
+
]
|
707
|
+
)
|
527
708
|
self.run_command(main_command)
|
528
709
|
|
529
710
|
# compare directory content
|
530
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
531
|
-
expected_output_dir_path)
|
711
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
532
712
|
|
533
713
|
dirs_cmp.report()
|
534
714
|
|
@@ -537,36 +717,41 @@ class TestFilmFileProd(CommandTestCase):
|
|
537
717
|
|
538
718
|
for filename in self.get_diff_files(dirs_cmp):
|
539
719
|
# compare only cdf files with differences
|
540
|
-
if filename.endswith(
|
720
|
+
if filename.endswith(".cdf"):
|
541
721
|
# use cdf compare to compute the differences between expected output and the command output
|
542
722
|
result = cdf_compare(
|
543
723
|
os.path.join(generated_output_dir_path, filename),
|
544
724
|
os.path.join(expected_output_dir_path, filename),
|
545
725
|
list_ignore_gatt=[
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
]
|
726
|
+
"File_ID",
|
727
|
+
"Generation_date",
|
728
|
+
"Pipeline_version",
|
729
|
+
"Pipeline_name",
|
730
|
+
"Software_version",
|
731
|
+
"IDB_version",
|
732
|
+
],
|
553
733
|
)
|
554
734
|
|
555
735
|
# compare the difference dict with the expected one
|
556
736
|
if result:
|
557
|
-
logger.error(
|
737
|
+
logger.error(
|
738
|
+
f"Differences between expected output and the command output: {pformat(result)}"
|
739
|
+
)
|
558
740
|
|
559
741
|
assert result == {}
|
560
742
|
|
561
|
-
@pytest.mark.parametrize(
|
562
|
-
|
563
|
-
|
564
|
-
|
743
|
+
@pytest.mark.parametrize(
|
744
|
+
"idb_source,idb_version",
|
745
|
+
[
|
746
|
+
("MIB", "20200131"),
|
747
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
748
|
+
],
|
749
|
+
)
|
565
750
|
def test_dds_to_l0(self, idb_source, idb_version):
|
566
751
|
from poppy.core.conf import Settings
|
567
752
|
|
568
753
|
# Name of the command to test
|
569
|
-
cmd =
|
754
|
+
cmd = "dds_to_l0"
|
570
755
|
|
571
756
|
input_dir_path, inputs = self.get_inputs(cmd)
|
572
757
|
expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -574,52 +759,85 @@ class TestFilmFileProd(CommandTestCase):
|
|
574
759
|
|
575
760
|
# extract spice kernels
|
576
761
|
spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
577
|
-
|
762
|
+
logger.debug(spice_kernel_dir_path)
|
763
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
578
764
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
579
765
|
|
580
|
-
date =
|
581
|
-
scos_header =
|
766
|
+
date = "20200303"
|
767
|
+
scos_header = "0"
|
582
768
|
|
583
769
|
# Build list of tm and tc input files
|
584
|
-
tm_inputs =
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
770
|
+
tm_inputs = " ".join(
|
771
|
+
[
|
772
|
+
os.path.join(input_dir_path, input_file)
|
773
|
+
for input_file in inputs
|
774
|
+
if os.path.basename(input_file).startswith("solo_TM_")
|
775
|
+
]
|
776
|
+
)
|
777
|
+
tc_inputs = " ".join(
|
778
|
+
[
|
779
|
+
os.path.join(input_dir_path, input_file)
|
780
|
+
for input_file in inputs
|
781
|
+
if os.path.basename(input_file).startswith("solo_TC_")
|
782
|
+
]
|
783
|
+
)
|
594
784
|
|
595
785
|
# initialize the main command
|
596
786
|
# Make sure that start-time/end_time keyword values are consistent with input DDS files
|
597
|
-
main_command = [
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
787
|
+
main_command = [
|
788
|
+
"pop",
|
789
|
+
"film",
|
790
|
+
"--idb-version",
|
791
|
+
idb_version,
|
792
|
+
"--idb-source",
|
793
|
+
idb_source,
|
794
|
+
"--scos-header",
|
795
|
+
scos_header,
|
796
|
+
"--cdag",
|
797
|
+
cmd,
|
798
|
+
date,
|
799
|
+
"--dds-tmraw-xml",
|
800
|
+
tm_inputs,
|
801
|
+
"--dds-tcreport-xml",
|
802
|
+
tc_inputs,
|
803
|
+
"--output-dir",
|
804
|
+
generated_output_dir_path,
|
805
|
+
"-ll",
|
806
|
+
"INFO",
|
807
|
+
]
|
607
808
|
|
608
809
|
# define the required plugins
|
609
|
-
plugin_list = [
|
810
|
+
plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
|
610
811
|
|
611
812
|
# run the command
|
612
813
|
# force the value of the plugin list
|
613
|
-
with mock.patch.object(
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
self.
|
814
|
+
with mock.patch.object(
|
815
|
+
Settings,
|
816
|
+
"configure",
|
817
|
+
autospec=True,
|
818
|
+
side_effect=self.mock_configure_settings(
|
819
|
+
dictionary={"PLUGINS": plugin_list}
|
820
|
+
),
|
821
|
+
):
|
822
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
823
|
+
self.run_command(
|
824
|
+
[
|
825
|
+
"pop",
|
826
|
+
"-ll",
|
827
|
+
"INFO",
|
828
|
+
"idb",
|
829
|
+
"install",
|
830
|
+
"-s",
|
831
|
+
idb_source,
|
832
|
+
"-v",
|
833
|
+
idb_version,
|
834
|
+
"--load",
|
835
|
+
]
|
836
|
+
)
|
618
837
|
self.run_command(main_command)
|
619
838
|
|
620
839
|
# compare directory content
|
621
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
622
|
-
expected_output_dir_path)
|
840
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
623
841
|
|
624
842
|
dirs_cmp.report()
|
625
843
|
|
@@ -650,15 +868,18 @@ class TestFilmFileProd(CommandTestCase):
|
|
650
868
|
#
|
651
869
|
# assert result == {}
|
652
870
|
|
653
|
-
@pytest.mark.parametrize(
|
654
|
-
|
655
|
-
|
656
|
-
|
871
|
+
@pytest.mark.parametrize(
|
872
|
+
"idb_source,idb_version",
|
873
|
+
[
|
874
|
+
("MIB", "20200131"),
|
875
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
876
|
+
],
|
877
|
+
)
|
657
878
|
def test_l0_to_l1_sbm(self, idb_source, idb_version):
|
658
879
|
from poppy.core.conf import Settings
|
659
880
|
|
660
881
|
# Name of the command to test
|
661
|
-
cmd =
|
882
|
+
cmd = "l0_to_l1_sbm"
|
662
883
|
|
663
884
|
input_dir_path, inputs = self.get_inputs(cmd)
|
664
885
|
expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -666,37 +887,62 @@ class TestFilmFileProd(CommandTestCase):
|
|
666
887
|
|
667
888
|
# extract spice kernels
|
668
889
|
spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
890
|
+
logger.debug(spice_kernel_dir_path)
|
669
891
|
|
670
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
892
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
671
893
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
672
894
|
|
673
895
|
# initialize the main command
|
674
|
-
main_command = [
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
896
|
+
main_command = [
|
897
|
+
"pop",
|
898
|
+
"film",
|
899
|
+
"--idb-version",
|
900
|
+
idb_version,
|
901
|
+
"--idb-source",
|
902
|
+
idb_source,
|
903
|
+
"--cdag",
|
904
|
+
cmd,
|
905
|
+
" ".join(
|
906
|
+
[os.path.join(input_dir_path, input_file) for input_file in inputs]
|
907
|
+
),
|
908
|
+
"--output-dir",
|
909
|
+
generated_output_dir_path,
|
910
|
+
"-ll",
|
911
|
+
"INFO",
|
912
|
+
]
|
684
913
|
|
685
914
|
# define the required plugins
|
686
|
-
plugin_list = [
|
915
|
+
plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
|
687
916
|
|
688
917
|
# run the command
|
689
918
|
# force the value of the plugin list
|
690
|
-
with mock.patch.object(
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
self.
|
919
|
+
with mock.patch.object(
|
920
|
+
Settings,
|
921
|
+
"configure",
|
922
|
+
autospec=True,
|
923
|
+
side_effect=self.mock_configure_settings(
|
924
|
+
dictionary={"PLUGINS": plugin_list}
|
925
|
+
),
|
926
|
+
):
|
927
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
928
|
+
self.run_command(
|
929
|
+
[
|
930
|
+
"pop",
|
931
|
+
"-ll",
|
932
|
+
"INFO",
|
933
|
+
"idb",
|
934
|
+
"install",
|
935
|
+
"-s",
|
936
|
+
idb_source,
|
937
|
+
"-v",
|
938
|
+
idb_version,
|
939
|
+
"--load",
|
940
|
+
]
|
941
|
+
)
|
695
942
|
self.run_command(main_command)
|
696
943
|
|
697
944
|
# compare directory content
|
698
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
699
|
-
expected_output_dir_path)
|
945
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
700
946
|
|
701
947
|
dirs_cmp.report()
|
702
948
|
|
@@ -705,36 +951,41 @@ class TestFilmFileProd(CommandTestCase):
|
|
705
951
|
|
706
952
|
for filename in self.get_diff_files(dirs_cmp):
|
707
953
|
# compare only cdf files with differences
|
708
|
-
if filename.endswith(
|
954
|
+
if filename.endswith(".cdf"):
|
709
955
|
# use cdf compare to compute the differences between expected output and the command output
|
710
956
|
result = cdf_compare(
|
711
957
|
os.path.join(generated_output_dir_path, filename),
|
712
958
|
os.path.join(expected_output_dir_path, filename),
|
713
959
|
list_ignore_gatt=[
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
]
|
960
|
+
"File_ID",
|
961
|
+
"Generation_date",
|
962
|
+
"Pipeline_version",
|
963
|
+
"Pipeline_name",
|
964
|
+
"Software_version",
|
965
|
+
"IDB_version",
|
966
|
+
],
|
721
967
|
)
|
722
968
|
|
723
969
|
# compare the difference dict with the expected one
|
724
970
|
if result:
|
725
|
-
logger.error(
|
971
|
+
logger.error(
|
972
|
+
f"Differences between expected output and the command output: {pformat(result)}"
|
973
|
+
)
|
726
974
|
|
727
975
|
assert result == {}
|
728
976
|
|
729
|
-
@pytest.mark.parametrize(
|
730
|
-
|
731
|
-
|
732
|
-
|
977
|
+
@pytest.mark.parametrize(
|
978
|
+
"idb_source,idb_version",
|
979
|
+
[
|
980
|
+
("MIB", "20200131"),
|
981
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
982
|
+
],
|
983
|
+
)
|
733
984
|
def test_l0_to_anc_bia_sweep_table(self, idb_source, idb_version):
|
734
985
|
from poppy.core.conf import Settings
|
735
986
|
|
736
987
|
# Name of the command to test
|
737
|
-
cmd =
|
988
|
+
cmd = "l0_to_anc_bia_sweep_table"
|
738
989
|
|
739
990
|
input_dir_path, inputs = self.get_inputs(cmd)
|
740
991
|
expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -742,82 +993,112 @@ class TestFilmFileProd(CommandTestCase):
|
|
742
993
|
|
743
994
|
# extract spice kernels
|
744
995
|
spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
996
|
+
logger.debug(spice_kernel_dir_path)
|
745
997
|
|
746
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
998
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
747
999
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
748
1000
|
|
749
1001
|
# initialize the main command
|
750
|
-
main_command = [
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
755
|
-
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
1002
|
+
main_command = [
|
1003
|
+
"pop",
|
1004
|
+
"film",
|
1005
|
+
"--idb-version",
|
1006
|
+
idb_version,
|
1007
|
+
"--idb-source",
|
1008
|
+
idb_source,
|
1009
|
+
"--monthly",
|
1010
|
+
cmd,
|
1011
|
+
"--l0-files",
|
1012
|
+
" ".join(
|
1013
|
+
[os.path.join(input_dir_path, input_file) for input_file in inputs]
|
1014
|
+
),
|
1015
|
+
"--output-dir",
|
1016
|
+
generated_output_dir_path,
|
1017
|
+
"-ll",
|
1018
|
+
"INFO",
|
1019
|
+
]
|
761
1020
|
|
762
1021
|
# define the required plugins
|
763
|
-
plugin_list = [
|
1022
|
+
plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
|
764
1023
|
|
765
1024
|
# run the command
|
766
1025
|
# force the value of the plugin list
|
767
|
-
with mock.patch.object(
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
self.
|
1026
|
+
with mock.patch.object(
|
1027
|
+
Settings,
|
1028
|
+
"configure",
|
1029
|
+
autospec=True,
|
1030
|
+
side_effect=self.mock_configure_settings(
|
1031
|
+
dictionary={"PLUGINS": plugin_list}
|
1032
|
+
),
|
1033
|
+
):
|
1034
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
1035
|
+
self.run_command(
|
1036
|
+
[
|
1037
|
+
"pop",
|
1038
|
+
"-ll",
|
1039
|
+
"INFO",
|
1040
|
+
"idb",
|
1041
|
+
"install",
|
1042
|
+
"-s",
|
1043
|
+
idb_source,
|
1044
|
+
"-v",
|
1045
|
+
idb_version,
|
1046
|
+
"--load",
|
1047
|
+
]
|
1048
|
+
)
|
772
1049
|
self.run_command(main_command)
|
773
1050
|
|
774
1051
|
# compare directory content
|
775
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
776
|
-
expected_output_dir_path)
|
1052
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
777
1053
|
|
778
1054
|
dirs_cmp.report()
|
779
1055
|
|
780
1056
|
# ensure that we have the same files in both directories
|
781
|
-
assert (len(dirs_cmp.left_only) == 0) and (
|
782
|
-
|
1057
|
+
assert (len(dirs_cmp.left_only) == 0) and (
|
1058
|
+
len(dirs_cmp.right_only) == 0
|
1059
|
+
), "Different files in expected and generated output directories!"
|
783
1060
|
|
784
1061
|
for filename in self.get_diff_files(dirs_cmp):
|
785
1062
|
# compare only cdf files with differences
|
786
|
-
if filename.endswith(
|
1063
|
+
if filename.endswith(".cdf"):
|
787
1064
|
# use cdf compare to compute the differences between expected output and the command output
|
788
1065
|
result = cdf_compare(
|
789
1066
|
os.path.join(generated_output_dir_path, filename),
|
790
1067
|
os.path.join(expected_output_dir_path, filename),
|
791
1068
|
list_ignore_gatt=[
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
]
|
1069
|
+
"File_ID",
|
1070
|
+
"Generation_date",
|
1071
|
+
"Pipeline_version",
|
1072
|
+
"Pipeline_name",
|
1073
|
+
"Software_version",
|
1074
|
+
"IDB_version",
|
1075
|
+
],
|
799
1076
|
)
|
800
|
-
elif filename.endswith(
|
1077
|
+
elif filename.endswith(".csv"):
|
801
1078
|
result = filecmp.cmpfiles(
|
802
1079
|
os.path.join(generated_output_dir_path, filename),
|
803
|
-
os.path.join(expected_output_dir_path, filename)
|
1080
|
+
os.path.join(expected_output_dir_path, filename),
|
804
1081
|
)
|
805
1082
|
else:
|
806
1083
|
result = {}
|
807
1084
|
|
808
1085
|
# compare the difference dict with the expected one
|
809
|
-
assert
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
|
815
|
-
|
1086
|
+
assert (
|
1087
|
+
result == {}
|
1088
|
+
), f"Differences between expected output and the command output: {pformat(result)}"
|
1089
|
+
|
1090
|
+
@pytest.mark.parametrize(
|
1091
|
+
"idb_source,idb_version",
|
1092
|
+
[
|
1093
|
+
("MIB", "20200131"),
|
1094
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
1095
|
+
],
|
1096
|
+
)
|
816
1097
|
def test_l0_to_l1_bia_sweep(self, idb_source, idb_version):
|
817
1098
|
from poppy.core.conf import Settings
|
818
1099
|
|
819
1100
|
# Name of the command to test
|
820
|
-
cmd =
|
1101
|
+
cmd = "l0_to_anc_bia_sweep_table"
|
821
1102
|
|
822
1103
|
input_dir_path, inputs = self.get_inputs(cmd)
|
823
1104
|
expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -825,91 +1106,130 @@ class TestFilmFileProd(CommandTestCase):
|
|
825
1106
|
|
826
1107
|
# extract spice kernels
|
827
1108
|
spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
1109
|
+
logger.debug(spice_kernel_dir_path)
|
828
1110
|
|
829
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
1111
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
830
1112
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
831
1113
|
|
832
1114
|
# Build list of inputs
|
833
|
-
l0_files =
|
834
|
-
|
835
|
-
|
836
|
-
|
837
|
-
|
838
|
-
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
1115
|
+
l0_files = " ".join(
|
1116
|
+
[
|
1117
|
+
os.path.join(input_dir_path, input_file)
|
1118
|
+
for input_file in inputs
|
1119
|
+
if os.path.basename(input_file).startswith("solo_L0_rpw")
|
1120
|
+
]
|
1121
|
+
)
|
1122
|
+
sweep_tables = " ".join(
|
1123
|
+
[
|
1124
|
+
os.path.join(input_dir_path, input_file)
|
1125
|
+
for input_file in inputs
|
1126
|
+
if os.path.basename(input_file).startswith(
|
1127
|
+
"solo_ANC_rpw-bia-sweep-table"
|
1128
|
+
)
|
1129
|
+
]
|
1130
|
+
)
|
843
1131
|
|
844
1132
|
# initialize the main command
|
845
|
-
main_command = [
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
1133
|
+
main_command = [
|
1134
|
+
"pop",
|
1135
|
+
"film",
|
1136
|
+
"--idb-version",
|
1137
|
+
idb_version,
|
1138
|
+
"--idb-source",
|
1139
|
+
idb_source,
|
1140
|
+
"--cdag",
|
1141
|
+
cmd,
|
1142
|
+
"--l0-files",
|
1143
|
+
l0_files,
|
1144
|
+
"--sweep-tables",
|
1145
|
+
sweep_tables,
|
1146
|
+
"--output-dir",
|
1147
|
+
generated_output_dir_path,
|
1148
|
+
"-ll",
|
1149
|
+
"INFO",
|
1150
|
+
]
|
854
1151
|
|
855
1152
|
# define the required plugins
|
856
|
-
plugin_list = [
|
1153
|
+
plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
|
857
1154
|
|
858
1155
|
# run the command
|
859
1156
|
# force the value of the plugin list
|
860
|
-
with mock.patch.object(
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
self.
|
1157
|
+
with mock.patch.object(
|
1158
|
+
Settings,
|
1159
|
+
"configure",
|
1160
|
+
autospec=True,
|
1161
|
+
side_effect=self.mock_configure_settings(
|
1162
|
+
dictionary={"PLUGINS": plugin_list}
|
1163
|
+
),
|
1164
|
+
):
|
1165
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
1166
|
+
self.run_command(
|
1167
|
+
[
|
1168
|
+
"pop",
|
1169
|
+
"-ll",
|
1170
|
+
"INFO",
|
1171
|
+
"idb",
|
1172
|
+
"install",
|
1173
|
+
"-s",
|
1174
|
+
idb_source,
|
1175
|
+
"-v",
|
1176
|
+
idb_version,
|
1177
|
+
"--load",
|
1178
|
+
]
|
1179
|
+
)
|
865
1180
|
self.run_command(main_command)
|
866
1181
|
|
867
1182
|
# compare directory content
|
868
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
869
|
-
expected_output_dir_path)
|
1183
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
870
1184
|
|
871
1185
|
dirs_cmp.report()
|
872
1186
|
|
873
1187
|
# ensure that we have the same files in both directories
|
874
|
-
assert (len(dirs_cmp.left_only) == 0) and (
|
875
|
-
|
1188
|
+
assert (len(dirs_cmp.left_only) == 0) and (
|
1189
|
+
len(dirs_cmp.right_only) == 0
|
1190
|
+
), "Different files in expected and generated output directories!"
|
876
1191
|
|
877
1192
|
for filename in self.get_diff_files(dirs_cmp):
|
878
1193
|
# compare only cdf files with differences
|
879
|
-
if filename.endswith(
|
1194
|
+
if filename.endswith(".cdf"):
|
880
1195
|
# use cdf compare to compute the differences between expected output and the command output
|
881
1196
|
result = cdf_compare(
|
882
1197
|
os.path.join(generated_output_dir_path, filename),
|
883
1198
|
os.path.join(expected_output_dir_path, filename),
|
884
1199
|
list_ignore_gatt=[
|
885
|
-
|
886
|
-
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
|
891
|
-
]
|
1200
|
+
"File_ID",
|
1201
|
+
"Generation_date",
|
1202
|
+
"Pipeline_version",
|
1203
|
+
"Pipeline_name",
|
1204
|
+
"Software_version",
|
1205
|
+
"IDB_version",
|
1206
|
+
],
|
892
1207
|
)
|
893
|
-
elif filename.endswith(
|
1208
|
+
elif filename.endswith(".csv"):
|
894
1209
|
result = filecmp.cmpfiles(
|
895
1210
|
os.path.join(generated_output_dir_path, filename),
|
896
|
-
os.path.join(expected_output_dir_path, filename)
|
1211
|
+
os.path.join(expected_output_dir_path, filename),
|
897
1212
|
)
|
898
1213
|
else:
|
899
1214
|
result = {}
|
900
1215
|
|
901
1216
|
# compare the difference dict with the expected one
|
902
|
-
assert
|
903
|
-
|
904
|
-
|
905
|
-
|
906
|
-
|
907
|
-
|
1217
|
+
assert (
|
1218
|
+
result == {}
|
1219
|
+
), f"Differences between expected output and the command output: {pformat(result)}"
|
1220
|
+
|
1221
|
+
@pytest.mark.parametrize(
|
1222
|
+
"idb_source,idb_version",
|
1223
|
+
[
|
1224
|
+
("MIB", "20200131"),
|
1225
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
1226
|
+
],
|
1227
|
+
)
|
908
1228
|
def test_l0_to_l1_bia_current(self, idb_source, idb_version):
|
909
1229
|
from poppy.core.conf import Settings
|
910
1230
|
|
911
1231
|
# Name of the command to test
|
912
|
-
cmd =
|
1232
|
+
cmd = "l0_to_anc_bia_sweep_table"
|
913
1233
|
|
914
1234
|
input_dir_path, inputs = self.get_inputs(cmd)
|
915
1235
|
expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -917,70 +1237,97 @@ class TestFilmFileProd(CommandTestCase):
|
|
917
1237
|
|
918
1238
|
# extract spice kernels
|
919
1239
|
spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
1240
|
+
logger.debug(spice_kernel_dir_path)
|
920
1241
|
|
921
|
-
generated_output_dir_path = os.path.join(self.tmp_dir_path,
|
1242
|
+
generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
|
922
1243
|
os.makedirs(generated_output_dir_path, exist_ok=True)
|
923
1244
|
|
924
1245
|
# initialize the main command
|
925
|
-
main_command = [
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
1246
|
+
main_command = [
|
1247
|
+
"pop",
|
1248
|
+
"film",
|
1249
|
+
"--idb-version",
|
1250
|
+
idb_version,
|
1251
|
+
"--idb-source",
|
1252
|
+
idb_source,
|
1253
|
+
"--cdag",
|
1254
|
+
"--monthly",
|
1255
|
+
cmd,
|
1256
|
+
"--l0-files",
|
1257
|
+
" ".join(
|
1258
|
+
[os.path.join(input_dir_path, input_file) for input_file in inputs]
|
1259
|
+
),
|
1260
|
+
"--output-dir",
|
1261
|
+
generated_output_dir_path,
|
1262
|
+
"-ll",
|
1263
|
+
"INFO",
|
1264
|
+
]
|
937
1265
|
|
938
1266
|
# define the required plugins
|
939
|
-
plugin_list = [
|
1267
|
+
plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
|
940
1268
|
|
941
1269
|
# run the command
|
942
1270
|
# force the value of the plugin list
|
943
|
-
with mock.patch.object(
|
944
|
-
|
945
|
-
|
946
|
-
|
947
|
-
self.
|
1271
|
+
with mock.patch.object(
|
1272
|
+
Settings,
|
1273
|
+
"configure",
|
1274
|
+
autospec=True,
|
1275
|
+
side_effect=self.mock_configure_settings(
|
1276
|
+
dictionary={"PLUGINS": plugin_list}
|
1277
|
+
),
|
1278
|
+
):
|
1279
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
1280
|
+
self.run_command(
|
1281
|
+
[
|
1282
|
+
"pop",
|
1283
|
+
"-ll",
|
1284
|
+
"INFO",
|
1285
|
+
"idb",
|
1286
|
+
"install",
|
1287
|
+
"-s",
|
1288
|
+
idb_source,
|
1289
|
+
"-v",
|
1290
|
+
idb_version,
|
1291
|
+
"--load",
|
1292
|
+
]
|
1293
|
+
)
|
948
1294
|
self.run_command(main_command)
|
949
1295
|
|
950
1296
|
# compare directory content
|
951
|
-
dirs_cmp = filecmp.dircmp(generated_output_dir_path,
|
952
|
-
expected_output_dir_path)
|
1297
|
+
dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
|
953
1298
|
|
954
1299
|
dirs_cmp.report()
|
955
1300
|
|
956
1301
|
# ensure that we have the same files in both directories
|
957
|
-
assert (len(dirs_cmp.left_only) == 0) and (
|
958
|
-
|
1302
|
+
assert (len(dirs_cmp.left_only) == 0) and (
|
1303
|
+
len(dirs_cmp.right_only) == 0
|
1304
|
+
), "Different files in expected and generated output directories!"
|
959
1305
|
|
960
1306
|
for filename in self.get_diff_files(dirs_cmp):
|
961
1307
|
# compare only cdf files with differences
|
962
|
-
if filename.endswith(
|
1308
|
+
if filename.endswith(".cdf"):
|
963
1309
|
# use cdf compare to compute the differences between expected output and the command output
|
964
1310
|
result = cdf_compare(
|
965
1311
|
os.path.join(generated_output_dir_path, filename),
|
966
1312
|
os.path.join(expected_output_dir_path, filename),
|
967
1313
|
list_ignore_gatt=[
|
968
|
-
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
|
973
|
-
|
974
|
-
]
|
1314
|
+
"File_ID",
|
1315
|
+
"Generation_date",
|
1316
|
+
"Pipeline_version",
|
1317
|
+
"Pipeline_name",
|
1318
|
+
"Software_version",
|
1319
|
+
"IDB_version",
|
1320
|
+
],
|
975
1321
|
)
|
976
|
-
elif filename.endswith(
|
1322
|
+
elif filename.endswith(".csv"):
|
977
1323
|
result = filecmp.cmpfiles(
|
978
1324
|
os.path.join(generated_output_dir_path, filename),
|
979
|
-
os.path.join(expected_output_dir_path, filename)
|
1325
|
+
os.path.join(expected_output_dir_path, filename),
|
980
1326
|
)
|
981
1327
|
else:
|
982
1328
|
result = {}
|
983
1329
|
|
984
1330
|
# compare the difference dict with the expected one
|
985
|
-
assert
|
986
|
-
|
1331
|
+
assert (
|
1332
|
+
result == {}
|
1333
|
+
), f"Differences between expected output and the command output: {pformat(result)}"
|