roc-film 1.13.5__py3-none-any.whl → 1.14.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. roc/__init__.py +2 -1
  2. roc/film/__init__.py +2 -2
  3. roc/film/commands.py +372 -323
  4. roc/film/config/__init__.py +0 -1
  5. roc/film/constants.py +101 -65
  6. roc/film/descriptor.json +127 -96
  7. roc/film/exceptions.py +28 -27
  8. roc/film/tasks/__init__.py +16 -16
  9. roc/film/tasks/cat_solo_hk.py +86 -74
  10. roc/film/tasks/cdf_postpro.py +438 -309
  11. roc/film/tasks/check_dds.py +39 -45
  12. roc/film/tasks/db_to_anc_bia_sweep_table.py +381 -0
  13. roc/film/tasks/dds_to_l0.py +232 -180
  14. roc/film/tasks/export_solo_coord.py +147 -0
  15. roc/film/tasks/file_handler.py +91 -75
  16. roc/film/tasks/l0_to_hk.py +117 -103
  17. roc/film/tasks/l0_to_l1_bia_current.py +38 -30
  18. roc/film/tasks/l0_to_l1_bia_sweep.py +417 -329
  19. roc/film/tasks/l0_to_l1_sbm.py +250 -208
  20. roc/film/tasks/l0_to_l1_surv.py +185 -130
  21. roc/film/tasks/make_daily_tm.py +40 -37
  22. roc/film/tasks/merge_tcreport.py +77 -71
  23. roc/film/tasks/merge_tmraw.py +101 -88
  24. roc/film/tasks/parse_dds_xml.py +21 -20
  25. roc/film/tasks/set_l0_utc.py +51 -49
  26. roc/film/tests/cdf_compare.py +565 -0
  27. roc/film/tests/hdf5_compare.py +84 -62
  28. roc/film/tests/test_dds_to_l0.py +93 -51
  29. roc/film/tests/test_dds_to_tc.py +8 -11
  30. roc/film/tests/test_dds_to_tm.py +8 -10
  31. roc/film/tests/test_film.py +161 -116
  32. roc/film/tests/test_l0_to_hk.py +64 -36
  33. roc/film/tests/test_l0_to_l1_bia.py +10 -14
  34. roc/film/tests/test_l0_to_l1_sbm.py +14 -19
  35. roc/film/tests/test_l0_to_l1_surv.py +68 -41
  36. roc/film/tests/test_metadata.py +21 -20
  37. roc/film/tests/tests.py +743 -396
  38. roc/film/tools/__init__.py +5 -5
  39. roc/film/tools/dataset_tasks.py +34 -2
  40. roc/film/tools/file_helpers.py +390 -269
  41. roc/film/tools/l0.py +402 -324
  42. roc/film/tools/metadata.py +147 -127
  43. roc/film/tools/skeleton.py +12 -17
  44. roc/film/tools/tools.py +109 -92
  45. roc/film/tools/xlsx2skt.py +161 -139
  46. {roc_film-1.13.5.dist-info → roc_film-1.14.0.dist-info}/LICENSE +127 -125
  47. roc_film-1.14.0.dist-info/METADATA +60 -0
  48. roc_film-1.14.0.dist-info/RECORD +50 -0
  49. {roc_film-1.13.5.dist-info → roc_film-1.14.0.dist-info}/WHEEL +1 -1
  50. roc/film/tasks/l0_to_anc_bia_sweep_table.py +0 -348
  51. roc_film-1.13.5.dist-info/METADATA +0 -120
  52. roc_film-1.13.5.dist-info/RECORD +0 -48
roc/film/tests/tests.py CHANGED
@@ -9,6 +9,20 @@ import subprocess
9
9
 
10
10
  from poppy.core.test import TaskTestCase
11
11
 
12
+ import filecmp
13
+ import os
14
+ import tempfile
15
+ from pprint import pformat
16
+
17
+ import pytest
18
+ import shutil
19
+ import unittest.mock as mock
20
+ from maser.tools.cdf.cdfcompare import cdf_compare
21
+ from poppy.core.generic.requests import download_file
22
+ from poppy.core.logger import logger
23
+ from poppy.core.test import CommandTestCase
24
+
25
+
12
26
  # Tests on roc.film.tasks.metadata methods
13
27
  class TestFilmMetadata(TaskTestCase):
14
28
  # create the pipeline
@@ -25,14 +39,14 @@ class TestFilmMetadata(TaskTestCase):
25
39
 
26
40
  # initialize meta dictionary with expected inputs
27
41
  meta = {
28
- 'Source_name': 'SOLO>Solar Orbiter',
29
- 'LEVEL': 'L1>Level 1 data processing',
30
- 'Descriptor': 'RPW-HFR-SURV> RPW HFR survey data',
31
- 'Datetime': '20220103',
32
- 'Data_version' : '01'
33
- }
42
+ "Source_name": "SOLO>Solar Orbiter",
43
+ "LEVEL": "L1>Level 1 data processing",
44
+ "Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
45
+ "Datetime": "20220103",
46
+ "Data_version": "01",
47
+ }
34
48
 
35
- assert build_file_basename(meta) == 'solo_L1_rpw-hfr-surv_20220103_V01'
49
+ assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
36
50
 
37
51
  def test_build_file_basename_with_logical(self):
38
52
  """
@@ -45,8 +59,8 @@ class TestFilmMetadata(TaskTestCase):
45
59
  # import method to test
46
60
  from roc.film.tools.file_helpers import build_file_basename
47
61
 
48
- meta = {'Logical_file_id': 'solo_L1_rpw-hfr-surv_20220103_V01'}
49
- assert build_file_basename(meta) == 'solo_L1_rpw-hfr-surv_20220103_V01'
62
+ meta = {"Logical_file_id": "solo_L1_rpw-hfr-surv_20220103_V01"}
63
+ assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
50
64
 
51
65
  # def test_generate_filepath(self):
52
66
 
@@ -60,7 +74,7 @@ class TestFilmMetadata(TaskTestCase):
60
74
  # import method to test
61
75
  from roc.film.tools.metadata import valid_data_version
62
76
 
63
- assert valid_data_version(1) == '01'
77
+ assert valid_data_version(1) == "01"
64
78
 
65
79
  def test_set_logical_file_id(self):
66
80
  """
@@ -74,36 +88,24 @@ class TestFilmMetadata(TaskTestCase):
74
88
 
75
89
  # initialize meta dictionary with expected inputs
76
90
  meta = {
77
- 'File_naming_convention': '<Source_name>_<LEVEL>_<Descriptor>_' \
78
- '<Datetime>_V<Data_version>',
79
- 'Source_name': 'SOLO>Solar Orbiter',
80
- 'LEVEL': 'L1>Level 1 data processing',
81
- 'Descriptor': 'RPW-HFR-SURV> RPW HFR survey data',
82
- 'Datetime': '20220103',
83
- 'Data_version' : '01'
84
- }
85
-
86
- assert set_logical_file_id(meta) == 'solo_L1_rpw-hfr-surv_20220103_V01'
87
-
88
-
89
- import filecmp
90
- import os
91
- import tempfile
92
- from pprint import pformat
91
+ "File_naming_convention": "<Source_name>_<LEVEL>_<Descriptor>_"
92
+ "<Datetime>_V<Data_version>",
93
+ "Source_name": "SOLO>Solar Orbiter",
94
+ "LEVEL": "L1>Level 1 data processing",
95
+ "Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
96
+ "Datetime": "20220103",
97
+ "Data_version": "01",
98
+ }
93
99
 
94
- import pytest
95
- import shutil
96
- import unittest.mock as mock
97
- from maser.tools.cdf.cdfcompare import cdf_compare
98
- from poppy.core.generic.requests import download_file
99
- from poppy.core.logger import logger
100
- from poppy.core.test import CommandTestCase
100
+ assert set_logical_file_id(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
101
101
 
102
102
 
103
103
  class TestFilmFileProd(CommandTestCase):
104
- base_url = 'https://rpw.lesia.obspm.fr/roc/data/private/devtest/roc/test_data/rodp/film'
104
+ base_url = (
105
+ "https://rpw.lesia.obspm.fr/roc/data/private/devtest/roc/test_data/rodp/film"
106
+ )
105
107
  # test credentials
106
- username = 'roctest'
108
+ username = "roctest"
107
109
  password = None
108
110
 
109
111
  @classmethod
@@ -113,10 +115,12 @@ class TestFilmFileProd(CommandTestCase):
113
115
  """
114
116
 
115
117
  try:
116
- cls.password = os.environ['ROC_TEST_PASSWORD']
118
+ cls.password = os.environ["ROC_TEST_PASSWORD"]
117
119
  except KeyError:
118
- raise KeyError('You have to define the test user password using the "ROC_TEST_PASSWORD" environment'
119
- 'variable ')
120
+ raise KeyError(
121
+ 'You have to define the test user password using the "ROC_TEST_PASSWORD" environment'
122
+ "variable "
123
+ )
120
124
 
121
125
  def setup_method(self, method):
122
126
  super().setup_method(method)
@@ -124,27 +128,28 @@ class TestFilmFileProd(CommandTestCase):
124
128
  self.tmp_dir_path = tempfile.mkdtemp()
125
129
 
126
130
  def load_manifest_file(self, manifest_filepath, manifest_file_url, auth=None):
127
-
128
131
  download_file(manifest_filepath, manifest_file_url, auth=auth)
129
132
 
130
133
  with open(manifest_filepath) as manifest_file:
131
134
  for line in manifest_file:
132
- yield line.strip('\n\r')
135
+ yield line.strip("\n\r")
133
136
 
134
137
  os.remove(manifest_filepath)
135
138
 
136
139
  def get_files(self, subdir, category=None):
137
- categories = ['input', 'expected_output', 'ancillarie']
140
+ categories = ["input", "expected_output", "ancillarie"]
138
141
  if category not in categories:
139
- raise ValueError('Invalid category. Expected one of: %s' % categories)
142
+ raise ValueError("Invalid category. Expected one of: %s" % categories)
140
143
 
141
144
  auth = (self.username, self.password)
142
145
 
143
146
  dir_path = os.path.join(self.tmp_dir_path, subdir, category)
144
147
  os.makedirs(dir_path, exist_ok=True)
145
- manifest_filepath = os.path.join(dir_path, 'manifest.txt')
146
- manifest_file_url = f'{self.base_url}/{subdir}/{category}s/manifest.txt'
147
- file_list = list(self.load_manifest_file(manifest_filepath, manifest_file_url, auth=auth))
148
+ manifest_filepath = os.path.join(dir_path, "manifest.txt")
149
+ manifest_file_url = f"{self.base_url}/{subdir}/{category}s/manifest.txt"
150
+ file_list = list(
151
+ self.load_manifest_file(manifest_filepath, manifest_file_url, auth=auth)
152
+ )
148
153
 
149
154
  for relative_filepath in file_list:
150
155
  # skip empty strings
@@ -154,17 +159,21 @@ class TestFilmFileProd(CommandTestCase):
154
159
  # get the complete filepath
155
160
  filepath = os.path.join(dir_path, relative_filepath)
156
161
  os.makedirs(os.path.dirname(filepath), exist_ok=True)
157
- download_file(filepath,
158
- f'{self.base_url}/{subdir}/{category}s/{relative_filepath}',
159
- auth=auth)
162
+ download_file(
163
+ filepath,
164
+ f"{self.base_url}/{subdir}/{category}s/{relative_filepath}",
165
+ auth=auth,
166
+ )
160
167
 
161
168
  return dir_path, file_list
162
169
 
163
- def get_diff_files(self, dirs_cmp, path=''):
170
+ def get_diff_files(self, dirs_cmp, path=""):
164
171
  for name in dirs_cmp.diff_files:
165
172
  yield os.path.join(path, name)
166
173
  for parent, sub_dirs_cmp in dirs_cmp.subdirs.items():
167
- for filepath in self.get_diff_files(sub_dirs_cmp, path=os.path.join(path, parent)):
174
+ for filepath in self.get_diff_files(
175
+ sub_dirs_cmp, path=os.path.join(path, parent)
176
+ ):
168
177
  yield filepath
169
178
 
170
179
  def teardown_method(self, method):
@@ -184,16 +193,15 @@ class TestFilmFileProd(CommandTestCase):
184
193
  shutil.rmtree(self.tmp_dir_path)
185
194
 
186
195
  def get_inputs(self, subdir):
187
- return self.get_files(subdir, category='input')
196
+ return self.get_files(subdir, category="input")
188
197
 
189
198
  def get_expected_outputs(self, subdir):
190
- return self.get_files(subdir, category='expected_output')
199
+ return self.get_files(subdir, category="expected_output")
191
200
 
192
201
  def get_ancillaries(self, subdir):
193
- return self.get_files(subdir, category='ancillarie')
202
+ return self.get_files(subdir, category="ancillarie")
194
203
 
195
- def unzip_kernels(self, zip_path,
196
- pipeline_dir='/pipeline'):
204
+ def unzip_kernels(self, zip_path, pipeline_dir="/pipeline"):
197
205
  """
198
206
  Unzip input SPICE kernel ZIP file.
199
207
 
@@ -208,74 +216,108 @@ class TestFilmFileProd(CommandTestCase):
208
216
 
209
217
  # check zip file existence
210
218
  if not os.path.isfile(zip_path):
211
- raise FileNotFoundError(f'ZIP file containing SPICE kernel not found ({zip_path})!')
219
+ raise FileNotFoundError(
220
+ f"ZIP file containing SPICE kernel not found ({zip_path})!"
221
+ )
212
222
 
213
223
  # Get zip directory
214
224
  zip_dir = os.path.dirname(zip_path)
215
225
 
216
226
  # Unzip ZIP file
217
227
  try:
218
- cmd = f'unzip -o {zip_path}'
219
- completed = subprocess.run(cmd,
220
- cwd=zip_dir,
221
- shell=True, timeout=3600)
228
+ cmd = f"unzip -o {zip_path}"
229
+ completed = subprocess.run(cmd, cwd=zip_dir, shell=True, timeout=3600)
230
+ logger.debug(completed)
222
231
  # Path of the unzipped kernels dir
223
- kernels_dir = os.path.join(zip_dir, 'kernels')
232
+ kernels_dir = os.path.join(zip_dir, "kernels")
224
233
  if not os.path.isdir(kernels_dir):
225
234
  raise FileNotFoundError
226
235
 
227
- except:
228
- logger.exception(f'Extracting {zip_dir} has failed: \n'
229
- f'{completed.stderr}')
236
+ except Exception as e:
237
+ logger.exception(f"Extracting {zip_dir} has failed: \n{e}")
230
238
  else:
231
239
  # Move unzipped "kernels" folder into /pipeline/data/spice_kernels
232
240
  if os.path.isdir(pipeline_dir):
233
- data_dir = os.path.join(pipeline_dir, 'data')
241
+ data_dir = os.path.join(pipeline_dir, "data")
234
242
  os.makedirs(data_dir, exist_ok=True)
235
- target_dir = os.path.join(data_dir, 'spice_kernels')
243
+ target_dir = os.path.join(data_dir, "spice_kernels")
236
244
  shutil.move(kernels_dir, target_dir)
237
245
  else:
238
246
  target_dir = kernels_dir
239
247
 
240
248
  return target_dir
241
249
 
242
- @pytest.mark.parametrize('idb_source,idb_version', [
243
- ('MIB', '20200131'),
244
- ])
250
+ @pytest.mark.parametrize(
251
+ "idb_source,idb_version",
252
+ [
253
+ ("MIB", "20200131"),
254
+ ],
255
+ )
245
256
  def test_classify_tmraw(self, idb_source, idb_version):
246
257
  from poppy.core.conf import Settings
247
258
 
248
- input_dir_path, inputs = self.get_inputs('classify_tmraw')
249
- expected_output_dir_path, expected_outputs = self.get_expected_outputs('classify_tmraw')
259
+ input_dir_path, inputs = self.get_inputs("classify_tmraw")
260
+ expected_output_dir_path, expected_outputs = self.get_expected_outputs(
261
+ "classify_tmraw"
262
+ )
250
263
 
251
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
264
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
252
265
  os.makedirs(generated_output_dir_path, exist_ok=True)
253
266
 
254
267
  # initialize the main command
255
- main_command = ['pop', 'film',
256
- 'classify_tmraw',
257
- '--dds-files',
258
- ' '.join([
259
- os.path.join(input_dir_path, input_file)
260
- for input_file in inputs]),
261
- '--output-dir', generated_output_dir_path,
262
- '-ll', 'INFO']
268
+ main_command = [
269
+ "pop",
270
+ "film",
271
+ "classify_tmraw",
272
+ "--dds-files",
273
+ " ".join(
274
+ [os.path.join(input_dir_path, input_file) for input_file in inputs]
275
+ ),
276
+ "--output-dir",
277
+ generated_output_dir_path,
278
+ "-ll",
279
+ "INFO",
280
+ ]
263
281
 
264
282
  # define the required plugins
265
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.dingo' ,'roc.film']
283
+ plugin_list = [
284
+ "poppy.pop",
285
+ "roc.idb",
286
+ "roc.rpl",
287
+ "roc.rap",
288
+ "roc.dingo",
289
+ "roc.film",
290
+ ]
266
291
 
267
292
  # run the command
268
293
  # force the value of the plugin list
269
- with mock.patch.object(Settings, 'configure',
270
- autospec=True,
271
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
272
- self.run_command('pop db upgrade heads -ll INFO')
273
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
294
+ with mock.patch.object(
295
+ Settings,
296
+ "configure",
297
+ autospec=True,
298
+ side_effect=self.mock_configure_settings(
299
+ dictionary={"PLUGINS": plugin_list}
300
+ ),
301
+ ):
302
+ self.run_command("pop db upgrade heads -ll INFO")
303
+ self.run_command(
304
+ [
305
+ "pop",
306
+ "-ll",
307
+ "INFO",
308
+ "idb",
309
+ "install",
310
+ "-s",
311
+ idb_source,
312
+ "-v",
313
+ idb_version,
314
+ "--load",
315
+ ]
316
+ )
274
317
  self.run_command(main_command)
275
318
 
276
319
  # compare directory content
277
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
278
- expected_output_dir_path)
320
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
279
321
 
280
322
  dirs_cmp.report()
281
323
 
@@ -284,7 +326,7 @@ class TestFilmFileProd(CommandTestCase):
284
326
 
285
327
  for filename in self.get_diff_files(dirs_cmp):
286
328
  # compare only xml files with differences
287
- if filename.endswith('.xml'):
329
+ if filename.endswith(".xml"):
288
330
  # use cdf compare to compute the differences between expected output and the command output
289
331
  result = filecmp.cmpfiles(
290
332
  os.path.join(generated_output_dir_path, filename),
@@ -292,48 +334,83 @@ class TestFilmFileProd(CommandTestCase):
292
334
  )
293
335
  # compare the difference dict with the expected one
294
336
  if result:
295
- logger.error(f'Differences between expected output and the command output: {pformat(result)}')
337
+ logger.error(
338
+ f"Differences between expected output and the command output: {pformat(result)}"
339
+ )
296
340
 
297
341
  assert result == {}
298
342
 
299
-
300
- @pytest.mark.parametrize('idb_source,idb_version', [
301
- ('MIB', '20200131'),
302
- ])
343
+ @pytest.mark.parametrize(
344
+ "idb_source,idb_version",
345
+ [
346
+ ("MIB", "20200131"),
347
+ ],
348
+ )
303
349
  def test_classify_tcreport(self, idb_source, idb_version):
304
350
  from poppy.core.conf import Settings
305
351
 
306
- input_dir_path, inputs = self.get_inputs('classify_tcreport')
307
- expected_output_dir_path, expected_outputs = self.get_expected_outputs('classify_tcreport')
352
+ input_dir_path, inputs = self.get_inputs("classify_tcreport")
353
+ expected_output_dir_path, expected_outputs = self.get_expected_outputs(
354
+ "classify_tcreport"
355
+ )
308
356
 
309
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
357
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
310
358
  os.makedirs(generated_output_dir_path, exist_ok=True)
311
359
 
312
360
  # initialize the main command
313
- main_command = ['pop', 'film',
314
- 'classify_tcreport',
315
- '--dds-files',
316
- ' '.join([
317
- os.path.join(input_dir_path, input_file)
318
- for input_file in inputs]),
319
- '--output-dir', generated_output_dir_path,
320
- '-ll', 'INFO']
361
+ main_command = [
362
+ "pop",
363
+ "film",
364
+ "classify_tcreport",
365
+ "--dds-files",
366
+ " ".join(
367
+ [os.path.join(input_dir_path, input_file) for input_file in inputs]
368
+ ),
369
+ "--output-dir",
370
+ generated_output_dir_path,
371
+ "-ll",
372
+ "INFO",
373
+ ]
321
374
 
322
375
  # define the required plugins
323
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.dingo' ,'roc.film']
376
+ plugin_list = [
377
+ "poppy.pop",
378
+ "roc.idb",
379
+ "roc.rpl",
380
+ "roc.rap",
381
+ "roc.dingo",
382
+ "roc.film",
383
+ ]
324
384
 
325
385
  # run the command
326
386
  # force the value of the plugin list
327
- with mock.patch.object(Settings, 'configure',
328
- autospec=True,
329
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
330
- self.run_command('pop db upgrade heads -ll INFO')
331
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
387
+ with mock.patch.object(
388
+ Settings,
389
+ "configure",
390
+ autospec=True,
391
+ side_effect=self.mock_configure_settings(
392
+ dictionary={"PLUGINS": plugin_list}
393
+ ),
394
+ ):
395
+ self.run_command("pop db upgrade heads -ll INFO")
396
+ self.run_command(
397
+ [
398
+ "pop",
399
+ "-ll",
400
+ "INFO",
401
+ "idb",
402
+ "install",
403
+ "-s",
404
+ idb_source,
405
+ "-v",
406
+ idb_version,
407
+ "--load",
408
+ ]
409
+ )
332
410
  self.run_command(main_command)
333
411
 
334
412
  # compare directory content
335
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
336
- expected_output_dir_path)
413
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
337
414
 
338
415
  dirs_cmp.report()
339
416
 
@@ -342,7 +419,7 @@ class TestFilmFileProd(CommandTestCase):
342
419
 
343
420
  for filename in self.get_diff_files(dirs_cmp):
344
421
  # compare only xml files with differences
345
- if filename.endswith('.xml'):
422
+ if filename.endswith(".xml"):
346
423
  # use cdf compare to compute the differences between expected output and the command output
347
424
  result = filecmp.cmpfiles(
348
425
  os.path.join(generated_output_dir_path, filename),
@@ -350,47 +427,83 @@ class TestFilmFileProd(CommandTestCase):
350
427
  )
351
428
  # compare the difference dict with the expected one
352
429
  if result:
353
- logger.error(f'Differences between expected output and the command output: {pformat(result)}')
430
+ logger.error(
431
+ f"Differences between expected output and the command output: {pformat(result)}"
432
+ )
354
433
 
355
434
  assert result == {}
356
435
 
357
- @pytest.mark.parametrize('idb_source,idb_version', [
358
- ('MIB', '20200131'),
359
- ])
436
+ @pytest.mark.parametrize(
437
+ "idb_source,idb_version",
438
+ [
439
+ ("MIB", "20200131"),
440
+ ],
441
+ )
360
442
  def test_classify_process_solohk(self, idb_source, idb_version):
361
443
  from poppy.core.conf import Settings
362
444
 
363
- input_dir_path, inputs = self.get_inputs('process_solohk')
364
- expected_output_dir_path, expected_outputs = self.get_expected_outputs('process_solohk')
445
+ input_dir_path, inputs = self.get_inputs("process_solohk")
446
+ expected_output_dir_path, expected_outputs = self.get_expected_outputs(
447
+ "process_solohk"
448
+ )
365
449
 
366
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
450
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
367
451
  os.makedirs(generated_output_dir_path, exist_ok=True)
368
452
 
369
453
  # initialize the main command
370
- main_command = ['pop', 'film',
371
- 'process_solohk',
372
- '--dds-files',
373
- ' '.join([
374
- os.path.join(input_dir_path, input_file)
375
- for input_file in inputs]),
376
- '--output-dir', generated_output_dir_path,
377
- '-ll', 'INFO']
454
+ main_command = [
455
+ "pop",
456
+ "film",
457
+ "process_solohk",
458
+ "--dds-files",
459
+ " ".join(
460
+ [os.path.join(input_dir_path, input_file) for input_file in inputs]
461
+ ),
462
+ "--output-dir",
463
+ generated_output_dir_path,
464
+ "-ll",
465
+ "INFO",
466
+ ]
378
467
 
379
468
  # define the required plugins
380
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.dingo' ,'roc.film']
469
+ plugin_list = [
470
+ "poppy.pop",
471
+ "roc.idb",
472
+ "roc.rpl",
473
+ "roc.rap",
474
+ "roc.dingo",
475
+ "roc.film",
476
+ ]
381
477
 
382
478
  # run the command
383
479
  # force the value of the plugin list
384
- with mock.patch.object(Settings, 'configure',
385
- autospec=True,
386
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
387
- self.run_command('pop db upgrade heads -ll INFO')
388
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
480
+ with mock.patch.object(
481
+ Settings,
482
+ "configure",
483
+ autospec=True,
484
+ side_effect=self.mock_configure_settings(
485
+ dictionary={"PLUGINS": plugin_list}
486
+ ),
487
+ ):
488
+ self.run_command("pop db upgrade heads -ll INFO")
489
+ self.run_command(
490
+ [
491
+ "pop",
492
+ "-ll",
493
+ "INFO",
494
+ "idb",
495
+ "install",
496
+ "-s",
497
+ idb_source,
498
+ "-v",
499
+ idb_version,
500
+ "--load",
501
+ ]
502
+ )
389
503
  self.run_command(main_command)
390
504
 
391
505
  # compare directory content
392
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
393
- expected_output_dir_path)
506
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
394
507
 
395
508
  dirs_cmp.report()
396
509
 
@@ -399,7 +512,7 @@ class TestFilmFileProd(CommandTestCase):
399
512
 
400
513
  for filename in self.get_diff_files(dirs_cmp):
401
514
  # compare only xml files with differences
402
- if filename.endswith('.xml'):
515
+ if filename.endswith(".xml"):
403
516
  # use cdf compare to compute the differences between expected output and the command output
404
517
  result = filecmp.cmpfiles(
405
518
  os.path.join(generated_output_dir_path, filename),
@@ -407,19 +520,24 @@ class TestFilmFileProd(CommandTestCase):
407
520
  )
408
521
  # compare the difference dict with the expected one
409
522
  if result:
410
- logger.error(f'Differences between expected output and the command output: {pformat(result)}')
523
+ logger.error(
524
+ f"Differences between expected output and the command output: {pformat(result)}"
525
+ )
411
526
 
412
527
  assert result == {}
413
528
 
414
- @pytest.mark.parametrize('idb_source,idb_version', [
415
- ('MIB', '20200131'),
416
- ('PALISADE', '4.3.5_MEB_PFM'),
417
- ])
529
+ @pytest.mark.parametrize(
530
+ "idb_source,idb_version",
531
+ [
532
+ ("MIB", "20200131"),
533
+ ("PALISADE", "4.3.5_MEB_PFM"),
534
+ ],
535
+ )
418
536
  def test_l0_to_l1_surv(self, idb_source, idb_version):
419
537
  from poppy.core.conf import Settings
420
538
 
421
539
  # Name of the command to test
422
- cmd = 'l0_to_l1_surv'
540
+ cmd = "l0_to_l1_surv"
423
541
 
424
542
  input_dir_path, inputs = self.get_inputs(cmd)
425
543
  expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -427,35 +545,60 @@ class TestFilmFileProd(CommandTestCase):
427
545
 
428
546
  # extract spice kernels
429
547
  spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
548
+ logger.debug(spice_kernel_dir_path)
430
549
 
431
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
550
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
432
551
  os.makedirs(generated_output_dir_path, exist_ok=True)
433
552
 
434
553
  # initialize the main command
435
- main_command = ['pop', 'film',
436
- '--idb-version', idb_version,
437
- '--idb-source', idb_source,
438
- '--cdag',
439
- cmd,
440
- os.path.join(input_dir_path, inputs[0]),
441
- '--output-dir', generated_output_dir_path,
442
- '-ll', 'INFO']
554
+ main_command = [
555
+ "pop",
556
+ "film",
557
+ "--idb-version",
558
+ idb_version,
559
+ "--idb-source",
560
+ idb_source,
561
+ "--cdag",
562
+ cmd,
563
+ os.path.join(input_dir_path, inputs[0]),
564
+ "--output-dir",
565
+ generated_output_dir_path,
566
+ "-ll",
567
+ "INFO",
568
+ ]
443
569
 
444
570
  # define the required plugins
445
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.film']
571
+ plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
446
572
 
447
573
  # run the command
448
574
  # force the value of the plugin list
449
- with mock.patch.object(Settings, 'configure',
450
- autospec=True,
451
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
452
- self.run_command('pop db upgrade heads -ll INFO')
453
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
575
+ with mock.patch.object(
576
+ Settings,
577
+ "configure",
578
+ autospec=True,
579
+ side_effect=self.mock_configure_settings(
580
+ dictionary={"PLUGINS": plugin_list}
581
+ ),
582
+ ):
583
+ self.run_command("pop db upgrade heads -ll INFO")
584
+ self.run_command(
585
+ [
586
+ "pop",
587
+ "-ll",
588
+ "INFO",
589
+ "idb",
590
+ "install",
591
+ "-s",
592
+ idb_source,
593
+ "-v",
594
+ idb_version,
595
+ "--load",
596
+ ]
597
+ )
454
598
  self.run_command(main_command)
455
599
 
456
600
  # compare directory content
457
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
458
- expected_output_dir_path)
601
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
459
602
 
460
603
  dirs_cmp.report()
461
604
 
@@ -464,36 +607,41 @@ class TestFilmFileProd(CommandTestCase):
464
607
 
465
608
  for filename in self.get_diff_files(dirs_cmp):
466
609
  # compare only cdf files with differences
467
- if filename.endswith('.cdf'):
610
+ if filename.endswith(".cdf"):
468
611
  # use cdf compare to compute the differences between expected output and the command output
469
612
  result = cdf_compare(
470
613
  os.path.join(generated_output_dir_path, filename),
471
614
  os.path.join(expected_output_dir_path, filename),
472
615
  list_ignore_gatt=[
473
- 'File_ID',
474
- 'Generation_date',
475
- 'Pipeline_version',
476
- 'Pipeline_name',
477
- 'Software_version',
478
- 'IDB_version'
479
- ]
616
+ "File_ID",
617
+ "Generation_date",
618
+ "Pipeline_version",
619
+ "Pipeline_name",
620
+ "Software_version",
621
+ "IDB_version",
622
+ ],
480
623
  )
481
624
 
482
625
  # compare the difference dict with the expected one
483
626
  if result:
484
- logger.error(f'Differences between expected output and the command output: {pformat(result)}')
627
+ logger.error(
628
+ f"Differences between expected output and the command output: {pformat(result)}"
629
+ )
485
630
 
486
631
  assert result == {}
487
632
 
488
- @pytest.mark.parametrize('idb_source,idb_version', [
489
- ('MIB', '20200131'),
490
- ('PALISADE', '4.3.5_MEB_PFM'),
491
- ])
633
+ @pytest.mark.parametrize(
634
+ "idb_source,idb_version",
635
+ [
636
+ ("MIB", "20200131"),
637
+ ("PALISADE", "4.3.5_MEB_PFM"),
638
+ ],
639
+ )
492
640
  def test_l0_to_hk(self, idb_source, idb_version):
493
641
  from poppy.core.conf import Settings
494
642
 
495
643
  # Name of the command to test
496
- cmd = 'l0_to_hk'
644
+ cmd = "l0_to_hk"
497
645
 
498
646
  input_dir_path, inputs = self.get_inputs(cmd)
499
647
  expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -501,34 +649,66 @@ class TestFilmFileProd(CommandTestCase):
501
649
 
502
650
  # extract spice kernels
503
651
  spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
652
+ logger.debug(spice_kernel_dir_path)
504
653
 
505
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
654
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
506
655
  os.makedirs(generated_output_dir_path, exist_ok=True)
507
656
 
508
657
  # initialize the main command
509
- main_command = ['pop', 'film',
510
- '--idb-version', idb_version,
511
- '--idb-source', idb_source,
512
- cmd,
513
- os.path.join(input_dir_path, inputs[0]),
514
- '--output-dir', generated_output_dir_path,
515
- '-ll', 'INFO']
658
+ main_command = [
659
+ "pop",
660
+ "film",
661
+ "--idb-version",
662
+ idb_version,
663
+ "--idb-source",
664
+ idb_source,
665
+ cmd,
666
+ os.path.join(input_dir_path, inputs[0]),
667
+ "--output-dir",
668
+ generated_output_dir_path,
669
+ "-ll",
670
+ "INFO",
671
+ ]
516
672
 
517
673
  # define the required plugins
518
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.dingo', 'roc.film']
674
+ plugin_list = [
675
+ "poppy.pop",
676
+ "roc.idb",
677
+ "roc.rpl",
678
+ "roc.rap",
679
+ "roc.dingo",
680
+ "roc.film",
681
+ ]
519
682
 
520
683
  # run the command
521
684
  # force the value of the plugin list
522
- with mock.patch.object(Settings, 'configure',
523
- autospec=True,
524
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
525
- self.run_command('pop db upgrade heads -ll INFO')
526
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
685
+ with mock.patch.object(
686
+ Settings,
687
+ "configure",
688
+ autospec=True,
689
+ side_effect=self.mock_configure_settings(
690
+ dictionary={"PLUGINS": plugin_list}
691
+ ),
692
+ ):
693
+ self.run_command("pop db upgrade heads -ll INFO")
694
+ self.run_command(
695
+ [
696
+ "pop",
697
+ "-ll",
698
+ "INFO",
699
+ "idb",
700
+ "install",
701
+ "-s",
702
+ idb_source,
703
+ "-v",
704
+ idb_version,
705
+ "--load",
706
+ ]
707
+ )
527
708
  self.run_command(main_command)
528
709
 
529
710
  # compare directory content
530
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
531
- expected_output_dir_path)
711
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
532
712
 
533
713
  dirs_cmp.report()
534
714
 
@@ -537,36 +717,41 @@ class TestFilmFileProd(CommandTestCase):
537
717
 
538
718
  for filename in self.get_diff_files(dirs_cmp):
539
719
  # compare only cdf files with differences
540
- if filename.endswith('.cdf'):
720
+ if filename.endswith(".cdf"):
541
721
  # use cdf compare to compute the differences between expected output and the command output
542
722
  result = cdf_compare(
543
723
  os.path.join(generated_output_dir_path, filename),
544
724
  os.path.join(expected_output_dir_path, filename),
545
725
  list_ignore_gatt=[
546
- 'File_ID',
547
- 'Generation_date',
548
- 'Pipeline_version',
549
- 'Pipeline_name',
550
- 'Software_version',
551
- 'IDB_version'
552
- ]
726
+ "File_ID",
727
+ "Generation_date",
728
+ "Pipeline_version",
729
+ "Pipeline_name",
730
+ "Software_version",
731
+ "IDB_version",
732
+ ],
553
733
  )
554
734
 
555
735
  # compare the difference dict with the expected one
556
736
  if result:
557
- logger.error(f'Differences between expected output and the command output: {pformat(result)}')
737
+ logger.error(
738
+ f"Differences between expected output and the command output: {pformat(result)}"
739
+ )
558
740
 
559
741
  assert result == {}
560
742
 
561
- @pytest.mark.parametrize('idb_source,idb_version', [
562
- ('MIB', '20200131'),
563
- ('PALISADE', '4.3.5_MEB_PFM'),
564
- ])
743
+ @pytest.mark.parametrize(
744
+ "idb_source,idb_version",
745
+ [
746
+ ("MIB", "20200131"),
747
+ ("PALISADE", "4.3.5_MEB_PFM"),
748
+ ],
749
+ )
565
750
  def test_dds_to_l0(self, idb_source, idb_version):
566
751
  from poppy.core.conf import Settings
567
752
 
568
753
  # Name of the command to test
569
- cmd = 'dds_to_l0'
754
+ cmd = "dds_to_l0"
570
755
 
571
756
  input_dir_path, inputs = self.get_inputs(cmd)
572
757
  expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -574,52 +759,85 @@ class TestFilmFileProd(CommandTestCase):
574
759
 
575
760
  # extract spice kernels
576
761
  spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
577
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
762
+ logger.debug(spice_kernel_dir_path)
763
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
578
764
  os.makedirs(generated_output_dir_path, exist_ok=True)
579
765
 
580
- date = '20200303'
581
- scos_header = '0'
766
+ date = "20200303"
767
+ scos_header = "0"
582
768
 
583
769
  # Build list of tm and tc input files
584
- tm_inputs = ' '.join([
585
- os.path.join(input_dir_path, input_file)
586
- for input_file in inputs
587
- if os.path.basename(input_file).startswith('solo_TM_')
588
- ])
589
- tc_inputs = ' '.join([
590
- os.path.join(input_dir_path, input_file)
591
- for input_file in inputs
592
- if os.path.basename(input_file).startswith('solo_TC_')
593
- ])
770
+ tm_inputs = " ".join(
771
+ [
772
+ os.path.join(input_dir_path, input_file)
773
+ for input_file in inputs
774
+ if os.path.basename(input_file).startswith("solo_TM_")
775
+ ]
776
+ )
777
+ tc_inputs = " ".join(
778
+ [
779
+ os.path.join(input_dir_path, input_file)
780
+ for input_file in inputs
781
+ if os.path.basename(input_file).startswith("solo_TC_")
782
+ ]
783
+ )
594
784
 
595
785
  # initialize the main command
596
786
  # Make sure that start-time/end_time keyword values are consistent with input DDS files
597
- main_command = ['pop', 'film',
598
- '--idb-version', idb_version,
599
- '--idb-source', idb_source,
600
- '--scos-header', scos_header,
601
- '--cdag',
602
- cmd, date,
603
- '--dds-tmraw-xml', tm_inputs,
604
- '--dds-tcreport-xml', tc_inputs,
605
- '--output-dir', generated_output_dir_path,
606
- '-ll', 'INFO']
787
+ main_command = [
788
+ "pop",
789
+ "film",
790
+ "--idb-version",
791
+ idb_version,
792
+ "--idb-source",
793
+ idb_source,
794
+ "--scos-header",
795
+ scos_header,
796
+ "--cdag",
797
+ cmd,
798
+ date,
799
+ "--dds-tmraw-xml",
800
+ tm_inputs,
801
+ "--dds-tcreport-xml",
802
+ tc_inputs,
803
+ "--output-dir",
804
+ generated_output_dir_path,
805
+ "-ll",
806
+ "INFO",
807
+ ]
607
808
 
608
809
  # define the required plugins
609
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.film']
810
+ plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
610
811
 
611
812
  # run the command
612
813
  # force the value of the plugin list
613
- with mock.patch.object(Settings, 'configure',
614
- autospec=True,
615
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
616
- self.run_command('pop db upgrade heads -ll INFO')
617
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
814
+ with mock.patch.object(
815
+ Settings,
816
+ "configure",
817
+ autospec=True,
818
+ side_effect=self.mock_configure_settings(
819
+ dictionary={"PLUGINS": plugin_list}
820
+ ),
821
+ ):
822
+ self.run_command("pop db upgrade heads -ll INFO")
823
+ self.run_command(
824
+ [
825
+ "pop",
826
+ "-ll",
827
+ "INFO",
828
+ "idb",
829
+ "install",
830
+ "-s",
831
+ idb_source,
832
+ "-v",
833
+ idb_version,
834
+ "--load",
835
+ ]
836
+ )
618
837
  self.run_command(main_command)
619
838
 
620
839
  # compare directory content
621
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
622
- expected_output_dir_path)
840
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
623
841
 
624
842
  dirs_cmp.report()
625
843
 
@@ -650,15 +868,18 @@ class TestFilmFileProd(CommandTestCase):
650
868
  #
651
869
  # assert result == {}
652
870
 
653
- @pytest.mark.parametrize('idb_source,idb_version', [
654
- ('MIB', '20200131'),
655
- ('PALISADE', '4.3.5_MEB_PFM'),
656
- ])
871
+ @pytest.mark.parametrize(
872
+ "idb_source,idb_version",
873
+ [
874
+ ("MIB", "20200131"),
875
+ ("PALISADE", "4.3.5_MEB_PFM"),
876
+ ],
877
+ )
657
878
  def test_l0_to_l1_sbm(self, idb_source, idb_version):
658
879
  from poppy.core.conf import Settings
659
880
 
660
881
  # Name of the command to test
661
- cmd = 'l0_to_l1_sbm'
882
+ cmd = "l0_to_l1_sbm"
662
883
 
663
884
  input_dir_path, inputs = self.get_inputs(cmd)
664
885
  expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -666,37 +887,62 @@ class TestFilmFileProd(CommandTestCase):
666
887
 
667
888
  # extract spice kernels
668
889
  spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
890
+ logger.debug(spice_kernel_dir_path)
669
891
 
670
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
892
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
671
893
  os.makedirs(generated_output_dir_path, exist_ok=True)
672
894
 
673
895
  # initialize the main command
674
- main_command = ['pop', 'film',
675
- '--idb-version', idb_version,
676
- '--idb-source', idb_source,
677
- '--cdag',
678
- cmd,
679
- ' '.join([
680
- os.path.join(input_dir_path, input_file)
681
- for input_file in inputs]),
682
- '--output-dir', generated_output_dir_path,
683
- '-ll', 'INFO']
896
+ main_command = [
897
+ "pop",
898
+ "film",
899
+ "--idb-version",
900
+ idb_version,
901
+ "--idb-source",
902
+ idb_source,
903
+ "--cdag",
904
+ cmd,
905
+ " ".join(
906
+ [os.path.join(input_dir_path, input_file) for input_file in inputs]
907
+ ),
908
+ "--output-dir",
909
+ generated_output_dir_path,
910
+ "-ll",
911
+ "INFO",
912
+ ]
684
913
 
685
914
  # define the required plugins
686
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.film']
915
+ plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
687
916
 
688
917
  # run the command
689
918
  # force the value of the plugin list
690
- with mock.patch.object(Settings, 'configure',
691
- autospec=True,
692
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
693
- self.run_command('pop db upgrade heads -ll INFO')
694
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
919
+ with mock.patch.object(
920
+ Settings,
921
+ "configure",
922
+ autospec=True,
923
+ side_effect=self.mock_configure_settings(
924
+ dictionary={"PLUGINS": plugin_list}
925
+ ),
926
+ ):
927
+ self.run_command("pop db upgrade heads -ll INFO")
928
+ self.run_command(
929
+ [
930
+ "pop",
931
+ "-ll",
932
+ "INFO",
933
+ "idb",
934
+ "install",
935
+ "-s",
936
+ idb_source,
937
+ "-v",
938
+ idb_version,
939
+ "--load",
940
+ ]
941
+ )
695
942
  self.run_command(main_command)
696
943
 
697
944
  # compare directory content
698
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
699
- expected_output_dir_path)
945
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
700
946
 
701
947
  dirs_cmp.report()
702
948
 
@@ -705,36 +951,41 @@ class TestFilmFileProd(CommandTestCase):
705
951
 
706
952
  for filename in self.get_diff_files(dirs_cmp):
707
953
  # compare only cdf files with differences
708
- if filename.endswith('.cdf'):
954
+ if filename.endswith(".cdf"):
709
955
  # use cdf compare to compute the differences between expected output and the command output
710
956
  result = cdf_compare(
711
957
  os.path.join(generated_output_dir_path, filename),
712
958
  os.path.join(expected_output_dir_path, filename),
713
959
  list_ignore_gatt=[
714
- 'File_ID',
715
- 'Generation_date',
716
- 'Pipeline_version',
717
- 'Pipeline_name',
718
- 'Software_version',
719
- 'IDB_version'
720
- ]
960
+ "File_ID",
961
+ "Generation_date",
962
+ "Pipeline_version",
963
+ "Pipeline_name",
964
+ "Software_version",
965
+ "IDB_version",
966
+ ],
721
967
  )
722
968
 
723
969
  # compare the difference dict with the expected one
724
970
  if result:
725
- logger.error(f'Differences between expected output and the command output: {pformat(result)}')
971
+ logger.error(
972
+ f"Differences between expected output and the command output: {pformat(result)}"
973
+ )
726
974
 
727
975
  assert result == {}
728
976
 
729
- @pytest.mark.parametrize('idb_source,idb_version', [
730
- ('MIB', '20200131'),
731
- ('PALISADE', '4.3.5_MEB_PFM'),
732
- ])
977
+ @pytest.mark.parametrize(
978
+ "idb_source,idb_version",
979
+ [
980
+ ("MIB", "20200131"),
981
+ ("PALISADE", "4.3.5_MEB_PFM"),
982
+ ],
983
+ )
733
984
  def test_l0_to_anc_bia_sweep_table(self, idb_source, idb_version):
734
985
  from poppy.core.conf import Settings
735
986
 
736
987
  # Name of the command to test
737
- cmd = 'l0_to_anc_bia_sweep_table'
988
+ cmd = "l0_to_anc_bia_sweep_table"
738
989
 
739
990
  input_dir_path, inputs = self.get_inputs(cmd)
740
991
  expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -742,82 +993,112 @@ class TestFilmFileProd(CommandTestCase):
742
993
 
743
994
  # extract spice kernels
744
995
  spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
996
+ logger.debug(spice_kernel_dir_path)
745
997
 
746
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
998
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
747
999
  os.makedirs(generated_output_dir_path, exist_ok=True)
748
1000
 
749
1001
  # initialize the main command
750
- main_command = ['pop', 'film',
751
- '--idb-version', idb_version,
752
- '--idb-source', idb_source,
753
- '--monthly',
754
- cmd,
755
- '--l0-files',
756
- ' '.join([
757
- os.path.join(input_dir_path, input_file)
758
- for input_file in inputs]),
759
- '--output-dir', generated_output_dir_path,
760
- '-ll', 'INFO']
1002
+ main_command = [
1003
+ "pop",
1004
+ "film",
1005
+ "--idb-version",
1006
+ idb_version,
1007
+ "--idb-source",
1008
+ idb_source,
1009
+ "--monthly",
1010
+ cmd,
1011
+ "--l0-files",
1012
+ " ".join(
1013
+ [os.path.join(input_dir_path, input_file) for input_file in inputs]
1014
+ ),
1015
+ "--output-dir",
1016
+ generated_output_dir_path,
1017
+ "-ll",
1018
+ "INFO",
1019
+ ]
761
1020
 
762
1021
  # define the required plugins
763
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.film']
1022
+ plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
764
1023
 
765
1024
  # run the command
766
1025
  # force the value of the plugin list
767
- with mock.patch.object(Settings, 'configure',
768
- autospec=True,
769
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
770
- self.run_command('pop db upgrade heads -ll INFO')
771
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
1026
+ with mock.patch.object(
1027
+ Settings,
1028
+ "configure",
1029
+ autospec=True,
1030
+ side_effect=self.mock_configure_settings(
1031
+ dictionary={"PLUGINS": plugin_list}
1032
+ ),
1033
+ ):
1034
+ self.run_command("pop db upgrade heads -ll INFO")
1035
+ self.run_command(
1036
+ [
1037
+ "pop",
1038
+ "-ll",
1039
+ "INFO",
1040
+ "idb",
1041
+ "install",
1042
+ "-s",
1043
+ idb_source,
1044
+ "-v",
1045
+ idb_version,
1046
+ "--load",
1047
+ ]
1048
+ )
772
1049
  self.run_command(main_command)
773
1050
 
774
1051
  # compare directory content
775
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
776
- expected_output_dir_path)
1052
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
777
1053
 
778
1054
  dirs_cmp.report()
779
1055
 
780
1056
  # ensure that we have the same files in both directories
781
- assert (len(dirs_cmp.left_only) == 0) and (len(dirs_cmp.right_only) == 0), \
782
- 'Different files in expected and generated output directories!'
1057
+ assert (len(dirs_cmp.left_only) == 0) and (
1058
+ len(dirs_cmp.right_only) == 0
1059
+ ), "Different files in expected and generated output directories!"
783
1060
 
784
1061
  for filename in self.get_diff_files(dirs_cmp):
785
1062
  # compare only cdf files with differences
786
- if filename.endswith('.cdf'):
1063
+ if filename.endswith(".cdf"):
787
1064
  # use cdf compare to compute the differences between expected output and the command output
788
1065
  result = cdf_compare(
789
1066
  os.path.join(generated_output_dir_path, filename),
790
1067
  os.path.join(expected_output_dir_path, filename),
791
1068
  list_ignore_gatt=[
792
- 'File_ID',
793
- 'Generation_date',
794
- 'Pipeline_version',
795
- 'Pipeline_name',
796
- 'Software_version',
797
- 'IDB_version'
798
- ]
1069
+ "File_ID",
1070
+ "Generation_date",
1071
+ "Pipeline_version",
1072
+ "Pipeline_name",
1073
+ "Software_version",
1074
+ "IDB_version",
1075
+ ],
799
1076
  )
800
- elif filename.endswith('.csv'):
1077
+ elif filename.endswith(".csv"):
801
1078
  result = filecmp.cmpfiles(
802
1079
  os.path.join(generated_output_dir_path, filename),
803
- os.path.join(expected_output_dir_path, filename)
1080
+ os.path.join(expected_output_dir_path, filename),
804
1081
  )
805
1082
  else:
806
1083
  result = {}
807
1084
 
808
1085
  # compare the difference dict with the expected one
809
- assert result == {}, \
810
- f'Differences between expected output and the command output: {pformat(result)}'
811
-
812
- @pytest.mark.parametrize('idb_source,idb_version', [
813
- ('MIB', '20200131'),
814
- ('PALISADE', '4.3.5_MEB_PFM'),
815
- ])
1086
+ assert (
1087
+ result == {}
1088
+ ), f"Differences between expected output and the command output: {pformat(result)}"
1089
+
1090
+ @pytest.mark.parametrize(
1091
+ "idb_source,idb_version",
1092
+ [
1093
+ ("MIB", "20200131"),
1094
+ ("PALISADE", "4.3.5_MEB_PFM"),
1095
+ ],
1096
+ )
816
1097
  def test_l0_to_l1_bia_sweep(self, idb_source, idb_version):
817
1098
  from poppy.core.conf import Settings
818
1099
 
819
1100
  # Name of the command to test
820
- cmd = 'l0_to_anc_bia_sweep_table'
1101
+ cmd = "l0_to_anc_bia_sweep_table"
821
1102
 
822
1103
  input_dir_path, inputs = self.get_inputs(cmd)
823
1104
  expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -825,91 +1106,130 @@ class TestFilmFileProd(CommandTestCase):
825
1106
 
826
1107
  # extract spice kernels
827
1108
  spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
1109
+ logger.debug(spice_kernel_dir_path)
828
1110
 
829
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
1111
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
830
1112
  os.makedirs(generated_output_dir_path, exist_ok=True)
831
1113
 
832
1114
  # Build list of inputs
833
- l0_files = ' '.join([
834
- os.path.join(input_dir_path, input_file)
835
- for input_file in inputs
836
- if os.path.basename(input_file).startswith('solo_L0_rpw')
837
- ])
838
- sweep_tables = ' '.join([
839
- os.path.join(input_dir_path, input_file)
840
- for input_file in inputs
841
- if os.path.basename(input_file).startswith('solo_ANC_rpw-bia-sweep-table')
842
- ])
1115
+ l0_files = " ".join(
1116
+ [
1117
+ os.path.join(input_dir_path, input_file)
1118
+ for input_file in inputs
1119
+ if os.path.basename(input_file).startswith("solo_L0_rpw")
1120
+ ]
1121
+ )
1122
+ sweep_tables = " ".join(
1123
+ [
1124
+ os.path.join(input_dir_path, input_file)
1125
+ for input_file in inputs
1126
+ if os.path.basename(input_file).startswith(
1127
+ "solo_ANC_rpw-bia-sweep-table"
1128
+ )
1129
+ ]
1130
+ )
843
1131
 
844
1132
  # initialize the main command
845
- main_command = ['pop', 'film',
846
- '--idb-version', idb_version,
847
- '--idb-source', idb_source,
848
- '--cdag',
849
- cmd,
850
- '--l0-files', l0_files,
851
- '--sweep-tables', sweep_tables,
852
- '--output-dir', generated_output_dir_path,
853
- '-ll', 'INFO']
1133
+ main_command = [
1134
+ "pop",
1135
+ "film",
1136
+ "--idb-version",
1137
+ idb_version,
1138
+ "--idb-source",
1139
+ idb_source,
1140
+ "--cdag",
1141
+ cmd,
1142
+ "--l0-files",
1143
+ l0_files,
1144
+ "--sweep-tables",
1145
+ sweep_tables,
1146
+ "--output-dir",
1147
+ generated_output_dir_path,
1148
+ "-ll",
1149
+ "INFO",
1150
+ ]
854
1151
 
855
1152
  # define the required plugins
856
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.film']
1153
+ plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
857
1154
 
858
1155
  # run the command
859
1156
  # force the value of the plugin list
860
- with mock.patch.object(Settings, 'configure',
861
- autospec=True,
862
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
863
- self.run_command('pop db upgrade heads -ll INFO')
864
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
1157
+ with mock.patch.object(
1158
+ Settings,
1159
+ "configure",
1160
+ autospec=True,
1161
+ side_effect=self.mock_configure_settings(
1162
+ dictionary={"PLUGINS": plugin_list}
1163
+ ),
1164
+ ):
1165
+ self.run_command("pop db upgrade heads -ll INFO")
1166
+ self.run_command(
1167
+ [
1168
+ "pop",
1169
+ "-ll",
1170
+ "INFO",
1171
+ "idb",
1172
+ "install",
1173
+ "-s",
1174
+ idb_source,
1175
+ "-v",
1176
+ idb_version,
1177
+ "--load",
1178
+ ]
1179
+ )
865
1180
  self.run_command(main_command)
866
1181
 
867
1182
  # compare directory content
868
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
869
- expected_output_dir_path)
1183
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
870
1184
 
871
1185
  dirs_cmp.report()
872
1186
 
873
1187
  # ensure that we have the same files in both directories
874
- assert (len(dirs_cmp.left_only) == 0) and (len(dirs_cmp.right_only) == 0), \
875
- 'Different files in expected and generated output directories!'
1188
+ assert (len(dirs_cmp.left_only) == 0) and (
1189
+ len(dirs_cmp.right_only) == 0
1190
+ ), "Different files in expected and generated output directories!"
876
1191
 
877
1192
  for filename in self.get_diff_files(dirs_cmp):
878
1193
  # compare only cdf files with differences
879
- if filename.endswith('.cdf'):
1194
+ if filename.endswith(".cdf"):
880
1195
  # use cdf compare to compute the differences between expected output and the command output
881
1196
  result = cdf_compare(
882
1197
  os.path.join(generated_output_dir_path, filename),
883
1198
  os.path.join(expected_output_dir_path, filename),
884
1199
  list_ignore_gatt=[
885
- 'File_ID',
886
- 'Generation_date',
887
- 'Pipeline_version',
888
- 'Pipeline_name',
889
- 'Software_version',
890
- 'IDB_version'
891
- ]
1200
+ "File_ID",
1201
+ "Generation_date",
1202
+ "Pipeline_version",
1203
+ "Pipeline_name",
1204
+ "Software_version",
1205
+ "IDB_version",
1206
+ ],
892
1207
  )
893
- elif filename.endswith('.csv'):
1208
+ elif filename.endswith(".csv"):
894
1209
  result = filecmp.cmpfiles(
895
1210
  os.path.join(generated_output_dir_path, filename),
896
- os.path.join(expected_output_dir_path, filename)
1211
+ os.path.join(expected_output_dir_path, filename),
897
1212
  )
898
1213
  else:
899
1214
  result = {}
900
1215
 
901
1216
  # compare the difference dict with the expected one
902
- assert result == {}, f'Differences between expected output and the command output: {pformat(result)}'
903
-
904
- @pytest.mark.parametrize('idb_source,idb_version', [
905
- ('MIB', '20200131'),
906
- ('PALISADE', '4.3.5_MEB_PFM'),
907
- ])
1217
+ assert (
1218
+ result == {}
1219
+ ), f"Differences between expected output and the command output: {pformat(result)}"
1220
+
1221
+ @pytest.mark.parametrize(
1222
+ "idb_source,idb_version",
1223
+ [
1224
+ ("MIB", "20200131"),
1225
+ ("PALISADE", "4.3.5_MEB_PFM"),
1226
+ ],
1227
+ )
908
1228
  def test_l0_to_l1_bia_current(self, idb_source, idb_version):
909
1229
  from poppy.core.conf import Settings
910
1230
 
911
1231
  # Name of the command to test
912
- cmd = 'l0_to_anc_bia_sweep_table'
1232
+ cmd = "l0_to_anc_bia_sweep_table"
913
1233
 
914
1234
  input_dir_path, inputs = self.get_inputs(cmd)
915
1235
  expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -917,70 +1237,97 @@ class TestFilmFileProd(CommandTestCase):
917
1237
 
918
1238
  # extract spice kernels
919
1239
  spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
1240
+ logger.debug(spice_kernel_dir_path)
920
1241
 
921
- generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
1242
+ generated_output_dir_path = os.path.join(self.tmp_dir_path, "generated_output")
922
1243
  os.makedirs(generated_output_dir_path, exist_ok=True)
923
1244
 
924
1245
  # initialize the main command
925
- main_command = ['pop', 'film',
926
- '--idb-version', idb_version,
927
- '--idb-source', idb_source,
928
- '--cdag',
929
- '--monthly',
930
- cmd,
931
- '--l0-files',
932
- ' '.join([
933
- os.path.join(input_dir_path, input_file)
934
- for input_file in inputs]),
935
- '--output-dir', generated_output_dir_path,
936
- '-ll', 'INFO']
1246
+ main_command = [
1247
+ "pop",
1248
+ "film",
1249
+ "--idb-version",
1250
+ idb_version,
1251
+ "--idb-source",
1252
+ idb_source,
1253
+ "--cdag",
1254
+ "--monthly",
1255
+ cmd,
1256
+ "--l0-files",
1257
+ " ".join(
1258
+ [os.path.join(input_dir_path, input_file) for input_file in inputs]
1259
+ ),
1260
+ "--output-dir",
1261
+ generated_output_dir_path,
1262
+ "-ll",
1263
+ "INFO",
1264
+ ]
937
1265
 
938
1266
  # define the required plugins
939
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.film']
1267
+ plugin_list = ["poppy.pop", "roc.idb", "roc.rpl", "roc.rap", "roc.film"]
940
1268
 
941
1269
  # run the command
942
1270
  # force the value of the plugin list
943
- with mock.patch.object(Settings, 'configure',
944
- autospec=True,
945
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
946
- self.run_command('pop db upgrade heads -ll INFO')
947
- self.run_command(['pop', '-ll', 'INFO', 'idb', 'install', '-s', idb_source, '-v', idb_version, '--load'])
1271
+ with mock.patch.object(
1272
+ Settings,
1273
+ "configure",
1274
+ autospec=True,
1275
+ side_effect=self.mock_configure_settings(
1276
+ dictionary={"PLUGINS": plugin_list}
1277
+ ),
1278
+ ):
1279
+ self.run_command("pop db upgrade heads -ll INFO")
1280
+ self.run_command(
1281
+ [
1282
+ "pop",
1283
+ "-ll",
1284
+ "INFO",
1285
+ "idb",
1286
+ "install",
1287
+ "-s",
1288
+ idb_source,
1289
+ "-v",
1290
+ idb_version,
1291
+ "--load",
1292
+ ]
1293
+ )
948
1294
  self.run_command(main_command)
949
1295
 
950
1296
  # compare directory content
951
- dirs_cmp = filecmp.dircmp(generated_output_dir_path,
952
- expected_output_dir_path)
1297
+ dirs_cmp = filecmp.dircmp(generated_output_dir_path, expected_output_dir_path)
953
1298
 
954
1299
  dirs_cmp.report()
955
1300
 
956
1301
  # ensure that we have the same files in both directories
957
- assert (len(dirs_cmp.left_only) == 0) and (len(dirs_cmp.right_only) == 0), \
958
- 'Different files in expected and generated output directories!'
1302
+ assert (len(dirs_cmp.left_only) == 0) and (
1303
+ len(dirs_cmp.right_only) == 0
1304
+ ), "Different files in expected and generated output directories!"
959
1305
 
960
1306
  for filename in self.get_diff_files(dirs_cmp):
961
1307
  # compare only cdf files with differences
962
- if filename.endswith('.cdf'):
1308
+ if filename.endswith(".cdf"):
963
1309
  # use cdf compare to compute the differences between expected output and the command output
964
1310
  result = cdf_compare(
965
1311
  os.path.join(generated_output_dir_path, filename),
966
1312
  os.path.join(expected_output_dir_path, filename),
967
1313
  list_ignore_gatt=[
968
- 'File_ID',
969
- 'Generation_date',
970
- 'Pipeline_version',
971
- 'Pipeline_name',
972
- 'Software_version',
973
- 'IDB_version'
974
- ]
1314
+ "File_ID",
1315
+ "Generation_date",
1316
+ "Pipeline_version",
1317
+ "Pipeline_name",
1318
+ "Software_version",
1319
+ "IDB_version",
1320
+ ],
975
1321
  )
976
- elif filename.endswith('.csv'):
1322
+ elif filename.endswith(".csv"):
977
1323
  result = filecmp.cmpfiles(
978
1324
  os.path.join(generated_output_dir_path, filename),
979
- os.path.join(expected_output_dir_path, filename)
1325
+ os.path.join(expected_output_dir_path, filename),
980
1326
  )
981
1327
  else:
982
1328
  result = {}
983
1329
 
984
1330
  # compare the difference dict with the expected one
985
- assert result == {}, \
986
- f'Differences between expected output and the command output: {pformat(result)}'
1331
+ assert (
1332
+ result == {}
1333
+ ), f"Differences between expected output and the command output: {pformat(result)}"