roc-film 1.13.4__py3-none-any.whl → 1.14.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. roc/__init__.py +2 -1
  2. roc/film/__init__.py +2 -2
  3. roc/film/commands.py +372 -323
  4. roc/film/config/__init__.py +0 -1
  5. roc/film/constants.py +101 -65
  6. roc/film/descriptor.json +126 -95
  7. roc/film/exceptions.py +28 -27
  8. roc/film/tasks/__init__.py +16 -16
  9. roc/film/tasks/cat_solo_hk.py +86 -74
  10. roc/film/tasks/cdf_postpro.py +438 -309
  11. roc/film/tasks/check_dds.py +39 -45
  12. roc/film/tasks/db_to_anc_bia_sweep_table.py +381 -0
  13. roc/film/tasks/dds_to_l0.py +232 -180
  14. roc/film/tasks/export_solo_coord.py +147 -0
  15. roc/film/tasks/file_handler.py +91 -75
  16. roc/film/tasks/l0_to_hk.py +117 -103
  17. roc/film/tasks/l0_to_l1_bia_current.py +38 -30
  18. roc/film/tasks/l0_to_l1_bia_sweep.py +417 -329
  19. roc/film/tasks/l0_to_l1_sbm.py +250 -208
  20. roc/film/tasks/l0_to_l1_surv.py +185 -130
  21. roc/film/tasks/make_daily_tm.py +40 -37
  22. roc/film/tasks/merge_tcreport.py +77 -71
  23. roc/film/tasks/merge_tmraw.py +102 -89
  24. roc/film/tasks/parse_dds_xml.py +21 -20
  25. roc/film/tasks/set_l0_utc.py +51 -49
  26. roc/film/tests/cdf_compare.py +565 -0
  27. roc/film/tests/hdf5_compare.py +84 -62
  28. roc/film/tests/test_dds_to_l0.py +93 -51
  29. roc/film/tests/test_dds_to_tc.py +8 -11
  30. roc/film/tests/test_dds_to_tm.py +8 -10
  31. roc/film/tests/test_film.py +161 -116
  32. roc/film/tests/test_l0_to_hk.py +64 -36
  33. roc/film/tests/test_l0_to_l1_bia.py +10 -14
  34. roc/film/tests/test_l0_to_l1_sbm.py +14 -19
  35. roc/film/tests/test_l0_to_l1_surv.py +68 -41
  36. roc/film/tests/test_metadata.py +21 -20
  37. roc/film/tests/tests.py +743 -396
  38. roc/film/tools/__init__.py +5 -5
  39. roc/film/tools/dataset_tasks.py +34 -2
  40. roc/film/tools/file_helpers.py +390 -269
  41. roc/film/tools/l0.py +402 -324
  42. roc/film/tools/metadata.py +147 -127
  43. roc/film/tools/skeleton.py +12 -17
  44. roc/film/tools/tools.py +109 -92
  45. roc/film/tools/xlsx2skt.py +161 -139
  46. {roc_film-1.13.4.dist-info → roc_film-1.14.0.dist-info}/LICENSE +127 -125
  47. roc_film-1.14.0.dist-info/METADATA +60 -0
  48. roc_film-1.14.0.dist-info/RECORD +50 -0
  49. {roc_film-1.13.4.dist-info → roc_film-1.14.0.dist-info}/WHEEL +1 -1
  50. roc/film/tasks/l0_to_anc_bia_sweep_table.py +0 -348
  51. roc_film-1.13.4.dist-info/METADATA +0 -120
  52. roc_film-1.13.4.dist-info/RECORD +0 -48
@@ -5,18 +5,12 @@
5
5
  Test all l0_to_l1_bia* commands of the roc.film plugin.
6
6
  """
7
7
 
8
- import filecmp
9
- import os
10
8
  import tempfile
11
- from pprint import pformat
12
9
 
13
10
  import pytest
14
11
  import shutil
15
- import unittest.mock as mock
16
12
 
17
- from maser.tools.cdf.cdfcompare import cdf_compare
18
13
 
19
- from poppy.core.logger import logger
20
14
  from poppy.core.test import CommandTestCase
21
15
 
22
16
  from roc.film.tests.test_film import FilmTest
@@ -46,16 +40,18 @@ class TestL0ToL1Bia(CommandTestCase):
46
40
  # clear the downloaded files
47
41
  shutil.rmtree(self.tmp_dir_path)
48
42
 
49
- @pytest.mark.skip(reason='Not working')
50
- @pytest.mark.parametrize('idb_source,idb_version', [
51
- ('MIB', '20200131'),
52
- ('PALISADE', '4.3.5_MEB_PFM'),
53
- ])
43
+ @pytest.mark.skip(reason="Not working")
44
+ @pytest.mark.parametrize(
45
+ "idb_source,idb_version",
46
+ [
47
+ ("MIB", "20200131"),
48
+ ("PALISADE", "4.3.5_MEB_PFM"),
49
+ ],
50
+ )
54
51
  def test_l0_to_anc_bia_sweep_table(self, idb_source, idb_version):
55
- from poppy.core.conf import Settings
56
-
52
+ pass
57
53
  # Name of the command to test
58
- cmd = 'l0_to_anc_bia_sweep_table'
54
+ # cmd = "l0_to_anc_bia_sweep_table"
59
55
 
60
56
  # input_dir_path, inputs = self.get_inputs(cmd)
61
57
  # expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
@@ -5,25 +5,18 @@
5
5
  Test l0_to_l1_sbm command of the roc.film plugin.
6
6
  """
7
7
 
8
- import filecmp
9
- import os
10
8
  import tempfile
11
- from pprint import pformat
12
9
 
13
10
  import pytest
14
11
  import shutil
15
- import unittest.mock as mock
16
12
 
17
- from maser.tools.cdf.cdfcompare import cdf_compare
18
13
 
19
- from poppy.core.logger import logger
20
14
  from poppy.core.test import CommandTestCase
21
15
 
22
16
  from roc.film.tests.test_film import FilmTest
23
17
 
24
18
 
25
19
  class TestL0ToL1Sbm(CommandTestCase):
26
-
27
20
  film = FilmTest()
28
21
 
29
22
  def setup_method(self, method):
@@ -47,23 +40,25 @@ class TestL0ToL1Sbm(CommandTestCase):
47
40
  # clear the downloaded files
48
41
  shutil.rmtree(self.tmp_dir_path)
49
42
 
50
- @pytest.mark.skip(reason='Not working')
51
- @pytest.mark.parametrize('idb_source,idb_version', [
52
- ('MIB', '20200131'),
53
- ('PALISADE', '4.3.5_MEB_PFM'),
54
- ])
43
+ @pytest.mark.skip(reason="Not working")
44
+ @pytest.mark.parametrize(
45
+ "idb_source,idb_version",
46
+ [
47
+ ("MIB", "20200131"),
48
+ ("PALISADE", "4.3.5_MEB_PFM"),
49
+ ],
50
+ )
55
51
  def test_l0_to_l1_sbm(self, idb_source, idb_version):
56
- from poppy.core.conf import Settings
57
-
52
+ pass
58
53
  # Name of the command to test
59
- cmd = 'l0_to_l1_sbm'
54
+ # cmd = "l0_to_l1_sbm"
60
55
 
61
- #input_dir_path, inputs = self.get_inputs(cmd)
62
- #expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
63
- #ancillary_dir_path, ancillaries = self.get_ancillaries(cmd)
56
+ # input_dir_path, inputs = self.get_inputs(cmd)
57
+ # expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
58
+ # ancillary_dir_path, ancillaries = self.get_ancillaries(cmd)
64
59
 
65
60
  # extract spice kernels
66
- #spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
61
+ # spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
67
62
 
68
63
  # generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
69
64
  # os.makedirs(generated_output_dir_path, exist_ok=True)
@@ -7,14 +7,13 @@ Test l0_to_l1_surv command of the roc.film plugin.
7
7
 
8
8
  import filecmp
9
9
  import os
10
- import tempfile
11
10
  from pprint import pformat
12
11
 
13
12
  import pytest
14
13
  import shutil
15
14
  import unittest.mock as mock
16
15
 
17
- from maser.tools.cdf.cdfcompare import cdf_compare
16
+ from roc.film.tests.cdf_compare import cdf_compare
18
17
 
19
18
  from poppy.core.logger import logger
20
19
  from poppy.core.test import CommandTestCase
@@ -45,14 +44,17 @@ class TestL0ToL1surv(CommandTestCase):
45
44
  shutil.rmtree(self.output_dir_path)
46
45
 
47
46
  # @pytest.mark.skip()
48
- @pytest.mark.parametrize('idb_source,idb_version', [
49
- ('MIB', '20200131'),
50
- ])
47
+ @pytest.mark.parametrize(
48
+ "idb_source,idb_version",
49
+ [
50
+ ("MIB", "20200131"),
51
+ ],
52
+ )
51
53
  def test_l0_to_l1_surv(self, idb_source, idb_version):
52
54
  from poppy.core.conf import Settings
53
55
 
54
56
  # Name of the command to test
55
- cmd = 'l0_to_l1_surv'
57
+ cmd = "l0_to_l1_surv"
56
58
 
57
59
  # Retrieve data for current test
58
60
  test_data_path = FilmTest().get_test_data(cmd, extract=True)
@@ -62,51 +64,74 @@ class TestL0ToL1surv(CommandTestCase):
62
64
  # Initialize inputs and expected outputs
63
65
  input_dir_path, inputs = FilmTest.get_inputs(test_data_dir)
64
66
  if not inputs:
65
- raise FileNotFoundError(f'No input found in {test_data_dir}!')
66
- expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(test_data_dir)
67
+ raise FileNotFoundError(f"No input found in {test_data_dir}!")
68
+ expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(
69
+ test_data_dir
70
+ )
67
71
  if not expected_outputs:
68
- raise FileNotFoundError(f'No expected output found in {test_data_dir}!')
72
+ raise FileNotFoundError(f"No expected output found in {test_data_dir}!")
69
73
 
70
74
  # Initialize directory where files produced during test will be saved
71
- output_dir_path = os.path.join(test_data_dir, 'output')
75
+ output_dir_path = os.path.join(test_data_dir, "output")
72
76
  self.output_dir_path = output_dir_path
73
77
 
74
78
  # Check that SPICE kernels are present in ./data/spice_kernels folder
75
79
  spice_kernels_dir = FilmTest.get_spice_kernel_dir()
76
80
  if not os.path.isdir(spice_kernels_dir):
77
- raise FileNotFoundError(f'No SPICE kernel set found in {spice_kernels_dir}!')
81
+ raise FileNotFoundError(
82
+ f"No SPICE kernel set found in {spice_kernels_dir}!"
83
+ )
78
84
 
79
85
  # # initialize the main command
80
- command_to_test = ['pop',
81
- '-ll', 'INFO',
82
- 'film',
83
- '--force',
84
- '--idb-version', idb_version,
85
- '--idb-source', idb_source,
86
- '--cdag',
87
- cmd,
88
- os.path.join(input_dir_path, inputs[-1]),
89
- '--output-dir', output_dir_path,
90
- ]
86
+ command_to_test = [
87
+ "pop",
88
+ "-ll",
89
+ "INFO",
90
+ "film",
91
+ "--force",
92
+ "--idb-version",
93
+ idb_version,
94
+ "--idb-source",
95
+ idb_source,
96
+ "--cdag",
97
+ cmd,
98
+ os.path.join(input_dir_path, inputs[-1]),
99
+ "--output-dir",
100
+ output_dir_path,
101
+ ]
91
102
 
92
103
  # define the required plugins
93
- plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.dingo', 'roc.film']
104
+ plugin_list = [
105
+ "poppy.pop",
106
+ "roc.idb",
107
+ "roc.rpl",
108
+ "roc.rap",
109
+ "roc.dingo",
110
+ "roc.film",
111
+ ]
94
112
 
95
113
  # run the command
96
114
  # force the value of the plugin list
97
- with mock.patch.object(Settings, 'configure',
98
- autospec=True,
99
- side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
100
- self.run_command('pop db upgrade heads -ll INFO')
115
+ with mock.patch.object(
116
+ Settings,
117
+ "configure",
118
+ autospec=True,
119
+ side_effect=self.mock_configure_settings(
120
+ dictionary={"PLUGINS": plugin_list}
121
+ ),
122
+ ):
123
+ self.run_command("pop db upgrade heads -ll INFO")
101
124
  FilmTest.load_idb(self, idb_version)
102
- FilmTest.load_l0(self, input_dir_path, idb_version,
103
- idb_source=idb_source,
104
- )
125
+ FilmTest.load_l0(
126
+ self,
127
+ input_dir_path,
128
+ idb_version,
129
+ idb_source=idb_source,
130
+ )
105
131
  self.run_command(command_to_test)
106
132
 
107
133
  # compare directory content
108
- dirs_cmp = filecmp.dircmp(output_dir_path,
109
- expected_output_dir_path)
134
+ dirs_cmp = filecmp.dircmp(output_dir_path, expected_output_dir_path)
110
135
 
111
136
  dirs_cmp.report()
112
137
 
@@ -115,23 +140,25 @@ class TestL0ToL1surv(CommandTestCase):
115
140
 
116
141
  for filename in FilmTest.get_diff_files(dirs_cmp):
117
142
  # compare only cdf files with differences
118
- if filename.endswith('.cdf'):
143
+ if filename.endswith(".cdf"):
119
144
  # use cdf compare to compute the differences between expected output and the command output
120
145
  result = cdf_compare(
121
146
  os.path.join(output_dir_path, filename),
122
147
  os.path.join(expected_output_dir_path, filename),
123
148
  list_ignore_gatt=[
124
- 'File_ID',
125
- 'Generation_date',
126
- 'Pipeline_version',
127
- 'Pipeline_name',
128
- 'Software_version',
129
- 'IDB_version'
130
- ]
149
+ "File_ID",
150
+ "Generation_date",
151
+ "Pipeline_version",
152
+ "Pipeline_name",
153
+ "Software_version",
154
+ "IDB_version",
155
+ ],
131
156
  )
132
157
 
133
158
  # compare the difference dict with the expected one
134
159
  if result:
135
- logger.error(f'Differences between expected output and the command output: {pformat(result)}')
160
+ logger.error(
161
+ f"Differences between expected output and the command output: {pformat(result)}"
162
+ )
136
163
 
137
164
  assert result == {}
@@ -7,6 +7,7 @@ Tests module for the roc.film plugin.
7
7
 
8
8
  from poppy.core.test import TaskTestCase
9
9
 
10
+
10
11
  # Tests on roc.film.tasks.metadata methods
11
12
  class TestFilmMetadata(TaskTestCase):
12
13
  # create the pipeline
@@ -23,14 +24,14 @@ class TestFilmMetadata(TaskTestCase):
23
24
 
24
25
  # initialize meta dictionary with expected inputs
25
26
  meta = {
26
- 'Source_name': 'SOLO>Solar Orbiter',
27
- 'LEVEL': 'L1>Level 1 data processing',
28
- 'Descriptor': 'RPW-HFR-SURV> RPW HFR survey data',
29
- 'Datetime': '20220103',
30
- 'Data_version' : '01'
31
- }
27
+ "Source_name": "SOLO>Solar Orbiter",
28
+ "LEVEL": "L1>Level 1 data processing",
29
+ "Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
30
+ "Datetime": "20220103",
31
+ "Data_version": "01",
32
+ }
32
33
 
33
- assert build_file_basename(meta) == 'solo_L1_rpw-hfr-surv_20220103_V01'
34
+ assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
34
35
 
35
36
  def test_build_file_basename_with_logical(self):
36
37
  """
@@ -43,8 +44,8 @@ class TestFilmMetadata(TaskTestCase):
43
44
  # import method to test
44
45
  from roc.film.tools.file_helpers import build_file_basename
45
46
 
46
- meta = {'Logical_file_id': 'solo_L1_rpw-hfr-surv_20220103_V01'}
47
- assert build_file_basename(meta) == 'solo_L1_rpw-hfr-surv_20220103_V01'
47
+ meta = {"Logical_file_id": "solo_L1_rpw-hfr-surv_20220103_V01"}
48
+ assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
48
49
 
49
50
  # def test_generate_filepath(self):
50
51
 
@@ -58,7 +59,7 @@ class TestFilmMetadata(TaskTestCase):
58
59
  # import method to test
59
60
  from roc.film.tools.metadata import valid_data_version
60
61
 
61
- assert valid_data_version(1) == '01'
62
+ assert valid_data_version(1) == "01"
62
63
 
63
64
  def test_set_logical_file_id(self):
64
65
  """
@@ -72,13 +73,13 @@ class TestFilmMetadata(TaskTestCase):
72
73
 
73
74
  # initialize meta dictionary with expected inputs
74
75
  meta = {
75
- 'File_naming_convention': '<Source_name>_<LEVEL>_<Descriptor>_' \
76
- '<Datetime>_V<Data_version>',
77
- 'Source_name': 'SOLO>Solar Orbiter',
78
- 'LEVEL': 'L1>Level 1 data processing',
79
- 'Descriptor': 'RPW-HFR-SURV> RPW HFR survey data',
80
- 'Datetime': '20220103',
81
- 'Data_version': '01'
82
- }
83
-
84
- assert set_logical_file_id(meta) == 'solo_L1_rpw-hfr-surv_20220103_V01'
76
+ "File_naming_convention": "<Source_name>_<LEVEL>_<Descriptor>_"
77
+ "<Datetime>_V<Data_version>",
78
+ "Source_name": "SOLO>Solar Orbiter",
79
+ "LEVEL": "L1>Level 1 data processing",
80
+ "Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
81
+ "Datetime": "20220103",
82
+ "Data_version": "01",
83
+ }
84
+
85
+ assert set_logical_file_id(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"