roc-film 1.13.5__py3-none-any.whl → 1.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- roc/__init__.py +2 -1
- roc/film/__init__.py +2 -2
- roc/film/commands.py +372 -323
- roc/film/config/__init__.py +0 -1
- roc/film/constants.py +101 -65
- roc/film/descriptor.json +127 -96
- roc/film/exceptions.py +28 -27
- roc/film/tasks/__init__.py +16 -16
- roc/film/tasks/cat_solo_hk.py +86 -74
- roc/film/tasks/cdf_postpro.py +438 -309
- roc/film/tasks/check_dds.py +39 -45
- roc/film/tasks/db_to_anc_bia_sweep_table.py +381 -0
- roc/film/tasks/dds_to_l0.py +232 -180
- roc/film/tasks/export_solo_coord.py +147 -0
- roc/film/tasks/file_handler.py +91 -75
- roc/film/tasks/l0_to_hk.py +117 -103
- roc/film/tasks/l0_to_l1_bia_current.py +38 -30
- roc/film/tasks/l0_to_l1_bia_sweep.py +417 -329
- roc/film/tasks/l0_to_l1_sbm.py +250 -208
- roc/film/tasks/l0_to_l1_surv.py +185 -130
- roc/film/tasks/make_daily_tm.py +40 -37
- roc/film/tasks/merge_tcreport.py +77 -71
- roc/film/tasks/merge_tmraw.py +101 -88
- roc/film/tasks/parse_dds_xml.py +21 -20
- roc/film/tasks/set_l0_utc.py +51 -49
- roc/film/tests/cdf_compare.py +565 -0
- roc/film/tests/hdf5_compare.py +84 -62
- roc/film/tests/test_dds_to_l0.py +93 -51
- roc/film/tests/test_dds_to_tc.py +8 -11
- roc/film/tests/test_dds_to_tm.py +8 -10
- roc/film/tests/test_film.py +161 -116
- roc/film/tests/test_l0_to_hk.py +64 -36
- roc/film/tests/test_l0_to_l1_bia.py +10 -14
- roc/film/tests/test_l0_to_l1_sbm.py +14 -19
- roc/film/tests/test_l0_to_l1_surv.py +68 -41
- roc/film/tests/test_metadata.py +21 -20
- roc/film/tests/tests.py +743 -396
- roc/film/tools/__init__.py +5 -5
- roc/film/tools/dataset_tasks.py +34 -2
- roc/film/tools/file_helpers.py +390 -269
- roc/film/tools/l0.py +402 -324
- roc/film/tools/metadata.py +147 -127
- roc/film/tools/skeleton.py +12 -17
- roc/film/tools/tools.py +109 -92
- roc/film/tools/xlsx2skt.py +161 -139
- {roc_film-1.13.5.dist-info → roc_film-1.14.0.dist-info}/LICENSE +127 -125
- roc_film-1.14.0.dist-info/METADATA +60 -0
- roc_film-1.14.0.dist-info/RECORD +50 -0
- {roc_film-1.13.5.dist-info → roc_film-1.14.0.dist-info}/WHEEL +1 -1
- roc/film/tasks/l0_to_anc_bia_sweep_table.py +0 -348
- roc_film-1.13.5.dist-info/METADATA +0 -120
- roc_film-1.13.5.dist-info/RECORD +0 -48
@@ -5,18 +5,12 @@
|
|
5
5
|
Test all l0_to_l1_bia* commands of the roc.film plugin.
|
6
6
|
"""
|
7
7
|
|
8
|
-
import filecmp
|
9
|
-
import os
|
10
8
|
import tempfile
|
11
|
-
from pprint import pformat
|
12
9
|
|
13
10
|
import pytest
|
14
11
|
import shutil
|
15
|
-
import unittest.mock as mock
|
16
12
|
|
17
|
-
from maser.tools.cdf.cdfcompare import cdf_compare
|
18
13
|
|
19
|
-
from poppy.core.logger import logger
|
20
14
|
from poppy.core.test import CommandTestCase
|
21
15
|
|
22
16
|
from roc.film.tests.test_film import FilmTest
|
@@ -46,16 +40,18 @@ class TestL0ToL1Bia(CommandTestCase):
|
|
46
40
|
# clear the downloaded files
|
47
41
|
shutil.rmtree(self.tmp_dir_path)
|
48
42
|
|
49
|
-
@pytest.mark.skip(reason=
|
50
|
-
@pytest.mark.parametrize(
|
51
|
-
|
52
|
-
|
53
|
-
|
43
|
+
@pytest.mark.skip(reason="Not working")
|
44
|
+
@pytest.mark.parametrize(
|
45
|
+
"idb_source,idb_version",
|
46
|
+
[
|
47
|
+
("MIB", "20200131"),
|
48
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
49
|
+
],
|
50
|
+
)
|
54
51
|
def test_l0_to_anc_bia_sweep_table(self, idb_source, idb_version):
|
55
|
-
|
56
|
-
|
52
|
+
pass
|
57
53
|
# Name of the command to test
|
58
|
-
cmd =
|
54
|
+
# cmd = "l0_to_anc_bia_sweep_table"
|
59
55
|
|
60
56
|
# input_dir_path, inputs = self.get_inputs(cmd)
|
61
57
|
# expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
@@ -5,25 +5,18 @@
|
|
5
5
|
Test l0_to_l1_sbm command of the roc.film plugin.
|
6
6
|
"""
|
7
7
|
|
8
|
-
import filecmp
|
9
|
-
import os
|
10
8
|
import tempfile
|
11
|
-
from pprint import pformat
|
12
9
|
|
13
10
|
import pytest
|
14
11
|
import shutil
|
15
|
-
import unittest.mock as mock
|
16
12
|
|
17
|
-
from maser.tools.cdf.cdfcompare import cdf_compare
|
18
13
|
|
19
|
-
from poppy.core.logger import logger
|
20
14
|
from poppy.core.test import CommandTestCase
|
21
15
|
|
22
16
|
from roc.film.tests.test_film import FilmTest
|
23
17
|
|
24
18
|
|
25
19
|
class TestL0ToL1Sbm(CommandTestCase):
|
26
|
-
|
27
20
|
film = FilmTest()
|
28
21
|
|
29
22
|
def setup_method(self, method):
|
@@ -47,23 +40,25 @@ class TestL0ToL1Sbm(CommandTestCase):
|
|
47
40
|
# clear the downloaded files
|
48
41
|
shutil.rmtree(self.tmp_dir_path)
|
49
42
|
|
50
|
-
@pytest.mark.skip(reason=
|
51
|
-
@pytest.mark.parametrize(
|
52
|
-
|
53
|
-
|
54
|
-
|
43
|
+
@pytest.mark.skip(reason="Not working")
|
44
|
+
@pytest.mark.parametrize(
|
45
|
+
"idb_source,idb_version",
|
46
|
+
[
|
47
|
+
("MIB", "20200131"),
|
48
|
+
("PALISADE", "4.3.5_MEB_PFM"),
|
49
|
+
],
|
50
|
+
)
|
55
51
|
def test_l0_to_l1_sbm(self, idb_source, idb_version):
|
56
|
-
|
57
|
-
|
52
|
+
pass
|
58
53
|
# Name of the command to test
|
59
|
-
cmd =
|
54
|
+
# cmd = "l0_to_l1_sbm"
|
60
55
|
|
61
|
-
#input_dir_path, inputs = self.get_inputs(cmd)
|
62
|
-
#expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
63
|
-
#ancillary_dir_path, ancillaries = self.get_ancillaries(cmd)
|
56
|
+
# input_dir_path, inputs = self.get_inputs(cmd)
|
57
|
+
# expected_output_dir_path, expected_outputs = self.get_expected_outputs(cmd)
|
58
|
+
# ancillary_dir_path, ancillaries = self.get_ancillaries(cmd)
|
64
59
|
|
65
60
|
# extract spice kernels
|
66
|
-
#spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
61
|
+
# spice_kernel_dir_path = self.unzip_kernels(ancillaries[0])
|
67
62
|
|
68
63
|
# generated_output_dir_path = os.path.join(self.tmp_dir_path, 'generated_output')
|
69
64
|
# os.makedirs(generated_output_dir_path, exist_ok=True)
|
@@ -7,14 +7,13 @@ Test l0_to_l1_surv command of the roc.film plugin.
|
|
7
7
|
|
8
8
|
import filecmp
|
9
9
|
import os
|
10
|
-
import tempfile
|
11
10
|
from pprint import pformat
|
12
11
|
|
13
12
|
import pytest
|
14
13
|
import shutil
|
15
14
|
import unittest.mock as mock
|
16
15
|
|
17
|
-
from
|
16
|
+
from roc.film.tests.cdf_compare import cdf_compare
|
18
17
|
|
19
18
|
from poppy.core.logger import logger
|
20
19
|
from poppy.core.test import CommandTestCase
|
@@ -45,14 +44,17 @@ class TestL0ToL1surv(CommandTestCase):
|
|
45
44
|
shutil.rmtree(self.output_dir_path)
|
46
45
|
|
47
46
|
# @pytest.mark.skip()
|
48
|
-
@pytest.mark.parametrize(
|
49
|
-
|
50
|
-
|
47
|
+
@pytest.mark.parametrize(
|
48
|
+
"idb_source,idb_version",
|
49
|
+
[
|
50
|
+
("MIB", "20200131"),
|
51
|
+
],
|
52
|
+
)
|
51
53
|
def test_l0_to_l1_surv(self, idb_source, idb_version):
|
52
54
|
from poppy.core.conf import Settings
|
53
55
|
|
54
56
|
# Name of the command to test
|
55
|
-
cmd =
|
57
|
+
cmd = "l0_to_l1_surv"
|
56
58
|
|
57
59
|
# Retrieve data for current test
|
58
60
|
test_data_path = FilmTest().get_test_data(cmd, extract=True)
|
@@ -62,51 +64,74 @@ class TestL0ToL1surv(CommandTestCase):
|
|
62
64
|
# Initialize inputs and expected outputs
|
63
65
|
input_dir_path, inputs = FilmTest.get_inputs(test_data_dir)
|
64
66
|
if not inputs:
|
65
|
-
raise FileNotFoundError(f
|
66
|
-
expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(
|
67
|
+
raise FileNotFoundError(f"No input found in {test_data_dir}!")
|
68
|
+
expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(
|
69
|
+
test_data_dir
|
70
|
+
)
|
67
71
|
if not expected_outputs:
|
68
|
-
raise FileNotFoundError(f
|
72
|
+
raise FileNotFoundError(f"No expected output found in {test_data_dir}!")
|
69
73
|
|
70
74
|
# Initialize directory where files produced during test will be saved
|
71
|
-
output_dir_path = os.path.join(test_data_dir,
|
75
|
+
output_dir_path = os.path.join(test_data_dir, "output")
|
72
76
|
self.output_dir_path = output_dir_path
|
73
77
|
|
74
78
|
# Check that SPICE kernels are present in ./data/spice_kernels folder
|
75
79
|
spice_kernels_dir = FilmTest.get_spice_kernel_dir()
|
76
80
|
if not os.path.isdir(spice_kernels_dir):
|
77
|
-
raise FileNotFoundError(
|
81
|
+
raise FileNotFoundError(
|
82
|
+
f"No SPICE kernel set found in {spice_kernels_dir}!"
|
83
|
+
)
|
78
84
|
|
79
85
|
# # initialize the main command
|
80
|
-
command_to_test = [
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
86
|
+
command_to_test = [
|
87
|
+
"pop",
|
88
|
+
"-ll",
|
89
|
+
"INFO",
|
90
|
+
"film",
|
91
|
+
"--force",
|
92
|
+
"--idb-version",
|
93
|
+
idb_version,
|
94
|
+
"--idb-source",
|
95
|
+
idb_source,
|
96
|
+
"--cdag",
|
97
|
+
cmd,
|
98
|
+
os.path.join(input_dir_path, inputs[-1]),
|
99
|
+
"--output-dir",
|
100
|
+
output_dir_path,
|
101
|
+
]
|
91
102
|
|
92
103
|
# define the required plugins
|
93
|
-
plugin_list = [
|
104
|
+
plugin_list = [
|
105
|
+
"poppy.pop",
|
106
|
+
"roc.idb",
|
107
|
+
"roc.rpl",
|
108
|
+
"roc.rap",
|
109
|
+
"roc.dingo",
|
110
|
+
"roc.film",
|
111
|
+
]
|
94
112
|
|
95
113
|
# run the command
|
96
114
|
# force the value of the plugin list
|
97
|
-
with mock.patch.object(
|
98
|
-
|
99
|
-
|
100
|
-
|
115
|
+
with mock.patch.object(
|
116
|
+
Settings,
|
117
|
+
"configure",
|
118
|
+
autospec=True,
|
119
|
+
side_effect=self.mock_configure_settings(
|
120
|
+
dictionary={"PLUGINS": plugin_list}
|
121
|
+
),
|
122
|
+
):
|
123
|
+
self.run_command("pop db upgrade heads -ll INFO")
|
101
124
|
FilmTest.load_idb(self, idb_version)
|
102
|
-
FilmTest.load_l0(
|
103
|
-
|
104
|
-
|
125
|
+
FilmTest.load_l0(
|
126
|
+
self,
|
127
|
+
input_dir_path,
|
128
|
+
idb_version,
|
129
|
+
idb_source=idb_source,
|
130
|
+
)
|
105
131
|
self.run_command(command_to_test)
|
106
132
|
|
107
133
|
# compare directory content
|
108
|
-
dirs_cmp = filecmp.dircmp(output_dir_path,
|
109
|
-
expected_output_dir_path)
|
134
|
+
dirs_cmp = filecmp.dircmp(output_dir_path, expected_output_dir_path)
|
110
135
|
|
111
136
|
dirs_cmp.report()
|
112
137
|
|
@@ -115,23 +140,25 @@ class TestL0ToL1surv(CommandTestCase):
|
|
115
140
|
|
116
141
|
for filename in FilmTest.get_diff_files(dirs_cmp):
|
117
142
|
# compare only cdf files with differences
|
118
|
-
if filename.endswith(
|
143
|
+
if filename.endswith(".cdf"):
|
119
144
|
# use cdf compare to compute the differences between expected output and the command output
|
120
145
|
result = cdf_compare(
|
121
146
|
os.path.join(output_dir_path, filename),
|
122
147
|
os.path.join(expected_output_dir_path, filename),
|
123
148
|
list_ignore_gatt=[
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
]
|
149
|
+
"File_ID",
|
150
|
+
"Generation_date",
|
151
|
+
"Pipeline_version",
|
152
|
+
"Pipeline_name",
|
153
|
+
"Software_version",
|
154
|
+
"IDB_version",
|
155
|
+
],
|
131
156
|
)
|
132
157
|
|
133
158
|
# compare the difference dict with the expected one
|
134
159
|
if result:
|
135
|
-
logger.error(
|
160
|
+
logger.error(
|
161
|
+
f"Differences between expected output and the command output: {pformat(result)}"
|
162
|
+
)
|
136
163
|
|
137
164
|
assert result == {}
|
roc/film/tests/test_metadata.py
CHANGED
@@ -7,6 +7,7 @@ Tests module for the roc.film plugin.
|
|
7
7
|
|
8
8
|
from poppy.core.test import TaskTestCase
|
9
9
|
|
10
|
+
|
10
11
|
# Tests on roc.film.tasks.metadata methods
|
11
12
|
class TestFilmMetadata(TaskTestCase):
|
12
13
|
# create the pipeline
|
@@ -23,14 +24,14 @@ class TestFilmMetadata(TaskTestCase):
|
|
23
24
|
|
24
25
|
# initialize meta dictionary with expected inputs
|
25
26
|
meta = {
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
27
|
+
"Source_name": "SOLO>Solar Orbiter",
|
28
|
+
"LEVEL": "L1>Level 1 data processing",
|
29
|
+
"Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
|
30
|
+
"Datetime": "20220103",
|
31
|
+
"Data_version": "01",
|
32
|
+
}
|
32
33
|
|
33
|
-
assert build_file_basename(meta) ==
|
34
|
+
assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
|
34
35
|
|
35
36
|
def test_build_file_basename_with_logical(self):
|
36
37
|
"""
|
@@ -43,8 +44,8 @@ class TestFilmMetadata(TaskTestCase):
|
|
43
44
|
# import method to test
|
44
45
|
from roc.film.tools.file_helpers import build_file_basename
|
45
46
|
|
46
|
-
meta = {
|
47
|
-
assert build_file_basename(meta) ==
|
47
|
+
meta = {"Logical_file_id": "solo_L1_rpw-hfr-surv_20220103_V01"}
|
48
|
+
assert build_file_basename(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
|
48
49
|
|
49
50
|
# def test_generate_filepath(self):
|
50
51
|
|
@@ -58,7 +59,7 @@ class TestFilmMetadata(TaskTestCase):
|
|
58
59
|
# import method to test
|
59
60
|
from roc.film.tools.metadata import valid_data_version
|
60
61
|
|
61
|
-
assert valid_data_version(1) ==
|
62
|
+
assert valid_data_version(1) == "01"
|
62
63
|
|
63
64
|
def test_set_logical_file_id(self):
|
64
65
|
"""
|
@@ -72,13 +73,13 @@ class TestFilmMetadata(TaskTestCase):
|
|
72
73
|
|
73
74
|
# initialize meta dictionary with expected inputs
|
74
75
|
meta = {
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
assert set_logical_file_id(meta) ==
|
76
|
+
"File_naming_convention": "<Source_name>_<LEVEL>_<Descriptor>_"
|
77
|
+
"<Datetime>_V<Data_version>",
|
78
|
+
"Source_name": "SOLO>Solar Orbiter",
|
79
|
+
"LEVEL": "L1>Level 1 data processing",
|
80
|
+
"Descriptor": "RPW-HFR-SURV> RPW HFR survey data",
|
81
|
+
"Datetime": "20220103",
|
82
|
+
"Data_version": "01",
|
83
|
+
}
|
84
|
+
|
85
|
+
assert set_logical_file_id(meta) == "solo_L1_rpw-hfr-surv_20220103_V01"
|