deep-code 0.0.1.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deep_code/__init__.py +24 -0
- deep_code/cli/__init__.py +3 -0
- deep_code/cli/main.py +22 -0
- deep_code/cli/publish.py +26 -0
- deep_code/constants.py +16 -0
- deep_code/tests/tools/__init__.py +3 -0
- deep_code/tests/tools/test_publish.py +120 -0
- deep_code/tests/utils/__init__.py +3 -0
- deep_code/tests/utils/test_dataset_stac_generator.py +219 -0
- deep_code/tests/utils/test_github_automation.py +120 -0
- deep_code/tests/utils/test_ogc_api_record.py +113 -0
- deep_code/tests/utils/test_ogc_record_generator.py +63 -0
- deep_code/tests/utils/test_osc_extension.py +117 -0
- deep_code/tools/__init__.py +3 -0
- deep_code/tools/check.py +4 -0
- deep_code/tools/new.py +5 -0
- deep_code/tools/publish.py +233 -0
- deep_code/tools/register.py +0 -0
- deep_code/tools/setup_ci.py +1 -0
- deep_code/tools/test.py +2 -0
- deep_code/utils/__init__.py +3 -0
- deep_code/utils/dataset_stac_generator.py +426 -0
- deep_code/utils/github_automation.py +122 -0
- deep_code/utils/ogc_api_record.py +94 -0
- deep_code/utils/ogc_record_generator.py +54 -0
- deep_code/utils/osc_extension.py +201 -0
- deep_code/version.py +22 -0
- deep_code-0.0.1.dev0.dist-info/LICENSE +21 -0
- deep_code-0.0.1.dev0.dist-info/METADATA +166 -0
- deep_code-0.0.1.dev0.dist-info/RECORD +33 -0
- deep_code-0.0.1.dev0.dist-info/WHEEL +5 -0
- deep_code-0.0.1.dev0.dist-info/entry_points.txt +2 -0
- deep_code-0.0.1.dev0.dist-info/top_level.txt +1 -0
deep_code/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# The MIT License (MIT)
|
|
2
|
+
# Copyright (c) 2024 by DeepESDL and Brockmann Consult GmbH
|
|
3
|
+
#
|
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
5
|
+
# copy of this software and associated documentation files (the "Software"),
|
|
6
|
+
# to deal in the Software without restriction, including without limitation
|
|
7
|
+
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
8
|
+
# and/or sell copies of the Software, and to permit persons to whom the
|
|
9
|
+
# Software is furnished to do so, subject to the following conditions:
|
|
10
|
+
#
|
|
11
|
+
# The above copyright notice and this permission notice shall be included in
|
|
12
|
+
# all copies or substantial portions of the Software.
|
|
13
|
+
#
|
|
14
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
15
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
16
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NON INFRINGEMENT. IN NO EVENT SHALL THE
|
|
17
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
18
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
19
|
+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
20
|
+
# DEALINGS IN THE SOFTWARE.
|
|
21
|
+
|
|
22
|
+
from .version import version
|
|
23
|
+
|
|
24
|
+
__version__ = version
|
deep_code/cli/main.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2025 by Brockmann Consult GmbH
|
|
4
|
+
# Permissions are hereby granted under the terms of the MIT License:
|
|
5
|
+
# https://opensource.org/licenses/MIT.
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from deep_code.cli.publish import publish_dataset, publish_workflow
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@click.group()
|
|
13
|
+
def main():
|
|
14
|
+
"""Deep Code CLI."""
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
main.add_command(publish_dataset)
|
|
19
|
+
main.add_command(publish_workflow)
|
|
20
|
+
|
|
21
|
+
if __name__ == "__main__":
|
|
22
|
+
main()
|
deep_code/cli/publish.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2025 by Brockmann Consult GmbH
|
|
4
|
+
# Permissions are hereby granted under the terms of the MIT License:
|
|
5
|
+
# https://opensource.org/licenses/MIT.
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from deep_code.tools.publish import DatasetPublisher, WorkflowPublisher
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@click.command(name="publish-dataset")
|
|
13
|
+
@click.argument("dataset_config", type=click.Path(exists=True))
|
|
14
|
+
def publish_dataset(dataset_config):
|
|
15
|
+
"""Request publishing a dataset to the open science catalogue.
|
|
16
|
+
"""
|
|
17
|
+
publisher = DatasetPublisher()
|
|
18
|
+
publisher.publish_dataset(dataset_config_path=dataset_config)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@click.command(name="publish-workflow")
|
|
22
|
+
@click.argument("workflow_metadata", type=click.Path(exists=True))
|
|
23
|
+
def publish_workflow(workflow_metadata):
|
|
24
|
+
|
|
25
|
+
workflow_publisher = WorkflowPublisher()
|
|
26
|
+
workflow_publisher.publish_workflow(workflow_config_path=workflow_metadata)
|
deep_code/constants.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2024 by Brockmann Consult GmbH
|
|
4
|
+
# Permissions are hereby granted under the terms of the MIT License:
|
|
5
|
+
# https://opensource.org/licenses/MIT.
|
|
6
|
+
|
|
7
|
+
OSC_SCHEMA_URI = "https://stac-extensions.github.io/osc/v1.0.0-rc.3/schema.json"
|
|
8
|
+
CF_SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json"
|
|
9
|
+
OSC_REPO_OWNER = "ESA-EarthCODE"
|
|
10
|
+
OSC_REPO_NAME = "open-science-catalog-metadata-testing"
|
|
11
|
+
OSC_BRANCH_NAME = "add-new-collection"
|
|
12
|
+
DEFAULT_THEME_SCHEME = (
|
|
13
|
+
"https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords"
|
|
14
|
+
)
|
|
15
|
+
OGC_API_RECORD_SPEC = "http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core"
|
|
16
|
+
WF_BRANCH_NAME = "add-new-workflow-from-deepesdl"
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from unittest.mock import MagicMock, mock_open, patch
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from deep_code.tools.publish import DatasetPublisher
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TestDatasetPublisher:
|
|
9
|
+
@patch("deep_code.tools.publish.fsspec.open")
|
|
10
|
+
def test_init_missing_credentials(self, mock_fsspec_open):
|
|
11
|
+
mock_fsspec_open.return_value.__enter__.return_value = mock_open(
|
|
12
|
+
read_data="{}"
|
|
13
|
+
)()
|
|
14
|
+
|
|
15
|
+
with pytest.raises(
|
|
16
|
+
ValueError, match="GitHub credentials are missing in the `.gitaccess` file."
|
|
17
|
+
):
|
|
18
|
+
DatasetPublisher()
|
|
19
|
+
|
|
20
|
+
@patch("deep_code.tools.publish.fsspec.open")
|
|
21
|
+
def test_publish_dataset_missing_ids(self, mock_fsspec_open):
|
|
22
|
+
git_yaml_content = """
|
|
23
|
+
github-username: test-user
|
|
24
|
+
github-token: test-token
|
|
25
|
+
"""
|
|
26
|
+
dataset_yaml_content = """
|
|
27
|
+
collection-id: test-collection
|
|
28
|
+
"""
|
|
29
|
+
mock_fsspec_open.side_effect = [
|
|
30
|
+
mock_open(read_data=git_yaml_content)(),
|
|
31
|
+
mock_open(read_data=dataset_yaml_content)(),
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
publisher = DatasetPublisher()
|
|
35
|
+
|
|
36
|
+
with pytest.raises(
|
|
37
|
+
ValueError, match="Dataset ID or Collection ID missing in the config."
|
|
38
|
+
):
|
|
39
|
+
publisher.publish_dataset("/path/to/dataset-config.yaml")
|
|
40
|
+
|
|
41
|
+
@patch("deep_code.utils.github_automation.os.chdir")
|
|
42
|
+
@patch("deep_code.utils.github_automation.subprocess.run")
|
|
43
|
+
@patch("deep_code.utils.github_automation.os.path.expanduser", return_value="/tmp")
|
|
44
|
+
@patch("requests.post")
|
|
45
|
+
@patch("deep_code.utils.github_automation.GitHubAutomation")
|
|
46
|
+
@patch("deep_code.tools.publish.fsspec.open")
|
|
47
|
+
def test_publish_dataset_success(
|
|
48
|
+
self,
|
|
49
|
+
mock_fsspec_open,
|
|
50
|
+
mock_github_automation,
|
|
51
|
+
mock_requests_post,
|
|
52
|
+
mock_expanduser,
|
|
53
|
+
mock_subprocess_run,
|
|
54
|
+
mock_chdir,
|
|
55
|
+
):
|
|
56
|
+
# Mock the YAML reads
|
|
57
|
+
git_yaml_content = """
|
|
58
|
+
github-username: test-user
|
|
59
|
+
github-token: test-token
|
|
60
|
+
"""
|
|
61
|
+
dataset_yaml_content = """
|
|
62
|
+
dataset_id: test-dataset
|
|
63
|
+
collection_id: test-collection
|
|
64
|
+
documentation_link: http://example.com/doc
|
|
65
|
+
access_link: http://example.com/access
|
|
66
|
+
dataset_status: ongoing
|
|
67
|
+
dataset_region: Global
|
|
68
|
+
osc_theme: ["climate"]
|
|
69
|
+
cf_parameter: []
|
|
70
|
+
"""
|
|
71
|
+
mock_fsspec_open.side_effect = [
|
|
72
|
+
mock_open(read_data=git_yaml_content)(),
|
|
73
|
+
mock_open(read_data=dataset_yaml_content)(),
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
# Mock GitHubAutomation methods
|
|
77
|
+
mock_git = mock_github_automation.return_value
|
|
78
|
+
mock_git.fork_repository.return_value = None
|
|
79
|
+
mock_git.clone_repository.return_value = None
|
|
80
|
+
mock_git.create_branch.return_value = None
|
|
81
|
+
mock_git.add_file.return_value = None
|
|
82
|
+
mock_git.commit_and_push.return_value = None
|
|
83
|
+
mock_git.create_pull_request.return_value = "http://example.com/pr"
|
|
84
|
+
mock_git.clean_up.return_value = None
|
|
85
|
+
|
|
86
|
+
# Mock subprocess.run & os.chdir
|
|
87
|
+
mock_subprocess_run.return_value = None
|
|
88
|
+
mock_chdir.return_value = None
|
|
89
|
+
|
|
90
|
+
# Mock STAC generator
|
|
91
|
+
mock_collection = MagicMock()
|
|
92
|
+
mock_collection.to_dict.return_value = {
|
|
93
|
+
"type": "Collection",
|
|
94
|
+
"id": "test-collection",
|
|
95
|
+
"description": "A test STAC collection",
|
|
96
|
+
"extent": {
|
|
97
|
+
"spatial": {"bbox": [[-180.0, -90.0, 180.0, 90.0]]},
|
|
98
|
+
"temporal": {"interval": [["2023-01-01T00:00:00Z", None]]},
|
|
99
|
+
},
|
|
100
|
+
"links": [],
|
|
101
|
+
"stac_version": "1.0.0",
|
|
102
|
+
}
|
|
103
|
+
with patch("deep_code.tools.publish.OscDatasetStacGenerator") as mock_generator:
|
|
104
|
+
mock_generator.return_value.build_dataset_stac_collection.return_value = (
|
|
105
|
+
mock_collection
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# Instantiate & publish
|
|
109
|
+
publisher = DatasetPublisher()
|
|
110
|
+
publisher.publish_dataset("/fake/path/to/dataset-config.yaml")
|
|
111
|
+
|
|
112
|
+
# Assert that we called git clone with /tmp/temp_repo
|
|
113
|
+
# Because expanduser("~") is now patched to /tmp, the actual path is /tmp/temp_repo
|
|
114
|
+
auth_url = "https://test-user:test-token@github.com/test-user/open-science-catalog-metadata-testing.git"
|
|
115
|
+
mock_subprocess_run.assert_any_call(
|
|
116
|
+
["git", "clone", auth_url, "/tmp/temp_repo"], check=True
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# Also confirm we changed directories to /tmp/temp_repo
|
|
120
|
+
mock_chdir.assert_any_call("/tmp/temp_repo")
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import unittest
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from unittest.mock import MagicMock, patch
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
from pystac import Collection
|
|
8
|
+
from xarray import Dataset
|
|
9
|
+
|
|
10
|
+
from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestOSCProductSTACGenerator(unittest.TestCase):
|
|
14
|
+
@patch("deep_code.utils.dataset_stac_generator.new_data_store")
|
|
15
|
+
def setUp(self, mock_data_store):
|
|
16
|
+
"""Set up a mock dataset and generator."""
|
|
17
|
+
self.mock_dataset = Dataset(
|
|
18
|
+
coords={
|
|
19
|
+
"lon": ("lon", np.linspace(-180, 180, 10)),
|
|
20
|
+
"lat": ("lat", np.linspace(-90, 90, 5)),
|
|
21
|
+
"time": (
|
|
22
|
+
"time",
|
|
23
|
+
[
|
|
24
|
+
np.datetime64(datetime(2023, 1, 1), "ns"),
|
|
25
|
+
np.datetime64(datetime(2023, 1, 2), "ns"),
|
|
26
|
+
],
|
|
27
|
+
),
|
|
28
|
+
},
|
|
29
|
+
attrs={"description": "Mock dataset for testing.", "title": "Mock Dataset"},
|
|
30
|
+
data_vars={
|
|
31
|
+
"var1": (
|
|
32
|
+
("time", "lat", "lon"),
|
|
33
|
+
np.random.rand(2, 5, 10),
|
|
34
|
+
{
|
|
35
|
+
"description": "dummy",
|
|
36
|
+
"standard_name": "var1",
|
|
37
|
+
"gcmd_keyword_url": "https://dummy",
|
|
38
|
+
},
|
|
39
|
+
),
|
|
40
|
+
"var2": (
|
|
41
|
+
("time", "lat", "lon"),
|
|
42
|
+
np.random.rand(2, 5, 10),
|
|
43
|
+
{
|
|
44
|
+
"description": "dummy",
|
|
45
|
+
"standard_name": "var2",
|
|
46
|
+
"gcmd_keyword_url": "https://dummy",
|
|
47
|
+
},
|
|
48
|
+
),
|
|
49
|
+
},
|
|
50
|
+
)
|
|
51
|
+
mock_store = MagicMock()
|
|
52
|
+
mock_store.open_data.return_value = self.mock_dataset
|
|
53
|
+
mock_data_store.return_value = mock_store
|
|
54
|
+
|
|
55
|
+
self.generator = OscDatasetStacGenerator(
|
|
56
|
+
dataset_id="mock-dataset-id",
|
|
57
|
+
collection_id="mock-collection-id",
|
|
58
|
+
access_link="s3://mock-bucket/mock-dataset",
|
|
59
|
+
documentation_link="https://example.com/docs",
|
|
60
|
+
osc_status="ongoing",
|
|
61
|
+
osc_region="Global",
|
|
62
|
+
osc_themes=["climate", "environment"],
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def test_open_dataset(self):
|
|
66
|
+
"""Test if the dataset is opened correctly."""
|
|
67
|
+
self.assertIsInstance(self.generator.dataset, Dataset)
|
|
68
|
+
self.assertIn("lon", self.generator.dataset.coords)
|
|
69
|
+
self.assertIn("lat", self.generator.dataset.coords)
|
|
70
|
+
self.assertIn("time", self.generator.dataset.coords)
|
|
71
|
+
|
|
72
|
+
def test_get_spatial_extent(self):
|
|
73
|
+
"""Test spatial extent extraction."""
|
|
74
|
+
extent = self.generator._get_spatial_extent()
|
|
75
|
+
self.assertEqual(extent.bboxes[0], [-180.0, -90.0, 180.0, 90.0])
|
|
76
|
+
|
|
77
|
+
def test_get_temporal_extent(self):
|
|
78
|
+
"""Test temporal extent extraction."""
|
|
79
|
+
extent = self.generator._get_temporal_extent()
|
|
80
|
+
expected_intervals = [datetime(2023, 1, 1, 0, 0), datetime(2023, 1, 2, 0, 0)]
|
|
81
|
+
self.assertEqual(extent.intervals[0], expected_intervals)
|
|
82
|
+
|
|
83
|
+
def test_get_variables(self):
|
|
84
|
+
"""Test variable extraction."""
|
|
85
|
+
variables = self.generator.get_variable_ids()
|
|
86
|
+
self.assertEqual(variables, ["var1", "var2"])
|
|
87
|
+
|
|
88
|
+
def test_get_general_metadata(self):
|
|
89
|
+
"""Test general metadata extraction."""
|
|
90
|
+
metadata = self.generator._get_general_metadata()
|
|
91
|
+
self.assertEqual(metadata["description"], "Mock dataset for testing.")
|
|
92
|
+
|
|
93
|
+
@patch("pystac.Collection.add_link")
|
|
94
|
+
@patch("pystac.Collection.set_self_href")
|
|
95
|
+
def test_build_stac_collection(self, mock_set_self_href, mock_add_link):
|
|
96
|
+
"""Test STAC collection creation."""
|
|
97
|
+
collection = self.generator.build_dataset_stac_collection()
|
|
98
|
+
self.assertIsInstance(collection, Collection)
|
|
99
|
+
self.assertEqual(collection.id, "mock-collection-id")
|
|
100
|
+
self.assertEqual(collection.description, "Mock dataset for testing.")
|
|
101
|
+
self.assertEqual(
|
|
102
|
+
collection.extent.spatial.bboxes[0], [-180.0, -90.0, 180.0, 90.0]
|
|
103
|
+
)
|
|
104
|
+
self.assertEqual(
|
|
105
|
+
collection.extent.temporal.intervals[0],
|
|
106
|
+
[datetime(2023, 1, 1, 0, 0), datetime(2023, 1, 2, 0, 0)],
|
|
107
|
+
)
|
|
108
|
+
mock_set_self_href.assert_called_once()
|
|
109
|
+
mock_add_link.assert_called()
|
|
110
|
+
|
|
111
|
+
def test_invalid_spatial_extent(self):
|
|
112
|
+
"""Test spatial extent extraction with missing coordinates."""
|
|
113
|
+
self.generator.dataset = Dataset(coords={"x": [], "y": []})
|
|
114
|
+
with self.assertRaises(ValueError):
|
|
115
|
+
self.generator._get_spatial_extent()
|
|
116
|
+
|
|
117
|
+
def test_invalid_temporal_extent(self):
|
|
118
|
+
"""Test temporal extent extraction with missing time."""
|
|
119
|
+
self.generator.dataset = Dataset(coords={})
|
|
120
|
+
with self.assertRaises(ValueError):
|
|
121
|
+
self.generator._get_temporal_extent()
|
|
122
|
+
|
|
123
|
+
@patch("deep_code.utils.dataset_stac_generator.new_data_store")
|
|
124
|
+
@patch("deep_code.utils.dataset_stac_generator.logging.getLogger")
|
|
125
|
+
def test_open_dataset_success_public_store(self, mock_logger, mock_new_data_store):
|
|
126
|
+
"""Test dataset opening with the public store configuration."""
|
|
127
|
+
# Create a mock store and mock its `open_data` method
|
|
128
|
+
mock_store = MagicMock()
|
|
129
|
+
mock_new_data_store.return_value = mock_store
|
|
130
|
+
mock_store.open_data.return_value = self.mock_dataset
|
|
131
|
+
|
|
132
|
+
# Instantiate the generator (this will implicitly call _open_dataset)
|
|
133
|
+
generator = OscDatasetStacGenerator("mock-dataset-id", "mock-collection-id")
|
|
134
|
+
|
|
135
|
+
# Validate that the dataset is assigned correctly
|
|
136
|
+
self.assertEqual(generator.dataset, "mock_dataset")
|
|
137
|
+
|
|
138
|
+
# Validate that `new_data_store` was called once with the correct parameters
|
|
139
|
+
mock_new_data_store.assert_called_once_with(
|
|
140
|
+
"s3", root="deep-esdl-public", storage_options={"anon": True}
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
# Ensure `open_data` was called once on the returned store
|
|
144
|
+
mock_store.open_data.assert_called_once_with("mock-dataset-id")
|
|
145
|
+
|
|
146
|
+
# Validate logging behavior
|
|
147
|
+
mock_logger().info.assert_any_call(
|
|
148
|
+
"Attempting to open dataset with configuration: Public store"
|
|
149
|
+
)
|
|
150
|
+
mock_logger().info.assert_any_call(
|
|
151
|
+
"Successfully opened dataset with configuration: Public store"
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
@patch("deep_code.utils.dataset_stac_generator.new_data_store")
|
|
155
|
+
@patch("deep_code.utils.dataset_stac_generator.logging.getLogger")
|
|
156
|
+
def test_open_dataset_success_authenticated_store(
|
|
157
|
+
self, mock_logger, mock_new_data_store
|
|
158
|
+
):
|
|
159
|
+
"""Test dataset opening with the authenticated store configuration."""
|
|
160
|
+
# Simulate public store failure
|
|
161
|
+
mock_store = MagicMock()
|
|
162
|
+
mock_new_data_store.side_effect = [
|
|
163
|
+
Exception("Public store failure"),
|
|
164
|
+
# First call (public store) raises an exception
|
|
165
|
+
mock_store,
|
|
166
|
+
# Second call (authenticated store) returns a mock store
|
|
167
|
+
]
|
|
168
|
+
mock_store.open_data.return_value = self.mock_dataset
|
|
169
|
+
|
|
170
|
+
os.environ["S3_USER_STORAGE_BUCKET"] = "mock-bucket"
|
|
171
|
+
os.environ["S3_USER_STORAGE_KEY"] = "mock-key"
|
|
172
|
+
os.environ["S3_USER_STORAGE_SECRET"] = "mock-secret"
|
|
173
|
+
|
|
174
|
+
generator = OscDatasetStacGenerator("mock-dataset-id", "mock-collection-id")
|
|
175
|
+
|
|
176
|
+
# Validate that the dataset was successfully opened with the authenticated store
|
|
177
|
+
self.assertEqual(generator.dataset, "mock_dataset")
|
|
178
|
+
self.assertEqual(mock_new_data_store.call_count, 2)
|
|
179
|
+
|
|
180
|
+
# Validate calls to `new_data_store`
|
|
181
|
+
mock_new_data_store.assert_any_call(
|
|
182
|
+
"s3", root="deep-esdl-public", storage_options={"anon": True}
|
|
183
|
+
)
|
|
184
|
+
mock_new_data_store.assert_any_call(
|
|
185
|
+
"s3",
|
|
186
|
+
root="mock-bucket",
|
|
187
|
+
storage_options={"anon": False, "key": "mock-key", "secret": "mock-secret"},
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
# Validate logging calls
|
|
191
|
+
mock_logger().info.assert_any_call(
|
|
192
|
+
"Attempting to open dataset with configuration: Public store"
|
|
193
|
+
)
|
|
194
|
+
mock_logger().info.assert_any_call(
|
|
195
|
+
"Attempting to open dataset with configuration: Authenticated store"
|
|
196
|
+
)
|
|
197
|
+
mock_logger().info.assert_any_call(
|
|
198
|
+
"Successfully opened dataset with configuration: Authenticated store"
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
@patch("deep_code.utils.dataset_stac_generator.new_data_store")
|
|
202
|
+
@patch("deep_code.utils.dataset_stac_generator.logging.getLogger")
|
|
203
|
+
def test_open_dataset_failure(self, mock_logger, mock_new_data_store):
|
|
204
|
+
"""Test dataset opening failure with all configurations."""
|
|
205
|
+
# Simulate all store failures
|
|
206
|
+
mock_new_data_store.side_effect = Exception("Store failure")
|
|
207
|
+
os.environ["S3_USER_STORAGE_BUCKET"] = "mock-bucket"
|
|
208
|
+
os.environ["S3_USER_STORAGE_KEY"] = "mock-key"
|
|
209
|
+
os.environ["S3_USER_STORAGE_SECRET"] = "mock-secret"
|
|
210
|
+
|
|
211
|
+
with self.assertRaises(ValueError) as context:
|
|
212
|
+
OscDatasetStacGenerator("mock-dataset-id", "mock-collection-id")
|
|
213
|
+
|
|
214
|
+
self.assertIn(
|
|
215
|
+
"Failed to open Zarr dataset with ID mock-dataset-id",
|
|
216
|
+
str(context.exception),
|
|
217
|
+
)
|
|
218
|
+
self.assertIn("Public store, Authenticated store", str(context.exception))
|
|
219
|
+
self.assertEqual(mock_new_data_store.call_count, 2)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import unittest
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from unittest.mock import MagicMock, patch
|
|
5
|
+
|
|
6
|
+
from deep_code.utils.github_automation import GitHubAutomation
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TestGitHubAutomation(unittest.TestCase):
|
|
10
|
+
def setUp(self):
|
|
11
|
+
self.github = GitHubAutomation(
|
|
12
|
+
username="test-user",
|
|
13
|
+
token="test-token",
|
|
14
|
+
repo_owner="test-owner",
|
|
15
|
+
repo_name="test-repo",
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
@patch("requests.post")
|
|
19
|
+
def test_fork_repository(self, mock_post):
|
|
20
|
+
"""Test the fork_repository method."""
|
|
21
|
+
mock_response = MagicMock()
|
|
22
|
+
mock_response.raise_for_status.return_value = None
|
|
23
|
+
mock_post.return_value = mock_response
|
|
24
|
+
|
|
25
|
+
self.github.fork_repository()
|
|
26
|
+
|
|
27
|
+
mock_post.assert_called_once_with(
|
|
28
|
+
"https://api.github.com/repos/test-owner/test-repo/forks",
|
|
29
|
+
headers={"Authorization": "token test-token"},
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
@patch("subprocess.run")
|
|
33
|
+
@patch("os.chdir")
|
|
34
|
+
def test_clone_repository(self, mock_chdir, mock_run):
|
|
35
|
+
"""Test the clone_repository method."""
|
|
36
|
+
self.github.clone_repository()
|
|
37
|
+
|
|
38
|
+
mock_run.assert_called_once_with(
|
|
39
|
+
["git", "clone", self.github.fork_repo_url, self.github.local_clone_dir],
|
|
40
|
+
check=True,
|
|
41
|
+
)
|
|
42
|
+
mock_chdir.assert_called_once_with(self.github.local_clone_dir)
|
|
43
|
+
|
|
44
|
+
@patch("subprocess.run")
|
|
45
|
+
def test_create_branch(self, mock_run):
|
|
46
|
+
"""Test the create_branch method."""
|
|
47
|
+
branch_name = "test-branch"
|
|
48
|
+
self.github.create_branch(branch_name)
|
|
49
|
+
|
|
50
|
+
mock_run.assert_called_once_with(
|
|
51
|
+
["git", "checkout", "-b", branch_name], check=True
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
@patch("subprocess.run")
|
|
55
|
+
@patch("builtins.open", new_callable=unittest.mock.mock_open)
|
|
56
|
+
@patch("pathlib.Path.mkdir")
|
|
57
|
+
def test_add_file(self, mock_mkdir, mock_open, mock_run):
|
|
58
|
+
"""Test the add_file method."""
|
|
59
|
+
file_path = "test-dir/test-file.json"
|
|
60
|
+
content = {"key": "value"}
|
|
61
|
+
|
|
62
|
+
self.github.add_file(file_path, content)
|
|
63
|
+
|
|
64
|
+
mock_mkdir.assert_called_once_with(parents=True, exist_ok=True)
|
|
65
|
+
mock_open.assert_called_once_with(
|
|
66
|
+
Path(self.github.local_clone_dir) / file_path, "w"
|
|
67
|
+
)
|
|
68
|
+
mock_open().write.assert_called_once_with(json.dumps(content, indent=2))
|
|
69
|
+
mock_run.assert_called_once_with(
|
|
70
|
+
["git", "add", str(Path(self.github.local_clone_dir) / file_path)],
|
|
71
|
+
check=True,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
@patch("subprocess.run")
|
|
75
|
+
def test_commit_and_push(self, mock_run):
|
|
76
|
+
"""Test the commit_and_push method."""
|
|
77
|
+
branch_name = "test-branch"
|
|
78
|
+
commit_message = "Test commit message"
|
|
79
|
+
|
|
80
|
+
self.github.commit_and_push(branch_name, commit_message)
|
|
81
|
+
|
|
82
|
+
mock_run.assert_any_call(["git", "commit", "-m", commit_message], check=True)
|
|
83
|
+
mock_run.assert_any_call(
|
|
84
|
+
["git", "push", "-u", "origin", branch_name], check=True
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
@patch("requests.post")
|
|
88
|
+
def test_create_pull_request(self, mock_post):
|
|
89
|
+
"""Test the create_pull_request method."""
|
|
90
|
+
branch_name = "test-branch"
|
|
91
|
+
pr_title = "Test PR"
|
|
92
|
+
pr_body = "This is a test PR"
|
|
93
|
+
base_branch = "main"
|
|
94
|
+
|
|
95
|
+
mock_response = MagicMock()
|
|
96
|
+
mock_response.json.return_value = {"html_url": "https://github.com/test-pr"}
|
|
97
|
+
mock_response.raise_for_status.return_value = None
|
|
98
|
+
mock_post.return_value = mock_response
|
|
99
|
+
|
|
100
|
+
self.github.create_pull_request(branch_name, pr_title, pr_body, base_branch)
|
|
101
|
+
|
|
102
|
+
mock_post.assert_called_once_with(
|
|
103
|
+
"https://api.github.com/repos/test-owner/test-repo/pulls",
|
|
104
|
+
headers={"Authorization": "token test-token"},
|
|
105
|
+
json={
|
|
106
|
+
"title": pr_title,
|
|
107
|
+
"head": f"test-user:{branch_name}",
|
|
108
|
+
"base": base_branch,
|
|
109
|
+
"body": pr_body,
|
|
110
|
+
},
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
@patch("subprocess.run")
|
|
114
|
+
@patch("os.chdir")
|
|
115
|
+
def test_clean_up(self, mock_chdir, mock_run):
|
|
116
|
+
"""Test the clean_up method."""
|
|
117
|
+
self.github.clean_up()
|
|
118
|
+
|
|
119
|
+
mock_chdir.assert_called_once_with("..")
|
|
120
|
+
mock_run.assert_called_once_with(["rm", "-rf", self.github.local_clone_dir])
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
|
|
3
|
+
from deep_code.constants import OGC_API_RECORD_SPEC
|
|
4
|
+
from deep_code.utils.ogc_api_record import (
|
|
5
|
+
Contact,
|
|
6
|
+
JupyterKernelInfo,
|
|
7
|
+
OgcRecord,
|
|
8
|
+
RecordProperties,
|
|
9
|
+
Theme,
|
|
10
|
+
ThemeConcept,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TestClasses(unittest.TestCase):
|
|
15
|
+
def test_contact_initialization(self):
|
|
16
|
+
contact = Contact(
|
|
17
|
+
name="Person-X",
|
|
18
|
+
organization="Organization X",
|
|
19
|
+
position="Researcher",
|
|
20
|
+
links=[{"url": "http://example.com", "type": "website"}],
|
|
21
|
+
contactInstructions="Contact via email",
|
|
22
|
+
roles=["developer", "reviewer"],
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
self.assertEqual(contact.name, "Person-X")
|
|
26
|
+
self.assertEqual(contact.organization, "Organization X")
|
|
27
|
+
self.assertEqual(contact.position, "Researcher")
|
|
28
|
+
self.assertEqual(len(contact.links), 1)
|
|
29
|
+
self.assertEqual(contact.contactInstructions, "Contact via email")
|
|
30
|
+
self.assertIn("developer", contact.roles)
|
|
31
|
+
|
|
32
|
+
def test_theme_concept_initialization(self):
|
|
33
|
+
theme_concept = ThemeConcept(id="concept1")
|
|
34
|
+
self.assertEqual(theme_concept.id, "concept1")
|
|
35
|
+
|
|
36
|
+
def test_theme_initialization(self):
|
|
37
|
+
theme_concepts = [ThemeConcept(id="concept1"), ThemeConcept(id="concept2")]
|
|
38
|
+
theme = Theme(concepts=theme_concepts, scheme="http://example.com/scheme")
|
|
39
|
+
|
|
40
|
+
self.assertEqual(len(theme.concepts), 2)
|
|
41
|
+
self.assertEqual(theme.scheme, "http://example.com/scheme")
|
|
42
|
+
|
|
43
|
+
def test_jupyter_kernel_info_initialization(self):
|
|
44
|
+
kernel_info = JupyterKernelInfo(
|
|
45
|
+
name="Python", python_version=3.9, env_file="env.yml"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
self.assertEqual(kernel_info.name, "Python")
|
|
49
|
+
self.assertEqual(kernel_info.python_version, 3.9)
|
|
50
|
+
self.assertEqual(kernel_info.env_file, "env.yml")
|
|
51
|
+
|
|
52
|
+
def test_record_properties_initialization(self):
|
|
53
|
+
kernel_info = JupyterKernelInfo(
|
|
54
|
+
name="Python", python_version=3.9, env_file="env.yml"
|
|
55
|
+
)
|
|
56
|
+
contacts = [Contact(name="Jane Doe", organization="Org Y")]
|
|
57
|
+
themes = [Theme(concepts=[ThemeConcept(id="concept1")], scheme="scheme1")]
|
|
58
|
+
|
|
59
|
+
record_properties = RecordProperties(
|
|
60
|
+
created="2025-01-01",
|
|
61
|
+
type="dataset",
|
|
62
|
+
title="Sample Dataset",
|
|
63
|
+
description="A sample dataset",
|
|
64
|
+
jupyter_kernel_info=kernel_info,
|
|
65
|
+
updated="2025-01-02",
|
|
66
|
+
contacts=contacts,
|
|
67
|
+
themes=themes,
|
|
68
|
+
keywords=["sample", "test"],
|
|
69
|
+
formats=[{"format": "JSON"}],
|
|
70
|
+
license="CC-BY",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
self.assertEqual(record_properties.created, "2025-01-01")
|
|
74
|
+
self.assertEqual(record_properties.updated, "2025-01-02")
|
|
75
|
+
self.assertEqual(record_properties.type, "dataset")
|
|
76
|
+
self.assertEqual(record_properties.title, "Sample Dataset")
|
|
77
|
+
self.assertEqual(record_properties.description, "A sample dataset")
|
|
78
|
+
self.assertEqual(record_properties.jupyter_kernel_info.name, "Python")
|
|
79
|
+
self.assertEqual(len(record_properties.contacts), 1)
|
|
80
|
+
self.assertEqual(len(record_properties.themes), 1)
|
|
81
|
+
self.assertIn("sample", record_properties.keywords)
|
|
82
|
+
self.assertEqual(record_properties.license, "CC-BY")
|
|
83
|
+
|
|
84
|
+
def test_ogc_record_initialization(self):
|
|
85
|
+
kernel_info = JupyterKernelInfo(
|
|
86
|
+
name="Python", python_version=3.9, env_file="env.yml"
|
|
87
|
+
)
|
|
88
|
+
properties = RecordProperties(
|
|
89
|
+
created="2025-01-01",
|
|
90
|
+
type="dataset",
|
|
91
|
+
title="Sample Dataset",
|
|
92
|
+
description="A sample dataset",
|
|
93
|
+
jupyter_kernel_info=kernel_info,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
ogc_record = OgcRecord(
|
|
97
|
+
id="record1",
|
|
98
|
+
type="Feature",
|
|
99
|
+
time={"start": "2025-01-01T00:00:00Z", "end": "2025-01-02T00:00:00Z"},
|
|
100
|
+
properties=properties,
|
|
101
|
+
links=[{"href": "http://example.com", "rel": "self"}],
|
|
102
|
+
linkTemplates=[{"template": "http://example.com/{id}"}],
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
self.assertEqual(ogc_record.id, "record1")
|
|
106
|
+
self.assertEqual(ogc_record.type, "Feature")
|
|
107
|
+
self.assertEqual(ogc_record.time["start"], "2025-01-01T00:00:00Z")
|
|
108
|
+
self.assertEqual(ogc_record.properties.title, "Sample Dataset")
|
|
109
|
+
self.assertEqual(len(ogc_record.links), 1)
|
|
110
|
+
self.assertEqual(
|
|
111
|
+
ogc_record.linkTemplates[0]["template"], "http://example.com/{id}"
|
|
112
|
+
)
|
|
113
|
+
self.assertEqual(ogc_record.conformsTo[0], OGC_API_RECORD_SPEC)
|