deep-code 0.1.4.dev1__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/PKG-INFO +2 -1
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/constants.py +10 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/tools/test_publish.py +58 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_dataset_stac_generator.py +3 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_ogc_api_record.py +16 -3
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tools/new.py +10 -3
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tools/publish.py +27 -9
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/custom_xrlint_rules.py +1 -1
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/dataset_stac_generator.py +17 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/ogc_api_record.py +124 -9
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/version.py +1 -1
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code.egg-info/PKG-INFO +2 -1
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code.egg-info/requires.txt +1 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/pyproject.toml +1 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/LICENSE +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/README.md +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/__init__.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/cli/__init__.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/cli/generate_config.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/cli/main.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/cli/publish.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/tools/__init__.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/__init__.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_custom_xrlint_rules.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_github_automation.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_helper.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_ogc_record_generator.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_osc_extension.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tools/__init__.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tools/lint.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tools/register.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tools/setup_ci.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tools/test.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/__init__.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/github_automation.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/helper.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/ogc_record_generator.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/utils/osc_extension.py +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code.egg-info/SOURCES.txt +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code.egg-info/dependency_links.txt +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code.egg-info/entry_points.txt +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code.egg-info/top_level.txt +0 -0
- {deep_code-0.1.4.dev1 → deep_code-0.1.5}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deep_code
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: deepesdl earthcode integration utility tool
|
|
5
5
|
Author-email: Tejas Morbagal Harish <tejas.morbagalharish@brockmann-consult.de>
|
|
6
6
|
License: MIT
|
|
@@ -14,6 +14,7 @@ License-File: LICENSE
|
|
|
14
14
|
Requires-Dist: click
|
|
15
15
|
Requires-Dist: fsspec
|
|
16
16
|
Requires-Dist: jsonschema
|
|
17
|
+
Requires-Dist: jsonpickle
|
|
17
18
|
Requires-Dist: requests
|
|
18
19
|
Requires-Dist: pandas
|
|
19
20
|
Requires-Dist: pystac
|
|
@@ -29,3 +29,13 @@ WORKFLOW_BASE_CATALOG_SELF_HREF = (
|
|
|
29
29
|
".json"
|
|
30
30
|
)
|
|
31
31
|
PROJECT_COLLECTION_NAME = "deep-earth-system-data-lab"
|
|
32
|
+
DEEPESDL_GIT_PULL_BASE = (
|
|
33
|
+
"https://deep.earthsystemdatalab.net/hub/user-redirect/git-pull"
|
|
34
|
+
)
|
|
35
|
+
APPLICATION_TYPE_JUPYTER_SPEC = (
|
|
36
|
+
"https://raw.githubusercontent.com/EOEPCA/metadata"
|
|
37
|
+
"-profile/refs/heads/1.0/schemas/application-type-jupyter-notebook"
|
|
38
|
+
)
|
|
39
|
+
APPLICATION_STAC_EXTENSION_SPEC = (
|
|
40
|
+
"https://stac-extensions.github.io/application/v0.1.0/schema.json"
|
|
41
|
+
)
|
|
@@ -4,10 +4,12 @@ import unittest
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from unittest.mock import MagicMock, mock_open, patch
|
|
6
6
|
|
|
7
|
+
import pytest
|
|
7
8
|
import yaml
|
|
8
9
|
from pystac import Catalog
|
|
9
10
|
|
|
10
11
|
from deep_code.tools.publish import Publisher
|
|
12
|
+
from deep_code.utils.ogc_api_record import LinksBuilder
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class TestPublisher(unittest.TestCase):
|
|
@@ -107,3 +109,59 @@ class TestPublisher(unittest.TestCase):
|
|
|
107
109
|
# Assertions
|
|
108
110
|
self.assertEqual(self.publisher.dataset_config, dataset_config)
|
|
109
111
|
self.assertEqual(self.publisher.workflow_config, workflow_config)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class TestParseGithubNotebookUrl:
|
|
115
|
+
@pytest.mark.parametrize(
|
|
116
|
+
"url,repo_url,repo_name,branch,file_path",
|
|
117
|
+
[
|
|
118
|
+
(
|
|
119
|
+
"https://github.com/deepesdl/cube-gen/blob/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb",
|
|
120
|
+
"https://github.com/deepesdl/cube-gen",
|
|
121
|
+
"cube-gen",
|
|
122
|
+
"main",
|
|
123
|
+
"Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb",
|
|
124
|
+
),
|
|
125
|
+
(
|
|
126
|
+
"https://github.com/deepesdl/cube-gen/tree/release-1.0/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb",
|
|
127
|
+
"https://github.com/deepesdl/cube-gen",
|
|
128
|
+
"cube-gen",
|
|
129
|
+
"release-1.0",
|
|
130
|
+
"Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb",
|
|
131
|
+
),
|
|
132
|
+
(
|
|
133
|
+
"https://raw.githubusercontent.com/deepesdl/cube-gen/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb",
|
|
134
|
+
"https://github.com/deepesdl/cube-gen",
|
|
135
|
+
"cube-gen",
|
|
136
|
+
"main",
|
|
137
|
+
"Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb",
|
|
138
|
+
),
|
|
139
|
+
],
|
|
140
|
+
)
|
|
141
|
+
def test_valid_urls(self, url, repo_url, repo_name, branch, file_path):
|
|
142
|
+
got_repo_url, got_repo_name, got_branch, got_file_path = LinksBuilder._parse_github_notebook_url(
|
|
143
|
+
url
|
|
144
|
+
)
|
|
145
|
+
assert got_repo_url == repo_url
|
|
146
|
+
assert got_repo_name == repo_name
|
|
147
|
+
assert got_branch == branch
|
|
148
|
+
assert got_file_path == file_path
|
|
149
|
+
|
|
150
|
+
def test_invalid_domain(self):
|
|
151
|
+
url = "https://gitlab.com/deepesdl/cube-gen/-/blob/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb"
|
|
152
|
+
with pytest.raises(ValueError) as e:
|
|
153
|
+
LinksBuilder._parse_github_notebook_url(url)
|
|
154
|
+
assert "Only GitHub URLs are supported" in str(e.value)
|
|
155
|
+
|
|
156
|
+
def test_unexpected_github_format_missing_blob_or_tree(self):
|
|
157
|
+
# Missing the "blob" or "tree" segment
|
|
158
|
+
url = "https://github.com/deepesdl/cube-gen/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb"
|
|
159
|
+
with pytest.raises(ValueError) as e:
|
|
160
|
+
LinksBuilder._parse_github_notebook_url(url)
|
|
161
|
+
assert "Unexpected GitHub URL format" in str(e.value)
|
|
162
|
+
|
|
163
|
+
def test_unexpected_raw_format_too_short(self):
|
|
164
|
+
url = "https://raw.githubusercontent.com/deepesdl/cube-gen/main"
|
|
165
|
+
with pytest.raises(ValueError) as e:
|
|
166
|
+
LinksBuilder._parse_github_notebook_url(url)
|
|
167
|
+
assert "Unexpected raw.githubusercontent URL format" in str(e.value)
|
{deep_code-0.1.4.dev1 → deep_code-0.1.5}/deep_code/tests/utils/test_dataset_stac_generator.py
RENAMED
|
@@ -65,8 +65,11 @@ class TestOSCProductSTACGenerator(unittest.TestCase):
|
|
|
65
65
|
self.generator = OscDatasetStacGenerator(
|
|
66
66
|
dataset_id="mock-dataset-id",
|
|
67
67
|
collection_id="mock-collection-id",
|
|
68
|
+
workflow_id="dummy",
|
|
69
|
+
workflow_title="test",
|
|
68
70
|
access_link="s3://mock-bucket/mock-dataset",
|
|
69
71
|
documentation_link="https://example.com/docs",
|
|
72
|
+
license_type="proprietary",
|
|
70
73
|
osc_status="ongoing",
|
|
71
74
|
osc_region="Global",
|
|
72
75
|
osc_themes=["climate", "environment"],
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import unittest
|
|
2
2
|
|
|
3
|
-
from deep_code.constants import
|
|
3
|
+
from deep_code.constants import (
|
|
4
|
+
APPLICATION_STAC_EXTENSION_SPEC,
|
|
5
|
+
APPLICATION_TYPE_JUPYTER_SPEC,
|
|
6
|
+
OGC_API_RECORD_SPEC,
|
|
7
|
+
)
|
|
4
8
|
from deep_code.utils.ogc_api_record import (
|
|
5
9
|
Contact,
|
|
6
10
|
ExperimentAsOgcRecord,
|
|
@@ -136,7 +140,9 @@ class TestRecordProperties(unittest.TestCase):
|
|
|
136
140
|
|
|
137
141
|
class TestLinksBuilder(unittest.TestCase):
|
|
138
142
|
def test_build_theme_links_for_records(self):
|
|
139
|
-
links_builder = LinksBuilder(
|
|
143
|
+
links_builder = LinksBuilder(
|
|
144
|
+
themes=["climate", "ocean"], jupyter_kernel_info={}
|
|
145
|
+
)
|
|
140
146
|
theme_links = links_builder.build_theme_links_for_records()
|
|
141
147
|
|
|
142
148
|
expected_links = [
|
|
@@ -201,7 +207,14 @@ class TestWorkflowAsOgcRecord(unittest.TestCase):
|
|
|
201
207
|
workflow_record.jupyter_notebook_url, "https://example.com/notebook.ipynb"
|
|
202
208
|
)
|
|
203
209
|
self.assertEqual(workflow_record.properties, record_properties)
|
|
204
|
-
self.assertEqual(
|
|
210
|
+
self.assertEqual(
|
|
211
|
+
workflow_record.conformsTo,
|
|
212
|
+
[
|
|
213
|
+
OGC_API_RECORD_SPEC,
|
|
214
|
+
APPLICATION_TYPE_JUPYTER_SPEC,
|
|
215
|
+
APPLICATION_STAC_EXTENSION_SPEC,
|
|
216
|
+
],
|
|
217
|
+
)
|
|
205
218
|
self.assertEqual(workflow_record.links[0]["rel"], "root")
|
|
206
219
|
self.assertEqual(workflow_record.links[-1]["rel"], "self")
|
|
207
220
|
|
|
@@ -20,7 +20,11 @@ class TemplateGenerator:
|
|
|
20
20
|
"title": "[Human-readable title of the workflow]",
|
|
21
21
|
"description": "[A concise summary of what the workflow does]",
|
|
22
22
|
"keywords": ["[KEYWORD1]", "[KEYWORD2]"],
|
|
23
|
-
"themes": [
|
|
23
|
+
"themes": [
|
|
24
|
+
"[Thematic area(s) of focus (e.g. land, ocean, atmosphere)]",
|
|
25
|
+
"[THEME1]",
|
|
26
|
+
"[THEME2]",
|
|
27
|
+
],
|
|
24
28
|
"license": "[License type (e.g. MIT, Apache-2.0, CC-BY-4.0, proprietary)]",
|
|
25
29
|
"jupyter_kernel_info": {
|
|
26
30
|
"name": "[Name of the execution environment or notebook kernel]",
|
|
@@ -61,8 +65,11 @@ class TemplateGenerator:
|
|
|
61
65
|
template = {
|
|
62
66
|
"dataset_id": "[The name of the dataset object within your S3 bucket].zarr",
|
|
63
67
|
"collection_id": "[A unique identifier for the dataset collection]",
|
|
64
|
-
"osc_themes": [
|
|
65
|
-
|
|
68
|
+
"osc_themes": [
|
|
69
|
+
"[Oceans]",
|
|
70
|
+
"[Open Science theme (choose from "
|
|
71
|
+
"https://opensciencedata.esa.int/themes/catalog)",
|
|
72
|
+
],
|
|
66
73
|
"osc_region": "[Geographical coverage, e.g. 'global']",
|
|
67
74
|
"dataset_status": "[Status of the dataset: 'ongoing', 'completed', or 'planned']",
|
|
68
75
|
"documentation_link": "[Link to relevant documentation, publication, or handbook]",
|
|
@@ -4,12 +4,12 @@
|
|
|
4
4
|
# https://opensource.org/licenses/MIT.
|
|
5
5
|
|
|
6
6
|
import copy
|
|
7
|
-
import json
|
|
8
7
|
import logging
|
|
9
8
|
from datetime import datetime
|
|
10
9
|
from pathlib import Path
|
|
11
10
|
|
|
12
11
|
import fsspec
|
|
12
|
+
import jsonpickle
|
|
13
13
|
import yaml
|
|
14
14
|
from pystac import Catalog, Link
|
|
15
15
|
|
|
@@ -22,7 +22,6 @@ from deep_code.constants import (
|
|
|
22
22
|
)
|
|
23
23
|
from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator
|
|
24
24
|
from deep_code.utils.github_automation import GitHubAutomation
|
|
25
|
-
from deep_code.utils.helper import serialize
|
|
26
25
|
from deep_code.utils.ogc_api_record import (
|
|
27
26
|
ExperimentAsOgcRecord,
|
|
28
27
|
LinksBuilder,
|
|
@@ -130,6 +129,7 @@ class Publisher:
|
|
|
130
129
|
self._read_config_files()
|
|
131
130
|
self.collection_id = self.dataset_config.get("collection_id")
|
|
132
131
|
self.workflow_title = self.workflow_config.get("properties", {}).get("title")
|
|
132
|
+
self.workflow_id = self.workflow_config.get("workflow_id")
|
|
133
133
|
|
|
134
134
|
if not self.collection_id:
|
|
135
135
|
raise ValueError("collection_id is missing in dataset config.")
|
|
@@ -151,11 +151,12 @@ class Publisher:
|
|
|
151
151
|
# Create the directory if it doesn't exist
|
|
152
152
|
Path(file_path).parent.mkdir(parents=True, exist_ok=True)
|
|
153
153
|
try:
|
|
154
|
-
|
|
154
|
+
# unpicklable=False -> plain JSON (drops type metadata); cycles are resolved.
|
|
155
|
+
json_content = jsonpickle.encode(data, unpicklable=False, indent=2)
|
|
155
156
|
except TypeError as e:
|
|
156
157
|
raise RuntimeError(f"JSON serialization failed: {e}")
|
|
157
158
|
|
|
158
|
-
with open(file_path, "w") as f:
|
|
159
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
159
160
|
f.write(json_content)
|
|
160
161
|
|
|
161
162
|
def _update_and_add_to_file_dict(
|
|
@@ -217,6 +218,7 @@ class Publisher:
|
|
|
217
218
|
osc_region = self.dataset_config.get("osc_region")
|
|
218
219
|
osc_themes = self.dataset_config.get("osc_themes")
|
|
219
220
|
cf_params = self.dataset_config.get("cf_parameter")
|
|
221
|
+
license_type = self.dataset_config.get("license_type")
|
|
220
222
|
|
|
221
223
|
if not dataset_id or not self.collection_id:
|
|
222
224
|
raise ValueError("Dataset ID or Collection ID missing in the config.")
|
|
@@ -226,6 +228,9 @@ class Publisher:
|
|
|
226
228
|
generator = OscDatasetStacGenerator(
|
|
227
229
|
dataset_id=dataset_id,
|
|
228
230
|
collection_id=self.collection_id,
|
|
231
|
+
workflow_id=self.workflow_id,
|
|
232
|
+
workflow_title=self.workflow_title,
|
|
233
|
+
license_type=license_type,
|
|
229
234
|
documentation_link=documentation_link,
|
|
230
235
|
access_link=access_link,
|
|
231
236
|
osc_status=dataset_status,
|
|
@@ -310,7 +315,7 @@ class Publisher:
|
|
|
310
315
|
|
|
311
316
|
return base_catalog
|
|
312
317
|
|
|
313
|
-
def
|
|
318
|
+
def generate_workflow_experiment_records(self, write_to_file: bool = False) -> None:
|
|
314
319
|
"""prepare workflow and experiment as ogc api record to publish it to the
|
|
315
320
|
specified GitHub repository."""
|
|
316
321
|
workflow_id = self._normalize_name(self.workflow_config.get("workflow_id"))
|
|
@@ -328,16 +333,23 @@ class Publisher:
|
|
|
328
333
|
wf_record_properties = rg.build_record_properties(properties_list, contacts)
|
|
329
334
|
# make a copy for experiment record
|
|
330
335
|
exp_record_properties = copy.deepcopy(wf_record_properties)
|
|
336
|
+
jupyter_kernel_info = wf_record_properties.jupyter_kernel_info.to_dict()
|
|
331
337
|
|
|
332
|
-
link_builder = LinksBuilder(osc_themes)
|
|
338
|
+
link_builder = LinksBuilder(osc_themes, jupyter_kernel_info)
|
|
333
339
|
theme_links = link_builder.build_theme_links_for_records()
|
|
340
|
+
application_link = link_builder.build_link_to_jnb(
|
|
341
|
+
self.workflow_title, jupyter_notebook_url
|
|
342
|
+
)
|
|
343
|
+
jnb_open_link = link_builder.make_related_link_for_opening_jnb_from_github(
|
|
344
|
+
jupyter_notebook_url=jupyter_notebook_url
|
|
345
|
+
)
|
|
334
346
|
|
|
335
347
|
workflow_record = WorkflowAsOgcRecord(
|
|
336
348
|
id=workflow_id,
|
|
337
349
|
type="Feature",
|
|
338
350
|
title=self.workflow_title,
|
|
339
351
|
properties=wf_record_properties,
|
|
340
|
-
links=links + theme_links,
|
|
352
|
+
links=links + theme_links + application_link + jnb_open_link,
|
|
341
353
|
jupyter_notebook_url=jupyter_notebook_url,
|
|
342
354
|
themes=osc_themes,
|
|
343
355
|
)
|
|
@@ -347,6 +359,7 @@ class Publisher:
|
|
|
347
359
|
del workflow_dict["jupyter_notebook_url"]
|
|
348
360
|
if "osc_workflow" in workflow_dict["properties"]:
|
|
349
361
|
del workflow_dict["properties"]["osc_workflow"]
|
|
362
|
+
# add workflow record to file_dict
|
|
350
363
|
wf_file_path = f"workflows/{workflow_id}/record.json"
|
|
351
364
|
file_dict = {wf_file_path: workflow_dict}
|
|
352
365
|
|
|
@@ -354,6 +367,8 @@ class Publisher:
|
|
|
354
367
|
exp_record_properties.type = "experiment"
|
|
355
368
|
exp_record_properties.osc_workflow = workflow_id
|
|
356
369
|
|
|
370
|
+
dataset_link = link_builder.build_link_to_dataset(self.collection_id)
|
|
371
|
+
|
|
357
372
|
experiment_record = ExperimentAsOgcRecord(
|
|
358
373
|
id=workflow_id,
|
|
359
374
|
title=self.workflow_title,
|
|
@@ -361,7 +376,7 @@ class Publisher:
|
|
|
361
376
|
jupyter_notebook_url=jupyter_notebook_url,
|
|
362
377
|
collection_id=self.collection_id,
|
|
363
378
|
properties=exp_record_properties,
|
|
364
|
-
links=links + theme_links,
|
|
379
|
+
links=links + theme_links + dataset_link,
|
|
365
380
|
)
|
|
366
381
|
# Convert to dictionary and cleanup
|
|
367
382
|
experiment_dict = experiment_record.to_dict()
|
|
@@ -371,6 +386,7 @@ class Publisher:
|
|
|
371
386
|
del experiment_dict["collection_id"]
|
|
372
387
|
if "osc:project" in experiment_dict["properties"]:
|
|
373
388
|
del experiment_dict["properties"]["osc:project"]
|
|
389
|
+
# add experiment record to file_dict
|
|
374
390
|
exp_file_path = f"experiments/{workflow_id}/record.json"
|
|
375
391
|
file_dict[exp_file_path] = experiment_dict
|
|
376
392
|
|
|
@@ -397,7 +413,9 @@ class Publisher:
|
|
|
397
413
|
"""Publish both dataset and workflow/experiment in a single PR."""
|
|
398
414
|
# Get file dictionaries from both methods
|
|
399
415
|
dataset_files = self.publish_dataset(write_to_file=write_to_file)
|
|
400
|
-
workflow_files = self.
|
|
416
|
+
workflow_files = self.generate_workflow_experiment_records(
|
|
417
|
+
write_to_file=write_to_file
|
|
418
|
+
)
|
|
401
419
|
|
|
402
420
|
# Combine the file dictionaries
|
|
403
421
|
combined_files = {**dataset_files, **workflow_files}
|
|
@@ -39,6 +39,9 @@ class OscDatasetStacGenerator:
|
|
|
39
39
|
self,
|
|
40
40
|
dataset_id: str,
|
|
41
41
|
collection_id: str,
|
|
42
|
+
workflow_id: str,
|
|
43
|
+
workflow_title: str,
|
|
44
|
+
license_type: str,
|
|
42
45
|
access_link: str | None = None,
|
|
43
46
|
documentation_link: str | None = None,
|
|
44
47
|
osc_status: str = "ongoing",
|
|
@@ -49,6 +52,9 @@ class OscDatasetStacGenerator:
|
|
|
49
52
|
):
|
|
50
53
|
self.dataset_id = dataset_id
|
|
51
54
|
self.collection_id = collection_id
|
|
55
|
+
self.workflow_id = workflow_id
|
|
56
|
+
self.workflow_title = workflow_title
|
|
57
|
+
self.license_type = license_type
|
|
52
58
|
self.access_link = access_link or f"s3://deep-esdl-public/{dataset_id}"
|
|
53
59
|
self.documentation_link = documentation_link
|
|
54
60
|
self.osc_status = osc_status
|
|
@@ -478,6 +484,17 @@ class OscDatasetStacGenerator:
|
|
|
478
484
|
)
|
|
479
485
|
)
|
|
480
486
|
|
|
487
|
+
collection.add_link(
|
|
488
|
+
Link(
|
|
489
|
+
rel="related",
|
|
490
|
+
target=f"../../experiments/{self.workflow_id}/record.json",
|
|
491
|
+
media_type="application/json",
|
|
492
|
+
title=f"Experiment: {self.workflow_title}",
|
|
493
|
+
)
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
collection.license = self.license_type
|
|
497
|
+
|
|
481
498
|
# Validate OSC extension fields
|
|
482
499
|
try:
|
|
483
500
|
osc_extension.validate_extension()
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
from typing import Any, Optional
|
|
1
|
+
from typing import Any, Optional, Tuple, List, Dict
|
|
2
|
+
from urllib.parse import quote, urlencode, urlparse
|
|
2
3
|
|
|
3
4
|
from xrlint.util.constructible import MappingConstructible
|
|
4
5
|
from xrlint.util.serializable import JsonSerializable, JsonValue
|
|
5
6
|
|
|
6
7
|
from deep_code.constants import (
|
|
8
|
+
APPLICATION_STAC_EXTENSION_SPEC,
|
|
9
|
+
APPLICATION_TYPE_JUPYTER_SPEC,
|
|
7
10
|
BASE_URL_OSC,
|
|
11
|
+
DEEPESDL_GIT_PULL_BASE,
|
|
8
12
|
OGC_API_RECORD_SPEC,
|
|
9
13
|
PROJECT_COLLECTION_NAME,
|
|
10
14
|
)
|
|
@@ -86,12 +90,16 @@ class RecordProperties(MappingConstructible["RecordProperties"], JsonSerializabl
|
|
|
86
90
|
if self.osc_project is not None:
|
|
87
91
|
data["osc:project"] = self.osc_project
|
|
88
92
|
del data["osc_project"]
|
|
93
|
+
data["application:type"] = "jupyter-notebook"
|
|
94
|
+
data["application:container"] = ("true",)
|
|
95
|
+
data["application:language"] = ("Python",)
|
|
89
96
|
return data
|
|
90
97
|
|
|
91
98
|
|
|
92
99
|
class LinksBuilder:
|
|
93
|
-
def __init__(self, themes: list[str]):
|
|
100
|
+
def __init__(self, themes: list[str], jupyter_kernel_info: dict[str]):
|
|
94
101
|
self.themes = themes
|
|
102
|
+
self.jupyter_kernel_info = jupyter_kernel_info
|
|
95
103
|
self.theme_links = []
|
|
96
104
|
|
|
97
105
|
def build_theme_links_for_records(self):
|
|
@@ -117,6 +125,99 @@ class LinksBuilder:
|
|
|
117
125
|
}
|
|
118
126
|
]
|
|
119
127
|
|
|
128
|
+
def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, Any]]:
|
|
129
|
+
return [
|
|
130
|
+
{
|
|
131
|
+
"rel": "application",
|
|
132
|
+
"title": f"Jupyter Notebook: {workflow_title}",
|
|
133
|
+
"href": jupyter_nb_url,
|
|
134
|
+
"type": "application/x-ipynb+json",
|
|
135
|
+
"application:type": "jupyter-notebook",
|
|
136
|
+
"application:container": "true",
|
|
137
|
+
"application:language": "Python",
|
|
138
|
+
"jupyter:kernel": {
|
|
139
|
+
"name": self.jupyter_kernel_info["name"],
|
|
140
|
+
"pythonVersion": self.jupyter_kernel_info["python_version"],
|
|
141
|
+
"envFile": self.jupyter_kernel_info["env_file"],
|
|
142
|
+
},
|
|
143
|
+
}
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
@staticmethod
|
|
147
|
+
def _parse_github_notebook_url(url: str) -> Tuple[str, str, str, str]:
|
|
148
|
+
"""
|
|
149
|
+
Returns (repo_url, repo_name, branch, file_path_in_repo) from a GitHub URL.
|
|
150
|
+
|
|
151
|
+
Supports:
|
|
152
|
+
- https://github.com/<owner>/<repo>/blob/<branch>/<path/to/notebook.ipynb>
|
|
153
|
+
- https://raw.githubusercontent.com/<owner>/<repo>/<branch>/<path/to/notebook.ipynb>
|
|
154
|
+
"""
|
|
155
|
+
p = urlparse(url)
|
|
156
|
+
parts = p.path.strip("/").split("/")
|
|
157
|
+
|
|
158
|
+
if p.netloc == "github.com":
|
|
159
|
+
if len(parts) >= 5 and parts[2] in ("blob", "tree"):
|
|
160
|
+
owner, repo, _blob_or_tree, branch = parts[:4]
|
|
161
|
+
file_path = "/".join(parts[4:])
|
|
162
|
+
else:
|
|
163
|
+
raise ValueError(f"Unexpected GitHub URL format: {url}")
|
|
164
|
+
repo_url = f"https://github.com/{owner}/{repo}"
|
|
165
|
+
repo_name = repo
|
|
166
|
+
|
|
167
|
+
elif p.netloc == "raw.githubusercontent.com":
|
|
168
|
+
if len(parts) >= 4:
|
|
169
|
+
owner, repo, branch = parts[:3]
|
|
170
|
+
file_path = "/".join(parts[3:])
|
|
171
|
+
else:
|
|
172
|
+
raise ValueError(f"Unexpected raw.githubusercontent URL format: {url}")
|
|
173
|
+
repo_url = f"https://github.com/{owner}/{repo}"
|
|
174
|
+
repo_name = repo
|
|
175
|
+
|
|
176
|
+
else:
|
|
177
|
+
raise ValueError(f"Only GitHub URLs are supported: {url}")
|
|
178
|
+
|
|
179
|
+
return repo_url, repo_name, branch, file_path
|
|
180
|
+
|
|
181
|
+
def build_deepesdl_notebook_href_from_github(
|
|
182
|
+
self,
|
|
183
|
+
jupyter_notebook_url: str,
|
|
184
|
+
base_redirect: str = DEEPESDL_GIT_PULL_BASE,
|
|
185
|
+
branch_override: str | None = None,
|
|
186
|
+
) -> str:
|
|
187
|
+
"""
|
|
188
|
+
Build DeepESDL git-pull redirect from a full GitHub notebook URL.
|
|
189
|
+
{base}?repo=<repo_url>&urlpath=lab/tree/<repo_name>/<path>&branch=<branch>
|
|
190
|
+
"""
|
|
191
|
+
repo_url, repo_name, branch, file_path = self._parse_github_notebook_url(
|
|
192
|
+
jupyter_notebook_url
|
|
193
|
+
)
|
|
194
|
+
if branch_override:
|
|
195
|
+
branch = branch_override
|
|
196
|
+
|
|
197
|
+
params = {
|
|
198
|
+
"repo": repo_url,
|
|
199
|
+
"urlpath": f"lab/tree/{repo_name}/{file_path}",
|
|
200
|
+
"branch": branch,
|
|
201
|
+
}
|
|
202
|
+
return f"{base_redirect}?{urlencode(params, quote_via=quote)}"
|
|
203
|
+
|
|
204
|
+
def make_related_link_for_opening_jnb_from_github(
|
|
205
|
+
self,
|
|
206
|
+
jupyter_notebook_url: str,
|
|
207
|
+
title: str = "Open notebook on the DeepESDL platform",
|
|
208
|
+
branch_override: str | None = None,
|
|
209
|
+
) -> dict[str, str]:
|
|
210
|
+
return [
|
|
211
|
+
{
|
|
212
|
+
"rel": "related",
|
|
213
|
+
"href": self.build_deepesdl_notebook_href_from_github(
|
|
214
|
+
jupyter_notebook_url, branch_override=branch_override
|
|
215
|
+
),
|
|
216
|
+
"type": "text/html",
|
|
217
|
+
"title": title,
|
|
218
|
+
}
|
|
219
|
+
]
|
|
220
|
+
|
|
120
221
|
|
|
121
222
|
class WorkflowAsOgcRecord(MappingConstructible["OgcRecord"], JsonSerializable):
|
|
122
223
|
def __init__(
|
|
@@ -133,7 +234,11 @@ class WorkflowAsOgcRecord(MappingConstructible["OgcRecord"], JsonSerializable):
|
|
|
133
234
|
themes: Optional[Any] = None,
|
|
134
235
|
):
|
|
135
236
|
if conformsTo is None:
|
|
136
|
-
conformsTo = [
|
|
237
|
+
conformsTo = [
|
|
238
|
+
OGC_API_RECORD_SPEC,
|
|
239
|
+
APPLICATION_TYPE_JUPYTER_SPEC,
|
|
240
|
+
APPLICATION_STAC_EXTENSION_SPEC,
|
|
241
|
+
]
|
|
137
242
|
self.id = id
|
|
138
243
|
self.type = type
|
|
139
244
|
self.title = title
|
|
@@ -172,6 +277,14 @@ class WorkflowAsOgcRecord(MappingConstructible["OgcRecord"], JsonSerializable):
|
|
|
172
277
|
"title": "Jupyter Notebook",
|
|
173
278
|
"href": f"{self.jupyter_notebook_url}",
|
|
174
279
|
},
|
|
280
|
+
{
|
|
281
|
+
"rel": "application-originating-platform",
|
|
282
|
+
"title": "DeepESDL platform",
|
|
283
|
+
"href": "https://deep.earthsystemdatalab.net/",
|
|
284
|
+
"type": "text/html",
|
|
285
|
+
"application:platform_supports": ["jupyter-notebook"],
|
|
286
|
+
"application:preferred_app": "JupyterLab",
|
|
287
|
+
},
|
|
175
288
|
{
|
|
176
289
|
"rel": "related",
|
|
177
290
|
"href": f"../../projects/{PROJECT_COLLECTION_NAME}/collection.json",
|
|
@@ -236,18 +349,20 @@ class ExperimentAsOgcRecord(MappingConstructible["OgcRecord"], JsonSerializable)
|
|
|
236
349
|
"type": "application/json",
|
|
237
350
|
"title": f"Workflow: {self.title}",
|
|
238
351
|
},
|
|
239
|
-
{
|
|
240
|
-
"rel": "child",
|
|
241
|
-
"href": f"../../products/{self.collection_id}/collection.json",
|
|
242
|
-
"type": "application/json",
|
|
243
|
-
"title": f"{self.collection_id}",
|
|
244
|
-
},
|
|
245
352
|
{
|
|
246
353
|
"rel": "related",
|
|
247
354
|
"href": f"../../projects/{PROJECT_COLLECTION_NAME}/collection.json",
|
|
248
355
|
"type": "application/json",
|
|
249
356
|
"title": "Project: DeepESDL",
|
|
250
357
|
},
|
|
358
|
+
{
|
|
359
|
+
"rel": "application-originating-platform",
|
|
360
|
+
"title": "DeepESDL platform",
|
|
361
|
+
"href": "https://deep.earthsystemdatalab.net/",
|
|
362
|
+
"type": "text/html",
|
|
363
|
+
"application:platform_supports": ["jupyter-notebook"],
|
|
364
|
+
"application:preferred_app": "JupyterLab",
|
|
365
|
+
},
|
|
251
366
|
{
|
|
252
367
|
"rel": "input",
|
|
253
368
|
"href": "./input.yaml",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deep_code
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: deepesdl earthcode integration utility tool
|
|
5
5
|
Author-email: Tejas Morbagal Harish <tejas.morbagalharish@brockmann-consult.de>
|
|
6
6
|
License: MIT
|
|
@@ -14,6 +14,7 @@ License-File: LICENSE
|
|
|
14
14
|
Requires-Dist: click
|
|
15
15
|
Requires-Dist: fsspec
|
|
16
16
|
Requires-Dist: jsonschema
|
|
17
|
+
Requires-Dist: jsonpickle
|
|
17
18
|
Requires-Dist: requests
|
|
18
19
|
Requires-Dist: pandas
|
|
19
20
|
Requires-Dist: pystac
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|