deep-code 0.0.1__tar.gz

deep_code-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 DeepESDL and Brockmann Consult GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

deep_code-0.0.1/PKG-INFO

@@ -0,0 +1,154 @@
+Metadata-Version: 2.2
+Name: deep_code
+Version: 0.0.1
+Summary: deepesdl earthcode integration utility tool
+Author-email: Tejas Morbagal Harish <tejas.morbagalharish@brockmann-consult.de>
+License: MIT
+Project-URL: Repository, https://github.com/deepesdl/deep-code
+Project-URL: Issues, https://github.com/deepesdl/deep-code/issues
+Project-URL: Changelog, https://github.com/deepesdl/deep-code/blob/main/CHANGES.md
+Keywords: analysis ready data,data science,datacube,xarray,zarr,xcube,stac,FAIR,reproducible workflow,DeepESDL
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click
+Requires-Dist: fsspec
+Requires-Dist: jsonschema
+Requires-Dist: requests
+Requires-Dist: pandas
+Requires-Dist: pystac
+Requires-Dist: pyyaml
+Requires-Dist: xcube-core
+Requires-Dist: xrlint
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: numpy; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-recording; extra == "dev"
+
+# deep-code
+
+[![Build Status](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml/badge.svg)](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml)
+[![codecov](https://codecov.io/gh/deepesdl/deep-code/graph/badge.svg?token=47MQXOXWOK)](https://codecov.io/gh/deepesdl/deep-code)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![License](https://img.shields.io/github/license/dcs4cop/xcube-smos)](https://github.com/deepesdl/deep-code/blob/main/LICENSE)
+
+`deep-code` is a lightweight python tool that comprises a command line interface(CLI) 
+and Python API providing utilities that aid integration of DeepESDL datasets, 
+experiments with EarthCODE.
+
+The first release will focus on implementing the publish feature of DeepESDL 
+experiments/workflow as OGC API record and Datasets as an OSC stac collection.
+
+## Setup
+
+## Install
+`deep-code` will be available in PyPI and conda-forge. Till the stable release,
+developers/contributors can follow the below steps to install deep-code.
+
+## Installing from the repository for Developers/Contributors
+
+To install deep-code directly from the git repository, clone the repository, and execute the steps below:
+
+```commandline
+conda env create
+conda activate deep-code
+pip install -e .
+```
+
+This installs all the dependencies of `deep-code` into a fresh conda environment, 
+and installs deep-code from the repository into the same environment.
+
+## Testing
+
+To run the unit test suite:
+
+```commandline
+pytest
+```
+
+To analyze test coverage
+```shell
+pytest --cov=deep-code
+```
+
+To produce an HTML coverage report
+
+```commandline
+pytest --cov-report html --cov=deep-code
+```
+
+## deep_code usage
+
+`deep_code` provides a command-line tool called deep-code, which has several subcommands 
+providing different utility functions.
+Use the --help option with these subcommands to get more details on usage.
+
+The CLI retrieves the Git username and personal access token from a hidden file named 
+.gitaccess. Ensure this file is located in the same directory where you execute the CLI
+command.
+
+#### .gitaccess example
+
+```
+github-username: your-git-user
+github-token: personal access token
+```
+
+###  deep-code publish
+
+Publish the experiment, workflow and dataset which is a result of an experiment to 
+the EarthCODE open-science catalog.
+
+```commandline
+ deep-code publish /path/to/dataset-config.yaml /path/to/workflow-config.yaml
+ ```
+
+#### dataset-config.yaml example
+
+```
+dataset_id: hydrology-1D-0.009deg-100x60x60-3.0.2.zarr
+collection_id: hydrology
+osc_themes:
+  - Land
+  - Oceans
+# non-mandatory
+documentation_link: https://deepesdl.readthedocs.io/en/latest/datasets/hydrology-1D-0.009deg-100x60x60-3.0.2.zarr/
+access_link: s3://test
+dataset_status: completed
+osc_region: global
+cf_parameter:
+  - name: hydrology
+```
+
+dataset-id has to be a valid dataset-id from `deep-esdl-public` s3 bucket or your team 
+bucket.
+
+#### workflow-config.yaml example
+
+```
+workflow_id: "esa-cci-permafrost"
+properties:
+  title: "ESA CCI permafrost"
+  description: "cube generation workflow for esa-cci-permafrost"
+  keywords:
+    - Earth Science
+  themes:
+      - cryosphere
+  license: proprietary
+  jupyter_kernel_info:
+    name: deepesdl-xcube-1.8.3
+    python_version: 3.11
+    env_file: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/environment.yml"
+jupyter_notebook_url: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb"
+contact:
+  - name: Tejas Morbagal Harish
+    organization: Brockmann Consult GmbH
+    links:
+      - rel: "about"
+        type: "text/html"
+        href: "https://www.brockmann-consult.de/"
+```

deep_code-0.0.1/README.md

@@ -0,0 +1,123 @@
+# deep-code
+
+[![Build Status](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml/badge.svg)](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml)
+[![codecov](https://codecov.io/gh/deepesdl/deep-code/graph/badge.svg?token=47MQXOXWOK)](https://codecov.io/gh/deepesdl/deep-code)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![License](https://img.shields.io/github/license/dcs4cop/xcube-smos)](https://github.com/deepesdl/deep-code/blob/main/LICENSE)
+
+`deep-code` is a lightweight python tool that comprises a command line interface(CLI) 
+and Python API providing utilities that aid integration of DeepESDL datasets, 
+experiments with EarthCODE.
+
+The first release will focus on implementing the publish feature of DeepESDL 
+experiments/workflow as OGC API record and Datasets as an OSC stac collection.
+
+## Setup
+
+## Install
+`deep-code` will be available in PyPI and conda-forge. Till the stable release,
+developers/contributors can follow the below steps to install deep-code.
+
+## Installing from the repository for Developers/Contributors
+
+To install deep-code directly from the git repository, clone the repository, and execute the steps below:
+
+```commandline
+conda env create
+conda activate deep-code
+pip install -e .
+```
+
+This installs all the dependencies of `deep-code` into a fresh conda environment, 
+and installs deep-code from the repository into the same environment.
+
+## Testing
+
+To run the unit test suite:
+
+```commandline
+pytest
+```
+
+To analyze test coverage
+```shell
+pytest --cov=deep-code
+```
+
+To produce an HTML coverage report
+
+```commandline
+pytest --cov-report html --cov=deep-code
+```
+
+## deep_code usage
+
+`deep_code` provides a command-line tool called deep-code, which has several subcommands 
+providing different utility functions.
+Use the --help option with these subcommands to get more details on usage.
+
+The CLI retrieves the Git username and personal access token from a hidden file named 
+.gitaccess. Ensure this file is located in the same directory where you execute the CLI
+command.
+
+#### .gitaccess example
+
+```
+github-username: your-git-user
+github-token: personal access token
+```
+
+###  deep-code publish
+
+Publish the experiment, workflow and dataset which is a result of an experiment to 
+the EarthCODE open-science catalog.
+
+```commandline
+ deep-code publish /path/to/dataset-config.yaml /path/to/workflow-config.yaml
+ ```
+
+#### dataset-config.yaml example
+
+```
+dataset_id: hydrology-1D-0.009deg-100x60x60-3.0.2.zarr
+collection_id: hydrology
+osc_themes:
+  - Land
+  - Oceans
+# non-mandatory
+documentation_link: https://deepesdl.readthedocs.io/en/latest/datasets/hydrology-1D-0.009deg-100x60x60-3.0.2.zarr/
+access_link: s3://test
+dataset_status: completed
+osc_region: global
+cf_parameter:
+  - name: hydrology
+```
+
+dataset-id has to be a valid dataset-id from `deep-esdl-public` s3 bucket or your team 
+bucket.
+
+#### workflow-config.yaml example
+
+```
+workflow_id: "esa-cci-permafrost"
+properties:
+  title: "ESA CCI permafrost"
+  description: "cube generation workflow for esa-cci-permafrost"
+  keywords:
+    - Earth Science
+  themes:
+      - cryosphere
+  license: proprietary
+  jupyter_kernel_info:
+    name: deepesdl-xcube-1.8.3
+    python_version: 3.11
+    env_file: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/environment.yml"
+jupyter_notebook_url: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb"
+contact:
+  - name: Tejas Morbagal Harish
+    organization: Brockmann Consult GmbH
+    links:
+      - rel: "about"
+        type: "text/html"
+        href: "https://www.brockmann-consult.de/"
+```

deep_code-0.0.1/deep_code/__init__.py

@@ -0,0 +1,24 @@
+# The MIT License (MIT)
+# Copyright (c) 2024 by DeepESDL and Brockmann Consult GmbH
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+from .version import version
+
+__version__ = version

deep_code-0.0.1/deep_code/cli/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.

deep_code-0.0.1/deep_code/cli/main.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+import click
+
+from deep_code.cli.publish import publish
+
+
+@click.group()
+def main():
+    """Deep Code CLI."""
+    pass
+
+
+main.add_command(publish)
+
+if __name__ == "__main__":
+    main()

deep_code-0.0.1/deep_code/cli/publish.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+import click
+
+from deep_code.tools.publish import Publisher
+
+
+@click.command(name="publish")
+@click.argument("dataset_config", type=click.Path(exists=True))
+@click.argument("workflow_config", type=click.Path(exists=True))
+def publish(dataset_config, workflow_config):
+    """Request publishing a dataset to the open science catalogue.
+    """
+    publisher = Publisher(
+        dataset_config_path=dataset_config, workflow_config_path=workflow_config
+    )
+    publisher.publish_all()

deep_code-0.0.1/deep_code/constants.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+OSC_SCHEMA_URI = "https://stac-extensions.github.io/osc/v1.0.0/schema.json"
+CF_SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json"
+THEMES_SCHEMA_URI = "https://stac-extensions.github.io/themes/v1.0.0/schema.json"
+OSC_THEME_SCHEME = "https://github.com/stac-extensions/osc#theme"
+OSC_REPO_OWNER = "ESA-EarthCODE"
+OSC_REPO_NAME = "open-science-catalog-metadata"
+OSC_BRANCH_NAME = "add-new-collection"
+DEFAULT_THEME_SCHEME = (
+    "https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords"
+)
+OGC_API_RECORD_SPEC = "http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core"
+WF_BRANCH_NAME = "add-new-workflow-from-deepesdl"
+VARIABLE_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/variables/catalog.json"
+PRODUCT_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/products/catalog.json"
+DEEPESDL_COLLECTION_SELF_HREF = (
+    "https://esa-earthcode.github.io/open-science-catalog-metadata/projects/deepesdl"
+    "/collection.json"
+)
+BASE_URL_OSC = "https://esa-earthcode.github.io/open-science-catalog-metadata"
+EXPERIMENT_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/experiments/catalog.json"
+WORKFLOW_BASE_CATALOG_SELF_HREF = (
+    "https://esa-earthcode.github.io/open-science-catalog-metadata/workflows/catalog"
+    ".json"
+)
+PROJECT_COLLECTION_NAME = "deep-earth-system-data-lab"

deep_code-0.0.1/deep_code/tests/tools/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.

deep_code-0.0.1/deep_code/tests/tools/test_publish.py

@@ -0,0 +1,108 @@
+import unittest
+from unittest.mock import patch, mock_open, MagicMock
+import json
+import yaml
+from pathlib import Path
+import tempfile
+from pystac import Catalog
+
+from deep_code.tools.publish import Publisher
+
+
+class TestPublisher(unittest.TestCase):
+    @patch("fsspec.open")
+    @patch("deep_code.tools.publish.GitHubPublisher")
+    def setUp(self, mock_github_publisher, mock_fsspec_open):
+        # Mock GitHubPublisher to avoid reading .gitaccess
+        self.mock_github_publisher_instance = MagicMock()
+        mock_github_publisher.return_value = self.mock_github_publisher_instance
+
+        # Mock dataset and workflow config files
+        self.dataset_config = {
+            "collection_id": "test-collection",
+            "dataset_id": "test-dataset",
+        }
+        self.workflow_config = {
+            "properties": {"title": "Test Workflow"},
+            "workflow_id": "test-workflow",
+        }
+
+        # Mock fsspec.open for config files
+        self.mock_fsspec_open = mock_fsspec_open
+        self.mock_fsspec_open.side_effect = [
+            mock_open(read_data=yaml.dump(self.dataset_config)).return_value,
+            mock_open(read_data=yaml.dump(self.workflow_config)).return_value,
+        ]
+
+        # Initialize Publisher
+        self.publisher = Publisher(
+            dataset_config_path="test-dataset-config.yaml",
+            workflow_config_path="test-workflow-config.yaml",
+        )
+
+    def test_normalize_name(self):
+        self.assertEqual(Publisher._normalize_name("Test Name"), "test-name")
+        self.assertEqual(Publisher._normalize_name("Test   Name"), "test---name")
+        self.assertIsNone(Publisher._normalize_name(""))
+        self.assertIsNone(Publisher._normalize_name(None))
+
+    def test_write_to_file(self):
+        # Create a temporary file
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            file_path = temp_file.name
+
+        # Test data
+        data = {"key": "value"}
+
+        # Call the method
+        Publisher._write_to_file(file_path, data)
+
+        # Read the file and verify its content
+        with open(file_path, "r") as f:
+            content = json.load(f)
+            self.assertEqual(content, data)
+
+        # Clean up
+        Path(file_path).unlink()
+
+    def test_update_base_catalog(self):
+        # Create a mock Catalog
+        catalog = Catalog(id="test-catalog", description="Test Catalog")
+
+        # Mock file path and item ID
+        catalog_path = "test-catalog.json"
+        item_id = "test-item"
+        self_href = "https://example.com/catalog.json"
+
+        self.publisher.workflow_title = "Test Workflow"
+
+        # Mock the Catalog.from_file method
+        with patch("pystac.Catalog.from_file", return_value=catalog):
+            updated_catalog = self.publisher._update_base_catalog(
+                catalog_path, item_id, self_href
+            )
+
+        # Assertions
+        self.assertEqual(updated_catalog.get_self_href(), self_href)
+        self.assertIsInstance(updated_catalog, Catalog)
+
+    def test_read_config_files(self):
+        # Mock dataset and workflow config files
+        dataset_config = {
+            "collection_id": "test-collection",
+            "dataset_id": "test-dataset",
+        }
+        workflow_config = {
+            "properties": {"title": "Test Workflow"},
+            "workflow_id": "test-workflow",
+        }
+
+        # Mock fsspec.open for config files
+        self.mock_fsspec_open.side_effect = [
+            mock_open(read_data=yaml.dump(dataset_config)).return_value,
+            mock_open(read_data=yaml.dump(workflow_config)).return_value,
+        ]
+
+        # Assertions
+        self.assertEqual(self.publisher.dataset_config, dataset_config)
+        self.assertEqual(self.publisher.workflow_config, workflow_config)

deep_code-0.0.1/deep_code/tests/utils/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.