deep-code 0.1.7__tar.gz → 0.1.9.dev0__tar.gz

deep_code-0.1.9.dev0/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.4
+Name: deep_code
+Version: 0.1.9.dev0
+Summary: deepesdl earthcode integration utility tool
+Author-email: Tejas Morbagal Harish <tejas.morbagalharish@brockmann-consult.de>
+License: MIT
+Project-URL: Repository, https://github.com/deepesdl/deep-code
+Project-URL: Issues, https://github.com/deepesdl/deep-code/issues
+Project-URL: Changelog, https://github.com/deepesdl/deep-code/blob/main/CHANGES.md
+Keywords: analysis ready data,data science,datacube,xarray,zarr,xcube,stac,FAIR,reproducible workflow,DeepESDL
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click
+Requires-Dist: fsspec
+Requires-Dist: jsonschema
+Requires-Dist: jsonpickle
+Requires-Dist: requests
+Requires-Dist: pandas
+Requires-Dist: pystac
+Requires-Dist: pyyaml
+Requires-Dist: xcube-core
+Requires-Dist: xrlint
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: numpy; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-recording; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.5; extra == "docs"
+Requires-Dist: mkdocs-autorefs; extra == "docs"
+Requires-Dist: mkdocs-material>=9.5; extra == "docs"
+Requires-Dist: mkdocstrings; extra == "docs"
+Requires-Dist: mkdocstrings-python; extra == "docs"
+Dynamic: license-file
+
+# deep-code
+
+[![Build Status](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml/badge.svg)](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml)
+[![codecov](https://codecov.io/gh/deepesdl/deep-code/graph/badge.svg?token=47MQXOXWOK)](https://codecov.io/gh/deepesdl/deep-code)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![PyPI version](https://img.shields.io/pypi/v/deep-code)](https://pypi.org/project/deep-code/)
+[![License](https://img.shields.io/github/license/dcs4cop/xcube-smos)](https://github.com/deepesdl/deep-code/blob/main/LICENSE)
+[![Docs](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://deepesdl.github.io/deep-code/)
+
+`deep-code` is a lightweight Python CLI and API that turns DeepESDL datasets and 
+workflows into EarthCODE Open Science Catalog metadata. It can generate starter configs,
+build STAC collections and OGC API records, and open pull requests to the target 
+EarthCODE metadata repository (production, staging, or testing).
+
+## Features
+- Generate starter dataset and workflow YAML templates.
+- Publish dataset collections, workflows, and experiments via a single command.
+- Build STAC collections and catalogs for Datasets and their corresponding variables 
+  automatically from the dataset metadata.
+- Build OGC API records for Workflows and Experiments from your configs.
+- Flexible publishling targets i.e production/staging/testing EarthCODE metadata
+  repositories with GitHub automation.
+
+More about deep-code can be found in its [documentation](https://deepesdl.github.io/deep-code/).

deep_code-0.1.9.dev0/README.md

@@ -0,0 +1,25 @@
+# deep-code
+
+[![Build Status](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml/badge.svg)](https://github.com/deepesdl/deep-code/actions/workflows/unittest-workflow.yaml)
+[![codecov](https://codecov.io/gh/deepesdl/deep-code/graph/badge.svg?token=47MQXOXWOK)](https://codecov.io/gh/deepesdl/deep-code)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![PyPI version](https://img.shields.io/pypi/v/deep-code)](https://pypi.org/project/deep-code/)
+[![License](https://img.shields.io/github/license/dcs4cop/xcube-smos)](https://github.com/deepesdl/deep-code/blob/main/LICENSE)
+[![Docs](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://deepesdl.github.io/deep-code/)
+
+`deep-code` is a lightweight Python CLI and API that turns DeepESDL datasets and 
+workflows into EarthCODE Open Science Catalog metadata. It can generate starter configs,
+build STAC collections and OGC API records, and open pull requests to the target 
+EarthCODE metadata repository (production, staging, or testing).
+
+## Features
+- Generate starter dataset and workflow YAML templates.
+- Publish dataset collections, workflows, and experiments via a single command.
+- Build STAC collections and catalogs for Datasets and their corresponding variables 
+  automatically from the dataset metadata.
+- Build OGC API records for Workflows and Experiments from your configs.
+- Flexible publishling targets i.e production/staging/testing EarthCODE metadata
+  repositories with GitHub automation.
+
+More about deep-code can be found in its [documentation](https://deepesdl.github.io/deep-code/).

{deep_code-0.1.7 → deep_code-0.1.9.dev0}/deep_code/constants.py

@@ -39,3 +39,4 @@ APPLICATION_TYPE_JUPYTER_SPEC = (
 APPLICATION_STAC_EXTENSION_SPEC = (
     "https://stac-extensions.github.io/application/v0.1.0/schema.json"
 )
+ZARR_MEDIA_TYPE = "application/vnd+zarr"

{deep_code-0.1.7 → deep_code-0.1.9.dev0}/deep_code/tests/tools/test_publish.py

@@ -1,4 +1,5 @@
 import json
+import os
 import tempfile
 import unittest
 from pathlib import Path
@@ -174,6 +175,206 @@ class TestPublisher(unittest.TestCase):
         assert "workflow/experiment: wf" in kwargs["pr_title"]
 
 
+    # ------------------------------------------------------------------
+    # S3 credential resolution
+    # ------------------------------------------------------------------
+
+    def test_get_stac_s3_storage_options_prefers_stac_env_vars(self):
+        env = {
+            "STAC_S3_KEY": "stac-key",
+            "STAC_S3_SECRET": "stac-secret",
+            "AWS_ACCESS_KEY_ID": "aws-key",
+            "AWS_SECRET_ACCESS_KEY": "aws-secret",
+        }
+        with patch.dict(os.environ, env):
+            opts = self.publisher._get_stac_s3_storage_options()
+        self.assertEqual(opts["key"], "stac-key")
+        self.assertEqual(opts["secret"], "stac-secret")
+        self.assertEqual(opts["s3_additional_kwargs"], {"ACL": ""})
+
+    def test_get_stac_s3_storage_options_falls_back_to_aws_env_vars(self):
+        env = {"AWS_ACCESS_KEY_ID": "aws-key", "AWS_SECRET_ACCESS_KEY": "aws-secret"}
+        patched_env = {
+            k: v
+            for k, v in os.environ.items()
+            if k not in ("STAC_S3_KEY", "STAC_S3_SECRET")
+        }
+        patched_env.update(env)
+        with patch.dict(os.environ, patched_env, clear=True):
+            opts = self.publisher._get_stac_s3_storage_options()
+        self.assertEqual(opts["key"], "aws-key")
+        self.assertEqual(opts["secret"], "aws-secret")
+        self.assertEqual(opts["s3_additional_kwargs"], {"ACL": ""})
+
+    def test_get_stac_s3_storage_options_returns_acl_suppression_for_boto3_chain(self):
+        no_cred_env = {
+            k: v
+            for k, v in os.environ.items()
+            if k
+            not in (
+                "STAC_S3_KEY",
+                "STAC_S3_SECRET",
+                "AWS_ACCESS_KEY_ID",
+                "AWS_SECRET_ACCESS_KEY",
+            )
+        }
+        with patch.dict(os.environ, no_cred_env, clear=True):
+            opts = self.publisher._get_stac_s3_storage_options()
+        self.assertEqual(opts, {"s3_additional_kwargs": {"ACL": ""}})
+
+    # ------------------------------------------------------------------
+    # S3 write helper
+    # ------------------------------------------------------------------
+
+    @patch("deep_code.tools.publish.fsspec.open")
+    def test_write_stac_catalog_to_s3(self, mock_fsspec_open):
+        mock_file = MagicMock()
+        mock_ctx = MagicMock()
+        mock_ctx.__enter__ = MagicMock(return_value=mock_file)
+        mock_ctx.__exit__ = MagicMock(return_value=False)
+        mock_fsspec_open.return_value = mock_ctx
+
+        file_dict = {
+            "s3://bucket/catalog.json": {"type": "Catalog", "id": "test"},
+            "s3://bucket/col/item.json": {"type": "Feature", "id": "item"},
+        }
+        self.publisher._write_stac_catalog_to_s3(
+            file_dict, {"key": "k", "secret": "s"}
+        )
+
+        self.assertEqual(mock_fsspec_open.call_count, 2)
+        mock_fsspec_open.assert_any_call(
+            "s3://bucket/catalog.json", "w", key="k", secret="s"
+        )
+        mock_fsspec_open.assert_any_call(
+            "s3://bucket/col/item.json", "w", key="k", secret="s"
+        )
+
+    # ------------------------------------------------------------------
+    # End-to-end zarr STAC publishing wired into publish()
+    # ------------------------------------------------------------------
+
+    @patch("deep_code.tools.publish.fsspec.open")
+    @patch.object(Publisher, "publish_dataset", return_value={"github_file.json": {}})
+    def test_publish_writes_zarr_stac_to_s3_when_configured(
+        self, mock_publish_ds, mock_fsspec_open
+    ):
+        self.publisher.dataset_config["stac_catalog_s3_root"] = (
+            "s3://test-bucket/stac/"
+        )
+
+        mock_ctx = MagicMock()
+        mock_ctx.__enter__ = MagicMock(return_value=MagicMock())
+        mock_ctx.__exit__ = MagicMock(return_value=False)
+        mock_fsspec_open.return_value = mock_ctx
+
+        mock_generator = MagicMock()
+        mock_generator.build_zarr_stac_catalog_file_dict.return_value = {
+            "s3://test-bucket/stac/catalog.json": {"type": "Catalog"},
+            "s3://test-bucket/stac/test-collection/item.json": {"type": "Feature"},
+        }
+        # Simulate what publish_dataset() normally does: store the generator
+        self.publisher._last_generator = mock_generator
+        self.publisher.gh_publisher.publish_files.return_value = "PR_URL"
+
+        self.publisher.publish(mode="dataset")
+
+        mock_generator.build_zarr_stac_catalog_file_dict.assert_called_once_with(
+            "s3://test-bucket/stac/"
+        )
+        # Two S3 files written: catalog.json + item.json
+        self.assertEqual(mock_fsspec_open.call_count, 2)
+
+    # ------------------------------------------------------------------
+    # Project collection create-vs-update branching
+    # ------------------------------------------------------------------
+
+    @patch("deep_code.tools.publish.OscDatasetStacGenerator")
+    def test_publish_dataset_creates_project_collection_when_missing(
+        self, MockGenerator
+    ):
+        """When the project collection does not exist, build_project_collection is
+        called and projects/catalog.json is updated via _update_and_add_to_file_dict."""
+        mock_gen = MagicMock()
+        mock_gen.osc_project = "test-project"
+        mock_gen.get_variable_ids.return_value = []
+        mock_gen.build_dataset_stac_collection.return_value.to_dict.return_value = {}
+        mock_gen.build_project_collection.return_value = {
+            "type": "Collection",
+            "id": "test-project",
+        }
+        MockGenerator.return_value = mock_gen
+
+        self.publisher.dataset_config = {
+            "dataset_id": "test-dataset",
+            "collection_id": "test-collection",
+            "license_type": "CC-BY-4.0",
+        }
+        self.publisher.collection_id = "test-collection"
+
+        # Project collection is missing; all other file_exists calls return True
+        self.publisher.gh_publisher.github_automation.file_exists.return_value = False
+
+        with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \
+                patch.object(self.publisher, "_update_variable_catalogs"):
+            file_dict = self.publisher.publish_dataset(write_to_file=False)
+
+        mock_gen.build_project_collection.assert_called_once()
+        self.assertIn("projects/test-project/collection.json", file_dict)
+        mock_gen.update_deepesdl_collection.assert_not_called()
+
+        # projects/catalog.json must be updated
+        updated_paths = [call.args[1] for call in mock_update.call_args_list]
+        self.assertIn("projects/catalog.json", updated_paths)
+
+    @patch("deep_code.tools.publish.OscDatasetStacGenerator")
+    def test_publish_dataset_updates_project_collection_when_exists(
+        self, MockGenerator
+    ):
+        """When the project collection exists, update_deepesdl_collection is called
+        via _update_and_add_to_file_dict and build_project_collection is not called."""
+        mock_gen = MagicMock()
+        mock_gen.osc_project = "test-project"
+        mock_gen.get_variable_ids.return_value = []
+        mock_gen.build_dataset_stac_collection.return_value.to_dict.return_value = {}
+        MockGenerator.return_value = mock_gen
+
+        self.publisher.dataset_config = {
+            "dataset_id": "test-dataset",
+            "collection_id": "test-collection",
+            "license_type": "CC-BY-4.0",
+        }
+        self.publisher.collection_id = "test-collection"
+
+        # Project collection already exists
+        self.publisher.gh_publisher.github_automation.file_exists.return_value = True
+
+        with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \
+                patch.object(self.publisher, "_update_variable_catalogs"):
+            self.publisher.publish_dataset(write_to_file=False)
+
+        mock_gen.build_project_collection.assert_not_called()
+
+        # update_deepesdl_collection passed to _update_and_add_to_file_dict
+        update_methods = [call.args[2] for call in mock_update.call_args_list]
+        self.assertIn(mock_gen.update_deepesdl_collection, update_methods)
+
+    @patch.object(Publisher, "publish_dataset", return_value={"github_file.json": {}})
+    def test_publish_skips_zarr_stac_when_not_configured(self, mock_publish_ds):
+        # No stac_catalog_s3_root in config
+        self.publisher.dataset_config = {
+            "collection_id": "test-collection",
+            "dataset_id": "test-dataset",
+        }
+        self.publisher.gh_publisher.publish_files.return_value = "PR_URL"
+
+        with patch.object(
+            self.publisher, "_write_stac_catalog_to_s3"
+        ) as mock_write:
+            self.publisher.publish(mode="dataset")
+            mock_write.assert_not_called()
+
+
 class TestParseGithubNotebookUrl:
     @pytest.mark.parametrize(
         "url,repo_url,repo_name,branch,file_path",