deep-code 0.0.1.dev0__py3-none-any.whl

deep_code/tests/utils/test_ogc_record_generator.py

@@ -0,0 +1,63 @@
+import unittest
+from datetime import datetime, timezone
+
+from deep_code.constants import DEFAULT_THEME_SCHEME
+from deep_code.utils.ogc_record_generator import OSCWorkflowOGCApiRecordGenerator
+
+
+class TestOSCWorkflowOGCApiRecordGenerator(unittest.TestCase):
+    def test_build_contact_objects(self):
+        contacts_list = [
+            {"name": "Alice", "organization": "Org A", "position": "Researcher"},
+            {"name": "Bob", "organization": "Org B", "position": "Developer"},
+        ]
+
+        result = OSCWorkflowOGCApiRecordGenerator.build_contact_objects(contacts_list)
+
+        self.assertEqual(len(result), 2)
+        self.assertEqual(result[0].name, "Alice")
+        self.assertEqual(result[0].organization, "Org A")
+        self.assertEqual(result[0].position, "Researcher")
+        self.assertEqual(result[1].name, "Bob")
+        self.assertEqual(result[1].organization, "Org B")
+        self.assertEqual(result[1].position, "Developer")
+
+    def test_build_theme(self):
+        osc_themes = ["theme1", "theme2"]
+
+        theme = OSCWorkflowOGCApiRecordGenerator.build_theme(osc_themes)
+
+        self.assertEqual(len(theme.concepts), 2)
+        self.assertEqual(theme.concepts[0].id, "theme1")
+        self.assertEqual(theme.concepts[1].id, "theme2")
+        self.assertEqual(theme.scheme, DEFAULT_THEME_SCHEME)
+
+    def test_build_record_properties(self):
+        generator = OSCWorkflowOGCApiRecordGenerator()
+        properties = {
+            "title": "Test Workflow",
+            "description": "A test description",
+            "themes": ["theme1"],
+            "jupyter_kernel_info": {
+                "name": "deepesdl-xcube-1.7.1",
+                "python_version": 3.11,
+                "env_file": "https://git/env.yml",
+            },
+        }
+        contacts = [
+            {"name": "Alice", "organization": "Org A", "position": "Researcher"}
+        ]
+
+        record_properties = generator.build_record_properties(properties, contacts)
+
+        now_iso = datetime.now(timezone.utc).isoformat()
+
+        self.assertEqual(record_properties.title, "Test Workflow")
+        self.assertEqual(record_properties.description, "A test description")
+        self.assertEqual(len(record_properties.contacts), 1)
+        self.assertEqual(record_properties.contacts[0].name, "Alice")
+        self.assertEqual(len(record_properties.themes), 1)
+        self.assertEqual(record_properties.themes[0].concepts[0].id, "theme1")
+        self.assertEqual(record_properties.type, "workflow")
+        self.assertTrue("created" in record_properties.__dict__)
+        self.assertTrue("updated" in record_properties.__dict__)

deep_code/tests/utils/test_osc_extension.py

@@ -0,0 +1,117 @@
+import unittest
+
+from pystac import Collection, Extent, SpatialExtent, TemporalExtent
+
+from deep_code.utils.osc_extension import OscExtension
+
+
+class TestOscExtension(unittest.TestCase):
+    def setUp(self):
+        """Set up a test Collection object and attach the OscExtension."""
+        self.collection = Collection(
+            id="test-collection",
+            description="Test collection for unit tests",
+            extent=Extent(
+                spatial=SpatialExtent([[-180, -90, 180, 90]]),
+                temporal=TemporalExtent(
+                    [["2022-01-01T00:00:00Z", "2023-01-01T00:00:00Z"]]
+                ),
+            ),
+            stac_extensions=[],
+        )
+        OscExtension.add_to(self.collection)
+
+    def test_osc_status(self):
+        """Test the osc:status property."""
+        extension = OscExtension.ext(self.collection)
+        extension.osc_status = "ongoing"
+        self.assertEqual(extension.osc_status, "ongoing")
+
+    def test_osc_region(self):
+        """Test the osc:region property."""
+        extension = OscExtension.ext(self.collection)
+        extension.osc_region = "Mediterranean region"
+        self.assertEqual(extension.osc_region, "Mediterranean region")
+
+    def test_osc_themes(self):
+        """Test the osc:themes property."""
+        extension = OscExtension.ext(self.collection)
+        extension.osc_themes = ["land", "ocean"]
+        self.assertEqual(extension.osc_themes, ["land", "ocean"])
+
+    def test_osc_missions(self):
+        """Test the osc:missions property."""
+        extension = OscExtension.ext(self.collection)
+        extension.osc_missions = ["mission1", "mission2"]
+        self.assertEqual(extension.osc_missions, ["mission1", "mission2"])
+
+    def test_keywords(self):
+        """Test the keywords property."""
+        extension = OscExtension.ext(self.collection)
+        extension.keywords = ["Hydrology", "Remote Sensing"]
+        self.assertEqual(extension.keywords, ["Hydrology", "Remote Sensing"])
+
+    def test_cf_parameters(self):
+        """Test the cf:parameter property."""
+        extension = OscExtension.ext(self.collection)
+        extension.cf_parameter = [{"name": "hydrology-4D"}]
+        self.assertEqual(extension.cf_parameter, [{"name": "hydrology-4D"}])
+
+    def test_created_updated(self):
+        """Test the created and updated properties."""
+        extension = OscExtension.ext(self.collection)
+        extension.created = "2023-12-21T11:50:17Z"
+        extension.updated = "2023-12-21T11:50:17Z"
+        self.assertEqual(extension.created, "2023-12-21T11:50:17Z")
+        self.assertEqual(extension.updated, "2023-12-21T11:50:17Z")
+
+    def test_set_extent(self):
+        """Test setting spatial and temporal extent."""
+        extension = OscExtension.ext(self.collection)
+        spatial = [[-5.7, 28.3, 37.7, 48.1]]
+        temporal = [["2014-12-31T12:00:00Z", "2022-10-06T12:00:00Z"]]
+        extension.set_extent(spatial, temporal)
+
+        self.assertEqual(self.collection.extent.spatial.bboxes, spatial)
+        self.assertEqual(self.collection.extent.temporal.intervals, temporal)
+
+    def test_validation_success(self):
+        """Test validation with all required fields."""
+        extension = OscExtension.ext(self.collection)
+        extension.osc_type = "product"
+        extension.osc_project = "test-project"
+        extension.osc_status = "ongoing"
+        extension.validate_extension()  # Should not raise an exception
+
+    def test_add_osc_extension(self):
+        osc_ext = OscExtension.add_to(self.collection)
+        self.assertEqual(OscExtension.get_schema_uri(), self.collection.stac_extensions)
+        self.assertIsInstance(osc_ext, OscExtension)
+
+    def test_has_extension(self):
+        self.collection.stac_extensions = []
+        self.assertFalse(OscExtension.has_extension(self.collection))
+        OscExtension.add_to(self.collection)
+        self.assertTrue(OscExtension.has_extension(self.collection))
+
+    def test_set_and_get_properties(self):
+        osc_ext = OscExtension.add_to(self.collection)
+        osc_ext.osc_type = "example-type"
+        osc_ext.osc_project = "example-project"
+        osc_ext.osc_product = "example-product"
+        osc_ext.osc_theme = ["example-theme"]
+        osc_ext.osc_variables = ["var1", "var2", "var3"]
+
+        self.assertEqual(osc_ext.osc_type, "example-type")
+        self.assertEqual(osc_ext.osc_project, "example-project")
+        self.assertEqual(osc_ext.osc_product, "example-product")
+        self.assertEqual(osc_ext.osc_theme, ["example-theme"])
+        self.assertListEqual(osc_ext.osc_variables, ["var1", "var2", "var3"])
+
+    def test_validation_missing_fields(self):
+        """Test validation with missing required fields."""
+        extension = OscExtension.ext(self.collection)
+        with self.assertRaises(ValueError) as context:
+            extension.validate_extension()
+        self.assertIn("Missing required fields", str(context.exception))
+        self.assertIn("osc:type", str(context.exception))

deep_code/tools/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.

deep_code/tools/check.py

@@ -0,0 +1,4 @@
+"""
+Verify the readiness of a dataset or an existing workflow repository for experiment
+publication by identifying any issues or missing components
+"""

deep_code/tools/new.py

@@ -0,0 +1,5 @@
+"""Logic for initializing repositories
+ Initialize a GitHub repository with the proposed configurations files, an initial
+ workflow notebook template (e.g. workflow.ipynb), a template Python package (code and
+pyproject.toml), and a template setup for documentation (e.g., using mkdocs),
+setup of thebuild pipeline"""

deep_code/tools/publish.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+import logging
+from pathlib import Path
+
+import fsspec
+import yaml
+
+from deep_code.constants import (
+    OSC_BRANCH_NAME,
+    OSC_REPO_NAME,
+    OSC_REPO_OWNER,
+    WF_BRANCH_NAME,
+)
+from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator
+from deep_code.utils.github_automation import GitHubAutomation
+from deep_code.utils.ogc_api_record import OgcRecord
+from deep_code.utils.ogc_record_generator import OSCWorkflowOGCApiRecordGenerator
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class GitHubPublisher:
+    """
+    Base class providing:
+      - Reading .gitaccess for credentials
+      - Common GitHub automation steps (fork, clone, branch, file commit, pull request)
+    """
+
+    def __init__(self):
+        with fsspec.open(".gitaccess", "r") as file:
+            git_config = yaml.safe_load(file) or {}
+        self.github_username = git_config.get("github-username")
+        self.github_token = git_config.get("github-token")
+        if not self.github_username or not self.github_token:
+            raise ValueError("GitHub credentials are missing in the `.gitaccess` file.")
+
+        self.github_automation = GitHubAutomation(
+            self.github_username, self.github_token, OSC_REPO_OWNER, OSC_REPO_NAME
+        )
+
+    def publish_files(
+        self,
+        branch_name: str,
+        file_dict: dict[str, dict],
+        commit_message: str,
+        pr_title: str,
+        pr_body: str,
+    ) -> str:
+        """Publish multiple files to a new branch and open a PR.
+
+        Args:
+            branch_name: Branch name to create (e.g. "osc-branch-collectionid").
+            file_dict: { file_path: file_content_dict } for each file to commit.
+            commit_message: Commit message for all changes.
+            pr_title: Title of the pull request.
+            pr_body: Description/body of the pull request.
+
+        Returns:
+            URL of the created pull request.
+        """
+        try:
+            logger.info("Forking and cloning repository...")
+            self.github_automation.fork_repository()
+            self.github_automation.clone_repository()
+            self.github_automation.create_branch(branch_name)
+
+            # Add each file to the branch
+            for file_path, content in file_dict.items():
+                logger.info(f"Adding {file_path} to {branch_name}")
+                self.github_automation.add_file(file_path, content)
+
+            # Commit and push
+            self.github_automation.commit_and_push(branch_name, commit_message)
+
+            # Create pull request
+            pr_url = self.github_automation.create_pull_request(
+                branch_name, pr_title, pr_body
+            )
+            logger.info(f"Pull request created: {pr_url}")
+            return pr_url
+
+        finally:
+            # Cleanup local clone
+            self.github_automation.clean_up()
+
+
+class DatasetPublisher:
+    """Publishes products (datasets) to the OSC GitHub repository.
+    Inherits from BasePublisher for GitHub publishing logic.
+    """
+
+    def __init__(self):
+        # Composition
+        self.gh_publisher = GitHubPublisher()
+
+    def publish_dataset(self, dataset_config_path: str):
+        """Publish a product collection to the specified GitHub repository."""
+        with fsspec.open(dataset_config_path, "r") as file:
+            dataset_config = yaml.safe_load(file) or {}
+
+        dataset_id = dataset_config.get("dataset_id")
+        collection_id = dataset_config.get("collection_id")
+        documentation_link = dataset_config.get("documentation_link")
+        access_link = dataset_config.get("access_link")
+        dataset_status = dataset_config.get("dataset_status")
+        osc_region = dataset_config.get("osc_region")
+        osc_themes = dataset_config.get("osc_themes")
+        cf_params = dataset_config.get("cf_parameter")
+
+        if not dataset_id or not collection_id:
+            raise ValueError("Dataset ID or Collection ID missing in the config.")
+
+        logger.info("Generating STAC collection...")
+
+        generator = OscDatasetStacGenerator(
+            dataset_id=dataset_id,
+            collection_id=collection_id,
+            documentation_link=documentation_link,
+            access_link=access_link,
+            osc_status=dataset_status,
+            osc_region=osc_region,
+            osc_themes=osc_themes,
+            cf_params=cf_params,
+        )
+
+        variable_ids = generator.get_variable_ids()
+        ds_collection = generator.build_dataset_stac_collection()
+
+        # Prepare a dictionary of file paths and content
+        file_dict = {}
+        product_path = f"products/{collection_id}/collection.json"
+        file_dict[product_path] = ds_collection.to_dict()
+
+        # Add or update variable files
+        for var_id in variable_ids:
+            var_file_path = f"variables/{var_id}/catalog.json"
+            if not self.gh_publisher.github_automation.file_exists(var_file_path):
+                logger.info(
+                    f"Variable catalog for {var_id} does not exist. Creating..."
+                )
+                var_metadata = generator.variables_metadata.get(var_id)
+                var_catalog = generator.build_variable_catalog(var_metadata)
+                file_dict[var_file_path] = var_catalog.to_dict()
+            else:
+                logger.info(
+                    f"Variable catalog already exists for {var_id}, adding product link."
+                )
+                full_path = (
+                    Path(self.gh_publisher.github_automation.local_clone_dir)
+                    / var_file_path
+                )
+                updated_catalog = generator.update_existing_variable_catalog(
+                    full_path, var_id
+                )
+                file_dict[var_file_path] = updated_catalog.to_dict()
+
+        # Create branch name, commit message, PR info
+        branch_name = f"{OSC_BRANCH_NAME}-{collection_id}"
+        commit_message = f"Add new dataset collection: {collection_id}"
+        pr_title = "Add new dataset collection"
+        pr_body = "This PR adds a new dataset collection to the repository."
+
+        # Publish all files in one go
+        pr_url = self.gh_publisher.publish_files(
+            branch_name=branch_name,
+            file_dict=file_dict,
+            commit_message=commit_message,
+            pr_title=pr_title,
+            pr_body=pr_body,
+        )
+
+        logger.info(f"Pull request created: {pr_url}")
+
+
+class WorkflowPublisher:
+    """Publishes workflows to the OSC GitHub repository."""
+
+    def __init__(self):
+        self.gh_publisher = GitHubPublisher()
+
+    @staticmethod
+    def _normalize_name(name: str | None) -> str | None:
+        return name.replace(" ", "-").lower() if name else None
+
+    def publish_workflow(self, workflow_config_path: str):
+        with fsspec.open(workflow_config_path, "r") as file:
+            workflow_config = yaml.safe_load(file) or {}
+
+        workflow_id = self._normalize_name(workflow_config.get("workflow_id"))
+        if not workflow_id:
+            raise ValueError("workflow_id is missing in workflow config.")
+
+        properties_list = workflow_config.get("properties", [])
+        contacts = workflow_config.get("contact", [])
+        links = workflow_config.get("links", [])
+
+        logger.info("Generating OGC API Record for the workflow...")
+        rg = OSCWorkflowOGCApiRecordGenerator()
+        wf_record_properties = rg.build_record_properties(properties_list, contacts)
+
+        ogc_record = OgcRecord(
+            id=workflow_id,
+            type="Feature",
+            time={},
+            properties=wf_record_properties,
+            links=links,
+        )
+
+        file_path = f"workflow/{workflow_id}/collection.json"
+
+        # Prepare the single file dict
+        file_dict = {file_path: ogc_record.to_dict()}
+
+        branch_name = f"{WF_BRANCH_NAME}-{workflow_id}"
+        commit_message = f"Add new workflow: {workflow_id}"
+        pr_title = "Add new workflow"
+        pr_body = "This PR adds a new workflow to the OSC repository."
+
+        pr_url = self.gh_publisher.publish_files(
+            branch_name=branch_name,
+            file_dict=file_dict,
+            commit_message=commit_message,
+            pr_title=pr_title,
+            pr_body=pr_body,
+        )
+
+        logger.info(f"Pull request created: {pr_url}")

deep_code/tools/setup_ci.py

@@ -0,0 +1 @@
+"""Logic for setting up build pipelines"""

deep_code/tools/test.py

@@ -0,0 +1,2 @@
+""" Execute the application package of a published experiment on a subset of input data
+to verify the reproducibility is achieved"""

deep_code/utils/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.