osc-metadata-client 0.3.0__py3-none-any.whl

osc_metadata_client/__about__.py

@@ -0,0 +1,15 @@
+# Copyright 2026 Terradue
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.3.0"

osc_metadata_client/__init__.py

@@ -0,0 +1,275 @@
+# Copyright 2026 Terradue
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from gzip import GzipFile
+from io import BytesIO, TextIOWrapper
+from loguru import logger
+from ogc_api_processes_client.api_client import ApiClient
+from ogc_api_processes_client.api.status_api import StatusApi
+from ogc_api_processes_client.configuration import Configuration
+from ogc_api_processes_client.models.status_code import StatusCode
+from ogc_api_processes_client.models.status_info import StatusInfo
+from pathlib import Path
+from pydantic import BaseModel
+from pystac import Catalog, Link as PystacLink, RelType
+from requests import Session
+from requests.adapters import BaseAdapter, HTTPAdapter
+from session_adapters.file_adapter import FileAdapter
+from session_adapters.http_conts import DEFAULT_ENCODING
+from session_adapters.oci_adapter import OCIAdapter
+from tempfile import NamedTemporaryFile
+from typing import Any, Mapping, TypeVar
+from transpiler_mate.metadata import MetadataManager
+from transpiler_mate.metadata.software_application_models import SoftwareApplication
+from transpiler_mate.ogcapi.records import OgcRecordsTranspiler
+from transpiler_mate.ogcapi.records.ogcapi_records_models import (
+    Coordinate,
+    Link,
+    MultipointGeoJSON,
+    RecordGeoJSON,
+)
+
+import yaml
+import json
+import time
+
+PENDING = {StatusCode.ACCEPTED, StatusCode.RUNNING}
+
+
+def create_client(ogc_api_endpoint: str, authorization_token: str | None) -> ApiClient:
+    return ApiClient(
+        configuration=Configuration(
+            host=ogc_api_endpoint,
+        ),
+        header_name="Authorization" if authorization_token else None,
+        header_value=f"Bearer {authorization_token}" if authorization_token else None,
+    )
+
+
+def retrieve_status_info(api_client: ApiClient, job_id: str) -> StatusInfo:
+    status_api = StatusApi(api_client=api_client)
+
+    logger.debug(f"Retrieving the Job {job_id} status...")
+
+    status_info: StatusInfo | None = None
+    while status_info is None or status_info.status in PENDING:
+        time.sleep(10)
+
+        status_info = status_api.get_status(job_id=job_id)
+
+        logger.debug(f"Job {job_id} status is {status_info.status}")
+
+    if StatusCode.SUCCESSFUL != status_info.status:
+        raise Exception(
+            f"Impossible to create the OGC API Records 'Experiment', job '{job_id}' terminated with status '{status_info.status}', report to your provider"
+        )
+
+    logger.success(f"Job {job_id} execution is complete.")
+    return status_info
+
+
+def load_record_geojson(
+    source: str, project_id: str, project_name: str
+) -> RecordGeoJSON:
+    session: Session = Session()
+
+    def mount_session(scheme: str, adapter: BaseAdapter):
+        logger.debug(f"Mounting '{scheme}' scheme to '{type(adapter).__name__}'...")
+        session.mount(scheme, adapter)
+        logger.debug(
+            f"Scheme '{scheme}' successfully mount to '{type(adapter).__name__}'"
+        )
+
+    http_adapter = HTTPAdapter()
+    mount_session("http://", http_adapter)
+    mount_session("https://", http_adapter)
+    mount_session("file://", FileAdapter())
+    mount_session("oci://", OCIAdapter())
+
+    logger.debug(f"> GET {source}...")
+
+    response = session.get(source, stream=True)
+    response.raise_for_status()
+
+    logger.debug(f"< {response.status_code} {response.reason}")
+    for k, v in response.headers.items():
+        logger.debug(f"< {k}: {v}")
+
+    # Read first 2 bytes to check for gzip
+    magic = response.raw.read(2)
+    remaining = response.raw.read()  # Read rest of the stream
+    combined = BytesIO(magic + remaining)
+
+    if b"\x1f\x8b" == magic:
+        logger.debug(f"gzip compression detected in response body from {source}")
+        buffer = GzipFile(fileobj=combined)
+    else:
+        buffer = combined
+
+    input_stream = TextIOWrapper(buffer, encoding=DEFAULT_ENCODING)
+
+    fd = NamedTemporaryFile(mode="w", suffix=".cwl", encoding=DEFAULT_ENCODING)
+
+    try:
+        tmp_path = Path(fd.name)
+
+        logger.debug(
+            f"Caching the CWL document to a temporary file on {tmp_path.absolute()}..."
+        )
+
+        with tmp_path.open("w") as output_stream:
+            output_stream.write(input_stream.read())
+
+        logger.success(f"CWL document stored to {tmp_path.absolute()} temporary file.")
+
+        logger.debug(
+            f"Reading Schema.org metadata from CWL document on {tmp_path.absolute()}..."
+        )
+
+        manager: MetadataManager = MetadataManager(tmp_path)
+        metadata: SoftwareApplication = manager.metadata
+
+        logger.success(
+            f"Schema.org metadata read from CWL document on {tmp_path.absolute()}."
+        )
+
+        logger.debug("Transpiling Schema.org metadata to OGCP API Records...")
+
+        transpiler: OgcRecordsTranspiler = OgcRecordsTranspiler()
+
+        data: Mapping[str, Any] = transpiler.transpile(metadata)
+
+        record_geojson: RecordGeoJSON = RecordGeoJSON.model_validate(
+            obj=data, by_alias=True
+        )
+
+        record_geojson.geometry = MultipointGeoJSON(
+            coordinates=[Coordinate([-180.0, -90.0, 180.0, 90.0])],
+        )
+
+        if record_geojson.properties.language:
+            record_geojson.properties.language.alternate = (
+                record_geojson.properties.language.name
+            )
+
+            record_geojson.properties.languages = [record_geojson.properties.language]
+
+        if record_geojson.properties.resource_languages:
+            for resource_language in record_geojson.properties.resource_languages:
+                resource_language.alternate = resource_language.name
+
+        if not record_geojson.links:
+            record_geojson.links = []
+
+        record_geojson.links.append(
+            Link(
+                rel="related",
+                href=f"../../projects/{project_id}/collection.json",
+                type="application/json",
+                title=f"Project: {project_name}",
+                hreflang="en-US",
+                created=None,
+                updated=None,
+            )
+        )
+        record_geojson.links.append(
+            Link(
+                rel="root",
+                href="../../catalog.json",
+                type="application/json",
+                title="Open Science Catalog",
+                hreflang="en-US",
+                created=None,
+                updated=None,
+            )
+        )
+
+        logger.success("Schema.org metadata transpiled to OGCP API Records.")
+
+        return record_geojson
+    finally:
+        fd.close()
+
+
+T = TypeVar("T", bound=BaseModel)
+
+
+def cast_model(src: BaseModel, dst_cls: type[T]) -> T:
+    # mode="python" preserves datetimes, URLs, etc.
+    data = src.model_dump(mode="python", by_alias=True, exclude_none=False)
+    return dst_cls.model_validate(data, by_alias=True)
+
+
+def dump_data(data: Mapping[str, Any], output: Path, rel: RelType = RelType.ITEM):
+    logger.info(f"Serializing OGC API Records to {output.absolute()}...")
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    with output.open("w") as output_stream:
+        json.dump(
+            data,
+            output_stream,
+            indent=2,
+        )
+
+    logger.success(f"OGC API Records serialized to {output.absolute()}.")
+
+    catalog_file = Path(output.parent.parent, "catalog.json")
+
+    if catalog_file.exists():
+        logger.info(f"Updating STAC Catalog from {output.absolute()}...")
+
+        href: str = f"./{data['id']}/record.json"
+
+        catalog: Catalog = Catalog.from_file(catalog_file)
+
+        # Check whether the same rel + href is already present
+        for link in catalog.links:
+            if link.get_href() == href:
+                logger.info(
+                    f"Link {href} already present in {output.absolute()}, update is not required."
+                )
+                return
+
+        catalog.add_link(
+            PystacLink(
+                rel=rel,
+                target=href,
+                media_type="application/json",
+                title=data["properties"]["title"]
+                if "properties" in data
+                else data["title"]
+                if "title" in data
+                else None,
+            )
+        )
+
+        logger.info(f"Saving STAC Catalog to {catalog_file.absolute()}...")
+        catalog.save_object(
+            include_self_link=False, dest_href=catalog_file.absolute().as_posix()
+        )
+        logger.success(f"STAC Catalog successfully saved to {catalog_file.absolute()}.")
+    else:
+        logger.warning(
+            f"Catalog file {catalog_file.absolute()} not found, skipping the update"
+        )
+
+
+def serialize_yaml(data: Any, target_file: Path):
+    target_file.parent.mkdir(parents=True, exist_ok=True)
+    with target_file.open("w") as output_stream:
+        yaml.dump(
+            data,
+            output_stream,
+        )

osc_metadata_client/cli.py

@@ -0,0 +1,161 @@
+# Copyright 2026 Terradue
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from osc_metadata_client import load_record_geojson
+from osc_metadata_client.experiment import execute as execute_experiment
+from osc_metadata_client.product import execute as execute_product
+from osc_metadata_client.workflow import execute as execute_workflow
+from pathlib import Path
+from transpiler_mate.cli.cli import _track
+from transpiler_mate.ogcapi.records.ogcapi_records_models import RecordGeoJSON
+
+import click
+
+
+@click.group()
+@click.argument("source", type=click.STRING, required=True)
+@click.option(
+    "--id", type=click.STRING, required=True, help="The OGC API Processes Job ID."
+)
+@click.option(
+    "--project-id",
+    type=click.STRING,
+    required=True,
+    help="The referencing Open Science Catalog project ID.",
+)
+@click.option(
+    "--project-name",
+    type=click.STRING,
+    required=True,
+    help="The referencing Open Science Catalog project Name.",
+)
+@click.option(
+    "--ogc-api-processes-endpoint",
+    type=click.STRING,
+    required=True,
+    help="The referencing OGC API Processes service URL.",
+)
+@click.option(
+    "--output",
+    type=click.Path(path_type=Path),
+    required=True,
+    help="The output directory path",
+)
+@click.pass_context
+def main(
+    ctx,
+    source: str,
+    id: str,
+    project_id: str,
+    project_name: str,
+    ogc_api_processes_endpoint: str,
+    output: Path,
+):
+    ctx.ensure_object(dict)
+    ctx.obj["source"] = source
+
+    record_geojson: RecordGeoJSON = load_record_geojson(
+        source, project_id, project_name
+    )
+    record_geojson.id = id
+    ctx.obj["record_geojson"] = record_geojson
+
+    ctx.obj["ogc-api-processes-endpoint"] = ogc_api_processes_endpoint
+    ctx.obj["project-id"] = project_id
+    ctx.obj["output"] = output
+
+
+@main.command(context_settings={"show_default": True})
+@click.pass_context
+def workflow(ctx):
+    source: str = ctx.obj["source"]
+    ogc_api_processes_endpoint = ctx.obj["ogc-api-processes-endpoint"]
+    record_geojson: RecordGeoJSON = ctx.obj["record_geojson"]
+    project_id: str = ctx.obj["project-id"]
+    output: Path = ctx.obj["output"]
+    execute_workflow(
+        source, ogc_api_processes_endpoint, record_geojson, project_id, output
+    )
+
+
+@main.command(context_settings={"show_default": True})
+@click.pass_context
+@click.option(
+    "--workflow-id",
+    type=click.STRING,
+    required=True,
+    help="The referencing OGC API Records workflow URL.",
+)
+@click.option(
+    "--authorization-token",
+    type=click.STRING,
+    required=False,
+    default=None,
+    help="Authorization JSON Web Token'",
+)
+def experiment(
+    ctx,
+    workflow_id: str,
+    authorization_token: str,
+):
+    ogc_api_processes_endpoint = ctx.obj["ogc-api-processes-endpoint"]
+    record_geojson: RecordGeoJSON = ctx.obj["record_geojson"]
+    project_id: str = ctx.obj["project-id"]
+    output: Path = ctx.obj["output"]
+    execute_experiment(
+        project_id=project_id,
+        workflow_id=workflow_id,
+        record_geojson=record_geojson,
+        ogc_api_processes_endpoint=ogc_api_processes_endpoint,
+        output=output,
+        authorization_token=authorization_token,
+    )
+
+
+@main.command(context_settings={"show_default": True})
+@click.pass_context
+@click.option(
+    "--experiment-id",
+    type=click.STRING,
+    required=True,
+    help="The referencing OGC API Records workflow ID.",
+)
+@click.option(
+    "--authorization-token",
+    type=click.STRING,
+    required=False,
+    default=None,
+    help="Authorization JSON Web Token'",
+)
+def products(
+    ctx,
+    experiment_id: str,
+    authorization_token: str,
+):
+    ogc_api_processes_endpoint = ctx.obj["ogc-api-processes-endpoint"]
+    record_geojson: RecordGeoJSON = ctx.obj["record_geojson"]
+    project_id: str = ctx.obj["project-id"]
+    output: Path = ctx.obj["output"]
+    execute_product(
+        ogc_api_processes_endpoint,
+        record_geojson,
+        project_id,
+        experiment_id,
+        output,
+        authorization_token,
+    )
+
+
+for command in [workflow, experiment, products]:
+    command.callback = _track(command.callback)

osc_metadata_client/experiment.py

@@ -0,0 +1,136 @@
+# Copyright 2026 Terradue
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from loguru import logger
+from osc_metadata_client import (
+    cast_model,
+    create_client,
+    retrieve_status_info,
+    dump_data,
+    serialize_yaml,
+)
+from ogc_api_processes_client.models.status_info import StatusInfo
+from osc_metadata_client.models import ExperimentProperties
+from pathlib import Path
+from transpiler_mate.ogcapi.records.ogcapi_records_models import Link, RecordGeoJSON
+
+
+def execute(
+    project_id: str,
+    workflow_id: str,
+    record_geojson: RecordGeoJSON,
+    ogc_api_processes_endpoint: str,
+    output: Path,
+    authorization_token: str,
+):
+    logger.debug("Enriching OGCP API Records...")
+
+    record_geojson.links.append(  # type: ignore see osc_metadata_client.load_record_geojson
+        Link(
+            rel="parent",
+            href="../catalog.json",
+            type="application/json",
+            title="Experiments",
+            hreflang="en-US",
+            created=None,
+            updated=None,
+        )
+    )
+    record_geojson.links.append(  # type: ignore see osc_metadata_client.load_record_geojson
+        Link(
+            href=f"{ogc_api_processes_endpoint}/jobs/{record_geojson.id}",
+            hreflang="en-US",
+            rel="via",
+            type="application/json",
+            title=f"OGC API Processes - Job: {record_geojson.id}",
+            created=None,
+            updated=None,
+        )
+    )
+
+    logger.debug("Reassembling OGC API Records 'Experiment' inputs...")
+
+    status_info: StatusInfo = retrieve_status_info(
+        create_client(ogc_api_processes_endpoint, authorization_token),
+        record_geojson.id,
+    )
+
+    logger.debug(status_info.properties)
+
+    target_file = Path(output, f"experiments/{record_geojson.id}/record.json")
+    input_files: Path = Path(target_file.parent, "input.yaml")
+
+    serialize_yaml(status_info.inputs, input_files)
+
+    record_geojson.links.append(  # type: ignore see osc_metadata_client.load_record_geojson
+        Link(
+            href=f"./{input_files.name}",
+            hreflang="en-US",
+            rel="input",
+            type="application/yaml",
+            title="Input parameters",
+            created=status_info.started,
+            updated=status_info.started,
+        )
+    )
+    record_geojson.links.append(  # type: ignore see osc_metadata_client.load_record_geojson
+        Link(
+            href="./environment.yaml",
+            hreflang="en-US",
+            rel="environment",
+            type="application/yaml",
+            title="Execution environment",
+            created=status_info.started,
+            updated=status_info.started,
+        )
+    )
+    record_geojson.links.append(  # type: ignore see osc_metadata_client.load_record_geojson
+        Link(
+            href=f"../../workflows/{workflow_id}/record.json",
+            hreflang="en-US",
+            rel="related",
+            type="application/json",
+            title=f"Workflow: {record_geojson.properties.title}",
+            created=status_info.started,
+            updated=status_info.started,
+        )
+    )
+
+    logger.success(
+        f"OGC API Records 'Experiment' inputs saved to {input_files.absolute()}"
+    )
+
+    record_geojson.properties.type = "experiment"
+    experiment_properties: ExperimentProperties = cast_model(
+        record_geojson.properties,
+        ExperimentProperties,
+    )
+    experiment_properties.osc_project = project_id
+    experiment_properties.osc_workflow = workflow_id
+    experiment_properties.osc_prov_described_by_workflow = workflow_id
+    # experiment_properties.osc_prov_generated = "TODO"
+    experiment_properties.osc_prov_generated_by = "osc-client"
+    experiment_properties.osc_prov_started_at_time = status_info.started
+    experiment_properties.osc_prov_ended_at_time = status_info.finished
+
+    record_geojson.properties = experiment_properties
+
+    logger.success("OGCP API Records enriched")
+
+    dump_data(
+        record_geojson.model_dump(
+            by_alias=True, exclude_none=True, serialize_as_any=True
+        ),
+        target_file,
+    )

osc_metadata_client/models.py

@@ -0,0 +1,78 @@
+# Copyright 2026 Terradue
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# generated by datamodel-codegen:
+#   filename:  osc_ogcapi_records_models.yaml
+#   timestamp: 2026-03-26T09:49:14+00:00
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import Field, RootModel
+from transpiler_mate.ogcapi.records.ogcapi_records_models import RecordCommonProperties
+
+
+class Model(RootModel[Any]):
+    root: Any
+
+
+class OscStatus(Enum):
+    PLANNED = "planned"
+    ONGOING = "ongoing"
+    COMPLETED = "completed"
+
+
+class WorkflowProperties(RecordCommonProperties):
+    osc_type: Optional[str] = Field("workflow", alias="osc:type")
+    osc_project: Optional[str] = Field(None, alias="osc:project")
+    osc_status: Optional[OscStatus] = Field(None, alias="osc:status")
+
+
+class ExperimentProperties(RecordCommonProperties):
+    osc_project: Optional[str] = Field(None, alias="osc:project")
+    osc_workflow: Optional[str] = Field(None, alias="osc:workflow")
+    osc_prov_generated_by: Optional[str] = Field(None, alias="osc-prov:generatedBy")
+    osc_prov_started_at_time: Optional[datetime] = Field(
+        None, alias="osc-prov:startedAtTime"
+    )
+    osc_prov_ended_at_time: Optional[datetime] = Field(
+        None, alias="osc-prov:endedAtTime"
+    )
+    osc_prov_generated: Optional[str] = Field(None, alias="osc-prov:generated")
+    osc_prov_described_by_workflow: Optional[str] = Field(
+        None, alias="osc-prov:describedByWorkflow"
+    )
+
+
+class ProductProperties(RecordCommonProperties):
+    osc_experiment: Optional[str] = Field(None, alias="osc:experiment")
+    osc_status: Optional[str] = Field(None, alias="osc:status")
+    osc_region: Optional[str] = Field(None, alias="osc:region")
+    osc_type: Optional[str] = Field(None, alias="osc:type")
+    osc_project: Optional[str] = Field(None, alias="osc:project")
+    osc_missions: Optional[str] = Field(None, alias="osc:missions")
+    osc_variables: Optional[str] = Field(None, alias="osc:variables")
+    osc_prov_type: Optional[str] = Field(None, alias="osc-prov:type")
+    osc_prov_was_derived_from: Optional[str] = Field(
+        None, alias="osc-prov:wasDerivedFrom"
+    )
+    osc_prov_was_output_from: Optional[str] = Field(
+        None, alias="osc-prov:wasOutputFrom"
+    )
+    osc_prov_described_by_parameter: Optional[str] = Field(
+        None, alias="osc-prov:describedByParameter"
+    )