marinerg-data 0.1.0__py3-none-any.whl

marinerg_data/__init__.py

@@ -0,0 +1,14 @@
+"""marinerg-data: facility-side convert / validate / publish tool.
+
+Design: docs/domain-metadata-roadmap.md "Submission & ingest architecture" in
+the data-access-service repo. Validation runs locally at the facility before any
+upload; publication is thin orchestration over the facility's own Zenodo account
+plus CKAN registration. Bulk data never transits the e-infrastructure.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("marinerg-data")
+except PackageNotFoundError:  # not installed (e.g. running from a source tree)
+    __version__ = "0.0.0"

marinerg_data/bundle.py

@@ -0,0 +1,159 @@
+"""A publishable bundle: one campaign directory.
+
+Layout expected by ``marinerg-data publish <dir>``:
+
+    <dir>/dataset.yaml           campaign-level metadata (this module's model)
+    <dir>/*.manifest.json        one run manifest per run (schema v0.1)
+    <dir>/...                    data files referenced by the manifests
+
+The CKAN dataset represents the campaign; runs are structured resources
+within it (domain-metadata-roadmap.md design principle 6).
+
+Publish state (Zenodo record id + DOIs) is written to
+``<dir>/.marinerg-publish.json`` so ``publish --new-version`` knows which
+record to version. The state file is local bookkeeping, not metadata — it
+never gets uploaded.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+from .manifest import (
+    ManifestError,
+    find_dataset_metadata,
+    find_manifests,
+    load_manifest,
+)
+
+STATE_FILE_NAME = ".marinerg-publish.json"
+
+
+class Creator(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    affiliation: str | None = None
+    orcid: str | None = None
+
+
+class DatasetMetadata(BaseModel):
+    """Campaign-level metadata; the manifest carries per-run detail."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str = Field(description="CKAN dataset slug, e.g. 'gwk-floatx-2026'")
+    title: str
+    description: str
+    creators: list[Creator] = Field(min_length=1)
+    license: str = "CC-BY-4.0"
+    keywords: list[str] = Field(default_factory=list)
+    visibility: str = "public"
+    community: str = "marinerg-i"
+
+    facility_ref: str | None = None
+    facility_name: str | None = None
+    equipment_name: str | None = None
+    owner_org: str | None = None
+
+    site: str | None = None
+    feature_type: str | None = None
+    processing_level: str | None = None
+    coordinate_reference_system: str | None = None
+    data_mode: str | None = None
+
+    # Test Context (domain-metadata-roadmap short term). Passed to CKAN as
+    # extras until the scheming field group lands, then as top-level fields.
+    test_type: str | None = None
+    device_type: str | None = None
+    device_scale: str | None = None
+    campaign_name: str | None = None
+    test_report_ref: str | None = None
+
+
+class PublishState(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    record_id: str
+    doi: str
+    concept_doi: str | None = None
+    ckan_id: str | None = None
+    published_at: str | None = None
+    sandbox: bool = False
+
+
+class BundleError(Exception):
+    pass
+
+
+class Bundle:
+    def __init__(
+        self,
+        directory: Path,
+        metadata: DatasetMetadata,
+        manifests: dict[Path, dict[str, Any]],
+    ):
+        self.directory = directory
+        self.metadata = metadata
+        self.manifests = manifests
+
+    @classmethod
+    def load(cls, directory: Path) -> Bundle:
+        if not directory.is_dir():
+            raise BundleError(f"Not a directory: {directory}")
+        metadata_path = find_dataset_metadata(directory)
+        if metadata_path is None:
+            raise BundleError(
+                f"No dataset.yaml in {directory} — the bundle needs "
+                "campaign-level metadata (title, creators, license...)"
+            )
+        raw = yaml.safe_load(metadata_path.read_text(encoding="utf-8")) or {}
+        metadata = DatasetMetadata(**raw)
+        manifests = {}
+        try:
+            for manifest_path in find_manifests(directory):
+                manifests[manifest_path] = load_manifest(manifest_path)
+        except ManifestError as exc:
+            raise BundleError(str(exc)) from exc
+        if not manifests:
+            raise BundleError(f"No *.manifest.json files in {directory}")
+        return cls(directory, metadata, manifests)
+
+    def upload_files(self) -> list[Path]:
+        """Files that belong in the Zenodo record: every manifest, plus all
+        zenodo-hosted resources, media, documents, and reference sidecars they
+        reference."""
+        files: list[Path] = sorted(self.manifests)
+        seen = set(files)
+        for manifest_path, manifest in sorted(self.manifests.items()):
+            base = manifest_path.parent
+            for section in ("resources", "media", "documents", "references"):
+                for entry in manifest.get(section, []):
+                    if entry.get("hosted", "zenodo") != "zenodo":
+                        continue
+                    local = base / entry["path"]
+                    if local not in seen:
+                        files.append(local)
+                        seen.add(local)
+        return files
+
+    @property
+    def state_path(self) -> Path:
+        return self.directory / STATE_FILE_NAME
+
+    def read_state(self) -> PublishState | None:
+        if not self.state_path.is_file():
+            return None
+        return PublishState(**json.loads(self.state_path.read_text(encoding="utf-8")))
+
+    def write_state(self, state: PublishState) -> None:
+        state.published_at = datetime.now(UTC).isoformat()
+        self.state_path.write_text(
+            json.dumps(state.model_dump(), indent=2) + "\n", encoding="utf-8"
+        )

marinerg_data/ckan.py

@@ -0,0 +1,272 @@
+"""CKAN catalogue registration.
+
+Maps a published bundle onto the marinerg scheming schema. Conventions
+follow infra/ckan/scripts/seed_ckan.py: scheming fields go as top-level
+package keys, everything else into the extras list (scheming rejects
+extras whose key collides with a schema field).
+
+Record semantics (data-access-service/CLAUDE.md): the version DOI is the
+record's primary identifier; the concept DOI travels as an extra until the
+record-semantics scheming fields land. Test Context and Forcing values are
+extras for the same reason — one home each, promoted when the scheming
+groups are added.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import requests
+
+from .bundle import Bundle
+from .zenodo import PublishedRecord
+
+# Keep in sync with ckanext-marinerg/ckanext/marinerg/schemas/dataset.yaml
+# (same list as infra/ckan/scripts/seed_ckan.py).
+SCHEMING_FIELDS = {
+    "zenodo_doi",
+    "publication_date",
+    "visibility",
+    "access_scope",
+    "data_mode",
+    "source_kind",
+    "facility_ref",
+    "facility_name",
+    "equipment_name",
+    "feature_type",
+    "site",
+    "time_coverage_start",
+    "time_coverage_end",
+    "processing_level",
+    "coordinate_reference_system",
+    # Test Context + Forcing groups
+    "test_type",
+    "device_type",
+    "device_scale",
+    "device_trl",
+    "campaign_name",
+    "campaign_id",
+    "test_report_ref",
+    "wave_type",
+    "wave_height_m",
+    "wave_period_s",
+    "water_depth_m",
+    "current_speed_ms",
+    "wind_speed_ms",
+    "sample_frequencies_hz",
+    "run_count",
+}
+
+
+class CkanError(Exception):
+    pass
+
+
+class CkanConflict(CkanError):  # noqa: N818 — established exception name
+    """package_create hit an existing dataset with the same name."""
+
+
+def _json_value(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, (str, int, float, bool)):
+        return str(value)
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _single_or_list(values: list[Any]) -> Any:
+    unique = sorted({value for value in values if value is not None}, key=str)
+    if not unique:
+        return None
+    if len(unique) == 1:
+        return unique[0]
+    return unique
+
+
+def _aggregate_manifests(bundle: Bundle) -> dict[str, Any]:
+    """Campaign-level summary of the per-run manifests."""
+    manifests = list(bundle.manifests.values())
+
+    starts = sorted(
+        manifest.get("run", {}).get("started")
+        for manifest in manifests
+        if manifest.get("run", {}).get("started")
+    )
+    values: dict[str, Any] = {"run_count": len(manifests)}
+    if starts:
+        values["time_coverage_start"] = starts[0]
+        values["time_coverage_end"] = starts[-1]
+
+    for key in (
+        "wave_type",
+        "wave_height_m",
+        "wave_period_s",
+        "water_depth_m",
+        "current_speed_ms",
+        "wind_speed_ms",
+    ):
+        value = _single_or_list(
+            [manifest.get("forcing", {}).get(key) for manifest in manifests]
+        )
+        if value is not None:
+            values[key] = value
+
+    rates = sorted(
+        {
+            channel.get("sample_rate_hz")
+            for manifest in manifests
+            for channel in manifest.get("channels", [])
+            if channel.get("sample_rate_hz")
+        }
+    )
+    if rates:
+        values["sample_frequencies_hz"] = rates
+
+    parameters = sorted(
+        {
+            channel.get("name")
+            for manifest in manifests
+            for channel in manifest.get("channels", [])
+            if channel.get("name")
+        }
+    )
+    if parameters:
+        values["measured_parameters"] = parameters
+    return values
+
+
+def _resources(bundle: Bundle, record: PublishedRecord) -> list[dict[str, Any]]:
+    resources = []
+    for manifest_path, manifest in sorted(bundle.manifests.items()):
+        run_id = manifest.get("run", {}).get("id", manifest_path.stem)
+        for entry in manifest.get("resources", []):
+            if entry.get("hosted", "zenodo") != "zenodo":
+                continue
+            name = entry["path"].rsplit("/", 1)[-1]
+            url = (
+                f"{record.html_url}/files/{name}"
+                if record.html_url
+                else f"https://doi.org/{record.doi}"
+            )
+            resource = {
+                "name": name,
+                "url": url,
+                "format": entry.get("format", ""),
+                "description": f"Run {run_id} ({entry.get('record_type', 'data')})",
+            }
+            resources.append({k: v for k, v in resource.items() if v})
+        manifest_url = (
+            f"{record.html_url}/files/{manifest_path.name}"
+            if record.html_url
+            else f"https://doi.org/{record.doi}"
+        )
+        resources.append(
+            {
+                "name": manifest_path.name,
+                "url": manifest_url,
+                "format": "JSON",
+                "description": f"Run manifest for run {run_id} "
+                "(MARINERG-i run-manifest schema)",
+            }
+        )
+    return resources
+
+
+def build_package(bundle: Bundle, record: PublishedRecord) -> dict[str, Any]:
+    metadata = bundle.metadata
+    package: dict[str, Any] = {
+        "name": metadata.name,
+        "title": metadata.title,
+        "notes": metadata.description.strip(),
+        "license_id": metadata.license,
+        "private": metadata.visibility == "private",
+        "visibility": metadata.visibility,
+        "url": f"https://doi.org/{record.doi}",
+        "tags": [{"name": keyword} for keyword in metadata.keywords],
+        "zenodo_doi": record.doi,
+        "source_kind": "zenodo",
+        "author": metadata.creators[0].name,
+        "resources": _resources(bundle, record),
+    }
+    if metadata.owner_org:
+        package["owner_org"] = metadata.owner_org
+
+    fields = {
+        "facility_ref": metadata.facility_ref,
+        "facility_name": metadata.facility_name,
+        "equipment_name": metadata.equipment_name,
+        "site": metadata.site,
+        "feature_type": metadata.feature_type,
+        "processing_level": metadata.processing_level,
+        "coordinate_reference_system": metadata.coordinate_reference_system,
+        "data_mode": metadata.data_mode,
+        "test_type": metadata.test_type,
+        "device_type": metadata.device_type,
+        "device_scale": metadata.device_scale,
+        "campaign_name": metadata.campaign_name,
+        "test_report_ref": metadata.test_report_ref,
+    }
+
+    extras: dict[str, Any] = {}
+    if record.concept_doi:
+        extras["concept_doi"] = record.concept_doi
+
+    for key, value in _aggregate_manifests(bundle).items():
+        # Aggregates that differ across runs come back as lists; a scheming
+        # numeric field holds one value, so lists stay in extras (except
+        # sample_frequencies_hz, whose scheming field is a list).
+        is_field_shaped = not isinstance(value, list) or key == "sample_frequencies_hz"
+        if key in SCHEMING_FIELDS and is_field_shaped:
+            fields[key] = value
+        else:
+            extras[key] = value
+
+    for key, value in fields.items():
+        if value is not None:
+            package[key] = _json_value(value)
+    package["extras"] = [
+        {"key": key, "value": _json_value(value)} for key, value in extras.items()
+    ]
+    return package
+
+
+class CkanRegistrar:
+    def __init__(self, url: str, token: str, session: requests.Session | None = None):
+        if not url:
+            raise CkanError("A CKAN URL is required (set CKAN_URL)")
+        if not token:
+            raise CkanError("A CKAN API token is required (set CKAN_TOKEN)")
+        self.base_url = url.rstrip("/")
+        self.session = session or requests.Session()
+        self.session.headers.update({"Authorization": token})
+
+    def _call(self, action: str, payload: dict[str, Any]) -> dict[str, Any]:
+        url = f"{self.base_url}/api/3/action/{action}"
+        response = self.session.post(url, json=payload, timeout=60)
+        if response.status_code >= 400 and response.status_code != 409:
+            try:
+                detail = response.json().get("error")
+            except ValueError:
+                detail = response.text
+            raise CkanError(f"{action} -> {response.status_code}: {detail}")
+        body = response.json()
+        if not body.get("success", False):
+            if response.status_code == 409:
+                raise CkanConflict(body.get("error", {}))
+            raise CkanError(body.get("error", {}).get("message", f"{action} failed"))
+        return body["result"]
+
+    def register(self, package: dict[str, Any]) -> dict[str, Any]:
+        try:
+            return self._call("package_create", package)
+        except CkanConflict:
+            existing = self._call("package_show", {"id": package["name"]})
+            payload = dict(package)
+            payload["id"] = existing["id"]
+            return self._call("package_update", payload)
+
+    def update(self, ckan_id: str, package: dict[str, Any]) -> dict[str, Any]:
+        payload = dict(package)
+        payload["id"] = ckan_id
+        return self._call("package_update", payload)