fairagro-middleware-shared 10.0.3.dev26__tar.gz → 11.0.1.dev27__tar.gz

{fairagro_middleware_shared-10.0.3.dev26 → fairagro_middleware_shared-11.0.1.dev27}/.gitignore

@@ -228,6 +228,3 @@ helmchart/**/client_ext.conf
 .cache_ggshield
 
 docker/Dockerfile.api.bak
-# Docker DinD: ignore runtime state in docker-config, keep minimal config.json tracked
-.devcontainer/docker-config/*
-!.devcontainer/docker-config/config.json

{fairagro_middleware_shared-10.0.3.dev26 → fairagro_middleware_shared-11.0.1.dev27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fairagro-middleware-shared
-Version: 10.0.3.dev26
+Version: 11.0.1.dev27
 Summary: The FAIRagro advanced middleware shared components
 Requires-Python: >=3.12
 Requires-Dist: opentelemetry-api>=1.26.0

fairagro_middleware_shared-11.0.1.dev27/src/middleware/shared/report/__init__.py

@@ -0,0 +1,19 @@
+"""Harvest run report model, serializers, and stdout emission."""
+
+from middleware.shared.report.emit import print_report
+from middleware.shared.report.formats.base import ReportSerializer
+from middleware.shared.report.formats.jsonld import (
+    FAIRAGRO_HARVEST_REPORT_NS,
+    JsonLdReportSerializer,
+)
+from middleware.shared.report.model import FailedRecord, HarvestReport, RepositoryReport
+
+__all__ = [
+    "FAIRAGRO_HARVEST_REPORT_NS",
+    "FailedRecord",
+    "HarvestReport",
+    "JsonLdReportSerializer",
+    "ReportSerializer",
+    "RepositoryReport",
+    "print_report",
+]

fairagro_middleware_shared-11.0.1.dev27/src/middleware/shared/report/emit.py

@@ -0,0 +1,26 @@
+"""Stdout emission for harvest reports."""
+
+from __future__ import annotations
+
+import logging
+
+from middleware.shared.report.formats.base import ReportSerializer
+from middleware.shared.report.formats.jsonld import JsonLdReportSerializer
+from middleware.shared.report.model import HarvestReport
+
+logger = logging.getLogger(__name__)
+
+
+def print_report(
+    report: HarvestReport,
+    serializer: ReportSerializer | None = None,
+) -> None:
+    """Serialize the report and print it to stdout.
+
+    Serialization or print failures are logged as warnings and do not raise.
+    """
+    active_serializer: ReportSerializer = serializer or JsonLdReportSerializer()
+    try:
+        print(active_serializer.render(report))
+    except Exception as exc:  # noqa: BLE001 — MUST NOT fail the process (harvest-report)
+        logger.warning("Failed to serialise harvest report: %s", exc)

fairagro_middleware_shared-11.0.1.dev27/src/middleware/shared/report/formats/__init__.py

@@ -0,0 +1,9 @@
+"""Report format serializers."""
+
+from middleware.shared.report.formats.base import ReportSerializer
+from middleware.shared.report.formats.jsonld import JsonLdReportSerializer
+
+__all__ = [
+    "JsonLdReportSerializer",
+    "ReportSerializer",
+]

fairagro_middleware_shared-11.0.1.dev27/src/middleware/shared/report/formats/base.py

@@ -0,0 +1,15 @@
+"""Report serializer protocol."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from middleware.shared.report.model import HarvestReport
+
+
+class ReportSerializer(Protocol):
+    """Serialize a harvest report to a string in a specific format."""
+
+    def render(self, report: HarvestReport) -> str:
+        """Return the serialized report document."""
+        raise NotImplementedError

fairagro_middleware_shared-11.0.1.dev27/src/middleware/shared/report/formats/jsonld.py

@@ -0,0 +1,86 @@
+"""JSON-LD harvest report serializer."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+from middleware.shared.report.model import FailedRecord, HarvestReport, RepositoryReport
+
+FAIRAGRO_HARVEST_REPORT_NS = "https://fairagro.github.io/m4.2_advanced_middleware_api/ns/harvest-report/v1/#"
+
+_JSON_LD_CONTEXT = {
+    "@vocab": "https://schema.org/",
+    "schema": "https://schema.org/",
+    "fairagro": FAIRAGRO_HARVEST_REPORT_NS,
+}
+
+
+def _format_iso_duration(seconds: float) -> str:
+    """Serialize a duration value as an ISO 8601 duration string."""
+    remainder = f"{seconds:.6f}".rstrip("0").rstrip(".")
+    if remainder == "":
+        remainder = "0"
+    return f"PT{remainder}S"
+
+
+def _format_iso_timestamp(value: datetime) -> str:
+    """Format a UTC datetime value as an ISO 8601 timestamp ending in Z."""
+    return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _failed_record_to_jsonld(record: FailedRecord) -> dict[str, Any]:
+    """Convert a failed record to its JSON-LD object, omitting unset fields."""
+    result: dict[str, Any] = {"fairagro:message": record.message}
+    if record.record_id:
+        result["fairagro:recordId"] = record.record_id
+    if record.url:
+        result["fairagro:url"] = record.url
+    return result
+
+
+def _repository_to_jsonld(report: RepositoryReport) -> dict[str, Any]:
+    """Convert a repository report to a schema.org EntryPoint JSON-LD object."""
+    result: dict[str, Any] = {
+        "@type": "schema:EntryPoint",
+        "name": report.rdi,
+        "identifier": report.rdi,
+        "schema:duration": _format_iso_duration(report.duration_seconds),
+        "fairagro:harvestId": report.harvest_id,
+        "fairagro:skippedDatasets": report.skipped_datasets,
+    }
+    if report.harvested_datasets is not None:
+        result["fairagro:harvestedDatasets"] = report.harvested_datasets
+    if report.expected_datasets is not None:
+        result["fairagro:expectedDatasets"] = report.expected_datasets
+    if report.failed_datasets is not None:
+        result["fairagro:failedDatasets"] = report.failed_datasets
+    if report.total_studies is not None:
+        result["fairagro:totalStudies"] = report.total_studies
+    if report.total_assays is not None:
+        result["fairagro:totalAssays"] = report.total_assays
+    if report.failed_records:
+        result["fairagro:failedRecords"] = [_failed_record_to_jsonld(record) for record in report.failed_records]
+    return result
+
+
+class JsonLdReportSerializer:
+    """Render a harvest report as an operator-readable JSON-LD document."""
+
+    def __init__(self, *, indent: int = 2) -> None:
+        """Configure JSON indentation for operator-readable output."""
+        self._indent = indent
+
+    def render(self, report: HarvestReport) -> str:
+        """Return the report as indented JSON-LD text."""
+        document: dict[str, Any] = {
+            "@context": _JSON_LD_CONTEXT,
+            "@type": "schema:Action",
+            "name": report.name,
+            "schema:startTime": _format_iso_timestamp(report.start_time),
+            "schema:endTime": _format_iso_timestamp(report.end_time),
+            "fairagro:harvestDurationSeconds": report.duration_seconds,
+            "schema:result": [_repository_to_jsonld(entry) for entry in report.repository_reports],
+        }
+        return json.dumps(document, ensure_ascii=False, indent=self._indent)

fairagro_middleware_shared-11.0.1.dev27/src/middleware/shared/report/model.py

@@ -0,0 +1,46 @@
+"""Format-neutral harvest run report domain model."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass(frozen=True)
+class FailedRecord:
+    """A single dataset that failed during harvesting."""
+
+    message: str
+    record_id: str | None = None
+    url: str | None = None
+
+
+@dataclass(frozen=True)
+class RepositoryReport:  # pylint: disable=too-many-instance-attributes
+    """Execution statistics for a single harvested repository."""
+
+    rdi: str
+    harvest_id: str | None
+    duration_seconds: float
+    expected_datasets: int | None = None
+    harvested_datasets: int | None = None
+    failed_datasets: int | None = None
+    skipped_datasets: int = 0
+    failed_records: tuple[FailedRecord, ...] = ()
+    total_studies: int | None = None
+    total_assays: int | None = None
+
+
+@dataclass(frozen=True)
+class HarvestReport:
+    """Summary statistics for an entire harvest run."""
+
+    start_time: datetime
+    end_time: datetime
+    repository_reports: tuple[RepositoryReport, ...] = ()
+    name: str = "FAIRagro Harvest Run"
+
+    @property
+    def duration_seconds(self) -> float:
+        """Return the total harvest run duration in seconds."""
+        return (self.end_time - self.start_time).total_seconds()

fairagro_middleware_shared-11.0.1.dev27/tests/unit/test_harvest_report.py

@@ -0,0 +1,224 @@
+"""Unit tests for the shared harvest report library."""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import fields
+from datetime import UTC, datetime
+from typing import Any
+
+import pytest
+
+from middleware.shared.report import (
+    FAIRAGRO_HARVEST_REPORT_NS,
+    FailedRecord,
+    HarvestReport,
+    JsonLdReportSerializer,
+    RepositoryReport,
+    print_report,
+)
+
+_START = datetime(2026, 5, 6, 14, 0, 0, tzinfo=UTC)
+_END = datetime(2026, 5, 6, 14, 3, 45, tzinfo=UTC)
+
+
+def _sample_repo(**overrides: Any) -> RepositoryReport:
+    """Build a repository report with sensible defaults."""
+    values: dict[str, Any] = {
+        "rdi": "bonares",
+        "harvest_id": "harvest-1",
+        "duration_seconds": 12.3,
+        "expected_datasets": 100,
+        "harvested_datasets": 95,
+        "failed_datasets": 5,
+        "skipped_datasets": 2,
+        "failed_records": (
+            FailedRecord(
+                message="map failed",
+                record_id="frl:123",
+                url="https://example.test/frl:123",
+            ),
+        ),
+    }
+    values.update(overrides)
+    return RepositoryReport(**values)
+
+
+def _sample_report(
+    repositories: tuple[RepositoryReport, ...] | None = None,
+) -> HarvestReport:
+    """Build a harvest run report spanning a fixed UTC window."""
+    return HarvestReport(
+        start_time=_START,
+        end_time=_END,
+        repository_reports=((_sample_repo(),) if repositories is None else repositories),
+    )
+
+
+def test_failed_record_with_optional_identifiers() -> None:
+    """Failed records expose message, record id, and URL when provided."""
+    record = FailedRecord(message="boom", record_id="id-1", url="https://x.test")
+    assert record.message == "boom"
+    assert record.record_id == "id-1"
+    assert record.url == "https://x.test"
+
+
+def test_failed_record_message_only() -> None:
+    """Optional identifiers are unset when omitted."""
+    record = FailedRecord(message="boom")
+    assert record.record_id is None
+    assert record.url is None
+
+
+def test_harvest_report_with_one_repository() -> None:
+    """A run report exposes timing and a single repository entry."""
+    report = _sample_report()
+    assert report.start_time == _START
+    assert len(report.repository_reports) == 1
+    assert report.repository_reports[0].rdi == "bonares"
+    assert report.duration_seconds == (_END - _START).total_seconds()
+
+
+def test_harvest_report_with_no_repositories() -> None:
+    """An empty repository list is preserved."""
+    report = _sample_report(repositories=())
+    assert not report.repository_reports
+    assert report.duration_seconds == (_END - _START).total_seconds()
+
+
+def test_optional_study_and_assay_counts_set() -> None:
+    """Study and assay totals are available when provided."""
+    total_studies = 10
+    total_assays = 20
+    repo = _sample_repo(total_studies=total_studies, total_assays=total_assays)
+    assert repo.total_studies == total_studies
+    assert repo.total_assays == total_assays
+
+
+def test_optional_study_and_assay_counts_unset() -> None:
+    """Unset study and assay totals are None, not zero."""
+    repo = _sample_repo()
+    assert repo.total_studies is None
+    assert repo.total_assays is None
+
+
+def test_model_is_format_neutral() -> None:
+    """The domain model does not embed a serialized document."""
+    report = _sample_report()
+    assert not hasattr(report, "to_jsonld")
+    assert "repository_reports" in {field.name for field in fields(report)}
+
+
+def test_jsonld_context_and_types() -> None:
+    """JSON-LD uses schema.org Action with EntryPoint results."""
+    document = json.loads(JsonLdReportSerializer().render(_sample_report()))
+    assert document["@context"]["@vocab"] == "https://schema.org/"
+    assert document["@context"]["schema"] == "https://schema.org/"
+    assert document["@context"]["fairagro"] == FAIRAGRO_HARVEST_REPORT_NS
+    assert document["@context"]["fairagro"].endswith("/ns/harvest-report/v1/#")
+    assert document["@type"] == "schema:Action"
+    assert document["schema:result"][0]["@type"] == "schema:EntryPoint"
+
+
+def test_jsonld_timestamps_and_durations() -> None:
+    """Timestamps end with Z and durations use ISO 8601 / seconds."""
+    report = _sample_report()
+    document = json.loads(JsonLdReportSerializer().render(report))
+    assert document["schema:startTime"].endswith("Z")
+    assert document["schema:endTime"].endswith("Z")
+    assert document["fairagro:harvestDurationSeconds"] == report.duration_seconds
+    assert document["schema:result"][0]["schema:duration"].startswith("PT")
+
+
+def test_jsonld_metrics_and_failed_records() -> None:
+    """Fairagro metrics and nested failed records are emitted."""
+    repo = _sample_repo()
+    entry = json.loads(JsonLdReportSerializer().render(_sample_report(repositories=(repo,))))["schema:result"][0]
+    assert entry["fairagro:harvestId"] == repo.harvest_id
+    assert entry["fairagro:expectedDatasets"] == repo.expected_datasets
+    assert entry["fairagro:harvestedDatasets"] == repo.harvested_datasets
+    assert entry["fairagro:failedDatasets"] == repo.failed_datasets
+    assert entry["fairagro:skippedDatasets"] == repo.skipped_datasets
+    failed = repo.failed_records[0]
+    assert entry["fairagro:failedRecords"] == [
+        {
+            "fairagro:message": failed.message,
+            "fairagro:recordId": failed.record_id,
+            "fairagro:url": failed.url,
+        }
+    ]
+
+
+def test_jsonld_optional_study_and_assay_totals() -> None:
+    """Optional study and assay totals appear as fairagro properties."""
+    total_studies = 3
+    total_assays = 7
+    report = _sample_report(repositories=(_sample_repo(total_studies=total_studies, total_assays=total_assays),))
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert entry["fairagro:totalStudies"] == total_studies
+    assert entry["fairagro:totalAssays"] == total_assays
+
+
+def test_jsonld_omits_unset_expected_datasets() -> None:
+    """Unset optional counts are omitted rather than null."""
+    report = _sample_report(
+        repositories=(
+            _sample_repo(
+                expected_datasets=None,
+                harvested_datasets=None,
+                failed_datasets=None,
+                total_studies=None,
+                total_assays=None,
+            ),
+        )
+    )
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert "fairagro:expectedDatasets" not in entry
+    assert "fairagro:harvestedDatasets" not in entry
+    assert "fairagro:failedDatasets" not in entry
+    assert "fairagro:totalStudies" not in entry
+    assert "fairagro:totalAssays" not in entry
+    assert "fairagro:harvestId" in entry
+
+
+def test_jsonld_omits_empty_failed_records() -> None:
+    """Empty failed-record lists are omitted from JSON-LD."""
+    report = _sample_report(repositories=(_sample_repo(failed_records=()),))
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert "fairagro:failedRecords" not in entry
+
+
+def test_jsonld_empty_result_array() -> None:
+    """A run with no repositories emits an empty result array."""
+    document = json.loads(JsonLdReportSerializer().render(_sample_report(repositories=())))
+    assert document["schema:result"] == []
+
+
+def test_print_report_writes_jsonld_to_stdout(capsys: pytest.CaptureFixture[str]) -> None:
+    """print_report writes a JSON-LD document to stdout."""
+    print_report(_sample_report())
+    captured = capsys.readouterr()
+    document = json.loads(captured.out)
+    assert document["@type"] == "schema:Action"
+    assert captured.err == ""
+
+
+def test_print_report_does_not_raise_on_serialization_failure(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Serialization failures are logged and do not propagate."""
+
+    class _BoomSerializer:
+        def __init__(self) -> None:
+            self._fail = True
+
+        def render(self, report: HarvestReport) -> str:
+            if self._fail:
+                raise RuntimeError(f"cannot serialise {type(report).__name__}")
+            return "{}"
+
+    with caplog.at_level(logging.WARNING):
+        print_report(_sample_report(), serializer=_BoomSerializer())
+
+    assert any("Failed to serialise harvest report" in message for message in caplog.messages)