fairagro-middleware-shared 11.0.1.dev28__tar.gz → 11.0.1.dev30__tar.gz

{fairagro_middleware_shared-11.0.1.dev28 → fairagro_middleware_shared-11.0.1.dev30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fairagro-middleware-shared
-Version: 11.0.1.dev28
+Version: 11.0.1.dev30
 Summary: The FAIRagro advanced middleware shared components
 Requires-Python: >=3.12
 Requires-Dist: opentelemetry-api>=1.26.0

{fairagro_middleware_shared-11.0.1.dev28 → fairagro_middleware_shared-11.0.1.dev30}/src/middleware/shared/report/__init__.py

@@ -1,12 +1,8 @@
-"""Harvest run report model, serializers, and stdout emission."""
+"""Harvest report models and serializers (format-neutral counting API)."""
 
-from middleware.shared.report.emit import print_report
 from middleware.shared.report.formats.base import ReportSerializer
-from middleware.shared.report.formats.jsonld import (
-    FAIRAGRO_HARVEST_REPORT_NS,
-    JsonLdReportSerializer,
-)
-from middleware.shared.report.model import FailedRecord, HarvestReport, RepositoryReport
+from middleware.shared.report.formats.jsonld import FAIRAGRO_HARVEST_REPORT_NS, JsonLdReportSerializer
+from middleware.shared.report.model import FailedRecord, HarvestReport, RepositoryReport, RepositoryScope
 
 __all__ = [
     "FAIRAGRO_HARVEST_REPORT_NS",
@@ -15,5 +11,5 @@ __all__ = [
     "JsonLdReportSerializer",
     "ReportSerializer",
     "RepositoryReport",
-    "print_report",
+    "RepositoryScope",
 ]

fairagro_middleware_shared-11.0.1.dev30/src/middleware/shared/report/formats/jsonld.py

@@ -0,0 +1,90 @@
+"""JSON-LD serializer for HarvestReport (harvester-compatible wire shape)."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+from middleware.shared.report.model import FailedRecord, HarvestReport, RepositoryReport
+
+FAIRAGRO_HARVEST_REPORT_NS = "https://fairagro.github.io/m4.2_advanced_middleware_api/ns/harvest-report/v1/#"
+
+_JSON_LD_CONTEXT: dict[str, Any] = {
+    "@vocab": "https://schema.org/",
+    "schema": "https://schema.org/",
+    "fairagro": FAIRAGRO_HARVEST_REPORT_NS,
+}
+
+
+def _format_iso_duration(seconds: float) -> str:
+    """Serialize a duration value as an ISO 8601 duration string."""
+    remainder = f"{seconds:.6f}".rstrip("0").rstrip(".")
+    if remainder == "":
+        remainder = "0"
+    return f"PT{remainder}S"
+
+
+def _format_iso_timestamp(value: datetime) -> str:
+    """Format a UTC datetime value as an ISO 8601 timestamp ending in Z."""
+    if value.tzinfo is None:
+        raise ValueError("HarvestReport timestamps must be timezone-aware UTC")
+    return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _failed_record_node(record: FailedRecord) -> dict[str, Any]:
+    node: dict[str, Any] = {"fairagro:message": record.message}
+    if record.record_id is not None:
+        node["fairagro:recordId"] = record.record_id
+    if record.url is not None:
+        node["fairagro:url"] = record.url
+    return node
+
+
+def _entry_point(repo: RepositoryReport) -> dict[str, Any]:
+    entry: dict[str, Any] = {
+        "@type": "schema:EntryPoint",
+        "@id": repo.rdi,
+        "name": repo.rdi,
+        "identifier": repo.rdi,
+        "schema:duration": _format_iso_duration(repo.duration_seconds),
+        "fairagro:harvestId": repo.harvest_id,
+        "fairagro:harvestedDatasets": repo.harvested_datasets,
+        "fairagro:failedDatasets": repo.failed_datasets,
+        "fairagro:skippedDatasets": repo.skipped_datasets,
+    }
+    if repo.expected_datasets is not None:
+        entry["fairagro:expectedDatasets"] = repo.expected_datasets
+    if repo.total_studies is not None:
+        entry["fairagro:totalStudies"] = repo.total_studies
+    if repo.total_assays is not None:
+        entry["fairagro:totalAssays"] = repo.total_assays
+    if repo.failed_records:
+        entry["fairagro:failedRecords"] = [_failed_record_node(r) for r in repo.failed_records]
+    return entry
+
+
+class JsonLdReportSerializer:
+    """Serialize a finished :class:`HarvestReport` to a JSON-LD document string."""
+
+    context: dict[str, Any] = _JSON_LD_CONTEXT
+
+    def render(self, report: HarvestReport) -> str:
+        """Return a JSON-LD document encoding the finished harvest run.
+
+        Raises:
+            ValueError: If :meth:`HarvestReport.finish` has not been called, or
+                timestamps are naive.
+        """
+        start = _format_iso_timestamp(report.start_time)
+        end = _format_iso_timestamp(report.end_time)
+        document: dict[str, Any] = {
+            "@context": self.context,
+            "@type": "schema:Action",
+            "name": report.name,
+            "schema:startTime": start,
+            "schema:endTime": end,
+            "fairagro:harvestDurationSeconds": report.duration_seconds,
+            "schema:result": [_entry_point(repo) for repo in report.repository_reports],
+        }
+        return json.dumps(document, indent=2, ensure_ascii=False) + "\n"

fairagro_middleware_shared-11.0.1.dev30/src/middleware/shared/report/model.py

@@ -0,0 +1,235 @@
+"""Mutable harvest-run report with counting methods and format-neutral snapshots."""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+
+def _require_aware_utc(value: datetime, *, what: str) -> datetime:
+    """Reject naive datetimes and normalize timezone-aware values to UTC."""
+    if value.tzinfo is None:
+        raise ValueError(f"{what} must be timezone-aware UTC")
+    return value.astimezone(UTC)
+
+
+@dataclass(frozen=True)
+class FailedRecord:
+    """A single dataset that failed during harvesting."""
+
+    message: str
+    record_id: str | None = None
+    url: str | None = None
+
+
+@dataclass(frozen=True)
+class RepositoryReport:  # pylint: disable=too-many-instance-attributes
+    """Immutable snapshot of statistics for one repository scope."""
+
+    rdi: str
+    harvest_id: str | None
+    duration_seconds: float
+    expected_datasets: int | None = None
+    harvested_datasets: int = 0
+    failed_datasets: int = 0
+    skipped_datasets: int = 0
+    failed_records: tuple[FailedRecord, ...] = ()
+    total_studies: int | None = None
+    total_assays: int | None = None
+
+
+@dataclass
+class _DatasetCounts:
+    """Mutable dataset-level counters for one repository scope."""
+
+    expected_datasets: int | None = None
+    harvest_id: str | None = None
+    harvested_datasets: int = 0
+    failed_datasets: int = 0
+    skipped_datasets: int = 0
+    failed_records: list[FailedRecord] = field(default_factory=list)
+
+
+@dataclass
+class _CompositionCounts:
+    """Mutable study/assay totals for one repository scope."""
+
+    total_studies: int = 0
+    total_assays: int = 0
+
+
+class RepositoryScope:
+    """Mutable counting handle for one RDI within a harvest run."""
+
+    def __init__(self, rdi: str, *, opened_at: datetime | None = None) -> None:
+        """Open a repository scope for ``rdi``; timing starts immediately."""
+        self._rdi = rdi
+        if opened_at is None:
+            self._opened_at = datetime.now(UTC)
+        else:
+            self._opened_at = _require_aware_utc(opened_at, what="RepositoryScope opened_at")
+        self._closed_at: datetime | None = None
+        self._lock = threading.Lock()
+        self._datasets = _DatasetCounts()
+        self._composition = _CompositionCounts()
+
+    @property
+    def rdi(self) -> str:
+        """RDI identifier for this scope."""
+        return self._rdi
+
+    def close(self, *, closed_at: datetime | None = None) -> None:
+        """Close the scope and record end time for duration."""
+        if closed_at is not None:
+            closed_at = _require_aware_utc(closed_at, what="RepositoryScope closed_at")
+        with self._lock:
+            if self._closed_at is None:
+                self._closed_at = closed_at or datetime.now(UTC)
+
+    def set_expected_datasets(self, count: int) -> None:
+        """Set the optional expected dataset count for this repository."""
+        if count < 0:
+            raise ValueError("expected dataset count must be non-negative")
+        with self._lock:
+            self._datasets.expected_datasets = count
+
+    def set_harvest_id(self, harvest_id: str | None) -> None:
+        """Set or clear the harvest identifier for this repository."""
+        with self._lock:
+            self._datasets.harvest_id = harvest_id
+
+    def record_harvested(self) -> None:
+        """Record one successfully harvested dataset."""
+        with self._lock:
+            self._datasets.harvested_datasets += 1
+
+    def record_failed(
+        self,
+        message: str,
+        *,
+        record_id: str | None = None,
+        url: str | None = None,
+    ) -> None:
+        """Record one failed dataset and append a failed-record detail."""
+        with self._lock:
+            self._datasets.failed_datasets += 1
+            self._datasets.failed_records.append(FailedRecord(message=message, record_id=record_id, url=url))
+
+    def record_skipped(self) -> None:
+        """Record one intentionally skipped dataset."""
+        with self._lock:
+            self._datasets.skipped_datasets += 1
+
+    def add_studies(self, count: int = 1) -> None:
+        """Add ``count`` studies to this repository's totals."""
+        if count < 0:
+            raise ValueError("study count must be non-negative")
+        with self._lock:
+            self._composition.total_studies += count
+
+    def add_assays(self, count: int = 1) -> None:
+        """Add ``count`` assays to this repository's totals."""
+        if count < 0:
+            raise ValueError("assay count must be non-negative")
+        with self._lock:
+            self._composition.total_assays += count
+
+    def snapshot(self) -> RepositoryReport:
+        """Return a format-neutral immutable view of current statistics."""
+        with self._lock:
+            datasets = self._datasets
+            composition = self._composition
+            closed_at = self._closed_at or datetime.now(UTC)
+            duration = (closed_at - self._opened_at).total_seconds()
+            if composition.total_studies == 0 and composition.total_assays == 0:
+                total_studies: int | None = None
+                total_assays: int | None = None
+            else:
+                total_studies = composition.total_studies
+                total_assays = composition.total_assays
+            return RepositoryReport(
+                rdi=self._rdi,
+                harvest_id=datasets.harvest_id,
+                duration_seconds=duration,
+                expected_datasets=datasets.expected_datasets,
+                harvested_datasets=datasets.harvested_datasets,
+                failed_datasets=datasets.failed_datasets,
+                skipped_datasets=datasets.skipped_datasets,
+                failed_records=tuple(datasets.failed_records),
+                total_studies=total_studies,
+                total_assays=total_assays,
+            )
+
+
+class HarvestReport:
+    """Mutable harvest-run accumulator; serializers read finished snapshots."""
+
+    def __init__(
+        self,
+        *,
+        name: str = "FAIRagro Harvest Run",
+        start_time: datetime | None = None,
+    ) -> None:
+        """Create a run report and record the authoritative start time."""
+        self.name = name
+        if start_time is None:
+            self._start_time = datetime.now(UTC)
+        else:
+            self._start_time = _require_aware_utc(start_time, what="HarvestReport start_time")
+        self._end_time: datetime | None = None
+        self._scopes: list[RepositoryScope] = []
+        self._scopes_lock = threading.Lock()
+
+    @property
+    def start_time(self) -> datetime:
+        """Run start timestamp."""
+        return self._start_time
+
+    @property
+    def end_time(self) -> datetime:
+        """Run end timestamp.
+
+        Raises:
+            ValueError: If :meth:`finish` has not been called yet.
+        """
+        if self._end_time is None:
+            raise ValueError("HarvestReport.finish() must be called before reading end_time")
+        return self._end_time
+
+    @property
+    def duration_seconds(self) -> float:
+        """Total harvest run duration in seconds."""
+        return (self.end_time - self.start_time).total_seconds()
+
+    @property
+    def repository_reports(self) -> tuple[RepositoryReport, ...]:
+        """Immutable snapshots of all repository scopes in open order."""
+        with self._scopes_lock:
+            scopes = list(self._scopes)
+        return tuple(scope.snapshot() for scope in scopes)
+
+    def open_repository(self, rdi: str, *, opened_at: datetime | None = None) -> RepositoryScope:
+        """Open a repository scope handle for ``rdi`` (multiple may be open)."""
+        scope = RepositoryScope(rdi, opened_at=opened_at)
+        with self._scopes_lock:
+            self._scopes.append(scope)
+        return scope
+
+    def finish(self, *, end_time: datetime | None = None) -> None:
+        """Finish the harvest run, close open scopes, and record the end timestamp.
+
+        Raises:
+            ValueError: If :meth:`finish` has already been called.
+        """
+        if self._end_time is not None:
+            raise ValueError("HarvestReport.finish() has already been called")
+        if end_time is None:
+            finished_at = datetime.now(UTC)
+        else:
+            finished_at = _require_aware_utc(end_time, what="HarvestReport end_time")
+        with self._scopes_lock:
+            scopes = list(self._scopes)
+        for scope in scopes:
+            scope.close(closed_at=finished_at)
+        self._end_time = finished_at

fairagro_middleware_shared-11.0.1.dev30/tests/unit/test_harvest_report.py

@@ -0,0 +1,421 @@
+"""Unit tests for the shared harvest report counting API and JSON-LD serializer."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime, timedelta
+from zoneinfo import ZoneInfo
+
+import pytest
+
+from middleware.shared.report import (
+    FAIRAGRO_HARVEST_REPORT_NS,
+    FailedRecord,
+    HarvestReport,
+    JsonLdReportSerializer,
+    RepositoryScope,
+)
+
+_START = datetime(2026, 5, 6, 14, 0, 0, tzinfo=UTC)
+_END = datetime(2026, 5, 6, 14, 3, 45, tzinfo=UTC)
+_SCOPE_CLOSE = datetime(2026, 5, 6, 14, 0, 17, tzinfo=UTC)
+
+_SCOPE_DURATION_SECONDS = 12.5
+_SAMPLE_EXPECTED = 100
+_SAMPLE_HARVESTED = 95
+_SAMPLE_FAILED = 5
+_SAMPLE_SKIPPED = 2
+_EXPECTED_COUNT = 10
+_HARVESTED_TWICE = 2
+_STUDIES = 3
+_ASSAYS = 7
+_REPO_COUNT = 2
+_ASYNC_HARVESTED = 100
+_ASYNC_FAILED = 40
+_PARALLEL_HARVESTED = 80
+_THREAD_HARVESTED = 200
+_THREAD_FAILED = 50
+
+
+def _finished_report(
+    *,
+    populate: Callable[[HarvestReport], None] | None = None,
+    start_time: datetime = _START,
+    end_time: datetime = _END,
+) -> HarvestReport:
+    """Build a finished report; optional ``populate(report)`` opens scopes."""
+    report = HarvestReport(start_time=start_time)
+    if populate is not None:
+        populate(report)
+    report.finish(end_time=end_time)
+    return report
+
+
+def _open_sample_scope(report: HarvestReport) -> None:
+    """Open one sample repository scope with typical counts."""
+    scope = report.open_repository("bonares")
+    scope.set_expected_datasets(_SAMPLE_EXPECTED)
+    scope.set_harvest_id("harvest-1")
+    for _ in range(_SAMPLE_HARVESTED):
+        scope.record_harvested()
+    for i in range(_SAMPLE_FAILED):
+        scope.record_failed(
+            "map failed" if i == 0 else f"fail-{i}",
+            record_id="frl:123" if i == 0 else None,
+            url="https://example.test/frl:123" if i == 0 else None,
+        )
+    for _ in range(_SAMPLE_SKIPPED):
+        scope.record_skipped()
+    scope.close(closed_at=_SCOPE_CLOSE)
+
+
+def test_failed_record_with_optional_identifiers() -> None:
+    """Failed records expose message, record id, and URL when provided."""
+    record = FailedRecord(message="boom", record_id="id-1", url="https://x.test")
+    assert record.message == "boom"
+    assert record.record_id == "id-1"
+    assert record.url == "https://x.test"
+
+
+def test_failed_record_message_only() -> None:
+    """Optional identifiers are unset when omitted."""
+    record = FailedRecord(message="boom")
+    assert record.record_id is None
+    assert record.url is None
+
+
+def test_mutable_run_records_start_time() -> None:
+    """Creating a report records an authoritative start timestamp."""
+    report = HarvestReport(start_time=_START)
+    assert report.start_time == _START
+
+
+def test_finish_records_end_time_and_duration() -> None:
+    """Finishing the run records end time used for duration."""
+    report = HarvestReport(start_time=_START)
+    report.finish(end_time=_END)
+    assert report.end_time == _END
+    assert report.duration_seconds == (_END - _START).total_seconds()
+
+
+def test_end_time_requires_finish() -> None:
+    """Reading end time before finish raises."""
+    report = HarvestReport(start_time=_START)
+    with pytest.raises(ValueError, match="finish"):
+        _ = report.end_time
+
+
+def test_open_and_close_repository_scope() -> None:
+    """Open/close records RDI and per-scope duration."""
+    report = HarvestReport(start_time=_START)
+    opened = datetime(2026, 5, 6, 14, 1, 0, tzinfo=UTC)
+    closed = opened + timedelta(seconds=_SCOPE_DURATION_SECONDS)
+    scope = report.open_repository("bonares", opened_at=opened)
+    scope.close(closed_at=closed)
+    report.finish(end_time=_END)
+    repos = report.repository_reports
+    assert len(repos) == 1
+    assert repos[0].rdi == "bonares"
+    assert repos[0].duration_seconds == _SCOPE_DURATION_SECONDS
+
+
+def test_concurrent_scopes_are_isolated() -> None:
+    """Counting on one handle does not change another handle's totals."""
+    report = HarvestReport(start_time=_START)
+    a = report.open_repository("rdi-a")
+    b = report.open_repository("rdi-b")
+    a.record_harvested()
+    a.record_harvested()
+    b.record_failed("oops")
+    snap_a = a.snapshot()
+    snap_b = b.snapshot()
+    assert snap_a.harvested_datasets == _HARVESTED_TWICE
+    assert snap_a.failed_datasets == 0
+    assert snap_b.harvested_datasets == 0
+    assert snap_b.failed_datasets == 1
+
+
+def test_counting_increments() -> None:
+    """Counting methods increment harvested, failed, and skipped."""
+    report = HarvestReport(start_time=_START)
+    scope = report.open_repository("bonares")
+    scope.set_expected_datasets(_EXPECTED_COUNT)
+    scope.set_harvest_id("h-1")
+    scope.record_harvested()
+    scope.record_harvested()
+    scope.record_failed("bad", record_id="r1", url="https://x.test/r1")
+    scope.record_skipped()
+    scope.add_studies(_STUDIES)
+    scope.add_assays(_ASSAYS)
+    snap = scope.snapshot()
+    assert snap.expected_datasets == _EXPECTED_COUNT
+    assert snap.harvest_id == "h-1"
+    assert snap.harvested_datasets == _HARVESTED_TWICE
+    assert snap.failed_datasets == 1
+    assert snap.skipped_datasets == 1
+    assert snap.total_studies == _STUDIES
+    assert snap.total_assays == _ASSAYS
+    assert snap.failed_records == (FailedRecord(message="bad", record_id="r1", url="https://x.test/r1"),)
+
+
+def test_study_assay_omit_when_both_zero() -> None:
+    """Unset study/assay totals are None when never incremented."""
+    report = HarvestReport(start_time=_START)
+    scope = report.open_repository("bonares")
+    snap = scope.snapshot()
+    assert snap.total_studies is None
+    assert snap.total_assays is None
+
+
+def test_render_requires_finish() -> None:
+    """JSON-LD render requires finish()."""
+    report = HarvestReport(start_time=_START)
+    with pytest.raises(ValueError, match="finish"):
+        JsonLdReportSerializer().render(report)
+
+
+def test_naive_start_time_rejected() -> None:
+    """Naive start times must not be silently treated as local time."""
+    with pytest.raises(ValueError, match="timezone-aware"):
+        HarvestReport(start_time=datetime(2026, 5, 6, 14, 0, 0))
+
+
+def test_naive_opened_at_rejected() -> None:
+    """Naive opened_at on a repository scope is rejected."""
+    with pytest.raises(ValueError, match="timezone-aware"):
+        RepositoryScope("bonares", opened_at=datetime(2026, 5, 6, 14, 0, 0))
+
+
+def test_naive_closed_at_rejected() -> None:
+    """Naive closed_at on a repository scope is rejected."""
+    scope = RepositoryScope("bonares", opened_at=_START)
+    with pytest.raises(ValueError, match="timezone-aware"):
+        scope.close(closed_at=datetime(2026, 5, 6, 14, 1, 0))
+
+
+def test_finish_closes_open_scopes_deterministically() -> None:
+    """finish() closes still-open scopes at the run end time."""
+    opened = datetime(2026, 5, 6, 14, 0, 0, tzinfo=UTC)
+    report = HarvestReport(start_time=opened)
+    scope = report.open_repository("bonares", opened_at=opened)
+    report.finish(end_time=_END)
+    assert scope.snapshot().duration_seconds == (_END - opened).total_seconds()
+    assert report.end_time == _END
+
+
+def test_finish_is_single_shot() -> None:
+    """Calling finish() twice raises so run and scope durations stay aligned."""
+    report = HarvestReport(start_time=_START)
+    report.open_repository("bonares", opened_at=_START)
+    report.finish(end_time=_END)
+    with pytest.raises(ValueError, match="already been called"):
+        report.finish(end_time=_END + timedelta(seconds=30))
+    assert report.end_time == _END
+
+
+def test_require_aware_utc_normalizes_non_utc() -> None:
+    """Timezone-aware non-UTC timestamps are stored as UTC equivalents."""
+    berlin = ZoneInfo("Europe/Berlin")
+    local_start = datetime(2026, 5, 6, 16, 0, 0, tzinfo=berlin)  # UTC+2 in May
+    report = HarvestReport(start_time=local_start)
+    assert report.start_time == datetime(2026, 5, 6, 14, 0, 0, tzinfo=UTC)
+    assert report.start_time.tzinfo == UTC
+
+
+def test_jsonld_context_and_types() -> None:
+    """JSON-LD uses schema.org Action with EntryPoint results."""
+    report = _finished_report(populate=_open_sample_scope)
+    document = json.loads(JsonLdReportSerializer().render(report))
+    assert document["@context"]["@vocab"] == "https://schema.org/"
+    assert document["@context"]["schema"] == "https://schema.org/"
+    assert document["@context"]["fairagro"] == FAIRAGRO_HARVEST_REPORT_NS
+    assert document["@context"]["fairagro"].endswith("/ns/harvest-report/v1/#")
+    assert document["@type"] == "schema:Action"
+    assert document["schema:result"][0]["@type"] == "schema:EntryPoint"
+
+
+def test_jsonld_timestamps_and_durations() -> None:
+    """Timestamps end with Z; run duration is seconds; scope duration is ISO."""
+    report = _finished_report(populate=_open_sample_scope)
+    document = json.loads(JsonLdReportSerializer().render(report))
+    assert document["schema:startTime"].endswith("Z")
+    assert document["schema:endTime"].endswith("Z")
+    assert document["fairagro:harvestDurationSeconds"] == report.duration_seconds
+    assert document["schema:result"][0]["schema:duration"].startswith("PT")
+
+
+def test_jsonld_metrics_and_failed_records() -> None:
+    """Fairagro metrics and nested failed records are emitted."""
+    report = _finished_report(populate=_open_sample_scope)
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert entry["@id"] == "bonares"
+    assert entry["fairagro:harvestId"] == "harvest-1"
+    assert entry["fairagro:expectedDatasets"] == _SAMPLE_EXPECTED
+    assert entry["fairagro:harvestedDatasets"] == _SAMPLE_HARVESTED
+    assert entry["fairagro:failedDatasets"] == _SAMPLE_FAILED
+    assert entry["fairagro:skippedDatasets"] == _SAMPLE_SKIPPED
+    assert entry["fairagro:failedRecords"][0] == {
+        "fairagro:message": "map failed",
+        "fairagro:recordId": "frl:123",
+        "fairagro:url": "https://example.test/frl:123",
+    }
+
+
+def test_jsonld_optional_study_and_assay_totals() -> None:
+    """Optional study and assay totals appear as fairagro properties."""
+
+    def populate(report: HarvestReport) -> None:
+        scope = report.open_repository("sql")
+        scope.add_studies(_STUDIES)
+        scope.add_assays(_ASSAYS)
+        scope.close(closed_at=_SCOPE_CLOSE)
+
+    report = _finished_report(populate=populate)
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert entry["fairagro:totalStudies"] == _STUDIES
+    assert entry["fairagro:totalAssays"] == _ASSAYS
+
+
+def test_jsonld_omits_unset_expected_and_study_assay() -> None:
+    """Unset expected and zero study/assay totals are omitted."""
+
+    def populate(report: HarvestReport) -> None:
+        scope = report.open_repository("bonares")
+        scope.set_harvest_id("harvest-1")
+        scope.record_harvested()
+        scope.close(closed_at=_SCOPE_CLOSE)
+
+    report = _finished_report(populate=populate)
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert "fairagro:expectedDatasets" not in entry
+    assert "fairagro:totalStudies" not in entry
+    assert "fairagro:totalAssays" not in entry
+    assert entry["fairagro:harvestedDatasets"] == 1
+    assert entry["fairagro:failedDatasets"] == 0
+    assert entry["fairagro:skippedDatasets"] == 0
+    assert entry["fairagro:harvestId"] == "harvest-1"
+
+
+def test_jsonld_harvest_id_null_when_unset() -> None:
+    """Unset harvest id is emitted as JSON null."""
+
+    def populate(report: HarvestReport) -> None:
+        scope = report.open_repository("bonares")
+        scope.close(closed_at=_SCOPE_CLOSE)
+
+    report = _finished_report(populate=populate)
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert entry["fairagro:harvestId"] is None
+
+
+def test_jsonld_omits_empty_failed_records() -> None:
+    """Empty failed-record lists are omitted from JSON-LD."""
+
+    def populate(report: HarvestReport) -> None:
+        scope = report.open_repository("bonares")
+        scope.close(closed_at=_SCOPE_CLOSE)
+
+    report = _finished_report(populate=populate)
+    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
+    assert "fairagro:failedRecords" not in entry
+
+
+def test_jsonld_empty_result_array() -> None:
+    """A run with no repositories emits an empty result array."""
+    document = json.loads(JsonLdReportSerializer().render(_finished_report()))
+    assert document["schema:result"] == []
+
+
+def test_jsonld_multi_repository_action() -> None:
+    """Two scopes produce two EntryPoints in schema:result."""
+
+    def populate(report: HarvestReport) -> None:
+        a = report.open_repository("rdi-a")
+        b = report.open_repository("rdi-b")
+        a.close(closed_at=_SCOPE_CLOSE)
+        b.close(closed_at=_SCOPE_CLOSE)
+
+    report = _finished_report(populate=populate)
+    document = json.loads(JsonLdReportSerializer().render(report))
+    assert len(document["schema:result"]) == _REPO_COUNT
+    assert all(e["@type"] == "schema:EntryPoint" for e in document["schema:result"])
+
+
+def test_serializer_returns_string() -> None:
+    """The JSON-LD serializer returns a document string, not a dict."""
+    rendered = JsonLdReportSerializer().render(_finished_report())
+    assert isinstance(rendered, str)
+    assert json.loads(rendered)["@type"] == "schema:Action"
+
+
+def test_thread_safe_counting_on_one_handle() -> None:
+    """Concurrent thread updates on one handle do not lose counts."""
+    report = HarvestReport(start_time=_START)
+    scope = report.open_repository("bonares")
+
+    def harvest(_i: int) -> None:
+        scope.record_harvested()
+
+    def fail(_i: int) -> None:
+        scope.record_failed("x")
+
+    with ThreadPoolExecutor(max_workers=16) as pool:
+        list(pool.map(harvest, range(_THREAD_HARVESTED)))
+        list(pool.map(fail, range(_THREAD_FAILED)))
+
+    snap = scope.snapshot()
+    assert snap.harvested_datasets == _THREAD_HARVESTED
+    assert snap.failed_datasets == _THREAD_FAILED
+
+
+def test_asyncio_interleaved_counting() -> None:
+    """Interleaved asyncio tasks on one handle preserve totals."""
+    report = HarvestReport(start_time=_START)
+    scope = report.open_repository("bonares")
+
+    async def run() -> None:
+        async def harvested() -> None:
+            for _ in range(_ASYNC_HARVESTED):
+                scope.record_harvested()
+                await asyncio.sleep(0)
+
+        async def failed() -> None:
+            for _ in range(_ASYNC_FAILED):
+                scope.record_failed("e")
+                await asyncio.sleep(0)
+
+        await asyncio.gather(harvested(), failed())
+
+    asyncio.run(run())
+    snap = scope.snapshot()
+    assert snap.harvested_datasets == _ASYNC_HARVESTED
+    assert snap.failed_datasets == _ASYNC_FAILED
+
+
+def test_parallel_rdis_do_not_cross_count() -> None:
+    """Concurrent tasks on different handles stay isolated."""
+    report = HarvestReport(start_time=_START)
+    first = report.open_repository("rdi-a")
+    second = report.open_repository("rdi-b")
+    barrier = threading.Barrier(_REPO_COUNT)
+
+    def only_first() -> None:
+        barrier.wait()
+        for _ in range(_PARALLEL_HARVESTED):
+            first.record_harvested()
+
+    def idle_second() -> None:
+        barrier.wait()
+
+    t1 = threading.Thread(target=only_first)
+    t2 = threading.Thread(target=idle_second)
+    t1.start()
+    t2.start()
+    t1.join()
+    t2.join()
+    assert first.snapshot().harvested_datasets == _PARALLEL_HARVESTED
+    assert second.snapshot().harvested_datasets == 0

fairagro_middleware_shared-11.0.1.dev28/src/middleware/shared/report/emit.py

@@ -1,26 +0,0 @@
-"""Stdout emission for harvest reports."""
-
-from __future__ import annotations
-
-import logging
-
-from middleware.shared.report.formats.base import ReportSerializer
-from middleware.shared.report.formats.jsonld import JsonLdReportSerializer
-from middleware.shared.report.model import HarvestReport
-
-logger = logging.getLogger(__name__)
-
-
-def print_report(
-    report: HarvestReport,
-    serializer: ReportSerializer | None = None,
-) -> None:
-    """Serialize the report and print it to stdout.
-
-    Serialization or print failures are logged as warnings and do not raise.
-    """
-    active_serializer: ReportSerializer = serializer or JsonLdReportSerializer()
-    try:
-        print(active_serializer.render(report))
-    except Exception as exc:  # noqa: BLE001 — MUST NOT fail the process (harvest-report)
-        logger.warning("Failed to serialise harvest report: %s", exc)

fairagro_middleware_shared-11.0.1.dev28/src/middleware/shared/report/formats/jsonld.py

@@ -1,94 +0,0 @@
-"""JSON-LD harvest report serializer."""
-
-from __future__ import annotations
-
-import json
-from datetime import UTC, datetime
-from typing import Any
-
-from middleware.shared.report.model import FailedRecord, HarvestReport, RepositoryReport
-
-FAIRAGRO_HARVEST_REPORT_NS = "https://fairagro.github.io/m4.2_advanced_middleware_api/ns/harvest-report/v1/#"
-
-_JSON_LD_CONTEXT = {
-    "@vocab": "https://schema.org/",
-    "schema": "https://schema.org/",
-    "fairagro": FAIRAGRO_HARVEST_REPORT_NS,
-}
-
-
-def _format_iso_duration(seconds: float) -> str:
-    """Serialize a duration value as an ISO 8601 duration string."""
-    remainder = f"{seconds:.6f}".rstrip("0").rstrip(".")
-    if remainder == "":
-        remainder = "0"
-    return f"PT{remainder}S"
-
-
-def _format_iso_timestamp(value: datetime) -> str:
-    """Format a timezone-aware datetime as an ISO 8601 UTC timestamp ending in Z.
-
-    Raises:
-        ValueError: If ``value`` is naive (no ``tzinfo``). Callers must supply
-            timezone-aware UTC times; treating naive values as local time would
-            silently shift the wall clock on non-UTC hosts.
-    """
-    if value.tzinfo is None:
-        raise ValueError("HarvestReport timestamps must be timezone-aware UTC")
-    return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
-
-
-def _failed_record_to_jsonld(record: FailedRecord) -> dict[str, Any]:
-    """Convert a failed record to its JSON-LD object, omitting unset fields."""
-    result: dict[str, Any] = {"fairagro:message": record.message}
-    if record.record_id:
-        result["fairagro:recordId"] = record.record_id
-    if record.url:
-        result["fairagro:url"] = record.url
-    return result
-
-
-def _repository_to_jsonld(report: RepositoryReport) -> dict[str, Any]:
-    """Convert a repository report to a schema.org EntryPoint JSON-LD object."""
-    result: dict[str, Any] = {
-        "@type": "schema:EntryPoint",
-        "name": report.rdi,
-        "identifier": report.rdi,
-        "schema:duration": _format_iso_duration(report.duration_seconds),
-        "fairagro:harvestId": report.harvest_id,
-        "fairagro:skippedDatasets": report.skipped_datasets,
-    }
-    if report.harvested_datasets is not None:
-        result["fairagro:harvestedDatasets"] = report.harvested_datasets
-    if report.expected_datasets is not None:
-        result["fairagro:expectedDatasets"] = report.expected_datasets
-    if report.failed_datasets is not None:
-        result["fairagro:failedDatasets"] = report.failed_datasets
-    if report.total_studies is not None:
-        result["fairagro:totalStudies"] = report.total_studies
-    if report.total_assays is not None:
-        result["fairagro:totalAssays"] = report.total_assays
-    if report.failed_records:
-        result["fairagro:failedRecords"] = [_failed_record_to_jsonld(record) for record in report.failed_records]
-    return result
-
-
-class JsonLdReportSerializer:
-    """Render a harvest report as an operator-readable JSON-LD document."""
-
-    def __init__(self, *, indent: int = 2) -> None:
-        """Configure JSON indentation for operator-readable output."""
-        self._indent = indent
-
-    def render(self, report: HarvestReport) -> str:
-        """Return the report as indented JSON-LD text."""
-        document: dict[str, Any] = {
-            "@context": _JSON_LD_CONTEXT,
-            "@type": "schema:Action",
-            "name": report.name,
-            "schema:startTime": _format_iso_timestamp(report.start_time),
-            "schema:endTime": _format_iso_timestamp(report.end_time),
-            "fairagro:harvestDurationSeconds": report.duration_seconds,
-            "schema:result": [_repository_to_jsonld(entry) for entry in report.repository_reports],
-        }
-        return json.dumps(document, ensure_ascii=False, indent=self._indent)

fairagro_middleware_shared-11.0.1.dev28/src/middleware/shared/report/model.py

@@ -1,46 +0,0 @@
-"""Format-neutral harvest run report domain model."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from datetime import datetime
-
-
-@dataclass(frozen=True)
-class FailedRecord:
-    """A single dataset that failed during harvesting."""
-
-    message: str
-    record_id: str | None = None
-    url: str | None = None
-
-
-@dataclass(frozen=True)
-class RepositoryReport:  # pylint: disable=too-many-instance-attributes
-    """Execution statistics for a single harvested repository."""
-
-    rdi: str
-    harvest_id: str | None
-    duration_seconds: float
-    expected_datasets: int | None = None
-    harvested_datasets: int | None = None
-    failed_datasets: int | None = None
-    skipped_datasets: int = 0
-    failed_records: tuple[FailedRecord, ...] = ()
-    total_studies: int | None = None
-    total_assays: int | None = None
-
-
-@dataclass(frozen=True)
-class HarvestReport:
-    """Summary statistics for an entire harvest run."""
-
-    start_time: datetime
-    end_time: datetime
-    repository_reports: tuple[RepositoryReport, ...] = ()
-    name: str = "FAIRagro Harvest Run"
-
-    @property
-    def duration_seconds(self) -> float:
-        """Return the total harvest run duration in seconds."""
-        return (self.end_time - self.start_time).total_seconds()

fairagro_middleware_shared-11.0.1.dev28/tests/unit/test_harvest_report.py

@@ -1,234 +0,0 @@
-"""Unit tests for the shared harvest report library."""
-
-from __future__ import annotations
-
-import json
-import logging
-from dataclasses import fields
-from datetime import UTC, datetime
-from typing import Any
-
-import pytest
-
-from middleware.shared.report import (
-    FAIRAGRO_HARVEST_REPORT_NS,
-    FailedRecord,
-    HarvestReport,
-    JsonLdReportSerializer,
-    RepositoryReport,
-    print_report,
-)
-
-_START = datetime(2026, 5, 6, 14, 0, 0, tzinfo=UTC)
-_END = datetime(2026, 5, 6, 14, 3, 45, tzinfo=UTC)
-
-
-def _sample_repo(**overrides: Any) -> RepositoryReport:
-    """Build a repository report with sensible defaults."""
-    values: dict[str, Any] = {
-        "rdi": "bonares",
-        "harvest_id": "harvest-1",
-        "duration_seconds": 12.3,
-        "expected_datasets": 100,
-        "harvested_datasets": 95,
-        "failed_datasets": 5,
-        "skipped_datasets": 2,
-        "failed_records": (
-            FailedRecord(
-                message="map failed",
-                record_id="frl:123",
-                url="https://example.test/frl:123",
-            ),
-        ),
-    }
-    values.update(overrides)
-    return RepositoryReport(**values)
-
-
-def _sample_report(
-    repositories: tuple[RepositoryReport, ...] | None = None,
-) -> HarvestReport:
-    """Build a harvest run report spanning a fixed UTC window."""
-    return HarvestReport(
-        start_time=_START,
-        end_time=_END,
-        repository_reports=((_sample_repo(),) if repositories is None else repositories),
-    )
-
-
-def test_failed_record_with_optional_identifiers() -> None:
-    """Failed records expose message, record id, and URL when provided."""
-    record = FailedRecord(message="boom", record_id="id-1", url="https://x.test")
-    assert record.message == "boom"
-    assert record.record_id == "id-1"
-    assert record.url == "https://x.test"
-
-
-def test_failed_record_message_only() -> None:
-    """Optional identifiers are unset when omitted."""
-    record = FailedRecord(message="boom")
-    assert record.record_id is None
-    assert record.url is None
-
-
-def test_harvest_report_with_one_repository() -> None:
-    """A run report exposes timing and a single repository entry."""
-    report = _sample_report()
-    assert report.start_time == _START
-    assert len(report.repository_reports) == 1
-    assert report.repository_reports[0].rdi == "bonares"
-    assert report.duration_seconds == (_END - _START).total_seconds()
-
-
-def test_harvest_report_with_no_repositories() -> None:
-    """An empty repository list is preserved."""
-    report = _sample_report(repositories=())
-    assert not report.repository_reports
-    assert report.duration_seconds == (_END - _START).total_seconds()
-
-
-def test_optional_study_and_assay_counts_set() -> None:
-    """Study and assay totals are available when provided."""
-    total_studies = 10
-    total_assays = 20
-    repo = _sample_repo(total_studies=total_studies, total_assays=total_assays)
-    assert repo.total_studies == total_studies
-    assert repo.total_assays == total_assays
-
-
-def test_optional_study_and_assay_counts_unset() -> None:
-    """Unset study and assay totals are None, not zero."""
-    repo = _sample_repo()
-    assert repo.total_studies is None
-    assert repo.total_assays is None
-
-
-def test_model_is_format_neutral() -> None:
-    """The domain model does not embed a serialized document."""
-    report = _sample_report()
-    assert not hasattr(report, "to_jsonld")
-    assert "repository_reports" in {field.name for field in fields(report)}
-
-
-def test_jsonld_context_and_types() -> None:
-    """JSON-LD uses schema.org Action with EntryPoint results."""
-    document = json.loads(JsonLdReportSerializer().render(_sample_report()))
-    assert document["@context"]["@vocab"] == "https://schema.org/"
-    assert document["@context"]["schema"] == "https://schema.org/"
-    assert document["@context"]["fairagro"] == FAIRAGRO_HARVEST_REPORT_NS
-    assert document["@context"]["fairagro"].endswith("/ns/harvest-report/v1/#")
-    assert document["@type"] == "schema:Action"
-    assert document["schema:result"][0]["@type"] == "schema:EntryPoint"
-
-
-def test_jsonld_timestamps_and_durations() -> None:
-    """Timestamps end with Z and durations use ISO 8601 / seconds."""
-    report = _sample_report()
-    document = json.loads(JsonLdReportSerializer().render(report))
-    assert document["schema:startTime"].endswith("Z")
-    assert document["schema:endTime"].endswith("Z")
-    assert document["fairagro:harvestDurationSeconds"] == report.duration_seconds
-    assert document["schema:result"][0]["schema:duration"].startswith("PT")
-
-
-def test_jsonld_rejects_naive_timestamps() -> None:
-    """Naive start/end times must not be silently treated as local time."""
-    report = HarvestReport(
-        start_time=datetime(2026, 5, 6, 14, 0, 0),
-        end_time=datetime(2026, 5, 6, 14, 3, 45),
-    )
-    with pytest.raises(ValueError, match="timezone-aware"):
-        JsonLdReportSerializer().render(report)
-
-
-def test_jsonld_metrics_and_failed_records() -> None:
-    """Fairagro metrics and nested failed records are emitted."""
-    repo = _sample_repo()
-    entry = json.loads(JsonLdReportSerializer().render(_sample_report(repositories=(repo,))))["schema:result"][0]
-    assert entry["fairagro:harvestId"] == repo.harvest_id
-    assert entry["fairagro:expectedDatasets"] == repo.expected_datasets
-    assert entry["fairagro:harvestedDatasets"] == repo.harvested_datasets
-    assert entry["fairagro:failedDatasets"] == repo.failed_datasets
-    assert entry["fairagro:skippedDatasets"] == repo.skipped_datasets
-    failed = repo.failed_records[0]
-    assert entry["fairagro:failedRecords"] == [
-        {
-            "fairagro:message": failed.message,
-            "fairagro:recordId": failed.record_id,
-            "fairagro:url": failed.url,
-        }
-    ]
-
-
-def test_jsonld_optional_study_and_assay_totals() -> None:
-    """Optional study and assay totals appear as fairagro properties."""
-    total_studies = 3
-    total_assays = 7
-    report = _sample_report(repositories=(_sample_repo(total_studies=total_studies, total_assays=total_assays),))
-    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
-    assert entry["fairagro:totalStudies"] == total_studies
-    assert entry["fairagro:totalAssays"] == total_assays
-
-
-def test_jsonld_omits_unset_expected_datasets() -> None:
-    """Unset optional counts are omitted rather than null."""
-    report = _sample_report(
-        repositories=(
-            _sample_repo(
-                expected_datasets=None,
-                harvested_datasets=None,
-                failed_datasets=None,
-                total_studies=None,
-                total_assays=None,
-            ),
-        )
-    )
-    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
-    assert "fairagro:expectedDatasets" not in entry
-    assert "fairagro:harvestedDatasets" not in entry
-    assert "fairagro:failedDatasets" not in entry
-    assert "fairagro:totalStudies" not in entry
-    assert "fairagro:totalAssays" not in entry
-    assert "fairagro:harvestId" in entry
-
-
-def test_jsonld_omits_empty_failed_records() -> None:
-    """Empty failed-record lists are omitted from JSON-LD."""
-    report = _sample_report(repositories=(_sample_repo(failed_records=()),))
-    entry = json.loads(JsonLdReportSerializer().render(report))["schema:result"][0]
-    assert "fairagro:failedRecords" not in entry
-
-
-def test_jsonld_empty_result_array() -> None:
-    """A run with no repositories emits an empty result array."""
-    document = json.loads(JsonLdReportSerializer().render(_sample_report(repositories=())))
-    assert document["schema:result"] == []
-
-
-def test_print_report_writes_jsonld_to_stdout(capsys: pytest.CaptureFixture[str]) -> None:
-    """print_report writes a JSON-LD document to stdout."""
-    print_report(_sample_report())
-    captured = capsys.readouterr()
-    document = json.loads(captured.out)
-    assert document["@type"] == "schema:Action"
-    assert captured.err == ""
-
-
-def test_print_report_does_not_raise_on_serialization_failure(
-    caplog: pytest.LogCaptureFixture,
-) -> None:
-    """Serialization failures are logged and do not propagate."""
-
-    class _BoomSerializer:
-        def __init__(self) -> None:
-            self._fail = True
-
-        def render(self, report: HarvestReport) -> str:
-            if self._fail:
-                raise RuntimeError(f"cannot serialise {type(report).__name__}")
-            return "{}"
-
-    with caplog.at_level(logging.WARNING):
-        print_report(_sample_report(), serializer=_BoomSerializer())
-
-    assert any("Failed to serialise harvest report" in message for message in caplog.messages)