fairagro-middleware-api-client 10.0.2.dev24__tar.gz → 10.0.3.dev26__tar.gz

{fairagro_middleware_api_client-10.0.2.dev24 → fairagro_middleware_api_client-10.0.3.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fairagro-middleware-api-client
-Version: 10.0.2.dev24
+Version: 10.0.3.dev26
 Summary: The FAIRagro advanced middleware API client
 Requires-Python: >=3.12
 Requires-Dist: httpx>=0.28.1

{fairagro_middleware_api_client-10.0.2.dev24 → fairagro_middleware_api_client-10.0.3.dev26}/spec/harvest-client/design.md

@@ -48,11 +48,13 @@ harvester
    appended rather than overwriting.
 
 5. **Duplicate detection is performed client-side before the HTTP request**
-   — Submitting both duplicates would cause the server to process two ARCs
-   with the same identifier in the same harvest run, resulting in an opaque
-   conflict. Client-side detection gives an explicit `DUPLICATE` error,
-   prevents the wasted round-trip, and avoids requiring the server to handle
-   intra-harvest identity conflicts.
+   — Within one harvest batch, two different ARC payloads that share an
+   identifier are a harvester bug. Detecting them client-side yields an
+   explicit `DUPLICATE` error and avoids the round-trip. The server still
+   enforces harvest-local identity: identical re-submits (e.g. transport
+   retries) return `200`; conflicting content for the same identifier returns
+   `409` (see `harvest-arc-upload/`). Client-side detection therefore covers
+   *intra-batch* duplicates; server idempotency covers *retry* duplicates.
 
 6. **Item-level failures are non-fatal; harvest-level failures are fatal**
    — A submission failure for one ARC (e.g. server 422 on bad content) must
@@ -60,3 +62,11 @@ harvester
    catastrophic failure (e.g. 401 Unauthorized, harvest already closed) means
    no further submissions will succeed, so the harvest is aborted, marked
    `FAILED`, and the exception propagates to the caller.
+
+7. **POST ARC endpoints retry transport failures**
+   — `POST /v3/arcs` and `POST /v3/harvests/{id}/arcs` are server-idempotent for
+   identical bodies, so the client retries `ConnectError` and transient gateway
+   statuses (`502`/`503`/`504`) on those paths without risking a second object.
+   Other POSTs (create/complete harvest) are not retried. Conflicting `409`
+   (same identifier, different content in one harvest) remains a real conflict
+   and is not treated as success.

{fairagro_middleware_api_client-10.0.2.dev24 → fairagro_middleware_api_client-10.0.3.dev26}/src/middleware/api_client/api_client.py

@@ -69,6 +69,29 @@ class ApiClient:
     _global_in_flight_requests: int = 0
     _global_state_lock = threading.Lock()
 
+    @classmethod
+    def _is_idempotent_arc_post_path(cls, path: str) -> bool:
+        """Return whether *path* is a server-idempotent ARC POST endpoint.
+
+        Covers ``POST /v3/arcs`` and ``POST /v3/harvests/{harvest_id}/arcs``.
+        Other POSTs (create/complete harvest) are not safe to retry.
+        """
+        normalized = path.lstrip("/")
+        if normalized == "v3/arcs":
+            return True
+        parts = normalized.split("/")
+        harvest_arc_path_parts = 4  # v3 / harvests / {id} / arcs
+        return (
+            len(parts) == harvest_arc_path_parts and parts[0] == "v3" and parts[1] == "harvests" and parts[3] == "arcs"
+        )
+
+    @classmethod
+    def _is_retryable_method(cls, method: str, path: str) -> bool:
+        """Return whether failures for this method/path may be retried."""
+        if method in cls._IDEMPOTENT_METHODS:
+            return True
+        return method == "POST" and cls._is_idempotent_arc_post_path(path)
+
     @classmethod
     def _configure_global_request_limiter(cls, max_concurrency: int) -> None:
         """Configure the package-wide request concurrency limiter."""
@@ -109,15 +132,15 @@ class ApiClient:
             limiter.release()
 
     @classmethod
-    def _should_retry_http_status(cls, method: str, status_code: int) -> bool:
-        """Return whether a response status is retryable for a method."""
+    def _should_retry_http_status(cls, method: str, path: str, status_code: int) -> bool:
+        """Return whether a response status is retryable for a method/path."""
         transient = {httpx.codes.BAD_GATEWAY, httpx.codes.SERVICE_UNAVAILABLE, httpx.codes.GATEWAY_TIMEOUT}
-        return method in cls._IDEMPOTENT_METHODS and status_code in transient
+        return cls._is_retryable_method(method, path) and status_code in transient
 
     @classmethod
-    def _should_retry_request_error(cls, method: str, error: httpx.RequestError) -> bool:
-        """Return whether a request error is retryable for a method."""
-        if method not in cls._IDEMPOTENT_METHODS:
+    def _should_retry_request_error(cls, method: str, path: str, error: httpx.RequestError) -> bool:
+        """Return whether a request error is retryable for a method/path."""
+        if not cls._is_retryable_method(method, path):
             return False
         return not isinstance(error, httpx.TimeoutException)
 
@@ -125,17 +148,35 @@ class ApiClient:
     def _should_retry_failure(
         cls,
         method: str,
+        path: str,
         *,
         status_code: int | None = None,
         request_error: httpx.RequestError | None = None,
     ) -> bool:
-        """Return whether an HTTP failure is retryable for a request method."""
+        """Return whether an HTTP failure is retryable for a request method/path."""
         if status_code is not None:
-            return cls._should_retry_http_status(method, status_code)
+            return cls._should_retry_http_status(method, path, status_code)
         if request_error is not None:
-            return cls._should_retry_request_error(method, request_error)
+            return cls._should_retry_request_error(method, path, request_error)
         return False
 
+    @classmethod
+    def _format_request_error(cls, error: BaseException) -> str:
+        """Return a non-empty description for a transport/request error.
+
+        ``httpx.ConnectError`` (and similar) often stringify to ``""``, which
+        produces useless log lines like ``Request error: ``. Fall back to the
+        exception type name, then ``repr``.
+        """
+        message = str(error).strip()
+        if message:
+            return message
+        type_name = type(error).__name__
+        representation = repr(error).strip()
+        if representation and representation != type_name:
+            return f"{type_name}: {representation}"
+        return type_name
+
     @classmethod
     def _build_failure_error_message(
         cls,
@@ -151,9 +192,10 @@ class ApiClient:
                 return f"Request failed after {max_retries} retries: HTTP {status_code}", status_code
             return cls._format_http_error_message(status_code, failure.response.text), status_code
 
+        detail = cls._format_request_error(failure)
         if retryable:
-            return f"Request failed after {max_retries} retries: {failure}", None
-        return f"Request failed: {failure}", None
+            return f"Request failed after {max_retries} retries: {detail}", None
+        return f"Request failed: {detail}", None
 
     @classmethod
     def _should_retry_or_raise_failure(
@@ -161,6 +203,7 @@ class ApiClient:
         failure: httpx.HTTPStatusError | httpx.RequestError,
         *,
         method: str,
+        path: str,
         attempt: int,
         max_retries: int,
     ) -> bool:
@@ -170,6 +213,7 @@ class ApiClient:
 
         should_retry = cls._should_retry_failure(
             method,
+            path,
             status_code=status_code,
             request_error=request_error,
         )
@@ -178,7 +222,7 @@ class ApiClient:
             if isinstance(failure, httpx.HTTPStatusError):
                 logger.warning("Transient HTTP error %d from server, will retry", failure.response.status_code)
             else:
-                logger.warning("Request error: %s. Retrying...", failure)
+                logger.warning("Request error: %s. Retrying...", cls._format_request_error(failure))
             return True
 
         msg, normalized_status_code = cls._build_failure_error_message(
@@ -445,7 +489,7 @@ class ApiClient:
                     resp = await client.request(method, path, **kwargs)
 
                 # Retry on transient server-side errors before raising
-                should_retry = self._should_retry_failure(method, status_code=resp.status_code)
+                should_retry = self._should_retry_failure(method, path, status_code=resp.status_code)
                 if should_retry and attempt < self._config.max_retries:
                     logger.warning("Transient HTTP error %d from server, will retry", resp.status_code)
                     continue
@@ -459,6 +503,7 @@ class ApiClient:
                 if self._should_retry_or_raise_failure(
                     e,
                     method=method,
+                    path=path,
                     attempt=attempt,
                     max_retries=self._config.max_retries,
                 ):

fairagro_middleware_api_client-10.0.3.dev26/tests/client_test_support.py

@@ -0,0 +1,50 @@
+"""Shared helpers for ApiClient unit tests (not a pytest plugin)."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from arctrl import ARC  # type: ignore[import-untyped]
+
+ARC_RESPONSE = {
+    "client_id": "test-client",
+    "message": "ARC processed successfully",
+    "arc_id": "arc-123",
+    "status": "created",
+    "metadata": {
+        "arc_hash": "abc123",
+        "status": "ACTIVE",
+        "first_seen": "2024-01-01T00:00:00Z",
+        "last_seen": "2024-01-01T00:00:00Z",
+    },
+    "events": [],
+}
+
+HARVEST_RESPONSE: dict[str, str | None | dict] = {
+    "client_id": "test-client",
+    "message": "Harvest created",
+    "harvest_id": "harvest-456",
+    "rdi": "test-rdi",
+    "status": "RUNNING",
+    "started_at": "2024-01-01T00:00:00Z",
+    "completed_at": None,
+    "statistics": {},
+}
+
+EXPECTED_ARC_UPLOADS = 3
+
+
+def rocrate_dict(identifier: str = "mock-arc") -> dict[str, Any]:
+    """Minimal valid RO-Crate payload for client tests."""
+    return {
+        "@context": "https://w3id.org/ro/crate/1.1/context",
+        "@graph": [{"@id": "./", "identifier": identifier}],
+    }
+
+
+async def arc_gen(*arcs: dict[str, Any] | str | ARC) -> AsyncGenerator[dict[str, Any] | str | ARC, None]:
+    """Yield the provided arc dicts, JSON strings, or ARC objects as an async generator."""
+    for arc in arcs:
+        yield arc

fairagro_middleware_api_client-10.0.3.dev26/tests/unit/conftest.py

@@ -0,0 +1,11 @@
+"""Shared fixtures for ApiClient unit tests."""
+
+import pytest
+
+from middleware.api_client import Config
+
+
+@pytest.fixture
+def client_config(test_config_dict: dict) -> Config:
+    """Create a Config instance for testing."""
+    return Config.from_data(test_config_dict)