parsimony-shared 0.7.0__tar.gz

parsimony_shared-0.7.0/.gitignore

@@ -0,0 +1,40 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+.Python
+build/
+dist/
+*.egg-info/
+*.egg
+
+.venv/
+.env
+.env.*
+!.env.example
+
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+coverage.xml
+
+uv.lock
+
+.vscode/
+.council/
+PLAN-*.md
+.idea/
+*.swp
+.DS_Store
+
+outputs/
+logs/
+# Recorded HTTP cassettes must never be committed — respx mocks are hand-authored
+# from upstream API documentation. A pre-commit / CI regex scan is the belt; this
+# ignore is the braces. Override per-file via `!` if you need a hand-authored
+# fixture checked in.
+packages/*/tests/fixtures/**
+!packages/*/tests/fixtures/README.md

parsimony_shared-0.7.0/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: parsimony-shared
+Version: 0.7.0
+Summary: Shared HTTP and metadata-enumeration helpers for the parsimony central-bank connector packages.
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27

parsimony_shared-0.7.0/parsimony_shared/__init__.py

@@ -0,0 +1,23 @@
+"""Internal shared helpers for official connector packages."""
+
+from parsimony_shared.cb_enumerate import (
+    DEFAULT_RETRY_BACKOFFS_S,
+    DEFAULT_RETRY_STATUSES,
+    DESCRIPTION_CHAR_CAP,
+    MetadataCrawlConfig,
+    ThrottledJsonFetcher,
+    enumerate_descriptions,
+    parse_retry_after,
+    truncate_description,
+)
+
+__all__ = [
+    "DESCRIPTION_CHAR_CAP",
+    "DEFAULT_RETRY_BACKOFFS_S",
+    "DEFAULT_RETRY_STATUSES",
+    "MetadataCrawlConfig",
+    "ThrottledJsonFetcher",
+    "enumerate_descriptions",
+    "parse_retry_after",
+    "truncate_description",
+]

parsimony_shared-0.7.0/parsimony_shared/cb_enumerate.py

@@ -0,0 +1,184 @@
+"""Shared helpers for central-bank catalog enumerators."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+DESCRIPTION_CHAR_CAP = 1500
+
+DEFAULT_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+DEFAULT_RETRY_BACKOFFS_S: tuple[float, ...] = (1.0, 2.0, 4.0)
+
+
+def truncate_description(text: str, *, cap: int = DESCRIPTION_CHAR_CAP) -> str:
+    """Cap a string at ``cap`` characters; return as-is if shorter."""
+    if not text:
+        return ""
+    if len(text) <= cap:
+        return text
+    return text[:cap].rstrip()
+
+
+def enumerate_descriptions(*parts: str, cap: int = DESCRIPTION_CHAR_CAP, sep: str = " ") -> str:
+    """Join non-empty description fragments and cap total length for embedders."""
+    joined = sep.join(p.strip() for p in parts if p and p.strip())
+    return truncate_description(joined, cap=cap)
+
+
+def parse_retry_after(response: httpx.Response) -> float | None:
+    """Parse the ``Retry-After`` header; ``None`` if absent or malformed."""
+    raw = response.headers.get("Retry-After")
+    if not raw:
+        return None
+    try:
+        return float(raw)
+    except ValueError:
+        return None
+
+
+@dataclass(frozen=True)
+class MetadataCrawlConfig:
+    """Throttling and retry policy for metadata enumeration crawls."""
+
+    concurrency: int = 4
+    inter_request_delay_s: float = 0.25
+    retry_statuses: frozenset[int] = field(default_factory=lambda: DEFAULT_RETRY_STATUSES)
+    retry_backoffs_s: tuple[float, ...] = DEFAULT_RETRY_BACKOFFS_S
+
+
+class ThrottledJsonFetcher:
+    """Async JSON GET helper with semaphore throttling, delay, and retries."""
+
+    def __init__(
+        self,
+        client: httpx.AsyncClient,
+        *,
+        provider: str,
+        config: MetadataCrawlConfig | None = None,
+        logger: logging.Logger | None = None,
+        accept_non_json: Callable[[httpx.Response], bool] | None = None,
+    ) -> None:
+        self._client = client
+        self._provider = provider
+        self._config = config or MetadataCrawlConfig()
+        self._logger = logger or logging.getLogger(__name__)
+        self._semaphore = asyncio.Semaphore(self._config.concurrency)
+        self._accept_non_json = accept_non_json
+
+    @property
+    def config(self) -> MetadataCrawlConfig:
+        return self._config
+
+    async def _get_with_retries(
+        self,
+        url: str,
+        *,
+        params: dict[str, str] | None = None,
+        label: str | None = None,
+    ) -> httpx.Response | None:
+        """GET *url* with throttling and retries; ``None`` after exhausted attempts."""
+        log_target = label or url
+        async with self._semaphore:
+            await asyncio.sleep(self._config.inter_request_delay_s)
+            last_status: int | None = None
+            last_error: str | None = None
+            for attempt, backoff in enumerate((*self._config.retry_backoffs_s, None)):
+                try:
+                    response = await self._client.get(url, params=params)
+                except httpx.HTTPError as exc:
+                    last_error = f"{type(exc).__name__}: {exc}"
+                    if backoff is None:
+                        break
+                    await asyncio.sleep(backoff)
+                    continue
+
+                if response.status_code == 200:
+                    if self._accept_non_json is not None and not self._accept_non_json(response):
+                        return None
+                    return response
+
+                last_status = response.status_code
+                if response.status_code in self._config.retry_statuses and backoff is not None:
+                    wait = parse_retry_after(response) or backoff
+                    self._logger.info(
+                        "%s %s returned %s (attempt %d); retrying in %.1fs",
+                        self._provider,
+                        log_target,
+                        response.status_code,
+                        attempt + 1,
+                        wait,
+                    )
+                    await asyncio.sleep(wait)
+                    continue
+                break
+
+            self._logger.warning(
+                "%s fetch failed for %s after retries (last_status=%s, last_error=%s)",
+                self._provider,
+                log_target,
+                last_status,
+                last_error,
+            )
+            return None
+
+    async def get_json(
+        self,
+        url: str,
+        *,
+        params: dict[str, str] | None = None,
+        label: str | None = None,
+    ) -> Any | None:
+        """GET *url* and return parsed JSON, or ``None`` after exhausted retries."""
+        log_target = label or url
+        response = await self._get_with_retries(url, params=params, label=label)
+        if response is None:
+            return None
+        try:
+            return response.json()
+        except ValueError as exc:
+            self._logger.warning("%s %s returned non-JSON body: %s", self._provider, log_target, exc)
+            return None
+
+    async def get_text(
+        self,
+        url: str,
+        *,
+        params: dict[str, str] | None = None,
+        label: str | None = None,
+    ) -> str | None:
+        """GET *url* and return response text, or ``None`` after exhausted retries."""
+        response = await self._get_with_retries(url, params=params, label=label)
+        if response is None:
+            return None
+        return response.text
+
+    async def get_content(
+        self,
+        url: str,
+        *,
+        params: dict[str, str] | None = None,
+        label: str | None = None,
+    ) -> bytes | None:
+        """GET *url* and return raw response bytes, or ``None`` after exhausted retries."""
+        response = await self._get_with_retries(url, params=params, label=label)
+        if response is None:
+            return None
+        return response.content
+
+
+__all__ = [
+    "DESCRIPTION_CHAR_CAP",
+    "DEFAULT_RETRY_BACKOFFS_S",
+    "DEFAULT_RETRY_STATUSES",
+    "MetadataCrawlConfig",
+    "ThrottledJsonFetcher",
+    "enumerate_descriptions",
+    "parse_retry_after",
+    "truncate_description",
+]

parsimony_shared-0.7.0/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "parsimony-shared"
+version = "0.7.0"
+description = "Shared HTTP and metadata-enumeration helpers for the parsimony central-bank connector packages."
+requires-python = ">=3.11"
+dependencies = [
+    "httpx>=0.27",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["parsimony_shared"]
+
+[tool.parsimony.conformance]
+# Not a `parsimony.providers` plugin — a shared helper library consumed by the
+# central-bank connectors (bde/bdf/bdp/boj/destatis). Opt out of the release
+# workflow's plugin-conformance gate (it has no provider entry point).
+skip = true

parsimony_shared-0.7.0/tests/test_smoke.py

@@ -0,0 +1,55 @@
+"""Offline smoke tests for parsimony-shared.
+
+parsimony-shared is a shared helper library, not a ``parsimony.providers``
+plugin, so it carries no conformance test. These checks pin the public helper
+surface so the release build has real (offline) coverage instead of failing
+pytest's "no tests collected" exit code.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+import httpx
+import pytest
+
+import parsimony_shared as ps
+from parsimony_shared import (
+    DESCRIPTION_CHAR_CAP,
+    MetadataCrawlConfig,
+    enumerate_descriptions,
+    parse_retry_after,
+    truncate_description,
+)
+
+
+def test_public_api_exports() -> None:
+    for name in ps.__all__:
+        assert hasattr(ps, name), f"missing export: {name}"
+
+
+def test_truncate_description() -> None:
+    assert truncate_description("") == ""
+    assert truncate_description("short") == "short"
+    out = truncate_description("x" * (DESCRIPTION_CHAR_CAP + 50))
+    assert len(out) <= DESCRIPTION_CHAR_CAP
+
+
+def test_enumerate_descriptions_joins_and_caps() -> None:
+    assert enumerate_descriptions("a", "", "  ", "b") == "a b"
+    capped = enumerate_descriptions("y" * 1000, "z" * 1000, cap=100)
+    assert len(capped) <= 100
+
+
+def test_metadata_crawl_config_defaults_are_frozen() -> None:
+    cfg = MetadataCrawlConfig()
+    assert cfg.concurrency >= 1
+    assert 429 in cfg.retry_statuses
+    with pytest.raises(dataclasses.FrozenInstanceError):
+        cfg.concurrency = 99  # type: ignore[misc]
+
+
+def test_parse_retry_after() -> None:
+    assert parse_retry_after(httpx.Response(429, headers={"Retry-After": "5"})) == 5.0
+    assert parse_retry_after(httpx.Response(429)) is None
+    assert parse_retry_after(httpx.Response(429, headers={"Retry-After": "soon"})) is None