statwrapper 0.1.0__py3-none-any.whl

statwrapper/api_clients/pxweb2_client.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from ..base_api_client import APIWrapper
+from ..models import DiscoveredDataset, Provider, ResolvedDatasetMetadata
+from ..parsers import parse_pxweb2_discovery_table, parse_pxweb2_metadata_payload
+from ..utils import parse_dt
+
+
+class PxWeb2Client(APIWrapper):
+    def __init__(self, provider: Provider, **kwargs: Any) -> None:
+        super().__init__(
+            provider_code=provider.provider_code,
+            label=provider.label,
+            language=kwargs.pop("language"),
+            json_request_handler=kwargs.pop("json_request_handler"),
+            text_request_handler=kwargs.pop("text_request_handler", None),
+            bytes_request_handler=kwargs.pop("bytes_request_handler", None),
+            logger=kwargs.pop("logger", None),
+        )
+        self.provider = provider
+        self.api_type = provider.api_type
+        self.base_api_url = provider.base_api_url or ""
+        self.base_web_url = provider.base_web_url
+
+    def _get_api_url(self, endpoint: str, dataset_code: str | None = None) -> str:
+        clean_api = self.base_api_url.rstrip("/")
+        if endpoint == "config":
+            return f"{clean_api}/config"
+        if endpoint == "tables":
+            return f"{clean_api}/tables"
+        if endpoint == "metadata" and dataset_code:
+            return f"{clean_api}/tables/{dataset_code}/metadata"
+        if endpoint == "data" and dataset_code:
+            return f"{clean_api}/tables/{dataset_code}/data"
+        raise ValueError(f"Unsupported endpoint: {endpoint}")
+
+    async def health_check(self, dataset_code: str | None = None) -> bool:
+        url = self._get_api_url("metadata" if dataset_code else "config", dataset_code)
+        payload = await self._get_json(url, params={"lang": self.language})
+        return payload is not None
+
+    async def discover_datasets(
+        self,
+        task_id: uuid.UUID,
+        **_: Any,
+    ) -> list[DiscoveredDataset]:
+        discovered: list[DiscoveredDataset] = []
+        page_number = 1
+        total_pages = 1
+        while page_number <= total_pages:
+            payload = await self._get_json(
+                self._get_api_url("tables"),
+                params={
+                    "lang": self.language,
+                    "pageSize": 1000,
+                    "pageNumber": page_number,
+                    "includeDiscontinued": "true",
+                },
+            )
+            if not isinstance(payload, dict):
+                break
+            tables = payload.get("tables")
+            page_info = payload.get("page", {})
+            if not isinstance(tables, list):
+                break
+            total_pages = int(page_info.get("totalPages", page_number))
+            for table in tables:
+                if not isinstance(table, dict):
+                    continue
+                parsed = parse_pxweb2_discovery_table(
+                    table,
+                    base_api_url=self.base_api_url,
+                    base_web_url=self.base_web_url,
+                    language=self.language,
+                )
+                if not parsed:
+                    continue
+                updated = parsed["updated"] or parse_dt("1970-01-01T00:00:00+00:00")
+                if updated is None:
+                    continue
+                discovered.append(
+                    DiscoveredDataset(
+                        task_id=task_id,
+                        provider_code=self.provider_code,
+                        dataset_code=parsed["dataset_code"],
+                        language=self.language,
+                        updated=updated,
+                        label=parsed["label"],
+                        source=parsed["source"],
+                        note=parsed["note"],
+                        description=parsed["description"],
+                        time_unit=parsed["time_unit"],
+                        first_period=parsed["first_period"],
+                        last_period=parsed["last_period"],
+                        discontinued=parsed["discontinued"],
+                        paths=parsed["paths"],
+                        subject_code=parsed["subject_code"],
+                        metadata_url=parsed["metadata_url"],
+                        data_url=parsed["data_url"],
+                        web_url=parsed["web_url"],
+                        extension=parsed["extension"],
+                    )
+                )
+            page_number += 1
+        return discovered
+
+    async def resolve_dataset_metadata(
+        self,
+        discovered: DiscoveredDataset,
+        task_id: uuid.UUID | None = None,
+        **_: Any,
+    ) -> ResolvedDatasetMetadata:
+        payload = await self._get_json(
+            self._get_api_url("metadata", discovered.dataset_code),
+            params={"lang": self.language, "outputFormat": "json-stat2"},
+        )
+        parsed = (
+            parse_pxweb2_metadata_payload(
+                payload,
+                default_note=discovered.note,
+                default_subject_label=discovered.subject_label,
+                default_official_statistics=discovered.official_statistics,
+                default_contact=discovered.contact,
+                default_extension=discovered.extension,
+            )
+            if isinstance(payload, dict)
+            else {
+                "dimensions": None,
+                "dimension_ids": None,
+                "required_dimensions": None,
+                "role": discovered.role,
+                "note": discovered.note,
+                "subject_label": discovered.subject_label,
+                "official_statistics": discovered.official_statistics,
+                "contact": discovered.contact,
+                "extension": dict(discovered.extension),
+            }
+        )
+        return ResolvedDatasetMetadata(
+            task_id=task_id or discovered.task_id,
+            provider_code=discovered.provider_code,
+            dataset_code=discovered.dataset_code,
+            language=discovered.language,
+            updated=discovered.updated,
+            label=discovered.label or discovered.dataset_code,
+            time_unit=discovered.time_unit or "Other",
+            first_period=discovered.first_period or "",
+            last_period=discovered.last_period or discovered.first_period or "",
+            paths=discovered.paths or [],
+            role=parsed["role"] or {},
+            metadata_url=discovered.metadata_url
+            or self._get_api_url("metadata", discovered.dataset_code),
+            data_url=discovered.data_url or self._get_api_url("data", discovered.dataset_code),
+            dimension=parsed["dimensions"] or {},
+            required_dimensions=parsed["required_dimensions"] or {},
+            note=parsed["note"],
+            source=discovered.source,
+            description=discovered.description,
+            discontinued=discovered.discontinued,
+            subject_code=discovered.subject_code,
+            subject_label=parsed["subject_label"],
+            web_url=discovered.web_url,
+            official_statistics=parsed["official_statistics"],
+            contact=parsed["contact"],
+            dimension_ids=parsed["dimension_ids"],
+            extension=parsed["extension"],
+        )

statwrapper/api_clients/pxweb_client.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import quote
+
+from ..base_api_client import APIWrapper
+from ..models import DiscoveredDataset, Provider, ResolvedDatasetMetadata
+from ..utils import detect_role, determine_time_unit, parse_dt
+
+
+def _encode_segment(value: str) -> str:
+    return quote(value, safe="")
+
+
+def _normalize_path(path: str) -> str:
+    ids = [segment.strip() for segment in path.split("/") if segment.strip()]
+    return "/".join(ids)
+
+
+@dataclass(slots=True)
+class _PxWebTableEntry:
+    id: str
+    title: str
+    path: str
+    db_id: str
+    language: str
+    provider_code: str
+    base_api_url: str
+    base_web_url: str | None
+    published: str | None = None
+    updated: str | None = None
+
+    @property
+    def api_url(self) -> str:
+        return (
+            f"{self.base_api_url.rstrip('/')}/{self.language}/"
+            f"{_encode_segment(self.db_id)}/{_normalize_path(self.path)}/{_encode_segment(self.id)}"
+        )
+
+    @property
+    def web_url(self) -> str | None:
+        if not self.base_web_url:
+            return None
+        path_segment = _normalize_path(self.path).replace("/", "__")
+        return (
+            f"{self.base_web_url.rstrip('/')}/{self.language}/"
+            f"{_encode_segment(self.db_id)}/{_encode_segment(self.db_id)}__{path_segment}/{_encode_segment(self.id)}/"
+        )
+
+
+class PxWebClient(APIWrapper):
+    def __init__(self, provider: Provider, **kwargs: Any) -> None:
+        super().__init__(
+            provider_code=provider.provider_code,
+            label=provider.label,
+            language=kwargs.pop("language"),
+            json_request_handler=kwargs.pop("json_request_handler"),
+            text_request_handler=kwargs.pop("text_request_handler", None),
+            bytes_request_handler=kwargs.pop("bytes_request_handler", None),
+            logger=kwargs.pop("logger", None),
+        )
+        self.provider = provider
+        self.api_type = provider.api_type
+        self.base_api_url = provider.base_api_url
+        self.base_web_url = provider.base_web_url
+        extension = provider.extension if isinstance(provider.extension, dict) else {}
+        dbid = extension.get("dbid", {})
+        self.db_ids = set(dbid.keys()) if isinstance(dbid, dict) else set()
+        if not self.base_api_url or not self.db_ids:
+            raise ValueError("PxWeb provider requires base_api_url and extension.dbid")
+
+    async def health_check(self, dataset_code: str | None = None) -> bool:
+        if dataset_code is None:
+            db_id = next(iter(self.db_ids))
+            url = f"{self.base_api_url.rstrip('/')}/{self.language}/{_encode_segment(db_id)}"
+        else:
+            url = dataset_code
+        payload = await self._get_json(url, params={"filter": "*", "query": "*"})
+        return payload is not None
+
+    async def discover_datasets(
+        self,
+        task_id: uuid.UUID,
+        **_: Any,
+    ) -> list[DiscoveredDataset]:
+        discovered: list[DiscoveredDataset] = []
+        for db_id in sorted(self.db_ids):
+            url = f"{self.base_api_url.rstrip('/')}/{self.language}/{_encode_segment(db_id)}"
+            payload = await self._get_json(url, params={"filter": "*", "query": "*"})
+            if not isinstance(payload, list):
+                continue
+            for entry_data in payload:
+                if not isinstance(entry_data, dict):
+                    continue
+                entry_id = str(entry_data.get("id") or "").strip()
+                path = str(entry_data.get("path") or "").strip()
+                if not entry_id or not path:
+                    continue
+                entry = _PxWebTableEntry(
+                    id=entry_id,
+                    title=str(entry_data.get("title") or entry_id).strip(),
+                    path=path,
+                    db_id=db_id,
+                    language=self.language,
+                    provider_code=self.provider_code,
+                    base_api_url=self.base_api_url,
+                    base_web_url=self.base_web_url,
+                    published=str(entry_data.get("published") or "").strip() or None,
+                    updated=str(entry_data.get("updated") or "").strip() or None,
+                )
+                updated = parse_dt(entry.updated) or parse_dt(entry.published)
+                if updated is None:
+                    continue
+                path_items = [
+                    {"id": segment, "label": segment}
+                    for segment in entry.path.split("/")
+                    if segment.strip()
+                ]
+                discovered.append(
+                    DiscoveredDataset(
+                        task_id=task_id,
+                        provider_code=self.provider_code,
+                        dataset_code=entry.id,
+                        language=self.language,
+                        updated=updated,
+                        label=entry.title,
+                        metadata_url=entry.api_url,
+                        data_url=entry.api_url,
+                        web_url=entry.web_url,
+                        paths=[path_items] if path_items else None,
+                        subject_code=path_items[-1]["id"] if path_items else None,
+                        subject_label=path_items[-1]["label"] if path_items else None,
+                    )
+                )
+        return discovered
+
+    async def resolve_dataset_metadata(
+        self,
+        discovered: DiscoveredDataset,
+        task_id: uuid.UUID | None = None,
+        **_: Any,
+    ) -> ResolvedDatasetMetadata:
+        metadata_url = discovered.metadata_url
+        if metadata_url is None:
+            raise ValueError("Discovered dataset is missing metadata_url")
+        payload = await self._get_json(metadata_url)
+        dimensions: dict[str, dict[str, Any]] = {}
+        dimension_ids: list[str] = []
+        required_dimensions: dict[str, bool | None] = {}
+        role: dict[str, list[str]] = {}
+        dimension_extensions: dict[str, dict[str, Any]] = {}
+        time_labels: list[str] = []
+        title = discovered.label or discovered.dataset_code
+        if isinstance(payload, dict):
+            payload_title = payload.get("title")
+            if isinstance(payload_title, str) and payload_title.strip():
+                title = payload_title.strip()
+            variables = payload.get("variables")
+            if isinstance(variables, list):
+                for position, var in enumerate(variables):
+                    if not isinstance(var, dict):
+                        continue
+                    code = str(var.get("code") or "").strip()
+                    if not code:
+                        continue
+                    label = str(var.get("text") or code).strip()
+                    values = var.get("values") if isinstance(var.get("values"), list) else []
+                    value_texts = (
+                        var.get("valueTexts")
+                        if isinstance(var.get("valueTexts"), list)
+                        else []
+                    )
+                    index = [str(value) for value in values]
+                    labels = {
+                        str(value): str(value_texts[idx] if idx < len(value_texts) else value)
+                        for idx, value in enumerate(values)
+                    }
+                    dimensions[code] = {
+                        "label": label,
+                        "category": {
+                            "index": {
+                                category_code: ordinal
+                                for ordinal, category_code in enumerate(index)
+                            },
+                            "label": labels,
+                        },
+                    }
+                    dimension_ids.append(code)
+                    elimination = var.get("elimination")
+                    required_dimensions[code] = (
+                        None if elimination is None else not bool(elimination)
+                    )
+                    inferred_role = detect_role(code, label, bool(var.get("time")))
+                    if inferred_role is not None:
+                        role.setdefault(inferred_role, []).append(code)
+                    if inferred_role == "time":
+                        time_labels = [label for label in labels.values() if label.strip()]
+                    extras = {
+                        key: value
+                        for key, value in var.items()
+                        if key
+                        not in {"code", "text", "values", "valueTexts", "time", "elimination"}
+                    }
+                    extras["position"] = position
+                    if extras:
+                        dimension_extensions[code] = {"extension": extras}
+        sorted_time_labels = sorted(time_labels)
+        first_period = discovered.first_period or (sorted_time_labels[0] if sorted_time_labels else "")
+        last_period = discovered.last_period or (sorted_time_labels[-1] if sorted_time_labels else first_period)
+        time_unit = discovered.time_unit or determine_time_unit(first_period, last_period)
+        extension = dict(discovered.extension)
+        if dimension_extensions:
+            extension["dimension_extensions"] = dimension_extensions
+        return ResolvedDatasetMetadata(
+            task_id=task_id or discovered.task_id,
+            provider_code=discovered.provider_code,
+            dataset_code=discovered.dataset_code,
+            language=discovered.language,
+            updated=discovered.updated,
+            label=title,
+            time_unit=time_unit,
+            first_period=first_period,
+            last_period=last_period,
+            paths=discovered.paths or [],
+            role=role,
+            metadata_url=metadata_url,
+            data_url=discovered.data_url or metadata_url,
+            dimension=dimensions,
+            required_dimensions=required_dimensions,
+            note=discovered.note,
+            source=discovered.source,
+            description=discovered.description,
+            discontinued=discovered.discontinued,
+            subject_code=discovered.subject_code,
+            subject_label=discovered.subject_label,
+            web_url=discovered.web_url,
+            doc_url=discovered.doc_url,
+            official_statistics=discovered.official_statistics,
+            contact=discovered.contact,
+            dimension_ids=dimension_ids or None,
+            extension=extension,
+        )

statwrapper/base_api_client.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from .models import DiscoveredDataset, ResolvedDatasetMetadata
+
+JsonResponse = dict[str, Any] | list[Any] | None
+JsonRequestHandler = Callable[..., Awaitable[JsonResponse]]
+TextRequestHandler = Callable[..., Awaitable[str | None]]
+BytesRequestHandler = Callable[..., Awaitable[bytes | None]]
+
+
+class APIWrapper(ABC):
+    """Base class for dependency-free API wrappers."""
+
+    def __init__(
+        self,
+        provider_code: str,
+        label: str,
+        language: str,
+        json_request_handler: JsonRequestHandler,
+        logger: logging.Logger | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self.provider_code = provider_code
+        self.label = label
+        self.language = language
+        self.api_type = "generic"
+        self.json_request_handler = json_request_handler
+        self.text_request_handler: TextRequestHandler | None = kwargs.pop(
+            "text_request_handler",
+            None,
+        )
+        self.bytes_request_handler: BytesRequestHandler | None = kwargs.pop(
+            "bytes_request_handler",
+            None,
+        )
+        self.logger = logger or logging.getLogger(__name__)
+        self._logger_prefix = f"[{self.provider_code}:{self.language}]"
+
+    def _log_prefix(self, message: str) -> str:
+        return f"{self._logger_prefix} {message}"
+
+    async def _get_json(self, url: str, **kwargs: Any) -> JsonResponse:
+        return await self.json_request_handler(url, **kwargs)
+
+    async def _get_text(self, url: str, **kwargs: Any) -> str | None:
+        if self.text_request_handler is None:
+            raise RuntimeError("text_request_handler is not configured")
+        return await self.text_request_handler(url, **kwargs)
+
+    async def _get_bytes(self, url: str, **kwargs: Any) -> bytes | None:
+        if self.bytes_request_handler is None:
+            raise RuntimeError("bytes_request_handler is not configured")
+        return await self.bytes_request_handler(url, **kwargs)
+
+    @abstractmethod
+    async def health_check(self, dataset_code: str | None = None) -> bool:
+        """Return whether the provider is reachable."""
+
+    @abstractmethod
+    async def discover_datasets(
+        self,
+        task_id: Any,
+        **kwargs: Any,
+    ) -> list[DiscoveredDataset]:
+        """Discover datasets from the provider."""
+
+    @abstractmethod
+    async def resolve_dataset_metadata(
+        self,
+        discovered: DiscoveredDataset,
+        task_id: Any | None = None,
+        **kwargs: Any,
+    ) -> ResolvedDatasetMetadata:
+        """Resolve a discovered dataset into persistence-ready metadata."""

statwrapper/exceptions.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+
+class StatwrapperError(Exception):
+    """Base exception for statwrapper."""
+
+
+class ProviderNotFoundError(StatwrapperError):
+    def __init__(self, provider_code: str) -> None:
+        super().__init__(f"Provider not found: {provider_code!r}")
+
+
+class UnsupportedAPITypeError(StatwrapperError):
+    def __init__(self, api_type: str) -> None:
+        super().__init__(f"No wrapper registered for api_type={api_type!r}")
+
+
+class UnsupportedLanguageError(StatwrapperError):
+    def __init__(self, provider_code: str, language: str) -> None:
+        super().__init__(
+            f"Language {language!r} is not supported by provider {provider_code!r}"
+        )

statwrapper/http.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlencode, urlparse
+from urllib.request import Request, urlopen
+
+
+@dataclass
+class _HostLimiter:
+    interval: float
+    lock: asyncio.Lock
+    last_request_time: float = 0.0
+
+
+class RateLimitedSession:
+    """Minimal stdlib-backed async HTTP client with per-host spacing."""
+
+    def __init__(
+        self,
+        *,
+        default_rate: float = 1.0,
+        host_rates: dict[str, float] | None = None,
+        timeout: float = 60.0,
+        headers: dict[str, str] | None = None,
+    ) -> None:
+        self._default_interval = 1.0 / default_rate if default_rate > 0 else 0.0
+        self._timeout = timeout
+        self._headers = headers or {"User-Agent": "statwrapper/0.1.0"}
+        self._limiters: dict[str, _HostLimiter] = {}
+        for host, rate in (host_rates or {}).items():
+            hostname = urlparse(host).hostname or host
+            interval = 1.0 / rate if rate > 0 else 0.0
+            self._limiters[hostname] = _HostLimiter(interval=interval, lock=asyncio.Lock())
+
+    async def __aenter__(self) -> RateLimitedSession:
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        return None
+
+    def _get_limiter(self, url: str) -> _HostLimiter:
+        hostname = urlparse(url).hostname or ""
+        limiter = self._limiters.get(hostname)
+        if limiter is None:
+            limiter = _HostLimiter(
+                interval=self._default_interval,
+                lock=asyncio.Lock(),
+            )
+            self._limiters[hostname] = limiter
+        return limiter
+
+    async def _wait_for_slot(self, url: str) -> None:
+        limiter = self._get_limiter(url)
+        async with limiter.lock:
+            if limiter.interval > 0:
+                now = time.monotonic()
+                elapsed = now - limiter.last_request_time
+                if elapsed < limiter.interval:
+                    await asyncio.sleep(limiter.interval - elapsed)
+                limiter.last_request_time = time.monotonic()
+
+    def _build_url(self, url: str, params: dict[str, Any] | None) -> str:
+        if not params:
+            return url
+        encoded = urlencode(
+            {
+                key: value
+                for key, value in params.items()
+                if value is not None
+            },
+            doseq=True,
+        )
+        separator = "&" if "?" in url else "?"
+        return f"{url}{separator}{encoded}"
+
+    async def _read(self, url: str, params: dict[str, Any] | None = None) -> bytes | None:
+        await self._wait_for_slot(url)
+        request = Request(self._build_url(url, params), headers=self._headers)
+        try:
+            return await asyncio.to_thread(
+                lambda: urlopen(request, timeout=self._timeout).read()
+            )
+        except Exception:
+            return None
+
+    async def get_bytes(
+        self,
+        url: str,
+        *,
+        params: dict[str, Any] | None = None,
+        **_: Any,
+    ) -> bytes | None:
+        return await self._read(url, params=params)
+
+    async def get_text(
+        self,
+        url: str,
+        *,
+        params: dict[str, Any] | None = None,
+        encoding: str = "utf-8",
+        **_: Any,
+    ) -> str | None:
+        payload = await self._read(url, params=params)
+        if payload is None:
+            return None
+        return payload.decode(encoding, errors="replace")
+
+    async def get_json(
+        self,
+        url: str,
+        *,
+        params: dict[str, Any] | None = None,
+        **_: Any,
+    ) -> dict[str, Any] | list[Any] | None:
+        payload = await self.get_text(url, params=params)
+        if payload is None:
+            return None
+        try:
+            decoded = json.loads(payload)
+        except json.JSONDecodeError:
+            return None
+        return decoded if isinstance(decoded, (dict, list)) else None