statwrapper 0.1.0__py3-none-any.whl

statwrapper/models.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+_VALID_TIME_UNITS = {"Annual", "Quarterly", "Monthly", "Weekly", "Other"}
+
+
+def _validate_time_unit(value: str | None) -> None:
+    if value is None:
+        return
+    if value not in _VALID_TIME_UNITS:
+        raise ValueError(
+            f"Invalid time_unit: {value}. Must be one of {_VALID_TIME_UNITS}."
+        )
+
+
+@dataclass(slots=True)
+class Provider:
+    provider_code: str
+    api_type: str
+    default_language: str
+    languages: list[str]
+    label: str
+    country_code: str
+    base_api_url: str | None = None
+    base_web_url: str | None = None
+    rate_limit: float = 1.0
+    cell_limit: int = 5000
+    variable_limit: int = 1000
+    max_concurrency: int | None = None
+    data_formats: list[str] = field(default_factory=list)
+    extension: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class DiscoveredDataset:
+    task_id: uuid.UUID
+    provider_code: str
+    dataset_code: str
+    language: str
+    updated: datetime
+    label: str | None = None
+    source: str | None = None
+    note: list[str] | None = None
+    role: dict[str, list[str]] | None = None
+    description: str | None = None
+    time_unit: str | None = None
+    first_period: str | None = None
+    last_period: str | None = None
+    discontinued: bool | None = None
+    paths: list[list[dict[str, str]]] | None = None
+    subject_code: str | None = None
+    subject_label: str | None = None
+    metadata_url: str | None = None
+    data_url: str | None = None
+    web_url: str | None = None
+    doc_url: str | None = None
+    dimension: dict[str, dict[str, Any]] | None = None
+    required_dimensions: dict[str, bool | None] | None = None
+    official_statistics: bool | None = None
+    contact: dict[str, Any] | None = None
+    dimension_ids: list[str] | None = None
+    extension: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        _validate_time_unit(self.time_unit)
+
+
+@dataclass(slots=True)
+class ResolvedDatasetMetadata:
+    task_id: uuid.UUID
+    provider_code: str
+    dataset_code: str
+    language: str
+    updated: datetime
+    label: str
+    time_unit: str
+    first_period: str
+    last_period: str
+    paths: list[list[dict[str, str]]]
+    role: dict[str, list[str]]
+    metadata_url: str
+    data_url: str
+    dimension: dict[str, dict[str, Any]]
+    required_dimensions: dict[str, bool | None]
+    note: list[str] | None = None
+    source: str | None = None
+    description: str | None = None
+    discontinued: bool | None = None
+    subject_code: str | None = None
+    subject_label: str | None = None
+    web_url: str | None = None
+    doc_url: str | None = None
+    official_statistics: bool | None = None
+    contact: dict[str, Any] | None = None
+    dimension_ids: list[str] | None = None
+    extension: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        _validate_time_unit(self.time_unit)

statwrapper/parsers.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+from typing import Any
+
+from .utils import normalize_contact, normalize_note, parse_dt
+
+_VALID_TIME_UNITS = {"Annual", "Quarterly", "Monthly", "Weekly", "Other"}
+
+
+def normalize_paths(raw_paths: Any) -> list[list[dict[str, str]]] | None:
+    if raw_paths is None:
+        return None
+
+    if isinstance(raw_paths, dict) and isinstance(raw_paths.get("paths"), list):
+        raw_paths = raw_paths["paths"]
+    elif isinstance(raw_paths, dict) and isinstance(raw_paths.get("path"), list):
+        raw_paths = [raw_paths]
+    elif not isinstance(raw_paths, list):
+        raw_paths = [[raw_paths]]
+
+    output: list[list[dict[str, str]]] = []
+    for raw_path in raw_paths:
+        items = raw_path.get("path") if isinstance(raw_path, dict) else raw_path
+        if not isinstance(items, list):
+            continue
+        normalized_path: list[dict[str, str]] = []
+        for index, item in enumerate(items):
+            if isinstance(item, dict):
+                item_id = str(
+                    item.get("id")
+                    or item.get("code")
+                    or item.get("key")
+                    or item.get("value")
+                    or index
+                ).strip()
+                if not item_id:
+                    continue
+                label = str(
+                    item.get("label")
+                    or item.get("title")
+                    or item.get("name")
+                    or item_id
+                ).strip()
+            else:
+                item_id = str(item).strip()
+                if not item_id:
+                    continue
+                label = item_id
+            normalized_path.append({"id": item_id, "label": label})
+        if normalized_path:
+            output.append(normalized_path)
+    return output or None
+
+
+def normalize_role(raw_role: Any) -> dict[str, list[str]] | None:
+    if not isinstance(raw_role, dict):
+        return None
+    output: dict[str, list[str]] = {}
+    for key in ("time", "geo", "metric"):
+        values = raw_role.get(key)
+        if not isinstance(values, list):
+            continue
+        deduped: list[str] = []
+        for value in values:
+            text = str(value).strip()
+            if text and text not in deduped:
+                deduped.append(text)
+        if deduped:
+            output[key] = deduped
+    return output or None
+
+
+def normalize_dimensions(
+    raw_dimensions: Any,
+    raw_dimension_ids: Any,
+) -> tuple[
+    dict[str, dict[str, Any]] | None,
+    list[str] | None,
+    dict[str, bool | None] | None,
+    dict[str, dict[str, Any]] | None,
+]:
+    if not isinstance(raw_dimensions, dict):
+        return None, None, None, None
+
+    if isinstance(raw_dimension_ids, list):
+        dimension_ids = [str(item).strip() for item in raw_dimension_ids if str(item).strip()]
+    else:
+        dimension_ids = [str(key).strip() for key in raw_dimensions if str(key).strip()]
+
+    dimensions: dict[str, dict[str, Any]] = {}
+    required_dimensions: dict[str, bool | None] = {}
+    dimension_extensions: dict[str, dict[str, Any]] = {}
+    for dimension_id in dimension_ids:
+        raw_dimension = raw_dimensions.get(dimension_id)
+        if not isinstance(raw_dimension, dict):
+            continue
+        category = raw_dimension.get("category")
+        category = category if isinstance(category, dict) else {}
+
+        raw_index = category.get("index")
+        if isinstance(raw_index, dict):
+            index = {str(key): int(value) for key, value in raw_index.items()}
+        elif isinstance(raw_index, list):
+            index = {str(value): ordinal for ordinal, value in enumerate(raw_index)}
+        else:
+            raw_labels = category.get("label")
+            index = (
+                {str(key): ordinal for ordinal, key in enumerate(raw_labels)}
+                if isinstance(raw_labels, dict)
+                else {}
+            )
+        if not index:
+            continue
+
+        raw_labels = category.get("label")
+        labels = (
+            {key: str(raw_labels.get(key, key)) for key in index}
+            if isinstance(raw_labels, dict)
+            else {key: key for key in index}
+        )
+
+        parsed_dimension: dict[str, Any] = {
+            "label": str(raw_dimension.get("label") or dimension_id),
+            "category": {"index": index, "label": labels},
+        }
+        note = normalize_note(raw_dimension.get("note"))
+        if note:
+            parsed_dimension["note"] = note
+        dimensions[dimension_id] = parsed_dimension
+
+        elimination = raw_dimension.get("elimination")
+        required_dimensions[dimension_id] = (
+            None if elimination is None else not bool(elimination)
+        )
+
+        extras = {
+            key: value
+            for key, value in raw_dimension.items()
+            if key not in {"label", "category", "note", "elimination"}
+        }
+        if extras:
+            dimension_extensions[dimension_id] = {"extension": extras}
+
+    return (
+        dimensions or None,
+        dimension_ids or None,
+        required_dimensions or None,
+        dimension_extensions or None,
+    )
+
+
+def parse_pxweb2_discovery_table(
+    table: dict[str, Any],
+    *,
+    base_api_url: str,
+    base_web_url: str | None,
+    language: str,
+) -> dict[str, Any] | None:
+    dataset_code = str(table.get("id") or "").strip()
+    if not dataset_code:
+        return None
+
+    raw_time_unit = table.get("timeUnit")
+    time_unit = (
+        raw_time_unit
+        if isinstance(raw_time_unit, str) and raw_time_unit in _VALID_TIME_UNITS
+        else "Other"
+    )
+
+    clean_api = base_api_url.rstrip("/")
+    return {
+        "dataset_code": dataset_code,
+        "label": str(table.get("label") or table.get("title") or dataset_code),
+        "updated": parse_dt(table.get("updated")),
+        "time_unit": time_unit,
+        "first_period": str(table.get("firstPeriod") or "").strip() or None,
+        "last_period": str(table.get("lastPeriod") or "").strip() or None,
+        "description": str(table.get("description")).strip()
+        if table.get("description") is not None
+        else None,
+        "source": str(table.get("source")).strip()
+        if table.get("source") is not None
+        else None,
+        "note": normalize_note(table.get("note")),
+        "subject_code": str(table.get("subjectCode")).strip()
+        if table.get("subjectCode") is not None
+        else None,
+        "paths": normalize_paths(table.get("paths")),
+        "discontinued": bool(table.get("discontinued", False)),
+        "metadata_url": f"{clean_api}/tables/{dataset_code}/metadata",
+        "data_url": f"{clean_api}/tables/{dataset_code}/data?lang={language}",
+        "web_url": (
+            f"{base_web_url.rstrip('/')}/{language}/table/{dataset_code}"
+            if base_web_url
+            else None
+        ),
+        "extension": {
+            "links": table.get("links"),
+            "variable_names": list(table.get("variableNames", []))
+            if isinstance(table.get("variableNames"), list)
+            else [],
+        },
+    }
+
+
+def parse_pxweb2_metadata_payload(
+    payload: dict[str, Any],
+    *,
+    default_note: list[str] | None,
+    default_subject_label: str | None,
+    default_official_statistics: bool | None,
+    default_contact: dict[str, Any] | None,
+    default_extension: dict[str, Any],
+) -> dict[str, Any]:
+    (
+        dimensions,
+        dimension_ids,
+        required_dimensions,
+        dimension_extensions,
+    ) = normalize_dimensions(payload.get("dimension"), payload.get("id"))
+
+    extension = dict(default_extension)
+    note = normalize_note(payload.get("note")) or default_note
+    role = normalize_role(payload.get("role"))
+    subject_label = default_subject_label
+    official_statistics = default_official_statistics
+    contact = default_contact
+
+    raw_extension = payload.get("extension")
+    if isinstance(raw_extension, dict):
+        extension.update(raw_extension)
+        px = raw_extension.get("px")
+        if isinstance(px, dict):
+            if subject_label is None and px.get("subject-area") is not None:
+                subject_label = str(px.get("subject-area")).strip() or None
+            if px.get("official-statistics") is not None:
+                official_statistics = bool(px.get("official-statistics"))
+        normalized_contact = normalize_contact(raw_extension.get("contact"))
+        if normalized_contact is not None:
+            contact = normalized_contact
+
+    if dimension_extensions is not None:
+        merged = {}
+        existing = extension.get("dimension_extensions")
+        if isinstance(existing, dict):
+            merged.update(existing)
+        merged.update(dimension_extensions)
+        extension["dimension_extensions"] = merged
+
+    return {
+        "dimensions": dimensions,
+        "dimension_ids": dimension_ids,
+        "required_dimensions": required_dimensions,
+        "role": role,
+        "note": note,
+        "subject_label": subject_label,
+        "official_statistics": official_statistics,
+        "contact": contact,
+        "extension": extension,
+    }

statwrapper/provider_registry.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+from importlib.resources import files
+from pathlib import Path
+from typing import Any
+
+from .api_clients import DstClient, EurostatClient, PxWeb2Client, PxWebClient
+from .exceptions import (
+    ProviderNotFoundError,
+    UnsupportedAPITypeError,
+    UnsupportedLanguageError,
+)
+from .models import Provider
+
+WRAPPER_REGISTRY = {
+    "dst": DstClient,
+    "eurostat": EurostatClient,
+    "pxweb": PxWebClient,
+    "pxweb2": PxWeb2Client,
+}
+
+
+def _default_provider_path() -> Path:
+    repo_file = Path(__file__).resolve().parent.parent / "PROVIDERS.json"
+    if repo_file.exists():
+        return repo_file
+    return Path(files("statwrapper").joinpath("providers.json"))
+
+
+def load_providers(path: str | Path | None = None) -> list[Provider]:
+    resolved_path = Path(path) if path is not None else _default_provider_path()
+    data = json.loads(resolved_path.read_text(encoding="utf-8"))
+    return [Provider(**item) for item in data]
+
+
+def get_provider(
+    provider_code: str,
+    providers: list[Provider] | None = None,
+) -> Provider:
+    candidates = providers or load_providers()
+    for provider in candidates:
+        if provider.provider_code == provider_code:
+            return provider
+    raise ProviderNotFoundError(provider_code)
+
+
+def get_wrapper_class(api_type: str) -> type:
+    wrapper_cls = WRAPPER_REGISTRY.get(api_type.lower())
+    if wrapper_cls is None:
+        raise UnsupportedAPITypeError(api_type)
+    return wrapper_cls
+
+
+def create_wrapper(
+    provider: Provider,
+    *,
+    language: str,
+    json_request_handler: Any,
+    text_request_handler: Any = None,
+    bytes_request_handler: Any = None,
+    logger: Any = None,
+):
+    if language not in provider.languages:
+        raise UnsupportedLanguageError(provider.provider_code, language)
+    wrapper_cls = get_wrapper_class(provider.api_type)
+    return wrapper_cls(
+        provider,
+        language=language,
+        json_request_handler=json_request_handler,
+        text_request_handler=text_request_handler,
+        bytes_request_handler=bytes_request_handler,
+        logger=logger,
+    )