kpubdata 0.1.0__py3-none-any.whl

kpubdata/core/models.py

@@ -0,0 +1,156 @@
+"""Canonical domain models shared across providers and adapters."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass as _stdlib_dataclass
+from dataclasses import field
+from types import MappingProxyType
+from typing import TypeVar
+
+from typing_extensions import dataclass_transform
+
+from kpubdata.core.capability import Operation, QuerySupport
+from kpubdata.core.representation import Representation
+
+_T = TypeVar("_T")
+
+
+@dataclass_transform()
+def _dataclass(
+    *,
+    slots: bool = False,
+    frozen: bool = False,
+) -> Callable[[type[_T]], type[_T]]:
+    """Return a decorator that applies ``dataclasses.dataclass`` with fixed options."""
+
+    def _decorate(cls: type[_T]) -> type[_T]:
+        return _stdlib_dataclass(slots=slots, frozen=frozen)(cls)  # pyright: ignore[reportCallIssue]
+
+    return _decorate
+
+
+def _empty_proxy() -> MappingProxyType[str, object]:
+    """Return an empty immutable string-keyed mapping proxy."""
+    return MappingProxyType({})
+
+
+def _empty_object_proxy() -> MappingProxyType[str, object]:
+    """Return an empty immutable object-valued mapping proxy."""
+    return MappingProxyType({})
+
+
+@_dataclass(slots=True, frozen=True)
+class DatasetRef:
+    """Canonical immutable reference to a provider dataset.
+
+    Attributes:
+        query_support: Structured list-query feature metadata, if known.
+        raw_metadata: Provider-native discovery metadata for debugging.
+    """
+
+    id: str
+    provider: str
+    dataset_key: str
+    name: str
+    representation: Representation
+    operations: frozenset[Operation] = frozenset()
+    query_support: QuerySupport | None = None
+    raw_metadata: MappingProxyType[str, object] = field(default_factory=_empty_proxy)
+
+    def supports(self, op: Operation) -> bool:
+        """Return whether this dataset supports the requested operation."""
+
+        return op in self.operations
+
+    def __repr__(self) -> str:
+        """Return a concise developer-friendly representation."""
+        ops = ", ".join(sorted(operation.value for operation in self.operations))
+        return f"DatasetRef(id={self.id!r}, provider={self.provider!r}, ops=[{ops}])"
+
+
+@_dataclass(slots=True)
+class Query:
+    """Provider-agnostic query object for listing records.
+
+    Attributes:
+        filters: Provider-specific filter payload merged into query translation.
+        extra: Additional provider-native parameters not covered canonically.
+    """
+
+    filters: dict[str, object] = field(default_factory=dict)
+    page: int | None = None
+    page_size: int | None = None
+    cursor: str | None = None
+    start_date: str | None = None
+    end_date: str | None = None
+    fields: list[str] | None = None
+    sort: list[str] | None = None
+    extra: dict[str, object] = field(default_factory=dict)
+
+
+@_dataclass(slots=True)
+class RecordBatch:
+    """Batch of normalized records returned from a dataset query.
+
+    Attributes:
+        next_page: Next offset page number for offset pagination.
+        next_cursor: Opaque cursor token for cursor pagination.
+        raw: Provider-native response payload used to derive this batch.
+        meta: Additional adapter metadata that does not fit canonical fields.
+    """
+
+    items: list[dict[str, object]]
+    dataset: DatasetRef
+    total_count: int | None = None
+    next_page: int | None = None
+    next_cursor: str | None = None
+    raw: object | None = None
+    meta: dict[str, object] = field(default_factory=dict)
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    def __iter__(self) -> Iterator[dict[str, object]]:
+        return iter(self.items)
+
+    def __bool__(self) -> bool:
+        return bool(self.items)
+
+
+@_dataclass(slots=True)
+class FieldDescriptor:
+    """Describe a single field in a dataset schema.
+
+    Attributes:
+        raw: Provider-native field metadata retained for advanced use.
+    """
+
+    name: str
+    title: str | None = None
+    type: str | None = None
+    description: str | None = None
+    nullable: bool | None = None
+    raw: MappingProxyType[str, object] = field(default_factory=_empty_object_proxy)
+
+
+@_dataclass(slots=True)
+class SchemaDescriptor:
+    """Describe schema metadata exposed for a dataset.
+
+    Attributes:
+        raw: Provider-native schema metadata retained without normalization.
+    """
+
+    dataset: DatasetRef
+    fields: list[FieldDescriptor]
+    raw: MappingProxyType[str, object] = field(default_factory=_empty_object_proxy)
+
+
+__all__ = [
+    "DatasetRef",
+    "FieldDescriptor",
+    "Query",
+    "RecordBatch",
+    "SchemaDescriptor",
+]

kpubdata/core/protocol.py

@@ -0,0 +1,60 @@
+"""Provider adapter protocol — the extension point for KPubData."""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from kpubdata.core.models import DatasetRef, Query, RecordBatch, SchemaDescriptor
+
+
+@runtime_checkable
+class ProviderAdapter(Protocol):
+    """Protocol that every provider adapter must satisfy.
+
+    Adapters are responsible for auth, discovery, translation, error mapping,
+    raw access, and truthful capability declaration.
+    """
+
+    @property
+    def name(self) -> str:
+        """Return provider identifier (for example ``datago`` or ``seoul``)."""
+
+        ...
+
+    def list_datasets(self) -> list[DatasetRef]:
+        """Return discoverable datasets for this provider."""
+
+        ...
+
+    def search_datasets(self, text: str) -> list[DatasetRef]:
+        """Return datasets matching free-text search for this provider."""
+
+        ...
+
+    def get_dataset(self, dataset_key: str) -> DatasetRef:
+        """Resolve a provider-local dataset key to a canonical dataset ref."""
+
+        ...
+
+    def query_records(self, dataset: DatasetRef, query: Query) -> RecordBatch:
+        """Execute a canonical list/query request for a dataset."""
+
+        ...
+
+    def get_record(self, dataset: DatasetRef, key: dict[str, object]) -> dict[str, object] | None:
+        """Return a single normalized record by key or ``None`` if missing."""
+
+        ...
+
+    def get_schema(self, dataset: DatasetRef) -> SchemaDescriptor | None:
+        """Return canonical schema metadata if supported."""
+
+        ...
+
+    def call_raw(self, dataset: DatasetRef, operation: str, params: dict[str, object]) -> object:
+        """Execute provider-native operation and return unnormalized response."""
+
+        ...
+
+
+__all__ = ["ProviderAdapter"]

kpubdata/core/representation.py

@@ -0,0 +1,19 @@
+"""Dataset representation types for source/shape modeling."""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class Representation(str, Enum):
+    """How a dataset is provisioned: source shape rather than access mode."""
+
+    API_JSON = "api_json"
+    API_XML = "api_xml"
+    FILE_CSV = "file_csv"
+    FILE_EXCEL = "file_excel"
+    SHEET = "sheet"
+    OTHER = "other"
+
+
+__all__ = ["Representation"]

kpubdata/exceptions.py

@@ -0,0 +1,116 @@
+"""KPubData exception hierarchy with structured error context."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class PublicDataError(Exception):
+    """Base for all KPubData errors with structured context attributes."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        provider: str | None = None,
+        dataset_id: str | None = None,
+        operation: str | None = None,
+        status_code: int | None = None,
+        provider_code: str | None = None,
+        retryable: bool = False,
+        detail: object = None,
+    ) -> None:
+        """Initialize an error with optional provider and transport metadata."""
+
+        super().__init__(message)
+        self.provider = provider
+        self.dataset_id = dataset_id
+        self.operation = operation
+        self.status_code = status_code
+        self.provider_code = provider_code
+        self.retryable = retryable
+        self.detail = detail
+
+    def __repr__(self) -> str:
+        """Return a structured repr including provider and transport metadata."""
+        parts = [f"{type(self).__name__}({self.args[0]!r}"]
+        if self.provider:
+            parts.append(f"provider={self.provider!r}")
+        if self.dataset_id:
+            parts.append(f"dataset={self.dataset_id!r}")
+        if self.status_code is not None:
+            parts.append(f"status={self.status_code}")
+        if self.retryable:
+            parts.append("retryable=True")
+        return ", ".join(parts) + ")"
+
+
+class ConfigError(PublicDataError):
+    """Raised when KPubData configuration is invalid or incomplete."""
+
+
+class AuthError(PublicDataError):
+    """Raised for authentication or authorization failures."""
+
+
+class TransportError(PublicDataError):
+    """Raised for network and transport-layer failures."""
+
+    def __init__(self, message: str, **kwargs: Any) -> None:
+        """Initialize a retryable transport error by default."""
+
+        kwargs.setdefault("retryable", True)
+        super().__init__(message, **kwargs)
+
+
+class TransportTimeoutError(TransportError):
+    """Raised when a provider request exceeds timeout limits."""
+
+
+class RateLimitError(TransportError):
+    """Raised when the provider rejects requests due to throttling."""
+
+
+class ServiceUnavailableError(TransportError):
+    """Raised when the upstream provider service is temporarily unavailable."""
+
+
+class ParseError(PublicDataError):
+    """Raised when provider payloads cannot be parsed safely."""
+
+
+class InvalidRequestError(PublicDataError):
+    """Raised when query or operation inputs are semantically invalid."""
+
+
+class ProviderResponseError(PublicDataError):
+    """Raised when provider responses violate contract expectations."""
+
+
+class UnsupportedCapabilityError(PublicDataError):
+    """Raised when a requested operation is unsupported for a dataset."""
+
+
+class DatasetNotFoundError(PublicDataError):
+    """Raised when a requested dataset identifier cannot be resolved."""
+
+
+class ProviderNotRegisteredError(PublicDataError):
+    """Raised when a provider key is not present in the registry."""
+
+
+__all__ = [
+    "AuthError",
+    "ConfigError",
+    "DatasetNotFoundError",
+    "InvalidRequestError",
+    "ParseError",
+    "ProviderNotRegisteredError",
+    "ProviderResponseError",
+    "PublicDataError",
+    "RateLimitError",
+    "ServiceUnavailableError",
+    "TransportError",
+    "TransportTimeoutError",
+    "UnsupportedCapabilityError",
+]

kpubdata/providers/__init__.py

@@ -0,0 +1,5 @@
+"""Built-in provider adapter packages."""
+
+from __future__ import annotations
+
+__all__: list[str] = []

kpubdata/providers/_common.py

@@ -0,0 +1,204 @@
+"""Shared catalogue and schema utilities for provider adapters."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from importlib.resources import files
+from types import MappingProxyType
+from typing import cast
+
+from kpubdata.core.capability import Operation, PaginationMode, QuerySupport
+from kpubdata.core.models import (
+    DatasetRef,
+    FieldDescriptor,
+    SchemaDescriptor,
+)
+from kpubdata.core.representation import Representation
+from kpubdata.exceptions import ConfigError
+
+
+def load_catalogue(package_name: str, provider: str) -> tuple[DatasetRef, ...]:
+    """Load and parse a catalogue.json from a provider package."""
+    package_files = files(package_name)
+    catalogue_text = package_files.joinpath("catalogue.json").read_text(encoding="utf-8")
+    parsed_catalogue = cast(object, json.loads(catalogue_text))
+    if not isinstance(parsed_catalogue, list):
+        msg = f"{provider} catalogue.json must contain a top-level JSON array"
+        raise ConfigError(msg, provider=provider)
+
+    catalogue_entries = cast(list[object], parsed_catalogue)
+    datasets: list[DatasetRef] = []
+    for entry_object in catalogue_entries:
+        if not isinstance(entry_object, dict):
+            msg = f"{provider} catalogue entries must be JSON objects"
+            raise ConfigError(msg, provider=provider)
+        typed_entry_object = cast(dict[object, object], entry_object)
+        entry: dict[str, object] = {}
+        for key, value in typed_entry_object.items():
+            if not isinstance(key, str):
+                msg = f"{provider} catalogue entry keys must be strings"
+                raise ConfigError(msg, provider=provider)
+            entry[key] = value
+        datasets.append(build_dataset_ref(provider, entry))
+
+    dataset_ids = [dataset.id for dataset in datasets]
+    duplicate_ids = sorted(
+        {dataset_id for dataset_id in dataset_ids if dataset_ids.count(dataset_id) > 1}
+    )
+    if duplicate_ids:
+        duplicates = ", ".join(duplicate_ids)
+        msg = f"{provider} catalogue contains duplicate dataset ids: {duplicates}"
+        raise ConfigError(msg, provider=provider)
+
+    return tuple(datasets)
+
+
+def build_dataset_ref(provider: str, entry: dict[str, object]) -> DatasetRef:
+    """Build a DatasetRef from a raw catalogue entry dict."""
+    dataset_key = require_string_field(entry, "dataset_key", provider)
+    name = require_string_field(entry, "name", provider)
+    representation_value = require_string_field(entry, "representation", provider)
+    dataset_id = f"{provider}.{dataset_key}"
+    try:
+        representation = Representation(representation_value)
+    except ValueError as exc:
+        msg = f"{provider} catalogue entry has invalid representation: {representation_value}"
+        raise ConfigError(msg, provider=provider, dataset_id=dataset_id) from exc
+
+    ops_raw_obj = entry.get("operations", [])
+    ops_raw = cast(list[object], ops_raw_obj) if isinstance(ops_raw_obj, list) else []
+    operations: set[Operation] = set()
+    for op_raw in ops_raw:
+        if not isinstance(op_raw, str):
+            msg = f"{provider} catalogue entry has non-string operation value: {op_raw!r}"
+            raise ConfigError(msg, provider=provider, dataset_id=dataset_id)
+        try:
+            operations.add(Operation(op_raw))
+        except ValueError as exc:
+            msg = f"{provider} catalogue entry has invalid operation: {op_raw}"
+            raise ConfigError(msg, provider=provider, dataset_id=dataset_id) from exc
+
+    query_support = _parse_query_support(entry, provider)
+
+    raw_metadata = MappingProxyType(
+        {
+            key: value
+            for key, value in entry.items()
+            if key not in ("dataset_key", "name", "representation", "operations", "query_support")
+        }
+    )
+
+    return DatasetRef(
+        id=dataset_id,
+        provider=provider,
+        dataset_key=dataset_key,
+        name=name,
+        representation=representation,
+        operations=frozenset(operations),
+        query_support=query_support,
+        raw_metadata=raw_metadata,
+    )
+
+
+def _parse_query_support(entry: dict[str, object], provider: str) -> QuerySupport | None:
+    """Parse query_support from a catalogue entry."""
+    qs_raw_obj = entry.get("query_support")
+    if not isinstance(qs_raw_obj, dict):
+        return None
+
+    qs_raw = cast(dict[str, object], qs_raw_obj)
+    pagination_raw = qs_raw.get("pagination", "none")
+    valid_pagination = {member.value for member in PaginationMode}
+    if isinstance(pagination_raw, str):
+        if pagination_raw not in valid_pagination:
+            msg = f"{provider} query_support.pagination has invalid value: {pagination_raw}"
+            raise ConfigError(msg, provider=provider)
+        pagination = PaginationMode(pagination_raw)
+    else:
+        pagination = PaginationMode.NONE
+
+    max_page_size = None
+    if "max_page_size" in qs_raw:
+        max_page_size_raw = qs_raw["max_page_size"]
+        if isinstance(max_page_size_raw, int):
+            max_page_size = max_page_size_raw
+        elif isinstance(max_page_size_raw, str):
+            max_page_size = int(max_page_size_raw)
+        else:
+            msg = f"{provider} query_support.max_page_size must be int-like"
+            raise ConfigError(msg, provider=provider)
+
+    return QuerySupport(pagination=pagination, max_page_size=max_page_size)
+
+
+def build_schema_from_metadata(dataset: DatasetRef) -> SchemaDescriptor | None:
+    """Build SchemaDescriptor from catalogue metadata fields."""
+    fields_raw = dataset.raw_metadata.get("fields")
+    if not isinstance(fields_raw, list) or not fields_raw:
+        return None
+
+    entries = cast(list[object], fields_raw)
+    field_descriptors: list[FieldDescriptor] = []
+    for entry_obj in entries:
+        if not isinstance(entry_obj, dict):
+            continue
+        entry = cast(dict[str, object], entry_obj)
+        name_raw = entry.get("name")
+        if not isinstance(name_raw, str) or not name_raw:
+            continue
+        title_raw = entry.get("title")
+        type_raw = entry.get("type")
+        desc_raw = entry.get("description")
+        nullable_raw = entry.get("nullable")
+        field_descriptors.append(
+            FieldDescriptor(
+                name=name_raw,
+                title=title_raw if isinstance(title_raw, str) else None,
+                type=type_raw if isinstance(type_raw, str) else None,
+                description=desc_raw if isinstance(desc_raw, str) else None,
+                nullable=nullable_raw if isinstance(nullable_raw, bool) else None,
+                raw=MappingProxyType({k: v for k, v in entry.items() if k != "name"}),
+            )
+        )
+
+    if not field_descriptors:
+        return None
+
+    return SchemaDescriptor(
+        dataset=dataset,
+        fields=field_descriptors,
+        raw=MappingProxyType({"source": "catalogue"}),
+    )
+
+
+def coerce_int(value: object, default: int) -> int:
+    """Coerce a value to int, returning default on failure."""
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str):
+        try:
+            return int(value)
+        except ValueError:
+            return default
+    return default
+
+
+def require_string_field(entry: Mapping[str, object], field_name: str, provider: str) -> str:
+    """Extract a required non-empty string field from a catalogue entry."""
+    value = entry.get(field_name)
+    if isinstance(value, str) and value:
+        return value
+    raise ConfigError(
+        f"{provider} catalogue entry missing non-empty string field: {field_name}",
+        provider=provider,
+    )
+
+
+__all__ = [
+    "build_dataset_ref",
+    "build_schema_from_metadata",
+    "coerce_int",
+    "load_catalogue",
+    "require_string_field",
+]

kpubdata/providers/bok/__init__.py

@@ -0,0 +1,7 @@
+"""Bank of Korea ECOS provider adapter."""
+
+from __future__ import annotations
+
+from .adapter import BokAdapter
+
+__all__ = ["BokAdapter"]