kpubdata 0.1.0__py3-none-any.whl

kpubdata/__init__.py

@@ -0,0 +1,52 @@
+"""KPubData — Korean public data access framework for Python 3.10+."""
+
+from __future__ import annotations
+
+from kpubdata.client import Client
+from kpubdata.core.capability import Operation, PaginationMode, QuerySupport
+from kpubdata.core.models import DatasetRef, FieldDescriptor, Query, RecordBatch, SchemaDescriptor
+from kpubdata.core.representation import Representation
+from kpubdata.exceptions import (
+    AuthError,
+    ConfigError,
+    DatasetNotFoundError,
+    InvalidRequestError,
+    ParseError,
+    ProviderNotRegisteredError,
+    ProviderResponseError,
+    PublicDataError,
+    RateLimitError,
+    ServiceUnavailableError,
+    TransportError,
+    TransportTimeoutError,
+    UnsupportedCapabilityError,
+)
+
+__all__ = [
+    "__version__",
+    "Client",
+    "DatasetRef",
+    "Query",
+    "RecordBatch",
+    "SchemaDescriptor",
+    "FieldDescriptor",
+    "Operation",
+    "PaginationMode",
+    "QuerySupport",
+    "Representation",
+    "PublicDataError",
+    "ConfigError",
+    "AuthError",
+    "TransportError",
+    "TransportTimeoutError",
+    "RateLimitError",
+    "ServiceUnavailableError",
+    "ParseError",
+    "InvalidRequestError",
+    "ProviderResponseError",
+    "UnsupportedCapabilityError",
+    "DatasetNotFoundError",
+    "ProviderNotRegisteredError",
+]
+
+__version__ = "0.1.0a0"

kpubdata/catalog.py

@@ -0,0 +1,101 @@
+"""Catalog — dataset discovery, search, and resolution."""
+
+from __future__ import annotations
+
+import builtins
+from typing import cast
+
+from kpubdata.core.models import DatasetRef
+from kpubdata.core.protocol import ProviderAdapter
+from kpubdata.exceptions import DatasetNotFoundError, ProviderNotRegisteredError
+from kpubdata.registry import ProviderRegistry
+
+
+class Catalog:
+    """Provides dataset discovery across registered providers."""
+
+    def __init__(self, registry: ProviderRegistry) -> None:
+        """Initialize catalog bound to a provider registry."""
+
+        self._registry = registry
+
+    def list(self, *, provider: str | None = None) -> builtins.list[DatasetRef]:
+        """Return discoverable datasets, optionally filtered by provider.
+
+        Raises:
+            ProviderNotRegisteredError: If ``provider`` is given but unknown.
+        """
+
+        if provider is not None:
+            adapter = self._get_adapter(provider)
+            return adapter.list_datasets()
+
+        datasets: builtins.list[DatasetRef] = []
+        for provider_name in self._registry:
+            adapter = self._get_adapter(provider_name)
+            datasets.extend(adapter.list_datasets())
+        return datasets
+
+    def search(self, text: str, *, provider: str | None = None) -> builtins.list[DatasetRef]:
+        """Search datasets by delegating to each adapter's search logic.
+
+        Each adapter implements its own ``search_datasets(text)`` method,
+        allowing provider-specific search semantics.
+
+        Raises:
+            ProviderNotRegisteredError: If ``provider`` is given but unknown.
+        """
+
+        if provider is not None:
+            adapter = self._get_adapter(provider)
+            return adapter.search_datasets(text)
+
+        results: builtins.list[DatasetRef] = []
+        for provider_name in self._registry:
+            adapter = self._get_adapter(provider_name)
+            results.extend(adapter.search_datasets(text))
+        return results
+
+    def resolve(self, dataset_id: str) -> tuple[ProviderAdapter, DatasetRef]:
+        """Resolve ``provider.dataset_key`` into an adapter and dataset ref.
+
+        Raises:
+            DatasetNotFoundError: If the dataset id is malformed or not found.
+            ProviderNotRegisteredError: If the provider is not registered.
+        """
+
+        provider_name, dataset_key = self._split_dataset_id(dataset_id)
+        adapter = self._get_adapter(provider_name)
+
+        try:
+            return adapter, adapter.get_dataset(dataset_key)
+        except DatasetNotFoundError:
+            raise
+        except Exception as exc:
+            raise DatasetNotFoundError(
+                f"Dataset not found: {dataset_id}",
+                provider=provider_name,
+                dataset_id=dataset_id,
+            ) from exc
+
+    def _get_adapter(self, provider: str) -> ProviderAdapter:
+        """Fetch provider adapter from registry or raise canonical error."""
+
+        try:
+            return cast(ProviderAdapter, self._registry.get(provider))
+        except ProviderNotRegisteredError:
+            raise
+
+    @staticmethod
+    def _split_dataset_id(dataset_id: str) -> tuple[str, str]:
+        """Split canonical dataset id into ``(provider, dataset_key)``."""
+
+        parts = dataset_id.split(".", 1)
+        if len(parts) != 2 or not parts[0] or not parts[1]:
+            raise DatasetNotFoundError(
+                f"Invalid dataset id format: {dataset_id}", dataset_id=dataset_id
+            )
+        return parts[0], parts[1]
+
+
+__all__ = ["Catalog"]

kpubdata/client.py

@@ -0,0 +1,179 @@
+"""Client — the top-level entry point for KPubData."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import cast
+
+from typing_extensions import override
+
+from kpubdata.catalog import Catalog
+from kpubdata.config import KPubDataConfig
+from kpubdata.core.dataset import Dataset
+from kpubdata.core.protocol import ProviderAdapter
+from kpubdata.registry import ProviderRegistry
+from kpubdata.transport.http import HttpTransport, TransportConfig, TransportRequirements
+
+_BUILTIN_PROVIDERS: tuple[tuple[str, str, str], ...] = (
+    ("datago", "kpubdata.providers.datago", "DataGoAdapter"),
+    ("bok", "kpubdata.providers.bok", "BokAdapter"),
+    ("kosis", "kpubdata.providers.kosis", "KosisAdapter"),
+    ("lofin", "kpubdata.providers.lofin", "LofinAdapter"),
+)
+
+
+class Client:
+    """Top-level entry point for dataset discovery and bound operations."""
+
+    def __init__(
+        self,
+        *,
+        provider_keys: dict[str, str] | None = None,
+        timeout: float = 30.0,
+        max_retries: int = 3,
+        **extra: object,
+    ) -> None:
+        """Initialize a client with explicit provider and transport configuration.
+
+        Use ``provider_keys`` to supply credentials directly, and configure
+        transport behavior with ``timeout`` and ``max_retries``.
+        Built-in providers (datago, bok, kosis, lofin) are lazily registered by default.
+        """
+
+        self._config: KPubDataConfig = KPubDataConfig(
+            provider_keys=provider_keys or {},
+            timeout=timeout,
+            max_retries=max_retries,
+            extra=dict(extra),
+        )
+        self._registry: ProviderRegistry = ProviderRegistry()
+        self._transport_config: TransportConfig = TransportConfig(
+            timeout=self._config.timeout,
+            max_retries=self._config.max_retries,
+        )
+        self._transport: HttpTransport = HttpTransport(self._transport_config)
+        self._provider_transports: list[HttpTransport] = []
+        self._register_builtin_providers()
+        self._catalog: Catalog = Catalog(self._registry)
+
+    @classmethod
+    def from_env(cls, **overrides: object) -> Client:
+        """Create a client from environment variables and explicit overrides."""
+
+        config = KPubDataConfig.from_env(**overrides)
+        return cls(
+            provider_keys=config.provider_keys,
+            timeout=config.timeout,
+            max_retries=config.max_retries,
+            **config.extra,
+        )
+
+    def __enter__(self) -> Client:
+        """Enter context manager and initialize transport client."""
+
+        _ = self._transport.__enter__()
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        """Exit context manager and close transport resources."""
+
+        self.close()
+
+    def close(self) -> None:
+        """Close underlying transport resources for this client."""
+
+        self._transport.close()
+        for provider_transport in self._provider_transports:
+            provider_transport.close()
+        self._provider_transports.clear()
+
+    @property
+    def datasets(self) -> Catalog:
+        """Return catalog interface for discovery, search, and resolution."""
+
+        return self._catalog
+
+    def dataset(self, dataset_id: str) -> Dataset:
+        """Bind and return a dataset object by canonical identifier.
+
+        Raises:
+            DatasetNotFoundError: If the dataset id is invalid or unknown.
+            ProviderNotRegisteredError: If the provider is not registered.
+        """
+
+        adapter, ref = self._catalog.resolve(dataset_id)
+        return Dataset(ref=ref, adapter=adapter)
+
+    def register_provider(self, adapter: object) -> None:
+        """Register a provider adapter in this client's registry.
+
+        Raises:
+            TypeError: If the adapter does not satisfy the required protocol.
+            ValueError: If the provider is already registered.
+        """
+
+        self._registry.register(adapter)
+
+    def _register_builtin_providers(self) -> None:
+        config = self._config
+        transport = self._transport
+        transport_config = self._transport_config
+        provider_transports = self._provider_transports
+
+        for provider_name, module_path, class_name in _BUILTIN_PROVIDERS:
+
+            def _make_factory(
+                mod: str,
+                cls: str,
+                cfg: KPubDataConfig,
+                tpt: HttpTransport,
+                base_transport_config: TransportConfig,
+                owned_transports: list[HttpTransport],
+            ) -> Callable[[], ProviderAdapter]:
+                def _factory() -> ProviderAdapter:
+                    import importlib
+
+                    module = importlib.import_module(mod)
+                    adapter_cls = cast(Callable[..., ProviderAdapter], getattr(module, cls))
+                    adapter = adapter_cls(config=cfg, transport=tpt)
+                    requirements = _get_transport_requirements(adapter)
+                    if requirements is None:
+                        return adapter
+
+                    custom_transport = HttpTransport.with_requirements(
+                        base_transport_config,
+                        requirements,
+                    )
+                    owned_transports.append(custom_transport)
+                    return adapter_cls(config=cfg, transport=custom_transport)
+
+                return _factory
+
+            self._registry.register_lazy(
+                provider_name,
+                _make_factory(
+                    module_path,
+                    class_name,
+                    config,
+                    transport,
+                    transport_config,
+                    provider_transports,
+                ),
+                skip_if_exists=True,
+            )
+
+    @override
+    def __repr__(self) -> str:
+        """Return concise representation with known providers."""
+
+        return f"Client(providers=[{', '.join(self._registry)}])"
+
+
+__all__ = ["Client"]
+
+
+def _get_transport_requirements(adapter: ProviderAdapter) -> TransportRequirements | None:
+    requirements = getattr(adapter, "transport_requirements", None)
+    if requirements is None:
+        return None
+    return cast(TransportRequirements | None, requirements)

kpubdata/config.py

@@ -0,0 +1,119 @@
+"""Configuration management — explicit construction and environment-based loading.
+
+Key lookup order for provider keys:
+1. Explicit `provider_keys` dict passed to constructor
+2. Environment variable: KPUBDATA_{PROVIDER}_API_KEY (uppercased)
+3. Environment variable: {PROVIDER}_API_KEY (uppercased, fallback)
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from kpubdata.exceptions import ConfigError
+
+_ENV_KEY_PATTERN = re.compile(r"^KPUBDATA_([A-Z0-9_]+)_API_KEY$")
+
+
+@dataclass
+class KPubDataConfig:
+    """Framework configuration."""
+
+    provider_keys: dict[str, str] = field(default_factory=dict)
+    timeout: float = 30.0
+    max_retries: int = 3
+    extra: dict[str, object] = field(default_factory=dict)
+
+    def __repr__(self) -> str:
+        """Return concise debug representation without exposing secrets."""
+        providers = sorted(self.provider_keys.keys())
+        return (
+            "KPubDataConfig("
+            f"providers={providers}, "
+            f"timeout={self.timeout}, "
+            f"max_retries={self.max_retries}, "
+            f"extra_keys={sorted(self.extra.keys())}"
+            ")"
+        )
+
+    def get_provider_key(self, provider: str) -> str | None:
+        """Look up API key for a provider following documented precedence."""
+        normalized_provider = _normalize_provider_name(provider)
+
+        explicit = _get_explicit_key(self.provider_keys, normalized_provider)
+        if explicit:
+            return explicit
+
+        provider_token = _provider_env_token(normalized_provider)
+        kpub_var = f"KPUBDATA_{provider_token}_API_KEY"
+        value = os.environ.get(kpub_var)
+        if value:
+            return value
+
+        fallback_var = f"{provider_token}_API_KEY"
+        fallback_value = os.environ.get(fallback_var)
+        if fallback_value:
+            return fallback_value
+
+        return None
+
+    def require_provider_key(self, provider: str) -> str:
+        """Like get_provider_key but raises ConfigError if missing."""
+        key = self.get_provider_key(provider)
+        if key is not None:
+            return key
+        raise ConfigError(f"Missing provider API key for '{provider}'")
+
+    @classmethod
+    def from_env(cls, **overrides: Any) -> KPubDataConfig:
+        """Build config from environment variables.
+
+        Scans for KPUBDATA_*_API_KEY patterns.
+        Overrides can be passed as kwargs.
+        """
+        scanned_keys: dict[str, str] = {}
+        for env_name, env_value in os.environ.items():
+            match = _ENV_KEY_PATTERN.match(env_name)
+            if match is None:
+                continue
+            if not env_value:
+                continue
+            provider_name = match.group(1).lower()
+            scanned_keys[provider_name] = env_value
+
+        provider_overrides_raw = overrides.pop("provider_keys", None)
+        provider_overrides: dict[str, str] = {}
+        if isinstance(provider_overrides_raw, dict):
+            for key, value in provider_overrides_raw.items():
+                if isinstance(key, str) and isinstance(value, str) and value:
+                    provider_overrides[_normalize_provider_name(key)] = value
+
+        merged_provider_keys = scanned_keys.copy()
+        merged_provider_keys.update(provider_overrides)
+
+        return cls(provider_keys=merged_provider_keys, **overrides)
+
+
+def _normalize_provider_name(provider: str) -> str:
+    return provider.strip().lower()
+
+
+def _provider_env_token(provider: str) -> str:
+    token = re.sub(r"[^A-Za-z0-9]", "_", provider)
+    return token.upper()
+
+
+def _get_explicit_key(provider_keys: dict[str, str], provider: str) -> str | None:
+    if provider in provider_keys and provider_keys[provider]:
+        return provider_keys[provider]
+
+    for name, value in provider_keys.items():
+        if name.lower() == provider and value:
+            return value
+    return None
+
+
+__all__ = ["KPubDataConfig"]

kpubdata/core/__init__.py

@@ -0,0 +1,25 @@
+"""Core canonical models and capability metadata for KPubData."""
+
+from __future__ import annotations
+
+from kpubdata.core.capability import Operation, PaginationMode, QuerySupport
+from kpubdata.core.models import (
+    DatasetRef,
+    FieldDescriptor,
+    Query,
+    RecordBatch,
+    SchemaDescriptor,
+)
+from kpubdata.core.representation import Representation
+
+__all__ = [
+    "DatasetRef",
+    "FieldDescriptor",
+    "Operation",
+    "PaginationMode",
+    "Query",
+    "QuerySupport",
+    "RecordBatch",
+    "Representation",
+    "SchemaDescriptor",
+]

kpubdata/core/capability.py

@@ -0,0 +1,56 @@
+"""Capability metadata describing dataset operations and query support."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass as _stdlib_dataclass
+from enum import Enum
+from typing import TypeVar
+
+from typing_extensions import dataclass_transform
+
+_T = TypeVar("_T")
+
+
+@dataclass_transform()
+def _dataclass(
+    *,
+    slots: bool = False,
+    frozen: bool = False,
+) -> Callable[[type[_T]], type[_T]]:
+    def _decorate(cls: type[_T]) -> type[_T]:
+        return _stdlib_dataclass(slots=slots, frozen=frozen)(cls)  # pyright: ignore[reportCallIssue]
+
+    return _decorate
+
+
+class Operation(str, Enum):
+    """Major operations a dataset can support."""
+
+    LIST = "list"
+    GET = "get"
+    SCHEMA = "schema"
+    RAW = "raw"
+    DOWNLOAD = "download"
+
+
+class PaginationMode(str, Enum):
+    """How a dataset supports pagination."""
+
+    OFFSET = "offset"
+    CURSOR = "cursor"
+    NONE = "none"
+
+
+@_dataclass(slots=True, frozen=True)
+class QuerySupport:
+    """Structured metadata about the list-query features a dataset supports."""
+
+    pagination: PaginationMode = PaginationMode.NONE
+    filterable_fields: frozenset[str] = frozenset()
+    sortable_fields: frozenset[str] = frozenset()
+    time_range: bool = False
+    max_page_size: int | None = None
+
+
+__all__ = ["Operation", "PaginationMode", "QuerySupport"]

kpubdata/core/dataset.py

@@ -0,0 +1,153 @@
+"""Bound Dataset — the user-facing object for dataset operations."""
+
+from __future__ import annotations
+
+from kpubdata.core.capability import Operation
+from kpubdata.core.models import DatasetRef, Query, RecordBatch, SchemaDescriptor
+from kpubdata.core.protocol import ProviderAdapter
+from kpubdata.exceptions import UnsupportedCapabilityError
+
+_CANONICAL_QUERY_KEYS = frozenset(
+    {"page", "page_size", "cursor", "start_date", "end_date", "fields", "sort"}
+)
+
+
+class Dataset:
+    """Bound dataset that routes operations to a provider adapter."""
+
+    def __init__(self, ref: DatasetRef, adapter: ProviderAdapter) -> None:
+        """Initialize a dataset bound to its canonical ref and adapter."""
+
+        self._ref = ref
+        self._adapter = adapter
+
+    @property
+    def ref(self) -> DatasetRef:
+        """Return immutable canonical dataset reference."""
+
+        return self._ref
+
+    @property
+    def id(self) -> str:
+        """Return canonical dataset identifier."""
+
+        return self._ref.id
+
+    @property
+    def name(self) -> str:
+        """Return human-readable dataset name."""
+
+        return self._ref.name
+
+    @property
+    def provider(self) -> str:
+        """Return provider identifier serving this dataset."""
+
+        return self._ref.provider
+
+    @property
+    def operations(self) -> frozenset[Operation]:
+        """Return declared operation capabilities for this dataset."""
+
+        return self._ref.operations
+
+    def list(self, **kwargs: object) -> RecordBatch:
+        """Query records from this dataset using canonical list semantics.
+
+        Canonical query parameters (``page``, ``page_size``, ``cursor``,
+        ``start_date``, ``end_date``, ``fields``, ``sort``) are extracted
+        into the corresponding ``Query`` fields.  Remaining kwargs are
+        passed as provider-specific ``filters``.
+
+        Raises:
+            UnsupportedCapabilityError: If this dataset does not support ``list``.
+        """
+
+        if Operation.LIST not in self._ref.operations:
+            raise UnsupportedCapabilityError(
+                f"Dataset does not support list: {self._ref.id}",
+                provider=self._ref.provider,
+                dataset_id=self._ref.id,
+                operation=Operation.LIST.value,
+            )
+
+        page: int | None = None
+        page_size: int | None = None
+        cursor: str | None = None
+        start_date: str | None = None
+        end_date: str | None = None
+        fields_list: list[str] | None = None
+        sort_list: list[str] | None = None
+        filters: dict[str, object] = {}
+
+        for key, value in kwargs.items():
+            if key == "page" and isinstance(value, int):
+                page = value
+            elif key == "page_size" and isinstance(value, int):
+                page_size = value
+            elif key == "cursor" and isinstance(value, str):
+                cursor = value
+            elif key == "start_date" and isinstance(value, str):
+                start_date = value
+            elif key == "end_date" and isinstance(value, str):
+                end_date = value
+            elif key == "fields" and isinstance(value, list):
+                fields_list = value
+            elif key == "sort" and isinstance(value, list):
+                sort_list = value
+            else:
+                filters[key] = value
+
+        query = Query(
+            filters=filters,
+            page=page,
+            page_size=page_size,
+            cursor=cursor,
+            start_date=start_date,
+            end_date=end_date,
+            fields=fields_list,
+            sort=sort_list,
+        )
+        return self._adapter.query_records(self._ref, query)
+
+    def get(self, **key: object) -> dict[str, object] | None:
+        """Return a single record matching the provided key fields.
+
+        Return ``None`` when no matching record is found.
+
+        Raises:
+            UnsupportedCapabilityError: If this dataset does not support ``get``.
+        """
+
+        if Operation.GET not in self._ref.operations:
+            raise UnsupportedCapabilityError(
+                f"Dataset does not support get: {self._ref.id}",
+                provider=self._ref.provider,
+                dataset_id=self._ref.id,
+                operation=Operation.GET.value,
+            )
+        key_payload: dict[str, object] = {k: v for k, v in key.items()}
+        return self._adapter.get_record(self._ref, key_payload)
+
+    def schema(self) -> SchemaDescriptor | None:
+        """Return canonical schema metadata when the provider exposes it."""
+
+        return self._adapter.get_schema(self._ref)
+
+    def call_raw(self, operation: str, **params: object) -> object:
+        """Execute a provider-native operation without canonical normalization.
+
+        Use this escape hatch for provider features not represented in the
+        canonical model.
+        """
+
+        payload: dict[str, object] = {k: v for k, v in params.items()}
+        return self._adapter.call_raw(self._ref, operation, payload)
+
+    def __repr__(self) -> str:
+        """Return concise debug representation."""
+
+        return f"Dataset({self._ref.id!r})"
+
+
+__all__ = ["Dataset"]