ds-resource-plugin-py-lib 0.1.0__py3-none-any.whl

ds_resource_plugin_py_lib/__init__.py

@@ -0,0 +1,28 @@
+"""
+**File:** ``__init__.py``
+**Region:** ``ds_resource_plugin_py_lib``
+
+Description
+-----------
+A Python package from the ds-resource-plugin library.
+
+Example
+-------
+.. code-block:: python
+
+    from ds_resource_plugin_py_lib import __version__
+
+    print(f"Package version: {__version__}")
+"""
+
+from importlib.metadata import version
+
+from . import common, libs
+
+__version__ = version("ds-resource-plugin-py-lib")
+
+__all__ = [
+    "__version__",
+    "common",
+    "libs",
+]

ds_resource_plugin_py_lib/common/__init__.py

@@ -0,0 +1,15 @@
+"""
+**File:** ``__init__.py``
+**Region:** ``ds_resource_plugin_py_lib/common``
+
+Description
+-----------
+Common utilities for resources and (de)serialization.
+"""
+
+from . import resource, serde
+
+__all__ = [
+    "resource",
+    "serde",
+]

ds_resource_plugin_py_lib/common/resource/__init__.py

@@ -0,0 +1,17 @@
+"""
+**File:** ``__init__.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource``
+
+Description
+-----------
+Public resource APIs: datasets, linked services, and the resource client.
+"""
+
+from . import dataset, linked_service
+from .client import ResourceClient
+
+__all__ = [
+    "ResourceClient",
+    "dataset",
+    "linked_service",
+]

ds_resource_plugin_py_lib/common/resource/client.py

@@ -0,0 +1,265 @@
+"""
+**File:** ``client.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource``
+
+Description
+-----------
+Resource client for discovering and managing datasets and linked services using entry points.
+
+Example
+-------
+.. code-block:: python
+
+    from ds_resource_plugin_py_lib.common.resource.client import ResourceClient
+
+    client = ResourceClient()
+
+    # Inspect discovered resources (populated via Python entry points).
+    print(client.resources.keys())
+    print(client.linked_services.keys())
+    print(client.datasets.keys())
+
+    dataset = client.dataset(config={"type": "dataset.example", "version": "1.0.0"})
+    linked_service = client.linked_service(config={"type": "linked_service.example", "version": "1.0.0"})
+
+    linked_service.connect()
+    print(linked_service.connection)
+    print(dataset.read())
+"""
+
+from functools import lru_cache
+from importlib import import_module
+from importlib.metadata import entry_points
+from pathlib import Path
+from typing import Any, cast
+
+import yaml
+from ds_common_logger_py_lib import Logger
+from ds_common_serde_py_lib.errors import DeserializationError
+
+from ...libs.utils import import_string, sanitize_version
+from ..resource.dataset.base import Dataset, DatasetInfo
+from ..resource.linked_service.base import LinkedService, LinkedServiceInfo
+
+logger = Logger.get_logger(__name__, package=True)
+
+
+class ResourceClient:
+    PROTOCOL_GROUP = "ds.protocols"
+    PROVIDER_GROUP = "ds.providers"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._resource_dict: dict[str, dict[str, Any]] = {}
+        self._linked_services: dict[tuple[str, str], LinkedServiceInfo] = {}
+        self._datasets: dict[tuple[str, str], DatasetInfo] = {}
+        self._discover_resources(self.PROTOCOL_GROUP)
+        self._discover_resources(self.PROVIDER_GROUP)
+        logger.debug(f"Loaded {len(self._resource_dict)} resources")
+        logger.debug(f"Loaded {len(self._linked_services)} linked services")
+        logger.debug(f"Loaded {len(self._datasets)} datasets")
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def get_instance(cls) -> "ResourceClient":
+        """Get the singleton instance of ResourceClient."""
+        return cls()
+
+    @property
+    def resources(self) -> dict[str, dict[str, Any]]:
+        return self._resource_dict
+
+    @property
+    def linked_services(self) -> dict[tuple[str, str], LinkedServiceInfo]:
+        return self._linked_services
+
+    @property
+    def datasets(self) -> dict[tuple[str, str], DatasetInfo]:
+        return self._datasets
+
+    def _discover_resources(self, group: str) -> None:
+        """
+        Discover protocol/provider packages via entry points.
+        Each entry point must target a Python package that contains resource.yaml in its root.
+        """
+        try:
+            eps = entry_points(group=group)
+        except Exception as exc:
+            logger.warning(f"Failed to read entry points for {group}: {exc}")
+            return
+
+        for ep in eps:
+            try:
+                module = import_module(ep.module)
+                module_path = getattr(module, "__file__", None)
+                if not module_path:
+                    logger.warning(f"Entry point {ep.name} has no __file__; skipping.")
+                    continue
+
+                real_path = str(Path(module_path).parent.resolve())
+                self._scan_resource_directory(real_path)
+
+            except Exception as exc:
+                logger.error(f"Error when loading entry point {ep.name} ({group}): {exc}")
+
+    def _scan_resource_directory(self, resource_dir: str) -> None:
+        """
+        Scan a resource directory for resource.yaml.
+        Checks root first (new behavior), then subdirectories (old behavior).
+        Args:
+            resource_dir: Path to the resource directory.
+        """
+        resource_path = Path(resource_dir)
+        if not resource_path.exists():
+            logger.debug(f"Resource directory {resource_dir} does not exist")
+            return
+
+        self._load_resource_from_path(str(resource_path))
+
+    def _load_resource_from_path(self, path: str) -> None:
+        """
+        Load resource configuration from a directory path.
+
+        Args:
+            path: Path to the resource directory.
+        """
+        resource_dir = Path(path)
+        resource_yaml = resource_dir / "resource.yaml"
+        if not resource_yaml.exists():
+            logger.debug(f"No resource.yaml found in {path}")
+            return
+
+        try:
+            with Path(resource_yaml).open() as f:
+                resource_config = yaml.safe_load(f)
+                if not resource_config:
+                    logger.warning(f"Empty resource configuration in {resource_yaml}")
+                    return
+
+                resource_name = resource_config.get("name", resource_dir.name)
+                self._resource_dict[resource_name] = resource_config
+                self._parse_linked_services(resource_config)
+                self._parse_datasets(resource_config)
+        except Exception as exc:
+            logger.error(f"Error loading resource configuration from {resource_yaml}: {exc}")
+
+    def _parse_linked_services(self, config: dict[str, Any]) -> None:
+        """
+        Parse linked services from resource configuration.
+
+        Args:
+            config: Resource configuration dictionary.
+        """
+        linked_services = config.get("linked_service", [])
+        for service in linked_services:
+            service_name = service.get("name")
+            if service_name:
+                type = service.get("type")
+                version = service.get("version", "1.0.0")
+                service_info = LinkedServiceInfo(
+                    type=type,
+                    name=service_name,
+                    class_name=service.get("class_name"),
+                    version=version,
+                    description=service.get("description"),
+                )
+                # Store by composite key (type, version) to support multiple versions
+                self._linked_services[service_info.key] = service_info
+
+    def _parse_datasets(self, config: dict[str, Any]) -> None:
+        """
+        Parse datasets from resource configuration.
+
+        Args:
+            config: Resource configuration dictionary.
+        """
+        datasets = config.get("dataset", [])
+        for dataset in datasets:
+            dataset_name = dataset.get("name")
+            if dataset_name:
+                type = dataset.get("type")
+                version = dataset.get("version", "1.0.0")
+                dataset_info = DatasetInfo(
+                    type=type,
+                    name=dataset_name,
+                    class_name=dataset.get("class_name"),
+                    version=version,
+                    description=dataset.get("description"),
+                )
+                self._datasets[dataset_info.key] = dataset_info
+
+    def _get_dataset_model_cls(self, _type: str, version: str) -> type[Dataset[Any, Any, Any, Any]]:
+        """
+        Get a dataset model class by type and optionally version.
+
+        Args:
+            _type: str
+            version: str
+        Returns:
+            Type[Dataset]
+        """
+        cls_name = self.datasets[(_type, version)].class_name
+        logger.debug("Dataset Class Name: %s", cls_name)
+        return cast("type[Dataset[Any, Any, Any, Any]]", import_string(cls_name))
+
+    def _get_linked_service_model_cls(self, _type: str, version: str) -> type[LinkedService[Any]]:
+        """
+        Get a linked service model class by type and version.
+
+        Args:
+            _type: The type of the linked service.
+            version: str version of the linked service.
+        Returns:
+            Type[LinkedService]
+        """
+        cls_name = self.linked_services[(_type, version)].class_name
+        logger.debug("Linked Service Class Name: %s", cls_name)
+        return cast("type[LinkedService[Any]]", import_string(cls_name))
+
+    def linked_service(self, config: dict[str, Any]) -> LinkedService[Any]:
+        """
+        Get a linked service instance by configuration.
+
+        Args:
+            config: dict containing at least 'type' and 'version'
+        Returns:
+            LinkedService
+        Raises:
+            DeserializationError: If the linked service cannot be deserialized.
+        """
+        try:
+            type = config["type"]
+            version = sanitize_version(config["version"])
+            model_cls = self._get_linked_service_model_cls(type, version)
+            linked_service: LinkedService[Any] = model_cls.deserialize(config)
+            return linked_service
+        except (TypeError, KeyError) as exc:
+            logger.error(f"Error deserializing linked service: {exc}")
+            raise DeserializationError(
+                message=f"Error deserializing linked service: {exc}",
+                details={"config": config, "error": str(exc)},
+            ) from exc
+
+    def dataset(self, config: dict[str, Any]) -> Dataset[Any, Any, Any, Any]:
+        """
+        Get a dataset instance by configuration.
+
+        Args:
+            config: dict containing at least 'type' and 'version'
+        Returns:
+            Dataset
+        Raises:
+            DeserializationError: If the dataset cannot be deserialized.
+        """
+        try:
+            type = config["type"]
+            version = sanitize_version(config["version"])
+            dataset_cls = self._get_dataset_model_cls(type, version)
+            dataset: Dataset[Any, Any, Any, Any] = dataset_cls.deserialize(config)
+            return dataset
+        except (TypeError, KeyError) as exc:
+            logger.error(f"Error deserializing dataset: {exc}")
+            raise DeserializationError(
+                message=f"Error deserializing dataset: {exc}",
+                details={"config": config, "error": str(exc)},
+            ) from exc

ds_resource_plugin_py_lib/common/resource/dataset/__init__.py

@@ -0,0 +1,25 @@
+"""
+**File:** ``__init__.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/dataset``
+
+Description
+-----------
+Dataset models, typed properties, and storage format helpers.
+"""
+
+from .base import Dataset, DatasetInfo, DatasetSettings, TabularDataset
+from .enums import DatasetMethod
+from .result import OperationError, OperationInfo
+from .storage_format import DatasetStorageFormat, DatasetStorageFormatType
+
+__all__ = [
+    "Dataset",
+    "DatasetInfo",
+    "DatasetMethod",
+    "DatasetSettings",
+    "DatasetStorageFormat",
+    "DatasetStorageFormatType",
+    "OperationError",
+    "OperationInfo",
+    "TabularDataset",
+]

ds_resource_plugin_py_lib/common/resource/dataset/base.py

@@ -0,0 +1,322 @@
+"""
+**File:** ``base.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/dataset``
+
+Description
+-----------
+Base dataset models and typed properties.
+"""
+
+import io
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import StrEnum
+from types import TracebackType
+from typing import Any, Generic, NamedTuple, Self, TypeVar
+
+import pandas as pd
+from ds_common_serde_py_lib import Serializable
+
+from ...resource.linked_service.base import LinkedService
+from ...serde.deserialize.base import DataDeserializer
+from ...serde.serialize.base import DataSerializer
+from .decorators import track_result
+from .enums import DatasetMethod
+from .result import OperationInfo
+
+
+class DatasetInfo(NamedTuple):
+    """
+    NamedTuple that represents the dataset information.
+    """
+
+    type: str
+    name: str
+    class_name: str
+    version: str
+    description: str | None = None
+
+    def __str__(self) -> str:
+        """
+        Return a string representation of the dataset info.
+
+        Returns:
+            A string representation of the dataset info.
+        """
+        return f"{self.type}:v{self.version}"
+
+    @property
+    def key(self) -> tuple[str, str]:
+        """
+        Return the composite key (type, version) for dictionary lookups.
+
+        Returns:
+            A tuple containing the type and version.
+        """
+        return (self.type, self.version)
+
+
+@dataclass(kw_only=True)
+class DatasetSettings(Serializable):
+    """
+    The object containing the settings of the dataset.
+    """
+
+    pass
+
+
+DatasetSettingsType = TypeVar("DatasetSettingsType", bound=DatasetSettings)
+LinkedServiceType = TypeVar("LinkedServiceType", bound=LinkedService[Any])
+SerializerType = TypeVar("SerializerType", bound=DataSerializer)
+DeserializerType = TypeVar("DeserializerType", bound=DataDeserializer)
+
+
+@dataclass(kw_only=True)
+class Dataset(
+    ABC,
+    Serializable,
+    Generic[LinkedServiceType, DatasetSettingsType, SerializerType, DeserializerType],
+):
+    """
+    The ds workflow nested object which identifies data within a data store,
+    such as table, files, folders and documents.
+
+    You probably want to use the subclasses and not this class directly.
+    """
+
+    id: uuid.UUID
+    name: str
+    description: str | None = None
+    version: str
+
+    settings: DatasetSettingsType
+    linked_service: LinkedServiceType
+
+    serializer: SerializerType | None = None
+    deserializer: DeserializerType | None = None
+
+    input: Any | None = field(default=None, metadata={"serialize": False})
+    output: Any | None = field(default=None, metadata={"serialize": False})
+
+    checkpoint: dict[str, Any] = field(default_factory=dict)
+    operation: OperationInfo = field(default_factory=OperationInfo)
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        """
+        Initialize the subclass.
+
+        Args:
+            kwargs: The keyword arguments.
+
+        Returns:
+            The subclass.
+        """
+        super().__init_subclass__(**kwargs)
+        for name in DatasetMethod.all_values():
+            method = cls.__dict__.get(name)
+            if method is not None and not getattr(method, "_tracked", False):
+                setattr(cls, name, track_result(method))
+
+    def __enter__(self) -> Self:
+        """
+        Context manager enter.
+
+        Returns:
+            The dataset.
+        """
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        """
+        Context manager exit.
+
+        Args:
+            exc_type: The type of the exception.
+            exc_value: The value of the exception.
+            traceback: The traceback of the exception.
+        """
+        self.close()
+
+    @property
+    def supports_checkpoint(self) -> bool:
+        """Whether this provider supports incremental loads via ``self.checkpoint``."""
+        return False
+
+    @property
+    @abstractmethod
+    def type(self) -> StrEnum:
+        """
+        Get the type of the dataset.
+        """
+        ...
+
+    @abstractmethod
+    def create(self) -> None:
+        """
+        Insert all rows in ``self.input`` into the target as a single atomic
+        transaction. Must not delete, update, or overwrite existing data.
+
+        Raises:
+            CreateError: If the operation fails.
+            NotSupportedError: If the provider does not support create.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``create()``
+        """
+        ...
+
+    @abstractmethod
+    def read(self) -> None:
+        """
+        Read data from the source and assign it to ``self.output``.
+        Pagination within a single call is handled internally.
+        Supports incremental loads via ``self.checkpoint``.
+
+        Raises:
+            ReadError: If the operation fails.
+            NotSupportedError: If the provider does not support read.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``read()``
+        """
+        ...
+
+    @abstractmethod
+    def update(self) -> None:
+        """
+        Update existing rows in the target matched by identity columns
+        defined in ``self.settings``. Atomic. Must not insert new rows.
+
+        Raises:
+            UpdateError: If the operation fails.
+            NotSupportedError: If the provider does not support update.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``update()``
+        """
+        ...
+
+    @abstractmethod
+    def upsert(self) -> None:
+        """
+        Insert rows that do not exist, update rows that do, matched by
+        identity columns defined in ``self.settings``. Atomic.
+
+        Raises:
+            UpsertError: If the operation fails.
+            NotSupportedError: If the provider does not support upsert.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``upsert()``
+        """
+        ...
+
+    @abstractmethod
+    def delete(self) -> None:
+        """
+        Remove specific rows from the target matched by identity columns
+        defined in ``self.settings``. Atomic. Idempotent.
+
+        Raises:
+            DeleteError: If the operation fails.
+            NotSupportedError: If the provider does not support delete.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``delete()``
+        """
+        ...
+
+    @abstractmethod
+    def purge(self) -> None:
+        """
+        Remove all content from the target. ``self.input`` is not used.
+        Atomic. Idempotent.
+
+        Raises:
+            PurgeError: If the operation fails.
+            NotSupportedError: If the provider does not support purge.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``purge()``
+        """
+        ...
+
+    @abstractmethod
+    def list(self) -> None:
+        """
+        Discover available resources and populate ``self.output`` with a
+        DataFrame of resources and their metadata. Idempotent.
+
+        Raises:
+            ListError: If the operation fails.
+            NotSupportedError: If the provider does not support listing.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``list()``
+        """
+        ...
+
+    @abstractmethod
+    def rename(self) -> None:
+        """
+        Rename the resource in the backend. Atomic. Not idempotent.
+
+        Raises:
+            RenameError: If the operation fails.
+            NotSupportedError: If the provider does not support renaming.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``rename()``
+        """
+        ...
+
+    @abstractmethod
+    def close(self) -> None:
+        """
+        Release any connections, sessions, or handles held by the linked
+        service. Must not raise if already closed. Idempotent.
+
+        See Also:
+            Full contract: ``docs/DATASET_CONTRACT.md`` -- ``close()``
+        """
+        ...
+
+
+@dataclass(kw_only=True)
+class BinaryDataset(
+    Dataset[LinkedServiceType, DatasetSettingsType, SerializerType, DeserializerType],
+    Generic[LinkedServiceType, DatasetSettingsType, SerializerType, DeserializerType],
+):
+    """
+    Binary dataset object which identifies data within a data store,
+    such as files, folders and documents.
+
+    The input of the dataset is a binary file.
+    The output of the dataset is a binary file.
+    """
+
+    input: io.BytesIO = field(default_factory=io.BytesIO, metadata={"serialize": False})
+    output: io.BytesIO = field(default_factory=io.BytesIO, metadata={"serialize": False})
+
+
+@dataclass(kw_only=True)
+class TabularDataset(
+    Dataset[LinkedServiceType, DatasetSettingsType, SerializerType, DeserializerType],
+    Generic[LinkedServiceType, DatasetSettingsType, SerializerType, DeserializerType],
+):
+    """
+    Tabular dataset object which identifies data within a data store,
+    such as table/csv/json/parquet/parquetdataset/ and other documents.
+
+    The input of the dataset is a pandas DataFrame.
+    The output of the dataset is a pandas DataFrame.
+    """
+
+    input: pd.DataFrame = field(default_factory=pd.DataFrame, metadata={"serialize": False})
+    output: pd.DataFrame = field(default_factory=pd.DataFrame, metadata={"serialize": False})