ds-resource-plugin-py-lib 0.1.0a1__py3-none-any.whl

ds_resource_plugin_py_lib/__init__.py

@@ -0,0 +1,27 @@
+"""
+A Python package from the ds-resource-plugin library.
+
+**File:** ``__init__.py``
+**Region:** ``ds-resource-plugin-py-lib``
+
+Example:
+
+.. code-block:: python
+
+    from ds_resource_plugin_py_lib import __version__
+
+    print(f"Package version: {__version__}")
+"""
+
+from pathlib import Path
+
+from . import common, libs
+
+_VERSION_FILE = Path(__file__).parent.parent.parent / "VERSION.txt"
+__version__ = _VERSION_FILE.read_text().strip() if _VERSION_FILE.exists() else "0.0.0"
+
+__all__ = [
+    "__version__",
+    "common",
+    "libs",
+]

ds_resource_plugin_py_lib/common/__init__.py

@@ -0,0 +1,6 @@
+from . import resource, serde
+
+__all__ = [
+    "resource",
+    "serde",
+]

ds_resource_plugin_py_lib/common/resource/__init__.py

@@ -0,0 +1,8 @@
+from . import dataset, linked_service
+from .client import ResourceClient
+
+__all__ = [
+    "ResourceClient",
+    "dataset",
+    "linked_service",
+]

ds_resource_plugin_py_lib/common/resource/client.py

@@ -0,0 +1,239 @@
+"""
+File: client.py
+Description: Resource client for discovering and managing datasets and linked services using entry points
+Region: packages/shared
+"""
+
+import logging
+from functools import lru_cache
+from importlib import import_module
+from importlib.metadata import entry_points
+from pathlib import Path
+from typing import Any, cast
+
+import yaml
+from ds_common_logger_py_lib.mixin import LoggingMixin
+
+from ...libs.utils.import_string import import_string
+from ..resource.dataset.base import Dataset, DatasetInfo
+from ..resource.errors import DeserializationException
+from ..resource.linked_service.base import LinkedService, LinkedServiceInfo
+
+
+class ResourceClient(LoggingMixin):
+    log_level = logging.DEBUG
+
+    PROTOCOL_GROUP = "ds.protocols"
+    PROVIDER_GROUP = "ds.providers"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._resource_dict: dict[str, dict[str, Any]] = {}
+        self._linked_services: dict[tuple[str, str], LinkedServiceInfo] = {}
+        self._datasets: dict[tuple[str, str], DatasetInfo] = {}
+        self._discover_resources(self.PROTOCOL_GROUP)
+        self._discover_resources(self.PROVIDER_GROUP)
+        self.log.debug(f"Loaded {len(self._resource_dict)} resources")
+        self.log.debug(f"Loaded {len(self._linked_services)} linked services")
+        self.log.debug(f"Loaded {len(self._datasets)} datasets")
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def get_instance(cls) -> "ResourceClient":
+        """Get the singleton instance of ResourceClient."""
+        return cls()
+
+    @property
+    def resources(self) -> dict[str, dict[str, Any]]:
+        return self._resource_dict
+
+    @property
+    def linked_services(self) -> dict[tuple[str, str], LinkedServiceInfo]:
+        return self._linked_services
+
+    @property
+    def datasets(self) -> dict[tuple[str, str], DatasetInfo]:
+        return self._datasets
+
+    def _discover_resources(self, group: str) -> None:
+        """
+        Discover protocol/provider packages via entry points.
+        Each entry point must target a Python package that contains resource.yaml in its root.
+        """
+        try:
+            eps = entry_points(group=group)
+        except Exception as exc:
+            self.log.warning(f"Failed to read entry points for {group}: {exc}")
+            return
+
+        for ep in eps:
+            try:
+                module = import_module(ep.module)
+                module_path = getattr(module, "__file__", None)
+                if not module_path:
+                    self.log.warning(f"Entry point {ep.name} has no __file__; skipping.")
+                    continue
+
+                real_path = str(Path(module_path).parent.resolve())
+                self._scan_resource_directory(real_path)
+
+            except Exception as exc:
+                self.log.error(f"Error when loading entry point {ep.name} ({group}): {exc}")
+
+    def _scan_resource_directory(self, resource_dir: str) -> None:
+        """
+        Scan a resource directory for resource.yaml.
+        Checks root first (new behavior), then subdirectories (old behavior).
+        Args:
+            resource_dir: Path to the resource directory.
+        """
+        resource_path = Path(resource_dir)
+        if not resource_path.exists():
+            self.log.debug(f"Resource directory {resource_dir} does not exist")
+            return
+
+        self._load_resource_from_path(str(resource_path))
+
+    def _load_resource_from_path(self, path: str) -> None:
+        """
+        Load resource configuration from a directory path.
+
+        Args:
+            path: Path to the resource directory.
+        """
+        resource_dir = Path(path)
+        resource_yaml = resource_dir / "resource.yaml"
+        if not resource_yaml.exists():
+            self.log.debug(f"No resource.yaml found in {path}")
+            return
+
+        try:
+            with Path(resource_yaml).open() as f:
+                resource_config = yaml.safe_load(f)
+                if not resource_config:
+                    self.log.warning(f"Empty resource configuration in {resource_yaml}")
+                    return
+
+                resource_name = resource_config.get("name", resource_dir.name)
+                self._resource_dict[resource_name] = resource_config
+                self._parse_linked_services(resource_config)
+                self._parse_datasets(resource_config)
+        except Exception as exc:
+            self.log.error(f"Error loading resource configuration from {resource_yaml}: {exc}")
+
+    def _parse_linked_services(self, config: dict[str, Any]) -> None:
+        """
+        Parse linked services from resource configuration.
+
+        Args:
+            config: Resource configuration dictionary.
+        """
+        linked_services = config.get("linked_service", [])
+        for service in linked_services:
+            service_name = service.get("name")
+            if service_name:
+                kind = service.get("kind")
+                version = service.get("version", "1.0.0")
+                service_info = LinkedServiceInfo(
+                    kind=kind,
+                    name=service_name,
+                    class_name=service.get("class_name"),
+                    version=version,
+                    description=service.get("description"),
+                )
+                # Store by composite key (kind, version) to support multiple versions
+                self._linked_services[service_info.key] = service_info
+
+    def _parse_datasets(self, config: dict[str, Any]) -> None:
+        """
+        Parse datasets from resource configuration.
+
+        Args:
+            config: Resource configuration dictionary.
+        """
+        datasets = config.get("dataset", [])
+        for dataset in datasets:
+            dataset_name = dataset.get("name")
+            if dataset_name:
+                kind = dataset.get("kind")
+                version = dataset.get("version", "1.0.0")
+                dataset_info = DatasetInfo(
+                    kind=kind,
+                    name=dataset_name,
+                    class_name=dataset.get("class_name"),
+                    version=version,
+                    description=dataset.get("description"),
+                )
+                self._datasets[dataset_info.key] = dataset_info
+
+    def _get_dataset_model_cls(self, kind: str, version: str) -> type[Dataset[Any, Any]]:
+        """
+        Get a dataset model class by kind and optionally version.
+
+        Args:
+            kind: str
+            version: str
+        Returns:
+            Type[Dataset]
+        """
+        cls_name = self.datasets[(kind, version)].class_name
+        self.log.debug("Dataset Class Name: %s", cls_name)
+        return cast("type[Dataset[Any, Any]]", import_string(cls_name))
+
+    def _get_linked_service_model_cls(self, kind: str, version: str) -> type[LinkedService[Any]]:
+        """
+        Get a linked service model class by kind and version.
+
+        Args:
+            kind: The kind of the linked service.
+            version: str version of the linked service.
+        Returns:
+            Type[LinkedService]
+        """
+        cls_name = self.linked_services[(kind, version)].class_name
+        self.log.debug("Linked Service Class Name: %s", cls_name)
+        return cast("type[LinkedService[Any]]", import_string(cls_name))
+
+    def linked_service(self, config: dict[str, Any]) -> LinkedService[Any]:
+        """
+        Get a linked service instance by configuration.
+
+        Args:
+            config: dict containing at least 'kind' and 'version'
+        Returns:
+            LinkedService
+        """
+        try:
+            kind = config["kind"]
+            version = config["version"]
+            model_cls = self._get_linked_service_model_cls(kind, version)
+            linked_service: LinkedService[Any] = model_cls.deserialize(config)
+            return linked_service
+        except (TypeError, KeyError) as exc:
+            self.log.exception(f"Error deserializing linked service: {exc}")
+            raise DeserializationException(
+                message=f"Error deserializing linked service: {exc}",
+                details={"config": config, "error": str(exc)},
+            ) from exc
+
+    def dataset(self, config: dict[str, Any]) -> Dataset[Any, Any]:
+        """
+        Get a dataset instance by configuration.
+
+        Args:
+            config: dict containing at least 'kind' and 'version'
+        Returns:
+            Dataset
+        """
+        try:
+            kind = config["kind"]
+            version = config["version"]
+            dataset_cls = self._get_dataset_model_cls(kind, version)
+            dataset: Dataset[Any, Any] = dataset_cls.deserialize(config)
+            return dataset
+        except (TypeError, KeyError) as exc:
+            self.log.exception(f"Error deserializing dataset: {exc}")
+            raise DeserializationException(
+                message=f"Error deserializing dataset: {exc}",
+                details={"config": config, "error": str(exc)},
+            ) from exc

ds_resource_plugin_py_lib/common/resource/dataset/__init__.py

@@ -0,0 +1,11 @@
+from .base import Dataset, DatasetInfo, DatasetTypedProperties, TabularDataset
+from .storage_format import DatasetStorageFormat, DatasetStorageFormatType
+
+__all__ = [
+    "Dataset",
+    "DatasetInfo",
+    "DatasetStorageFormat",
+    "DatasetStorageFormatType",
+    "DatasetTypedProperties",
+    "TabularDataset",
+]

ds_resource_plugin_py_lib/common/resource/dataset/base.py

@@ -0,0 +1,158 @@
+import io
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Generic, NamedTuple, TypeVar
+
+import pandas as pd
+
+from ....libs.models.serializable import Serializable
+from ...resource.linked_service.base import LinkedService
+from ...serde.deserialize.base import DataDeserializer
+from ...serde.serialize.base import DataSerializer
+
+
+class DatasetInfo(NamedTuple):
+    kind: str
+    name: str
+    class_name: str
+    version: str
+    description: str | None = None
+
+    def __str__(self) -> str:
+        """Return a string representation of the dataset info."""
+        return f"{self.kind}:v{self.version}"
+
+    @property
+    def key(self) -> tuple[str, str]:
+        """Return the composite key (kind, version) for dictionary lookups."""
+        return (self.kind, self.version)
+
+
+@dataclass(kw_only=True)
+class DatasetTypedProperties(Serializable):
+    """
+    The object containing the typed properties of the dataset.
+    """
+
+    pass
+
+
+DatasetTypedPropertiesType = TypeVar("DatasetTypedPropertiesType", bound=DatasetTypedProperties)
+LinkedServiceType = TypeVar("LinkedServiceType", bound=LinkedService[Any])
+
+
+@dataclass(kw_only=True)
+class Dataset(
+    ABC,
+    Serializable,
+    Generic[LinkedServiceType, DatasetTypedPropertiesType],
+):
+    """
+    The ds workflow nested object which identifies data within a data store,
+    such as table, files, folders and documents.
+
+    You probably want to use the subclasses and not this class directly.
+    """
+
+    typed_properties: DatasetTypedPropertiesType
+    linked_service: LinkedServiceType
+
+    serializer: DataSerializer | None = None
+    deserializer: DataDeserializer | None = None
+
+    post_fetch_callback: Callable[..., Any] | None = None
+    prepare_write_callback: Callable[..., Any] | None = None
+
+    content: Any | None = None
+    schema: dict[str, Any] | None = None
+    next: bool | None = True
+    cursor: str | None = None
+
+    @property
+    @abstractmethod
+    def kind(self) -> str:
+        """
+        Get the kind of the dataset.
+        """
+        raise NotImplementedError("Method (kind) not implemented")
+
+    @abstractmethod
+    def create(self, **kwargs: Any) -> Any:
+        """
+        Create the dataset.
+        :param kwargs: dict
+        :return: None
+        """
+        raise NotImplementedError("Method (create) not implemented")
+
+    @abstractmethod
+    def read(self, **kwargs: Any) -> Any:
+        """
+        Read the dataset.
+        :param kwargs: dict
+        :return: None
+        """
+        raise NotImplementedError("Method (read) not implemented")
+
+    @abstractmethod
+    def delete(self, **kwargs: Any) -> Any:
+        """
+        Delete the dataset.
+        :param kwargs: dict
+        :return: None
+        """
+        raise NotImplementedError("Method (delete) not implemented")
+
+    @abstractmethod
+    def update(self, **kwargs: Any) -> Any:
+        """
+        Update the dataset.
+        :param kwargs: dict
+        :return: None
+        """
+        raise NotImplementedError("Method (update) not implemented")
+
+    @abstractmethod
+    def rename(self, **kwargs: Any) -> Any:
+        """
+        Move the dataset.
+        :param kwargs: dict
+        :return: None
+        """
+        raise NotImplementedError("Method (move) not implemented")
+
+
+@dataclass(kw_only=True)
+class BinaryDataset(
+    Dataset[LinkedServiceType, DatasetTypedPropertiesType],
+    Generic[LinkedServiceType, DatasetTypedPropertiesType],
+):
+    """
+    Binary dataset object which identifies data within a data store,
+    such as files, folders and documents.
+
+    The content of the dataset is a binary file.
+    """
+
+    content: io.BytesIO = field(default_factory=io.BytesIO)
+    next: bool | None = True
+    cursor: str | None = None
+
+
+@dataclass(kw_only=True)
+class TabularDataset(
+    Dataset[LinkedServiceType, DatasetTypedPropertiesType],
+    Generic[LinkedServiceType, DatasetTypedPropertiesType],
+):
+    """
+    Tabular dataset object which identifies data within a data store,
+    such as table/csv/json/parquet/parquetdataset/ and other documents.
+
+    The content of the dataset is a pandas DataFrame.
+    """
+
+    schema: dict[str, Any] | None = None
+    content: pd.DataFrame = field(default_factory=pd.DataFrame)
+    next: bool | None = True
+    cursor: str | None = None

ds_resource_plugin_py_lib/common/resource/dataset/errors.py

@@ -0,0 +1,150 @@
+"""
+Exceptions for Datasets.
+"""
+
+from typing import Any
+
+from ..errors import ResourceException
+
+
+class DatasetException(ResourceException):
+    """Base exception for all dataset-related errors."""
+
+    def __init__(
+        self,
+        message: str = "Dataset operation failed",
+        code: str = "DATASET_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class MismatchedLinkedServiceException(DatasetException):
+    """Raised when a linked service does not match the dataset type."""
+
+    def __init__(
+        self,
+        message: str = "Mismatched linked service",
+        code: str = "MISMATCHED_LINKED_SERVICE",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class UnsupportedDatasetType(DatasetException):
+    """Raised when a dataset type is not supported."""
+
+    def __init__(
+        self,
+        message: str = "Dataset type is not supported",
+        code: str = "UNSUPPORTED_DATASET_TYPE",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class InvalidDatasetClass(DatasetException):
+    """Raised when a dataset type is invalid."""
+
+    def __init__(
+        self,
+        message: str = "Invalid dataset type",
+        code: str = "INVALID_DATASET_TYPE",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class FileNotFound(DatasetException):
+    """Raised when file not found."""
+
+    def __init__(
+        self,
+        message: str = "File not found",
+        code: str = "NOT_FOUND",
+        status_code: int = 404,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class WriteException(DatasetException):
+    """Raised when a write operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Write operation failed",
+        code: str = "WRITE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class UpdateException(DatasetException):
+    """Raised when a update operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Update operation failed",
+        code: str = "UPDATE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class ReadException(DatasetException):
+    """Raised when a read operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Read operation failed",
+        code: str = "READ_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class DeleteException(DatasetException):
+    """Raised when a delete operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Delete operation failed",
+        code: str = "DELETE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class JsonDecodeException(DatasetException):
+    """Raised when a operation fails due to JSONDecodeError."""
+
+    def __init__(
+        self,
+        message: str = "Failed to decode JSON.",
+        code: str = "JSON_DECODE_ERROR",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class BadRequestException(DatasetException):
+    """Raised when an operation fails due to invalid input."""
+
+    def __init__(
+        self,
+        message: str = "Bad Request.",
+        code: str = "BAD_REQUEST",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)

ds_resource_plugin_py_lib/common/resource/dataset/storage_format.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+from ....libs.models.serializable import Serializable
+
+
+class DatasetStorageFormatType(StrEnum):
+    """
+    Enum to define the storage format types.
+    """
+
+    PARQUET = "PARQUET"
+    CSV = "CSV"
+    JSON = "JSON"
+    EXCEL = "EXCEL"
+    SEMI_STRUCTURED_JSON = "SEMI_STRUCTURED_JSON"
+    XML = "XML"
+
+
+@dataclass(kw_only=True)
+class DatasetStorageFormat(Serializable):
+    """
+    The object containing the storage format of the dataset.
+    """
+
+    type: DatasetStorageFormatType
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(kw_only=True)
+class ParquetFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.PARQUET
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(kw_only=True)
+class CsvFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.CSV
+    args: dict[str, Any] = field(default_factory=lambda: {"delimiter": ","})
+
+
+@dataclass(kw_only=True)
+class SemiStructuredJsonFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.SEMI_STRUCTURED_JSON
+    args: dict[str, Any] = field(default_factory=lambda: {"record_path": None})
+
+
+@dataclass(kw_only=True)
+class XMLFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.XML
+    args: dict[str, Any] = field(default_factory=dict)

ds_resource_plugin_py_lib/common/resource/errors.py

@@ -0,0 +1,35 @@
+"""
+Exceptions for Resources.
+"""
+
+from typing import Any
+
+
+class ResourceException(Exception):
+    """Base exception for all resource-related errors."""
+
+    def __init__(
+        self,
+        message: str = "Resource operation failed",
+        code: str = "RESOURCE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.code = code
+        self.status_code = status_code
+        self.message = message
+        self.details = details or {}
+        super().__init__(self.message)
+
+
+class DeserializationException(ResourceException):
+    """Raised when a deserialization operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Deserialization operation failed",
+        code: str = "DESERIALIZATION_ERROR",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)

ds_resource_plugin_py_lib/common/resource/linked_service/__init__.py

@@ -0,0 +1,7 @@
+from .base import LinkedService, LinkedServiceInfo, LinkedServiceTypedProperties
+
+__all__ = [
+    "LinkedService",
+    "LinkedServiceInfo",
+    "LinkedServiceTypedProperties",
+]