ds-resource-plugin-py-lib 0.1.0__py3-none-any.whl

ds_resource_plugin_py_lib/common/resource/dataset/decorators.py

@@ -0,0 +1,73 @@
+"""
+**File:** ``decorators.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/dataset``
+
+Description
+-----------
+Decorators applied automatically to dataset method overrides via
+``Dataset.__init_subclass__``.
+"""
+
+import functools
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any
+
+from .enums import DatasetMethod
+from .result import OperationError, OperationInfo
+
+
+def track_result(fn: Callable[..., Any]) -> Callable[..., Any]:
+    """
+    Wrap a dataset method so that ``self.operation`` is initialised before the
+    call and enriched afterwards.
+
+    **Auto-populated fields:**
+
+    - ``method`` -- set to the function name.
+    - ``success`` -- ``True`` when the method returns without raising.
+    - ``error`` -- structured ``OperationError`` captured from
+      ``ResourceException`` on failure.
+    - ``started_at`` / ``ended_at`` / ``duration_ms`` -- wall-clock timing.
+    - ``row_count`` -- derived from ``len(self.output)`` when the
+      provider leaves the default (``0``).
+    - ``schema`` -- derived via PyArrow dtype inference on ``self.output``.
+
+    Providers may override any of these by assigning to ``self.operation``
+    inside their method body.
+
+    """
+
+    @functools.wraps(fn)
+    def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
+        self.operation = OperationInfo(method=DatasetMethod(fn.__name__))
+        self.operation.started_at = datetime.now(tz=UTC)
+        try:
+            fn(self, *args, **kwargs)
+
+            self.operation.success = True
+
+            if self.operation.row_count == 0 and hasattr(self.output, "__len__"):
+                self.operation.row_count = len(self.output)
+
+            if self.operation.schema is None and hasattr(self.output, "convert_dtypes"):
+                converted = self.output.convert_dtypes(dtype_backend="pyarrow")
+                self.operation.schema = {str(col): str(dtype) for col, dtype in converted.dtypes.to_dict().items()}
+
+        except Exception as exc:
+            self.operation.success = False
+            self.operation.error = OperationError(
+                message=getattr(exc, "message", str(exc)),
+                code=getattr(exc, "code", type(exc).__name__),
+                status_code=getattr(exc, "status_code", 500),
+                details=getattr(exc, "details", {}),
+            )
+            raise
+
+        finally:
+            self.operation.ended_at = datetime.now(tz=UTC)
+            delta = self.operation.ended_at - self.operation.started_at
+            self.operation.duration_ms = round(delta.total_seconds() * 1000, 3)
+
+    wrapper._tracked = True  # type: ignore[attr-defined]
+    return wrapper

ds_resource_plugin_py_lib/common/resource/dataset/enums.py

@@ -0,0 +1,36 @@
+"""
+**File:** ``enums.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/dataset``
+
+Description
+-----------
+Enumerations for dataset operations.
+"""
+
+from enum import StrEnum
+
+
+class DatasetMethod(StrEnum):
+    """Allowed dataset operation names."""
+
+    CREATE = "create"
+    """Insert rows into the target. Atomic. Not idempotent."""
+    READ = "read"
+    """Read all data from the source into ``self.output``. Idempotent."""
+    UPDATE = "update"
+    """Update existing rows matched by identity columns. Atomic. Idempotent."""
+    UPSERT = "upsert"
+    """Insert or update rows matched by identity columns. Atomic. Idempotent."""
+    DELETE = "delete"
+    """Remove specific rows matched by identity columns. Atomic. Idempotent."""
+    PURGE = "purge"
+    """Remove all content from the target. Atomic. Idempotent."""
+    LIST = "list"
+    """Discover available resources and populate ``self.output``. Idempotent."""
+    RENAME = "rename"
+    """Rename a resource in the backend. Atomic. Not idempotent."""
+
+    @staticmethod
+    def all_values() -> frozenset[str]:
+        """Return all operation method values as a frozen set."""
+        return frozenset(m.value for m in DatasetMethod)

ds_resource_plugin_py_lib/common/resource/dataset/errors.py

@@ -0,0 +1,181 @@
+"""
+**File:** ``errors.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/dataset``
+
+Description
+-----------
+Exceptions for datasets.
+"""
+
+from typing import Any
+
+from ..errors import ResourceException
+
+
+class DatasetException(ResourceException):
+    """Base exception for all dataset-related errors."""
+
+    def __init__(
+        self,
+        message: str = "Dataset operation failed",
+        code: str = "DS_DATASET_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class MismatchedLinkedServiceError(DatasetException):
+    """Raised when a linked service does not match the dataset type."""
+
+    def __init__(
+        self,
+        message: str = "Mismatched linked service",
+        code: str = "DS_DATASET_LINKED_SERVICE_MISMATCHED_ERROR",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class UnsupportedDatasetTypeError(DatasetException):
+    """Raised when a dataset type is not supported."""
+
+    def __init__(
+        self,
+        message: str = "Dataset type is not supported",
+        code: str = "DS_DATASET_UNSUPPORTED_TYPE_ERROR",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class InvalidDatasetClassError(DatasetException):
+    """Raised when a dataset type is invalid."""
+
+    def __init__(
+        self,
+        message: str = "Invalid dataset type",
+        code: str = "DS_DATASET_INVALID_CLASS_ERROR",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class NotFoundError(DatasetException):
+    """Raised when a resource is not found."""
+
+    def __init__(
+        self,
+        message: str = "Resource not found",
+        code: str = "DS_DATASET_NOT_FOUND_ERROR",
+        status_code: int = 404,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class ReadError(DatasetException):
+    """Raised when a read operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Read operation failed",
+        code: str = "DS_DATASET_READ_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class CreateError(DatasetException):
+    """Raised when a create operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Create operation failed",
+        code: str = "DS_DATASET_CREATE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class UpdateError(DatasetException):
+    """Raised when a update operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Update operation failed",
+        code: str = "DS_DATASET_UPDATE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class DeleteError(DatasetException):
+    """Raised when a delete operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Delete operation failed",
+        code: str = "DS_DATASET_DELETE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class RenameError(DatasetException):
+    """Raised when a rename operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Rename operation failed",
+        code: str = "DS_DATASET_RENAME_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class UpsertError(DatasetException):
+    """Raised when an upsert operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Upsert operation failed",
+        code: str = "DS_DATASET_UPSERT_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class PurgeError(DatasetException):
+    """Raised when a purge operation fails."""
+
+    def __init__(
+        self,
+        message: str = "Purge operation failed",
+        code: str = "DS_DATASET_PURGE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class ListError(DatasetException):
+    """Raised when a list operation fails."""
+
+    def __init__(
+        self,
+        message: str = "List operation failed",
+        code: str = "DS_DATASET_LIST_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)

ds_resource_plugin_py_lib/common/resource/dataset/result.py

@@ -0,0 +1,64 @@
+"""
+**File:** ``result.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/dataset``
+
+Description
+-----------
+Dataclass capturing operation details for every dataset method call.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+from ds_common_serde_py_lib import Serializable
+
+from .enums import DatasetMethod
+
+
+@dataclass(kw_only=True)
+class OperationError(Serializable):
+    """Structured error captured from a ``ResourceException``."""
+
+    message: str
+    """The error message."""
+    code: str
+    """The error code."""
+    status_code: int
+    """The HTTP status code."""
+    details: dict[str, Any] = field(default_factory=dict)
+    """The error details."""
+
+
+@dataclass(kw_only=True)
+class OperationInfo(Serializable):
+    """
+    Report produced by every dataset operation.
+
+    Timing fields (``started_at``, ``ended_at``, ``duration_ms``) are
+    populated automatically by the ``track_result`` decorator.  Providers
+    may set ``row_count``, ``schema``, or ``metadata`` inside their
+    method; any value left at its default will be auto-derived from
+    ``self.output`` after the method returns.
+
+    Accessible on the dataset instance as ``self.operation``.
+    """
+
+    method: DatasetMethod | None = None
+    """The method that was called."""
+    success: bool = False
+    """Whether the method call was successful."""
+    error: OperationError | None = None
+    """The error captured from a ``ResourceException``."""
+    row_count: int = 0
+    """The number of rows read, written, or discovered."""
+    started_at: datetime | None = None
+    """The timestamp when the method started."""
+    ended_at: datetime | None = None
+    """The timestamp when the method ended."""
+    duration_ms: float = 0.0
+    """The duration of the method in milliseconds."""
+    schema: dict[str, Any] | None = None
+    """The schema of the data."""
+    metadata: dict[str, Any] = field(default_factory=dict)
+    """The metadata of the data."""

ds_resource_plugin_py_lib/common/resource/dataset/storage_format.py

@@ -0,0 +1,80 @@
+"""
+**File:** ``storage_format.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/dataset``
+
+Description
+-----------
+Storage format models for datasets.
+
+Example
+-------
+.. code-block:: python
+
+    from ds_resource_plugin_py_lib.common.resource.dataset.storage_format import (
+        CsvFormat,
+        DatasetStorageFormatType,
+    )
+
+    fmt = CsvFormat(args={"delimiter": ";"})
+    assert fmt.type == DatasetStorageFormatType.CSV
+"""
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+from ds_common_serde_py_lib import Serializable
+
+
+class DatasetStorageFormatType(StrEnum):
+    """
+    Enum to define the storage format types.
+    """
+
+    PARQUET = "parquet"
+    CSV = "csv"
+    JSON = "json"
+    EXCEL = "excel"
+    SEMI_STRUCTURED_JSON = "semi-structured-json"
+    XML = "xml"
+    BINARY = "binary"
+
+
+@dataclass(kw_only=True)
+class DatasetStorageFormat(Serializable):
+    """
+    The object containing the storage format of the dataset.
+    """
+
+    type: DatasetStorageFormatType
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(kw_only=True)
+class ParquetFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.PARQUET
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(kw_only=True)
+class CsvFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.CSV
+    args: dict[str, Any] = field(default_factory=lambda: {"delimiter": ","})
+
+
+@dataclass(kw_only=True)
+class SemiStructuredJsonFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.SEMI_STRUCTURED_JSON
+    args: dict[str, Any] = field(default_factory=lambda: {"record_path": None})
+
+
+@dataclass(kw_only=True)
+class XMLFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.XML
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(kw_only=True)
+class BinaryFormat(DatasetStorageFormat):
+    type: DatasetStorageFormatType = DatasetStorageFormatType.BINARY
+    args: dict[str, Any] = field(default_factory=lambda: {"column": "binary", "row": 0})

ds_resource_plugin_py_lib/common/resource/errors.py

@@ -0,0 +1,53 @@
+"""
+**File:** ``errors.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource``
+
+Description
+-----------
+Exceptions for resources.
+"""
+
+from typing import Any
+
+
+class ResourceException(Exception):
+    """Base exception for all resource-related errors."""
+
+    def __init__(
+        self,
+        message: str = "Resource operation failed",
+        code: str = "DS_RESOURCE_ERROR",
+        status_code: int = 500,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        self.code = code
+        self.status_code = status_code
+        self.message = message
+        self.details = details or {}
+        super().__init__(self.message)
+
+
+class NotSupportedError(ResourceException):
+    """Raised when a provider does not support an optional method."""
+
+    def __init__(
+        self,
+        message: str = "Operation not supported",
+        code: str = "DS_RESOURCE_NOT_SUPPORTED_ERROR",
+        status_code: int = 501,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)
+
+
+class ValidationError(ResourceException):
+    """Raised when input fails validation before reaching the backend."""
+
+    def __init__(
+        self,
+        message: str = "Validation failed",
+        code: str = "DS_RESOURCE_VALIDATION_ERROR",
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message, code, status_code, details)

ds_resource_plugin_py_lib/common/resource/linked_service/__init__.py

@@ -0,0 +1,18 @@
+"""
+**File:** ``__init__.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/linked_service``
+
+Description
+-----------
+Linked service models and typed properties.
+"""
+
+from .base import LinkedService, LinkedServiceInfo, LinkedServiceSettings
+from .enums import LinkedServiceMethod
+
+__all__ = [
+    "LinkedService",
+    "LinkedServiceInfo",
+    "LinkedServiceMethod",
+    "LinkedServiceSettings",
+]

ds_resource_plugin_py_lib/common/resource/linked_service/base.py

@@ -0,0 +1,186 @@
+"""
+**File:** ``base.py``
+**Region:** ``ds_resource_plugin_py_lib/common/resource/linked_service``
+
+Description
+-----------
+Base models for linked services.
+"""
+
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import StrEnum
+from types import TracebackType
+from typing import Any, Generic, NamedTuple, Self, TypeVar
+
+from ds_common_serde_py_lib import Serializable
+
+
+class LinkedServiceInfo(NamedTuple):
+    """
+    NamedTuple that represents the linked service information.
+    """
+
+    type: str
+    name: str
+    class_name: str
+    version: str
+    description: str | None = None
+
+    def __str__(self) -> str:
+        """
+        Return a string representation of the linked service info.
+
+        Returns:
+            A string representation of the linked service info.
+        """
+        return f"{self.type}:v{self.version}"
+
+    @property
+    def key(self) -> tuple[str, str]:
+        """
+        Return the composite key (type, version) for dictionary lookups.
+
+        Returns:
+            A tuple containing the type and version.
+        """
+        return (self.type, self.version)
+
+
+@dataclass(kw_only=True)
+class LinkedServiceSettings(Serializable):
+    """
+    The object containing the settings of the linked service.
+    """
+
+    pass
+
+
+LinkedServiceSettingsType = TypeVar("LinkedServiceSettingsType", bound=LinkedServiceSettings)
+
+
+@dataclass(kw_only=True)
+class LinkedService(
+    ABC,
+    Serializable,
+    Generic[LinkedServiceSettingsType],
+):
+    """
+    The object containing the connection information to connect with related data store.
+
+    You probably want to use the subclasses and not this class directly.
+    Known subclasses are:
+    - SftpLinkedService
+    - S3LinkedService
+    - GraphQlLinkedService
+
+    All required parameters must be populated in the constructor in order to send to ds-workflow-service.
+    """
+
+    id: uuid.UUID
+    name: str
+    description: str | None = None
+    version: str
+
+    settings: LinkedServiceSettingsType
+
+    def __enter__(self) -> Self:
+        """
+        Context manager enter.
+
+        Returns:
+            The linked service.
+        """
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        """
+        Context manager exit.
+
+        Args:
+            exc_type: The type of the exception.
+            exc_value: The value of the exception.
+            traceback: The traceback of the exception.
+        """
+        self.close()
+
+    @property
+    @abstractmethod
+    def type(self) -> StrEnum:
+        """
+        Get the type of the linked service.
+
+        Returns:
+            The type of the linked service.
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def connection(self) -> Any:
+        """
+        Return the backend client or connection object. Must raise
+        ``ConnectionError`` if ``connect()`` has not been called.
+
+        Returns:
+            The backend client, session, or connection object.
+
+        Raises:
+            ConnectionError: If the connection has not been established.
+
+        See Also:
+            Full contract: ``docs/LINKED_SERVICE_CONTRACT.md`` -- ``connection``
+        """
+        ...
+
+    @abstractmethod
+    def connect(self) -> None:
+        """
+        Establish a connection to the backend data store. The result is
+        stored internally and accessible via the ``connection`` property.
+
+        Raises:
+            ConnectionError: If the connection cannot be established.
+            AuthenticationError: If credentials are invalid.
+            NotSupportedError: If the provider does not support connect.
+
+        See Also:
+            Full contract: ``docs/LINKED_SERVICE_CONTRACT.md`` -- ``connect()``
+        """
+        ...
+
+    @abstractmethod
+    def test_connection(self) -> tuple[bool, str]:
+        """
+        Verify that the connection to the backend is healthy. Does not
+        raise on connection failure -- returns ``(False, "reason")``
+        instead.
+
+        Returns:
+            A tuple ``(success, message)``. On success: ``(True, "")``.
+            On failure: ``(False, "error description")``.
+
+        Raises:
+            NotSupportedError: If the provider does not support test_connection.
+
+        See Also:
+            Full contract: ``docs/LINKED_SERVICE_CONTRACT.md`` -- ``test_connection()``
+        """
+        ...
+
+    @abstractmethod
+    def close(self) -> None:
+        """
+        Release connections, sessions, or handles. Must not raise if
+        already closed. Idempotent.
+
+        See Also:
+            Full contract: ``docs/LINKED_SERVICE_CONTRACT.md`` -- ``close()``
+        """
+        ...