UncountablePythonSDK 0.0.24__py3-none-any.whl → 0.0.131__py3-none-any.whl

pkgs/filesystem_utils/_sftp_session.py

@@ -0,0 +1,151 @@
+import os
+from collections.abc import Iterable
+from io import BytesIO
+
+import paramiko
+import pysftp
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemFileReference,
+    FileSystemObject,
+    FileSystemSFTPConfig,
+    FileTransfer,
+    IncompatibleFileReference,
+)
+
+from .filesystem_session import FileSystemSession
+
+
+def move_sftp_files(
+    connection: pysftp.Connection,
+    src_filepath: str,
+    dest_filepath: str,
+) -> None:
+    connection.rename(src_filepath, dest_filepath)
+
+
+def list_sftp_files(
+    connection: pysftp.Connection,
+    dir_path: str,
+    *,
+    valid_extensions: Iterable[str] | None = None,
+    parent_dir_path: str | None = None,
+    recursive: bool = True,
+) -> list[str]:
+    file_paths: list[str] = []
+    if recursive:
+
+        def _skip(name: str) -> None:
+            return
+
+        def _add_file(path: str) -> None:
+            if (
+                valid_extensions is None
+                or os.path.splitext(path)[1] in valid_extensions
+            ) and (parent_dir_path is None or os.path.dirname(path) == parent_dir_path):
+                file_paths.append(path)
+
+        connection.walktree(
+            dir_path, fcallback=_add_file, dcallback=_skip, ucallback=_skip
+        )
+    else:
+        file_paths.extend([
+            os.path.join(dir_path, file)
+            for file in connection.listdir(dir_path)
+            if connection.isfile(os.path.join(dir_path, file))
+            and (
+                valid_extensions is None
+                or os.path.splitext(file)[1] in valid_extensions
+            )
+        ])
+    return file_paths
+
+
+class SFTPSession(FileSystemSession):
+    def __init__(self, sftp_config: FileSystemSFTPConfig) -> None:
+        super().__init__()
+        self.host: str = sftp_config.ip
+        self.username: str = sftp_config.username
+        self.key_file: str | paramiko.RSAKey | None = (
+            sftp_config.pem_path
+            if sftp_config.pem_path is not None
+            else sftp_config.pem_key
+        )
+        self.password: str | None = sftp_config.password
+
+    def start(self) -> None:
+        cnopts = pysftp.CnOpts()
+        cnopts.hostkeys = None
+        if self.key_file is not None:
+            self.connection = pysftp.Connection(
+                self.host,
+                username=self.username,
+                private_key=self.key_file,
+                cnopts=cnopts,
+            )
+        elif self.password is not None:
+            self.connection = pysftp.Connection(
+                self.host,
+                username=self.username,
+                password=self.password,
+                cnopts=cnopts,
+            )
+        else:
+            raise pysftp.CredentialException(
+                "Must specify either a private key path or a password."
+            )
+
+    def __enter__(self) -> "SFTPSession":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+        self.connection.close()
+
+    def list_files(
+        self,
+        dir_path: FileSystemObject,
+        *,
+        recursive: bool = True,
+        valid_extensions: list[str] | None = None,
+    ) -> list[FileSystemObject]:
+        if not isinstance(
+            dir_path, FileSystemFileReference
+        ) or not self.connection.isdir(dir_path.filepath):
+            raise IncompatibleFileReference()
+
+        return [
+            FileSystemFileReference(file_path)
+            for file_path in list_sftp_files(
+                self.connection,
+                dir_path.filepath,
+                recursive=recursive,
+                valid_extensions=valid_extensions,
+            )
+        ]
+
+    def download_files(self, filepaths: list[FileSystemObject]) -> list[FileObjectData]:
+        downloaded_files: list[FileObjectData] = []
+        for file_object in filepaths:
+            if (
+                not isinstance(file_object, FileSystemFileReference)
+                or file_object.filename is None
+            ):
+                raise IncompatibleFileReference()
+            filepath = file_object.filepath
+            file_data = self.connection.open(filepath).read()
+            downloaded_file = FileObjectData(
+                file_data, BytesIO(file_data), file_object.filename, filepath=filepath
+            )
+            if downloaded_file is not None:
+                downloaded_files.append(downloaded_file)
+        return downloaded_files
+
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        for src_file, dest_file in file_mappings:
+            if not isinstance(src_file, FileSystemFileReference) or not isinstance(
+                dest_file, FileSystemFileReference
+            ):
+                raise IncompatibleFileReference()
+            move_sftp_files(self.connection, src_file.filepath, dest_file.filepath)

pkgs/filesystem_utils/file_type_utils.py

@@ -0,0 +1,91 @@
+import os
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Union
+
+import paramiko
+from azure.core.credentials import (
+    AzureNamedKeyCredential,
+    AzureSasCredential,
+    TokenCredential,
+)
+from azure.storage.blob import ContainerProperties
+
+
+@dataclass
+class FileObjectData:
+    file_data: bytes
+    file_IO: BytesIO
+    filename: str
+    filepath: str | None = None
+    mime_type: str | None = None
+    metadata: dict[str, str] | None = None
+
+
+@dataclass
+class FileSystemFileReference:
+    filepath: str
+
+    @property
+    def filename(self) -> str:
+        return os.path.basename(self.filepath)
+
+    @property
+    def dirname(self) -> str:
+        return os.path.dirname(self.filepath)
+
+
+@dataclass
+class RemoteObjectReference:
+    file_id: str
+    mime_type: str
+    filename: str | None = None
+
+    @property
+    def is_dir(self) -> bool:
+        return "folder" in self.mime_type
+
+
+FileSystemObject = Union[FileSystemFileReference, RemoteObjectReference, FileObjectData]
+
+
+FileTransfer = tuple[FileSystemObject, FileSystemObject]
+
+
+class IncompatibleFileReference(Exception):
+    pass
+
+
+@dataclass(frozen=True, kw_only=True)
+class FileSystemSFTPConfig:
+    ip: str
+    username: str
+    pem_path: str | None
+    pem_key: paramiko.RSAKey | None = None
+    password: str | None = None
+    valid_extensions: tuple[str] | None = None
+    recursive: bool = True
+
+
+@dataclass(kw_only=True)
+class FileSystemS3Config:
+    endpoint_url: str
+    bucket_name: str
+    region_name: str | None
+    access_key_id: str | None
+    secret_access_key: str | None
+    session_token: str | None
+
+
+@dataclass(kw_only=True)
+class FileSystemBlobConfig:
+    account_url: str
+    credential: (
+        str
+        | dict[str, str]
+        | AzureNamedKeyCredential
+        | AzureSasCredential
+        | TokenCredential
+        | None
+    )
+    container: ContainerProperties | str

pkgs/filesystem_utils/filesystem_session.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemObject,
+    FileTransfer,
+)
+
+
+class FileSystemSession(ABC):
+    def __init__(self) -> None:
+        return
+
+    @abstractmethod
+    def start(self) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def list_files(
+        self, dir_path: FileSystemObject, *, recursive: bool = True
+    ) -> list[FileSystemObject]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def download_files(self, filepaths: list[FileSystemObject]) -> list[FileObjectData]:
+        raise NotImplementedError
+
+    def delete_files(self, filepaths: list[FileSystemObject]) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def __enter__(self) -> "FileSystemSession": ...
+
+    @abstractmethod
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: ...

pkgs/serialization/__init__.py

@@ -1,10 +1,17 @@
-# flake8:noqa
+from .annotation import unwrap_annotated as unwrap_annotated
 from .missing_sentry import MISSING_SENTRY as MISSING_SENTRY
 from .missing_sentry import MissingSentryType as MissingSentryType
 from .missing_sentry import MissingType as MissingType
 from .missing_sentry import coalesce_missing_sentry as coalesce_missing_sentry
 from .opaque_key import OpaqueKey as OpaqueKey
+from .serial_alias import SerialAliasInspector as SerialAliasInspector
+from .serial_alias import get_serial_alias_data as get_serial_alias_data
+from .serial_alias import serial_alias_annotation as serial_alias_annotation
+from .serial_class import SerialClassDataInspector as SerialClassDataInspector
 from .serial_class import get_serial_class_data as get_serial_class_data
 from .serial_class import get_serial_string_enum_data as get_serial_string_enum_data
 from .serial_class import serial_class as serial_class
 from .serial_class import serial_string_enum as serial_string_enum
+from .serial_generic import get_serial_data as get_serial_data
+from .serial_union import get_serial_union_data as get_serial_union_data
+from .serial_union import serial_union_annotation as serial_union_annotation

pkgs/serialization/annotation.py

@@ -0,0 +1,64 @@
+import dataclasses
+import typing
+
+T = typing.TypeVar("T")
+
+
+@dataclasses.dataclass(kw_only=True, frozen=True, eq=True)
+class SerialBase:
+    named_type_path: str | None = None
+    # Indicates this type is allowed in dynamic lookups, such as via a named_type_path
+    # This isn't meant to be limting, but to catalog all the types where we need it
+    is_dynamic_allowed: bool = False
+    # Tracks if this data was provided as a decorator to the type.
+    # This is used to track "proper types" which are appropriate
+    # for serialization and/or dynamic discovery
+    from_decorator: bool = False
+
+
+def get_serial_annotation(parsed_type: type[T]) -> SerialBase | None:
+    if not hasattr(parsed_type, "__metadata__"):
+        return None
+    metadata = parsed_type.__metadata__  # type:ignore[attr-defined]
+    if not isinstance(metadata, tuple) or len(metadata) != 1:
+        return None
+    serial = metadata[0]
+    if not isinstance(serial, SerialBase):
+        return None
+    return serial
+
+
+class SerialInspector(typing.Generic[T]):
+    def __init__(self, parsed_type: type[T], serial_base: SerialBase) -> None:
+        self._parsed_type = parsed_type
+        self._serial_base = serial_base
+
+    @property
+    def named_type_path(self) -> str | None:
+        return self._serial_base.named_type_path
+
+    @property
+    def from_decorator(self) -> bool:
+        return self._serial_base.from_decorator
+
+    @property
+    def is_field_proper(self) -> bool:
+        return (
+            self._serial_base.from_decorator
+            and self._serial_base.named_type_path is not None
+        )
+
+    @property
+    def is_dynamic_allowed(self) -> bool:
+        return self._serial_base.is_dynamic_allowed
+
+
+def unwrap_annotated(parsed_type: type[T]) -> type[T]:
+    """
+    If the type is an annotated type then return the origin of it.
+    Otherwise return the original type.
+    """
+    if typing.get_origin(parsed_type) is typing.Annotated:
+        # It's unclear if there's anyway to type this correctly
+        return parsed_type.__origin__  # type:ignore[attr-defined, no-any-return]
+    return parsed_type

pkgs/serialization/missing_sentry.py

@@ -26,5 +26,5 @@ MISSING_SENTRY = MissingSentryType()
 MissingType = Union[MissingSentryType, ClassT]
 
 
-def coalesce_missing_sentry(value: MissingType[ClassT]) -> Optional[ClassT]:
+def coalesce_missing_sentry(value: MissingType[ClassT]) -> ClassT | None:
     return None if isinstance(value, MissingSentryType) else value

pkgs/serialization/opaque_key.py

@@ -1,4 +1,4 @@
 # Blocks a string key value from being interpreted for case conversion
-class OpaqueKey(str):
+class OpaqueKey(str):  # noqa: FURB189
     def __new__(cls, key: str) -> "OpaqueKey":
         return str.__new__(cls, key)

pkgs/serialization/serial_alias.py

@@ -0,0 +1,47 @@
+import dataclasses
+import typing
+
+from .annotation import SerialBase, SerialInspector, get_serial_annotation
+
+T = typing.TypeVar("T")
+
+
+@dataclasses.dataclass(kw_only=True, frozen=True, eq=True)
+class _SerialAlias(SerialBase):
+    """
+    This class is to be kept private, to provide flexibility in registration/lookup.
+    Places that need the data should access it via help classes/methods.
+    """
+
+
+def serial_alias_annotation(
+    *,
+    named_type_path: str | None = None,
+    is_dynamic_allowed: bool = False,
+) -> _SerialAlias:
+    return _SerialAlias(
+        named_type_path=named_type_path,
+        from_decorator=True,
+        is_dynamic_allowed=is_dynamic_allowed,
+    )
+
+
+def _get_serial_alias(parsed_type: type[T]) -> _SerialAlias | None:
+    serial = get_serial_annotation(parsed_type)
+    if not isinstance(serial, _SerialAlias):
+        return None
+    return serial
+
+
+class SerialAliasInspector(SerialInspector[T]):
+    def __init__(self, parsed_type: type[T], serial_alias: _SerialAlias) -> None:
+        super().__init__(parsed_type, serial_alias)
+        self._serial_alias = serial_alias
+
+
+def get_serial_alias_data(parsed_type: type[T]) -> SerialAliasInspector[T] | None:
+    serial = _get_serial_alias(parsed_type)
+    if serial is None:
+        return None
+
+    return SerialAliasInspector(parsed_type, serial)

pkgs/serialization/serial_class.py

@@ -3,26 +3,34 @@ from __future__ import annotations
 import dataclasses
 from collections.abc import Callable
 from enum import StrEnum
-from typing import Any, Optional, TypeVar, cast
+from typing import Any, TypeVar, cast
 
-_ClassT = TypeVar("_ClassT")
+from .annotation import SerialBase, SerialInspector
 
+ClassT = TypeVar("ClassT")
 
-@dataclasses.dataclass
-class _SerialClassData:
+
+@dataclasses.dataclass(kw_only=True, frozen=True, eq=True)
+class _SerialClassData(SerialBase):
     unconverted_keys: set[str] = dataclasses.field(default_factory=set)
     unconverted_values: set[str] = dataclasses.field(default_factory=set)
     to_string_values: set[str] = dataclasses.field(default_factory=set)
     parse_require: set[str] = dataclasses.field(default_factory=set)
+    named_type_path: str | None = None
+
+
+EMPTY_SERIAL_CLASS_DATA = _SerialClassData()
 
 
 def serial_class(
     *,
-    unconverted_keys: Optional[set[str]] = None,
-    unconverted_values: Optional[set[str]] = None,
-    to_string_values: Optional[set[str]] = None,
-    parse_require: Optional[set[str]] = None,
-) -> Callable[[_ClassT], _ClassT]:
+    unconverted_keys: set[str] | None = None,
+    unconverted_values: set[str] | None = None,
+    to_string_values: set[str] | None = None,
+    parse_require: set[str] | None = None,
+    named_type_path: str | None = None,
+    is_dynamic_allowed: bool = False,
+) -> Callable[[ClassT], ClassT]:
     """
     An additional decorator to a dataclass that specifies serialization options.
 
@@ -42,73 +50,80 @@ def serial_class(
         This field is always required while parsing, even if it has a default in the definition.
         This allows supporting literal type defaults for Python instantiation, but
         requiring them for the API input.
+    @param named_type_path
+        The type_spec type-path to this type. This applies only to named types.
     """
 
-    def decorate(orig_class: _ClassT) -> _ClassT:
+    def decorate(orig_class: ClassT) -> ClassT:
         cast(Any, orig_class).__unc_serial_data = _SerialClassData(
             unconverted_keys=unconverted_keys or set(),
             unconverted_values=unconverted_values or set(),
             to_string_values=to_string_values or set(),
             parse_require=parse_require or set(),
+            named_type_path=named_type_path,
+            from_decorator=True,
+            is_dynamic_allowed=is_dynamic_allowed,
         )
         return orig_class
 
     return decorate
 
 
-class SerialClassDataInspector:
-    bases: list[SerialClassDataInspector]
-
+class SerialClassDataInspector(SerialInspector[ClassT]):
     def __init__(
-        self, bases: list[SerialClassDataInspector], current: _SerialClassData
+        self,
+        parsed_type: type[ClassT],
+        current: _SerialClassData,
     ) -> None:
-        self.bases = bases
+        super().__init__(parsed_type, current)
         self.current = current
 
     def has_unconverted_key(self, key: str) -> bool:
-        if key in self.current.unconverted_keys:
-            return True
-        for base in self.bases:
-            if base.has_unconverted_key(key):
-                return True
-        return False
+        return key in self.current.unconverted_keys
 
     def has_unconverted_value(self, key: str) -> bool:
-        if key in self.current.unconverted_values:
-            return True
-        for base in self.bases:
-            if base.has_unconverted_value(key):
-                return True
-        return False
+        return key in self.current.unconverted_values
 
     def has_to_string_value(self, key: str) -> bool:
-        if key in self.current.to_string_values:
-            return True
-        for base in self.bases:
-            if base.has_to_string_value(key):
-                return True
-        return False
+        return key in self.current.to_string_values
 
     def has_parse_require(self, key: str) -> bool:
-        if key in self.current.parse_require:
-            return True
-        for base in self.bases:
-            if base.has_parse_require(key):
-                return True
-        return False
-
-
-def get_serial_class_data(type_class: type[Any]) -> SerialClassDataInspector:
-    bases = (
-        [get_serial_class_data(base) for base in type_class.__bases__]
-        if type_class.__bases__ is not None
-        else []
-    )
-    return SerialClassDataInspector(
-        bases,
+        return key in self.current.parse_require
+
+
+def get_merged_serial_class_data(type_class: type[Any]) -> _SerialClassData | None:
+    base_class_data = (
         cast(_SerialClassData, type_class.__unc_serial_data)
         if hasattr(type_class, "__unc_serial_data")
-        else _SerialClassData(),
+        else None
+    )
+    if base_class_data is None:
+        return None
+
+    # IMPROVE: We should cache this result on the type
+    if type_class.__bases__ is not None:
+        for base in type_class.__bases__:
+            curr_base_class_data = get_merged_serial_class_data(base)
+            if curr_base_class_data is not None:
+                base_class_data = dataclasses.replace(
+                    base_class_data,
+                    unconverted_keys=base_class_data.unconverted_keys
+                    | curr_base_class_data.unconverted_keys,
+                    unconverted_values=base_class_data.unconverted_values
+                    | curr_base_class_data.unconverted_values,
+                    to_string_values=base_class_data.to_string_values
+                    | curr_base_class_data.to_string_values,
+                    parse_require=base_class_data.parse_require
+                    | curr_base_class_data.parse_require,
+                )
+    return base_class_data
+
+
+def get_serial_class_data(
+    type_class: type[ClassT],
+) -> SerialClassDataInspector[ClassT]:
+    return SerialClassDataInspector(
+        type_class, get_merged_serial_class_data(type_class) or EMPTY_SERIAL_CLASS_DATA
     )
 
 
@@ -119,13 +134,13 @@ class _SerialStringEnumData:
 
 
 def serial_string_enum(
-    *, labels: Optional[dict[str, str]] = None, deprecated: Optional[set[str]] = None
-) -> Callable[[_ClassT], _ClassT]:
+    *, labels: dict[str, str] | None = None, deprecated: set[str] | None = None
+) -> Callable[[ClassT], ClassT]:
     """
     A decorator for enums to provide serialization data, including labels.
     """
 
-    def decorate(orig_class: _ClassT) -> _ClassT:
+    def decorate(orig_class: ClassT) -> ClassT:
         cast(Any, orig_class).__unc_serial_string_enum_data = _SerialStringEnumData(
             labels=labels or {}, deprecated=deprecated or set()
         )
@@ -138,7 +153,7 @@ class SerialStringEnumInspector:
     def __init__(self, current: _SerialStringEnumData) -> None:
         self.current = current
 
-    def get_label(self, value: str) -> Optional[str]:
+    def get_label(self, value: str) -> str | None:
         return self.current.labels.get(value)
 
     def get_deprecated(self, value: str) -> bool:

pkgs/serialization/serial_generic.py

@@ -0,0 +1,16 @@
+import typing
+
+from .annotation import SerialInspector, get_serial_annotation
+from .serial_class import get_merged_serial_class_data
+
+T = typing.TypeVar("T")
+
+
+def get_serial_data(parsed_type: type[T]) -> SerialInspector[T] | None:
+    serial = get_serial_annotation(parsed_type)
+    if serial is None:
+        serial = get_merged_serial_class_data(parsed_type)
+
+    if serial is not None:
+        return SerialInspector(parsed_type, serial)
+    return None