UncountablePythonSDK 0.0.8__py3-none-any.whl → 0.0.92__py3-none-any.whl

pkgs/filesystem_utils/file_type_utils.py

@@ -0,0 +1,91 @@
+import os
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Union
+
+import paramiko
+from azure.core.credentials import (
+    AzureNamedKeyCredential,
+    AzureSasCredential,
+    TokenCredential,
+)
+from azure.storage.blob import ContainerProperties
+
+
+@dataclass
+class FileObjectData:
+    file_data: bytes
+    file_IO: BytesIO
+    filename: str
+    filepath: str | None = None
+    mime_type: str | None = None
+    metadata: dict[str, str] | None = None
+
+
+@dataclass
+class FileSystemFileReference:
+    filepath: str
+
+    @property
+    def filename(self) -> str:
+        return os.path.basename(self.filepath)
+
+    @property
+    def dirname(self) -> str:
+        return os.path.dirname(self.filepath)
+
+
+@dataclass
+class RemoteObjectReference:
+    file_id: str
+    mime_type: str
+    filename: str | None = None
+
+    @property
+    def is_dir(self) -> bool:
+        return "folder" in self.mime_type
+
+
+FileSystemObject = Union[FileSystemFileReference, RemoteObjectReference, FileObjectData]
+
+
+FileTransfer = tuple[FileSystemObject, FileSystemObject]
+
+
+class IncompatibleFileReference(Exception):
+    pass
+
+
+@dataclass(frozen=True, kw_only=True)
+class FileSystemSFTPConfig:
+    ip: str
+    username: str
+    pem_path: str | None
+    pem_key: paramiko.RSAKey | None = None
+    password: str | None = None
+    valid_extensions: tuple[str] | None = None
+    recursive: bool = True
+
+
+@dataclass(kw_only=True)
+class FileSystemS3Config:
+    endpoint_url: str
+    bucket_name: str
+    region_name: str | None
+    access_key_id: str | None
+    secret_access_key: str | None
+    session_token: str | None
+
+
+@dataclass(kw_only=True)
+class FileSystemBlobConfig:
+    account_url: str
+    credential: (
+        str
+        | dict[str, str]
+        | AzureNamedKeyCredential
+        | AzureSasCredential
+        | TokenCredential
+        | None
+    )
+    container: ContainerProperties | str

pkgs/filesystem_utils/filesystem_session.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemObject,
+    FileTransfer,
+)
+
+
+class FileSystemSession(ABC):
+    def __init__(self) -> None:
+        return
+
+    @abstractmethod
+    def start(self) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def list_files(
+        self, dir_path: FileSystemObject, *, recursive: bool = True
+    ) -> list[FileSystemObject]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def download_files(self, filepaths: list[FileSystemObject]) -> list[FileObjectData]:
+        raise NotImplementedError
+
+    def delete_files(self, filepaths: list[FileSystemObject]) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def __enter__(self) -> "FileSystemSession": ...
+
+    @abstractmethod
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: ...

pkgs/serialization/__init__.py

@@ -1,10 +1,17 @@
-# flake8:noqa
+from .annotation import unwrap_annotated as unwrap_annotated
 from .missing_sentry import MISSING_SENTRY as MISSING_SENTRY
 from .missing_sentry import MissingSentryType as MissingSentryType
 from .missing_sentry import MissingType as MissingType
 from .missing_sentry import coalesce_missing_sentry as coalesce_missing_sentry
 from .opaque_key import OpaqueKey as OpaqueKey
+from .serial_alias import SerialAliasInspector as SerialAliasInspector
+from .serial_alias import get_serial_alias_data as get_serial_alias_data
+from .serial_alias import serial_alias_annotation as serial_alias_annotation
+from .serial_class import SerialClassDataInspector as SerialClassDataInspector
 from .serial_class import get_serial_class_data as get_serial_class_data
 from .serial_class import get_serial_string_enum_data as get_serial_string_enum_data
 from .serial_class import serial_class as serial_class
 from .serial_class import serial_string_enum as serial_string_enum
+from .serial_generic import get_serial_data as get_serial_data
+from .serial_union import get_serial_union_data as get_serial_union_data
+from .serial_union import serial_union_annotation as serial_union_annotation

pkgs/serialization/annotation.py

@@ -0,0 +1,64 @@
+import dataclasses
+import typing
+
+T = typing.TypeVar("T")
+
+
+@dataclasses.dataclass(kw_only=True, frozen=True, eq=True)
+class SerialBase:
+    named_type_path: str | None = None
+    # Indicates this type is allowed in dynamic lookups, such as via a named_type_path
+    # This isn't meant to be limting, but to catalog all the types where we need it
+    is_dynamic_allowed: bool = False
+    # Tracks if this data was provided as a decorator to the type.
+    # This is used to track "proper types" which are appropriate
+    # for serialization and/or dynamic discovery
+    from_decorator: bool = False
+
+
+def get_serial_annotation(parsed_type: type[T]) -> SerialBase | None:
+    if not hasattr(parsed_type, "__metadata__"):
+        return None
+    metadata = parsed_type.__metadata__  # type:ignore[attr-defined]
+    if not isinstance(metadata, tuple) or len(metadata) != 1:
+        return None
+    serial = metadata[0]
+    if not isinstance(serial, SerialBase):
+        return None
+    return serial
+
+
+class SerialInspector(typing.Generic[T]):
+    def __init__(self, parsed_type: type[T], serial_base: SerialBase) -> None:
+        self._parsed_type = parsed_type
+        self._serial_base = serial_base
+
+    @property
+    def named_type_path(self) -> str | None:
+        return self._serial_base.named_type_path
+
+    @property
+    def from_decorator(self) -> bool:
+        return self._serial_base.from_decorator
+
+    @property
+    def is_field_proper(self) -> bool:
+        return (
+            self._serial_base.from_decorator
+            and self._serial_base.named_type_path is not None
+        )
+
+    @property
+    def is_dynamic_allowed(self) -> bool:
+        return self._serial_base.is_dynamic_allowed
+
+
+def unwrap_annotated(parsed_type: type[T]) -> type[T]:
+    """
+    If the type is an annotated type then return the origin of it.
+    Otherwise return the original type.
+    """
+    if typing.get_origin(parsed_type) is typing.Annotated:
+        # It's unclear if there's anyway to type this correctly
+        return parsed_type.__origin__  # type:ignore[attr-defined, no-any-return]
+    return parsed_type

pkgs/serialization/missing_sentry.py

@@ -26,5 +26,5 @@ MISSING_SENTRY = MissingSentryType()
 MissingType = Union[MissingSentryType, ClassT]
 
 
-def coalesce_missing_sentry(value: MissingType[ClassT]) -> Optional[ClassT]:
+def coalesce_missing_sentry(value: MissingType[ClassT]) -> ClassT | None:
     return None if isinstance(value, MissingSentryType) else value

pkgs/serialization/opaque_key.py

@@ -1,4 +1,4 @@
 # Blocks a string key value from being interpreted for case conversion
-class OpaqueKey(str):
+class OpaqueKey(str):  # noqa: FURB189
     def __new__(cls, key: str) -> "OpaqueKey":
         return str.__new__(cls, key)

pkgs/serialization/serial_alias.py

@@ -0,0 +1,47 @@
+import dataclasses
+import typing
+
+from .annotation import SerialBase, SerialInspector, get_serial_annotation
+
+T = typing.TypeVar("T")
+
+
+@dataclasses.dataclass(kw_only=True, frozen=True, eq=True)
+class _SerialAlias(SerialBase):
+    """
+    This class is to be kept private, to provide flexibility in registration/lookup.
+    Places that need the data should access it via help classes/methods.
+    """
+
+
+def serial_alias_annotation(
+    *,
+    named_type_path: str | None = None,
+    is_dynamic_allowed: bool = False,
+) -> _SerialAlias:
+    return _SerialAlias(
+        named_type_path=named_type_path,
+        from_decorator=True,
+        is_dynamic_allowed=is_dynamic_allowed,
+    )
+
+
+def _get_serial_alias(parsed_type: type[T]) -> _SerialAlias | None:
+    serial = get_serial_annotation(parsed_type)
+    if not isinstance(serial, _SerialAlias):
+        return None
+    return serial
+
+
+class SerialAliasInspector(SerialInspector[T]):
+    def __init__(self, parsed_type: type[T], serial_alias: _SerialAlias) -> None:
+        super().__init__(parsed_type, serial_alias)
+        self._serial_alias = serial_alias
+
+
+def get_serial_alias_data(parsed_type: type[T]) -> SerialAliasInspector[T] | None:
+    serial = _get_serial_alias(parsed_type)
+    if serial is None:
+        return None
+
+    return SerialAliasInspector(parsed_type, serial)

pkgs/serialization/serial_class.py

@@ -1,27 +1,35 @@
 from __future__ import annotations
 
 import dataclasses
-from typing import Any, Callable, Optional, TypeVar, cast
+from collections.abc import Callable
+from enum import StrEnum
+from typing import Any, TypeVar, cast
 
-from pkgs.strenum_compat import StrEnum
+from .annotation import SerialBase, SerialInspector
 
 _ClassT = TypeVar("_ClassT")
 
 
-@dataclasses.dataclass
-class _SerialClassData:
+@dataclasses.dataclass(kw_only=True, frozen=True, eq=True)
+class _SerialClassData(SerialBase):
     unconverted_keys: set[str] = dataclasses.field(default_factory=set)
     unconverted_values: set[str] = dataclasses.field(default_factory=set)
     to_string_values: set[str] = dataclasses.field(default_factory=set)
     parse_require: set[str] = dataclasses.field(default_factory=set)
+    named_type_path: str | None = None
+
+
+EMPTY_SERIAL_CLASS_DATA = _SerialClassData()
 
 
 def serial_class(
     *,
-    unconverted_keys: Optional[set[str]] = None,
-    unconverted_values: Optional[set[str]] = None,
-    to_string_values: Optional[set[str]] = None,
-    parse_require: Optional[set[str]] = None,
+    unconverted_keys: set[str] | None = None,
+    unconverted_values: set[str] | None = None,
+    to_string_values: set[str] | None = None,
+    parse_require: set[str] | None = None,
+    named_type_path: str | None = None,
+    is_dynamic_allowed: bool = False,
 ) -> Callable[[_ClassT], _ClassT]:
     """
     An additional decorator to a dataclass that specifies serialization options.
@@ -42,6 +50,8 @@ def serial_class(
         This field is always required while parsing, even if it has a default in the definition.
         This allows supporting literal type defaults for Python instantiation, but
         requiring them for the API input.
+    @param named_type_path
+        The type_spec type-path to this type. This applies only to named types.
     """
 
     def decorate(orig_class: _ClassT) -> _ClassT:
@@ -50,65 +60,70 @@ def serial_class(
             unconverted_values=unconverted_values or set(),
             to_string_values=to_string_values or set(),
             parse_require=parse_require or set(),
+            named_type_path=named_type_path,
+            from_decorator=True,
+            is_dynamic_allowed=is_dynamic_allowed,
         )
         return orig_class
 
     return decorate
 
 
-class SerialClassDataInspector:
-    bases: list[SerialClassDataInspector]
-
+class SerialClassDataInspector(SerialInspector[_ClassT]):
     def __init__(
-        self, bases: list[SerialClassDataInspector], current: _SerialClassData
+        self,
+        parsed_type: type[_ClassT],
+        current: _SerialClassData,
     ) -> None:
-        self.bases = bases
+        super().__init__(parsed_type, current)
         self.current = current
 
     def has_unconverted_key(self, key: str) -> bool:
-        if key in self.current.unconverted_keys:
-            return True
-        for base in self.bases:
-            if base.has_unconverted_key(key):
-                return True
-        return False
+        return key in self.current.unconverted_keys
 
     def has_unconverted_value(self, key: str) -> bool:
-        if key in self.current.unconverted_values:
-            return True
-        for base in self.bases:
-            if base.has_unconverted_value(key):
-                return True
-        return False
+        return key in self.current.unconverted_values
 
     def has_to_string_value(self, key: str) -> bool:
-        if key in self.current.to_string_values:
-            return True
-        for base in self.bases:
-            if base.has_to_string_value(key):
-                return True
-        return False
+        return key in self.current.to_string_values
 
     def has_parse_require(self, key: str) -> bool:
-        if key in self.current.parse_require:
-            return True
-        for base in self.bases:
-            if base.has_parse_require(key):
-                return True
-        return False
-
-
-def get_serial_class_data(type_class: type[Any]) -> SerialClassDataInspector:
-    bases = (
-        [get_serial_class_data(base) for base in type_class.__bases__]
-        if type_class.__bases__ is not None
-        else []
-    )
-    return SerialClassDataInspector(
-        bases,
+        return key in self.current.parse_require
+
+
+def get_merged_serial_class_data(type_class: type[Any]) -> _SerialClassData | None:
+    base_class_data = (
         cast(_SerialClassData, type_class.__unc_serial_data)
         if hasattr(type_class, "__unc_serial_data")
-        else _SerialClassData(),
+        else None
+    )
+    if base_class_data is None:
+        return None
+
+    # IMPROVE: We should cache this result on the type
+    if type_class.__bases__ is not None:
+        for base in type_class.__bases__:
+            curr_base_class_data = get_merged_serial_class_data(base)
+            if curr_base_class_data is not None:
+                base_class_data = dataclasses.replace(
+                    base_class_data,
+                    unconverted_keys=base_class_data.unconverted_keys
+                    | curr_base_class_data.unconverted_keys,
+                    unconverted_values=base_class_data.unconverted_values
+                    | curr_base_class_data.unconverted_values,
+                    to_string_values=base_class_data.to_string_values
+                    | curr_base_class_data.to_string_values,
+                    parse_require=base_class_data.parse_require
+                    | curr_base_class_data.parse_require,
+                )
+    return base_class_data
+
+
+def get_serial_class_data(
+    type_class: type[_ClassT],
+) -> SerialClassDataInspector[_ClassT]:
+    return SerialClassDataInspector(
+        type_class, get_merged_serial_class_data(type_class) or EMPTY_SERIAL_CLASS_DATA
     )
 
 
@@ -119,7 +134,7 @@ class _SerialStringEnumData:
 
 
 def serial_string_enum(
-    *, labels: Optional[dict[str, str]] = None, deprecated: Optional[set[str]] = None
+    *, labels: dict[str, str] | None = None, deprecated: set[str] | None = None
 ) -> Callable[[_ClassT], _ClassT]:
     """
     A decorator for enums to provide serialization data, including labels.
@@ -138,7 +153,7 @@ class SerialStringEnumInspector:
     def __init__(self, current: _SerialStringEnumData) -> None:
         self.current = current
 
-    def get_label(self, value: str) -> Optional[str]:
+    def get_label(self, value: str) -> str | None:
         return self.current.labels.get(value)
 
     def get_deprecated(self, value: str) -> bool:

pkgs/serialization/serial_generic.py

@@ -0,0 +1,16 @@
+import typing
+
+from .annotation import SerialInspector, get_serial_annotation
+from .serial_class import get_merged_serial_class_data
+
+T = typing.TypeVar("T")
+
+
+def get_serial_data(parsed_type: type[T]) -> SerialInspector[T] | None:
+    serial = get_serial_annotation(parsed_type)
+    if serial is None:
+        serial = get_merged_serial_class_data(parsed_type)
+
+    if serial is not None:
+        return SerialInspector(parsed_type, serial)
+    return None

pkgs/serialization/serial_union.py

@@ -0,0 +1,84 @@
+import dataclasses
+import typing
+
+from .annotation import SerialBase, SerialInspector, get_serial_annotation
+
+T = typing.TypeVar("T")
+
+
+class IdentityHashWrapper(typing.Generic[T]):
+    """This allows unhashable types to be used in the SerialUnion, like dict.
+    Since we have only one copy of the types themselves, we rely on
+    object identity for the hashing."""
+
+    def __init__(self, inner: T) -> None:
+        self.inner = inner
+
+    def __hash__(self) -> int:
+        return id(self.inner)
+
+
+@dataclasses.dataclass(kw_only=True, frozen=True, eq=True)
+class _SerialUnion(SerialBase):
+    """
+    This class is to be kept private, to provide flexibility in registration/lookup.
+    Places that need the data should access it via help classes/methods.
+    """
+
+    # If specified, indicates the Union has a discriminator which should be used to
+    # determine which type to parse.
+    discriminator: str | None = None
+    discriminator_map: IdentityHashWrapper[dict[str, type]] | None = None
+
+
+def serial_union_annotation(
+    *,
+    discriminator: str | None = None,
+    discriminator_map: dict[str, type] | None = None,
+    named_type_path: str | None = None,
+    is_dynamic_allowed: bool = False,
+) -> _SerialUnion:
+    return _SerialUnion(
+        discriminator=discriminator,
+        discriminator_map=IdentityHashWrapper(discriminator_map)
+        if discriminator_map is not None
+        else None,
+        named_type_path=named_type_path,
+        from_decorator=True,
+        is_dynamic_allowed=is_dynamic_allowed,
+    )
+
+
+def _get_serial_union(parsed_type: type[T]) -> _SerialUnion | None:
+    serial = get_serial_annotation(parsed_type)
+    if not isinstance(serial, _SerialUnion):
+        return None
+    return serial
+
+
+class SerialClassInspector(SerialInspector[T]):
+    def __init__(self, parsed_type: type[T], serial_union: _SerialUnion) -> None:
+        super().__init__(parsed_type, serial_union)
+        self._parsed_type = parsed_type
+        self._serial_union = serial_union
+
+    def get_union_underlying(self) -> type[T]:
+        return typing.get_args(self._parsed_type)[0]  # type:ignore[no-any-return]
+
+    @property
+    def discriminator(self) -> str | None:
+        return self._serial_union.discriminator
+
+    @property
+    def discriminator_map(self) -> dict[str, type] | None:
+        if self._serial_union.discriminator_map is None:
+            return None
+        return self._serial_union.discriminator_map.inner
+
+
+def get_serial_union_data(parsed_type: type[T]) -> SerialClassInspector[T] | None:
+    serial = _get_serial_union(parsed_type)
+    if serial is None:
+        return None
+
+    return SerialClassInspector(parsed_type, serial)

pkgs/serialization/yaml.py

@@ -0,0 +1,57 @@
+from decimal import Decimal
+from typing import TYPE_CHECKING, Any
+
+import yaml
+
+if TYPE_CHECKING:
+    from _typeshed import SupportsRead as SupportsReadT
+    from _typeshed import SupportsWrite as SupportsWriteT
+
+    SupportsRead = SupportsReadT[Any]
+    SupportsWrite = SupportsWriteT[Any]
+else:
+    SupportsRead = object
+    SupportsWrite = object
+
+
+def _decimal_constructor(loader, node):  # type:ignore
+    value = loader.construct_scalar(node)
+    return Decimal(value)
+
+
+# A semi-acceptable patch to force a number to be parsed as a decimal, since pyyaml
+# parses them as lossy floats otherwise. Though a bit ugly, at least this way we have
+# support for decimal constants
+yaml.SafeLoader.add_constructor("!decimal", _decimal_constructor)
+
+
+class YAMLError(BaseException):
+    pass
+
+
+def dumps(obj: Any, sort_keys: bool = False) -> str:
+    return yaml.dump(obj, sort_keys=sort_keys)
+
+
+def dump(obj: Any, f: SupportsWrite, sort_keys: bool = False) -> None:
+    yaml.dump(obj, f, sort_keys=sort_keys)
+
+
+def safe_load(src: str | bytes | SupportsRead) -> Any:
+    try:
+        return yaml.safe_load(src)
+    except yaml.YAMLError as e:
+        raise YAMLError() from e
+
+
+def safe_dump(
+    obj: Any,
+    sort_keys: bool = False,
+    indent: int | None = None,
+    width: int | None = None,
+) -> str:
+    return yaml.safe_dump(obj, sort_keys=sort_keys, indent=indent, width=width)
+
+
+def c_load(f: SupportsRead) -> Any:
+    return yaml.load(f, Loader=yaml.CLoader)

pkgs/serialization_util/__init__.py

@@ -1,8 +1,8 @@
+from .convert_to_snakecase import convert_dict_to_snake_case
+from .dataclasses import dict_fields as dict_fields
+from .dataclasses import iterate_fields as iterate_fields
 from .serialization_helpers import (
-    convert_dict_to_snake_case,
-    convert_to_camelcase,
-    resolve_missing_to_none,
-    serialize,
+    JsonValue,
     serialize_for_api,
     serialize_for_storage,
     serialize_for_storage_dict,
@@ -10,10 +10,10 @@ from .serialization_helpers import (
 
 __all__: list[str] = [
     "convert_dict_to_snake_case",
-    "convert_to_camelcase",
-    "resolve_missing_to_none",
-    "serialize",
     "serialize_for_api",
     "serialize_for_storage",
     "serialize_for_storage_dict",
+    "iterate_fields",
+    "dict_fields",
+    "JsonValue",
 ]

pkgs/serialization_util/_get_type_for_serialization.py

@@ -1,11 +1,9 @@
 import dataclasses
 import datetime
 import functools
-from enum import Enum
+from enum import Enum, StrEnum
 from typing import Any
 
-from pkgs.strenum_compat import StrEnum
-
 
 class SerializationType(StrEnum):
     NAMED_TUPLE = "NAMED_TUPLE"

pkgs/serialization_util/convert_to_snakecase.py

@@ -0,0 +1,27 @@
+from typing import (
+    Any,
+)
+
+from pkgs.argument_parser import camel_to_snake_case
+from pkgs.serialization import (
+    MISSING_SENTRY,
+    OpaqueKey,
+)
+
+
+def _key_convert_to_snake_case(o: Any) -> Any:
+    if isinstance(o, OpaqueKey):
+        return o
+    if isinstance(o, str):
+        return camel_to_snake_case(o)
+    return o
+
+
+def convert_dict_to_snake_case(data: Any) -> Any:
+    return {
+        _key_convert_to_snake_case(k): convert_dict_to_snake_case(v)
+        if isinstance(v, dict)
+        else v
+        for k, v in data.items()
+        if v != MISSING_SENTRY
+    }