UncountablePythonSDK 0.0.8__py3-none-any.whl → 0.0.92__py3-none-any.whl

pkgs/argument_parser/argument_parser.py

@@ -1,16 +1,24 @@
+from __future__ import annotations
+
 import dataclasses
+import math
 import types
 import typing
 from collections import defaultdict
-from dataclasses import MISSING, dataclass
 from datetime import date, datetime
 from decimal import Decimal
+from enum import Enum, auto
 from importlib import resources
 
 import dateutil.parser
-import yaml
 
-from pkgs.serialization import MissingSentryType, OpaqueKey, get_serial_class_data
+from pkgs.serialization import (
+    MissingSentryType,
+    OpaqueKey,
+    get_serial_class_data,
+    get_serial_union_data,
+    yaml,
+)
 
 from ._is_enum import is_string_enum_class
 from ._is_namedtuple import is_namedtuple_type
@@ -18,38 +26,78 @@ from .case_convert import camel_to_snake_case, snake_to_camel_case
 
 T = typing.TypeVar("T")
 ParserFunction = typing.Callable[[typing.Any], T]
-ParserCache = dict[typing.Type[typing.Any], ParserFunction[typing.Any]]
+ParserCache = dict[type[typing.Any], ParserFunction[typing.Any]]
+
 
+class SourceEncoding(Enum):
+    API = auto()
+    STORAGE = auto()
 
-@dataclass(frozen=True, eq=True)
+
+@dataclasses.dataclass(frozen=True, eq=True)
 class ParserOptions:
-    convert_to_snake_case: bool
+    encoding: SourceEncoding
+    strict_property_parsing: bool = False
+
+    @staticmethod
+    def Api(*, strict_property_parsing: bool = False) -> ParserOptions:
+        return ParserOptions(
+            encoding=SourceEncoding.API, strict_property_parsing=strict_property_parsing
+        )
+
+    @staticmethod
+    def Storage(*, strict_property_parsing: bool = False) -> ParserOptions:
+        return ParserOptions(
+            encoding=SourceEncoding.STORAGE,
+            strict_property_parsing=strict_property_parsing,
+        )
 
+    @property
+    def from_camel_case(self) -> bool:
+        return self.encoding == SourceEncoding.API
 
-@dataclass(frozen=True)
+    @property
+    def allow_direct_dataclass(self) -> bool:
+        return self.encoding == SourceEncoding.STORAGE
+
+
+@dataclasses.dataclass(frozen=True)
 class ParserContext:
     options: ParserOptions
     cache: ParserCache
 
 
+class ParserError(BaseException): ...
+
+
+class ParserExtraFieldsError(ParserError):
+    extra_fields: set[str]
+
+    def __init__(self, extra_fields: set[str]) -> None:
+        self.extra_fields = extra_fields
+
+    def __str__(self) -> str:
+        return f"extra fields were provided: {', '.join(self.extra_fields)}"
+
+
+def is_union(field_type: typing.Any) -> bool:
+    origin = typing.get_origin(field_type)
+    return origin is typing.Union or origin is types.UnionType
+
+
 def is_optional(field_type: typing.Any) -> bool:
-    return typing.get_origin(field_type) is typing.Union and type(
-        None
-    ) in typing.get_args(field_type)
+    return is_union(field_type) and type(None) in typing.get_args(field_type)
 
 
 def is_missing(field_type: typing.Any) -> bool:
-    origin = typing.get_origin(field_type)
-    if origin is not typing.Union:
+    if not is_union(field_type):
         return False
     args = typing.get_args(field_type)
-    if len(args) == 0 or args[0] is not MissingSentryType:
-        return False
-    return True
+    return not (len(args) == 0 or args[0] is not MissingSentryType)
 
 
 def _invoke_tuple_parsers(
-    tuple_type: typing.Type[T],
+    tuple_type: type[T],
     arg_parsers: typing.Sequence[typing.Callable[[typing.Any], object]],
     has_ellipsis: bool,
     value: typing.Any,
@@ -68,22 +116,25 @@ def _invoke_tuple_parsers(
 
 
 def _invoke_fallback_parsers(
-    original_type: typing.Type[T],
+    original_type: type[T],
     arg_parsers: typing.Sequence[typing.Callable[[typing.Any], T]],
     value: typing.Any,
 ) -> T:
+    exceptions = []
+
     for parser in arg_parsers:
         try:
             return parser(value)
-        except Exception:
+        except Exception as e:
+            exceptions.append(e)
             continue
     raise ValueError(
         f"Unhandled value {value} cannot be cast to a member of {original_type}"
-    )
+    ) from ExceptionGroup("Fallback Parser Exception", exceptions)
 
 
 def _invoke_membership_parser(
-    expected_values: typing.Set[T],
+    expected_values: set[T],
     value: typing.Any,
 ) -> T:
     """
@@ -97,24 +148,54 @@ def _invoke_membership_parser(
     raise ValueError(f"Expected value from {expected_values} but got value {value}")
 
 
+def _build_parser_discriminated_union(
+    context: ParserContext,
+    discriminator: str,
+    discriminator_map: dict[str, ParserFunction[T]],
+) -> ParserFunction[T]:
+    def parse(value: typing.Any) -> typing.Any:
+        if context.options.allow_direct_dataclass and dataclasses.is_dataclass(value):
+            discriminant = getattr(value, discriminator)
+        else:
+            discriminant = value.get(discriminator)
+        if discriminant is None:
+            raise ValueError("missing-union-discriminant")
+        if not isinstance(discriminant, str):
+            raise ValueError("union-discriminant-is-not-string")
+        parser = discriminator_map.get(discriminant)
+        if parser is None:
+            raise ValueError("missing-type-for-union-discriminant", discriminant)
+        return parser(value)
+
+    return parse
+
+
 def _build_parser_inner(
-    parsed_type: typing.Type[T],
+    parsed_type: type[T],
     context: ParserContext,
-    *,
-    convert_string_to_snake_case: bool = False,
 ) -> ParserFunction[T]:
     """
-    convert_to_snake_case - internal flag
-                            if convert_to_snake_case is True, and parsed_type is str,
-                            then the generated parser will convert camel to snake case case
-                            should only be True for cases like dictionary keys
-                            should only be True if options.convert_to_snake_case is True
-
-    NOTE: This argument makes caching at this level difficult, as the cache-map
-    would need to vary based on this argument. For this reason only dataclasses
-    are cached now, as they don't use the argument, and they're known to be safe.
-    This is also enough to support some recursion.
+    IMPROVE: We can now cache at this level, to avoid producing redundant
+    internal parsers.
     """
+
+    serial_union = get_serial_union_data(parsed_type)
+    if serial_union is not None:
+        discriminator = serial_union.discriminator
+        discriminator_map = serial_union.discriminator_map
+        if discriminator is None or discriminator_map is None:
+            # fallback to standard union parsing
+            parsed_type = serial_union.get_union_underlying()
+        else:
+            return _build_parser_discriminated_union(
+                context,
+                discriminator,
+                {
+                    key: _build_parser_inner(value, context)
+                    for key, value in discriminator_map.items()
+                },
+            )
+
     if dataclasses.is_dataclass(parsed_type):
         return _build_parser_dataclass(parsed_type, context)  # type: ignore[arg-type]
 
@@ -129,7 +210,7 @@ def _build_parser_inner(
             field_name: field_parser(
                 value.get(
                     snake_to_camel_case(field_name)
-                    if context.options.convert_to_snake_case
+                    if context.options.from_camel_case
                     else field_name
                 )
             )
@@ -142,7 +223,7 @@ def _build_parser_inner(
     origin = typing.get_origin(parsed_type)
     if origin is tuple:
         args = typing.get_args(parsed_type)
-        element_parsers: typing.List[typing.Callable[[typing.Any], object]] = []
+        element_parsers: list[typing.Callable[[typing.Any], object]] = []
         has_ellipsis = False
         for arg in args:
             assert not has_ellipsis
@@ -164,7 +245,7 @@ def _build_parser_inner(
         arg_parsers = [_build_parser_inner(arg, context) for arg in sorted_args]
         return lambda value: _invoke_fallback_parsers(parsed_type, arg_parsers, value)
 
-    if parsed_type is typing.Any:
+    if parsed_type is typing.Any:  # type: ignore[comparison-overlap]
         return lambda value: value
 
     if origin in (list, set):
@@ -173,10 +254,16 @@ def _build_parser_inner(
             raise ValueError("List types only support one argument")
         arg_parser = _build_parser_inner(args[0], context)
 
+        def parse_element(value: typing.Any) -> typing.Any:
+            try:
+                return arg_parser(value)
+            except Exception as e:
+                raise ValueError("Failed to parse element", value) from e
+
         def parse(value: typing.Any) -> typing.Any:
             if not isinstance(value, list):
                 raise ValueError("value is not a list", parsed_type)
-            return origin(arg_parser(x) for x in value)
+            return origin(parse_element(x) for x in value)
 
         return parse
 
@@ -184,36 +271,47 @@ def _build_parser_inner(
         args = typing.get_args(parsed_type)
         if len(args) != 2:
             raise ValueError("Dict types only support two arguments for now")
-        k_parser = _build_parser_inner(
+        k_inner_parser = _build_parser_inner(
             args[0],
             context,
-            convert_string_to_snake_case=context.options.convert_to_snake_case,
         )
+
+        def key_parser(value: typing.Any) -> object:
+            inner = k_inner_parser(value)
+            if (
+                isinstance(inner, str)
+                # enum keys and OpaqueData's would also have string value types,
+                # but their explicit type is not a string, thus shouldn't be converted
+                and args[0] is str
+                and context.options.from_camel_case
+            ):
+                return camel_to_snake_case(value)
+            return inner
+
         v_parser = _build_parser_inner(args[1], context)
-        return lambda value: origin((k_parser(k), v_parser(v)) for k, v in value.items())
+        return lambda value: origin(
+            (key_parser(k), v_parser(v)) for k, v in value.items()
+        )
 
     if origin == typing.Literal:
-        valid_values: typing.Set[T] = set(typing.get_args(parsed_type))
+        valid_values: set[T] = set(typing.get_args(parsed_type))
         return lambda value: _invoke_membership_parser(valid_values, value)
 
-    if parsed_type is str and convert_string_to_snake_case:
-        return lambda value: camel_to_snake_case(value)  # type: ignore
-
     if parsed_type is int:
         # first parse ints to decimal to allow scientific notation and decimals
         # e.g. (1) 1e4 => 1000, (2) 3.0 => 3
 
         def parse_int(value: typing.Any) -> T:
             if isinstance(value, str):
-                assert (
-                    "_" not in value
-                ), "numbers with underscores not considered integers"
+                assert "_" not in value, (
+                    "numbers with underscores not considered integers"
+                )
 
             dec_value = Decimal(value)
             int_value = int(dec_value)
-            assert (
-                int_value == dec_value
-            ), f"value ({value}) cannot be parsed to int without discarding precision"
+            assert int_value == dec_value, (
+                f"value ({value}) cannot be parsed to int without discarding precision"
+            )
             return int_value  # type: ignore
 
         return parse_int
@@ -237,7 +335,18 @@ def _build_parser_inner(
 
         return parse_str
 
-    if parsed_type in (float, dict, bool, Decimal) or is_string_enum_class(parsed_type):
+    if parsed_type in (float, Decimal):
+
+        def parse_as_numeric_type(value: typing.Any) -> T:
+            numeric_value: Decimal | float = parsed_type(value)  # type: ignore
+            if math.isnan(numeric_value):
+                raise ValueError(f"Invalid numeric value: {numeric_value}")
+
+            return numeric_value  # type: ignore
+
+        return parse_as_numeric_type
+
+    if parsed_type in (dict, bool) or is_string_enum_class(parsed_type):
         return lambda value: parsed_type(value)  # type: ignore
 
     if parsed_type is MissingSentryType:
@@ -246,11 +355,21 @@ def _build_parser_inner(
             raise ValueError("Missing type cannot be parsed directly")
 
         return error
+
+    # Check last for generic annotated types and process them unwrapped
+    # this must be last, since some of the expected types, like Unions,
+    # will also be annotated, but have a special form
+    if typing.get_origin(parsed_type) is typing.Annotated:
+        return _build_parser_inner(
+            parsed_type.__origin__,  # type: ignore[attr-defined]
+            context,
+        )
+
     raise ValueError(f"Unhandled type {parsed_type}")
 
 
 def _build_parser_dataclass(
-    parsed_type: typing.Type[T],
+    parsed_type: type[T],
     context: ParserContext,
 ) -> ParserFunction[T]:
     """
@@ -264,36 +383,45 @@ def _build_parser_dataclass(
 
     type_hints = typing.get_type_hints(parsed_type)
     dc_field_parsers: list[
-        typing.Tuple[
+        tuple[
             dataclasses.Field[typing.Any],
-            typing.Type[typing.Any],
+            type[typing.Any],
             ParserFunction[typing.Any],
         ]
     ] = []
 
     serial_class_data = get_serial_class_data(parsed_type)
 
+    def resolve_serialized_field_name(*, field_name: str) -> str:
+        return (
+            snake_to_camel_case(field_name)
+            if (
+                context.options.from_camel_case
+                and not serial_class_data.has_unconverted_key(field_name)
+            )
+            else field_name
+        )
+
     def parse(value: typing.Any) -> typing.Any:
-        data: typing.Dict[typing.Any, typing.Any] = {}
+        # Use an exact type match to prevent base/derived class mismatches
+        if context.options.allow_direct_dataclass and type(value) is parsed_type:
+            return value
+
+        data: dict[typing.Any, typing.Any] = {}
         for field, field_type, field_parser in dc_field_parsers:
             field_raw_value = None
             try:
                 field_raw_value = value.get(
-                    snake_to_camel_case(field.name)
-                    if (
-                        context.options.convert_to_snake_case
-                        and not serial_class_data.has_unconverted_key(field.name)
-                    )
-                    else field.name,
-                    MISSING,
+                    resolve_serialized_field_name(field_name=field.name),
+                    dataclasses.MISSING,
                 )
                 field_value: typing.Any
-                if field_raw_value == MISSING:
+                if field_raw_value == dataclasses.MISSING:
                     if serial_class_data.has_parse_require(field.name):
                         raise ValueError("missing-required-field", field.name)
-                    if field.default != MISSING:
+                    if field.default != dataclasses.MISSING:
                         field_value = field.default
-                    elif field.default_factory != MISSING:
+                    elif field.default_factory != dataclasses.MISSING:
                         field_value = field.default_factory()
                     elif is_missing(field_type):
                         field_value = MissingSentryType()
@@ -315,9 +443,21 @@ def _build_parser_dataclass(
 
             except Exception as e:
                 raise ValueError(
-                    f"unable to parse field:{field.name}", field_raw_value
+                    f"unable-to-parse-field:{field.name}", field_raw_value
                 ) from e
 
+        if context.options.strict_property_parsing:
+            all_allowed_field_names = set(
+                resolve_serialized_field_name(field_name=field.name)
+                for (field, _, _) in dc_field_parsers
+            )
+            passed_field_names = set(value.keys())
+            disallowed_field_names = passed_field_names.difference(
+                all_allowed_field_names
+            )
+            if len(disallowed_field_names) > 0:
+                raise ParserExtraFieldsError(disallowed_field_names)
+
         return parsed_type(**data)
 
     # Add to cache before building inner types, to support recursion
@@ -340,7 +480,7 @@ _CACHE_MAP: dict[ParserOptions, ParserCache] = defaultdict(ParserCache)
 
 
 def build_parser(
-    parsed_type: typing.Type[T],
+    parsed_type: type[T],
     options: ParserOptions,
 ) -> ParserFunction[T]:
     """
@@ -364,11 +504,27 @@ def build_parser(
 class CachedParser(typing.Generic[T]):
     def __init__(
         self,
-        args: typing.Type[T],
+        args: type[T],
+        strict_property_parsing: bool = False,
     ):
         self.arguments = args
-        self.parser_api: typing.Optional[ParserFunction[T]] = None
-        self.parser_storage: typing.Optional[ParserFunction[T]] = None
+        self.parser_api: ParserFunction[T] | None = None
+        self.parser_storage: ParserFunction[T] | None = None
+        self.strict_property_parsing = strict_property_parsing
+
+    def parse_from_encoding(
+        self,
+        args: typing.Any,
+        *,
+        source_encoding: SourceEncoding,
+    ) -> T:
+        match source_encoding:
+            case SourceEncoding.API:
+                return self.parse_api(args)
+            case SourceEncoding.STORAGE:
+                return self.parse_storage(args)
+            case _:
+                typing.assert_never(source_encoding)
 
     def parse_api(self, args: typing.Any) -> T:
         """
@@ -381,8 +537,8 @@ class CachedParser(typing.Generic[T]):
         if self.parser_api is None:
             self.parser_api = build_parser(
                 self.arguments,
-                ParserOptions(
-                    convert_to_snake_case=True,
+                ParserOptions.Api(
+                    strict_property_parsing=self.strict_property_parsing,
                 ),
             )
         assert self.parser_api is not None
@@ -395,8 +551,8 @@ class CachedParser(typing.Generic[T]):
         if self.parser_storage is None:
             self.parser_storage = build_parser(
                 self.arguments,
-                ParserOptions(
-                    convert_to_snake_case=False,
+                ParserOptions.Storage(
+                    strict_property_parsing=self.strict_property_parsing,
                 ),
             )
         assert self.parser_storage is not None
@@ -407,5 +563,5 @@ class CachedParser(typing.Generic[T]):
             return self.parse_storage(yaml.safe_load(data_in))
 
     def parse_yaml_resource(self, package: resources.Package, resource: str) -> T:
-        raw_data = resources.read_text(package, resource)
-        return self.parse_storage(yaml.safe_load(raw_data))
+        with resources.open_text(package, resource) as fp:
+            return self.parse_storage(yaml.safe_load(fp))

pkgs/argument_parser/case_convert.py

@@ -4,9 +4,10 @@ import re
 
 @functools.lru_cache(maxsize=500000)
 def snake_to_camel_case(o: str) -> str:
-    return "".join([
-        part.title() if i > 0 else part for i, part in enumerate(o.split("_"))
-    ])
+    tokens = o.split("_")
+    if len(tokens) < 2:
+        return o
+    return "".join([part.title() if i > 0 else part for i, part in enumerate(tokens)])
 
 
 def kebab_to_pascal_case(o: str) -> str:

pkgs/filesystem_utils/__init__.py

@@ -0,0 +1,20 @@
+from ._gdrive_session import GDriveSession as GDriveSession
+from ._gdrive_session import delete_gdrive_file as delete_gdrive_file
+from ._gdrive_session import download_gdrive_file as download_gdrive_file
+from ._gdrive_session import list_gdrive_files as list_gdrive_files
+from ._gdrive_session import move_gdrive_file as move_gdrive_file
+from ._gdrive_session import upload_file_gdrive as upload_file_gdrive
+from ._local_session import LocalSession as LocalSession
+from ._s3_session import S3Session as S3Session
+from ._sftp_session import SFTPSession as SFTPSession
+from ._sftp_session import list_sftp_files as list_sftp_files
+from ._sftp_session import move_sftp_files as move_sftp_files
+from .file_type_utils import FileObjectData as FileObjectData
+from .file_type_utils import FileSystemFileReference as FileSystemFileReference
+from .file_type_utils import FileSystemObject as FileSystemObject
+from .file_type_utils import FileSystemS3Config as FileSystemS3Config
+from .file_type_utils import FileSystemSFTPConfig as FileSystemSFTPConfig
+from .file_type_utils import FileTransfer as FileTransfer
+from .file_type_utils import IncompatibleFileReference as IncompatibleFileReference
+from .file_type_utils import RemoteObjectReference as RemoteObjectReference
+from .filesystem_session import FileSystemSession as FileSystemSession

pkgs/filesystem_utils/_blob_session.py

@@ -0,0 +1,137 @@
+from io import BytesIO
+
+from azure.storage.blob import BlobServiceClient, ContainerClient
+
+from pkgs.filesystem_utils.file_type_utils import (
+    FileObjectData,
+    FileSystemBlobConfig,
+    FileSystemFileReference,
+    FileSystemObject,
+    FileTransfer,
+    IncompatibleFileReference,
+)
+
+from .filesystem_session import FileSystemSession
+
+
+def _add_slash(prefix: str) -> str:
+    if len(prefix) > 0 and prefix[-1] != "/":
+        prefix = prefix + "/"
+    return prefix
+
+
+class BlobSession(FileSystemSession):
+    config: FileSystemBlobConfig
+
+    def __init__(self, blob_config: FileSystemBlobConfig) -> None:
+        super().__init__()
+        self.config = blob_config
+
+    def start(self) -> None:
+        self.service_client: BlobServiceClient | None = BlobServiceClient(
+            self.config.account_url, credential=self.config.credential
+        )
+        self.container_client: ContainerClient | None = (
+            self.service_client.get_container_client(self.config.container)
+        )
+
+    def __enter__(self) -> "BlobSession":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
+        self.service_client = None
+        self.container_client = None
+
+    def list_files(
+        self,
+        dir_path: FileSystemObject,
+        *,
+        recursive: bool = False,
+        valid_extensions: list[str] | None = None,
+    ) -> list[FileSystemObject]:
+        if not isinstance(dir_path, FileSystemFileReference):
+            raise IncompatibleFileReference()
+
+        assert self.service_client is not None and self.container_client is not None, (
+            "call to list_files on uninitialized blob session"
+        )
+
+        filesystem_file_references: list[FileSystemObject] = []
+        prefix = _add_slash(dir_path.filepath)
+        for blob in self.container_client.list_blobs(name_starts_with=prefix):
+            if not recursive and (
+                blob.name == prefix or "/" in blob.name[len(prefix) :]
+            ):
+                continue
+            if valid_extensions is None or any(
+                blob.name.endswith(valid_extension)
+                for valid_extension in valid_extensions
+            ):
+                filesystem_file_references.append(
+                    FileSystemFileReference(
+                        filepath=blob.name,
+                    )
+                )
+
+        return filesystem_file_references
+
+    def download_files(
+        self,
+        filepaths: list[FileSystemObject],
+    ) -> list[FileObjectData]:
+        downloaded_files: list[FileObjectData] = []
+        assert self.service_client is not None and self.container_client is not None, (
+            "call to download_files on uninitialized blob session"
+        )
+
+        for file_object in filepaths:
+            if (
+                not isinstance(file_object, FileSystemFileReference)
+                or file_object.filename is None
+            ):
+                raise IncompatibleFileReference()
+
+            blob_client = self.container_client.get_blob_client(file_object.filepath)
+            download_stream = blob_client.download_blob()
+            file_data = download_stream.readall()
+            downloaded_files.append(
+                FileObjectData(
+                    file_data=file_data,
+                    file_IO=BytesIO(file_data),
+                    filename=file_object.filename,
+                    filepath=file_object.filepath,
+                )
+            )
+
+        return downloaded_files
+
+    def move_files(self, file_mappings: list[FileTransfer]) -> None:
+        assert self.service_client is not None and self.container_client is not None, (
+            "call to move_files on uninitialized blob session"
+        )
+
+        for src_file, dest_file in file_mappings:
+            if not isinstance(src_file, FileSystemFileReference) or not isinstance(
+                dest_file, FileSystemFileReference
+            ):
+                raise IncompatibleFileReference()
+
+            source_blob_client = self.container_client.get_blob_client(
+                src_file.filepath
+            )
+            dest_blob_client = self.container_client.get_blob_client(dest_file.filepath)
+
+            dest_blob_client.start_copy_from_url(source_blob_client.url)
+            source_blob_client.delete_blob()
+
+    def delete_files(self, filepaths: list[FileSystemObject]) -> None:
+        assert self.service_client is not None and self.container_client is not None, (
+            "call to delete_files on uninitialized blob session"
+        )
+        for file_object in filepaths:
+            if not isinstance(file_object, FileSystemFileReference):
+                raise IncompatibleFileReference()
+
+            blob_client = self.container_client.get_blob_client(file_object.filepath)
+            blob_client.delete_blob()