UncountablePythonSDK 0.0.24__py3-none-any.whl → 0.0.131__py3-none-any.whl

examples/invoke_uploader.py

@@ -0,0 +1,26 @@
+import os
+
+from uncountable.core import AuthDetailsApiKey, Client, MediaFileUpload
+from uncountable.types import generic_upload_t
+from uncountable.types.identifier_t import IdentifierKeyId
+
+client = Client(
+    base_url="http://localhost:5000",
+    auth_details=AuthDetailsApiKey(
+        api_id=os.environ["UNC_API_ID"],
+        api_secret_key=os.environ["UNC_API_SECRET_KEY"],
+    ),
+)
+uploaded_file = client.upload_files(
+    file_uploads=[
+        MediaFileUpload(path="~/Downloads/my_file_to_upload.csv"),
+    ]
+)[0]
+
+client.invoke_uploader(
+    file_id=uploaded_file.file_id,
+    uploader_key=IdentifierKeyId(id=48),
+    destination=generic_upload_t.UploadDestinationMaterialFamily(
+        material_family_key=IdentifierKeyId(id=7)
+    ),
+)

examples/oauth.py

@@ -0,0 +1,7 @@
+from uncountable.core.client import Client
+from uncountable.core.types import AuthDetailsOAuth
+
+client = Client(
+    base_url="https://app.uncountable.com",
+    auth_details=AuthDetailsOAuth(refresh_token="x"),
+)

examples/set_recipe_metadata_file.py

@@ -0,0 +1,40 @@
+import os
+
+from uncountable.core import (
+    AsyncBatchProcessor,
+    AuthDetailsApiKey,
+    Client,
+    MediaFileUpload,
+)
+from uncountable.types import recipe_metadata_t
+from uncountable.types.identifier_t import IdentifierKeyBatchReference
+
+client = Client(
+    base_url="http://localhost:5000",
+    auth_details=AuthDetailsApiKey(
+        api_id=os.environ["UNC_API_ID"],
+        api_secret_key=os.environ["UNC_API_SECRET_KEY"],
+    ),
+)
+uploaded_file = client.upload_files(
+    file_uploads=[
+        MediaFileUpload(path="Downloads/my_file_to_upload.csv"),
+    ]
+)[0]
+
+batch_processor = AsyncBatchProcessor(client=client)
+
+recipe_batch_identifier = batch_processor.create_recipe(
+    material_family_id=1, workflow_id=1
+).batch_reference
+
+batch_processor.set_recipe_metadata(
+    recipe_key=IdentifierKeyBatchReference(reference=recipe_batch_identifier),
+    recipe_metadata=[
+        recipe_metadata_t.MetadataValue(
+            metadata_id=102, value_file_ids=[uploaded_file.file_id]
+        )
+    ],
+)
+
+batch_processor.send()

examples/set_recipe_output_file_sdk.py

@@ -0,0 +1,26 @@
+import os
+from pathlib import Path
+
+import uncountable.types.api.recipes.set_recipe_output_file as set_recipe_output_file_t
+from uncountable.core import AuthDetailsApiKey, Client, MediaFileUpload
+
+client = Client(
+    base_url="http://localhost:5000",
+    auth_details=AuthDetailsApiKey(
+        api_id=os.environ["UNC_API_ID"],
+        api_secret_key=os.environ["UNC_API_SECRET_KEY"],
+    ),
+)
+uploaded_file = client.upload_files(
+    file_uploads=[
+        MediaFileUpload(
+            path=str((Path.home() / "Downloads" / "my_file_to_upload.csv").absolute())
+        ),
+    ]
+)[0]
+
+client.set_recipe_output_file(
+    output_file_data=set_recipe_output_file_t.RecipeOutputFileValue(
+        recipe_id=58070, output_id=148, file_id=uploaded_file.file_id, experiment_num=1
+    )
+)

examples/upload_files.py

@@ -1,7 +1,7 @@
 import os
 from pprint import pprint
 
-from uncountable import AuthDetailsApiKey, Client, MediaFileUpload
+from uncountable.core import AuthDetailsApiKey, Client, MediaFileUpload
 
 client = Client(
     base_url="http://localhost:5000",
@@ -12,8 +12,7 @@ client = Client(
 )
 uploaded = client.upload_files(
     file_uploads=[
-        MediaFileUpload(path="/my/file/path.doc"),
-        MediaFileUpload(path="/my/file/path2.doc"),
+        MediaFileUpload(path="Downloads/file"),
     ]
 )
 pprint(uploaded)

pkgs/argument_parser/__init__.py

@@ -1,6 +1,15 @@
+from ._is_enum import is_string_enum_class as is_string_enum_class
 from .argument_parser import CachedParser as CachedParser
+from .argument_parser import ParserBase as ParserBase
+from .argument_parser import ParserError as ParserError
+from .argument_parser import ParserExtraFieldsError as ParserExtraFieldsError
+from .argument_parser import ParserFunction as ParserFunction
 from .argument_parser import ParserOptions as ParserOptions
+from .argument_parser import SourceEncoding as SourceEncoding
 from .argument_parser import build_parser as build_parser
+from .argument_parser import is_missing as is_missing
+from .argument_parser import is_optional as is_optional
+from .argument_parser import is_union as is_union
 from .case_convert import camel_to_snake_case as camel_to_snake_case
 from .case_convert import kebab_to_pascal_case as kebab_to_pascal_case
 from .case_convert import snake_to_camel_case as snake_to_camel_case

pkgs/argument_parser/_is_namedtuple.py

@@ -5,6 +5,9 @@ def is_namedtuple_type(x: Any) -> bool:
     if not hasattr(x, "__annotations__"):
         return False
 
+    if not hasattr(x, "__bases__"):
+        return False
+
     b = x.__bases__
     if len(b) != 1 or b[0] is not tuple:
         return False

pkgs/argument_parser/argument_parser.py

@@ -1,16 +1,26 @@
+from __future__ import annotations
+
 import dataclasses
+import datetime
+import math
 import types
 import typing
+from abc import ABC, abstractmethod
 from collections import defaultdict
-from dataclasses import MISSING, dataclass
-from datetime import date, datetime
+from datetime import date
 from decimal import Decimal
+from enum import Enum, auto
 from importlib import resources
 
 import dateutil.parser
-import yaml
+import msgspec.yaml
 
-from pkgs.serialization import MissingSentryType, OpaqueKey, get_serial_class_data
+from pkgs.serialization import (
+    MissingSentryType,
+    OpaqueKey,
+    get_serial_class_data,
+    get_serial_union_data,
+)
 
 from ._is_enum import is_string_enum_class
 from ._is_namedtuple import is_namedtuple_type
@@ -21,31 +31,91 @@ ParserFunction = typing.Callable[[typing.Any], T]
 ParserCache = dict[type[typing.Any], ParserFunction[typing.Any]]
 
 
-@dataclass(frozen=True, eq=True)
+class SourceEncoding(Enum):
+    API = auto()
+    STORAGE = auto()
+
+
+@dataclasses.dataclass(frozen=True, eq=True)
 class ParserOptions:
-    convert_to_snake_case: bool
+    encoding: SourceEncoding
+    strict_property_parsing: bool = False
 
+    @staticmethod
+    def Api(*, strict_property_parsing: bool = False) -> ParserOptions:
+        return ParserOptions(
+            encoding=SourceEncoding.API, strict_property_parsing=strict_property_parsing
+        )
+
+    @staticmethod
+    def Storage(*, strict_property_parsing: bool = False) -> ParserOptions:
+        return ParserOptions(
+            encoding=SourceEncoding.STORAGE,
+            strict_property_parsing=strict_property_parsing,
+        )
 
-@dataclass(frozen=True)
+    @property
+    def from_camel_case(self) -> bool:
+        return self.encoding == SourceEncoding.API
+
+    @property
+    def allow_direct_type(self) -> bool:
+        """This allows parsing from a DB column without having to check whether it's
+        the native format of the type, a JSON column, or a string encoding."""
+        return self.encoding == SourceEncoding.STORAGE
+
+
+@dataclasses.dataclass(frozen=True)
 class ParserContext:
     options: ParserOptions
     cache: ParserCache
 
 
+class ParserError(BaseException): ...
+
+
+class ParserExtraFieldsError(ParserError):
+    extra_fields: set[str]
+
+    def __init__(self, extra_fields: set[str]) -> None:
+        self.extra_fields = extra_fields
+
+    def __str__(self) -> str:
+        return f"extra fields were provided: {', '.join(self.extra_fields)}"
+
+
+def is_union(field_type: typing.Any) -> bool:
+    origin = typing.get_origin(field_type)
+    return origin is typing.Union or origin is types.UnionType
+
+
 def is_optional(field_type: typing.Any) -> bool:
-    return typing.get_origin(field_type) is typing.Union and type(
-        None
-    ) in typing.get_args(field_type)
+    return is_union(field_type) and type(None) in typing.get_args(field_type)
 
 
 def is_missing(field_type: typing.Any) -> bool:
-    origin = typing.get_origin(field_type)
-    if origin is not typing.Union:
+    if not is_union(field_type):
         return False
     args = typing.get_args(field_type)
     return not (len(args) == 0 or args[0] is not MissingSentryType)
 
 
+def _has_field_default(field: dataclasses.Field[typing.Any]) -> bool:
+    return (
+        field.default != dataclasses.MISSING
+        and not isinstance(field.default, MissingSentryType)
+    ) or field.default_factory != dataclasses.MISSING
+
+
+def _get_field_default(
+    field: dataclasses.Field[typing.Any],
+) -> typing.Any:
+    if field.default != dataclasses.MISSING:
+        return field.default
+    assert field.default_factory != dataclasses.MISSING
+    return field.default_factory()
+
+
 def _invoke_tuple_parsers(
     tuple_type: type[T],
     arg_parsers: typing.Sequence[typing.Callable[[typing.Any], object]],
@@ -98,26 +168,79 @@ def _invoke_membership_parser(
     raise ValueError(f"Expected value from {expected_values} but got value {value}")
 
 
+# Uses `is` to compare
+def _build_identity_parser(
+    identity_value: T,
+) -> ParserFunction[T]:
+    def parse(value: typing.Any) -> T:
+        if value is identity_value:
+            return identity_value
+        raise ValueError(
+            f"Expected value {identity_value} (type: {type(identity_value)}) but got value {value} (type: {type(value)})"
+        )
+
+    return parse
+
+
+NONE_IDENTITY_PARSER = _build_identity_parser(None)
+
+
+def _build_parser_discriminated_union(
+    context: ParserContext,
+    discriminator_raw: str,
+    discriminator_map: dict[str, ParserFunction[T]],
+) -> ParserFunction[T]:
+    discriminator = (
+        snake_to_camel_case(discriminator_raw)
+        if context.options.from_camel_case
+        else discriminator_raw
+    )
+
+    def parse(value: typing.Any) -> typing.Any:
+        if context.options.allow_direct_type and dataclasses.is_dataclass(value):
+            discriminant = getattr(value, discriminator)
+        else:
+            discriminant = value.get(discriminator)
+        if discriminant is None:
+            raise ValueError("missing-union-discriminant")
+        if not isinstance(discriminant, str):
+            raise ValueError("union-discriminant-is-not-string")
+        parser = discriminator_map.get(discriminant)
+        if parser is None:
+            raise ValueError("missing-type-for-union-discriminant", discriminant)
+        return parser(value)
+
+    return parse
+
+
 def _build_parser_inner(
     parsed_type: type[T],
     context: ParserContext,
-    *,
-    convert_string_to_snake_case: bool = False,
 ) -> ParserFunction[T]:
     """
-    convert_to_snake_case - internal flag
-                            if convert_to_snake_case is True, and parsed_type is str,
-                            then the generated parser will convert camel to snake case case
-                            should only be True for cases like dictionary keys
-                            should only be True if options.convert_to_snake_case is True
-
-    NOTE: This argument makes caching at this level difficult, as the cache-map
-    would need to vary based on this argument. For this reason only dataclasses
-    are cached now, as they don't use the argument, and they're known to be safe.
-    This is also enough to support some recursion.
+    IMPROVE: We can now cache at this level, to avoid producing redundant
+    internal parsers.
     """
+
+    serial_union = get_serial_union_data(parsed_type)
+    if serial_union is not None:
+        discriminator = serial_union.discriminator
+        discriminator_map = serial_union.discriminator_map
+        if discriminator is None or discriminator_map is None:
+            # fallback to standard union parsing
+            parsed_type = serial_union.get_union_underlying()
+        else:
+            return _build_parser_discriminated_union(
+                context,
+                discriminator,
+                {
+                    key: _build_parser_inner(value, context)
+                    for key, value in discriminator_map.items()
+                },
+            )
+
     if dataclasses.is_dataclass(parsed_type):
-        return _build_parser_dataclass(parsed_type, context)  # type: ignore[arg-type]
+        return _build_parser_dataclass(parsed_type, context)
 
     # namedtuple support
     if is_namedtuple_type(parsed_type):
@@ -130,15 +253,17 @@ def _build_parser_inner(
             field_name: field_parser(
                 value.get(
                     snake_to_camel_case(field_name)
-                    if context.options.convert_to_snake_case
+                    if context.options.from_camel_case
                     else field_name
                 )
             )
             for field_name, field_parser in field_parsers
         })
 
+    # IMPROVE: unclear why we need == here
     if parsed_type == type(None):  # noqa: E721
-        return lambda value: _invoke_membership_parser({None}, value)  # type: ignore
+        # Need to convince type checker that parsed_type is type(None)
+        return typing.cast(ParserFunction[T], NONE_IDENTITY_PARSER)
 
     origin = typing.get_origin(parsed_type)
     if origin is tuple:
@@ -191,45 +316,70 @@ def _build_parser_inner(
         args = typing.get_args(parsed_type)
         if len(args) != 2:
             raise ValueError("Dict types only support two arguments for now")
-        k_parser = _build_parser_inner(
+        k_inner_parser = _build_parser_inner(
             args[0],
             context,
-            convert_string_to_snake_case=context.options.convert_to_snake_case,
         )
+
+        def key_parser(value: typing.Any) -> object:
+            inner = k_inner_parser(value)
+            if (
+                isinstance(inner, str)
+                # enum keys and OpaqueData's would also have string value types,
+                # but their explicit type is not a string, thus shouldn't be converted
+                and args[0] is str
+                and context.options.from_camel_case
+            ):
+                return camel_to_snake_case(value)
+            return inner
+
         v_parser = _build_parser_inner(args[1], context)
-        return lambda value: origin((k_parser(k), v_parser(v)) for k, v in value.items())
+        return lambda value: origin(
+            (key_parser(k), v_parser(v)) for k, v in value.items()
+        )
 
     if origin == typing.Literal:
         valid_values: set[T] = set(typing.get_args(parsed_type))
         return lambda value: _invoke_membership_parser(valid_values, value)
 
-    if parsed_type is str and convert_string_to_snake_case:
-        return lambda value: camel_to_snake_case(value)  # type: ignore
-
     if parsed_type is int:
         # first parse ints to decimal to allow scientific notation and decimals
         # e.g. (1) 1e4 => 1000, (2) 3.0 => 3
 
         def parse_int(value: typing.Any) -> T:
             if isinstance(value, str):
-                assert (
-                    "_" not in value
-                ), "numbers with underscores not considered integers"
+                assert "_" not in value, (
+                    "numbers with underscores not considered integers"
+                )
 
             dec_value = Decimal(value)
             int_value = int(dec_value)
-            assert (
-                int_value == dec_value
-            ), f"value ({value}) cannot be parsed to int without discarding precision"
+            assert int_value == dec_value, (
+                f"value ({value}) cannot be parsed to int without discarding precision"
+            )
             return int_value  # type: ignore
 
         return parse_int
 
-    if parsed_type is datetime:
-        return lambda value: dateutil.parser.isoparse(value)  # type:ignore
+    if parsed_type is datetime.datetime:
+
+        def parse_datetime(value: typing.Any) -> T:
+            if context.options.allow_direct_type and isinstance(
+                value, datetime.datetime
+            ):
+                return value  # type: ignore
+            return dateutil.parser.isoparse(value)  # type:ignore
+
+        return parse_datetime
 
     if parsed_type is date:
-        return lambda value: date.fromisoformat(value)  # type:ignore
+
+        def parse_date(value: typing.Any) -> T:
+            if context.options.allow_direct_type and isinstance(value, date):
+                return value  # type:ignore
+            return date.fromisoformat(value)  # type:ignore
+
+        return parse_date
 
     # MyPy: It's unclear why `parsed_type in (str, OpaqueKey)` is flagged as invalid
     # Thus an or statement is used instead, which isn't flagged as invalid.
@@ -244,7 +394,18 @@ def _build_parser_inner(
 
         return parse_str
 
-    if parsed_type in (float, dict, bool, Decimal) or is_string_enum_class(parsed_type):
+    if parsed_type in (float, Decimal):
+
+        def parse_as_numeric_type(value: typing.Any) -> T:
+            numeric_value: Decimal | float = parsed_type(value)  # type: ignore
+            if math.isnan(numeric_value):
+                raise ValueError(f"Invalid numeric value: {numeric_value}")
+
+            return numeric_value  # type: ignore
+
+        return parse_as_numeric_type
+
+    if parsed_type in (dict, bool) or is_string_enum_class(parsed_type):
         return lambda value: parsed_type(value)  # type: ignore
 
     if parsed_type is MissingSentryType:
@@ -253,7 +414,17 @@ def _build_parser_inner(
             raise ValueError("Missing type cannot be parsed directly")
 
         return error
-    raise ValueError(f"Unhandled type {parsed_type}")
+
+    # Check last for generic annotated types and process them unwrapped
+    # this must be last, since some of the expected types, like Unions,
+    # will also be annotated, but have a special form
+    if typing.get_origin(parsed_type) is typing.Annotated:
+        return _build_parser_inner(
+            parsed_type.__origin__,  # type: ignore[attr-defined]
+            context,
+        )
+
+    raise ValueError(f"Unhandled type {parsed_type}/{origin}")
 
 
 def _build_parser_dataclass(
@@ -268,8 +439,7 @@ def _build_parser_dataclass(
     cur_parser = context.cache.get(parsed_type)
     if cur_parser is not None:
         return cur_parser
-
-    type_hints = typing.get_type_hints(parsed_type)
+    type_hints = typing.get_type_hints(parsed_type, include_extras=True)
     dc_field_parsers: list[
         tuple[
             dataclasses.Field[typing.Any],
@@ -280,28 +450,35 @@ def _build_parser_dataclass(
 
     serial_class_data = get_serial_class_data(parsed_type)
 
+    def resolve_serialized_field_name(*, field_name: str) -> str:
+        return (
+            snake_to_camel_case(field_name)
+            if (
+                context.options.from_camel_case
+                and not serial_class_data.has_unconverted_key(field_name)
+            )
+            else field_name
+        )
+
     def parse(value: typing.Any) -> typing.Any:
+        # Use an exact type match to prevent base/derived class mismatches
+        if context.options.allow_direct_type and type(value) is parsed_type:
+            return value
+
         data: dict[typing.Any, typing.Any] = {}
         for field, field_type, field_parser in dc_field_parsers:
             field_raw_value = None
             try:
                 field_raw_value = value.get(
-                    snake_to_camel_case(field.name)
-                    if (
-                        context.options.convert_to_snake_case
-                        and not serial_class_data.has_unconverted_key(field.name)
-                    )
-                    else field.name,
-                    MISSING,
+                    resolve_serialized_field_name(field_name=field.name),
+                    dataclasses.MISSING,
                 )
                 field_value: typing.Any
-                if field_raw_value == MISSING:
+                if field_raw_value == dataclasses.MISSING:
                     if serial_class_data.has_parse_require(field.name):
                         raise ValueError("missing-required-field", field.name)
-                    if field.default != MISSING:
-                        field_value = field.default
-                    elif field.default_factory != MISSING:
-                        field_value = field.default_factory()
+                    if _has_field_default(field):
+                        field_value = _get_field_default(field)
                     elif is_missing(field_type):
                         field_value = MissingSentryType()
                     elif is_optional(field_type):
@@ -313,6 +490,13 @@ def _build_parser_dataclass(
                         field_value = False
                     else:
                         raise ValueError("missing-value-for-field", field.name)
+                elif (
+                    field_raw_value is None
+                    and not is_optional(field_type)
+                    and _has_field_default(field)
+                    and not serial_class_data.has_parse_require(field.name)
+                ):
+                    field_value = _get_field_default(field)
                 elif serial_class_data.has_unconverted_value(field.name):
                     field_value = field_raw_value
                 else:
@@ -322,9 +506,21 @@ def _build_parser_dataclass(
 
             except Exception as e:
                 raise ValueError(
-                    f"unable to parse field:{field.name}", field_raw_value
+                    f"unable-to-parse-field:{field.name}", field_raw_value
                 ) from e
 
+        if context.options.strict_property_parsing:
+            all_allowed_field_names = set(
+                resolve_serialized_field_name(field_name=field.name)
+                for (field, _, _) in dc_field_parsers
+            )
+            passed_field_names = set(value.keys())
+            disallowed_field_names = passed_field_names.difference(
+                all_allowed_field_names
+            )
+            if len(disallowed_field_names) > 0:
+                raise ParserExtraFieldsError(disallowed_field_names)
+
         return parsed_type(**data)
 
     # Add to cache before building inner types, to support recursion
@@ -368,14 +564,47 @@ def build_parser(
     return built_parser
 
 
-class CachedParser(typing.Generic[T]):
+class ParserBase(ABC, typing.Generic[T]):
+    def parse_from_encoding(
+        self,
+        args: typing.Any,
+        *,
+        source_encoding: SourceEncoding,
+    ) -> T:
+        match source_encoding:
+            case SourceEncoding.API:
+                return self.parse_api(args)
+            case SourceEncoding.STORAGE:
+                return self.parse_storage(args)
+            case _:
+                typing.assert_never(source_encoding)
+
+    # IMPROVE: Args would be better typed as "object"
+    @abstractmethod
+    def parse_storage(self, args: typing.Any) -> T: ...
+
+    @abstractmethod
+    def parse_api(self, args: typing.Any) -> T: ...
+
+    def parse_yaml_file(self, path: str) -> T:
+        with open(path, encoding="utf-8") as data_in:
+            return self.parse_storage(msgspec.yaml.decode(data_in.read()))
+
+    def parse_yaml_resource(self, package: resources.Package, resource: str) -> T:
+        with resources.open_text(package, resource) as fp:
+            return self.parse_storage(msgspec.yaml.decode(fp.read()))
+
+
+class CachedParser(ParserBase[T], typing.Generic[T]):
     def __init__(
         self,
         args: type[T],
+        strict_property_parsing: bool = False,
     ):
         self.arguments = args
-        self.parser_api: typing.Optional[ParserFunction[T]] = None
-        self.parser_storage: typing.Optional[ParserFunction[T]] = None
+        self.parser_api: ParserFunction[T] | None = None
+        self.parser_storage: ParserFunction[T] | None = None
+        self.strict_property_parsing = strict_property_parsing
 
     def parse_api(self, args: typing.Any) -> T:
         """
@@ -388,8 +617,8 @@ class CachedParser(typing.Generic[T]):
         if self.parser_api is None:
             self.parser_api = build_parser(
                 self.arguments,
-                ParserOptions(
-                    convert_to_snake_case=True,
+                ParserOptions.Api(
+                    strict_property_parsing=self.strict_property_parsing,
                 ),
             )
         assert self.parser_api is not None
@@ -402,17 +631,9 @@ class CachedParser(typing.Generic[T]):
         if self.parser_storage is None:
             self.parser_storage = build_parser(
                 self.arguments,
-                ParserOptions(
-                    convert_to_snake_case=False,
+                ParserOptions.Storage(
+                    strict_property_parsing=self.strict_property_parsing,
                 ),
             )
         assert self.parser_storage is not None
         return self.parser_storage(args)
-
-    def parse_yaml_file(self, path: str) -> T:
-        with open(path, encoding="utf-8") as data_in:
-            return self.parse_storage(yaml.safe_load(data_in))
-
-    def parse_yaml_resource(self, package: resources.Package, resource: str) -> T:
-        with resources.open_text(package, resource) as fp:
-            return self.parse_storage(yaml.safe_load(fp))