UncountablePythonSDK 0.0.188__py3-none-any.whl → 0.0.190__py3-none-any.whl

docs/conf.py

@@ -16,7 +16,9 @@ from sphinx.application import Sphinx  # type: ignore[import-not-found]
 from sphinx.environment import BuildEnvironment  # type: ignore[import-not-found]
 
 project = "Uncountable SDK"
-copyright = f"{datetime.datetime.now(tz=datetime.UTC).date().year}, Uncountable Inc"
+project_copyright = (
+    f"{datetime.datetime.now(tz=datetime.UTC).date().year}, Uncountable Inc"
+)
 author = "Uncountable Inc"
 
 # -- General configuration ---------------------------------------------------

docs/integration_examples/add_file_to_folder.md

@@ -1,7 +1,8 @@
 # Add a File to a Folder
 
 This is a two-step workflow: first upload the file using `upload_files`, then place it in a target
-folder using `add_file_to_folder`.
+folder using `modify_file_system` with an `upsert_file` operation.
+Each call creates a new document, so a retried call adds a duplicate file.
 
 ## Step 1: Upload the file
 
@@ -34,11 +35,13 @@ file_id = upload_file_response[0].file_id
 ## Step 2: Add the file to a folder
 
 ```{code-block} python
-from uncountable.types import identifier_t
+from uncountable.types import identifier_t, modify_file_system_t
 
-response = client.add_file_to_folder(
-    file_id=file_id,
-    folder_key=identifier_t.IdentifierKeyId(id=<FOLDER_ID>),
+response = client.modify_file_system(
+    operation=modify_file_system_t.UpsertFileOperation(
+        file_id=file_id,
+        folder_key=identifier_t.IdentifierKeyId(id=<FOLDER_ID>),
+    ),
 )
 ```
 
@@ -50,8 +53,10 @@ Data(modification_made=True, result_id=4521, entity=Entity(id=892, type=<EntityT
 To add the file to the root folder instead of a specific folder:
 
 ```{code-block} python
-response = client.add_file_to_folder(
-    file_id=file_id,
-    folder_key=identifier_t.IdentifierKeyRefName(ref_name="root"),
+response = client.modify_file_system(
+    operation=modify_file_system_t.UpsertFileOperation(
+        file_id=file_id,
+        folder_key=identifier_t.IdentifierKeyRefName(ref_name="root"),
+    ),
 )
 ```

examples/integration-server/jobs/materials_auto/example_scale_connection.py

@@ -65,7 +65,7 @@ def _resolve_equipment_id(*, client: Client, equipment_name: str) -> ObjectId:
         )
     equipment_id = results[0].column_values[0]
     if not isinstance(equipment_id, int):
-        raise Exception(
+        raise TypeError(
             f"expected an integer id for equipment named {equipment_name}, found {equipment_id!r}"
         )
     return ObjectId(equipment_id)

examples/stream_dataset_measurements.py

@@ -2,9 +2,9 @@ import argparse
 import datetime
 import json
 import os
-import random
 import time
 from decimal import Decimal
+from random import SystemRandom
 
 import requests
 from uncountable.types import dataset_measurements_t, identifier_t
@@ -17,11 +17,12 @@ API_SECRET_KEY_ENV_VAR = "UNC_API_SECRET_KEY"
 MEASUREMENTS_PER_SECOND = 500
 ONE_SECOND = datetime.timedelta(seconds=1)
 VALUE_QUANTUM = Decimal("0.0001")
+RNG = SystemRandom()
 
 
 def _random_value(*, min_value: Decimal, max_value: Decimal) -> Decimal:
     span = max_value - min_value
-    sampled = (min_value + span * Decimal(str(random.random()))).quantize(VALUE_QUANTUM)
+    sampled = (min_value + span * Decimal(str(RNG.random()))).quantize(VALUE_QUANTUM)
     return max(min_value, min(max_value, sampled))
 
 

pkgs/argument_parser/__init__.py

@@ -19,6 +19,7 @@ __all__ = [
     "is_string_enum_class",
     "is_union",
     "kebab_to_pascal_case",
+    "resolve_serialized_field_name",
     "snake_to_camel_case",
 ]
 
@@ -32,6 +33,9 @@ from .case_convert import (
     kebab_to_pascal_case as kebab_to_pascal_case,
     snake_to_camel_case as snake_to_camel_case,
 )
+from .field_name_resolution import (
+    resolve_serialized_field_name as resolve_serialized_field_name,
+)
 from .parser_base import ParserBase as ParserBase
 from .parser_builder import build_parser as build_parser
 from .parser_error import (

pkgs/argument_parser/field_name_resolution.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import TypeVar, assert_never
+
+from pkgs.serialization import SerialClassDataInspector
+
+from .case_convert import snake_to_camel_case
+from .parser_options import SourceEncoding
+
+T = TypeVar("T")
+
+
+def resolve_serialized_field_name(
+    *,
+    serial_class_data: SerialClassDataInspector[T],
+    field_name: str,
+    source_encoding: SourceEncoding,
+) -> str:
+    match source_encoding:
+        case SourceEncoding.API:
+            emit_api_override = serial_class_data.get_emit_api_override(field_name)
+            if emit_api_override is not None:
+                assert not serial_class_data.has_unconverted_key(field_name), (
+                    f"serial class field {field_name!r} appears in both "
+                    "emit_api_overrides and unconverted_keys; the features cannot "
+                    "be combined"
+                )
+                return emit_api_override
+            if not serial_class_data.has_unconverted_key(field_name):
+                return snake_to_camel_case(field_name)
+            return field_name
+        case SourceEncoding.STORAGE:
+            return field_name
+        case _ as unreachable:
+            assert_never(unreachable)

pkgs/argument_parser/parser_inner.py

@@ -21,6 +21,7 @@ from pkgs.serialization import (
 )
 
 from .case_convert import camel_to_snake_case, snake_to_camel_case
+from .field_name_resolution import resolve_serialized_field_name
 from .parser_error import (
     ParserEnumError,
     ParserError,
@@ -36,7 +37,11 @@ from .type_predicates import (
     is_string_enum_class,
 )
 
+if typing.TYPE_CHECKING:
+    from _typeshed import DataclassInstance
+
 T = typing.TypeVar("T")
+DataclassT = typing.TypeVar("DataclassT", bound="DataclassInstance")
 
 
 def _has_field_default(field: dataclasses.Field[typing.Any]) -> bool:
@@ -250,6 +255,40 @@ def _build_parser_for_type_alias(
     return build_parser_inner(alias.__value__, context, merged_alias_map)
 
 
+def _build_parser_for_collection(
+    *,
+    parsed_type: type[T]
+    | typing.TypeVar
+    | typing.TypeAliasType
+    | typing.NewType
+    | types.UnionType,
+    origin: type,
+    type_args: tuple[typing.Any, ...],
+    context: ParserContext,
+    type_var_map: dict[typing.TypeVar, type],
+) -> ParserFunction[T]:
+    if len(type_args) != 1:
+        raise ParserError(
+            "List types only support one argument", expected_type=parsed_type
+        )
+    arg_parser = build_parser_inner(type_args[0], context, type_var_map)
+
+    def parse_element(value: typing.Any) -> typing.Any:
+        try:
+            return arg_parser(value)
+        except Exception as e:
+            raise ParserError("Failed to parse element", value=value) from e
+
+    def parse(value: typing.Any) -> typing.Any:
+        if not isinstance(value, list):
+            raise ParserError(
+                "value is not a list", value=value, expected_type=parsed_type
+            )
+        return origin(parse_element(x) for x in value)
+
+    return parse
+
+
 def build_parser_inner(
     parsed_type: type[T]
     | typing.TypeVar
@@ -323,7 +362,7 @@ def build_parser_inner(
             type_var_map=merged_map,
         )
 
-        return _build_parser_dataclass(origin, context, merged_map)  # type: ignore[arg-type]
+        return _build_parser_dataclass(origin, context, merged_map)  # type: ignore[return-value]
 
     # namedtuple support
     if is_namedtuple_type(parsed_type):
@@ -383,27 +422,13 @@ def build_parser_inner(
         return lambda value: value
 
     if origin in (list, set):
-        args = typing.get_args(parsed_type)
-        if len(args) != 1:
-            raise ParserError(
-                "List types only support one argument", expected_type=parsed_type
-            )
-        arg_parser = build_parser_inner(args[0], context, type_var_map)
-
-        def parse_element(value: typing.Any) -> typing.Any:
-            try:
-                return arg_parser(value)
-            except Exception as e:
-                raise ParserError("Failed to parse element", value=value) from e
-
-        def parse(value: typing.Any) -> typing.Any:
-            if not isinstance(value, list):
-                raise ParserError(
-                    "value is not a list", value=value, expected_type=parsed_type
-                )
-            return origin(parse_element(x) for x in value)
-
-        return parse
+        return _build_parser_for_collection(
+            parsed_type=parsed_type,
+            origin=origin,
+            type_args=type_args,
+            context=context,
+            type_var_map=type_var_map,
+        )
 
     if origin is dict:
         args = typing.get_args(parsed_type)
@@ -445,7 +470,7 @@ def build_parser_inner(
         # first parse ints to decimal to allow scientific notation and decimals
         # e.g. (1) 1e4 => 1000, (2) 3.0 => 3
 
-        def parse_int(value: typing.Any) -> T:
+        def parse_int(value: typing.Any) -> int:
             if isinstance(value, str):
                 assert "_" not in value, (
                     "numbers with underscores not considered integers"
@@ -456,54 +481,57 @@ def build_parser_inner(
             assert int_value == dec_value, (
                 f"value ({value}) cannot be parsed to int without discarding precision"
             )
-            return int_value  # type: ignore
+            return int_value
 
-        return parse_int
+        return parse_int  # type: ignore[return-value]
 
     if parsed_type is datetime.datetime:
 
-        def parse_datetime(value: typing.Any) -> T:
+        def parse_datetime(value: typing.Any) -> datetime.datetime:
             if context.options.allow_direct_type and isinstance(
                 value, datetime.datetime
             ):
-                return value  # type: ignore
-            return dateutil.parser.isoparse(value)  # type:ignore
+                return value
+            return dateutil.parser.isoparse(value)
 
-        return parse_datetime
+        return parse_datetime  # type: ignore[return-value]
 
     if parsed_type is date:
 
-        def parse_date(value: typing.Any) -> T:
+        def parse_date(value: typing.Any) -> date:
             if context.options.allow_direct_type and isinstance(value, date):
                 if isinstance(value, datetime.datetime):
                     assert value.time() == datetime.time.min
-                    return value.date()  # type:ignore
-                return value  # type:ignore
-            return date.fromisoformat(value)  # type:ignore
+                    return value.date()
+                return value
+            return date.fromisoformat(value)
 
-        return parse_date
+        return parse_date  # type: ignore[return-value]
 
     # MyPy: It's unclear why `parsed_type in (str, OpaqueKey)` is flagged as invalid
     # Thus an or statement is used instead, which isn't flagged as invalid.
     if parsed_type is str or parsed_type is OpaqueKey:
 
-        def parse_str(value: typing.Any) -> T:
+        def parse_str(value: typing.Any) -> str:
             if isinstance(value, str):
-                return value  # type: ignore
-            if isinstance(value, (float, int)):
-                return str(value)  # type: ignore
+                return value
+            if isinstance(value, (Decimal, float, int)):
+                return str(value)
             raise ParserError(
                 f"Invalid string value: {type(value)}: {value}",
                 value=value,
                 expected_type=str,
             )
 
-        return parse_str
+        return parse_str  # type: ignore[return-value]
 
     if parsed_type in (float, Decimal):
+        numeric_type: type[Decimal | float] = (
+            Decimal if parsed_type is Decimal else float
+        )
 
-        def parse_as_numeric_type(value: typing.Any) -> T:
-            numeric_value: Decimal | float = parsed_type(value)  # type: ignore
+        def parse_as_numeric_type(value: typing.Any) -> Decimal | float:
+            numeric_value = numeric_type(value)
             if math.isnan(numeric_value):
                 raise ParserError(
                     f"Invalid numeric value: {numeric_value}",
@@ -511,12 +539,12 @@ def build_parser_inner(
                     expected_type=parsed_type,
                 )
 
-            return numeric_value  # type: ignore
+            return numeric_value
 
-        return parse_as_numeric_type
+        return parse_as_numeric_type  # type: ignore[return-value]
 
     if parsed_type in (dict, bool):
-        return lambda value: parsed_type(value)  # type: ignore
+        return lambda value: parsed_type(value)  # type: ignore[call-arg]  # ty: ignore[too-many-positional-arguments]
 
     if is_string_enum_class(parsed_type):
         assert issubclass(parsed_type, Enum)
@@ -550,7 +578,7 @@ def build_parser_inner(
     # will also be annotated, but have a special form
     if typing.get_origin(parsed_type) is typing.Annotated:
         return build_parser_inner(
-            parsed_type.__origin__,  # type: ignore[attr-defined]
+            typing.get_args(parsed_type)[0],
             context,
             type_var_map,
         )
@@ -562,10 +590,10 @@ def build_parser_inner(
 
 # Take in map of parameter name to materialized type
 def _build_parser_dataclass(
-    parsed_type: type[T],
+    parsed_type: type[DataclassT],
     context: ParserContext,
     type_var_map: dict[typing.TypeVar, type],
-) -> ParserFunction[T]:
+) -> ParserFunction[DataclassT]:
     """
     Use the cache so that recursion involve dataclasses is supported. This
     requires the build order is a bit inverted: the dataclass parser is added
@@ -609,33 +637,27 @@ def _build_parser_dataclass(
 
     serial_class_data = get_serial_class_data(parsed_type)
 
-    def resolve_serialized_field_name(*, field_name: str) -> str:
-        if context.options.is_api_encoding:
-            emit_api_override = serial_class_data.get_emit_api_override(field_name)
-            if emit_api_override is not None:
-                assert not serial_class_data.has_unconverted_key(field_name), (
-                    f"serial class field {field_name!r} appears in both "
-                    "emit_api_overrides and unconverted_keys; the features cannot "
-                    "be combined"
-                )
-                return emit_api_override
-            if not serial_class_data.has_unconverted_key(field_name):
-                return snake_to_camel_case(field_name)
-        return field_name
-
     def parse(value: typing.Any) -> typing.Any:
         # Use an exact type match to prevent base/derived class mismatches
         if context.options.allow_direct_type and type(value) is parsed_type:
             return value
 
         data: dict[typing.Any, typing.Any] = {}
+        all_allowed_field_names: set[str] = set()
         for field, field_type, field_parser in dc_field_parsers:
             field_raw_value = None
             try:
+                serialized_field_name = resolve_serialized_field_name(
+                    serial_class_data=serial_class_data,
+                    field_name=field.name,
+                    source_encoding=context.options.encoding,
+                )
                 field_raw_value = value.get(
-                    resolve_serialized_field_name(field_name=field.name),
+                    serialized_field_name,
                     dataclasses.MISSING,
                 )
+                if context.options.strict_property_parsing:
+                    all_allowed_field_names.add(serialized_field_name)
                 field_value: typing.Any
                 if field_raw_value == dataclasses.MISSING:
                     if serial_class_data.has_parse_require(field.name):
@@ -679,10 +701,6 @@ def _build_parser_dataclass(
                 ) from e
 
         if context.options.strict_property_parsing:
-            all_allowed_field_names = set(
-                resolve_serialized_field_name(field_name=field.name)
-                for (field, _, _) in dc_field_parsers
-            )
             passed_field_names = set(value.keys())
             disallowed_field_names = passed_field_names.difference(
                 all_allowed_field_names
@@ -697,7 +715,7 @@ def _build_parser_dataclass(
     context.cache.put(cache_key, parser_function)
 
     dc_field_parsers = []
-    for field in dataclasses.fields(parsed_type):  # type:ignore[arg-type]
+    for field in dataclasses.fields(parsed_type):
         field_type_hint = type_hints[field.name]
         if isinstance(field_type_hint, typing.TypeVar):
             field_type_hint = type_var_map[field_type_hint]

pkgs/filesystem_utils/_gdrive_session.py

@@ -187,7 +187,7 @@ class GDriveSession(FileSystemSession):
         self.service_account_json_path = service_account_json_path
 
     def start(self) -> None:
-        credentials = service_account.Credentials.from_service_account_file(  # type: ignore[no-untyped-call]
+        credentials = service_account.Credentials.from_service_account_file(
             self.service_account_json_path
         )
         gdrive_connection = build_gdrive_connection(

pkgs/filesystem_utils/_sftp_connection.py

@@ -21,6 +21,8 @@ def _open_ssh_and_sftp(
     if private_key is None and password is None:
         raise SFTPCredentialError("Must specify either a private key or a password.")
     ssh = paramiko.SSHClient()
+    # ruff: ignore[ssh-no-host-key-verification]
+    # The platform does not hold host keys for dynamic customer SFTP endpoints.
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
     try:
         if private_key is not None:

pkgs/filesystem_utils/file_type_utils.py

@@ -1,7 +1,6 @@
 import os
 from dataclasses import dataclass
 from io import BytesIO
-from typing import Union
 
 import paramiko
 from azure.core.credentials import (
@@ -46,7 +45,7 @@ class RemoteObjectReference:
         return "folder" in self.mime_type
 
 
-FileSystemObject = Union[FileSystemFileReference, RemoteObjectReference, FileObjectData]
+FileSystemObject = FileSystemFileReference | RemoteObjectReference | FileObjectData
 
 
 FileTransfer = tuple[FileSystemObject, FileSystemObject]

pkgs/serialization/__init__.py

@@ -20,7 +20,10 @@ from .serial_class import (
 )
 from .serial_generic import get_serial_data as get_serial_data
 from .serial_union import (
+    IdentityHashWrapper as IdentityHashWrapper,
     get_serial_union_data as get_serial_union_data,
+    get_serial_union_discriminator_map as get_serial_union_discriminator_map,
+    rebuild_serial_union_metadata as rebuild_serial_union_metadata,
     serial_union_annotation as serial_union_annotation,
 )
 from .unknown import UnknownBrand as UnknownBrand

pkgs/serialization/missing_sentry.py

@@ -1,4 +1,4 @@
-from typing import Optional, TypeVar, Union
+from typing import Optional, TypeAlias, TypeVar
 
 ClassT = TypeVar("ClassT")
 
@@ -23,7 +23,7 @@ class MissingSentryType:
 MISSING_SENTRY = MissingSentryType()
 
 # The MissingSentryType is expected to come first for serialization parsing
-MissingType = Union[MissingSentryType, ClassT]
+MissingType: TypeAlias = MissingSentryType | ClassT
 
 
 def coalesce_missing_sentry(value: MissingType[ClassT]) -> ClassT | None:

pkgs/serialization/serial_union.py

@@ -55,6 +55,26 @@ def serial_union_annotation(
     )
 
 
+def rebuild_serial_union_metadata(
+    serial_union_metadata: SerialBase,
+    *,
+    discriminator_map: IdentityHashWrapper[dict[str, type]],
+) -> SerialBase:
+    assert isinstance(serial_union_metadata, _SerialUnion)
+    return dataclasses.replace(
+        serial_union_metadata,
+        discriminator_map=discriminator_map,
+    )
+
+
+def get_serial_union_discriminator_map(
+    metadata: object,
+) -> IdentityHashWrapper[dict[str, type]] | None:
+    if not isinstance(metadata, _SerialUnion):
+        return None
+    return metadata.discriminator_map
+
+
 def _get_serial_union(parsed_type: SerialType[T]) -> _SerialUnion | None:
     serial = get_serial_annotation(parsed_type)
     if not isinstance(serial, _SerialUnion):

pkgs/serialization/yaml.py

@@ -1,4 +1,6 @@
+from collections.abc import Iterable
 from decimal import Decimal
+from enum import StrEnum
 from typing import TYPE_CHECKING, Any
 
 import yaml
@@ -29,12 +31,45 @@ class YAMLError(BaseException):
     pass
 
 
+def _represent_str_enum(
+    dumper: yaml.representer.SafeRepresenter, value: StrEnum
+) -> yaml.ScalarNode:
+    return dumper.represent_str(value.value)
+
+
+def _represent_decimal(
+    dumper: yaml.representer.SafeRepresenter, value: Decimal
+) -> yaml.ScalarNode:
+    return dumper.represent_str(str(value))
+
+
+class _StorageFormatDumper(yaml.Dumper):
+    pass
+
+
+class _StorageFormatSafeDumper(yaml.SafeDumper):
+    pass
+
+
+_StorageFormatDumper.add_multi_representer(StrEnum, _represent_str_enum)
+_StorageFormatSafeDumper.add_multi_representer(StrEnum, _represent_str_enum)
+_StorageFormatSafeDumper.add_representer(Decimal, _represent_decimal)
+
+
 def dumps(obj: Any, sort_keys: bool = False) -> str:
-    return yaml.dump(obj, sort_keys=sort_keys)
+    return yaml.dump(obj, Dumper=_StorageFormatDumper, sort_keys=sort_keys)
 
 
-def dump(obj: Any, f: SupportsWrite, sort_keys: bool = False) -> None:
-    yaml.dump(obj, f, sort_keys=sort_keys)
+def dump(
+    obj: Any, f: SupportsWrite, sort_keys: bool = False, *, allow_unicode: bool = False
+) -> None:
+    yaml.dump(
+        obj,
+        f,
+        Dumper=_StorageFormatDumper,
+        sort_keys=sort_keys,
+        allow_unicode=allow_unicode,
+    )
 
 
 def safe_load(src: str | bytes | SupportsRead) -> Any:
@@ -44,10 +79,26 @@ def safe_load(src: str | bytes | SupportsRead) -> Any:
         raise YAMLError() from e
 
 
+def safe_load_all(src: str | bytes | SupportsRead) -> list[object]:
+    return list(yaml.load_all(src, Loader=yaml.CSafeLoader))
+
+
+def safe_dump_all(documents: Iterable[object], sort_keys: bool = False) -> str:
+    return yaml.dump_all(
+        documents, Dumper=_StorageFormatSafeDumper, sort_keys=sort_keys
+    )
+
+
 def safe_dump(
     obj: Any,
     sort_keys: bool = False,
     indent: int | None = None,
     width: int | None = None,
 ) -> str:
-    return yaml.safe_dump(obj, sort_keys=sort_keys, indent=indent, width=width)
+    return yaml.dump(
+        obj,
+        Dumper=_StorageFormatSafeDumper,
+        sort_keys=sort_keys,
+        indent=indent,
+        width=width,
+    )

pkgs/serialization_util/__init__.py

@@ -9,11 +9,15 @@ from .serialization_helpers import (
     serialize_for_storage,
     serialize_for_storage_dict,
 )
+from .serialize_for_api_encoding import serialize_for_api_encoding_reusing_input
+from .source_encoding_serializer import serialize_for_source_encoding
 
 __all__: list[str] = [
     "convert_decimals_to_strings",
     "convert_dict_to_snake_case",
     "serialize_for_api",
+    "serialize_for_api_encoding_reusing_input",
+    "serialize_for_source_encoding",
     "serialize_for_storage",
     "serialize_for_storage_dict",
     "iterate_fields",