UncountablePythonSDK 0.0.115__py3-none-any.whl → 0.0.142.dev0__py3-none-any.whl

examples/integration-server/jobs/materials_auto/example_predictions.py

@@ -0,0 +1,61 @@
+import random
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from decimal import Decimal
+
+from uncountable.integration.job import JobArguments, WebhookJob, register_job
+from uncountable.types import (
+    base_t,
+    identifier_t,
+    job_definition_t,
+    recipe_links_t,
+    set_recipe_outputs_t,
+)
+
+
+@dataclass(kw_only=True)
+class PredictionsPayload:
+    output_id: base_t.ObjectId
+    recipe_ids: list[base_t.ObjectId]
+
+
+@register_job
+class PredictionsExample(WebhookJob[PredictionsPayload]):
+    def run(
+        self, args: JobArguments, payload: PredictionsPayload
+    ) -> job_definition_t.JobResult:
+        recipe_data = args.client.get_recipes_data(recipe_ids=payload.recipe_ids)
+        formatted_datetime = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S")
+
+        for recipe in recipe_data.recipes:
+            test_sample_name = f"Predictions Model ({formatted_datetime})"
+            created_recipe_id = args.client.create_recipe(
+                name=test_sample_name,
+                material_family_id=1,
+                workflow_id=1,
+                definition_key=identifier_t.IdentifierKeyRefName(
+                    ref_name="unc_test_sample"
+                ),
+            ).result_id
+            args.client.set_recipe_outputs(
+                output_data=[
+                    set_recipe_outputs_t.RecipeOutputValue(
+                        recipe_id=created_recipe_id,
+                        output_id=payload.output_id,
+                        experiment_num=1,
+                        value_numeric=Decimal(random.random() * 10),
+                    )
+                ]
+            )
+            args.client.create_recipe_link(
+                recipe_from_key=identifier_t.IdentifierKeyId(id=recipe.recipe_id),
+                recipe_to_key=identifier_t.IdentifierKeyId(id=created_recipe_id),
+                link_type=recipe_links_t.RecipeLinkType.CHILD,
+                name=test_sample_name,
+            )
+
+        return job_definition_t.JobResult(success=True)
+
+    @property
+    def webhook_payload_type(self) -> type:
+        return PredictionsPayload

examples/integration-server/jobs/materials_auto/example_runsheet_wh.py

@@ -1,8 +1,19 @@
 from io import BytesIO
 
+from openpyxl import Workbook, load_workbook
 from uncountable.core.file_upload import DataFileUpload, FileUpload
 from uncountable.integration.job import JobArguments, RunsheetWebhookJob, register_job
-from uncountable.types import entity_t
+from uncountable.types import (
+    download_file_t,
+    entity_t,
+    export_default_runsheet_t,
+    identifier_t,
+    webhook_job_t,
+)
+from uncountable.types.client_base import APIRequest
+
+RUNSHEET_REF_NAME = "recipe_export_runsheet"
+RUNSHEET_REF_NAME_2 = "recipe_export_runsheet_2"
 
 
 @register_job
@@ -11,25 +22,55 @@ class StandardRunsheetGenerator(RunsheetWebhookJob):
         self,
         *,
         args: JobArguments,
-        entities: list[entity_t.Entity],
+        payload: webhook_job_t.RunsheetWebhookPayload,
     ) -> FileUpload:
-        args.logger.log_info(f"Generating runsheet for {len(entities)} entities")
+        args.logger.log_info("Exporting default runsheets")
+
+        entity_identifiers: list[identifier_t.IdentifierKey] = [
+            identifier_t.IdentifierKeyId(id=entity.id) for entity in payload.entities
+        ]
+
+        combined_wb = Workbook()
+        combined_sheet = combined_wb.active or combined_wb.create_sheet(
+            title="Combined Runsheet"
+        )
+        combined_sheet.title = "Combined Runsheet"
+
+        for ref_name in [RUNSHEET_REF_NAME, RUNSHEET_REF_NAME_2]:
+            api_request = APIRequest(
+                method=export_default_runsheet_t.ENDPOINT_METHOD,
+                endpoint=export_default_runsheet_t.ENDPOINT_PATH,
+                args=export_default_runsheet_t.Arguments(
+                    entities=entity_identifiers,
+                    runsheet_key=identifier_t.IdentifierKeyRefName(ref_name=ref_name),
+                    entity_type=payload.entities[0].type
+                    if payload.entities
+                    else entity_t.EntityType.RECIPE,
+                ),
+            )
 
-        content = []
-        content.append("STANDARD LAB RUNSHEET\n")
-        content.append("=" * 30 + "\n\n")
+            response = args.client.do_request(
+                api_request=api_request,
+                return_type=export_default_runsheet_t.Data,
+            )
 
-        for entity in entities:
-            content.append(f"Type: {entity.type}\n")
-            content.append(f"ID: {entity.id}\n")
+            file_query = download_file_t.FileDownloadQueryTextDocumentId(
+                text_document_id=response.text_document_id,
+            )
 
-            if hasattr(entity, "field_values") and entity.field_values:
-                content.append("Field Values:\n")
-                for field in entity.field_values:
-                    content.append(f"  - {field.name}: {field.value}\n")
+            downloaded_files = args.client.download_files(file_query=file_query)
+            file_data = downloaded_files[0].data.read()
 
-            content.append("\n")
+            wb = load_workbook(filename=BytesIO(file_data))
+            for sheet_name in wb.sheetnames:
+                for row in wb[sheet_name].iter_rows(values_only=True):
+                    combined_sheet.append(row)
 
-        runsheet_data = "".join(content).encode("utf-8")
+        output = BytesIO()
+        combined_wb.save(output)
+        output.seek(0)
 
-        return DataFileUpload(name="lab_runsheet.txt", data=BytesIO(runsheet_data))
+        return DataFileUpload(
+            data=output,
+            name="combined_runsheet.xlsx",
+        )

examples/integration-server/jobs/materials_auto/profile.yaml

@@ -75,3 +75,30 @@ jobs:
     executor:
       type: script
       import_path: example_runsheet_wh
+  - id: example_instrument
+    type: webhook
+    name: Webhook Instrument Connection
+    signature_key_secret:
+      type: env
+      env_key: WH_INSTRUMENT_SIGNATURE_KEY
+    executor:
+      type: script
+      import_path: example_instrument
+  - id: example_predictions
+    type: webhook
+    name: Webook Predictions
+    signature_key_secret:
+      type: env
+      env_key: WH_PREDICTIONS_SIGNATURE_KEY
+    executor:
+      type: script
+      import_path: example_predictions
+  - id: example_parse
+    type: webhook
+    name: Webhook Parse
+    signature_key_secret:
+      type: env
+      env_key: WH_PARSE_SIGNATURE_KEY
+    executor:
+      type: script
+      import_path: example_parse

examples/integration-server/pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
   "ruff == 0.*",
   "openpyxl == 3.*",
   "more_itertools == 10.*",
-  "types-paramiko ==3.5.0.20240918",
+  "types-paramiko ==4.0.0.20250822",
   "types-openpyxl == 3.*",
   "types-pysftp == 0.*",
   "types-pytz ==2025.*",
@@ -17,7 +17,8 @@ dependencies = [
   "types-simplejson == 3.*",
   "pandas-stubs",
   "xlrd == 2.*",
-  "msgspec == 0.19.*"
+  "msgspec == 0.19.*",
+  "websockets==15.0.1",
 ]
 
 [tool.mypy]
@@ -114,7 +115,6 @@ lint.ignore = [
     "PD010", # .pivottable.  Should add
     "PD011", # use .to_numpy.  Skip
     "PD015", # use .merge.  Should add
-    "PD901", # avoid generic df name.  Skip
     "PERF203", # avoid try except in loop. Skip
     "PERF401", # use list comprehension. Skip
     "PERF402", # use list.copy.  Skip
@@ -194,7 +194,6 @@ lint.ignore = [
     "RUF022",  # __all__ is not sorted.  skip due to isort complication
     "UP017", # use datetime.UTC, TODO add back in
     "UP035", # replacing List with list, TODO add back in
-    "UP038", # isinstance X | Y instead of (X, Y), TODO add back in
     # ## FROM RUFF UPGRADE
     "PLC2701",  # private name imports.  should add
     "PLR1702",  # too many nested blocks -- add with config.  skip
@@ -213,6 +212,7 @@ exclude = [
 
 [tool.ruff.lint.isort]
 split-on-trailing-comma = true
+known-first-party = ["pkgs"]
 
 [tool.ruff.lint.mccabe]
 max-complexity = 130 # goal would be to bring this down to ~50 or so

examples/oauth.py

@@ -0,0 +1,7 @@
+from uncountable.core.client import Client
+from uncountable.core.types import AuthDetailsOAuth
+
+client = Client(
+    base_url="https://app.uncountable.com",
+    auth_details=AuthDetailsOAuth(refresh_token="x"),
+)

pkgs/argument_parser/__init__.py

@@ -7,6 +7,7 @@ from .argument_parser import ParserFunction as ParserFunction
 from .argument_parser import ParserOptions as ParserOptions
 from .argument_parser import SourceEncoding as SourceEncoding
 from .argument_parser import build_parser as build_parser
+from .argument_parser import is_missing as is_missing
 from .argument_parser import is_optional as is_optional
 from .argument_parser import is_union as is_union
 from .case_convert import camel_to_snake_case as camel_to_snake_case

pkgs/argument_parser/_is_namedtuple.py

@@ -5,6 +5,9 @@ def is_namedtuple_type(x: Any) -> bool:
     if not hasattr(x, "__annotations__"):
         return False
 
+    if not hasattr(x, "__bases__"):
+        return False
+
     b = x.__bases__
     if len(b) != 1 or b[0] is not tuple:
         return False

pkgs/argument_parser/argument_parser.py

@@ -168,6 +168,23 @@ def _invoke_membership_parser(
     raise ValueError(f"Expected value from {expected_values} but got value {value}")
 
 
+# Uses `is` to compare
+def _build_identity_parser(
+    identity_value: T,
+) -> ParserFunction[T]:
+    def parse(value: typing.Any) -> T:
+        if value is identity_value:
+            return identity_value
+        raise ValueError(
+            f"Expected value {identity_value} (type: {type(identity_value)}) but got value {value} (type: {type(value)})"
+        )
+
+    return parse
+
+
+NONE_IDENTITY_PARSER = _build_identity_parser(None)
+
+
 def _build_parser_discriminated_union(
     context: ParserContext,
     discriminator_raw: str,
@@ -223,7 +240,7 @@ def _build_parser_inner(
             )
 
     if dataclasses.is_dataclass(parsed_type):
-        return _build_parser_dataclass(parsed_type, context)  # type: ignore[arg-type]
+        return _build_parser_dataclass(parsed_type, context)
 
     # namedtuple support
     if is_namedtuple_type(parsed_type):
@@ -243,8 +260,10 @@ def _build_parser_inner(
             for field_name, field_parser in field_parsers
         })
 
+    # IMPROVE: unclear why we need == here
     if parsed_type == type(None):  # noqa: E721
-        return lambda value: _invoke_membership_parser({None}, value)  # type: ignore
+        # Need to convince type checker that parsed_type is type(None)
+        return typing.cast(ParserFunction[T], NONE_IDENTITY_PARSER)
 
     origin = typing.get_origin(parsed_type)
     if origin is tuple:
@@ -271,7 +290,7 @@ def _build_parser_inner(
         arg_parsers = [_build_parser_inner(arg, context) for arg in sorted_args]
         return lambda value: _invoke_fallback_parsers(parsed_type, arg_parsers, value)
 
-    if parsed_type is typing.Any:  # type: ignore[comparison-overlap]
+    if parsed_type is typing.Any:
         return lambda value: value
 
     if origin in (list, set):

pkgs/serialization_util/serialization_helpers.py

@@ -85,7 +85,9 @@ def _serialize_dataclass(d: Any) -> dict[str, JsonValue]:
 
 
 def _to_string_value(value: Any) -> str:
-    assert isinstance(value, (Decimal, int))
+    assert isinstance(value, (Decimal, int)), (
+        f"Expecting decimal or int, received: {value} (type={type(value)})"
+    )
     return str(value)
 
 

pkgs/type_spec/builder.py

@@ -13,14 +13,25 @@ from enum import Enum, StrEnum, auto
 from typing import Any, Self
 
 from . import util
-from .cross_output_links import CrossOutputPaths
+from .builder_types import CrossOutputPaths
 from .non_discriminated_union_exceptions import NON_DISCRIMINATED_UNION_EXCEPTIONS
-from .util import parse_type_str, unused
+from .util import parse_type_str
 
 RawDict = dict[Any, Any]
 EndpointKey = str
 
 
+class PathMapping(StrEnum):
+    NO_MAPPING = "no_mapping"
+    DEFAULT_MAPPING = "default_mapping"
+
+
+@dataclass(kw_only=True)
+class APIEndpointInfo:
+    root_path: str
+    path_mapping: PathMapping
+
+
 class StabilityLevel(StrEnum):
     """These are currently used for open api,
     see: https://github.com/Tufin/oasdiff/blob/main/docs/STABILITY.md
@@ -297,6 +308,7 @@ class SpecTypeDefn(SpecType):
         self._is_value_to_string = False
         self._is_valid_parameter = True
         self._is_dynamic_allowed = False
+        self._default_extant: PropertyExtant | None = None
         self.ext_info: Any = None
 
     def is_value_converted(self) -> bool:
@@ -329,6 +341,7 @@ class SpecTypeDefn(SpecType):
                 "ext_info",
                 "label",
                 "is_dynamic_allowed",
+                "default_extant",
             ]
             + extra_names,
         )
@@ -340,6 +353,10 @@ class SpecTypeDefn(SpecType):
         assert isinstance(is_dynamic_allowed, bool)
         self._is_dynamic_allowed = is_dynamic_allowed
 
+        default_extant = data.get("default_extant")
+        if default_extant is not None:
+            self._default_extant = PropertyExtant(default_extant)
+
     def _process_property(
         self, builder: SpecBuilder, spec_name: str, data: RawDict
     ) -> SpecProperty:
@@ -358,18 +375,18 @@ class SpecTypeDefn(SpecType):
             ],
         )
         try:
-            extant_type = data.get("extant")
+            extant_type_str = data.get("extant")
+            extant_type = (
+                PropertyExtant(extant_type_str) if extant_type_str is not None else None
+            )
+            extant = extant_type or self._default_extant
             if spec_name.endswith("?"):
-                if extant_type is not None:
+                if extant is not None:
                     raise Exception("cannot specify extant with ?")
                 extant = PropertyExtant.optional
                 name = spec_name[:-1]
             else:
-                extant = (
-                    PropertyExtant.required
-                    if extant_type is None
-                    else PropertyExtant(extant_type)
-                )
+                extant = extant or PropertyExtant.required
                 name = spec_name
 
             property_name_case = self.name_case
@@ -406,7 +423,16 @@ class SpecTypeDefn(SpecType):
             parse_require = False
             literal = unwrap_literal_type(ptype)
             if literal is not None:
-                default = literal.value
+                if isinstance(
+                    literal.value_type, SpecTypeDefnStringEnum
+                ) and isinstance(literal.value, str):
+                    resolved_value = literal.value_type.values.get(literal.value)
+                    assert resolved_value is not None, (
+                        f"Value {literal.value} not found in enum"
+                    )
+                    default = resolved_value.value
+                else:
+                    default = literal.value
                 has_default = True
                 parse_require = True
 
@@ -734,6 +760,8 @@ class SpecTypeDefnStringEnum(SpecTypeDefn):
                     builder.ensure(
                         isinstance(enum_value, str), "enum value should be string"
                     )
+                    assert isinstance(enum_value, str)
+
                     deprecated = value.get("deprecated", False)
                     builder.ensure(
                         isinstance(deprecated, bool),
@@ -835,7 +863,7 @@ class _EndpointPathDetails:
 
 
 def _resolve_endpoint_path(
-    path: str, api_endpoints: dict[EndpointKey, str]
+    path: str, api_endpoints: dict[EndpointKey, APIEndpointInfo]
 ) -> _EndpointPathDetails:
     root_path_source = path.split("/")[0]
     root_match = RE_ENDPOINT_ROOT.fullmatch(root_path_source)
@@ -843,7 +871,7 @@ def _resolve_endpoint_path(
         raise Exception(f"invalid-api-path-root:{root_path_source}")
 
     root_var = root_match.group(1)
-    root_path = api_endpoints[root_var]
+    root_path = api_endpoints[root_var].root_path
 
     _, *rest_path = path.split("/", 1)
     resolved_path = "/".join([root_path] + rest_path)
@@ -911,6 +939,7 @@ class SpecEndpoint:
     stability_level: StabilityLevel | None
     # Don't emit TypeScript endpoint code
     suppress_ts: bool
+    deprecated: bool = False
     async_batch_path: str | None = None
     result_type: ResultType = ResultType.json
     has_attachment: bool = False
@@ -928,13 +957,13 @@ class SpecEndpoint:
         pass
 
     def process(self, builder: SpecBuilder, data: RawDict) -> None:
-        unused(builder)
         util.check_fields(
             data,
             [
                 "method",
                 "path",
                 "data_loader",
+                "deprecated",
                 "is_sdk",
                 "stability_level",
                 "async_batch_path",
@@ -954,6 +983,7 @@ class SpecEndpoint:
         data_loader = data.get("data_loader", False)
         assert isinstance(data_loader, bool)
         self.data_loader = data_loader
+        self.deprecated = data.get("deprecated", False)
 
         is_sdk = data.get("is_sdk", EndpointEmitType.EMIT_NOTHING)
 
@@ -1080,7 +1110,7 @@ def _parse_const(
         elif const_type.defn_type.name == BaseTypeName.s_dict:
             assert isinstance(value, dict)
             builder.ensure(
-                len(const_type.parameters) == 2, "constant-dict-expects-one-type"
+                len(const_type.parameters) == 2, "constant-dict-expects-two-types"
             )
             key_type = const_type.parameters[0]
             value_type = const_type.parameters[1]
@@ -1129,6 +1159,11 @@ def _parse_const(
             )
             return value
 
+        if not const_type.is_base:
+            # IMPROVE: validate the object type properties before emission stage
+            builder.ensure(isinstance(value, dict), "invalid value for object constant")
+            return value
+
     raise Exception("unsupported-const-scalar-type", const_type)
 
 
@@ -1250,7 +1285,8 @@ class SpecNamespace:
 
             assert util.is_valid_type_name(name), f"{name} is not a valid type name"
             assert name not in self.types, f"{name} is duplicate"
-            defn_type = defn["type"]
+            defn_type = defn.get("type")
+            assert isinstance(defn_type, str), f"{name} requires a string type"
             spec_type: SpecTypeDefn
             if defn_type == DefnTypeName.s_alias:
                 spec_type = SpecTypeDefnAlias(self, name)
@@ -1346,7 +1382,7 @@ class SpecBuilder:
     def __init__(
         self,
         *,
-        api_endpoints: dict[EndpointKey, str],
+        api_endpoints: dict[EndpointKey, APIEndpointInfo],
         top_namespace: str,
         cross_output_paths: CrossOutputPaths | None,
     ) -> None:
@@ -1377,9 +1413,13 @@ class SpecBuilder:
         self.emit_id_source_enums: set[SpecTypeDefnStringEnum] = set()
 
         this_dir = os.path.dirname(os.path.realpath(__file__))
-        with open(f"{this_dir}/parts/base.py.prepart") as py_base_part:
+        with open(
+            f"{this_dir}/parts/base.py.prepart", encoding="utf-8"
+        ) as py_base_part:
             self.preparts["python"][base_namespace_name] = py_base_part.read()
-        with open(f"{this_dir}/parts/base.ts.prepart") as ts_base_part:
+        with open(
+            f"{this_dir}/parts/base.ts.prepart", encoding="utf-8"
+        ) as ts_base_part:
             self.preparts["typescript"][base_namespace_name] = ts_base_part.read()
 
         base_namespace.types["ObjectId"] = SpecTypeDefnObject(
@@ -1506,8 +1546,10 @@ class SpecBuilder:
         if len(path) == 2:
             if isinstance(defn_type, SpecTypeDefnStringEnum):
                 assert path[1].parameters is None
+                statement = f"$import: [{defn_type.namespace.name}]"
                 self.ensure(
-                    path[1].name in defn_type.values, f"missing-enum-value: {path}"
+                    path[1].name in defn_type.values,
+                    f"missing-enum-value: {path} have you specified the dependency in an import statement: {statement}",
                 )
                 return SpecTypeLiteralWrapper(
                     value=path[1].name,
@@ -1554,6 +1596,11 @@ class SpecBuilder:
                 f"'examples' in example files are expected to be a list, endpoint_path={path_details.resolved_path}"
             )
         for example in examples_data:
+            if not isinstance(example, dict):
+                raise Exception(
+                    f"each example in example file is expected to be a dict, endpoint_path={path_details.resolved_path}"
+                )
+
             arguments = example["arguments"]
             data_example = example["data"]
             if not isinstance(arguments, dict) or not isinstance(data_example, dict):

pkgs/type_spec/builder_types.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass
+
+
+@dataclass(kw_only=True, frozen=True)
+class CrossOutputPaths:
+    python_types_output: str
+    typescript_types_output: str
+    typescript_routes_output_by_endpoint: dict[str, str]
+    typespec_files_input: list[str]

pkgs/type_spec/config.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from typing import Self, TypeVar
 
 from pkgs.serialization import yaml
+from pkgs.type_spec.builder import APIEndpointInfo, EndpointKey
 
 ConfigValueType = str | None | Mapping[str, str | None] | list[str]
 
@@ -19,6 +20,22 @@ def _parse_string_lookup(
     }
 
 
+VT = TypeVar("VT")
+
+
+def _parse_data_lookup(
+    key: str,
+    raw_value: ConfigValueType,
+    conv_func: type[VT],
+) -> dict[str, VT]:
+    assert isinstance(raw_value, dict), f"{key} must be key/values"
+    return {
+        k: conv_func(**v)
+        for k, v in raw_value.items()
+        if v is not None and isinstance(v, dict)
+    }
+
+
 @dataclass(kw_only=True)
 class BaseLanguageConfig:
     types_output: (
@@ -31,7 +48,9 @@ class BaseLanguageConfig:
 
 @dataclass(kw_only=True)
 class TypeScriptConfig(BaseLanguageConfig):
-    routes_output: str  # folder for generate route files will be located.
+    endpoint_to_routes_output: dict[
+        EndpointKey, str
+    ]  # folder for generate route files will be located.
     type_info_output: str  # folder for generated type info files
     id_source_output: str | None = None  # folder for emitted id source maps.
     endpoint_to_frontend_app_type: dict[
@@ -39,7 +58,7 @@ class TypeScriptConfig(BaseLanguageConfig):
     ]  # map from api_endpoint to frontend app type
 
     def __post_init__(self: Self) -> None:
-        self.routes_output = self.routes_output
+        self.endpoint_to_routes_output = self.endpoint_to_routes_output
         self.type_info_output = os.path.abspath(self.type_info_output)
         self.id_source_output = (
             os.path.abspath(self.id_source_output)
@@ -100,7 +119,7 @@ class OpenAPIConfig(BaseLanguageConfig):
 class Config:
     top_namespace: str
     type_spec_types: list[str]  # folders containing the yaml type spec definitions
-    api_endpoint: dict[str, str]
+    api_endpoint: dict[str, APIEndpointInfo]
     # languages
     typescript: TypeScriptConfig | None
     python: PythonConfig
@@ -125,8 +144,10 @@ def parse_yaml_config(config_file: str) -> Config:
     )
     type_spec_types = [os.path.abspath(folder) for folder in raw_type_spec_types]
 
-    api_endpoint = _parse_string_lookup(
-        "api_endpoint", raw_config.get("api_endpoint", {}), lambda x: x
+    api_endpoint = _parse_data_lookup(
+        "api_endpoint",
+        raw_config.get("api_endpoint", {}),
+        APIEndpointInfo,
     )
 
     raw_typescript = raw_config.get("typescript")