UncountablePythonSDK 0.0.125__py3-none-any.whl → 0.0.127__py3-none-any.whl

examples/integration-server/jobs/materials_auto/example_instrument.py

@@ -1,17 +1,20 @@
+import json
 import time
 from dataclasses import dataclass
 from decimal import Decimal
 
+from pkgs.argument_parser.argument_parser import CachedParser
+from pkgs.serialization_util import serialize_for_api
 from uncountable.integration.job import JobArguments, WebhookJob, register_job
 from uncountable.types import (
     base_t,
-    entity_t,
     identifier_t,
-    integrations_t,
     job_definition_t,
-    publish_realtime_data_t,
+    sockets_t,
 )
-from uncountable.types.client_base import APIRequest
+from uncountable.types.integration_session_t import IntegrationSessionInstrument
+from websockets.sync.client import connect
+from websockets.typing import Data
 
 
 @dataclass(kw_only=True)
@@ -24,45 +27,71 @@ class InstrumentExample(WebhookJob[InstrumentPayload]):
     def run(
         self, args: JobArguments, payload: InstrumentPayload
     ) -> job_definition_t.JobResult:
-        equipment_data = args.client.get_entities_data(
-            entity_type=entity_t.EntityType.EQUIPMENT,
-            entity_ids=[payload.equipment_id],
-        ).entity_details[0]
+        parser: CachedParser[sockets_t.SocketResponse] = CachedParser(
+            sockets_t.SocketResponse  # type:ignore[arg-type]
+        )
 
-        # Load the instrument's connection details from the entity
-        instrument_id = None
-        for field in equipment_data.field_values:
-            if field.field_ref_name == "ins_instrument_id":
-                instrument_id = field.value
+        def parse_message(message: Data) -> sockets_t.SocketEventData | None:
+            try:
+                return parser.parse_api(json.loads(message)).data
+            except ValueError as e:
+                return None
 
-        if instrument_id is None:
-            args.logger.log_error("Could not find instrument ID")
-            return job_definition_t.JobResult(success=False)
-
-        args.logger.log_info(f"Instrument ID: {instrument_id}")
-
-        for i in range(10):
-            req_args = publish_realtime_data_t.Arguments(
-                data_package=integrations_t.DataPackageNumericReading(
-                    value=Decimal(i * 15),
-                    target_entity=entity_t.EntityIdentifier(
-                        identifier_key=identifier_t.IdentifierKeyId(
-                            id=payload.equipment_id
-                        ),
-                        type=entity_t.EntityType.EQUIPMENT,
-                    ),
-                ),
-            )
-            api_request = APIRequest(
-                method=publish_realtime_data_t.ENDPOINT_METHOD,
-                endpoint=publish_realtime_data_t.ENDPOINT_PATH,
-                args=req_args,
+        integration_session = IntegrationSessionInstrument(
+            equipment_key=identifier_t.IdentifierKeyId(id=payload.equipment_id)
+        )
+        registration_info = args.client.register_sockets_token(
+            socket_request=sockets_t.SocketRequestIntegrationSession(
+                integration_session=integration_session
             )
-            args.client.do_request(
-                api_request=api_request, return_type=publish_realtime_data_t.Data
+        ).response
+        token = registration_info.token
+        room_key = registration_info.room_key
+        args.logger.log_info(f"Token: {token}")
+
+        with connect(
+            "ws://host.docker.internal:8765",
+            additional_headers={
+                "Authorization": f"Bearer {token}",
+                "X-UNC-EXTERNAL": "true",
+            },
+        ) as ws:
+            ws.send(
+                json.dumps(
+                    serialize_for_api(
+                        sockets_t.JoinRoomWithTokenSocketClientMessage(token=token)
+                    )
+                )
             )
-            time.sleep(0.75)
+            for i in range(10):
+                args.logger.log_info("Sending reading...")
+                ws.send(
+                    json.dumps(
+                        serialize_for_api(
+                            sockets_t.SendInstrumentReadingClientMessage(
+                                value=Decimal(i * 100), room_key=room_key
+                            )
+                        )
+                    )
+                )
+                time.sleep(0.75)
+
+            while True:
+                message = parse_message(ws.recv())
+                match message:
+                    case sockets_t.UsersInRoomUpdatedEventData():
+                        num_users = len(message.user_ids)
+                        if num_users <= 1:
+                            break
+                        else:
+                            args.logger.log_info(
+                                f"Session still open, {num_users} users in room."
+                            )
+                    case _:
+                        args.logger.log_info("Session still open...")
+                        continue
 
+        args.logger.log_info("Session closed.")
         return job_definition_t.JobResult(success=True)
 
     @property

examples/integration-server/jobs/materials_auto/example_predictions.py

@@ -0,0 +1,61 @@
+import random
+from dataclasses import dataclass
+from datetime import datetime
+from decimal import Decimal
+
+from uncountable.integration.job import JobArguments, WebhookJob, register_job
+from uncountable.types import (
+    base_t,
+    identifier_t,
+    job_definition_t,
+    recipe_links_t,
+    set_recipe_outputs_t,
+)
+
+
+@dataclass(kw_only=True)
+class PredictionsPayload:
+    output_id: base_t.ObjectId
+    recipe_ids: list[base_t.ObjectId]
+
+
+@register_job
+class PredictionsExample(WebhookJob[PredictionsPayload]):
+    def run(
+        self, args: JobArguments, payload: PredictionsPayload
+    ) -> job_definition_t.JobResult:
+        recipe_data = args.client.get_recipes_data(recipe_ids=payload.recipe_ids)
+        formatted_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        for recipe in recipe_data.recipes:
+            test_sample_name = f"Predictions Model ({formatted_datetime})"
+            created_recipe_id = args.client.create_recipe(
+                name=test_sample_name,
+                material_family_id=1,
+                workflow_id=1,
+                definition_key=identifier_t.IdentifierKeyRefName(
+                    ref_name="unc_test_sample"
+                ),
+            ).result_id
+            args.client.set_recipe_outputs(
+                output_data=[
+                    set_recipe_outputs_t.RecipeOutputValue(
+                        recipe_id=created_recipe_id,
+                        output_id=payload.output_id,
+                        experiment_num=1,
+                        value_numeric=Decimal(random.random() * 10),
+                    )
+                ]
+            )
+            args.client.create_recipe_link(
+                recipe_from_key=identifier_t.IdentifierKeyId(id=recipe.recipe_id),
+                recipe_to_key=identifier_t.IdentifierKeyId(id=created_recipe_id),
+                link_type=recipe_links_t.RecipeLinkType.CHILD,
+                name=test_sample_name,
+            )
+
+        return job_definition_t.JobResult(success=True)
+
+    @property
+    def webhook_payload_type(self) -> type:
+        return PredictionsPayload

examples/integration-server/jobs/materials_auto/profile.yaml

@@ -84,3 +84,12 @@ jobs:
     executor:
       type: script
       import_path: example_instrument
+  - id: example_predictions
+    type: webhook
+    name: Webook Predictions
+    signature_key_secret:
+      type: env
+      env_key: WH_PREDICTIONS_SIGNATURE_KEY
+    executor:
+      type: script
+      import_path: example_predictions

examples/integration-server/pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
   "ruff == 0.*",
   "openpyxl == 3.*",
   "more_itertools == 10.*",
-  "types-paramiko ==3.5.0.20250801",
+  "types-paramiko ==4.0.0.20250822",
   "types-openpyxl == 3.*",
   "types-pysftp == 0.*",
   "types-pytz ==2025.*",
@@ -17,7 +17,8 @@ dependencies = [
   "types-simplejson == 3.*",
   "pandas-stubs",
   "xlrd == 2.*",
-  "msgspec == 0.19.*"
+  "msgspec == 0.19.*",
+  "websockets==15.0.1",
 ]
 
 [tool.mypy]
@@ -194,7 +195,6 @@ lint.ignore = [
     "RUF022",  # __all__ is not sorted.  skip due to isort complication
     "UP017", # use datetime.UTC, TODO add back in
     "UP035", # replacing List with list, TODO add back in
-    "UP038", # isinstance X | Y instead of (X, Y), TODO add back in
     # ## FROM RUFF UPGRADE
     "PLC2701",  # private name imports.  should add
     "PLR1702",  # too many nested blocks -- add with config.  skip

pkgs/type_spec/builder.py

@@ -308,6 +308,7 @@ class SpecTypeDefn(SpecType):
         self._is_value_to_string = False
         self._is_valid_parameter = True
         self._is_dynamic_allowed = False
+        self._default_extant: PropertyExtant | None = None
         self.ext_info: Any = None
 
     def is_value_converted(self) -> bool:
@@ -340,6 +341,7 @@ class SpecTypeDefn(SpecType):
                 "ext_info",
                 "label",
                 "is_dynamic_allowed",
+                "default_extant",
             ]
             + extra_names,
         )
@@ -351,6 +353,10 @@ class SpecTypeDefn(SpecType):
         assert isinstance(is_dynamic_allowed, bool)
         self._is_dynamic_allowed = is_dynamic_allowed
 
+        default_extant = data.get("default_extant")
+        if default_extant is not None:
+            self._default_extant = PropertyExtant(default_extant)
+
     def _process_property(
         self, builder: SpecBuilder, spec_name: str, data: RawDict
     ) -> SpecProperty:
@@ -369,18 +375,18 @@ class SpecTypeDefn(SpecType):
             ],
         )
         try:
-            extant_type = data.get("extant")
+            extant_type_str = data.get("extant")
+            extant_type = (
+                PropertyExtant(extant_type_str) if extant_type_str is not None else None
+            )
+            extant = extant_type or self._default_extant
             if spec_name.endswith("?"):
-                if extant_type is not None:
+                if extant is not None:
                     raise Exception("cannot specify extant with ?")
                 extant = PropertyExtant.optional
                 name = spec_name[:-1]
             else:
-                extant = (
-                    PropertyExtant.required
-                    if extant_type is None
-                    else PropertyExtant(extant_type)
-                )
+                extant = extant or PropertyExtant.required
                 name = spec_name
 
             property_name_case = self.name_case
@@ -1392,9 +1398,13 @@ class SpecBuilder:
         self.emit_id_source_enums: set[SpecTypeDefnStringEnum] = set()
 
         this_dir = os.path.dirname(os.path.realpath(__file__))
-        with open(f"{this_dir}/parts/base.py.prepart") as py_base_part:
+        with open(
+            f"{this_dir}/parts/base.py.prepart", encoding="utf-8"
+        ) as py_base_part:
             self.preparts["python"][base_namespace_name] = py_base_part.read()
-        with open(f"{this_dir}/parts/base.ts.prepart") as ts_base_part:
+        with open(
+            f"{this_dir}/parts/base.ts.prepart", encoding="utf-8"
+        ) as ts_base_part:
             self.preparts["typescript"][base_namespace_name] = ts_base_part.read()
 
         base_namespace.types["ObjectId"] = SpecTypeDefnObject(

pkgs/type_spec/emit_typescript.py

@@ -285,14 +285,14 @@ export const apiCall = {wrap_call}(
     index_path = f"{config.endpoint_to_routes_output[endpoint.default_endpoint_key]}/{'/'.join(namespace.path[0:-1])}/index.tsx"
     api_name = f"Api{ts_type_name(namespace.path[0 - 1])}"
     if os.path.exists(index_path):
-        with open(index_path) as index:
+        with open(index_path, encoding="utf-8") as index:
             index_data = index.read()
             need_index = index_data.find(api_name) == -1
     else:
         need_index = True
 
     if need_index:
-        with open(index_path, "a") as index:
+        with open(index_path, "a", encoding="utf-8") as index:
             print(f"Updated API Index {index_path}")
             index.write(f'import * as {api_name} from "./{namespace.path[-1]}"\n\n')
             index.write(f"export {{ {api_name} }}\n")

pkgs/type_spec/type_info/emit_type_info.py

@@ -41,10 +41,23 @@ def type_path_of(stype: builder.SpecType) -> object:  # NamePath
             parts: list[object] = ["$literal"]
             for parameter in stype.parameters:
                 assert isinstance(parameter, builder.SpecTypeLiteralWrapper)
+                emit_value = parameter.value
+                if isinstance(parameter.value_type, builder.SpecTypeDefnObject):
+                    emit_value = parameter.value
+                    assert isinstance(emit_value, (str, bool)), (
+                        f"invalid-literal-value:{emit_value}"
+                    )
+                elif isinstance(parameter.value_type, builder.SpecTypeDefnStringEnum):
+                    key = parameter.value
+                    assert isinstance(key, str)
+                    emit_value = parameter.value_type.values[key].value
+                else:
+                    raise Exception("unhandled-literal-type")
+
                 # This allows expansion to enum literal values later
                 parts.append([
                     "$value",
-                    parameter.value,
+                    emit_value,
                     type_path_of(parameter.value_type),
                 ])
             return parts

pkgs/type_spec/value_spec/__main__.py

@@ -20,7 +20,7 @@ The accepted argument type must accept "None", it is not implied.
 """
 
 import sys
-from typing import TypeVar, cast
+from typing import Match, Pattern, TypeVar, cast
 
 import regex as re
 
@@ -56,7 +56,7 @@ class Source:
     def has_more(self) -> bool:
         return self._at < len(self._text)
 
-    def match(self, expression: re.Pattern) -> re.Match | None:
+    def match(self, expression: Pattern[str]) -> Match[str] | None:
         self.skip_space()
         m = expression.match(self._text, self._at)
         if m is not None:

uncountable/integration/cli.py

@@ -8,6 +8,7 @@ from uncountable.core.environment import get_local_admin_server_port
 from uncountable.integration.queue_runner.command_server.command_client import (
     send_job_queue_message,
     send_list_queued_jobs_message,
+    send_retry_job_message,
 )
 from uncountable.integration.telemetry import Logger
 from uncountable.types import queued_job_t
@@ -70,12 +71,13 @@ def register_list_queued_jobs(
             port=get_local_admin_server_port(),
         )
 
-        headers = ["UUID", "Job Ref Name", "Attempts", "Submitted At"]
+        headers = ["UUID", "Job Ref Name", "Attempts", "Status", "Submitted At"]
         rows = [
             [
                 job.uuid,
                 job.job_ref_name,
                 job.num_attempts,
+                job.status,
                 job.submitted_at.ToDatetime(tz.UTC).astimezone(tz.tzlocal()),
             ]
             for job in queued_jobs
@@ -85,6 +87,31 @@ def register_list_queued_jobs(
     list_queued_jobs_parser.set_defaults(func=_handle_list_queued_jobs)
 
 
+def register_retry_job_parser(
+    sub_parser_manager: argparse._SubParsersAction,
+    parents: list[argparse.ArgumentParser],
+) -> None:
+    retry_failed_jobs_parser = sub_parser_manager.add_parser(
+        "retry-job",
+        parents=parents,
+        help="Retry failed job on the integration server",
+        description="Retry failed job on the integration server",
+    )
+
+    retry_failed_jobs_parser.add_argument(
+        "job_uuid", type=str, help="The uuid of the job to retry"
+    )
+
+    def _handle_retry_job(args: argparse.Namespace) -> None:
+        send_retry_job_message(
+            job_uuid=args.job_uuid,
+            host=args.host,
+            port=get_local_admin_server_port(),
+        )
+
+    retry_failed_jobs_parser.set_defaults(func=_handle_retry_job)
+
+
 def main() -> None:
     logger = Logger(get_current_span())
 
@@ -104,6 +131,7 @@ def main() -> None:
     )
 
     register_enqueue_job_parser(subparser_action, parents=[base_parser])
+    register_retry_job_parser(subparser_action, parents=[base_parser])
     register_list_queued_jobs(subparser_action, parents=[base_parser])
 
     args = main_parser.parse_args()

uncountable/integration/executors/executors.py

@@ -88,7 +88,6 @@ def execute_job(
     job_definition: job_definition_t.JobDefinition,
     profile_metadata: job_definition_t.ProfileMetadata,
     args: JobArguments,
-    job_uuid: str,
 ) -> job_definition_t.JobResult:
     with args.logger.push_scope(job_definition.name) as job_logger:
         job = resolve_executor(job_definition.executor, profile_metadata)
@@ -104,7 +103,7 @@ def execute_job(
                 run_entity = _create_run_entity(
                     client=args.client,
                     logging_settings=job_definition.logging_settings,
-                    job_uuid=job_uuid,
+                    job_uuid=args.job_uuid,
                 )
             result = job.run_outer(args=args)
         except Exception as e:

uncountable/integration/executors/generic_upload_executor.py

@@ -41,7 +41,7 @@ def _get_extension(filename: str) -> str | None:
 
 def _run_keyword_detection(data: io.BytesIO, keyword: str) -> bool:
     try:
-        text = io.TextIOWrapper(data)
+        text = io.TextIOWrapper(data, encoding="utf-8")
         for line in text:
             if (
                 keyword in line

uncountable/integration/job.py

@@ -49,6 +49,7 @@ class JobArguments:
     batch_processor: AsyncBatchProcessor
     logger: JobLogger
     payload: base_t.JsonValue
+    job_uuid: str
 
 
 # only for compatibility:

uncountable/integration/queue_runner/command_server/__init__.py

@@ -4,6 +4,8 @@ from .types import (
     CommandEnqueueJob,
     CommandEnqueueJobResponse,
     CommandQueue,
+    CommandRetryJob,
+    CommandRetryJobResponse,
     CommandServerBadResponse,
     CommandServerException,
     CommandServerTimeout,
@@ -16,6 +18,8 @@ __all__: list[str] = [
     "send_job_queue_message",
     "CommandEnqueueJob",
     "CommandEnqueueJobResponse",
+    "CommandRetryJob",
+    "CommandRetryJobResponse",
     "CommandTask",
     "CommandQueue",
     "CommandServerTimeout",

uncountable/integration/queue_runner/command_server/command_client.py

@@ -12,6 +12,10 @@ from uncountable.integration.queue_runner.command_server.protocol.command_server
     EnqueueJobResult,
     ListQueuedJobsRequest,
     ListQueuedJobsResult,
+    RetryJobRequest,
+    RetryJobResult,
+    VaccuumQueuedJobsRequest,
+    VaccuumQueuedJobsResult,
 )
 from uncountable.integration.queue_runner.command_server.types import (
     CommandServerBadResponse,
@@ -59,6 +63,26 @@ def send_job_queue_message(
         return response.queued_job_uuid
 
 
+def send_retry_job_message(
+    *,
+    job_uuid: str,
+    host: str = "localhost",
+    port: int,
+) -> str:
+    with command_server_connection(host=host, port=port) as stub:
+        request = RetryJobRequest(uuid=job_uuid)
+
+        try:
+            response = stub.RetryJob(request, timeout=_DEFAULT_MESSAGE_TIMEOUT_SECS)
+            assert isinstance(response, RetryJobResult)
+            if not response.successfully_queued:
+                raise CommandServerBadResponse("queue operation was not successful")
+
+            return response.queued_job_uuid
+        except grpc.RpcError as e:
+            raise ValueError(e.details())  # type: ignore
+
+
 def check_health(*, host: str = _LOCAL_RPC_HOST, port: int) -> bool:
     with command_server_connection(host=host, port=port) as stub:
         request = CheckHealthRequest()
@@ -92,3 +116,18 @@ def send_list_queued_jobs_message(
 
         assert isinstance(response, ListQueuedJobsResult)
         return list(response.queued_jobs)
+
+
+def send_vaccuum_queued_jobs_message(*, host: str = "localhost", port: int) -> None:
+    with command_server_connection(host=host, port=port) as stub:
+        request = VaccuumQueuedJobsRequest()
+
+        try:
+            response = stub.VaccuumQueuedJobs(
+                request, timeout=_DEFAULT_MESSAGE_TIMEOUT_SECS
+            )
+        except grpc.RpcError as e:
+            raise ValueError(e.details())  # type: ignore
+
+        assert isinstance(response, VaccuumQueuedJobsResult)
+        return None

uncountable/integration/queue_runner/command_server/command_server.py

@@ -13,11 +13,19 @@ from uncountable.integration.queue_runner.command_server.protocol.command_server
     EnqueueJobResult,
     ListQueuedJobsRequest,
     ListQueuedJobsResult,
+    RetryJobRequest,
+    RetryJobResult,
+    VaccuumQueuedJobsRequest,
+    VaccuumQueuedJobsResult,
 )
 from uncountable.integration.queue_runner.command_server.types import (
     CommandEnqueueJob,
     CommandEnqueueJobResponse,
     CommandQueue,
+    CommandRetryJob,
+    CommandRetryJobResponse,
+    CommandVaccuumQueuedJobs,
+    CommandVaccuumQueuedJobsResponse,
 )
 from uncountable.integration.queue_runner.datastore import DatastoreSqlite
 from uncountable.types import queued_job_t
@@ -54,6 +62,23 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
             )
             return result
 
+        async def RetryJob(
+            self, request: RetryJobRequest, context: aio.ServicerContext
+        ) -> RetryJobResult:
+            response_queue: asyncio.Queue[CommandRetryJobResponse] = asyncio.Queue()
+            await command_queue.put(
+                CommandRetryJob(
+                    queued_job_uuid=request.uuid, response_queue=response_queue
+                )
+            )
+            response = await response_queue.get()
+            if response.queued_job_uuid is not None:
+                return RetryJobResult(
+                    successfully_queued=True, queued_job_uuid=response.queued_job_uuid
+                )
+            else:
+                return RetryJobResult(successfully_queued=False, queued_job_uuid="")
+
         async def CheckHealth(
             self, request: CheckHealthRequest, context: aio.ServicerContext
         ) -> CheckHealthResult:
@@ -90,10 +115,22 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
                         job_ref_name=item.job_ref_name,
                         num_attempts=item.num_attempts,
                         submitted_at=proto_timestamp,
+                        status=item.status,
                     )
                 )
             return ListQueuedJobsResult(queued_jobs=response_list)
 
+        async def VaccuumQueuedJobs(
+            self, request: VaccuumQueuedJobsRequest, context: aio.ServicerContext
+        ) -> VaccuumQueuedJobsResult:
+            response_queue: asyncio.Queue[CommandVaccuumQueuedJobsResponse] = (
+                asyncio.Queue()
+            )
+            await command_queue.put(
+                CommandVaccuumQueuedJobs(response_queue=response_queue)
+            )
+            return VaccuumQueuedJobsResult()
+
     add_CommandServerServicer_to_server(CommandServerHandler(), server)
 
     listen_addr = f"[::]:{get_local_admin_server_port()}"

uncountable/integration/queue_runner/command_server/protocol/command_server.proto

@@ -3,8 +3,10 @@ import "google/protobuf/timestamp.proto";
 
 service CommandServer {
 	rpc EnqueueJob(EnqueueJobRequest) returns (EnqueueJobResult) {}
+	rpc RetryJob(RetryJobRequest) returns (RetryJobResult) {}
 	rpc CheckHealth(CheckHealthRequest) returns (CheckHealthResult) {}
 	rpc ListQueuedJobs(ListQueuedJobsRequest) returns (ListQueuedJobsResult) {}
+	rpc VaccuumQueuedJobs(VaccuumQueuedJobsRequest) returns (VaccuumQueuedJobsResult) {}
 }
 
 message EnqueueJobRequest {
@@ -17,6 +19,21 @@ message EnqueueJobResult {
 	string queued_job_uuid = 2;
 }
 
+message RetryJobRequest {
+	string uuid = 1;
+}
+
+message RetryJobResult {
+	bool successfully_queued = 1;
+	string queued_job_uuid = 2;
+}
+
+message VaccuumQueuedJobsRequest {
+}
+
+message VaccuumQueuedJobsResult {
+}
+
 message CheckHealthRequest {}
 
 message CheckHealthResult {
@@ -34,6 +51,7 @@ message ListQueuedJobsResult {
 	string job_ref_name = 2;
 	int64 num_attempts = 3;
 	google.protobuf.Timestamp submitted_at = 4;
+	string status = 5;
 	}
 
 	repeated ListQueuedJobsResultItem queued_jobs  = 1;