UncountablePythonSDK 0.0.69__py3-none-any.whl → 0.0.71__py3-none-any.whl

uncountable/integration/scheduler.py

@@ -3,10 +3,17 @@ import subprocess
 import sys
 import time
 from dataclasses import dataclass
+from datetime import datetime, timezone
 
 from opentelemetry.trace import get_current_span
 
+from uncountable.core.environment import get_local_admin_server_port
 from uncountable.integration.entrypoint import main as cron_target
+from uncountable.integration.queue_runner.command_server import (
+    CommandServerTimeout,
+    check_health,
+)
+from uncountable.integration.queue_runner.queue_runner import start_queue_runner
 from uncountable.integration.telemetry import Logger
 
 SHUTDOWN_TIMEOUT_SECS = 30
@@ -66,7 +73,7 @@ def handle_shutdown(logger: Logger, processes: list[ProcessInfo]) -> None:
         proc_info.process.kill()
 
 
-def check_process_health(logger: Logger, processes: list[ProcessInfo]) -> None:
+def check_process_alive(logger: Logger, processes: list[ProcessInfo]) -> None:
     for proc_info in processes:
         if not proc_info.is_alive:
             logger.log_error(
@@ -76,6 +83,25 @@ def check_process_health(logger: Logger, processes: list[ProcessInfo]) -> None:
             sys.exit(1)
 
 
+def _wait_queue_runner_online() -> None:
+    _MAX_QUEUE_RUNNER_HEALTH_CHECKS = 10
+    _QUEUE_RUNNER_HEALTH_CHECK_DELAY_SECS = 1
+
+    num_attempts = 0
+    before = datetime.now(timezone.utc)
+    while num_attempts < _MAX_QUEUE_RUNNER_HEALTH_CHECKS:
+        try:
+            if check_health(port=get_local_admin_server_port()):
+                return
+        except CommandServerTimeout:
+            pass
+        num_attempts += 1
+        time.sleep(_QUEUE_RUNNER_HEALTH_CHECK_DELAY_SECS)
+    after = datetime.now(timezone.utc)
+    duration_secs = (after - before).seconds
+    raise Exception(f"queue runner failed to come online after {duration_secs} seconds")
+
+
 def main() -> None:
     logger = Logger(get_current_span())
     processes: list[ProcessInfo] = []
@@ -84,7 +110,18 @@ def main() -> None:
         processes.append(process)
         logger.log_info(f"started process {process.name}")
 
-    cron_process = multiprocessing.Process(target=cron_target, args={"blocking": True})
+    runner_process = multiprocessing.Process(target=start_queue_runner)
+    runner_process.start()
+    add_process(ProcessInfo(name="queue runner", process=runner_process))
+
+    try:
+        _wait_queue_runner_online()
+    except Exception as e:
+        logger.log_exception(e)
+        handle_shutdown(logger, processes=processes)
+        return
+
+    cron_process = multiprocessing.Process(target=cron_target)
     cron_process.start()
     add_process(ProcessInfo(name="cron server", process=cron_process))
 
@@ -98,7 +135,7 @@ def main() -> None:
 
     try:
         while True:
-            check_process_health(logger, processes=processes)
+            check_process_alive(logger, processes=processes)
             time.sleep(1)
     except KeyboardInterrupt:
         handle_shutdown(logger, processes=processes)

uncountable/integration/webhook_server/entrypoint.py

@@ -1,4 +1,5 @@
 import hmac
+import typing
 from dataclasses import dataclass
 
 import flask
@@ -6,15 +7,21 @@ import simplejson
 from flask.typing import ResponseReturnValue
 from flask.wrappers import Response
 from opentelemetry.trace import get_current_span
-from uncountable.core.async_batch import AsyncBatchProcessor
-from uncountable.core.environment import get_integration_env, get_webhook_server_port
-from uncountable.integration.construct_client import construct_uncountable_client
-from uncountable.integration.executors.executors import execute_job
-from uncountable.integration.job import WebhookJobArguments
+from uncountable.core.environment import (
+    get_integration_env,
+    get_local_admin_server_port,
+    get_webhook_server_port,
+)
+from uncountable.integration.queue_runner.command_server.command_client import (
+    send_job_queue_message,
+)
+from uncountable.integration.queue_runner.command_server.types import (
+    CommandServerException,
+)
 from uncountable.integration.scan_profiles import load_profiles
 from uncountable.integration.secret_retrieval.retrieve_secret import retrieve_secret
-from uncountable.integration.telemetry import JobLogger, Logger, get_otel_tracer
-from uncountable.types import job_definition_t, webhook_job_t
+from uncountable.integration.telemetry import Logger
+from uncountable.types import base_t, job_definition_t, queued_job_t, webhook_job_t
 
 from pkgs.argument_parser import CachedParser
 
@@ -66,7 +73,7 @@ class WebhookException(BaseException):
 
 def _parse_webhook_payload(
     *, raw_request_body: bytes, signature_key: str, passed_signature: str
-) -> webhook_job_t.WebhookEventBody:
+) -> base_t.JsonValue:
     request_body_signature = hmac.new(
         signature_key.encode("utf-8"), msg=raw_request_body, digestmod="sha256"
     ).hexdigest()
@@ -76,7 +83,7 @@ def _parse_webhook_payload(
 
     try:
         request_body = simplejson.loads(raw_request_body.decode())
-        return webhook_payload_parser.parse_api(request_body)
+        return typing.cast(base_t.JsonValue, request_body)
     except (simplejson.JSONDecodeError, ValueError) as e:
         raise WebhookException.body_parse_error() from e
 
@@ -110,31 +117,20 @@ def register_route(
                     passed_signature=passed_signature,
                 )
 
-                with get_otel_tracer().start_as_current_span(
-                    "webhook_executor"
-                ) as span:
-                    job_logger = JobLogger(
-                        profile_metadata=profile_meta,
-                        job_definition=job,
-                        base_span=span,
-                    )
-                    client = construct_uncountable_client(
-                        profile_meta=profile_meta, job_logger=job_logger
-                    )
-                    execute_job(
-                        job_definition=job,
-                        profile_metadata=profile_meta,
-                        args=WebhookJobArguments(
-                            job_definition=job,
-                            profile_metadata=profile_meta,
-                            client=client,
-                            batch_processor=AsyncBatchProcessor(client=client),
-                            logger=job_logger,
-                            payload=webhook_payload,
+                try:
+                    send_job_queue_message(
+                        job_ref_name=job.id,
+                        payload=queued_job_t.QueuedJobPayload(
+                            invocation_context=queued_job_t.InvocationContextWebhook(
+                                webhook_payload=webhook_payload
+                            )
                         ),
+                        port=get_local_admin_server_port(),
                     )
+                except CommandServerException as e:
+                    raise WebhookException.unknown_error() from e
 
-                    return flask.jsonify(WebhookResponse())
+                return flask.jsonify(WebhookResponse())
             except WebhookException as e:
                 server_logger.log_exception(e)
                 return e.make_error_response()

uncountable/types/__init__.py

@@ -63,6 +63,7 @@ from . import overrides_t as overrides_t
 from . import permissions_t as permissions_t
 from . import phases_t as phases_t
 from . import post_base_t as post_base_t
+from . import queued_job_t as queued_job_t
 from . import recipe_identifiers_t as recipe_identifiers_t
 from . import recipe_inputs_t as recipe_inputs_t
 from . import recipe_links_t as recipe_links_t
@@ -163,6 +164,7 @@ __all__: list[str] = [
     "permissions_t",
     "phases_t",
     "post_base_t",
+    "queued_job_t",
     "recipe_identifiers_t",
     "recipe_inputs_t",
     "recipe_links_t",

uncountable/types/api/recipes/get_recipes_data.py

@@ -97,6 +97,7 @@ class RecipeInput:
     curve_id: typing.Optional[base_t.ObjectId]
     actual_quantity_json: base_t.JsonValue
     behavior: str
+    ingredient_role_id: typing.Optional[base_t.ObjectId]
     quantity_dec: typing.Optional[Decimal] = None
     actual_quantity_dec: typing.Optional[Decimal] = None
 

uncountable/types/api/recipes/set_recipe_outputs.py

@@ -10,6 +10,7 @@ from decimal import Decimal  # noqa: F401
 import dataclasses
 from pkgs.serialization import serial_class
 from ... import base_t
+from ... import field_values_t
 from ... import recipes_t
 from ... import response_t
 
@@ -47,6 +48,7 @@ class RecipeOutputValue:
     value_str: typing.Optional[str] = None
     value_curve: typing.Optional[CurveValues] = None
     formatting: typing.Optional[recipes_t.RecipeAttributeFormatting] = None
+    field_values: typing.Optional[list[typing.Union[field_values_t.ArgumentValueRefName, field_values_t.ArgumentValueId]]] = None
 
 
 # DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/queued_job.py

@@ -0,0 +1,16 @@
+# flake8: noqa: F821
+# ruff: noqa: E402 Q003
+# fmt: off
+# isort: skip_file
+# DO NOT MODIFY -- This file is generated by type_spec
+# Kept only for SDK backwards compatibility
+from .queued_job_t import InvocationContextType as InvocationContextType
+from .queued_job_t import InvocationContextBase as InvocationContextBase
+from .queued_job_t import InvocationContextCron as InvocationContextCron
+from .queued_job_t import InvocationContextManual as InvocationContextManual
+from .queued_job_t import InvocationContextWebhook as InvocationContextWebhook
+from .queued_job_t import InvocationContext as InvocationContext
+from .queued_job_t import QueuedJobPayload as QueuedJobPayload
+from .queued_job_t import QueuedJobResult as QueuedJobResult
+from .queued_job_t import QueuedJob as QueuedJob
+# DO NOT MODIFY -- This file is generated by type_spec

uncountable/types/queued_job_t.py

@@ -0,0 +1,107 @@
+# DO NOT MODIFY -- This file is generated by type_spec
+# flake8: noqa: F821
+# ruff: noqa: E402 Q003
+# fmt: off
+# isort: skip_file
+from __future__ import annotations
+import typing  # noqa: F401
+import datetime  # noqa: F401
+from decimal import Decimal  # noqa: F401
+from pkgs.strenum_compat import StrEnum
+import dataclasses
+from pkgs.serialization import serial_class
+from pkgs.serialization import serial_union_annotation
+from . import base_t
+from . import job_definition_t
+
+__all__: list[str] = [
+    "InvocationContext",
+    "InvocationContextBase",
+    "InvocationContextCron",
+    "InvocationContextManual",
+    "InvocationContextType",
+    "InvocationContextWebhook",
+    "QueuedJob",
+    "QueuedJobPayload",
+    "QueuedJobResult",
+]
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+class InvocationContextType(StrEnum):
+    CRON = "cron"
+    MANUAL = "manual"
+    WEBHOOK = "webhook"
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@dataclasses.dataclass(kw_only=True)
+class InvocationContextBase:
+    type: InvocationContextType
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    parse_require={"type"},
+)
+@dataclasses.dataclass(kw_only=True)
+class InvocationContextCron(InvocationContextBase):
+    type: typing.Literal[InvocationContextType.CRON] = InvocationContextType.CRON
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    parse_require={"type"},
+)
+@dataclasses.dataclass(kw_only=True)
+class InvocationContextManual(InvocationContextBase):
+    type: typing.Literal[InvocationContextType.MANUAL] = InvocationContextType.MANUAL
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@serial_class(
+    unconverted_values={"webhook_payload"},
+    parse_require={"type"},
+)
+@dataclasses.dataclass(kw_only=True)
+class InvocationContextWebhook(InvocationContextBase):
+    type: typing.Literal[InvocationContextType.WEBHOOK] = InvocationContextType.WEBHOOK
+    webhook_payload: base_t.JsonValue
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+InvocationContext = typing.Annotated[
+    typing.Union[InvocationContextCron, InvocationContextManual, InvocationContextWebhook],
+    serial_union_annotation(
+        discriminator="type",
+        discriminator_map={
+            "cron": InvocationContextCron,
+            "manual": InvocationContextManual,
+            "webhook": InvocationContextWebhook,
+        },
+    ),
+]
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@dataclasses.dataclass(kw_only=True)
+class QueuedJobPayload:
+    invocation_context: InvocationContext
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@dataclasses.dataclass(kw_only=True)
+class QueuedJobResult:
+    queued_job_uuid: str
+    job_result: job_definition_t.JobResult
+
+
+# DO NOT MODIFY -- This file is generated by type_spec
+@dataclasses.dataclass(kw_only=True)
+class QueuedJob:
+    queued_job_uuid: str
+    job_ref_name: str
+    num_attempts: int
+    submitted_at: datetime.datetime
+    payload: QueuedJobPayload
+# DO NOT MODIFY -- This file is generated by type_spec