UncountablePythonSDK 0.0.113__py3-none-any.whl → 0.0.115__py3-none-any.whl

examples/integration-server/jobs/materials_auto/concurrent_cron.py

@@ -0,0 +1,11 @@
+import time
+
+from uncountable.integration.job import CronJob, JobArguments, register_job
+from uncountable.types.job_definition_t import JobResult
+
+
+@register_job
+class MyConcurrentCronJob(CronJob):
+    def run(self, args: JobArguments) -> JobResult:
+        time.sleep(10)
+        return JobResult(success=True)

examples/integration-server/jobs/materials_auto/example_http.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+
+from uncountable.integration.http_server import (
+    GenericHttpRequest,
+    GenericHttpResponse,
+)
+from uncountable.integration.job import CustomHttpJob, register_job
+from uncountable.types import job_definition_t
+
+
+@dataclass(kw_only=True)
+class ExampleWebhookPayload:
+    id: int
+    message: str
+
+
+@register_job
+class HttpExample(CustomHttpJob):
+    @staticmethod
+    def validate_request(
+        *,
+        request: GenericHttpRequest,
+        job_definition: job_definition_t.HttpJobDefinitionBase,
+        profile_meta: job_definition_t.ProfileMetadata,
+    ) -> None:
+        return None
+
+    @staticmethod
+    def handle_request(
+        *,
+        request: GenericHttpRequest,
+        job_definition: job_definition_t.HttpJobDefinitionBase,
+        profile_meta: job_definition_t.ProfileMetadata,
+    ) -> GenericHttpResponse:
+        return GenericHttpResponse(response="OK", status_code=200)

examples/integration-server/jobs/materials_auto/profile.yaml

@@ -22,6 +22,25 @@ jobs:
         - admin:
           type: ref_name
           ref_name: admin
+  - id: concurrent_cron_1
+    enabled: true
+    type: cron
+    name: MyConcurrentCron - 1
+    cron_spec: "* * * * *"
+    executor:
+      type: script
+      import_path: concurrent_cron
+    subqueue_name: subqueue_1
+  - id: concurrent_cron_2
+    enabled: true
+    type: cron
+    name: MyConcurrentCron - 2
+    cron_spec: "* * * * *"
+    executor:
+      type: script
+      import_path: concurrent_cron
+    subqueue_name: subqueue_2
+
 
   - id: example_wh1
     type: webhook
@@ -41,6 +60,12 @@ jobs:
     executor:
       type: script
       import_path: example_wh
+  - id: example_http
+    type: custom_http
+    name: Custom HTTP
+    executor:
+      type: script
+      import_path: example_http
   - id: example_runsheet_wh
     type: webhook
     name: Runsheet Webhook

pkgs/argument_parser/argument_parser.py

@@ -170,9 +170,15 @@ def _invoke_membership_parser(
 
 def _build_parser_discriminated_union(
     context: ParserContext,
-    discriminator: str,
+    discriminator_raw: str,
     discriminator_map: dict[str, ParserFunction[T]],
 ) -> ParserFunction[T]:
+    discriminator = (
+        snake_to_camel_case(discriminator_raw)
+        if context.options.from_camel_case
+        else discriminator_raw
+    )
+
     def parse(value: typing.Any) -> typing.Any:
         if context.options.allow_direct_type and dataclasses.is_dataclass(value):
             discriminant = getattr(value, discriminator)
@@ -414,8 +420,7 @@ def _build_parser_dataclass(
     cur_parser = context.cache.get(parsed_type)
     if cur_parser is not None:
         return cur_parser
-
-    type_hints = typing.get_type_hints(parsed_type)
+    type_hints = typing.get_type_hints(parsed_type, include_extras=True)
     dc_field_parsers: list[
         tuple[
             dataclasses.Field[typing.Any],

pkgs/type_spec/builder.py

@@ -14,6 +14,7 @@ from typing import Any, Self
 
 from . import util
 from .cross_output_links import CrossOutputPaths
+from .non_discriminated_union_exceptions import NON_DISCRIMINATED_UNION_EXCEPTIONS
 from .util import parse_type_str, unused
 
 RawDict = dict[Any, Any]
@@ -636,6 +637,11 @@ class SpecTypeDefnUnion(SpecTypeDefn):
                 self.discriminator_map[discriminant] = sub_type
 
                 builder.pop_where()
+        elif (
+            f"{self.namespace.name}.{self.name}"
+            not in NON_DISCRIMINATED_UNION_EXCEPTIONS
+        ):
+            raise Exception(f"union requires a discriminator: {self.name}")
 
     def get_referenced_types(self) -> list[SpecType]:
         return self.types
@@ -960,8 +966,8 @@ class SpecEndpoint:
         elif isinstance(is_sdk, str):
             try:
                 is_sdk = EndpointEmitType(is_sdk)
-            except ValueError:
-                raise ValueError(f"Invalid value for is_sdk: {is_sdk}")
+            except ValueError as e:
+                raise ValueError(f"Invalid value for is_sdk: {is_sdk}") from e
 
         assert isinstance(is_sdk, EndpointEmitType)
 

pkgs/type_spec/non_discriminated_union_exceptions.py

@@ -0,0 +1,14 @@
+NON_DISCRIMINATED_UNION_EXCEPTIONS = [
+    "generate_tool_parameters.UnionWithoutDiscrim",
+    "output_calculation_entities.ConditionParameterFilterCondition",
+    "output_parameters.AnalyticalMethodParameterOptions",
+    "output_parameters.AnalyticalMethodLinkedOptionValue",
+    "recipes_redirect.RecipesRedirectResult",
+    "value_spec.ResolvedPathAll",
+    "weighted_sum.WeightedSumEntities",
+    "workflows.WorkflowTotalDisplay",
+    "type_info.TypeFormActionConstraint",
+    "structured_loading.CompatibleFilterNode",
+    "structured_display_element.DisplayElementColumn",
+    "field_values.FieldValue",
+]

pkgs/type_spec/parts/base.py.prepart

@@ -58,15 +58,12 @@ def is_pure_json_value(value: ExtJsonValue) -> bool:
         return True
 
     if isinstance(value, list):
-        for item in value:
-            if not is_pure_json_value(item):
-                return False
-        return True
+        return all(is_pure_json_value(item) for item in value)
 
     if isinstance(value, dict):
-        for key, item in value.items():
-            if not is_pure_json_value(key) or not is_pure_json_value(item):
-                return False
-        return True
+        return all(
+            is_pure_json_value(key) and is_pure_json_value(item)
+            for key, item in value.items()
+        )
 
     return False

pkgs/type_spec/ui_entry_actions/generate_ui_entry_actions.py

@@ -1,13 +1,14 @@
 import json
+import re
 from dataclasses import dataclass
 from io import StringIO
 from pathlib import Path
 from typing import assert_never
 
 from main.base.types import (
+    base_t,
     ui_entry_actions_t,
 )
-from pkgs.argument_parser import snake_to_camel_case
 from pkgs.serialization_util import serialize_for_api
 from pkgs.type_spec import emit_typescript_util
 from pkgs.type_spec.builder import (
@@ -45,8 +46,8 @@ def ui_entry_variable_to_type_spec_type(
     match variable:
         case ui_entry_actions_t.UiEntryActionVariableString():
             return BaseTypeName.s_string
-        case ui_entry_actions_t.UiEntryActionVariableEntityFields():
-            return "entity.Entity"
+        case ui_entry_actions_t.UiEntryActionVariableSingleEntity():
+            return "ObjectId"
         case _:
             assert_never(variable)
 
@@ -157,6 +158,15 @@ def emit_entry_action_definition(
     )
 
 
+def _validate_input(input: ui_entry_actions_t.UiEntryActionVariable) -> None:
+    if "_" in input.vs_var_name:
+        raise ValueError(f"Expected camelCase for variable {input.vs_var_name}")
+    if not re.fullmatch(base_t.REF_NAME_STRICT_REGEX, input.vs_var_name):
+        raise ValueError(
+            f"Variable {input.vs_var_name} has invalid syntax. See REF_NAME_STRICT_REGEX"
+        )
+
+
 def emit_query_index(
     ctx: emit_typescript_util.EmitTypescriptContext,
     defn_infos: list[EntryActionTypeInfo],
@@ -182,6 +192,7 @@ def emit_query_index(
             "type": BaseTypeName.s_object,
             "properties": {
                 "value_spec_var": {"type": "String"},
+                "type": {"type": "ui_entry_actions.UiEntryActionDataType"},
                 "variable": {"type": "ui_entry_actions.UiEntryActionVariable"},
             },
         },
@@ -215,10 +226,12 @@ def emit_query_index(
     for scope, defn in definitions.items():
         inputs = []
         outputs = []
-        for name, input in defn.inputs.items():
+        for input in defn.inputs.values():
+            _validate_input(input)
             inputs.append(
                 serialize_for_api({
-                    "value_spec_var": snake_to_camel_case(name),
+                    "value_spec_var": input.vs_var_name,
+                    "type": input.type,
                     "variable": input,
                 })
             )

pkgs/type_spec/value_spec/__main__.py

@@ -104,8 +104,8 @@ def parse_function_signature(text: str) -> ParsedFunctionSignature:
 
         type_str = source.extract_type()
         ref_name = arg_group.group(1)
-        is_missing = arg_group.group(2) == "?"
-        is_repeating = arg_group.group(2) == "+"
+        # is_missing = arg_group.group(2) == "?"
+        # is_repeating = arg_group.group(2) == "+"
         type_path = parse_type_str(type_str)
 
         match arg_group.group(3):

uncountable/core/environment.py

@@ -18,7 +18,7 @@ def get_server_env() -> str | None:
     return os.environ.get("UNC_SERVER_ENV")
 
 
-def get_webhook_server_port() -> int:
+def get_http_server_port() -> int:
     return int(os.environ.get("UNC_WEBHOOK_SERVER_PORT", "5001"))
 
 

uncountable/integration/http_server/__init__.py

@@ -0,0 +1,5 @@
+# CLOSED MODULE
+
+from .types import GenericHttpRequest as GenericHttpRequest
+from .types import GenericHttpResponse as GenericHttpResponse
+from .types import HttpException as HttpException

uncountable/integration/http_server/types.py

@@ -0,0 +1,67 @@
+import base64
+import functools
+from dataclasses import dataclass
+
+from flask.wrappers import Response
+
+
+class HttpException(Exception):
+    error_code: int
+    message: str
+
+    def __init__(self, *, error_code: int, message: str) -> None:
+        self.error_code = error_code
+        self.message = message
+
+    @staticmethod
+    def payload_failed_signature() -> "HttpException":
+        return HttpException(
+            error_code=401, message="webhook payload did not match signature"
+        )
+
+    @staticmethod
+    def no_signature_passed() -> "HttpException":
+        return HttpException(error_code=400, message="missing signature")
+
+    @staticmethod
+    def body_parse_error() -> "HttpException":
+        return HttpException(error_code=400, message="body parse error")
+
+    @staticmethod
+    def unknown_error() -> "HttpException":
+        return HttpException(error_code=500, message="internal server error")
+
+    @staticmethod
+    def configuration_error(
+        message: str = "internal configuration error",
+    ) -> "HttpException":
+        return HttpException(error_code=500, message=message)
+
+    def __str__(self) -> str:
+        return f"[{self.error_code}]: {self.message}"
+
+    def make_error_response(self) -> Response:
+        return Response(
+            status=self.error_code, response={"error": {"message": str(self)}}
+        )
+
+
+@dataclass(kw_only=True, frozen=True)
+class GenericHttpRequest:
+    body_base64: str
+    headers: dict[str, str]
+
+    @functools.cached_property
+    def body_bytes(self) -> bytes:
+        return base64.b64decode(self.body_base64)
+
+    @functools.cached_property
+    def body_text(self) -> str:
+        return self.body_bytes.decode()
+
+
+@dataclass(kw_only=True)
+class GenericHttpResponse:
+    response: str
+    status_code: int
+    headers: dict[str, str] | None = None

uncountable/integration/job.py

@@ -1,15 +1,43 @@
 import functools
+import hmac
 import typing
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
+import simplejson
+
 from pkgs.argument_parser import CachedParser
+from pkgs.serialization_util import serialize_for_api
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
+from uncountable.core.environment import get_local_admin_server_port
 from uncountable.core.file_upload import FileUpload
+from uncountable.integration.http_server import (
+    GenericHttpRequest,
+    GenericHttpResponse,
+    HttpException,
+)
+from uncountable.integration.queue_runner.command_server.command_client import (
+    send_job_queue_message,
+)
+from uncountable.integration.queue_runner.command_server.types import (
+    CommandServerException,
+)
+from uncountable.integration.secret_retrieval.retrieve_secret import retrieve_secret
 from uncountable.integration.telemetry import JobLogger
-from uncountable.types import base_t, entity_t, webhook_job_t
-from uncountable.types.job_definition_t import JobDefinition, JobResult, ProfileMetadata
+from uncountable.types import (
+    base_t,
+    entity_t,
+    job_definition_t,
+    queued_job_t,
+    webhook_job_t,
+)
+from uncountable.types.job_definition_t import (
+    HttpJobDefinitionBase,
+    JobDefinition,
+    JobResult,
+    ProfileMetadata,
+)
 
 
 @dataclass(kw_only=True)
@@ -26,9 +54,6 @@ class JobArguments:
 CronJobArguments = JobArguments
 
 
-PT = typing.TypeVar("PT")
-
-
 class Job[PT](ABC):
     _unc_job_registered: bool = False
 
@@ -63,6 +88,51 @@ class CronJob(Job):
 WPT = typing.TypeVar("WPT")
 
 
+@dataclass(kw_only=True)
+class WebhookResponse:
+    pass
+
+
+class CustomHttpJob(Job[GenericHttpRequest]):
+    @property
+    def payload_type(self) -> type[GenericHttpRequest]:
+        return GenericHttpRequest
+
+    @staticmethod
+    @abstractmethod
+    def validate_request(
+        *,
+        request: GenericHttpRequest,
+        job_definition: HttpJobDefinitionBase,
+        profile_meta: ProfileMetadata,
+    ) -> None:
+        """
+        Validate that the request is valid. If the request is invalid, raise an
+        exception.
+        """
+        ...
+
+    @staticmethod
+    @abstractmethod
+    def handle_request(
+        *,
+        request: GenericHttpRequest,
+        job_definition: HttpJobDefinitionBase,
+        profile_meta: ProfileMetadata,
+    ) -> GenericHttpResponse:
+        """
+        Handle the request synchronously. Normally this should just enqueue a job
+        and return immediately (see WebhookJob as an example).
+        """
+        ...
+
+    def run_outer(self, args: JobArguments) -> JobResult:
+        args.logger.log_warning(
+            message=f"Unexpected call to run_outer for CustomHttpJob: {args.job_definition.id}"
+        )
+        return JobResult(success=False)
+
+
 class WebhookJob[WPT](Job[webhook_job_t.WebhookEventPayload]):
     @property
     def payload_type(self) -> type[webhook_job_t.WebhookEventPayload]:
@@ -72,6 +142,60 @@ class WebhookJob[WPT](Job[webhook_job_t.WebhookEventPayload]):
     @abstractmethod
     def webhook_payload_type(self) -> type[WPT]: ...
 
+    @staticmethod
+    def validate_request(
+        *,
+        request: GenericHttpRequest,
+        job_definition: job_definition_t.HttpJobDefinitionBase,
+        profile_meta: ProfileMetadata,
+    ) -> None:
+        assert isinstance(job_definition, job_definition_t.WebhookJobDefinition)
+        signature_key = retrieve_secret(
+            profile_metadata=profile_meta,
+            secret_retrieval=job_definition.signature_key_secret,
+        )
+        passed_signature = request.headers.get("Uncountable-Webhook-Signature")
+        if passed_signature is None:
+            raise HttpException.no_signature_passed()
+
+        request_body_signature = hmac.new(
+            signature_key.encode("utf-8"), msg=request.body_bytes, digestmod="sha256"
+        ).hexdigest()
+
+        if request_body_signature != passed_signature:
+            raise HttpException.payload_failed_signature()
+
+    @staticmethod
+    def handle_request(
+        *,
+        request: GenericHttpRequest,
+        job_definition: job_definition_t.HttpJobDefinitionBase,
+        profile_meta: ProfileMetadata,  # noqa: ARG004
+    ) -> GenericHttpResponse:
+        try:
+            request_body = simplejson.loads(request.body_text)
+            webhook_payload = typing.cast(base_t.JsonValue, request_body)
+        except (simplejson.JSONDecodeError, ValueError) as e:
+            raise HttpException.body_parse_error() from e
+
+        try:
+            send_job_queue_message(
+                job_ref_name=job_definition.id,
+                payload=queued_job_t.QueuedJobPayload(
+                    invocation_context=queued_job_t.InvocationContextWebhook(
+                        webhook_payload=webhook_payload
+                    )
+                ),
+                port=get_local_admin_server_port(),
+            )
+        except CommandServerException as e:
+            raise HttpException.unknown_error() from e
+
+        return GenericHttpResponse(
+            response=simplejson.dumps(serialize_for_api(WebhookResponse())),
+            status_code=200,
+        )
+
     def run_outer(self, args: JobArguments) -> JobResult:
         webhook_body = self.get_payload(args.payload)
         inner_payload = CachedParser(self.webhook_payload_type).parse_api(

uncountable/integration/queue_runner/job_scheduler.py

@@ -1,4 +1,5 @@
 import asyncio
+import sys
 import typing
 from concurrent.futures import ProcessPoolExecutor
 from dataclasses import dataclass
@@ -34,6 +35,10 @@ class JobListenerKey:
 def _get_job_worker_key(
     job_definition: job_definition_t.JobDefinition, profile_name: str
 ) -> JobListenerKey:
+    if job_definition.subqueue_name is not None:
+        return JobListenerKey(
+            profile_name=profile_name, subqueue_name=job_definition.subqueue_name
+        )
     return JobListenerKey(profile_name=profile_name)
 
 
@@ -41,9 +46,12 @@ def on_worker_crash(
     worker_key: JobListenerKey,
 ) -> typing.Callable[[asyncio.Task], None]:
     def hook(task: asyncio.Task) -> None:
-        raise Exception(
-            f"worker {worker_key.profile_name}_{worker_key.subqueue_name} crashed unexpectedly"
+        Logger(get_current_span()).log_exception(
+            Exception(
+                f"worker {worker_key.profile_name}_{worker_key.subqueue_name} crashed unexpectedly"
+            )
         )
+        sys.exit(1)
 
     return hook
 

uncountable/integration/server.py

@@ -16,7 +16,7 @@ from uncountable.integration.telemetry import Logger
 from uncountable.types import base_t, job_definition_t
 from uncountable.types.job_definition_t import (
     CronJobDefinition,
-    WebhookJobDefinition,
+    HttpJobDefinitionBase,
 )
 
 _MAX_APSCHEDULER_CONCURRENT_JOBS = 1
@@ -86,7 +86,7 @@ class IntegrationServer:
                                 misfire_grace_time=None,
                                 **job_opts,
                             )
-                    case WebhookJobDefinition():
+                    case HttpJobDefinitionBase():
                         pass
                     case _:
                         assert_never(job_defn)

uncountable/integration/telemetry.py

@@ -177,7 +177,7 @@ class JobLogger(Logger):
                 patched_attributes["job.definition.cron_spec"] = (
                     self.job_definition.cron_spec
                 )
-            case job_definition_t.WebhookJobDefinition():
+            case job_definition_t.HttpJobDefinitionBase():
                 pass
             case _:
                 assert_never(self.job_definition)