UncountablePythonSDK 0.0.115__py3-none-any.whl → 0.0.142.dev0__py3-none-any.whl

pkgs/type_spec/emit_typescript_util.py

@@ -3,7 +3,7 @@ import typing
 from dataclasses import dataclass, field
 
 from . import builder, util
-from .cross_output_links import CrossOutputPaths
+from .builder_types import CrossOutputPaths
 
 INDENT = "  "
 
@@ -32,6 +32,7 @@ class EmitTypescriptContext:
     namespace: builder.SpecNamespace
     namespaces: set[builder.SpecNamespace] = field(default_factory=set)
     cross_output_paths: CrossOutputPaths | None = None
+    api_endpoints: dict[builder.EndpointKey, builder.APIEndpointInfo]
 
 
 def ts_type_name(name: str) -> str:
@@ -50,7 +51,10 @@ def ts_name(name: str, name_case: builder.NameCase) -> str:
 
 
 def emit_value_ts(
-    ctx: EmitTypescriptContext, stype: builder.SpecType, value: typing.Any
+    ctx: EmitTypescriptContext,
+    stype: builder.SpecType,
+    value: typing.Any,
+    indent: int = 0,
 ) -> str:
     """Mimics emit_python even if not all types are used in TypeScript yet"""
     literal = builder.unwrap_literal_type(stype)
@@ -80,18 +84,24 @@ def emit_value_ts(
         if stype.defn_type.is_base_type(builder.BaseTypeName.s_dict):
             key_type = stype.parameters[0]
             value_type = stype.parameters[1]
+
+            if not key_type.is_base_type(
+                builder.BaseTypeName.s_string
+            ) and not isinstance(key_type, builder.SpecTypeDefnStringEnum):
+                raise Exception("invalid dict keys -- dict keys must be string or enum")
+
             return (
-                "{\n\t"
-                + ",\n\t".join(
+                f"{{\n{INDENT * (indent + 1)}"
+                + f",\n{INDENT * (indent + 1)}".join(
                     (
                         f"[{emit_value_ts(ctx, key_type, dkey)}]: "
                         if not key_type.is_base_type(builder.BaseTypeName.s_string)
                         else f"{dkey}: "
                     )
-                    + emit_value_ts(ctx, value_type, dvalue)
+                    + emit_value_ts(ctx, value_type, dvalue, indent=indent + 1)
                     for dkey, dvalue in value.items()
                 )
-                + "\n}"
+                + f"\n{INDENT * (indent)}}}"
             )
 
         if stype.defn_type.is_base_type(builder.BaseTypeName.s_optional):
@@ -102,10 +112,37 @@ def emit_value_ts(
 
     elif isinstance(stype, builder.SpecTypeDefnStringEnum):
         return f"{refer_to(ctx, stype)}.{ts_enum_name(value, stype.name_case)}"
+    elif isinstance(stype, builder.SpecTypeDefnObject):
+        assert isinstance(value, dict), (
+            f"Expected dict value for {stype.name} but got {value}"
+        )
+        obj_out = "{"
+        did_emit = False
+        for prop_name, prop in (stype.properties or {}).items():
+            if prop_name not in value and prop.has_default:
+                value_to_emit = prop.default
+            elif prop_name not in value:
+                continue
+            else:
+                value_to_emit = value[prop_name]
+            did_emit = True
+            typescript_name = ts_name(prop.name, prop.name_case)
+            obj_out += f"\n{INDENT * (indent + 1)}{typescript_name}: {emit_value_ts(ctx, prop.spec_type, value_to_emit, indent=indent + 1)},"
+        whitespace = f"\n{INDENT * indent}" if did_emit else ""
+        obj_out += f"{whitespace}}} as const"
+        return obj_out
 
     raise Exception("invalid constant type", value, stype, type(stype))
 
 
+def emit_constant_ts(ctx: EmitTypescriptContext, sconst: builder.SpecConstant) -> None:
+    ctx.out.write("\n\n")
+    ctx.out.write(MODIFY_NOTICE)
+    value = emit_value_ts(ctx, sconst.value_type, sconst.value)
+    const_name = sconst.name.upper()
+    ctx.out.write(f"export const {const_name} = {value}\n")
+
+
 def emit_type_ts(ctx: EmitTypescriptContext, stype: builder.SpecType) -> None:
     if not isinstance(stype, builder.SpecTypeDefn):
         return
@@ -118,6 +155,7 @@ def emit_type_ts(ctx: EmitTypescriptContext, stype: builder.SpecType) -> None:
 
     if isinstance(stype, builder.SpecTypeDefnExternal):
         assert not stype.is_exported, "expecting private names"
+        ctx.out.write("\n")
         ctx.out.write(stype.external_map["ts"])
         ctx.out.write("\n")
         return

pkgs/type_spec/load_types.py

@@ -7,8 +7,8 @@ from shelljob import fs
 from pkgs.serialization import yaml
 
 from .builder import SpecBuilder
+from .builder_types import CrossOutputPaths
 from .config import Config
-from .cross_output_links import CrossOutputPaths
 
 ext_map = {
     ".ts": "typescript",
@@ -46,7 +46,7 @@ def load_types(config: Config) -> SpecBuilder | None:
         cross_output_paths = CrossOutputPaths(
             python_types_output=config.python.types_output,
             typescript_types_output=config.typescript.types_output,
-            typescript_routes_output=config.typescript.routes_output,
+            typescript_routes_output_by_endpoint=config.typescript.endpoint_to_routes_output,
             typespec_files_input=config.type_spec_types,
             # IMPROVE not sure how to know which one is the correct one in emit_typescript
         )

pkgs/type_spec/open_api_util.py

@@ -223,13 +223,28 @@ class OpenAPIUnionType(OpenAPIType):
         base_types: list[OpenAPIType],
         description: str | None = None,
         nullable: bool = False,
+        discriminator: str | None = None,
+        discriminator_map: dict[str, OpenAPIRefType] | None = None,
     ) -> None:
         self.base_types = base_types
+        self._discriminator = discriminator
+        self._discriminator_map = discriminator_map
         super().__init__(description=description, nullable=nullable)
 
     def asdict(self) -> dict[str, object]:
         # TODO: use parents description and nullable
-        return {"oneOf": [base_type.asdict() for base_type in self.base_types]}
+        return {
+            "oneOf": [base_type.asdict() for base_type in self.base_types],
+            "discriminator": {
+                "propertyName": self._discriminator,
+                "mapping": {
+                    discriminator_value: base_type.source
+                    for discriminator_value, base_type in self._discriminator_map.items()
+                },
+            }
+            if self._discriminator is not None and self._discriminator_map is not None
+            else None,
+        }
 
 
 class OpenAPIIntersectionType(OpenAPIType):

pkgs/type_spec/parts/base.ts.prepart

@@ -3,6 +3,7 @@
 // doesn't allow referring explicitly to global names (thus cannot override here)
 // IMPROVE: invert relationship for global.d.ts looks here instead
 import * as IO from 'io-ts';
+
 type localJsonScalar = JsonScalar
 type localJsonValue = JsonValue
 type localObjectId = ObjectId
@@ -28,3 +29,6 @@ export const IOJsonValue: IO.Type<JsonValue> = IO.recursion('JsonValue', () =>
 export interface nominal<T> {
   "nominal structural brand": T
 }
+
+// Ids matching a strict integer number are converted to integers
+export const ID_REGEX = /^-?[1-9][0-9]{0,20}$/

pkgs/type_spec/type_info/emit_type_info.py

@@ -41,10 +41,23 @@ def type_path_of(stype: builder.SpecType) -> object:  # NamePath
             parts: list[object] = ["$literal"]
             for parameter in stype.parameters:
                 assert isinstance(parameter, builder.SpecTypeLiteralWrapper)
+                emit_value = parameter.value
+                if isinstance(parameter.value_type, builder.SpecTypeDefnObject):
+                    emit_value = parameter.value
+                    assert isinstance(emit_value, (str, bool)), (
+                        f"invalid-literal-value:{emit_value}"
+                    )
+                elif isinstance(parameter.value_type, builder.SpecTypeDefnStringEnum):
+                    key = parameter.value
+                    assert isinstance(key, str)
+                    emit_value = parameter.value_type.values[key].value
+                else:
+                    raise Exception("unhandled-literal-type")
+
                 # This allows expansion to enum literal values later
                 parts.append([
                     "$value",
-                    parameter.value,
+                    emit_value,
                     type_path_of(parameter.value_type),
                 ])
             return parts
@@ -158,9 +171,16 @@ class MapTypeAlias(MapTypeBase):
     discriminator: str | None
 
 
+@dataclasses.dataclass
+class StringEnumValue:
+    value: str
+    label: str
+    deprecated: bool = False
+
+
 @dataclasses.dataclass
 class MapStringEnum(MapTypeBase):
-    values: dict[str, str]
+    values: dict[str, StringEnumValue]
 
 
 MapType = MapTypeObject | MapTypeAlias | MapStringEnum
@@ -267,7 +287,9 @@ def _extract_and_validate_layout(
     for group in ext_info.layout.groups:
         fields = set(group.fields or [])
         for field in fields:
-            assert field in stype.properties, f"layout-refers-to-missing-field:{field}"
+            assert field in stype.properties or field == DISCRIMINATOR_COMMON_NAME, (
+                f"layout-refers-to-missing-field:{field}"
+            )
 
         local_ref_name = None
         if group.ref_name is not None:
@@ -314,6 +336,9 @@ def _pull_property_from_type_recursively(
     return _pull_property_from_type_recursively(stype.base, property_name)
 
 
+DISCRIMINATOR_COMMON_NAME = "type"
+
+
 def _validate_type_ext_info(
     stype: builder.SpecTypeDefnObject,
 ) -> tuple[ExtInfoLayout | None, type_info_t.ExtInfo | None]:
@@ -324,12 +349,16 @@ def _validate_type_ext_info(
     if ext_info.label_fields is not None:
         assert stype.properties is not None
         for name in ext_info.label_fields:
+            if name == DISCRIMINATOR_COMMON_NAME:
+                continue
             prop = _pull_property_from_type_recursively(stype, name)
             assert prop is not None, f"missing-label-field:{name}"
 
     if ext_info.actions is not None:
         assert stype.properties is not None
         for action in ext_info.actions:
+            if action.property == DISCRIMINATOR_COMMON_NAME:
+                continue
             prop = _pull_property_from_type_recursively(stype, action.property)
             assert prop is not None, f"missing-action-field:{action.property}"
 
@@ -414,7 +443,11 @@ def _build_map_type(
             # IMPROVE: We probably want the label here, but this requires a change
             # to the front-end type-info and form code to handle
             values={
-                entry.value: (entry.label or entry.name)
+                entry.value: StringEnumValue(
+                    value=entry.value,
+                    label=entry.label or entry.name,
+                    deprecated=entry.deprecated,
+                )
                 for entry in stype.values.values()
             },
         )

pkgs/type_spec/ui_entry_actions/generate_ui_entry_actions.py

@@ -282,6 +282,7 @@ def generate_entry_actions_typescript(
     ctx = emit_typescript_util.EmitTypescriptContext(
         out=definition_buffer,
         namespace=index_namespace,
+        api_endpoints={},
     )
     builder.namespaces[index_namespace.name] = index_namespace
 

pkgs/type_spec/value_spec/__main__.py

@@ -20,7 +20,7 @@ The accepted argument type must accept "None", it is not implied.
 """
 
 import sys
-from typing import TypeVar, cast
+from typing import Match, Pattern, TypeVar, cast
 
 import regex as re
 
@@ -56,7 +56,7 @@ class Source:
     def has_more(self) -> bool:
         return self._at < len(self._text)
 
-    def match(self, expression: re.Pattern) -> re.Match | None:
+    def match(self, expression: Pattern[str]) -> Match[str] | None:
         self.skip_space()
         m = expression.match(self._text, self._at)
         if m is not None:

pkgs/type_spec/value_spec/emit_python.py

@@ -75,9 +75,14 @@ def _emit_function_wrapper(function: value_spec_t.Function) -> str:
             ):
                 python_type += " | None"
                 any_pass_null = True
+
+            if python_type.startswith("base_t.ExtJsonValue"):
+                return_statement = f"self._extract({index})"
+            else:
+                return_statement = f"cast({python_type}, self._extract({index}))"
             out.write(
                 f"""{INDENT}def get_{argument.ref_name}(self) -> {python_type}:
-{INDENT}{INDENT}return cast({python_type}, self._extract({index}))
+{INDENT}{INDENT}return {return_statement}
 """
             )
         out.write("\n")

uncountable/core/client.py

@@ -226,13 +226,15 @@ class Client(ClientMethods):
         except JSONDecodeError as e:
             raise SDKError("unable to process response", request_id=request_id) from e
 
-    def _send_request(self, request: requests.Request) -> requests.Response:
+    def _send_request(
+        self, request: requests.Request, *, timeout: float | None = None
+    ) -> requests.Response:
         if self._cfg.extra_headers is not None:
             request.headers = {**request.headers, **self._cfg.extra_headers}
         if self._cfg.transform_request is not None:
             request = self._cfg.transform_request(request)
         prepared_request = request.prepare()
-        response = self._session.send(prepared_request)
+        response = self._session.send(prepared_request, timeout=timeout)
         return response
 
     def do_request(self, *, api_request: APIRequest, return_type: type[DT]) -> DT:
@@ -257,7 +259,12 @@ class Client(ClientMethods):
         with push_scope_optional(self._cfg.logger, "api_call", attributes=attributes):
             if self._cfg.logger is not None:
                 self._cfg.logger.log_info(api_request.endpoint, attributes=attributes)
-            response = self._send_request(request)
+            timeout = (
+                api_request.request_options.timeout_secs
+                if api_request.request_options is not None
+                else None
+            )
+            response = self._send_request(request, timeout=timeout)
         response_data = self._get_response_json(response, request_id=request_id)
         cached_parser = self._get_cached_parser(return_type)
         try:

uncountable/integration/cli.py

@@ -1,49 +1,201 @@
 import argparse
+import json
+from typing import assert_never
 
+from dateutil import tz
 from opentelemetry.trace import get_current_span
+from tabulate import tabulate
 
 from uncountable.core.environment import get_local_admin_server_port
 from uncountable.integration.queue_runner.command_server.command_client import (
+    send_job_cancellation_message,
     send_job_queue_message,
+    send_list_queued_jobs_message,
+    send_retry_job_message,
+)
+from uncountable.integration.queue_runner.command_server.types import (
+    CommandCancelJobStatus,
 )
 from uncountable.integration.telemetry import Logger
 from uncountable.types import queued_job_t
 
 
-def main() -> None:
-    logger = Logger(get_current_span())
+def register_enqueue_job_parser(
+    sub_parser_manager: argparse._SubParsersAction,
+    parents: list[argparse.ArgumentParser],
+) -> None:
+    run_parser = sub_parser_manager.add_parser(
+        "run",
+        parents=parents,
+        help="Process a job with a given host and job ID",
+        description="Process a job with a given host and job ID",
+    )
+    run_parser.add_argument("job_id", type=str, help="The ID of the job to process")
+    run_parser.add_argument(
+        "--payload", type=str, help="JSON payload for webhook invocation context"
+    )
+
+    def _handle_enqueue_job(args: argparse.Namespace) -> None:
+        invocation_context: queued_job_t.InvocationContext
+
+        if args.payload is not None:
+            try:
+                webhook_payload = json.loads(args.payload)
+                invocation_context = queued_job_t.InvocationContextWebhook(
+                    webhook_payload=webhook_payload
+                )
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON payload: {e}")
+        else:
+            invocation_context = queued_job_t.InvocationContextManual()
+
+        send_job_queue_message(
+            job_ref_name=args.job_id,
+            payload=queued_job_t.QueuedJobPayload(
+                invocation_context=invocation_context
+            ),
+            host=args.host,
+            port=get_local_admin_server_port(),
+        )
+
+    run_parser.set_defaults(func=_handle_enqueue_job)
 
-    parser = argparse.ArgumentParser(
-        description="Process a job with a given command and job ID."
+
+def register_cancel_queued_job_parser(
+    sub_parser_manager: argparse._SubParsersAction,
+    parents: list[argparse.ArgumentParser],
+) -> None:
+    cancel_parser = sub_parser_manager.add_parser(
+        "cancel",
+        parents=parents,
+        help="Cancel a queued job with a given host and queued job UUID",
+        description="Cancel a job with a given host and queued job UUID",
+    )
+    cancel_parser.add_argument(
+        "uuid", type=str, help="The UUID of the queued job to cancel"
     )
 
-    parser.add_argument(
-        "command",
-        type=str,
-        choices=["run"],
-        help="The command to execute (e.g., 'run')",
+    def _handle_cancel_queued_job(args: argparse.Namespace) -> None:
+        resp = send_job_cancellation_message(
+            queued_job_uuid=args.uuid,
+            host=args.host,
+            port=get_local_admin_server_port(),
+        )
+
+        match resp:
+            case CommandCancelJobStatus.CANCELLED_WITH_RESTART:
+                print(
+                    "Job successfully cancelled. The integration server will restart."
+                )
+            case CommandCancelJobStatus.NO_JOB_FOUND:
+                print("Job not found.")
+            case CommandCancelJobStatus.JOB_ALREADY_COMPLETED:
+                print("Job already completed.")
+            case _:
+                assert_never(resp)
+
+    cancel_parser.set_defaults(func=_handle_cancel_queued_job)
+
+
+def register_list_queued_jobs(
+    sub_parser_manager: argparse._SubParsersAction,
+    parents: list[argparse.ArgumentParser],
+) -> None:
+    list_queued_jobs_parser = sub_parser_manager.add_parser(
+        "list-queued-jobs",
+        parents=parents,
+        help="List all jobs queued on the integration server",
+        description="List all jobs queued on the integration server",
+    )
+
+    list_queued_jobs_parser.add_argument(
+        "--offset",
+        type=int,
+        default=0,
+        help="Number of jobs to skip. Should be non-negative.",
+    )
+    list_queued_jobs_parser.add_argument(
+        "--limit",
+        type=int,
+        default=100,
+        help="A number between 1 and 100 specifying the number of jobs to return in the result set.",
     )
 
-    parser.add_argument("job_id", type=str, help="The ID of the job to process")
+    def _handle_list_queued_jobs(args: argparse.Namespace) -> None:
+        queued_jobs = send_list_queued_jobs_message(
+            offset=args.offset,
+            limit=args.limit,
+            host=args.host,
+            port=get_local_admin_server_port(),
+        )
 
-    parser.add_argument(
+        headers = ["UUID", "Job Ref Name", "Attempts", "Status", "Submitted At"]
+        rows = [
+            [
+                job.uuid,
+                job.job_ref_name,
+                job.num_attempts,
+                job.status,
+                job.submitted_at.ToDatetime(tz.UTC).astimezone(tz.tzlocal()),
+            ]
+            for job in queued_jobs
+        ]
+        print(tabulate(rows, headers=headers, tablefmt="grid"))
+
+    list_queued_jobs_parser.set_defaults(func=_handle_list_queued_jobs)
+
+
+def register_retry_job_parser(
+    sub_parser_manager: argparse._SubParsersAction,
+    parents: list[argparse.ArgumentParser],
+) -> None:
+    retry_failed_jobs_parser = sub_parser_manager.add_parser(
+        "retry-job",
+        parents=parents,
+        help="Retry failed job on the integration server",
+        description="Retry failed job on the integration server",
+    )
+
+    retry_failed_jobs_parser.add_argument(
+        "job_uuid", type=str, help="The uuid of the job to retry"
+    )
+
+    def _handle_retry_job(args: argparse.Namespace) -> None:
+        send_retry_job_message(
+            job_uuid=args.job_uuid,
+            host=args.host,
+            port=get_local_admin_server_port(),
+        )
+
+    retry_failed_jobs_parser.set_defaults(func=_handle_retry_job)
+
+
+def main() -> None:
+    logger = Logger(get_current_span())
+
+    main_parser = argparse.ArgumentParser(
+        description="Execute a given integrations server command."
+    )
+
+    base_parser = argparse.ArgumentParser(add_help=False)
+    base_parser.add_argument(
         "--host", type=str, default="localhost", nargs="?", help="The host to run on"
     )
 
-    args = parser.parse_args()
+    subparser_action = main_parser.add_subparsers(
+        dest="command",
+        required=True,
+        help="The command to execute (e.g., 'run')",
+    )
 
+    register_enqueue_job_parser(subparser_action, parents=[base_parser])
+    register_retry_job_parser(subparser_action, parents=[base_parser])
+    register_list_queued_jobs(subparser_action, parents=[base_parser])
+    register_cancel_queued_job_parser(subparser_action, parents=[base_parser])
+
+    args = main_parser.parse_args()
     with logger.push_scope(args.command):
-        if args.command == "run":
-            send_job_queue_message(
-                job_ref_name=args.job_id,
-                payload=queued_job_t.QueuedJobPayload(
-                    invocation_context=queued_job_t.InvocationContextManual()
-                ),
-                host=args.host,
-                port=get_local_admin_server_port(),
-            )
-        else:
-            parser.print_usage()
+        args.func(args)
 
 
 main()

uncountable/integration/executors/executors.py

@@ -88,7 +88,6 @@ def execute_job(
     job_definition: job_definition_t.JobDefinition,
     profile_metadata: job_definition_t.ProfileMetadata,
     args: JobArguments,
-    job_uuid: str,
 ) -> job_definition_t.JobResult:
     with args.logger.push_scope(job_definition.name) as job_logger:
         job = resolve_executor(job_definition.executor, profile_metadata)
@@ -104,7 +103,7 @@ def execute_job(
                 run_entity = _create_run_entity(
                     client=args.client,
                     logging_settings=job_definition.logging_settings,
-                    job_uuid=job_uuid,
+                    job_uuid=args.job_uuid,
                 )
             result = job.run_outer(args=args)
         except Exception as e:

uncountable/integration/executors/generic_upload_executor.py

@@ -41,7 +41,7 @@ def _get_extension(filename: str) -> str | None:
 
 def _run_keyword_detection(data: io.BytesIO, keyword: str) -> bool:
     try:
-        text = io.TextIOWrapper(data)
+        text = io.TextIOWrapper(data, encoding="utf-8")
         for line in text:
             if (
                 keyword in line

uncountable/integration/http_server/types.py

@@ -1,5 +1,6 @@
 import base64
 import functools
+import json
 from dataclasses import dataclass
 
 from flask.wrappers import Response
@@ -42,7 +43,8 @@ class HttpException(Exception):
 
     def make_error_response(self) -> Response:
         return Response(
-            status=self.error_code, response={"error": {"message": str(self)}}
+            status=self.error_code,
+            response=json.dumps({"error": {"message": str(self)}}),
         )