UncountablePythonSDK 0.0.190__py3-none-any.whl → 0.0.191__py3-none-any.whl

pkgs/type_spec/emit_typescript_util.py

@@ -175,6 +175,27 @@ def emit_constant_ts(ctx: EmitTypescriptContext, sconst: builder.SpecConstant) -
     ctx.out.write(f"export const {const_name} = {value}\n")
 
 
+def _collect_object_properties(
+    stype: builder.SpecTypeDefnObject,
+) -> dict[str, builder.SpecProperty]:
+    base_properties = (
+        _collect_object_properties(stype.base) if stype.base is not None else {}
+    )
+    return {**base_properties, **(stype.properties or {})}
+
+
+def _type_info_ref_type_ts(*, stype: builder.SpecTypeDefnObject, base_ref: str) -> str:
+    defaulted_names = sorted(
+        ts_property_name(prop)
+        for prop in _collect_object_properties(stype).values()
+        if prop.has_default and not prop.parse_require and prop.default is not None
+    )
+    if len(defaulted_names) == 0:
+        return f"{base_ref}.TypeInfoRef<{stype.name}>"
+    defaulted_keys = " | ".join(f'"{name}"' for name in defaulted_names)
+    return f"{base_ref}.TypeInfoRef<{stype.name}, {base_ref}.WithDefaultsApplied<{stype.name}, {defaulted_keys}>>"
+
+
 def emit_type_info_ref_ts(
     ctx: EmitTypescriptContext,
     stype: builder.SpecTypeDefn,
@@ -184,9 +205,9 @@ def emit_type_info_ref_ts(
     assert ctx.namespace.emit_type_info, (
         f"{ctx.namespace.name}: $emit_type_info_refs requires $emit_type_info"
     )
-    assert (
-        isinstance(stype, builder.SpecTypeDefnObject) and len(stype.parameters) == 0
-    ), (
+    if not isinstance(stype, builder.SpecTypeDefnObject):
+        return
+    assert len(stype.parameters) == 0, (
         f"{ctx.namespace.name}.{stype.name}: type info refs support non-generic objects only"
     )
 
@@ -195,7 +216,7 @@ def emit_type_info_ref_ts(
     ctx.out.write("\n")
     ctx.out.write(MODIFY_NOTICE)
     ctx.out.write(
-        f"export const {stype.name}TypeInfoRef: {base_ref}.TypeInfoRef<{stype.name}> = {{\n"
+        f"export const {stype.name}TypeInfoRef: {_type_info_ref_type_ts(stype=stype, base_ref=base_ref)} = {{\n"
     )
     ctx.out.write(f'{INDENT}typeName: "{ctx.namespace.name}.{stype.name}",\n')
     ctx.out.write("}\n")

pkgs/type_spec/parts/base.ts.prepart

@@ -40,11 +40,16 @@ declare const brand: unique symbol
 export type Branded<TBase, TBrand extends string> = TBase & { readonly [brand]: TBrand }
 
 declare const typeInfoValue: unique symbol
-export interface TypeInfoRef<T> {
+declare const typeInfoParsedValue: unique symbol
+export interface TypeInfoRef<T, TParsed extends T = T> {
   readonly typeName: string
   readonly [typeInfoValue]?: T
+  readonly [typeInfoParsedValue]?: TParsed
 }
 
+export type WithDefaultsApplied<T, TDefaultedKey extends keyof T> = Omit<T, TDefaultedKey> &
+  Required<Pick<T, TDefaultedKey>>
+
 // Ids matching a strict integer number are converted to integers
 export const ID_REGEX = /^-?[1-9][0-9]{0,20}$/
 // Matches ISO 8601 dates. Stricter than `Date.parse`.

uncountable/integration/cli.py

@@ -89,6 +89,8 @@ def register_cancel_queued_job_parser(
                 print(
                     "Job successfully cancelled. The integration server will restart."
                 )
+            case CommandCancelJobStatus.CANCELLED_QUEUED:
+                print("Queued job successfully cancelled.")
             case CommandCancelJobStatus.NO_JOB_FOUND:
                 print("Job not found.")
             case CommandCancelJobStatus.JOB_ALREADY_COMPLETED:

uncountable/integration/job.py

@@ -10,7 +10,6 @@ from pkgs.argument_parser import CachedParser
 from pkgs.serialization_util import serialize_for_api
 from uncountable.core.async_batch import AsyncBatchProcessor
 from uncountable.core.client import Client
-from uncountable.core.environment import get_local_admin_server_port
 from uncountable.core.file_upload import FileUpload
 from uncountable.core.types import AuthDetailsOAuth
 from uncountable.integration.http_server import (
@@ -18,12 +17,7 @@ from uncountable.integration.http_server import (
     GenericHttpResponse,
     HttpException,
 )
-from uncountable.integration.queue_runner.command_server.command_client import (
-    send_job_queue_message,
-)
-from uncountable.integration.queue_runner.command_server.types import (
-    CommandServerException,
-)
+from uncountable.integration.job_queue import JobQueue
 from uncountable.integration.request_context import extract_request_context
 from uncountable.integration.secret_retrieval import retrieve_secret
 from uncountable.integration.telemetry import JobLogger
@@ -111,11 +105,7 @@ class _RequestValidatorClient(Client):
         return self._oauth_bearer_token
 
 
-class CustomHttpJob(Job[GenericHttpRequest]):
-    @property
-    def payload_type(self) -> type[GenericHttpRequest]:
-        return GenericHttpRequest
-
+class HttpJob(ABC):
     @staticmethod
     @abstractmethod
     def validate_request(
@@ -130,6 +120,22 @@ class CustomHttpJob(Job[GenericHttpRequest]):
         """
         ...
 
+    @abstractmethod
+    def respond_to_request(
+        self,
+        *,
+        request: GenericHttpRequest,
+        job_definition: HttpJobDefinitionBase,
+        profile_meta: ProfileMetadata,
+        job_queue: JobQueue,
+    ) -> GenericHttpResponse: ...
+
+
+class CustomHttpJob(Job[GenericHttpRequest], HttpJob):
+    @property
+    def payload_type(self) -> type[GenericHttpRequest]:
+        return GenericHttpRequest
+
     @staticmethod
     def get_validated_oauth_request_user_id(
         *, profile_metadata: ProfileMetadata, request: GenericHttpRequest
@@ -162,6 +168,19 @@ class CustomHttpJob(Job[GenericHttpRequest]):
         """
         ...
 
+    @typing.override
+    def respond_to_request(
+        self,
+        *,
+        request: GenericHttpRequest,
+        job_definition: HttpJobDefinitionBase,
+        profile_meta: ProfileMetadata,
+        job_queue: JobQueue,
+    ) -> GenericHttpResponse:
+        return self.handle_request(
+            request=request, job_definition=job_definition, profile_meta=profile_meta
+        )
+
     def run_outer(self, args: JobArguments) -> JobResult:
         args.logger.log_warning(
             message=f"Unexpected call to run_outer for CustomHttpJob: {args.job_definition.id}"
@@ -189,7 +208,7 @@ def _resolve_webhook_signature_key(
     )
 
 
-class WebhookJob[WPT](Job[webhook_job_t.WebhookEventPayload]):
+class WebhookJob[WPT](Job[webhook_job_t.WebhookEventPayload], HttpJob):
     @property
     def payload_type(self) -> type[webhook_job_t.WebhookEventPayload]:
         return webhook_job_t.WebhookEventPayload
@@ -219,12 +238,14 @@ class WebhookJob[WPT](Job[webhook_job_t.WebhookEventPayload]):
         if not hmac.compare_digest(request_body_signature, passed_signature):
             raise HttpException.payload_failed_signature()
 
-    @staticmethod
-    def handle_request(
+    @typing.override
+    def respond_to_request(
+        self,
         *,
         request: GenericHttpRequest,
-        job_definition: job_definition_t.HttpJobDefinitionBase,
-        profile_meta: ProfileMetadata,  # noqa: ARG004
+        job_definition: HttpJobDefinitionBase,
+        profile_meta: ProfileMetadata,
+        job_queue: JobQueue,
     ) -> GenericHttpResponse:
         try:
             request_body = simplejson.loads(request.body_text)
@@ -234,19 +255,15 @@ class WebhookJob[WPT](Job[webhook_job_t.WebhookEventPayload]):
 
         request_context = extract_request_context(headers=request.headers)
 
-        try:
-            send_job_queue_message(
-                job_ref_name=job_definition.id,
-                payload=queued_job_t.QueuedJobPayload(
-                    invocation_context=queued_job_t.InvocationContextWebhook(
-                        webhook_payload=webhook_payload,
-                        request_context=request_context,
-                    )
-                ),
-                port=get_local_admin_server_port(),
-            )
-        except CommandServerException as e:
-            raise HttpException.unknown_error() from e
+        job_queue.enqueue(
+            job_ref_name=job_definition.id,
+            payload=queued_job_t.QueuedJobPayload(
+                invocation_context=queued_job_t.InvocationContextWebhook(
+                    webhook_payload=webhook_payload,
+                    request_context=request_context,
+                )
+            ),
+        )
 
         return GenericHttpResponse(
             response=simplejson.dumps(serialize_for_api(WebhookResponse())),

uncountable/integration/job_queue.py

@@ -0,0 +1,14 @@
+from uncountable.integration.queue_runner.datastore import Datastore
+from uncountable.types import queued_job_t
+
+
+class JobQueue:
+    def __init__(self, datastore: Datastore) -> None:
+        self._datastore = datastore
+
+    def enqueue(
+        self, *, job_ref_name: str, payload: queued_job_t.QueuedJobPayload
+    ) -> str:
+        return self._datastore.add_job_to_queue(
+            job_payload=payload, job_ref_name=job_ref_name
+        ).queued_job_uuid

uncountable/integration/ownership_lock.py

@@ -0,0 +1,77 @@
+import time
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from enum import StrEnum
+
+from opentelemetry.trace import get_current_span
+from sqlalchemy import Connection, text
+from sqlalchemy.engine import Engine
+
+from uncountable.integration.telemetry import Logger
+
+OWNERSHIP_ADVISORY_LOCK_KEY = 0x756E635F696E7467
+OWNERSHIP_LOCK_POLL_INTERVAL_SECS = 1
+
+
+class OwnershipOutcome(StrEnum):
+    ACQUIRED = "acquired"
+    SHUTDOWN_REQUESTED = "shutdown_requested"
+
+
+def _try_acquire_advisory_lock(connection: Connection) -> bool:
+    return bool(
+        connection.execute(
+            text("select pg_try_advisory_lock(:key)"),
+            {"key": OWNERSHIP_ADVISORY_LOCK_KEY},
+        ).scalar_one()
+    )
+
+
+def _release_advisory_lock(connection: Connection) -> None:
+    connection.execute(
+        text("select pg_advisory_unlock(:key)"),
+        {"key": OWNERSHIP_ADVISORY_LOCK_KEY},
+    )
+
+
+def _wait_for_advisory_lock(
+    *, connection: Connection, is_shutdown_requested: Callable[[], bool]
+) -> OwnershipOutcome:
+    logger = Logger(get_current_span())
+    if is_shutdown_requested():
+        return OwnershipOutcome.SHUTDOWN_REQUESTED
+    if _try_acquire_advisory_lock(connection):
+        return OwnershipOutcome.ACQUIRED
+    logger.log_info("waiting for the integration ownership lock held by another pod")
+    while not is_shutdown_requested():
+        time.sleep(OWNERSHIP_LOCK_POLL_INTERVAL_SECS)
+        if is_shutdown_requested():
+            return OwnershipOutcome.SHUTDOWN_REQUESTED
+        if _try_acquire_advisory_lock(connection):
+            logger.log_info("acquired the integration ownership lock")
+            return OwnershipOutcome.ACQUIRED
+    return OwnershipOutcome.SHUTDOWN_REQUESTED
+
+
+@contextmanager
+def hold_integration_ownership(
+    *, engine: Engine, is_shutdown_requested: Callable[[], bool]
+) -> Generator[OwnershipOutcome]:
+    dialect_name = engine.dialect.name
+    match dialect_name:
+        case "sqlite":
+            yield OwnershipOutcome.ACQUIRED
+        case "postgresql":
+            with engine.connect().execution_options(
+                isolation_level="AUTOCOMMIT"
+            ) as connection:
+                outcome = _wait_for_advisory_lock(
+                    connection=connection, is_shutdown_requested=is_shutdown_requested
+                )
+                try:
+                    yield outcome
+                finally:
+                    if outcome == OwnershipOutcome.ACQUIRED:
+                        _release_advisory_lock(connection)
+        case _:
+            raise ValueError(f"unsupported database dialect: {dialect_name!r}")

uncountable/integration/queue_runner/command_server/command_client.py

@@ -85,6 +85,8 @@ def send_job_cancellation_message(
                 return CommandCancelJobStatus.NO_JOB_FOUND
             case CancelJobStatus.CANCELLED_WITH_RESTART:
                 return CommandCancelJobStatus.CANCELLED_WITH_RESTART
+            case CancelJobStatus.CANCELLED_QUEUED:
+                return CommandCancelJobStatus.CANCELLED_QUEUED
             case CancelJobStatus.JOB_ALREADY_COMPLETED:
                 return CommandCancelJobStatus.JOB_ALREADY_COMPLETED
             case _:

uncountable/integration/queue_runner/command_server/command_server.py

@@ -94,6 +94,8 @@ async def serve(command_queue: CommandQueue, datastore: Datastore) -> None:
                     proto_status = CancelJobStatus.NO_JOB_FOUND
                 case CommandCancelJobStatus.CANCELLED_WITH_RESTART:
                     proto_status = CancelJobStatus.CANCELLED_WITH_RESTART
+                case CommandCancelJobStatus.CANCELLED_QUEUED:
+                    proto_status = CancelJobStatus.CANCELLED_QUEUED
                 case CommandCancelJobStatus.JOB_ALREADY_COMPLETED:
                     proto_status = CancelJobStatus.JOB_ALREADY_COMPLETED
                 case _:

uncountable/integration/queue_runner/command_server/protocol/command_server.proto

@@ -71,6 +71,7 @@ enum CancelJobStatus {
 	CANCELLED_WITH_RESTART = 0;
 	NO_JOB_FOUND = 1;
 	JOB_ALREADY_COMPLETED = 2;
+	CANCELLED_QUEUED = 3;
 }
 
 message CancelJobResult {

uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.py

@@ -29,7 +29,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\nQuncountable/integration/queue_runner/command_server/protocol/command_server.proto\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x11\x45nqueueJobRequest\x12\x14\n\x0cjob_ref_name\x18\x01 \x01(\t\x12\x1a\n\x12serialized_payload\x18\x02 \x01(\t"H\n\x10\x45nqueueJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1f\n\x0fRetryJobRequest\x12\x0c\n\x04uuid\x18\x01 \x01(\t"F\n\x0eRetryJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1a\n\x18VaccuumQueuedJobsRequest"\x19\n\x17VaccuumQueuedJobsResult"\x14\n\x12\x43heckHealthRequest"$\n\x11\x43heckHealthResult\x12\x0f\n\x07success\x18\x01 \x01(\x08"\xc2\x01\n\x15ListQueuedJobsRequest\x12\x0e\n\x06offset\x18\x01 \x01(\r\x12\r\n\x05limit\x18\x02 \x01(\r\x12\x0e\n\x06status\x18\x03 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x04 \x01(\t\x12\x32\n\x0esubmitted_from\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0csubmitted_to\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\xf4\x01\n\x14ListQueuedJobsResult\x12\x43\n\x0bqueued_jobs\x18\x01 \x03(\x0b\x32..ListQueuedJobsResult.ListQueuedJobsResultItem\x1a\x96\x01\n\x18ListQueuedJobsResultItem\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x02 \x01(\t\x12\x14\n\x0cnum_attempts\x18\x03 \x01(\x03\x12\x30\n\x0csubmitted_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06status\x18\x05 \x01(\t"$\n\x10\x43\x61ncelJobRequest\x12\x10\n\x08job_uuid\x18\x01 \x01(\t"3\n\x0f\x43\x61ncelJobResult\x12 \n\x06status\x18\x01 \x01(\x0e\x32\x10.CancelJobStatus")\n\x17ProvisionWebhookRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t".\n\x16ProvisionWebhookResult\x12\x14\n\x0cprofile_name\x18\x01 \x01(\t*Z\n\x0f\x43\x61ncelJobStatus\x12\x1a\n\x16\x43\x41NCELLED_WITH_RESTART\x10\x00\x12\x10\n\x0cNO_JOB_FOUND\x10\x01\x12\x19\n\x15JOB_ALREADY_COMPLETED\x10\x02\x32\xbd\x03\n\rCommandServer\x12\x35\n\nEnqueueJob\x12\x12.EnqueueJobRequest\x1a\x11.EnqueueJobResult"\x00\x12/\n\x08RetryJob\x12\x10.RetryJobRequest\x1a\x0f.RetryJobResult"\x00\x12\x38\n\x0b\x43heckHealth\x12\x13.CheckHealthRequest\x1a\x12.CheckHealthResult"\x00\x12\x41\n\x0eListQueuedJobs\x12\x16.ListQueuedJobsRequest\x1a\x15.ListQueuedJobsResult"\x00\x12J\n\x11VaccuumQueuedJobs\x12\x19.VaccuumQueuedJobsRequest\x1a\x18.VaccuumQueuedJobsResult"\x00\x12\x32\n\tCancelJob\x12\x11.CancelJobRequest\x1a\x10.CancelJobResult"\x00\x12G\n\x10ProvisionWebhook\x12\x18.ProvisionWebhookRequest\x1a\x17.ProvisionWebhookResult"\x00\x62\x06proto3'
+    b'\nQuncountable/integration/queue_runner/command_server/protocol/command_server.proto\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x11\x45nqueueJobRequest\x12\x14\n\x0cjob_ref_name\x18\x01 \x01(\t\x12\x1a\n\x12serialized_payload\x18\x02 \x01(\t"H\n\x10\x45nqueueJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1f\n\x0fRetryJobRequest\x12\x0c\n\x04uuid\x18\x01 \x01(\t"F\n\x0eRetryJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1a\n\x18VaccuumQueuedJobsRequest"\x19\n\x17VaccuumQueuedJobsResult"\x14\n\x12\x43heckHealthRequest"$\n\x11\x43heckHealthResult\x12\x0f\n\x07success\x18\x01 \x01(\x08"\xc2\x01\n\x15ListQueuedJobsRequest\x12\x0e\n\x06offset\x18\x01 \x01(\r\x12\r\n\x05limit\x18\x02 \x01(\r\x12\x0e\n\x06status\x18\x03 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x04 \x01(\t\x12\x32\n\x0esubmitted_from\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0csubmitted_to\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"\xf4\x01\n\x14ListQueuedJobsResult\x12\x43\n\x0bqueued_jobs\x18\x01 \x03(\x0b\x32..ListQueuedJobsResult.ListQueuedJobsResultItem\x1a\x96\x01\n\x18ListQueuedJobsResultItem\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x02 \x01(\t\x12\x14\n\x0cnum_attempts\x18\x03 \x01(\x03\x12\x30\n\x0csubmitted_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06status\x18\x05 \x01(\t"$\n\x10\x43\x61ncelJobRequest\x12\x10\n\x08job_uuid\x18\x01 \x01(\t"3\n\x0f\x43\x61ncelJobResult\x12 \n\x06status\x18\x01 \x01(\x0e\x32\x10.CancelJobStatus")\n\x17ProvisionWebhookRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t".\n\x16ProvisionWebhookResult\x12\x14\n\x0cprofile_name\x18\x01 \x01(\t*p\n\x0f\x43\x61ncelJobStatus\x12\x1a\n\x16\x43\x41NCELLED_WITH_RESTART\x10\x00\x12\x10\n\x0cNO_JOB_FOUND\x10\x01\x12\x19\n\x15JOB_ALREADY_COMPLETED\x10\x02\x12\x14\n\x10\x43\x41NCELLED_QUEUED\x10\x03\x32\xbd\x03\n\rCommandServer\x12\x35\n\nEnqueueJob\x12\x12.EnqueueJobRequest\x1a\x11.EnqueueJobResult"\x00\x12/\n\x08RetryJob\x12\x10.RetryJobRequest\x1a\x0f.RetryJobResult"\x00\x12\x38\n\x0b\x43heckHealth\x12\x13.CheckHealthRequest\x1a\x12.CheckHealthResult"\x00\x12\x41\n\x0eListQueuedJobs\x12\x16.ListQueuedJobsRequest\x1a\x15.ListQueuedJobsResult"\x00\x12J\n\x11VaccuumQueuedJobs\x12\x19.VaccuumQueuedJobsRequest\x1a\x18.VaccuumQueuedJobsResult"\x00\x12\x32\n\tCancelJob\x12\x11.CancelJobRequest\x1a\x10.CancelJobResult"\x00\x12G\n\x10ProvisionWebhook\x12\x18.ProvisionWebhookRequest\x1a\x17.ProvisionWebhookResult"\x00\x62\x06proto3'
 )
 
 _globals = globals()
@@ -42,7 +42,7 @@ _builder.BuildTopDescriptorsAndMessages(
 if not _descriptor._USE_C_DESCRIPTORS:
     DESCRIPTOR._loaded_options = None
     _globals["_CANCELJOBSTATUS"]._serialized_start = 1109
-    _globals["_CANCELJOBSTATUS"]._serialized_end = 1199
+    _globals["_CANCELJOBSTATUS"]._serialized_end = 1221
     _globals["_ENQUEUEJOBREQUEST"]._serialized_start = 118
     _globals["_ENQUEUEJOBREQUEST"]._serialized_end = 187
     _globals["_ENQUEUEJOBRESULT"]._serialized_start = 189
@@ -73,6 +73,6 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_PROVISIONWEBHOOKREQUEST"]._serialized_end = 1059
     _globals["_PROVISIONWEBHOOKRESULT"]._serialized_start = 1061
     _globals["_PROVISIONWEBHOOKRESULT"]._serialized_end = 1107
-    _globals["_COMMANDSERVER"]._serialized_start = 1202
-    _globals["_COMMANDSERVER"]._serialized_end = 1647
+    _globals["_COMMANDSERVER"]._serialized_start = 1224
+    _globals["_COMMANDSERVER"]._serialized_end = 1669
 # @@protoc_insertion_point(module_scope)

uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.pyi

@@ -16,10 +16,12 @@ class CancelJobStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     CANCELLED_WITH_RESTART: _ClassVar[CancelJobStatus]
     NO_JOB_FOUND: _ClassVar[CancelJobStatus]
     JOB_ALREADY_COMPLETED: _ClassVar[CancelJobStatus]
+    CANCELLED_QUEUED: _ClassVar[CancelJobStatus]
 
 CANCELLED_WITH_RESTART: CancelJobStatus
 NO_JOB_FOUND: CancelJobStatus
 JOB_ALREADY_COMPLETED: CancelJobStatus
+CANCELLED_QUEUED: CancelJobStatus
 
 class EnqueueJobRequest(_message.Message):
     __slots__ = ("job_ref_name", "serialized_payload")

uncountable/integration/queue_runner/command_server/types.py

@@ -8,6 +8,7 @@ from uncountable.types import queued_job_t
 
 class CommandCancelJobStatus(StrEnum):
     CANCELLED_WITH_RESTART = "cancelled_with_restart"
+    CANCELLED_QUEUED = "cancelled_queued"
     NO_JOB_FOUND = "no_job_found"
     JOB_ALREADY_COMPLETED = "job_already_completed"
 

uncountable/integration/queue_runner/datastore/datastore_sql_base.py

@@ -204,7 +204,19 @@ class DatastoreSqlBase(Datastore):
 
             return None
 
-    def load_job_queue(self) -> list[queued_job_t.QueuedJob]:
+    def list_queued_job_uuids(self) -> list[str]:
+        with self.session_maker() as session:
+            select_stmt = select(QueuedJob.id).filter(
+                or_(
+                    QueuedJob.status == queued_job_t.JobStatus.QUEUED,
+                    QueuedJob.status.is_(None),
+                )
+            )
+            return [row.id for row in session.execute(select_stmt)]
+
+    def load_queued_jobs(
+        self, *, queued_job_uuids: list[str]
+    ) -> list[queued_job_t.QueuedJob]:
         with self.session_maker() as session:
             select_stmt = (
                 select(
@@ -215,12 +227,7 @@ class DatastoreSqlBase(Datastore):
                     QueuedJob.status,
                     QueuedJob.submitted_at,
                 )
-                .filter(
-                    or_(
-                        QueuedJob.status == queued_job_t.JobStatus.QUEUED,
-                        QueuedJob.status.is_(None),
-                    )
-                )
+                .filter(QueuedJob.id.in_(queued_job_uuids))
                 .order_by(QueuedJob.submitted_at)
             )
 

uncountable/integration/queue_runner/datastore/interface.py

@@ -31,7 +31,12 @@ class Datastore(ABC):
     def increment_num_attempts(self, queued_job_uuid: str) -> int: ...
 
     @abstractmethod
-    def load_job_queue(self) -> list[queued_job_t.QueuedJob]: ...
+    def list_queued_job_uuids(self) -> list[str]: ...
+
+    @abstractmethod
+    def load_queued_jobs(
+        self, *, queued_job_uuids: list[str]
+    ) -> list[queued_job_t.QueuedJob]: ...
 
     @abstractmethod
     def get_next_queued_job_for_ref_name(

uncountable/integration/queue_runner/job_scheduler.py

@@ -31,6 +31,9 @@ from uncountable.types import job_definition_t, queued_job_t
 
 from .types import RESTART_EXIT_CODE, ResultQueue, ResultTask
 
+QUEUED_JOB_POLL_INTERVAL_SECS = 1
+RESTART_DELAY_SECS = 5
+
 
 @dataclass(kw_only=True, frozen=True)
 class JobListenerKey:
@@ -65,10 +68,12 @@ def on_worker_crash(
 
 
 def _start_workers(
+    *,
     process_pool: ProcessPoolExecutor,
     result_queue: ResultQueue,
     datastore: Datastore,
     worker_tasks: list[asyncio.Task],
+    running_job_uuids: set[str],
 ) -> dict[str, Worker]:
     profiles = load_profiles()
     job_queue_worker_lookup: dict[JobListenerKey, Worker] = {}
@@ -84,6 +89,7 @@ def _start_workers(
                     listen_queue=asyncio.Queue(),
                     result_queue=result_queue,
                     datastore=datastore,
+                    running_job_uuids=running_job_uuids,
                 )
                 task = asyncio.create_task(worker.run_worker_loop())
                 task.add_done_callback(on_worker_crash(job_worker_key))
@@ -108,11 +114,16 @@ async def start_scheduler(
     with ProcessPoolExecutor(
         max_workers=get_max_job_workers(), max_tasks_per_child=1
     ) as process_pool:
+        running_job_uuids: set[str] = set()
         job_worker_lookup = _start_workers(
-            process_pool, result_queue, datastore=datastore, worker_tasks=worker_tasks
+            process_pool=process_pool,
+            result_queue=result_queue,
+            datastore=datastore,
+            worker_tasks=worker_tasks,
+            running_job_uuids=running_job_uuids,
         )
 
-        queued_jobs = datastore.load_job_queue()
+        dispatched_job_uuids: set[str] = set()
 
         async def enqueue_queued_job(queued_job: queued_job_t.QueuedJob) -> None:
             try:
@@ -123,8 +134,22 @@ async def start_scheduler(
                     queued_job.queued_job_uuid, queued_job_t.JobStatus.FAILED
                 )
                 return
+            dispatched_job_uuids.add(queued_job.queued_job_uuid)
             await worker.listen_queue.put(queued_job)
 
+        async def _dispatch_undispatched_queued_jobs() -> None:
+            undispatched_job_uuids = [
+                queued_job_uuid
+                for queued_job_uuid in datastore.list_queued_job_uuids()
+                if queued_job_uuid not in dispatched_job_uuids
+            ]
+            if len(undispatched_job_uuids) == 0:
+                return
+            for queued_job in datastore.load_queued_jobs(
+                queued_job_uuids=undispatched_job_uuids
+            ):
+                await enqueue_queued_job(queued_job)
+
         async def _enqueue_or_deduplicate_job(
             job_ref_name: str,
             payload: queued_job_t.QueuedJobPayload,
@@ -172,30 +197,36 @@ async def start_scheduler(
                 )
                 return
 
-            if queued_job.status == queued_job_t.JobStatus.QUEUED:
-                datastore.remove_job_from_queue(command.queued_job_uuid)
+            if queued_job.status != queued_job_t.JobStatus.QUEUED:
                 await command.response_queue.put(
                     CommandCancelJobResponse(
-                        status=CommandCancelJobStatus.CANCELLED_WITH_RESTART
+                        status=CommandCancelJobStatus.JOB_ALREADY_COMPLETED
                     )
                 )
+                return
 
-                logger.log_debug(
-                    "Job cancelled successfully.",
-                    attributes={"job.queued_job_uuid": command.queued_job_uuid},
-                )
-
-                def delayed_exit() -> None:
-                    os._exit(RESTART_EXIT_CODE)
-
-                threading.Timer(interval=5, function=delayed_exit).start()
-
+            datastore.remove_job_from_queue(command.queued_job_uuid)
+            if command.queued_job_uuid in running_job_uuids:
+                cancel_status = CommandCancelJobStatus.CANCELLED_WITH_RESTART
+                threading.Timer(
+                    interval=RESTART_DELAY_SECS,
+                    function=os._exit,
+                    args=[RESTART_EXIT_CODE],
+                ).start()
             else:
-                await command.response_queue.put(
-                    CommandCancelJobResponse(
-                        status=CommandCancelJobStatus.JOB_ALREADY_COMPLETED
-                    )
-                )
+                dispatched_job_uuids.discard(command.queued_job_uuid)
+                cancel_status = CommandCancelJobStatus.CANCELLED_QUEUED
+
+            await command.response_queue.put(
+                CommandCancelJobResponse(status=cancel_status)
+            )
+            logger.log_debug(
+                "Job cancelled successfully.",
+                attributes={
+                    "job.queued_job_uuid": command.queued_job_uuid,
+                    "job.cancel_status": cancel_status,
+                },
+            )
 
         async def _handle_retry_job_command(command: CommandRetryJob) -> None:
             queued_job = datastore.retry_job(command.queued_job_uuid)
@@ -221,17 +252,21 @@ async def start_scheduler(
                 attributes={"job.queued_job_uuids": deleted_uuids},
             )
 
-        for queued_job in queued_jobs:
-            await enqueue_queued_job(queued_job)
+        await _dispatch_undispatched_queued_jobs()
 
         result_task: ResultTask = asyncio.create_task(result_queue.get())
         command_task: CommandTask = asyncio.create_task(command_queue.get())
+        poll_task = asyncio.create_task(asyncio.sleep(QUEUED_JOB_POLL_INTERVAL_SECS))
         shutdown_task: asyncio.Task[object] | None = None
         if shutdown_event is not None:
             shutdown_task = asyncio.create_task(shutdown_event.wait())
 
         while True:
-            waitables: list[asyncio.Task[object]] = [result_task, command_task]
+            waitables: list[asyncio.Task[object]] = [
+                result_task,
+                command_task,
+                poll_task,
+            ]
             if shutdown_task is not None:
                 waitables.append(shutdown_task)
 
@@ -246,6 +281,7 @@ async def start_scheduler(
                         worker_task.cancel()
                     command_task.cancel()
                     result_task.cancel()
+                    poll_task.cancel()
                     process_pool.shutdown(wait=False, cancel_futures=True)
                     return
                 if task == command_task:
@@ -275,4 +311,11 @@ async def start_scheduler(
                                 queued_job_result.queued_job_uuid,
                                 queued_job_t.JobStatus.FAILED,
                             )
+                    dispatched_job_uuids.discard(queued_job_result.queued_job_uuid)
+                    running_job_uuids.discard(queued_job_result.queued_job_uuid)
                     result_task = asyncio.create_task(result_queue.get())
+                elif task == poll_task:
+                    await _dispatch_undispatched_queued_jobs()
+                    poll_task = asyncio.create_task(
+                        asyncio.sleep(QUEUED_JOB_POLL_INTERVAL_SECS)
+                    )

uncountable/integration/queue_runner/worker.py

@@ -21,17 +21,25 @@ class Worker:
         listen_queue: ListenQueue,
         result_queue: ResultQueue,
         datastore: Datastore,
+        running_job_uuids: set[str],
     ) -> None:
         self.process_pool = process_pool
         self.listen_queue = listen_queue
         self.result_queue = result_queue
         self.datastore = datastore
+        self.running_job_uuids = running_job_uuids
 
     async def run_worker_loop(self) -> None:
         logger = Logger(get_current_span())
         while True:
             try:
                 queued_job = await self.listen_queue.get()
+                if (
+                    self.datastore.get_queued_job(uuid=queued_job.queued_job_uuid)
+                    is None
+                ):
+                    continue
+                self.running_job_uuids.add(queued_job.queued_job_uuid)
                 self.datastore.increment_num_attempts(queued_job.queued_job_uuid)
                 loop = asyncio.get_event_loop()
                 result = await loop.run_in_executor(

uncountable/integration/scheduler.py

@@ -12,7 +12,12 @@ from typing import assert_never
 from opentelemetry.trace import get_current_span
 
 from uncountable.core.environment import get_local_admin_server_port
+from uncountable.integration.db.connect import IntegrationDBService, create_db_engine
 from uncountable.integration.entrypoint import main as cron_target
+from uncountable.integration.ownership_lock import (
+    OwnershipOutcome,
+    hold_integration_ownership,
+)
 from uncountable.integration.queue_runner.command_server import (
     CommandServerTimeout,
     check_health,
@@ -259,15 +264,16 @@ def main() -> None:
             )
         )
 
-    _start_queue_runner()
+    shutdown_requested = False
 
-    cron_process = multiprocessing.Process(target=cron_target)
-    cron_process.start()
-    add_process(
-        ProcessInfo(
-            key=CoreProcessKey(name=ProcessName.CRON_SERVER), process=cron_process
-        )
-    )
+    def _sigterm_handler(signum: int, frame: object) -> None:
+        nonlocal shutdown_requested
+        shutdown_requested = True
+
+    def _is_shutdown_requested() -> bool:
+        return shutdown_requested
+
+    signal.signal(signal.SIGTERM, _sigterm_handler)
 
     webhook_server_process = subprocess.Popen([
         sys.executable,
@@ -284,59 +290,82 @@ def main() -> None:
         )
     )
 
-    services_by_id: dict[str, ServiceToStart] = {
-        service_to_start.service.id: service_to_start
-        for service_to_start in select_enabled_services(load_profiles())
-    }
-    for service_to_start in services_by_id.values():
-        _start_service(service_to_start)
+    def _run_owned_processes() -> None:
+        if _is_shutdown_requested():
+            return
+        _start_queue_runner()
+        if _is_shutdown_requested():
+            return
 
-    shutdown_requested = False
+        cron_process = multiprocessing.Process(target=cron_target)
+        cron_process.start()
+        add_process(
+            ProcessInfo(
+                key=CoreProcessKey(name=ProcessName.CRON_SERVER), process=cron_process
+            )
+        )
 
-    def _sigterm_handler(signum: int, frame: object) -> None:
-        nonlocal shutdown_requested
-        shutdown_requested = True
+        services_by_id: dict[str, ServiceToStart] = {
+            service_to_start.service.id: service_to_start
+            for service_to_start in select_enabled_services(load_profiles())
+        }
+        for service_to_start in services_by_id.values():
+            if _is_shutdown_requested():
+                return
+            _start_service(service_to_start)
 
-    signal.signal(signal.SIGTERM, _sigterm_handler)
+        try:
+            while not _is_shutdown_requested():
+                process_alarm = check_process_alarms(logger, processes=processes)
+                match process_alarm:
+                    case ProcessAlarmRestart():
+                        match process_alarm.process.key:
+                            case CoreProcessKey() as core_key:
+                                match core_key.name:
+                                    case ProcessName.QUEUE_RUNNER:
+                                        del processes[
+                                            CoreProcessKey(
+                                                name=ProcessName.QUEUE_RUNNER
+                                            )
+                                        ]
+                                        _start_queue_runner()
+                                    case (
+                                        ProcessName.CRON_SERVER
+                                        | ProcessName.WEBHOOK_SERVER
+                                    ):
+                                        raise NotImplementedError(
+                                            f"restarting {core_key.label} not yet implemented"
+                                        )
+                                    case _:
+                                        assert_never(core_key.name)
+                            case ServiceProcessKey() as service_key:
+                                del processes[service_key]
+                                _start_service(services_by_id[service_key.service_id])
+                            case _:
+                                assert_never(process_alarm.process.key)
+                    case ProcessAlarmShutdownAll():
+                        handle_shutdown(logger, processes)
+                        sys.exit(1)
+                    case None:
+                        pass
+                    case _:
+                        assert_never(process_alarm)
+                time.sleep(1)
+        except KeyboardInterrupt:
+            pass
 
-    try:
-        while not shutdown_requested:
-            process_alarm = check_process_alarms(logger, processes=processes)
-            match process_alarm:
-                case ProcessAlarmRestart():
-                    match process_alarm.process.key:
-                        case CoreProcessKey() as core_key:
-                            match core_key.name:
-                                case ProcessName.QUEUE_RUNNER:
-                                    del processes[
-                                        CoreProcessKey(name=ProcessName.QUEUE_RUNNER)
-                                    ]
-                                    _start_queue_runner()
-                                case (
-                                    ProcessName.CRON_SERVER | ProcessName.WEBHOOK_SERVER
-                                ):
-                                    raise NotImplementedError(
-                                        f"restarting {core_key.label} not yet implemented"
-                                    )
-                                case _:
-                                    assert_never(core_key.name)
-                        case ServiceProcessKey() as service_key:
-                            del processes[service_key]
-                            _start_service(services_by_id[service_key.service_id])
-                        case _:
-                            assert_never(process_alarm.process.key)
-                case ProcessAlarmShutdownAll():
-                    handle_shutdown(logger, processes)
-                    sys.exit(1)
-                case None:
-                    pass
-                case _:
-                    assert_never(process_alarm)
-            time.sleep(1)
-    except KeyboardInterrupt:
-        pass
-
-    handle_shutdown(logger, processes=processes)
+    with hold_integration_ownership(
+        engine=create_db_engine(IntegrationDBService.RUNNER),
+        is_shutdown_requested=_is_shutdown_requested,
+    ) as ownership_outcome:
+        match ownership_outcome:
+            case OwnershipOutcome.ACQUIRED:
+                _run_owned_processes()
+            case OwnershipOutcome.SHUTDOWN_REQUESTED:
+                pass
+            case _:
+                assert_never(ownership_outcome)
+        handle_shutdown(logger, processes=processes)
 
 
 if __name__ == "__main__":

uncountable/integration/webhook_server/entrypoint.py

@@ -8,9 +8,12 @@ from uncountable.core.environment import (
     get_http_server_port,
     get_server_env,
 )
+from uncountable.integration.db.connect import IntegrationDBService, create_db_engine
 from uncountable.integration.executors.script_executor import resolve_script_executor
 from uncountable.integration.http_server import GenericHttpRequest, HttpException
-from uncountable.integration.job import CustomHttpJob, WebhookJob
+from uncountable.integration.job import HttpJob
+from uncountable.integration.job_queue import JobQueue
+from uncountable.integration.queue_runner.datastore import create_datastore
 from uncountable.integration.scan_profiles import load_profiles
 from uncountable.integration.telemetry import Logger
 from uncountable.types import job_definition_t
@@ -42,6 +45,7 @@ def register_route(
     server_logger: Logger,
     profile_meta: job_definition_t.ProfileMetadata,
     job: HttpJobDefinition,
+    job_queue: JobQueue,
 ) -> None:
     route = f"/{profile_meta.name}/{job.id}"
     log_attributes = {
@@ -69,9 +73,9 @@ def register_route(
                 job_instance = resolve_script_executor(
                     executor=job.executor, profile_metadata=profile_meta
                 )
-                if not isinstance(job_instance, (CustomHttpJob, WebhookJob)):
+                if not isinstance(job_instance, HttpJob):
                     raise HttpException.configuration_error(
-                        message="[internal] http job must descend from CustomHttpJob"
+                        message="[internal] http job must descend from HttpJob"
                     )
                 http_request = GenericHttpRequest(
                     body_base64=base64.b64encode(flask.request.get_data()).decode(),
@@ -81,8 +85,11 @@ def register_route(
                 job_instance.validate_request(
                     request=http_request, job_definition=job, profile_meta=profile_meta
                 )
-                http_response = job_instance.handle_request(
-                    request=http_request, job_definition=job, profile_meta=profile_meta
+                http_response = job_instance.respond_to_request(
+                    request=http_request,
+                    job_definition=job,
+                    profile_meta=profile_meta,
+                    job_queue=job_queue,
                 )
 
                 return flask.make_response(
@@ -121,6 +128,10 @@ def register_route(
 def main() -> None:
     app.add_url_rule("/health", "health", lambda: ("OK", 200))
 
+    job_queue = JobQueue(
+        create_datastore(create_db_engine(IntegrationDBService.RUNNER))
+    )
+
     profiles = load_profiles()
     for profile_metadata in profiles:
         server_logger = Logger(get_current_span())
@@ -137,6 +148,7 @@ def main() -> None:
                         server_logger=server_logger,
                         profile_meta=profile_metadata,
                         job=job,
+                        job_queue=job_queue,
                     )
                 case _:
                     assert_never(job)

uncountable/types/entity_t.py

@@ -313,6 +313,7 @@ __all__: list[str] = [
         "user_group": "User Group",
         "user_group_assignment": "User Group Assignment",
         "user_list_set": "User List Set",
+        "user_preference": "User Preference",
         "value_spec_property": "Value Spec Property",
         "webhook": "Webhook",
         "webhook_event": "Webhook Event",
@@ -610,6 +611,7 @@ class EntityType(StrEnum):
     USER_GROUP = "user_group"
     USER_GROUP_ASSIGNMENT = "user_group_assignment"
     USER_LIST_SET = "user_list_set"
+    USER_PREFERENCE = "user_preference"
     VALUE_SPEC_PROPERTY = "value_spec_property"
     WEBHOOK = "webhook"
     WEBHOOK_EVENT = "webhook_event"

{uncountablepythonsdk-0.0.190.dist-info → uncountablepythonsdk-0.0.191.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: UncountablePythonSDK
-Version: 0.0.190
+Version: 0.0.191
 Summary: Uncountable SDK
 Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
 Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git

{uncountablepythonsdk-0.0.190.dist-info → uncountablepythonsdk-0.0.191.dist-info}/RECORD

@@ -104,7 +104,7 @@ pkgs/type_spec/emit_open_api_util.py,sha256=bTmRvrGP82-eB75hwf9ySI7pDEC87FNQTF18
 pkgs/type_spec/emit_python.py,sha256=6ZGh9F73Eqb4W8-bHLYkV5k8snEAQH9aXL9_T4onblM,65465
 pkgs/type_spec/emit_scss.py,sha256=6psmBAJyu7cvfQAIvIIMk2U6VFG_xeWy1Ea4afv0XyM,1497
 pkgs/type_spec/emit_typescript.py,sha256=XRMMovJz_g2j0rpn1Nrz4uzHRCs6O6HLIxdEB1QMNIE,11939
-pkgs/type_spec/emit_typescript_util.py,sha256=1-Q4p1ztLL3veQpMMeY3qtjqNFHlbr_IOCb7P7FzVZI,16764
+pkgs/type_spec/emit_typescript_util.py,sha256=X3EUGcZ8qJ2vkYZSjeV774fTa19t4VuYXNRKrFU9WDs,17653
 pkgs/type_spec/load_types.py,sha256=k5hWxFvhj8aT_9vAwtkjn9OdudQFK9IIUzZI7-Gpx2Y,5277
 pkgs/type_spec/non_discriminated_union_exceptions.py,sha256=jgQ4Naz8A6FbhgfR3c9SI4IMvkkft0MT6CyQXWu9RB0,559
 pkgs/type_spec/open_api_util.py,sha256=LX6ny4xNOWXQO3qWVM8elrSDTqWsj_7jFplax2POShE,7992
@@ -115,7 +115,7 @@ pkgs/type_spec/actions_registry/__main__.py,sha256=hWUnmiG68xeW5GhnIolfHOvPY2GI_
 pkgs/type_spec/actions_registry/emit_python.py,sha256=atY1_Z9Ue6r1nBsMeoeX02vQCc_UpAnDLK7gdVNemZ4,3753
 pkgs/type_spec/actions_registry/emit_typescript.py,sha256=XqmbX5OrgjdCGQf4-iNc_aZ7JiXgaVGt1wJcpAUh8SA,6787
 pkgs/type_spec/parts/base.py.prepart,sha256=ju50TWzxO7qgLjRwJSGJYnKGiLBT1g_8aC-01-f2fhk,2286
-pkgs/type_spec/parts/base.ts.prepart,sha256=bSnYpl7ZPTcaVwMdZOWONJM7_BtSkUoA9vwMN2Niqac,1959
+pkgs/type_spec/parts/base.ts.prepart,sha256=MOAkZgpF6KDWbIRIQODz8A-uXIcf04x3mNWG_hSfYpg,2203
 pkgs/type_spec/type_info/__main__.py,sha256=TLNvCHGcmaj_8Sj5bAQNpuNaaw2dpDzoFDWZds0V4Qo,1002
 pkgs/type_spec/type_info/emit_type_info.py,sha256=xoEFfK3H4r6w3Sur6-55I2efevdV9nKVHw1VtuCElR8,17990
 pkgs/type_spec/ui_entry_actions/__init__.py,sha256=WiHE_BexOEZWbkkbD7EnFau1aMLNmfgQywG9PTQNCkw,135
@@ -139,14 +139,16 @@ uncountable/core/query/column.py,sha256=WR5VRX30kGrObMxxWhnaH-ueJkn_P-K73kz0GSfZ
 uncountable/core/query/row.py,sha256=dy68YKvMz336nyOsj_g2oNTLKBuUnKRUz5n99sApfT0,1600
 uncountable/core/query/types.py,sha256=xZgc0_O1PIxZoJWQhxkg3qnSvHj3yt-bv9QX1UCOFGw,481
 uncountable/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-uncountable/integration/cli.py,sha256=5RzlfScaZHapiXp4XoaYh0QqSenIjkEf-kyjEue1ycg,9301
+uncountable/integration/cli.py,sha256=0s0cY9FcYHqWwpR7eeffGgsyrECOFacn6jMRJSTbD4o,9419
 uncountable/integration/construct_client.py,sha256=9c8ZqMjyXebsptiGkJbg53_af4TLOumY1hOQtIbC15w,2229
 uncountable/integration/cron.py,sha256=6eH-kIs3sdYPCyb62_L2M7U_uQTdMTdwY5hreEJb0hw,887
 uncountable/integration/entrypoint.py,sha256=seXMXxUPjBLzpP6UAB6fRVodkgGea6vnl2oy4FWvCyE,591
-uncountable/integration/job.py,sha256=Jeiiwvk-T3TxNYpesGbyTuQirw8gX1qQlmCIZ7r7Wuc,9187
+uncountable/integration/job.py,sha256=gMcPvV_8mS1mtyYCg7fkRPB0Nj4K1VCD2KrTILRwUpE,9460
+uncountable/integration/job_queue.py,sha256=6hnNWvifqtup2e3uG-EWKjcCNF-Xl4KW_432GaXBoAc,461
+uncountable/integration/ownership_lock.py,sha256=UDv9MKWiDRkTMUlznBLSd-hqtQcrbXEBQ5aU7PBA0E8,2638
 uncountable/integration/request_context.py,sha256=N_FJJxqvfUJ0yV9h3I3vFTGNJiDfyLYObczcYa44pw8,999
 uncountable/integration/scan_profiles.py,sha256=Fl8vr2jhW9BmnatLHZSvMnruUUXYgnXVVlbftYetY9g,3647
-uncountable/integration/scheduler.py,sha256=9nd2OZli9qQ9kQ5hIfR6nb1JBmjdSHu5yAS7jEyPq7o,11153
+uncountable/integration/scheduler.py,sha256=OvWK2VbqDM8LhuthVf90oJpCIRNDPnT4zyyi1TDx6wA,12425
 uncountable/integration/server.py,sha256=wnqpowr3xdAXfi4gCTdfFBvmQVyTro439kqrXewBauM,6002
 uncountable/integration/service.py,sha256=i3GSrNu7QzMfK1jOIKLdJQCM5Tzp9eF5nfWldj6Cx-M,2144
 uncountable/integration/service_runner.py,sha256=u9J90fh4CA9nluvfuTUaF6MhCJc2VKMD4Ahmk2x6Hzc,1435
@@ -165,32 +167,32 @@ uncountable/integration/http_server/types.py,sha256=0J59HhEyntDjy0o37Qze0v0o-wC8
 uncountable/integration/queue_runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 uncountable/integration/queue_runner/job_process.py,sha256=OONJ7aLsE80qIJ2ABLY3HYyKI7VdpBjI_pPQ3GCC13U,2586
 uncountable/integration/queue_runner/job_runner.py,sha256=Yh9hYfEYOya3BR6ypsal3LKPKrrEb4VQwfGuh4qyU2w,6115
-uncountable/integration/queue_runner/job_scheduler.py,sha256=ePGX7QXSDReU7NZX6YqtifQPEB0VH6NjZKHe3LGf7Zw,10798
+uncountable/integration/queue_runner/job_scheduler.py,sha256=1gdm0PRbjZBFlwWM0sLIkhVFlv9K0x5BjqRvbFMsZKA,12599
 uncountable/integration/queue_runner/queue_runner.py,sha256=G3HhYwKqjx863C_Kjg7rWe5BXy9d-0jkzIf9xuKCYGo,1181
 uncountable/integration/queue_runner/types.py,sha256=8HS6KnYMS_vc5XHeMpg0BFAQC-5P3QLzd-aDYDMMt3E,244
-uncountable/integration/queue_runner/worker.py,sha256=hyLJxBE5GQNIXo7YAYJfj0t1vvNR_6wMTrbBuCiBZjc,1688
+uncountable/integration/queue_runner/worker.py,sha256=b-tmOLNMaSOA3VQRtOvymBZYyZomq0kSA5LpQlWI_jM,2027
 uncountable/integration/queue_runner/command_server/__init__.py,sha256=hMCDLWct8zW4B2a9BaIAsMhtaEgFlxONjeow-6nf6dg,675
-uncountable/integration/queue_runner/command_server/command_client.py,sha256=s2E4v3ATGF1Ii1ZyP13Pa4Abdnj1ocB5tPUgP7J5-J0,6394
-uncountable/integration/queue_runner/command_server/command_server.py,sha256=-8FhL67-XQxx_sIIoQ0ImCOJYZHjwCpucCuayvRdocQ,8614
+uncountable/integration/queue_runner/command_server/command_client.py,sha256=9WTKWCD6o3h9xSOJP4lK8oyGzlQgXeoOLO8D7fWJZy4,6508
+uncountable/integration/queue_runner/command_server/command_server.py,sha256=ixcamAbYZ1R0C9-kV3aCUD0xKOov38bV7GW0UMyjcIM,8744
 uncountable/integration/queue_runner/command_server/constants.py,sha256=7J9mQIAMOfV50wnwpn7HgrPFEi3Ritj6HwrGYwxGLoU,88
 uncountable/integration/queue_runner/command_server/provision_webhook.py,sha256=TqH_6YjRZz8EHMKxOcCxvmK6vweR8nuY8ilWNEQoogk,1010
-uncountable/integration/queue_runner/command_server/types.py,sha256=ZNqJE6b6rfMZGdcPS-7umB_8x2a7dzTfRBsdGoCzDjY,2077
+uncountable/integration/queue_runner/command_server/types.py,sha256=PCEhvCMxLW1GlDjFHAXQmR5R3-0bd9To44XeOW3PLEQ,2119
 uncountable/integration/queue_runner/command_server/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-uncountable/integration/queue_runner/command_server/protocol/command_server.proto,sha256=mvgCHAIQocdtou7e8SZmmJIBK7jIzJ2fT4yDz6Bur3c,1869
-uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.py,sha256=voigJ1ScgpWx0FL59pKinN8MLj0seuqoZuOqvCtSPss,6013
-uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.pyi,sha256=CkaTBkhsGngUxD-am691wKFAARCrivqiJHxP1pLe7jw,5984
+uncountable/integration/queue_runner/command_server/protocol/command_server.proto,sha256=YltIWS_vVyuOuaj0HgIYONU0aApIcyVkqM5PF0s9TSQ,1892
+uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.py,sha256=2vzQZ8qLnSvhw4FhFVo7CcgvkFDuQUywamuBE55U-Jw,6057
+uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.pyi,sha256=CUurj3pVwW4Zact1i7uOW4vjHa46j6kjQaQpXmnkqrk,6067
 uncountable/integration/queue_runner/command_server/protocol/command_server_pb2_grpc.py,sha256=wzPYtvuccPVW1qEXaMxU-KB3ptnZYnjHsuit6L1Pk6U,17415
 uncountable/integration/queue_runner/datastore/__init__.py,sha256=VdoQquDSbXvlRbluebD_XavVt4AdKvmvg-8hV1yTbXo,298
 uncountable/integration/queue_runner/datastore/datastore_postgres.py,sha256=oKRwOWlxgodJyqKr-jDcP9VbXKKTrU26lI-7m4HFBuk,359
-uncountable/integration/queue_runner/datastore/datastore_sql_base.py,sha256=eDQFDzn8y1IHoA888Hy69dUNl9MeCFQWW_VmnFqKdfo,11107
+uncountable/integration/queue_runner/datastore/datastore_sql_base.py,sha256=UE2dc0QS2mx_tTVYxGAE0SXkRaXRhKEXlrqkWIb29nE,11390
 uncountable/integration/queue_runner/datastore/datastore_sqlite.py,sha256=W9hWKKvZrp3JyaO_tmK-qZL3ACnMnCfYwvDSJHHd6qs,1197
 uncountable/integration/queue_runner/datastore/factory.py,sha256=bv_7BxQpgjrggJKz5YDllgP_BihEJ0-CdpLvaG4ooj4,868
-uncountable/integration/queue_runner/datastore/interface.py,sha256=kuAU8lrjVN3NTllo99nfJwBojt5dg_wgdivyYCB4EbY,1581
+uncountable/integration/queue_runner/datastore/interface.py,sha256=xIToMWQmPELuEAznfQRbbNWyBjhr-4moeHP5NT5uzFs,1704
 uncountable/integration/queue_runner/datastore/model.py,sha256=06_fGS0QTALEA_381I1zz63yXYRooaAVJdfYi0ubSd0,851
 uncountable/integration/secret_retrieval/__init__.py,sha256=sisf8cxCYcJmCXCKjEPoT2xIxm1j-Ru9iTfMVxH6NM0,171
 uncountable/integration/secret_retrieval/basic_secret.py,sha256=PfpeaEDwdi8nFlHUJnzCob9GYgGG5O_3mmcaxPPY6zI,3272
 uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=c0VkCvppWKXwgxdblQuj2WhRaEGBPa7vLDoS6mlNOAQ,3919
-uncountable/integration/webhook_server/entrypoint.py,sha256=_IgZ98Fpigmx3VxYgKFC1jv55tz8M-u2CGgB9_q_04E,5440
+uncountable/integration/webhook_server/entrypoint.py,sha256=1vk6gjMbp7sGyGtcZBXewarP4OT1dSjF70sf-jznIZQ,5874
 uncountable/integration/webhook_server/gunicorn_config.py,sha256=F7XiS1z2BSfCMpwTmEFedk18WdGG5dwtCs_hDBV5TIE,232
 uncountable/types/__init__.py,sha256=d1t7dHkmZ--ykXDKV_IDtWXD4MHCC0bWflT_yUjkRBw,14462
 uncountable/types/async_batch.py,sha256=yCCWrrLQfxXVqZp-KskxLBNkNmuELdz4PJjx8ULppgs,662
@@ -218,7 +220,7 @@ uncountable/types/data_t.py,sha256=U_dIh5GcyaJlkrHEI4Qf4TOo1uJX07PYO70sYzzKtTo,2
 uncountable/types/dataset_measurements.py,sha256=CVwWePUC02B_R_UB86yfNBwnclR-5xpZaiCs2jWtu2A,512
 uncountable/types/dataset_measurements_t.py,sha256=RNGy6g-bjfnCeoYEbcqULC83bS4oZ46fbYKL9whYa8E,1697
 uncountable/types/entity.py,sha256=Zclk1LYcRaYrMDhqyCjMSLEg0fE6_q8LHvV22Qvscgs,566
-uncountable/types/entity_t.py,sha256=fbyHxW5sytqkgZPQC0w9xSJuzCt-AXNDZ4et85QC-9Q,31544
+uncountable/types/entity_t.py,sha256=4brCV8LvOBd8X-d_Jf4GW6quV_hEKiV5f8S5If1BXgY,31630
 uncountable/types/experiment_groups.py,sha256=qUpFOx1AKgzaT_4khCOv5Xs6jwiQGbvHH-GUh3v1nv4,288
 uncountable/types/experiment_groups_t.py,sha256=QjZBdjnbsD1D-VpPSSV6hBg-Q7mujjKvYtdMewtXNVU,712
 uncountable/types/exports.py,sha256=VMmxUO2PpV1Y63hZ2AnVor4H-B6aswJ7YpSru_u89lU,334
@@ -445,7 +447,7 @@ uncountable/types/api/uploader/complete_async_parse.py,sha256=Os1HAubxTlbut6Gylj
 uncountable/types/api/uploader/invoke_uploader.py,sha256=cmy7zl3AMF_AH5PCAB67rBeTE6JIcaUqPzm2NS4k_AE,1907
 uncountable/types/api/user/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
 uncountable/types/api/user/get_current_user_info.py,sha256=BT7bGp5SOHz3XdKWhhdZ6acnInafT2PgdJ-H4YH3W7c,1181
-uncountablepythonsdk-0.0.190.dist-info/METADATA,sha256=0d3sb3VUbxH281yHiHC-sGfc5qJ2A4rx8G_jeIVn_CU,2241
-uncountablepythonsdk-0.0.190.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-uncountablepythonsdk-0.0.190.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
-uncountablepythonsdk-0.0.190.dist-info/RECORD,,
+uncountablepythonsdk-0.0.191.dist-info/METADATA,sha256=dRu4VuL-ylGKivo1djoLKXA14upN9PpVWtbvDDar00E,2241
+uncountablepythonsdk-0.0.191.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+uncountablepythonsdk-0.0.191.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
+uncountablepythonsdk-0.0.191.dist-info/RECORD,,