UncountablePythonSDK 0.0.126__py3-none-any.whl → 0.0.128__py3-none-any.whl

uncountable/integration/queue_runner/command_server/command_server.py

@@ -13,11 +13,19 @@ from uncountable.integration.queue_runner.command_server.protocol.command_server
     EnqueueJobResult,
     ListQueuedJobsRequest,
     ListQueuedJobsResult,
+    RetryJobRequest,
+    RetryJobResult,
+    VaccuumQueuedJobsRequest,
+    VaccuumQueuedJobsResult,
 )
 from uncountable.integration.queue_runner.command_server.types import (
     CommandEnqueueJob,
     CommandEnqueueJobResponse,
     CommandQueue,
+    CommandRetryJob,
+    CommandRetryJobResponse,
+    CommandVaccuumQueuedJobs,
+    CommandVaccuumQueuedJobsResponse,
 )
 from uncountable.integration.queue_runner.datastore import DatastoreSqlite
 from uncountable.types import queued_job_t
@@ -54,6 +62,23 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
             )
             return result
 
+        async def RetryJob(
+            self, request: RetryJobRequest, context: aio.ServicerContext
+        ) -> RetryJobResult:
+            response_queue: asyncio.Queue[CommandRetryJobResponse] = asyncio.Queue()
+            await command_queue.put(
+                CommandRetryJob(
+                    queued_job_uuid=request.uuid, response_queue=response_queue
+                )
+            )
+            response = await response_queue.get()
+            if response.queued_job_uuid is not None:
+                return RetryJobResult(
+                    successfully_queued=True, queued_job_uuid=response.queued_job_uuid
+                )
+            else:
+                return RetryJobResult(successfully_queued=False, queued_job_uuid="")
+
         async def CheckHealth(
             self, request: CheckHealthRequest, context: aio.ServicerContext
         ) -> CheckHealthResult:
@@ -90,10 +115,22 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
                         job_ref_name=item.job_ref_name,
                         num_attempts=item.num_attempts,
                         submitted_at=proto_timestamp,
+                        status=item.status,
                     )
                 )
             return ListQueuedJobsResult(queued_jobs=response_list)
 
+        async def VaccuumQueuedJobs(
+            self, request: VaccuumQueuedJobsRequest, context: aio.ServicerContext
+        ) -> VaccuumQueuedJobsResult:
+            response_queue: asyncio.Queue[CommandVaccuumQueuedJobsResponse] = (
+                asyncio.Queue()
+            )
+            await command_queue.put(
+                CommandVaccuumQueuedJobs(response_queue=response_queue)
+            )
+            return VaccuumQueuedJobsResult()
+
     add_CommandServerServicer_to_server(CommandServerHandler(), server)
 
     listen_addr = f"[::]:{get_local_admin_server_port()}"

uncountable/integration/queue_runner/command_server/protocol/command_server.proto

@@ -3,8 +3,10 @@ import "google/protobuf/timestamp.proto";
 
 service CommandServer {
 	rpc EnqueueJob(EnqueueJobRequest) returns (EnqueueJobResult) {}
+	rpc RetryJob(RetryJobRequest) returns (RetryJobResult) {}
 	rpc CheckHealth(CheckHealthRequest) returns (CheckHealthResult) {}
 	rpc ListQueuedJobs(ListQueuedJobsRequest) returns (ListQueuedJobsResult) {}
+	rpc VaccuumQueuedJobs(VaccuumQueuedJobsRequest) returns (VaccuumQueuedJobsResult) {}
 }
 
 message EnqueueJobRequest {
@@ -17,6 +19,21 @@ message EnqueueJobResult {
 	string queued_job_uuid = 2;
 }
 
+message RetryJobRequest {
+	string uuid = 1;
+}
+
+message RetryJobResult {
+	bool successfully_queued = 1;
+	string queued_job_uuid = 2;
+}
+
+message VaccuumQueuedJobsRequest {
+}
+
+message VaccuumQueuedJobsResult {
+}
+
 message CheckHealthRequest {}
 
 message CheckHealthResult {
@@ -34,6 +51,7 @@ message ListQueuedJobsResult {
 	string job_ref_name = 2;
 	int64 num_attempts = 3;
 	google.protobuf.Timestamp submitted_at = 4;
+	string status = 5;
 	}
 
 	repeated ListQueuedJobsResultItem queued_jobs  = 1;

uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.py

@@ -18,7 +18,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\nQuncountable/integration/queue_runner/command_server/protocol/command_server.proto\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x11\x45nqueueJobRequest\x12\x14\n\x0cjob_ref_name\x18\x01 \x01(\t\x12\x1a\n\x12serialized_payload\x18\x02 \x01(\t"H\n\x10\x45nqueueJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x14\n\x12\x43heckHealthRequest"$\n\x11\x43heckHealthResult\x12\x0f\n\x07success\x18\x01 \x01(\x08"6\n\x15ListQueuedJobsRequest\x12\x0e\n\x06offset\x18\x01 \x01(\r\x12\r\n\x05limit\x18\x02 \x01(\r"\xe4\x01\n\x14ListQueuedJobsResult\x12\x43\n\x0bqueued_jobs\x18\x01 \x03(\x0b\x32..ListQueuedJobsResult.ListQueuedJobsResultItem\x1a\x86\x01\n\x18ListQueuedJobsResultItem\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x02 \x01(\t\x12\x14\n\x0cnum_attempts\x18\x03 \x01(\x03\x12\x30\n\x0csubmitted_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp2\xc3\x01\n\rCommandServer\x12\x35\n\nEnqueueJob\x12\x12.EnqueueJobRequest\x1a\x11.EnqueueJobResult"\x00\x12\x38\n\x0b\x43heckHealth\x12\x13.CheckHealthRequest\x1a\x12.CheckHealthResult"\x00\x12\x41\n\x0eListQueuedJobs\x12\x16.ListQueuedJobsRequest\x1a\x15.ListQueuedJobsResult"\x00\x62\x06proto3'
+    b'\nQuncountable/integration/queue_runner/command_server/protocol/command_server.proto\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x11\x45nqueueJobRequest\x12\x14\n\x0cjob_ref_name\x18\x01 \x01(\t\x12\x1a\n\x12serialized_payload\x18\x02 \x01(\t"H\n\x10\x45nqueueJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1f\n\x0fRetryJobRequest\x12\x0c\n\x04uuid\x18\x01 \x01(\t"F\n\x0eRetryJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1a\n\x18VaccuumQueuedJobsRequest"\x19\n\x17VaccuumQueuedJobsResult"\x14\n\x12\x43heckHealthRequest"$\n\x11\x43heckHealthResult\x12\x0f\n\x07success\x18\x01 \x01(\x08"6\n\x15ListQueuedJobsRequest\x12\x0e\n\x06offset\x18\x01 \x01(\r\x12\r\n\x05limit\x18\x02 \x01(\r"\xf4\x01\n\x14ListQueuedJobsResult\x12\x43\n\x0bqueued_jobs\x18\x01 \x03(\x0b\x32..ListQueuedJobsResult.ListQueuedJobsResultItem\x1a\x96\x01\n\x18ListQueuedJobsResultItem\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x02 \x01(\t\x12\x14\n\x0cnum_attempts\x18\x03 \x01(\x03\x12\x30\n\x0csubmitted_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06status\x18\x05 \x01(\t2\xc0\x02\n\rCommandServer\x12\x35\n\nEnqueueJob\x12\x12.EnqueueJobRequest\x1a\x11.EnqueueJobResult"\x00\x12/\n\x08RetryJob\x12\x10.RetryJobRequest\x1a\x0f.RetryJobResult"\x00\x12\x38\n\x0b\x43heckHealth\x12\x13.CheckHealthRequest\x1a\x12.CheckHealthResult"\x00\x12\x41\n\x0eListQueuedJobs\x12\x16.ListQueuedJobsRequest\x1a\x15.ListQueuedJobsResult"\x00\x12J\n\x11VaccuumQueuedJobs\x12\x19.VaccuumQueuedJobsRequest\x1a\x18.VaccuumQueuedJobsResult"\x00\x62\x06proto3'
 )
 
 _globals = globals()
@@ -34,16 +34,24 @@ if _descriptor._USE_C_DESCRIPTORS == False:
     _globals["_ENQUEUEJOBREQUEST"]._serialized_end = 187
     _globals["_ENQUEUEJOBRESULT"]._serialized_start = 189
     _globals["_ENQUEUEJOBRESULT"]._serialized_end = 261
-    _globals["_CHECKHEALTHREQUEST"]._serialized_start = 263
-    _globals["_CHECKHEALTHREQUEST"]._serialized_end = 283
-    _globals["_CHECKHEALTHRESULT"]._serialized_start = 285
-    _globals["_CHECKHEALTHRESULT"]._serialized_end = 321
-    _globals["_LISTQUEUEDJOBSREQUEST"]._serialized_start = 323
-    _globals["_LISTQUEUEDJOBSREQUEST"]._serialized_end = 377
-    _globals["_LISTQUEUEDJOBSRESULT"]._serialized_start = 380
-    _globals["_LISTQUEUEDJOBSRESULT"]._serialized_end = 608
-    _globals["_LISTQUEUEDJOBSRESULT_LISTQUEUEDJOBSRESULTITEM"]._serialized_start = 474
-    _globals["_LISTQUEUEDJOBSRESULT_LISTQUEUEDJOBSRESULTITEM"]._serialized_end = 608
-    _globals["_COMMANDSERVER"]._serialized_start = 611
-    _globals["_COMMANDSERVER"]._serialized_end = 806
+    _globals["_RETRYJOBREQUEST"]._serialized_start = 263
+    _globals["_RETRYJOBREQUEST"]._serialized_end = 294
+    _globals["_RETRYJOBRESULT"]._serialized_start = 296
+    _globals["_RETRYJOBRESULT"]._serialized_end = 366
+    _globals["_VACCUUMQUEUEDJOBSREQUEST"]._serialized_start = 368
+    _globals["_VACCUUMQUEUEDJOBSREQUEST"]._serialized_end = 394
+    _globals["_VACCUUMQUEUEDJOBSRESULT"]._serialized_start = 396
+    _globals["_VACCUUMQUEUEDJOBSRESULT"]._serialized_end = 421
+    _globals["_CHECKHEALTHREQUEST"]._serialized_start = 423
+    _globals["_CHECKHEALTHREQUEST"]._serialized_end = 443
+    _globals["_CHECKHEALTHRESULT"]._serialized_start = 445
+    _globals["_CHECKHEALTHRESULT"]._serialized_end = 481
+    _globals["_LISTQUEUEDJOBSREQUEST"]._serialized_start = 483
+    _globals["_LISTQUEUEDJOBSREQUEST"]._serialized_end = 537
+    _globals["_LISTQUEUEDJOBSRESULT"]._serialized_start = 540
+    _globals["_LISTQUEUEDJOBSRESULT"]._serialized_end = 784
+    _globals["_LISTQUEUEDJOBSRESULT_LISTQUEUEDJOBSRESULTITEM"]._serialized_start = 634
+    _globals["_LISTQUEUEDJOBSRESULT_LISTQUEUEDJOBSRESULTITEM"]._serialized_end = 784
+    _globals["_COMMANDSERVER"]._serialized_start = 787
+    _globals["_COMMANDSERVER"]._serialized_end = 1107
 # @@protoc_insertion_point(module_scope)

uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.pyi

@@ -35,6 +35,30 @@ class EnqueueJobResult(_message.Message):
         self, successfully_queued: bool = ..., queued_job_uuid: _Optional[str] = ...
     ) -> None: ...
 
+class RetryJobRequest(_message.Message):
+    __slots__ = ("uuid",)
+    UUID_FIELD_NUMBER: _ClassVar[int]
+    uuid: str
+    def __init__(self, uuid: _Optional[str] = ...) -> None: ...
+
+class RetryJobResult(_message.Message):
+    __slots__ = ("successfully_queued", "queued_job_uuid")
+    SUCCESSFULLY_QUEUED_FIELD_NUMBER: _ClassVar[int]
+    QUEUED_JOB_UUID_FIELD_NUMBER: _ClassVar[int]
+    successfully_queued: bool
+    queued_job_uuid: str
+    def __init__(
+        self, successfully_queued: bool = ..., queued_job_uuid: _Optional[str] = ...
+    ) -> None: ...
+
+class VaccuumQueuedJobsRequest(_message.Message):
+    __slots__ = ()
+    def __init__(self) -> None: ...
+
+class VaccuumQueuedJobsResult(_message.Message):
+    __slots__ = ()
+    def __init__(self) -> None: ...
+
 class CheckHealthRequest(_message.Message):
     __slots__ = ()
     def __init__(self) -> None: ...
@@ -58,21 +82,24 @@ class ListQueuedJobsRequest(_message.Message):
 class ListQueuedJobsResult(_message.Message):
     __slots__ = ("queued_jobs",)
     class ListQueuedJobsResultItem(_message.Message):
-        __slots__ = ("uuid", "job_ref_name", "num_attempts", "submitted_at")
+        __slots__ = ("uuid", "job_ref_name", "num_attempts", "submitted_at", "status")
         UUID_FIELD_NUMBER: _ClassVar[int]
         JOB_REF_NAME_FIELD_NUMBER: _ClassVar[int]
         NUM_ATTEMPTS_FIELD_NUMBER: _ClassVar[int]
         SUBMITTED_AT_FIELD_NUMBER: _ClassVar[int]
+        STATUS_FIELD_NUMBER: _ClassVar[int]
         uuid: str
         job_ref_name: str
         num_attempts: int
         submitted_at: _timestamp_pb2.Timestamp
+        status: str
         def __init__(
             self,
             uuid: _Optional[str] = ...,
             job_ref_name: _Optional[str] = ...,
             num_attempts: _Optional[int] = ...,
             submitted_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...,
+            status: _Optional[str] = ...,
         ) -> None: ...
 
     QUEUED_JOBS_FIELD_NUMBER: _ClassVar[int]

uncountable/integration/queue_runner/command_server/protocol/command_server_pb2_grpc.py

@@ -24,6 +24,11 @@ class CommandServerStub(object):
             request_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.EnqueueJobRequest.SerializeToString,
             response_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.EnqueueJobResult.FromString,
         )
+        self.RetryJob = channel.unary_unary(
+            "/CommandServer/RetryJob",
+            request_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.RetryJobRequest.SerializeToString,
+            response_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.RetryJobResult.FromString,
+        )
         self.CheckHealth = channel.unary_unary(
             "/CommandServer/CheckHealth",
             request_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CheckHealthRequest.SerializeToString,
@@ -34,6 +39,11 @@ class CommandServerStub(object):
             request_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.ListQueuedJobsRequest.SerializeToString,
             response_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.ListQueuedJobsResult.FromString,
         )
+        self.VaccuumQueuedJobs = channel.unary_unary(
+            "/CommandServer/VaccuumQueuedJobs",
+            request_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsRequest.SerializeToString,
+            response_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsResult.FromString,
+        )
 
 
 class CommandServerServicer(object):
@@ -45,6 +55,12 @@ class CommandServerServicer(object):
         context.set_details("Method not implemented!")
         raise NotImplementedError("Method not implemented!")
 
+    def RetryJob(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
     def CheckHealth(self, request, context):
         """Missing associated documentation comment in .proto file."""
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
@@ -57,6 +73,12 @@ class CommandServerServicer(object):
         context.set_details("Method not implemented!")
         raise NotImplementedError("Method not implemented!")
 
+    def VaccuumQueuedJobs(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
 
 def add_CommandServerServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -65,6 +87,11 @@ def add_CommandServerServicer_to_server(servicer, server):
             request_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.EnqueueJobRequest.FromString,
             response_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.EnqueueJobResult.SerializeToString,
         ),
+        "RetryJob": grpc.unary_unary_rpc_method_handler(
+            servicer.RetryJob,
+            request_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.RetryJobRequest.FromString,
+            response_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.RetryJobResult.SerializeToString,
+        ),
         "CheckHealth": grpc.unary_unary_rpc_method_handler(
             servicer.CheckHealth,
             request_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CheckHealthRequest.FromString,
@@ -75,6 +102,11 @@ def add_CommandServerServicer_to_server(servicer, server):
             request_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.ListQueuedJobsRequest.FromString,
             response_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.ListQueuedJobsResult.SerializeToString,
         ),
+        "VaccuumQueuedJobs": grpc.unary_unary_rpc_method_handler(
+            servicer.VaccuumQueuedJobs,
+            request_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsRequest.FromString,
+            response_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsResult.SerializeToString,
+        ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
         "CommandServer", rpc_method_handlers
@@ -115,6 +147,35 @@ class CommandServer(object):
             metadata,
         )
 
+    @staticmethod
+    def RetryJob(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/CommandServer/RetryJob",
+            uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.RetryJobRequest.SerializeToString,
+            uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.RetryJobResult.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )
+
     @staticmethod
     def CheckHealth(
         request,
@@ -172,3 +233,32 @@ class CommandServer(object):
             timeout,
             metadata,
         )
+
+    @staticmethod
+    def VaccuumQueuedJobs(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/CommandServer/VaccuumQueuedJobs",
+            uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsRequest.SerializeToString,
+            uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsResult.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+        )

uncountable/integration/queue_runner/command_server/types.py

@@ -8,6 +8,8 @@ from uncountable.types import queued_job_t
 
 class CommandType(StrEnum):
     ENQUEUE_JOB = "enqueue_job"
+    RETRY_JOB = "retry_job"
+    VACCUUM_QUEUED_JOBS = "vaccuum_queued_jobs"
 
 
 RT = typing.TypeVar("RT")
@@ -24,6 +26,16 @@ class CommandEnqueueJobResponse:
     queued_job_uuid: str
 
 
+@dataclass(kw_only=True)
+class CommandRetryJobResponse:
+    queued_job_uuid: str | None
+
+
+@dataclass(kw_only=True)
+class CommandVaccuumQueuedJobsResponse:
+    pass
+
+
 @dataclass(kw_only=True)
 class CommandEnqueueJob(CommandBase[CommandEnqueueJobResponse]):
     type: CommandType = CommandType.ENQUEUE_JOB
@@ -32,7 +44,18 @@ class CommandEnqueueJob(CommandBase[CommandEnqueueJobResponse]):
     response_queue: asyncio.Queue[CommandEnqueueJobResponse]
 
 
-_Command = CommandEnqueueJob
+@dataclass(kw_only=True)
+class CommandRetryJob(CommandBase[CommandRetryJobResponse]):
+    type: CommandType = CommandType.RETRY_JOB
+    queued_job_uuid: str
+
+
+@dataclass(kw_only=True)
+class CommandVaccuumQueuedJobs(CommandBase[CommandVaccuumQueuedJobsResponse]):
+    type: CommandType = CommandType.VACCUUM_QUEUED_JOBS
+
+
+_Command = CommandEnqueueJob | CommandRetryJob | CommandVaccuumQueuedJobs
 
 
 CommandQueue = asyncio.Queue[_Command]

uncountable/integration/queue_runner/datastore/datastore_sqlite.py

@@ -2,7 +2,7 @@ import datetime
 import uuid
 from datetime import UTC
 
-from sqlalchemy import delete, insert, select, update
+from sqlalchemy import delete, insert, or_, select, text, update
 from sqlalchemy.engine import Engine
 
 from pkgs.argument_parser import CachedParser
@@ -14,6 +14,8 @@ from uncountable.types import queued_job_t
 
 queued_job_payload_parser = CachedParser(queued_job_t.QueuedJobPayload)
 
+MAX_QUEUE_WINDOW_DAYS = 30
+
 
 class DatastoreSqlite(Datastore):
     def __init__(self, session_maker: DBSessionMaker) -> None:
@@ -23,6 +25,17 @@ class DatastoreSqlite(Datastore):
     @classmethod
     def setup(cls, engine: Engine) -> None:
         Base.metadata.create_all(engine)
+        with engine.connect() as connection:
+            if not bool(
+                connection.execute(
+                    text(
+                        "select exists (select 1 from pragma_table_info('queued_jobs') where name='status');"
+                    )
+                ).scalar()
+            ):
+                connection.execute(
+                    text("alter table queued_jobs add column status VARCHAR")
+                )
 
     def add_job_to_queue(
         self, job_payload: queued_job_t.QueuedJobPayload, job_ref_name: str
@@ -36,6 +49,7 @@ class DatastoreSqlite(Datastore):
                 QueuedJob.id.key: queued_job_uuid,
                 QueuedJob.job_ref_name.key: job_ref_name,
                 QueuedJob.payload.key: serialized_payload,
+                QueuedJob.status.key: queued_job_t.JobStatus.QUEUED,
                 QueuedJob.num_attempts: num_attempts,
                 QueuedJob.submitted_at: submitted_at,
             })
@@ -44,10 +58,48 @@ class DatastoreSqlite(Datastore):
                 queued_job_uuid=queued_job_uuid,
                 job_ref_name=job_ref_name,
                 payload=job_payload,
+                status=queued_job_t.JobStatus.QUEUED,
                 submitted_at=submitted_at,
                 num_attempts=num_attempts,
             )
 
+    def retry_job(
+        self,
+        queued_job_uuid: str,
+    ) -> queued_job_t.QueuedJob | None:
+        with self.session_maker() as session:
+            select_stmt = select(
+                QueuedJob.id,
+                QueuedJob.payload,
+                QueuedJob.num_attempts,
+                QueuedJob.job_ref_name,
+                QueuedJob.status,
+                QueuedJob.submitted_at,
+            ).filter(QueuedJob.id == queued_job_uuid)
+            existing_job = session.execute(select_stmt).one_or_none()
+
+            if (
+                existing_job is None
+                or existing_job.status != queued_job_t.JobStatus.FAILED
+            ):
+                return None
+
+            update_stmt = (
+                update(QueuedJob)
+                .values({QueuedJob.status.key: queued_job_t.JobStatus.QUEUED})
+                .filter(QueuedJob.id == queued_job_uuid)
+            )
+            session.execute(update_stmt)
+
+            return queued_job_t.QueuedJob(
+                queued_job_uuid=existing_job.id,
+                job_ref_name=existing_job.job_ref_name,
+                num_attempts=existing_job.num_attempts,
+                status=queued_job_t.JobStatus.QUEUED,
+                submitted_at=existing_job.submitted_at,
+                payload=queued_job_payload_parser.parse_storage(existing_job.payload),
+            )
+
     def increment_num_attempts(self, queued_job_uuid: str) -> int:
         with self.session_maker() as session:
             update_stmt = (
@@ -68,6 +120,17 @@ class DatastoreSqlite(Datastore):
             delete_stmt = delete(QueuedJob).filter(QueuedJob.id == queued_job_uuid)
             session.execute(delete_stmt)
 
+    def update_job_status(
+        self, queued_job_uuid: str, status: queued_job_t.JobStatus
+    ) -> None:
+        with self.session_maker() as session:
+            update_stmt = (
+                update(QueuedJob)
+                .values({QueuedJob.status.key: status})
+                .filter(QueuedJob.id == queued_job_uuid)
+            )
+            session.execute(update_stmt)
+
     def list_queued_job_metadata(
         self, offset: int = 0, limit: int | None = 100
     ) -> list[queued_job_t.QueuedJobMetadata]:
@@ -77,6 +140,7 @@ class DatastoreSqlite(Datastore):
                     QueuedJob.id,
                     QueuedJob.job_ref_name,
                     QueuedJob.num_attempts,
+                    QueuedJob.status,
                     QueuedJob.submitted_at,
                 )
                 .order_by(QueuedJob.submitted_at)
@@ -89,6 +153,7 @@ class DatastoreSqlite(Datastore):
                     queued_job_uuid=row.id,
                     job_ref_name=row.job_ref_name,
                     num_attempts=row.num_attempts,
+                    status=row.status or queued_job_t.JobStatus.QUEUED,
                     submitted_at=row.submitted_at,
                 )
                 for row in session.execute(select_statement)
@@ -106,9 +171,16 @@ class DatastoreSqlite(Datastore):
                     QueuedJob.payload,
                     QueuedJob.num_attempts,
                     QueuedJob.job_ref_name,
+                    QueuedJob.status,
                     QueuedJob.submitted_at,
                 )
                 .filter(QueuedJob.job_ref_name == job_ref_name)
+                .filter(
+                    or_(
+                        QueuedJob.status == queued_job_t.JobStatus.QUEUED,
+                        QueuedJob.status.is_(None),
+                    )
+                )
                 .limit(1)
                 .order_by(QueuedJob.submitted_at)
             )
@@ -119,6 +191,7 @@ class DatastoreSqlite(Datastore):
                     queued_job_uuid=row.id,
                     job_ref_name=row.job_ref_name,
                     num_attempts=row.num_attempts,
+                    status=row.status or queued_job_t.JobStatus.QUEUED,
                     submitted_at=row.submitted_at,
                     payload=parsed_payload,
                 )
@@ -127,13 +200,23 @@ class DatastoreSqlite(Datastore):
 
     def load_job_queue(self) -> list[queued_job_t.QueuedJob]:
         with self.session_maker() as session:
-            select_stmt = select(
-                QueuedJob.id,
-                QueuedJob.payload,
-                QueuedJob.num_attempts,
-                QueuedJob.job_ref_name,
-                QueuedJob.submitted_at,
-            ).order_by(QueuedJob.submitted_at)
+            select_stmt = (
+                select(
+                    QueuedJob.id,
+                    QueuedJob.payload,
+                    QueuedJob.num_attempts,
+                    QueuedJob.job_ref_name,
+                    QueuedJob.status,
+                    QueuedJob.submitted_at,
+                )
+                .filter(
+                    or_(
+                        QueuedJob.status == queued_job_t.JobStatus.QUEUED,
+                        QueuedJob.status.is_(None),
+                    )
+                )
+                .order_by(QueuedJob.submitted_at)
+            )
 
             queued_jobs: list[queued_job_t.QueuedJob] = []
             for row in session.execute(select_stmt):
@@ -143,9 +226,25 @@ class DatastoreSqlite(Datastore):
                         queued_job_uuid=row.id,
                         job_ref_name=row.job_ref_name,
                         num_attempts=row.num_attempts,
+                        status=row.status or queued_job_t.JobStatus.QUEUED,
                         submitted_at=row.submitted_at,
                         payload=parsed_payload,
                     )
                 )
 
             return queued_jobs
+
+    def vaccuum_queued_jobs(self) -> None:
+        with self.session_maker() as session:
+            delete_stmt = (
+                delete(QueuedJob)
+                .filter(QueuedJob.status == queued_job_t.JobStatus.QUEUED)
+                .filter(
+                    QueuedJob.submitted_at
+                    <= (
+                        datetime.datetime.now(UTC)
+                        - datetime.timedelta(days=MAX_QUEUE_WINDOW_DAYS)
+                    )
+                )
+            )
+            session.execute(delete_stmt)

uncountable/integration/queue_runner/datastore/model.py

@@ -1,7 +1,9 @@
-from sqlalchemy import JSON, BigInteger, Column, DateTime, Text
+from sqlalchemy import JSON, BigInteger, Column, DateTime, Enum, Text
 from sqlalchemy.orm import declarative_base
 from sqlalchemy.sql import func
 
+from uncountable.types import queued_job_t
+
 Base = declarative_base()
 
 
@@ -15,3 +17,8 @@ class QueuedJob(Base):
     )
     payload = Column(JSON, nullable=False)
     num_attempts = Column(BigInteger, nullable=False, default=0, server_default="0")
+    status = Column(
+        Enum(queued_job_t.JobStatus, length=None),
+        default=queued_job_t.JobStatus.QUEUED,
+        nullable=True,
+    )