UncountablePythonSDK 0.0.133__py3-none-any.whl → 0.0.136.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. examples/integration-server/jobs/materials_auto/example_cron.py +1 -1
  2. examples/integration-server/jobs/materials_auto/example_runsheet_wh.py +51 -14
  3. pkgs/type_spec/builder.py +15 -1
  4. uncountable/integration/cli.py +60 -1
  5. uncountable/integration/queue_runner/command_server/command_client.py +24 -0
  6. uncountable/integration/queue_runner/command_server/command_server.py +34 -0
  7. uncountable/integration/queue_runner/command_server/protocol/command_server.proto +15 -0
  8. uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.py +9 -3
  9. uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.pyi +25 -0
  10. uncountable/integration/queue_runner/command_server/protocol/command_server_pb2_grpc.py +45 -0
  11. uncountable/integration/queue_runner/command_server/types.py +21 -1
  12. uncountable/integration/queue_runner/datastore/datastore_sqlite.py +25 -0
  13. uncountable/integration/queue_runner/datastore/interface.py +3 -0
  14. uncountable/integration/queue_runner/job_scheduler.py +36 -1
  15. uncountable/integration/queue_runner/types.py +2 -0
  16. uncountable/integration/queue_runner/worker.py +28 -26
  17. uncountable/integration/scheduler.py +64 -13
  18. uncountable/integration/telemetry.py +79 -0
  19. uncountable/types/__init__.py +2 -2
  20. uncountable/types/api/files/download_file.py +15 -1
  21. uncountable/types/api/listing/fetch_listing.py +1 -2
  22. uncountable/types/api/runsheet/export_default_runsheet.py +44 -0
  23. uncountable/types/client_base.py +56 -0
  24. uncountable/types/listing.py +37 -0
  25. uncountable/types/listing_t.py +483 -1
  26. {uncountablepythonsdk-0.0.133.dist-info → uncountablepythonsdk-0.0.136.dev0.dist-info}/METADATA +2 -2
  27. {uncountablepythonsdk-0.0.133.dist-info → uncountablepythonsdk-0.0.136.dev0.dist-info}/RECORD +29 -30
  28. uncountable/types/structured_filters.py +0 -25
  29. uncountable/types/structured_filters_t.py +0 -248
  30. {uncountablepythonsdk-0.0.133.dist-info → uncountablepythonsdk-0.0.136.dev0.dist-info}/WHEEL +0 -0
  31. {uncountablepythonsdk-0.0.133.dist-info → uncountablepythonsdk-0.0.136.dev0.dist-info}/top_level.txt +0 -0
@@ -17,5 +17,5 @@ class MyCronJob(CronJob):
17
17
  if field_val.field_ref_name == "name":
18
18
  name = field_val.value
19
19
  args.logger.log_info(f"material family found with name: {name}")
20
- time.sleep(1.5)
20
+ time.sleep(20)
21
21
  return JobResult(success=True)
@@ -1,13 +1,19 @@
1
1
  from io import BytesIO
2
2
 
3
+ from openpyxl import Workbook, load_workbook
3
4
  from uncountable.core.file_upload import DataFileUpload, FileUpload
4
5
  from uncountable.integration.job import JobArguments, RunsheetWebhookJob, register_job
5
6
  from uncountable.types import (
6
7
  download_file_t,
7
8
  entity_t,
9
+ export_default_runsheet_t,
8
10
  identifier_t,
9
11
  webhook_job_t,
10
12
  )
13
+ from uncountable.types.client_base import APIRequest
14
+
15
+ RUNSHEET_REF_NAME = "recipe_export_runsheet"
16
+ RUNSHEET_REF_NAME_2 = "recipe_export_runsheet_2"
11
17
 
12
18
 
13
19
  @register_job
@@ -18,22 +24,53 @@ class StandardRunsheetGenerator(RunsheetWebhookJob):
18
24
  args: JobArguments,
19
25
  payload: webhook_job_t.RunsheetWebhookPayload,
20
26
  ) -> FileUpload:
21
- args.logger.log_info("Retrieving pre-exported runsheet file from async job")
22
-
23
- file_query = download_file_t.FileDownloadQueryEntityField(
24
- entity=entity_t.EntityIdentifier(
25
- type=entity_t.EntityType.ASYNC_JOB,
26
- identifier_key=identifier_t.IdentifierKeyId(id=payload.async_job_id),
27
- ),
28
- field_key=identifier_t.IdentifierKeyRefName(
29
- ref_name="unc_async_job_export_runsheet_recipe_export"
30
- ),
27
+ args.logger.log_info("Exporting default runsheets")
28
+
29
+ entity_identifiers: list[identifier_t.IdentifierKey] = [
30
+ identifier_t.IdentifierKeyId(id=entity.id) for entity in payload.entities
31
+ ]
32
+
33
+ combined_wb = Workbook()
34
+ combined_sheet = combined_wb.active or combined_wb.create_sheet(
35
+ title="Combined Runsheet"
31
36
  )
37
+ combined_sheet.title = "Combined Runsheet"
38
+
39
+ for ref_name in [RUNSHEET_REF_NAME, RUNSHEET_REF_NAME_2]:
40
+ api_request = APIRequest(
41
+ method=export_default_runsheet_t.ENDPOINT_METHOD,
42
+ endpoint=export_default_runsheet_t.ENDPOINT_PATH,
43
+ args=export_default_runsheet_t.Arguments(
44
+ entities=entity_identifiers,
45
+ runsheet_key=identifier_t.IdentifierKeyRefName(ref_name=ref_name),
46
+ entity_type=payload.entities[0].type
47
+ if payload.entities
48
+ else entity_t.EntityType.RECIPE,
49
+ ),
50
+ )
51
+
52
+ response = args.client.do_request(
53
+ api_request=api_request,
54
+ return_type=export_default_runsheet_t.Data,
55
+ )
56
+
57
+ file_query = download_file_t.FileDownloadQueryTextDocumentId(
58
+ text_document_id=response.text_document_id,
59
+ )
60
+
61
+ downloaded_files = args.client.download_files(file_query=file_query)
62
+ file_data = downloaded_files[0].data.read()
63
+
64
+ wb = load_workbook(filename=BytesIO(file_data))
65
+ for sheet_name in wb.sheetnames:
66
+ for row in wb[sheet_name].iter_rows(values_only=True):
67
+ combined_sheet.append(row)
32
68
 
33
- downloaded_files = args.client.download_files(file_query=file_query)
69
+ output = BytesIO()
70
+ combined_wb.save(output)
71
+ output.seek(0)
34
72
 
35
- file_data = downloaded_files[0].data.read()
36
73
  return DataFileUpload(
37
- data=BytesIO(file_data),
38
- name=downloaded_files[0].name,
74
+ data=output,
75
+ name="combined_runsheet.xlsx",
39
76
  )
pkgs/type_spec/builder.py CHANGED
@@ -423,7 +423,16 @@ class SpecTypeDefn(SpecType):
423
423
  parse_require = False
424
424
  literal = unwrap_literal_type(ptype)
425
425
  if literal is not None:
426
- default = literal.value
426
+ if isinstance(
427
+ literal.value_type, SpecTypeDefnStringEnum
428
+ ) and isinstance(literal.value, str):
429
+ resolved_value = literal.value_type.values.get(literal.value)
430
+ assert resolved_value is not None, (
431
+ f"Value {literal.value} not found in enum"
432
+ )
433
+ default = resolved_value.value
434
+ else:
435
+ default = literal.value
427
436
  has_default = True
428
437
  parse_require = True
429
438
 
@@ -1587,6 +1596,11 @@ class SpecBuilder:
1587
1596
  f"'examples' in example files are expected to be a list, endpoint_path={path_details.resolved_path}"
1588
1597
  )
1589
1598
  for example in examples_data:
1599
+ if not isinstance(example, dict):
1600
+ raise Exception(
1601
+ f"each example in example file is expected to be a dict, endpoint_path={path_details.resolved_path}"
1602
+ )
1603
+
1590
1604
  arguments = example["arguments"]
1591
1605
  data_example = example["data"]
1592
1606
  if not isinstance(arguments, dict) or not isinstance(data_example, dict):
@@ -1,4 +1,6 @@
1
1
  import argparse
2
+ import json
3
+ from typing import assert_never
2
4
 
3
5
  from dateutil import tz
4
6
  from opentelemetry.trace import get_current_span
@@ -6,10 +8,14 @@ from tabulate import tabulate
6
8
 
7
9
  from uncountable.core.environment import get_local_admin_server_port
8
10
  from uncountable.integration.queue_runner.command_server.command_client import (
11
+ send_job_cancellation_message,
9
12
  send_job_queue_message,
10
13
  send_list_queued_jobs_message,
11
14
  send_retry_job_message,
12
15
  )
16
+ from uncountable.integration.queue_runner.command_server.types import (
17
+ CommandCancelJobStatus,
18
+ )
13
19
  from uncountable.integration.telemetry import Logger
14
20
  from uncountable.types import queued_job_t
15
21
 
@@ -25,12 +31,28 @@ def register_enqueue_job_parser(
25
31
  description="Process a job with a given host and job ID",
26
32
  )
27
33
  run_parser.add_argument("job_id", type=str, help="The ID of the job to process")
34
+ run_parser.add_argument(
35
+ "--payload", type=str, help="JSON payload for webhook invocation context"
36
+ )
28
37
 
29
38
  def _handle_enqueue_job(args: argparse.Namespace) -> None:
39
+ invocation_context: queued_job_t.InvocationContext
40
+
41
+ if args.payload is not None:
42
+ try:
43
+ webhook_payload = json.loads(args.payload)
44
+ invocation_context = queued_job_t.InvocationContextWebhook(
45
+ webhook_payload=webhook_payload
46
+ )
47
+ except json.JSONDecodeError as e:
48
+ raise ValueError(f"Invalid JSON payload: {e}")
49
+ else:
50
+ invocation_context = queued_job_t.InvocationContextManual()
51
+
30
52
  send_job_queue_message(
31
53
  job_ref_name=args.job_id,
32
54
  payload=queued_job_t.QueuedJobPayload(
33
- invocation_context=queued_job_t.InvocationContextManual()
55
+ invocation_context=invocation_context
34
56
  ),
35
57
  host=args.host,
36
58
  port=get_local_admin_server_port(),
@@ -39,6 +61,42 @@ def register_enqueue_job_parser(
39
61
  run_parser.set_defaults(func=_handle_enqueue_job)
40
62
 
41
63
 
64
+ def register_cancel_queued_job_parser(
65
+ sub_parser_manager: argparse._SubParsersAction,
66
+ parents: list[argparse.ArgumentParser],
67
+ ) -> None:
68
+ cancel_parser = sub_parser_manager.add_parser(
69
+ "cancel",
70
+ parents=parents,
71
+ help="Cancel a queued job with a given host and queued job UUID",
72
+ description="Cancel a job with a given host and queued job UUID",
73
+ )
74
+ cancel_parser.add_argument(
75
+ "uuid", type=str, help="The UUID of the queued job to cancel"
76
+ )
77
+
78
+ def _handle_cancel_queued_job(args: argparse.Namespace) -> None:
79
+ resp = send_job_cancellation_message(
80
+ queued_job_uuid=args.uuid,
81
+ host=args.host,
82
+ port=get_local_admin_server_port(),
83
+ )
84
+
85
+ match resp:
86
+ case CommandCancelJobStatus.CANCELLED_WITH_RESTART:
87
+ print(
88
+ "Job successfully cancelled. The integration server will restart."
89
+ )
90
+ case CommandCancelJobStatus.NO_JOB_FOUND:
91
+ print("Job not found.")
92
+ case CommandCancelJobStatus.JOB_ALREADY_COMPLETED:
93
+ print("Job already completed.")
94
+ case _:
95
+ assert_never(resp)
96
+
97
+ cancel_parser.set_defaults(func=_handle_cancel_queued_job)
98
+
99
+
42
100
  def register_list_queued_jobs(
43
101
  sub_parser_manager: argparse._SubParsersAction,
44
102
  parents: list[argparse.ArgumentParser],
@@ -133,6 +191,7 @@ def main() -> None:
133
191
  register_enqueue_job_parser(subparser_action, parents=[base_parser])
134
192
  register_retry_job_parser(subparser_action, parents=[base_parser])
135
193
  register_list_queued_jobs(subparser_action, parents=[base_parser])
194
+ register_cancel_queued_job_parser(subparser_action, parents=[base_parser])
136
195
 
137
196
  args = main_parser.parse_args()
138
197
  with logger.push_scope(args.command):
@@ -6,6 +6,9 @@ import simplejson as json
6
6
 
7
7
  from pkgs.serialization_util import serialize_for_api
8
8
  from uncountable.integration.queue_runner.command_server.protocol.command_server_pb2 import (
9
+ CancelJobRequest,
10
+ CancelJobResult,
11
+ CancelJobStatus,
9
12
  CheckHealthRequest,
10
13
  CheckHealthResult,
11
14
  EnqueueJobRequest,
@@ -18,6 +21,7 @@ from uncountable.integration.queue_runner.command_server.protocol.command_server
18
21
  VaccuumQueuedJobsResult,
19
22
  )
20
23
  from uncountable.integration.queue_runner.command_server.types import (
24
+ CommandCancelJobStatus,
21
25
  CommandServerBadResponse,
22
26
  CommandServerTimeout,
23
27
  )
@@ -63,6 +67,26 @@ def send_job_queue_message(
63
67
  return response.queued_job_uuid
64
68
 
65
69
 
70
+ def send_job_cancellation_message(
71
+ *, queued_job_uuid: str, host: str = "localhost", port: int
72
+ ) -> CommandCancelJobStatus:
73
+ with command_server_connection(host=host, port=port) as stub:
74
+ request = CancelJobRequest(job_uuid=queued_job_uuid)
75
+
76
+ response = stub.CancelJob(request, timeout=_DEFAULT_MESSAGE_TIMEOUT_SECS)
77
+
78
+ assert isinstance(response, CancelJobResult)
79
+ match response.status:
80
+ case CancelJobStatus.NO_JOB_FOUND:
81
+ return CommandCancelJobStatus.NO_JOB_FOUND
82
+ case CancelJobStatus.CANCELLED_WITH_RESTART:
83
+ return CommandCancelJobStatus.CANCELLED_WITH_RESTART
84
+ case CancelJobStatus.JOB_ALREADY_COMPLETED:
85
+ return CommandCancelJobStatus.JOB_ALREADY_COMPLETED
86
+ case _:
87
+ raise CommandServerBadResponse(f"unknown status: {response.status}")
88
+
89
+
66
90
  def send_retry_job_message(
67
91
  *,
68
92
  job_uuid: str,
@@ -1,4 +1,5 @@
1
1
  import asyncio
2
+ from typing import assert_never
2
3
 
3
4
  import grpc.aio as grpc_aio
4
5
  import simplejson as json
@@ -8,6 +9,9 @@ from grpc import StatusCode
8
9
  from pkgs.argument_parser import CachedParser
9
10
  from uncountable.core.environment import get_local_admin_server_port
10
11
  from uncountable.integration.queue_runner.command_server.protocol.command_server_pb2 import (
12
+ CancelJobRequest,
13
+ CancelJobResult,
14
+ CancelJobStatus,
11
15
  CheckHealthRequest,
12
16
  CheckHealthResult,
13
17
  EnqueueJobRequest,
@@ -20,6 +24,9 @@ from uncountable.integration.queue_runner.command_server.protocol.command_server
20
24
  VaccuumQueuedJobsResult,
21
25
  )
22
26
  from uncountable.integration.queue_runner.command_server.types import (
27
+ CommandCancelJob,
28
+ CommandCancelJobResponse,
29
+ CommandCancelJobStatus,
23
30
  CommandEnqueueJob,
24
31
  CommandEnqueueJobResponse,
25
32
  CommandQueue,
@@ -63,6 +70,33 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
63
70
  )
64
71
  return result
65
72
 
73
+ async def CancelJob(
74
+ self, request: CancelJobRequest, context: grpc_aio.ServicerContext
75
+ ) -> CancelJobResult:
76
+ response_queue: asyncio.Queue[CommandCancelJobResponse] = asyncio.Queue()
77
+ await command_queue.put(
78
+ CommandCancelJob(
79
+ queued_job_uuid=request.job_uuid,
80
+ response_queue=response_queue,
81
+ )
82
+ )
83
+
84
+ response = await response_queue.get()
85
+
86
+ proto_status: CancelJobStatus
87
+ match response.status:
88
+ case CommandCancelJobStatus.NO_JOB_FOUND:
89
+ proto_status = CancelJobStatus.NO_JOB_FOUND
90
+ case CommandCancelJobStatus.CANCELLED_WITH_RESTART:
91
+ proto_status = CancelJobStatus.CANCELLED_WITH_RESTART
92
+ case CommandCancelJobStatus.JOB_ALREADY_COMPLETED:
93
+ proto_status = CancelJobStatus.JOB_ALREADY_COMPLETED
94
+ case _:
95
+ assert_never(response.status)
96
+
97
+ result = CancelJobResult(status=proto_status)
98
+ return result
99
+
66
100
  async def RetryJob(
67
101
  self, request: RetryJobRequest, context: grpc_aio.ServicerContext
68
102
  ) -> RetryJobResult:
@@ -7,6 +7,7 @@ service CommandServer {
7
7
  rpc CheckHealth(CheckHealthRequest) returns (CheckHealthResult) {}
8
8
  rpc ListQueuedJobs(ListQueuedJobsRequest) returns (ListQueuedJobsResult) {}
9
9
  rpc VaccuumQueuedJobs(VaccuumQueuedJobsRequest) returns (VaccuumQueuedJobsResult) {}
10
+ rpc CancelJob(CancelJobRequest) returns (CancelJobResult) {}
10
11
  }
11
12
 
12
13
  message EnqueueJobRequest {
@@ -56,3 +57,17 @@ message ListQueuedJobsResult {
56
57
 
57
58
  repeated ListQueuedJobsResultItem queued_jobs = 1;
58
59
  }
60
+
61
+ message CancelJobRequest {
62
+ string job_uuid = 1;
63
+ }
64
+
65
+ enum CancelJobStatus {
66
+ CANCELLED_WITH_RESTART = 0;
67
+ NO_JOB_FOUND = 1;
68
+ JOB_ALREADY_COMPLETED = 2;
69
+ }
70
+
71
+ message CancelJobResult {
72
+ CancelJobStatus status = 1;
73
+ }
@@ -18,7 +18,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__
18
18
 
19
19
 
20
20
  DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
21
- b'\nQuncountable/integration/queue_runner/command_server/protocol/command_server.proto\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x11\x45nqueueJobRequest\x12\x14\n\x0cjob_ref_name\x18\x01 \x01(\t\x12\x1a\n\x12serialized_payload\x18\x02 \x01(\t"H\n\x10\x45nqueueJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1f\n\x0fRetryJobRequest\x12\x0c\n\x04uuid\x18\x01 \x01(\t"F\n\x0eRetryJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1a\n\x18VaccuumQueuedJobsRequest"\x19\n\x17VaccuumQueuedJobsResult"\x14\n\x12\x43heckHealthRequest"$\n\x11\x43heckHealthResult\x12\x0f\n\x07success\x18\x01 \x01(\x08"6\n\x15ListQueuedJobsRequest\x12\x0e\n\x06offset\x18\x01 \x01(\r\x12\r\n\x05limit\x18\x02 \x01(\r"\xf4\x01\n\x14ListQueuedJobsResult\x12\x43\n\x0bqueued_jobs\x18\x01 \x03(\x0b\x32..ListQueuedJobsResult.ListQueuedJobsResultItem\x1a\x96\x01\n\x18ListQueuedJobsResultItem\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x02 \x01(\t\x12\x14\n\x0cnum_attempts\x18\x03 \x01(\x03\x12\x30\n\x0csubmitted_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06status\x18\x05 \x01(\t2\xc0\x02\n\rCommandServer\x12\x35\n\nEnqueueJob\x12\x12.EnqueueJobRequest\x1a\x11.EnqueueJobResult"\x00\x12/\n\x08RetryJob\x12\x10.RetryJobRequest\x1a\x0f.RetryJobResult"\x00\x12\x38\n\x0b\x43heckHealth\x12\x13.CheckHealthRequest\x1a\x12.CheckHealthResult"\x00\x12\x41\n\x0eListQueuedJobs\x12\x16.ListQueuedJobsRequest\x1a\x15.ListQueuedJobsResult"\x00\x12J\n\x11VaccuumQueuedJobs\x12\x19.VaccuumQueuedJobsRequest\x1a\x18.VaccuumQueuedJobsResult"\x00\x62\x06proto3'
21
+ b'\nQuncountable/integration/queue_runner/command_server/protocol/command_server.proto\x1a\x1fgoogle/protobuf/timestamp.proto"E\n\x11\x45nqueueJobRequest\x12\x14\n\x0cjob_ref_name\x18\x01 \x01(\t\x12\x1a\n\x12serialized_payload\x18\x02 \x01(\t"H\n\x10\x45nqueueJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1f\n\x0fRetryJobRequest\x12\x0c\n\x04uuid\x18\x01 \x01(\t"F\n\x0eRetryJobResult\x12\x1b\n\x13successfully_queued\x18\x01 \x01(\x08\x12\x17\n\x0fqueued_job_uuid\x18\x02 \x01(\t"\x1a\n\x18VaccuumQueuedJobsRequest"\x19\n\x17VaccuumQueuedJobsResult"\x14\n\x12\x43heckHealthRequest"$\n\x11\x43heckHealthResult\x12\x0f\n\x07success\x18\x01 \x01(\x08"6\n\x15ListQueuedJobsRequest\x12\x0e\n\x06offset\x18\x01 \x01(\r\x12\r\n\x05limit\x18\x02 \x01(\r"\xf4\x01\n\x14ListQueuedJobsResult\x12\x43\n\x0bqueued_jobs\x18\x01 \x03(\x0b\x32..ListQueuedJobsResult.ListQueuedJobsResultItem\x1a\x96\x01\n\x18ListQueuedJobsResultItem\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x14\n\x0cjob_ref_name\x18\x02 \x01(\t\x12\x14\n\x0cnum_attempts\x18\x03 \x01(\x03\x12\x30\n\x0csubmitted_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06status\x18\x05 \x01(\t"$\n\x10\x43\x61ncelJobRequest\x12\x10\n\x08job_uuid\x18\x01 \x01(\t"3\n\x0f\x43\x61ncelJobResult\x12 \n\x06status\x18\x01 \x01(\x0e\x32\x10.CancelJobStatus*Z\n\x0f\x43\x61ncelJobStatus\x12\x1a\n\x16\x43\x41NCELLED_WITH_RESTART\x10\x00\x12\x10\n\x0cNO_JOB_FOUND\x10\x01\x12\x19\n\x15JOB_ALREADY_COMPLETED\x10\x02\x32\xf4\x02\n\rCommandServer\x12\x35\n\nEnqueueJob\x12\x12.EnqueueJobRequest\x1a\x11.EnqueueJobResult"\x00\x12/\n\x08RetryJob\x12\x10.RetryJobRequest\x1a\x0f.RetryJobResult"\x00\x12\x38\n\x0b\x43heckHealth\x12\x13.CheckHealthRequest\x1a\x12.CheckHealthResult"\x00\x12\x41\n\x0eListQueuedJobs\x12\x16.ListQueuedJobsRequest\x1a\x15.ListQueuedJobsResult"\x00\x12J\n\x11VaccuumQueuedJobs\x12\x19.VaccuumQueuedJobsRequest\x1a\x18.VaccuumQueuedJobsResult"\x00\x12\x32\n\tCancelJob\x12\x11.CancelJobRequest\x1a\x10.CancelJobResult"\x00\x62\x06proto3'
22
22
  )
23
23
 
24
24
  _globals = globals()
@@ -30,6 +30,8 @@ _builder.BuildTopDescriptorsAndMessages(
30
30
  )
31
31
  if _descriptor._USE_C_DESCRIPTORS == False:
32
32
  DESCRIPTOR._options = None
33
+ _globals["_CANCELJOBSTATUS"]._serialized_start = 877
34
+ _globals["_CANCELJOBSTATUS"]._serialized_end = 967
33
35
  _globals["_ENQUEUEJOBREQUEST"]._serialized_start = 118
34
36
  _globals["_ENQUEUEJOBREQUEST"]._serialized_end = 187
35
37
  _globals["_ENQUEUEJOBRESULT"]._serialized_start = 189
@@ -52,6 +54,10 @@ if _descriptor._USE_C_DESCRIPTORS == False:
52
54
  _globals["_LISTQUEUEDJOBSRESULT"]._serialized_end = 784
53
55
  _globals["_LISTQUEUEDJOBSRESULT_LISTQUEUEDJOBSRESULTITEM"]._serialized_start = 634
54
56
  _globals["_LISTQUEUEDJOBSRESULT_LISTQUEUEDJOBSRESULTITEM"]._serialized_end = 784
55
- _globals["_COMMANDSERVER"]._serialized_start = 787
56
- _globals["_COMMANDSERVER"]._serialized_end = 1107
57
+ _globals["_CANCELJOBREQUEST"]._serialized_start = 786
58
+ _globals["_CANCELJOBREQUEST"]._serialized_end = 822
59
+ _globals["_CANCELJOBRESULT"]._serialized_start = 824
60
+ _globals["_CANCELJOBRESULT"]._serialized_end = 875
61
+ _globals["_COMMANDSERVER"]._serialized_start = 970
62
+ _globals["_COMMANDSERVER"]._serialized_end = 1342
57
63
  # @@protoc_insertion_point(module_scope)
@@ -1,6 +1,7 @@
1
1
  # ruff: noqa
2
2
  from google.protobuf import timestamp_pb2 as _timestamp_pb2
3
3
  from google.protobuf.internal import containers as _containers
4
+ from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
4
5
  from google.protobuf import descriptor as _descriptor
5
6
  from google.protobuf import message as _message
6
7
  from typing import (
@@ -13,6 +14,16 @@ from typing import (
13
14
 
14
15
  DESCRIPTOR: _descriptor.FileDescriptor
15
16
 
17
+ class CancelJobStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
18
+ __slots__ = ()
19
+ CANCELLED_WITH_RESTART: _ClassVar[CancelJobStatus]
20
+ NO_JOB_FOUND: _ClassVar[CancelJobStatus]
21
+ JOB_ALREADY_COMPLETED: _ClassVar[CancelJobStatus]
22
+
23
+ CANCELLED_WITH_RESTART: CancelJobStatus
24
+ NO_JOB_FOUND: CancelJobStatus
25
+ JOB_ALREADY_COMPLETED: CancelJobStatus
26
+
16
27
  class EnqueueJobRequest(_message.Message):
17
28
  __slots__ = ("job_ref_name", "serialized_payload")
18
29
  JOB_REF_NAME_FIELD_NUMBER: _ClassVar[int]
@@ -112,3 +123,17 @@ class ListQueuedJobsResult(_message.Message):
112
123
  _Iterable[_Union[ListQueuedJobsResult.ListQueuedJobsResultItem, _Mapping]]
113
124
  ] = ...,
114
125
  ) -> None: ...
126
+
127
+ class CancelJobRequest(_message.Message):
128
+ __slots__ = ("job_uuid",)
129
+ JOB_UUID_FIELD_NUMBER: _ClassVar[int]
130
+ job_uuid: str
131
+ def __init__(self, job_uuid: _Optional[str] = ...) -> None: ...
132
+
133
+ class CancelJobResult(_message.Message):
134
+ __slots__ = ("status",)
135
+ STATUS_FIELD_NUMBER: _ClassVar[int]
136
+ status: CancelJobStatus
137
+ def __init__(
138
+ self, status: _Optional[_Union[CancelJobStatus, str]] = ...
139
+ ) -> None: ...
@@ -44,6 +44,11 @@ class CommandServerStub(object):
44
44
  request_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsRequest.SerializeToString,
45
45
  response_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsResult.FromString,
46
46
  )
47
+ self.CancelJob = channel.unary_unary(
48
+ "/CommandServer/CancelJob",
49
+ request_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CancelJobRequest.SerializeToString,
50
+ response_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CancelJobResult.FromString,
51
+ )
47
52
 
48
53
 
49
54
  class CommandServerServicer(object):
@@ -79,6 +84,12 @@ class CommandServerServicer(object):
79
84
  context.set_details("Method not implemented!")
80
85
  raise NotImplementedError("Method not implemented!")
81
86
 
87
+ def CancelJob(self, request, context):
88
+ """Missing associated documentation comment in .proto file."""
89
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
90
+ context.set_details("Method not implemented!")
91
+ raise NotImplementedError("Method not implemented!")
92
+
82
93
 
83
94
  def add_CommandServerServicer_to_server(servicer, server):
84
95
  rpc_method_handlers = {
@@ -107,6 +118,11 @@ def add_CommandServerServicer_to_server(servicer, server):
107
118
  request_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsRequest.FromString,
108
119
  response_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.VaccuumQueuedJobsResult.SerializeToString,
109
120
  ),
121
+ "CancelJob": grpc.unary_unary_rpc_method_handler(
122
+ servicer.CancelJob,
123
+ request_deserializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CancelJobRequest.FromString,
124
+ response_serializer=uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CancelJobResult.SerializeToString,
125
+ ),
110
126
  }
111
127
  generic_handler = grpc.method_handlers_generic_handler(
112
128
  "CommandServer", rpc_method_handlers
@@ -262,3 +278,32 @@ class CommandServer(object):
262
278
  timeout,
263
279
  metadata,
264
280
  )
281
+
282
+ @staticmethod
283
+ def CancelJob(
284
+ request,
285
+ target,
286
+ options=(),
287
+ channel_credentials=None,
288
+ call_credentials=None,
289
+ insecure=False,
290
+ compression=None,
291
+ wait_for_ready=None,
292
+ timeout=None,
293
+ metadata=None,
294
+ ):
295
+ return grpc.experimental.unary_unary(
296
+ request,
297
+ target,
298
+ "/CommandServer/CancelJob",
299
+ uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CancelJobRequest.SerializeToString,
300
+ uncountable_dot_integration_dot_queue__runner_dot_command__server_dot_protocol_dot_command__server__pb2.CancelJobResult.FromString,
301
+ options,
302
+ channel_credentials,
303
+ insecure,
304
+ call_credentials,
305
+ compression,
306
+ wait_for_ready,
307
+ timeout,
308
+ metadata,
309
+ )
@@ -6,10 +6,17 @@ from enum import StrEnum
6
6
  from uncountable.types import queued_job_t
7
7
 
8
8
 
9
+ class CommandCancelJobStatus(StrEnum):
10
+ CANCELLED_WITH_RESTART = "cancelled_with_restart"
11
+ NO_JOB_FOUND = "no_job_found"
12
+ JOB_ALREADY_COMPLETED = "job_already_completed"
13
+
14
+
9
15
  class CommandType(StrEnum):
10
16
  ENQUEUE_JOB = "enqueue_job"
11
17
  RETRY_JOB = "retry_job"
12
18
  VACCUUM_QUEUED_JOBS = "vaccuum_queued_jobs"
19
+ CANCEL_JOB = "cancel_job"
13
20
 
14
21
 
15
22
  RT = typing.TypeVar("RT")
@@ -55,7 +62,20 @@ class CommandVaccuumQueuedJobs(CommandBase[CommandVaccuumQueuedJobsResponse]):
55
62
  type: CommandType = CommandType.VACCUUM_QUEUED_JOBS
56
63
 
57
64
 
58
- _Command = CommandEnqueueJob | CommandRetryJob | CommandVaccuumQueuedJobs
65
+ @dataclass(kw_only=True)
66
+ class CommandCancelJobResponse:
67
+ status: CommandCancelJobStatus
68
+
69
+
70
+ @dataclass(kw_only=True)
71
+ class CommandCancelJob(CommandBase[CommandCancelJobResponse]):
72
+ type: CommandType = CommandType.CANCEL_JOB
73
+ queued_job_uuid: str
74
+
75
+
76
+ _Command = (
77
+ CommandEnqueueJob | CommandRetryJob | CommandVaccuumQueuedJobs | CommandCancelJob
78
+ )
59
79
 
60
80
 
61
81
  CommandQueue = asyncio.Queue[_Command]
@@ -234,6 +234,31 @@ class DatastoreSqlite(Datastore):
234
234
 
235
235
  return queued_jobs
236
236
 
237
+ def get_queued_job(self, *, uuid: str) -> queued_job_t.QueuedJob | None:
238
+ with self.session_maker() as session:
239
+ select_stmt = select(
240
+ QueuedJob.id,
241
+ QueuedJob.payload,
242
+ QueuedJob.num_attempts,
243
+ QueuedJob.job_ref_name,
244
+ QueuedJob.status,
245
+ QueuedJob.submitted_at,
246
+ ).filter(QueuedJob.id == uuid)
247
+
248
+ row = session.execute(select_stmt).one_or_none()
249
+ return (
250
+ queued_job_t.QueuedJob(
251
+ queued_job_uuid=row.id,
252
+ job_ref_name=row.job_ref_name,
253
+ num_attempts=row.num_attempts,
254
+ status=row.status or queued_job_t.JobStatus.QUEUED,
255
+ submitted_at=row.submitted_at,
256
+ payload=queued_job_payload_parser.parse_storage(row.payload),
257
+ )
258
+ if row is not None
259
+ else None
260
+ )
261
+
237
262
  def vaccuum_queued_jobs(self) -> None:
238
263
  with self.session_maker() as session:
239
264
  delete_stmt = (
@@ -27,3 +27,6 @@ class Datastore(ABC):
27
27
  def list_queued_job_metadata(
28
28
  self, offset: int, limit: int | None
29
29
  ) -> list[queued_job_t.QueuedJobMetadata]: ...
30
+
31
+ @abstractmethod
32
+ def get_queued_job(self, *, uuid: str) -> queued_job_t.QueuedJob | None: ...
@@ -1,5 +1,7 @@
1
1
  import asyncio
2
+ import os
2
3
  import sys
4
+ import threading
3
5
  import typing
4
6
  from concurrent.futures import ProcessPoolExecutor
5
7
  from dataclasses import dataclass
@@ -15,6 +17,9 @@ from uncountable.integration.queue_runner.command_server import (
15
17
  CommandTask,
16
18
  )
17
19
  from uncountable.integration.queue_runner.command_server.types import (
20
+ CommandCancelJob,
21
+ CommandCancelJobResponse,
22
+ CommandCancelJobStatus,
18
23
  CommandVaccuumQueuedJobs,
19
24
  )
20
25
  from uncountable.integration.queue_runner.datastore import DatastoreSqlite
@@ -24,7 +29,7 @@ from uncountable.integration.scan_profiles import load_profiles
24
29
  from uncountable.integration.telemetry import Logger
25
30
  from uncountable.types import job_definition_t, queued_job_t
26
31
 
27
- from .types import ResultQueue, ResultTask
32
+ from .types import RESTART_EXIT_CODE, ResultQueue, ResultTask
28
33
 
29
34
  _MAX_JOB_WORKERS = 5
30
35
 
@@ -142,6 +147,34 @@ async def start_scheduler(
142
147
  CommandEnqueueJobResponse(queued_job_uuid=queued_job_uuid)
143
148
  )
144
149
 
150
+ async def _handle_cancel_job_command(command: CommandCancelJob) -> None:
151
+ queued_job = datastore.get_queued_job(uuid=command.queued_job_uuid)
152
+ if queued_job is None:
153
+ await command.response_queue.put(
154
+ CommandCancelJobResponse(status=CommandCancelJobStatus.NO_JOB_FOUND)
155
+ )
156
+ return
157
+
158
+ if queued_job.status == queued_job_t.JobStatus.QUEUED:
159
+ datastore.remove_job_from_queue(command.queued_job_uuid)
160
+ await command.response_queue.put(
161
+ CommandCancelJobResponse(
162
+ status=CommandCancelJobStatus.CANCELLED_WITH_RESTART
163
+ )
164
+ )
165
+
166
+ def delayed_exit() -> None:
167
+ os._exit(RESTART_EXIT_CODE)
168
+
169
+ threading.Timer(interval=5, function=delayed_exit).start()
170
+
171
+ else:
172
+ await command.response_queue.put(
173
+ CommandCancelJobResponse(
174
+ status=CommandCancelJobStatus.JOB_ALREADY_COMPLETED
175
+ )
176
+ )
177
+
145
178
  async def _handle_retry_job_command(command: CommandRetryJob) -> None:
146
179
  queued_job = datastore.retry_job(command.queued_job_uuid)
147
180
  if queued_job is None:
@@ -181,6 +214,8 @@ async def start_scheduler(
181
214
  await _handle_retry_job_command(command=command)
182
215
  case CommandVaccuumQueuedJobs():
183
216
  _handle_vaccuum_queued_jobs_command(command=command)
217
+ case CommandCancelJob():
218
+ await _handle_cancel_job_command(command=command)
184
219
  case _:
185
220
  typing.assert_never(command)
186
221
  command_task = asyncio.create_task(command_queue.get())
@@ -5,3 +5,5 @@ from uncountable.types import queued_job_t
5
5
  ListenQueue = Queue[queued_job_t.QueuedJob]
6
6
  ResultQueue = Queue[queued_job_t.QueuedJobResult]
7
7
  ResultTask = Task[queued_job_t.QueuedJobResult]
8
+
9
+ RESTART_EXIT_CODE = 147