UncountablePythonSDK 0.0.178__py3-none-any.whl → 0.0.179__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- examples/integration-server/pyproject.toml +1 -1
- pkgs/filesystem_utils/_file_share_session.py +1 -0
- uncountable/core/client.py +1 -1
- uncountable/integration/db/connect.py +35 -5
- uncountable/integration/queue_runner/command_server/command_server.py +2 -2
- uncountable/integration/queue_runner/datastore/__init__.py +9 -1
- uncountable/integration/queue_runner/datastore/datastore_postgres.py +12 -0
- uncountable/integration/queue_runner/datastore/datastore_sql_base.py +286 -0
- uncountable/integration/queue_runner/datastore/datastore_sqlite.py +6 -284
- uncountable/integration/queue_runner/datastore/factory.py +24 -0
- uncountable/integration/queue_runner/datastore/interface.py +17 -0
- uncountable/integration/queue_runner/datastore/model.py +1 -1
- uncountable/integration/queue_runner/job_scheduler.py +2 -5
- uncountable/integration/queue_runner/queue_runner.py +2 -6
- uncountable/integration/queue_runner/worker.py +1 -1
- uncountable/types/entity_t.py +8 -0
- {uncountablepythonsdk-0.0.178.dist-info → uncountablepythonsdk-0.0.179.dist-info}/METADATA +2 -1
- {uncountablepythonsdk-0.0.178.dist-info → uncountablepythonsdk-0.0.179.dist-info}/RECORD +20 -17
- {uncountablepythonsdk-0.0.178.dist-info → uncountablepythonsdk-0.0.179.dist-info}/WHEEL +1 -1
- {uncountablepythonsdk-0.0.178.dist-info → uncountablepythonsdk-0.0.179.dist-info}/top_level.txt +0 -0
uncountable/core/client.py
CHANGED
|
@@ -201,7 +201,7 @@ DownloadedFiles = list[DownloadedFile]
|
|
|
201
201
|
|
|
202
202
|
|
|
203
203
|
class Client(ClientMethods):
|
|
204
|
-
_parser_map: dict[type, CachedParser] = {}
|
|
204
|
+
_parser_map: typing.ClassVar[dict[type, CachedParser]] = {}
|
|
205
205
|
_auth_details: AuthDetailsAll
|
|
206
206
|
_base_url: str
|
|
207
207
|
_file_uploader: FileUploader
|
|
@@ -1,18 +1,48 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from enum import StrEnum
|
|
3
3
|
|
|
4
|
+
from opentelemetry.trace import get_current_span
|
|
4
5
|
from sqlalchemy import create_engine
|
|
5
6
|
from sqlalchemy.engine.base import Engine
|
|
6
7
|
|
|
8
|
+
from uncountable.integration.telemetry import Logger
|
|
9
|
+
|
|
7
10
|
|
|
8
11
|
class IntegrationDBService(StrEnum):
|
|
9
12
|
CRON = "cron"
|
|
10
13
|
RUNNER = "runner"
|
|
11
14
|
|
|
12
15
|
|
|
16
|
+
_DB_URI_ENV_VARS: dict[IntegrationDBService, str] = {
|
|
17
|
+
IntegrationDBService.CRON: "UNC_CRON_DB_URI",
|
|
18
|
+
IntegrationDBService.RUNNER: "UNC_RUNNER_DB_URI",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
_DEPRECATED_DB_URI_ENV_VARS: dict[IntegrationDBService, str] = {
|
|
22
|
+
IntegrationDBService.CRON: "UNC_CRON_SQLITE_URI",
|
|
23
|
+
IntegrationDBService.RUNNER: "UNC_RUNNER_SQLITE_URI",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _resolve_db_uri(service: IntegrationDBService) -> str:
|
|
28
|
+
logger = Logger(get_current_span())
|
|
29
|
+
env_var = _DB_URI_ENV_VARS[service]
|
|
30
|
+
uri = os.environ.get(env_var)
|
|
31
|
+
if uri is not None:
|
|
32
|
+
return uri
|
|
33
|
+
|
|
34
|
+
deprecated_env_var = _DEPRECATED_DB_URI_ENV_VARS[service]
|
|
35
|
+
deprecated_uri = os.environ.get(deprecated_env_var)
|
|
36
|
+
if deprecated_uri is not None:
|
|
37
|
+
logger.log_warning(
|
|
38
|
+
f"{deprecated_env_var} is deprecated and will be removed; set {env_var} instead.",
|
|
39
|
+
)
|
|
40
|
+
return deprecated_uri
|
|
41
|
+
|
|
42
|
+
raise KeyError(
|
|
43
|
+
f"no database URI configured for the {service} service: set {env_var}"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
13
47
|
def create_db_engine(service: IntegrationDBService) -> Engine:
|
|
14
|
-
|
|
15
|
-
case IntegrationDBService.CRON:
|
|
16
|
-
return create_engine(os.environ["UNC_CRON_SQLITE_URI"])
|
|
17
|
-
case IntegrationDBService.RUNNER:
|
|
18
|
-
return create_engine(os.environ["UNC_RUNNER_SQLITE_URI"])
|
|
48
|
+
return create_engine(_resolve_db_uri(service))
|
|
@@ -41,7 +41,7 @@ from uncountable.integration.queue_runner.command_server.types import (
|
|
|
41
41
|
CommandVaccuumQueuedJobs,
|
|
42
42
|
CommandVaccuumQueuedJobsResponse,
|
|
43
43
|
)
|
|
44
|
-
from uncountable.integration.queue_runner.datastore import
|
|
44
|
+
from uncountable.integration.queue_runner.datastore import Datastore
|
|
45
45
|
from uncountable.types import queued_job_t
|
|
46
46
|
|
|
47
47
|
from .constants import ListQueuedJobsConstants
|
|
@@ -53,7 +53,7 @@ from .protocol.command_server_pb2_grpc import (
|
|
|
53
53
|
queued_job_payload_parser = CachedParser(queued_job_t.QueuedJobPayload)
|
|
54
54
|
|
|
55
55
|
|
|
56
|
-
async def serve(command_queue: CommandQueue, datastore:
|
|
56
|
+
async def serve(command_queue: CommandQueue, datastore: Datastore) -> None:
|
|
57
57
|
server = grpc_aio.server()
|
|
58
58
|
|
|
59
59
|
class CommandServerHandler(CommandServerServicer):
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
from .datastore_postgres import DatastorePostgres
|
|
1
2
|
from .datastore_sqlite import DatastoreSqlite
|
|
3
|
+
from .factory import create_datastore
|
|
4
|
+
from .interface import Datastore
|
|
2
5
|
|
|
3
|
-
__all__: list[str] = [
|
|
6
|
+
__all__: list[str] = [
|
|
7
|
+
"Datastore",
|
|
8
|
+
"DatastorePostgres",
|
|
9
|
+
"DatastoreSqlite",
|
|
10
|
+
"create_datastore",
|
|
11
|
+
]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from sqlalchemy.engine import Engine
|
|
2
|
+
|
|
3
|
+
from uncountable.integration.queue_runner.datastore.datastore_sql_base import (
|
|
4
|
+
DatastoreSqlBase,
|
|
5
|
+
)
|
|
6
|
+
from uncountable.integration.queue_runner.datastore.model import Base
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class DatastorePostgres(DatastoreSqlBase):
|
|
10
|
+
@classmethod
|
|
11
|
+
def setup(cls, engine: Engine) -> None:
|
|
12
|
+
Base.metadata.create_all(engine)
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import uuid
|
|
3
|
+
from datetime import UTC
|
|
4
|
+
|
|
5
|
+
from sqlalchemy import delete, insert, or_, select, update
|
|
6
|
+
|
|
7
|
+
from pkgs.argument_parser import CachedParser
|
|
8
|
+
from pkgs.serialization_util import serialize_for_storage
|
|
9
|
+
from uncountable.integration.db.session import DBSessionMaker
|
|
10
|
+
from uncountable.integration.queue_runner.datastore.interface import Datastore
|
|
11
|
+
from uncountable.integration.queue_runner.datastore.model import QueuedJob
|
|
12
|
+
from uncountable.types import queued_job_t
|
|
13
|
+
|
|
14
|
+
queued_job_payload_parser = CachedParser(queued_job_t.QueuedJobPayload)
|
|
15
|
+
|
|
16
|
+
MAX_QUEUE_WINDOW_DAYS = 30
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DatastoreSqlBase(Datastore):
|
|
20
|
+
def __init__(self, session_maker: DBSessionMaker) -> None:
|
|
21
|
+
self.session_maker = session_maker
|
|
22
|
+
super().__init__()
|
|
23
|
+
|
|
24
|
+
def add_job_to_queue(
|
|
25
|
+
self, job_payload: queued_job_t.QueuedJobPayload, job_ref_name: str
|
|
26
|
+
) -> queued_job_t.QueuedJob:
|
|
27
|
+
with self.session_maker() as session:
|
|
28
|
+
serialized_payload = serialize_for_storage(job_payload)
|
|
29
|
+
queued_job_uuid = str(uuid.uuid4())
|
|
30
|
+
num_attempts = 0
|
|
31
|
+
submitted_at = datetime.datetime.now(UTC)
|
|
32
|
+
insert_stmt = insert(QueuedJob).values({
|
|
33
|
+
QueuedJob.id.key: queued_job_uuid,
|
|
34
|
+
QueuedJob.job_ref_name.key: job_ref_name,
|
|
35
|
+
QueuedJob.payload.key: serialized_payload,
|
|
36
|
+
QueuedJob.status.key: queued_job_t.JobStatus.QUEUED,
|
|
37
|
+
QueuedJob.num_attempts: num_attempts,
|
|
38
|
+
QueuedJob.submitted_at: submitted_at,
|
|
39
|
+
})
|
|
40
|
+
session.execute(insert_stmt)
|
|
41
|
+
return queued_job_t.QueuedJob(
|
|
42
|
+
queued_job_uuid=queued_job_uuid,
|
|
43
|
+
job_ref_name=job_ref_name,
|
|
44
|
+
payload=job_payload,
|
|
45
|
+
status=queued_job_t.JobStatus.QUEUED,
|
|
46
|
+
submitted_at=submitted_at,
|
|
47
|
+
num_attempts=num_attempts,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def retry_job(
|
|
51
|
+
self,
|
|
52
|
+
queued_job_uuid: str,
|
|
53
|
+
) -> queued_job_t.QueuedJob | None:
|
|
54
|
+
with self.session_maker() as session:
|
|
55
|
+
select_stmt = select(
|
|
56
|
+
QueuedJob.id,
|
|
57
|
+
QueuedJob.payload,
|
|
58
|
+
QueuedJob.num_attempts,
|
|
59
|
+
QueuedJob.job_ref_name,
|
|
60
|
+
QueuedJob.status,
|
|
61
|
+
QueuedJob.submitted_at,
|
|
62
|
+
).filter(QueuedJob.id == queued_job_uuid)
|
|
63
|
+
existing_job = session.execute(select_stmt).one_or_none()
|
|
64
|
+
|
|
65
|
+
if (
|
|
66
|
+
existing_job is None
|
|
67
|
+
or existing_job.status != queued_job_t.JobStatus.FAILED
|
|
68
|
+
):
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
update_stmt = (
|
|
72
|
+
update(QueuedJob)
|
|
73
|
+
.values({QueuedJob.status.key: queued_job_t.JobStatus.QUEUED})
|
|
74
|
+
.filter(QueuedJob.id == queued_job_uuid)
|
|
75
|
+
)
|
|
76
|
+
session.execute(update_stmt)
|
|
77
|
+
|
|
78
|
+
return queued_job_t.QueuedJob(
|
|
79
|
+
queued_job_uuid=existing_job.id,
|
|
80
|
+
job_ref_name=existing_job.job_ref_name,
|
|
81
|
+
num_attempts=existing_job.num_attempts,
|
|
82
|
+
status=queued_job_t.JobStatus.QUEUED,
|
|
83
|
+
submitted_at=existing_job.submitted_at,
|
|
84
|
+
payload=queued_job_payload_parser.parse_storage(existing_job.payload),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
def increment_num_attempts(self, queued_job_uuid: str) -> int:
|
|
88
|
+
with self.session_maker() as session:
|
|
89
|
+
update_stmt = (
|
|
90
|
+
update(QueuedJob)
|
|
91
|
+
.values({QueuedJob.num_attempts.key: QueuedJob.num_attempts + 1})
|
|
92
|
+
.filter(QueuedJob.id == queued_job_uuid)
|
|
93
|
+
)
|
|
94
|
+
session.execute(update_stmt)
|
|
95
|
+
session.flush()
|
|
96
|
+
# IMPROVE: python3's sqlite does not support the RETURNING clause
|
|
97
|
+
select_stmt = select(QueuedJob.num_attempts).filter(
|
|
98
|
+
QueuedJob.id == queued_job_uuid
|
|
99
|
+
)
|
|
100
|
+
return int(session.execute(select_stmt).one().num_attempts)
|
|
101
|
+
|
|
102
|
+
def remove_job_from_queue(self, queued_job_uuid: str) -> None:
|
|
103
|
+
with self.session_maker() as session:
|
|
104
|
+
delete_stmt = delete(QueuedJob).filter(QueuedJob.id == queued_job_uuid)
|
|
105
|
+
session.execute(delete_stmt)
|
|
106
|
+
|
|
107
|
+
def update_job_status(
|
|
108
|
+
self, queued_job_uuid: str, status: queued_job_t.JobStatus
|
|
109
|
+
) -> None:
|
|
110
|
+
with self.session_maker() as session:
|
|
111
|
+
update_stmt = (
|
|
112
|
+
update(QueuedJob)
|
|
113
|
+
.values({QueuedJob.status.key: status})
|
|
114
|
+
.filter(QueuedJob.id == queued_job_uuid)
|
|
115
|
+
)
|
|
116
|
+
session.execute(update_stmt)
|
|
117
|
+
|
|
118
|
+
def list_queued_job_metadata(
|
|
119
|
+
self,
|
|
120
|
+
*,
|
|
121
|
+
offset: int = 0,
|
|
122
|
+
limit: int | None = 100,
|
|
123
|
+
status: queued_job_t.JobStatus | None = None,
|
|
124
|
+
job_ref_name: str | None = None,
|
|
125
|
+
submitted_from: datetime.datetime | None = None,
|
|
126
|
+
submitted_to: datetime.datetime | None = None,
|
|
127
|
+
) -> list[queued_job_t.QueuedJobMetadata]:
|
|
128
|
+
with self.session_maker() as session:
|
|
129
|
+
select_statement = (
|
|
130
|
+
select(
|
|
131
|
+
QueuedJob.id,
|
|
132
|
+
QueuedJob.job_ref_name,
|
|
133
|
+
QueuedJob.num_attempts,
|
|
134
|
+
QueuedJob.status,
|
|
135
|
+
QueuedJob.submitted_at,
|
|
136
|
+
)
|
|
137
|
+
.order_by(QueuedJob.submitted_at.desc())
|
|
138
|
+
.offset(offset)
|
|
139
|
+
.limit(limit)
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
if status is not None:
|
|
143
|
+
select_statement = select_statement.filter(QueuedJob.status == status)
|
|
144
|
+
if job_ref_name is not None:
|
|
145
|
+
select_statement = select_statement.filter(
|
|
146
|
+
QueuedJob.job_ref_name == job_ref_name
|
|
147
|
+
)
|
|
148
|
+
if submitted_from is not None:
|
|
149
|
+
select_statement = select_statement.filter(
|
|
150
|
+
QueuedJob.submitted_at >= submitted_from
|
|
151
|
+
)
|
|
152
|
+
if submitted_to is not None:
|
|
153
|
+
select_statement = select_statement.filter(
|
|
154
|
+
QueuedJob.submitted_at < submitted_to
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
queued_job_metadata: list[queued_job_t.QueuedJobMetadata] = [
|
|
158
|
+
queued_job_t.QueuedJobMetadata(
|
|
159
|
+
queued_job_uuid=row.id,
|
|
160
|
+
job_ref_name=row.job_ref_name,
|
|
161
|
+
num_attempts=row.num_attempts,
|
|
162
|
+
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
163
|
+
submitted_at=row.submitted_at,
|
|
164
|
+
)
|
|
165
|
+
for row in session.execute(select_statement)
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
return queued_job_metadata
|
|
169
|
+
|
|
170
|
+
def get_next_queued_job_for_ref_name(
|
|
171
|
+
self, job_ref_name: str
|
|
172
|
+
) -> queued_job_t.QueuedJob | None:
|
|
173
|
+
with self.session_maker() as session:
|
|
174
|
+
select_stmt = (
|
|
175
|
+
select(
|
|
176
|
+
QueuedJob.id,
|
|
177
|
+
QueuedJob.payload,
|
|
178
|
+
QueuedJob.num_attempts,
|
|
179
|
+
QueuedJob.job_ref_name,
|
|
180
|
+
QueuedJob.status,
|
|
181
|
+
QueuedJob.submitted_at,
|
|
182
|
+
)
|
|
183
|
+
.filter(QueuedJob.job_ref_name == job_ref_name)
|
|
184
|
+
.filter(
|
|
185
|
+
or_(
|
|
186
|
+
QueuedJob.status == queued_job_t.JobStatus.QUEUED,
|
|
187
|
+
QueuedJob.status.is_(None),
|
|
188
|
+
)
|
|
189
|
+
)
|
|
190
|
+
.limit(1)
|
|
191
|
+
.order_by(QueuedJob.submitted_at)
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
for row in session.execute(select_stmt):
|
|
195
|
+
parsed_payload = queued_job_payload_parser.parse_storage(row.payload)
|
|
196
|
+
return queued_job_t.QueuedJob(
|
|
197
|
+
queued_job_uuid=row.id,
|
|
198
|
+
job_ref_name=row.job_ref_name,
|
|
199
|
+
num_attempts=row.num_attempts,
|
|
200
|
+
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
201
|
+
submitted_at=row.submitted_at,
|
|
202
|
+
payload=parsed_payload,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
return None
|
|
206
|
+
|
|
207
|
+
def load_job_queue(self) -> list[queued_job_t.QueuedJob]:
|
|
208
|
+
with self.session_maker() as session:
|
|
209
|
+
select_stmt = (
|
|
210
|
+
select(
|
|
211
|
+
QueuedJob.id,
|
|
212
|
+
QueuedJob.payload,
|
|
213
|
+
QueuedJob.num_attempts,
|
|
214
|
+
QueuedJob.job_ref_name,
|
|
215
|
+
QueuedJob.status,
|
|
216
|
+
QueuedJob.submitted_at,
|
|
217
|
+
)
|
|
218
|
+
.filter(
|
|
219
|
+
or_(
|
|
220
|
+
QueuedJob.status == queued_job_t.JobStatus.QUEUED,
|
|
221
|
+
QueuedJob.status.is_(None),
|
|
222
|
+
)
|
|
223
|
+
)
|
|
224
|
+
.order_by(QueuedJob.submitted_at)
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
queued_jobs: list[queued_job_t.QueuedJob] = []
|
|
228
|
+
for row in session.execute(select_stmt):
|
|
229
|
+
parsed_payload = queued_job_payload_parser.parse_storage(row.payload)
|
|
230
|
+
queued_jobs.append(
|
|
231
|
+
queued_job_t.QueuedJob(
|
|
232
|
+
queued_job_uuid=row.id,
|
|
233
|
+
job_ref_name=row.job_ref_name,
|
|
234
|
+
num_attempts=row.num_attempts,
|
|
235
|
+
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
236
|
+
submitted_at=row.submitted_at,
|
|
237
|
+
payload=parsed_payload,
|
|
238
|
+
)
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
return queued_jobs
|
|
242
|
+
|
|
243
|
+
def get_queued_job(self, *, uuid: str) -> queued_job_t.QueuedJob | None:
|
|
244
|
+
with self.session_maker() as session:
|
|
245
|
+
select_stmt = select(
|
|
246
|
+
QueuedJob.id,
|
|
247
|
+
QueuedJob.payload,
|
|
248
|
+
QueuedJob.num_attempts,
|
|
249
|
+
QueuedJob.job_ref_name,
|
|
250
|
+
QueuedJob.status,
|
|
251
|
+
QueuedJob.submitted_at,
|
|
252
|
+
).filter(QueuedJob.id == uuid)
|
|
253
|
+
|
|
254
|
+
row = session.execute(select_stmt).one_or_none()
|
|
255
|
+
return (
|
|
256
|
+
queued_job_t.QueuedJob(
|
|
257
|
+
queued_job_uuid=row.id,
|
|
258
|
+
job_ref_name=row.job_ref_name,
|
|
259
|
+
num_attempts=row.num_attempts,
|
|
260
|
+
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
261
|
+
submitted_at=row.submitted_at,
|
|
262
|
+
payload=queued_job_payload_parser.parse_storage(row.payload),
|
|
263
|
+
)
|
|
264
|
+
if row is not None
|
|
265
|
+
else None
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
def vaccuum_queued_jobs(self) -> list[str]:
|
|
269
|
+
with self.session_maker() as session:
|
|
270
|
+
select_stmt = (
|
|
271
|
+
select(QueuedJob.id)
|
|
272
|
+
.filter(QueuedJob.status == queued_job_t.JobStatus.QUEUED)
|
|
273
|
+
.filter(
|
|
274
|
+
QueuedJob.submitted_at
|
|
275
|
+
<= (
|
|
276
|
+
datetime.datetime.now(UTC)
|
|
277
|
+
- datetime.timedelta(days=MAX_QUEUE_WINDOW_DAYS)
|
|
278
|
+
)
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
deleted_uuids: list[str] = [row.id for row in session.execute(select_stmt)]
|
|
282
|
+
|
|
283
|
+
delete_stmt = delete(QueuedJob).filter(QueuedJob.id.in_(deleted_uuids))
|
|
284
|
+
session.execute(delete_stmt)
|
|
285
|
+
|
|
286
|
+
return deleted_uuids
|
|
@@ -1,27 +1,13 @@
|
|
|
1
|
-
import
|
|
2
|
-
import uuid
|
|
3
|
-
from datetime import UTC
|
|
4
|
-
|
|
5
|
-
from sqlalchemy import delete, insert, or_, select, text, update
|
|
1
|
+
from sqlalchemy import text
|
|
6
2
|
from sqlalchemy.engine import Engine
|
|
7
3
|
|
|
8
|
-
from
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
from uncountable.integration.queue_runner.datastore.
|
|
12
|
-
from uncountable.integration.queue_runner.datastore.model import Base, QueuedJob
|
|
13
|
-
from uncountable.types import queued_job_t
|
|
14
|
-
|
|
15
|
-
queued_job_payload_parser = CachedParser(queued_job_t.QueuedJobPayload)
|
|
16
|
-
|
|
17
|
-
MAX_QUEUE_WINDOW_DAYS = 30
|
|
18
|
-
|
|
4
|
+
from uncountable.integration.queue_runner.datastore.datastore_sql_base import (
|
|
5
|
+
DatastoreSqlBase,
|
|
6
|
+
)
|
|
7
|
+
from uncountable.integration.queue_runner.datastore.model import Base
|
|
19
8
|
|
|
20
|
-
class DatastoreSqlite(Datastore):
|
|
21
|
-
def __init__(self, session_maker: DBSessionMaker) -> None:
|
|
22
|
-
self.session_maker = session_maker
|
|
23
|
-
super().__init__()
|
|
24
9
|
|
|
10
|
+
class DatastoreSqlite(DatastoreSqlBase):
|
|
25
11
|
@classmethod
|
|
26
12
|
def setup(cls, engine: Engine) -> None:
|
|
27
13
|
Base.metadata.create_all(engine)
|
|
@@ -46,267 +32,3 @@ class DatastoreSqlite(Datastore):
|
|
|
46
32
|
"create index if not exists ix_queued_jobs_submitted_at on queued_jobs (submitted_at)"
|
|
47
33
|
)
|
|
48
34
|
)
|
|
49
|
-
|
|
50
|
-
def add_job_to_queue(
|
|
51
|
-
self, job_payload: queued_job_t.QueuedJobPayload, job_ref_name: str
|
|
52
|
-
) -> queued_job_t.QueuedJob:
|
|
53
|
-
with self.session_maker() as session:
|
|
54
|
-
serialized_payload = serialize_for_storage(job_payload)
|
|
55
|
-
queued_job_uuid = str(uuid.uuid4())
|
|
56
|
-
num_attempts = 0
|
|
57
|
-
submitted_at = datetime.datetime.now(UTC)
|
|
58
|
-
insert_stmt = insert(QueuedJob).values({
|
|
59
|
-
QueuedJob.id.key: queued_job_uuid,
|
|
60
|
-
QueuedJob.job_ref_name.key: job_ref_name,
|
|
61
|
-
QueuedJob.payload.key: serialized_payload,
|
|
62
|
-
QueuedJob.status.key: queued_job_t.JobStatus.QUEUED,
|
|
63
|
-
QueuedJob.num_attempts: num_attempts,
|
|
64
|
-
QueuedJob.submitted_at: submitted_at,
|
|
65
|
-
})
|
|
66
|
-
session.execute(insert_stmt)
|
|
67
|
-
return queued_job_t.QueuedJob(
|
|
68
|
-
queued_job_uuid=queued_job_uuid,
|
|
69
|
-
job_ref_name=job_ref_name,
|
|
70
|
-
payload=job_payload,
|
|
71
|
-
status=queued_job_t.JobStatus.QUEUED,
|
|
72
|
-
submitted_at=submitted_at,
|
|
73
|
-
num_attempts=num_attempts,
|
|
74
|
-
)
|
|
75
|
-
|
|
76
|
-
def retry_job(
|
|
77
|
-
self,
|
|
78
|
-
queued_job_uuid: str,
|
|
79
|
-
) -> queued_job_t.QueuedJob | None:
|
|
80
|
-
with self.session_maker() as session:
|
|
81
|
-
select_stmt = select(
|
|
82
|
-
QueuedJob.id,
|
|
83
|
-
QueuedJob.payload,
|
|
84
|
-
QueuedJob.num_attempts,
|
|
85
|
-
QueuedJob.job_ref_name,
|
|
86
|
-
QueuedJob.status,
|
|
87
|
-
QueuedJob.submitted_at,
|
|
88
|
-
).filter(QueuedJob.id == queued_job_uuid)
|
|
89
|
-
existing_job = session.execute(select_stmt).one_or_none()
|
|
90
|
-
|
|
91
|
-
if (
|
|
92
|
-
existing_job is None
|
|
93
|
-
or existing_job.status != queued_job_t.JobStatus.FAILED
|
|
94
|
-
):
|
|
95
|
-
return None
|
|
96
|
-
|
|
97
|
-
update_stmt = (
|
|
98
|
-
update(QueuedJob)
|
|
99
|
-
.values({QueuedJob.status.key: queued_job_t.JobStatus.QUEUED})
|
|
100
|
-
.filter(QueuedJob.id == queued_job_uuid)
|
|
101
|
-
)
|
|
102
|
-
session.execute(update_stmt)
|
|
103
|
-
|
|
104
|
-
return queued_job_t.QueuedJob(
|
|
105
|
-
queued_job_uuid=existing_job.id,
|
|
106
|
-
job_ref_name=existing_job.job_ref_name,
|
|
107
|
-
num_attempts=existing_job.num_attempts,
|
|
108
|
-
status=queued_job_t.JobStatus.QUEUED,
|
|
109
|
-
submitted_at=existing_job.submitted_at,
|
|
110
|
-
payload=queued_job_payload_parser.parse_storage(existing_job.payload),
|
|
111
|
-
)
|
|
112
|
-
|
|
113
|
-
def increment_num_attempts(self, queued_job_uuid: str) -> int:
|
|
114
|
-
with self.session_maker() as session:
|
|
115
|
-
update_stmt = (
|
|
116
|
-
update(QueuedJob)
|
|
117
|
-
.values({QueuedJob.num_attempts.key: QueuedJob.num_attempts + 1})
|
|
118
|
-
.filter(QueuedJob.id == queued_job_uuid)
|
|
119
|
-
)
|
|
120
|
-
session.execute(update_stmt)
|
|
121
|
-
session.flush()
|
|
122
|
-
# IMPROVE: python3's sqlite does not support the RETURNING clause
|
|
123
|
-
select_stmt = select(QueuedJob.num_attempts).filter(
|
|
124
|
-
QueuedJob.id == queued_job_uuid
|
|
125
|
-
)
|
|
126
|
-
return int(session.execute(select_stmt).one().num_attempts)
|
|
127
|
-
|
|
128
|
-
def remove_job_from_queue(self, queued_job_uuid: str) -> None:
|
|
129
|
-
with self.session_maker() as session:
|
|
130
|
-
delete_stmt = delete(QueuedJob).filter(QueuedJob.id == queued_job_uuid)
|
|
131
|
-
session.execute(delete_stmt)
|
|
132
|
-
|
|
133
|
-
def update_job_status(
|
|
134
|
-
self, queued_job_uuid: str, status: queued_job_t.JobStatus
|
|
135
|
-
) -> None:
|
|
136
|
-
with self.session_maker() as session:
|
|
137
|
-
update_stmt = (
|
|
138
|
-
update(QueuedJob)
|
|
139
|
-
.values({QueuedJob.status.key: status})
|
|
140
|
-
.filter(QueuedJob.id == queued_job_uuid)
|
|
141
|
-
)
|
|
142
|
-
session.execute(update_stmt)
|
|
143
|
-
|
|
144
|
-
def list_queued_job_metadata(
|
|
145
|
-
self,
|
|
146
|
-
*,
|
|
147
|
-
offset: int = 0,
|
|
148
|
-
limit: int | None = 100,
|
|
149
|
-
status: queued_job_t.JobStatus | None = None,
|
|
150
|
-
job_ref_name: str | None = None,
|
|
151
|
-
submitted_from: datetime.datetime | None = None,
|
|
152
|
-
submitted_to: datetime.datetime | None = None,
|
|
153
|
-
) -> list[queued_job_t.QueuedJobMetadata]:
|
|
154
|
-
with self.session_maker() as session:
|
|
155
|
-
select_statement = (
|
|
156
|
-
select(
|
|
157
|
-
QueuedJob.id,
|
|
158
|
-
QueuedJob.job_ref_name,
|
|
159
|
-
QueuedJob.num_attempts,
|
|
160
|
-
QueuedJob.status,
|
|
161
|
-
QueuedJob.submitted_at,
|
|
162
|
-
)
|
|
163
|
-
.order_by(QueuedJob.submitted_at.desc())
|
|
164
|
-
.offset(offset)
|
|
165
|
-
.limit(limit)
|
|
166
|
-
)
|
|
167
|
-
|
|
168
|
-
if status is not None:
|
|
169
|
-
select_statement = select_statement.filter(QueuedJob.status == status)
|
|
170
|
-
if job_ref_name is not None:
|
|
171
|
-
select_statement = select_statement.filter(
|
|
172
|
-
QueuedJob.job_ref_name == job_ref_name
|
|
173
|
-
)
|
|
174
|
-
if submitted_from is not None:
|
|
175
|
-
select_statement = select_statement.filter(
|
|
176
|
-
QueuedJob.submitted_at >= submitted_from
|
|
177
|
-
)
|
|
178
|
-
if submitted_to is not None:
|
|
179
|
-
select_statement = select_statement.filter(
|
|
180
|
-
QueuedJob.submitted_at < submitted_to
|
|
181
|
-
)
|
|
182
|
-
|
|
183
|
-
queued_job_metadata: list[queued_job_t.QueuedJobMetadata] = [
|
|
184
|
-
queued_job_t.QueuedJobMetadata(
|
|
185
|
-
queued_job_uuid=row.id,
|
|
186
|
-
job_ref_name=row.job_ref_name,
|
|
187
|
-
num_attempts=row.num_attempts,
|
|
188
|
-
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
189
|
-
submitted_at=row.submitted_at,
|
|
190
|
-
)
|
|
191
|
-
for row in session.execute(select_statement)
|
|
192
|
-
]
|
|
193
|
-
|
|
194
|
-
return queued_job_metadata
|
|
195
|
-
|
|
196
|
-
def get_next_queued_job_for_ref_name(
|
|
197
|
-
self, job_ref_name: str
|
|
198
|
-
) -> queued_job_t.QueuedJob | None:
|
|
199
|
-
with self.session_maker() as session:
|
|
200
|
-
select_stmt = (
|
|
201
|
-
select(
|
|
202
|
-
QueuedJob.id,
|
|
203
|
-
QueuedJob.payload,
|
|
204
|
-
QueuedJob.num_attempts,
|
|
205
|
-
QueuedJob.job_ref_name,
|
|
206
|
-
QueuedJob.status,
|
|
207
|
-
QueuedJob.submitted_at,
|
|
208
|
-
)
|
|
209
|
-
.filter(QueuedJob.job_ref_name == job_ref_name)
|
|
210
|
-
.filter(
|
|
211
|
-
or_(
|
|
212
|
-
QueuedJob.status == queued_job_t.JobStatus.QUEUED,
|
|
213
|
-
QueuedJob.status.is_(None),
|
|
214
|
-
)
|
|
215
|
-
)
|
|
216
|
-
.limit(1)
|
|
217
|
-
.order_by(QueuedJob.submitted_at)
|
|
218
|
-
)
|
|
219
|
-
|
|
220
|
-
for row in session.execute(select_stmt):
|
|
221
|
-
parsed_payload = queued_job_payload_parser.parse_storage(row.payload)
|
|
222
|
-
return queued_job_t.QueuedJob(
|
|
223
|
-
queued_job_uuid=row.id,
|
|
224
|
-
job_ref_name=row.job_ref_name,
|
|
225
|
-
num_attempts=row.num_attempts,
|
|
226
|
-
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
227
|
-
submitted_at=row.submitted_at,
|
|
228
|
-
payload=parsed_payload,
|
|
229
|
-
)
|
|
230
|
-
|
|
231
|
-
return None
|
|
232
|
-
|
|
233
|
-
def load_job_queue(self) -> list[queued_job_t.QueuedJob]:
|
|
234
|
-
with self.session_maker() as session:
|
|
235
|
-
select_stmt = (
|
|
236
|
-
select(
|
|
237
|
-
QueuedJob.id,
|
|
238
|
-
QueuedJob.payload,
|
|
239
|
-
QueuedJob.num_attempts,
|
|
240
|
-
QueuedJob.job_ref_name,
|
|
241
|
-
QueuedJob.status,
|
|
242
|
-
QueuedJob.submitted_at,
|
|
243
|
-
)
|
|
244
|
-
.filter(
|
|
245
|
-
or_(
|
|
246
|
-
QueuedJob.status == queued_job_t.JobStatus.QUEUED,
|
|
247
|
-
QueuedJob.status.is_(None),
|
|
248
|
-
)
|
|
249
|
-
)
|
|
250
|
-
.order_by(QueuedJob.submitted_at)
|
|
251
|
-
)
|
|
252
|
-
|
|
253
|
-
queued_jobs: list[queued_job_t.QueuedJob] = []
|
|
254
|
-
for row in session.execute(select_stmt):
|
|
255
|
-
parsed_payload = queued_job_payload_parser.parse_storage(row.payload)
|
|
256
|
-
queued_jobs.append(
|
|
257
|
-
queued_job_t.QueuedJob(
|
|
258
|
-
queued_job_uuid=row.id,
|
|
259
|
-
job_ref_name=row.job_ref_name,
|
|
260
|
-
num_attempts=row.num_attempts,
|
|
261
|
-
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
262
|
-
submitted_at=row.submitted_at,
|
|
263
|
-
payload=parsed_payload,
|
|
264
|
-
)
|
|
265
|
-
)
|
|
266
|
-
|
|
267
|
-
return queued_jobs
|
|
268
|
-
|
|
269
|
-
def get_queued_job(self, *, uuid: str) -> queued_job_t.QueuedJob | None:
|
|
270
|
-
with self.session_maker() as session:
|
|
271
|
-
select_stmt = select(
|
|
272
|
-
QueuedJob.id,
|
|
273
|
-
QueuedJob.payload,
|
|
274
|
-
QueuedJob.num_attempts,
|
|
275
|
-
QueuedJob.job_ref_name,
|
|
276
|
-
QueuedJob.status,
|
|
277
|
-
QueuedJob.submitted_at,
|
|
278
|
-
).filter(QueuedJob.id == uuid)
|
|
279
|
-
|
|
280
|
-
row = session.execute(select_stmt).one_or_none()
|
|
281
|
-
return (
|
|
282
|
-
queued_job_t.QueuedJob(
|
|
283
|
-
queued_job_uuid=row.id,
|
|
284
|
-
job_ref_name=row.job_ref_name,
|
|
285
|
-
num_attempts=row.num_attempts,
|
|
286
|
-
status=row.status or queued_job_t.JobStatus.QUEUED,
|
|
287
|
-
submitted_at=row.submitted_at,
|
|
288
|
-
payload=queued_job_payload_parser.parse_storage(row.payload),
|
|
289
|
-
)
|
|
290
|
-
if row is not None
|
|
291
|
-
else None
|
|
292
|
-
)
|
|
293
|
-
|
|
294
|
-
def vaccuum_queued_jobs(self) -> list[str]:
|
|
295
|
-
with self.session_maker() as session:
|
|
296
|
-
select_stmt = (
|
|
297
|
-
select(QueuedJob.id)
|
|
298
|
-
.filter(QueuedJob.status == queued_job_t.JobStatus.QUEUED)
|
|
299
|
-
.filter(
|
|
300
|
-
QueuedJob.submitted_at
|
|
301
|
-
<= (
|
|
302
|
-
datetime.datetime.now(UTC)
|
|
303
|
-
- datetime.timedelta(days=MAX_QUEUE_WINDOW_DAYS)
|
|
304
|
-
)
|
|
305
|
-
)
|
|
306
|
-
)
|
|
307
|
-
deleted_uuids: list[str] = [row.id for row in session.execute(select_stmt)]
|
|
308
|
-
|
|
309
|
-
delete_stmt = delete(QueuedJob).filter(QueuedJob.id.in_(deleted_uuids))
|
|
310
|
-
session.execute(delete_stmt)
|
|
311
|
-
|
|
312
|
-
return deleted_uuids
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from sqlalchemy.engine import Engine
|
|
2
|
+
|
|
3
|
+
from uncountable.integration.db.session import get_session_maker
|
|
4
|
+
from uncountable.integration.queue_runner.datastore.datastore_postgres import (
|
|
5
|
+
DatastorePostgres,
|
|
6
|
+
)
|
|
7
|
+
from uncountable.integration.queue_runner.datastore.datastore_sqlite import (
|
|
8
|
+
DatastoreSqlite,
|
|
9
|
+
)
|
|
10
|
+
from uncountable.integration.queue_runner.datastore.interface import Datastore
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def create_datastore(engine: Engine) -> Datastore:
|
|
14
|
+
session_maker = get_session_maker(engine)
|
|
15
|
+
dialect_name = engine.dialect.name
|
|
16
|
+
match dialect_name:
|
|
17
|
+
case "sqlite":
|
|
18
|
+
datastore: Datastore = DatastoreSqlite(session_maker)
|
|
19
|
+
case "postgresql":
|
|
20
|
+
datastore = DatastorePostgres(session_maker)
|
|
21
|
+
case _:
|
|
22
|
+
raise ValueError(f"unsupported database dialect: {dialect_name!r}")
|
|
23
|
+
datastore.setup(engine)
|
|
24
|
+
return datastore
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
|
|
4
|
+
from sqlalchemy.engine import Engine
|
|
5
|
+
|
|
4
6
|
from uncountable.types import queued_job_t
|
|
5
7
|
|
|
6
8
|
|
|
7
9
|
class Datastore(ABC):
|
|
10
|
+
@classmethod
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def setup(cls, engine: Engine) -> None: ...
|
|
13
|
+
|
|
8
14
|
@abstractmethod
|
|
9
15
|
def add_job_to_queue(
|
|
10
16
|
self, job_payload: queued_job_t.QueuedJobPayload, job_ref_name: str
|
|
@@ -13,6 +19,14 @@ class Datastore(ABC):
|
|
|
13
19
|
@abstractmethod
|
|
14
20
|
def remove_job_from_queue(self, queued_job_uuid: str) -> None: ...
|
|
15
21
|
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def update_job_status(
|
|
24
|
+
self, queued_job_uuid: str, status: queued_job_t.JobStatus
|
|
25
|
+
) -> None: ...
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def retry_job(self, queued_job_uuid: str) -> queued_job_t.QueuedJob | None: ...
|
|
29
|
+
|
|
16
30
|
@abstractmethod
|
|
17
31
|
def increment_num_attempts(self, queued_job_uuid: str) -> int: ...
|
|
18
32
|
|
|
@@ -38,3 +52,6 @@ class Datastore(ABC):
|
|
|
38
52
|
|
|
39
53
|
@abstractmethod
|
|
40
54
|
def get_queued_job(self, *, uuid: str) -> queued_job_t.QueuedJob | None: ...
|
|
55
|
+
|
|
56
|
+
@abstractmethod
|
|
57
|
+
def vaccuum_queued_jobs(self) -> list[str]: ...
|
|
@@ -21,7 +21,7 @@ class QueuedJob(Base):
|
|
|
21
21
|
payload = Column(JSON, nullable=False)
|
|
22
22
|
num_attempts = Column(BigInteger, nullable=False, default=0, server_default="0")
|
|
23
23
|
status = Column(
|
|
24
|
-
Enum(queued_job_t.JobStatus, length=None),
|
|
24
|
+
Enum(queued_job_t.JobStatus, length=None, native_enum=False),
|
|
25
25
|
default=queued_job_t.JobStatus.QUEUED,
|
|
26
26
|
nullable=True,
|
|
27
27
|
index=True,
|
|
@@ -23,8 +23,7 @@ from uncountable.integration.queue_runner.command_server.types import (
|
|
|
23
23
|
CommandCancelJobStatus,
|
|
24
24
|
CommandVaccuumQueuedJobs,
|
|
25
25
|
)
|
|
26
|
-
from uncountable.integration.queue_runner.datastore import
|
|
27
|
-
from uncountable.integration.queue_runner.datastore.interface import Datastore
|
|
26
|
+
from uncountable.integration.queue_runner.datastore import Datastore
|
|
28
27
|
from uncountable.integration.queue_runner.worker import Worker
|
|
29
28
|
from uncountable.integration.scan_profiles import load_profiles
|
|
30
29
|
from uncountable.integration.telemetry import Logger
|
|
@@ -90,9 +89,7 @@ def _start_workers(
|
|
|
90
89
|
return job_worker_lookup
|
|
91
90
|
|
|
92
91
|
|
|
93
|
-
async def start_scheduler(
|
|
94
|
-
command_queue: CommandQueue, datastore: DatastoreSqlite
|
|
95
|
-
) -> None:
|
|
92
|
+
async def start_scheduler(command_queue: CommandQueue, datastore: Datastore) -> None:
|
|
96
93
|
logger = Logger(get_current_span())
|
|
97
94
|
result_queue: ResultQueue = asyncio.Queue()
|
|
98
95
|
|
|
@@ -1,20 +1,16 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
|
|
3
3
|
from uncountable.integration.db.connect import IntegrationDBService, create_db_engine
|
|
4
|
-
from uncountable.integration.db.session import get_session_maker
|
|
5
4
|
from uncountable.integration.queue_runner.command_server import serve
|
|
6
5
|
from uncountable.integration.queue_runner.command_server.types import CommandQueue
|
|
7
|
-
from uncountable.integration.queue_runner.datastore import
|
|
6
|
+
from uncountable.integration.queue_runner.datastore import create_datastore
|
|
8
7
|
from uncountable.integration.queue_runner.job_scheduler import start_scheduler
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
async def queue_runner_loop() -> None:
|
|
12
11
|
command_queue: CommandQueue = asyncio.Queue()
|
|
13
12
|
engine = create_db_engine(IntegrationDBService.RUNNER)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
datastore = DatastoreSqlite(session_maker)
|
|
17
|
-
datastore.setup(engine)
|
|
13
|
+
datastore = create_datastore(engine)
|
|
18
14
|
|
|
19
15
|
command_server = asyncio.create_task(serve(command_queue, datastore))
|
|
20
16
|
|
|
@@ -21,7 +21,7 @@ from uncountable.core.async_batch import AsyncBatchProcessor
|
|
|
21
21
|
from uncountable.integration.construct_client import construct_uncountable_client
|
|
22
22
|
from uncountable.integration.executors.executors import execute_job
|
|
23
23
|
from uncountable.integration.job import JobArguments
|
|
24
|
-
from uncountable.integration.queue_runner.datastore
|
|
24
|
+
from uncountable.integration.queue_runner.datastore import Datastore
|
|
25
25
|
from uncountable.integration.queue_runner.types import ListenQueue, ResultQueue
|
|
26
26
|
from uncountable.integration.scan_profiles import load_profiles
|
|
27
27
|
from uncountable.integration.telemetry import JobLogger, Logger, get_otel_tracer
|
uncountable/types/entity_t.py
CHANGED
|
@@ -43,6 +43,7 @@ __all__: list[str] = [
|
|
|
43
43
|
"comment": "Comment",
|
|
44
44
|
"comment_thread": "Comment Thread",
|
|
45
45
|
"condition_match": "Condition Match",
|
|
46
|
+
"condition_nickname": "Condition Nickname",
|
|
46
47
|
"condition_parameter": "Condition Parameter",
|
|
47
48
|
"condition_parameter_mat_family": "Condition Parameter Material Family",
|
|
48
49
|
"condition_parameter_matches": "Condition Parameter Matches",
|
|
@@ -118,6 +119,7 @@ __all__: list[str] = [
|
|
|
118
119
|
"inventory_history": "Inventory History",
|
|
119
120
|
"inventory_reconciliation_action": "Inventory Reconciliation Action",
|
|
120
121
|
"inventory_reconciliation_session": "Inventory Reconciliation Session",
|
|
122
|
+
"it_admin_permission": "IT Admin Permission",
|
|
121
123
|
"lab_location": "Lab Location",
|
|
122
124
|
"lab_request": "Lab Request",
|
|
123
125
|
"license": "License",
|
|
@@ -130,6 +132,7 @@ __all__: list[str] = [
|
|
|
130
132
|
"measurement_timeline_type": "Measurement Timeline Type",
|
|
131
133
|
"milestone": "Milestone",
|
|
132
134
|
"ml_job": "Batch Processing Job",
|
|
135
|
+
"monitor": "Monitor",
|
|
133
136
|
"monitor_dataset": "Monitor Dataset",
|
|
134
137
|
"naming_scheme": "Naming Scheme",
|
|
135
138
|
"naming_counter": "Naming Counter",
|
|
@@ -230,6 +233,7 @@ __all__: list[str] = [
|
|
|
230
233
|
"timesheet_entry": "Timesheet Entry",
|
|
231
234
|
"training_module": "Training Module",
|
|
232
235
|
"training_module_answer": "Training Module Answer",
|
|
236
|
+
"training_module_attempt": "Training Module Attempt",
|
|
233
237
|
"training_module_block": "Training Module Block",
|
|
234
238
|
"training_module_page": "Training Module Page",
|
|
235
239
|
"training_module_question": "Training Module Question",
|
|
@@ -297,6 +301,7 @@ class EntityType(StrEnum):
|
|
|
297
301
|
COMMENT = "comment"
|
|
298
302
|
COMMENT_THREAD = "comment_thread"
|
|
299
303
|
CONDITION_MATCH = "condition_match"
|
|
304
|
+
CONDITION_NICKNAME = "condition_nickname"
|
|
300
305
|
CONDITION_PARAMETER = "condition_parameter"
|
|
301
306
|
CONDITION_PARAMETER_MAT_FAMILY = "condition_parameter_mat_family"
|
|
302
307
|
CONDITION_PARAMETER_MATCHES = "condition_parameter_matches"
|
|
@@ -372,6 +377,7 @@ class EntityType(StrEnum):
|
|
|
372
377
|
INVENTORY_HISTORY = "inventory_history"
|
|
373
378
|
INVENTORY_RECONCILIATION_ACTION = "inventory_reconciliation_action"
|
|
374
379
|
INVENTORY_RECONCILIATION_SESSION = "inventory_reconciliation_session"
|
|
380
|
+
IT_ADMIN_PERMISSION = "it_admin_permission"
|
|
375
381
|
LAB_LOCATION = "lab_location"
|
|
376
382
|
LAB_REQUEST = "lab_request"
|
|
377
383
|
LICENSE = "license"
|
|
@@ -384,6 +390,7 @@ class EntityType(StrEnum):
|
|
|
384
390
|
MEASUREMENT_TIMELINE_TYPE = "measurement_timeline_type"
|
|
385
391
|
MILESTONE = "milestone"
|
|
386
392
|
ML_JOB = "ml_job"
|
|
393
|
+
MONITOR = "monitor"
|
|
387
394
|
MONITOR_DATASET = "monitor_dataset"
|
|
388
395
|
NAMING_SCHEME = "naming_scheme"
|
|
389
396
|
NAMING_COUNTER = "naming_counter"
|
|
@@ -484,6 +491,7 @@ class EntityType(StrEnum):
|
|
|
484
491
|
TIMESHEET_ENTRY = "timesheet_entry"
|
|
485
492
|
TRAINING_MODULE = "training_module"
|
|
486
493
|
TRAINING_MODULE_ANSWER = "training_module_answer"
|
|
494
|
+
TRAINING_MODULE_ATTEMPT = "training_module_attempt"
|
|
487
495
|
TRAINING_MODULE_BLOCK = "training_module_block"
|
|
488
496
|
TRAINING_MODULE_PAGE = "training_module_page"
|
|
489
497
|
TRAINING_MODULE_QUESTION = "training_module_question"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: UncountablePythonSDK
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.179
|
|
4
4
|
Summary: Uncountable SDK
|
|
5
5
|
Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
|
|
6
6
|
Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
|
|
@@ -20,6 +20,7 @@ Requires-Dist: aiotus==1.*
|
|
|
20
20
|
Requires-Dist: aiohttp>=3.14.1
|
|
21
21
|
Requires-Dist: requests==2.*
|
|
22
22
|
Requires-Dist: SQLAlchemy>=1.4.0
|
|
23
|
+
Requires-Dist: psycopg[binary]==3.*
|
|
23
24
|
Requires-Dist: APScheduler==3.*
|
|
24
25
|
Requires-Dist: python-dateutil==2.*
|
|
25
26
|
Requires-Dist: PyYAML==6.*
|
|
@@ -32,7 +32,7 @@ examples/set_input_pdf_attribute.py,sha256=A-QqaWeeLyBXQ1zvq91cBZmvGJWEUuVF5TDj0
|
|
|
32
32
|
examples/set_recipe_metadata_file.py,sha256=cRVXGz4UN4aqnNrNSzyBmikYHpe63lMIuzOpMwD9EDU,1036
|
|
33
33
|
examples/set_recipe_output_file_sdk.py,sha256=Lz1amqppnWTX83z-C090wCJ4hcKmCD3kb-4v0uBRi0Y,782
|
|
34
34
|
examples/upload_files.py,sha256=qMaSvMSdTMPOOP55y1AwEurc0SOdZAMvEydlqJPsGpg,432
|
|
35
|
-
examples/integration-server/pyproject.toml,sha256=
|
|
35
|
+
examples/integration-server/pyproject.toml,sha256=qntU6IFi2nKoXcS4hr6J7x75EaD4WQW6KpjgHcKZzPo,9069
|
|
36
36
|
examples/integration-server/jobs/materials_auto/concurrent_cron.py,sha256=xsK3H9ZEaniedC2nJUB0rqOcFI8y-ojfl_nLSJb9AMM,312
|
|
37
37
|
examples/integration-server/jobs/materials_auto/example_cron.py,sha256=y1nAtGwbPJfIrfQsrHVmJLAHmQtCEHIy1g-0PjeXx04,735
|
|
38
38
|
examples/integration-server/jobs/materials_auto/example_http.py,sha256=htTR61XhG_fF2Zv2gXRGYE5Yc3mSDdigrOR89kcupZY,1380
|
|
@@ -57,7 +57,7 @@ pkgs/argument_parser/parser_options.py,sha256=NAwNZSIH8EjyfJXJSxbn4kYmF3K7D3HuOe
|
|
|
57
57
|
pkgs/argument_parser/type_predicates.py,sha256=CkQexiseN05XDz4e34K3gIOlER5LG5Ke8u0UHK4gC04,1157
|
|
58
58
|
pkgs/filesystem_utils/__init__.py,sha256=G-Nl0sxJsSgBB6yajzzY9aeSSBnkzJHGq3lj5LFn8fw,2056
|
|
59
59
|
pkgs/filesystem_utils/_blob_session.py,sha256=YB_idar5qVuF9QN_A6qjpTuyeZ7JDOLri1LGLfPHLow,5158
|
|
60
|
-
pkgs/filesystem_utils/_file_share_session.py,sha256=
|
|
60
|
+
pkgs/filesystem_utils/_file_share_session.py,sha256=Uos2ugL4mP8SUDNK5ILtAzEYwzOeaiaBYAEkdoyvn8M,5130
|
|
61
61
|
pkgs/filesystem_utils/_gdrive_session.py,sha256=4P2MSsA0GNxIYJxvmf7zKR8dM8oVMCCSzDIEkGA7XqE,11089
|
|
62
62
|
pkgs/filesystem_utils/_local_session.py,sha256=wbmcGqqus417lY_b6VzgEaZA0SUU8Ar5MrgMkXMp58M,2339
|
|
63
63
|
pkgs/filesystem_utils/_s3_session.py,sha256=8SClYopSg4Wgn6EuUZWe8SqO3oaZGzY-Ojj356AsLYg,4503
|
|
@@ -118,7 +118,7 @@ uncountable/__init__.py,sha256=8l8XWNCKsu7TG94c-xa2KHpDegvxDC2FyQISdWC763Y,89
|
|
|
118
118
|
uncountable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
119
119
|
uncountable/core/__init__.py,sha256=RFv0kO6rKFf1PtBPu83hCGmxqkJamRtsgQ9_-ztw7tA,341
|
|
120
120
|
uncountable/core/async_batch.py,sha256=9pYGFzVCQXt8059qFHgutweGIFPquJ5Xfq6NT5P-1K0,1206
|
|
121
|
-
uncountable/core/client.py,sha256=
|
|
121
|
+
uncountable/core/client.py,sha256=myEWbiYFMprqy7JMKMt1aBZAu7gQfGb9mmG9fLHBNgM,16743
|
|
122
122
|
uncountable/core/environment.py,sha256=Z9vu7JtnSDgQB_KKcZnjTFNyARXjRr_PDW9krwxNNAo,1132
|
|
123
123
|
uncountable/core/file_upload.py,sha256=IisvIBdm2f4CJTRQ9Awi5JuI16PSERjXrzZgmrUiCHY,5960
|
|
124
124
|
uncountable/core/types.py,sha256=s2CjqYJpsmbC7xMwxxT7kJ_V9bwokrjjWVVjpMcQpKI,333
|
|
@@ -140,7 +140,7 @@ uncountable/integration/server.py,sha256=P4RRGwU9jMselHPWbU6GxhRLgVtN7Ydcxr18sFn
|
|
|
140
140
|
uncountable/integration/telemetry.py,sha256=zvg-K8uy49vQxWgInXAx-Sz8jHC0IWT1kdzPQdp9ROA,16162
|
|
141
141
|
uncountable/integration/webhook_signature_key.py,sha256=vXlNtj0qLqZxjKDLmRvlChvLS5xoPAGmYcYcs4WIVaM,3882
|
|
142
142
|
uncountable/integration/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
143
|
-
uncountable/integration/db/connect.py,sha256=
|
|
143
|
+
uncountable/integration/db/connect.py,sha256=Rl1853wku0w3hEOsAA8TM4iydaLhfEE-BGEb6G6NJSw,1396
|
|
144
144
|
uncountable/integration/db/session.py,sha256=96cGQXpe6IugBTdSsjdP0S5yhJ6toSmbVB6qhc3FJzE,693
|
|
145
145
|
uncountable/integration/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
146
146
|
uncountable/integration/executors/executors.py,sha256=E3a0_yrQesfigDzjhUAwD7-l-oc9qnvPh-A8QTK6F3k,5149
|
|
@@ -149,13 +149,13 @@ uncountable/integration/executors/script_executor.py,sha256=BBQ9f0l7uH2hgKf60jtm
|
|
|
149
149
|
uncountable/integration/http_server/__init__.py,sha256=ld7hD3wOGwAbnVRBrkTUCP5zNG-kptMd4wGJyQvUWVc,170
|
|
150
150
|
uncountable/integration/http_server/types.py,sha256=3JJSulRfv784SbXnXo1Pywto7RwGxgS-iJ2-a6TOnDI,1869
|
|
151
151
|
uncountable/integration/queue_runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
152
|
-
uncountable/integration/queue_runner/job_scheduler.py,sha256=
|
|
153
|
-
uncountable/integration/queue_runner/queue_runner.py,sha256=
|
|
152
|
+
uncountable/integration/queue_runner/job_scheduler.py,sha256=HS6cpr863bho-EPBl4cQzK-AoBv7_Ti38Lx_0WGPx8k,9800
|
|
153
|
+
uncountable/integration/queue_runner/queue_runner.py,sha256=kB5W4jCbIEPMWfHxEaQ8suKMekT4vc4fmBOMFusGQMQ,994
|
|
154
154
|
uncountable/integration/queue_runner/types.py,sha256=8HS6KnYMS_vc5XHeMpg0BFAQC-5P3QLzd-aDYDMMt3E,244
|
|
155
|
-
uncountable/integration/queue_runner/worker.py,sha256=
|
|
155
|
+
uncountable/integration/queue_runner/worker.py,sha256=6qXP2Y9_yxYfvdC3pVL-a-W6fSSiKmzetB1oekK2ZUQ,7346
|
|
156
156
|
uncountable/integration/queue_runner/command_server/__init__.py,sha256=hMCDLWct8zW4B2a9BaIAsMhtaEgFlxONjeow-6nf6dg,675
|
|
157
157
|
uncountable/integration/queue_runner/command_server/command_client.py,sha256=N1UBs1Z9O6iDOEqXNzjkVFMqDUhm-0-M41I1y6s_5y0,6316
|
|
158
|
-
uncountable/integration/queue_runner/command_server/command_server.py,sha256=
|
|
158
|
+
uncountable/integration/queue_runner/command_server/command_server.py,sha256=Q9avIU5EPBPSeQJzZ6_zEg9PJez4BE5NNUkHsA2lHM8,8659
|
|
159
159
|
uncountable/integration/queue_runner/command_server/constants.py,sha256=7J9mQIAMOfV50wnwpn7HgrPFEi3Ritj6HwrGYwxGLoU,88
|
|
160
160
|
uncountable/integration/queue_runner/command_server/provision_webhook.py,sha256=yPYEqCi9lDFYOeGBJgwcLS3IdfYGcaZFEYo6_8sFiS8,1102
|
|
161
161
|
uncountable/integration/queue_runner/command_server/types.py,sha256=ZNqJE6b6rfMZGdcPS-7umB_8x2a7dzTfRBsdGoCzDjY,2077
|
|
@@ -164,10 +164,13 @@ uncountable/integration/queue_runner/command_server/protocol/command_server.prot
|
|
|
164
164
|
uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.py,sha256=voigJ1ScgpWx0FL59pKinN8MLj0seuqoZuOqvCtSPss,6013
|
|
165
165
|
uncountable/integration/queue_runner/command_server/protocol/command_server_pb2.pyi,sha256=CkaTBkhsGngUxD-am691wKFAARCrivqiJHxP1pLe7jw,5984
|
|
166
166
|
uncountable/integration/queue_runner/command_server/protocol/command_server_pb2_grpc.py,sha256=wzPYtvuccPVW1qEXaMxU-KB3ptnZYnjHsuit6L1Pk6U,17415
|
|
167
|
-
uncountable/integration/queue_runner/datastore/__init__.py,sha256=
|
|
168
|
-
uncountable/integration/queue_runner/datastore/
|
|
169
|
-
uncountable/integration/queue_runner/datastore/
|
|
170
|
-
uncountable/integration/queue_runner/datastore/
|
|
167
|
+
uncountable/integration/queue_runner/datastore/__init__.py,sha256=zwMMQbZ6FwAJU3yL8iVgKf9uTjpAjcVjqR1Akmhkrzw,282
|
|
168
|
+
uncountable/integration/queue_runner/datastore/datastore_postgres.py,sha256=oKRwOWlxgodJyqKr-jDcP9VbXKKTrU26lI-7m4HFBuk,359
|
|
169
|
+
uncountable/integration/queue_runner/datastore/datastore_sql_base.py,sha256=eDQFDzn8y1IHoA888Hy69dUNl9MeCFQWW_VmnFqKdfo,11107
|
|
170
|
+
uncountable/integration/queue_runner/datastore/datastore_sqlite.py,sha256=W9hWKKvZrp3JyaO_tmK-qZL3ACnMnCfYwvDSJHHd6qs,1197
|
|
171
|
+
uncountable/integration/queue_runner/datastore/factory.py,sha256=bv_7BxQpgjrggJKz5YDllgP_BihEJ0-CdpLvaG4ooj4,868
|
|
172
|
+
uncountable/integration/queue_runner/datastore/interface.py,sha256=kuAU8lrjVN3NTllo99nfJwBojt5dg_wgdivyYCB4EbY,1581
|
|
173
|
+
uncountable/integration/queue_runner/datastore/model.py,sha256=06_fGS0QTALEA_381I1zz63yXYRooaAVJdfYi0ubSd0,851
|
|
171
174
|
uncountable/integration/secret_retrieval/__init__.py,sha256=sisf8cxCYcJmCXCKjEPoT2xIxm1j-Ru9iTfMVxH6NM0,171
|
|
172
175
|
uncountable/integration/secret_retrieval/basic_secret.py,sha256=qjDhhBxhhQdMIsC3WJjFshJkqSC0v9hg3QJQoCi46oE,3286
|
|
173
176
|
uncountable/integration/secret_retrieval/retrieve_secret.py,sha256=c0VkCvppWKXwgxdblQuj2WhRaEGBPa7vLDoS6mlNOAQ,3919
|
|
@@ -196,7 +199,7 @@ uncountable/types/curves_t.py,sha256=WIML06C67yC4KWdPJR2HV_Ac7eBFTDRDUL3YgeXbFOs
|
|
|
196
199
|
uncountable/types/data.py,sha256=u2isf4XEug3Eu-xSIoqGaCQmW2dFaKBHCkP_WKYwwBc,500
|
|
197
200
|
uncountable/types/data_t.py,sha256=U_dIh5GcyaJlkrHEI4Qf4TOo1uJX07PYO70sYzzKtTo,2615
|
|
198
201
|
uncountable/types/entity.py,sha256=Zclk1LYcRaYrMDhqyCjMSLEg0fE6_q8LHvV22Qvscgs,566
|
|
199
|
-
uncountable/types/entity_t.py,sha256=
|
|
202
|
+
uncountable/types/entity_t.py,sha256=OxaVn2vuQRNHxVg_iU6d26c8xSYNNyt1hQ-N6XrEf1k,28224
|
|
200
203
|
uncountable/types/experiment_groups.py,sha256=qUpFOx1AKgzaT_4khCOv5Xs6jwiQGbvHH-GUh3v1nv4,288
|
|
201
204
|
uncountable/types/experiment_groups_t.py,sha256=QjZBdjnbsD1D-VpPSSV6hBg-Q7mujjKvYtdMewtXNVU,712
|
|
202
205
|
uncountable/types/exports.py,sha256=VMmxUO2PpV1Y63hZ2AnVor4H-B6aswJ7YpSru_u89lU,334
|
|
@@ -414,7 +417,7 @@ uncountable/types/api/uploader/complete_async_parse.py,sha256=Os1HAubxTlbut6Gylj
|
|
|
414
417
|
uncountable/types/api/uploader/invoke_uploader.py,sha256=QU1KmAFgGHSWskZEKfaAww8rCj-hVNP0i4kMQE3x3Fc,1822
|
|
415
418
|
uncountable/types/api/user/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
|
|
416
419
|
uncountable/types/api/user/get_current_user_info.py,sha256=BT7bGp5SOHz3XdKWhhdZ6acnInafT2PgdJ-H4YH3W7c,1181
|
|
417
|
-
uncountablepythonsdk-0.0.
|
|
418
|
-
uncountablepythonsdk-0.0.
|
|
419
|
-
uncountablepythonsdk-0.0.
|
|
420
|
-
uncountablepythonsdk-0.0.
|
|
420
|
+
uncountablepythonsdk-0.0.179.dist-info/METADATA,sha256=uTTLUVf_leXkYxohdAYWsCN81x1Xr7GsQfzJ_JG-tpY,2211
|
|
421
|
+
uncountablepythonsdk-0.0.179.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
422
|
+
uncountablepythonsdk-0.0.179.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
|
|
423
|
+
uncountablepythonsdk-0.0.179.dist-info/RECORD,,
|
{uncountablepythonsdk-0.0.178.dist-info → uncountablepythonsdk-0.0.179.dist-info}/top_level.txt
RENAMED
|
File without changes
|