UncountablePythonSDK 0.0.192__py3-none-any.whl → 0.0.193__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pkgs/codeowners.py +157 -0
- pkgs/type_spec/builder.py +32 -1
- pkgs/type_spec/emit_python.py +10 -0
- pkgs/type_spec/emit_typescript_util.py +8 -0
- uncountable/core/client.py +3 -3
- uncountable/core/types.py +4 -4
- uncountable/integration/queue_runner/command_handlers.py +144 -0
- uncountable/integration/queue_runner/job_dispatcher.py +44 -0
- uncountable/integration/queue_runner/job_scheduler.py +51 -172
- uncountable/integration/queue_runner/queue_runner.py +5 -1
- uncountable/types/api/files/download_file.py +18 -2
- {uncountablepythonsdk-0.0.192.dist-info → uncountablepythonsdk-0.0.193.dist-info}/METADATA +1 -1
- {uncountablepythonsdk-0.0.192.dist-info → uncountablepythonsdk-0.0.193.dist-info}/RECORD +15 -12
- {uncountablepythonsdk-0.0.192.dist-info → uncountablepythonsdk-0.0.193.dist-info}/WHEEL +1 -1
- {uncountablepythonsdk-0.0.192.dist-info → uncountablepythonsdk-0.0.193.dist-info}/top_level.txt +0 -0
pkgs/codeowners.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from collections.abc import Iterable, Mapping
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from functools import cached_property
|
|
5
|
+
from itertools import chain
|
|
6
|
+
|
|
7
|
+
GLOB_CHARS = "*?["
|
|
8
|
+
TEAM_OWNER_PREFIX = "@uncountableinc/"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def generate_team_username(team_slug: str) -> str:
|
|
12
|
+
return f"{TEAM_OWNER_PREFIX}{team_slug}"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _glob_regex(pattern: str) -> re.Pattern[str]:
|
|
16
|
+
directory_pattern = pattern.endswith("/**")
|
|
17
|
+
normalized_pattern = pattern[:-3] if directory_pattern else pattern
|
|
18
|
+
regex_parts: list[str] = []
|
|
19
|
+
index = 0
|
|
20
|
+
while index < len(normalized_pattern):
|
|
21
|
+
dotfile_guard = (
|
|
22
|
+
"(?!\\.)" if index == 0 or normalized_pattern[index - 1] == "/" else ""
|
|
23
|
+
)
|
|
24
|
+
if normalized_pattern.startswith("**/", index):
|
|
25
|
+
regex_parts.append("(?:(?!\\.)[^/]+/)*")
|
|
26
|
+
index += 3
|
|
27
|
+
elif normalized_pattern.startswith("**", index):
|
|
28
|
+
regex_parts.append(".*")
|
|
29
|
+
index += 2
|
|
30
|
+
elif normalized_pattern[index] == "*":
|
|
31
|
+
regex_parts.append(f"{dotfile_guard}[^/]*")
|
|
32
|
+
index += 1
|
|
33
|
+
elif normalized_pattern[index] == "?":
|
|
34
|
+
regex_parts.append(f"{dotfile_guard}[^/]")
|
|
35
|
+
index += 1
|
|
36
|
+
elif normalized_pattern[index] == "[":
|
|
37
|
+
closing_index = normalized_pattern.find("]", index + 1)
|
|
38
|
+
if closing_index == -1:
|
|
39
|
+
regex_parts.append(r"\[")
|
|
40
|
+
index += 1
|
|
41
|
+
else:
|
|
42
|
+
character_class = normalized_pattern[index + 1 : closing_index]
|
|
43
|
+
if character_class.startswith("!"):
|
|
44
|
+
character_class = f"^{character_class[1:]}"
|
|
45
|
+
regex_parts.append(f"{dotfile_guard}[{character_class}]")
|
|
46
|
+
index = closing_index + 1
|
|
47
|
+
else:
|
|
48
|
+
regex_parts.append(re.escape(normalized_pattern[index]))
|
|
49
|
+
index += 1
|
|
50
|
+
suffix = "(?:/(?!\\.)[^/]+)*" if directory_pattern else ""
|
|
51
|
+
return re.compile(f"^{''.join(regex_parts)}{suffix}$")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(kw_only=True)
|
|
55
|
+
class CodeownerRule:
|
|
56
|
+
owners: list[str]
|
|
57
|
+
glob: str
|
|
58
|
+
|
|
59
|
+
@cached_property
|
|
60
|
+
def normalized_globs(self) -> list[str]:
|
|
61
|
+
normalized_glob = self.glob.strip("/")
|
|
62
|
+
|
|
63
|
+
# /path/to/file should the file /path/to/file
|
|
64
|
+
globs = [normalized_glob]
|
|
65
|
+
|
|
66
|
+
# /path/to/file should also match files in subdirectory
|
|
67
|
+
# /path/to/file/foo
|
|
68
|
+
# heuristic: treat as directory if no wildcard and last path part has no dot
|
|
69
|
+
last_part = normalized_glob.split("/")[-1]
|
|
70
|
+
if (
|
|
71
|
+
"*" not in normalized_glob
|
|
72
|
+
and "?" not in normalized_glob
|
|
73
|
+
and "." not in last_part
|
|
74
|
+
):
|
|
75
|
+
globs.append(f"{normalized_glob}/**")
|
|
76
|
+
|
|
77
|
+
return globs
|
|
78
|
+
|
|
79
|
+
@cached_property
|
|
80
|
+
def matches_everything(self) -> bool:
|
|
81
|
+
return "*" in self.normalized_globs
|
|
82
|
+
|
|
83
|
+
@cached_property
|
|
84
|
+
def _compiled_matchers(self) -> list[re.Pattern[str]]:
|
|
85
|
+
return [_glob_regex(pattern) for pattern in self.normalized_globs]
|
|
86
|
+
|
|
87
|
+
def check_path(self, path: str) -> bool:
|
|
88
|
+
if self.matches_everything:
|
|
89
|
+
return True
|
|
90
|
+
return any(pattern.match(path) for pattern in self._compiled_matchers)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def clean_line(line: str) -> str:
|
|
94
|
+
return line.strip().split("#", 1)[0]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _rules_from_lines(lines: Iterable[str]) -> list[CodeownerRule]:
|
|
98
|
+
rules: list[CodeownerRule] = []
|
|
99
|
+
for raw_line in lines:
|
|
100
|
+
line = clean_line(raw_line)
|
|
101
|
+
parts = line.split()
|
|
102
|
+
if len(parts) > 1:
|
|
103
|
+
rules.append(CodeownerRule(glob=parts[0], owners=parts[1:]))
|
|
104
|
+
return rules
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def get_codeowner_rules(codeowners_path: str) -> list[CodeownerRule]:
|
|
108
|
+
with open(codeowners_path, encoding="utf-8") as codeowners_file:
|
|
109
|
+
return _rules_from_lines(codeowners_file)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def get_codeowner_rules_from_text(text: str) -> list[CodeownerRule]:
|
|
113
|
+
return _rules_from_lines(text.splitlines())
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def last_matching_rule(rules: list[CodeownerRule], path: str) -> CodeownerRule | None:
|
|
117
|
+
"""The rule that owns `path` under CODEOWNERS last-match-wins precedence."""
|
|
118
|
+
matching_rules = [rule for rule in rules if rule.check_path(path)]
|
|
119
|
+
return matching_rules[-1] if matching_rules else None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def get_owners_for_file(rules: list[CodeownerRule], path: str) -> list[str]:
|
|
123
|
+
rule = last_matching_rule(rules, path)
|
|
124
|
+
return rule.owners if rule is not None else []
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def evaluate_codeowner_rules(rules: list[CodeownerRule], files: list[str]) -> set[str]:
|
|
128
|
+
return set(chain.from_iterable(get_owners_for_file(rules, file) for file in files))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def get_owners(codeowners_path: str, files: list[str]) -> set[str]:
|
|
132
|
+
return evaluate_codeowner_rules(get_codeowner_rules(codeowners_path), files)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def get_codeowner_team_slugs(*, rules: list[CodeownerRule]) -> set[str]:
|
|
136
|
+
return {
|
|
137
|
+
owner.removeprefix(TEAM_OWNER_PREFIX)
|
|
138
|
+
for rule in rules
|
|
139
|
+
for owner in rule.owners
|
|
140
|
+
if owner.startswith(TEAM_OWNER_PREFIX)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def get_codeowner_team_slugs_for_files(
|
|
145
|
+
*,
|
|
146
|
+
files: list[str],
|
|
147
|
+
codeowner_rules: list[CodeownerRule],
|
|
148
|
+
all_teams_lookup: Mapping[str, object],
|
|
149
|
+
) -> list[str]:
|
|
150
|
+
if len(files) == 0:
|
|
151
|
+
return []
|
|
152
|
+
codeowners = evaluate_codeowner_rules(codeowner_rules, files)
|
|
153
|
+
return [
|
|
154
|
+
slug
|
|
155
|
+
for slug in all_teams_lookup
|
|
156
|
+
if generate_team_username(team_slug=slug) in codeowners
|
|
157
|
+
]
|
pkgs/type_spec/builder.py
CHANGED
|
@@ -103,6 +103,11 @@ class PropertyNameMode(StrEnum):
|
|
|
103
103
|
deprecated_non_snake_case = "deprecated_non_snake_case"
|
|
104
104
|
|
|
105
105
|
|
|
106
|
+
class StringEnumEmitMode(StrEnum):
|
|
107
|
+
STANDARD = "standard"
|
|
108
|
+
SOURCE_ENUM_ALIAS = "source_enum_alias"
|
|
109
|
+
|
|
110
|
+
|
|
106
111
|
class BaseTypeName(StrEnum):
|
|
107
112
|
"""
|
|
108
113
|
Base types that are supported.
|
|
@@ -912,6 +917,8 @@ class SpecTypeDefnStringEnum(SpecTypeDefn):
|
|
|
912
917
|
self.desc: str | None = None
|
|
913
918
|
self.sql_type_name: str | None = None
|
|
914
919
|
self.emit_id_source = False
|
|
920
|
+
self.emit_mode = StringEnumEmitMode.STANDARD
|
|
921
|
+
self.alias_source_enum: SpecTypeDefnStringEnum | None = None
|
|
915
922
|
self.source_enums: list[SpecType] = []
|
|
916
923
|
self.property_name_mode = PropertyNameMode.strict
|
|
917
924
|
|
|
@@ -1036,12 +1043,15 @@ class SpecTypeDefnStringEnum(SpecTypeDefn):
|
|
|
1036
1043
|
|
|
1037
1044
|
emit_data = data.get("emit")
|
|
1038
1045
|
if emit_data is not None:
|
|
1039
|
-
util.check_fields(emit_data, ["id_source"])
|
|
1046
|
+
util.check_fields(emit_data, ["id_source", "mode"])
|
|
1040
1047
|
emit_id_source = emit_data.get("id_source", False)
|
|
1041
1048
|
assert isinstance(emit_id_source, bool)
|
|
1042
1049
|
self.emit_id_source = emit_id_source
|
|
1043
1050
|
if emit_id_source:
|
|
1044
1051
|
builder.emit_id_source_enums.add(self)
|
|
1052
|
+
self.emit_mode = StringEnumEmitMode(
|
|
1053
|
+
emit_data.get("mode", StringEnumEmitMode.STANDARD.value)
|
|
1054
|
+
)
|
|
1045
1055
|
|
|
1046
1056
|
if self.emit_id_source:
|
|
1047
1057
|
assert len(self.namespace.path) == 1
|
|
@@ -1062,6 +1072,27 @@ class SpecTypeDefnStringEnum(SpecTypeDefn):
|
|
|
1062
1072
|
for name in self.values:
|
|
1063
1073
|
_validate_enum_name(name)
|
|
1064
1074
|
|
|
1075
|
+
if self.emit_mode is StringEnumEmitMode.SOURCE_ENUM_ALIAS:
|
|
1076
|
+
builder.ensure(
|
|
1077
|
+
len(self.source_enums) == 1,
|
|
1078
|
+
"source enum alias must reference one StringEnum",
|
|
1079
|
+
)
|
|
1080
|
+
source_enum = self.source_enums[0]
|
|
1081
|
+
assert isinstance(source_enum, SpecTypeDefnStringEnum)
|
|
1082
|
+
self.alias_source_enum = source_enum
|
|
1083
|
+
builder.ensure(
|
|
1084
|
+
self.values == source_enum.values,
|
|
1085
|
+
"source enum alias must preserve all source values",
|
|
1086
|
+
)
|
|
1087
|
+
builder.ensure(
|
|
1088
|
+
data_values is None,
|
|
1089
|
+
"source enum alias cannot define local values",
|
|
1090
|
+
)
|
|
1091
|
+
builder.ensure(
|
|
1092
|
+
self.sql_type_name is None,
|
|
1093
|
+
"source enum alias cannot define a SQL type",
|
|
1094
|
+
)
|
|
1095
|
+
|
|
1065
1096
|
def get_referenced_types(self) -> list[SpecType]:
|
|
1066
1097
|
return self.source_enums
|
|
1067
1098
|
|
pkgs/type_spec/emit_python.py
CHANGED
|
@@ -762,6 +762,16 @@ def _emit_endpoint_invocation_function(
|
|
|
762
762
|
|
|
763
763
|
|
|
764
764
|
def _emit_string_enum(ctx: Context, stype: builder.SpecTypeDefnStringEnum) -> None:
|
|
765
|
+
if stype.alias_source_enum is not None:
|
|
766
|
+
if stype.emit_mode is not builder.StringEnumEmitMode.SOURCE_ENUM_ALIAS:
|
|
767
|
+
raise ValueError("source enum alias has an invalid emission mode")
|
|
768
|
+
ctx.out.write(
|
|
769
|
+
f"{python_class_name(stype.name)} = {refer_to(ctx, stype.alias_source_enum)}\n"
|
|
770
|
+
)
|
|
771
|
+
return
|
|
772
|
+
if stype.emit_mode is not builder.StringEnumEmitMode.STANDARD:
|
|
773
|
+
raise ValueError("string enum has an unsupported emission mode")
|
|
774
|
+
|
|
765
775
|
ctx.use_enum = True
|
|
766
776
|
|
|
767
777
|
# Detect if we need serial_string_enum. Ideally we can just stick this on all enums
|
|
@@ -270,6 +270,14 @@ def emit_type_ts(ctx: EmitTypescriptContext, stype: builder.SpecType) -> None:
|
|
|
270
270
|
return
|
|
271
271
|
|
|
272
272
|
if isinstance(stype, builder.SpecTypeDefnStringEnum):
|
|
273
|
+
if stype.alias_source_enum is not None:
|
|
274
|
+
if stype.emit_mode is not builder.StringEnumEmitMode.SOURCE_ENUM_ALIAS:
|
|
275
|
+
raise ValueError("source enum alias has an invalid emission mode")
|
|
276
|
+
source_enum_ref = refer_to(ctx, stype.alias_source_enum)
|
|
277
|
+
ctx.out.write(f"export import {stype.name} = {source_enum_ref}\n")
|
|
278
|
+
return
|
|
279
|
+
if stype.emit_mode is not builder.StringEnumEmitMode.STANDARD:
|
|
280
|
+
raise ValueError("string enum has an unsupported emission mode")
|
|
273
281
|
ctx.out.write(f"export enum {stype.name} {{\n")
|
|
274
282
|
assert stype.values
|
|
275
283
|
for entry in stype.values.values():
|
uncountable/core/client.py
CHANGED
|
@@ -4,7 +4,7 @@ import pathlib
|
|
|
4
4
|
import re
|
|
5
5
|
import typing
|
|
6
6
|
import zipfile
|
|
7
|
-
from dataclasses import dataclass
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
8
|
from datetime import UTC, timedelta
|
|
9
9
|
from enum import StrEnum
|
|
10
10
|
from io import BytesIO
|
|
@@ -181,13 +181,13 @@ class SDKError(Exception):
|
|
|
181
181
|
|
|
182
182
|
@dataclass(kw_only=True)
|
|
183
183
|
class OAuthBearerTokenCache:
|
|
184
|
-
token: str
|
|
184
|
+
token: str = field(repr=False)
|
|
185
185
|
expires_at: datetime.datetime
|
|
186
186
|
|
|
187
187
|
|
|
188
188
|
@dataclass(kw_only=True)
|
|
189
189
|
class GetOauthBearerTokenData:
|
|
190
|
-
access_token: str
|
|
190
|
+
access_token: str = field(repr=False)
|
|
191
191
|
expires_in: int
|
|
192
192
|
token_type: str
|
|
193
193
|
scope: str
|
uncountable/core/types.py
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
@dataclass(kw_only=True)
|
|
5
5
|
class AuthDetailsApiKey:
|
|
6
6
|
api_id: str
|
|
7
|
-
api_secret_key: str
|
|
7
|
+
api_secret_key: str = field(repr=False)
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
@dataclass(kw_only=True)
|
|
11
11
|
class AuthDetailsOAuth:
|
|
12
|
-
refresh_token: str
|
|
12
|
+
refresh_token: str = field(repr=False)
|
|
13
13
|
scope: str = "unc.rnd"
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
@dataclass(kw_only=True)
|
|
17
17
|
class AuthDetailsBearerToken:
|
|
18
|
-
token: str
|
|
18
|
+
token: str = field(repr=False)
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
AuthDetails = AuthDetailsApiKey # Legacy Mapping
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import threading
|
|
3
|
+
import typing
|
|
4
|
+
|
|
5
|
+
from uncountable.integration.queue_runner.command_server import (
|
|
6
|
+
CommandEnqueueJob,
|
|
7
|
+
CommandEnqueueJobResponse,
|
|
8
|
+
CommandRetryJob,
|
|
9
|
+
CommandRetryJobResponse,
|
|
10
|
+
)
|
|
11
|
+
from uncountable.integration.queue_runner.command_server.types import (
|
|
12
|
+
CommandCancelJob,
|
|
13
|
+
CommandCancelJobResponse,
|
|
14
|
+
CommandCancelJobStatus,
|
|
15
|
+
)
|
|
16
|
+
from uncountable.integration.queue_runner.datastore import Datastore
|
|
17
|
+
from uncountable.integration.queue_runner.job_dispatcher import JobDispatcher
|
|
18
|
+
from uncountable.integration.queue_runner.types import RESTART_EXIT_CODE
|
|
19
|
+
from uncountable.integration.telemetry import Logger
|
|
20
|
+
from uncountable.types import queued_job_t
|
|
21
|
+
|
|
22
|
+
RESTART_DELAY_SECS = 5
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _should_deduplicate_job(
|
|
26
|
+
invocation_context: queued_job_t.InvocationContext,
|
|
27
|
+
) -> bool:
|
|
28
|
+
match invocation_context:
|
|
29
|
+
case (
|
|
30
|
+
queued_job_t.InvocationContextCron()
|
|
31
|
+
| queued_job_t.InvocationContextManual()
|
|
32
|
+
):
|
|
33
|
+
return True
|
|
34
|
+
case queued_job_t.InvocationContextWebhook():
|
|
35
|
+
return False
|
|
36
|
+
case _:
|
|
37
|
+
typing.assert_never(invocation_context)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def _enqueue_or_deduplicate_job(
|
|
41
|
+
*,
|
|
42
|
+
job_ref_name: str,
|
|
43
|
+
payload: queued_job_t.QueuedJobPayload,
|
|
44
|
+
job_dispatcher: JobDispatcher,
|
|
45
|
+
) -> str:
|
|
46
|
+
if _should_deduplicate_job(payload.invocation_context):
|
|
47
|
+
existing_queued_job = job_dispatcher.datastore.get_next_queued_job_for_ref_name(
|
|
48
|
+
job_ref_name=job_ref_name
|
|
49
|
+
)
|
|
50
|
+
if existing_queued_job is not None:
|
|
51
|
+
return existing_queued_job.queued_job_uuid
|
|
52
|
+
queued_job = job_dispatcher.datastore.add_job_to_queue(
|
|
53
|
+
job_payload=payload,
|
|
54
|
+
job_ref_name=job_ref_name,
|
|
55
|
+
)
|
|
56
|
+
job_dispatcher.logger.log_info(
|
|
57
|
+
"Job submitted successfully.",
|
|
58
|
+
attributes={
|
|
59
|
+
"job.id": queued_job.job_ref_name,
|
|
60
|
+
"job.queued_job_uuid": queued_job.queued_job_uuid,
|
|
61
|
+
},
|
|
62
|
+
)
|
|
63
|
+
await job_dispatcher.enqueue_queued_job(queued_job=queued_job)
|
|
64
|
+
return queued_job.queued_job_uuid
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
async def handle_enqueue_job_command(
|
|
68
|
+
*, command: CommandEnqueueJob, job_dispatcher: JobDispatcher
|
|
69
|
+
) -> None:
|
|
70
|
+
queued_job_uuid = await _enqueue_or_deduplicate_job(
|
|
71
|
+
job_ref_name=command.job_ref_name,
|
|
72
|
+
payload=command.payload,
|
|
73
|
+
job_dispatcher=job_dispatcher,
|
|
74
|
+
)
|
|
75
|
+
await command.response_queue.put(
|
|
76
|
+
CommandEnqueueJobResponse(queued_job_uuid=queued_job_uuid)
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
async def handle_cancel_job_command(
|
|
81
|
+
*,
|
|
82
|
+
command: CommandCancelJob,
|
|
83
|
+
job_dispatcher: JobDispatcher,
|
|
84
|
+
running_job_uuids: set[str],
|
|
85
|
+
) -> None:
|
|
86
|
+
queued_job = job_dispatcher.datastore.get_queued_job(uuid=command.queued_job_uuid)
|
|
87
|
+
if queued_job is None:
|
|
88
|
+
await command.response_queue.put(
|
|
89
|
+
CommandCancelJobResponse(status=CommandCancelJobStatus.NO_JOB_FOUND)
|
|
90
|
+
)
|
|
91
|
+
return
|
|
92
|
+
|
|
93
|
+
if queued_job.status != queued_job_t.JobStatus.QUEUED:
|
|
94
|
+
await command.response_queue.put(
|
|
95
|
+
CommandCancelJobResponse(
|
|
96
|
+
status=CommandCancelJobStatus.JOB_ALREADY_COMPLETED
|
|
97
|
+
)
|
|
98
|
+
)
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
job_dispatcher.datastore.remove_job_from_queue(command.queued_job_uuid)
|
|
102
|
+
if command.queued_job_uuid in running_job_uuids:
|
|
103
|
+
cancel_status = CommandCancelJobStatus.CANCELLED_WITH_RESTART
|
|
104
|
+
threading.Timer(
|
|
105
|
+
interval=RESTART_DELAY_SECS,
|
|
106
|
+
function=os._exit,
|
|
107
|
+
args=[RESTART_EXIT_CODE],
|
|
108
|
+
).start()
|
|
109
|
+
else:
|
|
110
|
+
job_dispatcher.dispatched_job_uuids.discard(command.queued_job_uuid)
|
|
111
|
+
cancel_status = CommandCancelJobStatus.CANCELLED_QUEUED
|
|
112
|
+
|
|
113
|
+
await command.response_queue.put(CommandCancelJobResponse(status=cancel_status))
|
|
114
|
+
job_dispatcher.logger.log_debug(
|
|
115
|
+
"Job cancelled successfully.",
|
|
116
|
+
attributes={
|
|
117
|
+
"job.queued_job_uuid": command.queued_job_uuid,
|
|
118
|
+
"job.cancel_status": cancel_status,
|
|
119
|
+
},
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
async def handle_retry_job_command(
|
|
124
|
+
*, command: CommandRetryJob, job_dispatcher: JobDispatcher
|
|
125
|
+
) -> None:
|
|
126
|
+
queued_job = job_dispatcher.datastore.retry_job(command.queued_job_uuid)
|
|
127
|
+
if queued_job is None:
|
|
128
|
+
await command.response_queue.put(CommandRetryJobResponse(queued_job_uuid=None))
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
await job_dispatcher.enqueue_queued_job(queued_job=queued_job)
|
|
132
|
+
await command.response_queue.put(
|
|
133
|
+
CommandRetryJobResponse(queued_job_uuid=queued_job.queued_job_uuid)
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def handle_vacuum_queued_jobs_command(*, datastore: Datastore, logger: Logger) -> None:
|
|
138
|
+
logger.log_debug("Vaccuuming queued jobs...")
|
|
139
|
+
deleted_uuids = datastore.vaccuum_queued_jobs()
|
|
140
|
+
|
|
141
|
+
logger.log_debug(
|
|
142
|
+
"Queued jobs vacuumed successfully.",
|
|
143
|
+
attributes={"job.queued_job_uuids": deleted_uuids},
|
|
144
|
+
)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
|
|
3
|
+
from uncountable.integration.queue_runner.datastore import Datastore
|
|
4
|
+
from uncountable.integration.queue_runner.worker import Worker
|
|
5
|
+
from uncountable.integration.telemetry import Logger
|
|
6
|
+
from uncountable.types import queued_job_t
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(kw_only=True, frozen=True)
|
|
10
|
+
class JobDispatcher:
|
|
11
|
+
datastore: Datastore
|
|
12
|
+
job_worker_lookup: dict[str, Worker]
|
|
13
|
+
logger: Logger
|
|
14
|
+
dispatched_job_uuids: set[str] = field(default_factory=set, init=False)
|
|
15
|
+
|
|
16
|
+
async def enqueue_queued_job(self, *, queued_job: queued_job_t.QueuedJob) -> None:
|
|
17
|
+
worker = self.job_worker_lookup.get(queued_job.job_ref_name)
|
|
18
|
+
if worker is None:
|
|
19
|
+
self.logger.log_error(
|
|
20
|
+
"No worker found for job.",
|
|
21
|
+
attributes={
|
|
22
|
+
"job.id": queued_job.job_ref_name,
|
|
23
|
+
"job.queued_job_uuid": queued_job.queued_job_uuid,
|
|
24
|
+
},
|
|
25
|
+
)
|
|
26
|
+
self.datastore.update_job_status(
|
|
27
|
+
queued_job.queued_job_uuid, queued_job_t.JobStatus.FAILED
|
|
28
|
+
)
|
|
29
|
+
return
|
|
30
|
+
self.dispatched_job_uuids.add(queued_job.queued_job_uuid)
|
|
31
|
+
await worker.listen_queue.put(queued_job)
|
|
32
|
+
|
|
33
|
+
async def dispatch_undispatched_queued_jobs(self) -> None:
|
|
34
|
+
undispatched_job_uuids = [
|
|
35
|
+
queued_job_uuid
|
|
36
|
+
for queued_job_uuid in self.datastore.list_queued_job_uuids()
|
|
37
|
+
if queued_job_uuid not in self.dispatched_job_uuids
|
|
38
|
+
]
|
|
39
|
+
if len(undispatched_job_uuids) == 0:
|
|
40
|
+
return
|
|
41
|
+
for queued_job in self.datastore.load_queued_jobs(
|
|
42
|
+
queued_job_uuids=undispatched_job_uuids
|
|
43
|
+
):
|
|
44
|
+
await self.enqueue_queued_job(queued_job=queued_job)
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
import os
|
|
3
2
|
import sys
|
|
4
|
-
import threading
|
|
5
3
|
import typing
|
|
6
4
|
from concurrent.futures import ProcessPoolExecutor
|
|
7
5
|
from dataclasses import dataclass
|
|
@@ -9,30 +7,32 @@ from dataclasses import dataclass
|
|
|
9
7
|
from opentelemetry.trace import get_current_span
|
|
10
8
|
|
|
11
9
|
from uncountable.core.environment import get_max_job_workers
|
|
10
|
+
from uncountable.integration.queue_runner.command_handlers import (
|
|
11
|
+
handle_cancel_job_command,
|
|
12
|
+
handle_enqueue_job_command,
|
|
13
|
+
handle_retry_job_command,
|
|
14
|
+
handle_vacuum_queued_jobs_command,
|
|
15
|
+
)
|
|
12
16
|
from uncountable.integration.queue_runner.command_server import (
|
|
13
17
|
CommandEnqueueJob,
|
|
14
|
-
CommandEnqueueJobResponse,
|
|
15
18
|
CommandQueue,
|
|
16
19
|
CommandRetryJob,
|
|
17
|
-
CommandRetryJobResponse,
|
|
18
20
|
CommandTask,
|
|
19
21
|
)
|
|
20
22
|
from uncountable.integration.queue_runner.command_server.types import (
|
|
21
23
|
CommandCancelJob,
|
|
22
|
-
CommandCancelJobResponse,
|
|
23
|
-
CommandCancelJobStatus,
|
|
24
24
|
CommandVaccuumQueuedJobs,
|
|
25
25
|
)
|
|
26
26
|
from uncountable.integration.queue_runner.datastore import Datastore
|
|
27
|
+
from uncountable.integration.queue_runner.job_dispatcher import JobDispatcher
|
|
27
28
|
from uncountable.integration.queue_runner.worker import Worker
|
|
28
29
|
from uncountable.integration.scan_profiles import load_profiles
|
|
29
30
|
from uncountable.integration.telemetry import Logger
|
|
30
31
|
from uncountable.types import job_definition_t, queued_job_t
|
|
31
32
|
|
|
32
|
-
from .types import
|
|
33
|
+
from .types import ResultQueue, ResultTask
|
|
33
34
|
|
|
34
35
|
QUEUED_JOB_POLL_INTERVAL_SECS = 1
|
|
35
|
-
RESTART_DELAY_SECS = 5
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
@dataclass(kw_only=True, frozen=True)
|
|
@@ -42,7 +42,7 @@ class JobListenerKey:
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
def _get_job_worker_key(
|
|
45
|
-
job_definition: job_definition_t.JobDefinition, profile_name: str
|
|
45
|
+
*, job_definition: job_definition_t.JobDefinition, profile_name: str
|
|
46
46
|
) -> JobListenerKey:
|
|
47
47
|
if job_definition.subqueue_name is not None:
|
|
48
48
|
return JobListenerKey(
|
|
@@ -52,6 +52,7 @@ def _get_job_worker_key(
|
|
|
52
52
|
|
|
53
53
|
|
|
54
54
|
def on_worker_crash(
|
|
55
|
+
*,
|
|
55
56
|
worker_key: JobListenerKey,
|
|
56
57
|
) -> typing.Callable[[asyncio.Task], None]:
|
|
57
58
|
def hook(task: asyncio.Task) -> None:
|
|
@@ -78,11 +79,11 @@ def _start_workers(
|
|
|
78
79
|
profiles = load_profiles()
|
|
79
80
|
job_queue_worker_lookup: dict[JobListenerKey, Worker] = {}
|
|
80
81
|
job_worker_lookup: dict[str, Worker] = {}
|
|
81
|
-
job_definition_lookup: dict[str, job_definition_t.JobDefinition] = {}
|
|
82
82
|
for profile in profiles:
|
|
83
83
|
for job_definition in profile.jobs:
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
job_worker_key = _get_job_worker_key(
|
|
85
|
+
job_definition=job_definition, profile_name=profile.name
|
|
86
|
+
)
|
|
86
87
|
if job_worker_key not in job_queue_worker_lookup:
|
|
87
88
|
worker = Worker(
|
|
88
89
|
process_pool=process_pool,
|
|
@@ -92,7 +93,7 @@ def _start_workers(
|
|
|
92
93
|
running_job_uuids=running_job_uuids,
|
|
93
94
|
)
|
|
94
95
|
task = asyncio.create_task(worker.run_worker_loop())
|
|
95
|
-
task.add_done_callback(on_worker_crash(job_worker_key))
|
|
96
|
+
task.add_done_callback(on_worker_crash(worker_key=job_worker_key))
|
|
96
97
|
worker_tasks.append(task)
|
|
97
98
|
job_queue_worker_lookup[job_worker_key] = worker
|
|
98
99
|
job_worker_lookup[job_definition.id] = job_queue_worker_lookup[
|
|
@@ -102,9 +103,10 @@ def _start_workers(
|
|
|
102
103
|
|
|
103
104
|
|
|
104
105
|
async def start_scheduler(
|
|
106
|
+
*,
|
|
105
107
|
command_queue: CommandQueue,
|
|
106
108
|
datastore: Datastore,
|
|
107
|
-
shutdown_event: asyncio.Event
|
|
109
|
+
shutdown_event: asyncio.Event,
|
|
108
110
|
) -> None:
|
|
109
111
|
logger = Logger(get_current_span())
|
|
110
112
|
result_queue: ResultQueue = asyncio.Queue()
|
|
@@ -123,155 +125,23 @@ async def start_scheduler(
|
|
|
123
125
|
running_job_uuids=running_job_uuids,
|
|
124
126
|
)
|
|
125
127
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
except KeyError as e:
|
|
132
|
-
logger.log_exception(e)
|
|
133
|
-
datastore.update_job_status(
|
|
134
|
-
queued_job.queued_job_uuid, queued_job_t.JobStatus.FAILED
|
|
135
|
-
)
|
|
136
|
-
return
|
|
137
|
-
dispatched_job_uuids.add(queued_job.queued_job_uuid)
|
|
138
|
-
await worker.listen_queue.put(queued_job)
|
|
139
|
-
|
|
140
|
-
async def _dispatch_undispatched_queued_jobs() -> None:
|
|
141
|
-
undispatched_job_uuids = [
|
|
142
|
-
queued_job_uuid
|
|
143
|
-
for queued_job_uuid in datastore.list_queued_job_uuids()
|
|
144
|
-
if queued_job_uuid not in dispatched_job_uuids
|
|
145
|
-
]
|
|
146
|
-
if len(undispatched_job_uuids) == 0:
|
|
147
|
-
return
|
|
148
|
-
for queued_job in datastore.load_queued_jobs(
|
|
149
|
-
queued_job_uuids=undispatched_job_uuids
|
|
150
|
-
):
|
|
151
|
-
await enqueue_queued_job(queued_job)
|
|
152
|
-
|
|
153
|
-
async def _enqueue_or_deduplicate_job(
|
|
154
|
-
job_ref_name: str,
|
|
155
|
-
payload: queued_job_t.QueuedJobPayload,
|
|
156
|
-
) -> str:
|
|
157
|
-
if isinstance(
|
|
158
|
-
payload.invocation_context,
|
|
159
|
-
(
|
|
160
|
-
queued_job_t.InvocationContextCron,
|
|
161
|
-
queued_job_t.InvocationContextManual,
|
|
162
|
-
),
|
|
163
|
-
):
|
|
164
|
-
existing_queued_job = datastore.get_next_queued_job_for_ref_name(
|
|
165
|
-
job_ref_name=job_ref_name
|
|
166
|
-
)
|
|
167
|
-
if existing_queued_job is not None:
|
|
168
|
-
return existing_queued_job.queued_job_uuid
|
|
169
|
-
queued_job = datastore.add_job_to_queue(
|
|
170
|
-
job_payload=payload,
|
|
171
|
-
job_ref_name=job_ref_name,
|
|
172
|
-
)
|
|
173
|
-
logger.log_info(
|
|
174
|
-
"Job submitted successfully.",
|
|
175
|
-
attributes={
|
|
176
|
-
"job.id": queued_job.job_ref_name,
|
|
177
|
-
"job.queued_job_uuid": queued_job.queued_job_uuid,
|
|
178
|
-
},
|
|
179
|
-
)
|
|
180
|
-
await enqueue_queued_job(queued_job)
|
|
181
|
-
return queued_job.queued_job_uuid
|
|
182
|
-
|
|
183
|
-
async def _handle_enqueue_job_command(command: CommandEnqueueJob) -> None:
|
|
184
|
-
queued_job_uuid = await _enqueue_or_deduplicate_job(
|
|
185
|
-
job_ref_name=command.job_ref_name,
|
|
186
|
-
payload=command.payload,
|
|
187
|
-
)
|
|
188
|
-
await command.response_queue.put(
|
|
189
|
-
CommandEnqueueJobResponse(queued_job_uuid=queued_job_uuid)
|
|
190
|
-
)
|
|
191
|
-
|
|
192
|
-
async def _handle_cancel_job_command(command: CommandCancelJob) -> None:
|
|
193
|
-
queued_job = datastore.get_queued_job(uuid=command.queued_job_uuid)
|
|
194
|
-
if queued_job is None:
|
|
195
|
-
await command.response_queue.put(
|
|
196
|
-
CommandCancelJobResponse(status=CommandCancelJobStatus.NO_JOB_FOUND)
|
|
197
|
-
)
|
|
198
|
-
return
|
|
199
|
-
|
|
200
|
-
if queued_job.status != queued_job_t.JobStatus.QUEUED:
|
|
201
|
-
await command.response_queue.put(
|
|
202
|
-
CommandCancelJobResponse(
|
|
203
|
-
status=CommandCancelJobStatus.JOB_ALREADY_COMPLETED
|
|
204
|
-
)
|
|
205
|
-
)
|
|
206
|
-
return
|
|
207
|
-
|
|
208
|
-
datastore.remove_job_from_queue(command.queued_job_uuid)
|
|
209
|
-
if command.queued_job_uuid in running_job_uuids:
|
|
210
|
-
cancel_status = CommandCancelJobStatus.CANCELLED_WITH_RESTART
|
|
211
|
-
threading.Timer(
|
|
212
|
-
interval=RESTART_DELAY_SECS,
|
|
213
|
-
function=os._exit,
|
|
214
|
-
args=[RESTART_EXIT_CODE],
|
|
215
|
-
).start()
|
|
216
|
-
else:
|
|
217
|
-
dispatched_job_uuids.discard(command.queued_job_uuid)
|
|
218
|
-
cancel_status = CommandCancelJobStatus.CANCELLED_QUEUED
|
|
219
|
-
|
|
220
|
-
await command.response_queue.put(
|
|
221
|
-
CommandCancelJobResponse(status=cancel_status)
|
|
222
|
-
)
|
|
223
|
-
logger.log_debug(
|
|
224
|
-
"Job cancelled successfully.",
|
|
225
|
-
attributes={
|
|
226
|
-
"job.queued_job_uuid": command.queued_job_uuid,
|
|
227
|
-
"job.cancel_status": cancel_status,
|
|
228
|
-
},
|
|
229
|
-
)
|
|
230
|
-
|
|
231
|
-
async def _handle_retry_job_command(command: CommandRetryJob) -> None:
|
|
232
|
-
queued_job = datastore.retry_job(command.queued_job_uuid)
|
|
233
|
-
if queued_job is None:
|
|
234
|
-
await command.response_queue.put(
|
|
235
|
-
CommandRetryJobResponse(queued_job_uuid=None)
|
|
236
|
-
)
|
|
237
|
-
return
|
|
238
|
-
|
|
239
|
-
await enqueue_queued_job(queued_job)
|
|
240
|
-
await command.response_queue.put(
|
|
241
|
-
CommandRetryJobResponse(queued_job_uuid=queued_job.queued_job_uuid)
|
|
242
|
-
)
|
|
243
|
-
|
|
244
|
-
def _handle_vaccuum_queued_jobs_command(
|
|
245
|
-
command: CommandVaccuumQueuedJobs,
|
|
246
|
-
) -> None:
|
|
247
|
-
logger.log_debug("Vaccuuming queued jobs...")
|
|
248
|
-
deleted_uuids = datastore.vaccuum_queued_jobs()
|
|
249
|
-
|
|
250
|
-
logger.log_debug(
|
|
251
|
-
"Queued jobs vacuumed successfully.",
|
|
252
|
-
attributes={"job.queued_job_uuids": deleted_uuids},
|
|
253
|
-
)
|
|
128
|
+
job_dispatcher = JobDispatcher(
|
|
129
|
+
datastore=datastore,
|
|
130
|
+
job_worker_lookup=job_worker_lookup,
|
|
131
|
+
logger=logger,
|
|
132
|
+
)
|
|
254
133
|
|
|
255
|
-
await
|
|
134
|
+
await job_dispatcher.dispatch_undispatched_queued_jobs()
|
|
256
135
|
|
|
257
136
|
result_task: ResultTask = asyncio.create_task(result_queue.get())
|
|
258
137
|
command_task: CommandTask = asyncio.create_task(command_queue.get())
|
|
259
138
|
poll_task = asyncio.create_task(asyncio.sleep(QUEUED_JOB_POLL_INTERVAL_SECS))
|
|
260
|
-
shutdown_task
|
|
261
|
-
if shutdown_event is not None:
|
|
262
|
-
shutdown_task = asyncio.create_task(shutdown_event.wait())
|
|
139
|
+
shutdown_task = asyncio.create_task(shutdown_event.wait())
|
|
263
140
|
|
|
264
141
|
while True:
|
|
265
|
-
waitables: list[asyncio.Task[object]] = [
|
|
266
|
-
result_task,
|
|
267
|
-
command_task,
|
|
268
|
-
poll_task,
|
|
269
|
-
]
|
|
270
|
-
if shutdown_task is not None:
|
|
271
|
-
waitables.append(shutdown_task)
|
|
272
|
-
|
|
273
142
|
finished, _ = await asyncio.wait(
|
|
274
|
-
|
|
143
|
+
[result_task, command_task, poll_task, shutdown_task],
|
|
144
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
275
145
|
)
|
|
276
146
|
|
|
277
147
|
for task in finished:
|
|
@@ -288,34 +158,43 @@ async def start_scheduler(
|
|
|
288
158
|
command = command_task.result()
|
|
289
159
|
match command:
|
|
290
160
|
case CommandEnqueueJob():
|
|
291
|
-
await
|
|
161
|
+
await handle_enqueue_job_command(
|
|
162
|
+
command=command, job_dispatcher=job_dispatcher
|
|
163
|
+
)
|
|
292
164
|
case CommandRetryJob():
|
|
293
|
-
await
|
|
165
|
+
await handle_retry_job_command(
|
|
166
|
+
command=command, job_dispatcher=job_dispatcher
|
|
167
|
+
)
|
|
294
168
|
case CommandVaccuumQueuedJobs():
|
|
295
|
-
|
|
169
|
+
handle_vacuum_queued_jobs_command(
|
|
170
|
+
datastore=datastore, logger=logger
|
|
171
|
+
)
|
|
296
172
|
case CommandCancelJob():
|
|
297
|
-
await
|
|
173
|
+
await handle_cancel_job_command(
|
|
174
|
+
command=command,
|
|
175
|
+
job_dispatcher=job_dispatcher,
|
|
176
|
+
running_job_uuids=running_job_uuids,
|
|
177
|
+
)
|
|
298
178
|
case _:
|
|
299
179
|
typing.assert_never(command)
|
|
300
180
|
command_task = asyncio.create_task(command_queue.get())
|
|
301
181
|
elif task == result_task:
|
|
302
182
|
queued_job_result = result_task.result()
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
dispatched_job_uuids.discard(queued_job_result.queued_job_uuid)
|
|
183
|
+
job_status = (
|
|
184
|
+
queued_job_t.JobStatus.SUCCESS
|
|
185
|
+
if queued_job_result.job_result.success
|
|
186
|
+
else queued_job_t.JobStatus.FAILED
|
|
187
|
+
)
|
|
188
|
+
datastore.update_job_status(
|
|
189
|
+
queued_job_result.queued_job_uuid, job_status
|
|
190
|
+
)
|
|
191
|
+
job_dispatcher.dispatched_job_uuids.discard(
|
|
192
|
+
queued_job_result.queued_job_uuid
|
|
193
|
+
)
|
|
315
194
|
running_job_uuids.discard(queued_job_result.queued_job_uuid)
|
|
316
195
|
result_task = asyncio.create_task(result_queue.get())
|
|
317
196
|
elif task == poll_task:
|
|
318
|
-
await
|
|
197
|
+
await job_dispatcher.dispatch_undispatched_queued_jobs()
|
|
319
198
|
poll_task = asyncio.create_task(
|
|
320
199
|
asyncio.sleep(QUEUED_JOB_POLL_INTERVAL_SECS)
|
|
321
200
|
)
|
|
@@ -20,7 +20,11 @@ async def queue_runner_loop() -> None:
|
|
|
20
20
|
loop.add_signal_handler(signal.SIGTERM, shutdown_event.set)
|
|
21
21
|
|
|
22
22
|
scheduler = asyncio.create_task(
|
|
23
|
-
start_scheduler(
|
|
23
|
+
start_scheduler(
|
|
24
|
+
command_queue=command_queue,
|
|
25
|
+
datastore=datastore,
|
|
26
|
+
shutdown_event=shutdown_event,
|
|
27
|
+
)
|
|
24
28
|
)
|
|
25
29
|
|
|
26
30
|
await scheduler
|
|
@@ -13,6 +13,7 @@ from pkgs.serialization import serial_union_annotation
|
|
|
13
13
|
from ... import base_t
|
|
14
14
|
from ... import entity_t
|
|
15
15
|
from ... import identifier_t
|
|
16
|
+
from ... import notebook_files_t
|
|
16
17
|
|
|
17
18
|
__all__: list[str] = [
|
|
18
19
|
"Arguments",
|
|
@@ -23,6 +24,7 @@ __all__: list[str] = [
|
|
|
23
24
|
"FileDownloadQueryBase",
|
|
24
25
|
"FileDownloadQueryEntityField",
|
|
25
26
|
"FileDownloadQueryFileFolderMembership",
|
|
27
|
+
"FileDownloadQueryNotebookFile",
|
|
26
28
|
"FileDownloadQueryTextDocumentId",
|
|
27
29
|
"FileDownloadQueryType",
|
|
28
30
|
"FileDownloadType",
|
|
@@ -31,7 +33,7 @@ __all__: list[str] = [
|
|
|
31
33
|
|
|
32
34
|
ENDPOINT_METHOD = "GET"
|
|
33
35
|
ENDPOINT_PATH = "api/external/files/download_file"
|
|
34
|
-
ENDPOINT_DESCRIPTION = "Download an attachment field value from an entity (e.g. a file attached to a form field) by entity + field reference, download a text document by its ID, or download a file stored in a file folder by its membership, optionally at a specific historical version rather than the current one. When querying by entity field, if the field contains a single file it is returned directly (X-File-Download-Type: raw); if multiple files are attached, they are bundled into a ZIP archive (X-File-Download-Type: archive). All other query modes return the file directly (X-File-Download-Type: raw). For downloading the binary file stored on a recipe output of type image or file, use recipes/external_download_recipe_output_file instead. A
|
|
36
|
+
ENDPOINT_DESCRIPTION = "Download an attachment field value from an entity (e.g. a file attached to a form field) by entity + field reference, download a text document by its ID, download a file from an image or file cell in a notebook, or download a file stored in a file folder by its membership, optionally at a specific historical version rather than the current one. When querying by entity field, if the field contains a single file it is returned directly (X-File-Download-Type: raw); if multiple files are attached, they are bundled into a ZIP archive (X-File-Download-Type: archive). All other query modes return the file directly (X-File-Download-Type: raw). For downloading the binary file stored on a recipe output of type image or file, use recipes/external_download_recipe_output_file instead. A text document that has not cleared virus scanning is not returned: a text document still being scanned responds with HTTP 422 indicating the scan is in progress, and a text document that failed the scan responds with HTTP 422 indicating a security-scan failure. Image-cell downloads return the stored image without a virus-scan check."
|
|
35
37
|
|
|
36
38
|
|
|
37
39
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -44,6 +46,7 @@ class FileDownloadType(StrEnum):
|
|
|
44
46
|
class FileDownloadQueryType(StrEnum):
|
|
45
47
|
ENTITY_FIELD = "entity_field"
|
|
46
48
|
TEXT_DOCUMENT_ID = "text_document_id"
|
|
49
|
+
NOTEBOOK_FILE = "notebook_file"
|
|
47
50
|
FILE_FOLDER_MEMBERSHIP = "file_folder_membership"
|
|
48
51
|
|
|
49
52
|
|
|
@@ -79,6 +82,18 @@ class FileDownloadQueryTextDocumentId(FileDownloadQueryBase):
|
|
|
79
82
|
text_document_id: base_t.ObjectId
|
|
80
83
|
|
|
81
84
|
|
|
85
|
+
# DO NOT MODIFY -- This file is generated by type_spec
|
|
86
|
+
@serial_class(
|
|
87
|
+
named_type_path="sdk.api.files.download_file.FileDownloadQueryNotebookFile",
|
|
88
|
+
parse_require={"type"},
|
|
89
|
+
)
|
|
90
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
91
|
+
class FileDownloadQueryNotebookFile(FileDownloadQueryBase):
|
|
92
|
+
type: typing.Literal[FileDownloadQueryType.NOTEBOOK_FILE] = FileDownloadQueryType.NOTEBOOK_FILE
|
|
93
|
+
notebook_key: identifier_t.IdentifierKey
|
|
94
|
+
file_key: notebook_files_t.NotebookFileKey
|
|
95
|
+
|
|
96
|
+
|
|
82
97
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
83
98
|
@serial_class(
|
|
84
99
|
named_type_path="sdk.api.files.download_file.FileDownloadQueryFileFolderMembership",
|
|
@@ -93,13 +108,14 @@ class FileDownloadQueryFileFolderMembership(FileDownloadQueryBase):
|
|
|
93
108
|
|
|
94
109
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
95
110
|
FileDownloadQuery = typing.Annotated[
|
|
96
|
-
FileDownloadQueryEntityField | FileDownloadQueryTextDocumentId | FileDownloadQueryFileFolderMembership,
|
|
111
|
+
FileDownloadQueryEntityField | FileDownloadQueryTextDocumentId | FileDownloadQueryNotebookFile | FileDownloadQueryFileFolderMembership,
|
|
97
112
|
serial_union_annotation(
|
|
98
113
|
named_type_path="sdk.api.files.download_file.FileDownloadQuery",
|
|
99
114
|
discriminator="type",
|
|
100
115
|
discriminator_map={
|
|
101
116
|
"entity_field": FileDownloadQueryEntityField,
|
|
102
117
|
"text_document_id": FileDownloadQueryTextDocumentId,
|
|
118
|
+
"notebook_file": FileDownloadQueryNotebookFile,
|
|
103
119
|
"file_folder_membership": FileDownloadQueryFileFolderMembership,
|
|
104
120
|
},
|
|
105
121
|
),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: UncountablePythonSDK
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.193
|
|
4
4
|
Summary: Uncountable SDK
|
|
5
5
|
Project-URL: Homepage, https://github.com/uncountableinc/uncountable-python-sdk
|
|
6
6
|
Project-URL: Repository, https://github.com/uncountableinc/uncountable-python-sdk.git
|
|
@@ -46,6 +46,7 @@ examples/integration-server/jobs/materials_auto/example_scale_connection.py,sha2
|
|
|
46
46
|
examples/integration-server/jobs/materials_auto/example_wh.py,sha256=PN-skP27yJwDZboWk5g5EZEc3AKfVayQLfnopjsDKJc,659
|
|
47
47
|
examples/integration-server/jobs/materials_auto/profile.yaml,sha256=Oga4GypqV4cBXwnOg7vGqDiu0Ha9KvRi555gBowzOI0,2888
|
|
48
48
|
pkgs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
49
|
+
pkgs/codeowners.py,sha256=wY1JXGUe-q5chVLgLbfm1bZR7hQ9_RS2BKWbJGeiUkE,5315
|
|
49
50
|
pkgs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
51
|
pkgs/argument_parser/__init__.py,sha256=7ZKnkttPloXGu6r-Uevdf1-ZNeeuw87dieg_tK-a6-c,1589
|
|
51
52
|
pkgs/argument_parser/cached_parser.py,sha256=8BImceYkv1ur0ZO4vtt-5CklJVkLPosQzrC9EV-Ezdw,4197
|
|
@@ -95,7 +96,7 @@ pkgs/type_spec/AGENTS.md,sha256=Y8blYMJlDbnBhRvBnbalgb2KNAmipCqubjNsESB3g7c,987
|
|
|
95
96
|
pkgs/type_spec/CLAUDE.md,sha256=U_Sc1Ovbo1QT0ca7a8bK5Sxo7u6aUwJqCujeTfQHUJc,206
|
|
96
97
|
pkgs/type_spec/__init__.py,sha256=h5DmJTca4QVV10sZR1x0-MlkZfuGYDfapR3zHvXfzto,19
|
|
97
98
|
pkgs/type_spec/__main__.py,sha256=snBwPNvzU46Rqp4HlYb5Urv10Rs36gUbsPoFZfQcWRY,1162
|
|
98
|
-
pkgs/type_spec/builder.py,sha256=
|
|
99
|
+
pkgs/type_spec/builder.py,sha256=6MeOoJnsAd99Q8VnfSQVXLsjvKeR7P4rd9cPqA3ESrQ,71614
|
|
99
100
|
pkgs/type_spec/builder_types.py,sha256=hBe6XnSbvADRVr1C0TZYyGNSFkY48KGjF0nnZ1yF904,362
|
|
100
101
|
pkgs/type_spec/config.py,sha256=8SQkCRAKzrcycR-9Tjt6J4-kNalN8IvbE7zo1FMlrZc,7226
|
|
101
102
|
pkgs/type_spec/cross_output_links.py,sha256=exmHrdr-Mrq6FyMXzWgmuKhE-uqo_rN5k_PcirQgrGU,3037
|
|
@@ -103,10 +104,10 @@ pkgs/type_spec/emit_io_ts.py,sha256=4F_8aQK29Cr8654AGDgfm8Lsx9FdnOd4XCb4BjzQn-o,
|
|
|
103
104
|
pkgs/type_spec/emit_open_api.py,sha256=jFxPuwo6H-qIw82hUqba7xtvCbSgI-pacwmUJ7jGh34,28309
|
|
104
105
|
pkgs/type_spec/emit_open_api_examples.py,sha256=FL419_4yLjzhUUGDnvvL9fXF3-dAFCBnXNXzrCY6aFQ,8582
|
|
105
106
|
pkgs/type_spec/emit_open_api_util.py,sha256=bTmRvrGP82-eB75hwf9ySI7pDEC87FNQTF18VKEWSXY,2367
|
|
106
|
-
pkgs/type_spec/emit_python.py,sha256=
|
|
107
|
+
pkgs/type_spec/emit_python.py,sha256=1cr93gdzTC7j17l_WZMF2PBYmUNvWiED4j0o7cXwMuM,63297
|
|
107
108
|
pkgs/type_spec/emit_scss.py,sha256=6psmBAJyu7cvfQAIvIIMk2U6VFG_xeWy1Ea4afv0XyM,1497
|
|
108
109
|
pkgs/type_spec/emit_typescript.py,sha256=XRMMovJz_g2j0rpn1Nrz4uzHRCs6O6HLIxdEB1QMNIE,11939
|
|
109
|
-
pkgs/type_spec/emit_typescript_util.py,sha256=
|
|
110
|
+
pkgs/type_spec/emit_typescript_util.py,sha256=wP79WfzyTcQWxGNIMnCDXpJkv1_0hhs0kbs5qLNg3zM,18094
|
|
110
111
|
pkgs/type_spec/load_types.py,sha256=k5hWxFvhj8aT_9vAwtkjn9OdudQFK9IIUzZI7-Gpx2Y,5277
|
|
111
112
|
pkgs/type_spec/non_discriminated_union_exceptions.py,sha256=jgQ4Naz8A6FbhgfR3c9SI4IMvkkft0MT6CyQXWu9RB0,559
|
|
112
113
|
pkgs/type_spec/open_api_util.py,sha256=LX6ny4xNOWXQO3qWVM8elrSDTqWsj_7jFplax2POShE,7992
|
|
@@ -131,11 +132,11 @@ uncountable/__init__.py,sha256=8l8XWNCKsu7TG94c-xa2KHpDegvxDC2FyQISdWC763Y,89
|
|
|
131
132
|
uncountable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
132
133
|
uncountable/core/__init__.py,sha256=58FSGc04XOyUXJ-uQ6Um73RCk4mFszgYaWcO2h_t5v8,395
|
|
133
134
|
uncountable/core/async_batch.py,sha256=9pYGFzVCQXt8059qFHgutweGIFPquJ5Xfq6NT5P-1K0,1206
|
|
134
|
-
uncountable/core/client.py,sha256=
|
|
135
|
+
uncountable/core/client.py,sha256=q7a10FZKVUNhtQX3FnFnhFAEiWsD75h0Ir6ZuaakuqY,17060
|
|
135
136
|
uncountable/core/environment.py,sha256=br-0pEsnTBZWj4Y4eIozvslf0MMOO30Z3wCh6d4YVzw,1222
|
|
136
137
|
uncountable/core/file_upload.py,sha256=tX4DAKLt5TEblK3EzypiToQEnGBqx0MnhHRKJFdaX_8,6106
|
|
137
138
|
uncountable/core/tls.py,sha256=mOWtDrGDbF_jy6kExkJQ3jjinxA8Fq8_JF1oUeW04eA,939
|
|
138
|
-
uncountable/core/types.py,sha256=
|
|
139
|
+
uncountable/core/types.py,sha256=UbT3Hcsa15YN5aG3fobojFWsGDxwoAW5hUm8jeWxx88,497
|
|
139
140
|
uncountable/core/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
140
141
|
uncountable/core/query/builder.py,sha256=MRaZYcSS6YssurRjMVhcTrGQhe5iRWjwYJzwt_9undY,8618
|
|
141
142
|
uncountable/core/query/column.py,sha256=WR5VRX30kGrObMxxWhnaH-ueJkn_P-K73kz0GSfZq08,45523
|
|
@@ -168,10 +169,12 @@ uncountable/integration/executors/script_executor.py,sha256=hKq09fOG89OAZ1Iq7wm2
|
|
|
168
169
|
uncountable/integration/http_server/__init__.py,sha256=ld7hD3wOGwAbnVRBrkTUCP5zNG-kptMd4wGJyQvUWVc,170
|
|
169
170
|
uncountable/integration/http_server/types.py,sha256=0J59HhEyntDjy0o37Qze0v0o-wC85QM5NQR8TuNrwV8,2672
|
|
170
171
|
uncountable/integration/queue_runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
172
|
+
uncountable/integration/queue_runner/command_handlers.py,sha256=s_GwAubFAi73Zo3DC3JW_cRp5_xOE32XHUJ_DrncVOw,4765
|
|
173
|
+
uncountable/integration/queue_runner/job_dispatcher.py,sha256=ukd94PDmlGWv5kjRZHCuTu1sMQeGL0SLwUK7riuv_po,1738
|
|
171
174
|
uncountable/integration/queue_runner/job_process.py,sha256=OONJ7aLsE80qIJ2ABLY3HYyKI7VdpBjI_pPQ3GCC13U,2586
|
|
172
175
|
uncountable/integration/queue_runner/job_runner.py,sha256=Yh9hYfEYOya3BR6ypsal3LKPKrrEb4VQwfGuh4qyU2w,6115
|
|
173
|
-
uncountable/integration/queue_runner/job_scheduler.py,sha256=
|
|
174
|
-
uncountable/integration/queue_runner/queue_runner.py,sha256=
|
|
176
|
+
uncountable/integration/queue_runner/job_scheduler.py,sha256=am81c5IArxQ0BTL44jH-bTUVRcSbfFBcMtC2SAVkXnc,7625
|
|
177
|
+
uncountable/integration/queue_runner/queue_runner.py,sha256=jUeVSNaQWGQpunVp-tvPNiAizG_r3DSTv5M1nGyJEhw,1267
|
|
175
178
|
uncountable/integration/queue_runner/types.py,sha256=8HS6KnYMS_vc5XHeMpg0BFAQC-5P3QLzd-aDYDMMt3E,244
|
|
176
179
|
uncountable/integration/queue_runner/worker.py,sha256=b-tmOLNMaSOA3VQRtOvymBZYyZomq0kSA5LpQlWI_jM,2027
|
|
177
180
|
uncountable/integration/queue_runner/command_server/__init__.py,sha256=hMCDLWct8zW4B2a9BaIAsMhtaEgFlxONjeow-6nf6dg,675
|
|
@@ -357,7 +360,7 @@ uncountable/types/api/file_folders/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHk
|
|
|
357
360
|
uncountable/types/api/file_folders/add_file_to_folder.py,sha256=NI34Jr_Mwra7uelaG6xWXW4XLvKNI0iV79mdyTCBg6g,1308
|
|
358
361
|
uncountable/types/api/file_folders/modify_file_system.py,sha256=7e671P6KMrmuQrxcUPyAz5klgjM8tNHs3R7a1cGwSPE,5088
|
|
359
362
|
uncountable/types/api/files/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
|
|
360
|
-
uncountable/types/api/files/download_file.py,sha256=
|
|
363
|
+
uncountable/types/api/files/download_file.py,sha256=nNJMJ-87H0bdQUBXd7-DeUH_zbWAco1D3ciO42O7xXQ,5789
|
|
361
364
|
uncountable/types/api/id_source/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
|
|
362
365
|
uncountable/types/api/id_source/list_id_source.py,sha256=tvrFKTynmfcLekUJnYSxSQi5V0N68ULjtn6xx0RmKec,1585
|
|
363
366
|
uncountable/types/api/id_source/match_id_source.py,sha256=xnKnhHQBYipJmMxfH-7MHyNM6QOTrkSApuN9oAlk2PA,1560
|
|
@@ -452,7 +455,7 @@ uncountable/types/api/uploader/complete_async_parse.py,sha256=Os1HAubxTlbut6Gylj
|
|
|
452
455
|
uncountable/types/api/uploader/invoke_uploader.py,sha256=cmy7zl3AMF_AH5PCAB67rBeTE6JIcaUqPzm2NS4k_AE,1907
|
|
453
456
|
uncountable/types/api/user/__init__.py,sha256=gCgbynxG3jA8FQHzercKtrHKHkiIKr8APdZYUniAor8,55
|
|
454
457
|
uncountable/types/api/user/get_current_user_info.py,sha256=BT7bGp5SOHz3XdKWhhdZ6acnInafT2PgdJ-H4YH3W7c,1181
|
|
455
|
-
uncountablepythonsdk-0.0.
|
|
456
|
-
uncountablepythonsdk-0.0.
|
|
457
|
-
uncountablepythonsdk-0.0.
|
|
458
|
-
uncountablepythonsdk-0.0.
|
|
458
|
+
uncountablepythonsdk-0.0.193.dist-info/METADATA,sha256=UVKkSALr9xvsG24IJzCbsq5Lcoz4c1lYnbt2yKDysmw,2241
|
|
459
|
+
uncountablepythonsdk-0.0.193.dist-info/WHEEL,sha256=a-udL2m4V_6Yn8b-uKzWb12RjRxwwMo6ohYFM24tpaY,91
|
|
460
|
+
uncountablepythonsdk-0.0.193.dist-info/top_level.txt,sha256=1UVGjAU-6hJY9qw2iJ7nCBeEwZ793AEN5ZfKX9A1uj4,31
|
|
461
|
+
uncountablepythonsdk-0.0.193.dist-info/RECORD,,
|
{uncountablepythonsdk-0.0.192.dist-info → uncountablepythonsdk-0.0.193.dist-info}/top_level.txt
RENAMED
|
File without changes
|