UncountablePythonSDK 0.0.128__py3-none-any.whl → 0.0.129__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of UncountablePythonSDK might be problematic. Click here for more details.
- examples/integration-server/jobs/materials_auto/example_instrument.py +4 -3
- examples/integration-server/jobs/materials_auto/example_parse.py +45 -2
- examples/integration-server/jobs/materials_auto/example_predictions.py +2 -2
- examples/integration-server/pyproject.toml +1 -1
- pkgs/serialization_util/serialization_helpers.py +3 -1
- pkgs/type_spec/builder.py +9 -3
- pkgs/type_spec/builder_types.py +9 -0
- pkgs/type_spec/cross_output_links.py +2 -10
- pkgs/type_spec/emit_open_api.py +0 -12
- pkgs/type_spec/emit_python.py +72 -11
- pkgs/type_spec/emit_typescript_util.py +28 -6
- pkgs/type_spec/load_types.py +1 -1
- pkgs/type_spec/type_info/emit_type_info.py +13 -2
- uncountable/core/client.py +10 -3
- uncountable/integration/queue_runner/command_server/command_server.py +8 -7
- uncountable/integration/webhook_server/entrypoint.py +2 -0
- uncountable/types/__init__.py +2 -0
- uncountable/types/api/entity/list_aggregate.py +79 -0
- uncountable/types/api/entity/list_entities.py +25 -0
- uncountable/types/api/recipes/get_recipes_data.py +13 -0
- uncountable/types/async_batch_processor.py +20 -0
- uncountable/types/client_base.py +194 -0
- uncountable/types/client_config.py +1 -0
- uncountable/types/client_config_t.py +10 -0
- uncountable/types/entity_t.py +2 -0
- {uncountablepythonsdk-0.0.128.dist-info → uncountablepythonsdk-0.0.129.dist-info}/METADATA +1 -1
- {uncountablepythonsdk-0.0.128.dist-info → uncountablepythonsdk-0.0.129.dist-info}/RECORD +29 -27
- {uncountablepythonsdk-0.0.128.dist-info → uncountablepythonsdk-0.0.129.dist-info}/WHEEL +0 -0
- {uncountablepythonsdk-0.0.128.dist-info → uncountablepythonsdk-0.0.129.dist-info}/top_level.txt +0 -0
|
@@ -3,8 +3,6 @@ import time
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from decimal import Decimal
|
|
5
5
|
|
|
6
|
-
from pkgs.argument_parser.argument_parser import CachedParser
|
|
7
|
-
from pkgs.serialization_util import serialize_for_api
|
|
8
6
|
from uncountable.integration.job import JobArguments, WebhookJob, register_job
|
|
9
7
|
from uncountable.types import (
|
|
10
8
|
base_t,
|
|
@@ -16,6 +14,9 @@ from uncountable.types.integration_session_t import IntegrationSessionInstrument
|
|
|
16
14
|
from websockets.sync.client import connect
|
|
17
15
|
from websockets.typing import Data
|
|
18
16
|
|
|
17
|
+
from pkgs.argument_parser.argument_parser import CachedParser
|
|
18
|
+
from pkgs.serialization_util import serialize_for_api
|
|
19
|
+
|
|
19
20
|
|
|
20
21
|
@dataclass(kw_only=True)
|
|
21
22
|
class InstrumentPayload:
|
|
@@ -34,7 +35,7 @@ class InstrumentExample(WebhookJob[InstrumentPayload]):
|
|
|
34
35
|
def parse_message(message: Data) -> sockets_t.SocketEventData | None:
|
|
35
36
|
try:
|
|
36
37
|
return parser.parse_api(json.loads(message)).data
|
|
37
|
-
except ValueError
|
|
38
|
+
except ValueError:
|
|
38
39
|
return None
|
|
39
40
|
|
|
40
41
|
integration_session = IntegrationSessionInstrument(
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
|
+
from decimal import Decimal
|
|
2
3
|
|
|
3
4
|
from uncountable.integration.job import JobArguments, WebhookJob, register_job
|
|
4
5
|
from uncountable.types import (
|
|
5
6
|
base_t,
|
|
7
|
+
entity_t,
|
|
6
8
|
generic_upload_t,
|
|
7
9
|
identifier_t,
|
|
8
10
|
job_definition_t,
|
|
11
|
+
notifications_t,
|
|
9
12
|
uploader_t,
|
|
10
13
|
)
|
|
11
14
|
|
|
@@ -68,18 +71,58 @@ class ParseExample(WebhookJob[ParsePayload]):
|
|
|
68
71
|
data=uploader_t.StringValue(value="my_file_to_upload.xlsx"),
|
|
69
72
|
),
|
|
70
73
|
),
|
|
74
|
+
uploader_t.HeaderEntry(
|
|
75
|
+
type=uploader_t.StructureElementType.HEADER,
|
|
76
|
+
value=uploader_t.NumericHeaderData(
|
|
77
|
+
name="file structure number",
|
|
78
|
+
data=uploader_t.DecimalValue(value=Decimal(99)),
|
|
79
|
+
),
|
|
80
|
+
),
|
|
71
81
|
],
|
|
72
82
|
)
|
|
73
83
|
]
|
|
74
84
|
|
|
75
|
-
|
|
85
|
+
user_id: base_t.ObjectId | None = None
|
|
86
|
+
recipe_id: base_t.ObjectId | None = None
|
|
87
|
+
data = args.client.get_entities_data(
|
|
88
|
+
entity_ids=[payload.async_job_id], entity_type=entity_t.EntityType.ASYNC_JOB
|
|
89
|
+
)
|
|
90
|
+
for field_value in data.entity_details[0].field_values:
|
|
91
|
+
if field_value.field_ref_name == "core_async_job_jobData":
|
|
92
|
+
assert isinstance(field_value.value, dict)
|
|
93
|
+
assert isinstance(field_value.value["user_id"], int)
|
|
94
|
+
user_id = field_value.value["user_id"]
|
|
95
|
+
if (
|
|
96
|
+
field_value.field_ref_name
|
|
97
|
+
== "unc_async_job_custom_parser_recipe_ids_in_view"
|
|
98
|
+
):
|
|
99
|
+
assert isinstance(field_value.value, list)
|
|
100
|
+
assert isinstance(field_value.value[0], int)
|
|
101
|
+
recipe_id = field_value.value[0]
|
|
102
|
+
|
|
103
|
+
assert user_id is not None
|
|
104
|
+
assert recipe_id is not None
|
|
105
|
+
|
|
106
|
+
complete_async_parse_req = args.batch_processor.complete_async_parse(
|
|
76
107
|
parsed_file_data=dummy_parsed_file_data,
|
|
77
108
|
async_job_key=identifier_t.IdentifierKeyId(id=payload.async_job_id),
|
|
78
109
|
upload_destination=generic_upload_t.UploadDestinationRecipe(
|
|
79
|
-
recipe_key=identifier_t.IdentifierKeyId(id=
|
|
110
|
+
recipe_key=identifier_t.IdentifierKeyId(id=recipe_id)
|
|
80
111
|
),
|
|
81
112
|
)
|
|
82
113
|
|
|
114
|
+
args.batch_processor.push_notification(
|
|
115
|
+
depends_on=[complete_async_parse_req.batch_reference],
|
|
116
|
+
notification_targets=[
|
|
117
|
+
notifications_t.NotificationTargetUser(
|
|
118
|
+
user_key=identifier_t.IdentifierKeyId(id=user_id)
|
|
119
|
+
)
|
|
120
|
+
],
|
|
121
|
+
subject="Upload complete",
|
|
122
|
+
message="Your file has been uploaded",
|
|
123
|
+
display_notice=True,
|
|
124
|
+
)
|
|
125
|
+
|
|
83
126
|
return job_definition_t.JobResult(success=True)
|
|
84
127
|
|
|
85
128
|
@property
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import random
|
|
2
2
|
from dataclasses import dataclass
|
|
3
|
-
from datetime import datetime
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
4
|
from decimal import Decimal
|
|
5
5
|
|
|
6
6
|
from uncountable.integration.job import JobArguments, WebhookJob, register_job
|
|
@@ -25,7 +25,7 @@ class PredictionsExample(WebhookJob[PredictionsPayload]):
|
|
|
25
25
|
self, args: JobArguments, payload: PredictionsPayload
|
|
26
26
|
) -> job_definition_t.JobResult:
|
|
27
27
|
recipe_data = args.client.get_recipes_data(recipe_ids=payload.recipe_ids)
|
|
28
|
-
formatted_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
28
|
+
formatted_datetime = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S")
|
|
29
29
|
|
|
30
30
|
for recipe in recipe_data.recipes:
|
|
31
31
|
test_sample_name = f"Predictions Model ({formatted_datetime})"
|
|
@@ -115,7 +115,6 @@ lint.ignore = [
|
|
|
115
115
|
"PD010", # .pivottable. Should add
|
|
116
116
|
"PD011", # use .to_numpy. Skip
|
|
117
117
|
"PD015", # use .merge. Should add
|
|
118
|
-
"PD901", # avoid generic df name. Skip
|
|
119
118
|
"PERF203", # avoid try except in loop. Skip
|
|
120
119
|
"PERF401", # use list comprehension. Skip
|
|
121
120
|
"PERF402", # use list.copy. Skip
|
|
@@ -213,6 +212,7 @@ exclude = [
|
|
|
213
212
|
|
|
214
213
|
[tool.ruff.lint.isort]
|
|
215
214
|
split-on-trailing-comma = true
|
|
215
|
+
known-first-party = ["pkgs"]
|
|
216
216
|
|
|
217
217
|
[tool.ruff.lint.mccabe]
|
|
218
218
|
max-complexity = 130 # goal would be to bring this down to ~50 or so
|
|
@@ -85,7 +85,9 @@ def _serialize_dataclass(d: Any) -> dict[str, JsonValue]:
|
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def _to_string_value(value: Any) -> str:
|
|
88
|
-
assert isinstance(value, (Decimal, int))
|
|
88
|
+
assert isinstance(value, (Decimal, int)), (
|
|
89
|
+
f"Expecting decimal or int, received: {value} (type={type(value)})"
|
|
90
|
+
)
|
|
89
91
|
return str(value)
|
|
90
92
|
|
|
91
93
|
|
pkgs/type_spec/builder.py
CHANGED
|
@@ -13,7 +13,7 @@ from enum import Enum, StrEnum, auto
|
|
|
13
13
|
from typing import Any, Self
|
|
14
14
|
|
|
15
15
|
from . import util
|
|
16
|
-
from .
|
|
16
|
+
from .builder_types import CrossOutputPaths
|
|
17
17
|
from .non_discriminated_union_exceptions import NON_DISCRIMINATED_UNION_EXCEPTIONS
|
|
18
18
|
from .util import parse_type_str
|
|
19
19
|
|
|
@@ -1101,7 +1101,7 @@ def _parse_const(
|
|
|
1101
1101
|
elif const_type.defn_type.name == BaseTypeName.s_dict:
|
|
1102
1102
|
assert isinstance(value, dict)
|
|
1103
1103
|
builder.ensure(
|
|
1104
|
-
len(const_type.parameters) == 2, "constant-dict-expects-
|
|
1104
|
+
len(const_type.parameters) == 2, "constant-dict-expects-two-types"
|
|
1105
1105
|
)
|
|
1106
1106
|
key_type = const_type.parameters[0]
|
|
1107
1107
|
value_type = const_type.parameters[1]
|
|
@@ -1150,6 +1150,11 @@ def _parse_const(
|
|
|
1150
1150
|
)
|
|
1151
1151
|
return value
|
|
1152
1152
|
|
|
1153
|
+
if not const_type.is_base:
|
|
1154
|
+
# IMPROVE: validate the object type properties before emission stage
|
|
1155
|
+
builder.ensure(isinstance(value, dict), "invalid value for object constant")
|
|
1156
|
+
return value
|
|
1157
|
+
|
|
1153
1158
|
raise Exception("unsupported-const-scalar-type", const_type)
|
|
1154
1159
|
|
|
1155
1160
|
|
|
@@ -1271,7 +1276,8 @@ class SpecNamespace:
|
|
|
1271
1276
|
|
|
1272
1277
|
assert util.is_valid_type_name(name), f"{name} is not a valid type name"
|
|
1273
1278
|
assert name not in self.types, f"{name} is duplicate"
|
|
1274
|
-
defn_type = defn
|
|
1279
|
+
defn_type = defn.get("type")
|
|
1280
|
+
assert isinstance(defn_type, str), f"{name} requires a string type"
|
|
1275
1281
|
spec_type: SpecTypeDefn
|
|
1276
1282
|
if defn_type == DefnTypeName.s_alias:
|
|
1277
1283
|
spec_type = SpecTypeDefnAlias(self, name)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
-
from dataclasses import dataclass
|
|
5
4
|
|
|
6
|
-
from
|
|
5
|
+
from . import builder
|
|
6
|
+
from .builder_types import CrossOutputPaths
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
def get_python_stub_file_path(
|
|
@@ -17,14 +17,6 @@ def get_python_stub_file_path(
|
|
|
17
17
|
return api_stub_file
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
@dataclass(kw_only=True, frozen=True)
|
|
21
|
-
class CrossOutputPaths:
|
|
22
|
-
python_types_output: str
|
|
23
|
-
typescript_types_output: str
|
|
24
|
-
typescript_routes_output_by_endpoint: dict[str, str]
|
|
25
|
-
typespec_files_input: list[str]
|
|
26
|
-
|
|
27
|
-
|
|
28
20
|
def get_python_api_file_path(
|
|
29
21
|
cross_output_paths: CrossOutputPaths,
|
|
30
22
|
namespace: builder.SpecNamespace,
|
pkgs/type_spec/emit_open_api.py
CHANGED
|
@@ -610,18 +610,6 @@ def _emit_type(
|
|
|
610
610
|
ctx.types[stype.name] = final_type
|
|
611
611
|
|
|
612
612
|
|
|
613
|
-
def _emit_constant(ctx: EmitOpenAPIContext, sconst: builder.SpecConstant) -> None:
|
|
614
|
-
if sconst.value_type.is_base_type(builder.BaseTypeName.s_string):
|
|
615
|
-
value = util.encode_common_string(cast(str, sconst.value))
|
|
616
|
-
elif sconst.value_type.is_base_type(builder.BaseTypeName.s_integer):
|
|
617
|
-
value = str(sconst.value)
|
|
618
|
-
else:
|
|
619
|
-
raise Exception("invalid constant type", sconst.name)
|
|
620
|
-
|
|
621
|
-
const_name = sconst.name.upper()
|
|
622
|
-
print("_emit_constant", value, const_name)
|
|
623
|
-
|
|
624
|
-
|
|
625
613
|
def _emit_endpoint(
|
|
626
614
|
gctx: EmitOpenAPIGlobalContext,
|
|
627
615
|
ctx: EmitOpenAPIContext,
|
pkgs/type_spec/emit_python.py
CHANGED
|
@@ -35,6 +35,11 @@ QUEUED_BATCH_REQUEST_STYPE = builder.SpecTypeDefnObject(
|
|
|
35
35
|
namespace=ASYNC_BATCH_TYPE_NAMESPACE, name="QueuedAsyncBatchRequest"
|
|
36
36
|
)
|
|
37
37
|
|
|
38
|
+
CLIENT_CONFIG_TYPE_NAMESPACE = builder.SpecNamespace(name="client_config")
|
|
39
|
+
REQUEST_OPTIONS_STYPE = builder.SpecTypeDefnObject(
|
|
40
|
+
namespace=CLIENT_CONFIG_TYPE_NAMESPACE, name="RequestOptions"
|
|
41
|
+
)
|
|
42
|
+
|
|
38
43
|
|
|
39
44
|
@dataclasses.dataclass(kw_only=True)
|
|
40
45
|
class TrackingContext:
|
|
@@ -117,26 +122,36 @@ def _check_type_match(stype: builder.SpecType, value: Any) -> bool:
|
|
|
117
122
|
raise Exception("invalid type", stype, value)
|
|
118
123
|
|
|
119
124
|
|
|
120
|
-
def _emit_value(
|
|
125
|
+
def _emit_value(
|
|
126
|
+
ctx: TrackingContext, stype: builder.SpecType, value: Any, indent: int = 0
|
|
127
|
+
) -> str:
|
|
121
128
|
literal = builder.unwrap_literal_type(stype)
|
|
122
129
|
if literal is not None:
|
|
123
130
|
return _emit_value(ctx, literal.value_type, literal.value)
|
|
124
131
|
|
|
125
132
|
if stype.is_base_type(builder.BaseTypeName.s_string):
|
|
126
|
-
assert isinstance(value, str)
|
|
133
|
+
assert isinstance(value, str), (
|
|
134
|
+
f"Expected str value for {stype.name} but got {value}"
|
|
135
|
+
)
|
|
127
136
|
return util.encode_common_string(value)
|
|
128
137
|
elif stype.is_base_type(builder.BaseTypeName.s_integer):
|
|
129
|
-
assert isinstance(value, int)
|
|
138
|
+
assert isinstance(value, int), (
|
|
139
|
+
f"Expected int value for {stype.name} but got {value}"
|
|
140
|
+
)
|
|
130
141
|
return str(value)
|
|
131
142
|
elif stype.is_base_type(builder.BaseTypeName.s_boolean):
|
|
132
|
-
assert isinstance(value, bool)
|
|
143
|
+
assert isinstance(value, bool), (
|
|
144
|
+
f"Expected bool value for {stype.name} but got {value}"
|
|
145
|
+
)
|
|
133
146
|
return "True" if value else "False"
|
|
134
147
|
elif stype.is_base_type(builder.BaseTypeName.s_decimal) or stype.is_base_type(
|
|
135
148
|
builder.BaseTypeName.s_lossy_decimal
|
|
136
149
|
):
|
|
137
150
|
# Note that decimal requires the `!decimal 123.12` style notation in the YAML
|
|
138
151
|
# file since PyYaml parses numbers as float, unfortuantely
|
|
139
|
-
assert isinstance(value, (Decimal, int))
|
|
152
|
+
assert isinstance(value, (Decimal, int)), (
|
|
153
|
+
f"Expected decimal value for {stype.name} but got {value} (type: {type(value)})"
|
|
154
|
+
)
|
|
140
155
|
if isinstance(value, int):
|
|
141
156
|
# skip quotes for integers
|
|
142
157
|
return f"Decimal({value})"
|
|
@@ -151,14 +166,14 @@ def _emit_value(ctx: TrackingContext, stype: builder.SpecType, value: Any) -> st
|
|
|
151
166
|
key_type = stype.parameters[0]
|
|
152
167
|
value_type = stype.parameters[1]
|
|
153
168
|
return (
|
|
154
|
-
"{\n
|
|
155
|
-
+ ",\n
|
|
169
|
+
f"{{\n{INDENT * (indent + 1)}"
|
|
170
|
+
+ f",\n{INDENT * (indent + 1)}".join(
|
|
156
171
|
_emit_value(ctx, key_type, dkey)
|
|
157
172
|
+ ": "
|
|
158
|
-
+ _emit_value(ctx, value_type, dvalue)
|
|
173
|
+
+ _emit_value(ctx, value_type, dvalue, indent=indent + 1)
|
|
159
174
|
for dkey, dvalue in value.items()
|
|
160
175
|
)
|
|
161
|
-
+ "\n}"
|
|
176
|
+
+ f"\n{INDENT * indent}}}"
|
|
162
177
|
)
|
|
163
178
|
|
|
164
179
|
if stype.defn_type.is_base_type(builder.BaseTypeName.s_optional):
|
|
@@ -184,6 +199,34 @@ def _emit_value(ctx: TrackingContext, stype: builder.SpecType, value: Any) -> st
|
|
|
184
199
|
return f"{refer_to(ctx, stype)}.{_resolve_enum_name(value, stype.name_case)}"
|
|
185
200
|
elif isinstance(stype, builder.SpecTypeDefnAlias):
|
|
186
201
|
return _emit_value(ctx, stype.alias, value)
|
|
202
|
+
elif isinstance(stype, builder.SpecTypeDefnObject):
|
|
203
|
+
assert isinstance(value, dict), (
|
|
204
|
+
f"Expected dict value for {stype.name} but got {value}"
|
|
205
|
+
)
|
|
206
|
+
if not stype.is_hashable:
|
|
207
|
+
raise Exception("invalid constant object type, non-hashable", value, stype)
|
|
208
|
+
obj_out = f"{refer_to(ctx, stype)}("
|
|
209
|
+
emitted_fields: set[str] = set()
|
|
210
|
+
for prop_name, prop in (stype.properties or {}).items():
|
|
211
|
+
if prop_name not in value:
|
|
212
|
+
continue
|
|
213
|
+
else:
|
|
214
|
+
value_to_emit = value[prop_name]
|
|
215
|
+
emitted_fields.add(prop_name)
|
|
216
|
+
py_name = python_field_name(prop.name, prop.name_case)
|
|
217
|
+
obj_out += f"\n{INDENT * (indent + 1)}{py_name}={_emit_value(ctx, prop.spec_type, value_to_emit, indent=indent + 1)},"
|
|
218
|
+
whitespace = f"\n{INDENT * indent}" if len(emitted_fields) > 0 else ""
|
|
219
|
+
obj_out += f"{whitespace})"
|
|
220
|
+
|
|
221
|
+
if emitted_fields != set(value.keys()):
|
|
222
|
+
raise Exception(
|
|
223
|
+
"invalid object type, extra fields found:",
|
|
224
|
+
value,
|
|
225
|
+
stype,
|
|
226
|
+
set(value.keys()) - emitted_fields,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
return obj_out
|
|
187
230
|
|
|
188
231
|
raise Exception("invalid constant type", value, stype)
|
|
189
232
|
|
|
@@ -471,6 +514,19 @@ def _emit_endpoint_invocation_function_signature(
|
|
|
471
514
|
else []
|
|
472
515
|
) + (extra_params if extra_params is not None else [])
|
|
473
516
|
|
|
517
|
+
request_options_property = builder.SpecProperty(
|
|
518
|
+
name="_request_options",
|
|
519
|
+
label="_request_options",
|
|
520
|
+
spec_type=REQUEST_OPTIONS_STYPE,
|
|
521
|
+
extant=builder.PropertyExtant.optional,
|
|
522
|
+
convert_value=builder.PropertyConvertValue.auto,
|
|
523
|
+
name_case=builder.NameCase.convert,
|
|
524
|
+
default=None,
|
|
525
|
+
has_default=True,
|
|
526
|
+
desc=None,
|
|
527
|
+
)
|
|
528
|
+
all_arguments.append(request_options_property)
|
|
529
|
+
|
|
474
530
|
# All endpoints share a function name
|
|
475
531
|
function = endpoint.path_per_api_endpoint[endpoint.default_endpoint_key].function
|
|
476
532
|
assert function is not None
|
|
@@ -641,6 +697,7 @@ def _emit_endpoint_invocation_function(
|
|
|
641
697
|
method={refer_to(ctx=ctx, stype=endpoint_method_stype)},
|
|
642
698
|
endpoint={refer_to(ctx=ctx, stype=endpoint_path_stype)},
|
|
643
699
|
args=args,
|
|
700
|
+
request_options=_request_options,
|
|
644
701
|
)
|
|
645
702
|
return self.do_request(api_request=api_request, return_type={refer_to(ctx=ctx, stype=data_type)})"""
|
|
646
703
|
)
|
|
@@ -1396,7 +1453,7 @@ CLIENT_CLASS_IMPORTS = [
|
|
|
1396
1453
|
"import dataclasses",
|
|
1397
1454
|
]
|
|
1398
1455
|
ASYNC_BATCH_PROCESSOR_FILENAME = "async_batch_processor"
|
|
1399
|
-
|
|
1456
|
+
ASYNC_BATCH_PROCESSOR_BASE_IMPORTS = [
|
|
1400
1457
|
"import uuid",
|
|
1401
1458
|
"from abc import ABC, abstractmethod",
|
|
1402
1459
|
"from pkgs.serialization_util import serialize_for_api",
|
|
@@ -1434,8 +1491,11 @@ def _emit_async_batch_processor(
|
|
|
1434
1491
|
config=config,
|
|
1435
1492
|
)
|
|
1436
1493
|
|
|
1494
|
+
imports = ASYNC_BATCH_PROCESSOR_BASE_IMPORTS.copy()
|
|
1495
|
+
if ctx.use_dataclass:
|
|
1496
|
+
imports.append("import dataclasses")
|
|
1437
1497
|
async_batch_processor_out.write(
|
|
1438
|
-
f"""{LINE_BREAK.join(
|
|
1498
|
+
f"""{LINE_BREAK.join(imports)}
|
|
1439
1499
|
|
|
1440
1500
|
|
|
1441
1501
|
class AsyncBatchProcessorBase(ABC):
|
|
@@ -1498,6 +1558,7 @@ class APIRequest:
|
|
|
1498
1558
|
method: str
|
|
1499
1559
|
endpoint: str
|
|
1500
1560
|
args: typing.Any
|
|
1561
|
+
request_options: {refer_to(ctx=ctx, stype=REQUEST_OPTIONS_STYPE)} | None = None
|
|
1501
1562
|
|
|
1502
1563
|
|
|
1503
1564
|
class ClientMethods(ABC):
|
|
@@ -3,7 +3,7 @@ import typing
|
|
|
3
3
|
from dataclasses import dataclass, field
|
|
4
4
|
|
|
5
5
|
from . import builder, util
|
|
6
|
-
from .
|
|
6
|
+
from .builder_types import CrossOutputPaths
|
|
7
7
|
|
|
8
8
|
INDENT = " "
|
|
9
9
|
|
|
@@ -51,7 +51,10 @@ def ts_name(name: str, name_case: builder.NameCase) -> str:
|
|
|
51
51
|
|
|
52
52
|
|
|
53
53
|
def emit_value_ts(
|
|
54
|
-
ctx: EmitTypescriptContext,
|
|
54
|
+
ctx: EmitTypescriptContext,
|
|
55
|
+
stype: builder.SpecType,
|
|
56
|
+
value: typing.Any,
|
|
57
|
+
indent: int = 0,
|
|
55
58
|
) -> str:
|
|
56
59
|
"""Mimics emit_python even if not all types are used in TypeScript yet"""
|
|
57
60
|
literal = builder.unwrap_literal_type(stype)
|
|
@@ -88,17 +91,17 @@ def emit_value_ts(
|
|
|
88
91
|
raise Exception("invalid dict keys -- dict keys must be string or enum")
|
|
89
92
|
|
|
90
93
|
return (
|
|
91
|
-
"{\n
|
|
92
|
-
+ ",\n
|
|
94
|
+
f"{{\n{INDENT * (indent + 1)}"
|
|
95
|
+
+ f",\n{INDENT * (indent + 1)}".join(
|
|
93
96
|
(
|
|
94
97
|
f"[{emit_value_ts(ctx, key_type, dkey)}]: "
|
|
95
98
|
if not key_type.is_base_type(builder.BaseTypeName.s_string)
|
|
96
99
|
else f"{dkey}: "
|
|
97
100
|
)
|
|
98
|
-
+ emit_value_ts(ctx, value_type, dvalue)
|
|
101
|
+
+ emit_value_ts(ctx, value_type, dvalue, indent=indent + 1)
|
|
99
102
|
for dkey, dvalue in value.items()
|
|
100
103
|
)
|
|
101
|
-
+ "\n}"
|
|
104
|
+
+ f"\n{INDENT * (indent)}}}"
|
|
102
105
|
)
|
|
103
106
|
|
|
104
107
|
if stype.defn_type.is_base_type(builder.BaseTypeName.s_optional):
|
|
@@ -109,6 +112,25 @@ def emit_value_ts(
|
|
|
109
112
|
|
|
110
113
|
elif isinstance(stype, builder.SpecTypeDefnStringEnum):
|
|
111
114
|
return f"{refer_to(ctx, stype)}.{ts_enum_name(value, stype.name_case)}"
|
|
115
|
+
elif isinstance(stype, builder.SpecTypeDefnObject):
|
|
116
|
+
assert isinstance(value, dict), (
|
|
117
|
+
f"Expected dict value for {stype.name} but got {value}"
|
|
118
|
+
)
|
|
119
|
+
obj_out = "{"
|
|
120
|
+
did_emit = False
|
|
121
|
+
for prop_name, prop in (stype.properties or {}).items():
|
|
122
|
+
if prop_name not in value and prop.has_default:
|
|
123
|
+
value_to_emit = prop.default
|
|
124
|
+
elif prop_name not in value:
|
|
125
|
+
continue
|
|
126
|
+
else:
|
|
127
|
+
value_to_emit = value[prop_name]
|
|
128
|
+
did_emit = True
|
|
129
|
+
typescript_name = ts_name(prop.name, prop.name_case)
|
|
130
|
+
obj_out += f"\n{INDENT * (indent + 1)}{typescript_name}: {emit_value_ts(ctx, prop.spec_type, value_to_emit, indent=indent + 1)},"
|
|
131
|
+
whitespace = f"\n{INDENT * indent}" if did_emit else ""
|
|
132
|
+
obj_out += f"{whitespace}}} as const"
|
|
133
|
+
return obj_out
|
|
112
134
|
|
|
113
135
|
raise Exception("invalid constant type", value, stype, type(stype))
|
|
114
136
|
|
pkgs/type_spec/load_types.py
CHANGED
|
@@ -7,8 +7,8 @@ from shelljob import fs
|
|
|
7
7
|
from pkgs.serialization import yaml
|
|
8
8
|
|
|
9
9
|
from .builder import SpecBuilder
|
|
10
|
+
from .builder_types import CrossOutputPaths
|
|
10
11
|
from .config import Config
|
|
11
|
-
from .cross_output_links import CrossOutputPaths
|
|
12
12
|
|
|
13
13
|
ext_map = {
|
|
14
14
|
".ts": "typescript",
|
|
@@ -171,9 +171,16 @@ class MapTypeAlias(MapTypeBase):
|
|
|
171
171
|
discriminator: str | None
|
|
172
172
|
|
|
173
173
|
|
|
174
|
+
@dataclasses.dataclass
|
|
175
|
+
class StringEnumValue:
|
|
176
|
+
value: str
|
|
177
|
+
label: str
|
|
178
|
+
deprecated: bool = False
|
|
179
|
+
|
|
180
|
+
|
|
174
181
|
@dataclasses.dataclass
|
|
175
182
|
class MapStringEnum(MapTypeBase):
|
|
176
|
-
values: dict[str,
|
|
183
|
+
values: dict[str, StringEnumValue]
|
|
177
184
|
|
|
178
185
|
|
|
179
186
|
MapType = MapTypeObject | MapTypeAlias | MapStringEnum
|
|
@@ -436,7 +443,11 @@ def _build_map_type(
|
|
|
436
443
|
# IMPROVE: We probably want the label here, but this requires a change
|
|
437
444
|
# to the front-end type-info and form code to handle
|
|
438
445
|
values={
|
|
439
|
-
entry.value: (
|
|
446
|
+
entry.value: StringEnumValue(
|
|
447
|
+
value=entry.value,
|
|
448
|
+
label=entry.label or entry.name,
|
|
449
|
+
deprecated=entry.deprecated,
|
|
450
|
+
)
|
|
440
451
|
for entry in stype.values.values()
|
|
441
452
|
},
|
|
442
453
|
)
|
uncountable/core/client.py
CHANGED
|
@@ -226,13 +226,15 @@ class Client(ClientMethods):
|
|
|
226
226
|
except JSONDecodeError as e:
|
|
227
227
|
raise SDKError("unable to process response", request_id=request_id) from e
|
|
228
228
|
|
|
229
|
-
def _send_request(
|
|
229
|
+
def _send_request(
|
|
230
|
+
self, request: requests.Request, *, timeout: float | None = None
|
|
231
|
+
) -> requests.Response:
|
|
230
232
|
if self._cfg.extra_headers is not None:
|
|
231
233
|
request.headers = {**request.headers, **self._cfg.extra_headers}
|
|
232
234
|
if self._cfg.transform_request is not None:
|
|
233
235
|
request = self._cfg.transform_request(request)
|
|
234
236
|
prepared_request = request.prepare()
|
|
235
|
-
response = self._session.send(prepared_request)
|
|
237
|
+
response = self._session.send(prepared_request, timeout=timeout)
|
|
236
238
|
return response
|
|
237
239
|
|
|
238
240
|
def do_request(self, *, api_request: APIRequest, return_type: type[DT]) -> DT:
|
|
@@ -257,7 +259,12 @@ class Client(ClientMethods):
|
|
|
257
259
|
with push_scope_optional(self._cfg.logger, "api_call", attributes=attributes):
|
|
258
260
|
if self._cfg.logger is not None:
|
|
259
261
|
self._cfg.logger.log_info(api_request.endpoint, attributes=attributes)
|
|
260
|
-
|
|
262
|
+
timeout = (
|
|
263
|
+
api_request.request_options.timeout_secs
|
|
264
|
+
if api_request.request_options is not None
|
|
265
|
+
else None
|
|
266
|
+
)
|
|
267
|
+
response = self._send_request(request, timeout=timeout)
|
|
261
268
|
response_data = self._get_response_json(response, request_id=request_id)
|
|
262
269
|
cached_parser = self._get_cached_parser(return_type)
|
|
263
270
|
try:
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
|
|
3
|
+
import grpc.aio as grpc_aio
|
|
3
4
|
import simplejson as json
|
|
4
5
|
from google.protobuf.timestamp_pb2 import Timestamp
|
|
5
|
-
from grpc import StatusCode
|
|
6
|
+
from grpc import StatusCode
|
|
6
7
|
|
|
7
8
|
from pkgs.argument_parser import CachedParser
|
|
8
9
|
from uncountable.core.environment import get_local_admin_server_port
|
|
@@ -40,11 +41,11 @@ queued_job_payload_parser = CachedParser(queued_job_t.QueuedJobPayload)
|
|
|
40
41
|
|
|
41
42
|
|
|
42
43
|
async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None:
|
|
43
|
-
server =
|
|
44
|
+
server = grpc_aio.server()
|
|
44
45
|
|
|
45
46
|
class CommandServerHandler(CommandServerServicer):
|
|
46
47
|
async def EnqueueJob(
|
|
47
|
-
self, request: EnqueueJobRequest, context:
|
|
48
|
+
self, request: EnqueueJobRequest, context: grpc_aio.ServicerContext
|
|
48
49
|
) -> EnqueueJobResult:
|
|
49
50
|
payload_json = json.loads(request.serialized_payload)
|
|
50
51
|
payload = queued_job_payload_parser.parse_api(payload_json)
|
|
@@ -63,7 +64,7 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
|
|
|
63
64
|
return result
|
|
64
65
|
|
|
65
66
|
async def RetryJob(
|
|
66
|
-
self, request: RetryJobRequest, context:
|
|
67
|
+
self, request: RetryJobRequest, context: grpc_aio.ServicerContext
|
|
67
68
|
) -> RetryJobResult:
|
|
68
69
|
response_queue: asyncio.Queue[CommandRetryJobResponse] = asyncio.Queue()
|
|
69
70
|
await command_queue.put(
|
|
@@ -80,12 +81,12 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
|
|
|
80
81
|
return RetryJobResult(successfully_queued=False, queued_job_uuid="")
|
|
81
82
|
|
|
82
83
|
async def CheckHealth(
|
|
83
|
-
self, request: CheckHealthRequest, context:
|
|
84
|
+
self, request: CheckHealthRequest, context: grpc_aio.ServicerContext
|
|
84
85
|
) -> CheckHealthResult:
|
|
85
86
|
return CheckHealthResult(success=True)
|
|
86
87
|
|
|
87
88
|
async def ListQueuedJobs(
|
|
88
|
-
self, request: ListQueuedJobsRequest, context:
|
|
89
|
+
self, request: ListQueuedJobsRequest, context: grpc_aio.ServicerContext
|
|
89
90
|
) -> ListQueuedJobsResult:
|
|
90
91
|
if (
|
|
91
92
|
request.limit < ListQueuedJobsConstants.LIMIT_MIN
|
|
@@ -121,7 +122,7 @@ async def serve(command_queue: CommandQueue, datastore: DatastoreSqlite) -> None
|
|
|
121
122
|
return ListQueuedJobsResult(queued_jobs=response_list)
|
|
122
123
|
|
|
123
124
|
async def VaccuumQueuedJobs(
|
|
124
|
-
self, request: VaccuumQueuedJobsRequest, context:
|
|
125
|
+
self, request: VaccuumQueuedJobsRequest, context: grpc_aio.ServicerContext
|
|
125
126
|
) -> VaccuumQueuedJobsResult:
|
|
126
127
|
response_queue: asyncio.Queue[CommandVaccuumQueuedJobsResponse] = (
|
|
127
128
|
asyncio.Queue()
|
uncountable/types/__init__.py
CHANGED
|
@@ -70,6 +70,7 @@ from . import integration_session_t as integration_session_t
|
|
|
70
70
|
from . import integrations_t as integrations_t
|
|
71
71
|
from .api.uploader import invoke_uploader as invoke_uploader_t
|
|
72
72
|
from . import job_definition_t as job_definition_t
|
|
73
|
+
from .api.entity import list_aggregate as list_aggregate_t
|
|
73
74
|
from .api.entity import list_entities as list_entities_t
|
|
74
75
|
from .api.id_source import list_id_source as list_id_source_t
|
|
75
76
|
from .api.entity import lock_entity as lock_entity_t
|
|
@@ -199,6 +200,7 @@ __all__: list[str] = [
|
|
|
199
200
|
"integrations_t",
|
|
200
201
|
"invoke_uploader_t",
|
|
201
202
|
"job_definition_t",
|
|
203
|
+
"list_aggregate_t",
|
|
202
204
|
"list_entities_t",
|
|
203
205
|
"list_id_source_t",
|
|
204
206
|
"lock_entity_t",
|