UncountablePythonSDK 0.0.91__py3-none-any.whl → 0.0.93__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of UncountablePythonSDK might be problematic. Click here for more details.
- {UncountablePythonSDK-0.0.91.dist-info → UncountablePythonSDK-0.0.93.dist-info}/METADATA +2 -2
- UncountablePythonSDK-0.0.93.dist-info/RECORD +301 -0
- {UncountablePythonSDK-0.0.91.dist-info → UncountablePythonSDK-0.0.93.dist-info}/WHEEL +1 -1
- examples/set_recipe_metadata_file.py +1 -1
- examples/upload_files.py +1 -2
- pkgs/argument_parser/__init__.py +3 -0
- pkgs/argument_parser/argument_parser.py +52 -23
- pkgs/filesystem_utils/__init__.py +1 -0
- pkgs/filesystem_utils/_gdrive_session.py +4 -4
- pkgs/filesystem_utils/_s3_session.py +2 -1
- pkgs/filesystem_utils/_sftp_session.py +2 -3
- pkgs/filesystem_utils/file_type_utils.py +10 -10
- pkgs/serialization/annotation.py +5 -5
- pkgs/serialization/missing_sentry.py +1 -1
- pkgs/serialization/serial_alias.py +3 -3
- pkgs/serialization/serial_class.py +10 -10
- pkgs/serialization/serial_generic.py +1 -1
- pkgs/serialization/serial_union.py +10 -10
- pkgs/serialization_util/__init__.py +2 -0
- pkgs/serialization_util/serialization_helpers.py +1 -4
- pkgs/type_spec/actions_registry/__main__.py +0 -4
- pkgs/type_spec/builder.py +121 -40
- pkgs/type_spec/config.py +10 -5
- pkgs/type_spec/emit_open_api.py +2 -2
- pkgs/type_spec/emit_open_api_util.py +1 -1
- pkgs/type_spec/emit_python.py +145 -63
- pkgs/type_spec/emit_typescript.py +57 -10
- pkgs/type_spec/load_types.py +1 -2
- pkgs/type_spec/open_api_util.py +1 -2
- pkgs/type_spec/parts/base.py.prepart +2 -0
- pkgs/type_spec/type_info/emit_type_info.py +8 -8
- pkgs/type_spec/util.py +5 -7
- pkgs/type_spec/value_spec/__main__.py +15 -5
- pkgs/type_spec/value_spec/emit_python.py +5 -2
- pkgs/type_spec/value_spec/types.py +1 -1
- uncountable/core/client.py +16 -15
- uncountable/core/file_upload.py +39 -15
- uncountable/integration/construct_client.py +3 -3
- uncountable/integration/executors/generic_upload_executor.py +1 -1
- uncountable/integration/job.py +2 -2
- uncountable/integration/queue_runner/command_server/types.py +1 -1
- uncountable/integration/queue_runner/worker.py +1 -1
- uncountable/integration/server.py +4 -4
- uncountable/integration/telemetry.py +11 -0
- uncountable/types/__init__.py +0 -1
- uncountable/types/api/batch/execute_batch.py +1 -2
- uncountable/types/api/batch/execute_batch_load_async.py +0 -1
- uncountable/types/api/chemical/convert_chemical_formats.py +0 -1
- uncountable/types/api/entity/create_entities.py +3 -4
- uncountable/types/api/entity/create_entity.py +4 -5
- uncountable/types/api/entity/get_entities_data.py +0 -1
- uncountable/types/api/entity/grant_entity_permissions.py +3 -4
- uncountable/types/api/entity/list_entities.py +5 -6
- uncountable/types/api/entity/lock_entity.py +1 -2
- uncountable/types/api/entity/resolve_entity_ids.py +2 -3
- uncountable/types/api/entity/set_entity_field_values.py +0 -1
- uncountable/types/api/entity/set_values.py +0 -1
- uncountable/types/api/entity/transition_entity_phase.py +1 -2
- uncountable/types/api/entity/unlock_entity.py +0 -1
- uncountable/types/api/equipment/associate_equipment_input.py +0 -1
- uncountable/types/api/field_options/upsert_field_options.py +3 -4
- uncountable/types/api/files/download_file.py +1 -2
- uncountable/types/api/id_source/list_id_source.py +3 -4
- uncountable/types/api/id_source/match_id_source.py +1 -2
- uncountable/types/api/input_groups/get_input_group_names.py +0 -1
- uncountable/types/api/inputs/create_inputs.py +4 -5
- uncountable/types/api/inputs/get_input_data.py +5 -6
- uncountable/types/api/inputs/get_input_names.py +3 -4
- uncountable/types/api/inputs/get_inputs_data.py +0 -1
- uncountable/types/api/inputs/set_input_attribute_values.py +2 -3
- uncountable/types/api/inputs/set_input_category.py +2 -3
- uncountable/types/api/inputs/set_input_subcategories.py +0 -1
- uncountable/types/api/inputs/set_intermediate_type.py +1 -2
- uncountable/types/api/material_families/update_entity_material_families.py +1 -2
- uncountable/types/api/outputs/get_output_data.py +6 -7
- uncountable/types/api/outputs/get_output_names.py +2 -3
- uncountable/types/api/outputs/resolve_output_conditions.py +2 -3
- uncountable/types/api/permissions/set_core_permissions.py +3 -4
- uncountable/types/api/project/get_projects.py +3 -4
- uncountable/types/api/project/get_projects_data.py +4 -5
- uncountable/types/api/recipe_links/create_recipe_link.py +1 -2
- uncountable/types/api/recipe_links/remove_recipe_link.py +1 -2
- uncountable/types/api/recipe_metadata/get_recipe_metadata_data.py +3 -4
- uncountable/types/api/recipes/add_recipe_to_project.py +0 -1
- uncountable/types/api/recipes/archive_recipes.py +1 -2
- uncountable/types/api/recipes/associate_recipe_as_input.py +2 -3
- uncountable/types/api/recipes/associate_recipe_as_lot.py +0 -1
- uncountable/types/api/recipes/clear_recipe_outputs.py +0 -1
- uncountable/types/api/recipes/create_recipe.py +6 -7
- uncountable/types/api/recipes/create_recipes.py +4 -5
- uncountable/types/api/recipes/disassociate_recipe_as_input.py +0 -1
- uncountable/types/api/recipes/edit_recipe_inputs.py +9 -10
- uncountable/types/api/recipes/get_column_calculation_values.py +1 -2
- uncountable/types/api/recipes/get_curve.py +2 -3
- uncountable/types/api/recipes/get_recipe_calculations.py +3 -4
- uncountable/types/api/recipes/get_recipe_links.py +1 -2
- uncountable/types/api/recipes/get_recipe_names.py +0 -1
- uncountable/types/api/recipes/get_recipe_output_metadata.py +0 -1
- uncountable/types/api/recipes/get_recipes_data.py +22 -23
- uncountable/types/api/recipes/lock_recipes.py +3 -4
- uncountable/types/api/recipes/remove_recipe_from_project.py +0 -1
- uncountable/types/api/recipes/set_recipe_inputs.py +6 -7
- uncountable/types/api/recipes/set_recipe_metadata.py +0 -1
- uncountable/types/api/recipes/set_recipe_output_annotations.py +5 -6
- uncountable/types/api/recipes/set_recipe_output_file.py +2 -3
- uncountable/types/api/recipes/set_recipe_outputs.py +8 -9
- uncountable/types/api/recipes/set_recipe_tags.py +2 -3
- uncountable/types/api/recipes/unarchive_recipes.py +0 -1
- uncountable/types/api/recipes/unlock_recipes.py +2 -3
- uncountable/types/api/triggers/run_trigger.py +1 -2
- uncountable/types/api/uploader/invoke_uploader.py +2 -3
- uncountable/types/async_batch.py +0 -1
- uncountable/types/async_batch_processor.py +23 -24
- uncountable/types/async_batch_t.py +5 -6
- uncountable/types/async_jobs.py +0 -1
- uncountable/types/async_jobs_t.py +1 -2
- uncountable/types/auth_retrieval.py +0 -1
- uncountable/types/auth_retrieval_t.py +2 -3
- uncountable/types/base.py +0 -1
- uncountable/types/base_t.py +2 -1
- uncountable/types/calculations.py +0 -1
- uncountable/types/calculations_t.py +0 -1
- uncountable/types/chemical_structure.py +0 -1
- uncountable/types/chemical_structure_t.py +3 -4
- uncountable/types/client_base.py +65 -66
- uncountable/types/client_config.py +0 -1
- uncountable/types/client_config_t.py +1 -2
- uncountable/types/curves.py +0 -1
- uncountable/types/curves_t.py +4 -5
- uncountable/types/entity.py +0 -1
- uncountable/types/entity_t.py +3 -4
- uncountable/types/experiment_groups.py +0 -1
- uncountable/types/experiment_groups_t.py +0 -1
- uncountable/types/field_values.py +0 -1
- uncountable/types/field_values_t.py +6 -7
- uncountable/types/fields.py +0 -1
- uncountable/types/fields_t.py +0 -1
- uncountable/types/generic_upload.py +0 -1
- uncountable/types/generic_upload_t.py +7 -8
- uncountable/types/id_source.py +0 -1
- uncountable/types/id_source_t.py +2 -3
- uncountable/types/identifier.py +0 -1
- uncountable/types/identifier_t.py +1 -2
- uncountable/types/input_attributes.py +0 -1
- uncountable/types/input_attributes_t.py +2 -3
- uncountable/types/inputs.py +0 -1
- uncountable/types/inputs_t.py +2 -3
- uncountable/types/integration_server.py +0 -1
- uncountable/types/integration_server_t.py +2 -3
- uncountable/types/job_definition.py +0 -1
- uncountable/types/job_definition_t.py +16 -17
- uncountable/types/outputs.py +0 -1
- uncountable/types/outputs_t.py +1 -2
- uncountable/types/overrides.py +0 -1
- uncountable/types/overrides_t.py +0 -1
- uncountable/types/permissions.py +0 -1
- uncountable/types/permissions_t.py +1 -2
- uncountable/types/phases.py +0 -1
- uncountable/types/phases_t.py +0 -1
- uncountable/types/post_base.py +0 -1
- uncountable/types/post_base_t.py +1 -2
- uncountable/types/queued_job.py +0 -1
- uncountable/types/queued_job_t.py +2 -3
- uncountable/types/recipe_identifiers.py +0 -1
- uncountable/types/recipe_identifiers_t.py +3 -4
- uncountable/types/recipe_inputs.py +0 -1
- uncountable/types/recipe_inputs_t.py +1 -2
- uncountable/types/recipe_links.py +0 -1
- uncountable/types/recipe_links_t.py +2 -3
- uncountable/types/recipe_metadata.py +0 -1
- uncountable/types/recipe_metadata_t.py +6 -7
- uncountable/types/recipe_output_metadata.py +0 -1
- uncountable/types/recipe_output_metadata_t.py +0 -1
- uncountable/types/recipe_tags.py +0 -1
- uncountable/types/recipe_tags_t.py +0 -1
- uncountable/types/recipe_workflow_steps.py +0 -1
- uncountable/types/recipe_workflow_steps_t.py +2 -3
- uncountable/types/recipes.py +0 -1
- uncountable/types/recipes_t.py +0 -1
- uncountable/types/response.py +0 -1
- uncountable/types/response_t.py +0 -1
- uncountable/types/secret_retrieval.py +0 -1
- uncountable/types/secret_retrieval_t.py +3 -4
- uncountable/types/units.py +0 -1
- uncountable/types/units_t.py +0 -1
- uncountable/types/users.py +0 -1
- uncountable/types/users_t.py +0 -1
- uncountable/types/webhook_job.py +0 -1
- uncountable/types/webhook_job_t.py +0 -1
- uncountable/types/workflows.py +0 -1
- uncountable/types/workflows_t.py +1 -2
- UncountablePythonSDK-0.0.91.dist-info/RECORD +0 -301
- {UncountablePythonSDK-0.0.91.dist-info → UncountablePythonSDK-0.0.93.dist-info}/top_level.txt +0 -0
|
@@ -3,7 +3,7 @@ import dataclasses
|
|
|
3
3
|
import decimal
|
|
4
4
|
import io
|
|
5
5
|
import json
|
|
6
|
-
from typing import Any,
|
|
6
|
+
from typing import Any, cast
|
|
7
7
|
|
|
8
8
|
from main.base.types import data_t, type_info_t
|
|
9
9
|
from main.base.types.base_t import PureJsonValue
|
|
@@ -134,7 +134,7 @@ class MapStringEnum(MapTypeBase):
|
|
|
134
134
|
values: dict[str, str]
|
|
135
135
|
|
|
136
136
|
|
|
137
|
-
type MapType =
|
|
137
|
+
type MapType = MapTypeObject | MapTypeAlias | MapStringEnum
|
|
138
138
|
|
|
139
139
|
|
|
140
140
|
@dataclasses.dataclass
|
|
@@ -172,9 +172,9 @@ class InheritablePropertyParts:
|
|
|
172
172
|
at that level, but that needs to be done in builder. When that is done, the
|
|
173
173
|
"label" and "desc" could probably be removed from this list."""
|
|
174
174
|
|
|
175
|
-
label:
|
|
176
|
-
desc:
|
|
177
|
-
ext_info:
|
|
175
|
+
label: str | None = None
|
|
176
|
+
desc: str | None = None
|
|
177
|
+
ext_info: type_info_t.ExtInfo | None = None
|
|
178
178
|
|
|
179
179
|
|
|
180
180
|
def _extract_inheritable_property_parts(
|
|
@@ -269,7 +269,7 @@ def _extract_and_validate_layout(
|
|
|
269
269
|
|
|
270
270
|
def _validate_type_ext_info(
|
|
271
271
|
stype: builder.SpecTypeDefnObject,
|
|
272
|
-
) -> tuple[ExtInfoLayout | None,
|
|
272
|
+
) -> tuple[ExtInfoLayout | None, type_info_t.ExtInfo | None]:
|
|
273
273
|
ext_info = _parse_ext_info(stype.ext_info)
|
|
274
274
|
if ext_info is None:
|
|
275
275
|
return None, None
|
|
@@ -369,7 +369,7 @@ def _build_map_type(
|
|
|
369
369
|
return None
|
|
370
370
|
|
|
371
371
|
|
|
372
|
-
def _parse_ext_info(in_ext: Any) ->
|
|
372
|
+
def _parse_ext_info(in_ext: Any) -> type_info_t.ExtInfo | None:
|
|
373
373
|
if in_ext is None:
|
|
374
374
|
return None
|
|
375
375
|
assert isinstance(in_ext, dict)
|
|
@@ -390,7 +390,7 @@ def _parse_ext_info(in_ext: Any) -> Optional[type_info_t.ExtInfo]:
|
|
|
390
390
|
return ext_info_parser.parse_storage(mod_ext)
|
|
391
391
|
|
|
392
392
|
|
|
393
|
-
def _convert_ext_info(in_ext: Any) ->
|
|
393
|
+
def _convert_ext_info(in_ext: Any) -> PureJsonValue | None:
|
|
394
394
|
# we need to convert this to API storage since it'll be used as-is in the UI
|
|
395
395
|
parsed = _parse_ext_info(in_ext)
|
|
396
396
|
return cast(PureJsonValue, serialize_for_api(parsed))
|
pkgs/type_spec/util.py
CHANGED
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
3
|
from dataclasses import dataclass
|
|
4
|
-
from typing import
|
|
4
|
+
from typing import Union
|
|
5
5
|
|
|
6
6
|
import regex as re
|
|
7
7
|
|
|
8
|
-
T = TypeVar("T")
|
|
9
|
-
|
|
10
8
|
|
|
11
9
|
def rewrite_file(filename: str, content: str) -> bool:
|
|
12
10
|
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
|
@@ -29,8 +27,8 @@ LiteralTypeValue = Union[str, bool]
|
|
|
29
27
|
class ParsedTypePart:
|
|
30
28
|
name: str
|
|
31
29
|
# An empty list is distinct from None
|
|
32
|
-
parameters:
|
|
33
|
-
literal_value:
|
|
30
|
+
parameters: list["ParsedTypePath"] | None = None
|
|
31
|
+
literal_value: LiteralTypeValue | None = None
|
|
34
32
|
|
|
35
33
|
|
|
36
34
|
ParsedTypePath = list[ParsedTypePart]
|
|
@@ -156,7 +154,7 @@ def is_valid_property_name(name: str) -> bool:
|
|
|
156
154
|
return re_pattern_property_name.match(name) is not None
|
|
157
155
|
|
|
158
156
|
|
|
159
|
-
def check_fields(data: dict[str, T], allowed: list[str]) -> None:
|
|
157
|
+
def check_fields[T](data: dict[str, T], allowed: list[str]) -> None:
|
|
160
158
|
for key in data:
|
|
161
159
|
if key not in allowed:
|
|
162
160
|
raise Exception(f"unexpected-field: {key}. Allowed: {allowed}")
|
|
@@ -180,7 +178,7 @@ def encode_common_string(value: str) -> str:
|
|
|
180
178
|
return rep
|
|
181
179
|
|
|
182
180
|
|
|
183
|
-
def unused(_arg: T) -> None:
|
|
181
|
+
def unused[T](_arg: T) -> None:
|
|
184
182
|
"""
|
|
185
183
|
Identifies that an argument is intended not be used, as opposed to
|
|
186
184
|
simply forgotten, or a remnant. This can happen in patterned calls
|
|
@@ -13,7 +13,10 @@ One of the following can be specified on the name of a argument:
|
|
|
13
13
|
After that you can also specify a `!` indicating the argument may not be null.
|
|
14
14
|
If this is not specified, then a null input on this argument should produce a null output.
|
|
15
15
|
We prefer not to use `!` as we want to encourage null pass-through where possible.
|
|
16
|
-
|
|
16
|
+
|
|
17
|
+
If null is allowed as a legitimate value, such as in conditionals like `is_null`,
|
|
18
|
+
then `!usenull` must be specified, this distinguishes it from the pass-through case.
|
|
19
|
+
The accepted argument type must accept "None", it is not implied.
|
|
17
20
|
"""
|
|
18
21
|
|
|
19
22
|
import sys
|
|
@@ -84,7 +87,7 @@ class Source:
|
|
|
84
87
|
return self._text[start : self._at]
|
|
85
88
|
|
|
86
89
|
|
|
87
|
-
_re_argument_name = re.compile(r"([a-z_]+)(\?|\+)?(
|
|
90
|
+
_re_argument_name = re.compile(r"([a-z_]+)(\?|\+)?(!|!usenull)?:")
|
|
88
91
|
|
|
89
92
|
|
|
90
93
|
def parse_function_signature(text: str) -> ParsedFunctionSignature:
|
|
@@ -103,9 +106,16 @@ def parse_function_signature(text: str) -> ParsedFunctionSignature:
|
|
|
103
106
|
ref_name = arg_group.group(1)
|
|
104
107
|
is_missing = arg_group.group(2) == "?"
|
|
105
108
|
is_repeating = arg_group.group(2) == "+"
|
|
106
|
-
pass_null = arg_group.group(3) is None
|
|
107
109
|
type_path = parse_type_str(type_str)
|
|
108
110
|
|
|
111
|
+
match arg_group.group(3):
|
|
112
|
+
case "!":
|
|
113
|
+
on_null = value_spec_t.OnNull.DISALLOW
|
|
114
|
+
case "!usenull":
|
|
115
|
+
on_null = value_spec_t.OnNull.USE
|
|
116
|
+
case _:
|
|
117
|
+
on_null = value_spec_t.OnNull.PASS
|
|
118
|
+
|
|
109
119
|
extant = value_spec_t.ArgumentExtant.REQUIRED
|
|
110
120
|
extant_marker = arg_group.group(2)
|
|
111
121
|
if extant_marker == "?":
|
|
@@ -116,7 +126,7 @@ def parse_function_signature(text: str) -> ParsedFunctionSignature:
|
|
|
116
126
|
arguments.append(
|
|
117
127
|
ParsedFunctionArgument(
|
|
118
128
|
ref_name=ref_name,
|
|
119
|
-
|
|
129
|
+
on_null=on_null,
|
|
120
130
|
extant=extant,
|
|
121
131
|
type_path=type_path,
|
|
122
132
|
)
|
|
@@ -208,7 +218,7 @@ def main() -> None:
|
|
|
208
218
|
name=arg_name,
|
|
209
219
|
description=arg_description,
|
|
210
220
|
type=convert_to_value_spec_type(in_argument.type_path),
|
|
211
|
-
|
|
221
|
+
on_null=in_argument.on_null,
|
|
212
222
|
extant=in_argument.extant,
|
|
213
223
|
)
|
|
214
224
|
)
|
|
@@ -70,7 +70,7 @@ def _emit_function_wrapper(function: value_spec_t.Function) -> str:
|
|
|
70
70
|
else:
|
|
71
71
|
python_type = _emit_python_type(argument.type)
|
|
72
72
|
if (
|
|
73
|
-
argument.
|
|
73
|
+
argument.on_null == value_spec_t.OnNull.PASS
|
|
74
74
|
or argument.extant == value_spec_t.ArgumentExtant.MISSING
|
|
75
75
|
):
|
|
76
76
|
python_type += " | None"
|
|
@@ -189,7 +189,10 @@ def _emit_argument(argument: value_spec_t.FunctionArgument, indent: str) -> str:
|
|
|
189
189
|
out.write(
|
|
190
190
|
f"{sub_indent}description={encode_common_string(argument.description)},\n"
|
|
191
191
|
)
|
|
192
|
-
|
|
192
|
+
# Quick enum emit since we have only one such type here
|
|
193
|
+
out.write(
|
|
194
|
+
f"{sub_indent}on_null=value_spec_t.OnNull.{str(argument.on_null).upper()},\n"
|
|
195
|
+
)
|
|
193
196
|
out.write(
|
|
194
197
|
f"{sub_indent}extant=value_spec_t.ArgumentExtant.{argument.extant.name},\n"
|
|
195
198
|
)
|
uncountable/core/client.py
CHANGED
|
@@ -14,10 +14,9 @@ from opentelemetry.sdk.resources import Attributes
|
|
|
14
14
|
from requests.exceptions import JSONDecodeError
|
|
15
15
|
|
|
16
16
|
from pkgs.argument_parser import CachedParser
|
|
17
|
-
from pkgs.serialization_util import serialize_for_api
|
|
18
|
-
from pkgs.serialization_util.serialization_helpers import JsonValue
|
|
17
|
+
from pkgs.serialization_util import JsonValue, serialize_for_api
|
|
19
18
|
from uncountable.core.environment import get_version
|
|
20
|
-
from uncountable.integration.telemetry import
|
|
19
|
+
from uncountable.integration.telemetry import Logger, push_scope_optional
|
|
21
20
|
from uncountable.types import download_file_t
|
|
22
21
|
from uncountable.types.client_base import APIRequest, ClientMethods
|
|
23
22
|
from uncountable.types.client_config import ClientConfigOptions
|
|
@@ -51,7 +50,7 @@ class HTTPGetRequest(HTTPRequestBase):
|
|
|
51
50
|
@dataclass(kw_only=True)
|
|
52
51
|
class HTTPPostRequest(HTTPRequestBase):
|
|
53
52
|
method = EndpointMethod.POST
|
|
54
|
-
body:
|
|
53
|
+
body: str | dict[str, str]
|
|
55
54
|
|
|
56
55
|
|
|
57
56
|
HTTPRequest = HTTPPostRequest | HTTPGetRequest
|
|
@@ -62,7 +61,7 @@ class ClientConfig(ClientConfigOptions):
|
|
|
62
61
|
transform_request: typing.Callable[[requests.Request], requests.Request] | None = (
|
|
63
62
|
None
|
|
64
63
|
)
|
|
65
|
-
|
|
64
|
+
logger: Logger | None = None
|
|
66
65
|
|
|
67
66
|
|
|
68
67
|
OAUTH_REFRESH_WINDOW_SECONDS = 60 * 5
|
|
@@ -192,7 +191,10 @@ class Client(ClientMethods):
|
|
|
192
191
|
self._session = requests.Session()
|
|
193
192
|
self._session.verify = not self._cfg.allow_insecure_tls
|
|
194
193
|
self._file_uploader = FileUploader(
|
|
195
|
-
self._base_url,
|
|
194
|
+
self._base_url,
|
|
195
|
+
self._auth_details,
|
|
196
|
+
self._cfg.allow_insecure_tls,
|
|
197
|
+
logger=self._cfg.logger,
|
|
196
198
|
)
|
|
197
199
|
|
|
198
200
|
def _get_response_json(
|
|
@@ -241,14 +243,13 @@ class Client(ClientMethods):
|
|
|
241
243
|
case _:
|
|
242
244
|
typing.assert_never(http_request)
|
|
243
245
|
request.headers = http_request.headers
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
else:
|
|
246
|
+
attributes: Attributes = {
|
|
247
|
+
"method": http_request.method,
|
|
248
|
+
"endpoint": api_request.endpoint,
|
|
249
|
+
}
|
|
250
|
+
with push_scope_optional(self._cfg.logger, "api_call", attributes=attributes):
|
|
251
|
+
if self._cfg.logger is not None:
|
|
252
|
+
self._cfg.logger.log_info(api_request.endpoint, attributes=attributes)
|
|
252
253
|
response = self._send_request(request)
|
|
253
254
|
response_data = self._get_response_json(response, request_id=request_id)
|
|
254
255
|
cached_parser = self._get_cached_parser(return_type)
|
|
@@ -326,7 +327,7 @@ class Client(ClientMethods):
|
|
|
326
327
|
case _:
|
|
327
328
|
raise ValueError(f"unsupported request method: {method}")
|
|
328
329
|
|
|
329
|
-
def _get_downloaded_filename(self, *, cd:
|
|
330
|
+
def _get_downloaded_filename(self, *, cd: str | None) -> str:
|
|
330
331
|
if not cd:
|
|
331
332
|
return "Unknown"
|
|
332
333
|
|
uncountable/core/file_upload.py
CHANGED
|
@@ -4,11 +4,13 @@ from dataclasses import dataclass
|
|
|
4
4
|
from enum import StrEnum
|
|
5
5
|
from io import BytesIO
|
|
6
6
|
from pathlib import Path
|
|
7
|
-
from typing import Generator, Literal, Self
|
|
7
|
+
from typing import Generator, Literal, Self, assert_never
|
|
8
8
|
|
|
9
9
|
import aiohttp
|
|
10
10
|
import aiotus
|
|
11
11
|
|
|
12
|
+
from uncountable.integration.telemetry import Logger, push_scope_optional
|
|
13
|
+
|
|
12
14
|
from .types import AuthDetailsAll, AuthDetailsApiKey
|
|
13
15
|
|
|
14
16
|
_CHUNK_SIZE = 5 * 1024 * 1024 # s3 requires 5MiB minimum
|
|
@@ -75,10 +77,12 @@ class FileUploader:
|
|
|
75
77
|
base_url: str,
|
|
76
78
|
auth_details: AuthDetailsAll,
|
|
77
79
|
allow_insecure_tls: bool = False,
|
|
80
|
+
logger: Logger | None = None,
|
|
78
81
|
) -> None:
|
|
79
82
|
self._base_url = base_url
|
|
80
83
|
self._auth_details = auth_details
|
|
81
84
|
self._allow_insecure_tls = allow_insecure_tls
|
|
85
|
+
self._logger = logger
|
|
82
86
|
|
|
83
87
|
async def _upload_file(self: Self, file_upload: FileUpload) -> UploadedFile:
|
|
84
88
|
creation_url = f"{self._base_url}/api/external/file_upload/files"
|
|
@@ -93,20 +97,40 @@ class FileUploader:
|
|
|
93
97
|
auth=auth, headers={"Origin": self._base_url}
|
|
94
98
|
) as session,
|
|
95
99
|
):
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
)
|
|
100
|
+
attributes = {}
|
|
101
|
+
match file_upload:
|
|
102
|
+
case MediaFileUpload():
|
|
103
|
+
attributes["file_path"] = file_upload.path
|
|
104
|
+
case DataFileUpload():
|
|
105
|
+
attributes["file_name"] = file_upload.name
|
|
106
|
+
case _:
|
|
107
|
+
assert_never(file_upload)
|
|
108
|
+
with push_scope_optional(
|
|
109
|
+
self._logger, "upload_file", attributes=attributes
|
|
110
|
+
):
|
|
111
|
+
if self._logger is not None:
|
|
112
|
+
self._logger.log_info("Uploading file", attributes=attributes)
|
|
113
|
+
with file_upload_data(file_upload) as file_bytes:
|
|
114
|
+
if file_bytes.bytes_data.read(1) == b"":
|
|
115
|
+
raise UploadFailed(
|
|
116
|
+
f"Failed to upload empty file: {file_bytes.name}"
|
|
117
|
+
)
|
|
118
|
+
file_bytes.bytes_data.seek(0)
|
|
119
|
+
location = await aiotus.upload(
|
|
120
|
+
creation_url,
|
|
121
|
+
file_bytes.bytes_data,
|
|
122
|
+
{"filename": file_bytes.name.encode()},
|
|
123
|
+
client_session=session,
|
|
124
|
+
config=aiotus.RetryConfiguration(
|
|
125
|
+
ssl=not self._allow_insecure_tls
|
|
126
|
+
),
|
|
127
|
+
chunksize=_CHUNK_SIZE,
|
|
128
|
+
)
|
|
129
|
+
if location is None:
|
|
130
|
+
raise UploadFailed(f"Failed to upload: {file_bytes.name}")
|
|
131
|
+
return UploadedFile(
|
|
132
|
+
name=file_bytes.name, file_id=int(location.path.split("/")[-1])
|
|
133
|
+
)
|
|
110
134
|
|
|
111
135
|
def upload_files(
|
|
112
136
|
self: Self, *, file_uploads: list[FileUpload]
|
|
@@ -37,15 +37,15 @@ def _construct_client_config(
|
|
|
37
37
|
return ClientConfig(
|
|
38
38
|
allow_insecure_tls=profile_meta.client_options.allow_insecure_tls,
|
|
39
39
|
extra_headers=profile_meta.client_options.extra_headers,
|
|
40
|
-
|
|
40
|
+
logger=job_logger,
|
|
41
41
|
)
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
def construct_uncountable_client(
|
|
45
|
-
profile_meta: ProfileMetadata,
|
|
45
|
+
profile_meta: ProfileMetadata, logger: JobLogger
|
|
46
46
|
) -> Client:
|
|
47
47
|
return Client(
|
|
48
48
|
base_url=profile_meta.base_url,
|
|
49
49
|
auth_details=_construct_auth_details(profile_meta),
|
|
50
|
-
config=_construct_client_config(profile_meta,
|
|
50
|
+
config=_construct_client_config(profile_meta, logger),
|
|
51
51
|
)
|
|
@@ -10,12 +10,12 @@ from pkgs.filesystem_utils import (
|
|
|
10
10
|
FileSystemFileReference,
|
|
11
11
|
FileSystemObject,
|
|
12
12
|
FileSystemS3Config,
|
|
13
|
+
FileSystemSession,
|
|
13
14
|
FileSystemSFTPConfig,
|
|
14
15
|
FileTransfer,
|
|
15
16
|
S3Session,
|
|
16
17
|
SFTPSession,
|
|
17
18
|
)
|
|
18
|
-
from pkgs.filesystem_utils.filesystem_session import FileSystemSession
|
|
19
19
|
from uncountable.core.file_upload import DataFileUpload, FileUpload
|
|
20
20
|
from uncountable.integration.job import Job, JobArguments
|
|
21
21
|
from uncountable.integration.secret_retrieval import retrieve_secret
|
uncountable/integration/job.py
CHANGED
|
@@ -28,7 +28,7 @@ CronJobArguments = JobArguments
|
|
|
28
28
|
PT = typing.TypeVar("PT")
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
class Job
|
|
31
|
+
class Job[PT](ABC):
|
|
32
32
|
_unc_job_registered: bool = False
|
|
33
33
|
|
|
34
34
|
@property
|
|
@@ -62,7 +62,7 @@ class CronJob(Job):
|
|
|
62
62
|
WPT = typing.TypeVar("WPT")
|
|
63
63
|
|
|
64
64
|
|
|
65
|
-
class WebhookJob(Job[webhook_job_t.WebhookEventPayload]
|
|
65
|
+
class WebhookJob[WPT](Job[webhook_job_t.WebhookEventPayload]):
|
|
66
66
|
@property
|
|
67
67
|
def payload_type(self) -> type[webhook_job_t.WebhookEventPayload]:
|
|
68
68
|
return webhook_job_t.WebhookEventPayload
|
|
@@ -90,7 +90,7 @@ def run_queued_job(
|
|
|
90
90
|
)
|
|
91
91
|
try:
|
|
92
92
|
client = construct_uncountable_client(
|
|
93
|
-
profile_meta=job_details.profile_metadata,
|
|
93
|
+
profile_meta=job_details.profile_metadata, logger=job_logger
|
|
94
94
|
)
|
|
95
95
|
batch_processor = AsyncBatchProcessor(client=client)
|
|
96
96
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import signal
|
|
2
2
|
from dataclasses import asdict
|
|
3
3
|
from types import TracebackType
|
|
4
|
-
from typing import
|
|
4
|
+
from typing import assert_never
|
|
5
5
|
|
|
6
6
|
from apscheduler.executors.pool import ThreadPoolExecutor
|
|
7
7
|
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
|
@@ -110,8 +110,8 @@ class IntegrationServer:
|
|
|
110
110
|
|
|
111
111
|
def __exit__(
|
|
112
112
|
self,
|
|
113
|
-
exc_type:
|
|
114
|
-
exc_val:
|
|
115
|
-
exc_tb:
|
|
113
|
+
exc_type: type[BaseException] | None,
|
|
114
|
+
exc_val: BaseException | None,
|
|
115
|
+
exc_tb: TracebackType | None,
|
|
116
116
|
) -> None:
|
|
117
117
|
self._stop_apscheduler()
|
|
@@ -196,3 +196,14 @@ class JobLogger(Logger):
|
|
|
196
196
|
case _:
|
|
197
197
|
assert_never(self.job_definition.executor)
|
|
198
198
|
return _cast_attributes(patched_attributes)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@contextmanager
|
|
202
|
+
def push_scope_optional(
|
|
203
|
+
logger: Logger | None, scope_name: str, *, attributes: Attributes | None = None
|
|
204
|
+
) -> Generator[None, None, None]:
|
|
205
|
+
if logger is None:
|
|
206
|
+
yield
|
|
207
|
+
else:
|
|
208
|
+
with logger.push_scope(scope_name, attributes=attributes):
|
|
209
|
+
yield
|
uncountable/types/__init__.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
2
|
-
# flake8: noqa: F821
|
|
3
2
|
# ruff: noqa: E402 Q003
|
|
4
3
|
# fmt: off
|
|
5
4
|
# isort: skip_file
|
|
@@ -7,7 +6,7 @@ from __future__ import annotations
|
|
|
7
6
|
import typing # noqa: F401
|
|
8
7
|
import datetime # noqa: F401
|
|
9
8
|
from decimal import Decimal # noqa: F401
|
|
10
|
-
from
|
|
9
|
+
from enum import StrEnum
|
|
11
10
|
import dataclasses
|
|
12
11
|
from pkgs.serialization import serial_class
|
|
13
12
|
from ... import base_t
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
2
|
-
# flake8: noqa: F821
|
|
3
2
|
# ruff: noqa: E402 Q003
|
|
4
3
|
# fmt: off
|
|
5
4
|
# isort: skip_file
|
|
@@ -32,7 +31,7 @@ ENDPOINT_PATH = "api/external/entity/external_create_entities"
|
|
|
32
31
|
)
|
|
33
32
|
@dataclasses.dataclass(kw_only=True)
|
|
34
33
|
class EntityToCreate:
|
|
35
|
-
field_values:
|
|
34
|
+
field_values: list[field_values_t.FieldRefNameValue] | None = None
|
|
36
35
|
|
|
37
36
|
|
|
38
37
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -43,8 +42,8 @@ class EntityToCreate:
|
|
|
43
42
|
class Arguments:
|
|
44
43
|
entity_type: entity_t.LimitedEntityType
|
|
45
44
|
entities_to_create: list[EntityToCreate]
|
|
46
|
-
definition_id:
|
|
47
|
-
definition_key:
|
|
45
|
+
definition_id: base_t.ObjectId | None = None
|
|
46
|
+
definition_key: identifier_t.IdentifierKey | None = None
|
|
48
47
|
|
|
49
48
|
|
|
50
49
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
2
|
-
# flake8: noqa: F821
|
|
3
2
|
# ruff: noqa: E402 Q003
|
|
4
3
|
# fmt: off
|
|
5
4
|
# isort: skip_file
|
|
@@ -35,7 +34,7 @@ ENDPOINT_PATH = "api/external/entity/external_create_entity"
|
|
|
35
34
|
class EntityFieldInitialValue:
|
|
36
35
|
field_ref_name: str
|
|
37
36
|
value: base_t.JsonValue
|
|
38
|
-
row_index:
|
|
37
|
+
row_index: int | None = None
|
|
39
38
|
|
|
40
39
|
|
|
41
40
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -45,9 +44,9 @@ class EntityFieldInitialValue:
|
|
|
45
44
|
@dataclasses.dataclass(kw_only=True)
|
|
46
45
|
class Arguments:
|
|
47
46
|
entity_type: entity_t.LimitedEntityType
|
|
48
|
-
definition_id:
|
|
49
|
-
definition_key:
|
|
50
|
-
field_values:
|
|
47
|
+
definition_id: base_t.ObjectId | None = None
|
|
48
|
+
definition_key: identifier_t.IdentifierKey | None = None
|
|
49
|
+
field_values: list[field_values_t.FieldRefNameValue] | None = None
|
|
51
50
|
|
|
52
51
|
|
|
53
52
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
2
|
-
# flake8: noqa: F821
|
|
3
2
|
# ruff: noqa: E402 Q003
|
|
4
3
|
# fmt: off
|
|
5
4
|
# isort: skip_file
|
|
@@ -33,9 +32,9 @@ class Arguments:
|
|
|
33
32
|
entity_type: entity_t.LimitedEntityType
|
|
34
33
|
entity_key: identifier_t.IdentifierKey
|
|
35
34
|
permission_types: list[entity_t.GrantableEntityPermissionType]
|
|
36
|
-
user_keys:
|
|
37
|
-
user_group_keys:
|
|
38
|
-
all_users:
|
|
35
|
+
user_keys: list[identifier_t.IdentifierKey] | None = None
|
|
36
|
+
user_group_keys: list[identifier_t.IdentifierKey] | None = None
|
|
37
|
+
all_users: bool | None = None
|
|
39
38
|
|
|
40
39
|
|
|
41
40
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
2
|
-
# flake8: noqa: F821
|
|
3
2
|
# ruff: noqa: E402 Q003
|
|
4
3
|
# fmt: off
|
|
5
4
|
# isort: skip_file
|
|
@@ -34,10 +33,10 @@ ENDPOINT_PATH = "api/external/entity/external_list_entities"
|
|
|
34
33
|
@dataclasses.dataclass(kw_only=True)
|
|
35
34
|
class Arguments:
|
|
36
35
|
config_reference: str
|
|
37
|
-
entity_type:
|
|
38
|
-
attributes:
|
|
39
|
-
offset:
|
|
40
|
-
limit:
|
|
36
|
+
entity_type: entity_t.EntityType | None = None
|
|
37
|
+
attributes: dict[OpaqueKey, base_t.JsonValue] | None = None
|
|
38
|
+
offset: int | None = None
|
|
39
|
+
limit: int | None = None
|
|
41
40
|
|
|
42
41
|
|
|
43
42
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -58,7 +57,7 @@ class EntityResult:
|
|
|
58
57
|
@dataclasses.dataclass(kw_only=True)
|
|
59
58
|
class ColumnAccess:
|
|
60
59
|
name: str
|
|
61
|
-
table_label:
|
|
60
|
+
table_label: str | None
|
|
62
61
|
|
|
63
62
|
|
|
64
63
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
2
|
-
# flake8: noqa: F821
|
|
3
2
|
# ruff: noqa: E402 Q003
|
|
4
3
|
# fmt: off
|
|
5
4
|
# isort: skip_file
|
|
@@ -32,7 +31,7 @@ ENDPOINT_PATH = "api/external/entity/external_lock_entity"
|
|
|
32
31
|
class Arguments:
|
|
33
32
|
entity_key: identifier_t.IdentifierKey
|
|
34
33
|
entity_type: entity_t.EntityType
|
|
35
|
-
globally_removable:
|
|
34
|
+
globally_removable: bool | None = None
|
|
36
35
|
|
|
37
36
|
|
|
38
37
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
2
|
-
# flake8: noqa: F821
|
|
3
2
|
# ruff: noqa: E402 Q003
|
|
4
3
|
# fmt: off
|
|
5
4
|
# isort: skip_file
|
|
@@ -30,7 +29,7 @@ ENDPOINT_PATH = "api/external/entity/external_resolve_entity_ids"
|
|
|
30
29
|
)
|
|
31
30
|
@dataclasses.dataclass(kw_only=True)
|
|
32
31
|
class Arguments:
|
|
33
|
-
entity_ids: list[
|
|
32
|
+
entity_ids: list[str | base_t.ObjectId]
|
|
34
33
|
entity_type: entity_t.EntityType
|
|
35
34
|
|
|
36
35
|
|
|
@@ -40,7 +39,7 @@ class Arguments:
|
|
|
40
39
|
)
|
|
41
40
|
@dataclasses.dataclass(kw_only=True)
|
|
42
41
|
class EntityNames:
|
|
43
|
-
id:
|
|
42
|
+
id: str | int
|
|
44
43
|
name: str
|
|
45
44
|
|
|
46
45
|
|