UncountablePythonSDK 0.0.168__py3-none-any.whl → 0.0.170__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- examples/integration-server/jobs/materials_auto/example_instrument.py +1 -1
- examples/integration-server/pyproject.toml +2 -2
- pkgs/serialization_util/serialization_helpers.py +1 -1
- pkgs/type_spec/emit_python.py +2 -9
- pkgs/type_spec/parts/base.py.prepart +0 -4
- pkgs/type_spec/type_info/emit_type_info.py +14 -5
- pkgs/type_spec/value_spec/__main__.py +2 -6
- pkgs/type_spec/value_spec/emit_python.py +1 -1
- uncountable/integration/queue_runner/command_server/command_client.py +3 -3
- uncountable/integration/telemetry.py +40 -45
- uncountable/types/api/batch/execute_batch.py +4 -4
- uncountable/types/api/batch/execute_batch_load_async.py +2 -2
- uncountable/types/api/chemical/convert_chemical_formats.py +3 -3
- uncountable/types/api/condition_parameters/upsert_condition_match.py +3 -4
- uncountable/types/api/condition_parameters/upsert_condition_matches.py +8 -9
- uncountable/types/api/entity/create_entities.py +3 -3
- uncountable/types/api/entity/create_entity.py +3 -3
- uncountable/types/api/entity/create_or_update_entity.py +2 -3
- uncountable/types/api/entity/export_entities.py +3 -3
- uncountable/types/api/entity/get_entities_data.py +3 -3
- uncountable/types/api/entity/grant_entity_permissions.py +2 -3
- uncountable/types/api/entity/list_aggregate.py +5 -5
- uncountable/types/api/entity/list_entities.py +6 -6
- uncountable/types/api/entity/lock_entity.py +2 -3
- uncountable/types/api/entity/lookup_entity.py +6 -6
- uncountable/types/api/entity/resolve_entity_ids.py +3 -3
- uncountable/types/api/entity/set_barcode.py +2 -3
- uncountable/types/api/entity/set_entity_field_values.py +2 -3
- uncountable/types/api/entity/set_values.py +2 -3
- uncountable/types/api/entity/transition_entity_phase.py +4 -5
- uncountable/types/api/entity/unlock_entity.py +2 -3
- uncountable/types/api/equipment/associate_equipment_input.py +2 -2
- uncountable/types/api/field_options/upsert_field_options.py +3 -4
- uncountable/types/api/file_folders/add_file_to_folder.py +2 -2
- uncountable/types/api/file_folders/modify_file_system.py +3 -4
- uncountable/types/api/files/download_file.py +4 -4
- uncountable/types/api/id_source/list_id_source.py +3 -3
- uncountable/types/api/id_source/match_id_source.py +3 -3
- uncountable/types/api/input_groups/get_input_group_names.py +3 -3
- uncountable/types/api/inputs/create_inputs.py +6 -5
- uncountable/types/api/inputs/get_input_data.py +6 -6
- uncountable/types/api/inputs/get_input_names.py +3 -3
- uncountable/types/api/inputs/get_inputs_data.py +6 -6
- uncountable/types/api/inputs/set_input_attribute_values.py +3 -3
- uncountable/types/api/inputs/set_input_category.py +2 -3
- uncountable/types/api/inputs/set_input_subcategories.py +2 -3
- uncountable/types/api/inputs/set_intermediate_type.py +2 -3
- uncountable/types/api/integrations/publish_realtime_data.py +3 -4
- uncountable/types/api/integrations/push_notification.py +2 -3
- uncountable/types/api/integrations/register_sockets_token.py +2 -3
- uncountable/types/api/listing/export_listing.py +2 -3
- uncountable/types/api/listing/fetch_listing.py +3 -3
- uncountable/types/api/material_families/update_entity_material_families.py +2 -2
- uncountable/types/api/notebooks/add_notebook_content.py +20 -20
- uncountable/types/api/notebooks/get_notebook_content.py +14 -15
- uncountable/types/api/output_parameters/swap_output_condition_parameters.py +7 -8
- uncountable/types/api/outputs/get_output_data.py +6 -6
- uncountable/types/api/outputs/get_output_names.py +3 -3
- uncountable/types/api/outputs/get_output_organization.py +9 -9
- uncountable/types/api/outputs/resolve_output_conditions.py +5 -5
- uncountable/types/api/outputs/update_output_condition_parameter.py +2 -3
- uncountable/types/api/permissions/set_core_permissions.py +6 -7
- uncountable/types/api/permissions/set_entity_permission.py +4 -5
- uncountable/types/api/project/get_projects.py +3 -3
- uncountable/types/api/project/get_projects_data.py +3 -3
- uncountable/types/api/recipe_links/create_recipe_link.py +2 -3
- uncountable/types/api/recipe_links/remove_recipe_link.py +2 -3
- uncountable/types/api/recipe_metadata/get_recipe_metadata_data.py +3 -3
- uncountable/types/api/recipes/add_recipe_to_project.py +2 -3
- uncountable/types/api/recipes/add_time_series_data.py +3 -4
- uncountable/types/api/recipes/archive_recipes.py +2 -3
- uncountable/types/api/recipes/associate_recipe_as_input.py +2 -3
- uncountable/types/api/recipes/associate_recipe_as_lot.py +2 -3
- uncountable/types/api/recipes/clear_recipe_outputs.py +2 -3
- uncountable/types/api/recipes/create_mix_order.py +2 -3
- uncountable/types/api/recipes/create_recipe.py +2 -2
- uncountable/types/api/recipes/create_recipes.py +4 -4
- uncountable/types/api/recipes/disassociate_recipe_as_input.py +2 -3
- uncountable/types/api/recipes/edit_recipe_inputs.py +20 -21
- uncountable/types/api/recipes/get_column_calculation_values.py +3 -3
- uncountable/types/api/recipes/get_curve.py +2 -2
- uncountable/types/api/recipes/get_recipe_calculations.py +3 -3
- uncountable/types/api/recipes/get_recipe_links.py +2 -2
- uncountable/types/api/recipes/get_recipe_names.py +3 -3
- uncountable/types/api/recipes/get_recipe_output_metadata.py +3 -3
- uncountable/types/api/recipes/get_recipes_data.py +13 -13
- uncountable/types/api/recipes/lock_recipes.py +3 -4
- uncountable/types/api/recipes/remove_recipe_from_project.py +2 -3
- uncountable/types/api/recipes/set_recipe_inputs.py +3 -3
- uncountable/types/api/recipes/set_recipe_metadata.py +2 -3
- uncountable/types/api/recipes/set_recipe_output_annotations.py +6 -6
- uncountable/types/api/recipes/set_recipe_output_file.py +3 -3
- uncountable/types/api/recipes/set_recipe_outputs.py +4 -4
- uncountable/types/api/recipes/set_recipe_tags.py +6 -6
- uncountable/types/api/recipes/set_recipe_total.py +3 -4
- uncountable/types/api/recipes/unarchive_recipes.py +2 -3
- uncountable/types/api/recipes/unlock_recipes.py +2 -3
- uncountable/types/api/recipes/upsert_recipe_workflow_step.py +2 -3
- uncountable/types/api/recipes/upsert_step_relationships.py +4 -5
- uncountable/types/api/runsheet/complete_async_upload.py +2 -2
- uncountable/types/api/runsheet/export_default_runsheet.py +2 -2
- uncountable/types/api/triggers/run_trigger.py +2 -3
- uncountable/types/api/uploader/complete_async_parse.py +2 -2
- uncountable/types/api/uploader/invoke_uploader.py +2 -2
- uncountable/types/api/user/get_current_user_info.py +2 -2
- uncountable/types/async_batch_t.py +5 -5
- uncountable/types/async_jobs_t.py +4 -0
- uncountable/types/auth_retrieval_t.py +3 -4
- uncountable/types/base_t.py +0 -4
- uncountable/types/calculations_t.py +1 -1
- uncountable/types/chemical_structure_t.py +1 -2
- uncountable/types/client_config_t.py +4 -4
- uncountable/types/curves_t.py +2 -2
- uncountable/types/data_t.py +4 -5
- uncountable/types/entity_t.py +2 -2
- uncountable/types/experiment_groups_t.py +1 -1
- uncountable/types/exports_t.py +1 -2
- uncountable/types/field_values_t.py +20 -20
- uncountable/types/fields_t.py +1 -1
- uncountable/types/generic_upload_t.py +6 -7
- uncountable/types/id_source_t.py +4 -5
- uncountable/types/identifier_t.py +3 -3
- uncountable/types/input_attributes_t.py +1 -1
- uncountable/types/inputs_t.py +1 -1
- uncountable/types/integration_server_t.py +1 -2
- uncountable/types/integration_session_t.py +2 -3
- uncountable/types/integrations_t.py +2 -3
- uncountable/types/job_definition_t.py +15 -16
- uncountable/types/listing_t.py +29 -29
- uncountable/types/notices_t.py +1 -2
- uncountable/types/notifications_t.py +3 -4
- uncountable/types/outputs_t.py +1 -1
- uncountable/types/overrides_t.py +2 -3
- uncountable/types/phases_t.py +1 -1
- uncountable/types/queued_job_t.py +8 -8
- uncountable/types/recipe_identifiers_t.py +3 -3
- uncountable/types/recipe_links_t.py +1 -1
- uncountable/types/recipe_metadata_t.py +3 -3
- uncountable/types/recipe_output_metadata_t.py +1 -1
- uncountable/types/recipe_tags_t.py +1 -1
- uncountable/types/recipe_workflow_step_types_t.py +5 -6
- uncountable/types/recipe_workflow_steps_t.py +4 -5
- uncountable/types/recipes_t.py +1 -2
- uncountable/types/response_t.py +1 -2
- uncountable/types/runsheet_format_t.py +3 -4
- uncountable/types/runsheet_t.py +10 -11
- uncountable/types/secret_retrieval_t.py +3 -4
- uncountable/types/sockets_t.py +8 -8
- uncountable/types/step_relationships_t.py +7 -8
- uncountable/types/units_t.py +1 -1
- uncountable/types/uploader_t.py +12 -13
- uncountable/types/users_t.py +1 -1
- uncountable/types/webhook_job_t.py +3 -3
- uncountable/types/workflows_t.py +2 -2
- {uncountablepythonsdk-0.0.168.dist-info → uncountablepythonsdk-0.0.170.dist-info}/METADATA +1 -1
- {uncountablepythonsdk-0.0.168.dist-info → uncountablepythonsdk-0.0.170.dist-info}/RECORD +158 -158
- {uncountablepythonsdk-0.0.168.dist-info → uncountablepythonsdk-0.0.170.dist-info}/WHEEL +0 -0
- {uncountablepythonsdk-0.0.168.dist-info → uncountablepythonsdk-0.0.170.dist-info}/top_level.txt +0 -0
|
@@ -29,7 +29,7 @@ class InstrumentExample(WebhookJob[InstrumentPayload]):
|
|
|
29
29
|
self, args: JobArguments, payload: InstrumentPayload
|
|
30
30
|
) -> job_definition_t.JobResult:
|
|
31
31
|
parser: CachedParser[sockets_t.SocketResponse] = CachedParser(
|
|
32
|
-
sockets_t.SocketResponse
|
|
32
|
+
sockets_t.SocketResponse
|
|
33
33
|
)
|
|
34
34
|
|
|
35
35
|
def parse_message(message: Data) -> sockets_t.SocketEventData | None:
|
|
@@ -8,8 +8,8 @@ dependencies = [
|
|
|
8
8
|
"mypy == 1.*",
|
|
9
9
|
"ruff == 0.*",
|
|
10
10
|
"openpyxl == 3.*",
|
|
11
|
-
"more_itertools ==
|
|
12
|
-
"types-paramiko ==4.0.0.
|
|
11
|
+
"more_itertools ==11.*",
|
|
12
|
+
"types-paramiko ==4.0.0.20260402",
|
|
13
13
|
"types-openpyxl == 3.*",
|
|
14
14
|
"types-pysftp == 0.*",
|
|
15
15
|
"types-pytz ==2026.*",
|
|
@@ -63,7 +63,7 @@ def convert_decimals_to_strings(obj: JsonValue) -> JsonValue:
|
|
|
63
63
|
|
|
64
64
|
|
|
65
65
|
class Dataclass(Protocol):
|
|
66
|
-
__dataclass_fields__: ClassVar[dict] # type: ignore[type-arg
|
|
66
|
+
__dataclass_fields__: ClassVar[dict] # type: ignore[type-arg]
|
|
67
67
|
|
|
68
68
|
|
|
69
69
|
def identity(x: T) -> T:
|
pkgs/type_spec/emit_python.py
CHANGED
|
@@ -9,7 +9,6 @@ from .builder import (
|
|
|
9
9
|
EndpointEmitType,
|
|
10
10
|
EndpointSpecificPath,
|
|
11
11
|
PropertyExtant,
|
|
12
|
-
base_namespace_name,
|
|
13
12
|
)
|
|
14
13
|
from .config import PythonConfig
|
|
15
14
|
from .cross_output_links import get_path_links
|
|
@@ -1016,19 +1015,13 @@ def _emit_type(ctx: Context, stype: builder.SpecType) -> None:
|
|
|
1016
1015
|
|
|
1017
1016
|
# Emit dataclass decorator
|
|
1018
1017
|
dataclass = "@dataclasses.dataclass"
|
|
1019
|
-
|
|
1020
|
-
ctx,
|
|
1021
|
-
builder.SpecTypeDefnAlias(
|
|
1022
|
-
namespace=ctx.builder.namespaces[base_namespace_name], name="ENABLE_SLOTS"
|
|
1023
|
-
),
|
|
1024
|
-
)
|
|
1025
|
-
dc_args = ["slots=base_t.ENABLE_SLOTS"]
|
|
1018
|
+
dc_args = ["slots=True"]
|
|
1026
1019
|
if stype.is_kw_only():
|
|
1027
1020
|
dc_args.append("kw_only=True")
|
|
1028
1021
|
if stype.is_hashable:
|
|
1029
1022
|
dc_args.extend(["frozen=True", "eq=True"])
|
|
1030
1023
|
if len(dc_args) > 0:
|
|
1031
|
-
dataclass += f"({', '.join(dc_args)})
|
|
1024
|
+
dataclass += f"({', '.join(dc_args)})"
|
|
1032
1025
|
|
|
1033
1026
|
ctx.out.write(f"{dataclass}\n")
|
|
1034
1027
|
ctx.out.write(class_out.getvalue())
|
|
@@ -30,10 +30,6 @@ REF_NAME_STRICT_REGEX = rf"{REF_NAME_STRICT_REGEX_STRING}"
|
|
|
30
30
|
ID_REGEX = r"-?[1-9][0-9]{0,20}"
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
# ENABLE_SLOTS should be removed after slots have been tested locally
|
|
34
|
-
import os
|
|
35
|
-
ENABLE_SLOTS = os.environ.get("UNC_ENABLE_DATACLASS_SLOTS") == "true"
|
|
36
|
-
|
|
37
33
|
if TYPE_CHECKING:
|
|
38
34
|
JsonValue = Union[JsonScalar, Mapping[str, "JsonValue"], Sequence["JsonValue"]]
|
|
39
35
|
ExtJsonValue = JsonValue
|
|
@@ -30,6 +30,7 @@ def type_path_of(stype: builder.SpecType) -> object: # NamePath
|
|
|
30
30
|
- Scoped Type: [ (namespace-string)..., type-string ]
|
|
31
31
|
- Instance Type: [ "$instance", Scoped-Type-Base, [TypePath-Parameters...] ]
|
|
32
32
|
- Literal Type: [ "$literal", [ "$value", value, value-type-string ]... ]
|
|
33
|
+
- Parameter Type: [ "$param", parameter-name-string ]
|
|
33
34
|
|
|
34
35
|
@return (string-specific, multiple-types)
|
|
35
36
|
"""
|
|
@@ -71,6 +72,9 @@ def type_path_of(stype: builder.SpecType) -> object: # NamePath
|
|
|
71
72
|
[type_path_of(parameter) for parameter in stype.parameters],
|
|
72
73
|
]
|
|
73
74
|
|
|
75
|
+
if isinstance(stype, builder.SpecTypeGenericParameter):
|
|
76
|
+
return ["$param", stype.name]
|
|
77
|
+
|
|
74
78
|
raise Exception("unhandled-SpecType")
|
|
75
79
|
|
|
76
80
|
|
|
@@ -177,12 +181,14 @@ class MapTypeBase:
|
|
|
177
181
|
class MapTypeObject(MapTypeBase):
|
|
178
182
|
base_type_path: object
|
|
179
183
|
properties: dict[OpaqueKey, MapProperty]
|
|
184
|
+
parameters: list[str] | None = None
|
|
180
185
|
|
|
181
186
|
|
|
182
187
|
@dataclasses.dataclass
|
|
183
188
|
class MapTypeAlias(MapTypeBase):
|
|
184
189
|
alias_type_path: object
|
|
185
190
|
discriminator: str | None
|
|
191
|
+
parameters: list[str] | None = None
|
|
186
192
|
|
|
187
193
|
|
|
188
194
|
@dataclasses.dataclass
|
|
@@ -387,23 +393,28 @@ def _validate_type_ext_info(
|
|
|
387
393
|
def _build_map_type(
|
|
388
394
|
build: builder.SpecBuilder, stype: builder.SpecTypeDefn, python: bool = False
|
|
389
395
|
) -> MapType | None:
|
|
390
|
-
# limited support for now
|
|
391
396
|
if (
|
|
392
397
|
isinstance(stype, builder.SpecTypeDefnObject)
|
|
393
|
-
and len(stype.parameters) == 0
|
|
394
398
|
and not stype.is_base
|
|
395
399
|
and stype.base is not None
|
|
396
400
|
):
|
|
397
401
|
_, ext_info = _validate_type_ext_info(stype)
|
|
398
402
|
|
|
399
403
|
properties: dict[OpaqueKey, MapProperty] = {}
|
|
404
|
+
params = stype.parameters if len(stype.parameters) > 0 else None
|
|
405
|
+
base_for_path = (
|
|
406
|
+
stype.base_type_instance
|
|
407
|
+
if stype.base_type_instance is not None
|
|
408
|
+
else stype.base
|
|
409
|
+
)
|
|
400
410
|
map_type = MapTypeObject(
|
|
401
411
|
type_name=stype.name,
|
|
402
412
|
label=stype.label,
|
|
403
413
|
properties=properties,
|
|
404
414
|
desc=stype.desc,
|
|
405
|
-
base_type_path=type_path_of(
|
|
415
|
+
base_type_path=type_path_of(base_for_path),
|
|
406
416
|
ext_info=ext_info,
|
|
417
|
+
parameters=params,
|
|
407
418
|
)
|
|
408
419
|
|
|
409
420
|
if stype.properties is not None:
|
|
@@ -441,8 +452,6 @@ def _build_map_type(
|
|
|
441
452
|
)
|
|
442
453
|
|
|
443
454
|
if isinstance(stype, builder.SpecTypeDefnUnion):
|
|
444
|
-
# Emit as a basic alias for now, as the front-end supports only those for now
|
|
445
|
-
# IMPROVE: We should emit a proper union type and support that
|
|
446
455
|
backing = stype.get_backing_type()
|
|
447
456
|
ext_info = _parse_ext_info(stype.ext_info)
|
|
448
457
|
return MapTypeAlias(
|
|
@@ -14,8 +14,6 @@ After that you can also specify a `!` indicating the argument may not be null.
|
|
|
14
14
|
If this is not specified, then a null input on this argument should produce a null output.
|
|
15
15
|
We prefer not to use `!` as we want to encourage null pass-through where possible.
|
|
16
16
|
|
|
17
|
-
If null is allowed as a legitimate value, such as in conditionals like `is_null`,
|
|
18
|
-
then `!usenull` must be specified, this distinguishes it from the pass-through case.
|
|
19
17
|
The accepted argument type must accept "None", it is not implied.
|
|
20
18
|
"""
|
|
21
19
|
|
|
@@ -87,7 +85,7 @@ class Source:
|
|
|
87
85
|
return self._text[start : self._at]
|
|
88
86
|
|
|
89
87
|
|
|
90
|
-
_re_argument_name = re.compile(r"([a-z_]+)(\?|\+)?(
|
|
88
|
+
_re_argument_name = re.compile(r"([a-z_]+)(\?|\+)?(!)?:")
|
|
91
89
|
|
|
92
90
|
|
|
93
91
|
def parse_function_signature(text: str) -> ParsedFunctionSignature:
|
|
@@ -111,10 +109,8 @@ def parse_function_signature(text: str) -> ParsedFunctionSignature:
|
|
|
111
109
|
match arg_group.group(3):
|
|
112
110
|
case "!":
|
|
113
111
|
on_null = value_spec_t.OnNull.DISALLOW
|
|
114
|
-
case "!usenull":
|
|
115
|
-
on_null = value_spec_t.OnNull.USE
|
|
116
112
|
case _:
|
|
117
|
-
on_null = value_spec_t.OnNull.
|
|
113
|
+
on_null = value_spec_t.OnNull.USE
|
|
118
114
|
|
|
119
115
|
extant = value_spec_t.ArgumentExtant.REQUIRED
|
|
120
116
|
extant_marker = arg_group.group(2)
|
|
@@ -70,7 +70,7 @@ def _emit_function_wrapper(function: value_spec_t.Function) -> str:
|
|
|
70
70
|
else:
|
|
71
71
|
python_type = _emit_python_type(argument.type)
|
|
72
72
|
if (
|
|
73
|
-
argument.on_null == value_spec_t.OnNull.
|
|
73
|
+
argument.on_null == value_spec_t.OnNull.USE
|
|
74
74
|
or argument.extant == value_spec_t.ArgumentExtant.MISSING
|
|
75
75
|
):
|
|
76
76
|
python_type += " | None"
|
|
@@ -105,7 +105,7 @@ def send_retry_job_message(
|
|
|
105
105
|
|
|
106
106
|
return response.queued_job_uuid
|
|
107
107
|
except grpc.RpcError as e:
|
|
108
|
-
raise ValueError(e.details())
|
|
108
|
+
raise ValueError(e.details())
|
|
109
109
|
|
|
110
110
|
|
|
111
111
|
def check_health(*, host: str = _LOCAL_RPC_HOST, port: int) -> bool:
|
|
@@ -152,7 +152,7 @@ def send_list_queued_jobs_message(
|
|
|
152
152
|
request, timeout=_DEFAULT_MESSAGE_TIMEOUT_SECS
|
|
153
153
|
)
|
|
154
154
|
except grpc.RpcError as e:
|
|
155
|
-
raise ValueError(e.details())
|
|
155
|
+
raise ValueError(e.details())
|
|
156
156
|
|
|
157
157
|
assert isinstance(response, ListQueuedJobsResult)
|
|
158
158
|
return list(response.queued_jobs)
|
|
@@ -167,7 +167,7 @@ def send_vaccuum_queued_jobs_message(*, host: str = "localhost", port: int) -> N
|
|
|
167
167
|
request, timeout=_DEFAULT_MESSAGE_TIMEOUT_SECS
|
|
168
168
|
)
|
|
169
169
|
except grpc.RpcError as e:
|
|
170
|
-
raise ValueError(e.details())
|
|
170
|
+
raise ValueError(e.details())
|
|
171
171
|
|
|
172
172
|
assert isinstance(response, VaccuumQueuedJobsResult)
|
|
173
173
|
return None
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import dataclasses
|
|
1
2
|
import datetime
|
|
2
3
|
import functools
|
|
3
4
|
import json
|
|
@@ -30,6 +31,7 @@ from opentelemetry.sdk.trace.export import (
|
|
|
30
31
|
)
|
|
31
32
|
from opentelemetry.trace import Span, Tracer
|
|
32
33
|
|
|
34
|
+
from pkgs.serialization_util import dict_fields
|
|
33
35
|
from uncountable.core.environment import (
|
|
34
36
|
get_otel_enabled,
|
|
35
37
|
get_server_env,
|
|
@@ -229,60 +231,75 @@ class Logger:
|
|
|
229
231
|
yield self
|
|
230
232
|
|
|
231
233
|
|
|
234
|
+
@dataclasses.dataclass(frozen=True, kw_only=True, slots=True)
|
|
235
|
+
class _JobResourceStats:
|
|
236
|
+
cpu_user_s: float
|
|
237
|
+
cpu_system_s: float
|
|
238
|
+
cpu_total_s: float
|
|
239
|
+
wall_time_s: float
|
|
240
|
+
current_rss_mb: float
|
|
241
|
+
peak_rss_mb: float
|
|
242
|
+
|
|
243
|
+
|
|
232
244
|
class PerJobResourceTracker:
|
|
233
245
|
def __init__(
|
|
234
246
|
self,
|
|
235
247
|
logger: "JobLogger",
|
|
248
|
+
*,
|
|
236
249
|
sample_interval: float = 0.5,
|
|
237
250
|
log_interval: float = 30.0,
|
|
251
|
+
bootup_delay: float = 3.0,
|
|
238
252
|
) -> None:
|
|
239
253
|
self.logger = logger
|
|
240
254
|
self.sample_interval = sample_interval
|
|
241
255
|
self.log_interval = log_interval
|
|
256
|
+
self.bootup_delay = bootup_delay
|
|
242
257
|
self._process = psutil.Process(os.getpid())
|
|
243
258
|
self._stop_event = threading.Event()
|
|
244
259
|
self._thread: threading.Thread | None = None
|
|
245
260
|
|
|
246
261
|
self.max_rss: int = 0
|
|
247
262
|
self.start_cpu_times: psutil._ntp.pcputimes | None = None
|
|
248
|
-
self.end_cpu_times: psutil._ntp.pcputimes | None = None
|
|
249
263
|
self.start_wall_time: float | None = None
|
|
250
|
-
self.end_wall_time: float | None = None
|
|
251
264
|
|
|
252
|
-
def _current_stats(self) ->
|
|
265
|
+
def _current_stats(self) -> _JobResourceStats:
|
|
253
266
|
assert self.start_cpu_times is not None
|
|
254
267
|
assert self.start_wall_time is not None
|
|
255
268
|
current_cpu_times = self._process.cpu_times()
|
|
256
269
|
cpu_user = current_cpu_times.user - self.start_cpu_times.user
|
|
257
270
|
cpu_sys = current_cpu_times.system - self.start_cpu_times.system
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
271
|
+
return _JobResourceStats(
|
|
272
|
+
cpu_user_s=round(cpu_user, 3),
|
|
273
|
+
cpu_system_s=round(cpu_sys, 3),
|
|
274
|
+
cpu_total_s=round(cpu_user + cpu_sys, 3),
|
|
275
|
+
wall_time_s=round(time.monotonic() - self.start_wall_time, 3),
|
|
276
|
+
current_rss_mb=round(self._process.memory_info().rss / (1024 * 1024), 2),
|
|
277
|
+
peak_rss_mb=round(self.max_rss / (1024 * 1024), 2),
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
def _log_stats(self, message: str) -> None:
|
|
281
|
+
self.logger.log_debug(
|
|
282
|
+
message, attributes=_cast_attributes(dict_fields(self._current_stats()))
|
|
283
|
+
)
|
|
268
284
|
|
|
269
285
|
def start(self) -> None:
|
|
270
286
|
self.start_cpu_times = self._process.cpu_times()
|
|
271
287
|
self.start_wall_time = time.monotonic()
|
|
272
288
|
|
|
273
289
|
def _monitor() -> None:
|
|
274
|
-
|
|
290
|
+
now = time.monotonic()
|
|
291
|
+
boot_at: float | None = now + self.bootup_delay
|
|
292
|
+
next_periodic_at = now + self.log_interval
|
|
275
293
|
try:
|
|
276
294
|
while not self._stop_event.is_set():
|
|
277
|
-
|
|
278
|
-
self.max_rss = max(self.max_rss, rss)
|
|
295
|
+
self.max_rss = max(self.max_rss, self._process.memory_info().rss)
|
|
279
296
|
now = time.monotonic()
|
|
280
|
-
if
|
|
281
|
-
self.
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
)
|
|
285
|
-
|
|
297
|
+
if boot_at is not None and now >= boot_at:
|
|
298
|
+
self._log_stats("Job resource usage (bootup)")
|
|
299
|
+
boot_at = None
|
|
300
|
+
if now >= next_periodic_at:
|
|
301
|
+
self._log_stats("Job resource usage (periodic)")
|
|
302
|
+
next_periodic_at = now + self.log_interval
|
|
286
303
|
time.sleep(self.sample_interval)
|
|
287
304
|
except Exception:
|
|
288
305
|
self._stop_event.set()
|
|
@@ -294,8 +311,6 @@ class PerJobResourceTracker:
|
|
|
294
311
|
self._stop_event.set()
|
|
295
312
|
if self._thread is not None:
|
|
296
313
|
self._thread.join()
|
|
297
|
-
self.end_cpu_times = self._process.cpu_times()
|
|
298
|
-
self.end_wall_time = time.monotonic()
|
|
299
314
|
|
|
300
315
|
def __enter__(self) -> typing.Self:
|
|
301
316
|
self.start()
|
|
@@ -308,7 +323,6 @@ class PerJobResourceTracker:
|
|
|
308
323
|
traceback: types.TracebackType | None,
|
|
309
324
|
) -> None:
|
|
310
325
|
self.stop()
|
|
311
|
-
stats = dict(self.summary())
|
|
312
326
|
if isinstance(exc_value, MemoryError):
|
|
313
327
|
if resource is not None:
|
|
314
328
|
limit, _ = resource.getrlimit(resource.RLIMIT_AS)
|
|
@@ -320,26 +334,7 @@ class PerJobResourceTracker:
|
|
|
320
334
|
),
|
|
321
335
|
)
|
|
322
336
|
return
|
|
323
|
-
self.
|
|
324
|
-
return
|
|
325
|
-
|
|
326
|
-
def summary(self) -> Attributes:
|
|
327
|
-
assert self.start_cpu_times is not None
|
|
328
|
-
assert self.end_cpu_times is not None
|
|
329
|
-
assert self.start_wall_time is not None
|
|
330
|
-
assert self.end_wall_time is not None
|
|
331
|
-
|
|
332
|
-
cpu_user = self.end_cpu_times.user - self.start_cpu_times.user
|
|
333
|
-
cpu_sys = self.end_cpu_times.system - self.start_cpu_times.system
|
|
334
|
-
cpu_total = cpu_user + cpu_sys
|
|
335
|
-
elapsed = self.end_wall_time - self.start_wall_time
|
|
336
|
-
return {
|
|
337
|
-
"cpu_user_s": round(cpu_user, 3),
|
|
338
|
-
"cpu_system_s": round(cpu_sys, 3),
|
|
339
|
-
"cpu_total_s": round(cpu_total, 3),
|
|
340
|
-
"wall_time_s": round(elapsed, 3),
|
|
341
|
-
"peak_rss_mb": round(self.max_rss / (1024 * 1024), 2),
|
|
342
|
-
}
|
|
337
|
+
self._log_stats("Job resource usage summary")
|
|
343
338
|
|
|
344
339
|
|
|
345
340
|
class JobLogger(Logger):
|
|
@@ -43,7 +43,7 @@ class RequestMethod(StrEnum):
|
|
|
43
43
|
named_type_path="sdk.api.batch.execute_batch.BatchRequest",
|
|
44
44
|
unconverted_values={"data"},
|
|
45
45
|
)
|
|
46
|
-
@dataclasses.dataclass(slots=
|
|
46
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
47
47
|
class BatchRequest:
|
|
48
48
|
path: str
|
|
49
49
|
method: RequestMethod
|
|
@@ -54,7 +54,7 @@ class BatchRequest:
|
|
|
54
54
|
@serial_class(
|
|
55
55
|
named_type_path="sdk.api.batch.execute_batch.Arguments",
|
|
56
56
|
)
|
|
57
|
-
@dataclasses.dataclass(slots=
|
|
57
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
58
58
|
class Arguments:
|
|
59
59
|
requests: list[BatchRequest]
|
|
60
60
|
|
|
@@ -64,7 +64,7 @@ class Arguments:
|
|
|
64
64
|
named_type_path="sdk.api.batch.execute_batch.BatchResponse",
|
|
65
65
|
unconverted_values={"response"},
|
|
66
66
|
)
|
|
67
|
-
@dataclasses.dataclass(slots=
|
|
67
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
68
68
|
class BatchResponse:
|
|
69
69
|
path: str
|
|
70
70
|
method: RequestMethod
|
|
@@ -76,7 +76,7 @@ class BatchResponse:
|
|
|
76
76
|
@serial_class(
|
|
77
77
|
named_type_path="sdk.api.batch.execute_batch.Data",
|
|
78
78
|
)
|
|
79
|
-
@dataclasses.dataclass(slots=
|
|
79
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
80
80
|
class Data:
|
|
81
81
|
responses: list[BatchResponse]
|
|
82
82
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -26,7 +26,7 @@ ENDPOINT_PATH = "api/external/batch/execute_batch_load_async"
|
|
|
26
26
|
@serial_class(
|
|
27
27
|
named_type_path="sdk.api.batch.execute_batch_load_async.Arguments",
|
|
28
28
|
)
|
|
29
|
-
@dataclasses.dataclass(slots=
|
|
29
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
30
30
|
class Arguments:
|
|
31
31
|
requests: list[async_batch_t.AsyncBatchRequest]
|
|
32
32
|
|
|
@@ -35,7 +35,7 @@ class Arguments:
|
|
|
35
35
|
@serial_class(
|
|
36
36
|
named_type_path="sdk.api.batch.execute_batch_load_async.Data",
|
|
37
37
|
)
|
|
38
|
-
@dataclasses.dataclass(slots=
|
|
38
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
39
39
|
class Data:
|
|
40
40
|
job_id: base_t.ObjectId
|
|
41
41
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -28,7 +28,7 @@ ENDPOINT_PATH = "api/external/chemical/convert_chemical_formats"
|
|
|
28
28
|
@serial_class(
|
|
29
29
|
named_type_path="sdk.api.chemical.convert_chemical_formats.ChemicalStructureFile",
|
|
30
30
|
)
|
|
31
|
-
@dataclasses.dataclass(slots=
|
|
31
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
32
32
|
class ChemicalStructureFile:
|
|
33
33
|
struct_file: str
|
|
34
34
|
|
|
@@ -37,7 +37,7 @@ class ChemicalStructureFile:
|
|
|
37
37
|
@serial_class(
|
|
38
38
|
named_type_path="sdk.api.chemical.convert_chemical_formats.Arguments",
|
|
39
39
|
)
|
|
40
|
-
@dataclasses.dataclass(slots=
|
|
40
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
41
41
|
class Arguments:
|
|
42
42
|
source_chemical_structures: list[ChemicalStructureFile]
|
|
43
43
|
|
|
@@ -56,7 +56,7 @@ UncountableChemicalStructure = typing.Annotated[
|
|
|
56
56
|
named_type_path="sdk.api.chemical.convert_chemical_formats.Data",
|
|
57
57
|
unconverted_values={"chemical_structures"},
|
|
58
58
|
)
|
|
59
|
-
@dataclasses.dataclass(slots=
|
|
59
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
60
60
|
class Data:
|
|
61
61
|
chemical_structures: list[UncountableChemicalStructure]
|
|
62
62
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -10,7 +10,6 @@ import dataclasses
|
|
|
10
10
|
from pkgs.serialization import serial_class
|
|
11
11
|
from pkgs.serialization import serial_alias_annotation
|
|
12
12
|
from ... import async_batch_t
|
|
13
|
-
from ... import base_t
|
|
14
13
|
from ... import condition_match_t
|
|
15
14
|
from ... import identifier_t
|
|
16
15
|
|
|
@@ -32,7 +31,7 @@ ENDPOINT_PATH = "api/external/condition_parameters/upsert_condition_match"
|
|
|
32
31
|
named_type_path="sdk.api.condition_parameters.upsert_condition_match.ConditionParameter",
|
|
33
32
|
to_string_values={"max_value", "min_value"},
|
|
34
33
|
)
|
|
35
|
-
@dataclasses.dataclass(slots=
|
|
34
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
36
35
|
class ConditionParameter:
|
|
37
36
|
condition_parameter_key: identifier_t.IdentifierKey
|
|
38
37
|
allow_null: bool
|
|
@@ -55,7 +54,7 @@ MatchType = typing.Annotated[
|
|
|
55
54
|
@serial_class(
|
|
56
55
|
named_type_path="sdk.api.condition_parameters.upsert_condition_match.Arguments",
|
|
57
56
|
)
|
|
58
|
-
@dataclasses.dataclass(slots=
|
|
57
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
59
58
|
class Arguments:
|
|
60
59
|
explicit_name: bool = False
|
|
61
60
|
name: str | None = None
|
|
@@ -68,7 +67,7 @@ class Arguments:
|
|
|
68
67
|
@serial_class(
|
|
69
68
|
named_type_path="sdk.api.condition_parameters.upsert_condition_match.Data",
|
|
70
69
|
)
|
|
71
|
-
@dataclasses.dataclass(slots=
|
|
70
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
72
71
|
class Data(async_batch_t.AsyncBatchActionReturn):
|
|
73
72
|
pass
|
|
74
73
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -10,7 +10,6 @@ import dataclasses
|
|
|
10
10
|
from pkgs.serialization import serial_class
|
|
11
11
|
from pkgs.serialization import serial_union_annotation
|
|
12
12
|
from ... import async_batch_t
|
|
13
|
-
from ... import base_t
|
|
14
13
|
from ... import condition_match_t
|
|
15
14
|
from ... import identifier_t
|
|
16
15
|
|
|
@@ -38,7 +37,7 @@ ENDPOINT_PATH = "api/external/condition_parameters/upsert_condition_matches"
|
|
|
38
37
|
named_type_path="sdk.api.condition_parameters.upsert_condition_matches.ConditionParameterAll",
|
|
39
38
|
parse_require={"match_type"},
|
|
40
39
|
)
|
|
41
|
-
@dataclasses.dataclass(slots=
|
|
40
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
42
41
|
class ConditionParameterAll:
|
|
43
42
|
condition_parameter_key: identifier_t.IdentifierKey
|
|
44
43
|
allow_null: bool
|
|
@@ -50,7 +49,7 @@ class ConditionParameterAll:
|
|
|
50
49
|
named_type_path="sdk.api.condition_parameters.upsert_condition_matches.ConditionParameterEqNumeric",
|
|
51
50
|
parse_require={"match_type"},
|
|
52
51
|
)
|
|
53
|
-
@dataclasses.dataclass(slots=
|
|
52
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
54
53
|
class ConditionParameterEqNumeric:
|
|
55
54
|
condition_parameter_key: identifier_t.IdentifierKey
|
|
56
55
|
allow_null: bool
|
|
@@ -63,7 +62,7 @@ class ConditionParameterEqNumeric:
|
|
|
63
62
|
named_type_path="sdk.api.condition_parameters.upsert_condition_matches.ConditionParameterEqCategorical",
|
|
64
63
|
parse_require={"match_type"},
|
|
65
64
|
)
|
|
66
|
-
@dataclasses.dataclass(slots=
|
|
65
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
67
66
|
class ConditionParameterEqCategorical:
|
|
68
67
|
condition_parameter_key: identifier_t.IdentifierKey
|
|
69
68
|
allow_null: bool
|
|
@@ -77,7 +76,7 @@ class ConditionParameterEqCategorical:
|
|
|
77
76
|
to_string_values={"max_value", "min_value"},
|
|
78
77
|
parse_require={"match_type"},
|
|
79
78
|
)
|
|
80
|
-
@dataclasses.dataclass(slots=
|
|
79
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
81
80
|
class ConditionParameterRange:
|
|
82
81
|
condition_parameter_key: identifier_t.IdentifierKey
|
|
83
82
|
allow_null: bool
|
|
@@ -107,7 +106,7 @@ ConditionParameter = typing.Annotated[
|
|
|
107
106
|
named_type_path="sdk.api.condition_parameters.upsert_condition_matches.UpsertArgsParameters",
|
|
108
107
|
parse_require={"kind"},
|
|
109
108
|
)
|
|
110
|
-
@dataclasses.dataclass(slots=
|
|
109
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
111
110
|
class UpsertArgsParameters:
|
|
112
111
|
condition_parameters: list[ConditionParameter]
|
|
113
112
|
kind: typing.Literal["condition_parameters"] = "condition_parameters"
|
|
@@ -120,7 +119,7 @@ class UpsertArgsParameters:
|
|
|
120
119
|
named_type_path="sdk.api.condition_parameters.upsert_condition_matches.UpsertArgsOutputConditions",
|
|
121
120
|
parse_require={"kind"},
|
|
122
121
|
)
|
|
123
|
-
@dataclasses.dataclass(slots=
|
|
122
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
124
123
|
class UpsertArgsOutputConditions:
|
|
125
124
|
output_condition_keys: list[identifier_t.IdentifierKey]
|
|
126
125
|
kind: typing.Literal["output_conditions"] = "output_conditions"
|
|
@@ -146,7 +145,7 @@ UpsertArgs = typing.Annotated[
|
|
|
146
145
|
@serial_class(
|
|
147
146
|
named_type_path="sdk.api.condition_parameters.upsert_condition_matches.Arguments",
|
|
148
147
|
)
|
|
149
|
-
@dataclasses.dataclass(slots=
|
|
148
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
150
149
|
class Arguments:
|
|
151
150
|
upserts: list[UpsertArgs]
|
|
152
151
|
|
|
@@ -155,7 +154,7 @@ class Arguments:
|
|
|
155
154
|
@serial_class(
|
|
156
155
|
named_type_path="sdk.api.condition_parameters.upsert_condition_matches.Data",
|
|
157
156
|
)
|
|
158
|
-
@dataclasses.dataclass(slots=
|
|
157
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
159
158
|
class Data:
|
|
160
159
|
results: list[async_batch_t.AsyncBatchActionReturn]
|
|
161
160
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -29,7 +29,7 @@ ENDPOINT_PATH = "api/external/entity/external_create_entities"
|
|
|
29
29
|
@serial_class(
|
|
30
30
|
named_type_path="sdk.api.entity.create_entities.EntityToCreate",
|
|
31
31
|
)
|
|
32
|
-
@dataclasses.dataclass(slots=
|
|
32
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
33
33
|
class EntityToCreate:
|
|
34
34
|
field_values: list[field_values_t.FieldRefNameValue] | None = None
|
|
35
35
|
parent_entity: entity_t.Entity | None = None
|
|
@@ -39,7 +39,7 @@ class EntityToCreate:
|
|
|
39
39
|
@serial_class(
|
|
40
40
|
named_type_path="sdk.api.entity.create_entities.Arguments",
|
|
41
41
|
)
|
|
42
|
-
@dataclasses.dataclass(slots=
|
|
42
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
43
43
|
class Arguments:
|
|
44
44
|
entity_type: entity_t.LimitedEntityType
|
|
45
45
|
entities_to_create: list[EntityToCreate]
|
|
@@ -51,7 +51,7 @@ class Arguments:
|
|
|
51
51
|
@serial_class(
|
|
52
52
|
named_type_path="sdk.api.entity.create_entities.Data",
|
|
53
53
|
)
|
|
54
|
-
@dataclasses.dataclass(slots=
|
|
54
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
55
55
|
class Data:
|
|
56
56
|
entities: list[entity_t.Entity]
|
|
57
57
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -30,7 +30,7 @@ ENDPOINT_PATH = "api/external/entity/external_create_entity"
|
|
|
30
30
|
named_type_path="sdk.api.entity.create_entity.EntityFieldInitialValue",
|
|
31
31
|
unconverted_values={"value"},
|
|
32
32
|
)
|
|
33
|
-
@dataclasses.dataclass(slots=
|
|
33
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
34
34
|
class EntityFieldInitialValue:
|
|
35
35
|
field_ref_name: str
|
|
36
36
|
value: base_t.JsonValue
|
|
@@ -41,7 +41,7 @@ class EntityFieldInitialValue:
|
|
|
41
41
|
@serial_class(
|
|
42
42
|
named_type_path="sdk.api.entity.create_entity.Arguments",
|
|
43
43
|
)
|
|
44
|
-
@dataclasses.dataclass(slots=
|
|
44
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
45
45
|
class Arguments:
|
|
46
46
|
entity_type: entity_t.LimitedEntityType
|
|
47
47
|
definition_id: base_t.ObjectId | None = None
|
|
@@ -54,7 +54,7 @@ class Arguments:
|
|
|
54
54
|
@serial_class(
|
|
55
55
|
named_type_path="sdk.api.entity.create_entity.Data",
|
|
56
56
|
)
|
|
57
|
-
@dataclasses.dataclass(slots=
|
|
57
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
58
58
|
class Data:
|
|
59
59
|
entity: entity_t.Entity
|
|
60
60
|
# DO NOT MODIFY -- This file is generated by type_spec
|
|
@@ -9,7 +9,6 @@ from decimal import Decimal # noqa: F401
|
|
|
9
9
|
import dataclasses
|
|
10
10
|
from pkgs.serialization import serial_class
|
|
11
11
|
from ... import async_batch_t
|
|
12
|
-
from ... import base_t
|
|
13
12
|
from ... import entity_t
|
|
14
13
|
from ... import field_values_t
|
|
15
14
|
from ... import identifier_t
|
|
@@ -29,7 +28,7 @@ ENDPOINT_PATH = "api/external/entity/create_or_update_entity"
|
|
|
29
28
|
@serial_class(
|
|
30
29
|
named_type_path="sdk.api.entity.create_or_update_entity.Arguments",
|
|
31
30
|
)
|
|
32
|
-
@dataclasses.dataclass(slots=
|
|
31
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
33
32
|
class Arguments:
|
|
34
33
|
entity_type: entity_t.EntityType
|
|
35
34
|
definition_key: identifier_t.IdentifierKey
|
|
@@ -42,7 +41,7 @@ class Arguments:
|
|
|
42
41
|
@serial_class(
|
|
43
42
|
named_type_path="sdk.api.entity.create_or_update_entity.Data",
|
|
44
43
|
)
|
|
45
|
-
@dataclasses.dataclass(slots=
|
|
44
|
+
@dataclasses.dataclass(slots=True, kw_only=True)
|
|
46
45
|
class Data(async_batch_t.AsyncBatchActionReturn):
|
|
47
46
|
pass
|
|
48
47
|
# DO NOT MODIFY -- This file is generated by type_spec
|