flwr-nightly 1.8.0.dev20240315__py3-none-any.whl → 1.15.0.dev20250114__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- flwr/cli/app.py +16 -2
- flwr/cli/build.py +181 -0
- flwr/cli/cli_user_auth_interceptor.py +90 -0
- flwr/cli/config_utils.py +343 -0
- flwr/cli/example.py +4 -1
- flwr/cli/install.py +253 -0
- flwr/cli/log.py +182 -0
- flwr/{server/superlink/state → cli/login}/__init__.py +4 -10
- flwr/cli/login/login.py +88 -0
- flwr/cli/ls.py +327 -0
- flwr/cli/new/__init__.py +1 -0
- flwr/cli/new/new.py +210 -66
- flwr/cli/new/templates/app/.gitignore.tpl +163 -0
- flwr/cli/new/templates/app/LICENSE.tpl +202 -0
- flwr/cli/new/templates/app/README.baseline.md.tpl +127 -0
- flwr/cli/new/templates/app/README.flowertune.md.tpl +66 -0
- flwr/cli/new/templates/app/README.md.tpl +16 -32
- flwr/cli/new/templates/app/code/__init__.baseline.py.tpl +1 -0
- flwr/cli/new/templates/app/code/__init__.py.tpl +1 -1
- flwr/cli/new/templates/app/code/client.baseline.py.tpl +58 -0
- flwr/cli/new/templates/app/code/client.huggingface.py.tpl +55 -0
- flwr/cli/new/templates/app/code/client.jax.py.tpl +50 -0
- flwr/cli/new/templates/app/code/client.mlx.py.tpl +73 -0
- flwr/cli/new/templates/app/code/client.numpy.py.tpl +7 -7
- flwr/cli/new/templates/app/code/client.pytorch.py.tpl +30 -21
- flwr/cli/new/templates/app/code/client.sklearn.py.tpl +63 -0
- flwr/cli/new/templates/app/code/client.tensorflow.py.tpl +57 -1
- flwr/cli/new/templates/app/code/dataset.baseline.py.tpl +36 -0
- flwr/cli/new/templates/app/code/flwr_tune/__init__.py +15 -0
- flwr/cli/new/templates/app/code/flwr_tune/client_app.py.tpl +126 -0
- flwr/cli/new/templates/app/code/flwr_tune/dataset.py.tpl +87 -0
- flwr/cli/new/templates/app/code/flwr_tune/models.py.tpl +78 -0
- flwr/cli/new/templates/app/code/flwr_tune/server_app.py.tpl +94 -0
- flwr/cli/new/templates/app/code/flwr_tune/strategy.py.tpl +83 -0
- flwr/cli/new/templates/app/code/model.baseline.py.tpl +80 -0
- flwr/cli/new/templates/app/code/server.baseline.py.tpl +46 -0
- flwr/cli/new/templates/app/code/server.huggingface.py.tpl +38 -0
- flwr/cli/new/templates/app/code/server.jax.py.tpl +26 -0
- flwr/cli/new/templates/app/code/server.mlx.py.tpl +31 -0
- flwr/cli/new/templates/app/code/server.numpy.py.tpl +22 -9
- flwr/cli/new/templates/app/code/server.pytorch.py.tpl +21 -18
- flwr/cli/new/templates/app/code/server.sklearn.py.tpl +36 -0
- flwr/cli/new/templates/app/code/server.tensorflow.py.tpl +29 -1
- flwr/cli/new/templates/app/code/strategy.baseline.py.tpl +1 -0
- flwr/cli/new/templates/app/code/task.huggingface.py.tpl +102 -0
- flwr/cli/new/templates/app/code/task.jax.py.tpl +57 -0
- flwr/cli/new/templates/app/code/task.mlx.py.tpl +102 -0
- flwr/cli/new/templates/app/code/task.numpy.py.tpl +7 -0
- flwr/cli/new/templates/app/code/task.pytorch.py.tpl +29 -24
- flwr/cli/new/templates/app/code/task.sklearn.py.tpl +67 -0
- flwr/cli/new/templates/app/code/task.tensorflow.py.tpl +53 -0
- flwr/cli/new/templates/app/code/utils.baseline.py.tpl +1 -0
- flwr/cli/new/templates/app/pyproject.baseline.toml.tpl +138 -0
- flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +68 -0
- flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl +46 -0
- flwr/cli/new/templates/app/pyproject.jax.toml.tpl +35 -0
- flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +39 -0
- flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +25 -12
- flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +29 -14
- flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +35 -0
- flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +29 -14
- flwr/cli/run/__init__.py +1 -0
- flwr/cli/run/run.py +212 -34
- flwr/cli/stop.py +130 -0
- flwr/cli/utils.py +240 -5
- flwr/client/__init__.py +3 -2
- flwr/client/app.py +432 -255
- flwr/client/client.py +1 -11
- flwr/client/client_app.py +74 -13
- flwr/client/clientapp/__init__.py +22 -0
- flwr/client/clientapp/app.py +259 -0
- flwr/client/clientapp/clientappio_servicer.py +244 -0
- flwr/client/clientapp/utils.py +115 -0
- flwr/client/dpfedavg_numpy_client.py +7 -8
- flwr/client/grpc_adapter_client/__init__.py +15 -0
- flwr/client/grpc_adapter_client/connection.py +98 -0
- flwr/client/grpc_client/connection.py +21 -7
- flwr/client/grpc_rere_client/__init__.py +1 -1
- flwr/client/grpc_rere_client/client_interceptor.py +176 -0
- flwr/client/grpc_rere_client/connection.py +163 -56
- flwr/client/grpc_rere_client/grpc_adapter.py +167 -0
- flwr/client/heartbeat.py +74 -0
- flwr/client/message_handler/__init__.py +1 -1
- flwr/client/message_handler/message_handler.py +10 -11
- flwr/client/mod/__init__.py +5 -5
- flwr/client/mod/centraldp_mods.py +4 -2
- flwr/client/mod/comms_mods.py +5 -4
- flwr/client/mod/localdp_mod.py +10 -5
- flwr/client/mod/secure_aggregation/__init__.py +1 -1
- flwr/client/mod/secure_aggregation/secaggplus_mod.py +26 -26
- flwr/client/mod/utils.py +2 -4
- flwr/client/nodestate/__init__.py +26 -0
- flwr/client/nodestate/in_memory_nodestate.py +38 -0
- flwr/client/nodestate/nodestate.py +31 -0
- flwr/client/nodestate/nodestate_factory.py +38 -0
- flwr/client/numpy_client.py +8 -31
- flwr/client/rest_client/__init__.py +1 -1
- flwr/client/rest_client/connection.py +199 -176
- flwr/client/run_info_store.py +112 -0
- flwr/client/supernode/__init__.py +24 -0
- flwr/client/supernode/app.py +321 -0
- flwr/client/typing.py +1 -0
- flwr/common/__init__.py +17 -11
- flwr/common/address.py +47 -3
- flwr/common/args.py +153 -0
- flwr/common/auth_plugin/__init__.py +24 -0
- flwr/common/auth_plugin/auth_plugin.py +121 -0
- flwr/common/config.py +243 -0
- flwr/common/constant.py +132 -1
- flwr/common/context.py +32 -2
- flwr/common/date.py +22 -4
- flwr/common/differential_privacy.py +2 -2
- flwr/common/dp.py +2 -4
- flwr/common/exit_handlers.py +3 -3
- flwr/common/grpc.py +164 -5
- flwr/common/logger.py +230 -12
- flwr/common/message.py +191 -106
- flwr/common/object_ref.py +179 -44
- flwr/common/pyproject.py +1 -0
- flwr/common/record/__init__.py +2 -1
- flwr/common/record/configsrecord.py +58 -18
- flwr/common/record/metricsrecord.py +57 -17
- flwr/common/record/parametersrecord.py +88 -20
- flwr/common/record/recordset.py +153 -30
- flwr/common/record/typeddict.py +30 -55
- flwr/common/recordset_compat.py +31 -12
- flwr/common/retry_invoker.py +123 -30
- flwr/common/secure_aggregation/__init__.py +1 -1
- flwr/common/secure_aggregation/crypto/__init__.py +1 -1
- flwr/common/secure_aggregation/crypto/shamir.py +11 -11
- flwr/common/secure_aggregation/crypto/symmetric_encryption.py +68 -4
- flwr/common/secure_aggregation/ndarrays_arithmetic.py +17 -17
- flwr/common/secure_aggregation/quantization.py +8 -8
- flwr/common/secure_aggregation/secaggplus_constants.py +1 -1
- flwr/common/secure_aggregation/secaggplus_utils.py +10 -12
- flwr/common/serde.py +298 -19
- flwr/common/telemetry.py +65 -29
- flwr/common/typing.py +120 -19
- flwr/common/version.py +17 -3
- flwr/proto/clientappio_pb2.py +45 -0
- flwr/proto/clientappio_pb2.pyi +132 -0
- flwr/proto/clientappio_pb2_grpc.py +135 -0
- flwr/proto/clientappio_pb2_grpc.pyi +53 -0
- flwr/proto/exec_pb2.py +62 -0
- flwr/proto/exec_pb2.pyi +212 -0
- flwr/proto/exec_pb2_grpc.py +237 -0
- flwr/proto/exec_pb2_grpc.pyi +93 -0
- flwr/proto/fab_pb2.py +31 -0
- flwr/proto/fab_pb2.pyi +65 -0
- flwr/proto/fab_pb2_grpc.py +4 -0
- flwr/proto/fab_pb2_grpc.pyi +4 -0
- flwr/proto/fleet_pb2.py +42 -23
- flwr/proto/fleet_pb2.pyi +123 -1
- flwr/proto/fleet_pb2_grpc.py +170 -0
- flwr/proto/fleet_pb2_grpc.pyi +61 -0
- flwr/proto/grpcadapter_pb2.py +32 -0
- flwr/proto/grpcadapter_pb2.pyi +43 -0
- flwr/proto/grpcadapter_pb2_grpc.py +66 -0
- flwr/proto/grpcadapter_pb2_grpc.pyi +24 -0
- flwr/proto/log_pb2.py +29 -0
- flwr/proto/log_pb2.pyi +39 -0
- flwr/proto/log_pb2_grpc.py +4 -0
- flwr/proto/log_pb2_grpc.pyi +4 -0
- flwr/proto/message_pb2.py +41 -0
- flwr/proto/message_pb2.pyi +128 -0
- flwr/proto/message_pb2_grpc.py +4 -0
- flwr/proto/message_pb2_grpc.pyi +4 -0
- flwr/proto/node_pb2.py +1 -1
- flwr/proto/recordset_pb2.py +35 -33
- flwr/proto/recordset_pb2.pyi +40 -14
- flwr/proto/run_pb2.py +64 -0
- flwr/proto/run_pb2.pyi +268 -0
- flwr/proto/run_pb2_grpc.py +4 -0
- flwr/proto/run_pb2_grpc.pyi +4 -0
- flwr/proto/serverappio_pb2.py +52 -0
- flwr/proto/{driver_pb2.pyi → serverappio_pb2.pyi} +62 -20
- flwr/proto/serverappio_pb2_grpc.py +410 -0
- flwr/proto/serverappio_pb2_grpc.pyi +160 -0
- flwr/proto/simulationio_pb2.py +38 -0
- flwr/proto/simulationio_pb2.pyi +65 -0
- flwr/proto/simulationio_pb2_grpc.py +239 -0
- flwr/proto/simulationio_pb2_grpc.pyi +94 -0
- flwr/proto/task_pb2.py +7 -8
- flwr/proto/task_pb2.pyi +8 -5
- flwr/proto/transport_pb2.py +8 -8
- flwr/proto/transport_pb2.pyi +9 -6
- flwr/server/__init__.py +2 -10
- flwr/server/app.py +579 -402
- flwr/server/client_manager.py +8 -6
- flwr/server/compat/app.py +6 -62
- flwr/server/compat/app_utils.py +14 -8
- flwr/server/compat/driver_client_proxy.py +25 -58
- flwr/server/compat/legacy_context.py +5 -4
- flwr/server/driver/__init__.py +2 -0
- flwr/server/driver/driver.py +36 -131
- flwr/server/driver/grpc_driver.py +217 -81
- flwr/server/driver/inmemory_driver.py +182 -0
- flwr/server/history.py +28 -29
- flwr/server/run_serverapp.py +15 -126
- flwr/server/server.py +50 -44
- flwr/server/server_app.py +59 -10
- flwr/server/serverapp/__init__.py +22 -0
- flwr/server/serverapp/app.py +256 -0
- flwr/server/serverapp_components.py +52 -0
- flwr/server/strategy/__init__.py +2 -2
- flwr/server/strategy/aggregate.py +37 -23
- flwr/server/strategy/bulyan.py +9 -9
- flwr/server/strategy/dp_adaptive_clipping.py +25 -25
- flwr/server/strategy/dp_fixed_clipping.py +23 -22
- flwr/server/strategy/dpfedavg_adaptive.py +8 -8
- flwr/server/strategy/dpfedavg_fixed.py +13 -12
- flwr/server/strategy/fault_tolerant_fedavg.py +11 -11
- flwr/server/strategy/fedadagrad.py +9 -9
- flwr/server/strategy/fedadam.py +20 -10
- flwr/server/strategy/fedavg.py +16 -16
- flwr/server/strategy/fedavg_android.py +17 -17
- flwr/server/strategy/fedavgm.py +9 -9
- flwr/server/strategy/fedmedian.py +5 -5
- flwr/server/strategy/fedopt.py +6 -6
- flwr/server/strategy/fedprox.py +7 -7
- flwr/server/strategy/fedtrimmedavg.py +8 -8
- flwr/server/strategy/fedxgb_bagging.py +12 -12
- flwr/server/strategy/fedxgb_cyclic.py +10 -10
- flwr/server/strategy/fedxgb_nn_avg.py +6 -6
- flwr/server/strategy/fedyogi.py +9 -9
- flwr/server/strategy/krum.py +9 -9
- flwr/server/strategy/qfedavg.py +16 -16
- flwr/server/strategy/strategy.py +10 -10
- flwr/server/superlink/driver/__init__.py +2 -2
- flwr/server/superlink/driver/serverappio_grpc.py +61 -0
- flwr/server/superlink/driver/serverappio_servicer.py +363 -0
- flwr/server/superlink/ffs/__init__.py +24 -0
- flwr/server/superlink/ffs/disk_ffs.py +108 -0
- flwr/server/superlink/ffs/ffs.py +79 -0
- flwr/server/superlink/ffs/ffs_factory.py +47 -0
- flwr/server/superlink/fleet/__init__.py +1 -1
- flwr/server/superlink/fleet/grpc_adapter/__init__.py +15 -0
- flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py +162 -0
- flwr/server/superlink/fleet/grpc_bidi/__init__.py +1 -1
- flwr/server/superlink/fleet/grpc_bidi/flower_service_servicer.py +4 -2
- flwr/server/superlink/fleet/grpc_bidi/grpc_bridge.py +3 -2
- flwr/server/superlink/fleet/grpc_bidi/grpc_client_proxy.py +1 -1
- flwr/server/superlink/fleet/grpc_bidi/grpc_server.py +5 -154
- flwr/server/superlink/fleet/grpc_rere/__init__.py +1 -1
- flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +120 -13
- flwr/server/superlink/fleet/grpc_rere/server_interceptor.py +228 -0
- flwr/server/superlink/fleet/message_handler/__init__.py +1 -1
- flwr/server/superlink/fleet/message_handler/message_handler.py +153 -9
- flwr/server/superlink/fleet/rest_rere/__init__.py +1 -1
- flwr/server/superlink/fleet/rest_rere/rest_api.py +119 -81
- flwr/server/superlink/fleet/vce/__init__.py +1 -0
- flwr/server/superlink/fleet/vce/backend/__init__.py +4 -4
- flwr/server/superlink/fleet/vce/backend/backend.py +8 -9
- flwr/server/superlink/fleet/vce/backend/raybackend.py +87 -68
- flwr/server/superlink/fleet/vce/vce_api.py +208 -146
- flwr/server/superlink/linkstate/__init__.py +28 -0
- flwr/server/superlink/linkstate/in_memory_linkstate.py +581 -0
- flwr/server/superlink/linkstate/linkstate.py +389 -0
- flwr/server/superlink/{state/state_factory.py → linkstate/linkstate_factory.py} +19 -10
- flwr/server/superlink/linkstate/sqlite_linkstate.py +1236 -0
- flwr/server/superlink/linkstate/utils.py +389 -0
- flwr/server/superlink/simulation/__init__.py +15 -0
- flwr/server/superlink/simulation/simulationio_grpc.py +65 -0
- flwr/server/superlink/simulation/simulationio_servicer.py +186 -0
- flwr/server/superlink/utils.py +65 -0
- flwr/server/typing.py +2 -0
- flwr/server/utils/__init__.py +1 -1
- flwr/server/utils/tensorboard.py +5 -5
- flwr/server/utils/validator.py +31 -11
- flwr/server/workflow/default_workflows.py +70 -26
- flwr/server/workflow/secure_aggregation/secagg_workflow.py +1 -0
- flwr/server/workflow/secure_aggregation/secaggplus_workflow.py +40 -27
- flwr/simulation/__init__.py +12 -5
- flwr/simulation/app.py +247 -315
- flwr/simulation/legacy_app.py +402 -0
- flwr/simulation/ray_transport/__init__.py +1 -1
- flwr/simulation/ray_transport/ray_actor.py +42 -67
- flwr/simulation/ray_transport/ray_client_proxy.py +37 -17
- flwr/simulation/ray_transport/utils.py +1 -0
- flwr/simulation/run_simulation.py +306 -163
- flwr/simulation/simulationio_connection.py +89 -0
- flwr/superexec/__init__.py +15 -0
- flwr/superexec/app.py +59 -0
- flwr/superexec/deployment.py +188 -0
- flwr/superexec/exec_grpc.py +80 -0
- flwr/superexec/exec_servicer.py +231 -0
- flwr/superexec/exec_user_auth_interceptor.py +101 -0
- flwr/superexec/executor.py +96 -0
- flwr/superexec/simulation.py +124 -0
- {flwr_nightly-1.8.0.dev20240315.dist-info → flwr_nightly-1.15.0.dev20250114.dist-info}/METADATA +33 -26
- flwr_nightly-1.15.0.dev20250114.dist-info/RECORD +328 -0
- flwr_nightly-1.15.0.dev20250114.dist-info/entry_points.txt +12 -0
- flwr/cli/flower_toml.py +0 -140
- flwr/cli/new/templates/app/flower.toml.tpl +0 -13
- flwr/cli/new/templates/app/requirements.numpy.txt.tpl +0 -2
- flwr/cli/new/templates/app/requirements.pytorch.txt.tpl +0 -4
- flwr/cli/new/templates/app/requirements.tensorflow.txt.tpl +0 -4
- flwr/client/node_state.py +0 -48
- flwr/client/node_state_tests.py +0 -65
- flwr/proto/driver_pb2.py +0 -44
- flwr/proto/driver_pb2_grpc.py +0 -169
- flwr/proto/driver_pb2_grpc.pyi +0 -66
- flwr/server/superlink/driver/driver_grpc.py +0 -54
- flwr/server/superlink/driver/driver_servicer.py +0 -129
- flwr/server/superlink/state/in_memory_state.py +0 -230
- flwr/server/superlink/state/sqlite_state.py +0 -630
- flwr/server/superlink/state/state.py +0 -154
- flwr_nightly-1.8.0.dev20240315.dist-info/RECORD +0 -211
- flwr_nightly-1.8.0.dev20240315.dist-info/entry_points.txt +0 -9
- {flwr_nightly-1.8.0.dev20240315.dist-info → flwr_nightly-1.15.0.dev20250114.dist-info}/LICENSE +0 -0
- {flwr_nightly-1.8.0.dev20240315.dist-info → flwr_nightly-1.15.0.dev20250114.dist-info}/WHEEL +0 -0
@@ -1,630 +0,0 @@
|
|
1
|
-
# Copyright 2023 Flower Labs GmbH. All Rights Reserved.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
# ==============================================================================
|
15
|
-
"""SQLite based implemenation of server state."""
|
16
|
-
|
17
|
-
|
18
|
-
import os
|
19
|
-
import re
|
20
|
-
import sqlite3
|
21
|
-
from datetime import datetime, timedelta
|
22
|
-
from logging import DEBUG, ERROR
|
23
|
-
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
|
24
|
-
from uuid import UUID, uuid4
|
25
|
-
|
26
|
-
from flwr.common import log, now
|
27
|
-
from flwr.proto.node_pb2 import Node # pylint: disable=E0611
|
28
|
-
from flwr.proto.recordset_pb2 import RecordSet # pylint: disable=E0611
|
29
|
-
from flwr.proto.task_pb2 import Task, TaskIns, TaskRes # pylint: disable=E0611
|
30
|
-
from flwr.server.utils.validator import validate_task_ins_or_res
|
31
|
-
|
32
|
-
from .state import State
|
33
|
-
|
34
|
-
SQL_CREATE_TABLE_NODE = """
|
35
|
-
CREATE TABLE IF NOT EXISTS node(
|
36
|
-
node_id INTEGER UNIQUE
|
37
|
-
);
|
38
|
-
"""
|
39
|
-
|
40
|
-
SQL_CREATE_TABLE_RUN = """
|
41
|
-
CREATE TABLE IF NOT EXISTS run(
|
42
|
-
run_id INTEGER UNIQUE
|
43
|
-
);
|
44
|
-
"""
|
45
|
-
|
46
|
-
SQL_CREATE_TABLE_TASK_INS = """
|
47
|
-
CREATE TABLE IF NOT EXISTS task_ins(
|
48
|
-
task_id TEXT UNIQUE,
|
49
|
-
group_id TEXT,
|
50
|
-
run_id INTEGER,
|
51
|
-
producer_anonymous BOOLEAN,
|
52
|
-
producer_node_id INTEGER,
|
53
|
-
consumer_anonymous BOOLEAN,
|
54
|
-
consumer_node_id INTEGER,
|
55
|
-
created_at TEXT,
|
56
|
-
delivered_at TEXT,
|
57
|
-
ttl TEXT,
|
58
|
-
ancestry TEXT,
|
59
|
-
task_type TEXT,
|
60
|
-
recordset BLOB,
|
61
|
-
FOREIGN KEY(run_id) REFERENCES run(run_id)
|
62
|
-
);
|
63
|
-
"""
|
64
|
-
|
65
|
-
|
66
|
-
SQL_CREATE_TABLE_TASK_RES = """
|
67
|
-
CREATE TABLE IF NOT EXISTS task_res(
|
68
|
-
task_id TEXT UNIQUE,
|
69
|
-
group_id TEXT,
|
70
|
-
run_id INTEGER,
|
71
|
-
producer_anonymous BOOLEAN,
|
72
|
-
producer_node_id INTEGER,
|
73
|
-
consumer_anonymous BOOLEAN,
|
74
|
-
consumer_node_id INTEGER,
|
75
|
-
created_at TEXT,
|
76
|
-
delivered_at TEXT,
|
77
|
-
ttl TEXT,
|
78
|
-
ancestry TEXT,
|
79
|
-
task_type TEXT,
|
80
|
-
recordset BLOB,
|
81
|
-
FOREIGN KEY(run_id) REFERENCES run(run_id)
|
82
|
-
);
|
83
|
-
"""
|
84
|
-
|
85
|
-
DictOrTuple = Union[Tuple[Any], Dict[str, Any]]
|
86
|
-
|
87
|
-
|
88
|
-
class SqliteState(State):
|
89
|
-
"""SQLite-based state implementation."""
|
90
|
-
|
91
|
-
def __init__(
|
92
|
-
self,
|
93
|
-
database_path: str,
|
94
|
-
) -> None:
|
95
|
-
"""Initialize an SqliteState.
|
96
|
-
|
97
|
-
Parameters
|
98
|
-
----------
|
99
|
-
database : (path-like object)
|
100
|
-
The path to the database file to be opened. Pass ":memory:" to open
|
101
|
-
a connection to a database that is in RAM, instead of on disk.
|
102
|
-
"""
|
103
|
-
self.database_path = database_path
|
104
|
-
self.conn: Optional[sqlite3.Connection] = None
|
105
|
-
|
106
|
-
def initialize(self, log_queries: bool = False) -> List[Tuple[str]]:
|
107
|
-
"""Create tables if they don't exist yet.
|
108
|
-
|
109
|
-
Parameters
|
110
|
-
----------
|
111
|
-
log_queries : bool
|
112
|
-
Log each query which is executed.
|
113
|
-
"""
|
114
|
-
self.conn = sqlite3.connect(self.database_path)
|
115
|
-
self.conn.execute("PRAGMA foreign_keys = ON;")
|
116
|
-
self.conn.row_factory = dict_factory
|
117
|
-
if log_queries:
|
118
|
-
self.conn.set_trace_callback(lambda query: log(DEBUG, query))
|
119
|
-
cur = self.conn.cursor()
|
120
|
-
|
121
|
-
# Create each table if not exists queries
|
122
|
-
cur.execute(SQL_CREATE_TABLE_RUN)
|
123
|
-
cur.execute(SQL_CREATE_TABLE_TASK_INS)
|
124
|
-
cur.execute(SQL_CREATE_TABLE_TASK_RES)
|
125
|
-
cur.execute(SQL_CREATE_TABLE_NODE)
|
126
|
-
res = cur.execute("SELECT name FROM sqlite_schema;")
|
127
|
-
|
128
|
-
return res.fetchall()
|
129
|
-
|
130
|
-
def query(
|
131
|
-
self,
|
132
|
-
query: str,
|
133
|
-
data: Optional[Union[List[DictOrTuple], DictOrTuple]] = None,
|
134
|
-
) -> List[Dict[str, Any]]:
|
135
|
-
"""Execute a SQL query."""
|
136
|
-
if self.conn is None:
|
137
|
-
raise AttributeError("State is not initialized.")
|
138
|
-
|
139
|
-
if data is None:
|
140
|
-
data = []
|
141
|
-
|
142
|
-
# Clean up whitespace to make the logs nicer
|
143
|
-
query = re.sub(r"\s+", " ", query)
|
144
|
-
|
145
|
-
try:
|
146
|
-
with self.conn:
|
147
|
-
if (
|
148
|
-
len(data) > 0
|
149
|
-
and isinstance(data, (tuple, list))
|
150
|
-
and isinstance(data[0], (tuple, dict))
|
151
|
-
):
|
152
|
-
rows = self.conn.executemany(query, data)
|
153
|
-
else:
|
154
|
-
rows = self.conn.execute(query, data)
|
155
|
-
|
156
|
-
# Extract results before committing to support
|
157
|
-
# INSERT/UPDATE ... RETURNING
|
158
|
-
# style queries
|
159
|
-
result = rows.fetchall()
|
160
|
-
except KeyError as exc:
|
161
|
-
log(ERROR, {"query": query, "data": data, "exception": exc})
|
162
|
-
|
163
|
-
return result
|
164
|
-
|
165
|
-
def store_task_ins(self, task_ins: TaskIns) -> Optional[UUID]:
|
166
|
-
"""Store one TaskIns.
|
167
|
-
|
168
|
-
Usually, the Driver API calls this to schedule instructions.
|
169
|
-
|
170
|
-
Stores the value of the task_ins in the state and, if successful, returns the
|
171
|
-
task_id (UUID) of the task_ins. If, for any reason, storing the task_ins fails,
|
172
|
-
`None` is returned.
|
173
|
-
|
174
|
-
Constraints
|
175
|
-
-----------
|
176
|
-
If `task_ins.task.consumer.anonymous` is `True`, then
|
177
|
-
`task_ins.task.consumer.node_id` MUST NOT be set (equal 0).
|
178
|
-
|
179
|
-
If `task_ins.task.consumer.anonymous` is `False`, then
|
180
|
-
`task_ins.task.consumer.node_id` MUST be set (not 0)
|
181
|
-
"""
|
182
|
-
# Validate task
|
183
|
-
errors = validate_task_ins_or_res(task_ins)
|
184
|
-
if any(errors):
|
185
|
-
log(ERROR, errors)
|
186
|
-
return None
|
187
|
-
|
188
|
-
# Create task_id, created_at and ttl
|
189
|
-
task_id = uuid4()
|
190
|
-
created_at: datetime = now()
|
191
|
-
ttl: datetime = created_at + timedelta(hours=24)
|
192
|
-
|
193
|
-
# Store TaskIns
|
194
|
-
task_ins.task_id = str(task_id)
|
195
|
-
task_ins.task.created_at = created_at.isoformat()
|
196
|
-
task_ins.task.ttl = ttl.isoformat()
|
197
|
-
data = (task_ins_to_dict(task_ins),)
|
198
|
-
columns = ", ".join([f":{key}" for key in data[0]])
|
199
|
-
query = f"INSERT INTO task_ins VALUES({columns});"
|
200
|
-
|
201
|
-
# Only invalid run_id can trigger IntegrityError.
|
202
|
-
# This may need to be changed in the future version with more integrity checks.
|
203
|
-
try:
|
204
|
-
self.query(query, data)
|
205
|
-
except sqlite3.IntegrityError:
|
206
|
-
log(ERROR, "`run` is invalid")
|
207
|
-
return None
|
208
|
-
|
209
|
-
return task_id
|
210
|
-
|
211
|
-
def get_task_ins(
|
212
|
-
self, node_id: Optional[int], limit: Optional[int]
|
213
|
-
) -> List[TaskIns]:
|
214
|
-
"""Get undelivered TaskIns for one node (either anonymous or with ID).
|
215
|
-
|
216
|
-
Usually, the Fleet API calls this for Nodes planning to work on one or more
|
217
|
-
TaskIns.
|
218
|
-
|
219
|
-
Constraints
|
220
|
-
-----------
|
221
|
-
If `node_id` is not `None`, retrieve all TaskIns where
|
222
|
-
|
223
|
-
1. the `task_ins.task.consumer.node_id` equals `node_id` AND
|
224
|
-
2. the `task_ins.task.consumer.anonymous` equals `False` AND
|
225
|
-
3. the `task_ins.task.delivered_at` equals `""`.
|
226
|
-
|
227
|
-
If `node_id` is `None`, retrieve all TaskIns where the
|
228
|
-
`task_ins.task.consumer.node_id` equals `0` and
|
229
|
-
`task_ins.task.consumer.anonymous` is set to `True`.
|
230
|
-
|
231
|
-
`delivered_at` MUST BE set (i.e., not `""`) otherwise the TaskIns MUST not be in
|
232
|
-
the result.
|
233
|
-
|
234
|
-
If `limit` is not `None`, return, at most, `limit` number of `task_ins`. If
|
235
|
-
`limit` is set, it has to be greater than zero.
|
236
|
-
"""
|
237
|
-
if limit is not None and limit < 1:
|
238
|
-
raise AssertionError("`limit` must be >= 1")
|
239
|
-
|
240
|
-
if node_id == 0:
|
241
|
-
msg = (
|
242
|
-
"`node_id` must be >= 1"
|
243
|
-
"\n\n For requesting anonymous tasks use `node_id` equal `None`"
|
244
|
-
)
|
245
|
-
raise AssertionError(msg)
|
246
|
-
|
247
|
-
data: Dict[str, Union[str, int]] = {}
|
248
|
-
|
249
|
-
if node_id is None:
|
250
|
-
# Retrieve all anonymous Tasks
|
251
|
-
query = """
|
252
|
-
SELECT task_id
|
253
|
-
FROM task_ins
|
254
|
-
WHERE consumer_anonymous == 1
|
255
|
-
AND consumer_node_id == 0
|
256
|
-
AND delivered_at = ""
|
257
|
-
"""
|
258
|
-
else:
|
259
|
-
# Retrieve all TaskIns for node_id
|
260
|
-
query = """
|
261
|
-
SELECT task_id
|
262
|
-
FROM task_ins
|
263
|
-
WHERE consumer_anonymous == 0
|
264
|
-
AND consumer_node_id == :node_id
|
265
|
-
AND delivered_at = ""
|
266
|
-
"""
|
267
|
-
data["node_id"] = node_id
|
268
|
-
|
269
|
-
if limit is not None:
|
270
|
-
query += " LIMIT :limit"
|
271
|
-
data["limit"] = limit
|
272
|
-
|
273
|
-
query += ";"
|
274
|
-
|
275
|
-
rows = self.query(query, data)
|
276
|
-
|
277
|
-
if rows:
|
278
|
-
# Prepare query
|
279
|
-
task_ids = [row["task_id"] for row in rows]
|
280
|
-
placeholders: str = ",".join([f":id_{i}" for i in range(len(task_ids))])
|
281
|
-
query = f"""
|
282
|
-
UPDATE task_ins
|
283
|
-
SET delivered_at = :delivered_at
|
284
|
-
WHERE task_id IN ({placeholders})
|
285
|
-
RETURNING *;
|
286
|
-
"""
|
287
|
-
|
288
|
-
# Prepare data for query
|
289
|
-
delivered_at = now().isoformat()
|
290
|
-
data = {"delivered_at": delivered_at}
|
291
|
-
for index, task_id in enumerate(task_ids):
|
292
|
-
data[f"id_{index}"] = str(task_id)
|
293
|
-
|
294
|
-
# Run query
|
295
|
-
rows = self.query(query, data)
|
296
|
-
|
297
|
-
result = [dict_to_task_ins(row) for row in rows]
|
298
|
-
|
299
|
-
return result
|
300
|
-
|
301
|
-
def store_task_res(self, task_res: TaskRes) -> Optional[UUID]:
|
302
|
-
"""Store one TaskRes.
|
303
|
-
|
304
|
-
Usually, the Fleet API calls this when Nodes return their results.
|
305
|
-
|
306
|
-
Stores the TaskRes and, if successful, returns the `task_id` (UUID) of
|
307
|
-
the `task_res`. If storing the `task_res` fails, `None` is returned.
|
308
|
-
|
309
|
-
Constraints
|
310
|
-
-----------
|
311
|
-
If `task_res.task.consumer.anonymous` is `True`, then
|
312
|
-
`task_res.task.consumer.node_id` MUST NOT be set (equal 0).
|
313
|
-
|
314
|
-
If `task_res.task.consumer.anonymous` is `False`, then
|
315
|
-
`task_res.task.consumer.node_id` MUST be set (not 0)
|
316
|
-
"""
|
317
|
-
# Validate task
|
318
|
-
errors = validate_task_ins_or_res(task_res)
|
319
|
-
if any(errors):
|
320
|
-
log(ERROR, errors)
|
321
|
-
return None
|
322
|
-
|
323
|
-
# Create task_id, created_at and ttl
|
324
|
-
task_id = uuid4()
|
325
|
-
created_at: datetime = now()
|
326
|
-
ttl: datetime = created_at + timedelta(hours=24)
|
327
|
-
|
328
|
-
# Store TaskIns
|
329
|
-
task_res.task_id = str(task_id)
|
330
|
-
task_res.task.created_at = created_at.isoformat()
|
331
|
-
task_res.task.ttl = ttl.isoformat()
|
332
|
-
data = (task_res_to_dict(task_res),)
|
333
|
-
columns = ", ".join([f":{key}" for key in data[0]])
|
334
|
-
query = f"INSERT INTO task_res VALUES({columns});"
|
335
|
-
|
336
|
-
# Only invalid run_id can trigger IntegrityError.
|
337
|
-
# This may need to be changed in the future version with more integrity checks.
|
338
|
-
try:
|
339
|
-
self.query(query, data)
|
340
|
-
except sqlite3.IntegrityError:
|
341
|
-
log(ERROR, "`run` is invalid")
|
342
|
-
return None
|
343
|
-
|
344
|
-
return task_id
|
345
|
-
|
346
|
-
def get_task_res(self, task_ids: Set[UUID], limit: Optional[int]) -> List[TaskRes]:
|
347
|
-
"""Get TaskRes for task_ids.
|
348
|
-
|
349
|
-
Usually, the Driver API calls this method to get results for instructions it has
|
350
|
-
previously scheduled.
|
351
|
-
|
352
|
-
Retrieves all TaskRes for the given `task_ids` and returns and empty list if
|
353
|
-
none could be found.
|
354
|
-
|
355
|
-
Constraints
|
356
|
-
-----------
|
357
|
-
If `limit` is not `None`, return, at most, `limit` number of TaskRes. The limit
|
358
|
-
will only take effect if enough task_ids are in the set AND are currently
|
359
|
-
available. If `limit` is set, it has to be greater than zero.
|
360
|
-
"""
|
361
|
-
if limit is not None and limit < 1:
|
362
|
-
raise AssertionError("`limit` must be >= 1")
|
363
|
-
|
364
|
-
# Retrieve all anonymous Tasks
|
365
|
-
if len(task_ids) == 0:
|
366
|
-
return []
|
367
|
-
|
368
|
-
placeholders = ",".join([f":id_{i}" for i in range(len(task_ids))])
|
369
|
-
query = f"""
|
370
|
-
SELECT *
|
371
|
-
FROM task_res
|
372
|
-
WHERE ancestry IN ({placeholders})
|
373
|
-
AND delivered_at = ""
|
374
|
-
"""
|
375
|
-
|
376
|
-
data: Dict[str, Union[str, int]] = {}
|
377
|
-
|
378
|
-
if limit is not None:
|
379
|
-
query += " LIMIT :limit"
|
380
|
-
data["limit"] = limit
|
381
|
-
|
382
|
-
query += ";"
|
383
|
-
|
384
|
-
for index, task_id in enumerate(task_ids):
|
385
|
-
data[f"id_{index}"] = str(task_id)
|
386
|
-
|
387
|
-
rows = self.query(query, data)
|
388
|
-
|
389
|
-
if rows:
|
390
|
-
# Prepare query
|
391
|
-
found_task_ids = [row["task_id"] for row in rows]
|
392
|
-
placeholders = ",".join([f":id_{i}" for i in range(len(found_task_ids))])
|
393
|
-
query = f"""
|
394
|
-
UPDATE task_res
|
395
|
-
SET delivered_at = :delivered_at
|
396
|
-
WHERE task_id IN ({placeholders})
|
397
|
-
RETURNING *;
|
398
|
-
"""
|
399
|
-
|
400
|
-
# Prepare data for query
|
401
|
-
delivered_at = now().isoformat()
|
402
|
-
data = {"delivered_at": delivered_at}
|
403
|
-
for index, task_id in enumerate(found_task_ids):
|
404
|
-
data[f"id_{index}"] = str(task_id)
|
405
|
-
|
406
|
-
# Run query
|
407
|
-
rows = self.query(query, data)
|
408
|
-
|
409
|
-
result = [dict_to_task_res(row) for row in rows]
|
410
|
-
return result
|
411
|
-
|
412
|
-
def num_task_ins(self) -> int:
|
413
|
-
"""Calculate the number of task_ins in store.
|
414
|
-
|
415
|
-
This includes delivered but not yet deleted task_ins.
|
416
|
-
"""
|
417
|
-
query = "SELECT count(*) AS num FROM task_ins;"
|
418
|
-
rows = self.query(query)
|
419
|
-
result = rows[0]
|
420
|
-
num = cast(int, result["num"])
|
421
|
-
return num
|
422
|
-
|
423
|
-
def num_task_res(self) -> int:
|
424
|
-
"""Calculate the number of task_res in store.
|
425
|
-
|
426
|
-
This includes delivered but not yet deleted task_res.
|
427
|
-
"""
|
428
|
-
query = "SELECT count(*) AS num FROM task_res;"
|
429
|
-
rows = self.query(query)
|
430
|
-
result: Dict[str, int] = rows[0]
|
431
|
-
return result["num"]
|
432
|
-
|
433
|
-
def delete_tasks(self, task_ids: Set[UUID]) -> None:
|
434
|
-
"""Delete all delivered TaskIns/TaskRes pairs."""
|
435
|
-
ids = list(task_ids)
|
436
|
-
if len(ids) == 0:
|
437
|
-
return None
|
438
|
-
|
439
|
-
placeholders = ",".join([f":id_{index}" for index in range(len(task_ids))])
|
440
|
-
data = {f"id_{index}": str(task_id) for index, task_id in enumerate(task_ids)}
|
441
|
-
|
442
|
-
# 1. Query: Delete task_ins which have a delivered task_res
|
443
|
-
query_1 = f"""
|
444
|
-
DELETE FROM task_ins
|
445
|
-
WHERE delivered_at != ''
|
446
|
-
AND task_id IN (
|
447
|
-
SELECT ancestry
|
448
|
-
FROM task_res
|
449
|
-
WHERE ancestry IN ({placeholders})
|
450
|
-
AND delivered_at != ''
|
451
|
-
);
|
452
|
-
"""
|
453
|
-
|
454
|
-
# 2. Query: Delete delivered task_res to be run after 1. Query
|
455
|
-
query_2 = f"""
|
456
|
-
DELETE FROM task_res
|
457
|
-
WHERE ancestry IN ({placeholders})
|
458
|
-
AND delivered_at != '';
|
459
|
-
"""
|
460
|
-
|
461
|
-
if self.conn is None:
|
462
|
-
raise AttributeError("State not intitialized")
|
463
|
-
|
464
|
-
with self.conn:
|
465
|
-
self.conn.execute(query_1, data)
|
466
|
-
self.conn.execute(query_2, data)
|
467
|
-
|
468
|
-
return None
|
469
|
-
|
470
|
-
def create_node(self) -> int:
|
471
|
-
"""Create, store in state, and return `node_id`."""
|
472
|
-
# Sample a random int64 as node_id
|
473
|
-
node_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
|
474
|
-
|
475
|
-
query = "INSERT INTO node VALUES(:node_id);"
|
476
|
-
try:
|
477
|
-
self.query(query, {"node_id": node_id})
|
478
|
-
except sqlite3.IntegrityError:
|
479
|
-
log(ERROR, "Unexpected node registration failure.")
|
480
|
-
return 0
|
481
|
-
return node_id
|
482
|
-
|
483
|
-
def delete_node(self, node_id: int) -> None:
|
484
|
-
"""Delete a client node."""
|
485
|
-
query = "DELETE FROM node WHERE node_id = :node_id;"
|
486
|
-
self.query(query, {"node_id": node_id})
|
487
|
-
|
488
|
-
def get_nodes(self, run_id: int) -> Set[int]:
|
489
|
-
"""Retrieve all currently stored node IDs as a set.
|
490
|
-
|
491
|
-
Constraints
|
492
|
-
-----------
|
493
|
-
If the provided `run_id` does not exist or has no matching nodes,
|
494
|
-
an empty `Set` MUST be returned.
|
495
|
-
"""
|
496
|
-
# Validate run ID
|
497
|
-
query = "SELECT COUNT(*) FROM run WHERE run_id = ?;"
|
498
|
-
if self.query(query, (run_id,))[0]["COUNT(*)"] == 0:
|
499
|
-
return set()
|
500
|
-
|
501
|
-
# Get nodes
|
502
|
-
query = "SELECT * FROM node;"
|
503
|
-
rows = self.query(query)
|
504
|
-
result: Set[int] = {row["node_id"] for row in rows}
|
505
|
-
return result
|
506
|
-
|
507
|
-
def create_run(self) -> int:
|
508
|
-
"""Create one run and store it in state."""
|
509
|
-
# Sample a random int64 as run_id
|
510
|
-
run_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
|
511
|
-
|
512
|
-
# Check conflicts
|
513
|
-
query = "SELECT COUNT(*) FROM run WHERE run_id = ?;"
|
514
|
-
# If run_id does not exist
|
515
|
-
if self.query(query, (run_id,))[0]["COUNT(*)"] == 0:
|
516
|
-
query = "INSERT INTO run VALUES(:run_id);"
|
517
|
-
self.query(query, {"run_id": run_id})
|
518
|
-
return run_id
|
519
|
-
log(ERROR, "Unexpected run creation failure.")
|
520
|
-
return 0
|
521
|
-
|
522
|
-
|
523
|
-
def dict_factory(
|
524
|
-
cursor: sqlite3.Cursor,
|
525
|
-
row: sqlite3.Row,
|
526
|
-
) -> Dict[str, Any]:
|
527
|
-
"""Turn SQLite results into dicts.
|
528
|
-
|
529
|
-
Less efficent for retrival of large amounts of data but easier to use.
|
530
|
-
"""
|
531
|
-
fields = [column[0] for column in cursor.description]
|
532
|
-
return dict(zip(fields, row))
|
533
|
-
|
534
|
-
|
535
|
-
def task_ins_to_dict(task_msg: TaskIns) -> Dict[str, Any]:
|
536
|
-
"""Transform TaskIns to dict."""
|
537
|
-
result = {
|
538
|
-
"task_id": task_msg.task_id,
|
539
|
-
"group_id": task_msg.group_id,
|
540
|
-
"run_id": task_msg.run_id,
|
541
|
-
"producer_anonymous": task_msg.task.producer.anonymous,
|
542
|
-
"producer_node_id": task_msg.task.producer.node_id,
|
543
|
-
"consumer_anonymous": task_msg.task.consumer.anonymous,
|
544
|
-
"consumer_node_id": task_msg.task.consumer.node_id,
|
545
|
-
"created_at": task_msg.task.created_at,
|
546
|
-
"delivered_at": task_msg.task.delivered_at,
|
547
|
-
"ttl": task_msg.task.ttl,
|
548
|
-
"ancestry": ",".join(task_msg.task.ancestry),
|
549
|
-
"task_type": task_msg.task.task_type,
|
550
|
-
"recordset": task_msg.task.recordset.SerializeToString(),
|
551
|
-
}
|
552
|
-
return result
|
553
|
-
|
554
|
-
|
555
|
-
def task_res_to_dict(task_msg: TaskRes) -> Dict[str, Any]:
|
556
|
-
"""Transform TaskRes to dict."""
|
557
|
-
result = {
|
558
|
-
"task_id": task_msg.task_id,
|
559
|
-
"group_id": task_msg.group_id,
|
560
|
-
"run_id": task_msg.run_id,
|
561
|
-
"producer_anonymous": task_msg.task.producer.anonymous,
|
562
|
-
"producer_node_id": task_msg.task.producer.node_id,
|
563
|
-
"consumer_anonymous": task_msg.task.consumer.anonymous,
|
564
|
-
"consumer_node_id": task_msg.task.consumer.node_id,
|
565
|
-
"created_at": task_msg.task.created_at,
|
566
|
-
"delivered_at": task_msg.task.delivered_at,
|
567
|
-
"ttl": task_msg.task.ttl,
|
568
|
-
"ancestry": ",".join(task_msg.task.ancestry),
|
569
|
-
"task_type": task_msg.task.task_type,
|
570
|
-
"recordset": task_msg.task.recordset.SerializeToString(),
|
571
|
-
}
|
572
|
-
return result
|
573
|
-
|
574
|
-
|
575
|
-
def dict_to_task_ins(task_dict: Dict[str, Any]) -> TaskIns:
|
576
|
-
"""Turn task_dict into protobuf message."""
|
577
|
-
recordset = RecordSet()
|
578
|
-
recordset.ParseFromString(task_dict["recordset"])
|
579
|
-
|
580
|
-
result = TaskIns(
|
581
|
-
task_id=task_dict["task_id"],
|
582
|
-
group_id=task_dict["group_id"],
|
583
|
-
run_id=task_dict["run_id"],
|
584
|
-
task=Task(
|
585
|
-
producer=Node(
|
586
|
-
node_id=task_dict["producer_node_id"],
|
587
|
-
anonymous=task_dict["producer_anonymous"],
|
588
|
-
),
|
589
|
-
consumer=Node(
|
590
|
-
node_id=task_dict["consumer_node_id"],
|
591
|
-
anonymous=task_dict["consumer_anonymous"],
|
592
|
-
),
|
593
|
-
created_at=task_dict["created_at"],
|
594
|
-
delivered_at=task_dict["delivered_at"],
|
595
|
-
ttl=task_dict["ttl"],
|
596
|
-
ancestry=task_dict["ancestry"].split(","),
|
597
|
-
task_type=task_dict["task_type"],
|
598
|
-
recordset=recordset,
|
599
|
-
),
|
600
|
-
)
|
601
|
-
return result
|
602
|
-
|
603
|
-
|
604
|
-
def dict_to_task_res(task_dict: Dict[str, Any]) -> TaskRes:
|
605
|
-
"""Turn task_dict into protobuf message."""
|
606
|
-
recordset = RecordSet()
|
607
|
-
recordset.ParseFromString(task_dict["recordset"])
|
608
|
-
|
609
|
-
result = TaskRes(
|
610
|
-
task_id=task_dict["task_id"],
|
611
|
-
group_id=task_dict["group_id"],
|
612
|
-
run_id=task_dict["run_id"],
|
613
|
-
task=Task(
|
614
|
-
producer=Node(
|
615
|
-
node_id=task_dict["producer_node_id"],
|
616
|
-
anonymous=task_dict["producer_anonymous"],
|
617
|
-
),
|
618
|
-
consumer=Node(
|
619
|
-
node_id=task_dict["consumer_node_id"],
|
620
|
-
anonymous=task_dict["consumer_anonymous"],
|
621
|
-
),
|
622
|
-
created_at=task_dict["created_at"],
|
623
|
-
delivered_at=task_dict["delivered_at"],
|
624
|
-
ttl=task_dict["ttl"],
|
625
|
-
ancestry=task_dict["ancestry"].split(","),
|
626
|
-
task_type=task_dict["task_type"],
|
627
|
-
recordset=recordset,
|
628
|
-
),
|
629
|
-
)
|
630
|
-
return result
|