flwr 1.22.0__py3-none-any.whl → 1.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flwr/cli/app.py +15 -1
- flwr/cli/auth_plugin/__init__.py +15 -6
- flwr/cli/auth_plugin/auth_plugin.py +95 -0
- flwr/cli/auth_plugin/noop_auth_plugin.py +58 -0
- flwr/cli/auth_plugin/oidc_cli_plugin.py +16 -25
- flwr/cli/build.py +118 -47
- flwr/cli/{cli_user_auth_interceptor.py → cli_account_auth_interceptor.py} +6 -5
- flwr/cli/log.py +2 -2
- flwr/cli/login/login.py +34 -23
- flwr/cli/ls.py +13 -9
- flwr/cli/new/new.py +187 -35
- flwr/cli/new/templates/app/pyproject.baseline.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.jax.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.pytorch_legacy_api.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.xgboost.toml.tpl +1 -1
- flwr/cli/pull.py +2 -2
- flwr/cli/run/run.py +11 -7
- flwr/cli/stop.py +2 -2
- flwr/cli/supernode/__init__.py +25 -0
- flwr/cli/supernode/ls.py +260 -0
- flwr/cli/supernode/register.py +185 -0
- flwr/cli/supernode/unregister.py +138 -0
- flwr/cli/utils.py +92 -69
- flwr/client/__init__.py +2 -1
- flwr/client/grpc_adapter_client/connection.py +6 -8
- flwr/client/grpc_rere_client/connection.py +59 -31
- flwr/client/grpc_rere_client/grpc_adapter.py +28 -12
- flwr/client/grpc_rere_client/{client_interceptor.py → node_auth_client_interceptor.py} +3 -6
- flwr/client/mod/secure_aggregation/secaggplus_mod.py +7 -5
- flwr/client/rest_client/connection.py +82 -37
- flwr/clientapp/__init__.py +1 -2
- flwr/{client/clientapp → clientapp}/utils.py +1 -1
- flwr/common/constant.py +53 -13
- flwr/common/exit/exit_code.py +20 -10
- flwr/common/inflatable_utils.py +10 -10
- flwr/common/record/array.py +3 -3
- flwr/common/record/arrayrecord.py +10 -1
- flwr/common/secure_aggregation/crypto/symmetric_encryption.py +1 -89
- flwr/common/serde.py +4 -2
- flwr/common/typing.py +7 -6
- flwr/compat/client/app.py +1 -1
- flwr/compat/client/grpc_client/connection.py +2 -2
- flwr/proto/control_pb2.py +48 -35
- flwr/proto/control_pb2.pyi +71 -5
- flwr/proto/control_pb2_grpc.py +102 -0
- flwr/proto/control_pb2_grpc.pyi +39 -0
- flwr/proto/fab_pb2.py +11 -7
- flwr/proto/fab_pb2.pyi +21 -1
- flwr/proto/fleet_pb2.py +31 -23
- flwr/proto/fleet_pb2.pyi +63 -23
- flwr/proto/fleet_pb2_grpc.py +98 -28
- flwr/proto/fleet_pb2_grpc.pyi +45 -13
- flwr/proto/node_pb2.py +3 -1
- flwr/proto/node_pb2.pyi +48 -0
- flwr/server/app.py +139 -114
- flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py +17 -7
- flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +132 -38
- flwr/server/superlink/fleet/grpc_rere/{server_interceptor.py → node_auth_server_interceptor.py} +27 -51
- flwr/server/superlink/fleet/message_handler/message_handler.py +67 -22
- flwr/server/superlink/fleet/rest_rere/rest_api.py +52 -31
- flwr/server/superlink/fleet/vce/backend/backend.py +1 -1
- flwr/server/superlink/fleet/vce/backend/raybackend.py +1 -1
- flwr/server/superlink/fleet/vce/vce_api.py +18 -5
- flwr/server/superlink/linkstate/in_memory_linkstate.py +167 -73
- flwr/server/superlink/linkstate/linkstate.py +107 -24
- flwr/server/superlink/linkstate/linkstate_factory.py +2 -1
- flwr/server/superlink/linkstate/sqlite_linkstate.py +306 -255
- flwr/server/superlink/linkstate/utils.py +3 -54
- flwr/server/superlink/serverappio/serverappio_servicer.py +2 -2
- flwr/server/superlink/simulation/simulationio_servicer.py +1 -1
- flwr/server/utils/validator.py +2 -3
- flwr/server/workflow/secure_aggregation/secaggplus_workflow.py +4 -2
- flwr/simulation/ray_transport/ray_actor.py +1 -1
- flwr/simulation/ray_transport/ray_client_proxy.py +1 -1
- flwr/simulation/run_simulation.py +3 -2
- flwr/supercore/constant.py +22 -0
- flwr/supercore/object_store/in_memory_object_store.py +0 -4
- flwr/supercore/object_store/object_store_factory.py +26 -6
- flwr/supercore/object_store/sqlite_object_store.py +252 -0
- flwr/{client/clientapp → supercore/primitives}/__init__.py +1 -1
- flwr/supercore/primitives/asymmetric.py +117 -0
- flwr/supercore/primitives/asymmetric_ed25519.py +165 -0
- flwr/supercore/sqlite_mixin.py +156 -0
- flwr/supercore/utils.py +20 -0
- flwr/{common → superlink}/auth_plugin/__init__.py +6 -6
- flwr/superlink/auth_plugin/auth_plugin.py +91 -0
- flwr/superlink/auth_plugin/noop_auth_plugin.py +87 -0
- flwr/superlink/servicer/control/{control_user_auth_interceptor.py → control_account_auth_interceptor.py} +19 -19
- flwr/superlink/servicer/control/control_event_log_interceptor.py +1 -1
- flwr/superlink/servicer/control/control_grpc.py +13 -11
- flwr/superlink/servicer/control/control_servicer.py +152 -60
- flwr/supernode/cli/flower_supernode.py +19 -26
- flwr/supernode/runtime/run_clientapp.py +2 -2
- flwr/supernode/servicer/clientappio/clientappio_servicer.py +1 -1
- flwr/supernode/start_client_internal.py +17 -9
- {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/METADATA +1 -1
- {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/RECORD +107 -96
- flwr/common/auth_plugin/auth_plugin.py +0 -149
- /flwr/{client → clientapp}/client_app.py +0 -0
- {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/WHEEL +0 -0
- {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/entry_points.txt +0 -0
|
@@ -33,6 +33,7 @@ from flwr.common.typing import RunStatus
|
|
|
33
33
|
# pylint: disable=E0611
|
|
34
34
|
from flwr.proto.message_pb2 import Context as ProtoContext
|
|
35
35
|
from flwr.proto.recorddict_pb2 import ConfigRecord as ProtoConfigRecord
|
|
36
|
+
from flwr.supercore.utils import int64_to_uint64, uint64_to_int64
|
|
36
37
|
|
|
37
38
|
# pylint: enable=E0611
|
|
38
39
|
VALID_RUN_STATUS_TRANSITIONS = {
|
|
@@ -76,58 +77,6 @@ def generate_rand_int_from_bytes(
|
|
|
76
77
|
return num
|
|
77
78
|
|
|
78
79
|
|
|
79
|
-
def convert_uint64_to_sint64(u: int) -> int:
|
|
80
|
-
"""Convert a uint64 value to a sint64 value with the same bit sequence.
|
|
81
|
-
|
|
82
|
-
Parameters
|
|
83
|
-
----------
|
|
84
|
-
u : int
|
|
85
|
-
The unsigned 64-bit integer to convert.
|
|
86
|
-
|
|
87
|
-
Returns
|
|
88
|
-
-------
|
|
89
|
-
int
|
|
90
|
-
The signed 64-bit integer equivalent.
|
|
91
|
-
|
|
92
|
-
The signed 64-bit integer will have the same bit pattern as the
|
|
93
|
-
unsigned 64-bit integer but may have a different decimal value.
|
|
94
|
-
|
|
95
|
-
For numbers within the range [0, `sint64` max value], the decimal
|
|
96
|
-
value remains the same. However, for numbers greater than the `sint64`
|
|
97
|
-
max value, the decimal value will differ due to the wraparound caused
|
|
98
|
-
by the sign bit.
|
|
99
|
-
"""
|
|
100
|
-
if u >= (1 << 63):
|
|
101
|
-
return u - (1 << 64)
|
|
102
|
-
return u
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def convert_sint64_to_uint64(s: int) -> int:
|
|
106
|
-
"""Convert a sint64 value to a uint64 value with the same bit sequence.
|
|
107
|
-
|
|
108
|
-
Parameters
|
|
109
|
-
----------
|
|
110
|
-
s : int
|
|
111
|
-
The signed 64-bit integer to convert.
|
|
112
|
-
|
|
113
|
-
Returns
|
|
114
|
-
-------
|
|
115
|
-
int
|
|
116
|
-
The unsigned 64-bit integer equivalent.
|
|
117
|
-
|
|
118
|
-
The unsigned 64-bit integer will have the same bit pattern as the
|
|
119
|
-
signed 64-bit integer but may have a different decimal value.
|
|
120
|
-
|
|
121
|
-
For negative `sint64` values, the conversion adds 2^64 to the
|
|
122
|
-
signed value to obtain the equivalent `uint64` value. For non-negative
|
|
123
|
-
`sint64` values, the decimal value remains unchanged in the `uint64`
|
|
124
|
-
representation.
|
|
125
|
-
"""
|
|
126
|
-
if s < 0:
|
|
127
|
-
return s + (1 << 64)
|
|
128
|
-
return s
|
|
129
|
-
|
|
130
|
-
|
|
131
80
|
def convert_uint64_values_in_dict_to_sint64(
|
|
132
81
|
data_dict: dict[str, int], keys: list[str]
|
|
133
82
|
) -> None:
|
|
@@ -142,7 +91,7 @@ def convert_uint64_values_in_dict_to_sint64(
|
|
|
142
91
|
"""
|
|
143
92
|
for key in keys:
|
|
144
93
|
if key in data_dict:
|
|
145
|
-
data_dict[key] =
|
|
94
|
+
data_dict[key] = uint64_to_int64(data_dict[key])
|
|
146
95
|
|
|
147
96
|
|
|
148
97
|
def convert_sint64_values_in_dict_to_uint64(
|
|
@@ -159,7 +108,7 @@ def convert_sint64_values_in_dict_to_uint64(
|
|
|
159
108
|
"""
|
|
160
109
|
for key in keys:
|
|
161
110
|
if key in data_dict:
|
|
162
|
-
data_dict[key] =
|
|
111
|
+
data_dict[key] = int64_to_uint64(data_dict[key])
|
|
163
112
|
|
|
164
113
|
|
|
165
114
|
def context_to_bytes(context: Context) -> bytes:
|
|
@@ -316,7 +316,7 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
|
316
316
|
|
|
317
317
|
ffs: Ffs = self.ffs_factory.ffs()
|
|
318
318
|
if result := ffs.get(request.hash_str):
|
|
319
|
-
fab = Fab(request.hash_str, result[0])
|
|
319
|
+
fab = Fab(request.hash_str, result[0], result[1])
|
|
320
320
|
return GetFabResponse(fab=fab_to_proto(fab))
|
|
321
321
|
|
|
322
322
|
raise ValueError(f"Found no FAB with hash: {request.hash_str}")
|
|
@@ -343,7 +343,7 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
|
|
|
343
343
|
fab = None
|
|
344
344
|
if run and run.fab_hash:
|
|
345
345
|
if result := ffs.get(run.fab_hash):
|
|
346
|
-
fab = Fab(run.fab_hash, result[0])
|
|
346
|
+
fab = Fab(run.fab_hash, result[0], result[1])
|
|
347
347
|
if run and fab and serverapp_ctxt:
|
|
348
348
|
# Update run status to STARTING
|
|
349
349
|
if state.update_run_status(run_id, RunStatus(Status.STARTING, "", "")):
|
|
@@ -150,7 +150,7 @@ class SimulationIoServicer(simulationio_pb2_grpc.SimulationIoServicer):
|
|
|
150
150
|
fab = None
|
|
151
151
|
if run and run.fab_hash:
|
|
152
152
|
if result := ffs.get(run.fab_hash):
|
|
153
|
-
fab = Fab(run.fab_hash, result[0])
|
|
153
|
+
fab = Fab(run.fab_hash, result[0], result[1])
|
|
154
154
|
if run and fab and serverapp_ctxt:
|
|
155
155
|
# Update run status to STARTING
|
|
156
156
|
if state.update_run_status(run_id, RunStatus(Status.STARTING, "", "")):
|
flwr/server/utils/validator.py
CHANGED
|
@@ -15,10 +15,9 @@
|
|
|
15
15
|
"""Validators."""
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
import time
|
|
19
|
-
|
|
20
18
|
from flwr.common import Message
|
|
21
19
|
from flwr.common.constant import SUPERLINK_NODE_ID
|
|
20
|
+
from flwr.common.date import now
|
|
22
21
|
|
|
23
22
|
|
|
24
23
|
# pylint: disable-next=too-many-branches
|
|
@@ -44,7 +43,7 @@ def validate_message(message: Message, is_reply_message: bool) -> list[str]:
|
|
|
44
43
|
validation_errors.append("`metadata.ttl` must be higher than zero")
|
|
45
44
|
|
|
46
45
|
# Verify TTL and created_at time
|
|
47
|
-
current_time =
|
|
46
|
+
current_time = now().timestamp()
|
|
48
47
|
if metadata.created_at + metadata.ttl <= current_time:
|
|
49
48
|
validation_errors.append("Message TTL has expired")
|
|
50
49
|
|
|
@@ -35,8 +35,6 @@ from flwr.common import (
|
|
|
35
35
|
)
|
|
36
36
|
from flwr.common.secure_aggregation.crypto.shamir import combine_shares
|
|
37
37
|
from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
|
|
38
|
-
bytes_to_private_key,
|
|
39
|
-
bytes_to_public_key,
|
|
40
38
|
generate_shared_key,
|
|
41
39
|
)
|
|
42
40
|
from flwr.common.secure_aggregation.ndarrays_arithmetic import (
|
|
@@ -56,6 +54,10 @@ from flwr.common.secure_aggregation.secaggplus_utils import pseudo_rand_gen
|
|
|
56
54
|
from flwr.server.client_proxy import ClientProxy
|
|
57
55
|
from flwr.server.compat.legacy_context import LegacyContext
|
|
58
56
|
from flwr.server.grid import Grid
|
|
57
|
+
from flwr.supercore.primitives.asymmetric import (
|
|
58
|
+
bytes_to_private_key,
|
|
59
|
+
bytes_to_public_key,
|
|
60
|
+
)
|
|
59
61
|
|
|
60
62
|
from ..constant import MAIN_CONFIGS_RECORD, MAIN_PARAMS_RECORD
|
|
61
63
|
from ..constant import Key as WorkflowKey
|
|
@@ -24,7 +24,7 @@ import ray
|
|
|
24
24
|
from ray import ObjectRef
|
|
25
25
|
from ray.util.actor_pool import ActorPool
|
|
26
26
|
|
|
27
|
-
from flwr.
|
|
27
|
+
from flwr.clientapp.client_app import ClientApp, ClientAppException, LoadClientAppError
|
|
28
28
|
from flwr.common import Context, Message
|
|
29
29
|
from flwr.common.logger import log
|
|
30
30
|
|
|
@@ -21,8 +21,8 @@ from typing import Optional
|
|
|
21
21
|
|
|
22
22
|
from flwr import common
|
|
23
23
|
from flwr.client import ClientFnExt
|
|
24
|
-
from flwr.client.client_app import ClientApp
|
|
25
24
|
from flwr.client.run_info_store import DeprecatedRunInfoStore
|
|
25
|
+
from flwr.clientapp.client_app import ClientApp
|
|
26
26
|
from flwr.common import DEFAULT_TTL, Message, Metadata, RecordDict, now
|
|
27
27
|
from flwr.common.constant import (
|
|
28
28
|
NUM_PARTITIONS_KEY,
|
|
@@ -30,7 +30,7 @@ from typing import Any, Optional
|
|
|
30
30
|
|
|
31
31
|
from flwr.cli.config_utils import load_and_validate
|
|
32
32
|
from flwr.cli.utils import get_sha256_hash
|
|
33
|
-
from flwr.
|
|
33
|
+
from flwr.clientapp import ClientApp
|
|
34
34
|
from flwr.common import Context, EventType, RecordDict, event, log, now
|
|
35
35
|
from flwr.common.config import get_fused_config_from_dir, parse_config_args
|
|
36
36
|
from flwr.common.constant import RUN_ID_NUM_BYTES, Status
|
|
@@ -51,6 +51,7 @@ from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
|
|
|
51
51
|
from flwr.simulation.ray_transport.utils import (
|
|
52
52
|
enable_tf_gpu_growth as enable_gpu_growth,
|
|
53
53
|
)
|
|
54
|
+
from flwr.supercore.constant import FLWR_IN_MEMORY_DB_NAME
|
|
54
55
|
|
|
55
56
|
|
|
56
57
|
def _replace_keys(d: Any, match: str, target: str) -> Any:
|
|
@@ -336,7 +337,7 @@ def _main_loop(
|
|
|
336
337
|
) -> Context:
|
|
337
338
|
"""Start ServerApp on a separate thread, then launch Simulation Engine."""
|
|
338
339
|
# Initialize StateFactory
|
|
339
|
-
state_factory = LinkStateFactory(
|
|
340
|
+
state_factory = LinkStateFactory(FLWR_IN_MEMORY_DB_NAME)
|
|
340
341
|
|
|
341
342
|
f_stop = threading.Event()
|
|
342
343
|
# A Threading event to indicate if an exception was raised in the ServerApp thread
|
flwr/supercore/constant.py
CHANGED
|
@@ -15,5 +15,27 @@
|
|
|
15
15
|
"""Constants for Flower infrastructure."""
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
18
20
|
# Top-level key in YAML config for exec plugin settings
|
|
19
21
|
EXEC_PLUGIN_SECTION = "exec_plugin"
|
|
22
|
+
|
|
23
|
+
# Flower in-memory Python-based database name
|
|
24
|
+
FLWR_IN_MEMORY_DB_NAME = ":flwr-in-memory:"
|
|
25
|
+
|
|
26
|
+
# Constants for Hub
|
|
27
|
+
APP_ID_PATTERN = r"^@(?P<user>[^/]+)/(?P<app>[^/]+)$"
|
|
28
|
+
PLATFORM_API_URL = "https://api.flower.ai/v1"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class NodeStatus:
|
|
32
|
+
"""Event log writer types."""
|
|
33
|
+
|
|
34
|
+
REGISTERED = "registered"
|
|
35
|
+
ONLINE = "online"
|
|
36
|
+
OFFLINE = "offline"
|
|
37
|
+
UNREGISTERED = "unregistered"
|
|
38
|
+
|
|
39
|
+
def __new__(cls) -> NodeStatus:
|
|
40
|
+
"""Prevent instantiation."""
|
|
41
|
+
raise TypeError(f"{cls.__name__} cannot be instantiated.")
|
|
@@ -48,9 +48,6 @@ class InMemoryObjectStore(ObjectStore):
|
|
|
48
48
|
self.verify = verify
|
|
49
49
|
self.store: dict[str, ObjectEntry] = {}
|
|
50
50
|
self.lock_store = threading.RLock()
|
|
51
|
-
# Mapping the Object ID of a message to the list of descendant object IDs
|
|
52
|
-
self.msg_descendant_objects_mapping: dict[str, list[str]] = {}
|
|
53
|
-
self.lock_msg_mapping = threading.RLock()
|
|
54
51
|
# Mapping each run ID to a set of object IDs that are used in that run
|
|
55
52
|
self.run_objects_mapping: dict[int, set[str]] = {}
|
|
56
53
|
|
|
@@ -215,7 +212,6 @@ class InMemoryObjectStore(ObjectStore):
|
|
|
215
212
|
"""Clear the store."""
|
|
216
213
|
with self.lock_store:
|
|
217
214
|
self.store.clear()
|
|
218
|
-
self.msg_descendant_objects_mapping.clear()
|
|
219
215
|
self.run_objects_mapping.clear()
|
|
220
216
|
|
|
221
217
|
def __contains__(self, object_id: str) -> bool:
|
|
@@ -19,15 +19,27 @@ from logging import DEBUG
|
|
|
19
19
|
from typing import Optional
|
|
20
20
|
|
|
21
21
|
from flwr.common.logger import log
|
|
22
|
+
from flwr.supercore.constant import FLWR_IN_MEMORY_DB_NAME
|
|
22
23
|
|
|
23
24
|
from .in_memory_object_store import InMemoryObjectStore
|
|
24
25
|
from .object_store import ObjectStore
|
|
26
|
+
from .sqlite_object_store import SqliteObjectStore
|
|
25
27
|
|
|
26
28
|
|
|
27
29
|
class ObjectStoreFactory:
|
|
28
|
-
"""Factory class that creates ObjectStore instances.
|
|
30
|
+
"""Factory class that creates ObjectStore instances.
|
|
29
31
|
|
|
30
|
-
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
database : str (default: FLWR_IN_MEMORY_DB_NAME)
|
|
35
|
+
A string representing the path to the database file that will be opened.
|
|
36
|
+
Note that passing ":memory:" will open a connection to a database that is
|
|
37
|
+
in RAM, instead of on disk. And FLWR_IN_MEMORY_DB_NAME will create an
|
|
38
|
+
Python-based in-memory ObjectStore.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, database: str = FLWR_IN_MEMORY_DB_NAME) -> None:
|
|
42
|
+
self.database = database
|
|
31
43
|
self.store_instance: Optional[ObjectStore] = None
|
|
32
44
|
|
|
33
45
|
def store(self) -> ObjectStore:
|
|
@@ -38,7 +50,15 @@ class ObjectStoreFactory:
|
|
|
38
50
|
ObjectStore
|
|
39
51
|
An ObjectStore instance for storing objects by object_id.
|
|
40
52
|
"""
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
53
|
+
# InMemoryObjectStore
|
|
54
|
+
if self.database == FLWR_IN_MEMORY_DB_NAME:
|
|
55
|
+
if self.store_instance is None:
|
|
56
|
+
self.store_instance = InMemoryObjectStore()
|
|
57
|
+
log(DEBUG, "Using InMemoryObjectStore")
|
|
58
|
+
return self.store_instance
|
|
59
|
+
|
|
60
|
+
# SqliteObjectStore
|
|
61
|
+
store = SqliteObjectStore(self.database)
|
|
62
|
+
store.initialize()
|
|
63
|
+
log(DEBUG, "Using SqliteObjectStore")
|
|
64
|
+
return store
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
# ==============================================================================
|
|
15
|
+
"""Flower SQLite ObjectStore implementation."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
from typing import Optional, cast
|
|
19
|
+
|
|
20
|
+
from flwr.common.inflatable import (
|
|
21
|
+
get_object_id,
|
|
22
|
+
is_valid_sha256_hash,
|
|
23
|
+
iterate_object_tree,
|
|
24
|
+
)
|
|
25
|
+
from flwr.common.inflatable_utils import validate_object_content
|
|
26
|
+
from flwr.proto.message_pb2 import ObjectTree # pylint: disable=E0611
|
|
27
|
+
from flwr.supercore.sqlite_mixin import SqliteMixin
|
|
28
|
+
from flwr.supercore.utils import uint64_to_int64
|
|
29
|
+
|
|
30
|
+
from .object_store import NoObjectInStoreError, ObjectStore
|
|
31
|
+
|
|
32
|
+
SQL_CREATE_OBJECTS = """
|
|
33
|
+
CREATE TABLE IF NOT EXISTS objects (
|
|
34
|
+
object_id TEXT PRIMARY KEY,
|
|
35
|
+
content BLOB,
|
|
36
|
+
is_available INTEGER NOT NULL CHECK (is_available IN (0,1)),
|
|
37
|
+
ref_count INTEGER NOT NULL
|
|
38
|
+
);
|
|
39
|
+
"""
|
|
40
|
+
SQL_CREATE_OBJECT_CHILDREN = """
|
|
41
|
+
CREATE TABLE IF NOT EXISTS object_children (
|
|
42
|
+
parent_id TEXT NOT NULL,
|
|
43
|
+
child_id TEXT NOT NULL,
|
|
44
|
+
FOREIGN KEY (parent_id) REFERENCES objects(object_id) ON DELETE CASCADE,
|
|
45
|
+
FOREIGN KEY (child_id) REFERENCES objects(object_id) ON DELETE CASCADE,
|
|
46
|
+
PRIMARY KEY (parent_id, child_id)
|
|
47
|
+
);
|
|
48
|
+
"""
|
|
49
|
+
SQL_CREATE_RUN_OBJECTS = """
|
|
50
|
+
CREATE TABLE IF NOT EXISTS run_objects (
|
|
51
|
+
run_id INTEGER NOT NULL,
|
|
52
|
+
object_id TEXT NOT NULL,
|
|
53
|
+
FOREIGN KEY (object_id) REFERENCES objects(object_id) ON DELETE CASCADE,
|
|
54
|
+
PRIMARY KEY (run_id, object_id)
|
|
55
|
+
);
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class SqliteObjectStore(ObjectStore, SqliteMixin):
|
|
60
|
+
"""SQLite-based implementation of the ObjectStore interface."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, database_path: str, verify: bool = True) -> None:
|
|
63
|
+
super().__init__(database_path)
|
|
64
|
+
self.verify = verify
|
|
65
|
+
|
|
66
|
+
def initialize(self, log_queries: bool = False) -> list[tuple[str]]:
|
|
67
|
+
"""Connect to the DB, enable FK support, and create tables if needed."""
|
|
68
|
+
return self._ensure_initialized(
|
|
69
|
+
SQL_CREATE_OBJECTS,
|
|
70
|
+
SQL_CREATE_OBJECT_CHILDREN,
|
|
71
|
+
SQL_CREATE_RUN_OBJECTS,
|
|
72
|
+
log_queries=log_queries,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
def preregister(self, run_id: int, object_tree: ObjectTree) -> list[str]:
|
|
76
|
+
"""Identify and preregister missing objects in the `ObjectStore`."""
|
|
77
|
+
new_objects = []
|
|
78
|
+
for tree_node in iterate_object_tree(object_tree):
|
|
79
|
+
obj_id = tree_node.object_id
|
|
80
|
+
if not is_valid_sha256_hash(obj_id):
|
|
81
|
+
raise ValueError(f"Invalid object ID format: {obj_id}")
|
|
82
|
+
|
|
83
|
+
child_ids = [child.object_id for child in tree_node.children]
|
|
84
|
+
with self.conn:
|
|
85
|
+
row = self.conn.execute(
|
|
86
|
+
"SELECT object_id, is_available FROM objects WHERE object_id=?",
|
|
87
|
+
(obj_id,),
|
|
88
|
+
).fetchone()
|
|
89
|
+
if row is None:
|
|
90
|
+
# Insert new object
|
|
91
|
+
self.conn.execute(
|
|
92
|
+
"INSERT INTO objects"
|
|
93
|
+
"(object_id, content, is_available, ref_count) "
|
|
94
|
+
"VALUES (?, ?, ?, ?)",
|
|
95
|
+
(obj_id, b"", 0, 0),
|
|
96
|
+
)
|
|
97
|
+
for cid in child_ids:
|
|
98
|
+
self.conn.execute(
|
|
99
|
+
"INSERT INTO object_children(parent_id, child_id) "
|
|
100
|
+
"VALUES (?, ?)",
|
|
101
|
+
(obj_id, cid),
|
|
102
|
+
)
|
|
103
|
+
self.conn.execute(
|
|
104
|
+
"UPDATE objects SET ref_count = ref_count + 1 "
|
|
105
|
+
"WHERE object_id = ?",
|
|
106
|
+
(cid,),
|
|
107
|
+
)
|
|
108
|
+
new_objects.append(obj_id)
|
|
109
|
+
else:
|
|
110
|
+
# Add to the list of new objects if not available
|
|
111
|
+
if not row["is_available"]:
|
|
112
|
+
new_objects.append(obj_id)
|
|
113
|
+
|
|
114
|
+
# Ensure run mapping
|
|
115
|
+
self.conn.execute(
|
|
116
|
+
"INSERT OR IGNORE INTO run_objects(run_id, object_id) "
|
|
117
|
+
"VALUES (?, ?)",
|
|
118
|
+
(uint64_to_int64(run_id), obj_id),
|
|
119
|
+
)
|
|
120
|
+
return new_objects
|
|
121
|
+
|
|
122
|
+
def get_object_tree(self, object_id: str) -> ObjectTree:
|
|
123
|
+
"""Get the object tree for a given object ID."""
|
|
124
|
+
with self.conn:
|
|
125
|
+
row = self.conn.execute(
|
|
126
|
+
"SELECT object_id FROM objects WHERE object_id=?", (object_id,)
|
|
127
|
+
).fetchone()
|
|
128
|
+
if not row:
|
|
129
|
+
raise NoObjectInStoreError(f"Object {object_id} not found.")
|
|
130
|
+
children = self.query(
|
|
131
|
+
"SELECT child_id FROM object_children WHERE parent_id=?", (object_id,)
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
# Build the object trees of all children
|
|
135
|
+
try:
|
|
136
|
+
child_trees = [self.get_object_tree(ch["child_id"]) for ch in children]
|
|
137
|
+
except NoObjectInStoreError as e:
|
|
138
|
+
# Raise an error if any child object is missing
|
|
139
|
+
# This indicates an integrity issue
|
|
140
|
+
raise NoObjectInStoreError(
|
|
141
|
+
f"Object tree for object ID '{object_id}' contains missing "
|
|
142
|
+
"children. This may indicate a corrupted object store."
|
|
143
|
+
) from e
|
|
144
|
+
|
|
145
|
+
# Create and return the ObjectTree for the current object
|
|
146
|
+
return ObjectTree(object_id=object_id, children=child_trees)
|
|
147
|
+
|
|
148
|
+
def put(self, object_id: str, object_content: bytes) -> None:
|
|
149
|
+
"""Put an object into the store."""
|
|
150
|
+
if self.verify:
|
|
151
|
+
# Verify object_id and object_content match
|
|
152
|
+
object_id_from_content = get_object_id(object_content)
|
|
153
|
+
if object_id != object_id_from_content:
|
|
154
|
+
raise ValueError(f"Object ID {object_id} does not match content hash")
|
|
155
|
+
|
|
156
|
+
# Validate object content
|
|
157
|
+
validate_object_content(content=object_content)
|
|
158
|
+
|
|
159
|
+
with self.conn:
|
|
160
|
+
# Only allow adding the object if it has been preregistered
|
|
161
|
+
row = self.conn.execute(
|
|
162
|
+
"SELECT is_available FROM objects WHERE object_id=?", (object_id,)
|
|
163
|
+
).fetchone()
|
|
164
|
+
if row is None:
|
|
165
|
+
raise NoObjectInStoreError(
|
|
166
|
+
f"Object with ID '{object_id}' was not pre-registered."
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
# Return if object is already present in the store
|
|
170
|
+
if row["is_available"]:
|
|
171
|
+
return
|
|
172
|
+
|
|
173
|
+
# Update the object entry in the store
|
|
174
|
+
self.conn.execute(
|
|
175
|
+
"UPDATE objects SET content=?, is_available=1 WHERE object_id=?",
|
|
176
|
+
(object_content, object_id),
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
def get(self, object_id: str) -> Optional[bytes]:
|
|
180
|
+
"""Get an object from the store."""
|
|
181
|
+
rows = self.query("SELECT content FROM objects WHERE object_id=?", (object_id,))
|
|
182
|
+
return rows[0]["content"] if rows else None
|
|
183
|
+
|
|
184
|
+
def delete(self, object_id: str) -> None:
|
|
185
|
+
"""Delete an object and its unreferenced descendants from the store."""
|
|
186
|
+
with self.conn:
|
|
187
|
+
row = self.conn.execute(
|
|
188
|
+
"SELECT ref_count FROM objects WHERE object_id=?", (object_id,)
|
|
189
|
+
).fetchone()
|
|
190
|
+
|
|
191
|
+
# If the object is not in the store, nothing to delete
|
|
192
|
+
if row is None:
|
|
193
|
+
return
|
|
194
|
+
|
|
195
|
+
# Skip deletion if there are still references
|
|
196
|
+
if row["ref_count"] > 0:
|
|
197
|
+
return
|
|
198
|
+
|
|
199
|
+
# Deleting will cascade via FK, but we need to decrement children first
|
|
200
|
+
children = self.conn.execute(
|
|
201
|
+
"SELECT child_id FROM object_children WHERE parent_id=?", (object_id,)
|
|
202
|
+
).fetchall()
|
|
203
|
+
child_ids = [child["child_id"] for child in children]
|
|
204
|
+
|
|
205
|
+
if child_ids:
|
|
206
|
+
placeholders = ", ".join("?" for _ in child_ids)
|
|
207
|
+
query = f"""
|
|
208
|
+
UPDATE objects SET ref_count = ref_count - 1
|
|
209
|
+
WHERE object_id IN ({placeholders})
|
|
210
|
+
"""
|
|
211
|
+
self.conn.execute(query, child_ids)
|
|
212
|
+
|
|
213
|
+
self.conn.execute("DELETE FROM objects WHERE object_id=?", (object_id,))
|
|
214
|
+
|
|
215
|
+
# Recursively clean children
|
|
216
|
+
for child_id in child_ids:
|
|
217
|
+
self.delete(child_id)
|
|
218
|
+
|
|
219
|
+
def delete_objects_in_run(self, run_id: int) -> None:
|
|
220
|
+
"""Delete all objects that were registered in a specific run."""
|
|
221
|
+
run_id_sint = uint64_to_int64(run_id)
|
|
222
|
+
with self.conn:
|
|
223
|
+
objs = self.conn.execute(
|
|
224
|
+
"SELECT object_id FROM run_objects WHERE run_id=?", (run_id_sint,)
|
|
225
|
+
).fetchall()
|
|
226
|
+
for obj in objs:
|
|
227
|
+
object_id = obj["object_id"]
|
|
228
|
+
row = self.conn.execute(
|
|
229
|
+
"SELECT ref_count FROM objects WHERE object_id=?", (object_id,)
|
|
230
|
+
).fetchone()
|
|
231
|
+
if row and row["ref_count"] == 0:
|
|
232
|
+
self.delete(object_id)
|
|
233
|
+
self.conn.execute("DELETE FROM run_objects WHERE run_id=?", (run_id_sint,))
|
|
234
|
+
|
|
235
|
+
def clear(self) -> None:
|
|
236
|
+
"""Clear the store."""
|
|
237
|
+
with self.conn:
|
|
238
|
+
self.conn.execute("DELETE FROM object_children;")
|
|
239
|
+
self.conn.execute("DELETE FROM run_objects;")
|
|
240
|
+
self.conn.execute("DELETE FROM objects;")
|
|
241
|
+
|
|
242
|
+
def __contains__(self, object_id: str) -> bool:
|
|
243
|
+
"""Check if an object_id is in the store."""
|
|
244
|
+
row = self.conn.execute(
|
|
245
|
+
"SELECT 1 FROM objects WHERE object_id=?", (object_id,)
|
|
246
|
+
).fetchone()
|
|
247
|
+
return row is not None
|
|
248
|
+
|
|
249
|
+
def __len__(self) -> int:
|
|
250
|
+
"""Return the number of objects in the store."""
|
|
251
|
+
row = self.conn.execute("SELECT COUNT(*) AS cnt FROM objects;").fetchone()
|
|
252
|
+
return cast(int, row["cnt"])
|
|
@@ -12,4 +12,4 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
# ==============================================================================
|
|
15
|
-
"""Flower
|
|
15
|
+
"""Cryptographic primitives for the Flower infrastructure."""
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
# ==============================================================================
|
|
15
|
+
"""Asymmetric cryptography utilities."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
from typing import cast
|
|
19
|
+
|
|
20
|
+
from cryptography.exceptions import InvalidSignature
|
|
21
|
+
from cryptography.hazmat.primitives import hashes, serialization
|
|
22
|
+
from cryptography.hazmat.primitives.asymmetric import ec
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def generate_key_pairs() -> (
|
|
26
|
+
tuple[ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]
|
|
27
|
+
):
|
|
28
|
+
"""Generate private and public key pairs with Cryptography."""
|
|
29
|
+
private_key = ec.generate_private_key(ec.SECP384R1())
|
|
30
|
+
public_key = private_key.public_key()
|
|
31
|
+
return private_key, public_key
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def private_key_to_bytes(private_key: ec.EllipticCurvePrivateKey) -> bytes:
|
|
35
|
+
"""Serialize private key to bytes."""
|
|
36
|
+
return private_key.private_bytes(
|
|
37
|
+
encoding=serialization.Encoding.PEM,
|
|
38
|
+
format=serialization.PrivateFormat.PKCS8,
|
|
39
|
+
encryption_algorithm=serialization.NoEncryption(),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def bytes_to_private_key(private_key_bytes: bytes) -> ec.EllipticCurvePrivateKey:
|
|
44
|
+
"""Deserialize private key from bytes."""
|
|
45
|
+
return cast(
|
|
46
|
+
ec.EllipticCurvePrivateKey,
|
|
47
|
+
serialization.load_pem_private_key(data=private_key_bytes, password=None),
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def public_key_to_bytes(public_key: ec.EllipticCurvePublicKey) -> bytes:
|
|
52
|
+
"""Serialize public key to bytes."""
|
|
53
|
+
return public_key.public_bytes(
|
|
54
|
+
encoding=serialization.Encoding.PEM,
|
|
55
|
+
format=serialization.PublicFormat.SubjectPublicKeyInfo,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def bytes_to_public_key(public_key_bytes: bytes) -> ec.EllipticCurvePublicKey:
|
|
60
|
+
"""Deserialize public key from bytes."""
|
|
61
|
+
return cast(
|
|
62
|
+
ec.EllipticCurvePublicKey,
|
|
63
|
+
serialization.load_pem_public_key(data=public_key_bytes),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def sign_message(private_key: ec.EllipticCurvePrivateKey, message: bytes) -> bytes:
|
|
68
|
+
"""Sign a message using the provided EC private key.
|
|
69
|
+
|
|
70
|
+
Parameters
|
|
71
|
+
----------
|
|
72
|
+
private_key : ec.EllipticCurvePrivateKey
|
|
73
|
+
The EC private key to sign the message with.
|
|
74
|
+
message : bytes
|
|
75
|
+
The message to be signed.
|
|
76
|
+
|
|
77
|
+
Returns
|
|
78
|
+
-------
|
|
79
|
+
bytes
|
|
80
|
+
The signature of the message.
|
|
81
|
+
"""
|
|
82
|
+
signature = private_key.sign(message, ec.ECDSA(hashes.SHA256()))
|
|
83
|
+
return signature
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def verify_signature(
|
|
87
|
+
public_key: ec.EllipticCurvePublicKey, message: bytes, signature: bytes
|
|
88
|
+
) -> bool:
|
|
89
|
+
"""Verify a signature against a message using the provided EC public key.
|
|
90
|
+
|
|
91
|
+
Parameters
|
|
92
|
+
----------
|
|
93
|
+
public_key : ec.EllipticCurvePublicKey
|
|
94
|
+
The EC public key to verify the signature.
|
|
95
|
+
message : bytes
|
|
96
|
+
The original message.
|
|
97
|
+
signature : bytes
|
|
98
|
+
The signature to verify.
|
|
99
|
+
|
|
100
|
+
Returns
|
|
101
|
+
-------
|
|
102
|
+
bool
|
|
103
|
+
True if the signature is valid, False otherwise.
|
|
104
|
+
"""
|
|
105
|
+
try:
|
|
106
|
+
public_key.verify(signature, message, ec.ECDSA(hashes.SHA256()))
|
|
107
|
+
return True
|
|
108
|
+
except InvalidSignature:
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def uses_nist_ec_curve(public_key: ec.EllipticCurvePublicKey) -> bool:
|
|
113
|
+
"""Return True if the provided key uses a NIST EC curve."""
|
|
114
|
+
return isinstance(
|
|
115
|
+
public_key.curve,
|
|
116
|
+
(ec.SECP192R1, ec.SECP224R1, ec.SECP256R1, ec.SECP384R1, ec.SECP521R1),
|
|
117
|
+
)
|