flwr 1.22.0__py3-none-any.whl → 1.23.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. flwr/cli/app.py +15 -1
  2. flwr/cli/auth_plugin/__init__.py +15 -6
  3. flwr/cli/auth_plugin/auth_plugin.py +95 -0
  4. flwr/cli/auth_plugin/noop_auth_plugin.py +58 -0
  5. flwr/cli/auth_plugin/oidc_cli_plugin.py +16 -25
  6. flwr/cli/build.py +118 -47
  7. flwr/cli/{cli_user_auth_interceptor.py → cli_account_auth_interceptor.py} +6 -5
  8. flwr/cli/log.py +2 -2
  9. flwr/cli/login/login.py +34 -23
  10. flwr/cli/ls.py +13 -9
  11. flwr/cli/new/new.py +187 -35
  12. flwr/cli/new/templates/app/pyproject.baseline.toml.tpl +1 -1
  13. flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +1 -1
  14. flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl +1 -1
  15. flwr/cli/new/templates/app/pyproject.jax.toml.tpl +1 -1
  16. flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +1 -1
  17. flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +1 -1
  18. flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +1 -1
  19. flwr/cli/new/templates/app/pyproject.pytorch_legacy_api.toml.tpl +1 -1
  20. flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +1 -1
  21. flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +1 -1
  22. flwr/cli/new/templates/app/pyproject.xgboost.toml.tpl +1 -1
  23. flwr/cli/pull.py +2 -2
  24. flwr/cli/run/run.py +11 -7
  25. flwr/cli/stop.py +2 -2
  26. flwr/cli/supernode/__init__.py +25 -0
  27. flwr/cli/supernode/ls.py +260 -0
  28. flwr/cli/supernode/register.py +185 -0
  29. flwr/cli/supernode/unregister.py +138 -0
  30. flwr/cli/utils.py +92 -69
  31. flwr/client/__init__.py +2 -1
  32. flwr/client/grpc_adapter_client/connection.py +6 -8
  33. flwr/client/grpc_rere_client/connection.py +59 -31
  34. flwr/client/grpc_rere_client/grpc_adapter.py +28 -12
  35. flwr/client/grpc_rere_client/{client_interceptor.py → node_auth_client_interceptor.py} +3 -6
  36. flwr/client/mod/secure_aggregation/secaggplus_mod.py +7 -5
  37. flwr/client/rest_client/connection.py +82 -37
  38. flwr/clientapp/__init__.py +1 -2
  39. flwr/{client/clientapp → clientapp}/utils.py +1 -1
  40. flwr/common/constant.py +53 -13
  41. flwr/common/exit/exit_code.py +20 -10
  42. flwr/common/inflatable_utils.py +10 -10
  43. flwr/common/record/array.py +3 -3
  44. flwr/common/record/arrayrecord.py +10 -1
  45. flwr/common/secure_aggregation/crypto/symmetric_encryption.py +1 -89
  46. flwr/common/serde.py +4 -2
  47. flwr/common/typing.py +7 -6
  48. flwr/compat/client/app.py +1 -1
  49. flwr/compat/client/grpc_client/connection.py +2 -2
  50. flwr/proto/control_pb2.py +48 -35
  51. flwr/proto/control_pb2.pyi +71 -5
  52. flwr/proto/control_pb2_grpc.py +102 -0
  53. flwr/proto/control_pb2_grpc.pyi +39 -0
  54. flwr/proto/fab_pb2.py +11 -7
  55. flwr/proto/fab_pb2.pyi +21 -1
  56. flwr/proto/fleet_pb2.py +31 -23
  57. flwr/proto/fleet_pb2.pyi +63 -23
  58. flwr/proto/fleet_pb2_grpc.py +98 -28
  59. flwr/proto/fleet_pb2_grpc.pyi +45 -13
  60. flwr/proto/node_pb2.py +3 -1
  61. flwr/proto/node_pb2.pyi +48 -0
  62. flwr/server/app.py +139 -114
  63. flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py +17 -7
  64. flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +132 -38
  65. flwr/server/superlink/fleet/grpc_rere/{server_interceptor.py → node_auth_server_interceptor.py} +27 -51
  66. flwr/server/superlink/fleet/message_handler/message_handler.py +67 -22
  67. flwr/server/superlink/fleet/rest_rere/rest_api.py +52 -31
  68. flwr/server/superlink/fleet/vce/backend/backend.py +1 -1
  69. flwr/server/superlink/fleet/vce/backend/raybackend.py +1 -1
  70. flwr/server/superlink/fleet/vce/vce_api.py +18 -5
  71. flwr/server/superlink/linkstate/in_memory_linkstate.py +167 -73
  72. flwr/server/superlink/linkstate/linkstate.py +107 -24
  73. flwr/server/superlink/linkstate/linkstate_factory.py +2 -1
  74. flwr/server/superlink/linkstate/sqlite_linkstate.py +306 -255
  75. flwr/server/superlink/linkstate/utils.py +3 -54
  76. flwr/server/superlink/serverappio/serverappio_servicer.py +2 -2
  77. flwr/server/superlink/simulation/simulationio_servicer.py +1 -1
  78. flwr/server/utils/validator.py +2 -3
  79. flwr/server/workflow/secure_aggregation/secaggplus_workflow.py +4 -2
  80. flwr/simulation/ray_transport/ray_actor.py +1 -1
  81. flwr/simulation/ray_transport/ray_client_proxy.py +1 -1
  82. flwr/simulation/run_simulation.py +3 -2
  83. flwr/supercore/constant.py +22 -0
  84. flwr/supercore/object_store/in_memory_object_store.py +0 -4
  85. flwr/supercore/object_store/object_store_factory.py +26 -6
  86. flwr/supercore/object_store/sqlite_object_store.py +252 -0
  87. flwr/{client/clientapp → supercore/primitives}/__init__.py +1 -1
  88. flwr/supercore/primitives/asymmetric.py +117 -0
  89. flwr/supercore/primitives/asymmetric_ed25519.py +165 -0
  90. flwr/supercore/sqlite_mixin.py +156 -0
  91. flwr/supercore/utils.py +20 -0
  92. flwr/{common → superlink}/auth_plugin/__init__.py +6 -6
  93. flwr/superlink/auth_plugin/auth_plugin.py +91 -0
  94. flwr/superlink/auth_plugin/noop_auth_plugin.py +87 -0
  95. flwr/superlink/servicer/control/{control_user_auth_interceptor.py → control_account_auth_interceptor.py} +19 -19
  96. flwr/superlink/servicer/control/control_event_log_interceptor.py +1 -1
  97. flwr/superlink/servicer/control/control_grpc.py +13 -11
  98. flwr/superlink/servicer/control/control_servicer.py +152 -60
  99. flwr/supernode/cli/flower_supernode.py +19 -26
  100. flwr/supernode/runtime/run_clientapp.py +2 -2
  101. flwr/supernode/servicer/clientappio/clientappio_servicer.py +1 -1
  102. flwr/supernode/start_client_internal.py +17 -9
  103. {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/METADATA +1 -1
  104. {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/RECORD +107 -96
  105. flwr/common/auth_plugin/auth_plugin.py +0 -149
  106. /flwr/{client → clientapp}/client_app.py +0 -0
  107. {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/WHEEL +0 -0
  108. {flwr-1.22.0.dist-info → flwr-1.23.0.dist-info}/entry_points.txt +0 -0
@@ -33,6 +33,7 @@ from flwr.common.typing import RunStatus
33
33
  # pylint: disable=E0611
34
34
  from flwr.proto.message_pb2 import Context as ProtoContext
35
35
  from flwr.proto.recorddict_pb2 import ConfigRecord as ProtoConfigRecord
36
+ from flwr.supercore.utils import int64_to_uint64, uint64_to_int64
36
37
 
37
38
  # pylint: enable=E0611
38
39
  VALID_RUN_STATUS_TRANSITIONS = {
@@ -76,58 +77,6 @@ def generate_rand_int_from_bytes(
76
77
  return num
77
78
 
78
79
 
79
- def convert_uint64_to_sint64(u: int) -> int:
80
- """Convert a uint64 value to a sint64 value with the same bit sequence.
81
-
82
- Parameters
83
- ----------
84
- u : int
85
- The unsigned 64-bit integer to convert.
86
-
87
- Returns
88
- -------
89
- int
90
- The signed 64-bit integer equivalent.
91
-
92
- The signed 64-bit integer will have the same bit pattern as the
93
- unsigned 64-bit integer but may have a different decimal value.
94
-
95
- For numbers within the range [0, `sint64` max value], the decimal
96
- value remains the same. However, for numbers greater than the `sint64`
97
- max value, the decimal value will differ due to the wraparound caused
98
- by the sign bit.
99
- """
100
- if u >= (1 << 63):
101
- return u - (1 << 64)
102
- return u
103
-
104
-
105
- def convert_sint64_to_uint64(s: int) -> int:
106
- """Convert a sint64 value to a uint64 value with the same bit sequence.
107
-
108
- Parameters
109
- ----------
110
- s : int
111
- The signed 64-bit integer to convert.
112
-
113
- Returns
114
- -------
115
- int
116
- The unsigned 64-bit integer equivalent.
117
-
118
- The unsigned 64-bit integer will have the same bit pattern as the
119
- signed 64-bit integer but may have a different decimal value.
120
-
121
- For negative `sint64` values, the conversion adds 2^64 to the
122
- signed value to obtain the equivalent `uint64` value. For non-negative
123
- `sint64` values, the decimal value remains unchanged in the `uint64`
124
- representation.
125
- """
126
- if s < 0:
127
- return s + (1 << 64)
128
- return s
129
-
130
-
131
80
  def convert_uint64_values_in_dict_to_sint64(
132
81
  data_dict: dict[str, int], keys: list[str]
133
82
  ) -> None:
@@ -142,7 +91,7 @@ def convert_uint64_values_in_dict_to_sint64(
142
91
  """
143
92
  for key in keys:
144
93
  if key in data_dict:
145
- data_dict[key] = convert_uint64_to_sint64(data_dict[key])
94
+ data_dict[key] = uint64_to_int64(data_dict[key])
146
95
 
147
96
 
148
97
  def convert_sint64_values_in_dict_to_uint64(
@@ -159,7 +108,7 @@ def convert_sint64_values_in_dict_to_uint64(
159
108
  """
160
109
  for key in keys:
161
110
  if key in data_dict:
162
- data_dict[key] = convert_sint64_to_uint64(data_dict[key])
111
+ data_dict[key] = int64_to_uint64(data_dict[key])
163
112
 
164
113
 
165
114
  def context_to_bytes(context: Context) -> bytes:
@@ -316,7 +316,7 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
316
316
 
317
317
  ffs: Ffs = self.ffs_factory.ffs()
318
318
  if result := ffs.get(request.hash_str):
319
- fab = Fab(request.hash_str, result[0])
319
+ fab = Fab(request.hash_str, result[0], result[1])
320
320
  return GetFabResponse(fab=fab_to_proto(fab))
321
321
 
322
322
  raise ValueError(f"Found no FAB with hash: {request.hash_str}")
@@ -343,7 +343,7 @@ class ServerAppIoServicer(serverappio_pb2_grpc.ServerAppIoServicer):
343
343
  fab = None
344
344
  if run and run.fab_hash:
345
345
  if result := ffs.get(run.fab_hash):
346
- fab = Fab(run.fab_hash, result[0])
346
+ fab = Fab(run.fab_hash, result[0], result[1])
347
347
  if run and fab and serverapp_ctxt:
348
348
  # Update run status to STARTING
349
349
  if state.update_run_status(run_id, RunStatus(Status.STARTING, "", "")):
@@ -150,7 +150,7 @@ class SimulationIoServicer(simulationio_pb2_grpc.SimulationIoServicer):
150
150
  fab = None
151
151
  if run and run.fab_hash:
152
152
  if result := ffs.get(run.fab_hash):
153
- fab = Fab(run.fab_hash, result[0])
153
+ fab = Fab(run.fab_hash, result[0], result[1])
154
154
  if run and fab and serverapp_ctxt:
155
155
  # Update run status to STARTING
156
156
  if state.update_run_status(run_id, RunStatus(Status.STARTING, "", "")):
@@ -15,10 +15,9 @@
15
15
  """Validators."""
16
16
 
17
17
 
18
- import time
19
-
20
18
  from flwr.common import Message
21
19
  from flwr.common.constant import SUPERLINK_NODE_ID
20
+ from flwr.common.date import now
22
21
 
23
22
 
24
23
  # pylint: disable-next=too-many-branches
@@ -44,7 +43,7 @@ def validate_message(message: Message, is_reply_message: bool) -> list[str]:
44
43
  validation_errors.append("`metadata.ttl` must be higher than zero")
45
44
 
46
45
  # Verify TTL and created_at time
47
- current_time = time.time()
46
+ current_time = now().timestamp()
48
47
  if metadata.created_at + metadata.ttl <= current_time:
49
48
  validation_errors.append("Message TTL has expired")
50
49
 
@@ -35,8 +35,6 @@ from flwr.common import (
35
35
  )
36
36
  from flwr.common.secure_aggregation.crypto.shamir import combine_shares
37
37
  from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
38
- bytes_to_private_key,
39
- bytes_to_public_key,
40
38
  generate_shared_key,
41
39
  )
42
40
  from flwr.common.secure_aggregation.ndarrays_arithmetic import (
@@ -56,6 +54,10 @@ from flwr.common.secure_aggregation.secaggplus_utils import pseudo_rand_gen
56
54
  from flwr.server.client_proxy import ClientProxy
57
55
  from flwr.server.compat.legacy_context import LegacyContext
58
56
  from flwr.server.grid import Grid
57
+ from flwr.supercore.primitives.asymmetric import (
58
+ bytes_to_private_key,
59
+ bytes_to_public_key,
60
+ )
59
61
 
60
62
  from ..constant import MAIN_CONFIGS_RECORD, MAIN_PARAMS_RECORD
61
63
  from ..constant import Key as WorkflowKey
@@ -24,7 +24,7 @@ import ray
24
24
  from ray import ObjectRef
25
25
  from ray.util.actor_pool import ActorPool
26
26
 
27
- from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError
27
+ from flwr.clientapp.client_app import ClientApp, ClientAppException, LoadClientAppError
28
28
  from flwr.common import Context, Message
29
29
  from flwr.common.logger import log
30
30
 
@@ -21,8 +21,8 @@ from typing import Optional
21
21
 
22
22
  from flwr import common
23
23
  from flwr.client import ClientFnExt
24
- from flwr.client.client_app import ClientApp
25
24
  from flwr.client.run_info_store import DeprecatedRunInfoStore
25
+ from flwr.clientapp.client_app import ClientApp
26
26
  from flwr.common import DEFAULT_TTL, Message, Metadata, RecordDict, now
27
27
  from flwr.common.constant import (
28
28
  NUM_PARTITIONS_KEY,
@@ -30,7 +30,7 @@ from typing import Any, Optional
30
30
 
31
31
  from flwr.cli.config_utils import load_and_validate
32
32
  from flwr.cli.utils import get_sha256_hash
33
- from flwr.client import ClientApp
33
+ from flwr.clientapp import ClientApp
34
34
  from flwr.common import Context, EventType, RecordDict, event, log, now
35
35
  from flwr.common.config import get_fused_config_from_dir, parse_config_args
36
36
  from flwr.common.constant import RUN_ID_NUM_BYTES, Status
@@ -51,6 +51,7 @@ from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
51
51
  from flwr.simulation.ray_transport.utils import (
52
52
  enable_tf_gpu_growth as enable_gpu_growth,
53
53
  )
54
+ from flwr.supercore.constant import FLWR_IN_MEMORY_DB_NAME
54
55
 
55
56
 
56
57
  def _replace_keys(d: Any, match: str, target: str) -> Any:
@@ -336,7 +337,7 @@ def _main_loop(
336
337
  ) -> Context:
337
338
  """Start ServerApp on a separate thread, then launch Simulation Engine."""
338
339
  # Initialize StateFactory
339
- state_factory = LinkStateFactory(":flwr-in-memory-state:")
340
+ state_factory = LinkStateFactory(FLWR_IN_MEMORY_DB_NAME)
340
341
 
341
342
  f_stop = threading.Event()
342
343
  # A Threading event to indicate if an exception was raised in the ServerApp thread
@@ -15,5 +15,27 @@
15
15
  """Constants for Flower infrastructure."""
16
16
 
17
17
 
18
+ from __future__ import annotations
19
+
18
20
  # Top-level key in YAML config for exec plugin settings
19
21
  EXEC_PLUGIN_SECTION = "exec_plugin"
22
+
23
+ # Flower in-memory Python-based database name
24
+ FLWR_IN_MEMORY_DB_NAME = ":flwr-in-memory:"
25
+
26
+ # Constants for Hub
27
+ APP_ID_PATTERN = r"^@(?P<user>[^/]+)/(?P<app>[^/]+)$"
28
+ PLATFORM_API_URL = "https://api.flower.ai/v1"
29
+
30
+
31
+ class NodeStatus:
32
+ """Event log writer types."""
33
+
34
+ REGISTERED = "registered"
35
+ ONLINE = "online"
36
+ OFFLINE = "offline"
37
+ UNREGISTERED = "unregistered"
38
+
39
+ def __new__(cls) -> NodeStatus:
40
+ """Prevent instantiation."""
41
+ raise TypeError(f"{cls.__name__} cannot be instantiated.")
@@ -48,9 +48,6 @@ class InMemoryObjectStore(ObjectStore):
48
48
  self.verify = verify
49
49
  self.store: dict[str, ObjectEntry] = {}
50
50
  self.lock_store = threading.RLock()
51
- # Mapping the Object ID of a message to the list of descendant object IDs
52
- self.msg_descendant_objects_mapping: dict[str, list[str]] = {}
53
- self.lock_msg_mapping = threading.RLock()
54
51
  # Mapping each run ID to a set of object IDs that are used in that run
55
52
  self.run_objects_mapping: dict[int, set[str]] = {}
56
53
 
@@ -215,7 +212,6 @@ class InMemoryObjectStore(ObjectStore):
215
212
  """Clear the store."""
216
213
  with self.lock_store:
217
214
  self.store.clear()
218
- self.msg_descendant_objects_mapping.clear()
219
215
  self.run_objects_mapping.clear()
220
216
 
221
217
  def __contains__(self, object_id: str) -> bool:
@@ -19,15 +19,27 @@ from logging import DEBUG
19
19
  from typing import Optional
20
20
 
21
21
  from flwr.common.logger import log
22
+ from flwr.supercore.constant import FLWR_IN_MEMORY_DB_NAME
22
23
 
23
24
  from .in_memory_object_store import InMemoryObjectStore
24
25
  from .object_store import ObjectStore
26
+ from .sqlite_object_store import SqliteObjectStore
25
27
 
26
28
 
27
29
  class ObjectStoreFactory:
28
- """Factory class that creates ObjectStore instances."""
30
+ """Factory class that creates ObjectStore instances.
29
31
 
30
- def __init__(self) -> None:
32
+ Parameters
33
+ ----------
34
+ database : str (default: FLWR_IN_MEMORY_DB_NAME)
35
+ A string representing the path to the database file that will be opened.
36
+ Note that passing ":memory:" will open a connection to a database that is
37
+ in RAM, instead of on disk. And FLWR_IN_MEMORY_DB_NAME will create an
38
+ Python-based in-memory ObjectStore.
39
+ """
40
+
41
+ def __init__(self, database: str = FLWR_IN_MEMORY_DB_NAME) -> None:
42
+ self.database = database
31
43
  self.store_instance: Optional[ObjectStore] = None
32
44
 
33
45
  def store(self) -> ObjectStore:
@@ -38,7 +50,15 @@ class ObjectStoreFactory:
38
50
  ObjectStore
39
51
  An ObjectStore instance for storing objects by object_id.
40
52
  """
41
- if self.store_instance is None:
42
- self.store_instance = InMemoryObjectStore()
43
- log(DEBUG, "Using InMemoryObjectStore")
44
- return self.store_instance
53
+ # InMemoryObjectStore
54
+ if self.database == FLWR_IN_MEMORY_DB_NAME:
55
+ if self.store_instance is None:
56
+ self.store_instance = InMemoryObjectStore()
57
+ log(DEBUG, "Using InMemoryObjectStore")
58
+ return self.store_instance
59
+
60
+ # SqliteObjectStore
61
+ store = SqliteObjectStore(self.database)
62
+ store.initialize()
63
+ log(DEBUG, "Using SqliteObjectStore")
64
+ return store
@@ -0,0 +1,252 @@
1
+ # Copyright 2025 Flower Labs GmbH. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ==============================================================================
15
+ """Flower SQLite ObjectStore implementation."""
16
+
17
+
18
+ from typing import Optional, cast
19
+
20
+ from flwr.common.inflatable import (
21
+ get_object_id,
22
+ is_valid_sha256_hash,
23
+ iterate_object_tree,
24
+ )
25
+ from flwr.common.inflatable_utils import validate_object_content
26
+ from flwr.proto.message_pb2 import ObjectTree # pylint: disable=E0611
27
+ from flwr.supercore.sqlite_mixin import SqliteMixin
28
+ from flwr.supercore.utils import uint64_to_int64
29
+
30
+ from .object_store import NoObjectInStoreError, ObjectStore
31
+
32
+ SQL_CREATE_OBJECTS = """
33
+ CREATE TABLE IF NOT EXISTS objects (
34
+ object_id TEXT PRIMARY KEY,
35
+ content BLOB,
36
+ is_available INTEGER NOT NULL CHECK (is_available IN (0,1)),
37
+ ref_count INTEGER NOT NULL
38
+ );
39
+ """
40
+ SQL_CREATE_OBJECT_CHILDREN = """
41
+ CREATE TABLE IF NOT EXISTS object_children (
42
+ parent_id TEXT NOT NULL,
43
+ child_id TEXT NOT NULL,
44
+ FOREIGN KEY (parent_id) REFERENCES objects(object_id) ON DELETE CASCADE,
45
+ FOREIGN KEY (child_id) REFERENCES objects(object_id) ON DELETE CASCADE,
46
+ PRIMARY KEY (parent_id, child_id)
47
+ );
48
+ """
49
+ SQL_CREATE_RUN_OBJECTS = """
50
+ CREATE TABLE IF NOT EXISTS run_objects (
51
+ run_id INTEGER NOT NULL,
52
+ object_id TEXT NOT NULL,
53
+ FOREIGN KEY (object_id) REFERENCES objects(object_id) ON DELETE CASCADE,
54
+ PRIMARY KEY (run_id, object_id)
55
+ );
56
+ """
57
+
58
+
59
+ class SqliteObjectStore(ObjectStore, SqliteMixin):
60
+ """SQLite-based implementation of the ObjectStore interface."""
61
+
62
+ def __init__(self, database_path: str, verify: bool = True) -> None:
63
+ super().__init__(database_path)
64
+ self.verify = verify
65
+
66
+ def initialize(self, log_queries: bool = False) -> list[tuple[str]]:
67
+ """Connect to the DB, enable FK support, and create tables if needed."""
68
+ return self._ensure_initialized(
69
+ SQL_CREATE_OBJECTS,
70
+ SQL_CREATE_OBJECT_CHILDREN,
71
+ SQL_CREATE_RUN_OBJECTS,
72
+ log_queries=log_queries,
73
+ )
74
+
75
+ def preregister(self, run_id: int, object_tree: ObjectTree) -> list[str]:
76
+ """Identify and preregister missing objects in the `ObjectStore`."""
77
+ new_objects = []
78
+ for tree_node in iterate_object_tree(object_tree):
79
+ obj_id = tree_node.object_id
80
+ if not is_valid_sha256_hash(obj_id):
81
+ raise ValueError(f"Invalid object ID format: {obj_id}")
82
+
83
+ child_ids = [child.object_id for child in tree_node.children]
84
+ with self.conn:
85
+ row = self.conn.execute(
86
+ "SELECT object_id, is_available FROM objects WHERE object_id=?",
87
+ (obj_id,),
88
+ ).fetchone()
89
+ if row is None:
90
+ # Insert new object
91
+ self.conn.execute(
92
+ "INSERT INTO objects"
93
+ "(object_id, content, is_available, ref_count) "
94
+ "VALUES (?, ?, ?, ?)",
95
+ (obj_id, b"", 0, 0),
96
+ )
97
+ for cid in child_ids:
98
+ self.conn.execute(
99
+ "INSERT INTO object_children(parent_id, child_id) "
100
+ "VALUES (?, ?)",
101
+ (obj_id, cid),
102
+ )
103
+ self.conn.execute(
104
+ "UPDATE objects SET ref_count = ref_count + 1 "
105
+ "WHERE object_id = ?",
106
+ (cid,),
107
+ )
108
+ new_objects.append(obj_id)
109
+ else:
110
+ # Add to the list of new objects if not available
111
+ if not row["is_available"]:
112
+ new_objects.append(obj_id)
113
+
114
+ # Ensure run mapping
115
+ self.conn.execute(
116
+ "INSERT OR IGNORE INTO run_objects(run_id, object_id) "
117
+ "VALUES (?, ?)",
118
+ (uint64_to_int64(run_id), obj_id),
119
+ )
120
+ return new_objects
121
+
122
+ def get_object_tree(self, object_id: str) -> ObjectTree:
123
+ """Get the object tree for a given object ID."""
124
+ with self.conn:
125
+ row = self.conn.execute(
126
+ "SELECT object_id FROM objects WHERE object_id=?", (object_id,)
127
+ ).fetchone()
128
+ if not row:
129
+ raise NoObjectInStoreError(f"Object {object_id} not found.")
130
+ children = self.query(
131
+ "SELECT child_id FROM object_children WHERE parent_id=?", (object_id,)
132
+ )
133
+
134
+ # Build the object trees of all children
135
+ try:
136
+ child_trees = [self.get_object_tree(ch["child_id"]) for ch in children]
137
+ except NoObjectInStoreError as e:
138
+ # Raise an error if any child object is missing
139
+ # This indicates an integrity issue
140
+ raise NoObjectInStoreError(
141
+ f"Object tree for object ID '{object_id}' contains missing "
142
+ "children. This may indicate a corrupted object store."
143
+ ) from e
144
+
145
+ # Create and return the ObjectTree for the current object
146
+ return ObjectTree(object_id=object_id, children=child_trees)
147
+
148
+ def put(self, object_id: str, object_content: bytes) -> None:
149
+ """Put an object into the store."""
150
+ if self.verify:
151
+ # Verify object_id and object_content match
152
+ object_id_from_content = get_object_id(object_content)
153
+ if object_id != object_id_from_content:
154
+ raise ValueError(f"Object ID {object_id} does not match content hash")
155
+
156
+ # Validate object content
157
+ validate_object_content(content=object_content)
158
+
159
+ with self.conn:
160
+ # Only allow adding the object if it has been preregistered
161
+ row = self.conn.execute(
162
+ "SELECT is_available FROM objects WHERE object_id=?", (object_id,)
163
+ ).fetchone()
164
+ if row is None:
165
+ raise NoObjectInStoreError(
166
+ f"Object with ID '{object_id}' was not pre-registered."
167
+ )
168
+
169
+ # Return if object is already present in the store
170
+ if row["is_available"]:
171
+ return
172
+
173
+ # Update the object entry in the store
174
+ self.conn.execute(
175
+ "UPDATE objects SET content=?, is_available=1 WHERE object_id=?",
176
+ (object_content, object_id),
177
+ )
178
+
179
+ def get(self, object_id: str) -> Optional[bytes]:
180
+ """Get an object from the store."""
181
+ rows = self.query("SELECT content FROM objects WHERE object_id=?", (object_id,))
182
+ return rows[0]["content"] if rows else None
183
+
184
+ def delete(self, object_id: str) -> None:
185
+ """Delete an object and its unreferenced descendants from the store."""
186
+ with self.conn:
187
+ row = self.conn.execute(
188
+ "SELECT ref_count FROM objects WHERE object_id=?", (object_id,)
189
+ ).fetchone()
190
+
191
+ # If the object is not in the store, nothing to delete
192
+ if row is None:
193
+ return
194
+
195
+ # Skip deletion if there are still references
196
+ if row["ref_count"] > 0:
197
+ return
198
+
199
+ # Deleting will cascade via FK, but we need to decrement children first
200
+ children = self.conn.execute(
201
+ "SELECT child_id FROM object_children WHERE parent_id=?", (object_id,)
202
+ ).fetchall()
203
+ child_ids = [child["child_id"] for child in children]
204
+
205
+ if child_ids:
206
+ placeholders = ", ".join("?" for _ in child_ids)
207
+ query = f"""
208
+ UPDATE objects SET ref_count = ref_count - 1
209
+ WHERE object_id IN ({placeholders})
210
+ """
211
+ self.conn.execute(query, child_ids)
212
+
213
+ self.conn.execute("DELETE FROM objects WHERE object_id=?", (object_id,))
214
+
215
+ # Recursively clean children
216
+ for child_id in child_ids:
217
+ self.delete(child_id)
218
+
219
+ def delete_objects_in_run(self, run_id: int) -> None:
220
+ """Delete all objects that were registered in a specific run."""
221
+ run_id_sint = uint64_to_int64(run_id)
222
+ with self.conn:
223
+ objs = self.conn.execute(
224
+ "SELECT object_id FROM run_objects WHERE run_id=?", (run_id_sint,)
225
+ ).fetchall()
226
+ for obj in objs:
227
+ object_id = obj["object_id"]
228
+ row = self.conn.execute(
229
+ "SELECT ref_count FROM objects WHERE object_id=?", (object_id,)
230
+ ).fetchone()
231
+ if row and row["ref_count"] == 0:
232
+ self.delete(object_id)
233
+ self.conn.execute("DELETE FROM run_objects WHERE run_id=?", (run_id_sint,))
234
+
235
+ def clear(self) -> None:
236
+ """Clear the store."""
237
+ with self.conn:
238
+ self.conn.execute("DELETE FROM object_children;")
239
+ self.conn.execute("DELETE FROM run_objects;")
240
+ self.conn.execute("DELETE FROM objects;")
241
+
242
+ def __contains__(self, object_id: str) -> bool:
243
+ """Check if an object_id is in the store."""
244
+ row = self.conn.execute(
245
+ "SELECT 1 FROM objects WHERE object_id=?", (object_id,)
246
+ ).fetchone()
247
+ return row is not None
248
+
249
+ def __len__(self) -> int:
250
+ """Return the number of objects in the store."""
251
+ row = self.conn.execute("SELECT COUNT(*) AS cnt FROM objects;").fetchone()
252
+ return cast(int, row["cnt"])
@@ -12,4 +12,4 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  # ==============================================================================
15
- """Flower AppIO service."""
15
+ """Cryptographic primitives for the Flower infrastructure."""
@@ -0,0 +1,117 @@
1
+ # Copyright 2025 Flower Labs GmbH. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ==============================================================================
15
+ """Asymmetric cryptography utilities."""
16
+
17
+
18
+ from typing import cast
19
+
20
+ from cryptography.exceptions import InvalidSignature
21
+ from cryptography.hazmat.primitives import hashes, serialization
22
+ from cryptography.hazmat.primitives.asymmetric import ec
23
+
24
+
25
+ def generate_key_pairs() -> (
26
+ tuple[ec.EllipticCurvePrivateKey, ec.EllipticCurvePublicKey]
27
+ ):
28
+ """Generate private and public key pairs with Cryptography."""
29
+ private_key = ec.generate_private_key(ec.SECP384R1())
30
+ public_key = private_key.public_key()
31
+ return private_key, public_key
32
+
33
+
34
+ def private_key_to_bytes(private_key: ec.EllipticCurvePrivateKey) -> bytes:
35
+ """Serialize private key to bytes."""
36
+ return private_key.private_bytes(
37
+ encoding=serialization.Encoding.PEM,
38
+ format=serialization.PrivateFormat.PKCS8,
39
+ encryption_algorithm=serialization.NoEncryption(),
40
+ )
41
+
42
+
43
+ def bytes_to_private_key(private_key_bytes: bytes) -> ec.EllipticCurvePrivateKey:
44
+ """Deserialize private key from bytes."""
45
+ return cast(
46
+ ec.EllipticCurvePrivateKey,
47
+ serialization.load_pem_private_key(data=private_key_bytes, password=None),
48
+ )
49
+
50
+
51
+ def public_key_to_bytes(public_key: ec.EllipticCurvePublicKey) -> bytes:
52
+ """Serialize public key to bytes."""
53
+ return public_key.public_bytes(
54
+ encoding=serialization.Encoding.PEM,
55
+ format=serialization.PublicFormat.SubjectPublicKeyInfo,
56
+ )
57
+
58
+
59
+ def bytes_to_public_key(public_key_bytes: bytes) -> ec.EllipticCurvePublicKey:
60
+ """Deserialize public key from bytes."""
61
+ return cast(
62
+ ec.EllipticCurvePublicKey,
63
+ serialization.load_pem_public_key(data=public_key_bytes),
64
+ )
65
+
66
+
67
+ def sign_message(private_key: ec.EllipticCurvePrivateKey, message: bytes) -> bytes:
68
+ """Sign a message using the provided EC private key.
69
+
70
+ Parameters
71
+ ----------
72
+ private_key : ec.EllipticCurvePrivateKey
73
+ The EC private key to sign the message with.
74
+ message : bytes
75
+ The message to be signed.
76
+
77
+ Returns
78
+ -------
79
+ bytes
80
+ The signature of the message.
81
+ """
82
+ signature = private_key.sign(message, ec.ECDSA(hashes.SHA256()))
83
+ return signature
84
+
85
+
86
+ def verify_signature(
87
+ public_key: ec.EllipticCurvePublicKey, message: bytes, signature: bytes
88
+ ) -> bool:
89
+ """Verify a signature against a message using the provided EC public key.
90
+
91
+ Parameters
92
+ ----------
93
+ public_key : ec.EllipticCurvePublicKey
94
+ The EC public key to verify the signature.
95
+ message : bytes
96
+ The original message.
97
+ signature : bytes
98
+ The signature to verify.
99
+
100
+ Returns
101
+ -------
102
+ bool
103
+ True if the signature is valid, False otherwise.
104
+ """
105
+ try:
106
+ public_key.verify(signature, message, ec.ECDSA(hashes.SHA256()))
107
+ return True
108
+ except InvalidSignature:
109
+ return False
110
+
111
+
112
+ def uses_nist_ec_curve(public_key: ec.EllipticCurvePublicKey) -> bool:
113
+ """Return True if the provided key uses a NIST EC curve."""
114
+ return isinstance(
115
+ public_key.curve,
116
+ (ec.SECP192R1, ec.SECP224R1, ec.SECP256R1, ec.SECP384R1, ec.SECP521R1),
117
+ )