flwr 1.21.0__py3-none-any.whl → 1.23.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. flwr/cli/app.py +17 -1
  2. flwr/cli/auth_plugin/__init__.py +15 -6
  3. flwr/cli/auth_plugin/auth_plugin.py +95 -0
  4. flwr/cli/auth_plugin/noop_auth_plugin.py +58 -0
  5. flwr/cli/auth_plugin/oidc_cli_plugin.py +16 -25
  6. flwr/cli/build.py +118 -47
  7. flwr/cli/{cli_user_auth_interceptor.py → cli_account_auth_interceptor.py} +6 -5
  8. flwr/cli/log.py +2 -2
  9. flwr/cli/login/login.py +34 -23
  10. flwr/cli/ls.py +13 -9
  11. flwr/cli/new/new.py +196 -42
  12. flwr/cli/new/templates/app/README.flowertune.md.tpl +1 -1
  13. flwr/cli/new/templates/app/code/client.baseline.py.tpl +64 -47
  14. flwr/cli/new/templates/app/code/client.huggingface.py.tpl +68 -30
  15. flwr/cli/new/templates/app/code/client.jax.py.tpl +63 -42
  16. flwr/cli/new/templates/app/code/client.mlx.py.tpl +80 -51
  17. flwr/cli/new/templates/app/code/client.numpy.py.tpl +36 -13
  18. flwr/cli/new/templates/app/code/client.pytorch.py.tpl +71 -46
  19. flwr/cli/new/templates/app/code/client.pytorch_legacy_api.py.tpl +55 -0
  20. flwr/cli/new/templates/app/code/client.sklearn.py.tpl +75 -30
  21. flwr/cli/new/templates/app/code/client.tensorflow.py.tpl +69 -44
  22. flwr/cli/new/templates/app/code/client.xgboost.py.tpl +110 -0
  23. flwr/cli/new/templates/app/code/flwr_tune/client_app.py.tpl +56 -90
  24. flwr/cli/new/templates/app/code/flwr_tune/models.py.tpl +1 -23
  25. flwr/cli/new/templates/app/code/flwr_tune/server_app.py.tpl +37 -58
  26. flwr/cli/new/templates/app/code/flwr_tune/strategy.py.tpl +39 -44
  27. flwr/cli/new/templates/app/code/model.baseline.py.tpl +0 -14
  28. flwr/cli/new/templates/app/code/server.baseline.py.tpl +27 -29
  29. flwr/cli/new/templates/app/code/server.huggingface.py.tpl +23 -19
  30. flwr/cli/new/templates/app/code/server.jax.py.tpl +27 -14
  31. flwr/cli/new/templates/app/code/server.mlx.py.tpl +29 -19
  32. flwr/cli/new/templates/app/code/server.numpy.py.tpl +30 -17
  33. flwr/cli/new/templates/app/code/server.pytorch.py.tpl +36 -26
  34. flwr/cli/new/templates/app/code/server.pytorch_legacy_api.py.tpl +31 -0
  35. flwr/cli/new/templates/app/code/server.sklearn.py.tpl +29 -21
  36. flwr/cli/new/templates/app/code/server.tensorflow.py.tpl +28 -19
  37. flwr/cli/new/templates/app/code/server.xgboost.py.tpl +56 -0
  38. flwr/cli/new/templates/app/code/task.huggingface.py.tpl +16 -20
  39. flwr/cli/new/templates/app/code/task.jax.py.tpl +1 -1
  40. flwr/cli/new/templates/app/code/task.numpy.py.tpl +1 -1
  41. flwr/cli/new/templates/app/code/task.pytorch.py.tpl +14 -27
  42. flwr/cli/new/templates/app/code/{task.pytorch_msg_api.py.tpl → task.pytorch_legacy_api.py.tpl} +27 -14
  43. flwr/cli/new/templates/app/code/task.tensorflow.py.tpl +1 -2
  44. flwr/cli/new/templates/app/code/task.xgboost.py.tpl +67 -0
  45. flwr/cli/new/templates/app/pyproject.baseline.toml.tpl +4 -4
  46. flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +2 -2
  47. flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl +4 -4
  48. flwr/cli/new/templates/app/pyproject.jax.toml.tpl +1 -1
  49. flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +2 -2
  50. flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +1 -1
  51. flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +3 -3
  52. flwr/cli/new/templates/app/{pyproject.pytorch_msg_api.toml.tpl → pyproject.pytorch_legacy_api.toml.tpl} +3 -3
  53. flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +1 -1
  54. flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +1 -1
  55. flwr/cli/new/templates/app/pyproject.xgboost.toml.tpl +61 -0
  56. flwr/cli/pull.py +100 -0
  57. flwr/cli/run/run.py +11 -7
  58. flwr/cli/stop.py +2 -2
  59. flwr/cli/supernode/__init__.py +25 -0
  60. flwr/cli/supernode/ls.py +260 -0
  61. flwr/cli/supernode/register.py +185 -0
  62. flwr/cli/supernode/unregister.py +138 -0
  63. flwr/cli/utils.py +109 -69
  64. flwr/client/__init__.py +2 -1
  65. flwr/client/grpc_adapter_client/connection.py +6 -8
  66. flwr/client/grpc_rere_client/connection.py +59 -31
  67. flwr/client/grpc_rere_client/grpc_adapter.py +28 -12
  68. flwr/client/grpc_rere_client/{client_interceptor.py → node_auth_client_interceptor.py} +3 -6
  69. flwr/client/mod/secure_aggregation/secaggplus_mod.py +7 -5
  70. flwr/client/rest_client/connection.py +82 -37
  71. flwr/clientapp/__init__.py +1 -2
  72. flwr/clientapp/mod/__init__.py +4 -1
  73. flwr/clientapp/mod/centraldp_mods.py +156 -40
  74. flwr/clientapp/mod/localdp_mod.py +169 -0
  75. flwr/clientapp/typing.py +22 -0
  76. flwr/{client/clientapp → clientapp}/utils.py +1 -1
  77. flwr/common/constant.py +56 -13
  78. flwr/common/exit/exit_code.py +24 -10
  79. flwr/common/inflatable_utils.py +10 -10
  80. flwr/common/record/array.py +3 -3
  81. flwr/common/record/arrayrecord.py +10 -1
  82. flwr/common/record/typeddict.py +12 -0
  83. flwr/common/secure_aggregation/crypto/symmetric_encryption.py +1 -89
  84. flwr/common/serde.py +4 -2
  85. flwr/common/typing.py +7 -6
  86. flwr/compat/client/app.py +1 -1
  87. flwr/compat/client/grpc_client/connection.py +2 -2
  88. flwr/proto/control_pb2.py +48 -31
  89. flwr/proto/control_pb2.pyi +95 -5
  90. flwr/proto/control_pb2_grpc.py +136 -0
  91. flwr/proto/control_pb2_grpc.pyi +52 -0
  92. flwr/proto/fab_pb2.py +11 -7
  93. flwr/proto/fab_pb2.pyi +21 -1
  94. flwr/proto/fleet_pb2.py +31 -23
  95. flwr/proto/fleet_pb2.pyi +63 -23
  96. flwr/proto/fleet_pb2_grpc.py +98 -28
  97. flwr/proto/fleet_pb2_grpc.pyi +45 -13
  98. flwr/proto/node_pb2.py +3 -1
  99. flwr/proto/node_pb2.pyi +48 -0
  100. flwr/server/app.py +152 -114
  101. flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py +17 -7
  102. flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +132 -38
  103. flwr/server/superlink/fleet/grpc_rere/{server_interceptor.py → node_auth_server_interceptor.py} +27 -51
  104. flwr/server/superlink/fleet/message_handler/message_handler.py +67 -22
  105. flwr/server/superlink/fleet/rest_rere/rest_api.py +52 -31
  106. flwr/server/superlink/fleet/vce/backend/backend.py +1 -1
  107. flwr/server/superlink/fleet/vce/backend/raybackend.py +1 -1
  108. flwr/server/superlink/fleet/vce/vce_api.py +18 -5
  109. flwr/server/superlink/linkstate/in_memory_linkstate.py +167 -73
  110. flwr/server/superlink/linkstate/linkstate.py +107 -24
  111. flwr/server/superlink/linkstate/linkstate_factory.py +2 -1
  112. flwr/server/superlink/linkstate/sqlite_linkstate.py +306 -255
  113. flwr/server/superlink/linkstate/utils.py +3 -54
  114. flwr/server/superlink/serverappio/serverappio_servicer.py +2 -2
  115. flwr/server/superlink/simulation/simulationio_servicer.py +1 -1
  116. flwr/server/utils/validator.py +2 -3
  117. flwr/server/workflow/secure_aggregation/secaggplus_workflow.py +4 -2
  118. flwr/serverapp/strategy/__init__.py +26 -0
  119. flwr/serverapp/strategy/bulyan.py +238 -0
  120. flwr/serverapp/strategy/dp_adaptive_clipping.py +335 -0
  121. flwr/serverapp/strategy/dp_fixed_clipping.py +71 -49
  122. flwr/serverapp/strategy/fedadagrad.py +0 -3
  123. flwr/serverapp/strategy/fedadam.py +0 -3
  124. flwr/serverapp/strategy/fedavg.py +89 -64
  125. flwr/serverapp/strategy/fedavgm.py +198 -0
  126. flwr/serverapp/strategy/fedmedian.py +105 -0
  127. flwr/serverapp/strategy/fedprox.py +174 -0
  128. flwr/serverapp/strategy/fedtrimmedavg.py +176 -0
  129. flwr/serverapp/strategy/fedxgb_bagging.py +117 -0
  130. flwr/serverapp/strategy/fedxgb_cyclic.py +220 -0
  131. flwr/serverapp/strategy/fedyogi.py +0 -3
  132. flwr/serverapp/strategy/krum.py +112 -0
  133. flwr/serverapp/strategy/multikrum.py +247 -0
  134. flwr/serverapp/strategy/qfedavg.py +252 -0
  135. flwr/serverapp/strategy/strategy_utils.py +48 -0
  136. flwr/simulation/app.py +1 -1
  137. flwr/simulation/ray_transport/ray_actor.py +1 -1
  138. flwr/simulation/ray_transport/ray_client_proxy.py +1 -1
  139. flwr/simulation/run_simulation.py +28 -32
  140. flwr/supercore/cli/flower_superexec.py +26 -1
  141. flwr/supercore/constant.py +41 -0
  142. flwr/supercore/object_store/in_memory_object_store.py +0 -4
  143. flwr/supercore/object_store/object_store_factory.py +26 -6
  144. flwr/supercore/object_store/sqlite_object_store.py +252 -0
  145. flwr/{client/clientapp → supercore/primitives}/__init__.py +1 -1
  146. flwr/supercore/primitives/asymmetric.py +117 -0
  147. flwr/supercore/primitives/asymmetric_ed25519.py +165 -0
  148. flwr/supercore/sqlite_mixin.py +156 -0
  149. flwr/supercore/superexec/plugin/exec_plugin.py +11 -1
  150. flwr/supercore/superexec/run_superexec.py +16 -2
  151. flwr/supercore/utils.py +20 -0
  152. flwr/superlink/artifact_provider/__init__.py +22 -0
  153. flwr/superlink/artifact_provider/artifact_provider.py +37 -0
  154. flwr/{common → superlink}/auth_plugin/__init__.py +6 -6
  155. flwr/superlink/auth_plugin/auth_plugin.py +91 -0
  156. flwr/superlink/auth_plugin/noop_auth_plugin.py +87 -0
  157. flwr/superlink/servicer/control/{control_user_auth_interceptor.py → control_account_auth_interceptor.py} +19 -19
  158. flwr/superlink/servicer/control/control_event_log_interceptor.py +1 -1
  159. flwr/superlink/servicer/control/control_grpc.py +16 -11
  160. flwr/superlink/servicer/control/control_servicer.py +207 -58
  161. flwr/supernode/cli/flower_supernode.py +19 -26
  162. flwr/supernode/runtime/run_clientapp.py +2 -2
  163. flwr/supernode/servicer/clientappio/clientappio_servicer.py +1 -1
  164. flwr/supernode/start_client_internal.py +17 -9
  165. {flwr-1.21.0.dist-info → flwr-1.23.0.dist-info}/METADATA +6 -16
  166. {flwr-1.21.0.dist-info → flwr-1.23.0.dist-info}/RECORD +170 -140
  167. flwr/cli/new/templates/app/code/client.pytorch_msg_api.py.tpl +0 -80
  168. flwr/cli/new/templates/app/code/server.pytorch_msg_api.py.tpl +0 -41
  169. flwr/common/auth_plugin/auth_plugin.py +0 -149
  170. flwr/serverapp/dp_fixed_clipping.py +0 -352
  171. flwr/serverapp/strategy/strategy_utils_tests.py +0 -304
  172. /flwr/cli/new/templates/app/code/{__init__.pytorch_msg_api.py.tpl → __init__.pytorch_legacy_api.py.tpl} +0 -0
  173. /flwr/{client → clientapp}/client_app.py +0 -0
  174. {flwr-1.21.0.dist-info → flwr-1.23.0.dist-info}/WHEEL +0 -0
  175. {flwr-1.21.0.dist-info → flwr-1.23.0.dist-info}/entry_points.txt +0 -0
@@ -24,7 +24,7 @@ import ray
24
24
  from ray import ObjectRef
25
25
  from ray.util.actor_pool import ActorPool
26
26
 
27
- from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError
27
+ from flwr.clientapp.client_app import ClientApp, ClientAppException, LoadClientAppError
28
28
  from flwr.common import Context, Message
29
29
  from flwr.common.logger import log
30
30
 
@@ -21,8 +21,8 @@ from typing import Optional
21
21
 
22
22
  from flwr import common
23
23
  from flwr.client import ClientFnExt
24
- from flwr.client.client_app import ClientApp
25
24
  from flwr.client.run_info_store import DeprecatedRunInfoStore
25
+ from flwr.clientapp.client_app import ClientApp
26
26
  from flwr.common import DEFAULT_TTL, Message, Metadata, RecordDict, now
27
27
  from flwr.common.constant import (
28
28
  NUM_PARTITIONS_KEY,
@@ -30,7 +30,7 @@ from typing import Any, Optional
30
30
 
31
31
  from flwr.cli.config_utils import load_and_validate
32
32
  from flwr.cli.utils import get_sha256_hash
33
- from flwr.client import ClientApp
33
+ from flwr.clientapp import ClientApp
34
34
  from flwr.common import Context, EventType, RecordDict, event, log, now
35
35
  from flwr.common.config import get_fused_config_from_dir, parse_config_args
36
36
  from flwr.common.constant import RUN_ID_NUM_BYTES, Status
@@ -51,6 +51,7 @@ from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
51
51
  from flwr.simulation.ray_transport.utils import (
52
52
  enable_tf_gpu_growth as enable_gpu_growth,
53
53
  )
54
+ from flwr.supercore.constant import FLWR_IN_MEMORY_DB_NAME
54
55
 
55
56
 
56
57
  def _replace_keys(d: Any, match: str, target: str) -> Any:
@@ -143,6 +144,15 @@ def run_simulation_from_cli() -> None:
143
144
  run = Run.create_empty(run_id)
144
145
  run.override_config = override_config
145
146
 
147
+ # Create Context
148
+ server_app_context = Context(
149
+ run_id=run_id,
150
+ node_id=0,
151
+ node_config=UserConfig(),
152
+ state=RecordDict(),
153
+ run_config=fused_config,
154
+ )
155
+
146
156
  _ = _run_simulation(
147
157
  server_app_attr=server_app_attr,
148
158
  client_app_attr=client_app_attr,
@@ -153,7 +163,7 @@ def run_simulation_from_cli() -> None:
153
163
  run=run,
154
164
  enable_tf_gpu_growth=args.enable_tf_gpu_growth,
155
165
  verbose_logging=args.verbose,
156
- server_app_run_config=fused_config,
166
+ server_app_context=server_app_context,
157
167
  is_app=True,
158
168
  exit_event=EventType.CLI_FLOWER_SIMULATION_LEAVE,
159
169
  )
@@ -241,13 +251,12 @@ def run_simulation(
241
251
  def run_serverapp_th(
242
252
  server_app_attr: Optional[str],
243
253
  server_app: Optional[ServerApp],
244
- server_app_run_config: UserConfig,
254
+ server_app_context: Context,
245
255
  grid: Grid,
246
256
  app_dir: str,
247
257
  f_stop: threading.Event,
248
258
  has_exception: threading.Event,
249
259
  enable_tf_gpu_growth: bool,
250
- run_id: int,
251
260
  ctx_queue: "Queue[Context]",
252
261
  ) -> threading.Thread:
253
262
  """Run SeverApp in a thread."""
@@ -258,7 +267,6 @@ def run_serverapp_th(
258
267
  exception_event: threading.Event,
259
268
  _grid: Grid,
260
269
  _server_app_dir: str,
261
- _server_app_run_config: UserConfig,
262
270
  _server_app_attr: Optional[str],
263
271
  _server_app: Optional[ServerApp],
264
272
  _ctx_queue: "Queue[Context]",
@@ -272,19 +280,10 @@ def run_serverapp_th(
272
280
  log(INFO, "Enabling GPU growth for Tensorflow on the server thread.")
273
281
  enable_gpu_growth()
274
282
 
275
- # Initialize Context
276
- context = Context(
277
- run_id=run_id,
278
- node_id=0,
279
- node_config={},
280
- state=RecordDict(),
281
- run_config=_server_app_run_config,
282
- )
283
-
284
283
  # Run ServerApp
285
284
  updated_context = _run(
286
285
  grid=_grid,
287
- context=context,
286
+ context=server_app_context,
288
287
  server_app_dir=_server_app_dir,
289
288
  server_app_attr=_server_app_attr,
290
289
  loaded_server_app=_server_app,
@@ -310,7 +309,6 @@ def run_serverapp_th(
310
309
  has_exception,
311
310
  grid,
312
311
  app_dir,
313
- server_app_run_config,
314
312
  server_app_attr,
315
313
  server_app,
316
314
  ctx_queue,
@@ -335,24 +333,26 @@ def _main_loop(
335
333
  client_app_attr: Optional[str] = None,
336
334
  server_app: Optional[ServerApp] = None,
337
335
  server_app_attr: Optional[str] = None,
338
- server_app_run_config: Optional[UserConfig] = None,
336
+ server_app_context: Optional[Context] = None,
339
337
  ) -> Context:
340
338
  """Start ServerApp on a separate thread, then launch Simulation Engine."""
341
339
  # Initialize StateFactory
342
- state_factory = LinkStateFactory(":flwr-in-memory-state:")
340
+ state_factory = LinkStateFactory(FLWR_IN_MEMORY_DB_NAME)
343
341
 
344
342
  f_stop = threading.Event()
345
343
  # A Threading event to indicate if an exception was raised in the ServerApp thread
346
344
  server_app_thread_has_exception = threading.Event()
347
345
  serverapp_th = None
348
346
  success = True
349
- updated_context = Context(
350
- run_id=run.run_id,
351
- node_id=0,
352
- node_config=UserConfig(),
353
- state=RecordDict(),
354
- run_config=UserConfig(),
355
- )
347
+ if server_app_context is None:
348
+ server_app_context = Context(
349
+ run_id=run.run_id,
350
+ node_id=0,
351
+ node_config=UserConfig(),
352
+ state=RecordDict(),
353
+ run_config=UserConfig(),
354
+ )
355
+ updated_context = server_app_context
356
356
  try:
357
357
  # Register run
358
358
  log(DEBUG, "Pre-registering run with id %s", run.run_id)
@@ -361,9 +361,6 @@ def _main_loop(
361
361
  run.running_at = run.starting_at
362
362
  state_factory.state().run_ids[run.run_id] = RunRecord(run=run) # type: ignore
363
363
 
364
- if server_app_run_config is None:
365
- server_app_run_config = {}
366
-
367
364
  # Initialize Grid
368
365
  grid = InMemoryGrid(state_factory=state_factory)
369
366
  grid.set_run(run_id=run.run_id)
@@ -373,13 +370,12 @@ def _main_loop(
373
370
  serverapp_th = run_serverapp_th(
374
371
  server_app_attr=server_app_attr,
375
372
  server_app=server_app,
376
- server_app_run_config=server_app_run_config,
373
+ server_app_context=server_app_context,
377
374
  grid=grid,
378
375
  app_dir=app_dir,
379
376
  f_stop=f_stop,
380
377
  has_exception=server_app_thread_has_exception,
381
378
  enable_tf_gpu_growth=enable_tf_gpu_growth,
382
- run_id=run.run_id,
383
379
  ctx_queue=output_context_queue,
384
380
  )
385
381
 
@@ -438,7 +434,7 @@ def _run_simulation(
438
434
  backend_config: Optional[BackendConfig] = None,
439
435
  client_app_attr: Optional[str] = None,
440
436
  server_app_attr: Optional[str] = None,
441
- server_app_run_config: Optional[UserConfig] = None,
437
+ server_app_context: Optional[Context] = None,
442
438
  app_dir: str = "",
443
439
  flwr_dir: Optional[str] = None,
444
440
  run: Optional[Run] = None,
@@ -502,7 +498,7 @@ def _run_simulation(
502
498
  client_app_attr,
503
499
  server_app,
504
500
  server_app_attr,
505
- server_app_run_config,
501
+ server_app_context,
506
502
  )
507
503
  # Detect if there is an Asyncio event loop already running.
508
504
  # If yes, disable logger propagation. In environmnets
@@ -17,7 +17,9 @@
17
17
 
18
18
  import argparse
19
19
  from logging import INFO
20
- from typing import Optional
20
+ from typing import Any, Optional
21
+
22
+ import yaml
21
23
 
22
24
  from flwr.common import EventType, event
23
25
  from flwr.common.constant import ExecPluginType
@@ -26,6 +28,7 @@ from flwr.common.logger import log
26
28
  from flwr.proto.clientappio_pb2_grpc import ClientAppIoStub
27
29
  from flwr.proto.serverappio_pb2_grpc import ServerAppIoStub
28
30
  from flwr.proto.simulationio_pb2_grpc import SimulationIoStub
31
+ from flwr.supercore.constant import EXEC_PLUGIN_SECTION
29
32
  from flwr.supercore.grpc_health import add_args_health
30
33
  from flwr.supercore.superexec.plugin import (
31
34
  ClientAppExecPlugin,
@@ -36,6 +39,7 @@ from flwr.supercore.superexec.plugin import (
36
39
  from flwr.supercore.superexec.run_superexec import run_superexec
37
40
 
38
41
  try:
42
+ from flwr.ee import add_ee_args_superexec
39
43
  from flwr.ee.constant import ExecEePluginType
40
44
  from flwr.ee.exec_plugin import get_ee_plugin_and_stub_class
41
45
  except ImportError:
@@ -54,6 +58,10 @@ except ImportError:
54
58
  """Get the EE plugin class and stub class based on the plugin type."""
55
59
  return None
56
60
 
61
+ # pylint: disable-next=unused-argument
62
+ def add_ee_args_superexec(parser: argparse.ArgumentParser) -> None:
63
+ """Add EE-specific arguments to the parser."""
64
+
57
65
 
58
66
  def flower_superexec() -> None:
59
67
  """Run `flower-superexec` command."""
@@ -70,12 +78,28 @@ def flower_superexec() -> None:
70
78
  # Trigger telemetry event
71
79
  event(EventType.RUN_SUPEREXEC_ENTER, {"plugin_type": args.plugin_type})
72
80
 
81
+ # Load plugin config from YAML file if provided
82
+ plugin_config = None
83
+ if plugin_config_path := getattr(args, "plugin_config", None):
84
+ try:
85
+ with open(plugin_config_path, encoding="utf-8") as file:
86
+ yaml_config: Optional[dict[str, Any]] = yaml.safe_load(file)
87
+ if yaml_config is None or EXEC_PLUGIN_SECTION not in yaml_config:
88
+ raise ValueError(f"Missing '{EXEC_PLUGIN_SECTION}' section.")
89
+ plugin_config = yaml_config[EXEC_PLUGIN_SECTION]
90
+ except (FileNotFoundError, yaml.YAMLError, ValueError) as e:
91
+ flwr_exit(
92
+ ExitCode.SUPEREXEC_INVALID_PLUGIN_CONFIG,
93
+ f"Failed to load plugin config from '{plugin_config_path}': {e!r}",
94
+ )
95
+
73
96
  # Get the plugin class and stub class based on the plugin type
74
97
  plugin_class, stub_class = _get_plugin_and_stub_class(args.plugin_type)
75
98
  run_superexec(
76
99
  plugin_class=plugin_class,
77
100
  stub_class=stub_class, # type: ignore
78
101
  appio_api_address=args.appio_api_address,
102
+ plugin_config=plugin_config,
79
103
  flwr_dir=args.flwr_dir,
80
104
  parent_pid=args.parent_pid,
81
105
  health_server_address=args.health_server_address,
@@ -122,6 +146,7 @@ def _parse_args() -> argparse.ArgumentParser:
122
146
  help="The PID of the parent process. When set, the process will terminate "
123
147
  "when the parent process exits.",
124
148
  )
149
+ add_ee_args_superexec(parser)
125
150
  add_args_health(parser)
126
151
  return parser
127
152
 
@@ -0,0 +1,41 @@
1
+ # Copyright 2025 Flower Labs GmbH. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ==============================================================================
15
+ """Constants for Flower infrastructure."""
16
+
17
+
18
+ from __future__ import annotations
19
+
20
+ # Top-level key in YAML config for exec plugin settings
21
+ EXEC_PLUGIN_SECTION = "exec_plugin"
22
+
23
+ # Flower in-memory Python-based database name
24
+ FLWR_IN_MEMORY_DB_NAME = ":flwr-in-memory:"
25
+
26
+ # Constants for Hub
27
+ APP_ID_PATTERN = r"^@(?P<user>[^/]+)/(?P<app>[^/]+)$"
28
+ PLATFORM_API_URL = "https://api.flower.ai/v1"
29
+
30
+
31
+ class NodeStatus:
32
+ """Event log writer types."""
33
+
34
+ REGISTERED = "registered"
35
+ ONLINE = "online"
36
+ OFFLINE = "offline"
37
+ UNREGISTERED = "unregistered"
38
+
39
+ def __new__(cls) -> NodeStatus:
40
+ """Prevent instantiation."""
41
+ raise TypeError(f"{cls.__name__} cannot be instantiated.")
@@ -48,9 +48,6 @@ class InMemoryObjectStore(ObjectStore):
48
48
  self.verify = verify
49
49
  self.store: dict[str, ObjectEntry] = {}
50
50
  self.lock_store = threading.RLock()
51
- # Mapping the Object ID of a message to the list of descendant object IDs
52
- self.msg_descendant_objects_mapping: dict[str, list[str]] = {}
53
- self.lock_msg_mapping = threading.RLock()
54
51
  # Mapping each run ID to a set of object IDs that are used in that run
55
52
  self.run_objects_mapping: dict[int, set[str]] = {}
56
53
 
@@ -215,7 +212,6 @@ class InMemoryObjectStore(ObjectStore):
215
212
  """Clear the store."""
216
213
  with self.lock_store:
217
214
  self.store.clear()
218
- self.msg_descendant_objects_mapping.clear()
219
215
  self.run_objects_mapping.clear()
220
216
 
221
217
  def __contains__(self, object_id: str) -> bool:
@@ -19,15 +19,27 @@ from logging import DEBUG
19
19
  from typing import Optional
20
20
 
21
21
  from flwr.common.logger import log
22
+ from flwr.supercore.constant import FLWR_IN_MEMORY_DB_NAME
22
23
 
23
24
  from .in_memory_object_store import InMemoryObjectStore
24
25
  from .object_store import ObjectStore
26
+ from .sqlite_object_store import SqliteObjectStore
25
27
 
26
28
 
27
29
  class ObjectStoreFactory:
28
- """Factory class that creates ObjectStore instances."""
30
+ """Factory class that creates ObjectStore instances.
29
31
 
30
- def __init__(self) -> None:
32
+ Parameters
33
+ ----------
34
+ database : str (default: FLWR_IN_MEMORY_DB_NAME)
35
+ A string representing the path to the database file that will be opened.
36
+ Note that passing ":memory:" will open a connection to a database that is
37
+ in RAM, instead of on disk. And FLWR_IN_MEMORY_DB_NAME will create an
38
+ Python-based in-memory ObjectStore.
39
+ """
40
+
41
+ def __init__(self, database: str = FLWR_IN_MEMORY_DB_NAME) -> None:
42
+ self.database = database
31
43
  self.store_instance: Optional[ObjectStore] = None
32
44
 
33
45
  def store(self) -> ObjectStore:
@@ -38,7 +50,15 @@ class ObjectStoreFactory:
38
50
  ObjectStore
39
51
  An ObjectStore instance for storing objects by object_id.
40
52
  """
41
- if self.store_instance is None:
42
- self.store_instance = InMemoryObjectStore()
43
- log(DEBUG, "Using InMemoryObjectStore")
44
- return self.store_instance
53
+ # InMemoryObjectStore
54
+ if self.database == FLWR_IN_MEMORY_DB_NAME:
55
+ if self.store_instance is None:
56
+ self.store_instance = InMemoryObjectStore()
57
+ log(DEBUG, "Using InMemoryObjectStore")
58
+ return self.store_instance
59
+
60
+ # SqliteObjectStore
61
+ store = SqliteObjectStore(self.database)
62
+ store.initialize()
63
+ log(DEBUG, "Using SqliteObjectStore")
64
+ return store
@@ -0,0 +1,252 @@
1
+ # Copyright 2025 Flower Labs GmbH. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ==============================================================================
15
+ """Flower SQLite ObjectStore implementation."""
16
+
17
+
18
+ from typing import Optional, cast
19
+
20
+ from flwr.common.inflatable import (
21
+ get_object_id,
22
+ is_valid_sha256_hash,
23
+ iterate_object_tree,
24
+ )
25
+ from flwr.common.inflatable_utils import validate_object_content
26
+ from flwr.proto.message_pb2 import ObjectTree # pylint: disable=E0611
27
+ from flwr.supercore.sqlite_mixin import SqliteMixin
28
+ from flwr.supercore.utils import uint64_to_int64
29
+
30
+ from .object_store import NoObjectInStoreError, ObjectStore
31
+
32
+ SQL_CREATE_OBJECTS = """
33
+ CREATE TABLE IF NOT EXISTS objects (
34
+ object_id TEXT PRIMARY KEY,
35
+ content BLOB,
36
+ is_available INTEGER NOT NULL CHECK (is_available IN (0,1)),
37
+ ref_count INTEGER NOT NULL
38
+ );
39
+ """
40
+ SQL_CREATE_OBJECT_CHILDREN = """
41
+ CREATE TABLE IF NOT EXISTS object_children (
42
+ parent_id TEXT NOT NULL,
43
+ child_id TEXT NOT NULL,
44
+ FOREIGN KEY (parent_id) REFERENCES objects(object_id) ON DELETE CASCADE,
45
+ FOREIGN KEY (child_id) REFERENCES objects(object_id) ON DELETE CASCADE,
46
+ PRIMARY KEY (parent_id, child_id)
47
+ );
48
+ """
49
+ SQL_CREATE_RUN_OBJECTS = """
50
+ CREATE TABLE IF NOT EXISTS run_objects (
51
+ run_id INTEGER NOT NULL,
52
+ object_id TEXT NOT NULL,
53
+ FOREIGN KEY (object_id) REFERENCES objects(object_id) ON DELETE CASCADE,
54
+ PRIMARY KEY (run_id, object_id)
55
+ );
56
+ """
57
+
58
+
59
+ class SqliteObjectStore(ObjectStore, SqliteMixin):
60
+ """SQLite-based implementation of the ObjectStore interface."""
61
+
62
+ def __init__(self, database_path: str, verify: bool = True) -> None:
63
+ super().__init__(database_path)
64
+ self.verify = verify
65
+
66
+ def initialize(self, log_queries: bool = False) -> list[tuple[str]]:
67
+ """Connect to the DB, enable FK support, and create tables if needed."""
68
+ return self._ensure_initialized(
69
+ SQL_CREATE_OBJECTS,
70
+ SQL_CREATE_OBJECT_CHILDREN,
71
+ SQL_CREATE_RUN_OBJECTS,
72
+ log_queries=log_queries,
73
+ )
74
+
75
+ def preregister(self, run_id: int, object_tree: ObjectTree) -> list[str]:
76
+ """Identify and preregister missing objects in the `ObjectStore`."""
77
+ new_objects = []
78
+ for tree_node in iterate_object_tree(object_tree):
79
+ obj_id = tree_node.object_id
80
+ if not is_valid_sha256_hash(obj_id):
81
+ raise ValueError(f"Invalid object ID format: {obj_id}")
82
+
83
+ child_ids = [child.object_id for child in tree_node.children]
84
+ with self.conn:
85
+ row = self.conn.execute(
86
+ "SELECT object_id, is_available FROM objects WHERE object_id=?",
87
+ (obj_id,),
88
+ ).fetchone()
89
+ if row is None:
90
+ # Insert new object
91
+ self.conn.execute(
92
+ "INSERT INTO objects"
93
+ "(object_id, content, is_available, ref_count) "
94
+ "VALUES (?, ?, ?, ?)",
95
+ (obj_id, b"", 0, 0),
96
+ )
97
+ for cid in child_ids:
98
+ self.conn.execute(
99
+ "INSERT INTO object_children(parent_id, child_id) "
100
+ "VALUES (?, ?)",
101
+ (obj_id, cid),
102
+ )
103
+ self.conn.execute(
104
+ "UPDATE objects SET ref_count = ref_count + 1 "
105
+ "WHERE object_id = ?",
106
+ (cid,),
107
+ )
108
+ new_objects.append(obj_id)
109
+ else:
110
+ # Add to the list of new objects if not available
111
+ if not row["is_available"]:
112
+ new_objects.append(obj_id)
113
+
114
+ # Ensure run mapping
115
+ self.conn.execute(
116
+ "INSERT OR IGNORE INTO run_objects(run_id, object_id) "
117
+ "VALUES (?, ?)",
118
+ (uint64_to_int64(run_id), obj_id),
119
+ )
120
+ return new_objects
121
+
122
+ def get_object_tree(self, object_id: str) -> ObjectTree:
123
+ """Get the object tree for a given object ID."""
124
+ with self.conn:
125
+ row = self.conn.execute(
126
+ "SELECT object_id FROM objects WHERE object_id=?", (object_id,)
127
+ ).fetchone()
128
+ if not row:
129
+ raise NoObjectInStoreError(f"Object {object_id} not found.")
130
+ children = self.query(
131
+ "SELECT child_id FROM object_children WHERE parent_id=?", (object_id,)
132
+ )
133
+
134
+ # Build the object trees of all children
135
+ try:
136
+ child_trees = [self.get_object_tree(ch["child_id"]) for ch in children]
137
+ except NoObjectInStoreError as e:
138
+ # Raise an error if any child object is missing
139
+ # This indicates an integrity issue
140
+ raise NoObjectInStoreError(
141
+ f"Object tree for object ID '{object_id}' contains missing "
142
+ "children. This may indicate a corrupted object store."
143
+ ) from e
144
+
145
+ # Create and return the ObjectTree for the current object
146
+ return ObjectTree(object_id=object_id, children=child_trees)
147
+
148
+ def put(self, object_id: str, object_content: bytes) -> None:
149
+ """Put an object into the store."""
150
+ if self.verify:
151
+ # Verify object_id and object_content match
152
+ object_id_from_content = get_object_id(object_content)
153
+ if object_id != object_id_from_content:
154
+ raise ValueError(f"Object ID {object_id} does not match content hash")
155
+
156
+ # Validate object content
157
+ validate_object_content(content=object_content)
158
+
159
+ with self.conn:
160
+ # Only allow adding the object if it has been preregistered
161
+ row = self.conn.execute(
162
+ "SELECT is_available FROM objects WHERE object_id=?", (object_id,)
163
+ ).fetchone()
164
+ if row is None:
165
+ raise NoObjectInStoreError(
166
+ f"Object with ID '{object_id}' was not pre-registered."
167
+ )
168
+
169
+ # Return if object is already present in the store
170
+ if row["is_available"]:
171
+ return
172
+
173
+ # Update the object entry in the store
174
+ self.conn.execute(
175
+ "UPDATE objects SET content=?, is_available=1 WHERE object_id=?",
176
+ (object_content, object_id),
177
+ )
178
+
179
+ def get(self, object_id: str) -> Optional[bytes]:
180
+ """Get an object from the store."""
181
+ rows = self.query("SELECT content FROM objects WHERE object_id=?", (object_id,))
182
+ return rows[0]["content"] if rows else None
183
+
184
+ def delete(self, object_id: str) -> None:
185
+ """Delete an object and its unreferenced descendants from the store."""
186
+ with self.conn:
187
+ row = self.conn.execute(
188
+ "SELECT ref_count FROM objects WHERE object_id=?", (object_id,)
189
+ ).fetchone()
190
+
191
+ # If the object is not in the store, nothing to delete
192
+ if row is None:
193
+ return
194
+
195
+ # Skip deletion if there are still references
196
+ if row["ref_count"] > 0:
197
+ return
198
+
199
+ # Deleting will cascade via FK, but we need to decrement children first
200
+ children = self.conn.execute(
201
+ "SELECT child_id FROM object_children WHERE parent_id=?", (object_id,)
202
+ ).fetchall()
203
+ child_ids = [child["child_id"] for child in children]
204
+
205
+ if child_ids:
206
+ placeholders = ", ".join("?" for _ in child_ids)
207
+ query = f"""
208
+ UPDATE objects SET ref_count = ref_count - 1
209
+ WHERE object_id IN ({placeholders})
210
+ """
211
+ self.conn.execute(query, child_ids)
212
+
213
+ self.conn.execute("DELETE FROM objects WHERE object_id=?", (object_id,))
214
+
215
+ # Recursively clean children
216
+ for child_id in child_ids:
217
+ self.delete(child_id)
218
+
219
+ def delete_objects_in_run(self, run_id: int) -> None:
220
+ """Delete all objects that were registered in a specific run."""
221
+ run_id_sint = uint64_to_int64(run_id)
222
+ with self.conn:
223
+ objs = self.conn.execute(
224
+ "SELECT object_id FROM run_objects WHERE run_id=?", (run_id_sint,)
225
+ ).fetchall()
226
+ for obj in objs:
227
+ object_id = obj["object_id"]
228
+ row = self.conn.execute(
229
+ "SELECT ref_count FROM objects WHERE object_id=?", (object_id,)
230
+ ).fetchone()
231
+ if row and row["ref_count"] == 0:
232
+ self.delete(object_id)
233
+ self.conn.execute("DELETE FROM run_objects WHERE run_id=?", (run_id_sint,))
234
+
235
+ def clear(self) -> None:
236
+ """Clear the store."""
237
+ with self.conn:
238
+ self.conn.execute("DELETE FROM object_children;")
239
+ self.conn.execute("DELETE FROM run_objects;")
240
+ self.conn.execute("DELETE FROM objects;")
241
+
242
+ def __contains__(self, object_id: str) -> bool:
243
+ """Check if an object_id is in the store."""
244
+ row = self.conn.execute(
245
+ "SELECT 1 FROM objects WHERE object_id=?", (object_id,)
246
+ ).fetchone()
247
+ return row is not None
248
+
249
+ def __len__(self) -> int:
250
+ """Return the number of objects in the store."""
251
+ row = self.conn.execute("SELECT COUNT(*) AS cnt FROM objects;").fetchone()
252
+ return cast(int, row["cnt"])
@@ -12,4 +12,4 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  # ==============================================================================
15
- """Flower AppIO service."""
15
+ """Cryptographic primitives for the Flower infrastructure."""