flwr-nightly 1.13.0.dev20241106__py3-none-any.whl → 1.13.0.dev20241111__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of flwr-nightly might be problematic. Click here for more details.

Files changed (45) hide show
  1. flwr/cli/run/run.py +16 -5
  2. flwr/client/app.py +10 -6
  3. flwr/client/clientapp/app.py +21 -16
  4. flwr/client/nodestate/__init__.py +25 -0
  5. flwr/client/nodestate/in_memory_nodestate.py +38 -0
  6. flwr/client/nodestate/nodestate.py +30 -0
  7. flwr/client/nodestate/nodestate_factory.py +37 -0
  8. flwr/common/args.py +83 -0
  9. flwr/common/config.py +10 -0
  10. flwr/common/constant.py +0 -1
  11. flwr/common/logger.py +6 -2
  12. flwr/common/object_ref.py +47 -16
  13. flwr/common/typing.py +1 -1
  14. flwr/proto/exec_pb2.py +14 -17
  15. flwr/proto/exec_pb2.pyi +6 -20
  16. flwr/proto/run_pb2.py +32 -27
  17. flwr/proto/run_pb2.pyi +26 -0
  18. flwr/proto/simulationio_pb2.py +2 -2
  19. flwr/proto/simulationio_pb2_grpc.py +34 -0
  20. flwr/proto/simulationio_pb2_grpc.pyi +13 -0
  21. flwr/server/app.py +45 -20
  22. flwr/server/driver/driver.py +1 -1
  23. flwr/server/driver/grpc_driver.py +2 -6
  24. flwr/server/driver/inmemory_driver.py +1 -3
  25. flwr/server/run_serverapp.py +2 -2
  26. flwr/server/serverapp/app.py +16 -72
  27. flwr/server/strategy/aggregate.py +4 -4
  28. flwr/server/superlink/linkstate/in_memory_linkstate.py +5 -16
  29. flwr/server/superlink/linkstate/linkstate.py +5 -4
  30. flwr/server/superlink/linkstate/sqlite_linkstate.py +6 -15
  31. flwr/server/superlink/linkstate/utils.py +2 -33
  32. flwr/server/superlink/simulation/simulationio_servicer.py +22 -1
  33. flwr/simulation/__init__.py +3 -1
  34. flwr/simulation/app.py +273 -345
  35. flwr/simulation/legacy_app.py +382 -0
  36. flwr/simulation/run_simulation.py +1 -1
  37. flwr/superexec/deployment.py +1 -1
  38. flwr/superexec/exec_servicer.py +2 -2
  39. flwr/superexec/executor.py +4 -3
  40. flwr/superexec/simulation.py +44 -102
  41. {flwr_nightly-1.13.0.dev20241106.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/METADATA +5 -4
  42. {flwr_nightly-1.13.0.dev20241106.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/RECORD +45 -39
  43. {flwr_nightly-1.13.0.dev20241106.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/entry_points.txt +1 -0
  44. {flwr_nightly-1.13.0.dev20241106.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/LICENSE +0 -0
  45. {flwr_nightly-1.13.0.dev20241106.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/WHEEL +0 -0
@@ -0,0 +1,382 @@
1
+ # Copyright 2024 Flower Labs GmbH. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ==============================================================================
15
+ """Flower simulation app."""
16
+
17
+
18
+ import asyncio
19
+ import logging
20
+ import sys
21
+ import threading
22
+ import traceback
23
+ import warnings
24
+ from logging import ERROR, INFO
25
+ from typing import Any, Optional, Union
26
+
27
+ import ray
28
+ from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
29
+
30
+ from flwr.client import ClientFnExt
31
+ from flwr.common import EventType, event
32
+ from flwr.common.constant import NODE_ID_NUM_BYTES
33
+ from flwr.common.logger import log, set_logger_propagation, warn_unsupported_feature
34
+ from flwr.server.client_manager import ClientManager
35
+ from flwr.server.history import History
36
+ from flwr.server.server import Server, init_defaults, run_fl
37
+ from flwr.server.server_config import ServerConfig
38
+ from flwr.server.strategy import Strategy
39
+ from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
40
+ from flwr.simulation.ray_transport.ray_actor import (
41
+ ClientAppActor,
42
+ VirtualClientEngineActor,
43
+ VirtualClientEngineActorPool,
44
+ pool_size_from_resources,
45
+ )
46
+ from flwr.simulation.ray_transport.ray_client_proxy import RayActorClientProxy
47
+
48
+ INVALID_ARGUMENTS_START_SIMULATION = """
49
+ INVALID ARGUMENTS ERROR
50
+
51
+ Invalid Arguments in method:
52
+
53
+ `start_simulation(
54
+ *,
55
+ client_fn: ClientFn,
56
+ num_clients: int,
57
+ clients_ids: Optional[List[str]] = None,
58
+ client_resources: Optional[Dict[str, float]] = None,
59
+ server: Optional[Server] = None,
60
+ config: ServerConfig = None,
61
+ strategy: Optional[Strategy] = None,
62
+ client_manager: Optional[ClientManager] = None,
63
+ ray_init_args: Optional[Dict[str, Any]] = None,
64
+ ) -> None:`
65
+
66
+ REASON:
67
+ Method requires:
68
+ - Either `num_clients`[int] or `clients_ids`[List[str]]
69
+ to be set exclusively.
70
+ OR
71
+ - `len(clients_ids)` == `num_clients`
72
+
73
+ """
74
+
75
+ NodeToPartitionMapping = dict[int, int]
76
+
77
+
78
+ def _create_node_id_to_partition_mapping(
79
+ num_clients: int,
80
+ ) -> NodeToPartitionMapping:
81
+ """Generate a node_id:partition_id mapping."""
82
+ nodes_mapping: NodeToPartitionMapping = {} # {node-id; partition-id}
83
+ for i in range(num_clients):
84
+ while True:
85
+ node_id = generate_rand_int_from_bytes(NODE_ID_NUM_BYTES)
86
+ if node_id not in nodes_mapping:
87
+ break
88
+ nodes_mapping[node_id] = i
89
+ return nodes_mapping
90
+
91
+
92
+ # pylint: disable=too-many-arguments,too-many-statements,too-many-branches
93
+ def start_simulation(
94
+ *,
95
+ client_fn: ClientFnExt,
96
+ num_clients: int,
97
+ clients_ids: Optional[list[str]] = None, # UNSUPPORTED, WILL BE REMOVED
98
+ client_resources: Optional[dict[str, float]] = None,
99
+ server: Optional[Server] = None,
100
+ config: Optional[ServerConfig] = None,
101
+ strategy: Optional[Strategy] = None,
102
+ client_manager: Optional[ClientManager] = None,
103
+ ray_init_args: Optional[dict[str, Any]] = None,
104
+ keep_initialised: Optional[bool] = False,
105
+ actor_type: type[VirtualClientEngineActor] = ClientAppActor,
106
+ actor_kwargs: Optional[dict[str, Any]] = None,
107
+ actor_scheduling: Union[str, NodeAffinitySchedulingStrategy] = "DEFAULT",
108
+ ) -> History:
109
+ """Start a Ray-based Flower simulation server.
110
+
111
+ Parameters
112
+ ----------
113
+ client_fn : ClientFnExt
114
+ A function creating `Client` instances. The function must have the signature
115
+ `client_fn(context: Context). It should return
116
+ a single client instance of type `Client`. Note that the created client
117
+ instances are ephemeral and will often be destroyed after a single method
118
+ invocation. Since client instances are not long-lived, they should not attempt
119
+ to carry state over method invocations. Any state required by the instance
120
+ (model, dataset, hyperparameters, ...) should be (re-)created in either the
121
+ call to `client_fn` or the call to any of the client methods (e.g., load
122
+ evaluation data in the `evaluate` method itself).
123
+ num_clients : int
124
+ The total number of clients in this simulation.
125
+ clients_ids : Optional[List[str]]
126
+ UNSUPPORTED, WILL BE REMOVED. USE `num_clients` INSTEAD.
127
+ List `client_id`s for each client. This is only required if
128
+ `num_clients` is not set. Setting both `num_clients` and `clients_ids`
129
+ with `len(clients_ids)` not equal to `num_clients` generates an error.
130
+ Using this argument will raise an error.
131
+ client_resources : Optional[Dict[str, float]] (default: `{"num_cpus": 1, "num_gpus": 0.0}`)
132
+ CPU and GPU resources for a single client. Supported keys
133
+ are `num_cpus` and `num_gpus`. To understand the GPU utilization caused by
134
+ `num_gpus`, as well as using custom resources, please consult the Ray
135
+ documentation.
136
+ server : Optional[flwr.server.Server] (default: None).
137
+ An implementation of the abstract base class `flwr.server.Server`. If no
138
+ instance is provided, then `start_server` will create one.
139
+ config: ServerConfig (default: None).
140
+ Currently supported values are `num_rounds` (int, default: 1) and
141
+ `round_timeout` in seconds (float, default: None).
142
+ strategy : Optional[flwr.server.Strategy] (default: None)
143
+ An implementation of the abstract base class `flwr.server.Strategy`. If
144
+ no strategy is provided, then `start_server` will use
145
+ `flwr.server.strategy.FedAvg`.
146
+ client_manager : Optional[flwr.server.ClientManager] (default: None)
147
+ An implementation of the abstract base class `flwr.server.ClientManager`.
148
+ If no implementation is provided, then `start_simulation` will use
149
+ `flwr.server.client_manager.SimpleClientManager`.
150
+ ray_init_args : Optional[Dict[str, Any]] (default: None)
151
+ Optional dictionary containing arguments for the call to `ray.init`.
152
+ If ray_init_args is None (the default), Ray will be initialized with
153
+ the following default args:
154
+
155
+ { "ignore_reinit_error": True, "include_dashboard": False }
156
+
157
+ An empty dictionary can be used (ray_init_args={}) to prevent any
158
+ arguments from being passed to ray.init.
159
+ keep_initialised: Optional[bool] (default: False)
160
+ Set to True to prevent `ray.shutdown()` in case `ray.is_initialized()=True`.
161
+
162
+ actor_type: VirtualClientEngineActor (default: ClientAppActor)
163
+ Optionally specify the type of actor to use. The actor object, which
164
+ persists throughout the simulation, will be the process in charge of
165
+ executing a ClientApp wrapping input argument `client_fn`.
166
+
167
+ actor_kwargs: Optional[Dict[str, Any]] (default: None)
168
+ If you want to create your own Actor classes, you might need to pass
169
+ some input argument. You can use this dictionary for such purpose.
170
+
171
+ actor_scheduling: Optional[Union[str, NodeAffinitySchedulingStrategy]]
172
+ (default: "DEFAULT")
173
+ Optional string ("DEFAULT" or "SPREAD") for the VCE to choose in which
174
+ node the actor is placed. If you are an advanced user needed more control
175
+ you can use lower-level scheduling strategies to pin actors to specific
176
+ compute nodes (e.g. via NodeAffinitySchedulingStrategy). Please note this
177
+ is an advanced feature. For all details, please refer to the Ray documentation:
178
+ https://docs.ray.io/en/latest/ray-core/scheduling/index.html
179
+
180
+ Returns
181
+ -------
182
+ hist : flwr.server.history.History
183
+ Object containing metrics from training.
184
+ """ # noqa: E501
185
+ # pylint: disable-msg=too-many-locals
186
+ event(
187
+ EventType.START_SIMULATION_ENTER,
188
+ {"num_clients": len(clients_ids) if clients_ids is not None else num_clients},
189
+ )
190
+
191
+ if clients_ids is not None:
192
+ warn_unsupported_feature(
193
+ "Passing `clients_ids` to `start_simulation` is deprecated and not longer "
194
+ "used by `start_simulation`. Use `num_clients` exclusively instead."
195
+ )
196
+ log(ERROR, "`clients_ids` argument used.")
197
+ sys.exit()
198
+
199
+ # Set logger propagation
200
+ loop: Optional[asyncio.AbstractEventLoop] = None
201
+ try:
202
+ loop = asyncio.get_running_loop()
203
+ except RuntimeError:
204
+ loop = None
205
+ finally:
206
+ if loop and loop.is_running():
207
+ # Set logger propagation to False to prevent duplicated log output in Colab.
208
+ logger = logging.getLogger("flwr")
209
+ _ = set_logger_propagation(logger, False)
210
+
211
+ # Initialize server and server config
212
+ initialized_server, initialized_config = init_defaults(
213
+ server=server,
214
+ config=config,
215
+ strategy=strategy,
216
+ client_manager=client_manager,
217
+ )
218
+
219
+ log(
220
+ INFO,
221
+ "Starting Flower simulation, config: %s",
222
+ initialized_config,
223
+ )
224
+
225
+ # Create node-id to partition-id mapping
226
+ nodes_mapping = _create_node_id_to_partition_mapping(num_clients)
227
+
228
+ # Default arguments for Ray initialization
229
+ if not ray_init_args:
230
+ ray_init_args = {
231
+ "ignore_reinit_error": True,
232
+ "include_dashboard": False,
233
+ }
234
+
235
+ # Shut down Ray if it has already been initialized (unless asked not to)
236
+ if ray.is_initialized() and not keep_initialised:
237
+ ray.shutdown()
238
+
239
+ # Initialize Ray
240
+ ray.init(**ray_init_args)
241
+ cluster_resources = ray.cluster_resources()
242
+ log(
243
+ INFO,
244
+ "Flower VCE: Ray initialized with resources: %s",
245
+ cluster_resources,
246
+ )
247
+
248
+ log(
249
+ INFO,
250
+ "Optimize your simulation with Flower VCE: "
251
+ "https://flower.ai/docs/framework/how-to-run-simulations.html",
252
+ )
253
+
254
+ # Log the resources that a single client will be able to use
255
+ if client_resources is None:
256
+ log(
257
+ INFO,
258
+ "No `client_resources` specified. Using minimal resources for clients.",
259
+ )
260
+ client_resources = {"num_cpus": 1, "num_gpus": 0.0}
261
+
262
+ # Each client needs at the very least one CPU
263
+ if "num_cpus" not in client_resources:
264
+ warnings.warn(
265
+ "No `num_cpus` specified in `client_resources`. "
266
+ "Using `num_cpus=1` for each client.",
267
+ stacklevel=2,
268
+ )
269
+ client_resources["num_cpus"] = 1
270
+
271
+ log(
272
+ INFO,
273
+ "Flower VCE: Resources for each Virtual Client: %s",
274
+ client_resources,
275
+ )
276
+
277
+ actor_args = {} if actor_kwargs is None else actor_kwargs
278
+
279
+ # An actor factory. This is called N times to add N actors
280
+ # to the pool. If at some point the pool can accommodate more actors
281
+ # this will be called again.
282
+ def create_actor_fn() -> type[VirtualClientEngineActor]:
283
+ return actor_type.options( # type: ignore
284
+ **client_resources,
285
+ scheduling_strategy=actor_scheduling,
286
+ ).remote(**actor_args)
287
+
288
+ # Instantiate ActorPool
289
+ pool = VirtualClientEngineActorPool(
290
+ create_actor_fn=create_actor_fn,
291
+ client_resources=client_resources,
292
+ )
293
+
294
+ f_stop = threading.Event()
295
+
296
+ # Periodically, check if the cluster has grown (i.e. a new
297
+ # node has been added). If this happens, we likely want to grow
298
+ # the actor pool by adding more Actors to it.
299
+ def update_resources(f_stop: threading.Event) -> None:
300
+ """Periodically check if more actors can be added to the pool.
301
+
302
+ If so, extend the pool.
303
+ """
304
+ if not f_stop.is_set():
305
+ num_max_actors = pool_size_from_resources(client_resources)
306
+ if num_max_actors > pool.num_actors:
307
+ num_new = num_max_actors - pool.num_actors
308
+ log(
309
+ INFO, "The cluster expanded. Adding %s actors to the pool.", num_new
310
+ )
311
+ pool.add_actors_to_pool(num_actors=num_new)
312
+
313
+ threading.Timer(10, update_resources, [f_stop]).start()
314
+
315
+ update_resources(f_stop)
316
+
317
+ log(
318
+ INFO,
319
+ "Flower VCE: Creating %s with %s actors",
320
+ pool.__class__.__name__,
321
+ pool.num_actors,
322
+ )
323
+
324
+ # Register one RayClientProxy object for each client with the ClientManager
325
+ for node_id, partition_id in nodes_mapping.items():
326
+ client_proxy = RayActorClientProxy(
327
+ client_fn=client_fn,
328
+ node_id=node_id,
329
+ partition_id=partition_id,
330
+ num_partitions=num_clients,
331
+ actor_pool=pool,
332
+ )
333
+ initialized_server.client_manager().register(client=client_proxy)
334
+
335
+ hist = History()
336
+ # pylint: disable=broad-except
337
+ try:
338
+ # Start training
339
+ hist = run_fl(
340
+ server=initialized_server,
341
+ config=initialized_config,
342
+ )
343
+ except Exception as ex:
344
+ log(ERROR, ex)
345
+ log(ERROR, traceback.format_exc())
346
+ log(
347
+ ERROR,
348
+ "Your simulation crashed :(. This could be because of several reasons. "
349
+ "The most common are: "
350
+ "\n\t > Sometimes, issues in the simulation code itself can cause crashes. "
351
+ "It's always a good idea to double-check your code for any potential bugs "
352
+ "or inconsistencies that might be contributing to the problem. "
353
+ "For example: "
354
+ "\n\t\t - You might be using a class attribute in your clients that "
355
+ "hasn't been defined."
356
+ "\n\t\t - There could be an incorrect method call to a 3rd party library "
357
+ "(e.g., PyTorch)."
358
+ "\n\t\t - The return types of methods in your clients/strategies might be "
359
+ "incorrect."
360
+ "\n\t > Your system couldn't fit a single VirtualClient: try lowering "
361
+ "`client_resources`."
362
+ "\n\t > All the actors in your pool crashed. This could be because: "
363
+ "\n\t\t - You clients hit an out-of-memory (OOM) error and actors couldn't "
364
+ "recover from it. Try launching your simulation with more generous "
365
+ "`client_resources` setting (i.e. it seems %s is "
366
+ "not enough for your run). Use fewer concurrent actors. "
367
+ "\n\t\t - You were running a multi-node simulation and all worker nodes "
368
+ "disconnected. The head node might still be alive but cannot accommodate "
369
+ "any actor with resources: %s."
370
+ "\nTake a look at the Flower simulation examples for guidance "
371
+ "<https://flower.ai/docs/framework/how-to-run-simulations.html>.",
372
+ client_resources,
373
+ client_resources,
374
+ )
375
+ raise RuntimeError("Simulation crashed.") from ex
376
+
377
+ finally:
378
+ # Stop time monitoring resources in cluster
379
+ f_stop.set()
380
+ event(EventType.START_SIMULATION_LEAVE)
381
+
382
+ return hist
@@ -347,7 +347,7 @@ def _main_loop(
347
347
 
348
348
  # Initialize Driver
349
349
  driver = InMemoryDriver(state_factory=state_factory)
350
- driver.init_run(run_id=run.run_id)
350
+ driver.set_run(run_id=run.run_id)
351
351
 
352
352
  # Get and run ServerApp thread
353
353
  serverapp_th = run_serverapp_th(
@@ -153,7 +153,7 @@ class DeploymentEngine(Executor):
153
153
  self,
154
154
  fab_file: bytes,
155
155
  override_config: UserConfig,
156
- federation_config: UserConfig,
156
+ federation_options: ConfigsRecord,
157
157
  ) -> Optional[int]:
158
158
  """Start run using the Flower Deployment Engine."""
159
159
  run_id = None
@@ -24,7 +24,7 @@ import grpc
24
24
 
25
25
  from flwr.common.constant import LOG_STREAM_INTERVAL, Status
26
26
  from flwr.common.logger import log
27
- from flwr.common.serde import user_config_from_proto
27
+ from flwr.common.serde import configs_record_from_proto, user_config_from_proto
28
28
  from flwr.proto import exec_pb2_grpc # pylint: disable=E0611
29
29
  from flwr.proto.exec_pb2 import ( # pylint: disable=E0611
30
30
  StartRunRequest,
@@ -61,7 +61,7 @@ class ExecServicer(exec_pb2_grpc.ExecServicer):
61
61
  run_id = self.executor.start_run(
62
62
  request.fab.content,
63
63
  user_config_from_proto(request.override_config),
64
- user_config_from_proto(request.federation_config),
64
+ configs_record_from_proto(request.federation_options),
65
65
  )
66
66
 
67
67
  if run_id is None:
@@ -19,6 +19,7 @@ from dataclasses import dataclass, field
19
19
  from subprocess import Popen
20
20
  from typing import Optional
21
21
 
22
+ from flwr.common import ConfigsRecord
22
23
  from flwr.common.typing import UserConfig
23
24
  from flwr.server.superlink.ffs.ffs_factory import FfsFactory
24
25
  from flwr.server.superlink.linkstate import LinkStateFactory
@@ -71,7 +72,7 @@ class Executor(ABC):
71
72
  self,
72
73
  fab_file: bytes,
73
74
  override_config: UserConfig,
74
- federation_config: UserConfig,
75
+ federation_options: ConfigsRecord,
75
76
  ) -> Optional[int]:
76
77
  """Start a run using the given Flower FAB ID and version.
77
78
 
@@ -84,8 +85,8 @@ class Executor(ABC):
84
85
  The Flower App Bundle file bytes.
85
86
  override_config: UserConfig
86
87
  The config overrides dict sent by the user (using `flwr run`).
87
- federation_config: UserConfig
88
- The federation options dict sent by the user (using `flwr run`).
88
+ federation_options: ConfigsRecord
89
+ The federation options sent by the user (using `flwr run`).
89
90
 
90
91
  Returns
91
92
  -------
@@ -15,46 +15,22 @@
15
15
  """Simulation engine executor."""
16
16
 
17
17
 
18
- import json
19
- import subprocess
20
- import sys
21
- from logging import ERROR, INFO, WARN
18
+ import hashlib
19
+ from logging import ERROR, INFO
22
20
  from typing import Optional
23
21
 
24
22
  from typing_extensions import override
25
23
 
26
- from flwr.cli.config_utils import load_and_validate
27
- from flwr.cli.install import install_from_fab
28
- from flwr.common.config import unflatten_dict
29
- from flwr.common.constant import RUN_ID_NUM_BYTES
24
+ from flwr.common import ConfigsRecord, Context, RecordSet
30
25
  from flwr.common.logger import log
31
- from flwr.common.typing import UserConfig
26
+ from flwr.common.typing import Fab, UserConfig
27
+ from flwr.server.superlink.ffs import Ffs
32
28
  from flwr.server.superlink.ffs.ffs_factory import FfsFactory
33
- from flwr.server.superlink.linkstate import LinkStateFactory
34
- from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
29
+ from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
35
30
 
36
31
  from .executor import Executor
37
32
 
38
33
 
39
- def _user_config_to_str(user_config: UserConfig) -> str:
40
- """Convert override user config to string."""
41
- user_config_list_str = []
42
- for key, value in user_config.items():
43
- if isinstance(value, bool):
44
- user_config_list_str.append(f"{key}={str(value).lower()}")
45
- elif isinstance(value, (int, float)):
46
- user_config_list_str.append(f"{key}={value}")
47
- elif isinstance(value, str):
48
- user_config_list_str.append(f'{key}="{value}"')
49
- else:
50
- raise ValueError(
51
- "Only types `bool`, `float`, `int` and `str` are supported"
52
- )
53
-
54
- user_config_str = ",".join(user_config_list_str)
55
- return user_config_str
56
-
57
-
58
34
  class SimulationEngine(Executor):
59
35
  """Simulation engine executor.
60
36
 
@@ -71,12 +47,30 @@ class SimulationEngine(Executor):
71
47
  ) -> None:
72
48
  self.num_supernodes = num_supernodes
73
49
  self.verbose = verbose
50
+ self.linkstate_factory: Optional[LinkStateFactory] = None
51
+ self.ffs_factory: Optional[FfsFactory] = None
74
52
 
75
53
  @override
76
54
  def initialize(
77
55
  self, linkstate_factory: LinkStateFactory, ffs_factory: FfsFactory
78
56
  ) -> None:
79
57
  """Initialize the executor with the necessary factories."""
58
+ self.linkstate_factory = linkstate_factory
59
+ self.ffs_factory = ffs_factory
60
+
61
+ @property
62
+ def linkstate(self) -> LinkState:
63
+ """Return the LinkState."""
64
+ if self.linkstate_factory is None:
65
+ raise RuntimeError("Executor is not initialized.")
66
+ return self.linkstate_factory.state()
67
+
68
+ @property
69
+ def ffs(self) -> Ffs:
70
+ """Return the Flower File Storage (FFS)."""
71
+ if self.ffs_factory is None:
72
+ raise RuntimeError("Executor is not initialized.")
73
+ return self.ffs_factory.ffs()
80
74
 
81
75
  @override
82
76
  def set_config(
@@ -124,87 +118,35 @@ class SimulationEngine(Executor):
124
118
  self,
125
119
  fab_file: bytes,
126
120
  override_config: UserConfig,
127
- federation_config: UserConfig,
121
+ federation_options: ConfigsRecord,
128
122
  ) -> Optional[int]:
129
123
  """Start run using the Flower Simulation Engine."""
130
- if self.num_supernodes is None:
131
- raise ValueError(
132
- "Error in `SuperExec` (`SimulationEngine` executor):\n\n"
133
- "`num-supernodes` must not be `None`, it must be a valid "
134
- "positive integer. In order to start this simulation executor "
135
- "with a specified number of `SuperNodes`, you can either provide "
136
- "a `--executor` that has been initialized with a number of nodes "
137
- "to the `flower-superexec` CLI, or `--executor-config num-supernodes=N`"
138
- "to the `flower-superexec` CLI."
139
- )
140
124
  try:
141
-
142
- # Install FAB to flwr dir
143
- fab_path = install_from_fab(fab_file, None, True)
144
-
145
- # Install FAB Python package
146
- subprocess.run(
147
- [sys.executable, "-m", "pip", "install", "--no-deps", str(fab_path)],
148
- stdout=None if self.verbose else subprocess.DEVNULL,
149
- stderr=None if self.verbose else subprocess.DEVNULL,
150
- check=True,
151
- )
152
-
153
- # Load and validate config
154
- config, errors, warnings = load_and_validate(fab_path / "pyproject.toml")
155
- if errors:
156
- raise ValueError(errors)
157
-
158
- if warnings:
159
- log(WARN, warnings)
160
-
161
- if config is None:
162
- raise ValueError(
163
- "Config extracted from FAB's pyproject.toml is not valid"
125
+ # Create run
126
+ fab = Fab(hashlib.sha256(fab_file).hexdigest(), fab_file)
127
+ fab_hash = self.ffs.put(fab.content, {})
128
+ if fab_hash != fab.hash_str:
129
+ raise RuntimeError(
130
+ f"FAB ({fab.hash_str}) hash from request doesn't match contents"
164
131
  )
165
132
 
166
- # Flatten federated config
167
- federation_config_flat = unflatten_dict(federation_config)
168
-
169
- num_supernodes = federation_config_flat.get(
170
- "num-supernodes", self.num_supernodes
133
+ run_id = self.linkstate.create_run(
134
+ None, None, fab_hash, override_config, federation_options
171
135
  )
172
- backend_cfg = federation_config_flat.get("backend", {})
173
- verbose: Optional[bool] = federation_config_flat.get("verbose")
174
-
175
- # In Simulation there is no SuperLink, still we create a run_id
176
- run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
177
- log(INFO, "Created run %s", str(run_id))
178
136
 
179
- # Prepare commnand
180
- command = [
181
- "flower-simulation",
182
- "--app",
183
- f"{str(fab_path)}",
184
- "--num-supernodes",
185
- f"{num_supernodes}",
186
- "--run-id",
187
- str(run_id),
188
- ]
189
-
190
- if backend_cfg:
191
- # Stringify as JSON
192
- command.extend(["--backend-config", json.dumps(backend_cfg)])
193
-
194
- if verbose:
195
- command.extend(["--verbose"])
196
-
197
- if override_config:
198
- override_config_str = _user_config_to_str(override_config)
199
- command.extend(["--run-config", f"{override_config_str}"])
200
-
201
- # Start Simulation
202
- _ = subprocess.Popen( # pylint: disable=consider-using-with
203
- command,
204
- text=True,
137
+ # Create an empty context for the Run
138
+ context = Context(
139
+ run_id=run_id,
140
+ node_id=0,
141
+ node_config={},
142
+ state=RecordSet(),
143
+ run_config={},
205
144
  )
206
145
 
207
- log(INFO, "Started run %s", str(run_id))
146
+ # Register the context at the LinkState
147
+ self.linkstate.set_serverapp_context(run_id=run_id, context=context)
148
+
149
+ log(INFO, "Created run %s", str(run_id))
208
150
 
209
151
  return run_id
210
152
 
@@ -1,10 +1,10 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: flwr-nightly
3
- Version: 1.13.0.dev20241106
3
+ Version: 1.13.0.dev20241111
4
4
  Summary: Flower: A Friendly Federated Learning Framework
5
5
  Home-page: https://flower.ai
6
6
  License: Apache-2.0
7
- Keywords: flower,fl,federated learning,federated analytics,federated evaluation,machine learning
7
+ Keywords: Artificial Intelligence,Federated AI,Federated Analytics,Federated Evaluation,Federated Learning,Flower,Machine Learning
8
8
  Author: The Flower Authors
9
9
  Author-email: hello@flower.ai
10
10
  Requires-Python: >=3.9,<4.0
@@ -21,6 +21,7 @@ Classifier: Programming Language :: Python :: 3.10
21
21
  Classifier: Programming Language :: Python :: 3.11
22
22
  Classifier: Programming Language :: Python :: 3.12
23
23
  Classifier: Programming Language :: Python :: 3 :: Only
24
+ Classifier: Programming Language :: Python :: 3.13
24
25
  Classifier: Programming Language :: Python :: Implementation :: CPython
25
26
  Classifier: Topic :: Scientific/Engineering
26
27
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -32,9 +33,9 @@ Classifier: Typing :: Typed
32
33
  Provides-Extra: rest
33
34
  Provides-Extra: simulation
34
35
  Requires-Dist: cryptography (>=42.0.4,<43.0.0)
35
- Requires-Dist: grpcio (>=1.60.0,<2.0.0,!=1.64.2,!=1.65.1,!=1.65.2,!=1.65.4,!=1.65.5,!=1.66.0,!=1.66.1)
36
+ Requires-Dist: grpcio (>=1.60.0,<2.0.0,!=1.64.2,<=1.64.3)
36
37
  Requires-Dist: iterators (>=0.0.2,<0.0.3)
37
- Requires-Dist: numpy (>=1.21.0,<2.0.0)
38
+ Requires-Dist: numpy (>=1.26.0,<3.0.0)
38
39
  Requires-Dist: pathspec (>=0.12.1,<0.13.0)
39
40
  Requires-Dist: protobuf (>=4.25.2,<5.0.0)
40
41
  Requires-Dist: pycryptodome (>=3.18.0,<4.0.0)