flwr-nightly 1.13.0.dev20241021__py3-none-any.whl → 1.13.0.dev20241111__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flwr-nightly might be problematic. Click here for more details.
- flwr/cli/build.py +2 -2
- flwr/cli/config_utils.py +97 -0
- flwr/cli/log.py +63 -97
- flwr/cli/new/templates/app/code/flwr_tune/dataset.py.tpl +1 -1
- flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +1 -0
- flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +1 -1
- flwr/cli/run/run.py +34 -88
- flwr/client/app.py +23 -20
- flwr/client/clientapp/app.py +22 -18
- flwr/client/nodestate/__init__.py +25 -0
- flwr/client/nodestate/in_memory_nodestate.py +38 -0
- flwr/client/nodestate/nodestate.py +30 -0
- flwr/client/nodestate/nodestate_factory.py +37 -0
- flwr/client/{node_state.py → run_info_store.py} +4 -3
- flwr/client/supernode/app.py +6 -8
- flwr/common/args.py +83 -0
- flwr/common/config.py +10 -0
- flwr/common/constant.py +39 -5
- flwr/common/context.py +9 -4
- flwr/common/date.py +3 -3
- flwr/common/logger.py +108 -1
- flwr/common/object_ref.py +47 -16
- flwr/common/serde.py +24 -0
- flwr/common/telemetry.py +0 -6
- flwr/common/typing.py +10 -1
- flwr/proto/exec_pb2.py +14 -17
- flwr/proto/exec_pb2.pyi +14 -22
- flwr/proto/log_pb2.py +29 -0
- flwr/proto/log_pb2.pyi +39 -0
- flwr/proto/log_pb2_grpc.py +4 -0
- flwr/proto/log_pb2_grpc.pyi +4 -0
- flwr/proto/message_pb2.py +8 -8
- flwr/proto/message_pb2.pyi +4 -1
- flwr/proto/run_pb2.py +32 -27
- flwr/proto/run_pb2.pyi +26 -0
- flwr/proto/serverappio_pb2.py +52 -0
- flwr/proto/{driver_pb2.pyi → serverappio_pb2.pyi} +54 -0
- flwr/proto/serverappio_pb2_grpc.py +376 -0
- flwr/proto/serverappio_pb2_grpc.pyi +147 -0
- flwr/proto/simulationio_pb2.py +38 -0
- flwr/proto/simulationio_pb2.pyi +65 -0
- flwr/proto/simulationio_pb2_grpc.py +205 -0
- flwr/proto/simulationio_pb2_grpc.pyi +81 -0
- flwr/server/app.py +272 -105
- flwr/server/driver/driver.py +15 -1
- flwr/server/driver/grpc_driver.py +25 -36
- flwr/server/driver/inmemory_driver.py +6 -16
- flwr/server/run_serverapp.py +29 -23
- flwr/server/{superlink/state → serverapp}/__init__.py +3 -9
- flwr/server/serverapp/app.py +214 -0
- flwr/server/strategy/aggregate.py +4 -4
- flwr/server/strategy/fedadam.py +11 -1
- flwr/server/superlink/driver/__init__.py +1 -1
- flwr/server/superlink/driver/{driver_grpc.py → serverappio_grpc.py} +19 -16
- flwr/server/superlink/driver/{driver_servicer.py → serverappio_servicer.py} +125 -39
- flwr/server/superlink/fleet/grpc_adapter/grpc_adapter_servicer.py +4 -2
- flwr/server/superlink/fleet/grpc_bidi/grpc_server.py +2 -2
- flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py +4 -2
- flwr/server/superlink/fleet/grpc_rere/server_interceptor.py +2 -2
- flwr/server/superlink/fleet/message_handler/message_handler.py +7 -7
- flwr/server/superlink/fleet/rest_rere/rest_api.py +7 -7
- flwr/server/superlink/fleet/vce/vce_api.py +23 -23
- flwr/server/superlink/linkstate/__init__.py +28 -0
- flwr/server/superlink/{state/in_memory_state.py → linkstate/in_memory_linkstate.py} +184 -36
- flwr/server/superlink/{state/state.py → linkstate/linkstate.py} +149 -19
- flwr/server/superlink/{state/state_factory.py → linkstate/linkstate_factory.py} +9 -9
- flwr/server/superlink/{state/sqlite_state.py → linkstate/sqlite_linkstate.py} +306 -65
- flwr/server/superlink/{state → linkstate}/utils.py +81 -30
- flwr/server/superlink/simulation/__init__.py +15 -0
- flwr/server/superlink/simulation/simulationio_grpc.py +65 -0
- flwr/server/superlink/simulation/simulationio_servicer.py +153 -0
- flwr/simulation/__init__.py +5 -1
- flwr/simulation/app.py +273 -345
- flwr/simulation/legacy_app.py +382 -0
- flwr/simulation/ray_transport/ray_client_proxy.py +2 -2
- flwr/simulation/run_simulation.py +57 -131
- flwr/simulation/simulationio_connection.py +86 -0
- flwr/superexec/app.py +6 -134
- flwr/superexec/deployment.py +61 -66
- flwr/superexec/exec_grpc.py +15 -8
- flwr/superexec/exec_servicer.py +36 -65
- flwr/superexec/executor.py +26 -7
- flwr/superexec/simulation.py +54 -107
- {flwr_nightly-1.13.0.dev20241021.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/METADATA +5 -4
- {flwr_nightly-1.13.0.dev20241021.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/RECORD +88 -69
- {flwr_nightly-1.13.0.dev20241021.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/entry_points.txt +2 -0
- flwr/client/node_state_tests.py +0 -66
- flwr/proto/driver_pb2.py +0 -42
- flwr/proto/driver_pb2_grpc.py +0 -239
- flwr/proto/driver_pb2_grpc.pyi +0 -94
- {flwr_nightly-1.13.0.dev20241021.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/LICENSE +0 -0
- {flwr_nightly-1.13.0.dev20241021.dist-info → flwr_nightly-1.13.0.dev20241111.dist-info}/WHEEL +0 -0
|
@@ -12,28 +12,30 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
# ==============================================================================
|
|
15
|
-
"""Abstract base class
|
|
15
|
+
"""Abstract base class LinkState."""
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
import abc
|
|
19
19
|
from typing import Optional
|
|
20
20
|
from uuid import UUID
|
|
21
21
|
|
|
22
|
-
from flwr.common
|
|
22
|
+
from flwr.common import Context
|
|
23
|
+
from flwr.common.record import ConfigsRecord
|
|
24
|
+
from flwr.common.typing import Run, RunStatus, UserConfig
|
|
23
25
|
from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611
|
|
24
26
|
|
|
25
27
|
|
|
26
|
-
class
|
|
27
|
-
"""Abstract
|
|
28
|
+
class LinkState(abc.ABC): # pylint: disable=R0904
|
|
29
|
+
"""Abstract LinkState."""
|
|
28
30
|
|
|
29
31
|
@abc.abstractmethod
|
|
30
32
|
def store_task_ins(self, task_ins: TaskIns) -> Optional[UUID]:
|
|
31
33
|
"""Store one TaskIns.
|
|
32
34
|
|
|
33
|
-
Usually, the
|
|
35
|
+
Usually, the ServerAppIo API calls this to schedule instructions.
|
|
34
36
|
|
|
35
|
-
Stores the value of the `task_ins` in the state and, if successful,
|
|
36
|
-
`task_id` (UUID) of the `task_ins`. If, for any reason,
|
|
37
|
+
Stores the value of the `task_ins` in the link state and, if successful,
|
|
38
|
+
returns the `task_id` (UUID) of the `task_ins`. If, for any reason,
|
|
37
39
|
storing the `task_ins` fails, `None` is returned.
|
|
38
40
|
|
|
39
41
|
Constraints
|
|
@@ -101,8 +103,8 @@ class State(abc.ABC): # pylint: disable=R0904
|
|
|
101
103
|
def get_task_res(self, task_ids: set[UUID]) -> list[TaskRes]:
|
|
102
104
|
"""Get TaskRes for task_ids.
|
|
103
105
|
|
|
104
|
-
Usually, the
|
|
105
|
-
previously scheduled.
|
|
106
|
+
Usually, the ServerAppIo API calls this method to get results for instructions
|
|
107
|
+
it has previously scheduled.
|
|
106
108
|
|
|
107
109
|
Retrieves all TaskRes for the given `task_ids` and returns and empty list of
|
|
108
110
|
none could be found.
|
|
@@ -130,11 +132,11 @@ class State(abc.ABC): # pylint: disable=R0904
|
|
|
130
132
|
def create_node(
|
|
131
133
|
self, ping_interval: float, public_key: Optional[bytes] = None
|
|
132
134
|
) -> int:
|
|
133
|
-
"""Create, store in state, and return `node_id`."""
|
|
135
|
+
"""Create, store in the link state, and return `node_id`."""
|
|
134
136
|
|
|
135
137
|
@abc.abstractmethod
|
|
136
138
|
def delete_node(self, node_id: int, public_key: Optional[bytes] = None) -> None:
|
|
137
|
-
"""Remove `node_id` from state."""
|
|
139
|
+
"""Remove `node_id` from the link state."""
|
|
138
140
|
|
|
139
141
|
@abc.abstractmethod
|
|
140
142
|
def get_nodes(self, run_id: int) -> set[int]:
|
|
@@ -151,15 +153,20 @@ class State(abc.ABC): # pylint: disable=R0904
|
|
|
151
153
|
"""Retrieve stored `node_id` filtered by `node_public_keys`."""
|
|
152
154
|
|
|
153
155
|
@abc.abstractmethod
|
|
154
|
-
def create_run(
|
|
156
|
+
def create_run( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
155
157
|
self,
|
|
156
158
|
fab_id: Optional[str],
|
|
157
159
|
fab_version: Optional[str],
|
|
158
160
|
fab_hash: Optional[str],
|
|
159
161
|
override_config: UserConfig,
|
|
162
|
+
federation_options: ConfigsRecord,
|
|
160
163
|
) -> int:
|
|
161
164
|
"""Create a new run for the specified `fab_hash`."""
|
|
162
165
|
|
|
166
|
+
@abc.abstractmethod
|
|
167
|
+
def get_run_ids(self) -> set[int]:
|
|
168
|
+
"""Retrieve all run IDs."""
|
|
169
|
+
|
|
163
170
|
@abc.abstractmethod
|
|
164
171
|
def get_run(self, run_id: int) -> Optional[Run]:
|
|
165
172
|
"""Retrieve information about the run with the specified `run_id`.
|
|
@@ -172,17 +179,77 @@ class State(abc.ABC): # pylint: disable=R0904
|
|
|
172
179
|
Returns
|
|
173
180
|
-------
|
|
174
181
|
Optional[Run]
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
182
|
+
The `Run` instance if found; otherwise, `None`.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
@abc.abstractmethod
|
|
186
|
+
def get_run_status(self, run_ids: set[int]) -> dict[int, RunStatus]:
|
|
187
|
+
"""Retrieve the statuses for the specified runs.
|
|
188
|
+
|
|
189
|
+
Parameters
|
|
190
|
+
----------
|
|
191
|
+
run_ids : set[int]
|
|
192
|
+
A set of run identifiers for which to retrieve statuses.
|
|
193
|
+
|
|
194
|
+
Returns
|
|
195
|
+
-------
|
|
196
|
+
dict[int, RunStatus]
|
|
197
|
+
A dictionary mapping each valid run ID to its corresponding status.
|
|
198
|
+
|
|
199
|
+
Notes
|
|
200
|
+
-----
|
|
201
|
+
Only valid run IDs that exist in the State will be included in the returned
|
|
202
|
+
dictionary. If a run ID is not found, it will be omitted from the result.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
@abc.abstractmethod
|
|
206
|
+
def update_run_status(self, run_id: int, new_status: RunStatus) -> bool:
|
|
207
|
+
"""Update the status of the run with the specified `run_id`.
|
|
208
|
+
|
|
209
|
+
Parameters
|
|
210
|
+
----------
|
|
211
|
+
run_id : int
|
|
212
|
+
The identifier of the run.
|
|
213
|
+
new_status : RunStatus
|
|
214
|
+
The new status to be assigned to the run.
|
|
215
|
+
|
|
216
|
+
Returns
|
|
217
|
+
-------
|
|
218
|
+
bool
|
|
219
|
+
True if the status update is successful; False otherwise.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
@abc.abstractmethod
|
|
223
|
+
def get_pending_run_id(self) -> Optional[int]:
|
|
224
|
+
"""Get the `run_id` of a run with `Status.PENDING` status.
|
|
225
|
+
|
|
226
|
+
Returns
|
|
227
|
+
-------
|
|
228
|
+
Optional[int]
|
|
229
|
+
The `run_id` of a `Run` that is pending to be started; None if
|
|
230
|
+
there is no Run pending.
|
|
231
|
+
"""
|
|
232
|
+
|
|
233
|
+
@abc.abstractmethod
|
|
234
|
+
def get_federation_options(self, run_id: int) -> Optional[ConfigsRecord]:
|
|
235
|
+
"""Retrieve the federation options for the specified `run_id`.
|
|
236
|
+
|
|
237
|
+
Parameters
|
|
238
|
+
----------
|
|
239
|
+
run_id : int
|
|
240
|
+
The identifier of the run.
|
|
241
|
+
|
|
242
|
+
Returns
|
|
243
|
+
-------
|
|
244
|
+
Optional[ConfigsRecord]
|
|
245
|
+
The federation options for the run if it exists; None otherwise.
|
|
179
246
|
"""
|
|
180
247
|
|
|
181
248
|
@abc.abstractmethod
|
|
182
249
|
def store_server_private_public_key(
|
|
183
250
|
self, private_key: bytes, public_key: bytes
|
|
184
251
|
) -> None:
|
|
185
|
-
"""Store `server_private_key` and `server_public_key` in state."""
|
|
252
|
+
"""Store `server_private_key` and `server_public_key` in the link state."""
|
|
186
253
|
|
|
187
254
|
@abc.abstractmethod
|
|
188
255
|
def get_server_private_key(self) -> Optional[bytes]:
|
|
@@ -194,11 +261,11 @@ class State(abc.ABC): # pylint: disable=R0904
|
|
|
194
261
|
|
|
195
262
|
@abc.abstractmethod
|
|
196
263
|
def store_node_public_keys(self, public_keys: set[bytes]) -> None:
|
|
197
|
-
"""Store a set of `node_public_keys` in state."""
|
|
264
|
+
"""Store a set of `node_public_keys` in the link state."""
|
|
198
265
|
|
|
199
266
|
@abc.abstractmethod
|
|
200
267
|
def store_node_public_key(self, public_key: bytes) -> None:
|
|
201
|
-
"""Store a `node_public_key` in state."""
|
|
268
|
+
"""Store a `node_public_key` in the link state."""
|
|
202
269
|
|
|
203
270
|
@abc.abstractmethod
|
|
204
271
|
def get_node_public_keys(self) -> set[bytes]:
|
|
@@ -222,3 +289,66 @@ class State(abc.ABC): # pylint: disable=R0904
|
|
|
222
289
|
is_acknowledged : bool
|
|
223
290
|
True if the ping is successfully acknowledged; otherwise, False.
|
|
224
291
|
"""
|
|
292
|
+
|
|
293
|
+
@abc.abstractmethod
|
|
294
|
+
def get_serverapp_context(self, run_id: int) -> Optional[Context]:
|
|
295
|
+
"""Get the context for the specified `run_id`.
|
|
296
|
+
|
|
297
|
+
Parameters
|
|
298
|
+
----------
|
|
299
|
+
run_id : int
|
|
300
|
+
The identifier of the run for which to retrieve the context.
|
|
301
|
+
|
|
302
|
+
Returns
|
|
303
|
+
-------
|
|
304
|
+
Optional[Context]
|
|
305
|
+
The context associated with the specified `run_id`, or `None` if no context
|
|
306
|
+
exists for the given `run_id`.
|
|
307
|
+
"""
|
|
308
|
+
|
|
309
|
+
@abc.abstractmethod
|
|
310
|
+
def set_serverapp_context(self, run_id: int, context: Context) -> None:
|
|
311
|
+
"""Set the context for the specified `run_id`.
|
|
312
|
+
|
|
313
|
+
Parameters
|
|
314
|
+
----------
|
|
315
|
+
run_id : int
|
|
316
|
+
The identifier of the run for which to set the context.
|
|
317
|
+
context : Context
|
|
318
|
+
The context to be associated with the specified `run_id`.
|
|
319
|
+
"""
|
|
320
|
+
|
|
321
|
+
@abc.abstractmethod
|
|
322
|
+
def add_serverapp_log(self, run_id: int, log_message: str) -> None:
|
|
323
|
+
"""Add a log entry to the ServerApp logs for the specified `run_id`.
|
|
324
|
+
|
|
325
|
+
Parameters
|
|
326
|
+
----------
|
|
327
|
+
run_id : int
|
|
328
|
+
The identifier of the run for which to add a log entry.
|
|
329
|
+
log_message : str
|
|
330
|
+
The log entry to be added to the ServerApp logs.
|
|
331
|
+
"""
|
|
332
|
+
|
|
333
|
+
@abc.abstractmethod
|
|
334
|
+
def get_serverapp_log(
|
|
335
|
+
self, run_id: int, after_timestamp: Optional[float]
|
|
336
|
+
) -> tuple[str, float]:
|
|
337
|
+
"""Get the ServerApp logs for the specified `run_id`.
|
|
338
|
+
|
|
339
|
+
Parameters
|
|
340
|
+
----------
|
|
341
|
+
run_id : int
|
|
342
|
+
The identifier of the run for which to retrieve the ServerApp logs.
|
|
343
|
+
|
|
344
|
+
after_timestamp : Optional[float]
|
|
345
|
+
Retrieve logs after this timestamp. If set to `None`, retrieve all logs.
|
|
346
|
+
|
|
347
|
+
Returns
|
|
348
|
+
-------
|
|
349
|
+
tuple[str, float]
|
|
350
|
+
A tuple containing:
|
|
351
|
+
- The ServerApp logs associated with the specified `run_id`.
|
|
352
|
+
- The timestamp of the latest log entry in the returned logs.
|
|
353
|
+
Returns `0` if no logs are returned.
|
|
354
|
+
"""
|
|
@@ -20,13 +20,13 @@ from typing import Optional
|
|
|
20
20
|
|
|
21
21
|
from flwr.common.logger import log
|
|
22
22
|
|
|
23
|
-
from .
|
|
24
|
-
from .
|
|
25
|
-
from .
|
|
23
|
+
from .in_memory_linkstate import InMemoryLinkState
|
|
24
|
+
from .linkstate import LinkState
|
|
25
|
+
from .sqlite_linkstate import SqliteLinkState
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
class
|
|
29
|
-
"""Factory class that creates
|
|
28
|
+
class LinkStateFactory:
|
|
29
|
+
"""Factory class that creates LinkState instances.
|
|
30
30
|
|
|
31
31
|
Parameters
|
|
32
32
|
----------
|
|
@@ -39,19 +39,19 @@ class StateFactory:
|
|
|
39
39
|
|
|
40
40
|
def __init__(self, database: str) -> None:
|
|
41
41
|
self.database = database
|
|
42
|
-
self.state_instance: Optional[
|
|
42
|
+
self.state_instance: Optional[LinkState] = None
|
|
43
43
|
|
|
44
|
-
def state(self) ->
|
|
44
|
+
def state(self) -> LinkState:
|
|
45
45
|
"""Return a State instance and create it, if necessary."""
|
|
46
46
|
# InMemoryState
|
|
47
47
|
if self.database == ":flwr-in-memory-state:":
|
|
48
48
|
if self.state_instance is None:
|
|
49
|
-
self.state_instance =
|
|
49
|
+
self.state_instance = InMemoryLinkState()
|
|
50
50
|
log(DEBUG, "Using InMemoryState")
|
|
51
51
|
return self.state_instance
|
|
52
52
|
|
|
53
53
|
# SqliteState
|
|
54
|
-
state =
|
|
54
|
+
state = SqliteLinkState(self.database)
|
|
55
55
|
state.initialize()
|
|
56
56
|
log(DEBUG, "Using SqliteState")
|
|
57
57
|
return state
|