flwr-nightly 1.13.0.dev20241024__tar.gz → 1.13.0.dev20241028__tar.gz

{flwr_nightly-1.13.0.dev20241024 → flwr_nightly-1.13.0.dev20241028}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.13.0.dev20241024
+Version: 1.13.0.dev20241028
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.13.0.dev20241024 → flwr_nightly-1.13.0.dev20241028}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.13.0.dev20241024"
+version = "1.13.0.dev20241028"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]

{flwr_nightly-1.13.0.dev20241024 → flwr_nightly-1.13.0.dev20241028}/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl

@@ -10,7 +10,7 @@ license = "Apache-2.0"
 dependencies = [
     "flwr[simulation]>=1.12.0",
     "flwr-datasets[vision]>=0.3.0",
-    "tensorflow>=2.11.1",
+    "tensorflow>=2.11.1,<2.18.0",
 ]
 
 [tool.hatch.build.targets.wheel]

{flwr_nightly-1.13.0.dev20241024 → flwr_nightly-1.13.0.dev20241028}/src/py/flwr/server/app.py

@@ -50,7 +50,6 @@ from flwr.common.constant import (
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
     TRANSPORT_TYPE_REST,
-    Status,
 )
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log
@@ -58,7 +57,6 @@ from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
     private_key_to_bytes,
     public_key_to_bytes,
 )
-from flwr.common.typing import RunStatus
 from flwr.proto.fleet_pb2_grpc import (  # pylint: disable=E0611
     add_FleetServicer_to_server,
 )
@@ -345,7 +343,7 @@ def run_superlink() -> None:
         # Scheduler thread
         scheduler_th = threading.Thread(
             target=_flwr_serverapp_scheduler,
-            args=(state_factory, args.driver_api_address),
+            args=(state_factory, args.driver_api_address, args.ssl_ca_certfile),
         )
         scheduler_th.start()
         bckg_threads.append(scheduler_th)
@@ -367,7 +365,9 @@ def run_superlink() -> None:
 
 
 def _flwr_serverapp_scheduler(
-    state_factory: LinkStateFactory, driver_api_address: str
+    state_factory: LinkStateFactory,
+    driver_api_address: str,
+    ssl_ca_certfile: Optional[str],
 ) -> None:
     log(DEBUG, "Started flwr-serverapp scheduler thread.")
 
@@ -380,10 +380,6 @@ def _flwr_serverapp_scheduler(
 
         if pending_run_id:
 
-            # Set run as starting
-            state.update_run_status(
-                run_id=pending_run_id, new_status=RunStatus(Status.STARTING, "", "")
-            )
             log(
                 INFO,
                 "Launching `flwr-serverapp` subprocess with run-id %d. "
@@ -399,6 +395,12 @@ def _flwr_serverapp_scheduler(
                 "--run-id",
                 str(pending_run_id),
             ]
+            if ssl_ca_certfile:
+                command.append("--root-certificates")
+                command.append(ssl_ca_certfile)
+            else:
+                command.append("--insecure")
+
             subprocess.run(
                 command,
                 stdout=None,

flwr_nightly-1.13.0.dev20241028/src/py/flwr/server/serverapp/app.py

@@ -0,0 +1,141 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower ServerApp process."""
+
+import argparse
+import sys
+from logging import DEBUG, INFO, WARN
+from os.path import isfile
+from pathlib import Path
+from typing import Optional
+
+from flwr.common.logger import log
+from flwr.server.driver.grpc_driver import GrpcDriver
+
+
+def flwr_serverapp() -> None:
+    """Run process-isolated Flower ServerApp."""
+    log(INFO, "Starting Flower ServerApp")
+
+    parser = argparse.ArgumentParser(
+        description="Run a Flower ServerApp",
+    )
+    parser.add_argument(
+        "--superlink",
+        type=str,
+        help="Address of SuperLink's DriverAPI",
+    )
+    parser.add_argument(
+        "--run-id",
+        type=int,
+        required=False,
+        help="Id of the Run this process should start. If not supplied, this "
+        "function will request a pending run to the LinkState.",
+    )
+    parser.add_argument(
+        "--flwr-dir",
+        default=None,
+        help="""The path containing installed Flower Apps.
+    By default, this value is equal to:
+
+        - `$FLWR_HOME/` if `$FLWR_HOME` is defined
+        - `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined
+        - `$HOME/.flwr/` in all other cases
+    """,
+    )
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Run the server without HTTPS, regardless of whether certificate "
+        "paths are provided. By default, the server runs with HTTPS enabled. "
+        "Use this flag only if you understand the risks.",
+    )
+    parser.add_argument(
+        "--root-certificates",
+        metavar="ROOT_CERT",
+        type=str,
+        help="Specifies the path to the PEM-encoded root certificate file for "
+        "establishing secure HTTPS connections.",
+    )
+    args = parser.parse_args()
+
+    certificates = _try_obtain_certificates(args)
+
+    log(
+        DEBUG,
+        "Staring isolated `ServerApp` connected to SuperLink DriverAPI at %s "
+        "for run-id %s",
+        args.superlink,
+        args.run_id,
+    )
+    run_serverapp(
+        superlink=args.superlink,
+        run_id=args.run_id,
+        flwr_dir_=args.flwr_dir,
+        certificates=certificates,
+    )
+
+
+def _try_obtain_certificates(
+    args: argparse.Namespace,
+) -> Optional[bytes]:
+
+    if args.insecure:
+        if args.root_certificates is not None:
+            sys.exit(
+                "Conflicting options: The '--insecure' flag disables HTTPS, "
+                "but '--root-certificates' was also specified. Please remove "
+                "the '--root-certificates' option when running in insecure mode, "
+                "or omit '--insecure' to use HTTPS."
+            )
+        log(
+            WARN,
+            "Option `--insecure` was set. Starting insecure HTTP channel to %s.",
+            args.superlink,
+        )
+        root_certificates = None
+    else:
+        # Load the certificates if provided, or load the system certificates
+        if not isfile(args.root_certificates):
+            sys.exit("Path argument `--root-certificates` does not point to a file.")
+        root_certificates = Path(args.root_certificates).read_bytes()
+        log(
+            DEBUG,
+            "Starting secure HTTPS channel to %s "
+            "with the following certificates: %s.",
+            args.superlink,
+            args.root_certificates,
+        )
+    return root_certificates
+
+
+def run_serverapp(  # pylint: disable=R0914
+    superlink: str,
+    run_id: Optional[int] = None,
+    flwr_dir_: Optional[str] = None,
+    certificates: Optional[bytes] = None,
+) -> None:
+    """Run Flower ServerApp process."""
+    _ = GrpcDriver(
+        run_id=run_id if run_id else 0,
+        driver_service_address=superlink,
+        root_certificates=certificates,
+    )
+
+    log(INFO, "%s", flwr_dir_)
+
+    # Then, GetServerInputs
+
+    # Then, run ServerApp

{flwr_nightly-1.13.0.dev20241024 → flwr_nightly-1.13.0.dev20241028}/src/py/flwr/server/superlink/driver/driver_servicer.py

@@ -15,21 +15,26 @@
 """Driver API servicer."""
 
 
+import threading
 import time
-from logging import DEBUG
+from logging import DEBUG, INFO
 from typing import Optional
 from uuid import UUID
 
 import grpc
+from google.protobuf.message import Message as GrpcMessage
 
+from flwr.common.constant import Status
 from flwr.common.logger import log
 from flwr.common.serde import (
+    context_from_proto,
+    context_to_proto,
     fab_from_proto,
     fab_to_proto,
+    run_to_proto,
     user_config_from_proto,
-    user_config_to_proto,
 )
-from flwr.common.typing import Fab
+from flwr.common.typing import Fab, RunStatus
 from flwr.proto import driver_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
     GetNodesRequest,
@@ -50,7 +55,6 @@ from flwr.proto.run_pb2 import (  # pylint: disable=E0611
     CreateRunResponse,
     GetRunRequest,
     GetRunResponse,
-    Run,
 )
 from flwr.proto.task_pb2 import TaskRes  # pylint: disable=E0611
 from flwr.server.superlink.ffs.ffs import Ffs
@@ -67,6 +71,7 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
     ) -> None:
         self.state_factory = state_factory
         self.ffs_factory = ffs_factory
+        self.lock = threading.RLock()
 
     def GetNodes(
         self, request: GetNodesRequest, context: grpc.ServicerContext
@@ -181,15 +186,7 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         if run is None:
             return GetRunResponse()
 
-        return GetRunResponse(
-            run=Run(
-                run_id=run.run_id,
-                fab_id=run.fab_id,
-                fab_version=run.fab_version,
-                override_config=user_config_to_proto(run.override_config),
-                fab_hash=run.fab_hash,
-            )
-        )
+        return GetRunResponse(run=run_to_proto(run))
 
     def GetFab(
         self, request: GetFabRequest, context: grpc.ServicerContext
@@ -208,15 +205,60 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         self, request: PullServerAppInputsRequest, context: grpc.ServicerContext
     ) -> PullServerAppInputsResponse:
         """Pull ServerApp process inputs."""
-        raise NotImplementedError()
+        log(DEBUG, "DriverServicer.PullServerAppInputs")
+        # Init access to LinkState and Ffs
+        state = self.state_factory.state()
+        ffs = self.ffs_factory.ffs()
+
+        # Lock access to LinkState, preventing obtaining the same pending run_id
+        with self.lock:
+            # If run_id is provided, use it, otherwise use the pending run_id
+            if _has_field(request, "run_id"):
+                run_id: Optional[int] = request.run_id
+            else:
+                run_id = state.get_pending_run_id()
+            # If there's no pending run, return an empty response
+            if run_id is None:
+                return PullServerAppInputsResponse()
+
+            # Retrieve Context, Run and Fab for the run_id
+            serverapp_ctxt = state.get_serverapp_context(run_id)
+            run = state.get_run(run_id)
+            fab = None
+            if run and run.fab_hash:
+                if result := ffs.get(run.fab_hash):
+                    fab = Fab(run.fab_hash, result[0])
+            if run and fab:
+                # Update run status to STARTING
+                if state.update_run_status(run_id, RunStatus(Status.STARTING, "", "")):
+                    log(INFO, "Starting run %d", run_id)
+                    return PullServerAppInputsResponse(
+                        context=(
+                            context_to_proto(serverapp_ctxt) if serverapp_ctxt else None
+                        ),
+                        run=run_to_proto(run),
+                        fab=fab_to_proto(fab),
+                    )
+
+        # Raise an exception if the Run or Fab is not found,
+        # or if the status cannot be updated to STARTING
+        raise RuntimeError(f"Failed to start run {run_id}")
 
     def PushServerAppOutputs(
         self, request: PushServerAppOutputsRequest, context: grpc.ServicerContext
     ) -> PushServerAppOutputsResponse:
         """Push ServerApp process outputs."""
-        raise NotImplementedError()
+        log(DEBUG, "DriverServicer.PushServerAppOutputs")
+        state = self.state_factory.state()
+        state.set_serverapp_context(request.run_id, context_from_proto(request.context))
+        return PushServerAppOutputsResponse()
 
 
 def _raise_if(validation_error: bool, detail: str) -> None:
     if validation_error:
         raise ValueError(f"Malformed PushTaskInsRequest: {detail}")
+
+
+def _has_field(message: GrpcMessage, field_name: str) -> bool:
+    """Check if a certain field is set for the message, including scalar fields."""
+    return field_name in {fld.name for fld, _ in message.ListFields()}

flwr_nightly-1.13.0.dev20241024/src/py/flwr/server/serverapp/app.py

@@ -1,78 +0,0 @@
-# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Flower ServerApp process."""
-
-import argparse
-from logging import DEBUG, INFO
-from typing import Optional
-
-from flwr.common.logger import log
-from flwr.server.driver.grpc_driver import GrpcDriver
-
-
-def flwr_serverapp() -> None:
-    """Run process-isolated Flower ServerApp."""
-    log(INFO, "Starting Flower ServerApp")
-
-    parser = argparse.ArgumentParser(
-        description="Run a Flower ServerApp",
-    )
-    parser.add_argument(
-        "--superlink",
-        type=str,
-        help="Address of SuperLink's DriverAPI",
-    )
-    parser.add_argument(
-        "--run-id",
-        type=int,
-        required=False,
-        help="Id of the Run this process should start. If not supplied, this "
-        "function will request a pending run to the LinkState.",
-    )
-    args = parser.parse_args()
-
-    log(
-        DEBUG,
-        "Staring isolated `ServerApp` connected to SuperLink DriverAPI at %s "
-        "for run-id %s",
-        args.superlink,
-        args.run_id,
-    )
-    run_serverapp(superlink=args.superlink, run_id=args.run_id)
-
-
-def run_serverapp(  # pylint: disable=R0914
-    superlink: str,
-    run_id: Optional[int] = None,
-) -> None:
-    """Run Flower ServerApp process.
-
-    Parameters
-    ----------
-    superlink : str
-        Address of SuperLink
-    run_id : Optional[int] (default: None)
-        Unique identifier of a Run registered at the LinkState. If not supplied,
-        this function will request a pending run to the LinkState.
-    """
-    _ = GrpcDriver(
-        run_id=run_id if run_id else 0,
-        driver_service_address=superlink,
-        root_certificates=None,
-    )
-
-    # Then, GetServerInputs
-
-    # Then, run ServerApp