flwr-nightly 1.13.0.dev20241114__py3-none-any.whl → 1.13.0.dev20241116__py3-none-any.whl

flwr/server/run_serverapp.py

@@ -15,33 +15,15 @@
 """Run ServerApp."""
 
 
-import argparse
 import sys
-from logging import DEBUG, INFO, WARN
-from pathlib import Path
+from logging import DEBUG, ERROR
 from typing import Optional
 
-from flwr.cli.config_utils import get_fab_metadata
-from flwr.cli.install import install_from_fab
-from flwr.common import Context, EventType, RecordSet, event
-from flwr.common.config import (
-    get_flwr_dir,
-    get_fused_config_from_dir,
-    get_metadata_from_config,
-    get_project_config,
-    get_project_dir,
-)
-from flwr.common.constant import SERVERAPPIO_API_DEFAULT_ADDRESS
-from flwr.common.logger import log, update_console_handler, warn_deprecated_feature
+from flwr.common import Context
+from flwr.common.logger import log, warn_unsupported_feature
 from flwr.common.object_ref import load_app
-from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
-from flwr.proto.run_pb2 import (  # pylint: disable=E0611
-    CreateRunRequest,
-    CreateRunResponse,
-)
 
 from .driver import Driver
-from .driver.grpc_driver import GrpcDriver
 from .server_app import LoadServerAppError, ServerApp
 
 
@@ -87,221 +69,9 @@ def run(
 # pylint: disable-next=too-many-branches,too-many-statements,too-many-locals
 def run_server_app() -> None:
     """Run Flower server app."""
-    event(EventType.RUN_SERVER_APP_ENTER)
-
-    args = _parse_args_run_server_app().parse_args()
-
-    # Check if the server app reference is passed.
-    # Since Flower 1.11, passing a reference is not allowed.
-    app_path: Optional[str] = args.app
-    # If the provided app_path doesn't exist, and contains a ":",
-    # it is likely to be a server app reference instead of a path.
-    if app_path is not None and not Path(app_path).exists() and ":" in app_path:
-        sys.exit(
-            "It appears you've passed a reference like `server:app`.\n\n"
-            "Note that since version `1.11.0`, `flower-server-app` no longer supports "
-            "passing a reference to a `ServerApp` attribute. Instead, you need to pass "
-            "the path to Flower app via the argument `--app`. This is the path to a "
-            "directory containing a `pyproject.toml`. You can create a valid Flower "
-            "app by executing `flwr new` and following the prompt."
-        )
-
-    if args.server != SERVERAPPIO_API_DEFAULT_ADDRESS:
-        warn = "Passing flag --server is deprecated. Use --superlink instead."
-        warn_deprecated_feature(warn)
-
-        if args.superlink != SERVERAPPIO_API_DEFAULT_ADDRESS:
-            # if `--superlink` also passed, then
-            # warn user that this argument overrides what was passed with `--server`
-            log(
-                WARN,
-                "Both `--server` and `--superlink` were passed. "
-                "`--server` will be ignored. Connecting to the "
-                "SuperLink ServerAppIo API at %s.",
-                args.superlink,
-            )
-        else:
-            args.superlink = args.server
-
-    update_console_handler(
-        level=DEBUG if args.verbose else INFO,
-        timestamps=args.verbose,
-        colored=True,
-    )
-
-    # Obtain certificates
-    if args.insecure:
-        if args.root_certificates is not None:
-            sys.exit(
-                "Conflicting options: The '--insecure' flag disables HTTPS, "
-                "but '--root-certificates' was also specified. Please remove "
-                "the '--root-certificates' option when running in insecure mode, "
-                "or omit '--insecure' to use HTTPS."
-            )
-        log(
-            WARN,
-            "Option `--insecure` was set. "
-            "Starting insecure HTTP client connected to %s.",
-            args.superlink,
-        )
-        root_certificates = None
-    else:
-        # Load the certificates if provided, or load the system certificates
-        cert_path = args.root_certificates
-        if cert_path is None:
-            root_certificates = None
-        else:
-            root_certificates = Path(cert_path).read_bytes()
-        log(
-            DEBUG,
-            "Starting secure HTTPS client connected to %s "
-            "with the following certificates: %s.",
-            args.superlink,
-            cert_path,
-        )
-
-    if not (app_path is None) ^ (args.run_id is None):
-        raise sys.exit(
-            "Please provide either a Flower App path or a Run ID, but not both. "
-            "For more details, use: ``flower-server-app -h``"
-        )
-
-    # Initialize GrpcDriver
-    if app_path is None:
-        # User provided `--run-id`, but not `app_dir`
-        driver = GrpcDriver(
-            serverappio_service_address=args.superlink,
-            root_certificates=root_certificates,
-        )
-        flwr_dir = get_flwr_dir(args.flwr_dir)
-        driver.set_run(args.run_id)
-        run_ = driver.run
-        if not run_.fab_hash:
-            raise ValueError("FAB hash not provided.")
-        fab_req = GetFabRequest(hash_str=run_.fab_hash)
-        # pylint: disable-next=W0212
-        fab_res: GetFabResponse = driver._stub.GetFab(fab_req)
-        if fab_res.fab.hash_str != run_.fab_hash:
-            raise ValueError("FAB hashes don't match.")
-        install_from_fab(fab_res.fab.content, flwr_dir, True)
-        fab_id, fab_version = get_fab_metadata(fab_res.fab.content)
-
-        app_path = str(get_project_dir(fab_id, fab_version, run_.fab_hash, flwr_dir))
-        config = get_project_config(app_path)
-        run_id = run_.run_id
-    else:
-        # User provided `app_dir`, but not `--run-id`
-        # Create run if run_id is not provided
-        driver = GrpcDriver(
-            serverappio_service_address=args.superlink,
-            root_certificates=root_certificates,
-        )
-        # Load config from the project directory
-        config = get_project_config(app_path)
-        fab_version, fab_id = get_metadata_from_config(config)
-
-        # Create run
-        req = CreateRunRequest(fab_id=fab_id, fab_version=fab_version)
-        res: CreateRunResponse = driver._stub.CreateRun(req)  # pylint: disable=W0212
-        # Fetch full `Run` using `run_id`
-        driver.set_run(res.run_id)  # pylint: disable=W0212
-        run_id = res.run_id
-
-    # Obtain server app reference and the run config
-    server_app_attr = config["tool"]["flwr"]["app"]["components"]["serverapp"]
-    server_app_run_config = get_fused_config_from_dir(
-        Path(app_path), driver.run.override_config
+    warn_unsupported_feature(
+        "The command `flower-server-app` is deprecated and no longer in use. "
+        "Use the `flwr-serverapp` exclusively instead."
     )
-
-    log(DEBUG, "Flower will load ServerApp `%s` in %s", server_app_attr, app_path)
-
-    log(
-        DEBUG,
-        "root_certificates: `%s`",
-        root_certificates,
-    )
-
-    # Initialize Context
-    context = Context(
-        run_id=run_id,
-        node_id=0,
-        node_config={},
-        state=RecordSet(),
-        run_config=server_app_run_config,
-    )
-
-    # Run the ServerApp with the Driver
-    run(
-        driver=driver,
-        context=context,
-        server_app_dir=app_path,
-        server_app_attr=server_app_attr,
-    )
-
-    # Clean up
-    driver.close()
-
-    event(EventType.RUN_SERVER_APP_LEAVE)
-
-
-def _parse_args_run_server_app() -> argparse.ArgumentParser:
-    """Parse flower-server-app command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="Start a Flower server app",
-    )
-
-    parser.add_argument(
-        "app",
-        nargs="?",
-        default=None,
-        help="Load and run the `ServerApp` from the specified Flower App path. "
-        "The `pyproject.toml` file must be located in the root of this path.",
-    )
-    parser.add_argument(
-        "--insecure",
-        action="store_true",
-        help="Run the `ServerApp` without HTTPS. By default, the app runs with "
-        "HTTPS enabled. Use this flag only if you understand the risks.",
-    )
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        help="Set the logging to `DEBUG`.",
-    )
-    parser.add_argument(
-        "--root-certificates",
-        metavar="ROOT_CERT",
-        type=str,
-        help="Specifies the path to the PEM-encoded root certificate file for "
-        "establishing secure HTTPS connections.",
-    )
-    parser.add_argument(
-        "--server",
-        default=SERVERAPPIO_API_DEFAULT_ADDRESS,
-        help="Server address",
-    )
-    parser.add_argument(
-        "--superlink",
-        default=SERVERAPPIO_API_DEFAULT_ADDRESS,
-        help="SuperLink ServerAppIo API (gRPC-rere) address "
-        "(IPv4, IPv6, or a domain name)",
-    )
-    parser.add_argument(
-        "--run-id",
-        default=None,
-        type=int,
-        help="The identifier of the run.",
-    )
-    parser.add_argument(
-        "--flwr-dir",
-        default=None,
-        help="""The path containing installed Flower Apps.
-    By default, this value is equal to:
-
-        - `$FLWR_HOME/` if `$FLWR_HOME` is defined
-        - `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined
-        - `$HOME/.flwr/` in all other cases
-    """,
-    )
-
-    return parser
+    log(ERROR, "`flower-server-app` used.")
+    sys.exit()

flwr/server/serverapp/app.py

@@ -30,7 +30,11 @@ from flwr.common.config import (
     get_project_config,
     get_project_dir,
 )
-from flwr.common.constant import Status, SubStatus
+from flwr.common.constant import (
+    SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
+    Status,
+    SubStatus,
+)
 from flwr.common.logger import (
     log,
     mirror_output_to_queue,
@@ -62,33 +66,18 @@ def flwr_serverapp() -> None:
     log_queue: Queue[Optional[str]] = Queue()
     mirror_output_to_queue(log_queue)
 
-    parser = argparse.ArgumentParser(
-        description="Run a Flower ServerApp",
-    )
-    parser.add_argument(
-        "--superlink",
-        type=str,
-        help="Address of SuperLink's ServerAppIo API",
-    )
-    parser.add_argument(
-        "--run-once",
-        action="store_true",
-        help="When set, this process will start a single ServerApp for a pending Run. "
-        "If there is no pending Run, the process will exit.",
-    )
-    add_args_flwr_app_common(parser=parser)
-    args = parser.parse_args()
+    args = _parse_args_run_flwr_serverapp().parse_args()
 
     log(INFO, "Starting Flower ServerApp")
-    certificates = try_obtain_root_certificates(args, args.superlink)
+    certificates = try_obtain_root_certificates(args, args.serverappio_api_address)
 
     log(
         DEBUG,
         "Starting isolated `ServerApp` connected to SuperLink's ServerAppIo API at %s",
-        args.superlink,
+        args.serverappio_api_address,
     )
     run_serverapp(
-        superlink=args.superlink,
+        serverappio_api_address=args.serverappio_api_address,
         log_queue=log_queue,
         run_once=args.run_once,
         flwr_dir=args.flwr_dir,
@@ -100,7 +89,7 @@ def flwr_serverapp() -> None:
 
 
 def run_serverapp(  # pylint: disable=R0914, disable=W0212
-    superlink: str,
+    serverappio_api_address: str,
     log_queue: Queue[Optional[str]],
     run_once: bool,
     flwr_dir: Optional[str] = None,
@@ -108,7 +97,7 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
 ) -> None:
     """Run Flower ServerApp process."""
     driver = GrpcDriver(
-        serverappio_service_address=superlink,
+        serverappio_service_address=serverappio_api_address,
         root_certificates=certificates,
     )
 
@@ -212,3 +201,25 @@ def run_serverapp(  # pylint: disable=R0914, disable=W0212
         # Stop the loop if `flwr-serverapp` is expected to process a single run
         if run_once:
             break
+
+
+def _parse_args_run_flwr_serverapp() -> argparse.ArgumentParser:
+    """Parse flwr-serverapp command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Run a Flower ServerApp",
+    )
+    parser.add_argument(
+        "--serverappio-api-address",
+        default=SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS,
+        type=str,
+        help="Address of SuperLink's ServerAppIo API (IPv4, IPv6, or a domain name)."
+        f"By default, it is set to {SERVERAPPIO_API_DEFAULT_CLIENT_ADDRESS}.",
+    )
+    parser.add_argument(
+        "--run-once",
+        action="store_true",
+        help="When set, this process will start a single ServerApp for a pending Run. "
+        "If there is no pending Run, the process will exit.",
+    )
+    add_args_flwr_app_common(parser=parser)
+    return parser

flwr/server/superlink/linkstate/in_memory_linkstate.py

@@ -40,6 +40,8 @@ from .utils import (
     generate_rand_int_from_bytes,
     has_valid_sub_status,
     is_valid_transition,
+    verify_found_taskres,
+    verify_taskins_ids,
 )
 
 
@@ -67,12 +69,13 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
         self.federation_options: dict[int, ConfigsRecord] = {}
         self.task_ins_store: dict[UUID, TaskIns] = {}
         self.task_res_store: dict[UUID, TaskRes] = {}
+        self.task_ins_id_to_task_res_id: dict[UUID, UUID] = {}
 
         self.node_public_keys: set[bytes] = set()
         self.server_public_key: Optional[bytes] = None
         self.server_private_key: Optional[bytes] = None
 
-        self.lock = threading.Lock()
+        self.lock = threading.RLock()
 
     def store_task_ins(self, task_ins: TaskIns) -> Optional[UUID]:
         """Store one TaskIns."""
@@ -222,42 +225,50 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
         task_res.task_id = str(task_id)
         with self.lock:
             self.task_res_store[task_id] = task_res
+            self.task_ins_id_to_task_res_id[UUID(task_ins_id)] = task_id
 
         # Return the new task_id
         return task_id
 
     def get_task_res(self, task_ids: set[UUID]) -> list[TaskRes]:
-        """Get all TaskRes that have not been delivered yet."""
+        """Get TaskRes for the given TaskIns IDs."""
+        ret: dict[UUID, TaskRes] = {}
+
         with self.lock:
-            # Find TaskRes that were not delivered yet
-            task_res_list: list[TaskRes] = []
-            replied_task_ids: set[UUID] = set()
-            for _, task_res in self.task_res_store.items():
-                reply_to = UUID(task_res.task.ancestry[0])
-
-                # Check if corresponding TaskIns exists and is not expired
-                task_ins = self.task_ins_store.get(reply_to)
-                if task_ins is None:
-                    log(WARNING, "TaskIns with task_id %s does not exist.", reply_to)
-                    task_ids.remove(reply_to)
-                    continue
-
-                if task_ins.task.created_at + task_ins.task.ttl <= time.time():
-                    log(WARNING, "TaskIns with task_id %s is expired.", reply_to)
-                    task_ids.remove(reply_to)
-                    continue
-
-                if reply_to in task_ids and task_res.task.delivered_at == "":
-                    task_res_list.append(task_res)
-                    replied_task_ids.add(reply_to)
-
-            # Mark all of them as delivered
+            current = time.time()
+
+            # Verify TaskIns IDs
+            ret = verify_taskins_ids(
+                inquired_taskins_ids=task_ids,
+                found_taskins_dict=self.task_ins_store,
+                current_time=current,
+            )
+
+            # Find all TaskRes
+            task_res_found: list[TaskRes] = []
+            for task_id in task_ids:
+                # If TaskRes exists and is not delivered, add it to the list
+                if task_res_id := self.task_ins_id_to_task_res_id.get(task_id):
+                    task_res = self.task_res_store[task_res_id]
+                    if task_res.task.delivered_at == "":
+                        task_res_found.append(task_res)
+            tmp_ret_dict = verify_found_taskres(
+                inquired_taskins_ids=task_ids,
+                found_taskins_dict=self.task_ins_store,
+                found_taskres_list=task_res_found,
+                current_time=current,
+            )
+            ret.update(tmp_ret_dict)
+
+            # Mark existing TaskRes to be returned as delivered
             delivered_at = now().isoformat()
-            for task_res in task_res_list:
+            for task_res in task_res_found:
                 task_res.task.delivered_at = delivered_at
 
-            # Return TaskRes
-            return task_res_list
+            # Cleanup
+            self._force_delete_tasks_by_ids(set(ret.keys()))
+
+        return list(ret.values())
 
     def delete_tasks(self, task_ids: set[UUID]) -> None:
         """Delete all delivered TaskIns/TaskRes pairs."""
@@ -278,9 +289,25 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
 
             for task_id in task_ins_to_be_deleted:
                 del self.task_ins_store[task_id]
+                del self.task_ins_id_to_task_res_id[task_id]
             for task_id in task_res_to_be_deleted:
                 del self.task_res_store[task_id]
 
+    def _force_delete_tasks_by_ids(self, task_ids: set[UUID]) -> None:
+        """Delete tasks based on a set of TaskIns IDs."""
+        if not task_ids:
+            return
+
+        with self.lock:
+            for task_id in task_ids:
+                # Delete TaskIns
+                if task_id in self.task_ins_store:
+                    del self.task_ins_store[task_id]
+                # Delete TaskRes
+                if task_id in self.task_ins_id_to_task_res_id:
+                    task_res_id = self.task_ins_id_to_task_res_id.pop(task_id)
+                    del self.task_res_store[task_res_id]
+
     def num_task_ins(self) -> int:
         """Calculate the number of task_ins in store.
 

flwr/server/superlink/linkstate/linkstate.py

@@ -101,13 +101,27 @@ class LinkState(abc.ABC):  # pylint: disable=R0904
 
     @abc.abstractmethod
     def get_task_res(self, task_ids: set[UUID]) -> list[TaskRes]:
-        """Get TaskRes for task_ids.
+        """Get TaskRes for the given TaskIns IDs.
 
-        Usually, the ServerAppIo API calls this method to get results for instructions
-        it has previously scheduled.
+        This method is typically called by the ServerAppIo API to obtain
+        results (TaskRes) for previously scheduled instructions (TaskIns).
+        For each task_id provided, this method returns one of the following responses:
 
-        Retrieves all TaskRes for the given `task_ids` and returns and empty list of
-        none could be found.
+        - An error TaskRes if the corresponding TaskIns does not exist or has expired.
+        - An error TaskRes if the corresponding TaskRes exists but has expired.
+        - The valid TaskRes if the TaskIns has a corresponding valid TaskRes.
+        - Nothing if the TaskIns is still valid and waiting for a TaskRes.
+
+        Parameters
+        ----------
+        task_ids : set[UUID]
+            A set of TaskIns IDs for which to retrieve results (TaskRes).
+
+        Returns
+        -------
+        list[TaskRes]
+            A list of TaskRes corresponding to the given task IDs. If no
+            TaskRes could be found for any of the task IDs, an empty list is returned.
         """
 
     @abc.abstractmethod