flwr-nightly 1.10.0.dev20240611__tar.gz → 1.10.0.dev20240613__tar.gz

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.10.0.dev20240611
+Version: 1.10.0.dev20240613
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.10.0.dev20240611"
+version = "1.10.0.dev20240613"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]
@@ -53,9 +53,8 @@ exclude = [
 
 [tool.poetry.scripts]
 flwr = "flwr.cli.app:app"
-flower-driver-api = "flwr.server:run_driver_api"
-flower-fleet-api = "flwr.server:run_fleet_api"
 flower-superlink = "flwr.server:run_superlink"
+flower-superexec = "flwr.superexec:run_superexec"
 flower-supernode = "flwr.client:run_supernode"
 flower-client-app = "flwr.client:run_client_app"
 flower-server-app = "flwr.server:run_server_app"

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/cli/app.py

@@ -18,6 +18,7 @@ import typer
 
 from .build import build
 from .example import example
+from .install import install
 from .new import new
 from .run import run
 
@@ -34,6 +35,7 @@ app.command()(new)
 app.command()(example)
 app.command()(run)
 app.command()(build)
+app.command()(install)
 
 if __name__ == "__main__":
     app()

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/cli/build.py

@@ -14,7 +14,6 @@
 # ==============================================================================
 """Flower command line interface `build` command."""
 
-import hashlib
 import os
 import zipfile
 from pathlib import Path
@@ -25,7 +24,7 @@ import typer
 from typing_extensions import Annotated
 
 from .config_utils import load_and_validate
-from .utils import is_valid_project_name
+from .utils import get_sha256_hash, is_valid_project_name
 
 
 # pylint: disable=too-many-locals
@@ -34,7 +33,7 @@ def build(
         Optional[Path],
         typer.Option(help="The Flower project directory to bundle into a FAB"),
     ] = None,
-) -> None:
+) -> str:
     """Build a Flower project into a Flower App Bundle (FAB).
 
     You can run `flwr build` without any argument to bundle the current directory:
@@ -115,7 +114,7 @@ def build(
                 fab_file.write(file_path, archive_path)
 
                 # Calculate file info
-                sha256_hash = _get_sha256_hash(file_path)
+                sha256_hash = get_sha256_hash(file_path)
                 file_size_bits = os.path.getsize(file_path) * 8  # size in bits
                 list_file_content += f"{archive_path},{sha256_hash},{file_size_bits}\n"
 
@@ -126,17 +125,7 @@ def build(
         f"🎊 Successfully built {fab_filename}.", fg=typer.colors.GREEN, bold=True
     )
 
-
-def _get_sha256_hash(file_path: Path) -> str:
-    """Calculate the SHA-256 hash of a file."""
-    sha256 = hashlib.sha256()
-    with open(file_path, "rb") as f:
-        while True:
-            data = f.read(65536)  # Read in 64kB blocks
-            if not data:
-                break
-            sha256.update(data)
-    return sha256.hexdigest()
+    return fab_filename
 
 
 def _load_gitignore(directory: Path) -> pathspec.PathSpec:

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/cli/config_utils.py

@@ -14,16 +14,59 @@
 # ==============================================================================
 """Utility to validate the `pyproject.toml` file."""
 
+import zipfile
+from io import BytesIO
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import IO, Any, Dict, List, Optional, Tuple, Union
 
 import tomli
 
 from flwr.common import object_ref
 
 
+def get_fab_metadata(fab_file: Union[Path, bytes]) -> Tuple[str, str]:
+    """Extract the fab_id and the fab_version from a FAB file or path.
+
+    Parameters
+    ----------
+    fab_file : Union[Path, bytes]
+        The Flower App Bundle file to validate and extract the metadata from.
+        It can either be a path to the file or the file itself as bytes.
+
+    Returns
+    -------
+    Tuple[str, str]
+        The `fab_version` and `fab_id` of the given Flower App Bundle.
+    """
+    fab_file_archive: Union[Path, IO[bytes]]
+    if isinstance(fab_file, bytes):
+        fab_file_archive = BytesIO(fab_file)
+    elif isinstance(fab_file, Path):
+        fab_file_archive = fab_file
+    else:
+        raise ValueError("fab_file must be either a Path or bytes")
+
+    with zipfile.ZipFile(fab_file_archive, "r") as zipf:
+        with zipf.open("pyproject.toml") as file:
+            toml_content = file.read().decode("utf-8")
+
+        conf = load_from_string(toml_content)
+        if conf is None:
+            raise ValueError("Invalid TOML content in pyproject.toml")
+
+        is_valid, errors, _ = validate(conf)
+        if not is_valid:
+            raise ValueError(errors)
+
+        return (
+            conf["project"]["version"],
+            f"{conf['flower']['publisher']}/{conf['project']['name']}",
+        )
+
+
 def load_and_validate(
     path: Optional[Path] = None,
+    check_module: bool = True,
 ) -> Tuple[Optional[Dict[str, Any]], List[str], List[str]]:
     """Load and validate pyproject.toml as dict.
 
@@ -42,7 +85,7 @@ def load_and_validate(
         ]
         return (None, errors, [])
 
-    is_valid, errors, warnings = validate(config)
+    is_valid, errors, warnings = validate(config, check_module)
 
     if not is_valid:
         return (None, errors, warnings)
@@ -62,8 +105,7 @@ def load(path: Optional[Path] = None) -> Optional[Dict[str, Any]]:
         return None
 
     with toml_path.open(encoding="utf-8") as toml_file:
-        data = tomli.loads(toml_file.read())
-        return data
+        return load_from_string(toml_file.read())
 
 
 # pylint: disable=too-many-branches
@@ -102,7 +144,9 @@ def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]
     return len(errors) == 0, errors, warnings
 
 
-def validate(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
+def validate(
+    config: Dict[str, Any], check_module: bool = True
+) -> Tuple[bool, List[str], List[str]]:
     """Validate pyproject.toml."""
     is_valid, errors, warnings = validate_fields(config)
 
@@ -110,14 +154,27 @@ def validate(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
         return False, errors, warnings
 
     # Validate serverapp
-    is_valid, reason = object_ref.validate(config["flower"]["components"]["serverapp"])
+    is_valid, reason = object_ref.validate(
+        config["flower"]["components"]["serverapp"], check_module
+    )
     if not is_valid and isinstance(reason, str):
         return False, [reason], []
 
     # Validate clientapp
-    is_valid, reason = object_ref.validate(config["flower"]["components"]["clientapp"])
+    is_valid, reason = object_ref.validate(
+        config["flower"]["components"]["clientapp"], check_module
+    )
 
     if not is_valid and isinstance(reason, str):
         return False, [reason], []
 
     return True, [], []
+
+
+def load_from_string(toml_content: str) -> Optional[Dict[str, Any]]:
+    """Load TOML content from a string and return as dict."""
+    try:
+        data = tomli.loads(toml_content)
+        return data
+    except tomli.TOMLDecodeError:
+        return None

flwr_nightly-1.10.0.dev20240613/src/py/flwr/cli/install.py

@@ -0,0 +1,211 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower command line interface `install` command."""
+
+
+import shutil
+import tempfile
+import zipfile
+from io import BytesIO
+from pathlib import Path
+from typing import IO, Optional, Union
+
+import typer
+from typing_extensions import Annotated
+
+from flwr.common.config import get_flwr_dir
+
+from .config_utils import load_and_validate
+from .utils import get_sha256_hash
+
+
+def install(
+    source: Annotated[
+        Optional[Path],
+        typer.Argument(metavar="source", help="The source FAB file to install."),
+    ] = None,
+    flwr_dir: Annotated[
+        Optional[Path],
+        typer.Option(help="The desired install path."),
+    ] = None,
+) -> None:
+    """Install a Flower App Bundle.
+
+    It can be ran with a single FAB file argument:
+
+        ``flwr install ./target_project.fab``
+
+    The target install directory can be specified with ``--flwr-dir``:
+
+        ``flwr install ./target_project.fab --flwr-dir ./docs/flwr``
+
+    This will install ``target_project`` to ``./docs/flwr/``. By default,
+    ``flwr-dir`` is equal to:
+
+        - ``$FLWR_HOME/`` if ``$FLWR_HOME`` is defined
+        - ``$XDG_DATA_HOME/.flwr/`` if ``$XDG_DATA_HOME`` is defined
+        - ``$HOME/.flwr/`` in all other cases
+    """
+    if source is None:
+        source = Path(typer.prompt("Enter the source FAB file"))
+
+    source = source.resolve()
+    if not source.exists() or not source.is_file():
+        typer.secho(
+            f"❌ The source {source} does not exist or is not a file.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    if source.suffix != ".fab":
+        typer.secho(
+            f"❌ The source {source} is not a `.fab` file.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    install_from_fab(source, flwr_dir)
+
+
+def install_from_fab(
+    fab_file: Union[Path, bytes],
+    flwr_dir: Optional[Path],
+    skip_prompt: bool = False,
+) -> Path:
+    """Install from a FAB file after extracting and validating."""
+    fab_file_archive: Union[Path, IO[bytes]]
+    fab_name: Optional[str]
+    if isinstance(fab_file, bytes):
+        fab_file_archive = BytesIO(fab_file)
+        fab_name = None
+    elif isinstance(fab_file, Path):
+        fab_file_archive = fab_file
+        fab_name = fab_file.stem
+    else:
+        raise ValueError("fab_file must be either a Path or bytes")
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with zipfile.ZipFile(fab_file_archive, "r") as zipf:
+            zipf.extractall(tmpdir)
+            tmpdir_path = Path(tmpdir)
+            info_dir = tmpdir_path / ".info"
+            if not info_dir.exists():
+                typer.secho(
+                    "❌ FAB file has incorrect format.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
+                raise typer.Exit(code=1)
+
+            content_file = info_dir / "CONTENT"
+
+            if not content_file.exists() or not _verify_hashes(
+                content_file.read_text(), tmpdir_path
+            ):
+                typer.secho(
+                    "❌ File hashes couldn't be verified.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
+                raise typer.Exit(code=1)
+
+            shutil.rmtree(info_dir)
+
+            installed_path = validate_and_install(
+                tmpdir_path, fab_name, flwr_dir, skip_prompt
+            )
+
+    return installed_path
+
+
+def validate_and_install(
+    project_dir: Path,
+    fab_name: Optional[str],
+    flwr_dir: Optional[Path],
+    skip_prompt: bool = False,
+) -> Path:
+    """Validate TOML files and install the project to the desired directory."""
+    config, _, _ = load_and_validate(project_dir / "pyproject.toml", check_module=False)
+
+    if config is None:
+        typer.secho(
+            "❌ Invalid config inside FAB file.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    publisher = config["flower"]["publisher"]
+    project_name = config["project"]["name"]
+    version = config["project"]["version"]
+
+    if (
+        fab_name
+        and fab_name != f"{publisher}.{project_name}.{version.replace('.', '-')}"
+    ):
+        typer.secho(
+            "❌ FAB file has incorrect name. The file name must follow the format "
+            "`<publisher>.<project_name>.<version>.fab`.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    install_dir: Path = (
+        (get_flwr_dir() if not flwr_dir else flwr_dir)
+        / "apps"
+        / publisher
+        / project_name
+        / version
+    )
+    if install_dir.exists() and not skip_prompt:
+        if not typer.confirm(
+            typer.style(
+                f"\n💬 {project_name} version {version} is already installed, "
+                "do you want to reinstall it?",
+                fg=typer.colors.MAGENTA,
+                bold=True,
+            )
+        ):
+            return install_dir
+
+    install_dir.mkdir(parents=True, exist_ok=True)
+
+    # Move contents from source directory
+    for item in project_dir.iterdir():
+        if item.is_dir():
+            shutil.copytree(item, install_dir / item.name, dirs_exist_ok=True)
+        else:
+            shutil.copy2(item, install_dir / item.name)
+
+    typer.secho(
+        f"🎊 Successfully installed {project_name} to {install_dir}.",
+        fg=typer.colors.GREEN,
+        bold=True,
+    )
+
+    return install_dir
+
+
+def _verify_hashes(list_content: str, tmpdir: Path) -> bool:
+    """Verify file hashes based on the LIST content."""
+    for line in list_content.strip().split("\n"):
+        rel_path, hash_expected, _ = line.split(",")
+        file_path = tmpdir / rel_path
+        if not file_path.exists() or get_sha256_hash(file_path) != hash_expected:
+            return False
+    return True

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/cli/utils.py

@@ -14,7 +14,9 @@
 # ==============================================================================
 """Flower command line interface utils."""
 
+import hashlib
 import re
+from pathlib import Path
 from typing import Callable, List, Optional, cast
 
 import typer
@@ -122,3 +124,15 @@ def sanitize_project_name(name: str) -> str:
         sanitized_name = sanitized_name[1:]
 
     return sanitized_name
+
+
+def get_sha256_hash(file_path: Path) -> str:
+    """Calculate the SHA-256 hash of a file."""
+    sha256 = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        while True:
+            data = f.read(65536)  # Read in 64kB blocks
+            if not data:
+                break
+            sha256.update(data)
+    return sha256.hexdigest()

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/client/app.py

@@ -19,7 +19,7 @@ import sys
 import time
 from dataclasses import dataclass
 from logging import DEBUG, ERROR, INFO, WARN
-from typing import Callable, ContextManager, Optional, Tuple, Type, Union
+from typing import Callable, ContextManager, Dict, Optional, Tuple, Type, Union
 
 from cryptography.hazmat.primitives.asymmetric import ec
 from grpc import RpcError
@@ -177,7 +177,7 @@ def start_client(
 def _start_client_internal(
     *,
     server_address: str,
-    load_client_app_fn: Optional[Callable[[], ClientApp]] = None,
+    load_client_app_fn: Optional[Callable[[str, str], ClientApp]] = None,
     client_fn: Optional[ClientFn] = None,
     client: Optional[Client] = None,
     grpc_max_message_length: int = GRPC_MAX_MESSAGE_LENGTH,
@@ -252,7 +252,7 @@ def _start_client_internal(
 
             client_fn = single_client_factory
 
-        def _load_client_app() -> ClientApp:
+        def _load_client_app(_1: str, _2: str) -> ClientApp:
             return ClientApp(client_fn=client_fn)
 
         load_client_app_fn = _load_client_app
@@ -308,6 +308,8 @@ def _start_client_internal(
     )
 
     node_state = NodeState()
+    # run_id -> (fab_id, fab_version)
+    run_info: Dict[int, Tuple[str, str]] = {}
 
     while not app_state_tracker.interrupt:
         sleep_duration: int = 0
@@ -319,7 +321,6 @@ def _start_client_internal(
             root_certificates,
             authentication_keys,
         ) as conn:
-            # pylint: disable-next=W0612
             receive, send, create_node, delete_node, get_run = conn
 
             # Register node
@@ -356,13 +357,20 @@ def _start_client_internal(
                         send(out_message)
                         break
 
+                    # Get run info
+                    run_id = message.metadata.run_id
+                    if run_id not in run_info:
+                        if get_run is not None:
+                            run_info[run_id] = get_run(run_id)
+                        # If get_run is None, i.e., in grpc-bidi mode
+                        else:
+                            run_info[run_id] = ("", "")
+
                     # Register context for this run
-                    node_state.register_context(run_id=message.metadata.run_id)
+                    node_state.register_context(run_id=run_id)
 
                     # Retrieve context for this run
-                    context = node_state.retrieve_context(
-                        run_id=message.metadata.run_id
-                    )
+                    context = node_state.retrieve_context(run_id=run_id)
 
                     # Create an error reply message that will never be used to prevent
                     # the used-before-assignment linting error
@@ -373,7 +381,7 @@ def _start_client_internal(
                     # Handle app loading and task message
                     try:
                         # Load ClientApp instance
-                        client_app: ClientApp = load_client_app_fn()
+                        client_app: ClientApp = load_client_app_fn(*run_info[run_id])
 
                         # Execute ClientApp
                         reply_message = client_app(message=message, context=context)
@@ -411,7 +419,7 @@ def _start_client_internal(
                     else:
                         # No exception, update node state
                         node_state.update_context(
-                            run_id=message.metadata.run_id,
+                            run_id=run_id,
                             context=context,
                         )
 

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/client/grpc_rere_client/client_interceptor.py

@@ -31,11 +31,11 @@ from flwr.common.secure_aggregation.crypto.symmetric_encryption import (
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
-    GetRunRequest,
     PingRequest,
     PullTaskInsRequest,
     PushTaskResRequest,
 )
+from flwr.proto.run_pb2 import GetRunRequest  # pylint: disable=E0611
 
 _PUBLIC_KEY_HEADER = "public-key"
 _AUTH_TOKEN_HEADER = "auth-token"

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/client/grpc_rere_client/connection.py

@@ -44,8 +44,6 @@ from flwr.common.serde import message_from_taskins, message_to_taskres
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
-    GetRunRequest,
-    GetRunResponse,
     PingRequest,
     PingResponse,
     PullTaskInsRequest,
@@ -53,6 +51,7 @@ from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
 )
 from flwr.proto.fleet_pb2_grpc import FleetStub  # pylint: disable=E0611
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
 from flwr.proto.task_pb2 import TaskIns  # pylint: disable=E0611
 
 from .client_interceptor import AuthenticateClientInterceptor

{flwr_nightly-1.10.0.dev20240611 → flwr_nightly-1.10.0.dev20240613}/src/py/flwr/client/rest_client/connection.py

@@ -46,8 +46,6 @@ from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeResponse,
     DeleteNodeRequest,
     DeleteNodeResponse,
-    GetRunRequest,
-    GetRunResponse,
     PingRequest,
     PingResponse,
     PullTaskInsRequest,
@@ -56,6 +54,7 @@ from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     PushTaskResResponse,
 )
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
 from flwr.proto.task_pb2 import TaskIns  # pylint: disable=E0611
 
 try: