flwr-nightly 1.9.0.dev20240531__tar.gz → 1.10.0.dev20240619__tar.gz

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.9.0.dev20240531
+Version: 1.10.0.dev20240619
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.9.0.dev20240531"
+version = "1.10.0.dev20240619"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]
@@ -53,9 +53,8 @@ exclude = [
 
 [tool.poetry.scripts]
 flwr = "flwr.cli.app:app"
-flower-driver-api = "flwr.server:run_driver_api"
-flower-fleet-api = "flwr.server:run_fleet_api"
 flower-superlink = "flwr.server:run_superlink"
+flower-superexec = "flwr.superexec:run_superexec"
 flower-supernode = "flwr.client:run_supernode"
 flower-client-app = "flwr.client:run_client_app"
 flower-server-app = "flwr.server:run_server_app"
@@ -105,7 +104,7 @@ jupyterlab = "==4.0.12"
 rope = "==1.11.0"
 semver = "==3.0.2"
 sphinx = "==6.2.1"
-sphinx-intl = "==2.1.0"
+sphinx-intl = "==2.2.0"
 myst-parser = "==1.0.0"
 sphinx-design = "==0.5.0"
 sphinx-copybutton = "==0.5.2"
@@ -113,7 +112,7 @@ sphinxcontrib-mermaid = "==0.9.2"
 sphinxcontrib-youtube = "==1.4.1"
 furo = "==2023.9.10"
 sphinx-reredirects = "==0.1.3"
-nbsphinx = "==0.9.3"
+nbsphinx = "==0.9.4"
 nbstripout = "==0.6.1"
 ruff = "==0.1.9"
 sphinx-argparse = "==0.4.0"

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/app.py

@@ -18,6 +18,7 @@ import typer
 
 from .build import build
 from .example import example
+from .install import install
 from .new import new
 from .run import run
 
@@ -34,6 +35,7 @@ app.command()(new)
 app.command()(example)
 app.command()(run)
 app.command()(build)
+app.command()(install)
 
 if __name__ == "__main__":
     app()

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/build.py

@@ -14,7 +14,6 @@
 # ==============================================================================
 """Flower command line interface `build` command."""
 
-import hashlib
 import os
 import zipfile
 from pathlib import Path
@@ -25,7 +24,7 @@ import typer
 from typing_extensions import Annotated
 
 from .config_utils import load_and_validate
-from .utils import is_valid_project_name
+from .utils import get_sha256_hash, is_valid_project_name
 
 
 # pylint: disable=too-many-locals
@@ -34,7 +33,7 @@ def build(
         Optional[Path],
         typer.Option(help="The Flower project directory to bundle into a FAB"),
     ] = None,
-) -> None:
+) -> str:
     """Build a Flower project into a Flower App Bundle (FAB).
 
     You can run `flwr build` without any argument to bundle the current directory:
@@ -115,7 +114,7 @@ def build(
                 fab_file.write(file_path, archive_path)
 
                 # Calculate file info
-                sha256_hash = _get_sha256_hash(file_path)
+                sha256_hash = get_sha256_hash(file_path)
                 file_size_bits = os.path.getsize(file_path) * 8  # size in bits
                 list_file_content += f"{archive_path},{sha256_hash},{file_size_bits}\n"
 
@@ -126,17 +125,7 @@ def build(
         f"🎊 Successfully built {fab_filename}.", fg=typer.colors.GREEN, bold=True
     )
 
-
-def _get_sha256_hash(file_path: Path) -> str:
-    """Calculate the SHA-256 hash of a file."""
-    sha256 = hashlib.sha256()
-    with open(file_path, "rb") as f:
-        while True:
-            data = f.read(65536)  # Read in 64kB blocks
-            if not data:
-                break
-            sha256.update(data)
-    return sha256.hexdigest()
+    return fab_filename
 
 
 def _load_gitignore(directory: Path) -> pathspec.PathSpec:

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/config_utils.py

@@ -14,16 +14,59 @@
 # ==============================================================================
 """Utility to validate the `pyproject.toml` file."""
 
+import zipfile
+from io import BytesIO
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import IO, Any, Dict, List, Optional, Tuple, Union
 
 import tomli
 
 from flwr.common import object_ref
 
 
+def get_fab_metadata(fab_file: Union[Path, bytes]) -> Tuple[str, str]:
+    """Extract the fab_id and the fab_version from a FAB file or path.
+
+    Parameters
+    ----------
+    fab_file : Union[Path, bytes]
+        The Flower App Bundle file to validate and extract the metadata from.
+        It can either be a path to the file or the file itself as bytes.
+
+    Returns
+    -------
+    Tuple[str, str]
+        The `fab_version` and `fab_id` of the given Flower App Bundle.
+    """
+    fab_file_archive: Union[Path, IO[bytes]]
+    if isinstance(fab_file, bytes):
+        fab_file_archive = BytesIO(fab_file)
+    elif isinstance(fab_file, Path):
+        fab_file_archive = fab_file
+    else:
+        raise ValueError("fab_file must be either a Path or bytes")
+
+    with zipfile.ZipFile(fab_file_archive, "r") as zipf:
+        with zipf.open("pyproject.toml") as file:
+            toml_content = file.read().decode("utf-8")
+
+        conf = load_from_string(toml_content)
+        if conf is None:
+            raise ValueError("Invalid TOML content in pyproject.toml")
+
+        is_valid, errors, _ = validate(conf, check_module=False)
+        if not is_valid:
+            raise ValueError(errors)
+
+        return (
+            conf["project"]["version"],
+            f"{conf['flower']['publisher']}/{conf['project']['name']}",
+        )
+
+
 def load_and_validate(
     path: Optional[Path] = None,
+    check_module: bool = True,
 ) -> Tuple[Optional[Dict[str, Any]], List[str], List[str]]:
     """Load and validate pyproject.toml as dict.
 
@@ -42,7 +85,7 @@ def load_and_validate(
         ]
         return (None, errors, [])
 
-    is_valid, errors, warnings = validate(config)
+    is_valid, errors, warnings = validate(config, check_module)
 
     if not is_valid:
         return (None, errors, warnings)
@@ -62,8 +105,7 @@ def load(path: Optional[Path] = None) -> Optional[Dict[str, Any]]:
         return None
 
     with toml_path.open(encoding="utf-8") as toml_file:
-        data = tomli.loads(toml_file.read())
-        return data
+        return load_from_string(toml_file.read())
 
 
 # pylint: disable=too-many-branches
@@ -102,7 +144,9 @@ def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]
     return len(errors) == 0, errors, warnings
 
 
-def validate(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
+def validate(
+    config: Dict[str, Any], check_module: bool = True
+) -> Tuple[bool, List[str], List[str]]:
     """Validate pyproject.toml."""
     is_valid, errors, warnings = validate_fields(config)
 
@@ -110,14 +154,27 @@ def validate(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
         return False, errors, warnings
 
     # Validate serverapp
-    is_valid, reason = object_ref.validate(config["flower"]["components"]["serverapp"])
+    is_valid, reason = object_ref.validate(
+        config["flower"]["components"]["serverapp"], check_module
+    )
     if not is_valid and isinstance(reason, str):
         return False, [reason], []
 
     # Validate clientapp
-    is_valid, reason = object_ref.validate(config["flower"]["components"]["clientapp"])
+    is_valid, reason = object_ref.validate(
+        config["flower"]["components"]["clientapp"], check_module
+    )
 
     if not is_valid and isinstance(reason, str):
         return False, [reason], []
 
     return True, [], []
+
+
+def load_from_string(toml_content: str) -> Optional[Dict[str, Any]]:
+    """Load TOML content from a string and return as dict."""
+    try:
+        data = tomli.loads(toml_content)
+        return data
+    except tomli.TOMLDecodeError:
+        return None

flwr_nightly-1.10.0.dev20240619/src/py/flwr/cli/install.py

@@ -0,0 +1,211 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower command line interface `install` command."""
+
+
+import shutil
+import tempfile
+import zipfile
+from io import BytesIO
+from pathlib import Path
+from typing import IO, Optional, Union
+
+import typer
+from typing_extensions import Annotated
+
+from flwr.common.config import get_flwr_dir
+
+from .config_utils import load_and_validate
+from .utils import get_sha256_hash
+
+
+def install(
+    source: Annotated[
+        Optional[Path],
+        typer.Argument(metavar="source", help="The source FAB file to install."),
+    ] = None,
+    flwr_dir: Annotated[
+        Optional[Path],
+        typer.Option(help="The desired install path."),
+    ] = None,
+) -> None:
+    """Install a Flower App Bundle.
+
+    It can be ran with a single FAB file argument:
+
+        ``flwr install ./target_project.fab``
+
+    The target install directory can be specified with ``--flwr-dir``:
+
+        ``flwr install ./target_project.fab --flwr-dir ./docs/flwr``
+
+    This will install ``target_project`` to ``./docs/flwr/``. By default,
+    ``flwr-dir`` is equal to:
+
+        - ``$FLWR_HOME/`` if ``$FLWR_HOME`` is defined
+        - ``$XDG_DATA_HOME/.flwr/`` if ``$XDG_DATA_HOME`` is defined
+        - ``$HOME/.flwr/`` in all other cases
+    """
+    if source is None:
+        source = Path(typer.prompt("Enter the source FAB file"))
+
+    source = source.resolve()
+    if not source.exists() or not source.is_file():
+        typer.secho(
+            f"❌ The source {source} does not exist or is not a file.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    if source.suffix != ".fab":
+        typer.secho(
+            f"❌ The source {source} is not a `.fab` file.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    install_from_fab(source, flwr_dir)
+
+
+def install_from_fab(
+    fab_file: Union[Path, bytes],
+    flwr_dir: Optional[Path],
+    skip_prompt: bool = False,
+) -> Path:
+    """Install from a FAB file after extracting and validating."""
+    fab_file_archive: Union[Path, IO[bytes]]
+    fab_name: Optional[str]
+    if isinstance(fab_file, bytes):
+        fab_file_archive = BytesIO(fab_file)
+        fab_name = None
+    elif isinstance(fab_file, Path):
+        fab_file_archive = fab_file
+        fab_name = fab_file.stem
+    else:
+        raise ValueError("fab_file must be either a Path or bytes")
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with zipfile.ZipFile(fab_file_archive, "r") as zipf:
+            zipf.extractall(tmpdir)
+            tmpdir_path = Path(tmpdir)
+            info_dir = tmpdir_path / ".info"
+            if not info_dir.exists():
+                typer.secho(
+                    "❌ FAB file has incorrect format.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
+                raise typer.Exit(code=1)
+
+            content_file = info_dir / "CONTENT"
+
+            if not content_file.exists() or not _verify_hashes(
+                content_file.read_text(), tmpdir_path
+            ):
+                typer.secho(
+                    "❌ File hashes couldn't be verified.",
+                    fg=typer.colors.RED,
+                    bold=True,
+                )
+                raise typer.Exit(code=1)
+
+            shutil.rmtree(info_dir)
+
+            installed_path = validate_and_install(
+                tmpdir_path, fab_name, flwr_dir, skip_prompt
+            )
+
+    return installed_path
+
+
+def validate_and_install(
+    project_dir: Path,
+    fab_name: Optional[str],
+    flwr_dir: Optional[Path],
+    skip_prompt: bool = False,
+) -> Path:
+    """Validate TOML files and install the project to the desired directory."""
+    config, _, _ = load_and_validate(project_dir / "pyproject.toml", check_module=False)
+
+    if config is None:
+        typer.secho(
+            "❌ Invalid config inside FAB file.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    publisher = config["flower"]["publisher"]
+    project_name = config["project"]["name"]
+    version = config["project"]["version"]
+
+    if (
+        fab_name
+        and fab_name != f"{publisher}.{project_name}.{version.replace('.', '-')}"
+    ):
+        typer.secho(
+            "❌ FAB file has incorrect name. The file name must follow the format "
+            "`<publisher>.<project_name>.<version>.fab`.",
+            fg=typer.colors.RED,
+            bold=True,
+        )
+        raise typer.Exit(code=1)
+
+    install_dir: Path = (
+        (get_flwr_dir() if not flwr_dir else flwr_dir)
+        / "apps"
+        / publisher
+        / project_name
+        / version
+    )
+    if install_dir.exists() and not skip_prompt:
+        if not typer.confirm(
+            typer.style(
+                f"\n💬 {project_name} version {version} is already installed, "
+                "do you want to reinstall it?",
+                fg=typer.colors.MAGENTA,
+                bold=True,
+            )
+        ):
+            return install_dir
+
+    install_dir.mkdir(parents=True, exist_ok=True)
+
+    # Move contents from source directory
+    for item in project_dir.iterdir():
+        if item.is_dir():
+            shutil.copytree(item, install_dir / item.name, dirs_exist_ok=True)
+        else:
+            shutil.copy2(item, install_dir / item.name)
+
+    typer.secho(
+        f"🎊 Successfully installed {project_name} to {install_dir}.",
+        fg=typer.colors.GREEN,
+        bold=True,
+    )
+
+    return install_dir
+
+
+def _verify_hashes(list_content: str, tmpdir: Path) -> bool:
+    """Verify file hashes based on the LIST content."""
+    for line in list_content.strip().split("\n"):
+        rel_path, hash_expected, _ = line.split(",")
+        file_path = tmpdir / rel_path
+        if not file_path.exists() or get_sha256_hash(file_path) != hash_expected:
+            return False
+    return True

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl

@@ -11,7 +11,7 @@ authors = [
 ]
 license = { text = "Apache License (2.0)" }
 dependencies = [
-    "flwr[simulation]>=1.8.0,<2.0",
+    "flwr[simulation]>=1.9.0,<2.0",
     "flwr-datasets>=0.0.2,<1.0.0",
     "torch==2.2.1",
     "transformers>=4.30.0,<5.0"

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl

@@ -11,7 +11,7 @@ authors = [
 ]
 license = {text = "Apache License (2.0)"}
 dependencies = [
-    "flwr[simulation]>=1.8.0,<2.0",
+    "flwr[simulation]>=1.9.0,<2.0",
     "jax==0.4.26",
     "jaxlib==0.4.26",
     "scikit-learn==1.4.2",

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl

@@ -11,7 +11,7 @@ authors = [
 ]
 license = { text = "Apache License (2.0)" }
 dependencies = [
-    "flwr[simulation]>=1.8.0,<2.0",
+    "flwr[simulation]>=1.9.0,<2.0",
     "flwr-datasets[vision]>=0.0.2,<1.0.0",
     "mlx==0.10.0",
     "numpy==1.24.4",

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl

@@ -11,7 +11,7 @@ authors = [
 ]
 license = { text = "Apache License (2.0)" }
 dependencies = [
-    "flwr[simulation]>=1.8.0,<2.0",
+    "flwr[simulation]>=1.9.0,<2.0",
     "numpy>=1.21.0",
 ]
 

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl

@@ -11,7 +11,7 @@ authors = [
 ]
 license = { text = "Apache License (2.0)" }
 dependencies = [
-    "flwr[simulation]>=1.8.0,<2.0",
+    "flwr[simulation]>=1.9.0,<2.0",
     "flwr-datasets[vision]>=0.0.2,<1.0.0",
     "torch==2.2.1",
     "torchvision==0.17.1",

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl

@@ -11,7 +11,7 @@ authors = [
 ]
 license = { text = "Apache License (2.0)" }
 dependencies = [
-    "flwr[simulation]>=1.8.0,<2.0",
+    "flwr[simulation]>=1.9.0,<2.0",
     "flwr-datasets[vision]>=0.0.2,<1.0.0",
     "scikit-learn>=1.1.1",
 ]

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl

@@ -11,7 +11,7 @@ authors = [
 ]
 license = { text = "Apache License (2.0)" }
 dependencies = [
-    "flwr[simulation]>=1.8.0,<2.0",
+    "flwr[simulation]>=1.9.0,<2.0",
     "flwr-datasets[vision]>=0.0.2,<1.0.0",
     "tensorflow>=2.11.1",
 ]

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/run/run.py

@@ -16,12 +16,18 @@
 
 import sys
 from enum import Enum
+from logging import DEBUG
 from typing import Optional
 
 import typer
 from typing_extensions import Annotated
 
 from flwr.cli import config_utils
+from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS
+from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel
+from flwr.common.logger import log
+from flwr.proto.exec_pb2 import StartRunRequest  # pylint: disable=E0611
+from flwr.proto.exec_pb2_grpc import ExecStub
 from flwr.simulation.run_simulation import _run_simulation
 
 
@@ -31,20 +37,32 @@ class Engine(str, Enum):
     SIMULATION = "simulation"
 
 
+# pylint: disable-next=too-many-locals
 def run(
     engine: Annotated[
         Optional[Engine],
-        typer.Option(case_sensitive=False, help="The ML framework to use"),
+        typer.Option(case_sensitive=False, help="The execution engine to run the app"),
     ] = None,
+    use_superexec: Annotated[
+        bool,
+        typer.Option(
+            case_sensitive=False, help="Use this flag to use the new SuperExec API"
+        ),
+    ] = False,
 ) -> None:
     """Run Flower project."""
+    if use_superexec:
+        _start_superexec_run()
+        return
+
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
 
     config, errors, warnings = config_utils.load_and_validate()
 
     if config is None:
         typer.secho(
-            "Project configuration could not be loaded.\npyproject.toml is invalid:\n"
+            "Project configuration could not be loaded.\n"
+            "pyproject.toml is invalid:\n"
             + "\n".join([f"- {line}" for line in errors]),
             fg=typer.colors.RED,
             bold=True,
@@ -82,3 +100,22 @@ def run(
             fg=typer.colors.RED,
             bold=True,
         )
+
+
+def _start_superexec_run() -> None:
+    def on_channel_state_change(channel_connectivity: str) -> None:
+        """Log channel connectivity."""
+        log(DEBUG, channel_connectivity)
+
+    channel = create_channel(
+        server_address=SUPEREXEC_DEFAULT_ADDRESS,
+        insecure=True,
+        root_certificates=None,
+        max_message_length=GRPC_MAX_MESSAGE_LENGTH,
+        interceptors=None,
+    )
+    channel.subscribe(on_channel_state_change)
+    stub = ExecStub(channel)
+
+    req = StartRunRequest()
+    stub.StartRun(req)

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/cli/utils.py

@@ -14,7 +14,9 @@
 # ==============================================================================
 """Flower command line interface utils."""
 
+import hashlib
 import re
+from pathlib import Path
 from typing import Callable, List, Optional, cast
 
 import typer
@@ -122,3 +124,15 @@ def sanitize_project_name(name: str) -> str:
         sanitized_name = sanitized_name[1:]
 
     return sanitized_name
+
+
+def get_sha256_hash(file_path: Path) -> str:
+    """Calculate the SHA-256 hash of a file."""
+    sha256 = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        while True:
+            data = f.read(65536)  # Read in 64kB blocks
+            if not data:
+                break
+            sha256.update(data)
+    return sha256.hexdigest()

{flwr_nightly-1.9.0.dev20240531 → flwr_nightly-1.10.0.dev20240619}/src/py/flwr/client/__init__.py

@@ -29,6 +29,7 @@ __all__ = [
     "ClientApp",
     "ClientFn",
     "NumPyClient",
+    "mod",
     "run_client_app",
     "run_supernode",
     "start_client",