flwr-nightly 1.9.0.dev20240416__tar.gz → 1.9.0.dev20240418__tar.gz

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.9.0.dev20240416
+Version: 1.9.0.dev20240418
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.9.0-dev20240416"
+version = "1.9.0.dev20240418"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]

flwr_nightly-1.9.0.dev20240416/src/py/flwr/cli/flower_toml.py → flwr_nightly-1.9.0.dev20240418/src/py/flwr/cli/config_utils.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Utility to validate the `flower.toml` file."""
+"""Utility to validate the `pyproject.toml` file."""
 
 import os
 from typing import Any, Dict, List, Optional, Tuple
@@ -25,7 +25,7 @@ from flwr.common import object_ref
 def load_and_validate_with_defaults(
     path: Optional[str] = None,
 ) -> Tuple[Optional[Dict[str, Any]], List[str], List[str]]:
-    """Load and validate flower.toml as dict.
+    """Load and validate pyproject.toml as dict.
 
     Returns
     -------
@@ -37,7 +37,7 @@ def load_and_validate_with_defaults(
 
     if config is None:
         errors = [
-            "Project configuration could not be loaded. flower.toml does not exist."
+            "Project configuration could not be loaded. pyproject.toml does not exist."
         ]
         return (None, errors, [])
 
@@ -58,10 +58,10 @@ def load_and_validate_with_defaults(
 
 
 def load(path: Optional[str] = None) -> Optional[Dict[str, Any]]:
-    """Load flower.toml and return as dict."""
+    """Load pyproject.toml and return as dict."""
     if path is None:
         cur_dir = os.getcwd()
-        toml_path = os.path.join(cur_dir, "flower.toml")
+        toml_path = os.path.join(cur_dir, "pyproject.toml")
     else:
         toml_path = path
 
@@ -74,7 +74,7 @@ def load(path: Optional[str] = None) -> Optional[Dict[str, Any]]:
 
 
 def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
-    """Validate flower.toml fields."""
+    """Validate pyproject.toml fields."""
     errors = []
     warnings = []
 
@@ -106,7 +106,7 @@ def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]
 
 
 def validate(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
-    """Validate flower.toml."""
+    """Validate pyproject.toml."""
     is_valid, errors, warnings = validate_fields(config)
 
     if not is_valid:

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/cli/new/new.py

@@ -58,8 +58,9 @@ def render_template(template: str, data: Dict[str, str]) -> str:
     """Render template."""
     tpl_file = load_template(template)
     tpl = Template(tpl_file)
-    result = tpl.substitute(data)
-    return result
+    if ".gitignore" not in template:
+        return tpl.substitute(data)
+    return tpl.template
 
 
 def create_file(file_path: str, content: str) -> None:
@@ -127,16 +128,19 @@ def new(
 
     # List of files to render
     files = {
+        ".gitignore": {"template": "app/.gitignore.tpl"},
         "README.md": {"template": "app/README.md.tpl"},
-        "flower.toml": {"template": "app/flower.toml.tpl"},
         "pyproject.toml": {"template": f"app/pyproject.{framework_str}.toml.tpl"},
         f"{pnl}/__init__.py": {"template": "app/code/__init__.py.tpl"},
         f"{pnl}/server.py": {"template": f"app/code/server.{framework_str}.py.tpl"},
         f"{pnl}/client.py": {"template": f"app/code/client.{framework_str}.py.tpl"},
     }
 
-    # In case framework is MlFramework.PYTORCH generate additionally the task.py file
-    if framework_str == MlFramework.PYTORCH.value.lower():
+    # Depending on the framework, generate task.py file
+    frameworks_with_tasks = [
+        MlFramework.PYTORCH.value.lower(),
+    ]
+    if framework_str in frameworks_with_tasks:
         files[f"{pnl}/task.py"] = {"template": f"app/code/task.{framework_str}.py.tpl"}
 
     context = {"project_name": project_name}

flwr_nightly-1.9.0.dev20240418/src/py/flwr/cli/new/templates/app/.gitignore.tpl

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl

@@ -17,3 +17,7 @@ dependencies = [
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]
+
+[flower.components]
+serverapp = "$project_name.server:app"
+clientapp = "$project_name.client:app"

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl

@@ -19,3 +19,7 @@ dependencies = [
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]
+
+[flower.components]
+serverapp = "$project_name.server:app"
+clientapp = "$project_name.client:app"

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl

@@ -18,3 +18,7 @@ dependencies = [
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]
+
+[flower.components]
+serverapp = "$project_name.server:app"
+clientapp = "$project_name.client:app"

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/cli/run/run.py

@@ -18,7 +18,7 @@ import sys
 
 import typer
 
-from flwr.cli import flower_toml
+from flwr.cli import config_utils
 from flwr.simulation.run_simulation import _run_simulation
 
 
@@ -26,7 +26,7 @@ def run() -> None:
     """Run Flower project."""
     typer.secho("Loading project configuration... ", fg=typer.colors.BLUE)
 
-    config, errors, warnings = flower_toml.load_and_validate_with_defaults()
+    config, errors, warnings = config_utils.load_and_validate_with_defaults()
 
     if config is None:
         typer.secho(

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/client/rest_client/connection.py

@@ -173,14 +173,14 @@ def http_request_response(  # pylint: disable=R0914, R0915
             log(
                 WARN,
                 "[Node] POST /%s: missing header `Content-Type`",
-                PATH_PULL_TASK_INS,
+                PATH_PING,
             )
             return
         if res.headers["content-type"] != "application/protobuf":
             log(
                 WARN,
                 "[Node] POST /%s: header `Content-Type` has wrong value",
-                PATH_PULL_TASK_INS,
+                PATH_PING,
             )
             return
 
@@ -223,14 +223,14 @@ def http_request_response(  # pylint: disable=R0914, R0915
             log(
                 WARN,
                 "[Node] POST /%s: missing header `Content-Type`",
-                PATH_PULL_TASK_INS,
+                PATH_CREATE_NODE,
             )
             return
         if res.headers["content-type"] != "application/protobuf":
             log(
                 WARN,
                 "[Node] POST /%s: header `Content-Type` has wrong value",
-                PATH_PULL_TASK_INS,
+                PATH_CREATE_NODE,
             )
             return
 
@@ -277,14 +277,14 @@ def http_request_response(  # pylint: disable=R0914, R0915
             log(
                 WARN,
                 "[Node] POST /%s: missing header `Content-Type`",
-                PATH_PULL_TASK_INS,
+                PATH_DELETE_NODE,
             )
             return
         if res.headers["content-type"] != "application/protobuf":
             log(
                 WARN,
                 "[Node] POST /%s: header `Content-Type` has wrong value",
-                PATH_PULL_TASK_INS,
+                PATH_DELETE_NODE,
             )
 
         # Cleanup

flwr_nightly-1.9.0.dev20240418/src/py/flwr/common/record/recordset.py

@@ -0,0 +1,118 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""RecordSet."""
+
+
+from dataclasses import dataclass
+from typing import Dict, Optional, cast
+
+from .configsrecord import ConfigsRecord
+from .metricsrecord import MetricsRecord
+from .parametersrecord import ParametersRecord
+from .typeddict import TypedDict
+
+
+class RecordSetData:
+    """Inner data container for the RecordSet class."""
+
+    parameters_records: TypedDict[str, ParametersRecord]
+    metrics_records: TypedDict[str, MetricsRecord]
+    configs_records: TypedDict[str, ConfigsRecord]
+
+    def __init__(
+        self,
+        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
+        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
+        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+    ) -> None:
+        self.parameters_records = TypedDict[str, ParametersRecord](
+            self._check_fn_str, self._check_fn_params
+        )
+        self.metrics_records = TypedDict[str, MetricsRecord](
+            self._check_fn_str, self._check_fn_metrics
+        )
+        self.configs_records = TypedDict[str, ConfigsRecord](
+            self._check_fn_str, self._check_fn_configs
+        )
+        if parameters_records is not None:
+            self.parameters_records.update(parameters_records)
+        if metrics_records is not None:
+            self.metrics_records.update(metrics_records)
+        if configs_records is not None:
+            self.configs_records.update(configs_records)
+
+    def _check_fn_str(self, key: str) -> None:
+        if not isinstance(key, str):
+            raise TypeError(
+                f"Expected `{str.__name__}`, but "
+                f"received `{type(key).__name__}` for the key."
+            )
+
+    def _check_fn_params(self, record: ParametersRecord) -> None:
+        if not isinstance(record, ParametersRecord):
+            raise TypeError(
+                f"Expected `{ParametersRecord.__name__}`, but "
+                f"received `{type(record).__name__}` for the value."
+            )
+
+    def _check_fn_metrics(self, record: MetricsRecord) -> None:
+        if not isinstance(record, MetricsRecord):
+            raise TypeError(
+                f"Expected `{MetricsRecord.__name__}`, but "
+                f"received `{type(record).__name__}` for the value."
+            )
+
+    def _check_fn_configs(self, record: ConfigsRecord) -> None:
+        if not isinstance(record, ConfigsRecord):
+            raise TypeError(
+                f"Expected `{ConfigsRecord.__name__}`, but "
+                f"received `{type(record).__name__}` for the value."
+            )
+
+
+@dataclass
+class RecordSet:
+    """RecordSet stores groups of parameters, metrics and configs."""
+
+    def __init__(
+        self,
+        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
+        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
+        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+    ) -> None:
+        data = RecordSetData(
+            parameters_records=parameters_records,
+            metrics_records=metrics_records,
+            configs_records=configs_records,
+        )
+        setattr(self, "_data", data)  # noqa
+
+    @property
+    def parameters_records(self) -> TypedDict[str, ParametersRecord]:
+        """Dictionary holding ParametersRecord instances."""
+        data = cast(RecordSetData, getattr(self, "_data"))  # noqa
+        return data.parameters_records
+
+    @property
+    def metrics_records(self) -> TypedDict[str, MetricsRecord]:
+        """Dictionary holding MetricsRecord instances."""
+        data = cast(RecordSetData, getattr(self, "_data"))  # noqa
+        return data.metrics_records
+
+    @property
+    def configs_records(self) -> TypedDict[str, ConfigsRecord]:
+        """Dictionary holding ConfigsRecord instances."""
+        data = cast(RecordSetData, getattr(self, "_data"))  # noqa
+        return data.configs_records

flwr_nightly-1.9.0.dev20240418/src/py/flwr/server/driver/abc_driver.py

@@ -0,0 +1,140 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Driver (abstract base class)."""
+
+
+from abc import ABC, abstractmethod
+from typing import Iterable, List, Optional
+
+from flwr.common import Message, RecordSet
+
+
+class Driver(ABC):
+    """Abstract base Driver class for the Driver API."""
+
+    @abstractmethod
+    def create_message(  # pylint: disable=too-many-arguments
+        self,
+        content: RecordSet,
+        message_type: str,
+        dst_node_id: int,
+        group_id: str,
+        ttl: Optional[float] = None,
+    ) -> Message:
+        """Create a new message with specified parameters.
+
+        This method constructs a new `Message` with given content and metadata.
+        The `run_id` and `src_node_id` will be set automatically.
+
+        Parameters
+        ----------
+        content : RecordSet
+            The content for the new message. This holds records that are to be sent
+            to the destination node.
+        message_type : str
+            The type of the message, defining the action to be executed on
+            the receiving end.
+        dst_node_id : int
+            The ID of the destination node to which the message is being sent.
+        group_id : str
+            The ID of the group to which this message is associated. In some settings,
+            this is used as the FL round.
+        ttl : Optional[float] (default: None)
+            Time-to-live for the round trip of this message, i.e., the time from sending
+            this message to receiving a reply. It specifies in seconds the duration for
+            which the message and its potential reply are considered valid. If unset,
+            the default TTL (i.e., `common.DEFAULT_TTL`) will be used.
+
+        Returns
+        -------
+        message : Message
+            A new `Message` instance with the specified content and metadata.
+        """
+
+    @abstractmethod
+    def get_node_ids(self) -> List[int]:
+        """Get node IDs."""
+
+    @abstractmethod
+    def push_messages(self, messages: Iterable[Message]) -> Iterable[str]:
+        """Push messages to specified node IDs.
+
+        This method takes an iterable of messages and sends each message
+        to the node specified in `dst_node_id`.
+
+        Parameters
+        ----------
+        messages : Iterable[Message]
+            An iterable of messages to be sent.
+
+        Returns
+        -------
+        message_ids : Iterable[str]
+            An iterable of IDs for the messages that were sent, which can be used
+            to pull replies.
+        """
+
+    @abstractmethod
+    def pull_messages(self, message_ids: Iterable[str]) -> Iterable[Message]:
+        """Pull messages based on message IDs.
+
+        This method is used to collect messages from the SuperLink
+        that correspond to a set of given message IDs.
+
+        Parameters
+        ----------
+        message_ids : Iterable[str]
+            An iterable of message IDs for which reply messages are to be retrieved.
+
+        Returns
+        -------
+        messages : Iterable[Message]
+            An iterable of messages received.
+        """
+
+    @abstractmethod
+    def send_and_receive(
+        self,
+        messages: Iterable[Message],
+        *,
+        timeout: Optional[float] = None,
+    ) -> Iterable[Message]:
+        """Push messages to specified node IDs and pull the reply messages.
+
+        This method sends a list of messages to their destination node IDs and then
+        waits for the replies. It continues to pull replies until either all
+        replies are received or the specified timeout duration is exceeded.
+
+        Parameters
+        ----------
+        messages : Iterable[Message]
+            An iterable of messages to be sent.
+        timeout : Optional[float] (default: None)
+            The timeout duration in seconds. If specified, the method will wait for
+            replies for this duration. If `None`, there is no time limit and the method
+            will wait until replies for all messages are received.
+
+        Returns
+        -------
+        replies : Iterable[Message]
+            An iterable of reply messages received from the SuperLink.
+
+        Notes
+        -----
+        This method uses `push_messages` to send the messages and `pull_messages`
+        to collect the replies. If `timeout` is set, the method may not return
+        replies for all sent messages. A message remains valid until its TTL,
+        which is not affected by `timeout`.
+        """

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/server/superlink/driver/driver_servicer.py

@@ -64,7 +64,7 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         """Create run ID."""
         log(INFO, "DriverServicer.CreateRun")
         state: State = self.state_factory.state()
-        run_id = state.create_run()
+        run_id = state.create_run("None/None", "None")
         return CreateRunResponse(run_id=run_id)
 
     def PushTaskIns(

{flwr_nightly-1.9.0.dev20240416 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/server/superlink/state/in_memory_state.py

@@ -36,7 +36,8 @@ class InMemoryState(State):
     def __init__(self) -> None:
         # Map node_id to (online_until, ping_interval)
         self.node_ids: Dict[int, Tuple[float, float]] = {}
-        self.run_ids: Set[int] = set()
+        # Map run_id to (fab_id, fab_version)
+        self.run_ids: Dict[int, Tuple[str, str]] = {}
         self.task_ins_store: Dict[UUID, TaskIns] = {}
         self.task_res_store: Dict[UUID, TaskRes] = {}
         self.lock = threading.Lock()
@@ -238,18 +239,26 @@ class InMemoryState(State):
                 if online_until > current_time
             }
 
-    def create_run(self) -> int:
-        """Create one run."""
+    def create_run(self, fab_id: str, fab_version: str) -> int:
+        """Create a new run for the specified `fab_id` and `fab_version`."""
         # Sample a random int64 as run_id
         with self.lock:
             run_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
 
             if run_id not in self.run_ids:
-                self.run_ids.add(run_id)
+                self.run_ids[run_id] = (fab_id, fab_version)
                 return run_id
         log(ERROR, "Unexpected run creation failure.")
         return 0
 
+    def get_run(self, run_id: int) -> Tuple[int, str, str]:
+        """Retrieve information about the run with the specified `run_id`."""
+        with self.lock:
+            if run_id not in self.run_ids:
+                log(ERROR, "`run_id` is invalid")
+                return 0, "", ""
+            return run_id, *self.run_ids[run_id]
+
     def acknowledge_ping(self, node_id: int, ping_interval: float) -> bool:
         """Acknowledge a ping received from a node, serving as a heartbeat."""
         with self.lock: