flwr-nightly 1.9.0.dev20240417__tar.gz → 1.9.0.dev20240418__tar.gz

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.9.0.dev20240417
+Version: 1.9.0.dev20240418
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.9.0.dev20240417"
+version = "1.9.0.dev20240418"
 description = "Flower: A Friendly Federated Learning Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/cli/new/new.py

@@ -58,8 +58,9 @@ def render_template(template: str, data: Dict[str, str]) -> str:
     """Render template."""
     tpl_file = load_template(template)
     tpl = Template(tpl_file)
-    result = tpl.substitute(data)
-    return result
+    if ".gitignore" not in template:
+        return tpl.substitute(data)
+    return tpl.template
 
 
 def create_file(file_path: str, content: str) -> None:
@@ -127,6 +128,7 @@ def new(
 
     # List of files to render
     files = {
+        ".gitignore": {"template": "app/.gitignore.tpl"},
         "README.md": {"template": "app/README.md.tpl"},
         "pyproject.toml": {"template": f"app/pyproject.{framework_str}.toml.tpl"},
         f"{pnl}/__init__.py": {"template": "app/code/__init__.py.tpl"},
@@ -134,8 +136,11 @@ def new(
         f"{pnl}/client.py": {"template": f"app/code/client.{framework_str}.py.tpl"},
     }
 
-    # In case framework is MlFramework.PYTORCH generate additionally the task.py file
-    if framework_str == MlFramework.PYTORCH.value.lower():
+    # Depending on the framework, generate task.py file
+    frameworks_with_tasks = [
+        MlFramework.PYTORCH.value.lower(),
+    ]
+    if framework_str in frameworks_with_tasks:
         files[f"{pnl}/task.py"] = {"template": f"app/code/task.{framework_str}.py.tpl"}
 
     context = {"project_name": project_name}

flwr_nightly-1.9.0.dev20240418/src/py/flwr/cli/new/templates/app/.gitignore.tpl

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/client/rest_client/connection.py

@@ -173,14 +173,14 @@ def http_request_response(  # pylint: disable=R0914, R0915
             log(
                 WARN,
                 "[Node] POST /%s: missing header `Content-Type`",
-                PATH_PULL_TASK_INS,
+                PATH_PING,
             )
             return
         if res.headers["content-type"] != "application/protobuf":
             log(
                 WARN,
                 "[Node] POST /%s: header `Content-Type` has wrong value",
-                PATH_PULL_TASK_INS,
+                PATH_PING,
             )
             return
 
@@ -223,14 +223,14 @@ def http_request_response(  # pylint: disable=R0914, R0915
             log(
                 WARN,
                 "[Node] POST /%s: missing header `Content-Type`",
-                PATH_PULL_TASK_INS,
+                PATH_CREATE_NODE,
             )
             return
         if res.headers["content-type"] != "application/protobuf":
             log(
                 WARN,
                 "[Node] POST /%s: header `Content-Type` has wrong value",
-                PATH_PULL_TASK_INS,
+                PATH_CREATE_NODE,
             )
             return
 
@@ -277,14 +277,14 @@ def http_request_response(  # pylint: disable=R0914, R0915
             log(
                 WARN,
                 "[Node] POST /%s: missing header `Content-Type`",
-                PATH_PULL_TASK_INS,
+                PATH_DELETE_NODE,
             )
             return
         if res.headers["content-type"] != "application/protobuf":
             log(
                 WARN,
                 "[Node] POST /%s: header `Content-Type` has wrong value",
-                PATH_PULL_TASK_INS,
+                PATH_DELETE_NODE,
             )
 
         # Cleanup

flwr_nightly-1.9.0.dev20240418/src/py/flwr/common/record/recordset.py

@@ -0,0 +1,118 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""RecordSet."""
+
+
+from dataclasses import dataclass
+from typing import Dict, Optional, cast
+
+from .configsrecord import ConfigsRecord
+from .metricsrecord import MetricsRecord
+from .parametersrecord import ParametersRecord
+from .typeddict import TypedDict
+
+
+class RecordSetData:
+    """Inner data container for the RecordSet class."""
+
+    parameters_records: TypedDict[str, ParametersRecord]
+    metrics_records: TypedDict[str, MetricsRecord]
+    configs_records: TypedDict[str, ConfigsRecord]
+
+    def __init__(
+        self,
+        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
+        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
+        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+    ) -> None:
+        self.parameters_records = TypedDict[str, ParametersRecord](
+            self._check_fn_str, self._check_fn_params
+        )
+        self.metrics_records = TypedDict[str, MetricsRecord](
+            self._check_fn_str, self._check_fn_metrics
+        )
+        self.configs_records = TypedDict[str, ConfigsRecord](
+            self._check_fn_str, self._check_fn_configs
+        )
+        if parameters_records is not None:
+            self.parameters_records.update(parameters_records)
+        if metrics_records is not None:
+            self.metrics_records.update(metrics_records)
+        if configs_records is not None:
+            self.configs_records.update(configs_records)
+
+    def _check_fn_str(self, key: str) -> None:
+        if not isinstance(key, str):
+            raise TypeError(
+                f"Expected `{str.__name__}`, but "
+                f"received `{type(key).__name__}` for the key."
+            )
+
+    def _check_fn_params(self, record: ParametersRecord) -> None:
+        if not isinstance(record, ParametersRecord):
+            raise TypeError(
+                f"Expected `{ParametersRecord.__name__}`, but "
+                f"received `{type(record).__name__}` for the value."
+            )
+
+    def _check_fn_metrics(self, record: MetricsRecord) -> None:
+        if not isinstance(record, MetricsRecord):
+            raise TypeError(
+                f"Expected `{MetricsRecord.__name__}`, but "
+                f"received `{type(record).__name__}` for the value."
+            )
+
+    def _check_fn_configs(self, record: ConfigsRecord) -> None:
+        if not isinstance(record, ConfigsRecord):
+            raise TypeError(
+                f"Expected `{ConfigsRecord.__name__}`, but "
+                f"received `{type(record).__name__}` for the value."
+            )
+
+
+@dataclass
+class RecordSet:
+    """RecordSet stores groups of parameters, metrics and configs."""
+
+    def __init__(
+        self,
+        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
+        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
+        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+    ) -> None:
+        data = RecordSetData(
+            parameters_records=parameters_records,
+            metrics_records=metrics_records,
+            configs_records=configs_records,
+        )
+        setattr(self, "_data", data)  # noqa
+
+    @property
+    def parameters_records(self) -> TypedDict[str, ParametersRecord]:
+        """Dictionary holding ParametersRecord instances."""
+        data = cast(RecordSetData, getattr(self, "_data"))  # noqa
+        return data.parameters_records
+
+    @property
+    def metrics_records(self) -> TypedDict[str, MetricsRecord]:
+        """Dictionary holding MetricsRecord instances."""
+        data = cast(RecordSetData, getattr(self, "_data"))  # noqa
+        return data.metrics_records
+
+    @property
+    def configs_records(self) -> TypedDict[str, ConfigsRecord]:
+        """Dictionary holding ConfigsRecord instances."""
+        data = cast(RecordSetData, getattr(self, "_data"))  # noqa
+        return data.configs_records

flwr_nightly-1.9.0.dev20240418/src/py/flwr/server/driver/abc_driver.py

@@ -0,0 +1,140 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Driver (abstract base class)."""
+
+
+from abc import ABC, abstractmethod
+from typing import Iterable, List, Optional
+
+from flwr.common import Message, RecordSet
+
+
+class Driver(ABC):
+    """Abstract base Driver class for the Driver API."""
+
+    @abstractmethod
+    def create_message(  # pylint: disable=too-many-arguments
+        self,
+        content: RecordSet,
+        message_type: str,
+        dst_node_id: int,
+        group_id: str,
+        ttl: Optional[float] = None,
+    ) -> Message:
+        """Create a new message with specified parameters.
+
+        This method constructs a new `Message` with given content and metadata.
+        The `run_id` and `src_node_id` will be set automatically.
+
+        Parameters
+        ----------
+        content : RecordSet
+            The content for the new message. This holds records that are to be sent
+            to the destination node.
+        message_type : str
+            The type of the message, defining the action to be executed on
+            the receiving end.
+        dst_node_id : int
+            The ID of the destination node to which the message is being sent.
+        group_id : str
+            The ID of the group to which this message is associated. In some settings,
+            this is used as the FL round.
+        ttl : Optional[float] (default: None)
+            Time-to-live for the round trip of this message, i.e., the time from sending
+            this message to receiving a reply. It specifies in seconds the duration for
+            which the message and its potential reply are considered valid. If unset,
+            the default TTL (i.e., `common.DEFAULT_TTL`) will be used.
+
+        Returns
+        -------
+        message : Message
+            A new `Message` instance with the specified content and metadata.
+        """
+
+    @abstractmethod
+    def get_node_ids(self) -> List[int]:
+        """Get node IDs."""
+
+    @abstractmethod
+    def push_messages(self, messages: Iterable[Message]) -> Iterable[str]:
+        """Push messages to specified node IDs.
+
+        This method takes an iterable of messages and sends each message
+        to the node specified in `dst_node_id`.
+
+        Parameters
+        ----------
+        messages : Iterable[Message]
+            An iterable of messages to be sent.
+
+        Returns
+        -------
+        message_ids : Iterable[str]
+            An iterable of IDs for the messages that were sent, which can be used
+            to pull replies.
+        """
+
+    @abstractmethod
+    def pull_messages(self, message_ids: Iterable[str]) -> Iterable[Message]:
+        """Pull messages based on message IDs.
+
+        This method is used to collect messages from the SuperLink
+        that correspond to a set of given message IDs.
+
+        Parameters
+        ----------
+        message_ids : Iterable[str]
+            An iterable of message IDs for which reply messages are to be retrieved.
+
+        Returns
+        -------
+        messages : Iterable[Message]
+            An iterable of messages received.
+        """
+
+    @abstractmethod
+    def send_and_receive(
+        self,
+        messages: Iterable[Message],
+        *,
+        timeout: Optional[float] = None,
+    ) -> Iterable[Message]:
+        """Push messages to specified node IDs and pull the reply messages.
+
+        This method sends a list of messages to their destination node IDs and then
+        waits for the replies. It continues to pull replies until either all
+        replies are received or the specified timeout duration is exceeded.
+
+        Parameters
+        ----------
+        messages : Iterable[Message]
+            An iterable of messages to be sent.
+        timeout : Optional[float] (default: None)
+            The timeout duration in seconds. If specified, the method will wait for
+            replies for this duration. If `None`, there is no time limit and the method
+            will wait until replies for all messages are received.
+
+        Returns
+        -------
+        replies : Iterable[Message]
+            An iterable of reply messages received from the SuperLink.
+
+        Notes
+        -----
+        This method uses `push_messages` to send the messages and `pull_messages`
+        to collect the replies. If `timeout` is set, the method may not return
+        replies for all sent messages. A message remains valid until its TTL,
+        which is not affected by `timeout`.
+        """

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/server/superlink/driver/driver_servicer.py

@@ -64,7 +64,7 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         """Create run ID."""
         log(INFO, "DriverServicer.CreateRun")
         state: State = self.state_factory.state()
-        run_id = state.create_run()
+        run_id = state.create_run("None/None", "None")
         return CreateRunResponse(run_id=run_id)
 
     def PushTaskIns(

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/server/superlink/state/in_memory_state.py

@@ -36,7 +36,8 @@ class InMemoryState(State):
     def __init__(self) -> None:
         # Map node_id to (online_until, ping_interval)
         self.node_ids: Dict[int, Tuple[float, float]] = {}
-        self.run_ids: Set[int] = set()
+        # Map run_id to (fab_id, fab_version)
+        self.run_ids: Dict[int, Tuple[str, str]] = {}
         self.task_ins_store: Dict[UUID, TaskIns] = {}
         self.task_res_store: Dict[UUID, TaskRes] = {}
         self.lock = threading.Lock()
@@ -238,18 +239,26 @@ class InMemoryState(State):
                 if online_until > current_time
             }
 
-    def create_run(self) -> int:
-        """Create one run."""
+    def create_run(self, fab_id: str, fab_version: str) -> int:
+        """Create a new run for the specified `fab_id` and `fab_version`."""
         # Sample a random int64 as run_id
         with self.lock:
             run_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
 
             if run_id not in self.run_ids:
-                self.run_ids.add(run_id)
+                self.run_ids[run_id] = (fab_id, fab_version)
                 return run_id
         log(ERROR, "Unexpected run creation failure.")
         return 0
 
+    def get_run(self, run_id: int) -> Tuple[int, str, str]:
+        """Retrieve information about the run with the specified `run_id`."""
+        with self.lock:
+            if run_id not in self.run_ids:
+                log(ERROR, "`run_id` is invalid")
+                return 0, "", ""
+            return run_id, *self.run_ids[run_id]
+
     def acknowledge_ping(self, node_id: int, ping_interval: float) -> bool:
         """Acknowledge a ping received from a node, serving as a heartbeat."""
         with self.lock:

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/server/superlink/state/sqlite_state.py

@@ -46,7 +46,9 @@ CREATE INDEX IF NOT EXISTS idx_online_until ON node (online_until);
 
 SQL_CREATE_TABLE_RUN = """
 CREATE TABLE IF NOT EXISTS run(
-    run_id INTEGER UNIQUE
+    run_id          INTEGER UNIQUE,
+    fab_id          TEXT,
+    fab_version     TEXT
 );
 """
 
@@ -558,8 +560,8 @@ class SqliteState(State):
         result: Set[int] = {row["node_id"] for row in rows}
         return result
 
-    def create_run(self) -> int:
-        """Create one run and store it in state."""
+    def create_run(self, fab_id: str, fab_version: str) -> int:
+        """Create a new run for the specified `fab_id` and `fab_version`."""
         # Sample a random int64 as run_id
         run_id: int = int.from_bytes(os.urandom(8), "little", signed=True)
 
@@ -567,12 +569,22 @@ class SqliteState(State):
         query = "SELECT COUNT(*) FROM run WHERE run_id = ?;"
         # If run_id does not exist
         if self.query(query, (run_id,))[0]["COUNT(*)"] == 0:
-            query = "INSERT INTO run VALUES(:run_id);"
-            self.query(query, {"run_id": run_id})
+            query = "INSERT INTO run (run_id, fab_id, fab_version) VALUES (?, ?, ?);"
+            self.query(query, (run_id, fab_id, fab_version))
             return run_id
         log(ERROR, "Unexpected run creation failure.")
         return 0
 
+    def get_run(self, run_id: int) -> Tuple[int, str, str]:
+        """Retrieve information about the run with the specified `run_id`."""
+        query = "SELECT * FROM run WHERE run_id = ?;"
+        try:
+            row = self.query(query, (run_id,))[0]
+            return run_id, row["fab_id"], row["fab_version"]
+        except sqlite3.IntegrityError:
+            log(ERROR, "`run_id` does not exist.")
+            return 0, "", ""
+
     def acknowledge_ping(self, node_id: int, ping_interval: float) -> bool:
         """Acknowledge a ping received from a node, serving as a heartbeat."""
         # Update `online_until` and `ping_interval` for the given `node_id`

{flwr_nightly-1.9.0.dev20240417 → flwr_nightly-1.9.0.dev20240418}/src/py/flwr/server/superlink/state/state.py

@@ -16,7 +16,7 @@
 
 
 import abc
-from typing import List, Optional, Set
+from typing import List, Optional, Set, Tuple
 from uuid import UUID
 
 from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
@@ -150,8 +150,26 @@ class State(abc.ABC):
         """
 
     @abc.abstractmethod
-    def create_run(self) -> int:
-        """Create one run."""
+    def create_run(self, fab_id: str, fab_version: str) -> int:
+        """Create a new run for the specified `fab_id` and `fab_version`."""
+
+    @abc.abstractmethod
+    def get_run(self, run_id: int) -> Tuple[int, str, str]:
+        """Retrieve information about the run with the specified `run_id`.
+
+        Parameters
+        ----------
+        run_id : int
+            The identifier of the run.
+
+        Returns
+        -------
+        Tuple[int, str, str]
+            A tuple containing three elements:
+            - `run_id`: The identifier of the run, same as the specified `run_id`.
+            - `fab_id`: The identifier of the FAB used in the specified run.
+            - `fab_version`: The version of the FAB used in the specified run.
+        """
 
     @abc.abstractmethod
     def acknowledge_ping(self, node_id: int, ping_interval: float) -> bool: