flwr-nightly 1.8.0.dev20240314__py3-none-any.whl → 1.15.0.dev20250114__py3-none-any.whl

flwr/common/object_ref.py

@@ -17,8 +17,14 @@
 
 import ast
 import importlib
+import sys
 from importlib.util import find_spec
-from typing import Any, Optional, Tuple, Type
+from logging import WARN
+from pathlib import Path
+from threading import Lock
+from typing import Any, Optional, Union
+
+from .logger import log
 
 OBJECT_REF_HELP_STR = """
 \n\nThe object reference string should have the form <module>:<attribute>. Valid
@@ -28,21 +34,42 @@ attribute.
 """
 
 
+_current_sys_path: Optional[str] = None
+_import_lock = Lock()
+
+
 def validate(
     module_attribute_str: str,
-) -> Tuple[bool, Optional[str]]:
+    check_module: bool = True,
+    project_dir: Optional[Union[str, Path]] = None,
+) -> tuple[bool, Optional[str]]:
     """Validate object reference.
 
-    The object reference string should have the form <module>:<attribute>. Valid
-    examples include `client:app` and `project.package.module:wrapper.app`. It must
-    refer to a module on the PYTHONPATH and the module needs to have the specified
-    attribute.
+    Parameters
+    ----------
+    module_attribute_str : str
+        The reference to the object. It should have the form `<module>:<attribute>`.
+        Valid examples include `client:app` and `project.package.module:wrapper.app`.
+        It must refer to a module on the PYTHONPATH or in the provided `project_dir`
+        and the module needs to have the specified attribute.
+    check_module : bool (default: True)
+        Flag indicating whether to verify the existence of the module and the
+        specified attribute within it.
+    project_dir : Optional[Union[str, Path]] (default: None)
+        The directory containing the module. If None, the current working directory
+        is used. If `check_module` is True, the `project_dir` will be temporarily
+        inserted into the system path and then removed after the validation is complete.
 
     Returns
     -------
     Tuple[bool, Optional[str]]
         A boolean indicating whether an object reference is valid and
         the reason why it might not be.
+
+    Note
+    ----
+    This function will temporarily modify `sys.path` by inserting the provided
+    `project_dir`, which will be removed after the validation is complete.
     """
     module_str, _, attributes_str = module_attribute_str.partition(":")
     if not module_str:
@@ -56,15 +83,29 @@ def validate(
             f"Missing attribute in {module_attribute_str}{OBJECT_REF_HELP_STR}",
         )
 
-    # Load module
-    module = find_spec(module_str)
-    if module and module.origin:
-        if not _find_attribute_in_module(module.origin, attributes_str):
-            return (
-                False,
-                f"Unable to find attribute {attributes_str} in module {module_str}"
-                f"{OBJECT_REF_HELP_STR}",
-            )
+    if check_module:
+        if project_dir is None:
+            project_dir = Path.cwd()
+        project_dir = Path(project_dir).absolute()
+        # Set the system path
+        sys.path.insert(0, str(project_dir))
+
+        # Load module
+        module = find_spec(module_str)
+
+        # Unset the system path
+        sys.path.remove(str(project_dir))
+
+        # Check if the module and the attribute exist
+        if module and module.origin:
+            if not _find_attribute_in_module(module.origin, attributes_str):
+                return (
+                    False,
+                    f"Unable to find attribute {attributes_str} in module {module_str}"
+                    f"{OBJECT_REF_HELP_STR}",
+                )
+            return (True, None)
+    else:
         return (True, None)
 
     return (
@@ -73,42 +114,136 @@ def validate(
     )
 
 
-def load_app(
+def load_app(  # pylint: disable= too-many-branches
     module_attribute_str: str,
-    error_type: Type[Exception],
+    error_type: type[Exception],
+    project_dir: Optional[Union[str, Path]] = None,
 ) -> Any:
     """Return the object specified in a module attribute string.
 
-    The module/attribute string should have the form <module>:<attribute>. Valid
-    examples include `client:app` and `project.package.module:wrapper.app`. It must
-    refer to a module on the PYTHONPATH, the module needs to have the specified
-    attribute.
+    Parameters
+    ----------
+    module_attribute_str : str
+        The reference to the object. It should have the form `<module>:<attribute>`.
+        Valid examples include `client:app` and `project.package.module:wrapper.app`.
+        It must refer to a module on the PYTHONPATH or in the provided `project_dir`
+        and the module needs to have the specified attribute.
+    error_type : Type[Exception]
+        The type of exception to be raised if the provided `module_attribute_str` is
+        in an invalid format.
+    project_dir : Optional[Union[str, Path]], optional (default=None)
+        The directory containing the module. If None, the current working directory
+        is used. The `project_dir` will be inserted into the system path, and the
+        previously inserted `project_dir` will be removed.
+
+    Returns
+    -------
+    Any
+        The object specified by the module attribute string.
+
+    Note
+    ----
+    - This function will unload all modules in the previously provided `project_dir`,
+      if it is invoked again.
+    - This function will modify `sys.path` by inserting the provided `project_dir`
+      and removing the previously inserted `project_dir`.
     """
-    valid, error_msg = validate(module_attribute_str)
-    if not valid and error_msg:
-        raise error_type(error_msg) from None
+    with _import_lock:
+        valid, error_msg = validate(module_attribute_str, check_module=False)
+        if not valid and error_msg:
+            raise error_type(error_msg) from None
 
-    module_str, _, attributes_str = module_attribute_str.partition(":")
+        module_str, _, attributes_str = module_attribute_str.partition(":")
+
+        try:
+            # Initialize project path
+            if project_dir is None:
+                project_dir = Path.cwd()
+            project_dir = Path(project_dir).absolute()
+
+            # Unload modules if the project directory has changed
+            if _current_sys_path and _current_sys_path != str(project_dir):
+                _unload_modules(Path(_current_sys_path))
+
+            # Set the system path
+            _set_sys_path(project_dir)
+
+            # Import the module
+            if module_str not in sys.modules:
+                module = importlib.import_module(module_str)
+            # Hack: `tabnet` does not work with `importlib.reload`
+            elif "tabnet" in sys.modules:
+                log(
+                    WARN,
+                    "Cannot reload module `%s` from disk due to compatibility issues "
+                    "with the `tabnet` library. The module will be loaded from the "
+                    "cache instead. If you experience issues, consider restarting "
+                    "the application.",
+                    module_str,
+                )
+                module = sys.modules[module_str]
+            else:
+                module = sys.modules[module_str]
+                _reload_modules(project_dir)
+
+        except ModuleNotFoundError as err:
+            raise error_type(
+                f"Unable to load module {module_str}{OBJECT_REF_HELP_STR}",
+            ) from err
+
+        # Recursively load attribute
+        attribute = module
+        try:
+            for attribute_str in attributes_str.split("."):
+                attribute = getattr(attribute, attribute_str)
+        except AttributeError as err:
+            raise error_type(
+                f"Unable to load attribute {attributes_str} from module {module_str}"
+                f"{OBJECT_REF_HELP_STR}",
+            ) from err
+
+        return attribute
+
+
+def _unload_modules(project_dir: Path) -> None:
+    """Unload modules from the project directory."""
+    dir_str = str(project_dir.absolute())
+    for name, m in list(sys.modules.items()):
+        path: Optional[str] = getattr(m, "__file__", None)
+        if path is not None and path.startswith(dir_str):
+            del sys.modules[name]
+
+
+def _reload_modules(project_dir: Path) -> None:
+    """Reload modules from the project directory."""
+    dir_str = str(project_dir.absolute())
+    for m in list(sys.modules.values()):
+        path: Optional[str] = getattr(m, "__file__", None)
+        if path is not None and path.startswith(dir_str):
+            importlib.reload(m)
+
+
+def _set_sys_path(directory: Optional[Union[str, Path]]) -> None:
+    """Set the system path."""
+    if directory is None:
+        directory = Path.cwd()
+    else:
+        directory = Path(directory).absolute()
+
+    # If the directory has already been added to `sys.path`, return
+    if str(directory) in sys.path:
+        return
+
+    # Remove the old path if it exists and is not `""`.
+    global _current_sys_path  # pylint: disable=global-statement
+    if _current_sys_path is not None:
+        sys.path.remove(_current_sys_path)
+
+    # Add the new path to sys.path
+    sys.path.insert(0, str(directory))
 
-    try:
-        module = importlib.import_module(module_str)
-    except ModuleNotFoundError:
-        raise error_type(
-            f"Unable to load module {module_str}{OBJECT_REF_HELP_STR}",
-        ) from None
-
-    # Recursively load attribute
-    attribute = module
-    try:
-        for attribute_str in attributes_str.split("."):
-            attribute = getattr(attribute, attribute_str)
-    except AttributeError:
-        raise error_type(
-            f"Unable to load attribute {attributes_str} from module {module_str}"
-            f"{OBJECT_REF_HELP_STR}",
-        ) from None
-
-    return attribute
+    # Update the current_sys_path
+    _current_sys_path = str(directory)
 
 
 def _find_attribute_in_module(file_path: str, attribute_name: str) -> bool:

flwr/common/pyproject.py

@@ -14,6 +14,7 @@
 # ==============================================================================
 """Validates the project's name property."""
 
+
 import re
 
 

flwr/common/record/__init__.py

@@ -14,6 +14,7 @@
 # ==============================================================================
 """Record APIs."""
 
+
 from .configsrecord import ConfigsRecord
 from .conversion_utils import array_from_numpy
 from .metricsrecord import MetricsRecord
@@ -22,9 +23,9 @@ from .recordset import RecordSet
 
 __all__ = [
     "Array",
-    "array_from_numpy",
     "ConfigsRecord",
     "MetricsRecord",
     "ParametersRecord",
     "RecordSet",
+    "array_from_numpy",
 ]

flwr/common/record/configsrecord.py

@@ -15,7 +15,7 @@
 """ConfigsRecord."""
 
 
-from typing import Dict, List, Optional, get_args
+from typing import Optional, get_args
 
 from flwr.common.typing import ConfigsRecordValues, ConfigsScalar
 
@@ -58,27 +58,61 @@ def _check_value(value: ConfigsRecordValues) -> None:
 
 
 class ConfigsRecord(TypedDict[str, ConfigsRecordValues]):
-    """Configs record."""
+    """Configs record.
+
+    A :code:`ConfigsRecord` is a Python dictionary designed to ensure that
+    each key-value pair adheres to specified data types. A :code:`ConfigsRecord`
+    is one of the types of records that a
+    `flwr.common.RecordSet <flwr.common.RecordSet.html#recordset>`_ supports and
+    can therefore be used to construct :code:`common.Message` objects.
+
+    Parameters
+    ----------
+    configs_dict : Optional[Dict[str, ConfigsRecordValues]]
+        A dictionary that stores basic types (i.e. `str`, `int`, `float`, `bytes` as
+        defined in `ConfigsScalar`) and lists of such types (see
+        `ConfigsScalarList`).
+    keep_input : bool (default: True)
+        A boolean indicating whether config passed should be deleted from the input
+        dictionary immediately after adding them to the record. When set
+        to True, the data is duplicated in memory. If memory is a concern, set
+        it to False.
+
+    Examples
+    --------
+    The usage of a :code:`ConfigsRecord` is envisioned for sending configuration values
+    telling the target node how to perform a certain action (e.g. train/evaluate a model
+    ). You can use standard Python built-in types such as :code:`float`, :code:`str`
+    , :code:`bytes`. All types allowed are defined in
+    :code:`flwr.common.ConfigsRecordValues`. While lists are supported, we
+    encourage you to use a :code:`ParametersRecord` instead if these are of high
+    dimensionality.
+
+    Let's see some examples of how to construct a :code:`ConfigsRecord` from scratch:
+
+    >>> from flwr.common import ConfigsRecord
+    >>>
+    >>> # A `ConfigsRecord` is a specialized Python dictionary
+    >>> record = ConfigsRecord({"lr": 0.1, "batch-size": 128})
+    >>> # You can add more content to an existing record
+    >>> record["compute-average"] = True
+    >>> # It also supports lists
+    >>> record["loss-fn-coefficients"] = [0.4, 0.25, 0.35]
+    >>> # And string values (among other types)
+    >>> record["path-to-S3"] = "s3://bucket_name/folder1/fileA.json"
+
+    Just like the other types of records in a :code:`flwr.common.RecordSet`, types are
+    enforced. If you need to add a custom data structure or object, we recommend to
+    serialise it into bytes and save it as such (bytes are allowed in a
+    :code:`ConfigsRecord`)
+    """
 
     def __init__(
         self,
-        configs_dict: Optional[Dict[str, ConfigsRecordValues]] = None,
+        configs_dict: Optional[dict[str, ConfigsRecordValues]] = None,
         keep_input: bool = True,
     ) -> None:
-        """Construct a ConfigsRecord object.
-
-        Parameters
-        ----------
-        configs_dict : Optional[Dict[str, ConfigsRecordValues]]
-            A dictionary that stores basic types (i.e. `str`, `int`, `float`, `bytes` as
-            defined in `ConfigsScalar`) and lists of such types (see
-            `ConfigsScalarList`).
-        keep_input : bool (default: True)
-            A boolean indicating whether config passed should be deleted from the input
-            dictionary immediately after adding them to the record. When set
-            to True, the data is duplicated in memory. If memory is a concern, set
-            it to False.
-        """
+
         super().__init__(_check_key, _check_value)
         if configs_dict:
             for k in list(configs_dict.keys()):
@@ -94,6 +128,7 @@ class ConfigsRecord(TypedDict[str, ConfigsRecordValues]):
 
         def get_var_bytes(value: ConfigsScalar) -> int:
             """Return Bytes of value passed."""
+            var_bytes = 0
             if isinstance(value, bool):
                 var_bytes = 1
             elif isinstance(value, (int, float)):
@@ -102,12 +137,17 @@ class ConfigsRecord(TypedDict[str, ConfigsRecordValues]):
                 )
             if isinstance(value, (str, bytes)):
                 var_bytes = len(value)
+            if var_bytes == 0:
+                raise ValueError(
+                    "Config values must be either `bool`, `int`, `float`, "
+                    "`str`, or `bytes`"
+                )
             return var_bytes
 
         num_bytes = 0
 
         for k, v in self.items():
-            if isinstance(v, List):
+            if isinstance(v, list):
                 if isinstance(v[0], (bytes, str)):
                     # not all str are of equal length necessarily
                     # for both the footprint of each element is 1 Byte

flwr/common/record/metricsrecord.py

@@ -15,7 +15,7 @@
 """MetricsRecord."""
 
 
-from typing import Dict, List, Optional, get_args
+from typing import Optional, get_args
 
 from flwr.common.typing import MetricsRecordValues, MetricsScalar
 
@@ -58,26 +58,66 @@ def _check_value(value: MetricsRecordValues) -> None:
 
 
 class MetricsRecord(TypedDict[str, MetricsRecordValues]):
-    """Metrics record."""
+    """Metrics recod.
+
+    A :code:`MetricsRecord` is a Python dictionary designed to ensure that
+    each key-value pair adheres to specified data types. A :code:`MetricsRecord`
+    is one of the types of records that a
+    `flwr.common.RecordSet <flwr.common.RecordSet.html#recordset>`_ supports and
+    can therefore be used to construct :code:`common.Message` objects.
+
+    Parameters
+    ----------
+    metrics_dict : Optional[Dict[str, MetricsRecordValues]]
+        A dictionary that stores basic types (i.e. `int`, `float` as defined
+        in `MetricsScalar`) and list of such types (see `MetricsScalarList`).
+    keep_input : bool (default: True)
+        A boolean indicating whether metrics should be deleted from the input
+        dictionary immediately after adding them to the record. When set
+        to True, the data is duplicated in memory. If memory is a concern, set
+        it to False.
+
+    Examples
+    --------
+    The usage of a :code:`MetricsRecord` is envisioned for communicating results
+    obtained when a node performs an action. A few typical examples include:
+    communicating the training accuracy after a model is trained locally by a
+    :code:`ClientApp`, reporting the validation loss obtained at a :code:`ClientApp`,
+    or, more generally, the output of executing a query by the :code:`ClientApp`.
+    Common to these examples is that the output can be typically represented by
+    a single scalar (:code:`int`, :code:`float`) or list of scalars.
+
+    Let's see some examples of how to construct a :code:`MetricsRecord` from scratch:
+
+    >>> from flwr.common import MetricsRecord
+    >>>
+    >>> # A `MetricsRecord` is a specialized Python dictionary
+    >>> record = MetricsRecord({"accuracy": 0.94})
+    >>> # You can add more content to an existing record
+    >>> record["loss"] = 0.01
+    >>> # It also supports lists
+    >>> record["loss-historic"] = [0.9, 0.5, 0.01]
+
+    Since types are enforced, the types of the objects inserted are checked. For a
+    :code:`MetricsRecord`, value types allowed are those in defined in
+    :code:`flwr.common.MetricsRecordValues`. Similarly, only :code:`str` keys are
+    allowed.
+
+    >>> from flwr.common import MetricsRecord
+    >>>
+    >>> record = MetricsRecord() # an empty record
+    >>> # Add unsupported value
+    >>> record["something-unsupported"] = {'a': 123} # Will throw a `TypeError`
+
+    If you need a more versatily type of record try :code:`ConfigsRecord` or
+    :code:`ParametersRecord`.
+    """
 
     def __init__(
         self,
-        metrics_dict: Optional[Dict[str, MetricsRecordValues]] = None,
+        metrics_dict: Optional[dict[str, MetricsRecordValues]] = None,
         keep_input: bool = True,
     ):
-        """Construct a MetricsRecord object.
-
-        Parameters
-        ----------
-        metrics_dict : Optional[Dict[str, MetricsRecordValues]]
-            A dictionary that stores basic types (i.e. `int`, `float` as defined
-            in `MetricsScalar`) and list of such types (see `MetricsScalarList`).
-        keep_input : bool (default: True)
-            A boolean indicating whether metrics should be deleted from the input
-            dictionary immediately after adding them to the record. When set
-            to True, the data is duplicated in memory. If memory is a concern, set
-            it to False.
-        """
         super().__init__(_check_key, _check_value)
         if metrics_dict:
             for k in list(metrics_dict.keys()):
@@ -90,7 +130,7 @@ class MetricsRecord(TypedDict[str, MetricsRecordValues]):
         num_bytes = 0
 
         for k, v in self.items():
-            if isinstance(v, List):
+            if isinstance(v, list):
                 # both int and float normally take 4 bytes
                 # But MetricRecords are mapped to 64bit int/float
                 # during protobuffing

flwr/common/record/parametersrecord.py

@@ -14,9 +14,11 @@
 # ==============================================================================
 """ParametersRecord and Array."""
 
+
+from collections import OrderedDict
 from dataclasses import dataclass
 from io import BytesIO
-from typing import List, Optional, OrderedDict, cast
+from typing import Optional, cast
 
 import numpy as np
 
@@ -51,7 +53,7 @@ class Array:
     """
 
     dtype: str
-    shape: List[int]
+    shape: list[int]
     stype: str
     data: bytes
 
@@ -82,13 +84,94 @@ def _check_value(value: Array) -> None:
         )
 
 
-@dataclass
 class ParametersRecord(TypedDict[str, Array]):
-    """Parameters record.
+    r"""Parameters record.
 
     A dataclass storing named Arrays in order. This means that it holds entries as an
     OrderedDict[str, Array]. ParametersRecord objects can be viewed as an equivalent to
-    PyTorch's state_dict, but holding serialised tensors instead.
+    PyTorch's state_dict, but holding serialised tensors instead. A
+    :code:`ParametersRecord`  is one of the types of records that a
+    `flwr.common.RecordSet <flwr.common.RecordSet.html#recordset>`_ supports and
+    can therefore be used to construct :code:`common.Message` objects.
+
+    Parameters
+    ----------
+    array_dict : Optional[OrderedDict[str, Array]]
+        A dictionary that stores serialized array-like or tensor-like objects.
+    keep_input : bool (default: False)
+        A boolean indicating whether parameters should be deleted from the input
+        dictionary immediately after adding them to the record. If False, the
+        dictionary passed to `set_parameters()` will be empty once exiting from that
+        function. This is the desired behaviour when working with very large
+        models/tensors/arrays. However, if you plan to continue working with your
+        parameters after adding it to the record, set this flag to True. When set
+        to True, the data is duplicated in memory.
+
+    Examples
+    --------
+    The usage of :code:`ParametersRecord` is envisioned for storing data arrays (e.g.
+    parameters of a machine learning model). These first need to be serialized into
+    a :code:`flwr.common.Array` data structure.
+
+    Let's see some examples:
+
+    >>> import numpy as np
+    >>> from flwr.common import ParametersRecord
+    >>> from flwr.common import array_from_numpy
+    >>>
+    >>> # Let's create a simple NumPy array
+    >>> arr_np = np.random.randn(3, 3)
+    >>>
+    >>> # If we print it
+    >>> array([[-1.84242409, -1.01539537, -0.46528405],
+    >>>      [ 0.32991896,  0.55540414,  0.44085534],
+    >>>      [-0.10758364,  1.97619858, -0.37120501]])
+    >>>
+    >>> # Let's create an Array out of it
+    >>> arr = array_from_numpy(arr_np)
+    >>>
+    >>> # If we print it you'll see (note the binary data)
+    >>> Array(dtype='float64', shape=[3,3], stype='numpy.ndarray', data=b'@\x99\x18...')
+    >>>
+    >>> # Adding it to a ParametersRecord:
+    >>> p_record = ParametersRecord({"my_array": arr})
+
+    Now that the NumPy array is embedded into a :code:`ParametersRecord` it could be
+    sent if added as part of a :code:`common.Message` or it could be saved as a
+    persistent state of a :code:`ClientApp` via its context. Regardless of the usecase,
+    we will sooner or later want to recover the array in its original NumPy
+    representation. For the example above, where the array was serialized using the
+    built-in utility function, deserialization can be done as follows:
+
+    >>> # Use the Array's built-in method
+    >>> arr_np_d = arr.numpy()
+    >>>
+    >>> # If printed, it will show the exact same data as above:
+    >>> array([[-1.84242409, -1.01539537, -0.46528405],
+    >>>      [ 0.32991896,  0.55540414,  0.44085534],
+    >>>      [-0.10758364,  1.97619858, -0.37120501]])
+
+    If you need finer control on how your arrays are serialized and deserialized, you
+    can construct :code:`Array` objects directly like this:
+
+    >>> from flwr.common import Array
+    >>> # Serialize your array and construct Array object
+    >>> arr = Array(
+    >>>         data=ndarray.tobytes(),
+    >>>         dtype=str(ndarray.dtype),
+    >>>         stype="",  # Could be used in a deserialization function
+    >>>         shape=list(ndarray.shape),
+    >>>       )
+    >>>
+    >>> # Then you can deserialize it like this
+    >>> arr_np_d = np.frombuffer(
+    >>>             buffer=array.data,
+    >>>             dtype=array.dtype,
+    >>>            ).reshape(array.shape)
+
+    Note that different arrays (e.g. from PyTorch, Tensorflow) might require different
+    serialization mechanism. Howerver, they often support a conversion to NumPy,
+    therefore allowing to use the same or similar steps as in the example above.
     """
 
     def __init__(
@@ -96,21 +179,6 @@ class ParametersRecord(TypedDict[str, Array]):
         array_dict: Optional[OrderedDict[str, Array]] = None,
         keep_input: bool = False,
     ) -> None:
-        """Construct a ParametersRecord object.
-
-        Parameters
-        ----------
-        array_dict : Optional[OrderedDict[str, Array]]
-            A dictionary that stores serialized array-like or tensor-like objects.
-        keep_input : bool (default: False)
-            A boolean indicating whether parameters should be deleted from the input
-            dictionary immediately after adding them to the record. If False, the
-            dictionary passed to `set_parameters()` will be empty once exiting from that
-            function. This is the desired behaviour when working with very large
-            models/tensors/arrays. However, if you plan to continue working with your
-            parameters after adding it to the record, set this flag to True. When set
-            to True, the data is duplicated in memory.
-        """
         super().__init__(_check_key, _check_value)
         if array_dict:
             for k in list(array_dict.keys()):