flwr-nightly 1.20.0.dev20250627__tar.gz → 1.20.0.dev20250630__tar.gz

{flwr_nightly-1.20.0.dev20250627 → flwr_nightly-1.20.0.dev20250630}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: flwr-nightly
-Version: 1.20.0.dev20250627
+Version: 1.20.0.dev20250630
 Summary: Flower: A Friendly Federated AI Framework
 License: Apache-2.0
 Keywords: Artificial Intelligence,Federated AI,Federated Analytics,Federated Evaluation,Federated Learning,Flower,Machine Learning

{flwr_nightly-1.20.0.dev20250627 → flwr_nightly-1.20.0.dev20250630}/py/flwr/common/constant.py

@@ -135,6 +135,7 @@ GC_THRESHOLD = 200_000_000  # 200 MB
 # Constants for Inflatable
 HEAD_BODY_DIVIDER = b"\x00"
 HEAD_VALUE_DIVIDER = " "
+MAX_ARRAY_CHUNK_SIZE = 20_971_520  # 20 MB
 
 # Constants for serialization
 INT64_MAX_VALUE = 9223372036854775807  # (1 << 63) - 1

{flwr_nightly-1.20.0.dev20250627 → flwr_nightly-1.20.0.dev20250630}/py/flwr/common/inflatable_utils.py

@@ -40,9 +40,11 @@ from .inflatable import (
 )
 from .message import Message
 from .record import Array, ArrayRecord, ConfigRecord, MetricRecord, RecordDict
+from .record.arraychunk import ArrayChunk
 
 # Helper registry that maps names of classes to their type
 inflatable_class_registry: dict[str, type[InflatableObject]] = {
+    ArrayChunk.__qualname__: ArrayChunk,
     Array.__qualname__: Array,
     ArrayRecord.__qualname__: ArrayRecord,
     ConfigRecord.__qualname__: ConfigRecord,

{flwr_nightly-1.20.0.dev20250627 → flwr_nightly-1.20.0.dev20250630}/py/flwr/common/record/array.py

@@ -17,6 +17,7 @@
 
 from __future__ import annotations
 
+import json
 import sys
 from dataclasses import dataclass
 from io import BytesIO
@@ -24,11 +25,15 @@ from typing import TYPE_CHECKING, Any, cast, overload
 
 import numpy as np
 
-from flwr.proto.recorddict_pb2 import Array as ArrayProto  # pylint: disable=E0611
-
-from ..constant import SType
-from ..inflatable import InflatableObject, add_header_to_object_body, get_object_body
+from ..constant import MAX_ARRAY_CHUNK_SIZE, SType
+from ..inflatable import (
+    InflatableObject,
+    add_header_to_object_body,
+    get_object_body,
+    get_object_children_ids_from_object_content,
+)
 from ..typing import NDArray
+from .arraychunk import ArrayChunk
 
 if TYPE_CHECKING:
     import torch
@@ -252,16 +257,48 @@ class Array(InflatableObject):
         ndarray_deserialized = np.load(bytes_io, allow_pickle=False)
         return cast(NDArray, ndarray_deserialized)
 
+    @property
+    def children(self) -> dict[str, InflatableObject]:
+        """Return a dictionary of ArrayChunks with their Object IDs as keys."""
+        return dict(self.slice_array())
+
+    def slice_array(self) -> list[tuple[str, InflatableObject]]:
+        """Slice Array data and construct a list of ArrayChunks."""
+        children: list[tuple[str, InflatableObject]] = []
+        # memoryview allows for zero-copy slicing
+        data_view = memoryview(self.data)
+        for start in range(0, len(data_view), MAX_ARRAY_CHUNK_SIZE):
+            end = min(start + MAX_ARRAY_CHUNK_SIZE, len(data_view))
+            ac = ArrayChunk(data_view[start:end])
+            children.append((ac.object_id, ac))
+        return children
+
     def deflate(self) -> bytes:
         """Deflate the Array."""
-        array_proto = ArrayProto(
-            dtype=self.dtype,
-            shape=self.shape,
-            stype=self.stype,
-            data=self.data,
-        )
-
-        obj_body = array_proto.SerializeToString(deterministic=True)
+        array_metadata: dict[str, str | tuple[int, ...] | list[int]] = {}
+
+        # We want to record all object_id even if repeated
+        # it can happend that chunks carry the exact same data
+        # for example when the array has only zeros
+        children_list = self.slice_array()
+        # Let's not save the entire object_id but a mapping to those
+        # that will be carried in the object head
+        # (replace a long object_id with a single scalar)
+        unique_children = list(self.children.keys())
+        arraychunk_ids = [unique_children.index(ch_id) for ch_id, _ in children_list]
+
+        # The deflated Array carries everything but the data
+        # The `arraychunk_ids` will be used during Array inflation
+        # to rematerialize the data from ArrayChunk objects.
+        array_metadata = {
+            "dtype": self.dtype,
+            "shape": self.shape,
+            "stype": self.stype,
+            "arraychunk_ids": arraychunk_ids,
+        }
+
+        # Serialize metadata dict
+        obj_body = json.dumps(array_metadata).encode("utf-8")
         return add_header_to_object_body(object_body=obj_body, obj=self)
 
     @classmethod
@@ -276,26 +313,55 @@ class Array(InflatableObject):
             The deflated object content of the Array.
 
         children : Optional[dict[str, InflatableObject]] (default: None)
-            Must be ``None``. ``Array`` does not support child objects.
-            Providing any children will raise a ``ValueError``.
+            Must be ``None``. ``Array`` must have child objects.
+            Providing no children will raise a ``ValueError``.
 
         Returns
         -------
         Array
             The inflated Array.
         """
-        if children:
-            raise ValueError("`Array` objects do not have children.")
+        if not children:
+            raise ValueError("`Array` objects must have children.")
 
         obj_body = get_object_body(object_content, cls)
-        proto_array = ArrayProto.FromString(obj_body)
-        return cls(
-            dtype=proto_array.dtype,
-            shape=tuple(proto_array.shape),
-            stype=proto_array.stype,
-            data=proto_array.data,
+
+        # Extract children IDs from head
+        children_ids = get_object_children_ids_from_object_content(object_content)
+        # Decode the Array body
+        array_metadata: dict[str, str | tuple[int, ...] | list[int]] = json.loads(
+            obj_body.decode(encoding="utf-8")
         )
 
+        # Verify children ids in body match those passed for inflation
+        chunk_ids_indices = cast(list[int], array_metadata["arraychunk_ids"])
+        # Convert indices back to IDs
+        chunk_ids = [children_ids[i] for i in chunk_ids_indices]
+        # Check consistency
+        unique_arrayschunks = set(chunk_ids)
+        children_obj_ids = set(children.keys())
+        if unique_arrayschunks != children_obj_ids:
+            raise ValueError(
+                "Unexpected set of `children`. "
+                f"Expected {unique_arrayschunks} but got {children_obj_ids}."
+            )
+
+        # Materialize Array with empty data
+        array = cls(
+            dtype=cast(str, array_metadata["dtype"]),
+            shape=cast(tuple[int], tuple(array_metadata["shape"])),
+            stype=cast(str, array_metadata["stype"]),
+            data=b"",
+        )
+
+        # Now inject data from chunks
+        buff = bytearray()
+        for ch_id in chunk_ids:
+            buff += cast(ArrayChunk, children[ch_id]).data
+
+        array.data = bytes(buff)
+        return array
+
     @property
     def object_id(self) -> str:
         """Get object ID."""

flwr_nightly-1.20.0.dev20250630/py/flwr/common/record/arraychunk.py

@@ -0,0 +1,62 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ArrayChunk."""
+
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..inflatable import InflatableObject, add_header_to_object_body, get_object_body
+
+
+@dataclass
+class ArrayChunk(InflatableObject):
+    """ArrayChunk type."""
+
+    data: memoryview
+
+    def __init__(self, data: bytes) -> None:
+        self.data = memoryview(data)
+
+    def deflate(self) -> bytes:
+        """Deflate the ArrayChunk."""
+        return add_header_to_object_body(object_body=self.data, obj=self)
+
+    @classmethod
+    def inflate(
+        cls, object_content: bytes, children: dict[str, InflatableObject] | None = None
+    ) -> ArrayChunk:
+        """Inflate an ArrayChunk from bytes.
+
+        Parameters
+        ----------
+        object_content : bytes
+            The deflated object content of the ArrayChunk.
+
+        children : Optional[dict[str, InflatableObject]] (default: None)
+            Must be ``None``. ``ArrayChunk`` does not support child objects.
+            Providing any children will raise a ``ValueError``.
+
+        Returns
+        -------
+        ArrayChunk
+            The inflated ArrayChunk.
+        """
+        if children:
+            raise ValueError("`ArrayChunk` objects do not have children.")
+
+        obj_body = get_object_body(object_content, cls)
+        return cls(data=memoryview(obj_body))

{flwr_nightly-1.20.0.dev20250627 → flwr_nightly-1.20.0.dev20250630}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.20.0.dev20250627"
+version = "1.20.0.dev20250630"
 description = "Flower: A Friendly Federated AI Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]