flwr-nightly 1.20.0.dev20250627__py3-none-any.whl → 1.20.0.dev20250630__py3-none-any.whl

flwr/common/constant.py

@@ -135,6 +135,7 @@ GC_THRESHOLD = 200_000_000  # 200 MB
 # Constants for Inflatable
 HEAD_BODY_DIVIDER = b"\x00"
 HEAD_VALUE_DIVIDER = " "
+MAX_ARRAY_CHUNK_SIZE = 20_971_520  # 20 MB
 
 # Constants for serialization
 INT64_MAX_VALUE = 9223372036854775807  # (1 << 63) - 1

flwr/common/inflatable_utils.py

@@ -40,9 +40,11 @@ from .inflatable import (
 )
 from .message import Message
 from .record import Array, ArrayRecord, ConfigRecord, MetricRecord, RecordDict
+from .record.arraychunk import ArrayChunk
 
 # Helper registry that maps names of classes to their type
 inflatable_class_registry: dict[str, type[InflatableObject]] = {
+    ArrayChunk.__qualname__: ArrayChunk,
     Array.__qualname__: Array,
     ArrayRecord.__qualname__: ArrayRecord,
     ConfigRecord.__qualname__: ConfigRecord,

flwr/common/record/array.py

@@ -17,6 +17,7 @@
 
 from __future__ import annotations
 
+import json
 import sys
 from dataclasses import dataclass
 from io import BytesIO
@@ -24,11 +25,15 @@ from typing import TYPE_CHECKING, Any, cast, overload
 
 import numpy as np
 
-from flwr.proto.recorddict_pb2 import Array as ArrayProto  # pylint: disable=E0611
-
-from ..constant import SType
-from ..inflatable import InflatableObject, add_header_to_object_body, get_object_body
+from ..constant import MAX_ARRAY_CHUNK_SIZE, SType
+from ..inflatable import (
+    InflatableObject,
+    add_header_to_object_body,
+    get_object_body,
+    get_object_children_ids_from_object_content,
+)
 from ..typing import NDArray
+from .arraychunk import ArrayChunk
 
 if TYPE_CHECKING:
     import torch
@@ -252,16 +257,48 @@ class Array(InflatableObject):
         ndarray_deserialized = np.load(bytes_io, allow_pickle=False)
         return cast(NDArray, ndarray_deserialized)
 
+    @property
+    def children(self) -> dict[str, InflatableObject]:
+        """Return a dictionary of ArrayChunks with their Object IDs as keys."""
+        return dict(self.slice_array())
+
+    def slice_array(self) -> list[tuple[str, InflatableObject]]:
+        """Slice Array data and construct a list of ArrayChunks."""
+        children: list[tuple[str, InflatableObject]] = []
+        # memoryview allows for zero-copy slicing
+        data_view = memoryview(self.data)
+        for start in range(0, len(data_view), MAX_ARRAY_CHUNK_SIZE):
+            end = min(start + MAX_ARRAY_CHUNK_SIZE, len(data_view))
+            ac = ArrayChunk(data_view[start:end])
+            children.append((ac.object_id, ac))
+        return children
+
     def deflate(self) -> bytes:
         """Deflate the Array."""
-        array_proto = ArrayProto(
-            dtype=self.dtype,
-            shape=self.shape,
-            stype=self.stype,
-            data=self.data,
-        )
-
-        obj_body = array_proto.SerializeToString(deterministic=True)
+        array_metadata: dict[str, str | tuple[int, ...] | list[int]] = {}
+
+        # We want to record all object_id even if repeated
+        # it can happend that chunks carry the exact same data
+        # for example when the array has only zeros
+        children_list = self.slice_array()
+        # Let's not save the entire object_id but a mapping to those
+        # that will be carried in the object head
+        # (replace a long object_id with a single scalar)
+        unique_children = list(self.children.keys())
+        arraychunk_ids = [unique_children.index(ch_id) for ch_id, _ in children_list]
+
+        # The deflated Array carries everything but the data
+        # The `arraychunk_ids` will be used during Array inflation
+        # to rematerialize the data from ArrayChunk objects.
+        array_metadata = {
+            "dtype": self.dtype,
+            "shape": self.shape,
+            "stype": self.stype,
+            "arraychunk_ids": arraychunk_ids,
+        }
+
+        # Serialize metadata dict
+        obj_body = json.dumps(array_metadata).encode("utf-8")
         return add_header_to_object_body(object_body=obj_body, obj=self)
 
     @classmethod
@@ -276,26 +313,55 @@ class Array(InflatableObject):
             The deflated object content of the Array.
 
         children : Optional[dict[str, InflatableObject]] (default: None)
-            Must be ``None``. ``Array`` does not support child objects.
-            Providing any children will raise a ``ValueError``.
+            Must be ``None``. ``Array`` must have child objects.
+            Providing no children will raise a ``ValueError``.
 
         Returns
         -------
         Array
             The inflated Array.
         """
-        if children:
-            raise ValueError("`Array` objects do not have children.")
+        if not children:
+            raise ValueError("`Array` objects must have children.")
 
         obj_body = get_object_body(object_content, cls)
-        proto_array = ArrayProto.FromString(obj_body)
-        return cls(
-            dtype=proto_array.dtype,
-            shape=tuple(proto_array.shape),
-            stype=proto_array.stype,
-            data=proto_array.data,
+
+        # Extract children IDs from head
+        children_ids = get_object_children_ids_from_object_content(object_content)
+        # Decode the Array body
+        array_metadata: dict[str, str | tuple[int, ...] | list[int]] = json.loads(
+            obj_body.decode(encoding="utf-8")
         )
 
+        # Verify children ids in body match those passed for inflation
+        chunk_ids_indices = cast(list[int], array_metadata["arraychunk_ids"])
+        # Convert indices back to IDs
+        chunk_ids = [children_ids[i] for i in chunk_ids_indices]
+        # Check consistency
+        unique_arrayschunks = set(chunk_ids)
+        children_obj_ids = set(children.keys())
+        if unique_arrayschunks != children_obj_ids:
+            raise ValueError(
+                "Unexpected set of `children`. "
+                f"Expected {unique_arrayschunks} but got {children_obj_ids}."
+            )
+
+        # Materialize Array with empty data
+        array = cls(
+            dtype=cast(str, array_metadata["dtype"]),
+            shape=cast(tuple[int], tuple(array_metadata["shape"])),
+            stype=cast(str, array_metadata["stype"]),
+            data=b"",
+        )
+
+        # Now inject data from chunks
+        buff = bytearray()
+        for ch_id in chunk_ids:
+            buff += cast(ArrayChunk, children[ch_id]).data
+
+        array.data = bytes(buff)
+        return array
+
     @property
     def object_id(self) -> str:
         """Get object ID."""

flwr/common/record/arraychunk.py

@@ -0,0 +1,62 @@
+# Copyright 2025 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ArrayChunk."""
+
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..inflatable import InflatableObject, add_header_to_object_body, get_object_body
+
+
+@dataclass
+class ArrayChunk(InflatableObject):
+    """ArrayChunk type."""
+
+    data: memoryview
+
+    def __init__(self, data: bytes) -> None:
+        self.data = memoryview(data)
+
+    def deflate(self) -> bytes:
+        """Deflate the ArrayChunk."""
+        return add_header_to_object_body(object_body=self.data, obj=self)
+
+    @classmethod
+    def inflate(
+        cls, object_content: bytes, children: dict[str, InflatableObject] | None = None
+    ) -> ArrayChunk:
+        """Inflate an ArrayChunk from bytes.
+
+        Parameters
+        ----------
+        object_content : bytes
+            The deflated object content of the ArrayChunk.
+
+        children : Optional[dict[str, InflatableObject]] (default: None)
+            Must be ``None``. ``ArrayChunk`` does not support child objects.
+            Providing any children will raise a ``ValueError``.
+
+        Returns
+        -------
+        ArrayChunk
+            The inflated ArrayChunk.
+        """
+        if children:
+            raise ValueError("`ArrayChunk` objects do not have children.")
+
+        obj_body = get_object_body(object_content, cls)
+        return cls(data=memoryview(obj_body))

{flwr_nightly-1.20.0.dev20250627.dist-info → flwr_nightly-1.20.0.dev20250630.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: flwr-nightly
-Version: 1.20.0.dev20250627
+Version: 1.20.0.dev20250630
 Summary: Flower: A Friendly Federated AI Framework
 License: Apache-2.0
 Keywords: Artificial Intelligence,Federated AI,Federated Analytics,Federated Evaluation,Federated Learning,Flower,Machine Learning

{flwr_nightly-1.20.0.dev20250627.dist-info → flwr_nightly-1.20.0.dev20250630.dist-info}/RECORD

@@ -108,7 +108,7 @@ flwr/common/args.py,sha256=-aX_jVnSaDrJR2KZ8Wq0Y3dQHII4R4MJtJOIXzVUA0c,5417
 flwr/common/auth_plugin/__init__.py,sha256=3rzPkVLn9WyB5n7HLk1XGDw3SLCqRWAU1_CnglcWPfw,970
 flwr/common/auth_plugin/auth_plugin.py,sha256=kXx5o39vJchaPv28sK9qO6H_UXSWym6zRBbCa7sUwtQ,4825
 flwr/common/config.py,sha256=glcZDjco-amw1YfQcYTFJ4S1pt9APoexT-mf1QscuHs,13960
-flwr/common/constant.py,sha256=HO1y0YYHZUCIDt5QQnvCTSYVeKhkIJve0RbQ3nW7jHU,8191
+flwr/common/constant.py,sha256=HMLDvtiWGcGUfHpyyMx1rau34eXdMuBsvn-koRD8-pQ,8234
 flwr/common/context.py,sha256=Be8obQR_OvEDy1OmshuUKxGRQ7Qx89mf5F4xlhkR10s,2407
 flwr/common/date.py,sha256=1ZT2cRSpC2DJqprOVTLXYCR_O2_OZR0zXO_brJ3LqWc,1554
 flwr/common/differential_privacy.py,sha256=FdlpdpPl_H_2HJa8CQM1iCUGBBQ5Dc8CzxmHERM-EoE,6148
@@ -125,14 +125,15 @@ flwr/common/heartbeat.py,sha256=SyEpNDnmJ0lni0cWO67rcoJVKasCLmkNHm3dKLeNrLU,5749
 flwr/common/inflatable.py,sha256=GDL9oBKs16_yyVdlH6kBf493O5xll_h9V7XB5Mpx1Hc,9524
 flwr/common/inflatable_grpc_utils.py,sha256=ZpwtgF1tGD6NwQkCidbhbeBPDBZ1Nx9eGMHQ04eNEE8,3554
 flwr/common/inflatable_rest_utils.py,sha256=KiZd06XRiXcl_WewOrag0JTvUQt5kZ74UIsQ3FCAXGc,3580
-flwr/common/inflatable_utils.py,sha256=-GTdgR1zLS9WtXrbOGJMpaoyVEL8KmoQ2yF4HeLxTI0,12406
+flwr/common/inflatable_utils.py,sha256=XfgkPl8GKuR7tuOpYVZ3zK4n-LbgMZtwoNxMfUbpiCE,12489
 flwr/common/logger.py,sha256=JbRf6E2vQxXzpDBq1T8IDUJo_usu3gjWEBPQ6uKcmdg,13049
 flwr/common/message.py,sha256=xAL7iZN5-n-xPQpgoSFvxNrzs8fmiiPfoU0DjNQEhRw,19953
 flwr/common/object_ref.py,sha256=p3SfTeqo3Aj16SkB-vsnNn01zswOPdGNBitcbRnqmUk,9134
 flwr/common/parameter.py,sha256=UVw6sOgehEFhFs4uUCMl2kfVq1PD6ncmWgPLMsZPKPE,2095
 flwr/common/pyproject.py,sha256=2SU6yJW7059SbMXgzjOdK1GZRWO6AixDH7BmdxbMvHI,1386
 flwr/common/record/__init__.py,sha256=cNGccdDoxttqgnUgyKRIqLWULjW-NaSmOufVxtXq-sw,1197
-flwr/common/record/array.py,sha256=3K01tAf_jedub2r2-vkRshbsjBSiKErAO4KqDgdDaSo,11776
+flwr/common/record/array.py,sha256=oApWRwxh1bY2MivQcAz9YHkrspkYZVvy1OkBsWolMec,14541
+flwr/common/record/arraychunk.py,sha256=2ubMzsRNZFL4cc-wXNrJkcTSJUD3nL8CeX5PZpEhNSo,2019
 flwr/common/record/arrayrecord.py,sha256=CpoqYXM6Iv4XEc9SryCMYmw-bIvP8ut6xWJzRwYJzdU,18008
 flwr/common/record/configrecord.py,sha256=G7U0q39kB0Kyi0zMxFmPxcVemL9NgwVS1qjvI4BRQuU,9763
 flwr/common/record/conversion_utils.py,sha256=wbNCzy7oAqaA3-arhls_EqRZYXRC4YrWIoE-Gy82fJ0,1191
@@ -360,7 +361,7 @@ flwr/supernode/servicer/__init__.py,sha256=lucTzre5WPK7G1YLCfaqg3rbFWdNSb7ZTt-ca
 flwr/supernode/servicer/clientappio/__init__.py,sha256=7Oy62Y_oijqF7Dxi6tpcUQyOpLc_QpIRZ83NvwmB0Yg,813
 flwr/supernode/servicer/clientappio/clientappio_servicer.py,sha256=d3GdIabycUoDBDL_eVlt513knGSjQW3-9lG6Cw4QEk4,5719
 flwr/supernode/start_client_internal.py,sha256=DAXuReZ1FCXt9Y1KbM0p-dI50ROWPEJXzfKrl14OE6k,18233
-flwr_nightly-1.20.0.dev20250627.dist-info/METADATA,sha256=FV-_wPKa9RQ3rLCjBW6TJht1mmGqT_-OU2Fi2DvCmIM,15910
-flwr_nightly-1.20.0.dev20250627.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-flwr_nightly-1.20.0.dev20250627.dist-info/entry_points.txt,sha256=jNpDXGBGgs21RqUxelF_jwGaxtqFwm-MQyfz-ZqSjrA,367
-flwr_nightly-1.20.0.dev20250627.dist-info/RECORD,,
+flwr_nightly-1.20.0.dev20250630.dist-info/METADATA,sha256=B3mOdd5CdmCCEJsFc4A65ifNMx4_vvr7UHDi-OuIAzY,15910
+flwr_nightly-1.20.0.dev20250630.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+flwr_nightly-1.20.0.dev20250630.dist-info/entry_points.txt,sha256=jNpDXGBGgs21RqUxelF_jwGaxtqFwm-MQyfz-ZqSjrA,367
+flwr_nightly-1.20.0.dev20250630.dist-info/RECORD,,