flwr-nightly 1.11.0.dev20240813__py3-none-any.whl → 1.11.0.dev20240816__py3-none-any.whl

flwr/client/process/utils.py

@@ -0,0 +1,108 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower ClientApp loading utils."""
+
+from logging import DEBUG
+from pathlib import Path
+from typing import Callable, Optional
+
+from flwr.client.client_app import ClientApp, LoadClientAppError
+from flwr.common.config import (
+    get_flwr_dir,
+    get_metadata_from_config,
+    get_project_config,
+    get_project_dir,
+)
+from flwr.common.logger import log
+from flwr.common.object_ref import load_app, validate
+
+
+def get_load_client_app_fn(
+    default_app_ref: str,
+    app_path: Optional[str],
+    multi_app: bool,
+    flwr_dir: Optional[str] = None,
+) -> Callable[[str, str], ClientApp]:
+    """Get the load_client_app_fn function.
+
+    If `multi_app` is True, this function loads the specified ClientApp
+    based on `fab_id` and `fab_version`. If `fab_id` is empty, a default
+    ClientApp will be loaded.
+
+    If `multi_app` is False, it ignores `fab_id` and `fab_version` and
+    loads a default ClientApp.
+    """
+    if not multi_app:
+        log(
+            DEBUG,
+            "Flower SuperNode will load and validate ClientApp `%s`",
+            default_app_ref,
+        )
+
+        valid, error_msg = validate(default_app_ref, project_dir=app_path)
+        if not valid and error_msg:
+            raise LoadClientAppError(error_msg) from None
+
+    def _load(fab_id: str, fab_version: str) -> ClientApp:
+        runtime_app_dir = Path(app_path if app_path else "").absolute()
+        # If multi-app feature is disabled
+        if not multi_app:
+            # Set app reference
+            client_app_ref = default_app_ref
+        # If multi-app feature is enabled but app directory is provided
+        elif app_path is not None:
+            config = get_project_config(runtime_app_dir)
+            this_fab_version, this_fab_id = get_metadata_from_config(config)
+
+            if this_fab_version != fab_version or this_fab_id != fab_id:
+                raise LoadClientAppError(
+                    f"FAB ID or version mismatch: Expected FAB ID '{this_fab_id}' and "
+                    f"FAB version '{this_fab_version}', but received FAB ID '{fab_id}' "
+                    f"and FAB version '{fab_version}'.",
+                ) from None
+
+            # log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
+
+            # Set app reference
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
+        # If multi-app feature is enabled
+        else:
+            try:
+                runtime_app_dir = get_project_dir(
+                    fab_id, fab_version, get_flwr_dir(flwr_dir)
+                )
+                config = get_project_config(runtime_app_dir)
+            except Exception as e:
+                raise LoadClientAppError("Failed to load ClientApp") from e
+
+            # Set app reference
+            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
+
+        # Load ClientApp
+        log(
+            DEBUG,
+            "Loading ClientApp `%s`",
+            client_app_ref,
+        )
+        client_app = load_app(client_app_ref, LoadClientAppError, runtime_app_dir)
+
+        if not isinstance(client_app, ClientApp):
+            raise LoadClientAppError(
+                f"Attribute {client_app_ref} is not of type {ClientApp}",
+            ) from None
+
+        return client_app
+
+    return _load

flwr/client/rest_client/connection.py

@@ -46,6 +46,7 @@ from flwr.common.serde import (
     user_config_from_proto,
 )
 from flwr.common.typing import Fab, Run
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
@@ -74,6 +75,7 @@ PATH_PULL_TASK_INS: str = "api/v0/fleet/pull-task-ins"
 PATH_PUSH_TASK_RES: str = "api/v0/fleet/push-task-res"
 PATH_PING: str = "api/v0/fleet/ping"
 PATH_GET_RUN: str = "/api/v0/fleet/get-run"
+PATH_GET_FAB: str = "/api/v0/fleet/get-fab"
 
 T = TypeVar("T", bound=GrpcMessage)
 
@@ -358,18 +360,29 @@ def http_request_response(  # pylint: disable=,R0913, R0914, R0915
         # Send the request
         res = _request(req, GetRunResponse, PATH_GET_RUN)
         if res is None:
-            return Run(run_id, "", "", {})
+            return Run(run_id, "", "", "", {})
 
         return Run(
             run_id,
             res.run.fab_id,
             res.run.fab_version,
+            res.run.fab_hash,
             user_config_from_proto(res.run.override_config),
         )
 
     def get_fab(fab_hash: str) -> Fab:
-        # Call FleetAPI
-        raise NotImplementedError
+        # Construct the request
+        req = GetFabRequest(hash_str=fab_hash)
+
+        # Send the request
+        res = _request(req, GetFabResponse, PATH_GET_FAB)
+        if res is None:
+            return Fab("", b"")
+
+        return Fab(
+            res.fab.hash_str,
+            res.fab.content,
+        )
 
     try:
         # Yield methods

flwr/client/supernode/app.py

@@ -18,7 +18,7 @@ import argparse
 import sys
 from logging import DEBUG, INFO, WARN
 from pathlib import Path
-from typing import Callable, Optional, Tuple
+from typing import Optional, Tuple
 
 from cryptography.exceptions import UnsupportedAlgorithm
 from cryptography.hazmat.primitives.asymmetric import ec
@@ -27,15 +27,8 @@ from cryptography.hazmat.primitives.serialization import (
     load_ssh_public_key,
 )
 
-from flwr.client.client_app import ClientApp, LoadClientAppError
 from flwr.common import EventType, event
-from flwr.common.config import (
-    get_flwr_dir,
-    get_metadata_from_config,
-    get_project_config,
-    get_project_dir,
-    parse_config_args,
-)
+from flwr.common.config import parse_config_args
 from flwr.common.constant import (
     TRANSPORT_TYPE_GRPC_ADAPTER,
     TRANSPORT_TYPE_GRPC_RERE,
@@ -43,9 +36,10 @@ from flwr.common.constant import (
 )
 from flwr.common.exit_handlers import register_exit_handlers
 from flwr.common.logger import log, warn_deprecated_feature
-from flwr.common.object_ref import load_app, validate
 
-from ..app import _start_client_internal
+from ..app import start_client_internal
+from ..process.process import run_clientapp
+from ..process.utils import get_load_client_app_fn
 
 ADDRESS_FLEET_API_GRPC_RERE = "0.0.0.0:9092"
 
@@ -61,7 +55,7 @@ def run_supernode() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(
+    load_fn = get_load_client_app_fn(
         default_app_ref="",
         app_path=args.app,
         flwr_dir=args.flwr_dir,
@@ -69,7 +63,7 @@ def run_supernode() -> None:
     )
     authentication_keys = _try_setup_client_authentication(args)
 
-    _start_client_internal(
+    start_client_internal(
         server_address=args.superlink,
         load_client_app_fn=load_fn,
         transport=args.transport,
@@ -79,7 +73,8 @@ def run_supernode() -> None:
         max_retries=args.max_retries,
         max_wait_time=args.max_wait_time,
         node_config=parse_config_args([args.node_config]),
-        flwr_path=get_flwr_dir(args.flwr_dir),
+        isolate=args.isolate,
+        supernode_address=args.supernode_address,
     )
 
     # Graceful shutdown
@@ -99,14 +94,14 @@ def run_client_app() -> None:
     _warn_deprecated_server_arg(args)
 
     root_certificates = _get_certificates(args)
-    load_fn = _get_load_client_app_fn(
+    load_fn = get_load_client_app_fn(
         default_app_ref=getattr(args, "client-app"),
         app_path=args.dir,
         multi_app=False,
     )
     authentication_keys = _try_setup_client_authentication(args)
 
-    _start_client_internal(
+    start_client_internal(
         server_address=args.superlink,
         node_config=parse_config_args([args.node_config]),
         load_client_app_fn=load_fn,
@@ -128,7 +123,7 @@ def flwr_clientapp() -> None:
         description="Run a Flower ClientApp",
     )
     parser.add_argument(
-        "--address",
+        "--supernode",
         help="Address of SuperNode ClientAppIo gRPC servicer",
     )
     parser.add_argument(
@@ -140,9 +135,10 @@ def flwr_clientapp() -> None:
         DEBUG,
         "Staring isolated `ClientApp` connected to SuperNode ClientAppIo at %s "
         "with the token %s",
-        args.address,
+        args.supernode,
         args.token,
     )
+    run_clientapp(supernode=args.supernode, token=int(args.token))
 
 
 def _warn_deprecated_server_arg(args: argparse.Namespace) -> None:
@@ -200,85 +196,6 @@ def _get_certificates(args: argparse.Namespace) -> Optional[bytes]:
     return root_certificates
 
 
-def _get_load_client_app_fn(
-    default_app_ref: str,
-    app_path: Optional[str],
-    multi_app: bool,
-    flwr_dir: Optional[str] = None,
-) -> Callable[[str, str], ClientApp]:
-    """Get the load_client_app_fn function.
-
-    If `multi_app` is True, this function loads the specified ClientApp
-    based on `fab_id` and `fab_version`. If `fab_id` is empty, a default
-    ClientApp will be loaded.
-
-    If `multi_app` is False, it ignores `fab_id` and `fab_version` and
-    loads a default ClientApp.
-    """
-    if not multi_app:
-        log(
-            DEBUG,
-            "Flower SuperNode will load and validate ClientApp `%s`",
-            default_app_ref,
-        )
-
-        valid, error_msg = validate(default_app_ref, project_dir=app_path)
-        if not valid and error_msg:
-            raise LoadClientAppError(error_msg) from None
-
-    def _load(fab_id: str, fab_version: str) -> ClientApp:
-        runtime_app_dir = Path(app_path if app_path else "").absolute()
-        # If multi-app feature is disabled
-        if not multi_app:
-            # Set app reference
-            client_app_ref = default_app_ref
-        # If multi-app feature is enabled but app directory is provided
-        elif app_path is not None:
-            config = get_project_config(runtime_app_dir)
-            this_fab_version, this_fab_id = get_metadata_from_config(config)
-
-            if this_fab_version != fab_version or this_fab_id != fab_id:
-                raise LoadClientAppError(
-                    f"FAB ID or version mismatch: Expected FAB ID '{this_fab_id}' and "
-                    f"FAB version '{this_fab_version}', but received FAB ID '{fab_id}' "
-                    f"and FAB version '{fab_version}'.",
-                ) from None
-
-            # log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.")
-
-            # Set app reference
-            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
-        # If multi-app feature is enabled
-        else:
-            try:
-                runtime_app_dir = get_project_dir(
-                    fab_id, fab_version, get_flwr_dir(flwr_dir)
-                )
-                config = get_project_config(runtime_app_dir)
-            except Exception as e:
-                raise LoadClientAppError("Failed to load ClientApp") from e
-
-            # Set app reference
-            client_app_ref = config["tool"]["flwr"]["app"]["components"]["clientapp"]
-
-        # Load ClientApp
-        log(
-            DEBUG,
-            "Loading ClientApp `%s`",
-            client_app_ref,
-        )
-        client_app = load_app(client_app_ref, LoadClientAppError, runtime_app_dir)
-
-        if not isinstance(client_app, ClientApp):
-            raise LoadClientAppError(
-                f"Attribute {client_app_ref} is not of type {ClientApp}",
-            ) from None
-
-        return client_app
-
-    return _load
-
-
 def _parse_args_run_supernode() -> argparse.ArgumentParser:
     """Parse flower-supernode command line arguments."""
     parser = argparse.ArgumentParser(
@@ -308,6 +225,17 @@ def _parse_args_run_supernode() -> argparse.ArgumentParser:
         - `$HOME/.flwr/` in all other cases
     """,
     )
+    parser.add_argument(
+        "--isolate",
+        action="store_true",
+        help="Run `ClientApp` in an isolated subprocess. By default, `ClientApp` "
+        "runs in the same process that executes the SuperNode.",
+    )
+    parser.add_argument(
+        "--supernode-address",
+        default="0.0.0.0:9094",
+        help="Set the SuperNode gRPC server address. Defaults to `0.0.0.0:9094`.",
+    )
 
     return parser
 

flwr/common/config.py

@@ -74,10 +74,15 @@ def get_project_config(project_dir: Union[str, Path]) -> Dict[str, Any]:
     return config
 
 
-def _fuse_dicts(
+def fuse_dicts(
     main_dict: UserConfig,
     override_dict: UserConfig,
 ) -> UserConfig:
+    """Merge a config with the overrides.
+
+    Remove the nesting by adding the nested keys as prefixes separated by dots, and fuse
+    it with the override dict.
+    """
     fused_dict = main_dict.copy()
 
     for key, value in override_dict.items():
@@ -96,7 +101,7 @@ def get_fused_config_from_dir(
     )
     flat_default_config = flatten_dict(default_config)
 
-    return _fuse_dicts(flat_default_config, override_config)
+    return fuse_dicts(flat_default_config, override_config)
 
 
 def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> UserConfig:

flwr/common/record/recordset.py

@@ -15,8 +15,10 @@
 """RecordSet."""
 
 
+from __future__ import annotations
+
 from dataclasses import dataclass
-from typing import Dict, Optional, cast
+from typing import cast
 
 from .configsrecord import ConfigsRecord
 from .metricsrecord import MetricsRecord
@@ -34,9 +36,9 @@ class RecordSetData:
 
     def __init__(
         self,
-        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
-        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
-        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+        parameters_records: dict[str, ParametersRecord] | None = None,
+        metrics_records: dict[str, MetricsRecord] | None = None,
+        configs_records: dict[str, ConfigsRecord] | None = None,
     ) -> None:
         self.parameters_records = TypedDict[str, ParametersRecord](
             self._check_fn_str, self._check_fn_params
@@ -88,9 +90,9 @@ class RecordSet:
 
     def __init__(
         self,
-        parameters_records: Optional[Dict[str, ParametersRecord]] = None,
-        metrics_records: Optional[Dict[str, MetricsRecord]] = None,
-        configs_records: Optional[Dict[str, ConfigsRecord]] = None,
+        parameters_records: dict[str, ParametersRecord] | None = None,
+        metrics_records: dict[str, MetricsRecord] | None = None,
+        configs_records: dict[str, ConfigsRecord] | None = None,
     ) -> None:
         data = RecordSetData(
             parameters_records=parameters_records,

flwr/common/record/typeddict.py

@@ -15,99 +15,61 @@
 """Typed dict base class for *Records."""
 
 
-from typing import Any, Callable, Dict, Generic, Iterator, Tuple, TypeVar, cast
+from typing import Callable, Dict, Generic, Iterator, MutableMapping, TypeVar, cast
 
 K = TypeVar("K")  # Key type
 V = TypeVar("V")  # Value type
 
 
-class TypedDict(Generic[K, V]):
+class TypedDict(MutableMapping[K, V], Generic[K, V]):
     """Typed dictionary."""
 
     def __init__(
         self, check_key_fn: Callable[[K], None], check_value_fn: Callable[[V], None]
     ):
-        self._data: Dict[K, V] = {}
-        self._check_key_fn = check_key_fn
-        self._check_value_fn = check_value_fn
+        self.__dict__["_check_key_fn"] = check_key_fn
+        self.__dict__["_check_value_fn"] = check_value_fn
+        self.__dict__["_data"] = {}
 
     def __setitem__(self, key: K, value: V) -> None:
         """Set the given key to the given value after type checking."""
         # Check the types of key and value
-        self._check_key_fn(key)
-        self._check_value_fn(value)
+        cast(Callable[[K], None], self.__dict__["_check_key_fn"])(key)
+        cast(Callable[[V], None], self.__dict__["_check_value_fn"])(value)
+
         # Set key-value pair
-        self._data[key] = value
+        cast(Dict[K, V], self.__dict__["_data"])[key] = value
 
     def __delitem__(self, key: K) -> None:
         """Remove the item with the specified key."""
-        del self._data[key]
+        del cast(Dict[K, V], self.__dict__["_data"])[key]
 
     def __getitem__(self, item: K) -> V:
         """Return the value for the specified key."""
-        return self._data[item]
+        return cast(Dict[K, V], self.__dict__["_data"])[item]
 
     def __iter__(self) -> Iterator[K]:
         """Yield an iterator over the keys of the dictionary."""
-        return iter(self._data)
+        return iter(cast(Dict[K, V], self.__dict__["_data"]))
 
     def __repr__(self) -> str:
         """Return a string representation of the dictionary."""
-        return self._data.__repr__()
+        return cast(Dict[K, V], self.__dict__["_data"]).__repr__()
 
     def __len__(self) -> int:
         """Return the number of items in the dictionary."""
-        return len(self._data)
+        return len(cast(Dict[K, V], self.__dict__["_data"]))
 
-    def __contains__(self, key: K) -> bool:
+    def __contains__(self, key: object) -> bool:
         """Check if the dictionary contains the specified key."""
-        return key in self._data
+        return key in cast(Dict[K, V], self.__dict__["_data"])
 
     def __eq__(self, other: object) -> bool:
         """Compare this instance to another dictionary or TypedDict."""
+        data = cast(Dict[K, V], self.__dict__["_data"])
         if isinstance(other, TypedDict):
-            return self._data == other._data
+            other_data = cast(Dict[K, V], other.__dict__["_data"])
+            return data == other_data
         if isinstance(other, dict):
-            return self._data == other
+            return data == other
         return NotImplemented
-
-    def items(self) -> Iterator[Tuple[K, V]]:
-        """R.items() -> a set-like object providing a view on R's items."""
-        return cast(Iterator[Tuple[K, V]], self._data.items())
-
-    def keys(self) -> Iterator[K]:
-        """R.keys() -> a set-like object providing a view on R's keys."""
-        return cast(Iterator[K], self._data.keys())
-
-    def values(self) -> Iterator[V]:
-        """R.values() -> an object providing a view on R's values."""
-        return cast(Iterator[V], self._data.values())
-
-    def update(self, *args: Any, **kwargs: Any) -> None:
-        """R.update([E, ]**F) -> None.
-
-        Update R from dict/iterable E and F.
-        """
-        for key, value in dict(*args, **kwargs).items():
-            self[key] = value
-
-    def pop(self, key: K) -> V:
-        """R.pop(k[,d]) -> v, remove specified key and return the corresponding value.
-
-        If key is not found, d is returned if given, otherwise KeyError is raised.
-        """
-        return self._data.pop(key)
-
-    def get(self, key: K, default: V) -> V:
-        """R.get(k[,d]) -> R[k] if k in R, else d.
-
-        d defaults to None.
-        """
-        return self._data.get(key, default)
-
-    def clear(self) -> None:
-        """R.clear() -> None.
-
-        Remove all items from R.
-        """
-        self._data.clear()

flwr/common/recordset_compat.py

@@ -145,7 +145,7 @@ def _recordset_to_fit_or_evaluate_ins_components(
     # get config dict
     config_record = recordset.configs_records[f"{ins_str}.config"]
     # pylint: disable-next=protected-access
-    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     return parameters, config_dict
 
@@ -213,7 +213,7 @@ def recordset_to_fitres(recordset: RecordSet, keep_input: bool) -> FitRes:
     )
     configs_record = recordset.configs_records[f"{ins_str}.metrics"]
     # pylint: disable-next=protected-access
-    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record._data)
+    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record)
     status = _extract_status_from_recordset(ins_str, recordset)
 
     return FitRes(
@@ -274,7 +274,7 @@ def recordset_to_evaluateres(recordset: RecordSet) -> EvaluateRes:
     configs_record = recordset.configs_records[f"{ins_str}.metrics"]
 
     # pylint: disable-next=protected-access
-    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record._data)
+    metrics = _check_mapping_from_recordscalartype_to_scalar(configs_record)
     status = _extract_status_from_recordset(ins_str, recordset)
 
     return EvaluateRes(
@@ -314,7 +314,7 @@ def recordset_to_getparametersins(recordset: RecordSet) -> GetParametersIns:
     """Derive GetParametersIns from a RecordSet object."""
     config_record = recordset.configs_records["getparametersins.config"]
     # pylint: disable-next=protected-access
-    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     return GetParametersIns(config=config_dict)
 
@@ -365,7 +365,7 @@ def recordset_to_getpropertiesins(recordset: RecordSet) -> GetPropertiesIns:
     """Derive GetPropertiesIns from a RecordSet object."""
     config_record = recordset.configs_records["getpropertiesins.config"]
     # pylint: disable-next=protected-access
-    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    config_dict = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     return GetPropertiesIns(config=config_dict)
 
@@ -384,7 +384,7 @@ def recordset_to_getpropertiesres(recordset: RecordSet) -> GetPropertiesRes:
     res_str = "getpropertiesres"
     config_record = recordset.configs_records[f"{res_str}.properties"]
     # pylint: disable-next=protected-access
-    properties = _check_mapping_from_recordscalartype_to_scalar(config_record._data)
+    properties = _check_mapping_from_recordscalartype_to_scalar(config_record)
 
     status = _extract_status_from_recordset(res_str, recordset=recordset)
 

flwr/common/serde.py

@@ -22,6 +22,7 @@ from google.protobuf.message import Message as GrpcMessage
 # pylint: disable=E0611
 from flwr.proto.clientappio_pb2 import ClientAppOutputCode, ClientAppOutputStatus
 from flwr.proto.error_pb2 import Error as ProtoError
+from flwr.proto.fab_pb2 import Fab as ProtoFab
 from flwr.proto.message_pb2 import Context as ProtoContext
 from flwr.proto.message_pb2 import Message as ProtoMessage
 from flwr.proto.message_pb2 import Metadata as ProtoMetadata
@@ -686,6 +687,19 @@ def message_from_taskres(taskres: TaskRes) -> Message:
     return message
 
 
+# === FAB ===
+
+
+def fab_to_proto(fab: typing.Fab) -> ProtoFab:
+    """Create a proto Fab object from a Python Fab."""
+    return ProtoFab(hash_str=fab.hash_str, content=fab.content)
+
+
+def fab_from_proto(fab: ProtoFab) -> typing.Fab:
+    """Create a Python Fab object from a proto Fab."""
+    return typing.Fab(fab.hash_str, fab.content)
+
+
 # === User configs ===
 
 
@@ -745,6 +759,7 @@ def metadata_to_proto(metadata: Metadata) -> ProtoMetadata:
         group_id=metadata.group_id,
         ttl=metadata.ttl,
         message_type=metadata.message_type,
+        created_at=metadata.created_at,
     )
     return proto
 
@@ -771,7 +786,9 @@ def message_to_proto(message: Message) -> ProtoMessage:
     """Serialize `Message` to ProtoBuf."""
     proto = ProtoMessage(
         metadata=metadata_to_proto(message.metadata),
-        content=recordset_to_proto(message.content),
+        content=(
+            recordset_to_proto(message.content) if message.has_content() else None
+        ),
         error=error_to_proto(message.error) if message.has_error() else None,
     )
     return proto
@@ -779,6 +796,7 @@ def message_to_proto(message: Message) -> ProtoMessage:
 
 def message_from_proto(message_proto: ProtoMessage) -> Message:
     """Deserialize `Message` from ProtoBuf."""
+    created_at = message_proto.metadata.created_at
     message = Message(
         metadata=metadata_from_proto(message_proto.metadata),
         content=(
@@ -792,6 +810,9 @@ def message_from_proto(message_proto: ProtoMessage) -> Message:
             else None
         ),
     )
+    # `.created_at` is set upon Message object construction
+    # we need to manually set it to the original value
+    message.metadata.created_at = created_at
     return message
 
 
@@ -829,8 +850,8 @@ def run_to_proto(run: typing.Run) -> ProtoRun:
         run_id=run.run_id,
         fab_id=run.fab_id,
         fab_version=run.fab_version,
+        fab_hash=run.fab_hash,
         override_config=user_config_to_proto(run.override_config),
-        fab_hash="",
     )
     return proto
 
@@ -841,6 +862,7 @@ def run_from_proto(run_proto: ProtoRun) -> typing.Run:
         run_id=run_proto.run_id,
         fab_id=run_proto.fab_id,
         fab_version=run_proto.fab_version,
+        fab_hash=run_proto.fab_hash,
         override_config=user_config_from_proto(run_proto.override_config),
     )
     return run