flwr-nightly 1.11.0.dev20240811__py3-none-any.whl → 1.11.0.dev20240821__py3-none-any.whl

flwr/server/driver/grpc_driver.py

@@ -131,6 +131,7 @@ class GrpcDriver(Driver):
             run_id=res.run.run_id,
             fab_id=res.run.fab_id,
             fab_version=res.run.fab_version,
+            fab_hash=res.run.fab_hash,
             override_config=user_config_from_proto(res.run.override_config),
         )
 

flwr/server/run_serverapp.py

@@ -21,6 +21,8 @@ from logging import DEBUG, INFO, WARN
 from pathlib import Path
 from typing import Optional
 
+from flwr.cli.config_utils import get_fab_metadata
+from flwr.cli.install import install_from_fab
 from flwr.common import Context, EventType, RecordSet, event
 from flwr.common.config import (
     get_flwr_dir,
@@ -36,6 +38,7 @@ from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
     CreateRunRequest,
     CreateRunResponse,
 )
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 
 from .driver import Driver
 from .driver.grpc_driver import GrpcDriver
@@ -87,7 +90,8 @@ def run(
     log(DEBUG, "ServerApp finished running.")
 
 
-def run_server_app() -> None:  # pylint: disable=too-many-branches
+# pylint: disable-next=too-many-branches,too-many-statements,too-many-locals
+def run_server_app() -> None:
     """Run Flower server app."""
     event(EventType.RUN_SERVER_APP_ENTER)
 
@@ -164,7 +168,19 @@ def run_server_app() -> None:  # pylint: disable=too-many-branches
         )
         flwr_dir = get_flwr_dir(args.flwr_dir)
         run_ = driver.run
-        app_path = str(get_project_dir(run_.fab_id, run_.fab_version, flwr_dir))
+        if run_.fab_hash:
+            fab_req = GetFabRequest(hash_str=run_.fab_hash)
+            # pylint: disable-next=W0212
+            fab_res: GetFabResponse = driver._stub.GetFab(fab_req)
+            if fab_res.fab.hash_str != run_.fab_hash:
+                raise ValueError("FAB hashes don't match.")
+
+            install_from_fab(fab_res.fab.content, flwr_dir, True)
+            fab_id, fab_version = get_fab_metadata(fab_res.fab.content)
+        else:
+            fab_id, fab_version = run_.fab_id, run_.fab_version
+
+        app_path = str(get_project_dir(fab_id, fab_version, flwr_dir))
         config = get_project_config(app_path)
     else:
         # User provided `app_dir`, but not `--run-id`

flwr/server/server.py

@@ -91,7 +91,7 @@ class Server:
         # Initialize parameters
         log(INFO, "[INIT]")
         self.parameters = self._get_initial_parameters(server_round=0, timeout=timeout)
-        log(INFO, "Evaluating initial global parameters")
+        log(INFO, "Starting evaluation of initial global parameters")
         res = self.strategy.evaluate(0, parameters=self.parameters)
         if res is not None:
             log(
@@ -102,6 +102,8 @@ class Server:
             )
             history.add_loss_centralized(server_round=0, loss=res[0])
             history.add_metrics_centralized(server_round=0, metrics=res[1])
+        else:
+            log(INFO, "Evaluation returned no results (`None`)")
 
         # Run federated learning for num_rounds
         start_time = timeit.default_timer()

flwr/server/superlink/driver/driver_grpc.py

@@ -24,6 +24,7 @@ from flwr.common.logger import log
 from flwr.proto.driver_pb2_grpc import (  # pylint: disable=E0611
     add_DriverServicer_to_server,
 )
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.state import StateFactory
 
 from ..fleet.grpc_bidi.grpc_server import generic_create_grpc_server
@@ -33,12 +34,14 @@ from .driver_servicer import DriverServicer
 def run_driver_api_grpc(
     address: str,
     state_factory: StateFactory,
+    ffs_factory: FfsFactory,
     certificates: Optional[Tuple[bytes, bytes, bytes]],
 ) -> grpc.Server:
     """Run Driver API (gRPC, request-response)."""
     # Create Driver API gRPC server
     driver_servicer: grpc.Server = DriverServicer(
         state_factory=state_factory,
+        ffs_factory=ffs_factory,
     )
     driver_add_servicer_to_server_fn = add_DriverServicer_to_server
     driver_grpc_server = generic_create_grpc_server(

flwr/server/superlink/driver/driver_servicer.py

@@ -23,7 +23,13 @@ from uuid import UUID
 import grpc
 
 from flwr.common.logger import log
-from flwr.common.serde import user_config_from_proto, user_config_to_proto
+from flwr.common.serde import (
+    fab_from_proto,
+    fab_to_proto,
+    user_config_from_proto,
+    user_config_to_proto,
+)
+from flwr.common.typing import Fab
 from flwr.proto import driver_pb2_grpc  # pylint: disable=E0611
 from flwr.proto.driver_pb2 import (  # pylint: disable=E0611
     CreateRunRequest,
@@ -43,6 +49,8 @@ from flwr.proto.run_pb2 import (  # pylint: disable=E0611
     Run,
 )
 from flwr.proto.task_pb2 import TaskRes  # pylint: disable=E0611
+from flwr.server.superlink.ffs.ffs import Ffs
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.state import State, StateFactory
 from flwr.server.utils.validator import validate_task_ins_or_res
 
@@ -50,8 +58,9 @@ from flwr.server.utils.validator import validate_task_ins_or_res
 class DriverServicer(driver_pb2_grpc.DriverServicer):
     """Driver API servicer."""
 
-    def __init__(self, state_factory: StateFactory) -> None:
+    def __init__(self, state_factory: StateFactory, ffs_factory: FfsFactory) -> None:
         self.state_factory = state_factory
+        self.ffs_factory = ffs_factory
 
     def GetNodes(
         self, request: GetNodesRequest, context: grpc.ServicerContext
@@ -71,9 +80,20 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
         """Create run ID."""
         log(DEBUG, "DriverServicer.CreateRun")
         state: State = self.state_factory.state()
+        if request.HasField("fab"):
+            fab = fab_from_proto(request.fab)
+            ffs: Ffs = self.ffs_factory.ffs()
+            fab_hash = ffs.put(fab.content, {})
+            _raise_if(
+                fab_hash != fab.hash_str,
+                f"FAB ({fab.hash_str}) hash from request doesn't match contents",
+            )
+        else:
+            fab_hash = ""
         run_id = state.create_run(
             request.fab_id,
             request.fab_version,
+            fab_hash,
             user_config_from_proto(request.override_config),
         )
         return CreateRunResponse(run_id=run_id)
@@ -161,14 +181,22 @@ class DriverServicer(driver_pb2_grpc.DriverServicer):
                 fab_id=run.fab_id,
                 fab_version=run.fab_version,
                 override_config=user_config_to_proto(run.override_config),
+                fab_hash=run.fab_hash,
             )
         )
 
     def GetFab(
         self, request: GetFabRequest, context: grpc.ServicerContext
     ) -> GetFabResponse:
-        """Will be implemented later."""
-        raise NotImplementedError
+        """Get FAB from Ffs."""
+        log(DEBUG, "DriverServicer.GetFab")
+
+        ffs: Ffs = self.ffs_factory.ffs()
+        if result := ffs.get(request.hash_str):
+            fab = Fab(request.hash_str, result[0])
+            return GetFabResponse(fab=fab_to_proto(fab))
+
+        raise ValueError(f"Found no FAB with hash: {request.hash_str}")
 
 
 def _raise_if(validation_error: bool, detail: str) -> None:

flwr/server/superlink/ffs/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower File Storage for large objects."""
+
+
+from .disk_ffs import DiskFfs as DiskFfs
+from .ffs import Ffs as Ffs
+
+__all__ = [
+    "DiskFfs",
+    "Ffs",
+]

flwr/server/superlink/ffs/disk_ffs.py

@@ -0,0 +1,107 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Disk based Flower File Storage."""
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from flwr.server.superlink.ffs.ffs import Ffs
+
+
+class DiskFfs(Ffs):  # pylint: disable=R0904
+    """Disk-based Flower File Storage interface for large objects."""
+
+    def __init__(self, base_dir: str) -> None:
+        """Create a new DiskFfs instance.
+
+        Parameters
+        ----------
+        base_dir : str
+            The base directory to store the objects.
+        """
+        self.base_dir = Path(base_dir)
+
+    def put(self, content: bytes, meta: Dict[str, str]) -> str:
+        """Store bytes and metadata and return key (hash of content).
+
+        Parameters
+        ----------
+        content : bytes
+            The content to be stored.
+        meta : Dict[str, str]
+            The metadata to be stored.
+
+        Returns
+        -------
+        key : str
+            The key (sha256hex hash) of the content.
+        """
+        content_hash = hashlib.sha256(content).hexdigest()
+
+        self.base_dir.mkdir(exist_ok=True, parents=True)
+        (self.base_dir / content_hash).write_bytes(content)
+        (self.base_dir / f"{content_hash}.META").write_text(json.dumps(meta))
+
+        return content_hash
+
+    def get(self, key: str) -> Optional[Tuple[bytes, Dict[str, str]]]:
+        """Return tuple containing the object content and metadata.
+
+        Parameters
+        ----------
+        key : str
+            The sha256hex hash of the object to be retrieved.
+
+        Returns
+        -------
+        Optional[Tuple[bytes, Dict[str, str]]]
+            A tuple containing the object content and metadata.
+        """
+        if not (self.base_dir / key).exists():
+            return None
+
+        content = (self.base_dir / key).read_bytes()
+        meta = json.loads((self.base_dir / f"{key}.META").read_text())
+
+        return content, meta
+
+    def delete(self, key: str) -> None:
+        """Delete object with hash.
+
+        Parameters
+        ----------
+        key : str
+            The sha256hex hash of the object to be deleted.
+        """
+        (self.base_dir / key).unlink()
+        (self.base_dir / f"{key}.META").unlink()
+
+    def list(self) -> List[str]:
+        """List all keys.
+
+        Return all available keys in this `Ffs` instance.
+        This can be combined with, for example,
+        the `delete` method to delete objects.
+
+        Returns
+        -------
+        List[str]
+            A list of all available keys.
+        """
+        return [
+            item.name for item in self.base_dir.iterdir() if not item.suffix == ".META"
+        ]

flwr/server/superlink/ffs/ffs.py

@@ -0,0 +1,79 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Abstract base class for Flower File Storage interface."""
+
+
+import abc
+from typing import Dict, List, Optional, Tuple
+
+
+class Ffs(abc.ABC):  # pylint: disable=R0904
+    """Abstract Flower File Storage interface for large objects."""
+
+    @abc.abstractmethod
+    def put(self, content: bytes, meta: Dict[str, str]) -> str:
+        """Store bytes and metadata and return sha256hex hash of data as str.
+
+        Parameters
+        ----------
+        content : bytes
+            The content to be stored.
+        meta : Dict[str, str]
+            The metadata to be stored.
+
+        Returns
+        -------
+        key : str
+            The key (sha256hex hash) of the content.
+        """
+
+    @abc.abstractmethod
+    def get(self, key: str) -> Optional[Tuple[bytes, Dict[str, str]]]:
+        """Return tuple containing the object content and metadata.
+
+        Parameters
+        ----------
+        key : str
+            The key (sha256hex hash) of the object to be retrieved.
+
+        Returns
+        -------
+        Optional[Tuple[bytes, Dict[str, str]]]
+            A tuple containing the object content and metadata.
+        """
+
+    @abc.abstractmethod
+    def delete(self, key: str) -> None:
+        """Delete object with hash.
+
+        Parameters
+        ----------
+        key : str
+            The key (sha256hex hash) of the object to be deleted.
+        """
+
+    @abc.abstractmethod
+    def list(self) -> List[str]:
+        """List keys of all stored objects.
+
+        Return all available keys in this `Ffs` instance.
+        This can be combined with, for example,
+        the `delete` method to delete objects.
+
+        Returns
+        -------
+        List[str]
+            A list of all available keys.
+        """

flwr/server/superlink/ffs/ffs_factory.py

@@ -0,0 +1,47 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Factory class that creates Ffs instances."""
+
+
+from logging import DEBUG
+from typing import Optional
+
+from flwr.common.logger import log
+
+from .disk_ffs import DiskFfs
+from .ffs import Ffs
+
+
+class FfsFactory:
+    """Factory class that creates Ffs instances.
+
+    Parameters
+    ----------
+    base_dir : str
+        The base directory used by DiskFfs to store objects.
+    """
+
+    def __init__(self, base_dir: str) -> None:
+        self.base_dir = base_dir
+        self.ffs_instance: Optional[Ffs] = None
+
+    def ffs(self) -> Ffs:
+        """Return a Ffs instance and create it, if necessary."""
+        if not self.ffs_instance:
+            log(DEBUG, "Initializing DiskFfs")
+            self.ffs_instance = DiskFfs(self.base_dir)
+
+        log(DEBUG, "Using DiskFfs")
+        return self.ffs_instance

flwr/server/superlink/fleet/grpc_rere/fleet_servicer.py

@@ -35,6 +35,7 @@ from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     PushTaskResResponse,
 )
 from flwr.proto.run_pb2 import GetRunRequest, GetRunResponse  # pylint: disable=E0611
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.fleet.message_handler import message_handler
 from flwr.server.superlink.state import StateFactory
 
@@ -42,18 +43,21 @@ from flwr.server.superlink.state import StateFactory
 class FleetServicer(fleet_pb2_grpc.FleetServicer):
     """Fleet API servicer."""
 
-    def __init__(self, state_factory: StateFactory) -> None:
+    def __init__(self, state_factory: StateFactory, ffs_factory: FfsFactory) -> None:
         self.state_factory = state_factory
+        self.ffs_factory = ffs_factory
 
     def CreateNode(
         self, request: CreateNodeRequest, context: grpc.ServicerContext
     ) -> CreateNodeResponse:
         """."""
         log(INFO, "FleetServicer.CreateNode")
-        return message_handler.create_node(
+        response = message_handler.create_node(
             request=request,
             state=self.state_factory.state(),
         )
+        log(INFO, "FleetServicer: Created node_id=%s", response.node.node_id)
+        return response
 
     def DeleteNode(
         self, request: DeleteNodeRequest, context: grpc.ServicerContext
@@ -106,5 +110,9 @@ class FleetServicer(fleet_pb2_grpc.FleetServicer):
     def GetFab(
         self, request: GetFabRequest, context: grpc.ServicerContext
     ) -> GetFabResponse:
-        """Will be implemented later."""
-        raise NotImplementedError
+        """Get FAB."""
+        log(DEBUG, "DriverServicer.GetFab")
+        return message_handler.get_fab(
+            request=request,
+            ffs=self.ffs_factory.ffs(),
+        )

flwr/server/superlink/fleet/grpc_rere/server_interceptor.py

@@ -16,7 +16,7 @@
 
 
 import base64
-from logging import WARNING
+from logging import INFO, WARNING
 from typing import Any, Callable, Optional, Sequence, Tuple, Union
 
 import grpc
@@ -128,9 +128,15 @@ class AuthenticateServerInterceptor(grpc.ServerInterceptor):  # type: ignore
                 context.abort(grpc.StatusCode.UNAUTHENTICATED, "Access denied")
 
             if isinstance(request, CreateNodeRequest):
-                return self._create_authenticated_node(
+                response = self._create_authenticated_node(
                     client_public_key_bytes, request, context
                 )
+                log(
+                    INFO,
+                    "AuthenticateServerInterceptor: Created node_id=%s",
+                    response.node.node_id,
+                )
+                return response
 
             # Verify hmac value
             hmac_value = base64.urlsafe_b64decode(

flwr/server/superlink/fleet/message_handler/message_handler.py

@@ -19,7 +19,9 @@ import time
 from typing import List, Optional
 from uuid import UUID
 
-from flwr.common.serde import user_config_to_proto
+from flwr.common.serde import fab_to_proto, user_config_to_proto
+from flwr.common.typing import Fab
+from flwr.proto.fab_pb2 import GetFabRequest, GetFabResponse  # pylint: disable=E0611
 from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
@@ -40,6 +42,7 @@ from flwr.proto.run_pb2 import (  # pylint: disable=E0611
     Run,
 )
 from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
+from flwr.server.superlink.ffs.ffs import Ffs
 from flwr.server.superlink.state import State
 
 
@@ -124,5 +127,17 @@ def get_run(
             fab_id=run.fab_id,
             fab_version=run.fab_version,
             override_config=user_config_to_proto(run.override_config),
+            fab_hash=run.fab_hash,
         )
     )
+
+
+def get_fab(
+    request: GetFabRequest, ffs: Ffs  # pylint: disable=W0613
+) -> GetFabResponse:
+    """Get FAB."""
+    if result := ffs.get(request.hash_str):
+        fab = Fab(request.hash_str, result[0])
+        return GetFabResponse(fab=fab_to_proto(fab))
+
+    raise ValueError(f"Found no FAB with hash: {request.hash_str}")

flwr/server/superlink/fleet/vce/backend/raybackend.py

@@ -14,6 +14,7 @@
 # ==============================================================================
 """Ray backend for the Fleet API using the Simulation Engine."""
 
+import sys
 from logging import DEBUG, ERROR
 from typing import Callable, Dict, Tuple, Union
 
@@ -111,8 +112,10 @@ class RayBackend(Backend):
             if backend_config.get(self.init_args_key):
                 for k, v in backend_config[self.init_args_key].items():
                     ray_init_args[k] = v
-
-            ray.init(**ray_init_args)
+            ray.init(
+                runtime_env={"env_vars": {"PYTHONPATH": ":".join(sys.path)}},
+                **ray_init_args,
+            )
 
     @property
     def num_workers(self) -> int:

flwr/server/superlink/fleet/vce/vce_api.py

@@ -27,8 +27,8 @@ from time import sleep
 from typing import Callable, Dict, Optional
 
 from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError
+from flwr.client.clientapp.utils import get_load_client_app_fn
 from flwr.client.node_state import NodeState
-from flwr.client.supernode.app import _get_load_client_app_fn
 from flwr.common.constant import (
     NUM_PARTITIONS_KEY,
     PARTITION_ID_KEY,
@@ -345,7 +345,7 @@ def start_vce(
     def _load() -> ClientApp:
 
         if client_app_attr:
-            app = _get_load_client_app_fn(
+            app = get_load_client_app_fn(
                 default_app_ref=client_app_attr,
                 app_path=app_dir,
                 flwr_dir=flwr_dir,

flwr/server/superlink/state/in_memory_state.py

@@ -277,11 +277,12 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
 
     def create_run(
         self,
-        fab_id: str,
-        fab_version: str,
+        fab_id: Optional[str],
+        fab_version: Optional[str],
+        fab_hash: Optional[str],
         override_config: UserConfig,
     ) -> int:
-        """Create a new run for the specified `fab_id` and `fab_version`."""
+        """Create a new run for the specified `fab_hash`."""
         # Sample a random int64 as run_id
         with self.lock:
             run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES)
@@ -289,8 +290,9 @@ class InMemoryState(State):  # pylint: disable=R0902,R0904
             if run_id not in self.run_ids:
                 self.run_ids[run_id] = Run(
                     run_id=run_id,
-                    fab_id=fab_id,
-                    fab_version=fab_version,
+                    fab_id=fab_id if fab_id else "",
+                    fab_version=fab_version if fab_version else "",
+                    fab_hash=fab_hash if fab_hash else "",
                     override_config=override_config,
                 )
                 return run_id

flwr/server/superlink/state/sqlite_state.py

@@ -65,6 +65,7 @@ CREATE TABLE IF NOT EXISTS run(
     run_id                INTEGER UNIQUE,
     fab_id                TEXT,
     fab_version           TEXT,
+    fab_hash              TEXT,
     override_config       TEXT
 );
 """
@@ -617,8 +618,9 @@ class SqliteState(State):  # pylint: disable=R0904
 
     def create_run(
         self,
-        fab_id: str,
-        fab_version: str,
+        fab_id: Optional[str],
+        fab_version: Optional[str],
+        fab_hash: Optional[str],
         override_config: UserConfig,
     ) -> int:
         """Create a new run for the specified `fab_id` and `fab_version`."""
@@ -630,12 +632,19 @@ class SqliteState(State):  # pylint: disable=R0904
         # If run_id does not exist
         if self.query(query, (run_id,))[0]["COUNT(*)"] == 0:
             query = (
-                "INSERT INTO run (run_id, fab_id, fab_version, override_config)"
-                "VALUES (?, ?, ?, ?);"
-            )
-            self.query(
-                query, (run_id, fab_id, fab_version, json.dumps(override_config))
+                "INSERT INTO run "
+                "(run_id, fab_id, fab_version, fab_hash, override_config)"
+                "VALUES (?, ?, ?, ?, ?);"
             )
+            if fab_hash:
+                self.query(
+                    query, (run_id, "", "", fab_hash, json.dumps(override_config))
+                )
+            else:
+                self.query(
+                    query,
+                    (run_id, fab_id, fab_version, "", json.dumps(override_config)),
+                )
             return run_id
         log(ERROR, "Unexpected run creation failure.")
         return 0
@@ -702,6 +711,7 @@ class SqliteState(State):  # pylint: disable=R0904
                 run_id=run_id,
                 fab_id=row["fab_id"],
                 fab_version=row["fab_version"],
+                fab_hash=row["fab_hash"],
                 override_config=json.loads(row["override_config"]),
             )
         except sqlite3.IntegrityError: