flwr-nightly 1.13.0.dev20241023__py3-none-any.whl → 1.13.0.dev20241025__py3-none-any.whl

flwr/server/superlink/linkstate/sqlite_linkstate.py

@@ -19,13 +19,14 @@
 import json
 import re
 import sqlite3
+import threading
 import time
 from collections.abc import Sequence
 from logging import DEBUG, ERROR, WARNING
 from typing import Any, Optional, Union, cast
 from uuid import UUID, uuid4
 
-from flwr.common import log, now
+from flwr.common import Context, log, now
 from flwr.common.constant import (
     MESSAGE_TTL_TOLERANCE,
     NODE_ID_NUM_BYTES,
@@ -33,13 +34,19 @@ from flwr.common.constant import (
     Status,
 )
 from flwr.common.typing import Run, RunStatus, UserConfig
-from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
-from flwr.proto.recordset_pb2 import RecordSet  # pylint: disable=E0611
-from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
+
+# pylint: disable=E0611
+from flwr.proto.node_pb2 import Node
+from flwr.proto.recordset_pb2 import RecordSet as ProtoRecordSet
+from flwr.proto.task_pb2 import Task, TaskIns, TaskRes
+
+# pylint: enable=E0611
 from flwr.server.utils.validator import validate_task_ins_or_res
 
 from .linkstate import LinkState
 from .utils import (
+    context_from_bytes,
+    context_to_bytes,
     convert_sint64_to_uint64,
     convert_sint64_values_in_dict_to_uint64,
     convert_uint64_to_sint64,
@@ -92,6 +99,14 @@ CREATE TABLE IF NOT EXISTS run(
 );
 """
 
+SQL_CREATE_TABLE_CONTEXT = """
+CREATE TABLE IF NOT EXISTS context(
+    run_id                INTEGER UNIQUE,
+    context               BLOB,
+    FOREIGN KEY(run_id) REFERENCES run(run_id)
+);
+"""
+
 SQL_CREATE_TABLE_TASK_INS = """
 CREATE TABLE IF NOT EXISTS task_ins(
     task_id                 TEXT UNIQUE,
@@ -152,6 +167,7 @@ class SqliteLinkState(LinkState):  # pylint: disable=R0904
         """
         self.database_path = database_path
         self.conn: Optional[sqlite3.Connection] = None
+        self.lock = threading.RLock()
 
     def initialize(self, log_queries: bool = False) -> list[tuple[str]]:
         """Create tables if they don't exist yet.
@@ -175,6 +191,7 @@ class SqliteLinkState(LinkState):  # pylint: disable=R0904
 
         # Create each table if not exists queries
         cur.execute(SQL_CREATE_TABLE_RUN)
+        cur.execute(SQL_CREATE_TABLE_CONTEXT)
         cur.execute(SQL_CREATE_TABLE_TASK_INS)
         cur.execute(SQL_CREATE_TABLE_TASK_RES)
         cur.execute(SQL_CREATE_TABLE_NODE)
@@ -970,6 +987,34 @@ class SqliteLinkState(LinkState):  # pylint: disable=R0904
             log(ERROR, "`node_id` does not exist.")
             return False
 
+    def get_serverapp_context(self, run_id: int) -> Optional[Context]:
+        """Get the context for the specified `run_id`."""
+        # Retrieve context if any
+        query = "SELECT context FROM context WHERE run_id = ?;"
+        rows = self.query(query, (convert_uint64_to_sint64(run_id),))
+        context = context_from_bytes(rows[0]["context"]) if rows else None
+        return context
+
+    def set_serverapp_context(self, run_id: int, context: Context) -> None:
+        """Set the context for the specified `run_id`."""
+        # Convert context to bytes
+        context_bytes = context_to_bytes(context)
+        sint_run_id = convert_uint64_to_sint64(run_id)
+
+        # Check if any existing Context assigned to the run_id
+        query = "SELECT COUNT(*) FROM context WHERE run_id = ?;"
+        if self.query(query, (sint_run_id,))[0]["COUNT(*)"] > 0:
+            # Update context
+            query = "UPDATE context SET context = ? WHERE run_id = ?;"
+            self.query(query, (context_bytes, sint_run_id))
+        else:
+            try:
+                # Store context
+                query = "INSERT INTO context (run_id, context) VALUES (?, ?);"
+                self.query(query, (sint_run_id, context_bytes))
+            except sqlite3.IntegrityError:
+                raise ValueError(f"Run {run_id} not found") from None
+
     def get_valid_task_ins(self, task_id: str) -> Optional[dict[str, Any]]:
         """Check if the TaskIns exists and is valid (not expired).
 
@@ -1054,7 +1099,7 @@ def task_res_to_dict(task_msg: TaskRes) -> dict[str, Any]:
 
 def dict_to_task_ins(task_dict: dict[str, Any]) -> TaskIns:
     """Turn task_dict into protobuf message."""
-    recordset = RecordSet()
+    recordset = ProtoRecordSet()
     recordset.ParseFromString(task_dict["recordset"])
 
     result = TaskIns(
@@ -1084,7 +1129,7 @@ def dict_to_task_ins(task_dict: dict[str, Any]) -> TaskIns:
 
 def dict_to_task_res(task_dict: dict[str, Any]) -> TaskRes:
     """Turn task_dict into protobuf message."""
-    recordset = RecordSet()
+    recordset = ProtoRecordSet()
     recordset.ParseFromString(task_dict["recordset"])
 
     result = TaskRes(

flwr/server/superlink/linkstate/utils.py

@@ -20,10 +20,11 @@ from logging import ERROR
 from os import urandom
 from uuid import uuid4
 
-from flwr.common import log
+from flwr.common import Context, log, serde
 from flwr.common.constant import ErrorCode, Status, SubStatus
 from flwr.common.typing import RunStatus
 from flwr.proto.error_pb2 import Error  # pylint: disable=E0611
+from flwr.proto.message_pb2 import Context as ProtoContext  # pylint: disable=E0611
 from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
 from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
 
@@ -135,6 +136,16 @@ def convert_sint64_values_in_dict_to_uint64(
             data_dict[key] = convert_sint64_to_uint64(data_dict[key])
 
 
+def context_to_bytes(context: Context) -> bytes:
+    """Serialize `Context` to bytes."""
+    return serde.context_to_proto(context).SerializeToString()
+
+
+def context_from_bytes(context_bytes: bytes) -> Context:
+    """Deserialize `Context` from bytes."""
+    return serde.context_from_proto(ProtoContext.FromString(context_bytes))
+
+
 def make_node_unavailable_taskres(ref_taskins: TaskIns) -> TaskRes:
     """Generate a TaskRes with a node unavailable error from a TaskIns."""
     current_time = time.time()

flwr/simulation/run_simulation.py

@@ -29,7 +29,7 @@ from typing import Any, Optional
 
 from flwr.cli.config_utils import load_and_validate
 from flwr.client import ClientApp
-from flwr.common import EventType, event, log, now
+from flwr.common import Context, EventType, RecordSet, event, log, now
 from flwr.common.config import get_fused_config_from_dir, parse_config_args
 from flwr.common.constant import RUN_ID_NUM_BYTES, Status
 from flwr.common.logger import (
@@ -40,7 +40,7 @@ from flwr.common.logger import (
 )
 from flwr.common.typing import Run, RunStatus, UserConfig
 from flwr.server.driver import Driver, InMemoryDriver
-from flwr.server.run_serverapp import run as run_server_app
+from flwr.server.run_serverapp import run as _run
 from flwr.server.server_app import ServerApp
 from flwr.server.superlink.fleet import vce
 from flwr.server.superlink.fleet.vce.backend.backend import BackendConfig
@@ -333,11 +333,19 @@ def run_serverapp_th(
                 log(INFO, "Enabling GPU growth for Tensorflow on the server thread.")
                 enable_gpu_growth()
 
+            # Initialize Context
+            context = Context(
+                node_id=0,
+                node_config={},
+                state=RecordSet(),
+                run_config=_server_app_run_config,
+            )
+
             # Run ServerApp
-            run_server_app(
+            _run(
                 driver=_driver,
+                context=context,
                 server_app_dir=_server_app_dir,
-                server_app_run_config=_server_app_run_config,
                 server_app_attr=_server_app_attr,
                 loaded_server_app=_server_app,
             )

flwr/superexec/app.py

@@ -16,21 +16,11 @@
 
 import argparse
 import sys
-from logging import INFO, WARN
-from pathlib import Path
-from typing import Optional
+from logging import INFO
 
-import grpc
-
-from flwr.common import EventType, event, log
-from flwr.common.address import parse_address
-from flwr.common.config import parse_config_args
-from flwr.common.constant import EXEC_API_DEFAULT_ADDRESS
-from flwr.common.exit_handlers import register_exit_handlers
-from flwr.common.logger import warn_deprecated_feature
+from flwr.common import log
 from flwr.common.object_ref import load_app, validate
 
-from .exec_grpc import run_superexec_api_grpc
 from .executor import Executor
 
 
@@ -38,137 +28,12 @@ def run_superexec() -> None:
     """Run Flower SuperExec."""
     log(INFO, "Starting Flower SuperExec")
 
-    warn_deprecated_feature(
+    sys.exit(
         "Manually launching the SuperExec is deprecated. Since `flwr 1.13.0` "
         "the executor service runs in the SuperLink. Launching it manually is not "
         "recommended."
     )
 
-    event(EventType.RUN_SUPEREXEC_ENTER)
-
-    args = _parse_args_run_superexec().parse_args()
-
-    # Parse IP address
-    parsed_address = parse_address(args.address)
-    if not parsed_address:
-        sys.exit(f"SuperExec IP address ({args.address}) cannot be parsed.")
-    host, port, is_v6 = parsed_address
-    address = f"[{host}]:{port}" if is_v6 else f"{host}:{port}"
-
-    # Obtain certificates
-    certificates = _try_obtain_certificates(args)
-
-    # Start SuperExec API
-    superexec_server: grpc.Server = run_superexec_api_grpc(
-        address=address,
-        executor=load_executor(args),
-        certificates=certificates,
-        config=parse_config_args(
-            [args.executor_config] if args.executor_config else args.executor_config
-        ),
-    )
-
-    grpc_servers = [superexec_server]
-
-    # Graceful shutdown
-    register_exit_handlers(
-        event_type=EventType.RUN_SUPEREXEC_LEAVE,
-        grpc_servers=grpc_servers,
-        bckg_threads=None,
-    )
-
-    superexec_server.wait_for_termination()
-
-
-def _parse_args_run_superexec() -> argparse.ArgumentParser:
-    """Parse command line arguments for SuperExec."""
-    parser = argparse.ArgumentParser(
-        description="Start a Flower SuperExec",
-    )
-    parser.add_argument(
-        "--address",
-        help="SuperExec (gRPC) server address (IPv4, IPv6, or a domain name)",
-        default=EXEC_API_DEFAULT_ADDRESS,
-    )
-    parser.add_argument(
-        "--executor",
-        help="For example: `deployment:exec` or `project.package.module:wrapper.exec`.",
-        default="flwr.superexec.deployment:executor",
-    )
-    parser.add_argument(
-        "--executor-dir",
-        help="The directory for the executor.",
-        default=".",
-    )
-    parser.add_argument(
-        "--executor-config",
-        help="Key-value pairs for the executor config, separated by spaces. "
-        'For example:\n\n`--executor-config \'superlink="superlink:9091" '
-        'root-certificates="certificates/superlink-ca.crt"\'`',
-    )
-    parser.add_argument(
-        "--insecure",
-        action="store_true",
-        help="Run the SuperExec without HTTPS, regardless of whether certificate "
-        "paths are provided. By default, the server runs with HTTPS enabled. "
-        "Use this flag only if you understand the risks.",
-    )
-    parser.add_argument(
-        "--ssl-certfile",
-        help="SuperExec server SSL certificate file (as a path str) "
-        "to create a secure connection.",
-        type=str,
-        default=None,
-    )
-    parser.add_argument(
-        "--ssl-keyfile",
-        help="SuperExec server SSL private key file (as a path str) "
-        "to create a secure connection.",
-        type=str,
-    )
-    parser.add_argument(
-        "--ssl-ca-certfile",
-        help="SuperExec server SSL CA certificate file (as a path str) "
-        "to create a secure connection.",
-        type=str,
-    )
-    return parser
-
-
-def _try_obtain_certificates(
-    args: argparse.Namespace,
-) -> Optional[tuple[bytes, bytes, bytes]]:
-    # Obtain certificates
-    if args.insecure:
-        log(WARN, "Option `--insecure` was set. Starting insecure HTTP server.")
-        return None
-    # Check if certificates are provided
-    if args.ssl_certfile and args.ssl_keyfile and args.ssl_ca_certfile:
-        if not Path(args.ssl_ca_certfile).is_file():
-            sys.exit("Path argument `--ssl-ca-certfile` does not point to a file.")
-        if not Path(args.ssl_certfile).is_file():
-            sys.exit("Path argument `--ssl-certfile` does not point to a file.")
-        if not Path(args.ssl_keyfile).is_file():
-            sys.exit("Path argument `--ssl-keyfile` does not point to a file.")
-        certificates = (
-            Path(args.ssl_ca_certfile).read_bytes(),  # CA certificate
-            Path(args.ssl_certfile).read_bytes(),  # server certificate
-            Path(args.ssl_keyfile).read_bytes(),  # server private key
-        )
-        return certificates
-    if args.ssl_certfile or args.ssl_keyfile or args.ssl_ca_certfile:
-        sys.exit(
-            "You need to provide valid file paths to `--ssl-certfile`, "
-            "`--ssl-keyfile`, and `—-ssl-ca-certfile` to create a secure "
-            "connection in SuperExec server (gRPC-rere)."
-        )
-    sys.exit(
-        "Certificates are required unless running in insecure mode. "
-        "Please provide certificate paths to `--ssl-certfile`, "
-        "`--ssl-keyfile`, and `—-ssl-ca-certfile` or run the server "
-        "in insecure mode using '--insecure' if you understand the risks."
-    )
-
 
 def load_executor(
     args: argparse.Namespace,

flwr/superexec/deployment.py

@@ -24,12 +24,11 @@ from typing_extensions import override
 
 from flwr.cli.install import install_from_fab
 from flwr.common.constant import DRIVER_API_DEFAULT_ADDRESS
-from flwr.common.grpc import create_channel
 from flwr.common.logger import log
-from flwr.common.serde import fab_to_proto, user_config_to_proto
 from flwr.common.typing import Fab, UserConfig
-from flwr.proto.driver_pb2_grpc import DriverStub
-from flwr.proto.run_pb2 import CreateRunRequest  # pylint: disable=E0611
+from flwr.server.superlink.ffs import Ffs
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkState, LinkStateFactory
 
 from .executor import Executor, RunTracker
 
@@ -62,7 +61,30 @@ class DeploymentEngine(Executor):
             self.root_certificates = root_certificates
             self.root_certificates_bytes = Path(root_certificates).read_bytes()
         self.flwr_dir = flwr_dir
-        self.stub: Optional[DriverStub] = None
+        self.linkstate_factory: Optional[LinkStateFactory] = None
+        self.ffs_factory: Optional[FfsFactory] = None
+
+    @override
+    def initialize(
+        self, linkstate_factory: LinkStateFactory, ffs_factory: FfsFactory
+    ) -> None:
+        """Initialize the executor with the necessary factories."""
+        self.linkstate_factory = linkstate_factory
+        self.ffs_factory = ffs_factory
+
+    @property
+    def linkstate(self) -> LinkState:
+        """Return the LinkState."""
+        if self.linkstate_factory is None:
+            raise RuntimeError("Executor is not initialized.")
+        return self.linkstate_factory.state()
+
+    @property
+    def ffs(self) -> Ffs:
+        """Return the Flower File Storage (FFS)."""
+        if self.ffs_factory is None:
+            raise RuntimeError("Executor is not initialized.")
+        return self.ffs_factory.ffs()
 
     @override
     def set_config(
@@ -101,32 +123,19 @@ class DeploymentEngine(Executor):
                 raise ValueError("The `flwr-dir` value should be of type `str`.")
             self.flwr_dir = str(flwr_dir)
 
-    def _connect(self) -> None:
-        if self.stub is not None:
-            return
-        channel = create_channel(
-            server_address=self.superlink,
-            insecure=(self.root_certificates_bytes is None),
-            root_certificates=self.root_certificates_bytes,
-        )
-        self.stub = DriverStub(channel)
-
     def _create_run(
         self,
         fab: Fab,
         override_config: UserConfig,
     ) -> int:
-        if self.stub is None:
-            self._connect()
-
-        assert self.stub is not None
+        fab_hash = self.ffs.put(fab.content, {})
+        if fab_hash != fab.hash_str:
+            raise RuntimeError(
+                f"FAB ({fab.hash_str}) hash from request doesn't match contents"
+            )
 
-        req = CreateRunRequest(
-            fab=fab_to_proto(fab),
-            override_config=user_config_to_proto(override_config),
-        )
-        res = self.stub.CreateRun(request=req)
-        return int(res.run_id)
+        run_id = self.linkstate.create_run(None, None, fab_hash, override_config)
+        return run_id
 
     @override
     def start_run(

flwr/superexec/exec_grpc.py

@@ -23,33 +23,40 @@ from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.logger import log
 from flwr.common.typing import UserConfig
 from flwr.proto.exec_pb2_grpc import add_ExecServicer_to_server
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
 from flwr.server.superlink.fleet.grpc_bidi.grpc_server import generic_create_grpc_server
+from flwr.server.superlink.linkstate import LinkStateFactory
 
 from .exec_servicer import ExecServicer
 from .executor import Executor
 
 
-def run_superexec_api_grpc(
+# pylint: disable-next=too-many-arguments, too-many-positional-arguments
+def run_exec_api_grpc(
     address: str,
     executor: Executor,
+    state_factory: LinkStateFactory,
+    ffs_factory: FfsFactory,
     certificates: Optional[tuple[bytes, bytes, bytes]],
     config: UserConfig,
 ) -> grpc.Server:
-    """Run SuperExec API (gRPC, request-response)."""
+    """Run Exec API (gRPC, request-response)."""
     executor.set_config(config)
 
     exec_servicer: grpc.Server = ExecServicer(
+        linkstate_factory=state_factory,
+        ffs_factory=ffs_factory,
         executor=executor,
     )
-    superexec_add_servicer_to_server_fn = add_ExecServicer_to_server
-    superexec_grpc_server = generic_create_grpc_server(
-        servicer_and_add_fn=(exec_servicer, superexec_add_servicer_to_server_fn),
+    exec_add_servicer_to_server_fn = add_ExecServicer_to_server
+    exec_grpc_server = generic_create_grpc_server(
+        servicer_and_add_fn=(exec_servicer, exec_add_servicer_to_server_fn),
         server_address=address,
         max_message_length=GRPC_MAX_MESSAGE_LENGTH,
         certificates=certificates,
     )
 
-    log(INFO, "Starting Flower SuperExec gRPC server on %s", address)
-    superexec_grpc_server.start()
+    log(INFO, "Flower Deployment Engine: Starting Exec API on %s", address)
+    exec_grpc_server.start()
 
-    return superexec_grpc_server
+    return exec_grpc_server

flwr/superexec/exec_servicer.py

@@ -34,6 +34,8 @@ from flwr.proto.exec_pb2 import (  # pylint: disable=E0611
     StreamLogsRequest,
     StreamLogsResponse,
 )
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkStateFactory
 
 from .executor import Executor, RunTracker
 
@@ -43,8 +45,16 @@ SELECT_TIMEOUT = 1  # Timeout for selecting ready-to-read file descriptors (in s
 class ExecServicer(exec_pb2_grpc.ExecServicer):
     """SuperExec API servicer."""
 
-    def __init__(self, executor: Executor) -> None:
+    def __init__(
+        self,
+        linkstate_factory: LinkStateFactory,
+        ffs_factory: FfsFactory,
+        executor: Executor,
+    ) -> None:
+        self.linkstate_factory = linkstate_factory
+        self.ffs_factory = ffs_factory
         self.executor = executor
+        self.executor.initialize(linkstate_factory, ffs_factory)
         self.runs: dict[int, RunTracker] = {}
 
     def StartRun(

flwr/superexec/executor.py

@@ -20,6 +20,8 @@ from subprocess import Popen
 from typing import Optional
 
 from flwr.common.typing import UserConfig
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkStateFactory
 
 
 @dataclass
@@ -34,6 +36,23 @@ class RunTracker:
 class Executor(ABC):
     """Execute and monitor a Flower run."""
 
+    @abstractmethod
+    def initialize(
+        self, linkstate_factory: LinkStateFactory, ffs_factory: FfsFactory
+    ) -> None:
+        """Initialize the executor with the necessary factories.
+
+        This method sets up the executor by providing it with the factories required
+        to access the LinkState and the Flower File Storage (FFS) in the SuperLink.
+
+        Parameters
+        ----------
+        linkstate_factory : LinkStateFactory
+            The factory to create access to the LinkState.
+        ffs_factory : FfsFactory
+            The factory to create access to the Flower File Storage (FFS).
+        """
+
     @abstractmethod
     def set_config(
         self,

flwr/superexec/simulation.py

@@ -29,6 +29,8 @@ from flwr.common.config import unflatten_dict
 from flwr.common.constant import RUN_ID_NUM_BYTES
 from flwr.common.logger import log
 from flwr.common.typing import UserConfig
+from flwr.server.superlink.ffs.ffs_factory import FfsFactory
+from flwr.server.superlink.linkstate import LinkStateFactory
 from flwr.server.superlink.linkstate.utils import generate_rand_int_from_bytes
 
 from .executor import Executor, RunTracker
@@ -70,6 +72,12 @@ class SimulationEngine(Executor):
         self.num_supernodes = num_supernodes
         self.verbose = verbose
 
+    @override
+    def initialize(
+        self, linkstate_factory: LinkStateFactory, ffs_factory: FfsFactory
+    ) -> None:
+        """Initialize the executor with the necessary factories."""
+
     @override
     def set_config(
         self,

{flwr_nightly-1.13.0.dev20241023.dist-info → flwr_nightly-1.13.0.dev20241025.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.13.0.dev20241023
+Version: 1.13.0.dev20241025
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0