flwr-nightly 1.23.0.dev20251027__tar.gz → 1.23.0.dev20251028__tar.gz

{flwr_nightly-1.23.0.dev20251027 → flwr_nightly-1.23.0.dev20251028}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: flwr-nightly
-Version: 1.23.0.dev20251027
+Version: 1.23.0.dev20251028
 Summary: Flower: A Friendly Federated AI Framework
 License: Apache-2.0
 Keywords: Artificial Intelligence,Federated AI,Federated Analytics,Federated Evaluation,Federated Learning,Flower,Machine Learning

{flwr_nightly-1.23.0.dev20251027 → flwr_nightly-1.23.0.dev20251028}/py/flwr/cli/build.py

@@ -17,22 +17,25 @@
 
 import hashlib
 import zipfile
+from collections.abc import Iterable
 from io import BytesIO
 from pathlib import Path
 from typing import Annotated, Any, Optional, Union
 
 import pathspec
+import tomli
 import tomli_w
 import typer
 
 from flwr.common.constant import (
-    FAB_ALLOWED_EXTENSIONS,
+    FAB_CONFIG_FILE,
     FAB_DATE,
+    FAB_EXCLUDE_PATTERNS,
     FAB_HASH_TRUNCATION,
+    FAB_INCLUDE_PATTERNS,
     FAB_MAX_SIZE,
 )
 
-from .config_utils import load as load_toml
 from .config_utils import load_and_validate
 from .utils import is_valid_project_name
 
@@ -112,7 +115,10 @@ def build(
         )
 
     # Build FAB
-    fab_bytes, fab_hash, _ = build_fab(app)
+    fab_bytes = build_fab_from_disk(app)
+
+    # Calculate hash for filename
+    fab_hash = hashlib.sha256(fab_bytes).hexdigest()
 
     # Get the name of the zip file
     fab_filename = get_fab_filename(config, fab_hash)
@@ -125,11 +131,10 @@ def build(
     )
 
 
-def build_fab(app: Path) -> tuple[bytes, str, dict[str, Any]]:
-    """Build a FAB in memory and return the bytes, hash, and config.
+def build_fab_from_disk(app: Path) -> bytes:
+    """Build a FAB from files on disk and return the FAB as bytes.
 
-    This function assumes that the provided path points to a valid Flower app and
-    bundles it into a FAB without performing additional validation.
+    This function reads files from disk and bundles them into a FAB.
 
     Parameters
     ----------
@@ -138,18 +143,67 @@ def build_fab(app: Path) -> tuple[bytes, str, dict[str, Any]]:
 
     Returns
     -------
-    tuple[bytes, str, dict[str, Any]]
-        A tuple containing:
-        - the FAB as bytes
-        - the SHA256 hash of the FAB
-        - the project configuration (with the 'federations' field removed)
+    bytes
+        The FAB as bytes.
     """
     app = app.resolve()
 
-    # Load the pyproject.toml file
-    config = load_toml(app / "pyproject.toml")
-    if config is None:
-        raise ValueError("Project configuration could not be loaded.")
+    # Collect all files recursively (including pyproject.toml and .gitignore)
+    all_files = [f for f in app.rglob("*") if f.is_file()]
+
+    # Create dict mapping relative paths to Path objects
+    files_dict: dict[str, Union[bytes, Path]] = {
+        # Ensure consistent path separators across platforms
+        str(file_path.relative_to(app)).replace("\\", "/"): file_path
+        for file_path in all_files
+    }
+
+    # Build FAB from the files dict
+    return build_fab_from_files(files_dict)
+
+
+def build_fab_from_files(files: dict[str, Union[bytes, Path]]) -> bytes:
+    r"""Build a FAB from in-memory files and return the FAB as bytes.
+
+    This is the core FAB building function that works with in-memory data.
+    It accepts either bytes or Path objects as file contents, applies filtering
+    rules (include/exclude patterns), and builds the FAB.
+
+    Parameters
+    ----------
+    files : dict[str, Union[bytes, Path]]
+        Dictionary mapping relative file paths to their contents.
+        - Keys: Relative paths (strings)
+        - Values: Either bytes (file contents) or Path (will be read)
+        Must include "pyproject.toml" and optionally ".gitignore".
+
+    Returns
+    -------
+    bytes
+        The FAB as bytes.
+
+    Examples
+    --------
+    Build a FAB from in-memory files::
+
+        files = {
+            "pyproject.toml": b"[project]\nname = 'myapp'\n...",
+            ".gitignore": b"*.pyc\n__pycache__/\n",
+            "src/client.py": Path("/path/to/client.py"),
+            "src/server.py": b"print('hello')",
+            "README.md": b"# My App\n",
+        }
+        fab_bytes = build_fab_from_files(files)
+    """
+
+    def to_bytes(content: Union[bytes, Path]) -> bytes:
+        return content.read_bytes() if isinstance(content, Path) else content
+
+    # Extract, load, and parse pyproject.toml
+    if FAB_CONFIG_FILE not in files:
+        raise ValueError(f"{FAB_CONFIG_FILE} not found in files")
+    pyproject_content = to_bytes(files[FAB_CONFIG_FILE])
+    config = tomli.loads(pyproject_content.decode("utf-8"))
 
     # Remove the 'federations' field if it exists
     if (
@@ -159,18 +213,22 @@ def build_fab(app: Path) -> tuple[bytes, str, dict[str, Any]]:
     ):
         del config["tool"]["flwr"]["federations"]
 
-    # Load .gitignore rules if present
-    ignore_spec = _load_gitignore(app)
+    # Extract and load .gitignore if present
+    gitignore_content = None
+    if ".gitignore" in files:
+        gitignore_content = to_bytes(files[".gitignore"])
+
+    # Get exclude and include specs
+    exclude_spec = get_fab_exclude_pathspec(gitignore_content)
+    include_spec = get_fab_include_pathspec()
 
-    # Search for all files in the app directory
-    all_files = [
-        f
-        for f in app.rglob("*")
-        if not ignore_spec.match_file(f)
-        and f.suffix in FAB_ALLOWED_EXTENSIONS
-        and f.name != "pyproject.toml"  # Exclude the original pyproject.toml
+    # Filter files based on include/exclude specs
+    filtered_paths = [
+        path.replace("\\", "/")  # Ensure consistent path separators across platforms
+        for path in files.keys()
+        if include_spec.match_file(path) and not exclude_spec.match_file(path)
     ]
-    all_files.sort()
+    filtered_paths.sort()  # Sort for deterministic output
 
     # Create a zip file in memory
     list_file_content = ""
@@ -178,41 +236,54 @@ def build_fab(app: Path) -> tuple[bytes, str, dict[str, Any]]:
     fab_buffer = BytesIO()
     with zipfile.ZipFile(fab_buffer, "w", zipfile.ZIP_DEFLATED) as fab_file:
         # Add pyproject.toml
-        write_to_zip(fab_file, "pyproject.toml", tomli_w.dumps(config))
+        write_to_zip(fab_file, FAB_CONFIG_FILE, tomli_w.dumps(config))
 
-        for file_path in all_files:
-            # Read the file content manually
-            file_contents = file_path.read_bytes()
+        for file_path in filtered_paths:
 
-            archive_path = str(file_path.relative_to(app)).replace("\\", "/")
-            write_to_zip(fab_file, archive_path, file_contents)
+            # Get file contents as bytes
+            file_content = to_bytes(files[file_path])
 
-            # Calculate file info
-            sha256_hash = hashlib.sha256(file_contents).hexdigest()
-            file_size_bits = len(file_contents) * 8  # size in bits
-            list_file_content += f"{archive_path},{sha256_hash},{file_size_bits}\n"
+            # Write file to FAB
+            write_to_zip(fab_file, file_path, file_content)
 
-        # Add CONTENT and CONTENT.jwt to the zip file
+            # Calculate file info for CONTENT manifest
+            sha256_hash = hashlib.sha256(file_content).hexdigest()
+            file_size_bits = len(file_content) * 8  # size in bits
+            list_file_content += f"{file_path},{sha256_hash},{file_size_bits}\n"
+
+        # Add CONTENT manifest to the zip file
         write_to_zip(fab_file, ".info/CONTENT", list_file_content)
 
     fab_bytes = fab_buffer.getvalue()
+
+    # Validate FAB size
     if len(fab_bytes) > FAB_MAX_SIZE:
         raise ValueError(
-            f"FAB size exceeds maximum allowed size of {FAB_MAX_SIZE:,} bytes."
+            f"FAB size exceeds maximum allowed size of {FAB_MAX_SIZE:,} bytes. "
             "To reduce the package size, consider ignoring unnecessary files "
             "via your `.gitignore` file or excluding them from the build."
         )
 
-    fab_hash = hashlib.sha256(fab_bytes).hexdigest()
+    return fab_bytes
 
-    return fab_bytes, fab_hash, config
 
+def build_pathspec(patterns: Iterable[str]) -> pathspec.PathSpec:
+    """Build a PathSpec from a list of patterns."""
+    return pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+
+
+def get_fab_include_pathspec() -> pathspec.PathSpec:
+    """Get the PathSpec for files to include in a FAB."""
+    return build_pathspec(FAB_INCLUDE_PATTERNS)
 
-def _load_gitignore(app: Path) -> pathspec.PathSpec:
-    """Load and parse .gitignore file, returning a pathspec."""
-    gitignore_path = app / ".gitignore"
-    patterns = ["__pycache__/"]  # Default pattern
-    if gitignore_path.exists():
-        with open(gitignore_path, encoding="UTF-8") as file:
-            patterns.extend(file.readlines())
+
+def get_fab_exclude_pathspec(gitignore_content: Optional[bytes]) -> pathspec.PathSpec:
+    """Get the PathSpec for files to exclude from a FAB.
+
+    If gitignore_content is provided, its patterns will be combined with the default
+    exclude patterns.
+    """
+    patterns = list(FAB_EXCLUDE_PATTERNS)
+    if gitignore_content:
+        patterns += gitignore_content.decode("UTF-8").splitlines()
     return pathspec.PathSpec.from_lines("gitwildmatch", patterns)

{flwr_nightly-1.23.0.dev20251027 → flwr_nightly-1.23.0.dev20251028}/py/flwr/cli/run/run.py

@@ -15,16 +15,18 @@
 """Flower command line interface `run` command."""
 
 
+import hashlib
 import io
 import json
 import subprocess
 from pathlib import Path
-from typing import Annotated, Any, Optional
+from typing import Annotated, Any, Optional, cast
 
 import typer
 from rich.console import Console
 
-from flwr.cli.build import build_fab, get_fab_filename
+from flwr.cli.build import build_fab_from_disk, get_fab_filename
+from flwr.cli.config_utils import load as load_toml
 from flwr.cli.config_utils import (
     load_and_validate,
     process_loaded_project_config,
@@ -37,7 +39,7 @@ from flwr.common.config import (
     parse_config_args,
     user_config_to_configrecord,
 )
-from flwr.common.constant import CliOutputFormat
+from flwr.common.constant import FAB_CONFIG_FILE, CliOutputFormat
 from flwr.common.logger import print_json_error, redirect_output, restore_output
 from flwr.common.serde import config_record_to_proto, fab_to_proto, user_config_to_proto
 from flwr.common.typing import Fab
@@ -152,7 +154,9 @@ def _run_with_control_api(
         channel = init_channel(app, federation_config, auth_plugin)
         stub = ControlStub(channel)
 
-        fab_bytes, fab_hash, config = build_fab(app)
+        fab_bytes = build_fab_from_disk(app)
+        fab_hash = hashlib.sha256(fab_bytes).hexdigest()
+        config = cast(dict[str, Any], load_toml(app / FAB_CONFIG_FILE))
         fab_id, fab_version = get_metadata_from_config(config)
 
         fab = Fab(fab_hash, fab_bytes, {})

{flwr_nightly-1.23.0.dev20251027 → flwr_nightly-1.23.0.dev20251028}/py/flwr/common/constant.py

@@ -72,13 +72,23 @@ NODE_ID_NUM_BYTES = 8
 
 # Constants for FAB
 APP_DIR = "apps"
-FAB_ALLOWED_EXTENSIONS = {".py", ".toml", ".md"}
 FAB_CONFIG_FILE = "pyproject.toml"
 FAB_DATE = (2024, 10, 1, 0, 0, 0)
 FAB_HASH_TRUNCATION = 8
 FAB_MAX_SIZE = 10 * 1024 * 1024  # 10 MB
 FLWR_DIR = ".flwr"  # The default Flower directory: ~/.flwr/
 FLWR_HOME = "FLWR_HOME"  # If set, override the default Flower directory
+# FAB file include patterns (gitignore-style patterns)
+FAB_INCLUDE_PATTERNS = (
+    "**/*.py",
+    "**/*.toml",
+    "**/*.md",
+)
+# FAB file exclude patterns (gitignore-style patterns)
+FAB_EXCLUDE_PATTERNS = (
+    "**/__pycache__/**",
+    FAB_CONFIG_FILE,  # Exclude the original pyproject.toml
+)
 
 # Constant for SuperLink
 SUPERLINK_NODE_ID = 1

{flwr_nightly-1.23.0.dev20251027 → flwr_nightly-1.23.0.dev20251028}/py/flwr/server/superlink/linkstate/in_memory_linkstate.py

@@ -411,10 +411,10 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
     ) -> Sequence[NodeInfo]:
         """Retrieve information about nodes based on the specified filters."""
         with self.lock:
-            self._check_and_tag_deactivated_nodes()
+            self._check_and_tag_offline_nodes()
             result = []
-            for node in self.nodes.values():
-                if node_ids is not None and node.node_id not in node_ids:
+            for node_id in self.nodes.keys() if node_ids is None else node_ids:
+                if (node := self.nodes.get(node_id)) is None:
                     continue
                 if owner_aids is not None and node.owner_aid not in owner_aids:
                     continue
@@ -423,11 +423,15 @@ class InMemoryLinkState(LinkState):  # pylint: disable=R0902,R0904
                 result.append(node)
             return result
 
-    def _check_and_tag_deactivated_nodes(self) -> None:
+    def _check_and_tag_offline_nodes(
+        self, node_ids: Optional[list[int]] = None
+    ) -> None:
         with self.lock:
             # Set all nodes of "online" status to "offline" if they've offline
             current_ts = now().timestamp()
-            for node in self.nodes.values():
+            for node_id in node_ids or self.nodes.keys():
+                if (node := self.nodes.get(node_id)) is None:
+                    continue
                 if node.status == NodeStatus.ONLINE:
                     if node.online_until <= current_ts:
                         node.status = NodeStatus.OFFLINE

{flwr_nightly-1.23.0.dev20251027 → flwr_nightly-1.23.0.dev20251028}/py/flwr/server/superlink/linkstate/sqlite_linkstate.py

@@ -99,6 +99,10 @@ SQL_CREATE_INDEX_OWNER_AID = """
 CREATE INDEX IF NOT EXISTS idx_node_owner_aid ON node(owner_aid);
 """
 
+SQL_CREATE_INDEX_NODE_STATUS = """
+CREATE INDEX IF NOT EXISTS idx_node_status ON node(status);
+"""
+
 SQL_CREATE_TABLE_RUN = """
 CREATE TABLE IF NOT EXISTS run(
     run_id                INTEGER UNIQUE,
@@ -199,6 +203,7 @@ class SqliteLinkState(LinkState, SqliteMixin):  # pylint: disable=R0904
             SQL_CREATE_TABLE_TOKEN_STORE,
             SQL_CREATE_INDEX_ONLINE_UNTIL,
             SQL_CREATE_INDEX_OWNER_AID,
+            SQL_CREATE_INDEX_NODE_STATUS,
             log_queries=log_queries,
         )
 
@@ -638,6 +643,28 @@ class SqliteLinkState(LinkState, SqliteMixin):  # pylint: disable=R0904
             node.node_id for node in self.get_node_info(statuses=[NodeStatus.ONLINE])
         }
 
+    def _check_and_tag_offline_nodes(
+        self, node_ids: Optional[list[int]] = None
+    ) -> None:
+        """Check and tag offline nodes."""
+        # strftime will convert POSIX timestamp to ISO format
+        query = """
+            UPDATE node SET status = ?,
+            last_deactivated_at =
+            strftime("%Y-%m-%dT%H:%M:%f+00:00", online_until, "unixepoch")
+            WHERE online_until <= ? AND status == ?
+        """
+        params = [
+            NodeStatus.OFFLINE,
+            now().timestamp(),
+            NodeStatus.ONLINE,
+        ]
+        if node_ids is not None:
+            placeholders = ",".join(["?"] * len(node_ids))
+            query += f" AND node_id IN ({placeholders})"
+            params.extend(uint64_to_int64(node_id) for node_id in node_ids)
+        self.conn.execute(query, params)
+
     def get_node_info(
         self,
         *,
@@ -646,29 +673,12 @@ class SqliteLinkState(LinkState, SqliteMixin):  # pylint: disable=R0904
         statuses: Optional[Sequence[str]] = None,
     ) -> Sequence[NodeInfo]:
         """Retrieve information about nodes based on the specified filters."""
-        if self.conn is None:
-            raise AttributeError("LinkState is not initialized.")
-
         with self.conn:
-            # Check and tag offline nodes
-            current_dt = now()
-            # strftime will convert POSIX timestamp to ISO format
-            query = """
-                UPDATE node SET status = ?,
-                last_deactivated_at =
-                strftime("%Y-%m-%dT%H:%M:%f+00:00", online_until, "unixepoch")
-                WHERE online_until <= ? AND status == ?
-            """
-            params: list[Any] = [
-                NodeStatus.OFFLINE,
-                current_dt.timestamp(),
-                NodeStatus.ONLINE,
-            ]
-            self.conn.execute(query, params)
+            self._check_and_tag_offline_nodes()
 
             # Build the WHERE clause based on provided filters
             conditions = []
-            params = []
+            params: list[Any] = []
             if node_ids is not None:
                 sint64_node_ids = [uint64_to_int64(node_id) for node_id in node_ids]
                 placeholders = ",".join(["?"] * len(sint64_node_ids))

{flwr_nightly-1.23.0.dev20251027 → flwr_nightly-1.23.0.dev20251028}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "flwr-nightly"
-version = "1.23.0.dev20251027"
+version = "1.23.0.dev20251028"
 description = "Flower: A Friendly Federated AI Framework"
 license = "Apache-2.0"
 authors = ["The Flower Authors <hello@flower.ai>"]