boneio 1.5.0.dev12__tar.gz → 1.5.0.dev14__tar.gz

{boneio-1.5.0.dev12/boneio.egg-info → boneio-1.5.0.dev14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boneio
-Version: 1.5.0.dev12
+Version: 1.5.0.dev14
 Summary: Python App for BoneIO
 Author-email: Paweł Szafer <pszafer@gmail.com>
 License: GNU General Public License v3.0

{boneio-1.5.0.dev12 → boneio-1.5.0.dev14}/boneio/core/config/migrations/__init__.py

@@ -17,7 +17,7 @@ from dataclasses import dataclass, field
 _LOGGER = logging.getLogger(__name__)
 
 # Current schema version — bump this when adding new migrations
-CURRENT_SCHEMA_VERSION = 3
+CURRENT_SCHEMA_VERSION = 4
 
 # Minimum app version that introduced each schema version.
 # Used by the WebUI to warn before rolling back to an incompatible version.
@@ -26,6 +26,7 @@ SCHEMA_VERSION_APP_MAP: dict[int, str] = {
     1: "1.2.0dev1",   # proxy_port rename
     2: "1.3.0dev1",   # transition timeperiod
     3: "1.3.0dev7",   # irrigation section
+    4: "1.5.0dev13",  # WLED effects/palettes → JSON cache
 }
 
 
@@ -230,4 +231,5 @@ def _persist_config_version(config_file: str, version: int) -> None:
 from boneio.core.config.migrations import (
     v1_proxy_port,  # noqa: E402, F401
     v2_transition,  # noqa: E402, F401
+    v4_wled_cache,  # noqa: E402, F401
 )

boneio-1.5.0.dev14/boneio/core/config/migrations/v4_wled_cache.py

@@ -0,0 +1,219 @@
+"""Migration v4: Extract WLED effects/palettes/segments from config to JSON cache.
+
+Before: WLED effects (219 items), palettes (72 items), and segments are stored
+        inline in each remote_device's ``wled`` section inside config.yaml.
+        This causes 3-5 second YAML serialization delays on ARM hardware.
+
+After:  These fields are stripped from config and saved to a lightweight
+        ``.wled_cache.json`` file.  The cache auto-regenerates from WLED API
+        on device connect, so no data is permanently lost.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from pathlib import Path
+
+from boneio.core.config.migrations import register_migration
+
+_LOGGER = logging.getLogger(__name__)
+
+# Fields to strip from each WLED device config — these are device metadata,
+# not user configuration.
+_WLED_CACHE_FIELDS = ("effects", "palettes", "segments")
+
+
+def _persist_wled_cache_strip(config_file: str) -> None:
+    """Strip effects/palettes/segments from WLED devices in the YAML file.
+
+    Reads the config file(s), extracts WLED metadata into a JSON cache,
+    and rewrites the YAML without those bulky lists.
+
+    Args:
+        config_file: Path to the main config.yaml file.
+    """
+    config_dir = Path(config_file).parent
+
+    # First, find and extract WLED data from the in-memory parse
+    # to save into .wled_cache.json before removing from YAML
+    try:
+        from yaml import SafeLoader, load
+
+        class IncludeLoader(SafeLoader):
+            """YAML loader that preserves !include tags."""
+
+        def _include_constructor(
+            loader: IncludeLoader, node: object
+        ) -> object:
+            filename = loader.construct_scalar(node)  # type: ignore[arg-type]
+            return type(
+                "Include", (), {"filename": filename, "tag": "!include"}
+            )()
+
+        IncludeLoader.add_constructor("!include", _include_constructor)
+
+        with open(config_file, encoding="utf-8") as f:
+            config = load(f, Loader=IncludeLoader)  # noqa: S506
+
+        if not config:
+            return
+
+        # Determine where remote_devices lives
+        rd_section = config.get("remote_devices")
+        if rd_section is None:
+            return
+
+        # If it's an !include, load the included file
+        if hasattr(rd_section, "tag") and rd_section.tag == "!include":
+            include_path = config_dir / rd_section.filename
+            if include_path.exists():
+                with open(include_path, encoding="utf-8") as f:
+                    rd_list = load(f, Loader=SafeLoader)  # noqa: S506
+                _strip_wled_fields_from_file(include_path, rd_list, config_dir)
+            return
+
+        # Inline remote_devices — strip from main config file
+        if isinstance(rd_section, list):
+            _strip_wled_fields_from_file(
+                Path(config_file), rd_section, config_dir
+            )
+
+    except Exception as exc:
+        _LOGGER.error("Failed to strip WLED cache fields from file: %s", exc)
+
+
+def _strip_wled_fields_from_file(
+    yaml_path: Path, devices: list[dict] | None, config_dir: Path
+) -> None:
+    """Extract WLED metadata to JSON cache, then strip from YAML file.
+
+    Args:
+        yaml_path: Path to the YAML file containing remote_devices.
+        devices: Parsed list of remote device dicts.
+        config_dir: Config directory for placing the cache file.
+    """
+    if not devices:
+        return
+
+    # Extract cache data
+    cache_data: dict[str, dict[str, list[dict[str, str | int]]]] = {}
+    has_data = False
+
+    for device in devices:
+        if not isinstance(device, dict):
+            continue
+        wled = device.get("wled")
+        if not isinstance(wled, dict):
+            continue
+
+        device_id = device.get("id", "")
+        if not device_id:
+            continue
+
+        device_cache: dict[str, list[dict[str, str | int]]] = {}
+        for field in _WLED_CACHE_FIELDS:
+            if field in wled and isinstance(wled[field], list) and wled[field]:
+                device_cache[field] = wled[field]
+                has_data = True
+        if device_cache:
+            cache_data[device_id] = device_cache
+
+    # Save extracted data to JSON cache
+    if has_data:
+        cache_path = config_dir / ".wled_cache.json"
+        try:
+            # Merge with existing cache (other migrations may have run)
+            existing: dict[str, dict[str, list[dict[str, str | int]]]] = {}
+            if cache_path.exists():
+                with open(cache_path, encoding="utf-8") as f:
+                    existing = json.load(f)
+
+            existing.update(cache_data)
+
+            with open(cache_path, "w", encoding="utf-8") as f:
+                json.dump(existing, f, indent=2, ensure_ascii=False)
+
+            _LOGGER.info(
+                "Saved WLED metadata for %d device(s) to %s",
+                len(cache_data),
+                cache_path,
+            )
+        except Exception as exc:
+            _LOGGER.warning("Failed to write WLED cache: %s", exc)
+
+    # Now strip the fields from the YAML file using regex
+    # This is safer than re-serializing the entire file
+    with open(yaml_path, encoding="utf-8") as f:
+        content = f.read()
+
+    original_content = content
+
+    # Remove effects/palettes/segments blocks from YAML
+    # Pattern: indented key followed by a list of dicts
+    for field in _WLED_CACHE_FIELDS:
+        # Match the field key line and all following lines that are more indented
+        # or are list items (- ) at deeper indent
+        pattern = re.compile(
+            rf"^( +){field}:\s*\n"        # field key line
+            rf"((?:\1 .+\n|\1- .+\n)*)",  # indented content below
+            re.MULTILINE,
+        )
+        content = pattern.sub("", content)
+
+    if content != original_content:
+        with open(yaml_path, "w", encoding="utf-8") as f:
+            f.write(content)
+        _LOGGER.info(
+            "Stripped WLED cache fields from %s (saved %d bytes)",
+            yaml_path.name,
+            len(original_content) - len(content),
+        )
+
+
+@register_migration(
+    version=4,
+    name="Extract WLED effects/palettes/segments to JSON cache",
+    migrate_file=_persist_wled_cache_strip,
+)
+def migrate(doc: dict) -> dict:
+    """Strip effects, palettes, and segments from remote_devices in memory.
+
+    These fields are device metadata auto-discovered from the WLED API,
+    not user configuration.  Storing them in config.yaml causes slow saves
+    on ARM hardware due to YAML serialization overhead (~64KB per 7 devices).
+
+    After migration, this data lives in ``.wled_cache.json`` and is
+    auto-populated from the WLED API on device connect.
+
+    Args:
+        doc: Raw config dict.
+
+    Returns:
+        Migrated config dict with WLED metadata fields removed.
+    """
+    remote_devices = doc.get("remote_devices")
+    if not isinstance(remote_devices, list):
+        return doc
+
+    for device in remote_devices:
+        if not isinstance(device, dict):
+            continue
+        wled = device.get("wled")
+        if not isinstance(wled, dict):
+            continue
+
+        for field in _WLED_CACHE_FIELDS:
+            if field in wled:
+                items = wled[field]
+                count = len(items) if isinstance(items, list) else 0
+                _LOGGER.info(
+                    "Stripping wled.%s (%d items) from device '%s'",
+                    field,
+                    count,
+                    device.get("id", "?"),
+                )
+                del wled[field]
+
+    return doc

{boneio-1.5.0.dev12 → boneio-1.5.0.dev14}/boneio/core/config/yaml_util.py

@@ -1227,11 +1227,17 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
     Returns:
         dict: Status response with success/error message
     """
+    import time
     from pathlib import Path
 
+    t_start = time.perf_counter()
     config_dir = Path(config_file).parent
 
-    _LOGGER.info(f"Updating section '{section}' with data: {data}")
+    data_size = len(str(data)) if data else 0
+    _LOGGER.info(
+        "[SAVE] START section='%s' data_size=%d bytes",
+        section, data_size,
+    )
 
     # Special handling for mcp23017 - convert hex strings to integers
     # This ensures YAML writes them as integers which are then read back as hex
@@ -1246,9 +1252,25 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
                         with contextlib.suppress(ValueError):
                             entry["address"] = int(addr, 10)
 
+    # Strip WLED device metadata (effects/palettes/segments) from remote_devices.
+    # These are auto-discovered from WLED API and cached in .wled_cache.json,
+    # NOT stored in config.yaml (defense in depth).
+    if section == "remote_devices" and isinstance(data, list):
+        _wled_cache_fields = ("effects", "palettes", "segments")
+        for entry in data:
+            if isinstance(entry, dict):
+                wled = entry.get("wled")
+                if isinstance(wled, dict):
+                    for field in _wled_cache_fields:
+                        wled.pop(field, None)
+
     # Strip default values to keep YAML clean
+    t_strip = time.perf_counter()
     cleaned_data = strip_default_values(data, {}, section)
-    _LOGGER.info(f"Cleaned data (defaults removed): {cleaned_data}")
+    _LOGGER.info(
+        "[SAVE] strip_default_values took %.3fs",
+        time.perf_counter() - t_strip,
+    )
 
     # Custom YAML loader that preserves !include tags
     class IncludeLoader(SafeLoader):
@@ -1265,8 +1287,13 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
 
     try:
         # Read current config.yaml with custom loader
+        t_load = time.perf_counter()
         with open(config_file, encoding="utf-8") as f:
             config_content = load(f, Loader=IncludeLoader)
+        _LOGGER.info(
+            "[SAVE] yaml.load(config.yaml) took %.3fs",
+            time.perf_counter() - t_load,
+        )
 
         if config_content is None:
             config_content = {}
@@ -1284,13 +1311,21 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
                 _LOGGER.info(f"Section '{section}' uses !include '{include_filename}', updating {include_file_path}")
 
                 # Save cleaned data to the included file
+                t_dump = time.perf_counter()
                 content = dump(
                     cleaned_data, Dumper=TimePeriodDumper, default_flow_style=False, allow_unicode=True, sort_keys=False
                 )
+                _LOGGER.info(
+                    "[SAVE] yaml.dump(!include data) took %.3fs (%d bytes)",
+                    time.perf_counter() - t_dump, len(content),
+                )
+                t_write = time.perf_counter()
                 with open(include_file_path, "w", encoding="utf-8") as f:
                     f.write(content)
-
-                _LOGGER.info(f"Successfully updated included file: {include_file_path}")
+                _LOGGER.info(
+                    "[SAVE] file write (!include) took %.3fs",
+                    time.perf_counter() - t_write,
+                )
 
             else:
                 # It's a regular section - replace in config.yaml
@@ -1299,14 +1334,20 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
 
                 # Save updated config.yaml (need to handle !include when saving)
                 # Read original file as text to preserve !include syntax
+                t_reread = time.perf_counter()
                 with open(config_file, encoding="utf-8") as f:
                     original_lines = f.readlines()
+                _LOGGER.info(
+                    "[SAVE] re-read config as text took %.3fs (%d lines)",
+                    time.perf_counter() - t_reread, len(original_lines),
+                )
 
                 # Find and replace the section in the original text
                 updated_lines = []
                 in_section = False
                 section_indent = 0
 
+                t_dump = time.perf_counter()
                 for line in original_lines:
                     stripped = line.strip()
                     # Check if this line starts the target section
@@ -1340,12 +1381,21 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
                         # Skip lines that are part of the old section
                     else:
                         updated_lines.append(line)
+                _LOGGER.info(
+                    "[SAVE] yaml.dump(section) + line replace took %.3fs",
+                    time.perf_counter() - t_dump,
+                )
 
                 # Write updated config
+                t_write = time.perf_counter()
                 with open(config_file, "w", encoding="utf-8") as f:
                     f.writelines(updated_lines)
+                _LOGGER.info(
+                    "[SAVE] file write (config.yaml) took %.3fs",
+                    time.perf_counter() - t_write,
+                )
 
-                _LOGGER.info(f"Successfully updated section '{section}' in config.yaml")
+                _LOGGER.info("[SAVE] section '%s' inline update done", section)
         else:
             # Section doesn't exist - add it to config.yaml
             _LOGGER.info(f"Section '{section}' doesn't exist, adding to config.yaml")
@@ -1363,6 +1413,16 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
 
             _LOGGER.info(f"Successfully added new section '{section}' to config.yaml")
 
+        t_total = time.perf_counter() - t_start
+        _LOGGER.info(
+            "[SAVE] DONE section='%s' total=%.3fs",
+            section, t_total,
+        )
+        if t_total > 1.0:
+            _LOGGER.warning(
+                "[SAVE] SLOW save detected: section='%s' took %.3fs",
+                section, t_total,
+            )
         return {"status": "success", "message": f"Section '{section}' saved successfully"}
 
     except Exception as e:

{boneio-1.5.0.dev12 → boneio-1.5.0.dev14}/boneio/core/remote/wled.py

@@ -530,12 +530,24 @@ class WLEDRemoteDevice(RemoteDevice):
                 # Update internal segments list
                 self._segments = segments
                 
+                # Persist effects/palettes/segments to JSON cache
+                # (they are no longer stored in config.yaml)
+                from boneio.core.remote.wled_cache import update_device_metadata
+                update_device_metadata(
+                    self._id,
+                    effects=effects,
+                    palettes=palettes,
+                    segments=segments,
+                )
+                
                 _LOGGER.info(
-                    "Discovered WLED '%s' v%s with %d segments, %d LEDs",
+                    "Discovered WLED '%s' v%s with %d segments, %d LEDs, %d effects, %d palettes",
                     self._device_info["name"],
                     self._device_info["version"],
                     len(segments),
-                    self._device_info["led_count"]
+                    self._device_info["led_count"],
+                    len(effects),
+                    len(palettes),
                 )
                 
                 return self._device_info

boneio-1.5.0.dev14/boneio/core/remote/wled_cache.py

@@ -0,0 +1,143 @@
+"""WLED metadata cache — JSON-based storage for effects, palettes, and segments.
+
+Stores auto-discovered WLED device metadata in ``<config_dir>/.wled_cache.json``
+instead of config.yaml.  This avoids slow YAML serialization of ~64KB data
+(219 effects × 72 palettes × N devices) on ARM hardware.
+
+The cache is populated from the WLED ``/json`` API on device connect and is
+**not** included in config backups — it auto-regenerates.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+_LOGGER = logging.getLogger(__name__)
+
+# In-memory cache, lazily populated from disk
+_cache: dict[str, dict[str, list[dict[str, str | int]]]] | None = None
+_cache_path: Path | None = None
+
+
+def init_cache(config_dir: str | Path) -> None:
+    """Initialize cache path and load from disk if available.
+
+    Call this once at startup with the config directory path.
+
+    Args:
+        config_dir: Path to the boneIO config directory.
+    """
+    global _cache_path, _cache
+    _cache_path = Path(config_dir) / ".wled_cache.json"
+    _cache = None  # Force reload on next access
+    _LOGGER.debug("WLED cache path: %s", _cache_path)
+
+
+def _load_cache() -> dict[str, dict[str, list[dict[str, str | int]]]]:
+    """Load cache from disk.
+
+    Returns:
+        Dict mapping device_id to {effects, palettes, segments}.
+    """
+    global _cache
+    if _cache is not None:
+        return _cache
+
+    _cache = {}
+    if _cache_path and _cache_path.exists():
+        try:
+            with open(_cache_path, encoding="utf-8") as f:
+                _cache = json.load(f)
+            _LOGGER.debug(
+                "Loaded WLED cache: %d device(s)", len(_cache)
+            )
+        except Exception as exc:
+            _LOGGER.warning("Failed to load WLED cache: %s", exc)
+            _cache = {}
+
+    return _cache
+
+
+def get_device_metadata(device_id: str) -> dict[str, list[dict[str, str | int]]]:
+    """Get cached metadata for a WLED device.
+
+    Args:
+        device_id: WLED device identifier.
+
+    Returns:
+        Dict with 'effects', 'palettes', 'segments' lists.
+        Empty dict if not cached.
+    """
+    cache = _load_cache()
+    return cache.get(device_id, {})
+
+
+def get_all_metadata() -> dict[str, dict[str, list[dict[str, str | int]]]]:
+    """Get cached metadata for all WLED devices.
+
+    Returns:
+        Dict mapping device_id to metadata.
+    """
+    return _load_cache()
+
+
+def update_device_metadata(
+    device_id: str,
+    *,
+    effects: list[dict[str, str | int]] | None = None,
+    palettes: list[dict[str, str | int]] | None = None,
+    segments: list[dict[str, str | int]] | None = None,
+) -> None:
+    """Update cached metadata for a device and persist to disk.
+
+    Only updates fields that are provided (not None).
+
+    Args:
+        device_id: WLED device identifier.
+        effects: List of effect dicts with 'id' and 'name'.
+        palettes: List of palette dicts with 'id' and 'name'.
+        segments: List of segment dicts with 'id', 'name', etc.
+    """
+    cache = _load_cache()
+
+    if device_id not in cache:
+        cache[device_id] = {}
+
+    if effects is not None:
+        cache[device_id]["effects"] = effects
+    if palettes is not None:
+        cache[device_id]["palettes"] = palettes
+    if segments is not None:
+        cache[device_id]["segments"] = segments
+
+    _persist_cache(cache)
+
+    _LOGGER.info(
+        "Updated WLED cache for '%s' (effects=%s, palettes=%s, segments=%s)",
+        device_id,
+        len(effects) if effects is not None else "-",
+        len(palettes) if palettes is not None else "-",
+        len(segments) if segments is not None else "-",
+    )
+
+
+def _persist_cache(
+    cache: dict[str, dict[str, list[dict[str, str | int]]]]
+) -> None:
+    """Write cache to disk as JSON.
+
+    Args:
+        cache: Full cache dict to persist.
+    """
+    if not _cache_path:
+        _LOGGER.warning("WLED cache path not initialized, skipping persist")
+        return
+
+    try:
+        with open(_cache_path, "w", encoding="utf-8") as f:
+            json.dump(cache, f, indent=2, ensure_ascii=False)
+    except Exception as exc:
+        _LOGGER.warning("Failed to write WLED cache: %s", exc)

{boneio-1.5.0.dev12 → boneio-1.5.0.dev14}/boneio/schema/remote_devices.yaml

@@ -350,84 +350,8 @@ schema:
           default: 80
           meta:
             label: Port
-        segments:
-          type: list
-          required: False
-          meta:
-            label: Available Segments
-            description: "List of LED segments discovered or manually configured"
-          schema:
-            type: dict
-            schema:
-              id:
-                type: integer
-                required: True
-                meta:
-                  label: Segment ID
-              name:
-                type: string
-                required: False
-                meta:
-                  label: Segment Name
-              start:
-                type: integer
-                required: False
-                meta:
-                  label: Start LED
-              stop:
-                type: integer
-                required: False
-                meta:
-                  label: Stop LED
-              len:
-                type: integer
-                required: False
-                meta:
-                  label: LED Count
-              supports_rgb:
-                type: boolean
-                required: False
-                default: true
-                meta:
-                  label: Supports RGB Color
-        effects:
-          type: list
-          required: False
-          meta:
-            label: Available Effects
-            description: "List of WLED effects discovered from the device"
-          schema:
-            type: dict
-            schema:
-              id:
-                type: integer
-                required: True
-                meta:
-                  label: Effect ID
-              name:
-                type: string
-                required: True
-                meta:
-                  label: Effect Name
-        palettes:
-          type: list
-          required: False
-          meta:
-            label: Available Palettes
-            description: "List of WLED color palettes discovered from the device"
-          schema:
-            type: dict
-            schema:
-              id:
-                type: integer
-                required: True
-                meta:
-                  label: Palette ID
-              name:
-                type: string
-                required: True
-                meta:
-                  label: Palette Name
+        # NOTE: segments, effects, and palettes are auto-discovered from WLED API
+        # and cached in .wled_cache.json — they are NOT stored in config.yaml.
     
     # # Lox Parameters (future)
     # lox:

boneio-1.5.0.dev14/boneio/version.py

@@ -0,0 +1,2 @@
+# flake8: noqa
+__version__ = "1.5.0dev14"

{boneio-1.5.0.dev12 → boneio-1.5.0.dev14}/boneio/webui/app.py

@@ -701,6 +701,12 @@ def init_app(
         _config_cache["mtime"] = _get_config_mtime(yaml_config_file)
         _LOGGER.info("Config cache pre-populated from initial_config")
 
+    # Initialize WLED metadata cache (effects/palettes/segments)
+    from boneio.core.remote.wled_cache import init_cache as init_wled_cache
+
+    config_dir = os.path.dirname(os.path.abspath(yaml_config_file))
+    init_wled_cache(config_dir)
+
     # Add auth middleware if configured
     if auth_config:
         username = auth_config.get("username")