boneio 1.6.0.dev15__tar.gz → 1.6.0.dev17__tar.gz

{boneio-1.6.0.dev15/boneio.egg-info → boneio-1.6.0.dev17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boneio
-Version: 1.6.0.dev15
+Version: 1.6.0.dev17
 Summary: Python App for BoneIO
 Author-email: Paweł Szafer <pszafer@gmail.com>
 License: GNU General Public License v3.0

{boneio-1.6.0.dev15 → boneio-1.6.0.dev17}/boneio/core/config/migrations/__init__.py

@@ -19,7 +19,7 @@ from boneio.core.atomic_file import write_atomically
 _LOGGER = logging.getLogger(__name__)
 
 # Current schema version — bump this when adding new migrations
-CURRENT_SCHEMA_VERSION = 6
+CURRENT_SCHEMA_VERSION = 7
 
 # Minimum app version that introduced each schema version.
 # Used by the WebUI to warn before rolling back to an incompatible version.
@@ -31,6 +31,7 @@ SCHEMA_VERSION_APP_MAP: dict[int, str] = {
     4: "1.5.0dev13",  # WLED effects/palettes → JSON cache
     5: "1.5.0dev35",  # OLED screen name: ina219 → ina
     6: "1.6.0dev15",  # gpio_mode, clear_message removed from inputs
+    7: "1.6.0dev16",  # MQTT password moved to secrets.yaml
 }
 
 
@@ -165,6 +166,16 @@ def _has_legacy_fields(doc: dict) -> bool:
         ):
             return True
 
+    # v7: a broker password written in the config itself. A value that came
+    # through !secret is a SecretStr, so only a plain one counts.
+    from boneio.core.config.yaml_util import SecretStr
+
+    mqtt = doc.get("mqtt")
+    if isinstance(mqtt, dict):
+        password = mqtt.get("password")
+        if password not in (None, "") and not isinstance(password, SecretStr):
+            return True
+
     return False
 
 
@@ -322,4 +333,5 @@ from boneio.core.config.migrations import (
     v4_wled_cache,  # noqa: E402, F401
     v5_ina_screen,  # noqa: E402, F401
     v6_input_keys,  # noqa: E402, F401
+    v7_mqtt_password_secret,  # noqa: E402, F401
 )

boneio-1.6.0.dev17/boneio/core/config/migrations/v7_mqtt_password_secret.py

@@ -0,0 +1,54 @@
+"""Migration v7: Keep the broker password in secrets.yaml.
+
+Images draw the broker password per device at first boot and write it to
+secrets.yaml, so the shipped mqtt.yaml says ``password: !secret mqtt_password``.
+Controllers installed before that keep the password in mqtt.yaml itself, which
+then travels with every config backup and every copy someone pastes into a
+support thread. This moves it next to the device's other secrets.
+
+Before (mqtt.yaml):
+    host: localhost
+    password: "s3cret"
+After:
+    host: localhost
+    password: !secret mqtt_password
+and secrets.yaml:
+    mqtt_password: "s3cret"
+
+A password already given as ``!secret`` is left alone. If secrets.yaml already
+holds a different ``mqtt_password``, the value goes under ``mqtt_password_2``.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from boneio.core.config.migrations import register_migration
+
+_LOGGER = logging.getLogger(__name__)
+
+_FIELD = ("mqtt", "password")
+_SECRET_NAME = "mqtt_password"
+
+
+def _persist_mqtt_password_secret(config_file: str) -> None:
+    from boneio.core.config.yaml_patch import move_to_secret
+
+    move_to_secret(config_file, _FIELD, _SECRET_NAME)
+
+
+@register_migration(
+    version=7,
+    name="Move the MQTT password to secrets.yaml",
+    migrate_file=_persist_mqtt_password_secret,
+)
+def migrate_v7_mqtt_password_secret(doc: dict) -> dict:
+    """Nothing changes in memory: the password is the same, only where it is kept.
+
+    Args:
+        doc: Raw config dictionary.
+
+    Returns:
+        The same dictionary.
+    """
+    return doc

{boneio-1.6.0.dev15 → boneio-1.6.0.dev17}/boneio/core/config/yaml_patch.py

@@ -587,3 +587,79 @@ def set_secret(secrets_file: str | Path, name: str, value: str) -> None:
 
     write_atomically(file_path, "".join(lines))
     _LOGGER.info("Wrote the secret %s", name)
+
+
+@_locked
+def move_to_secret(
+    config_file: str | Path, path: tuple[str, ...], name: str, *, overwrite: bool = False
+) -> str | None:
+    """Move a plain value into secrets.yaml and leave a ``!secret`` in its place.
+
+    The secrets file is the one beside the file holding the field — that is
+    where BoneIOLoader looks for it — and is created, 0600, when missing. A
+    name already taken by a different value is not overwritten; the value goes
+    under ``<name>_2``, ``<name>_3``… instead.
+
+    Args:
+        config_file: Path to config.yaml.
+        path: Field path, outermost first, e.g. ``("mqtt", "password")``.
+        name: The secret's preferred name.
+        overwrite: Store the value under ``name`` whatever it held — for a
+            field that referred to that secret before a save wrote it out.
+
+    Returns:
+        The name the value went under, or None when there was nothing to move:
+        the field is missing, empty, or already a reference.
+
+    Raises:
+        YamlPatchError: If secrets.yaml is there but is not a mapping — adding
+            a line to a file the loader cannot read would hide the value.
+    """
+    import os
+
+    import yaml
+
+    file_path, inner = resolve_field(config_file, path)
+    if not file_path.is_file():
+        return None
+    lines = file_path.read_text(encoding="utf-8").splitlines(keepends=True)
+    index = _find_section_line(lines, inner)
+    if index is None:
+        return None
+    parsed = _line_key(lines[index])
+    if parsed is None or _SECRET_REFERENCE.match(_strip_comment(parsed[2])):
+        return None
+    try:
+        value = yaml.safe_load(parsed[2]) if parsed[2] else None
+    except yaml.YAMLError as err:
+        raise YamlPatchError(f"{file_path.name}: cannot read {'.'.join(inner)}") from err
+    if value is None or value == "" or isinstance(value, (dict, list)):
+        return None
+    value = str(value)
+
+    secrets_file = file_path.parent / "secrets.yaml"
+    existing: dict = {}
+    if secrets_file.is_file():
+        try:
+            loaded = yaml.safe_load(secrets_file.read_text(encoding="utf-8"))
+        except yaml.YAMLError as err:
+            raise YamlPatchError(f"{secrets_file} is not valid YAML") from err
+        if loaded is not None and not isinstance(loaded, dict):
+            raise YamlPatchError(f"{secrets_file} is not a mapping")
+        existing = loaded or {}
+    else:
+        write_atomically(secrets_file, "", mode=0o600)
+
+    chosen, suffix = name, 1
+    while not overwrite and chosen in existing and str(existing[chosen]) != value:
+        suffix += 1
+        chosen = f"{name}_{suffix}"
+    if chosen not in existing or str(existing[chosen]) != value:
+        set_secret(secrets_file, chosen, value)
+    # It holds a password now, whatever it held before.
+    os.chmod(secrets_file, 0o600)
+
+    lines[index] = f"{' ' * parsed[1]}{inner[-1]}: !secret {chosen}\n"
+    write_atomically(file_path, "".join(lines))
+    _LOGGER.info("Moved %s in %s to secrets.yaml as %s", ".".join(inner), file_path.name, chosen)
+    return chosen

{boneio-1.6.0.dev15 → boneio-1.6.0.dev17}/boneio/core/config/yaml_util.py

@@ -1,4 +1,5 @@
 import contextlib
+import copy
 import fnmatch
 import logging
 import os
@@ -12,6 +13,7 @@ from typing import Any
 
 from cerberus import TypeDefinition, Validator
 from cerberus.schema import DefinitionSchema
+import yaml
 from yaml import MarkedYAMLError, YAMLError, dump, load
 
 from boneio.const import OUTPUT, VIRTUAL_SWITCH
@@ -28,6 +30,44 @@ _LOGGER = logging.getLogger(__name__)
 
 SECRET_YAML = "secrets.yaml"
 _SECRET_VALUES = {}
+#: Names of every secret a YAML load resolved, whatever its type — so the cache
+#: can tell a secret it kept as a reference from one validation turned into a
+#: number and it had to keep by value.
+_SECRET_NAMES_LOADED: set[str] = set()
+
+
+class SecretStr(str):
+    """A string that came from ``!secret``, and knows which secret it was.
+
+    The config cache holds the validated config. It used to hold it with every
+    secret already substituted — the passwords sat in ``.cache.pkl``, and
+    secrets.yaml was not part of the cache key, so changing a password there did
+    nothing until some other edit invalidated the cache. The cache now stores a
+    :class:`_SecretRef` in place of each ``SecretStr`` and resolves it against
+    secrets.yaml as it is read back. Validation keeps the subclass (it checks
+    type, it does not copy), which is what lets the cache find them.
+    """
+
+    secret_name: str
+
+    def __new__(cls, value: str, name: str) -> "SecretStr":
+        obj = super().__new__(cls, value)
+        obj.secret_name = name
+        return obj
+
+    def __reduce__(self):
+        # Anything else that pickles a config gets the plain value, never a
+        # boneIO type that another build may not define.
+        return (str, (str(self),))
+
+
+class _SecretRef:
+    """What the config cache stores instead of a secret's value."""
+
+    __slots__ = ("name",)
+
+    def __init__(self, name: str) -> None:
+        self.name = name
 
 #: Cerberus checks that the schema is itself a legal cerberus schema every time
 #: a Validator is built, and that - not validating anybody's config - is where a
@@ -342,6 +382,17 @@ def represent_time_period(dumper, data):
 TimePeriodDumper.add_representer(TimePeriod, represent_time_period)
 
 
+def _represent_secret_str(dumper, data):
+    # Written as the plain string it always was: a SecretStr only exists so the
+    # config cache can keep secrets out of .cache.pkl, and must not change what
+    # a save puts in a YAML file.
+    return dumper.represent_scalar("tag:yaml.org,2002:str", str(data))
+
+
+for _dumper in {TimePeriodDumper, FastSafeDumper, yaml.SafeDumper, yaml.Dumper}:
+    _dumper.add_representer(SecretStr, _represent_secret_str)
+
+
 class BoneIOLoader(FastSafeLoader):
     """Loader which support for include in yaml files."""
 
@@ -365,7 +416,8 @@ class BoneIOLoader(FastSafeLoader):
             raise MarkedYAMLError(f"Secret '{node.value}' not defined", node.start_mark)
         val = secrets[node.value]
         _SECRET_VALUES[str(val)] = node.value
-        return val
+        _SECRET_NAMES_LOADED.add(node.value)
+        return SecretStr(val, node.value) if isinstance(val, str) else val
 
     def represent_stringify(self, value):
         # Type: ignore[attr-defined] - represent_scalar is inherited from the loader base
@@ -1432,6 +1484,60 @@ def _compute_config_dir_hash(config_file: str) -> str:
     return h.hexdigest()
 
 
+def _secret_digest(value: Any) -> str:
+    import hashlib
+
+    return hashlib.sha256(repr(value).encode("utf-8")).hexdigest()
+
+
+def _secrets_to_refs(node: Any, found: set[str]) -> Any:
+    """A copy of *node* with every :class:`SecretStr` replaced by a reference."""
+    if isinstance(node, SecretStr):
+        found.add(node.secret_name)
+        return _SecretRef(node.secret_name)
+    if isinstance(node, dict):
+        out = copy.copy(node)
+        for key, value in node.items():
+            out[key] = _secrets_to_refs(value, found)
+        return out
+    if isinstance(node, list):
+        return [_secrets_to_refs(item, found) for item in node]
+    if isinstance(node, tuple):
+        return tuple(_secrets_to_refs(item, found) for item in node)
+    return node
+
+
+def _refs_to_secrets(node: Any, secrets: dict) -> Any:
+    """Resolve the cache's secret references against secrets.yaml, in place.
+
+    Raises:
+        KeyError: A reference names a secret secrets.yaml no longer defines.
+    """
+    if isinstance(node, _SecretRef):
+        value = secrets[node.name]
+        _SECRET_VALUES[str(value)] = node.name
+        return SecretStr(value, node.name) if isinstance(value, str) else value
+    if isinstance(node, dict):
+        for key in list(node):
+            node[key] = _refs_to_secrets(node[key], secrets)
+        return node
+    if isinstance(node, list):
+        for index, item in enumerate(node):
+            node[index] = _refs_to_secrets(item, secrets)
+        return node
+    if isinstance(node, tuple):
+        return tuple(_refs_to_secrets(item, secrets) for item in node)
+    return node
+
+
+def _load_secrets_for(config_file: str) -> dict:
+    path = os.path.join(os.path.dirname(config_file), SECRET_YAML)
+    if not os.path.isfile(path):
+        return {}
+    secrets = load_yaml_file(path)
+    return secrets if isinstance(secrets, dict) else {}
+
+
 def _try_load_cached_config(config_file: str) -> dict | None:
     """Try to load validated config from cache.
 
@@ -1499,12 +1605,31 @@ def _try_load_cached_config(config_file: str) -> dict | None:
                 _LOGGER.debug("Schema file changed, cache invalidated")
                 return None
 
+            # A secret validation turned into something other than a string is
+            # in the payload by value, so its current value has to match.
+            baked = header.get("baked_secrets", {})
+            secrets = _load_secrets_for(config_file) if baked or header.get("secret_refs") else {}
+            for name, digest in baked.items():
+                # A name the loader saw but secrets.yaml lacks was recorded as
+                # the digest of None; lacking it still is not a change.
+                if _secret_digest(secrets.get(name)) != digest:
+                    _LOGGER.debug("Secret %r changed, cache invalidated", name)
+                    return None
+
             data = pickle.load(f)
 
         if not isinstance(data, dict):
             _LOGGER.warning("Config cache %s holds no config, revalidating", cache_path)
             return None
 
+        try:
+            data = _refs_to_secrets(data, secrets)
+        except KeyError as missing:
+            _LOGGER.info(
+                "secrets.yaml no longer defines %s, revalidating config", missing
+            )
+            return None
+
         _LOGGER.debug("Loading validated config from cache (skipping Cerberus validation)")
         return data
     except FileNotFoundError:
@@ -1536,16 +1661,30 @@ def _save_config_cache(config_file: str, validated_config: dict) -> None:
 
     cache_path = _get_config_cache_path(config_file)
     try:
+        # No secret's value goes into the payload when it can be avoided:
+        # strings from !secret become references, resolved from secrets.yaml
+        # when the cache is read. A secret validation coerced (a port given as
+        # !secret, say) stays by value, and its digest in the header makes a
+        # change to it invalidate the cache instead of going unnoticed.
+        as_refs: set[str] = set()
+        payload = _secrets_to_refs(validated_config, as_refs)
+        baked: dict[str, str] = {}
+        leftover = _SECRET_NAMES_LOADED - as_refs
+        if leftover:
+            secrets = _load_secrets_for(config_file)
+            baked = {name: _secret_digest(secrets.get(name)) for name in sorted(leftover)}
         header = {
             "version": __version__,
             "config_hash": _compute_config_dir_hash(config_file),
             "schema_hash": _compute_file_hash(schema_file),
+            "secret_refs": sorted(as_refs),
+            "baked_secrets": baked,
         }
         with open(cache_path, "wb") as f:
             # Header first, payload second, so the reader can reject a cache
             # from another build without unpickling boneIO objects at all.
             pickle.dump(header, f, protocol=pickle.HIGHEST_PROTOCOL)
-            pickle.dump(validated_config, f, protocol=pickle.HIGHEST_PROTOCOL)
+            pickle.dump(payload, f, protocol=pickle.HIGHEST_PROTOCOL)
         _LOGGER.debug("Saved validated config cache to %s", cache_path)
     except (OSError, pickle.PicklingError) as e:
         _LOGGER.debug("Could not save config cache: %s", e)
@@ -1778,9 +1917,16 @@ def _full_config_validation(
     if not _is_downgraded:
         _check_virtual_switch_references(merged_doc)
 
-    # Save to cache for next startup
-    _progress("Saving cache...")
-    _save_config_cache(config_file, merged_doc)
+    # Save to cache for next startup — unless a migration just rewrote the
+    # files. The document in hand was read before that, so it no longer says
+    # what they say (v7 moves a password behind !secret: cached from here, it
+    # would sit in .cache.pkl by value and a later change in secrets.yaml would
+    # not be seen). The next start reads the files as they are now.
+    if migrations_applied:
+        _LOGGER.info("Config migrated; the cache is written on the next start")
+    else:
+        _progress("Saving cache...")
+        _save_config_cache(config_file, merged_doc)
 
     return merged_doc
 
@@ -1812,6 +1958,7 @@ def load_config_from_file(
 
     # Cache miss — full validation (slow path)
     _LOGGER.info("Config cache miss, running full validation...")
+    _SECRET_NAMES_LOADED.clear()
     if progress_callback:
         progress_callback("Parsing YAML...")
     try:
@@ -1951,6 +2098,52 @@ def strip_default_values(data: Any, schema: dict | None = None, section: str | N
         return data
 
 
+def _section_secret_refs(config_file: str, section: str, data: Any) -> dict[str, str]:
+    """Which of a section's fields defer to secrets.yaml, before a save.
+
+    A save dumps the section from the values the panel sent back, and the
+    dumper writes a secret as the plain string it is. Recorded here, each
+    reference is put back afterwards by :func:`_keep_secret_refs`.
+    """
+    if not isinstance(data, dict):
+        return {}
+    from boneio.core.config.yaml_patch import secret_reference
+
+    refs: dict[str, str] = {}
+    for key in data:
+        try:
+            ref = secret_reference(config_file, (section, str(key)))
+        except OSError:
+            continue
+        if ref is not None:
+            refs[str(key)] = ref
+    return refs
+
+
+def _keep_secret_refs(config_file: str, section: str, data: Any, refs: dict[str, str]) -> None:
+    """Move secret values a save wrote out back into secrets.yaml.
+
+    Every field that was ``!secret name`` gets its (possibly new) value stored
+    under that name and the reference back. The broker password goes to
+    secrets.yaml even when it was plain, as migration v7 does on upgrade.
+    """
+    if not isinstance(data, dict):
+        return
+    from boneio.core.config.yaml_patch import YamlPatchError, move_to_secret
+
+    fields = dict(refs)
+    if section == "mqtt" and "password" not in fields:
+        fields["password"] = "mqtt_password"
+    for key, name in fields.items():
+        value = data.get(key)
+        if not isinstance(value, str) or not value:
+            continue
+        try:
+            move_to_secret(config_file, (section, key), name, overwrite=key in refs)
+        except (OSError, YamlPatchError) as err:
+            _LOGGER.error("%s.%s saved in the config instead of secrets.yaml: %s", section, key, err)
+
+
 def update_config_section(config_file: str, section: str, data: dict | list) -> dict:
     """
     Update content of a configuration section with intelligent !include handling.
@@ -1973,6 +2166,8 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
             section, data_size,
         )
 
+        secret_refs = _section_secret_refs(config_file, section, data)
+
         # Special handling for mcp23017 - convert hex strings to integers
         # This ensures YAML writes them as integers which are then read back as hex
         if section == "mcp23017" and isinstance(data, list):
@@ -2145,6 +2340,8 @@ def update_config_section(config_file: str, section: str, data: dict | list) ->
 
                 _LOGGER.info(f"Successfully added new section '{section}' to config.yaml")
 
+            _keep_secret_refs(config_file, section, data, secret_refs)
+
             t_total = time.perf_counter() - t_start
             _LOGGER.info(
                 "[SAVE] DONE section='%s' total=%.3fs",

{boneio-1.6.0.dev15 → boneio-1.6.0.dev17}/boneio/core/manager/display.py

@@ -316,8 +316,8 @@ class DisplayManager:
             self._oled._sleep_timeout = TimePeriod(seconds=int(timeout_seconds))
 
             # Reset to first screen and refresh
-            if self._configured_screen_order:
-                self._oled._current_screen = self._configured_screen_order[0]
+            # The notice, if one stands, stays first.
+            self._oled._current_screen = self._oled.first_screen()
             self._oled.render_display()
 
             _LOGGER.info(