updates2mqtt 1.6.0__py3-none-any.whl

updates2mqtt/__init__.py

@@ -0,0 +1,5 @@
+"""Detect component version change availability and broadcast on MQTT topic, suitable for HomeAssistant autodetect"""
+
+from usingversion import getattr_with_version
+
+__getattr__ = getattr_with_version("updates2mqtt", __file__, __name__)

updates2mqtt/__main__.py

@@ -0,0 +1,6 @@
+"""Detect component version change availability and broadcast on MQTT topic, suitable for HomeAssistant autodetect"""
+
+if __name__ == "__main__":
+    from updates2mqtt.app import run
+
+    run()

updates2mqtt/app.py

@@ -0,0 +1,233 @@
+import asyncio
+import logging
+import sys
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime
+from pathlib import Path
+from threading import Event
+from typing import Any
+
+import structlog
+
+import updates2mqtt
+from updates2mqtt.model import Discovery, ReleaseProvider
+
+from .config import Config, PackageUpdateInfo, load_app_config, load_package_info
+from .integrations.docker import DockerProvider
+from .mqtt import MqttPublisher
+
+log = structlog.get_logger()
+
+CONF_FILE = Path("conf/config.yaml")
+PKG_INFO_FILE = Path("./common_packages.yaml")
+UPDATE_INTERVAL = 60 * 60 * 4
+
+# #TODO:
+#  - Set install in progress
+#  - Support apt
+#  - Retry on registry fetch fail
+#  - Fetcher in subproc or thread
+#  - Clear command message after install
+#  - use git hash as alt to img ref for builds, or daily builds
+
+
+class App:
+    def __init__(self) -> None:
+        self.startup_timestamp: str = datetime.now(UTC).isoformat()
+        self.last_scan_timestamp: str | None = None
+        app_config: Config | None = load_app_config(CONF_FILE)
+        if app_config is None:
+            log.error(f"Invalid configuration at {CONF_FILE}")
+            log.error("Edit config to fix missing or invalid values and restart")
+            log.error("Alternately supply correct MQTT_HOST,MQTT_USER,MQTT_PASSWORD environment variables")
+            log.error("Exiting app")
+            sys.exit(1)
+        self.cfg: Config = app_config
+        self.self_bounce: Event = Event()
+
+        structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, str(self.cfg.log.level))))
+        log.debug("Logging initialized", level=self.cfg.log.level)
+        self.common_pkg: dict[str, PackageUpdateInfo] = load_package_info(PKG_INFO_FILE)
+
+        self.publisher = MqttPublisher(self.cfg.mqtt, self.cfg.node, self.cfg.homeassistant)
+
+        self.scanners: list[ReleaseProvider] = []
+        self.scan_count: int = 0
+        self.last_scan: str | None = None
+        if self.cfg.docker.enabled:
+            self.scanners.append(DockerProvider(self.cfg.docker, self.common_pkg, self.cfg.node, self.self_bounce))
+        self.stopped = Event()
+        self.healthcheck_topic = self.cfg.node.healthcheck.topic_template.format(node_name=self.cfg.node.name)
+
+        log.info(
+            "App configured",
+            node=self.cfg.node.name,
+            scan_interval=self.cfg.scan_interval,
+            healthcheck_topic=self.healthcheck_topic,
+        )
+
+    async def scan(self) -> None:
+        session = uuid.uuid4().hex
+        for scanner in self.scanners:
+            slog = log.bind(source_type=scanner.source_type, session=session)
+            slog.info("Cleaning topics before scan")
+            if self.scan_count == 0:
+                await self.publisher.clean_topics(scanner, None, force=True)
+            if self.stopped.is_set():
+                break
+            slog.info("Scanning ...")
+            async with asyncio.TaskGroup() as tg:
+                # xtype: ignore[attr-defined]
+                async for discovery in scanner.scan(session):
+                    tg.create_task(self.on_discovery(discovery), name=f"discovery-{discovery.name}")
+            if self.stopped.is_set():
+                slog.debug("Breaking scan loop on stopped event")
+                break
+            await self.publisher.clean_topics(scanner, session, force=False)
+            self.scan_count += 1
+            slog.info(f"Scan #{self.scan_count} complete")
+        self.last_scan_timestamp = datetime.now(UTC).isoformat()
+
+    async def main_loop(self) -> None:
+        log.debug("Starting run loop")
+        self.publisher.start()
+
+        if self.cfg.node.healthcheck.enabled:
+            await self.healthcheck()  # initial eager healthcheck
+            log.info(
+                f"Setting up healthcheck every {self.cfg.node.healthcheck.interval} seconds to topic {self.healthcheck_topic}"
+            )
+            self.healthcheck_loop_task = asyncio.create_task(
+                repeated_call(self.healthcheck, interval=self.cfg.node.healthcheck.interval), name="healthcheck"
+            )
+
+        for scanner in self.scanners:
+            self.publisher.subscribe_hass_command(scanner)
+
+        while not self.stopped.is_set() and self.publisher.is_available():
+            await self.scan()
+            if not self.stopped.is_set() and self.publisher.is_available():
+                await asyncio.sleep(self.cfg.scan_interval)
+            else:
+                log.info("Stop requested, exiting run loop and skipping sleep")
+
+        if not self.publisher.is_available():
+            log.error("MQTT fatal connection error - check host,port,user,password in config")
+            self.shutdown(exit_code=1)
+
+        log.debug("Exiting run loop")
+
+    async def on_discovery(self, discovery: Discovery) -> None:
+        dlog = log.bind(name=discovery.name)
+        try:
+            if self.cfg.homeassistant.discovery.enabled:
+                self.publisher.publish_hass_config(discovery)
+
+            self.publisher.publish_hass_state(discovery)
+            if (
+                discovery.update_policy == "Auto"
+                and discovery.can_update
+                and discovery.latest_version != discovery.current_version
+            ):
+                # TODO: review auto update, trigger by version, use update interval as throttle
+                elapsed: float = (
+                    time.time() - discovery.update_last_attempt if discovery.update_last_attempt is not None else -1
+                )
+                if elapsed == -1 or elapsed > UPDATE_INTERVAL:
+                    dlog.info(
+                        "Initiate auto update (last:%s, elapsed:%s, max:%s)",
+                        discovery.update_last_attempt,
+                        elapsed,
+                        UPDATE_INTERVAL,
+                    )
+                    self.publisher.local_message(discovery, "install")
+                else:
+                    dlog.info("Skipping auto update")
+        except asyncio.CancelledError:
+            dlog.info("Discovery handling cancelled")
+        except Exception:
+            dlog.exception("Discovery handling failed")
+            raise
+
+    async def interrupt_tasks(self) -> None:
+        running_tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
+        log.info(f"Cancelling {len(running_tasks)} tasks")
+        for t in running_tasks:
+            log.debug("Cancelling task", task=t.get_name())
+            if t.get_name() == "healthcheck" or t.get_name().startswith("discovery-"):
+                t.cancel()
+        await asyncio.gather(*running_tasks, return_exceptions=True)
+        log.debug("Cancellation task completed")
+
+    def shutdown(self, *args, exit_code: int = 143) -> None:  # noqa: ANN002, ARG002
+        if self.self_bounce.is_set():
+            exit_code = 1
+            log.info("Self bouncing, overriding exit_code: %s", exit_code)
+        else:
+            log.info("Shutting down, exit_code: %s", exit_code)
+        self.stopped.set()
+        for scanner in self.scanners:
+            scanner.stop()
+        interrupt_task = asyncio.get_event_loop().create_task(
+            self.interrupt_tasks(),
+            eager_start=True,  # type: ignore[call-arg] # pyright: ignore[reportCallIssue]
+            name="interrupt",
+        )
+        for t in asyncio.all_tasks():
+            log.debug("Tasks waiting = %s", t)
+        self.publisher.stop()
+        log.debug("Interrupt: %s", interrupt_task.done())
+        log.info("Shutdown handling complete")
+        sys.exit(exit_code)  # SIGTERM Graceful Exit = 143
+
+    async def healthcheck(self) -> None:
+        if not self.publisher.is_available():
+            return
+        heartbeat_stamp: str = datetime.now(UTC).isoformat()
+        log.debug("Publishing health check", heartbeat_stamp=heartbeat_stamp)
+        self.publisher.publish(
+            topic=self.healthcheck_topic,
+            payload={
+                "version": updates2mqtt.version,  # pyright: ignore[reportAttributeAccessIssue]
+                "node": self.cfg.node.name,
+                "heartbeat_raw": time.time(),
+                "heartbeat_stamp": heartbeat_stamp,
+                "startup_stamp": self.startup_timestamp,
+                "last_scan_stamp": self.last_scan_timestamp,
+                "scan_count": self.scan_count,
+            },
+        )
+
+
+async def repeated_call(func: Callable, interval: int = 60, *args: Any, **kwargs: Any) -> None:
+    # run a task periodically indefinitely
+    while True:
+        try:
+            await func(*args, **kwargs)
+            await asyncio.sleep(interval)
+        except asyncio.CancelledError:
+            log.debug("Periodic task cancelled", func=func)
+        except Exception:
+            log.exception("Periodic task failed")
+
+
+def run() -> None:
+    import asyncio
+    import signal
+
+    # pyright: ignore[reportAttributeAccessIssue]
+    log.debug(f"Starting updates2mqtt v{updates2mqtt.version}")  # pyright: ignore[reportAttributeAccessIssue]
+    app = App()
+
+    signal.signal(signal.SIGTERM, app.shutdown)
+    try:
+        asyncio.run(app.main_loop(), debug=False)
+        log.debug("App exited gracefully")
+    except asyncio.CancelledError:
+        log.debug("App exited on cancelled task")
+
+
+if __name__ == "__main__":
+    run()

updates2mqtt/config.py

@@ -0,0 +1,176 @@
+import os
+import typing
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+
+import structlog
+from omegaconf import MISSING, DictConfig, MissingMandatoryValue, OmegaConf, ValidationError
+
+log = structlog.get_logger()
+
+
+class LogLevel(StrEnum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+@dataclass
+class MqttConfig:
+    host: str = "${oc.env:MQTT_HOST,localhost}"
+    user: str = f"${{oc.env:MQTT_USER,{MISSING}}}"
+    password: str = f"${{oc.env:MQTT_PASS,{MISSING}}}"
+    port: int = "${oc.decode:${oc.env:MQTT_PORT,1883}}"  # type: ignore[assignment]
+    topic_root: str = "updates2mqtt"
+    protocol: str = "${oc.env:MQTT_VERSION,3.11}"
+
+
+@dataclass
+class MetadataSourceConfig:
+    enabled: bool = True
+    cache_ttl: int = 60 * 60 * 24 * 7  # 1 week
+
+
+@dataclass
+class DockerConfig:
+    enabled: bool = True
+    allow_pull: bool = True
+    allow_restart: bool = True
+    allow_build: bool = True
+    compose_version: str = "v2"
+    default_entity_picture_url: str = "https://www.docker.com/wp-content/uploads/2022/03/Moby-logo.png"
+    # Icon to show when browsing entities in Home Assistant
+    device_icon: str = "mdi:docker"
+    discover_metadata: dict[str, MetadataSourceConfig] = field(
+        default_factory=lambda: {"linuxserver.io": MetadataSourceConfig(enabled=True)}
+    )
+    api_throttle_wait: int = 60 * 15
+
+
+@dataclass
+class HomeAssistantDiscoveryConfig:
+    prefix: str = "homeassistant"
+    enabled: bool = True
+
+
+@dataclass
+class HomeAssistantConfig:
+    discovery: HomeAssistantDiscoveryConfig = field(default_factory=HomeAssistantDiscoveryConfig)
+    state_topic_suffix: str = "state"
+    device_creation: bool = True
+    force_command_topic: bool = False
+    area: str | None = None
+
+
+@dataclass
+class HealthCheckConfig:
+    enabled: bool = True
+    interval: int = 300  # Interval in seconds to publish healthcheck message, 0 to disable
+    topic_template: str = "healthcheck/{node_name}/updates2mqtt"
+
+
+@dataclass
+class NodeConfig:
+    name: str = field(default_factory=lambda: os.uname().nodename.replace(".local", ""))
+    git_path: str = "/usr/bin/git"
+    healthcheck: HealthCheckConfig = field(default_factory=HealthCheckConfig)
+
+
+@dataclass
+class LogConfig:
+    level: LogLevel = "${oc.decode:${oc.env:U2M_LOG_LEVEL,INFO}}"  # type: ignore[assignment] # pyright: ignore[reportAssignmentType]
+
+
+@dataclass
+class Config:
+    log: LogConfig = field(default_factory=LogConfig)  # pyright: ignore[reportArgumentType, reportCallIssue]
+    node: NodeConfig = field(default_factory=NodeConfig)
+    mqtt: MqttConfig = field(default_factory=MqttConfig)  # pyright: ignore[reportArgumentType, reportCallIssue]
+    homeassistant: HomeAssistantConfig = field(default_factory=HomeAssistantConfig)
+    docker: DockerConfig = field(default_factory=DockerConfig)
+    scan_interval: int = 60 * 60 * 3
+
+
+@dataclass
+class DockerPackageUpdateInfo:
+    image_name: str = MISSING
+
+
+@dataclass
+class PackageUpdateInfo:
+    docker: DockerPackageUpdateInfo | None = field(default_factory=DockerPackageUpdateInfo)
+    logo_url: str | None = None
+    release_notes_url: str | None = None
+
+
+@dataclass
+class UpdateInfoConfig:
+    common_packages: dict[str, PackageUpdateInfo] = field(default_factory=lambda: {})
+
+
+class IncompleteConfigException(BaseException):
+    pass
+
+
+def load_package_info(pkginfo_file_path: Path) -> dict[str, PackageUpdateInfo]:
+    if pkginfo_file_path.exists():
+        log.debug("Loading common package update info", path=pkginfo_file_path)
+        cfg = OmegaConf.load(pkginfo_file_path)
+    else:
+        log.warn("No common package update info found", path=pkginfo_file_path)
+        cfg = OmegaConf.structured(UpdateInfoConfig)
+    try:
+        # omegaconf broken-ness on optional fields and converting to backclasses
+        pkg_conf: dict[str, PackageUpdateInfo] = {
+            pkg: PackageUpdateInfo(**pkg_cfg) for pkg, pkg_cfg in cfg.common_packages.items()
+        }
+        return pkg_conf
+    except (MissingMandatoryValue, ValidationError) as e:
+        log.error("Configuration error %s", e, path=pkginfo_file_path.as_posix())
+        raise
+
+
+def is_autogen_config() -> bool:
+    env_var: str | None = os.environ.get("U2M_AUTOGEN_CONFIG")
+    return not (env_var and env_var.lower() in ("no", "0", "false"))
+
+
+def load_app_config(conf_file_path: Path, return_invalid: bool = False) -> Config | None:
+    base_cfg: DictConfig = OmegaConf.structured(Config)
+    if conf_file_path.exists():
+        cfg: DictConfig = typing.cast("DictConfig", OmegaConf.merge(base_cfg, OmegaConf.load(conf_file_path)))
+    elif is_autogen_config():
+        if not conf_file_path.parent.exists():
+            try:
+                log.debug(f"Creating config directory {conf_file_path.parent} if not already present")
+                conf_file_path.parent.mkdir(parents=True, exist_ok=True)
+            except Exception:
+                log.warning("Unable to create config directory", path=conf_file_path.parent)
+        try:
+            conf_file_path.write_text(OmegaConf.to_yaml(base_cfg))
+            log.info(f"Auto-generated a new config file at {conf_file_path}")
+        except Exception:
+            log.warning("Unable to write config file", path=conf_file_path)
+        cfg = base_cfg
+    else:
+        cfg = base_cfg
+
+    try:
+        # Validate that all required fields are present, throw exception now rather than when config first used
+        OmegaConf.to_container(cfg, throw_on_missing=True)
+        OmegaConf.set_readonly(cfg, True)
+        config: Config = typing.cast("Config", cfg)
+
+        if config.mqtt.user in ("", MISSING) or config.mqtt.password in ("", MISSING):
+            log.info("The config has place holders for MQTT user and/or password")
+            if not return_invalid:
+                return None
+        return config
+    except (MissingMandatoryValue, ValidationError) as e:
+        log.error("Configuration error %s", e, path=conf_file_path.as_posix())
+        if return_invalid and cfg is not None:
+            return typing.cast("Config", cfg)
+        raise

updates2mqtt/hass_formatter.py

@@ -0,0 +1,89 @@
+from typing import Any
+
+import structlog
+
+import updates2mqtt
+from updates2mqtt.model import Discovery
+
+log = structlog.get_logger()
+HASS_UPDATE_SCHEMA = [
+    "installed_version",
+    "latest_version",
+    "title",
+    "release_summary",
+    "release_url",
+    "entity_picture",
+    "in_progress",
+    "update_percentage",
+]
+
+
+def hass_format_config(
+    discovery: Discovery,
+    object_id: str,
+    state_topic: str,
+    command_topic: str | None,
+    force_command_topic: bool | None,
+    device_creation: bool = True,
+    area: str | None = None,
+    session: str | None = None,
+) -> dict[str, Any]:
+    config: dict[str, Any] = {
+        "name": discovery.title,
+        "device_class": None,  # not firmware, so defaults to null
+        "unique_id": object_id,
+        "state_topic": state_topic,
+        "source_session": session,
+        "supported_features": discovery.features,
+        "can_update": discovery.can_update,
+        "can_build": discovery.can_build,
+        "can_restart": discovery.can_restart,
+        "update_policy": discovery.update_policy,
+        "origin": {
+            "name": f"{discovery.node} updates2mqtt",
+            "sw_version": updates2mqtt.version,  # pyright: ignore[reportAttributeAccessIssue]
+            "support_url": "https://github.com/rhizomatics/updates2mqtt/issues",
+        },
+    }
+    if discovery.entity_picture_url:
+        config["entity_picture"] = discovery.entity_picture_url
+    if discovery.device_icon:
+        config["icon"] = discovery.device_icon
+    if device_creation:
+        config["device"] = {
+            "name": f"{discovery.node} updates2mqtt",
+            "sw_version": updates2mqtt.version,  # pyright: ignore[reportAttributeAccessIssue]
+            "manufacturer": "rhizomatics",
+            "identifiers": [f"{discovery.node}.updates2mqtt"],
+        }
+        if area:
+            config["device"]["suggested_area"] = area
+    if command_topic and (discovery.can_update or force_command_topic):
+        config["command_topic"] = command_topic
+        if discovery.can_update:
+            config["payload_install"] = f"{discovery.source_type}|{discovery.name}|install"
+    config["custom"] = {}
+    config["custom"][discovery.source_type] = discovery.custom
+    config.update(discovery.provider.hass_config_format(discovery))
+    return config
+
+
+def hass_format_state(discovery: Discovery, session: str, in_progress: bool = False) -> dict[str, Any]:  # noqa: ARG001
+    state = {
+        "installed_version": discovery.current_version,
+        "latest_version": discovery.latest_version,
+        "title": discovery.title,
+        "in_progress": in_progress,
+    }
+    if discovery.release_summary:
+        state["release_summary"] = discovery.release_summary
+    if discovery.release_url:
+        state["release_url"] = discovery.release_url
+    custom_state = discovery.provider.hass_state_format(discovery)
+    if custom_state:
+        state.update(custom_state)
+    invalid_keys = [k for k in state if k not in HASS_UPDATE_SCHEMA]
+    if invalid_keys:
+        log.warning(f"Invalid keys in state: {invalid_keys}")
+        state = {k: v for k, v in state.items() if k in HASS_UPDATE_SCHEMA}
+    return state

updates2mqtt/integrations/__init__.py

@@ -0,0 +1 @@
+"""System integrations to detect component version availability and optionally execute update and restart"""