updates2mqtt 1.7.3__py3-none-any.whl → 1.8.1__py3-none-any.whl

updates2mqtt/app.py

@@ -21,7 +21,6 @@ from .mqtt import MqttPublisher
 log = structlog.get_logger()
 
 CONF_FILE = Path("conf/config.yaml")
-PKG_INFO_FILE = Path("./common_packages.yaml")
 UPDATE_INTERVAL = 60 * 60 * 4
 
 # #TODO:
@@ -56,7 +55,15 @@ class App:
         self.scan_count: int = 0
         self.last_scan: str | None = None
         if self.cfg.docker.enabled:
-            self.scanners.append(DockerProvider(self.cfg.docker, self.cfg.node, self.self_bounce))
+            self.scanners.append(
+                DockerProvider(
+                    self.cfg.docker,
+                    self.cfg.node,
+                    packages=self.cfg.packages,
+                    github_cfg=self.cfg.github,
+                    self_bounce=self.self_bounce,
+                )
+            )
         self.stopped = Event()
         self.healthcheck_topic = self.cfg.node.healthcheck.topic_template.format(node_name=self.cfg.node.name)
 
@@ -71,9 +78,6 @@ class App:
         session = uuid.uuid4().hex
         for scanner in self.scanners:
             slog = log.bind(source_type=scanner.source_type, session=session)
-            slog.info("Cleaning topics before scan")
-            if self.scan_count == 0:
-                await self.publisher.clean_topics(scanner, None, force=True)
             if self.stopped.is_set():
                 break
             slog.info("Scanning ...")
@@ -84,7 +88,7 @@ class App:
             if self.stopped.is_set():
                 slog.debug("Breaking scan loop on stopped event")
                 break
-            await self.publisher.clean_topics(scanner, session, force=False)
+            await self.publisher.clean_topics(scanner)
             self.scan_count += 1
             slog.info(f"Scan #{self.scan_count} complete")
         self.last_scan_timestamp = datetime.now(UTC).isoformat()

updates2mqtt/cli.py

@@ -4,7 +4,7 @@ import structlog
 from omegaconf import DictConfig, OmegaConf
 from rich import print_json
 
-from updates2mqtt.config import DockerConfig, NodeConfig, RegistryConfig
+from updates2mqtt.config import DockerConfig, GitHubConfig, NodeConfig, RegistryConfig
 from updates2mqtt.helpers import Throttler
 from updates2mqtt.integrations.docker import DockerProvider
 from updates2mqtt.integrations.docker_enrich import (
@@ -24,18 +24,20 @@ log = structlog.get_logger()
 """
 Super simple CLI
 
-python updates2mqtt.cli container=frigate
+Command can be `container`,`tags`,`manifest` or `blob`
 
-python updates2mqtt.cli container=frigate api=docker_client log_level=DEBUG
+* `container=container-name`
+* `container=hash`
+* `tags=ghcr.io/
+* `blob=mcr.microsoft.com/dotnet/sdk:latest`
+* `tags=quay.io/linuxserver.io/babybuddy`
+* `blob=ghcr.io/blakeblackshear/frigate@sha256:759c36ee869e3e60258350a2e221eae1a4ba1018613e0334f1bc84eb09c4bbbc`
 
-ython3 updates2mqtt/cli.py blob=ghcr.io/homarr-labs/homarr@sha256:af79a3339de5ed8ef7f5a0186ff3deb86f40b213ba75249291f2f68aef082a25 | jq '.config.Labels'
+In addition, a `log_level=DEBUG` or other level can be added, `github_token` to try a personal access
+token for GitHub release info retrieval, or `api=docker_client` to use the older API (defaults to `api=OCI_V2`)
 
-python3 updates2mqtt/cli.py manifest=ghcr.io/blakeblackshear/frigate:stable
 
-python3 updates2mqtt/cli.py blob=ghcr.io/blakeblackshear/frigate@sha256:ef8d56a7d50b545af176e950ce328aec7f0b7bc5baebdca189fe661d97924980
-
-python3 updates2mqtt/cli.py manifest=ghcr.io/blakeblackshear/frigate@sha256:c68fd78fd3237c9ba81b5aa927f17b54f46705990f43b4b5d5596cfbbb626af4
-"""  # noqa: E501
+"""
 
 OCI_MANIFEST_TYPES: list[str] = [
     "application/vnd.oci.image.manifest.v1+json",
@@ -91,7 +93,9 @@ ALL_OCI_MEDIA_TYPES: list[str] = (
 )
 
 
-def dump_url(doc_type: str, img_ref: str) -> None:
+def dump_url(doc_type: str, img_ref: str, cli_conf: DictConfig) -> None:
+    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(cli_conf.get("log_level", "WARNING")))
+
     lookup = ContainerDistributionAPIVersionLookup(Throttler(), RegistryConfig())
     img_info = DockerImageInfo(img_ref)
     if not img_info.index_name or not img_info.name:
@@ -110,35 +114,49 @@ def dump_url(doc_type: str, img_ref: str) -> None:
             log.warning("No tag or digest found in %s", img_ref)
             return
         url = f"https://{api_host}/v2/{img_info.name}/manifests/{img_info.tag_or_digest}"
+    elif doc_type == "tags":
+        url = f"https://{api_host}/v2/{img_info.name}/tags/list"
     else:
         return
 
     token: str | None = lookup.fetch_token(img_info.index_name, img_info.name)
 
     response: Response | None = fetch_url(url, bearer_token=token, follow_redirects=True, response_type=ALL_OCI_MEDIA_TYPES)
-    if response:
+    if response and response.is_error:
+        log.warning(f"{response.status_code}: {url}")
+        log.warning(response.text)
+    elif response and response.is_success:
         log.debug(f"{response.status_code}: {url}")
         log.debug("HEADERS")
         for k, v in response.headers.items():
             log.debug(f"{k}: {v}")
         log.debug("CONTENTS")
+
         print_json(response.text)
 
 
 def main() -> None:
     # will be a proper cli someday
     cli_conf: DictConfig = OmegaConf.from_cli()
-    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(cli_conf.get("log_level", "WARNING")))
 
     if cli_conf.get("blob"):
-        dump_url("blob", cli_conf.get("blob"))
+        dump_url("blob", cli_conf.get("blob"), cli_conf)
     elif cli_conf.get("manifest"):
-        dump_url("manifest", cli_conf.get("manifest"))
+        dump_url("manifest", cli_conf.get("manifest"), cli_conf)
+    elif cli_conf.get("tags"):
+        dump_url("tags", cli_conf.get("tags"), cli_conf)
 
     else:
+        structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(cli_conf.get("log_level", "INFO")))
+
         docker_scanner = DockerProvider(
-            DockerConfig(registry=RegistryConfig(api=cli_conf.get("api", "OCI_V2"))), NodeConfig(), None
+            DockerConfig(registry=RegistryConfig(api=cli_conf.get("api", "OCI_V2"))),
+            NodeConfig(),
+            packages={},
+            github_cfg=GitHubConfig(access_token=cli_conf.get("github_token")),
+            self_bounce=None,
         )
+        docker_scanner.initialize()
         discovery: Discovery | None = docker_scanner.rescan(
             Discovery(docker_scanner, cli_conf.get("container", "frigate"), "cli", "manual")
         )

updates2mqtt/config.py

@@ -67,6 +67,11 @@ class MqttConfig:
     protocol: str = "${oc.env:MQTT_VERSION,3.11}"
 
 
+@dataclass
+class GitHubConfig:
+    access_token: str | None = None
+
+
 @dataclass
 class MetadataSourceConfig:
     enabled: bool = True
@@ -84,6 +89,21 @@ class VersionPolicy(StrEnum):
     VERSION = "VERSION"
     DIGEST = "DIGEST"
     VERSION_DIGEST = "VERSION_DIGEST"
+    TIMESTAMP = "TIMESTAMP"
+
+
+@dataclass
+class DockerPackageUpdateInfo:
+    image_name: str = MISSING  # untagged image ref
+    version_policy: VersionPolicy = VersionPolicy.AUTO
+
+
+@dataclass
+class PackageUpdateInfo:
+    docker: DockerPackageUpdateInfo | None = field(default_factory=DockerPackageUpdateInfo)
+    logo_url: str | None = None
+    release_notes_url: str | None = None
+    source_repo_url: str | None = None
 
 
 @dataclass
@@ -149,25 +169,14 @@ class Config:
     mqtt: MqttConfig = field(default_factory=MqttConfig)  # pyright: ignore[reportArgumentType, reportCallIssue]
     homeassistant: HomeAssistantConfig = field(default_factory=HomeAssistantConfig)
     docker: DockerConfig = field(default_factory=DockerConfig)
+    github: GitHubConfig = field(default_factory=GitHubConfig)
     scan_interval: int = 60 * 60 * 3
+    packages: dict[str, PackageUpdateInfo] = field(default_factory=dict)
 
 
 @dataclass
-class DockerPackageUpdateInfo:
-    image_name: str = MISSING  # untagged image ref
-
-
-@dataclass
-class PackageUpdateInfo:
-    docker: DockerPackageUpdateInfo | None = field(default_factory=DockerPackageUpdateInfo)
-    logo_url: str | None = None
-    release_notes_url: str | None = None
-    source_repo_url: str | None = None
-
-
-@dataclass
-class UpdateInfoConfig:
-    common_packages: dict[str, PackageUpdateInfo] = field(default_factory=lambda: {})
+class CommonPackages:
+    common_packages: dict[str, PackageUpdateInfo] = field(default_factory=dict)
 
 
 class IncompleteConfigException(BaseException):
@@ -188,13 +197,13 @@ def load_app_config(conf_file_path: Path, return_invalid: bool = False) -> Confi
             try:
                 log.debug(f"Creating config directory {conf_file_path.parent} if not already present")
                 conf_file_path.parent.mkdir(parents=True, exist_ok=True)
-            except Exception:
-                log.warning("Unable to create config directory", path=conf_file_path.parent)
+            except Exception as e:
+                log.warning("Unable to create config directory: %s", e, path=conf_file_path.parent)
         try:
             conf_file_path.write_text(OmegaConf.to_yaml(base_cfg))
             log.info(f"Auto-generated a new config file at {conf_file_path}")
-        except Exception:
-            log.warning("Unable to write config file", path=conf_file_path)
+        except Exception as e:
+            log.warning("Unable to write config file: %s", e, path=conf_file_path)
         cfg = base_cfg
     else:
         cfg = base_cfg

updates2mqtt/helpers.py

@@ -145,8 +145,8 @@ class APIStats:
         """Log line friendly string summary"""
         return (
             f"fetches: {self.fetches}, cache ratio: {self.hit_ratio():.2%}, revalidated: {self.revalidated}, "
-            + f"errors: {', '.join(f'{status_code}:{fails}' for status_code, fails in self.failed.items())}, "
-            + f"oldest cache hit: {self.max_cache_age:.2f}, avg elapsed: {self.average_elapsed()}"
+            + f"errors: {', '.join(f'{status_code}:{fails}' for status_code, fails in self.failed.items()) or '0'}, "
+            + f"oldest cache hit: {self.max_cache_age:.2f}s, avg elapsed: {self.average_elapsed()}s"
         )
 
 
@@ -196,8 +196,9 @@ def fetch_url(
                 allow_stale=allow_stale,
             )
         )
+        log_headers: list[tuple[str, str]] = [h for h in headers if len(h) > 1 and h[0] != "Authorization"]
         with SyncCacheClient(headers=headers, follow_redirects=follow_redirects, policy=cache_policy) as client:
-            log.debug(f"Fetching URL {url}, redirects={follow_redirects}, headers={headers}, cache_ttl={cache_ttl}")
+            log.debug(f"Fetching URL {url}, redirects={follow_redirects}, headers={log_headers}, cache_ttl={cache_ttl}")
             response: Response = client.request(method=method, url=url, extensions={"hishel_ttl": cache_ttl})
             cache_metadata: CacheMetadata = CacheMetadata(response)
             if not response.is_success:
@@ -221,6 +222,45 @@ def fetch_url(
     return None
 
 
-def validate_url(url: str, cache_ttl: int = 300) -> bool:
+def validate_url(url: str, cache_ttl: int = 1500) -> bool:
     response: Response | None = fetch_url(url, method="HEAD", cache_ttl=cache_ttl, follow_redirects=True)
     return response is not None and response.status_code != 404
+
+
+def sanitize_name(name: str, replacement: str = "_", max_len: int = 64) -> str:
+    """Strict sanitization that removes/replaces common problematic characters for MQTT or HA
+
+    - Replaces spaces with underscores
+    - Removes control characters
+    - Ensures alphanumeric safety for broader compatibility
+
+    Args:
+        name: The topic component string to sanitize
+        replacement: Character to replace invalid characters with (default: "_")
+        max_len: Largest acceptable name size
+
+    Returns:
+        Sanitized topic string safe for most MQTT brokers
+
+    """
+    if not name:
+        raise ValueError("Name cannot be empty")
+    orig_name: str = name
+    name = re.sub(r"[^A-Za-z0-9_\-\.]+", replacement, name)
+
+    # Replace multiple consecutive replacement chars with single one
+    if replacement:
+        pattern = re.escape(replacement) + "+"
+        name = re.sub(pattern, replacement, name)
+
+    # Trim to max length
+    topic_bytes = name.encode("utf-8")
+    if len(topic_bytes) > max_len:
+        name = topic_bytes[:max_len].decode("utf-8", errors="ignore")
+
+    if not name:
+        raise ValueError("Topic became empty after sanitization")
+    if name != orig_name:
+        log.info("Component name %s changed to %s for MQTT/HA compatibility", orig_name, name)
+
+    return name

updates2mqtt/integrations/docker.py

@@ -19,6 +19,7 @@ from updates2mqtt.config import (
     UNKNOWN_VERSION,
     VERSION_RE,
     DockerConfig,
+    GitHubConfig,
     NodeConfig,
     PackageUpdateInfo,
     PublishPolicy,
@@ -127,6 +128,8 @@ class DockerProvider(ReleaseProvider):
         self,
         cfg: DockerConfig,
         node_cfg: NodeConfig,
+        packages: dict[str, PackageUpdateInfo] | None = None,
+        github_cfg: GitHubConfig | None = None,
         self_bounce: Event | None = None,
     ) -> None:
         super().__init__(node_cfg, "docker")
@@ -136,8 +139,9 @@ class DockerProvider(ReleaseProvider):
         # TODO: refresh discovered packages periodically
         self.throttler = Throttler(self.cfg.default_api_backoff, self.log, self.stopped)
         self.self_bounce: Event | None = self_bounce
+
         self.pkg_enrichers: list[PackageEnricher] = [
-            CommonPackageEnricher(self.cfg),
+            CommonPackageEnricher(self.cfg, packages),
             LinuxServerIOPackageEnricher(self.cfg),
             DefaultPackageEnricher(self.cfg),
         ]
@@ -145,12 +149,13 @@ class DockerProvider(ReleaseProvider):
             self.client, self.throttler, self.cfg.registry, self.cfg.default_api_backoff
         )
         self.registry_image_lookup = ContainerDistributionAPIVersionLookup(self.throttler, self.cfg.registry)
-        self.release_enricher = SourceReleaseEnricher()
+        self.release_enricher = SourceReleaseEnricher(github_cfg)
         self.local_info_builder = LocalContainerInfo()
 
     def initialize(self) -> None:
         for enricher in self.pkg_enrichers:
             enricher.initialize()
+        self.log.debug("Docker provider initialized")
 
     def update(self, discovery: Discovery) -> bool:
         logger: Any = self.log.bind(container=discovery.name, action="update")
@@ -267,11 +272,11 @@ class DockerProvider(ReleaseProvider):
         )
 
     def rescan(self, discovery: Discovery) -> Discovery | None:
-        logger = self.log.bind(container=discovery.name, action="rescan")
+        logger: Any = self.log.bind(container=discovery.name, action="rescan")
         try:
             c: Container = self.client.containers.get(discovery.name)
             if c:
-                rediscovery = self.analyze(c, discovery.session, previous_discovery=discovery)
+                rediscovery: Discovery | None = self.analyze(c, discovery.session, previous_discovery=discovery)
                 if rediscovery and not rediscovery.throttled:
                     self.discoveries[rediscovery.name] = rediscovery
                     return rediscovery
@@ -296,9 +301,7 @@ class DockerProvider(ReleaseProvider):
         if customization.ignore:
             logger.info("Container ignored due to UPD2MQTT_IGNORE setting")
             return None
-        version_policy: VersionPolicy = (
-            self.cfg.version_policy if not customization.version_policy else customization.version_policy
-        )
+
         if customization.update == UpdatePolicy.AUTO:
             logger.debug("Auto update policy detected")
         update_policy: UpdatePolicy = customization.update or UpdatePolicy.PASSIVE
@@ -308,6 +311,20 @@ class DockerProvider(ReleaseProvider):
         local_info, service_info = self.local_info_builder.build_image_info(c)
         pkg_info: PackageUpdateInfo = self.default_metadata(local_info)
 
+        version_policy: VersionPolicy
+        if customization.version_policy:
+            logger.debug("Overriding version_policy to local customization: %s", customization.version_policy)
+            version_policy = customization.version_policy
+        else:
+            if self.cfg.version_policy == VersionPolicy.AUTO and pkg_info.docker:
+                logger.debug(
+                    "Version policy, pkg level %s, config level: %s", pkg_info.docker.version_policy, self.cfg.version_policy
+                )
+                version_policy = pkg_info.docker.version_policy or self.cfg.version_policy
+            else:
+                logger.debug("Version policy, fixed config level: %s", self.cfg.version_policy)
+                version_policy = self.cfg.version_policy
+
         try:
             service_info.git_repo_path = customization.git_repo_path
 
@@ -516,7 +533,7 @@ class DockerProvider(ReleaseProvider):
 
     def default_metadata(self, image_info: DockerImageInfo) -> PackageUpdateInfo:
         for enricher in self.pkg_enrichers:
-            pkg_info = enricher.enrich(image_info)
+            pkg_info: PackageUpdateInfo | None = enricher.enrich(image_info)
             if pkg_info is not None:
                 return pkg_info
         raise ValueError("No enricher could provide metadata, not even default enricher")
@@ -583,30 +600,41 @@ def select_versions(version_policy: VersionPolicy, installed: DockerImageInfo, l
             basis("version-digest"),
         )
 
+    if (
+        version_policy == VersionPolicy.TIMESTAMP
+        and installed.created
+        and latest.created
+        and (
+            (latest.created > installed.created and latest.short_digest != installed.short_digest)
+            or (latest.created == installed.created and latest.short_digest == installed.short_digest)
+        )
+    ):
+        return installed.created, latest.created, basis("timestamp")
+
     phase = 1
     if version_policy == VersionPolicy.AUTO and (
         (installed.version == latest.version and installed.short_digest == latest.short_digest)
         or (installed.version != latest.version and installed.short_digest != latest.short_digest)
     ):
-        # detect semver, or casual semver (e.g. v1.030)
+        # detect semver, or v semver (e.g. v1.030)
         # only use this if both version and digest are consistently agreeing or disagreeing
         # if the strict conditions work, people see nice version numbers on screen rather than hashes
         if (
             installed.version
-            and re.match(SEMVER_RE, installed.version or "")
+            and re.fullmatch(SEMVER_RE, installed.version or "")
             and latest.version
-            and re.match(SEMVER_RE, latest.version or "")
+            and re.fullmatch(SEMVER_RE, latest.version or "")
         ):
             # Smells like semver, override if not using version_policy
             return installed.version, latest.version, basis("semver")
         if (
             installed.version
-            and re.match(VERSION_RE, installed.version or "")
+            and re.fullmatch(VERSION_RE, installed.version or "")
             and latest.version
-            and re.match(VERSION_RE, latest.version or "")
+            and re.fullmatch(VERSION_RE, latest.version or "")
         ):
             # Smells like casual semver, override if not using version_policy
-            return installed.version, latest.version, basis("causualver")
+            return installed.version, latest.version, basis("casualver")
 
     # AUTO or fallback
     phase = 2
@@ -629,6 +657,15 @@ def select_versions(version_policy: VersionPolicy, installed: DockerImageInfo, l
 
     # Fall back to digests, image or repo index
     phase = 4
+    if (
+        installed.created
+        and latest.created
+        and (
+            (latest.created > installed.created and latest.short_digest != installed.short_digest)
+            or (latest.created == installed.created and latest.short_digest == installed.short_digest)
+        )
+    ):
+        return installed.created, latest.created, basis("timestamp")
     if installed_digest_available and latest_digest_available:
         return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
     if installed.version and not latest.version and not latest.short_digest and not latest.repo_digest:
@@ -655,7 +692,8 @@ def select_versions(version_policy: VersionPolicy, installed: DockerImageInfo, l
         return condense_repo_id(latest), condense_repo_id(latest), basis("repo-digest")
 
     if installed_digest_available and not latest_digest_available:
-        return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+        # no new digest, so latest is the current
+        return installed.short_digest, installed.short_digest, basis("digest")  # type: ignore[return-value]
 
     log.warn("No versions can be determined for %s", installed.ref)
     phase = 999