updates2mqtt 1.7.0__py3-none-any.whl → 1.7.3__py3-none-any.whl

updates2mqtt/cli.py

@@ -0,0 +1,150 @@
+from typing import TYPE_CHECKING
+
+import structlog
+from omegaconf import DictConfig, OmegaConf
+from rich import print_json
+
+from updates2mqtt.config import DockerConfig, NodeConfig, RegistryConfig
+from updates2mqtt.helpers import Throttler
+from updates2mqtt.integrations.docker import DockerProvider
+from updates2mqtt.integrations.docker_enrich import (
+    REGISTRIES,
+    ContainerDistributionAPIVersionLookup,
+    DockerImageInfo,
+    fetch_url,
+)
+from updates2mqtt.model import Discovery
+
+if TYPE_CHECKING:
+    from httpx import Response
+
+log = structlog.get_logger()
+
+
+"""
+Super simple CLI
+
+python updates2mqtt.cli container=frigate
+
+python updates2mqtt.cli container=frigate api=docker_client log_level=DEBUG
+
+ython3 updates2mqtt/cli.py blob=ghcr.io/homarr-labs/homarr@sha256:af79a3339de5ed8ef7f5a0186ff3deb86f40b213ba75249291f2f68aef082a25 | jq '.config.Labels'
+
+python3 updates2mqtt/cli.py manifest=ghcr.io/blakeblackshear/frigate:stable
+
+python3 updates2mqtt/cli.py blob=ghcr.io/blakeblackshear/frigate@sha256:ef8d56a7d50b545af176e950ce328aec7f0b7bc5baebdca189fe661d97924980
+
+python3 updates2mqtt/cli.py manifest=ghcr.io/blakeblackshear/frigate@sha256:c68fd78fd3237c9ba81b5aa927f17b54f46705990f43b4b5d5596cfbbb626af4
+"""  # noqa: E501
+
+OCI_MANIFEST_TYPES: list[str] = [
+    "application/vnd.oci.image.manifest.v1+json",
+    "application/vnd.oci.image.index.v1+json",
+    "application/vnd.oci.descriptor.v1+json",
+    "application/vnd.oci.empty.v1+json",
+]
+
+OCI_CONFIG_TYPES: list[str] = [
+    "application/vnd.oci.image.config.v1+json",
+]
+
+OCI_LAYER_TYPES: list[str] = [
+    "application/vnd.oci.image.layer.v1.tar",
+    "application/vnd.oci.image.layer.v1.tar+gzip",
+    "application/vnd.oci.image.layer.v1.tar+zstd",
+]
+
+OCI_NONDISTRIBUTABLE_LAYER_TYPES: list[str] = [
+    "application/vnd.oci.image.layer.nondistributable.v1.tar",
+    "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+    "application/vnd.oci.image.layer.nondistributable.v1.tar+zstd",
+]
+
+# Docker Compatibility MIME Types
+DOCKER_MANIFEST_TYPES: list[str] = [
+    "application/vnd.docker.distribution.manifest.v2+json",
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.docker.distribution.manifest.v1+json",
+    "application/vnd.docker.distribution.manifest.v1+prettyjws",
+]
+
+DOCKER_CONFIG_TYPES: list[str] = [
+    "application/vnd.docker.container.image.v1+json",
+]
+
+DOCKER_LAYER_TYPES: list[str] = [
+    "application/vnd.docker.image.rootfs.diff.tar.gzip",
+    "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+]
+
+# Combined constants
+ALL_MANIFEST_TYPES: list[str] = OCI_MANIFEST_TYPES + DOCKER_MANIFEST_TYPES
+ALL_CONFIG_TYPES: list[str] = OCI_CONFIG_TYPES + DOCKER_CONFIG_TYPES
+ALL_LAYER_TYPES: list[str] = OCI_LAYER_TYPES + OCI_NONDISTRIBUTABLE_LAYER_TYPES + DOCKER_LAYER_TYPES
+
+# All content types that might be returned by the API
+ALL_OCI_MEDIA_TYPES: list[str] = (
+    ALL_MANIFEST_TYPES
+    + ALL_CONFIG_TYPES
+    + ALL_LAYER_TYPES
+    + ["application/octet-stream", "application/json"]  # Error responses
+)
+
+
+def dump_url(doc_type: str, img_ref: str) -> None:
+    lookup = ContainerDistributionAPIVersionLookup(Throttler(), RegistryConfig())
+    img_info = DockerImageInfo(img_ref)
+    if not img_info.index_name or not img_info.name:
+        log.error("Unable to parse %ss", img_ref)
+        return
+
+    api_host: str | None = REGISTRIES.get(img_info.index_name, (img_info.index_name, img_info.index_name))[1]
+
+    if doc_type == "blob":
+        if not img_info.pinned_digest:
+            log.warning("No digest found in %s", img_ref)
+            return
+        url: str = f"https://{api_host}/v2/{img_info.name}/blobs/{img_info.pinned_digest}"
+    elif doc_type == "manifest":
+        if not img_info.tag_or_digest:
+            log.warning("No tag or digest found in %s", img_ref)
+            return
+        url = f"https://{api_host}/v2/{img_info.name}/manifests/{img_info.tag_or_digest}"
+    else:
+        return
+
+    token: str | None = lookup.fetch_token(img_info.index_name, img_info.name)
+
+    response: Response | None = fetch_url(url, bearer_token=token, follow_redirects=True, response_type=ALL_OCI_MEDIA_TYPES)
+    if response:
+        log.debug(f"{response.status_code}: {url}")
+        log.debug("HEADERS")
+        for k, v in response.headers.items():
+            log.debug(f"{k}: {v}")
+        log.debug("CONTENTS")
+        print_json(response.text)
+
+
+def main() -> None:
+    # will be a proper cli someday
+    cli_conf: DictConfig = OmegaConf.from_cli()
+    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(cli_conf.get("log_level", "WARNING")))
+
+    if cli_conf.get("blob"):
+        dump_url("blob", cli_conf.get("blob"))
+    elif cli_conf.get("manifest"):
+        dump_url("manifest", cli_conf.get("manifest"))
+
+    else:
+        docker_scanner = DockerProvider(
+            DockerConfig(registry=RegistryConfig(api=cli_conf.get("api", "OCI_V2"))), NodeConfig(), None
+        )
+        discovery: Discovery | None = docker_scanner.rescan(
+            Discovery(docker_scanner, cli_conf.get("container", "frigate"), "cli", "manual")
+        )
+        if discovery:
+            log.info(discovery.as_dict())
+
+
+if __name__ == "__main__":
+    main()

updates2mqtt/config.py

@@ -10,7 +10,10 @@ from omegaconf import MISSING, DictConfig, MissingMandatoryValue, OmegaConf, Val
 log = structlog.get_logger()
 
 PKG_INFO_FILE = Path("./common_packages.yaml")
-NO_KNOWN_IMAGE = "UNKNOWN"
+UNKNOWN_VERSION = "UNKNOWN"
+VERSION_RE = r"[vVr]?[0-9]+(\.[0-9]+)*"
+# source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+SEMVER_RE = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"  # noqa: E501
 
 
 class UpdatePolicy(StrEnum):
@@ -32,6 +35,13 @@ class LogLevel(StrEnum):
     CRITICAL = "CRITICAL"
 
 
+class RegistryAPI(StrEnum):
+    OCI_V2 = "OCI_V2"
+    OCI_V2_MINIMAL = "OCI_V2"
+    DOCKER_CLIENT = "DOCKER_CLIENT"
+    DISABLED = "DISABLED"
+
+
 class VersionType:
     SHORT_SHA = "short_sha"
     FULL_SHA = "full_sha"
@@ -39,6 +49,14 @@ class VersionType:
     VERSION = "version"
 
 
+@dataclass
+class RegistryConfig:
+    api: RegistryAPI = RegistryAPI.OCI_V2
+    mutable_cache_ttl: int | None = None  # default to server cache hint
+    immutable_cache_ttl: int | None = 7776000  # 90 days
+    token_cache_ttl: int | None = None  # default to server cache hint
+
+
 @dataclass
 class MqttConfig:
     host: str = "${oc.env:MQTT_HOST,localhost}"
@@ -61,6 +79,13 @@ class Selector:
     exclude: list[str] | None = None
 
 
+class VersionPolicy(StrEnum):
+    AUTO = "AUTO"
+    VERSION = "VERSION"
+    DIGEST = "DIGEST"
+    VERSION_DIGEST = "VERSION_DIGEST"
+
+
 @dataclass
 class DockerConfig:
     enabled: bool = True
@@ -74,8 +99,12 @@ class DockerConfig:
     discover_metadata: dict[str, MetadataSourceConfig] = field(
         default_factory=lambda: {"linuxserver.io": MetadataSourceConfig(enabled=True)}
     )
+    registry: RegistryConfig = field(default_factory=lambda: RegistryConfig())
     default_api_backoff: int = 60 * 15
     image_ref_select: Selector = field(default_factory=lambda: Selector())
+    version_select: Selector = field(default_factory=lambda: Selector())
+    version_policy: VersionPolicy = VersionPolicy.AUTO
+    registry_select: Selector = field(default_factory=lambda: Selector())
 
 
 @dataclass
@@ -125,7 +154,7 @@ class Config:
 
 @dataclass
 class DockerPackageUpdateInfo:
-    image_name: str = MISSING
+    image_name: str = MISSING  # untagged image ref
 
 
 @dataclass
@@ -133,6 +162,7 @@ class PackageUpdateInfo:
     docker: DockerPackageUpdateInfo | None = field(default_factory=DockerPackageUpdateInfo)
     logo_url: str | None = None
     release_notes_url: str | None = None
+    source_repo_url: str | None = None
 
 
 @dataclass

updates2mqtt/hass_formatter.py

@@ -71,9 +71,10 @@ def hass_format_state(discovery: Discovery, session: str, in_progress: bool = Fa
         "title": discovery.title,
         "in_progress": in_progress,
     }
-    if discovery.release_summary:
-        state["release_summary"] = discovery.release_summary
-    if discovery.release_url:
-        state["release_url"] = discovery.release_url
+    if discovery.release_detail:
+        if discovery.release_detail.summary:
+            state["release_summary"] = discovery.release_detail.summary
+        if discovery.release_detail.notes_url:
+            state["release_url"] = discovery.release_detail.notes_url
 
     return state

updates2mqtt/helpers.py

@@ -0,0 +1,226 @@
+import datetime as dt
+import re
+import time
+from threading import Event
+from typing import Any
+from urllib.parse import urlparse
+
+import structlog
+from hishel import CacheOptions, SpecificationPolicy  # pyright: ignore[reportAttributeAccessIssue]
+from hishel.httpx import SyncCacheClient
+from httpx import Response
+from tzlocal import get_localzone
+
+from updates2mqtt.config import Selector
+
+log = structlog.get_logger()
+
+
+def timestamp(time_value: float | None) -> str | None:
+    if time_value is None:
+        return None
+    try:
+        return dt.datetime.fromtimestamp(time_value, tz=get_localzone()).isoformat()
+    except:  # noqa: E722
+        return None
+
+
+class Selection:
+    def __init__(self, selector: Selector, value: str | None) -> None:
+        self.result: bool = True
+        self.matched: str | None = None
+        if value is None:
+            self.result = selector.include is None
+            return
+        if selector.exclude is not None:
+            self.result = True
+            if any(re.search(pat, value) for pat in selector.exclude):
+                self.matched = value
+                self.result = False
+        if selector.include is not None:
+            self.result = False
+            if any(re.search(pat, value) for pat in selector.include):
+                self.matched = value
+                self.result = True
+
+    def __bool__(self) -> bool:
+        """Expose the actual boolean so objects can be appropriately truthy"""
+        return self.result
+
+
+class ThrottledError(Exception):
+    def __init__(self, message: str, retry_secs: int) -> None:
+        super().__init__(message)
+        self.retry_secs = retry_secs
+
+
+class Throttler:
+    DEFAULT_SITE = "DEFAULT_SITE"
+
+    def __init__(self, api_throttle_pause: int = 30, logger: Any | None = None, semaphore: Event | None = None) -> None:
+        self.log: Any = logger or log
+        self.pause_api_until: dict[str, float] = {}
+        self.api_throttle_pause: int = api_throttle_pause
+        self.semaphore = semaphore
+
+    def check_throttle(self, index_name: str | None = None) -> bool:
+        if self.semaphore and self.semaphore.is_set():
+            return True
+        index_name = index_name or self.DEFAULT_SITE
+        if self.pause_api_until.get(index_name) is not None:
+            if self.pause_api_until[index_name] < time.time():
+                del self.pause_api_until[index_name]
+                self.log.info("%s throttling wait complete", index_name)
+            else:
+                self.log.debug("%s throttling has %0.3f secs left", index_name, self.pause_api_until[index_name] - time.time())
+                return True
+        return False
+
+    def throttle(
+        self,
+        index_name: str | None = None,
+        retry_secs: int | None = None,
+        explanation: str | None = None,
+        raise_exception: bool = False,
+    ) -> None:
+        index_name = index_name or self.DEFAULT_SITE
+        retry_secs = retry_secs if retry_secs and retry_secs > 0 else self.api_throttle_pause
+        self.log.warn("%s throttling requests for %s seconds, %s", index_name, retry_secs, explanation)
+        self.pause_api_until[index_name] = time.time() + retry_secs
+        if raise_exception:
+            raise ThrottledError(explanation or f"{index_name} throttled request", retry_secs)
+
+
+class CacheMetadata:
+    """Cache metadata extracted from hishel response extensions"""
+
+    def __init__(self, response: Response) -> None:
+        self.from_cache: bool = response.extensions.get("hishel_from_cache", False)
+        self.revalidated: bool = response.extensions.get("hishel_revalidated", False)
+        self.created_at: float | None = response.extensions.get("hishel_created_at")
+        self.stored: bool = response.extensions.get("hishel_stored", False)
+        self.age: float | None = None
+        if self.created_at is not None:
+            self.age = time.time() - self.created_at
+
+    def __str__(self) -> str:
+        """Summarize in a string"""
+        return f"cached: {self.from_cache}, revalidated: {self.revalidated}, age:{self.age}, stored:{self.stored}"
+
+
+class APIStats:
+    def __init__(self) -> None:
+        self.fetches: int = 0
+        self.cached: int = 0
+        self.revalidated: int = 0
+        self.failed: dict[int, int] = {}
+        self.elapsed: float = 0
+        self.max_cache_age: float = 0
+
+    def tick(self, response: Response | None) -> None:
+        self.fetches += 1
+        if response is None:
+            self.failed.setdefault(0, 0)
+            self.failed[0] += 1
+            return
+        cache_metadata: CacheMetadata = CacheMetadata(response)
+        self.cached += 1 if cache_metadata.from_cache else 0
+        self.revalidated += 1 if cache_metadata.revalidated else 0
+        if response.elapsed:
+            self.elapsed += response.elapsed.microseconds / 1000000
+            self.elapsed += response.elapsed.seconds
+        if not response.is_success:
+            self.failed.setdefault(response.status_code, 0)
+            self.failed[response.status_code] += 1
+        if cache_metadata.age is not None and (self.max_cache_age is None or cache_metadata.age > self.max_cache_age):
+            self.max_cache_age = cache_metadata.age
+
+    def hit_ratio(self) -> float:
+        return round(self.cached / self.fetches, 2) if self.cached and self.fetches else 0
+
+    def average_elapsed(self) -> float:
+        return round(self.elapsed / self.fetches, 2) if self.elapsed and self.fetches else 0
+
+    def __str__(self) -> str:
+        """Log line friendly string summary"""
+        return (
+            f"fetches: {self.fetches}, cache ratio: {self.hit_ratio():.2%}, revalidated: {self.revalidated}, "
+            + f"errors: {', '.join(f'{status_code}:{fails}' for status_code, fails in self.failed.items())}, "
+            + f"oldest cache hit: {self.max_cache_age:.2f}, avg elapsed: {self.average_elapsed()}"
+        )
+
+
+class APIStatsCounter:
+    def __init__(self) -> None:
+        self.stats_report_interval: int = 100
+        self.host_stats: dict[str, APIStats] = {}
+        self.fetches: int = 0
+        self.log: Any = structlog.get_logger().bind()
+
+    def stats(self, url: str, response: Response | None) -> None:
+        try:
+            host: str = urlparse(url).hostname or "UNKNOWN"
+            api_stats: APIStats = self.host_stats.setdefault(host, APIStats())
+            api_stats.tick(response)
+            self.fetches += 1
+            if self.fetches % self.stats_report_interval == 0:
+                self.log.info(
+                    "OCI_V2 API Stats Summary\n%s", "\n".join(f"{host} {stats}" for host, stats in self.host_stats.items())
+                )
+        except Exception as e:
+            self.log.warning("Failed to tick stats: %s", e)
+
+
+def fetch_url(
+    url: str,
+    cache_ttl: int | None = None,  # default to server responses for cache ttl
+    bearer_token: str | None = None,
+    response_type: str | list[str] | None = None,
+    follow_redirects: bool = False,
+    allow_stale: bool = False,
+    method: str = "GET",
+    api_stats_counter: APIStatsCounter | None = None,
+) -> Response | None:
+    try:
+        headers = [("cache-control", f"max-age={cache_ttl}")]
+        if bearer_token:
+            headers.append(("Authorization", f"Bearer {bearer_token}"))
+        if response_type:
+            response_type = [response_type] if isinstance(response_type, str) else response_type
+            if response_type and isinstance(response_type, (tuple, list)):
+                headers.extend(("Accept", mime_type) for mime_type in response_type)
+
+        cache_policy = SpecificationPolicy(
+            cache_options=CacheOptions(
+                shared=False,  # Private browser cache
+                allow_stale=allow_stale,
+            )
+        )
+        with SyncCacheClient(headers=headers, follow_redirects=follow_redirects, policy=cache_policy) as client:
+            log.debug(f"Fetching URL {url}, redirects={follow_redirects}, headers={headers}, cache_ttl={cache_ttl}")
+            response: Response = client.request(method=method, url=url, extensions={"hishel_ttl": cache_ttl})
+            cache_metadata: CacheMetadata = CacheMetadata(response)
+            if not response.is_success:
+                log.debug("URL %s fetch returned non-success status: %s, %s", url, response.status_code, cache_metadata.stored)
+            elif response:
+                log.debug(
+                    "URL response: status: %s, cached: %s, revalidated: %s, cache age: %s, stored: %s",
+                    response.status_code,
+                    cache_metadata.from_cache,
+                    cache_metadata.revalidated,
+                    cache_metadata.age,
+                    cache_metadata.stored,
+                )
+            if api_stats_counter:
+                api_stats_counter.stats(url, response)
+            return response
+    except Exception as e:
+        log.debug("URL %s failed to fetch: %s", url, e)
+        if api_stats_counter:
+            api_stats_counter.stats(url, None)
+    return None
+
+
+def validate_url(url: str, cache_ttl: int = 300) -> bool:
+    response: Response | None = fetch_url(url, method="HEAD", cache_ttl=cache_ttl, follow_redirects=True)
+    return response is not None and response.status_code != 404