updates2mqtt 1.6.0__py3-none-any.whl → 1.7.2__py3-none-any.whl

updates2mqtt/app.py

@@ -14,7 +14,7 @@ import structlog
 import updates2mqtt
 from updates2mqtt.model import Discovery, ReleaseProvider
 
-from .config import Config, PackageUpdateInfo, load_app_config, load_package_info
+from .config import Config, PublishPolicy, UpdatePolicy, load_app_config
 from .integrations.docker import DockerProvider
 from .mqtt import MqttPublisher
 
@@ -49,7 +49,6 @@ class App:
 
         structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, str(self.cfg.log.level))))
         log.debug("Logging initialized", level=self.cfg.log.level)
-        self.common_pkg: dict[str, PackageUpdateInfo] = load_package_info(PKG_INFO_FILE)
 
         self.publisher = MqttPublisher(self.cfg.mqtt, self.cfg.node, self.cfg.homeassistant)
 
@@ -57,7 +56,7 @@ class App:
         self.scan_count: int = 0
         self.last_scan: str | None = None
         if self.cfg.docker.enabled:
-            self.scanners.append(DockerProvider(self.cfg.docker, self.common_pkg, self.cfg.node, self.self_bounce))
+            self.scanners.append(DockerProvider(self.cfg.docker, self.cfg.node, self.self_bounce))
         self.stopped = Event()
         self.healthcheck_topic = self.cfg.node.healthcheck.topic_template.format(node_name=self.cfg.node.name)
 
@@ -104,6 +103,7 @@ class App:
             )
 
         for scanner in self.scanners:
+            scanner.initialize()
             self.publisher.subscribe_hass_command(scanner)
 
         while not self.stopped.is_set() and self.publisher.is_available():
@@ -122,12 +122,15 @@ class App:
     async def on_discovery(self, discovery: Discovery) -> None:
         dlog = log.bind(name=discovery.name)
         try:
-            if self.cfg.homeassistant.discovery.enabled:
+            if discovery.publish_policy == PublishPolicy.HOMEASSISTANT and self.cfg.homeassistant.discovery.enabled:
+                # Switch off MQTT discovery if not Home Assistant enabled
                 self.publisher.publish_hass_config(discovery)
-
-            self.publisher.publish_hass_state(discovery)
+            if discovery.publish_policy in (PublishPolicy.HOMEASSISTANT):
+                self.publisher.publish_hass_state(discovery)
+            if discovery.publish_policy in (PublishPolicy.HOMEASSISTANT, PublishPolicy.MQTT):
+                self.publisher.publish_discovery(discovery)
             if (
-                discovery.update_policy == "Auto"
+                discovery.update_policy == UpdatePolicy.AUTO
                 and discovery.can_update
                 and discovery.latest_version != discovery.current_version
             ):

updates2mqtt/cli.py

@@ -0,0 +1,150 @@
+from typing import TYPE_CHECKING
+
+import structlog
+from omegaconf import DictConfig, OmegaConf
+from rich import print_json
+
+from updates2mqtt.config import DockerConfig, NodeConfig, RegistryConfig
+from updates2mqtt.helpers import Throttler
+from updates2mqtt.integrations.docker import DockerProvider
+from updates2mqtt.integrations.docker_enrich import (
+    REGISTRIES,
+    ContainerDistributionAPIVersionLookup,
+    DockerImageInfo,
+    fetch_url,
+)
+from updates2mqtt.model import Discovery
+
+if TYPE_CHECKING:
+    from httpx import Response
+
+log = structlog.get_logger()
+
+
+"""
+Super simple CLI
+
+python updates2mqtt.cli container=frigate
+
+python updates2mqtt.cli container=frigate api=docker_client log_level=DEBUG
+
+ython3 updates2mqtt/cli.py blob=ghcr.io/homarr-labs/homarr@sha256:af79a3339de5ed8ef7f5a0186ff3deb86f40b213ba75249291f2f68aef082a25 | jq '.config.Labels'
+
+python3 updates2mqtt/cli.py manifest=ghcr.io/blakeblackshear/frigate:stable
+
+python3 updates2mqtt/cli.py blob=ghcr.io/blakeblackshear/frigate@sha256:ef8d56a7d50b545af176e950ce328aec7f0b7bc5baebdca189fe661d97924980
+
+python3 updates2mqtt/cli.py manifest=ghcr.io/blakeblackshear/frigate@sha256:c68fd78fd3237c9ba81b5aa927f17b54f46705990f43b4b5d5596cfbbb626af4
+"""  # noqa: E501
+
+OCI_MANIFEST_TYPES: list[str] = [
+    "application/vnd.oci.image.manifest.v1+json",
+    "application/vnd.oci.image.index.v1+json",
+    "application/vnd.oci.descriptor.v1+json",
+    "application/vnd.oci.empty.v1+json",
+]
+
+OCI_CONFIG_TYPES: list[str] = [
+    "application/vnd.oci.image.config.v1+json",
+]
+
+OCI_LAYER_TYPES: list[str] = [
+    "application/vnd.oci.image.layer.v1.tar",
+    "application/vnd.oci.image.layer.v1.tar+gzip",
+    "application/vnd.oci.image.layer.v1.tar+zstd",
+]
+
+OCI_NONDISTRIBUTABLE_LAYER_TYPES: list[str] = [
+    "application/vnd.oci.image.layer.nondistributable.v1.tar",
+    "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+    "application/vnd.oci.image.layer.nondistributable.v1.tar+zstd",
+]
+
+# Docker Compatibility MIME Types
+DOCKER_MANIFEST_TYPES: list[str] = [
+    "application/vnd.docker.distribution.manifest.v2+json",
+    "application/vnd.docker.distribution.manifest.list.v2+json",
+    "application/vnd.docker.distribution.manifest.v1+json",
+    "application/vnd.docker.distribution.manifest.v1+prettyjws",
+]
+
+DOCKER_CONFIG_TYPES: list[str] = [
+    "application/vnd.docker.container.image.v1+json",
+]
+
+DOCKER_LAYER_TYPES: list[str] = [
+    "application/vnd.docker.image.rootfs.diff.tar.gzip",
+    "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+]
+
+# Combined constants
+ALL_MANIFEST_TYPES: list[str] = OCI_MANIFEST_TYPES + DOCKER_MANIFEST_TYPES
+ALL_CONFIG_TYPES: list[str] = OCI_CONFIG_TYPES + DOCKER_CONFIG_TYPES
+ALL_LAYER_TYPES: list[str] = OCI_LAYER_TYPES + OCI_NONDISTRIBUTABLE_LAYER_TYPES + DOCKER_LAYER_TYPES
+
+# All content types that might be returned by the API
+ALL_OCI_MEDIA_TYPES: list[str] = (
+    ALL_MANIFEST_TYPES
+    + ALL_CONFIG_TYPES
+    + ALL_LAYER_TYPES
+    + ["application/octet-stream", "application/json"]  # Error responses
+)
+
+
+def dump_url(doc_type: str, img_ref: str) -> None:
+    lookup = ContainerDistributionAPIVersionLookup(Throttler(), RegistryConfig())
+    img_info = DockerImageInfo(img_ref)
+    if not img_info.index_name or not img_info.name:
+        log.error("Unable to parse %ss", img_ref)
+        return
+
+    api_host: str | None = REGISTRIES.get(img_info.index_name, (img_info.index_name, img_info.index_name))[1]
+
+    if doc_type == "blob":
+        if not img_info.pinned_digest:
+            log.warning("No digest found in %s", img_ref)
+            return
+        url: str = f"https://{api_host}/v2/{img_info.name}/blobs/{img_info.pinned_digest}"
+    elif doc_type == "manifest":
+        if not img_info.tag_or_digest:
+            log.warning("No tag or digest found in %s", img_ref)
+            return
+        url = f"https://{api_host}/v2/{img_info.name}/manifests/{img_info.tag_or_digest}"
+    else:
+        return
+
+    token: str | None = lookup.fetch_token(img_info.index_name, img_info.name)
+
+    response: Response | None = fetch_url(url, bearer_token=token, follow_redirects=True, response_type=ALL_OCI_MEDIA_TYPES)
+    if response:
+        log.debug(f"{response.status_code}: {url}")
+        log.debug("HEADERS")
+        for k, v in response.headers.items():
+            log.debug(f"{k}: {v}")
+        log.debug("CONTENTS")
+        print_json(response.text)
+
+
+def main() -> None:
+    # will be a proper cli someday
+    cli_conf: DictConfig = OmegaConf.from_cli()
+    structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(cli_conf.get("log_level", "WARNING")))
+
+    if cli_conf.get("blob"):
+        dump_url("blob", cli_conf.get("blob"))
+    elif cli_conf.get("manifest"):
+        dump_url("manifest", cli_conf.get("manifest"))
+
+    else:
+        docker_scanner = DockerProvider(
+            DockerConfig(registry=RegistryConfig(api=cli_conf.get("api", "OCI_V2"))), NodeConfig(), None
+        )
+        discovery: Discovery | None = docker_scanner.rescan(
+            Discovery(docker_scanner, cli_conf.get("container", "frigate"), "cli", "manual")
+        )
+        if discovery:
+            log.info(discovery.as_dict())
+
+
+if __name__ == "__main__":
+    main()

updates2mqtt/config.py

@@ -9,6 +9,23 @@ from omegaconf import MISSING, DictConfig, MissingMandatoryValue, OmegaConf, Val
 
 log = structlog.get_logger()
 
+PKG_INFO_FILE = Path("./common_packages.yaml")
+UNKNOWN_VERSION = "UNKNOWN"
+VERSION_RE = r"[vVr]?[0-9]+(\.[0-9]+)*"
+# source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+SEMVER_RE = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"  # noqa: E501
+
+
+class UpdatePolicy(StrEnum):
+    AUTO = "Auto"
+    PASSIVE = "Passive"
+
+
+class PublishPolicy(StrEnum):
+    HOMEASSISTANT = "HomeAssistant"
+    MQTT = "MQTT"
+    SILENT = "Silent"
+
 
 class LogLevel(StrEnum):
     DEBUG = "DEBUG"
@@ -18,6 +35,28 @@ class LogLevel(StrEnum):
     CRITICAL = "CRITICAL"
 
 
+class RegistryAPI(StrEnum):
+    OCI_V2 = "OCI_V2"
+    OCI_V2_MINIMAL = "OCI_V2"
+    DOCKER_CLIENT = "DOCKER_CLIENT"
+    DISABLED = "DISABLED"
+
+
+class VersionType:
+    SHORT_SHA = "short_sha"
+    FULL_SHA = "full_sha"
+    VERSION_REVISION = "version_revision"
+    VERSION = "version"
+
+
+@dataclass
+class RegistryConfig:
+    api: RegistryAPI = RegistryAPI.OCI_V2
+    mutable_cache_ttl: int | None = None  # default to server cache hint
+    immutable_cache_ttl: int | None = 7776000  # 90 days
+    token_cache_ttl: int | None = None  # default to server cache hint
+
+
 @dataclass
 class MqttConfig:
     host: str = "${oc.env:MQTT_HOST,localhost}"
@@ -34,6 +73,19 @@ class MetadataSourceConfig:
     cache_ttl: int = 60 * 60 * 24 * 7  # 1 week
 
 
+@dataclass
+class Selector:
+    include: list[str] | None = None
+    exclude: list[str] | None = None
+
+
+class VersionPolicy(StrEnum):
+    AUTO = "AUTO"
+    VERSION = "VERSION"
+    DIGEST = "DIGEST"
+    VERSION_DIGEST = "VERSION_DIGEST"
+
+
 @dataclass
 class DockerConfig:
     enabled: bool = True
@@ -47,7 +99,12 @@ class DockerConfig:
     discover_metadata: dict[str, MetadataSourceConfig] = field(
         default_factory=lambda: {"linuxserver.io": MetadataSourceConfig(enabled=True)}
     )
-    api_throttle_wait: int = 60 * 15
+    registry: RegistryConfig = field(default_factory=lambda: RegistryConfig())
+    default_api_backoff: int = 60 * 15
+    image_ref_select: Selector = field(default_factory=lambda: Selector())
+    version_select: Selector = field(default_factory=lambda: Selector())
+    version_policy: VersionPolicy = VersionPolicy.AUTO
+    registry_select: Selector = field(default_factory=lambda: Selector())
 
 
 @dataclass
@@ -62,6 +119,7 @@ class HomeAssistantConfig:
     state_topic_suffix: str = "state"
     device_creation: bool = True
     force_command_topic: bool = False
+    extra_attributes: bool = True
     area: str | None = None
 
 
@@ -96,7 +154,7 @@ class Config:
 
 @dataclass
 class DockerPackageUpdateInfo:
-    image_name: str = MISSING
+    image_name: str = MISSING  # untagged image ref
 
 
 @dataclass
@@ -104,6 +162,7 @@ class PackageUpdateInfo:
     docker: DockerPackageUpdateInfo | None = field(default_factory=DockerPackageUpdateInfo)
     logo_url: str | None = None
     release_notes_url: str | None = None
+    source_repo_url: str | None = None
 
 
 @dataclass
@@ -115,24 +174,6 @@ class IncompleteConfigException(BaseException):
     pass
 
 
-def load_package_info(pkginfo_file_path: Path) -> dict[str, PackageUpdateInfo]:
-    if pkginfo_file_path.exists():
-        log.debug("Loading common package update info", path=pkginfo_file_path)
-        cfg = OmegaConf.load(pkginfo_file_path)
-    else:
-        log.warn("No common package update info found", path=pkginfo_file_path)
-        cfg = OmegaConf.structured(UpdateInfoConfig)
-    try:
-        # omegaconf broken-ness on optional fields and converting to backclasses
-        pkg_conf: dict[str, PackageUpdateInfo] = {
-            pkg: PackageUpdateInfo(**pkg_cfg) for pkg, pkg_cfg in cfg.common_packages.items()
-        }
-        return pkg_conf
-    except (MissingMandatoryValue, ValidationError) as e:
-        log.error("Configuration error %s", e, path=pkginfo_file_path.as_posix())
-        raise
-
-
 def is_autogen_config() -> bool:
     env_var: str | None = os.environ.get("U2M_AUTOGEN_CONFIG")
     return not (env_var and env_var.lower() in ("no", "0", "false"))

updates2mqtt/hass_formatter.py

@@ -23,28 +23,26 @@ def hass_format_config(
     object_id: str,
     state_topic: str,
     command_topic: str | None,
+    attrs_topic: str | None,
     force_command_topic: bool | None,
     device_creation: bool = True,
     area: str | None = None,
-    session: str | None = None,
 ) -> dict[str, Any]:
     config: dict[str, Any] = {
         "name": discovery.title,
         "device_class": None,  # not firmware, so defaults to null
         "unique_id": object_id,
         "state_topic": state_topic,
-        "source_session": session,
         "supported_features": discovery.features,
-        "can_update": discovery.can_update,
-        "can_build": discovery.can_build,
-        "can_restart": discovery.can_restart,
-        "update_policy": discovery.update_policy,
+        "default_entity_id": f"update.{discovery.node}_{discovery.provider.source_type}_{discovery.name}",
         "origin": {
             "name": f"{discovery.node} updates2mqtt",
             "sw_version": updates2mqtt.version,  # pyright: ignore[reportAttributeAccessIssue]
             "support_url": "https://github.com/rhizomatics/updates2mqtt/issues",
         },
     }
+    if attrs_topic:
+        config["json_attributes_topic"] = attrs_topic
     if discovery.entity_picture_url:
         config["entity_picture"] = discovery.entity_picture_url
     if discovery.device_icon:
@@ -62,28 +60,21 @@ def hass_format_config(
         config["command_topic"] = command_topic
         if discovery.can_update:
             config["payload_install"] = f"{discovery.source_type}|{discovery.name}|install"
-    config["custom"] = {}
-    config["custom"][discovery.source_type] = discovery.custom
-    config.update(discovery.provider.hass_config_format(discovery))
+
     return config
 
 
 def hass_format_state(discovery: Discovery, session: str, in_progress: bool = False) -> dict[str, Any]:  # noqa: ARG001
-    state = {
+    state: dict[str, str | dict | list | bool | None] = {
         "installed_version": discovery.current_version,
         "latest_version": discovery.latest_version,
         "title": discovery.title,
         "in_progress": in_progress,
     }
-    if discovery.release_summary:
-        state["release_summary"] = discovery.release_summary
-    if discovery.release_url:
-        state["release_url"] = discovery.release_url
-    custom_state = discovery.provider.hass_state_format(discovery)
-    if custom_state:
-        state.update(custom_state)
-    invalid_keys = [k for k in state if k not in HASS_UPDATE_SCHEMA]
-    if invalid_keys:
-        log.warning(f"Invalid keys in state: {invalid_keys}")
-        state = {k: v for k, v in state.items() if k in HASS_UPDATE_SCHEMA}
+    if discovery.release_detail:
+        if discovery.release_detail.summary:
+            state["release_summary"] = discovery.release_detail.summary
+        if discovery.release_detail.notes_url:
+            state["release_url"] = discovery.release_detail.notes_url
+
     return state

updates2mqtt/helpers.py

@@ -0,0 +1,226 @@
+import datetime as dt
+import re
+import time
+from threading import Event
+from typing import Any
+from urllib.parse import urlparse
+
+import structlog
+from hishel import CacheOptions, SpecificationPolicy  # pyright: ignore[reportAttributeAccessIssue]
+from hishel.httpx import SyncCacheClient
+from httpx import Response
+from tzlocal import get_localzone
+
+from updates2mqtt.config import Selector
+
+log = structlog.get_logger()
+
+
+def timestamp(time_value: float | None) -> str | None:
+    if time_value is None:
+        return None
+    try:
+        return dt.datetime.fromtimestamp(time_value, tz=get_localzone()).isoformat()
+    except:  # noqa: E722
+        return None
+
+
+class Selection:
+    def __init__(self, selector: Selector, value: str | None) -> None:
+        self.result: bool = True
+        self.matched: str | None = None
+        if value is None:
+            self.result = selector.include is None
+            return
+        if selector.exclude is not None:
+            self.result = True
+            if any(re.search(pat, value) for pat in selector.exclude):
+                self.matched = value
+                self.result = False
+        if selector.include is not None:
+            self.result = False
+            if any(re.search(pat, value) for pat in selector.include):
+                self.matched = value
+                self.result = True
+
+    def __bool__(self) -> bool:
+        """Expose the actual boolean so objects can be appropriately truthy"""
+        return self.result
+
+
+class ThrottledError(Exception):
+    def __init__(self, message: str, retry_secs: int) -> None:
+        super().__init__(message)
+        self.retry_secs = retry_secs
+
+
+class Throttler:
+    DEFAULT_SITE = "DEFAULT_SITE"
+
+    def __init__(self, api_throttle_pause: int = 30, logger: Any | None = None, semaphore: Event | None = None) -> None:
+        self.log: Any = logger or log
+        self.pause_api_until: dict[str, float] = {}
+        self.api_throttle_pause: int = api_throttle_pause
+        self.semaphore = semaphore
+
+    def check_throttle(self, index_name: str | None = None) -> bool:
+        if self.semaphore and self.semaphore.is_set():
+            return True
+        index_name = index_name or self.DEFAULT_SITE
+        if self.pause_api_until.get(index_name) is not None:
+            if self.pause_api_until[index_name] < time.time():
+                del self.pause_api_until[index_name]
+                self.log.info("%s throttling wait complete", index_name)
+            else:
+                self.log.debug("%s throttling has %0.3f secs left", index_name, self.pause_api_until[index_name] - time.time())
+                return True
+        return False
+
+    def throttle(
+        self,
+        index_name: str | None = None,
+        retry_secs: int | None = None,
+        explanation: str | None = None,
+        raise_exception: bool = False,
+    ) -> None:
+        index_name = index_name or self.DEFAULT_SITE
+        retry_secs = retry_secs if retry_secs and retry_secs > 0 else self.api_throttle_pause
+        self.log.warn("%s throttling requests for %s seconds, %s", index_name, retry_secs, explanation)
+        self.pause_api_until[index_name] = time.time() + retry_secs
+        if raise_exception:
+            raise ThrottledError(explanation or f"{index_name} throttled request", retry_secs)
+
+
+class CacheMetadata:
+    """Cache metadata extracted from hishel response extensions"""
+
+    def __init__(self, response: Response) -> None:
+        self.from_cache: bool = response.extensions.get("hishel_from_cache", False)
+        self.revalidated: bool = response.extensions.get("hishel_revalidated", False)
+        self.created_at: float | None = response.extensions.get("hishel_created_at")
+        self.stored: bool = response.extensions.get("hishel_stored", False)
+        self.age: float | None = None
+        if self.created_at is not None:
+            self.age = time.time() - self.created_at
+
+    def __str__(self) -> str:
+        """Summarize in a string"""
+        return f"cached: {self.from_cache}, revalidated: {self.revalidated}, age:{self.age}, stored:{self.stored}"
+
+
+class APIStats:
+    def __init__(self) -> None:
+        self.fetches: int = 0
+        self.cached: int = 0
+        self.revalidated: int = 0
+        self.failed: dict[int, int] = {}
+        self.elapsed: float = 0
+        self.max_cache_age: float | None = None
+
+    def tick(self, response: Response | None) -> None:
+        self.fetches += 1
+        if response is None:
+            self.failed.setdefault(0, 0)
+            self.failed[0] += 1
+            return
+        cache_metadata: CacheMetadata = CacheMetadata(response)
+        self.cached += 1 if cache_metadata.from_cache else 0
+        self.revalidated += 1 if cache_metadata.revalidated else 0
+        if response.elapsed:
+            self.elapsed += response.elapsed.microseconds / 1000000
+            self.elapsed += response.elapsed.seconds
+        if not response.is_success:
+            self.failed.setdefault(response.status_code, 0)
+            self.failed[response.status_code] += 1
+        if cache_metadata.age is not None and (self.max_cache_age is None or cache_metadata.age > self.max_cache_age):
+            self.max_cache_age = cache_metadata.age
+
+    def hit_ratio(self) -> float:
+        return round(self.cached / self.fetches, 2) if self.cached and self.fetches else 0
+
+    def average_elapsed(self) -> float:
+        return round(self.elapsed / self.fetches, 2) if self.elapsed and self.fetches else 0
+
+    def __str__(self) -> str:
+        """Log line friendly string summary"""
+        return (
+            f"fetches: {self.fetches}, cache ratio: {self.hit_ratio():.2%}, revalidated: {self.revalidated}, "
+            + f"errors: {', '.join(f'{status_code}:{fails}' for status_code, fails in self.failed.items())}, "
+            + f"oldest cache hit: {self.max_cache_age:.2f}, avg elapsed: {self.average_elapsed()}"
+        )
+
+
+class APIStatsCounter:
+    def __init__(self) -> None:
+        self.stats_report_interval: int = 100
+        self.host_stats: dict[str, APIStats] = {}
+        self.fetches: int = 0
+        self.log: Any = structlog.get_logger().bind()
+
+    def stats(self, url: str, response: Response | None) -> None:
+        try:
+            host: str = urlparse(url).hostname or "UNKNOWN"
+            api_stats: APIStats = self.host_stats.setdefault(host, APIStats())
+            api_stats.tick(response)
+            self.fetches += 1
+            if self.fetches % self.stats_report_interval == 0:
+                self.log.info(
+                    "OCI_V2 API Stats Summary\n%s", "\n".join(f"{host} {stats}" for host, stats in self.host_stats.items())
+                )
+        except Exception as e:
+            self.log.warning("Failed to tick stats: %s", e)
+
+
+def fetch_url(
+    url: str,
+    cache_ttl: int | None = None,  # default to server responses for cache ttl
+    bearer_token: str | None = None,
+    response_type: str | list[str] | None = None,
+    follow_redirects: bool = False,
+    allow_stale: bool = False,
+    method: str = "GET",
+    api_stats_counter: APIStatsCounter | None = None,
+) -> Response | None:
+    try:
+        headers = [("cache-control", f"max-age={cache_ttl}")]
+        if bearer_token:
+            headers.append(("Authorization", f"Bearer {bearer_token}"))
+        if response_type:
+            response_type = [response_type] if isinstance(response_type, str) else response_type
+            if response_type and isinstance(response_type, (tuple, list)):
+                headers.extend(("Accept", mime_type) for mime_type in response_type)
+
+        cache_policy = SpecificationPolicy(
+            cache_options=CacheOptions(
+                shared=False,  # Private browser cache
+                allow_stale=allow_stale,
+            )
+        )
+        with SyncCacheClient(headers=headers, follow_redirects=follow_redirects, policy=cache_policy) as client:
+            log.debug(f"Fetching URL {url}, redirects={follow_redirects}, headers={headers}, cache_ttl={cache_ttl}")
+            response: Response = client.request(method=method, url=url, extensions={"hishel_ttl": cache_ttl})
+            cache_metadata: CacheMetadata = CacheMetadata(response)
+            if not response.is_success:
+                log.debug("URL %s fetch returned non-success status: %s, %s", url, response.status_code, cache_metadata.stored)
+            elif response:
+                log.debug(
+                    "URL response: status: %s, cached: %s, revalidated: %s, cache age: %s, stored: %s",
+                    response.status_code,
+                    cache_metadata.from_cache,
+                    cache_metadata.revalidated,
+                    cache_metadata.age,
+                    cache_metadata.stored,
+                )
+            if api_stats_counter:
+                api_stats_counter.stats(url, response)
+            return response
+    except Exception as e:
+        log.debug("URL %s failed to fetch: %s", url, e)
+        if api_stats_counter:
+            api_stats_counter.stats(url, None)
+    return None
+
+
+def validate_url(url: str, cache_ttl: int = 300) -> bool:
+    response: Response | None = fetch_url(url, method="HEAD", cache_ttl=cache_ttl, follow_redirects=True)
+    return response is not None and response.status_code != 404