ap-client 0.5.1.dev1__tar.gz → 0.5.1.dev3__tar.gz

{ap_client-0.5.1.dev1 → ap_client-0.5.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ap-client
-Version: 0.5.1.dev1
+Version: 0.5.1.dev3
 Summary: Agent Platform API Client & CLI
 Requires-Python: >=3.10
 Requires-Dist: pyyaml>=6.0
@@ -9,9 +9,9 @@ Requires-Dist: rich>=13.0.0
 Requires-Dist: typer>=0.9.0
 Requires-Dist: websockets>=13.0
 Provides-Extra: all
-Requires-Dist: instance-repo[oss]==1.3.2.dev2; extra == 'all'
+Requires-Dist: instance-repo[oss]==1.3.2.dev3; extra == 'all'
 Provides-Extra: dataset
-Requires-Dist: instance-repo[oss]==1.3.2.dev2; extra == 'dataset'
+Requires-Dist: instance-repo[oss]==1.3.2.dev3; extra == 'dataset'
 Description-Content-Type: text/markdown
 
 A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.

{ap_client-0.5.1.dev1 → ap_client-0.5.1.dev3}/ap_client/irepo_sdk.py

@@ -29,11 +29,22 @@
 
 from __future__ import annotations
 
+import hashlib
 import inspect
 import json
+import math
 import os
+import platform
+import re
+import stat
+import subprocess
+import tempfile
+import threading
 import time
-from typing import TYPE_CHECKING, Any, NoReturn
+from contextlib import contextmanager
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Iterator, NoReturn
+from urllib.parse import urlsplit
 
 import typer
 
@@ -82,6 +93,565 @@ class SdkMissing(Exception):
         super().__init__(message)
 
 
+_OSS_PROBE_CACHE_VERSION = 1
+_OSS_PROBE_CACHE_TTL = 300.0
+_OSS_PROBE_CACHE_MAX_BYTES = 64 * 1024
+_OSS_PROBE_CACHE_MAX_ENTRIES = 128
+_NETWORK_FINGERPRINT_UNSET = object()
+
+
+def _oss_probe_cache_path() -> Path:
+    return Path.home() / ".ap" / ".cache" / "oss-probe.json"
+
+
+def _run_network_command(command: tuple[str, ...]) -> bytes | None:
+    if not command or not os.path.isabs(command[0]):
+        return None
+    environment = {
+        key: value
+        for key, value in os.environ.items()
+        if key.upper() in {"LANG", "LC_ALL", "SYSTEMROOT", "WINDIR"}
+    }
+    environment.update({"LANG": "C", "LC_ALL": "C"})
+    try:
+        result = subprocess.run(
+            command,
+            check=False,
+            capture_output=True,
+            env=environment,
+            timeout=1.0,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if result.returncode != 0 or not result.stdout.strip():
+        return None
+    return result.stdout[:32768]
+
+
+def _read_network_file(path: str) -> bytes | None:
+    try:
+        with open(path, "rb") as handle:
+            value = handle.read(32769)
+    except OSError:
+        return None
+    if not value or len(value) > 32768:
+        return None
+    return value
+
+
+def _network_context_parts(system: str) -> list[bytes]:
+    if system == "darwin":
+        primary = _run_network_command(("/usr/sbin/scutil", "--nwi"))
+        if primary is None:
+            return []
+        route = _run_network_command(("/sbin/route", "-n", "get", "default"))
+        return [primary, *([route] if route is not None else [])]
+    if system == "linux":
+        route = _read_network_file("/proc/net/route")
+        if route is None:
+            return []
+        parts = [route]
+        for path in ("/proc/net/ipv6_route", "/etc/resolv.conf"):
+            value = _read_network_file(path)
+            if value is not None:
+                parts.append(value)
+        try:
+            parts.append(f"netns:{os.stat('/proc/self/ns/net').st_ino}".encode("ascii"))
+        except OSError:
+            pass
+        try:
+            interfaces = sorted(Path("/sys/class/net").glob("*/address"))
+        except OSError:
+            interfaces = []
+        for address_path in interfaces:
+            value = _read_network_file(str(address_path))
+            if value is not None:
+                parts.append(address_path.parent.name.encode("utf-8") + b"=" + value)
+        return parts
+    return []
+
+
+def _network_fingerprint() -> str | None:
+    system = platform.system().strip().lower()
+    parts = _network_context_parts(system)
+    if not parts:
+        return None
+    digest = hashlib.sha256()
+    digest.update(system.encode("ascii", errors="replace"))
+    for part in parts:
+        digest.update(b"\0")
+        digest.update(part)
+    return f"{system}:{digest.hexdigest()}"
+
+
+def _cache_endpoint(endpoint: str) -> str | None:
+    value = str(endpoint or "").strip()
+    if not value or len(value) > 2048:
+        return None
+    try:
+        parsed = urlsplit(value if "://" in value else "//" + value)
+        hostname = parsed.hostname
+        port = parsed.port
+    except ValueError:
+        return None
+    if not hostname:
+        return None
+    host = hostname.lower()
+    if ":" in host:
+        host = f"[{host}]"
+    scheme = parsed.scheme.lower()
+    authority = f"{host}:{port}" if port is not None else host
+    return f"{scheme}://{authority}" if scheme else authority
+
+
+def _validate_oss_probe_cache(value: Any) -> list[dict[str, Any]]:
+    if (
+        not isinstance(value, dict)
+        or set(value) != {"version", "entries"}
+        or type(value.get("version")) is not int
+        or value["version"] != _OSS_PROBE_CACHE_VERSION
+    ):
+        raise ValueError("invalid OSS probe cache version")
+    entries = value.get("entries")
+    if not isinstance(entries, list) or len(entries) > _OSS_PROBE_CACHE_MAX_ENTRIES:
+        raise ValueError("invalid OSS probe cache entries")
+    validated = []
+    required = {"endpoint", "network", "reachable", "timestamp"}
+    for entry in entries:
+        if not isinstance(entry, dict) or set(entry) != required:
+            raise ValueError("invalid OSS probe cache entry")
+        endpoint = entry["endpoint"]
+        network = entry["network"]
+        reachable = entry["reachable"]
+        timestamp = entry["timestamp"]
+        if (
+            not isinstance(endpoint, str)
+            or not endpoint
+            or len(endpoint) > 2048
+            or _cache_endpoint(endpoint) != endpoint
+        ):
+            raise ValueError("invalid OSS probe cache endpoint")
+        if (
+            not isinstance(network, str)
+            or len(network) > 128
+            or re.fullmatch(r"[a-z0-9_-]+:[0-9a-f]{64}", network) is None
+        ):
+            raise ValueError("invalid OSS probe cache network")
+        if not isinstance(reachable, bool):
+            raise ValueError("invalid OSS probe cache result")
+        if (
+            isinstance(timestamp, bool)
+            or not isinstance(timestamp, (int, float))
+            or not math.isfinite(timestamp)
+            or timestamp < 0
+        ):
+            raise ValueError("invalid OSS probe cache timestamp")
+        validated.append(entry)
+    return validated
+
+
+def _private_path_stat(path: Path, *, directory: bool) -> os.stat_result:
+    path_stat = path.lstat()
+    expected = stat.S_ISDIR if directory else stat.S_ISREG
+    if not expected(path_stat.st_mode):
+        raise PermissionError(f"unsafe OSS probe cache path: {path}")
+    getuid = getattr(os, "getuid", None)
+    if callable(getuid) and path_stat.st_uid != getuid():
+        raise PermissionError(f"OSS probe cache is not owned by current user: {path}")
+    if stat.S_IMODE(path_stat.st_mode) & 0o077:
+        raise PermissionError(f"OSS probe cache permissions are too broad: {path}")
+    return path_stat
+
+
+def _prepare_oss_probe_cache_dir(path: Path) -> None:
+    path.mkdir(mode=0o700, parents=True, exist_ok=True)
+    path_stat = path.lstat()
+    if not stat.S_ISDIR(path_stat.st_mode):
+        raise PermissionError(f"unsafe OSS probe cache directory: {path}")
+    getuid = getattr(os, "getuid", None)
+    if callable(getuid) and path_stat.st_uid != getuid():
+        raise PermissionError(f"OSS probe cache directory is not owned by current user: {path}")
+    os.chmod(path, 0o700)
+
+
+@contextmanager
+def _oss_probe_cache_lock(
+    path: Path,
+    *,
+    name: str = ".oss-probe.lock",
+    timeout: float = 1.0,
+) -> Iterator[bool]:
+    try:
+        import fcntl
+
+        _prepare_oss_probe_cache_dir(path.parent)
+        flags = os.O_CREAT | os.O_RDWR
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        fd = os.open(path.parent / name, flags, 0o600)
+        lock_stat = os.fstat(fd)
+        getuid = getattr(os, "getuid", None)
+        if not stat.S_ISREG(lock_stat.st_mode) or (
+            callable(getuid) and lock_stat.st_uid != getuid()
+        ):
+            raise PermissionError("unsafe OSS probe cache lock")
+        os.fchmod(fd, 0o600)
+        deadline = time.monotonic() + timeout
+        while True:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if time.monotonic() >= deadline:
+                    raise TimeoutError("timed out waiting for OSS probe cache lock")
+                time.sleep(0.01)
+    except Exception:
+        if "fd" in locals():
+            os.close(fd)
+        yield False
+        return
+    try:
+        yield True
+    finally:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+        os.close(fd)
+
+
+def _oss_probe_endpoint_lock_name(endpoint: str) -> str:
+    digest = hashlib.sha256(endpoint.encode("utf-8")).hexdigest()[:24]
+    return f".oss-probe-{digest}.lock"
+
+
+def _read_oss_probe_cache(path: Path) -> list[dict[str, Any]]:
+    file_stat = _private_path_stat(path, directory=False)
+    if file_stat.st_size > _OSS_PROBE_CACHE_MAX_BYTES:
+        raise ValueError("invalid OSS probe cache file")
+    flags = os.O_RDONLY
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    fd = os.open(path, flags)
+    try:
+        rendered = os.read(fd, _OSS_PROBE_CACHE_MAX_BYTES + 1)
+    finally:
+        os.close(fd)
+    if len(rendered) > _OSS_PROBE_CACHE_MAX_BYTES:
+        raise ValueError("invalid OSS probe cache file")
+    return _validate_oss_probe_cache(json.loads(rendered.decode("utf-8")))
+
+
+def _write_oss_probe_cache(path: Path, entries: list[dict[str, Any]]) -> None:
+    cache_dir = path.parent
+    _prepare_oss_probe_cache_dir(cache_dir)
+    fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=cache_dir)
+    temporary_path = Path(temporary)
+    try:
+        os.fchmod(fd, 0o600)
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            fd = -1
+            json.dump(
+                {"version": _OSS_PROBE_CACHE_VERSION, "entries": entries},
+                handle,
+                separators=(",", ":"),
+                sort_keys=True,
+            )
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary_path, path)
+        os.chmod(path, 0o600)
+    finally:
+        if fd >= 0:
+            os.close(fd)
+        try:
+            temporary_path.unlink()
+        except FileNotFoundError:
+            pass
+
+
+def _probe_cache_log(verbose: bool, status: str, endpoint: str) -> None:
+    if verbose:
+        typer.echo(f"OSS probe cache {status}: {endpoint}", err=True)
+
+
+def _cached_oss_probe(
+    endpoint: str,
+    probe: Any,
+    *,
+    verbose: bool,
+    fingerprints: dict[str, str],
+    state_lock: Any,
+    fingerprint: Any = _NETWORK_FINGERPRINT_UNSET,
+    checked_at: Callable[[float], None] | None = None,
+) -> bool:
+    def run_probe() -> bool:
+        reachable = bool(probe(endpoint))
+        if checked_at is not None:
+            checked_at(time.time())
+        return reachable
+
+    cache_endpoint = _cache_endpoint(endpoint)
+    if fingerprint is _NETWORK_FINGERPRINT_UNSET:
+        fingerprint = _network_fingerprint()
+    if cache_endpoint is None or fingerprint is None:
+        status = "bypass (network context unavailable)" if fingerprint is None else "bypass"
+        _probe_cache_log(verbose, status, cache_endpoint or "invalid endpoint")
+        return run_probe()
+
+    with state_lock:
+        fingerprints[cache_endpoint] = fingerprint
+    path = _oss_probe_cache_path()
+    endpoint_lock_name = _oss_probe_endpoint_lock_name(cache_endpoint)
+    with _oss_probe_cache_lock(path, name=endpoint_lock_name, timeout=4.0) as endpoint_locked:
+        if not endpoint_locked:
+            _probe_cache_log(verbose, "bypass (cache busy)", cache_endpoint)
+            return run_probe()
+
+        now = time.time()
+        status = "miss"
+        with _oss_probe_cache_lock(path) as read_locked:
+            if not read_locked:
+                _probe_cache_log(verbose, "bypass (cache unavailable)", cache_endpoint)
+                return run_probe()
+            try:
+                entries = _read_oss_probe_cache(path)
+            except FileNotFoundError:
+                entries = []
+            except Exception:
+                entries = []
+                status = "invalid"
+
+        for entry in entries:
+            if entry["endpoint"] != cache_endpoint or entry["network"] != fingerprint:
+                continue
+            age = now - entry["timestamp"]
+            if 0 <= age < _OSS_PROBE_CACHE_TTL:
+                _probe_cache_log(verbose, "hit", cache_endpoint)
+                if checked_at is not None:
+                    checked_at(entry["timestamp"])
+                return entry["reachable"]
+            status = "stale"
+            break
+
+        _probe_cache_log(verbose, status, cache_endpoint)
+        reachable = run_probe()
+        now = time.time()
+        with _oss_probe_cache_lock(path) as write_locked:
+            if not write_locked:
+                _probe_cache_log(verbose, "write skipped (cache busy)", cache_endpoint)
+                return reachable
+            try:
+                latest_entries = _read_oss_probe_cache(path)
+            except Exception:
+                latest_entries = []
+            fresh_entries = [
+                entry
+                for entry in latest_entries
+                if 0 <= now - entry["timestamp"] < _OSS_PROBE_CACHE_TTL
+                and not (entry["endpoint"] == cache_endpoint and entry["network"] == fingerprint)
+            ]
+            fresh_entries.append(
+                {
+                    "endpoint": cache_endpoint,
+                    "network": fingerprint,
+                    "reachable": reachable,
+                    "timestamp": now,
+                }
+            )
+            try:
+                _write_oss_probe_cache(path, fresh_entries[-_OSS_PROBE_CACHE_MAX_ENTRIES:])
+            except Exception:
+                _probe_cache_log(verbose, "write failed", cache_endpoint)
+        return reachable
+
+
+def _invalidate_oss_probe_cache(
+    endpoint: str,
+    fingerprint: str | None,
+    *,
+    verbose: bool,
+) -> None:
+    cache_endpoint = _cache_endpoint(endpoint)
+    if cache_endpoint is None:
+        return
+    path = _oss_probe_cache_path()
+    endpoint_lock_name = _oss_probe_endpoint_lock_name(cache_endpoint)
+    with _oss_probe_cache_lock(path, name=endpoint_lock_name, timeout=4.0) as endpoint_locked:
+        if not endpoint_locked:
+            return
+        with _oss_probe_cache_lock(path) as locked:
+            if not locked:
+                return
+            try:
+                entries = _read_oss_probe_cache(path)
+            except Exception:
+                return
+            remaining = [
+                entry
+                for entry in entries
+                if not (
+                    entry["endpoint"] == cache_endpoint
+                    and (fingerprint is None or entry["network"] == fingerprint)
+                )
+            ]
+            if len(remaining) == len(entries):
+                return
+            try:
+                _write_oss_probe_cache(path, remaining)
+            except Exception:
+                return
+    _probe_cache_log(verbose, "invalidated after connection error", cache_endpoint)
+
+
+def _sdk_reachability_classifier() -> Callable[[BaseException], bool] | None:
+    try:
+        from instance_repo._routing import _is_reachability_error
+    except (ImportError, AttributeError):
+        return None
+    return _is_reachability_error
+
+
+def _install_connection_error_invalidation(
+    content: Any,
+    fingerprints: dict[str, str],
+    *,
+    verbose: bool,
+    classifier: Callable[[BaseException], bool],
+    state_lock: Any,
+) -> bool:
+    open_bucket = getattr(content, "_open_bucket", None)
+    execute = getattr(content, "_do", None)
+    if not callable(open_bucket) or not callable(execute):
+        return False
+
+    active = threading.local()
+
+    def tracked_open_bucket(*args: Any, **kwargs: Any) -> Any:
+        endpoint = kwargs.get("endpoint")
+        if endpoint is None and len(args) >= 3:
+            endpoint = args[2]
+        active.endpoint = endpoint if isinstance(endpoint, str) else None
+        try:
+            return open_bucket(*args, **kwargs)
+        except Exception:
+            active.endpoint = None
+            raise
+
+    def tracked_execute(*args: Any, **kwargs: Any) -> Any:
+        try:
+            return execute(*args, **kwargs)
+        except Exception as exc:
+            endpoint = getattr(active, "endpoint", None)
+            if endpoint and classifier(exc):
+                cache_endpoint = _cache_endpoint(endpoint)
+                with state_lock:
+                    fingerprint = fingerprints.get(cache_endpoint or "")
+                    probe_cache = getattr(content, "_probe_cache", None)
+                    if isinstance(probe_cache, dict):
+                        probe_cache.pop(endpoint, None)
+                _invalidate_oss_probe_cache(endpoint, fingerprint, verbose=verbose)
+            raise
+        finally:
+            active.endpoint = None
+
+    try:
+        content._open_bucket = tracked_open_bucket
+        content._do = tracked_execute
+    except (AttributeError, TypeError):
+        try:
+            content._open_bucket = open_bucket
+            content._do = execute
+        except (AttributeError, TypeError):
+            pass
+        return False
+    return True
+
+
+def _install_oss_probe_cache(repo: Any, *, verbose: bool) -> bool:
+    content = getattr(repo, "_content", None)
+    probe = getattr(content, "_probe_private_endpoint", None)
+    sdk_cached_probe = getattr(content, "_probe_cached", None)
+    probe_cache = getattr(content, "_probe_cache", None)
+    open_bucket = getattr(content, "_open_bucket", None)
+    execute = getattr(content, "_do", None)
+    classifier = _sdk_reachability_classifier()
+    if (
+        getattr(content, "_ap_probe_cache_installed", False)
+        or not callable(probe)
+        or not callable(sdk_cached_probe)
+        or not isinstance(probe_cache, dict)
+        or not callable(open_bucket)
+        or not callable(execute)
+        or classifier is None
+    ):
+        return False
+
+    fingerprints: dict[str, str] = {}
+    fingerprint_context = threading.local()
+    state_lock = threading.RLock()
+
+    def cached_probe(endpoint: str) -> bool:
+        fingerprint = getattr(fingerprint_context, "value", _NETWORK_FINGERPRINT_UNSET)
+        return _cached_oss_probe(
+            endpoint,
+            probe,
+            verbose=verbose,
+            fingerprints=fingerprints,
+            state_lock=state_lock,
+            fingerprint=fingerprint,
+            checked_at=lambda timestamp: setattr(fingerprint_context, "checked_at", timestamp),
+        )
+
+    memory_networks: dict[str, str] = {}
+
+    def network_aware_cached_probe(endpoint: str) -> bool:
+        with state_lock:
+            fingerprint = _network_fingerprint()
+            if fingerprint is not None and memory_networks.get(endpoint) != fingerprint:
+                probe_cache.pop(endpoint, None)
+                if fingerprint is not None:
+                    memory_networks[endpoint] = fingerprint
+            fingerprint_context.value = fingerprint
+            fingerprint_context.checked_at = None
+            try:
+                reachable = bool(sdk_cached_probe(endpoint))
+                # SDK timestamps newly acquired results. A disk-cache hit must
+                # retain its original age instead of receiving another 300s.
+                if fingerprint_context.checked_at is not None and endpoint in probe_cache:
+                    probe_cache[endpoint] = (reachable, fingerprint_context.checked_at)
+                return reachable
+            finally:
+                del fingerprint_context.value
+                del fingerprint_context.checked_at
+
+    try:
+        content._probe_private_endpoint = cached_probe
+        content._probe_cached = network_aware_cached_probe
+        content._ap_probe_cache_installed = True
+    except (AttributeError, TypeError):
+        try:
+            content._probe_private_endpoint = probe
+            content._probe_cached = sdk_cached_probe
+            if hasattr(content, "_ap_probe_cache_installed"):
+                del content._ap_probe_cache_installed
+        except (AttributeError, TypeError):
+            pass
+        return False
+    if not _install_connection_error_invalidation(
+        content,
+        fingerprints,
+        verbose=verbose,
+        classifier=classifier,
+        state_lock=state_lock,
+    ):
+        try:
+            content._probe_private_endpoint = probe
+            content._probe_cached = sdk_cached_probe
+            del content._ap_probe_cache_installed
+        except (AttributeError, TypeError):
+            pass
+        return False
+    return True
+
+
 # ---------------------------------------------------------------------------
 # Repo 构造
 # ---------------------------------------------------------------------------
@@ -284,7 +854,9 @@ def load_repo(
         _setdefault_env(_LEGACY_TOKEN_VAR, cfg.token_key or "")
         _setdefault_env(_LEGACY_STORAGE_ENV_VAR, effective_storage_env)
 
-    return _Repo(**kwargs)
+    repo = _Repo(**kwargs)
+    _install_oss_probe_cache(repo, verbose=cfg.verbose)
+    return repo
 
 
 def _init_parameters(repo_cls: type) -> frozenset[str]:

{ap_client-0.5.1.dev1 → ap_client-0.5.1.dev3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ap-client"
-version = "0.5.1.dev1"
+version = "0.5.1.dev3"
 description = "Agent Platform API Client & CLI"
 readme = { text = "A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.", content-type = "text/markdown" }
 requires-python = ">=3.10"
@@ -18,10 +18,10 @@ dependencies = [
 
 [project.optional-dependencies]
 dataset = [
-    "instance-repo[oss]==1.3.2.dev2",
+    "instance-repo[oss]==1.3.2.dev3",
 ]
 all = [
-    "instance-repo[oss]==1.3.2.dev2",
+    "instance-repo[oss]==1.3.2.dev3",
 ]
 
 [project.scripts]