ap-client 0.1.4.dev0__py3-none-any.whl

ap_client/config.py

@@ -0,0 +1,65 @@
+"""Configuration management."""
+
+import os
+from dataclasses import dataclass
+
+
+def _parse_bool(value: str | None) -> bool:
+    """Parse a boolean environment variable."""
+    if value is None:
+        return False
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _parse_positive_int(value: str | None, default: int) -> int:
+    """Parse a positive-integer environment variable."""
+    if value is None or not value.strip():
+        return default
+    parsed = int(value)
+    if parsed <= 0:
+        raise ValueError("environment variable must be a positive integer")
+    return parsed
+
+
+@dataclass
+class Config:
+    """CLI configuration."""
+
+    base_url: str
+    headers: dict
+    agenthub_ref: str | None
+    verbose: bool = False
+    verbose_body_limit: int = 4096
+    verbose_full_body: bool = False
+
+    @classmethod
+    def from_env(cls) -> "Config":
+        """Load configuration from environment variables."""
+        base_url = os.environ.get("AP_BASE_URL")
+        if not base_url:
+            raise ValueError("AP_BASE_URL environment variable must be set")
+
+        headers = {}
+        headers_str = os.environ.get("AP_HEADERS", "")
+        if headers_str:
+            for h in headers_str.split(","):
+                if ":" in h:
+                    k, v = h.split(":", 1)
+                    headers[k.strip()] = v.strip()
+
+        return cls(
+            base_url=base_url.rstrip("/"),
+            headers=headers,
+            agenthub_ref=os.environ.get("AP_AGENTHUB_REF"),
+            verbose=_parse_bool(os.environ.get("AP_VERBOSE")),
+            verbose_body_limit=_parse_positive_int(os.environ.get("AP_VERBOSE_BODY_LIMIT"), 4096),
+            verbose_full_body=_parse_bool(os.environ.get("AP_VERBOSE_FULL_BODY")),
+        )
+
+
+def get_config(verbose: bool | None = None) -> Config:
+    """Return the current configuration."""
+    config = Config.from_env()
+    if verbose is not None:
+        config.verbose = verbose
+    return config

ap_client/exporter.py

@@ -0,0 +1,368 @@
+"""Export job/group data from Agent Platform to local directories."""
+
+from __future__ import annotations
+import concurrent.futures
+from functools import lru_cache
+import json
+import os
+import socket
+import sys
+import tarfile
+import threading
+from pathlib import Path
+from typing import Callable, Optional
+from urllib.parse import urlparse, urlunparse
+
+import requests
+
+from .api import APIClient
+
+DEFAULT_CACHE_DIR = "~/.cache/agentplatform"
+_DOWNLOAD_TIMEOUT = (10, 300)
+_DOWNLOAD_CHUNK_SIZE = 1024 * 1024 * 4
+ProgressCallback = Callable[[int, int, str, str], None]
+StageCallback = Callable[[str], None]
+
+
+def default_cache_dir() -> Path:
+    """Return the export cache root."""
+    raw = os.environ.get("AP_CACHE_DIR", DEFAULT_CACHE_DIR)
+    return Path(raw).expanduser()
+
+
+def default_job_export_dir(job_id: str) -> Path:
+    """Return the default export path for one job."""
+    return default_cache_dir() / "jobs" / job_id
+
+
+def default_group_export_dir(group_id: str) -> Path:
+    """Return the default export path for one group."""
+    return default_cache_dir() / "groups" / group_id
+
+
+def export_job(
+    client: APIClient,
+    job_id: str,
+    output_dir: Optional[Path] = None,
+) -> Path:
+    """Export a single job to disk."""
+    dest = Path(output_dir) if output_dir else default_job_export_dir(job_id)
+    dest.mkdir(parents=True, exist_ok=True)
+    _export_job_into_dir(client, job_id, dest)
+    return dest
+
+
+def export_group(
+    client: APIClient,
+    group_id: str,
+    output_dir: Optional[Path] = None,
+    progress_callback: Optional[ProgressCallback] = None,
+    workers: int = 4,
+) -> Path:
+    """Export a group and all of its jobs to disk."""
+    dest = Path(output_dir) if output_dir else default_group_export_dir(group_id)
+    dest.mkdir(parents=True, exist_ok=True)
+
+    group = client.get_group_stats(group_id)
+    _write_json(dest / "group.json", group)
+    try:
+        group_eval = client.get_group_eval(group_id)
+        _write_json(dest / "eval.json", group_eval)
+        _write_json(dest / "eval_summary.json", group_eval.get("summary", {}))
+        _write_json(dest / "eval_tasks.json", group_eval.get("results", []))
+    except Exception:
+        # Some groups are non-eval workloads, or older servers may not expose eval endpoints.
+        pass
+
+    jobs_dir = dest / "jobs"
+    jobs_dir.mkdir(parents=True, exist_ok=True)
+
+    workers = max(1, int(workers))
+    jobs = _list_group_jobs(client, group_id)
+    total = len(jobs)
+    done = 0
+    done_lock = threading.Lock()
+
+    if progress_callback is not None:
+        progress_callback(done, total, "", "listing jobs")
+
+    def _notify(job_id: str, stage: str) -> None:
+        if progress_callback is None:
+            return
+        with done_lock:
+            current_done = done
+        progress_callback(current_done, total, job_id, stage)
+
+    def _mark_done(job_id: str) -> None:
+        nonlocal done
+        with done_lock:
+            done += 1
+            current_done = done
+        if progress_callback is not None:
+            progress_callback(current_done, total, job_id, "done")
+
+    if workers == 1 or total <= 1:
+        for job in jobs:
+            job_id = job["job_id"]
+            _notify(job_id, "starting")
+            _export_job_into_dir(
+                client,
+                job_id,
+                jobs_dir / job_id,
+                job=job,
+                stage_callback=lambda stage, current_job_id=job_id: _notify(current_job_id, stage),
+            )
+            _mark_done(job_id)
+        return dest
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+        future_map = {
+            executor.submit(
+                _export_job_with_fresh_client,
+                client.config,
+                job,
+                jobs_dir / job["job_id"],
+                lambda stage, current_job_id=job["job_id"]: _notify(current_job_id, stage),
+            ): job["job_id"]
+            for job in jobs
+        }
+        for future in concurrent.futures.as_completed(future_map):
+            job_id = future_map[future]
+            future.result()
+            _mark_done(job_id)
+
+    return dest
+
+
+def _list_group_jobs(client: APIClient, group_id: str) -> list[dict]:
+    """Fetch all jobs in a group."""
+    skip = 0
+    limit = 200
+    jobs: list[dict] = []
+
+    while True:
+        page = client.list_jobs(group_id=group_id, skip=skip, limit=limit)
+        page_jobs = page.get("jobs", [])
+        jobs.extend(page_jobs)
+        if len(page_jobs) < limit:
+            return jobs
+        skip += limit
+
+
+def _export_job_with_fresh_client(
+    config,
+    job: dict,
+    dest: Path,
+    stage_callback: Optional[StageCallback] = None,
+) -> None:
+    """Export one job with a dedicated client/session for thread safety."""
+    worker_client = APIClient(config)
+    _export_job_into_dir(worker_client, job["job_id"], dest, job=job, stage_callback=stage_callback)
+
+
+def _export_job_into_dir(
+    client: APIClient,
+    job_id: str,
+    dest: Path,
+    job: Optional[dict] = None,
+    stage_callback: Optional[StageCallback] = None,
+) -> None:
+    """Export one job into an existing directory."""
+    dest.mkdir(parents=True, exist_ok=True)
+    current_stage = "job metadata"
+
+    def _set_stage(stage: str) -> None:
+        nonlocal current_stage
+        current_stage = stage
+        if stage_callback is not None:
+            stage_callback(stage)
+
+    try:
+        _set_stage("job metadata")
+        job_data = job or client.get_job(job_id)
+
+        _set_stage("metrics")
+        metrics = client.get_job_metrics(job_id)
+
+        _set_stage("artifact metadata")
+        artifact = _get_job_artifact(client, job_id)
+
+        _set_stage("logs")
+        logs = client.get_job_logs(job_id)
+
+        _set_stage("events")
+        events = client.get_job_events(job_id)
+
+        _set_stage("writing files")
+        _write_json(dest / "job.json", job_data)
+        _write_json(dest / "metrics.json", metrics)
+        _write_json(dest / "artifacts.json", artifact)
+        _write_text(dest / "events.txt", _format_events(events))
+        _write_logs(dest / "logs", logs)
+
+        artifact_url = artifact.get("url")
+        if artifact_url:
+            archive_path = dest / "result.tgz"
+            try:
+                _set_stage("downloading artifact")
+                _download_file(artifact_url, archive_path)
+                _set_stage("extracting artifact")
+                _extract_tgz(archive_path, dest / "artifacts")
+            except Exception as exc:
+                archive_path.unlink(missing_ok=True)
+                if _is_missing_artifact_error(exc):
+                    _warn_missing_artifact(job_id)
+                else:
+                    raise
+    except Exception as exc:
+        raise RuntimeError(f"Failed to export job {job_id} at stage '{current_stage}': {exc}") from exc
+
+
+def _get_job_artifact(client: APIClient, job_id: str) -> dict:
+    """Return artifact metadata for one job."""
+    items = client.get_job_artifacts([job_id])
+    return items[0] if items else {"job_id": job_id, "url": None}
+
+
+def _write_json(path: Path, data: object) -> None:
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _write_text(path: Path, content: str) -> None:
+    path.write_text(content, encoding="utf-8")
+
+
+def _write_logs(logs_dir: Path, logs: dict) -> None:
+    logs_dir.mkdir(parents=True, exist_ok=True)
+
+    if "containers" in logs:
+        containers = logs.get("containers") or {}
+    else:
+        container = logs.get("container") or "main"
+        containers = {container: logs.get("logs", "")}
+
+    for container, content in containers.items():
+        filename = _sanitize_name(container or "default") + ".log"
+        _write_text(logs_dir / filename, str(content or ""))
+
+
+def _format_events(events: dict) -> str:
+    if "containers" in events:
+        containers = events.get("containers") or {}
+    else:
+        container = events.get("container") or "default"
+        containers = {container: events.get("events", [])}
+
+    lines = []
+    for container in sorted(containers):
+        values = containers.get(container) or []
+        if lines:
+            lines.append("")
+        lines.append("[{}]".format(container))
+        lines.extend(str(item) for item in values)
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+def _download_file(url: str, dest: Path) -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    temp_path = dest.with_name(dest.name + ".part")
+    public_url = url
+    download_url = _prefer_internal_oss_url(url)
+    response = None
+    try:
+        try:
+            response = requests.get(download_url, timeout=_DOWNLOAD_TIMEOUT, stream=True)
+            response.raise_for_status()
+        except (requests.ConnectionError, requests.Timeout):
+            if response is not None:
+                response.close()
+            if download_url == public_url:
+                raise
+            response = requests.get(public_url, timeout=_DOWNLOAD_TIMEOUT, stream=True)
+        response.raise_for_status()
+        with temp_path.open("wb") as f:
+            for chunk in response.iter_content(chunk_size=_DOWNLOAD_CHUNK_SIZE):
+                if chunk:
+                    f.write(chunk)
+        temp_path.replace(dest)
+    except Exception:
+        if temp_path.exists():
+            temp_path.unlink()
+        raise
+    finally:
+        if response is not None:
+            response.close()
+
+
+def _prefer_internal_oss_url(url: str) -> str:
+    parsed = urlparse(url)
+    hostname = parsed.hostname
+    if not hostname:
+        return url
+    internal_host = _to_internal_oss_host(hostname)
+    if internal_host == hostname:
+        return url
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    if not can_connect(internal_host, port=port):
+        return url
+    netloc = internal_host
+    if parsed.port is not None:
+        netloc = f"{netloc}:{parsed.port}"
+    return urlunparse(parsed._replace(netloc=netloc))
+
+
+def _to_internal_oss_host(hostname: str) -> str:
+    if ".oss-" not in hostname or hostname.endswith("-internal.aliyuncs.com"):
+        return hostname
+    suffix = ".aliyuncs.com"
+    if not hostname.endswith(suffix):
+        return hostname
+    return hostname[: -len(suffix)] + "-internal.aliyuncs.com"
+
+
+@lru_cache()
+def can_connect(endpoint: str, port: int = 80, timeout: int = 1) -> bool:
+    """Check whether the host is reachable."""
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
+    try:
+        ip = socket.gethostbyname(endpoint)
+        sock.connect((ip, port))
+        return True
+    except (socket.timeout, socket.gaierror, socket.error):
+        return False
+    finally:
+        sock.close()
+
+
+def _is_missing_artifact_error(exc: Exception) -> bool:
+    if isinstance(exc, requests.HTTPError):
+        response = exc.response
+        if response is not None:
+            if response.status_code == 404:
+                return True
+            try:
+                body = response.text or ""
+            except Exception:
+                body = ""
+            if "NoSuchKey" in body:
+                return True
+    return "NoSuchKey" in str(exc)
+
+
+def _warn_missing_artifact(job_id: str) -> None:
+    print(
+        f"[ap export] warning: artifact missing for job {job_id}; "
+        "skipping result.tgz download",
+        file=sys.stderr,
+    )
+
+
+def _extract_tgz(archive_path: Path, dest_dir: Path) -> None:
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    with tarfile.open(archive_path, "r:gz") as tar:
+        tar.extractall(dest_dir)
+
+
+def _sanitize_name(name: str) -> str:
+    return "".join(ch if ch.isalnum() or ch in ("-", "_", ".") else "-" for ch in name)

ap_client/waiter.py

@@ -0,0 +1,70 @@
+"""Polling helpers for waiting on jobs/groups."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime
+from typing import Callable
+
+from .api import APIClient
+
+
+def wait_for_job(
+    client: APIClient,
+    job_id: str,
+    interval: float = 5.0,
+    printer: Callable[[str], None] = print,
+) -> dict:
+    """Poll until a job reaches a terminal status."""
+    _TERMINAL = {"Succeeded", "Failed", "Cancelled", "Unknown"}
+    while True:
+        job = client.get_job(job_id)
+        _print_job_status(job, printer)
+        if job.get("status") in _TERMINAL:
+            return job
+        time.sleep(interval)
+
+
+def wait_for_group(
+    client: APIClient,
+    group_id: str,
+    interval: float = 5.0,
+    printer: Callable[[str], None] = print,
+) -> dict:
+    """Poll until all jobs in a group are finished."""
+    while True:
+        group = client.get_group_stats(group_id)
+        _print_group_status(group, printer)
+        stats = group.get("stats") or {}
+        total = int(stats.get("total", 0) or 0)
+        finished = int(stats.get("finished", 0) or 0)
+        if total == 0 or finished >= total:
+            return group
+        time.sleep(interval)
+
+
+def _print_job_status(job: dict, printer: Callable[[str], None]) -> None:
+    refreshed_at = _now_str()
+    status = job.get("status", "Unknown")
+    failed_reason = job.get("failed_reason") or ""
+    extra = f" failed_reason={failed_reason}" if failed_reason else ""
+    printer(
+        f"[{refreshed_at}] job={job.get('job_id')} "
+        f"status={status}{extra}"
+    )
+
+
+def _print_group_status(group: dict, printer: Callable[[str], None]) -> None:
+    refreshed_at = _now_str()
+    stats = group.get("stats") or {}
+    printer(
+        f"[{refreshed_at}] group={group.get('group_id')} "
+        f"total={stats.get('total', 0)} queued={stats.get('queued', 0)} "
+        f"pending={stats.get('pending', 0)} running={stats.get('running', 0)} "
+        f"finished={stats.get('finished', 0)} succeeded={stats.get('succeeded', 0)} "
+        f"failed={stats.get('failed', 0)} cancelled={stats.get('cancelled', 0)}"
+    )
+
+
+def _now_str() -> str:
+    return datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S")

ap_client-0.1.4.dev0.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: ap-client
+Version: 0.1.4.dev0
+Summary: Agent Platform API Client & CLI
+Requires-Python: >=3.10
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.28.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.9.0

ap_client-0.1.4.dev0.dist-info/RECORD

@@ -0,0 +1,10 @@
+ap_client/__init__.py,sha256=X-OBg7lVh4rEKgCMF29G20FvxcezmiK4rxT0tKlyOd4,351
+ap_client/api.py,sha256=5iG3Czg4YGvUZUd9NY_cfGCrUynsg3GL68BQB2nDp_0,17071
+ap_client/cli.py,sha256=D6UZwOr-VV7YVJegMgrV3pefu3FpsfFo9-fCw-9Xjt0,36075
+ap_client/config.py,sha256=-1a6dAyz4IhWX-jT-tHKvhmhXdZSIRhjWmf_wTejK4I,1977
+ap_client/exporter.py,sha256=z0NUXt5GDwsaaDr87NtlgAgld4u6cn-OkalgZbOccTA,11732
+ap_client/waiter.py,sha256=Iho34PZc0KzSUR2sLv0FS3Xwmb7JOJzjAqbIge_Gsfs,2200
+ap_client-0.1.4.dev0.dist-info/METADATA,sha256=aqKeqmwBNUdNE3-88_3zQKPGk5OCXVATCPcFItyzZHI,238
+ap_client-0.1.4.dev0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+ap_client-0.1.4.dev0.dist-info/entry_points.txt,sha256=gsL9nrj3zDkk6baW4tzt5NSDNAdQFyNNjKZbi3-wBbI,41
+ap_client-0.1.4.dev0.dist-info/RECORD,,

ap_client-0.1.4.dev0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

ap_client-0.1.4.dev0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ap = ap_client.cli:app