ap-client 0.1.24.dev0__tar.gz → 1.0.0.dev0__tar.gz

{ap_client-0.1.24.dev0 → ap_client-1.0.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ap-client
-Version: 0.1.24.dev0
+Version: 1.0.0.dev0
 Summary: Agent Platform API Client & CLI
 Requires-Python: >=3.10
 Requires-Dist: pyyaml>=6.0

{ap_client-0.1.24.dev0 → ap_client-1.0.0.dev0}/ap_client/api.py

@@ -2062,6 +2062,24 @@ class APIClient:
         """Get artifact download links for one or more jobs."""
         return self._post("/jobs/artifacts", {"job_ids": job_ids})
 
+    def get_job_artifacts_manifest(
+        self,
+        job_id: str,
+        *,
+        require_file_download: bool = False,
+    ) -> dict:
+        """Get the file manifest for one job's artifacts."""
+        params = {"require_file_download": "true"} if require_file_download else None
+        return self._get(
+            f"/jobs/{quote(job_id, safe='')}/artifacts-manifest",
+            params=params,
+        )
+
+    def get_artifact_file_download_urls(self, job_id: str, paths: list[str]) -> dict:
+        """Get download URLs for multiple artifact files."""
+        endpoint = f"/jobs/{quote(job_id, safe='')}/artifacts/files/download"
+        return self._post(endpoint, {"paths": paths})
+
     def render_template(self, name: str, params: dict) -> dict:
         """Preview the rendered template output."""
         query_params = {}
@@ -2718,6 +2736,20 @@ _SENSITIVE_SUBSTRINGS = (
     "passwd",
 )
 _SENSITIVE_SEGMENT_RE = re.compile(r"(?:^|[_-])(?:ak|sk)(?:[_-]|$)", re.IGNORECASE)
+_SENSITIVE_URL_QUERY_KEYS = frozenset(
+    {
+        "credential",
+        "ossaccesskeyid",
+        "security-token",
+        "signature",
+        "token",
+        "x-amz-credential",
+        "x-amz-security-token",
+        "x-amz-signature",
+        "x-oss-security-token",
+        "x-oss-signature",
+    }
+)
 
 
 def _redact_headers(headers: dict, enabled: bool = True) -> dict:
@@ -2790,7 +2822,7 @@ def _redact_url_query_secrets(value: str) -> str:
     changed = False
     query = []
     for key, item_value in parse_qsl(parsed.query, keep_blank_values=True):
-        if _is_sensitive_key(key) and item_value:
+        if (_is_sensitive_key(key) or key.lower() in _SENSITIVE_URL_QUERY_KEYS) and item_value:
             query.append((key, _mask_middle(item_value)))
             changed = True
         else:

{ap_client-0.1.24.dev0 → ap_client-1.0.0.dev0}/ap_client/cli.py

@@ -342,6 +342,14 @@ def _parse_datasources_option(raw: Optional[str]) -> Optional[list]:
     return [item.strip() for item in value.split(",") if item.strip()] or None
 
 
+def _parse_file_selectors(values: Optional[list[str]]) -> Optional[list[str]]:
+    """Expand repeatable, comma-separated file selector options."""
+    selectors = [
+        item.strip() for value in values or [] for item in value.split(",") if item.strip()
+    ]
+    return list(dict.fromkeys(selectors)) or None
+
+
 def _read_json_object_option(
     raw: Optional[str], file_path: Optional[str], option_name: str
 ) -> Optional[dict]:
@@ -4263,13 +4271,15 @@ _RESOURCE_ROLE_IDS = {
     ("job", "writer"): "built-in:job-writer",
     ("group", "reader"): "built-in:group-reader",
     ("group", "writer"): "built-in:group-writer",
+    ("group", "cloner"): "built-in:group-cloner",
 }
 
 
 def _resource_role_id(resource: str, role: str) -> str:
     key = (resource, (role or "").strip().lower())
     if key not in _RESOURCE_ROLE_IDS:
-        raise typer.BadParameter("role must be one of: reader, writer")
+        choices = "reader, writer, cloner" if resource == "group" else "reader, writer"
+        raise typer.BadParameter(f"role must be one of: {choices}")
     return _RESOURCE_ROLE_IDS[key]
 
 
@@ -5840,7 +5850,9 @@ def group_grant(
         None, "--user-group", help="Grant to this user_group_id"
     ),
     role: str = typer.Option(
-        "reader", "--role", help="reader (view) or writer (view+cancel/update)"
+        "reader",
+        "--role",
+        help="reader (view), cloner (view+clone), or writer (view+cancel/update)",
     ),
 ):
     """Grant a user/user-group access to a specific group (RBAC v2 instance delegation)."""
@@ -5854,7 +5866,7 @@ def group_revoke(
     user_group: Optional[str] = typer.Option(
         None, "--user-group", help="Revoke from this user_group_id"
     ),
-    role: str = typer.Option("reader", "--role", help="reader or writer"),
+    role: str = typer.Option("reader", "--role", help="reader, cloner, or writer"),
 ):
     """Revoke a user/user-group's instance access to a specific group."""
     _revoke_instance_access("group", group_id, role, user, user_group)
@@ -5880,8 +5892,30 @@ def group_export(
         "--no-extract-artifacts",
         help="Download result.tgz without extracting it into artifacts/",
     ),
+    include_files: Optional[list[str]] = typer.Option(
+        None,
+        "--include-files",
+        help="Only download matching artifact files or folders (repeatable; comma-separated)",
+    ),
+    exclude_files: Optional[list[str]] = typer.Option(
+        None,
+        "--exclude-files",
+        help="Skip matching artifact files or folders (repeatable; comma-separated)",
+    ),
 ):
     """Export artifacts of all jobs under a Group to a local directory. Use --logs/--events to include logs and events."""
+    include_file_selectors = _parse_file_selectors(include_files)
+    exclude_file_selectors = _parse_file_selectors(exclude_files)
+    if include_files and not include_file_selectors:
+        _emit_error("--include-files requires at least one non-empty file or folder name")
+        raise typer.Exit(1)
+    if exclude_files and not exclude_file_selectors:
+        _emit_error("--exclude-files requires at least one non-empty file or folder name")
+        raise typer.Exit(1)
+    if no_extract_artifacts and (include_file_selectors or exclude_file_selectors):
+        _emit_error("--include-files/--exclude-files cannot be used with --no-extract-artifacts")
+        raise typer.Exit(1)
+
     client = get_client()
     from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn
 
@@ -5920,6 +5954,8 @@ def group_export(
             include_logs=logs,
             include_events=events,
             extract_artifacts=not no_extract_artifacts,
+            include_files=include_file_selectors,
+            exclude_files=exclude_file_selectors,
         )
     print(f"[green]Group exported:[/] {group_id}")
     print(f"  path: {dest}")

{ap_client-0.1.24.dev0 → ap_client-1.0.0.dev0}/ap_client/exporter.py

@@ -10,7 +10,10 @@ import shutil
 import socket
 import sys
 import tarfile
+import tempfile
 import threading
+import time
+import uuid
 from dataclasses import dataclass, field
 from functools import lru_cache
 from pathlib import Path
@@ -26,6 +29,8 @@ logger = logging.getLogger(__name__)
 DEFAULT_CACHE_DIR = "~/.cache/agentplatform"
 _DOWNLOAD_TIMEOUT = (10, 300)
 _DOWNLOAD_CHUNK_SIZE = 1024 * 1024 * 4
+_FILE_URL_BATCH_SIZE = 500
+_FILE_URL_EXPIRY_MARGIN_SECONDS = 30
 ProgressCallback = Callable[[int, int, str, str], None]
 StageCallback = Callable[[str], None]
 
@@ -91,12 +96,19 @@ def export_group(
     include_logs: bool = False,
     include_events: bool = False,
     extract_artifacts: bool = True,
+    include_files: Optional[list[str]] = None,
+    exclude_files: Optional[list[str]] = None,
 ) -> tuple[Path, ExportSummary]:
     """Export a group and all of its jobs to disk.
 
     Jobs without artifact URLs are skipped: no directory is created for them.
     Returns (dest_path, ExportSummary) with download/skip statistics.
     """
+    include_files = _normalize_file_selectors(include_files)
+    exclude_files = _normalize_file_selectors(exclude_files)
+    if not extract_artifacts and (include_files or exclude_files):
+        raise ValueError("file filters cannot be used when artifact extraction is disabled")
+
     dest = Path(output_dir) if output_dir else default_group_export_dir(group_id)
     dest.mkdir(parents=True, exist_ok=True)
 
@@ -153,6 +165,8 @@ def export_group(
                 include_events=include_events,
                 extract_artifacts=extract_artifacts,
                 skip_no_artifact=True,
+                include_files=include_files,
+                exclude_files=exclude_files,
             )
             if result is not None:
                 skipped_jobs.append(result)
@@ -177,6 +191,8 @@ def export_group(
                 include_events=include_events,
                 extract_artifacts=extract_artifacts,
                 skip_no_artifact=True,
+                include_files=include_files,
+                exclude_files=exclude_files,
             ): job["job_id"]
             for job in jobs
         }
@@ -250,6 +266,8 @@ def _export_job_with_fresh_client(
     include_events: bool = False,
     extract_artifacts: bool = True,
     skip_no_artifact: bool = False,
+    include_files: Optional[list[str]] = None,
+    exclude_files: Optional[list[str]] = None,
 ) -> Optional[str]:
     """Export one job with a dedicated client/session for thread safety."""
     worker_client = APIClient(config)
@@ -263,6 +281,8 @@ def _export_job_with_fresh_client(
         include_events=include_events,
         extract_artifacts=extract_artifacts,
         skip_no_artifact=skip_no_artifact,
+        include_files=include_files,
+        exclude_files=exclude_files,
     )
 
 
@@ -277,6 +297,8 @@ def _export_job_into_dir(
     include_events: bool = False,
     extract_artifacts: bool = True,
     skip_no_artifact: bool = False,
+    include_files: Optional[list[str]] = None,
+    exclude_files: Optional[list[str]] = None,
 ) -> Optional[str]:
     """Export one job into a directory.
 
@@ -288,6 +310,7 @@ def _export_job_into_dir(
     Returns job_name if skipped, None if exported successfully.
     """
     current_stage = "job metadata"
+    selective_artifacts = bool(include_files or exclude_files)
 
     def _set_stage(stage: str) -> None:
         nonlocal current_stage
@@ -300,10 +323,21 @@ def _export_job_into_dir(
         job_data = job or client.get_job(job_id)
 
         _set_stage("artifact metadata")
-        artifact = _get_job_artifact(client, job_id)
+        manifest_payload = None
+        if selective_artifacts:
+            manifest_payload = _get_job_artifact_manifest(client, job_id)
+            artifact = {
+                "job_id": job_id,
+                "url": None,
+                "mode": "selected_files",
+            }
+            artifact_available = manifest_payload is not None
+        else:
+            artifact = _get_job_artifact(client, job_id)
+            artifact_available = bool(artifact.get("url"))
 
         # When skip_no_artifact is enabled, skip jobs without artifact URLs
-        if skip_no_artifact and not artifact.get("url"):
+        if skip_no_artifact and not artifact_available:
             _set_stage("skipped")
             job_name = job_data.get("job_name") or job_id
             job_status = job_data.get("status", "")
@@ -322,7 +356,8 @@ def _export_job_into_dir(
 
         _write_json(dest / "job.json", job_data)
         _write_json(dest / "metrics.json", metrics)
-        _write_json(dest / "artifacts.json", artifact)
+        if not selective_artifacts:
+            _write_json(dest / "artifacts.json", artifact)
 
         if include_events:
             _set_stage("events")
@@ -334,7 +369,19 @@ def _export_job_into_dir(
             _write_logs(dest / "logs", client, job_id, job_data)
 
         artifact_url = artifact.get("url")
-        if artifact_url:
+        if selective_artifacts:
+            _set_stage("downloading selected artifact files")
+            _download_selected_artifact_files(
+                client,
+                job_id,
+                manifest_payload.get("manifest"),
+                dest / "artifacts",
+                include_files=include_files,
+                exclude_files=exclude_files,
+            )
+            _write_json(dest / "artifacts.json", artifact)
+            _write_json(dest / "artifacts-manifest.json", manifest_payload)
+        elif artifact_url:
             archive_path = dest / "result.tgz"
             try:
                 _set_stage("downloading artifact")
@@ -378,6 +425,288 @@ def _get_job_artifact(client: APIClient, job_id: str) -> dict:
     return items[0] if items else {"job_id": job_id, "url": None}
 
 
+def _get_job_artifact_manifest(client: APIClient, job_id: str) -> Optional[dict]:
+    """Return a manifest payload, or None when the job has no artifact."""
+    payload = client.get_job_artifacts_manifest(job_id, require_file_download=True)
+    if not isinstance(payload, dict):
+        raise ValueError("Invalid artifact manifest response")
+    if "manifest" not in payload:
+        raise ValueError("Invalid artifact manifest response")
+    manifest = payload["manifest"]
+    if manifest is None:
+        return None
+    if not isinstance(manifest, list):
+        raise ValueError("Invalid artifact manifest response")
+    if payload.get("truncated") is True:
+        max_entries = payload.get("max_entries", "unknown")
+        raise ValueError(
+            f"Artifact manifest is truncated (max_entries={max_entries}); "
+            "selective export cannot guarantee a complete result"
+        )
+    return payload
+
+
+def _normalize_file_selectors(values: Optional[list[str]]) -> Optional[list[str]]:
+    selectors: list[str] = []
+    for raw in values or []:
+        if not isinstance(raw, str):
+            raise ValueError("Artifact file selectors must be strings")
+        selector = raw.strip()
+        while selector.startswith("./"):
+            selector = selector[2:]
+        selector = selector.rstrip("/")
+        _validate_relative_artifact_path(selector, "artifact file selector")
+        if selector not in selectors:
+            selectors.append(selector)
+    return selectors or None
+
+
+def _validate_relative_artifact_path(path: str, label: str) -> None:
+    invalid = (
+        not path
+        or path.startswith("/")
+        or "\\" in path
+        or "\x00" in path
+        or any(part in {"", ".", ".."} for part in path.split("/"))
+        or (len(path) >= 2 and path[0].isalpha() and path[1] == ":")
+    )
+    if invalid:
+        prefix = "Unsafe" if label == "artifact manifest path" else "Invalid"
+        raise ValueError(f"{prefix} {label}: {path!r}")
+
+
+def _parse_artifact_manifest(manifest: list[dict]) -> tuple[list[str], set[str]]:
+    files: list[str] = []
+    directories: set[str] = set()
+    seen_files: set[str] = set()
+    for item in manifest:
+        if not isinstance(item, dict):
+            raise ValueError("Invalid artifact manifest entry")
+        path = item.get("path")
+        if not isinstance(path, str):
+            raise ValueError("Invalid artifact manifest entry")
+        _validate_relative_artifact_path(path, "artifact manifest path")
+        item_type = item.get("type")
+        if item_type == "dir":
+            directories.add(path)
+        elif item_type == "file" and path not in seen_files:
+            seen_files.add(path)
+            files.append(path)
+    return files, directories
+
+
+def _selector_matches(path: str, selector: str) -> bool:
+    if "/" in selector:
+        return path == selector or path.startswith(selector + "/")
+    return selector in path.split("/")
+
+
+def _manifest_path_matches(path: str, selector: str, strip_root: Optional[str]) -> bool:
+    if _selector_matches(path, selector):
+        return True
+    if strip_root and path.startswith(strip_root + "/"):
+        logical_path = path[len(strip_root) + 1 :]
+        return _selector_matches(logical_path, selector)
+    return False
+
+
+def _select_artifact_files(
+    files: list[str],
+    *,
+    include_files: Optional[list[str]],
+    exclude_files: Optional[list[str]],
+    strip_root: Optional[str] = None,
+) -> list[str]:
+    if include_files:
+        selected = [
+            path
+            for path in files
+            if any(_manifest_path_matches(path, selector, strip_root) for selector in include_files)
+        ]
+    else:
+        selected = list(files)
+    if exclude_files:
+        selected = [
+            path
+            for path in selected
+            if not any(
+                _manifest_path_matches(path, selector, strip_root) for selector in exclude_files
+            )
+        ]
+    return selected
+
+
+def _manifest_strip_root(files: list[str], directories: set[str]) -> Optional[str]:
+    roots = {path.split("/", 1)[0] for path in files if "/" in path}
+    if len(roots) != 1:
+        return None
+    root = next(iter(roots))
+    if root not in directories or any("/" not in path for path in files):
+        return None
+    return root
+
+
+def _download_selected_artifact_files(
+    client: APIClient,
+    job_id: str,
+    manifest: list[dict],
+    artifacts_dir: Path,
+    *,
+    include_files: Optional[list[str]],
+    exclude_files: Optional[list[str]],
+) -> None:
+    files, directories = _parse_artifact_manifest(manifest)
+    strip_root = _manifest_strip_root(files, directories)
+    selected = _select_artifact_files(
+        files,
+        include_files=include_files,
+        exclude_files=exclude_files,
+        strip_root=strip_root,
+    )
+
+    artifacts_dir.parent.mkdir(parents=True, exist_ok=True)
+    staging_dir = Path(
+        tempfile.mkdtemp(
+            prefix=f".{artifacts_dir.name}.downloading-",
+            dir=artifacts_dir.parent,
+        )
+    )
+    output_dir = staging_dir / "output"
+    output_dir.mkdir()
+
+    destinations = {
+        path: output_dir / (path[len(strip_root) + 1 :] if strip_root else path)
+        for path in selected
+    }
+
+    try:
+        for start in range(0, len(selected), _FILE_URL_BATCH_SIZE):
+            paths = selected[start : start + _FILE_URL_BATCH_SIZE]
+            index = 0
+            urls, refresh_at = _issue_artifact_file_urls(client, job_id, paths)
+            issued_at_index = index
+            refreshed_after_auth_error: set[str] = set()
+            while index < len(paths):
+                if (
+                    index > issued_at_index
+                    and refresh_at is not None
+                    and time.monotonic() >= refresh_at
+                ):
+                    urls, refresh_at = _issue_artifact_file_urls(client, job_id, paths[index:])
+                    issued_at_index = index
+                path = paths[index]
+                try:
+                    _download_file(urls[path], destinations[path])
+                except requests.HTTPError as exc:
+                    response = exc.response
+                    if (
+                        response is not None
+                        and response.status_code in {401, 403}
+                        and path not in refreshed_after_auth_error
+                    ):
+                        refreshed_after_auth_error.add(path)
+                        urls, refresh_at = _issue_artifact_file_urls(client, job_id, paths[index:])
+                        issued_at_index = index
+                        continue
+                    _raise_sanitized_artifact_download_error(path, exc)
+                except Exception as exc:
+                    _raise_sanitized_artifact_download_error(path, exc)
+                index += 1
+        _commit_staged_artifacts(staging_dir, artifacts_dir)
+    finally:
+        if _path_exists(staging_dir):
+            _remove_path(staging_dir)
+
+
+def _raise_sanitized_artifact_download_error(path: str, exc: Exception) -> None:
+    """Raise a diagnostic error without retaining a capability URL in its text."""
+    detail = f"Artifact file download failed for {path!r}"
+    if isinstance(exc, requests.HTTPError) and exc.response is not None:
+        detail += f" (HTTP {exc.response.status_code})"
+    else:
+        detail += f" ({type(exc).__name__})"
+    raise RuntimeError(detail) from None
+
+
+def _issue_artifact_file_urls(
+    client: APIClient, job_id: str, paths: list[str]
+) -> tuple[dict[str, str], Optional[float]]:
+    payload = client.get_artifact_file_download_urls(job_id, paths)
+    if not isinstance(payload, dict) or not isinstance(payload.get("files"), list):
+        raise ValueError("Invalid artifact file download URL response")
+    urls = {
+        item.get("path"): item.get("url")
+        for item in payload["files"]
+        if isinstance(item, dict)
+        and isinstance(item.get("path"), str)
+        and isinstance(item.get("url"), str)
+        and item.get("url")
+    }
+    missing = [path for path in paths if path not in urls]
+    if missing:
+        raise ValueError(
+            "Artifact file download URL response is missing: " + ", ".join(missing[:10])
+        )
+
+    expires_in = payload.get("expires_in")
+    refresh_at = None
+    if isinstance(expires_in, (int, float)) and not isinstance(expires_in, bool) and expires_in > 0:
+        margin = min(
+            _FILE_URL_EXPIRY_MARGIN_SECONDS,
+            max(1.0, float(expires_in) * 0.1),
+        )
+        refresh_at = time.monotonic() + max(0.0, float(expires_in) - margin)
+    return urls, refresh_at
+
+
+def _path_exists(path: Path) -> bool:
+    return path.exists() or path.is_symlink()
+
+
+def _remove_path(path: Path) -> None:
+    if path.is_symlink() or path.is_file():
+        path.unlink()
+    elif path.exists():
+        shutil.rmtree(path)
+
+
+def _commit_staged_artifacts(staging_dir: Path, artifacts_dir: Path) -> None:
+    """Atomically publish a complete selective download and retire stale archives."""
+    suffix = uuid.uuid4().hex
+    artifacts_backup = artifacts_dir.with_name(f".{artifacts_dir.name}.previous-{suffix}")
+    archive_path = artifacts_dir.parent / "result.tgz"
+    archive_backup = archive_path.with_name(f".{archive_path.name}.previous-{suffix}")
+    moved_artifacts = False
+    moved_archive = False
+    installed = False
+
+    try:
+        if _path_exists(artifacts_dir):
+            artifacts_dir.replace(artifacts_backup)
+            moved_artifacts = True
+        if _path_exists(archive_path):
+            archive_path.replace(archive_backup)
+            moved_archive = True
+        staging_dir.replace(artifacts_dir)
+        installed = True
+    except Exception:
+        if installed and _path_exists(artifacts_dir):
+            _remove_path(artifacts_dir)
+        if moved_artifacts and _path_exists(artifacts_backup):
+            artifacts_backup.replace(artifacts_dir)
+        if moved_archive and _path_exists(archive_backup):
+            archive_backup.replace(archive_path)
+        raise
+
+    for backup in (artifacts_backup, archive_backup):
+        if not _path_exists(backup):
+            continue
+        try:
+            _remove_path(backup)
+        except OSError as exc:
+            logger.warning("Failed to remove selective export backup %s: %s", backup, exc)
+
+
 def _write_json(path: Path, data: object) -> None:
     path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
 

{ap_client-0.1.24.dev0 → ap_client-1.0.0.dev0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ap-client"
-version = "0.1.24.dev0"
+version = "1.0.0.dev0"
 description = "Agent Platform API Client & CLI"
 readme = { text = "A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.", content-type = "text/markdown" }
 requires-python = ">=3.10"