ap-client 0.6.1.dev12__tar.gz → 0.6.1.dev13__tar.gz

{ap_client-0.6.1.dev12 → ap_client-0.6.1.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ap-client
-Version: 0.6.1.dev12
+Version: 0.6.1.dev13
 Summary: Agent Platform API Client & CLI
 Requires-Python: >=3.10
 Requires-Dist: irepo-artifact==0.2.2

{ap_client-0.6.1.dev12 → ap_client-0.6.1.dev13}/ap_client/fs_commands.py

@@ -39,6 +39,8 @@ from __future__ import annotations
 
 import inspect
 import os
+import sys
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Optional
 
@@ -270,6 +272,32 @@ def _user_data_prefix(uid: str) -> str:
     return f"swe/user-data/users/{uid}/"
 
 
+def _existing_policy(on_exist: str, overwrite: bool, skip_existing: bool):
+    policy = _check_choice(on_exist, ("prompt", "overwrite", "skip", "error"), "--on-exist")
+    if overwrite and skip_existing or (policy != "prompt" and (overwrite or skip_existing)):
+        raise _fail(
+            "Conflicting existing-file options",
+            "Choose one of --on-exist, --overwrite, --skip-existing",
+            2,
+        )
+    return "overwrite" if overwrite else "skip" if skip_existing else policy
+
+
+def _replace_existing(path: str, policy: str) -> bool:
+    if policy == "overwrite":
+        return True
+    if policy == "skip":
+        return False
+    if policy == "error" or not all(
+        stream.isatty() for stream in (sys.stdin, sys.stdout, sys.stderr)
+    ):
+        raise FileExistsError(
+            f"Destination already exists: {path}. Use --overwrite or --skip-existing; "
+            "interactive confirmation requires a terminal."
+        )
+    return typer.confirm(f"Overwrite {path}?", default=False, err=True)
+
+
 # ---------------------------------------------------------------------------
 # ap fs ls / get / put
 # ---------------------------------------------------------------------------
@@ -277,28 +305,116 @@ def _user_data_prefix(uid: str) -> str:
 
 @fs_app.command("ls")
 def fs_ls(
+    path: Optional[str] = typer.Argument(None, help="Directory to list (default: space root)"),
     directory: Optional[str] = typer.Option(None, "--dir", help="Sub-directory prefix"),
+    recursive: bool = typer.Option(False, "--recursive", "-R", help="List all descendants"),
+    all_entries: bool = typer.Option(
+        False, "--all", "-a", help="Include names starting with a dot"
+    ),
+    reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse name sort order"),
+    long: bool = typer.Option(
+        False, "--long", "-l", help="Show type, size in bytes and modified time"
+    ),
+    human_readable: bool = typer.Option(
+        False, "--human-readable", "-h", help="Use readable sizes with -l"
+    ),
+    sort: str = typer.Option(
+        "name", "--sort", help="Sort by name/size/time (size and time: largest/newest first)"
+    ),
+    directories_only: bool = typer.Option(False, "--dirs-only", help="Show directories only"),
+    files_only: bool = typer.Option(False, "--files-only", help="Show files only"),
+    limit: Optional[int] = typer.Option(
+        None, "--limit", min=1, help="Maximum number of displayed entries"
+    ),
     user: Optional[str] = typer.Option(None, "--user", help=_USER_HELP),
     output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
 ):
-    """List objects under a user storage space."""
+    """List one directory level; use -R to include descendants."""
+    from .fs_storage import UserDataFiles
     from .irepo_sdk import sdk_error_exit
 
     output_format = _fmt(output_format)
+    sort = _check_choice(sort, ("name", "size", "time"), "--sort")
+    if path is not None and directory is not None:
+        raise _fail("Specify a path or --dir, not both", "Example: ap fs ls probe", 2)
+    if directories_only and files_only:
+        raise _fail("--dirs-only conflicts with --files-only", "Choose one type filter", 2)
+    directory = path if path is not None else directory or ""
     repo = _load_repo_or_exit()
     uid = _resolve_uid(repo, user)
     try:
-        keys = repo.list_user_data(uid, subpath=(directory or ""))
+        rows = UserDataFiles(repo).list(uid, directory, recursive=recursive)
     except Exception as exc:  # noqa: BLE001
         sdk_error_exit(exc, what=f"user storage {uid}", stage="list")
 
-    prefix = _user_data_prefix(uid)
-    rows = []
-    for key in list(keys or []):
-        text = str(key)
-        rows.append({"path": text[len(prefix) :] if text.startswith(prefix) else text, "key": text})
+    def visible(row):
+        return (
+            row.get("explicit", False)
+            or all_entries
+            or not any(part.startswith(".") for part in row["name"].split("/"))
+        )
+
+    rows = [
+        row
+        for row in rows
+        if visible(row)
+        and (not directories_only or row["type"] == "directory")
+        and (not files_only or row["type"] == "file")
+    ]
+    rows.sort(key=lambda row: row["path"])
+    if sort != "name":
+        field = "size" if sort == "size" else "modified"
+        rows.sort(key=lambda row: row[field] or 0, reverse=True)
+    if reverse:
+        rows.reverse()
+    if limit is not None:
+        rows = rows[:limit]
     payload = {"user": uid, "dir": directory or "", "files": rows}
-    _emit(payload, rows, (("path", "PATH"), ("key", "KEY")), output_format)
+    display_rows = []
+    for row in rows:
+        shown = dict(row)
+        size = row.get("size")
+        if human_readable and size is not None:
+            value = float(size)
+            for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+                if value < 1024 or unit == "TiB":
+                    shown["size"] = f"{value:g} {unit}"
+                    break
+                value /= 1024
+        if row.get("modified") is not None:
+            shown["modified"] = datetime.fromtimestamp(row["modified"], tz=timezone.utc).strftime(
+                "%Y-%m-%d %H:%M:%SZ"
+            )
+        display_rows.append(shown)
+    columns = (
+        (("name", "PATH"), ("type", "TYPE"), ("size", "BYTES"), ("modified", "MODIFIED"))
+        if long
+        else (("name", "PATH"),)
+    )
+    _emit(payload, display_rows, columns, output_format)
+
+
+@fs_app.command("mkdir")
+def fs_mkdir(
+    path: str = typer.Argument(..., help="Directory path inside the user storage space"),
+    parents: bool = typer.Option(
+        False, "--parents", "-p", help="Create missing parents; accept existing directories"
+    ),
+    user: Optional[str] = typer.Option(None, "--user", help=_USER_HELP),
+    output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
+):
+    """Create an empty directory using an OSS directory marker."""
+    from .fs_storage import UserDataFiles
+    from .irepo_sdk import sdk_error_exit
+
+    output_format = _fmt(output_format)
+    repo = _load_repo_or_exit()
+    uid = _resolve_uid(repo, user)
+    try:
+        result = UserDataFiles(repo).mkdir(uid, path, parents=parents)
+    except Exception as exc:
+        sdk_error_exit(exc, what=f"user storage {uid} {path}", stage="mkdir")
+    _emit_detail({"user": uid, **result}, output_format)
 
 
 @fs_app.command("get")
@@ -309,6 +425,15 @@ def fs_get(
     target: Optional[str] = typer.Option(
         None, "--target", help="Local target path (default: the remote file name in CWD)"
     ),
+    overwrite: bool = typer.Option(
+        False, "--overwrite", "-f", help="Replace an existing destination"
+    ),
+    skip_existing: bool = typer.Option(
+        False, "--skip-existing", "-n", help="Keep existing destinations"
+    ),
+    on_exist: str = typer.Option(
+        "prompt", "--on-exist", help="Existing destination: prompt/overwrite/skip/error"
+    ),
     concurrency: int = typer.Option(
         8, "--concurrency", "-j", min=1, max=64, help="Maximum concurrent transfer requests"
     ),
@@ -324,23 +449,41 @@ def fs_get(
     from .irepo_sdk import sdk_error_exit
 
     output_format = _fmt(output_format)
+    policy = _existing_policy(on_exist, overwrite, skip_existing)
     repo = _load_repo_or_exit()
     uid = _resolve_uid(repo, user)
     remote = _remote_subpath(filepath, directory)
     local = Path(target) if target else Path(Path(remote).name)
-    if local.is_dir():
+    if local.is_dir() or (target and target.endswith("/")):
         local = local / Path(remote).name
+    skipped = False
     try:
+        if local.is_symlink():
+            raise ValueError("Download target must not be a symbolic link")
+        exists = local.exists()
+        if exists and not local.is_file():
+            raise ValueError("Download target must be a regular file")
+        skipped = exists and not _replace_existing(str(local), policy)
         if local.parent and str(local.parent) not in ("", "."):
             local.parent.mkdir(parents=True, exist_ok=True)
-        written = UserDataTransfer(repo, concurrency=concurrency, progress=progress).get(
-            uid, remote, local
+        written = (
+            str(local)
+            if skipped
+            else UserDataTransfer(repo, concurrency=concurrency, progress=progress).get(
+                uid, remote, local, overwrite=exists or policy == "overwrite"
+            )
         )
     except Exception as exc:  # noqa: BLE001
         sdk_error_exit(exc, what=f"user storage {uid} {remote}", stage="download")
 
     _emit_detail(
-        {"user": uid, "remote": remote, "local": str(written or local)},
+        {
+            "user": uid,
+            "remote": remote,
+            "local": str(written or local),
+            "downloaded": 0 if skipped else 1,
+            "skipped": int(skipped),
+        },
         output_format,
     )
 
@@ -353,7 +496,15 @@ def fs_put(
     source: Optional[str] = typer.Option(
         None, "--source", help="Local source path (default: <filepath> relative to CWD)"
     ),
-    overwrite: bool = typer.Option(False, "--overwrite", help="Allow overwriting remote objects"),
+    overwrite: bool = typer.Option(
+        False, "--overwrite", "-f", help="Allow overwriting remote objects"
+    ),
+    skip_existing: bool = typer.Option(
+        False, "--skip-existing", "-n", help="Keep existing remote objects"
+    ),
+    on_exist: str = typer.Option(
+        "prompt", "--on-exist", help="Existing destination: prompt/overwrite/skip/error"
+    ),
     concurrency: int = typer.Option(
         8, "--concurrency", "-j", min=1, max=64, help="Maximum concurrent transfer requests"
     ),
@@ -369,6 +520,7 @@ def fs_put(
     from .irepo_sdk import sdk_error_exit
 
     output_format = _fmt(output_format)
+    policy = _existing_policy(on_exist, overwrite, skip_existing)
     local = Path(source or filepath)
     if not local.exists():
         raise _fail(
@@ -381,21 +533,30 @@ def fs_put(
     uid = _resolve_uid(repo, user)
     remote = _remote_subpath(filepath, directory)
     is_dir = local.is_dir()
+    if filepath.endswith("/") and not is_dir:
+        remote = remote.rstrip("/") + "/" + local.name
     try:
         outcome: Any = UserDataTransfer(repo, concurrency=concurrency, progress=progress).put(
-            uid, local, remote, overwrite=overwrite
+            uid,
+            local,
+            remote,
+            overwrite=policy == "overwrite",
+            on_conflict=lambda key: _replace_existing(key, policy),
         )
     except Exception as exc:  # noqa: BLE001
         sdk_error_exit(exc, what=f"user storage {uid} {remote}", stage="upload")
 
-    uploaded = len(outcome) if isinstance(outcome, list) else 1
+    results = outcome if isinstance(outcome, list) else [outcome]
+    skipped = sum(isinstance(item, dict) and item.get("skipped", False) for item in results)
     _emit_detail(
         {
             "user": uid,
             "remote": remote,
             "local": str(local),
             "kind": "directory" if is_dir else "file",
-            "objects": uploaded,
+            "objects": len(results),
+            "uploaded": len(results) - skipped,
+            "skipped": skipped,
         },
         output_format,
     )

ap_client-0.6.1.dev13/ap_client/fs_storage.py

@@ -0,0 +1,146 @@
+"""Directory operations using the SDK's scoped OSS authorization and routing."""
+
+from __future__ import annotations
+
+from .fs_transfer import UserDataTransfer, _scope_denied
+
+
+class UserDataFiles(UserDataTransfer):
+    def list(self, uid: str, path: str = "", *, recursive: bool = False):
+        from instance_repo._routing import _OperationIntent
+
+        root = self.repo._user_data_prefix(uid)
+        prefix = root if path in ("", ".", "/") else self._key(uid, path) + "/"
+
+        def listing(bucket):
+            entries = {}
+            found = False
+            marker = ""
+            while True:
+                page = bucket.list_objects(
+                    prefix=prefix, delimiter="" if recursive else "/", marker=marker, max_keys=1000
+                )
+                found = found or bool(page.object_list or page.prefix_list)
+                for obj in page.object_list:
+                    if obj.key == prefix:
+                        continue
+                    kind = "directory" if obj.key.endswith("/") else "file"
+                    entries[obj.key] = dict(
+                        path=obj.key[len(root) :],
+                        name=obj.key[len(prefix) :],
+                        key=obj.key,
+                        type=kind,
+                        size=obj.size,
+                        modified=obj.last_modified,
+                    )
+                    if recursive:
+                        # OSS may have no marker object for a directory. Rebuild
+                        # its ancestors so -R --dirs-only includes these too.
+                        parts = obj.key[len(prefix) :].split("/")
+                        for depth in range(1, len(parts)):
+                            name = "/".join(parts[:depth]) + "/"
+                            key = prefix + name
+                            entries.setdefault(
+                                key,
+                                dict(
+                                    path=key[len(root) :],
+                                    name=name,
+                                    key=key,
+                                    type="directory",
+                                    size=None,
+                                    modified=None,
+                                ),
+                            )
+                for common in page.prefix_list:
+                    entries[common] = dict(
+                        path=common[len(root) :],
+                        name=common[len(prefix) :],
+                        key=common,
+                        type="directory",
+                        size=None,
+                        modified=None,
+                    )
+                if not page.is_truncated:
+                    break
+                if not page.next_marker or page.next_marker == marker:
+                    raise ValueError("OSS listing did not advance its pagination marker")
+                marker = page.next_marker
+            return list(entries.values()), found
+
+        try:
+            rows, found = self.content._execute_oss(
+                _OperationIntent.NON_DOWNLOAD, prefix, None, listing
+            )
+        except Exception as exc:
+            if prefix == root or not _scope_denied(exc):
+                raise
+            # A file-only grant need not permit listing the same-name directory.
+            rows, found = [], False
+        if found or prefix == root:
+            return rows
+        key = prefix.rstrip("/")
+
+        def file_entry(bucket):
+            if not bucket.object_exists(key):
+                raise FileNotFoundError(f"No such file or directory: {path}")
+            meta = bucket.head_object(key)
+            return [
+                dict(
+                    path=key[len(root) :],
+                    name=key.rsplit("/", 1)[-1],
+                    key=key,
+                    type="file",
+                    size=meta.content_length,
+                    modified=meta.last_modified,
+                    explicit=True,
+                )
+            ]
+
+        if path.endswith("/"):
+            raise FileNotFoundError(f"No such directory: {path}")
+        return self.content._execute_oss(_OperationIntent.NON_DOWNLOAD, key, None, file_entry)
+
+    def mkdir(self, uid: str, path: str, *, parents: bool = False):
+        from instance_repo._routing import _OperationIntent
+
+        clean = self.repo._clean_user_data_subpath(path)
+        segments = clean.split("/")
+        paths = ["/".join(segments[:i]) for i in range(1, len(segments) + 1)]
+
+        def exists(key):
+            return self.content._execute_oss(
+                _OperationIntent.NON_DOWNLOAD, key, None, lambda bucket: bucket.object_exists(key)
+            )
+
+        def directory_exists(key):
+            return self.content._execute_oss(
+                _OperationIntent.NON_DOWNLOAD,
+                key,
+                None,
+                lambda bucket: bool(bucket.list_objects(prefix=key, max_keys=1).object_list),
+            )
+
+        # Check all ancestors before creating any marker.
+        present = []
+        for part in paths:
+            key = self._key(uid, part)
+            if exists(key):
+                raise FileExistsError(f"A file already exists at {part}")
+            present.append(directory_exists(key + "/"))
+        if present[-1] and not parents:
+            raise FileExistsError(f"Directory already exists: {clean}; use --parents to keep it")
+        if not parents and not all(present[:-1]):
+            raise FileNotFoundError("Parent directory does not exist; use --parents/-p")
+        created = []
+        for part, found in zip(paths, present):
+            if found:
+                continue
+            key = self._key(uid, part) + "/"
+
+            def create(bucket, key=key):
+                if not bucket.object_exists(key):
+                    bucket.put_object(key, b"", headers={"x-oss-forbid-overwrite": "true"})
+
+            self.content._execute_oss(_OperationIntent.NON_DOWNLOAD, key, None, create)
+            created.append(part)
+        return {"path": clean, "created": created}

{ap_client-0.6.1.dev12 → ap_client-0.6.1.dev13}/ap_client/fs_transfer.py

@@ -6,6 +6,7 @@ import os
 import tempfile
 import threading
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from dataclasses import replace
 from pathlib import Path
 
 from rich.console import Console
@@ -24,6 +25,25 @@ from .ossutil_io import OssOperationError, _WriteBucket
 PART_SIZE = 8 * 1024 * 1024
 
 
+def _visual_task(task):
+    # Rich treats total=0 as 0%. Count processed files for zero-byte work,
+    # while retaining actual bytes for DownloadColumn and speed calculation.
+    if task.total == 0:
+        count = task.fields["files_total"]
+        return replace(task, total=count or 1, completed=task.fields["files_done"] if count else 1)
+    return task
+
+
+class _TransferBar(BarColumn):
+    def render(self, task):
+        return super().render(_visual_task(task))
+
+
+class _TransferPercent(TaskProgressColumn):
+    def render(self, task):
+        return super().render(_visual_task(task))
+
+
 def _scope_denied(exc: Exception) -> bool:
     """Only explicit scope refusals allow a more specific authorization request."""
     seen = set()
@@ -43,12 +63,14 @@ class TransferProgress:
         console = Console(stderr=True, markup=False, highlight=False)
         self.display = Progress(
             TextColumn("{task.description}"),
-            BarColumn(),
-            TaskProgressColumn(),
+            _TransferBar(),
+            _TransferPercent(),
             DownloadColumn(),
             TransferSpeedColumn(),
             TimeRemainingColumn(),
-            TextColumn("files {task.fields[files_done]}/{task.fields[files_total]}"),
+            TextColumn(
+                "files {task.fields[files_done]}/{task.fields[files_total]} skipped {task.fields[skipped]}"
+            ),
             console=console,
             disable=not (console.is_terminal if enabled is None else enabled),
         )
@@ -57,6 +79,7 @@ class TransferProgress:
         self.sizes: dict[str, int] = {}
         self.done: dict[str, int] = {}
         self.files_done = 0
+        self.files_skipped = 0
         self.total_bytes = 0
         self.completed_bytes = 0
 
@@ -69,6 +92,7 @@ class TransferProgress:
             total=None if unknown else self.total_bytes,
             files_done=0,
             files_total=len(sizes),
+            skipped=0,
         )
         return self
 
@@ -100,6 +124,7 @@ class TransferProgress:
         with self.lock:
             self.known_total = True
             if skipped:
+                self.files_skipped += 1
                 # Skipped bytes were not transferred; do not inflate the speed.
                 self.total_bytes -= self.sizes[key]
                 self.sizes[key] = 0
@@ -115,6 +140,7 @@ class TransferProgress:
             completed=self.completed_bytes,
             total=self.total_bytes if self.known_total else None,
             files_done=self.files_done,
+            skipped=self.files_skipped,
         )
 
 
@@ -135,17 +161,23 @@ class UserDataTransfer:
             raise ValueError("uid must not be empty")
         return self.repo._user_data_prefix(uid) + self.repo._clean_user_data_subpath(path)
 
-    def put(self, uid: str, source: Path, remote: str, *, overwrite: bool = False):
+    def put(
+        self, uid: str, source: Path, remote: str, *, overwrite: bool = False, on_conflict=None
+    ):
         from instance_repo._routing import _OperationIntent
 
         source = Path(source)
         remote = self.repo._clean_user_data_subpath(remote)
         base = self._key(uid, remote)
         directory = source.is_dir()
+        if source.is_symlink():
+            raise ValueError("Symbolic link upload sources are not supported")
         files = []
         if directory:
             for root, dirs, names in os.walk(source):
                 dirs.sort()
+                if any((Path(root) / name).is_symlink() for name in dirs + names):
+                    raise ValueError("Symbolic links inside an upload directory are not supported")
                 for name in sorted(names):
                     local = Path(root) / name
                     key = self._key(
@@ -161,27 +193,33 @@ class UserDataTransfer:
         part_threads = max(1, self.concurrency // workers)
         report = TransferProgress("Uploading", self.progress).start(sizes)
         directory_scope = base.rstrip("/") + "/" if directory else None
+        conflict_lock = threading.Lock()
 
         def transfer(item):
             local, key = item
             write_started = False
+            allowed_overwrite = overwrite
             # Keep the SDK's resume records across failures and later invocations.
             store = self.oss.ResumableStore()
 
             def upload(bucket):
-                nonlocal write_started
+                nonlocal write_started, allowed_overwrite
                 if not overwrite and not write_started and bucket.object_exists(key):
-                    return {"key": key, "skipped": True}
+                    with conflict_lock:
+                        # Serialize prompts even when multiple workers find collisions.
+                        allowed_overwrite = bool(on_conflict(key)) if on_conflict else False
+                    if not allowed_overwrite:
+                        return {"key": key, "skipped": True}
                 write_started = True
                 self.oss.resumable_upload(
-                    _WriteBucket(bucket, overwrite),
+                    _WriteBucket(bucket, allowed_overwrite),
                     key,
                     str(local),
                     store=store,
                     num_threads=part_threads,
                     part_size=PART_SIZE,
                     multipart_threshold=PART_SIZE,
-                    headers={} if overwrite else {"x-oss-forbid-overwrite": "true"},
+                    headers={} if allowed_overwrite else {"x-oss-forbid-overwrite": "true"},
                     progress_callback=report.callback(key),
                 )
                 return {"key": key, "skipped": False}
@@ -251,12 +289,12 @@ class UserDataTransfer:
                 raise
         return outcomes
 
-    def get(self, uid: str, remote: str, target: Path) -> str:
+    def get(self, uid: str, remote: str, target: Path, *, overwrite: bool = True) -> str:
         from instance_repo._routing import _OperationIntent
 
         key = self._key(uid, remote)
         target = Path(target)
-        if target.exists() and not target.is_file():
+        if target.is_symlink() or (target.exists() and not target.is_file()):
             raise ValueError("Download target must be a regular file")
         target.parent.mkdir(parents=True, exist_ok=True)
         report = TransferProgress("Downloading", self.progress).start({key: 0}, unknown=True)
@@ -282,6 +320,10 @@ class UserDataTransfer:
 
             self.content._execute_oss(_OperationIntent.DOWNLOAD, key, None, download)
             partial.chmod(0o600)
-            os.replace(partial, target)
+            if overwrite:
+                os.replace(partial, target)
+            else:
+                # Atomically refuse a destination created since the CLI preflight.
+                os.link(partial, target)
             report.finish(key)
         return str(target)

{ap_client-0.6.1.dev12 → ap_client-0.6.1.dev13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ap-client"
-version = "0.6.1.dev12"
+version = "0.6.1.dev13"
 description = "Agent Platform API Client & CLI"
 readme = { text = "A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.", content-type = "text/markdown" }
 requires-python = ">=3.10"