ap-client 0.6.1.dev11__tar.gz → 0.6.1.dev13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/PKG-INFO +1 -1
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/fs_commands.py +177 -16
- ap_client-0.6.1.dev13/ap_client/fs_storage.py +146 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/fs_transfer.py +97 -13
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/pyproject.toml +1 -1
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/__init__.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/api.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/cli.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/config.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/dataset_commands.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/dataset_name.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/dataset_register.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/dfs_commands.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/exporter.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/instance_commands.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/instance_options.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/irepo_sdk.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/managed_ak.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/manifest_view.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/ossutil_hash.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/ossutil_io.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/profile_commands.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/repo_read_guard.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/share_api.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/split_publish.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/tbb.py +0 -0
- {ap_client-0.6.1.dev11 → ap_client-0.6.1.dev13}/ap_client/waiter.py +0 -0
|
@@ -39,6 +39,8 @@ from __future__ import annotations
|
|
|
39
39
|
|
|
40
40
|
import inspect
|
|
41
41
|
import os
|
|
42
|
+
import sys
|
|
43
|
+
from datetime import datetime, timezone
|
|
42
44
|
from pathlib import Path
|
|
43
45
|
from typing import Any, Optional
|
|
44
46
|
|
|
@@ -270,6 +272,32 @@ def _user_data_prefix(uid: str) -> str:
|
|
|
270
272
|
return f"swe/user-data/users/{uid}/"
|
|
271
273
|
|
|
272
274
|
|
|
275
|
+
def _existing_policy(on_exist: str, overwrite: bool, skip_existing: bool):
|
|
276
|
+
policy = _check_choice(on_exist, ("prompt", "overwrite", "skip", "error"), "--on-exist")
|
|
277
|
+
if overwrite and skip_existing or (policy != "prompt" and (overwrite or skip_existing)):
|
|
278
|
+
raise _fail(
|
|
279
|
+
"Conflicting existing-file options",
|
|
280
|
+
"Choose one of --on-exist, --overwrite, --skip-existing",
|
|
281
|
+
2,
|
|
282
|
+
)
|
|
283
|
+
return "overwrite" if overwrite else "skip" if skip_existing else policy
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _replace_existing(path: str, policy: str) -> bool:
|
|
287
|
+
if policy == "overwrite":
|
|
288
|
+
return True
|
|
289
|
+
if policy == "skip":
|
|
290
|
+
return False
|
|
291
|
+
if policy == "error" or not all(
|
|
292
|
+
stream.isatty() for stream in (sys.stdin, sys.stdout, sys.stderr)
|
|
293
|
+
):
|
|
294
|
+
raise FileExistsError(
|
|
295
|
+
f"Destination already exists: {path}. Use --overwrite or --skip-existing; "
|
|
296
|
+
"interactive confirmation requires a terminal."
|
|
297
|
+
)
|
|
298
|
+
return typer.confirm(f"Overwrite {path}?", default=False, err=True)
|
|
299
|
+
|
|
300
|
+
|
|
273
301
|
# ---------------------------------------------------------------------------
|
|
274
302
|
# ap fs ls / get / put
|
|
275
303
|
# ---------------------------------------------------------------------------
|
|
@@ -277,28 +305,116 @@ def _user_data_prefix(uid: str) -> str:
|
|
|
277
305
|
|
|
278
306
|
@fs_app.command("ls")
|
|
279
307
|
def fs_ls(
|
|
308
|
+
path: Optional[str] = typer.Argument(None, help="Directory to list (default: space root)"),
|
|
280
309
|
directory: Optional[str] = typer.Option(None, "--dir", help="Sub-directory prefix"),
|
|
310
|
+
recursive: bool = typer.Option(False, "--recursive", "-R", help="List all descendants"),
|
|
311
|
+
all_entries: bool = typer.Option(
|
|
312
|
+
False, "--all", "-a", help="Include names starting with a dot"
|
|
313
|
+
),
|
|
314
|
+
reverse: bool = typer.Option(False, "--reverse", "-r", help="Reverse name sort order"),
|
|
315
|
+
long: bool = typer.Option(
|
|
316
|
+
False, "--long", "-l", help="Show type, size in bytes and modified time"
|
|
317
|
+
),
|
|
318
|
+
human_readable: bool = typer.Option(
|
|
319
|
+
False, "--human-readable", "-h", help="Use readable sizes with -l"
|
|
320
|
+
),
|
|
321
|
+
sort: str = typer.Option(
|
|
322
|
+
"name", "--sort", help="Sort by name/size/time (size and time: largest/newest first)"
|
|
323
|
+
),
|
|
324
|
+
directories_only: bool = typer.Option(False, "--dirs-only", help="Show directories only"),
|
|
325
|
+
files_only: bool = typer.Option(False, "--files-only", help="Show files only"),
|
|
326
|
+
limit: Optional[int] = typer.Option(
|
|
327
|
+
None, "--limit", min=1, help="Maximum number of displayed entries"
|
|
328
|
+
),
|
|
281
329
|
user: Optional[str] = typer.Option(None, "--user", help=_USER_HELP),
|
|
282
330
|
output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
|
|
283
331
|
):
|
|
284
|
-
"""List
|
|
332
|
+
"""List one directory level; use -R to include descendants."""
|
|
333
|
+
from .fs_storage import UserDataFiles
|
|
285
334
|
from .irepo_sdk import sdk_error_exit
|
|
286
335
|
|
|
287
336
|
output_format = _fmt(output_format)
|
|
337
|
+
sort = _check_choice(sort, ("name", "size", "time"), "--sort")
|
|
338
|
+
if path is not None and directory is not None:
|
|
339
|
+
raise _fail("Specify a path or --dir, not both", "Example: ap fs ls probe", 2)
|
|
340
|
+
if directories_only and files_only:
|
|
341
|
+
raise _fail("--dirs-only conflicts with --files-only", "Choose one type filter", 2)
|
|
342
|
+
directory = path if path is not None else directory or ""
|
|
288
343
|
repo = _load_repo_or_exit()
|
|
289
344
|
uid = _resolve_uid(repo, user)
|
|
290
345
|
try:
|
|
291
|
-
|
|
346
|
+
rows = UserDataFiles(repo).list(uid, directory, recursive=recursive)
|
|
292
347
|
except Exception as exc: # noqa: BLE001
|
|
293
348
|
sdk_error_exit(exc, what=f"user storage {uid}", stage="list")
|
|
294
349
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
350
|
+
def visible(row):
|
|
351
|
+
return (
|
|
352
|
+
row.get("explicit", False)
|
|
353
|
+
or all_entries
|
|
354
|
+
or not any(part.startswith(".") for part in row["name"].split("/"))
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
rows = [
|
|
358
|
+
row
|
|
359
|
+
for row in rows
|
|
360
|
+
if visible(row)
|
|
361
|
+
and (not directories_only or row["type"] == "directory")
|
|
362
|
+
and (not files_only or row["type"] == "file")
|
|
363
|
+
]
|
|
364
|
+
rows.sort(key=lambda row: row["path"])
|
|
365
|
+
if sort != "name":
|
|
366
|
+
field = "size" if sort == "size" else "modified"
|
|
367
|
+
rows.sort(key=lambda row: row[field] or 0, reverse=True)
|
|
368
|
+
if reverse:
|
|
369
|
+
rows.reverse()
|
|
370
|
+
if limit is not None:
|
|
371
|
+
rows = rows[:limit]
|
|
300
372
|
payload = {"user": uid, "dir": directory or "", "files": rows}
|
|
301
|
-
|
|
373
|
+
display_rows = []
|
|
374
|
+
for row in rows:
|
|
375
|
+
shown = dict(row)
|
|
376
|
+
size = row.get("size")
|
|
377
|
+
if human_readable and size is not None:
|
|
378
|
+
value = float(size)
|
|
379
|
+
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
|
|
380
|
+
if value < 1024 or unit == "TiB":
|
|
381
|
+
shown["size"] = f"{value:g} {unit}"
|
|
382
|
+
break
|
|
383
|
+
value /= 1024
|
|
384
|
+
if row.get("modified") is not None:
|
|
385
|
+
shown["modified"] = datetime.fromtimestamp(row["modified"], tz=timezone.utc).strftime(
|
|
386
|
+
"%Y-%m-%d %H:%M:%SZ"
|
|
387
|
+
)
|
|
388
|
+
display_rows.append(shown)
|
|
389
|
+
columns = (
|
|
390
|
+
(("name", "PATH"), ("type", "TYPE"), ("size", "BYTES"), ("modified", "MODIFIED"))
|
|
391
|
+
if long
|
|
392
|
+
else (("name", "PATH"),)
|
|
393
|
+
)
|
|
394
|
+
_emit(payload, display_rows, columns, output_format)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@fs_app.command("mkdir")
|
|
398
|
+
def fs_mkdir(
|
|
399
|
+
path: str = typer.Argument(..., help="Directory path inside the user storage space"),
|
|
400
|
+
parents: bool = typer.Option(
|
|
401
|
+
False, "--parents", "-p", help="Create missing parents; accept existing directories"
|
|
402
|
+
),
|
|
403
|
+
user: Optional[str] = typer.Option(None, "--user", help=_USER_HELP),
|
|
404
|
+
output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
|
|
405
|
+
):
|
|
406
|
+
"""Create an empty directory using an OSS directory marker."""
|
|
407
|
+
from .fs_storage import UserDataFiles
|
|
408
|
+
from .irepo_sdk import sdk_error_exit
|
|
409
|
+
|
|
410
|
+
output_format = _fmt(output_format)
|
|
411
|
+
repo = _load_repo_or_exit()
|
|
412
|
+
uid = _resolve_uid(repo, user)
|
|
413
|
+
try:
|
|
414
|
+
result = UserDataFiles(repo).mkdir(uid, path, parents=parents)
|
|
415
|
+
except Exception as exc:
|
|
416
|
+
sdk_error_exit(exc, what=f"user storage {uid} {path}", stage="mkdir")
|
|
417
|
+
_emit_detail({"user": uid, **result}, output_format)
|
|
302
418
|
|
|
303
419
|
|
|
304
420
|
@fs_app.command("get")
|
|
@@ -309,6 +425,15 @@ def fs_get(
|
|
|
309
425
|
target: Optional[str] = typer.Option(
|
|
310
426
|
None, "--target", help="Local target path (default: the remote file name in CWD)"
|
|
311
427
|
),
|
|
428
|
+
overwrite: bool = typer.Option(
|
|
429
|
+
False, "--overwrite", "-f", help="Replace an existing destination"
|
|
430
|
+
),
|
|
431
|
+
skip_existing: bool = typer.Option(
|
|
432
|
+
False, "--skip-existing", "-n", help="Keep existing destinations"
|
|
433
|
+
),
|
|
434
|
+
on_exist: str = typer.Option(
|
|
435
|
+
"prompt", "--on-exist", help="Existing destination: prompt/overwrite/skip/error"
|
|
436
|
+
),
|
|
312
437
|
concurrency: int = typer.Option(
|
|
313
438
|
8, "--concurrency", "-j", min=1, max=64, help="Maximum concurrent transfer requests"
|
|
314
439
|
),
|
|
@@ -324,23 +449,41 @@ def fs_get(
|
|
|
324
449
|
from .irepo_sdk import sdk_error_exit
|
|
325
450
|
|
|
326
451
|
output_format = _fmt(output_format)
|
|
452
|
+
policy = _existing_policy(on_exist, overwrite, skip_existing)
|
|
327
453
|
repo = _load_repo_or_exit()
|
|
328
454
|
uid = _resolve_uid(repo, user)
|
|
329
455
|
remote = _remote_subpath(filepath, directory)
|
|
330
456
|
local = Path(target) if target else Path(Path(remote).name)
|
|
331
|
-
if local.is_dir():
|
|
457
|
+
if local.is_dir() or (target and target.endswith("/")):
|
|
332
458
|
local = local / Path(remote).name
|
|
459
|
+
skipped = False
|
|
333
460
|
try:
|
|
461
|
+
if local.is_symlink():
|
|
462
|
+
raise ValueError("Download target must not be a symbolic link")
|
|
463
|
+
exists = local.exists()
|
|
464
|
+
if exists and not local.is_file():
|
|
465
|
+
raise ValueError("Download target must be a regular file")
|
|
466
|
+
skipped = exists and not _replace_existing(str(local), policy)
|
|
334
467
|
if local.parent and str(local.parent) not in ("", "."):
|
|
335
468
|
local.parent.mkdir(parents=True, exist_ok=True)
|
|
336
|
-
written =
|
|
337
|
-
|
|
469
|
+
written = (
|
|
470
|
+
str(local)
|
|
471
|
+
if skipped
|
|
472
|
+
else UserDataTransfer(repo, concurrency=concurrency, progress=progress).get(
|
|
473
|
+
uid, remote, local, overwrite=exists or policy == "overwrite"
|
|
474
|
+
)
|
|
338
475
|
)
|
|
339
476
|
except Exception as exc: # noqa: BLE001
|
|
340
477
|
sdk_error_exit(exc, what=f"user storage {uid} {remote}", stage="download")
|
|
341
478
|
|
|
342
479
|
_emit_detail(
|
|
343
|
-
{
|
|
480
|
+
{
|
|
481
|
+
"user": uid,
|
|
482
|
+
"remote": remote,
|
|
483
|
+
"local": str(written or local),
|
|
484
|
+
"downloaded": 0 if skipped else 1,
|
|
485
|
+
"skipped": int(skipped),
|
|
486
|
+
},
|
|
344
487
|
output_format,
|
|
345
488
|
)
|
|
346
489
|
|
|
@@ -353,7 +496,15 @@ def fs_put(
|
|
|
353
496
|
source: Optional[str] = typer.Option(
|
|
354
497
|
None, "--source", help="Local source path (default: <filepath> relative to CWD)"
|
|
355
498
|
),
|
|
356
|
-
overwrite: bool = typer.Option(
|
|
499
|
+
overwrite: bool = typer.Option(
|
|
500
|
+
False, "--overwrite", "-f", help="Allow overwriting remote objects"
|
|
501
|
+
),
|
|
502
|
+
skip_existing: bool = typer.Option(
|
|
503
|
+
False, "--skip-existing", "-n", help="Keep existing remote objects"
|
|
504
|
+
),
|
|
505
|
+
on_exist: str = typer.Option(
|
|
506
|
+
"prompt", "--on-exist", help="Existing destination: prompt/overwrite/skip/error"
|
|
507
|
+
),
|
|
357
508
|
concurrency: int = typer.Option(
|
|
358
509
|
8, "--concurrency", "-j", min=1, max=64, help="Maximum concurrent transfer requests"
|
|
359
510
|
),
|
|
@@ -369,6 +520,7 @@ def fs_put(
|
|
|
369
520
|
from .irepo_sdk import sdk_error_exit
|
|
370
521
|
|
|
371
522
|
output_format = _fmt(output_format)
|
|
523
|
+
policy = _existing_policy(on_exist, overwrite, skip_existing)
|
|
372
524
|
local = Path(source or filepath)
|
|
373
525
|
if not local.exists():
|
|
374
526
|
raise _fail(
|
|
@@ -381,21 +533,30 @@ def fs_put(
|
|
|
381
533
|
uid = _resolve_uid(repo, user)
|
|
382
534
|
remote = _remote_subpath(filepath, directory)
|
|
383
535
|
is_dir = local.is_dir()
|
|
536
|
+
if filepath.endswith("/") and not is_dir:
|
|
537
|
+
remote = remote.rstrip("/") + "/" + local.name
|
|
384
538
|
try:
|
|
385
539
|
outcome: Any = UserDataTransfer(repo, concurrency=concurrency, progress=progress).put(
|
|
386
|
-
uid,
|
|
540
|
+
uid,
|
|
541
|
+
local,
|
|
542
|
+
remote,
|
|
543
|
+
overwrite=policy == "overwrite",
|
|
544
|
+
on_conflict=lambda key: _replace_existing(key, policy),
|
|
387
545
|
)
|
|
388
546
|
except Exception as exc: # noqa: BLE001
|
|
389
547
|
sdk_error_exit(exc, what=f"user storage {uid} {remote}", stage="upload")
|
|
390
548
|
|
|
391
|
-
|
|
549
|
+
results = outcome if isinstance(outcome, list) else [outcome]
|
|
550
|
+
skipped = sum(isinstance(item, dict) and item.get("skipped", False) for item in results)
|
|
392
551
|
_emit_detail(
|
|
393
552
|
{
|
|
394
553
|
"user": uid,
|
|
395
554
|
"remote": remote,
|
|
396
555
|
"local": str(local),
|
|
397
556
|
"kind": "directory" if is_dir else "file",
|
|
398
|
-
"objects":
|
|
557
|
+
"objects": len(results),
|
|
558
|
+
"uploaded": len(results) - skipped,
|
|
559
|
+
"skipped": skipped,
|
|
399
560
|
},
|
|
400
561
|
output_format,
|
|
401
562
|
)
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Directory operations using the SDK's scoped OSS authorization and routing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .fs_transfer import UserDataTransfer, _scope_denied
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class UserDataFiles(UserDataTransfer):
|
|
9
|
+
def list(self, uid: str, path: str = "", *, recursive: bool = False):
|
|
10
|
+
from instance_repo._routing import _OperationIntent
|
|
11
|
+
|
|
12
|
+
root = self.repo._user_data_prefix(uid)
|
|
13
|
+
prefix = root if path in ("", ".", "/") else self._key(uid, path) + "/"
|
|
14
|
+
|
|
15
|
+
def listing(bucket):
|
|
16
|
+
entries = {}
|
|
17
|
+
found = False
|
|
18
|
+
marker = ""
|
|
19
|
+
while True:
|
|
20
|
+
page = bucket.list_objects(
|
|
21
|
+
prefix=prefix, delimiter="" if recursive else "/", marker=marker, max_keys=1000
|
|
22
|
+
)
|
|
23
|
+
found = found or bool(page.object_list or page.prefix_list)
|
|
24
|
+
for obj in page.object_list:
|
|
25
|
+
if obj.key == prefix:
|
|
26
|
+
continue
|
|
27
|
+
kind = "directory" if obj.key.endswith("/") else "file"
|
|
28
|
+
entries[obj.key] = dict(
|
|
29
|
+
path=obj.key[len(root) :],
|
|
30
|
+
name=obj.key[len(prefix) :],
|
|
31
|
+
key=obj.key,
|
|
32
|
+
type=kind,
|
|
33
|
+
size=obj.size,
|
|
34
|
+
modified=obj.last_modified,
|
|
35
|
+
)
|
|
36
|
+
if recursive:
|
|
37
|
+
# OSS may have no marker object for a directory. Rebuild
|
|
38
|
+
# its ancestors so -R --dirs-only includes these too.
|
|
39
|
+
parts = obj.key[len(prefix) :].split("/")
|
|
40
|
+
for depth in range(1, len(parts)):
|
|
41
|
+
name = "/".join(parts[:depth]) + "/"
|
|
42
|
+
key = prefix + name
|
|
43
|
+
entries.setdefault(
|
|
44
|
+
key,
|
|
45
|
+
dict(
|
|
46
|
+
path=key[len(root) :],
|
|
47
|
+
name=name,
|
|
48
|
+
key=key,
|
|
49
|
+
type="directory",
|
|
50
|
+
size=None,
|
|
51
|
+
modified=None,
|
|
52
|
+
),
|
|
53
|
+
)
|
|
54
|
+
for common in page.prefix_list:
|
|
55
|
+
entries[common] = dict(
|
|
56
|
+
path=common[len(root) :],
|
|
57
|
+
name=common[len(prefix) :],
|
|
58
|
+
key=common,
|
|
59
|
+
type="directory",
|
|
60
|
+
size=None,
|
|
61
|
+
modified=None,
|
|
62
|
+
)
|
|
63
|
+
if not page.is_truncated:
|
|
64
|
+
break
|
|
65
|
+
if not page.next_marker or page.next_marker == marker:
|
|
66
|
+
raise ValueError("OSS listing did not advance its pagination marker")
|
|
67
|
+
marker = page.next_marker
|
|
68
|
+
return list(entries.values()), found
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
rows, found = self.content._execute_oss(
|
|
72
|
+
_OperationIntent.NON_DOWNLOAD, prefix, None, listing
|
|
73
|
+
)
|
|
74
|
+
except Exception as exc:
|
|
75
|
+
if prefix == root or not _scope_denied(exc):
|
|
76
|
+
raise
|
|
77
|
+
# A file-only grant need not permit listing the same-name directory.
|
|
78
|
+
rows, found = [], False
|
|
79
|
+
if found or prefix == root:
|
|
80
|
+
return rows
|
|
81
|
+
key = prefix.rstrip("/")
|
|
82
|
+
|
|
83
|
+
def file_entry(bucket):
|
|
84
|
+
if not bucket.object_exists(key):
|
|
85
|
+
raise FileNotFoundError(f"No such file or directory: {path}")
|
|
86
|
+
meta = bucket.head_object(key)
|
|
87
|
+
return [
|
|
88
|
+
dict(
|
|
89
|
+
path=key[len(root) :],
|
|
90
|
+
name=key.rsplit("/", 1)[-1],
|
|
91
|
+
key=key,
|
|
92
|
+
type="file",
|
|
93
|
+
size=meta.content_length,
|
|
94
|
+
modified=meta.last_modified,
|
|
95
|
+
explicit=True,
|
|
96
|
+
)
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
if path.endswith("/"):
|
|
100
|
+
raise FileNotFoundError(f"No such directory: {path}")
|
|
101
|
+
return self.content._execute_oss(_OperationIntent.NON_DOWNLOAD, key, None, file_entry)
|
|
102
|
+
|
|
103
|
+
def mkdir(self, uid: str, path: str, *, parents: bool = False):
|
|
104
|
+
from instance_repo._routing import _OperationIntent
|
|
105
|
+
|
|
106
|
+
clean = self.repo._clean_user_data_subpath(path)
|
|
107
|
+
segments = clean.split("/")
|
|
108
|
+
paths = ["/".join(segments[:i]) for i in range(1, len(segments) + 1)]
|
|
109
|
+
|
|
110
|
+
def exists(key):
|
|
111
|
+
return self.content._execute_oss(
|
|
112
|
+
_OperationIntent.NON_DOWNLOAD, key, None, lambda bucket: bucket.object_exists(key)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def directory_exists(key):
|
|
116
|
+
return self.content._execute_oss(
|
|
117
|
+
_OperationIntent.NON_DOWNLOAD,
|
|
118
|
+
key,
|
|
119
|
+
None,
|
|
120
|
+
lambda bucket: bool(bucket.list_objects(prefix=key, max_keys=1).object_list),
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
# Check all ancestors before creating any marker.
|
|
124
|
+
present = []
|
|
125
|
+
for part in paths:
|
|
126
|
+
key = self._key(uid, part)
|
|
127
|
+
if exists(key):
|
|
128
|
+
raise FileExistsError(f"A file already exists at {part}")
|
|
129
|
+
present.append(directory_exists(key + "/"))
|
|
130
|
+
if present[-1] and not parents:
|
|
131
|
+
raise FileExistsError(f"Directory already exists: {clean}; use --parents to keep it")
|
|
132
|
+
if not parents and not all(present[:-1]):
|
|
133
|
+
raise FileNotFoundError("Parent directory does not exist; use --parents/-p")
|
|
134
|
+
created = []
|
|
135
|
+
for part, found in zip(paths, present):
|
|
136
|
+
if found:
|
|
137
|
+
continue
|
|
138
|
+
key = self._key(uid, part) + "/"
|
|
139
|
+
|
|
140
|
+
def create(bucket, key=key):
|
|
141
|
+
if not bucket.object_exists(key):
|
|
142
|
+
bucket.put_object(key, b"", headers={"x-oss-forbid-overwrite": "true"})
|
|
143
|
+
|
|
144
|
+
self.content._execute_oss(_OperationIntent.NON_DOWNLOAD, key, None, create)
|
|
145
|
+
created.append(part)
|
|
146
|
+
return {"path": clean, "created": created}
|
|
@@ -6,6 +6,7 @@ import os
|
|
|
6
6
|
import tempfile
|
|
7
7
|
import threading
|
|
8
8
|
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
|
9
|
+
from dataclasses import replace
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
|
|
11
12
|
from rich.console import Console
|
|
@@ -24,17 +25,52 @@ from .ossutil_io import OssOperationError, _WriteBucket
|
|
|
24
25
|
PART_SIZE = 8 * 1024 * 1024
|
|
25
26
|
|
|
26
27
|
|
|
28
|
+
def _visual_task(task):
|
|
29
|
+
# Rich treats total=0 as 0%. Count processed files for zero-byte work,
|
|
30
|
+
# while retaining actual bytes for DownloadColumn and speed calculation.
|
|
31
|
+
if task.total == 0:
|
|
32
|
+
count = task.fields["files_total"]
|
|
33
|
+
return replace(task, total=count or 1, completed=task.fields["files_done"] if count else 1)
|
|
34
|
+
return task
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class _TransferBar(BarColumn):
|
|
38
|
+
def render(self, task):
|
|
39
|
+
return super().render(_visual_task(task))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _TransferPercent(TaskProgressColumn):
|
|
43
|
+
def render(self, task):
|
|
44
|
+
return super().render(_visual_task(task))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _scope_denied(exc: Exception) -> bool:
|
|
48
|
+
"""Only explicit scope refusals allow a more specific authorization request."""
|
|
49
|
+
seen = set()
|
|
50
|
+
cause = exc
|
|
51
|
+
while cause is not None and id(cause) not in seen:
|
|
52
|
+
seen.add(id(cause))
|
|
53
|
+
if getattr(cause, "biz_code", None) == 94023: # user-data has no grant
|
|
54
|
+
return True
|
|
55
|
+
if getattr(cause, "status", None) == 403 and getattr(cause, "code", None) == "AccessDenied":
|
|
56
|
+
return True
|
|
57
|
+
cause = cause.__cause__
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
|
|
27
61
|
class TransferProgress:
|
|
28
62
|
def __init__(self, description: str, enabled: bool | None):
|
|
29
63
|
console = Console(stderr=True, markup=False, highlight=False)
|
|
30
64
|
self.display = Progress(
|
|
31
65
|
TextColumn("{task.description}"),
|
|
32
|
-
|
|
33
|
-
|
|
66
|
+
_TransferBar(),
|
|
67
|
+
_TransferPercent(),
|
|
34
68
|
DownloadColumn(),
|
|
35
69
|
TransferSpeedColumn(),
|
|
36
70
|
TimeRemainingColumn(),
|
|
37
|
-
TextColumn(
|
|
71
|
+
TextColumn(
|
|
72
|
+
"files {task.fields[files_done]}/{task.fields[files_total]} skipped {task.fields[skipped]}"
|
|
73
|
+
),
|
|
38
74
|
console=console,
|
|
39
75
|
disable=not (console.is_terminal if enabled is None else enabled),
|
|
40
76
|
)
|
|
@@ -43,6 +79,7 @@ class TransferProgress:
|
|
|
43
79
|
self.sizes: dict[str, int] = {}
|
|
44
80
|
self.done: dict[str, int] = {}
|
|
45
81
|
self.files_done = 0
|
|
82
|
+
self.files_skipped = 0
|
|
46
83
|
self.total_bytes = 0
|
|
47
84
|
self.completed_bytes = 0
|
|
48
85
|
|
|
@@ -55,6 +92,7 @@ class TransferProgress:
|
|
|
55
92
|
total=None if unknown else self.total_bytes,
|
|
56
93
|
files_done=0,
|
|
57
94
|
files_total=len(sizes),
|
|
95
|
+
skipped=0,
|
|
58
96
|
)
|
|
59
97
|
return self
|
|
60
98
|
|
|
@@ -86,6 +124,7 @@ class TransferProgress:
|
|
|
86
124
|
with self.lock:
|
|
87
125
|
self.known_total = True
|
|
88
126
|
if skipped:
|
|
127
|
+
self.files_skipped += 1
|
|
89
128
|
# Skipped bytes were not transferred; do not inflate the speed.
|
|
90
129
|
self.total_bytes -= self.sizes[key]
|
|
91
130
|
self.sizes[key] = 0
|
|
@@ -101,6 +140,7 @@ class TransferProgress:
|
|
|
101
140
|
completed=self.completed_bytes,
|
|
102
141
|
total=self.total_bytes if self.known_total else None,
|
|
103
142
|
files_done=self.files_done,
|
|
143
|
+
skipped=self.files_skipped,
|
|
104
144
|
)
|
|
105
145
|
|
|
106
146
|
|
|
@@ -121,16 +161,23 @@ class UserDataTransfer:
|
|
|
121
161
|
raise ValueError("uid must not be empty")
|
|
122
162
|
return self.repo._user_data_prefix(uid) + self.repo._clean_user_data_subpath(path)
|
|
123
163
|
|
|
124
|
-
def put(
|
|
164
|
+
def put(
|
|
165
|
+
self, uid: str, source: Path, remote: str, *, overwrite: bool = False, on_conflict=None
|
|
166
|
+
):
|
|
125
167
|
from instance_repo._routing import _OperationIntent
|
|
126
168
|
|
|
127
169
|
source = Path(source)
|
|
170
|
+
remote = self.repo._clean_user_data_subpath(remote)
|
|
128
171
|
base = self._key(uid, remote)
|
|
129
172
|
directory = source.is_dir()
|
|
173
|
+
if source.is_symlink():
|
|
174
|
+
raise ValueError("Symbolic link upload sources are not supported")
|
|
130
175
|
files = []
|
|
131
176
|
if directory:
|
|
132
177
|
for root, dirs, names in os.walk(source):
|
|
133
178
|
dirs.sort()
|
|
179
|
+
if any((Path(root) / name).is_symlink() for name in dirs + names):
|
|
180
|
+
raise ValueError("Symbolic links inside an upload directory are not supported")
|
|
134
181
|
for name in sorted(names):
|
|
135
182
|
local = Path(root) / name
|
|
136
183
|
key = self._key(
|
|
@@ -145,39 +192,72 @@ class UserDataTransfer:
|
|
|
145
192
|
workers = min(self.concurrency, max(1, len(files)))
|
|
146
193
|
part_threads = max(1, self.concurrency // workers)
|
|
147
194
|
report = TransferProgress("Uploading", self.progress).start(sizes)
|
|
195
|
+
directory_scope = base.rstrip("/") + "/" if directory else None
|
|
196
|
+
conflict_lock = threading.Lock()
|
|
148
197
|
|
|
149
198
|
def transfer(item):
|
|
150
199
|
local, key = item
|
|
151
200
|
write_started = False
|
|
201
|
+
allowed_overwrite = overwrite
|
|
152
202
|
# Keep the SDK's resume records across failures and later invocations.
|
|
153
203
|
store = self.oss.ResumableStore()
|
|
154
204
|
|
|
155
205
|
def upload(bucket):
|
|
156
|
-
nonlocal write_started
|
|
206
|
+
nonlocal write_started, allowed_overwrite
|
|
157
207
|
if not overwrite and not write_started and bucket.object_exists(key):
|
|
158
|
-
|
|
208
|
+
with conflict_lock:
|
|
209
|
+
# Serialize prompts even when multiple workers find collisions.
|
|
210
|
+
allowed_overwrite = bool(on_conflict(key)) if on_conflict else False
|
|
211
|
+
if not allowed_overwrite:
|
|
212
|
+
return {"key": key, "skipped": True}
|
|
159
213
|
write_started = True
|
|
160
214
|
self.oss.resumable_upload(
|
|
161
|
-
_WriteBucket(bucket,
|
|
215
|
+
_WriteBucket(bucket, allowed_overwrite),
|
|
162
216
|
key,
|
|
163
217
|
str(local),
|
|
164
218
|
store=store,
|
|
165
219
|
num_threads=part_threads,
|
|
166
220
|
part_size=PART_SIZE,
|
|
167
221
|
multipart_threshold=PART_SIZE,
|
|
168
|
-
headers={} if
|
|
222
|
+
headers={} if allowed_overwrite else {"x-oss-forbid-overwrite": "true"},
|
|
169
223
|
progress_callback=report.callback(key),
|
|
170
224
|
)
|
|
171
225
|
return {"key": key, "skipped": False}
|
|
172
226
|
|
|
173
|
-
|
|
174
|
-
|
|
227
|
+
scope = directory_scope or key
|
|
228
|
+
try:
|
|
229
|
+
result = self.content._execute_oss(
|
|
230
|
+
_OperationIntent.NON_DOWNLOAD, scope, None, upload
|
|
231
|
+
)
|
|
232
|
+
except Exception as exc:
|
|
233
|
+
if scope == key or not _scope_denied(exc):
|
|
234
|
+
raise
|
|
235
|
+
# A directory token may deny a reader subtree that contains a
|
|
236
|
+
# more specific writer grant. Re-authorize only this object;
|
|
237
|
+
# never ignore a Deny or request a broader ancestor token.
|
|
238
|
+
result = self.content._execute_oss(_OperationIntent.NON_DOWNLOAD, key, None, upload)
|
|
175
239
|
report.finish(key, skipped=result["skipped"])
|
|
176
240
|
return result
|
|
177
241
|
|
|
178
242
|
with report:
|
|
179
243
|
if not files:
|
|
180
244
|
return []
|
|
245
|
+
if directory_scope is not None:
|
|
246
|
+
try:
|
|
247
|
+
# Prime the SDK's thread-safe, expiry-aware credential cache.
|
|
248
|
+
# The server includes descendant reader Deny rules in this
|
|
249
|
+
# directory policy. Never reconstruct that policy locally.
|
|
250
|
+
credentials = self.content._credentials(directory_scope, None)
|
|
251
|
+
except Exception as exc:
|
|
252
|
+
if not _scope_denied(exc):
|
|
253
|
+
raise
|
|
254
|
+
# File-only grants remain usable without a parent grant.
|
|
255
|
+
directory_scope = None
|
|
256
|
+
else:
|
|
257
|
+
if credentials.get("prefix") != directory_scope:
|
|
258
|
+
raise OssOperationError(
|
|
259
|
+
"Directory credential scope differs from the requested directory"
|
|
260
|
+
)
|
|
181
261
|
if workers == 1:
|
|
182
262
|
outcomes = [transfer(item) for item in files]
|
|
183
263
|
else:
|
|
@@ -209,12 +289,12 @@ class UserDataTransfer:
|
|
|
209
289
|
raise
|
|
210
290
|
return outcomes
|
|
211
291
|
|
|
212
|
-
def get(self, uid: str, remote: str, target: Path) -> str:
|
|
292
|
+
def get(self, uid: str, remote: str, target: Path, *, overwrite: bool = True) -> str:
|
|
213
293
|
from instance_repo._routing import _OperationIntent
|
|
214
294
|
|
|
215
295
|
key = self._key(uid, remote)
|
|
216
296
|
target = Path(target)
|
|
217
|
-
if target.exists() and not target.is_file():
|
|
297
|
+
if target.is_symlink() or (target.exists() and not target.is_file()):
|
|
218
298
|
raise ValueError("Download target must be a regular file")
|
|
219
299
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
220
300
|
report = TransferProgress("Downloading", self.progress).start({key: 0}, unknown=True)
|
|
@@ -240,6 +320,10 @@ class UserDataTransfer:
|
|
|
240
320
|
|
|
241
321
|
self.content._execute_oss(_OperationIntent.DOWNLOAD, key, None, download)
|
|
242
322
|
partial.chmod(0o600)
|
|
243
|
-
|
|
323
|
+
if overwrite:
|
|
324
|
+
os.replace(partial, target)
|
|
325
|
+
else:
|
|
326
|
+
# Atomically refuse a destination created since the CLI preflight.
|
|
327
|
+
os.link(partial, target)
|
|
244
328
|
report.finish(key)
|
|
245
329
|
return str(target)
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ap-client"
|
|
7
|
-
version = "0.6.1.
|
|
7
|
+
version = "0.6.1.dev13"
|
|
8
8
|
description = "Agent Platform API Client & CLI"
|
|
9
9
|
readme = { text = "A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.", content-type = "text/markdown" }
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|