accli 2.0.0b4__tar.gz → 2.0.0rc4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {accli-2.0.0b4 → accli-2.0.0rc4}/PKG-INFO +4 -4
- {accli-2.0.0b4 → accli-2.0.0rc4}/README.md +2 -2
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/AcceleratorTaskDispatcher.py +2 -2
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/CsvRegionalTimeseriesValidator.py +2 -2
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/_version.py +1 -1
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/cleanup-nfs.ps1 +5 -3
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/cli.py +342 -123
- accli-2.0.0rc4/accli/gui.py +630 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/mount_downloader.py +3 -3
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/setup-nfs.ps1 +72 -19
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/token.py +15 -1
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli.egg-info/PKG-INFO +4 -4
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli.egg-info/SOURCES.txt +1 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli.egg-info/entry_points.txt +1 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli.egg-info/requires.txt +1 -1
- {accli-2.0.0b4 → accli-2.0.0rc4}/pyproject.toml +2 -1
- {accli-2.0.0b4 → accli-2.0.0rc4}/LICENSE +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/AcceleratorJobProjectService.py +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/AcceleratorTerminalCliProjectService.py +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/__init__.py +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/__main__.py +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli/common.py +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli.egg-info/dependency_links.txt +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/accli.egg-info/top_level.txt +0 -0
- {accli-2.0.0b4 → accli-2.0.0rc4}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: accli
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.0rc4
|
|
4
4
|
Summary: IIASA Accelerator Client
|
|
5
5
|
Author-email: Wrufesh S <wrufesh@gmail.com>
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -39,7 +39,7 @@ Requires-Dist: requests>=2.25.1
|
|
|
39
39
|
Requires-Dist: jsonschema>=4.5.1
|
|
40
40
|
Requires-Dist: pydantic>=2.0
|
|
41
41
|
Requires-Dist: GitPython
|
|
42
|
-
Requires-Dist: hf-xet
|
|
42
|
+
Requires-Dist: hf-xet==1.5.2
|
|
43
43
|
Dynamic: license-file
|
|
44
44
|
|
|
45
45
|
# Accelerator Terminal Client and Python API
|
|
@@ -138,8 +138,8 @@ Usage: accli [OPTIONS] COMMAND [ARGS]...
|
|
|
138
138
|
|
|
139
139
|
### Handling self-signed certificates
|
|
140
140
|
|
|
141
|
-
1. Copy and paste certificates from `<control_services_backend>/certs/
|
|
142
|
-
`./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/
|
|
141
|
+
1. Copy and paste certificates from `<control_services_backend>/certs/ca.crt` as
|
|
142
|
+
`./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/ca.crt` as
|
|
143
143
|
`./certs/dev-minio-ca.crt`.
|
|
144
144
|
2. Run
|
|
145
145
|
```
|
|
@@ -94,8 +94,8 @@ Usage: accli [OPTIONS] COMMAND [ARGS]...
|
|
|
94
94
|
|
|
95
95
|
### Handling self-signed certificates
|
|
96
96
|
|
|
97
|
-
1. Copy and paste certificates from `<control_services_backend>/certs/
|
|
98
|
-
`./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/
|
|
97
|
+
1. Copy and paste certificates from `<control_services_backend>/certs/ca.crt` as
|
|
98
|
+
`./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/ca.crt` as
|
|
99
99
|
`./certs/dev-minio-ca.crt`.
|
|
100
100
|
2. Run
|
|
101
101
|
```
|
|
@@ -139,10 +139,10 @@ class JobDispatchModel(BaseModel):
|
|
|
139
139
|
class WKubeTaskMeta(BaseModel):
|
|
140
140
|
required_cores: float
|
|
141
141
|
required_ram: float
|
|
142
|
-
required_storage_local: float
|
|
142
|
+
required_storage_local: Optional[float] = None
|
|
143
143
|
|
|
144
144
|
# is ignored if it is a callback and child of non-free node jobs
|
|
145
|
-
required_storage_workflow: float
|
|
145
|
+
required_storage_workflow: Optional[float] = None
|
|
146
146
|
|
|
147
147
|
job_secrets: Optional[dict] = {}
|
|
148
148
|
|
|
@@ -205,8 +205,8 @@ class CsvRegionalTimeseriesValidator:
|
|
|
205
205
|
for key in self.errors:
|
|
206
206
|
print(f"[bold red]Invalid data:[/bold red] {self.errors[key]}")
|
|
207
207
|
print(f"[bold red]Error:[/bold red] {key}")
|
|
208
|
-
print("[bold red]Data is not valid against selected template.
|
|
208
|
+
print("[bold red]Data is not valid against selected template.[/bold red]")
|
|
209
209
|
typer.Exit(1)
|
|
210
210
|
else:
|
|
211
|
-
print("[bold green]Data validated against selected template.[/bold green]
|
|
211
|
+
print("[bold green]Data validated against selected template.[/bold green]")
|
|
212
212
|
typer.Exit(0)
|
|
@@ -22,14 +22,16 @@ if (-not $isAdmin) {
|
|
|
22
22
|
|
|
23
23
|
Write-Host "=== Reverting accli Windows NFS Client Configuration ===" -ForegroundColor Cyan
|
|
24
24
|
|
|
25
|
-
# 1. Unregister Scheduled
|
|
26
|
-
Write-Host "Removing
|
|
25
|
+
# 1. Unregister Scheduled Tasks
|
|
26
|
+
Write-Host "Removing Scheduled Tasks..." -ForegroundColor Yellow
|
|
27
27
|
if (Get-Command Unregister-ScheduledTask -ErrorAction SilentlyContinue) {
|
|
28
28
|
Unregister-ScheduledTask -TaskName "accli-mount-nfs" -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
|
|
29
|
+
Unregister-ScheduledTask -TaskName "accli-umount-nfs" -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
|
|
29
30
|
} else {
|
|
30
31
|
schtasks /delete /tn "accli-mount-nfs" /f 2> $null | Out-Null
|
|
32
|
+
schtasks /delete /tn "accli-umount-nfs" /f 2> $null | Out-Null
|
|
31
33
|
}
|
|
32
|
-
Write-Host "[OK] Scheduled
|
|
34
|
+
Write-Host "[OK] Scheduled Tasks removed." -ForegroundColor Green
|
|
33
35
|
|
|
34
36
|
# 2. Delete Registry Policy Keys
|
|
35
37
|
Write-Host "Removing registry policies (EnableLinkedConnections, AnonymousUid, AnonymousGid)..." -ForegroundColor Yellow
|
|
@@ -141,7 +141,7 @@ def dispatch(
|
|
|
141
141
|
verify_cert=(not ACCLI_DEBUG)
|
|
142
142
|
)
|
|
143
143
|
|
|
144
|
-
#
|
|
144
|
+
# Resolve the workflow file path properly
|
|
145
145
|
if os.path.isabs(workflow_filename):
|
|
146
146
|
workflow_filepath = workflow_filename
|
|
147
147
|
else:
|
|
@@ -152,9 +152,9 @@ def dispatch(
|
|
|
152
152
|
|
|
153
153
|
workflow_dir = os.path.dirname(workflow_filepath)
|
|
154
154
|
|
|
155
|
-
#
|
|
155
|
+
# Temporarily switch to the workflow file's directory
|
|
156
156
|
with pushd(workflow_dir):
|
|
157
|
-
#
|
|
157
|
+
# Import the module dynamically
|
|
158
158
|
spec = importlib.util.spec_from_file_location("workflow", workflow_filepath)
|
|
159
159
|
module = importlib.util.module_from_spec(spec)
|
|
160
160
|
spec.loader.exec_module(module)
|
|
@@ -300,12 +300,6 @@ def copy(
|
|
|
300
300
|
verify_cert=(not ACCLI_DEBUG),
|
|
301
301
|
)
|
|
302
302
|
|
|
303
|
-
def make_token_refresher(slug: str):
|
|
304
|
-
def refresher() -> tuple[str, int]:
|
|
305
|
-
token_str, _, exp_time = exchange_refresh_token(slug)
|
|
306
|
-
return token_str, exp_time
|
|
307
|
-
return refresher
|
|
308
|
-
|
|
309
303
|
if is_src_remote:
|
|
310
304
|
# Download flow: acc://[project_slug]/[remote_prefix] -> local destination
|
|
311
305
|
source_parsed = source[len("acc://"):]
|
|
@@ -336,7 +330,7 @@ def copy(
|
|
|
336
330
|
else:
|
|
337
331
|
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
338
332
|
|
|
339
|
-
|
|
333
|
+
files_to_download = []
|
|
340
334
|
for filename in filenames:
|
|
341
335
|
# Filename returned usually starts with project_slug/
|
|
342
336
|
if filename.startswith(f"{project_slug}/"):
|
|
@@ -378,32 +372,98 @@ def copy(
|
|
|
378
372
|
|
|
379
373
|
local_dest.parent.mkdir(parents=True, exist_ok=True)
|
|
380
374
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
destination_path=str(local_dest.resolve()),
|
|
384
|
-
hash=merkle_hash,
|
|
385
|
-
file_size=file_size
|
|
386
|
-
)
|
|
375
|
+
files_to_download.append(
|
|
376
|
+
(str(local_dest.resolve()), merkle_hash, file_size)
|
|
387
377
|
)
|
|
388
378
|
|
|
389
|
-
if not
|
|
379
|
+
if not files_to_download:
|
|
390
380
|
print("[bold red]ERROR: No valid files identified for download.[/bold red]")
|
|
391
381
|
raise typer.Exit(1)
|
|
392
382
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
383
|
+
MAX_FILES_PER_BATCH = 100
|
|
384
|
+
MAX_BYTES_PER_BATCH = 10 * 1024 * 1024 * 1024 # 10 GB
|
|
385
|
+
|
|
386
|
+
batches = []
|
|
387
|
+
current_batch = []
|
|
388
|
+
current_batch_size = 0
|
|
389
|
+
|
|
390
|
+
for item in files_to_download:
|
|
391
|
+
size = item[2]
|
|
392
|
+
if size >= MAX_BYTES_PER_BATCH and not current_batch:
|
|
393
|
+
batches.append([item])
|
|
394
|
+
continue
|
|
395
|
+
|
|
396
|
+
if len(current_batch) >= MAX_FILES_PER_BATCH or current_batch_size + size > MAX_BYTES_PER_BATCH:
|
|
397
|
+
batches.append(current_batch)
|
|
398
|
+
current_batch = []
|
|
399
|
+
current_batch_size = 0
|
|
400
|
+
|
|
401
|
+
current_batch.append(item)
|
|
402
|
+
current_batch_size += size
|
|
403
|
+
|
|
404
|
+
if current_batch:
|
|
405
|
+
batches.append(current_batch)
|
|
406
|
+
|
|
407
|
+
print(f"[bold cyan]Downloading {len(files_to_download)} files in {len(batches)} batches using hf-xet...[/bold cyan]")
|
|
408
|
+
|
|
409
|
+
refresh_url = f"{server_url.rstrip('/')}/api/v1/oauth/device/cas-token/"
|
|
410
|
+
refresh_headers = {"Authorization": f"Bearer {cas_token}"}
|
|
411
|
+
|
|
412
|
+
with Progress(
|
|
413
|
+
TextColumn("[progress.description]{task.description}"),
|
|
414
|
+
BarColumn(),
|
|
415
|
+
TaskProgressColumn(),
|
|
416
|
+
TimeElapsedColumn(),
|
|
417
|
+
) as progress:
|
|
418
|
+
total_size = sum(item[2] for item in files_to_download)
|
|
419
|
+
overall_task = progress.add_task("[green]Total Download Progress...", total=total_size)
|
|
420
|
+
|
|
421
|
+
completed_in_previous_batches = 0
|
|
422
|
+
|
|
423
|
+
session = hf_xet.XetSession()
|
|
424
|
+
for batch_index, batch_items in enumerate(batches, 1):
|
|
425
|
+
progress.print(f"[cyan]Processing batch {batch_index}/{len(batches)}...[/cyan]")
|
|
426
|
+
|
|
427
|
+
# capture current completed_in_previous_batches in a local scope to avoid late binding issues
|
|
428
|
+
file_tasks = {}
|
|
429
|
+
|
|
430
|
+
def make_progress_handler(base_completed):
|
|
431
|
+
def on_progress(group_report, item_reports):
|
|
432
|
+
progress.update(overall_task, completed=base_completed + group_report.total_bytes_completed)
|
|
433
|
+
for item_id, item in item_reports.items():
|
|
434
|
+
if item_id not in file_tasks:
|
|
435
|
+
file_tasks[item_id] = progress.add_task(f"[blue]{item.item_name}", total=item.total_bytes)
|
|
436
|
+
|
|
437
|
+
progress.update(file_tasks[item_id], completed=item.bytes_completed)
|
|
438
|
+
|
|
439
|
+
# Hide completed tasks to keep the UI clean when downloading thousands of files
|
|
440
|
+
if item.bytes_completed >= item.total_bytes:
|
|
441
|
+
progress.update(file_tasks[item_id], visible=False)
|
|
442
|
+
return on_progress
|
|
443
|
+
|
|
444
|
+
try:
|
|
445
|
+
with session.new_file_download_group(
|
|
446
|
+
endpoint=cas_endpoint,
|
|
447
|
+
token=cas_token,
|
|
448
|
+
token_expiry_unix_secs=expires_at,
|
|
449
|
+
token_refresh_url=refresh_url,
|
|
450
|
+
token_refresh_headers=refresh_headers,
|
|
451
|
+
progress_callback=make_progress_handler(completed_in_previous_batches),
|
|
452
|
+
progress_interval_ms=500,
|
|
453
|
+
) as group:
|
|
454
|
+
for dest_path, hash_val, size_val in batch_items:
|
|
455
|
+
info = hf_xet.XetFileInfo(hash_val, size_val)
|
|
456
|
+
group.start_download_file(info, dest_path)
|
|
457
|
+
|
|
458
|
+
batch_size = sum(item[2] for item in batch_items)
|
|
459
|
+
completed_in_previous_batches += batch_size
|
|
460
|
+
progress.update(overall_task, completed=completed_in_previous_batches)
|
|
461
|
+
progress.print(f"[bold green][OK] Batch {batch_index} completed successfully![/bold green]")
|
|
462
|
+
except Exception as e:
|
|
463
|
+
progress.print(f"[bold red]ERROR: Download failed for batch {batch_index}: {e}[/bold red]")
|
|
464
|
+
raise typer.Exit(1)
|
|
465
|
+
|
|
466
|
+
print("[bold green][OK] Download completed successfully![/bold green]")
|
|
407
467
|
|
|
408
468
|
else:
|
|
409
469
|
# Upload flow: local source -> acc://[project_slug]/[remote_path]
|
|
@@ -456,65 +516,137 @@ def copy(
|
|
|
456
516
|
print("[bold yellow]No files to upload.[/bold yellow]")
|
|
457
517
|
raise typer.Exit(0)
|
|
458
518
|
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
519
|
+
MAX_FILES_PER_BATCH = 100
|
|
520
|
+
MAX_BYTES_PER_BATCH = 10 * 1024 * 1024 * 1024 # 10 GB
|
|
521
|
+
|
|
522
|
+
batches = []
|
|
523
|
+
current_batch_paths = []
|
|
524
|
+
current_batch_remote = []
|
|
525
|
+
current_batch_size = 0
|
|
526
|
+
|
|
527
|
+
for local_p, remote_n in zip(local_paths, remote_filenames):
|
|
528
|
+
size = local_p.stat().st_size
|
|
529
|
+
if size >= MAX_BYTES_PER_BATCH and not current_batch_paths:
|
|
530
|
+
batches.append(([local_p], [remote_n]))
|
|
531
|
+
continue
|
|
532
|
+
|
|
533
|
+
if len(current_batch_paths) >= MAX_FILES_PER_BATCH or current_batch_size + size > MAX_BYTES_PER_BATCH:
|
|
534
|
+
batches.append((current_batch_paths, current_batch_remote))
|
|
535
|
+
current_batch_paths = []
|
|
536
|
+
current_batch_remote = []
|
|
537
|
+
current_batch_size = 0
|
|
538
|
+
|
|
539
|
+
current_batch_paths.append(local_p)
|
|
540
|
+
current_batch_remote.append(remote_n)
|
|
541
|
+
current_batch_size += size
|
|
542
|
+
|
|
543
|
+
if current_batch_paths:
|
|
544
|
+
batches.append((current_batch_paths, current_batch_remote))
|
|
475
545
|
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
546
|
+
print(f"[bold cyan]Uploading {len(local_paths)} files in {len(batches)} batches using hf-xet...[/bold cyan]")
|
|
547
|
+
|
|
548
|
+
refresh_url = f"{server_url.rstrip('/')}/api/v1/oauth/device/cas-token/"
|
|
549
|
+
refresh_headers = {"Authorization": f"Bearer {cas_token}"}
|
|
550
|
+
|
|
551
|
+
with Progress(
|
|
552
|
+
TextColumn("[progress.description]{task.description}"),
|
|
553
|
+
BarColumn(),
|
|
554
|
+
TaskProgressColumn(),
|
|
555
|
+
TimeElapsedColumn(),
|
|
556
|
+
) as progress:
|
|
557
|
+
total_size = sum(p.stat().st_size for p in local_paths)
|
|
558
|
+
overall_task = progress.add_task("[green]Total Upload Progress...", total=total_size)
|
|
559
|
+
|
|
560
|
+
completed_in_previous_batches = 0
|
|
561
|
+
|
|
562
|
+
session = hf_xet.XetSession()
|
|
563
|
+
for batch_index, (b_paths, b_remotes) in enumerate(batches, 1):
|
|
564
|
+
progress.print(f"[cyan]Processing batch {batch_index}/{len(batches)}...[/cyan]")
|
|
565
|
+
|
|
566
|
+
file_tasks = {}
|
|
567
|
+
|
|
568
|
+
def make_progress_handler(base_completed):
|
|
569
|
+
def on_progress(group_report, item_reports):
|
|
570
|
+
progress.update(overall_task, completed=base_completed + group_report.total_bytes_completed)
|
|
571
|
+
for item_id, item in item_reports.items():
|
|
572
|
+
if item_id not in file_tasks:
|
|
573
|
+
file_tasks[item_id] = progress.add_task(f"[blue]{item.item_name}", total=item.total_bytes)
|
|
574
|
+
|
|
575
|
+
progress.update(file_tasks[item_id], completed=item.bytes_completed)
|
|
576
|
+
|
|
577
|
+
# Hide completed tasks to keep the UI clean
|
|
578
|
+
if item.bytes_completed >= item.total_bytes:
|
|
579
|
+
progress.update(file_tasks[item_id], visible=False)
|
|
580
|
+
return on_progress
|
|
581
|
+
|
|
582
|
+
try:
|
|
583
|
+
with session.new_upload_commit(
|
|
584
|
+
endpoint=cas_endpoint,
|
|
585
|
+
token=cas_token,
|
|
586
|
+
token_expiry_unix_secs=expires_at,
|
|
587
|
+
token_refresh_url=refresh_url,
|
|
588
|
+
token_refresh_headers=refresh_headers,
|
|
589
|
+
progress_callback=make_progress_handler(completed_in_previous_batches),
|
|
590
|
+
progress_interval_ms=500,
|
|
591
|
+
) as commit:
|
|
592
|
+
handles = []
|
|
593
|
+
for p in b_paths:
|
|
594
|
+
sha = compute_sha256(str(p))
|
|
595
|
+
h = commit.start_upload_file(str(p), sha256=sha)
|
|
596
|
+
handles.append((p, sha, h))
|
|
597
|
+
|
|
598
|
+
upload_results = [(p, sha, h.result()) for p, sha, h in handles]
|
|
599
|
+
except Exception as e:
|
|
600
|
+
progress.print(f"[bold red]ERROR: Upload failed for batch {batch_index}: {e}[/bold red]")
|
|
601
|
+
raise typer.Exit(1)
|
|
494
602
|
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
603
|
+
progress.print("[cyan]Computing SHA-256 hashes and registering metadata for batch...[/cyan]")
|
|
604
|
+
registration_items = []
|
|
605
|
+
for local_path, remote_filename, (p, sha256_hash, upload_info) in zip(b_paths, b_remotes, upload_results):
|
|
606
|
+
registration_items.append({
|
|
607
|
+
"filename": f"{project_slug}/{remote_filename}",
|
|
608
|
+
"merkle_hash": upload_info.xet_info.hash,
|
|
609
|
+
"sha256": sha256_hash,
|
|
610
|
+
"file_size": upload_info.xet_info.file_size,
|
|
611
|
+
"content_type": "application/octet-stream"
|
|
612
|
+
})
|
|
613
|
+
|
|
614
|
+
headers = {
|
|
615
|
+
"Content-Type": "application/json",
|
|
616
|
+
"X-Project-Slug": project_slug,
|
|
617
|
+
"Authorization": f"Bearer {cas_token}"
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
try:
|
|
621
|
+
response = requests.post(
|
|
622
|
+
register_url,
|
|
623
|
+
json={"items": registration_items},
|
|
624
|
+
headers=headers,
|
|
625
|
+
verify=(not ACCLI_DEBUG)
|
|
626
|
+
)
|
|
627
|
+
response.raise_for_status()
|
|
628
|
+
|
|
629
|
+
batch_size = sum(p.stat().st_size for p in b_paths)
|
|
630
|
+
completed_in_previous_batches += batch_size
|
|
631
|
+
progress.update(overall_task, completed=completed_in_previous_batches)
|
|
632
|
+
|
|
633
|
+
progress.print(f"[bold green][OK] Batch {batch_index} registered successfully![/bold green]")
|
|
634
|
+
except requests.exceptions.HTTPError as e:
|
|
635
|
+
detail = None
|
|
636
|
+
try:
|
|
637
|
+
detail = e.response.json().get("detail")
|
|
638
|
+
except Exception:
|
|
639
|
+
pass
|
|
640
|
+
if detail:
|
|
641
|
+
progress.print(f"[bold red]ERROR: Bulk metadata registration failed for batch {batch_index}: {detail}[/bold red]")
|
|
642
|
+
else:
|
|
643
|
+
progress.print(f"[bold red]ERROR: Bulk metadata registration failed for batch {batch_index}: {e}[/bold red]")
|
|
644
|
+
raise typer.Exit(1)
|
|
645
|
+
except Exception as e:
|
|
646
|
+
progress.print(f"[bold red]ERROR: Bulk metadata registration failed for batch {batch_index}: {e}[/bold red]")
|
|
647
|
+
raise typer.Exit(1)
|
|
648
|
+
|
|
649
|
+
print("[bold green][OK] Upload completed successfully![/bold green]")
|
|
518
650
|
|
|
519
651
|
|
|
520
652
|
# --- Mount commands group ---
|
|
@@ -710,6 +842,7 @@ def mount_start(
|
|
|
710
842
|
from . import mount_downloader
|
|
711
843
|
import platform
|
|
712
844
|
import re
|
|
845
|
+
import json
|
|
713
846
|
|
|
714
847
|
sys_name = platform.system()
|
|
715
848
|
|
|
@@ -738,6 +871,27 @@ def mount_start(
|
|
|
738
871
|
|
|
739
872
|
# Standardize mount_point and handle defaults / auto-selection
|
|
740
873
|
if sys_name == "Windows":
|
|
874
|
+
# Check if the daemon is already running to prevent double mount
|
|
875
|
+
import subprocess
|
|
876
|
+
try:
|
|
877
|
+
task_check = subprocess.run(["tasklist", "/FI", "IMAGENAME eq hf-mount-nfs.exe"], capture_output=True, text=True)
|
|
878
|
+
if "hf-mount-nfs.exe" in task_check.stdout:
|
|
879
|
+
config_file = Path("C:/ProgramData/accli/mount_config.json")
|
|
880
|
+
active_msg = ""
|
|
881
|
+
if config_file.is_file():
|
|
882
|
+
try:
|
|
883
|
+
import json
|
|
884
|
+
cfg = json.loads(config_file.read_text(encoding="utf-8"))
|
|
885
|
+
active_msg = f" for project '{cfg.get('project_slug')}' at '{cfg.get('mount_point')}'"
|
|
886
|
+
except Exception:
|
|
887
|
+
pass
|
|
888
|
+
print(f"[bold red]ERROR: A mount daemon is already running{active_msg}.[/bold red]")
|
|
889
|
+
print("[yellow]Please run 'accli mount stop' to unmount before starting a new mount.[/yellow]")
|
|
890
|
+
raise typer.Exit(1)
|
|
891
|
+
except Exception as e:
|
|
892
|
+
if isinstance(e, typer.Exit):
|
|
893
|
+
raise e
|
|
894
|
+
|
|
741
895
|
if mount_point is None:
|
|
742
896
|
# Auto-detect available drive letter starting with W
|
|
743
897
|
try:
|
|
@@ -917,6 +1071,8 @@ def mount_start(
|
|
|
917
1071
|
print("[yellow]Falling back to UAC elevation...[/yellow]")
|
|
918
1072
|
has_task = False
|
|
919
1073
|
except Exception as e:
|
|
1074
|
+
if isinstance(e, typer.Exit) or type(e).__name__ == "Exit":
|
|
1075
|
+
raise e
|
|
920
1076
|
print(f"[bold red]ERROR: Task Scheduler trigger failed: {e}[/bold red]")
|
|
921
1077
|
print("[yellow]Falling back to UAC elevation...[/yellow]")
|
|
922
1078
|
has_task = False
|
|
@@ -1108,8 +1264,9 @@ def mount_stop(
|
|
|
1108
1264
|
print(f"[bold red]ERROR: You requested to stop '{mount_point_abs}', but the active daemon is mounted at '{active_mount}'.[/bold red]")
|
|
1109
1265
|
print(f"[yellow]To stop the server, please run: accli mount stop {active_mount}[/yellow]")
|
|
1110
1266
|
raise typer.Exit(1)
|
|
1111
|
-
except Exception:
|
|
1112
|
-
|
|
1267
|
+
except Exception as e:
|
|
1268
|
+
if isinstance(e, typer.Exit) or type(e).__name__ == "Exit":
|
|
1269
|
+
raise e
|
|
1113
1270
|
else:
|
|
1114
1271
|
if mount_point is None:
|
|
1115
1272
|
try:
|
|
@@ -1128,28 +1285,71 @@ def mount_stop(
|
|
|
1128
1285
|
print(f"[bold cyan]Stopping mount at '[white]{mount_point_abs}[/white]' on Windows...[/bold cyan]")
|
|
1129
1286
|
import subprocess
|
|
1130
1287
|
|
|
1131
|
-
#
|
|
1132
|
-
res = subprocess.run(["C:\\Windows\\System32\\umount.exe", "-f", str(mount_point_abs)], capture_output=True, text=True)
|
|
1133
|
-
|
|
1134
|
-
# 2. Stop the scheduled task (which terminates the elevated daemon background process tree)
|
|
1135
|
-
try:
|
|
1136
|
-
subprocess.run(["schtasks", "/end", "/tn", "accli-mount-nfs"], capture_output=True)
|
|
1137
|
-
except Exception:
|
|
1138
|
-
pass
|
|
1139
|
-
|
|
1140
|
-
# 2. Selective Kill: Target and terminate only the specific process using the target mount point using a PowerShell command-line filter
|
|
1288
|
+
# Determine if current user is admin
|
|
1141
1289
|
try:
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
"powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command",
|
|
1145
|
-
f"Get-CimInstance Win32_Process -Filter \"name='hf-mount-nfs.exe'\" | "
|
|
1146
|
-
f"Where-Object {{ $_.CommandLine -like '*{escaped_mount_point}*' }} | "
|
|
1147
|
-
f"ForEach-Object {{ Stop-Process -Id $_.ProcessId -Force }}"
|
|
1148
|
-
]
|
|
1149
|
-
subprocess.run(kill_cmd, capture_output=True)
|
|
1290
|
+
import ctypes
|
|
1291
|
+
is_admin = ctypes.windll.shell32.IsUserAnAdmin() != 0
|
|
1150
1292
|
except Exception:
|
|
1151
|
-
|
|
1152
|
-
|
|
1293
|
+
is_admin = False
|
|
1294
|
+
|
|
1295
|
+
# 1. Unmap the network drive in the user session
|
|
1296
|
+
res = subprocess.run(["C:\\Windows\\System32\\umount.exe", "-f", str(mount_point_abs)], capture_output=True, text=True)
|
|
1297
|
+
if res.returncode != 0:
|
|
1298
|
+
# Fallback to force-releasing using net use
|
|
1299
|
+
subprocess.run(["net", "use", str(mount_point_abs), "/delete", "/y"], capture_output=True)
|
|
1300
|
+
|
|
1301
|
+
# 2. Terminate the background daemon
|
|
1302
|
+
has_umount_task = False
|
|
1303
|
+
if not is_admin:
|
|
1304
|
+
try:
|
|
1305
|
+
task_check = subprocess.run(["schtasks", "/query", "/tn", "accli-umount-nfs"], capture_output=True, text=True)
|
|
1306
|
+
if task_check.returncode == 0:
|
|
1307
|
+
has_umount_task = True
|
|
1308
|
+
except Exception:
|
|
1309
|
+
pass
|
|
1310
|
+
|
|
1311
|
+
if has_umount_task:
|
|
1312
|
+
print("[cyan]Triggering elevated NFS daemon termination via Task Scheduler...[/cyan]")
|
|
1313
|
+
subprocess.run(["schtasks", "/run", "/tn", "accli-umount-nfs"], capture_output=True)
|
|
1314
|
+
# Poll for up to 5 seconds to ensure hf-mount-nfs.exe has fully terminated
|
|
1315
|
+
import time
|
|
1316
|
+
for _ in range(10):
|
|
1317
|
+
time.sleep(0.5)
|
|
1318
|
+
task_check = subprocess.run(["tasklist", "/FI", "IMAGENAME eq hf-mount-nfs.exe"], capture_output=True, text=True)
|
|
1319
|
+
if "hf-mount-nfs.exe" not in task_check.stdout:
|
|
1320
|
+
break
|
|
1321
|
+
else:
|
|
1322
|
+
# If admin or task scheduler gateway is not registered, stop/kill directly
|
|
1323
|
+
if is_admin:
|
|
1324
|
+
try:
|
|
1325
|
+
subprocess.run(["schtasks", "/end", "/tn", "accli-mount-nfs"], capture_output=True)
|
|
1326
|
+
except Exception:
|
|
1327
|
+
pass
|
|
1328
|
+
try:
|
|
1329
|
+
escaped_mount_point = str(mount_point_abs).replace("'", "''")
|
|
1330
|
+
kill_cmd = [
|
|
1331
|
+
"powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command",
|
|
1332
|
+
f"Get-CimInstance Win32_Process -Filter \"name='hf-mount-nfs.exe'\" | "
|
|
1333
|
+
f"Where-Object {{ $_.CommandLine -like '*{escaped_mount_point}*' }} | "
|
|
1334
|
+
f"ForEach-Object {{ Stop-Process -Id $_.ProcessId -Force }}"
|
|
1335
|
+
]
|
|
1336
|
+
subprocess.run(kill_cmd, capture_output=True)
|
|
1337
|
+
except Exception:
|
|
1338
|
+
# Fallback to killing all instances if selective filter fails
|
|
1339
|
+
subprocess.run(["taskkill", "/f", "/im", "hf-mount-nfs.exe"], capture_output=True)
|
|
1340
|
+
else:
|
|
1341
|
+
# Standard user with no task registered - need UAC to end task/kill process
|
|
1342
|
+
print("[yellow]Notice: Task Scheduler gateway is not active. Requesting UAC elevation to terminate daemon...[/yellow]")
|
|
1343
|
+
ps_command = (
|
|
1344
|
+
"Start-Process powershell.exe -ArgumentList '-NoProfile -ExecutionPolicy Bypass -Command "
|
|
1345
|
+
"\"Stop-ScheduledTask -TaskName accli-mount-nfs -ErrorAction SilentlyContinue; "
|
|
1346
|
+
"Get-CimInstance Win32_Process -Filter \\\"name=''hf-mount-nfs.exe''\\\" | Stop-Process -Force -ErrorAction SilentlyContinue\"' "
|
|
1347
|
+
"-Verb RunAs -Wait"
|
|
1348
|
+
)
|
|
1349
|
+
subprocess.run([
|
|
1350
|
+
"powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command",
|
|
1351
|
+
ps_command
|
|
1352
|
+
], capture_output=True)
|
|
1153
1353
|
|
|
1154
1354
|
if res.returncode == 0 or "successfully" in res.stdout or "successfully" in res.stderr:
|
|
1155
1355
|
print("[bold green][OK] Mount stopped successfully.[/bold green]")
|
|
@@ -1246,21 +1446,40 @@ def mount_status(
|
|
|
1246
1446
|
args = [str(hf_mount_bin), "status"]
|
|
1247
1447
|
|
|
1248
1448
|
import subprocess
|
|
1449
|
+
active_mounts = []
|
|
1450
|
+
|
|
1451
|
+
# 1. Query hf-mount status directly
|
|
1249
1452
|
try:
|
|
1250
1453
|
result = subprocess.run(args, capture_output=True, text=True)
|
|
1251
1454
|
if result.returncode == 0:
|
|
1252
1455
|
stdout = result.stdout.strip()
|
|
1253
|
-
if
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1456
|
+
if stdout:
|
|
1457
|
+
for line in stdout.splitlines():
|
|
1458
|
+
if line.strip():
|
|
1459
|
+
active_mounts.append(line.strip())
|
|
1460
|
+
except Exception:
|
|
1461
|
+
pass
|
|
1462
|
+
|
|
1463
|
+
# 2. Fallback: parse system mount table on Linux to find localhost NFS or FUSE hf-mounts
|
|
1464
|
+
if not active_mounts and platform.system() == "Linux":
|
|
1465
|
+
try:
|
|
1466
|
+
if os.path.exists("/proc/mounts"):
|
|
1467
|
+
with open("/proc/mounts", "r") as f:
|
|
1468
|
+
for line in f:
|
|
1469
|
+
parts = line.strip().split()
|
|
1470
|
+
if len(parts) >= 3:
|
|
1471
|
+
dev, mnt_point, fstype = parts[0], parts[1], parts[2]
|
|
1472
|
+
is_nfs_localhost = (fstype == "nfs" and dev.startswith("127.0.0.1:"))
|
|
1473
|
+
is_fuse_hf = ("hf-mount" in dev or "hf-mount" in fstype)
|
|
1474
|
+
if is_nfs_localhost or is_fuse_hf:
|
|
1475
|
+
active_mounts.append(f"{mnt_point} ({fstype} via {dev})")
|
|
1476
|
+
except Exception:
|
|
1477
|
+
pass
|
|
1478
|
+
|
|
1479
|
+
if not active_mounts:
|
|
1480
|
+
print("[cyan]No running mounts found.[/cyan]")
|
|
1481
|
+
else:
|
|
1482
|
+
print("[bold cyan]Active Mounts:[/bold cyan]")
|
|
1483
|
+
for m in active_mounts:
|
|
1484
|
+
print(m)
|
|
1266
1485
|
|