accli 2.0.0rc1__tar.gz → 2.0.0rc4__tar.gz

{accli-2.0.0rc1 → accli-2.0.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: accli
-Version: 2.0.0rc1
+Version: 2.0.0rc4
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -39,7 +39,7 @@ Requires-Dist: requests>=2.25.1
 Requires-Dist: jsonschema>=4.5.1
 Requires-Dist: pydantic>=2.0
 Requires-Dist: GitPython
-Requires-Dist: hf-xet>=1.4.3
+Requires-Dist: hf-xet==1.5.2
 Dynamic: license-file
 
 # Accelerator Terminal Client and Python API
@@ -138,8 +138,8 @@ Usage: accli [OPTIONS] COMMAND [ARGS]...
 
 ### Handling self-signed certificates
 
-1. Copy and paste certificates from `<control_services_backend>/certs/public.crt` as
-   `./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/public.crt` as
+1. Copy and paste certificates from `<control_services_backend>/certs/ca.crt` as
+   `./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/ca.crt` as
    `./certs/dev-minio-ca.crt`.
 2. Run
    ```

{accli-2.0.0rc1 → accli-2.0.0rc4}/README.md

@@ -94,8 +94,8 @@ Usage: accli [OPTIONS] COMMAND [ARGS]...
 
 ### Handling self-signed certificates
 
-1. Copy and paste certificates from `<control_services_backend>/certs/public.crt` as
-   `./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/public.crt` as
+1. Copy and paste certificates from `<control_services_backend>/certs/ca.crt` as
+   `./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/ca.crt` as
    `./certs/dev-minio-ca.crt`.
 2. Run
    ```

{accli-2.0.0rc1 → accli-2.0.0rc4}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read a developer guide for git tag command with regex
-VERSION = "v2.0.0-c1"
+VERSION = "v2.0.0-c4"

{accli-2.0.0rc1 → accli-2.0.0rc4}/accli/cli.py

@@ -300,12 +300,6 @@ def copy(
         verify_cert=(not ACCLI_DEBUG),
     )
 
-    def make_token_refresher(slug: str):
-        def refresher() -> tuple[str, int]:
-            token_str, _, exp_time = exchange_refresh_token(slug)
-            return token_str, exp_time
-        return refresher
-
     if is_src_remote:
         # Download flow: acc://[project_slug]/[remote_prefix] -> local destination
         source_parsed = source[len("acc://"):]
@@ -336,7 +330,7 @@ def copy(
         else:
             dest_path.parent.mkdir(parents=True, exist_ok=True)
 
-        download_infos = []
+        files_to_download = []
         for filename in filenames:
             # Filename returned usually starts with project_slug/
             if filename.startswith(f"{project_slug}/"):
@@ -378,32 +372,98 @@ def copy(
 
             local_dest.parent.mkdir(parents=True, exist_ok=True)
 
-            download_infos.append(
-                hf_xet.PyXetDownloadInfo(
-                    destination_path=str(local_dest.resolve()),
-                    hash=merkle_hash,
-                    file_size=file_size
-                )
+            files_to_download.append(
+                (str(local_dest.resolve()), merkle_hash, file_size)
             )
 
-        if not download_infos:
+        if not files_to_download:
             print("[bold red]ERROR: No valid files identified for download.[/bold red]")
             raise typer.Exit(1)
 
-        print(f"[bold cyan]Downloading {len(download_infos)} files using hf-xet...[/bold cyan]")
-        try:
-            hf_xet.download_files(
-                files=download_infos,
-                endpoint=cas_endpoint,
-                token_info=(cas_token, expires_at),
-                token_refresher=make_token_refresher(project_slug),
-                progress_updater=None,
-                request_headers=None
-            )
-            print("[bold green][OK] Download completed successfully![/bold green]")
-        except Exception as e:
-            print(f"[bold red]ERROR: Download failed: {e}[/bold red]")
-            raise typer.Exit(1)
+        MAX_FILES_PER_BATCH = 100
+        MAX_BYTES_PER_BATCH = 10 * 1024 * 1024 * 1024 # 10 GB
+        
+        batches = []
+        current_batch = []
+        current_batch_size = 0
+        
+        for item in files_to_download:
+            size = item[2]
+            if size >= MAX_BYTES_PER_BATCH and not current_batch:
+                batches.append([item])
+                continue
+                
+            if len(current_batch) >= MAX_FILES_PER_BATCH or current_batch_size + size > MAX_BYTES_PER_BATCH:
+                batches.append(current_batch)
+                current_batch = []
+                current_batch_size = 0
+                
+            current_batch.append(item)
+            current_batch_size += size
+            
+        if current_batch:
+            batches.append(current_batch)
+
+        print(f"[bold cyan]Downloading {len(files_to_download)} files in {len(batches)} batches using hf-xet...[/bold cyan]")
+        
+        refresh_url = f"{server_url.rstrip('/')}/api/v1/oauth/device/cas-token/"
+        refresh_headers = {"Authorization": f"Bearer {cas_token}"}
+        
+        with Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+        ) as progress:
+            total_size = sum(item[2] for item in files_to_download)
+            overall_task = progress.add_task("[green]Total Download Progress...", total=total_size)
+            
+            completed_in_previous_batches = 0
+            
+            session = hf_xet.XetSession()
+            for batch_index, batch_items in enumerate(batches, 1):
+                progress.print(f"[cyan]Processing batch {batch_index}/{len(batches)}...[/cyan]")
+                
+                # capture current completed_in_previous_batches in a local scope to avoid late binding issues
+                file_tasks = {}
+                
+                def make_progress_handler(base_completed):
+                    def on_progress(group_report, item_reports):
+                        progress.update(overall_task, completed=base_completed + group_report.total_bytes_completed)
+                        for item_id, item in item_reports.items():
+                            if item_id not in file_tasks:
+                                file_tasks[item_id] = progress.add_task(f"[blue]{item.item_name}", total=item.total_bytes)
+                            
+                            progress.update(file_tasks[item_id], completed=item.bytes_completed)
+                            
+                            # Hide completed tasks to keep the UI clean when downloading thousands of files
+                            if item.bytes_completed >= item.total_bytes:
+                                progress.update(file_tasks[item_id], visible=False)
+                    return on_progress
+                    
+                try:
+                    with session.new_file_download_group(
+                        endpoint=cas_endpoint,
+                        token=cas_token,
+                        token_expiry_unix_secs=expires_at,
+                        token_refresh_url=refresh_url,
+                        token_refresh_headers=refresh_headers,
+                        progress_callback=make_progress_handler(completed_in_previous_batches),
+                        progress_interval_ms=500,
+                    ) as group:
+                        for dest_path, hash_val, size_val in batch_items:
+                            info = hf_xet.XetFileInfo(hash_val, size_val)
+                            group.start_download_file(info, dest_path)
+                            
+                    batch_size = sum(item[2] for item in batch_items)
+                    completed_in_previous_batches += batch_size
+                    progress.update(overall_task, completed=completed_in_previous_batches)
+                    progress.print(f"[bold green][OK] Batch {batch_index} completed successfully![/bold green]")
+                except Exception as e:
+                    progress.print(f"[bold red]ERROR: Download failed for batch {batch_index}: {e}[/bold red]")
+                    raise typer.Exit(1)
+
+        print("[bold green][OK] Download completed successfully![/bold green]")
 
     else:
         # Upload flow: local source -> acc://[project_slug]/[remote_path]
@@ -456,65 +516,137 @@ def copy(
             print("[bold yellow]No files to upload.[/bold yellow]")
             raise typer.Exit(0)
 
-        print(f"[bold cyan]Uploading {len(local_paths)} files using hf-xet...[/bold cyan]")
-        try:
-            upload_results = hf_xet.upload_files(
-                file_paths=[str(p) for p in local_paths],
-                endpoint=cas_endpoint,
-                token_info=(cas_token, expires_at),
-                token_refresher=make_token_refresher(project_slug),
-                progress_updater=None,
-                _repo_type=None,
-                request_headers=None,
-                sha256s=None,
-                skip_sha256=False
-            )
-        except Exception as e:
-            print(f"[bold red]ERROR: Upload failed: {e}[/bold red]")
-            raise typer.Exit(1)
+        MAX_FILES_PER_BATCH = 100
+        MAX_BYTES_PER_BATCH = 10 * 1024 * 1024 * 1024 # 10 GB
+        
+        batches = []
+        current_batch_paths = []
+        current_batch_remote = []
+        current_batch_size = 0
+        
+        for local_p, remote_n in zip(local_paths, remote_filenames):
+            size = local_p.stat().st_size
+            if size >= MAX_BYTES_PER_BATCH and not current_batch_paths:
+                batches.append(([local_p], [remote_n]))
+                continue
+                
+            if len(current_batch_paths) >= MAX_FILES_PER_BATCH or current_batch_size + size > MAX_BYTES_PER_BATCH:
+                batches.append((current_batch_paths, current_batch_remote))
+                current_batch_paths = []
+                current_batch_remote = []
+                current_batch_size = 0
+                
+            current_batch_paths.append(local_p)
+            current_batch_remote.append(remote_n)
+            current_batch_size += size
+            
+        if current_batch_paths:
+            batches.append((current_batch_paths, current_batch_remote))
+
+        print(f"[bold cyan]Uploading {len(local_paths)} files in {len(batches)} batches using hf-xet...[/bold cyan]")
+        
+        refresh_url = f"{server_url.rstrip('/')}/api/v1/oauth/device/cas-token/"
+        refresh_headers = {"Authorization": f"Bearer {cas_token}"}
+        
+        with Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TimeElapsedColumn(),
+        ) as progress:
+            total_size = sum(p.stat().st_size for p in local_paths)
+            overall_task = progress.add_task("[green]Total Upload Progress...", total=total_size)
+            
+            completed_in_previous_batches = 0
+            
+            session = hf_xet.XetSession()
+            for batch_index, (b_paths, b_remotes) in enumerate(batches, 1):
+                progress.print(f"[cyan]Processing batch {batch_index}/{len(batches)}...[/cyan]")
+                
+                file_tasks = {}
+                
+                def make_progress_handler(base_completed):
+                    def on_progress(group_report, item_reports):
+                        progress.update(overall_task, completed=base_completed + group_report.total_bytes_completed)
+                        for item_id, item in item_reports.items():
+                            if item_id not in file_tasks:
+                                file_tasks[item_id] = progress.add_task(f"[blue]{item.item_name}", total=item.total_bytes)
+                            
+                            progress.update(file_tasks[item_id], completed=item.bytes_completed)
+                            
+                            # Hide completed tasks to keep the UI clean
+                            if item.bytes_completed >= item.total_bytes:
+                                progress.update(file_tasks[item_id], visible=False)
+                    return on_progress
+                    
+                try:
+                    with session.new_upload_commit(
+                        endpoint=cas_endpoint,
+                        token=cas_token,
+                        token_expiry_unix_secs=expires_at,
+                        token_refresh_url=refresh_url,
+                        token_refresh_headers=refresh_headers,
+                        progress_callback=make_progress_handler(completed_in_previous_batches),
+                        progress_interval_ms=500,
+                    ) as commit:
+                        handles = []
+                        for p in b_paths:
+                            sha = compute_sha256(str(p))
+                            h = commit.start_upload_file(str(p), sha256=sha)
+                            handles.append((p, sha, h))
+                        
+                    upload_results = [(p, sha, h.result()) for p, sha, h in handles]
+                except Exception as e:
+                    progress.print(f"[bold red]ERROR: Upload failed for batch {batch_index}: {e}[/bold red]")
+                    raise typer.Exit(1)
 
-        # Compute SHA-256 locally and register metadata in database
-        print("[cyan]Computing SHA-256 hashes and registering metadata...[/cyan]")
-        registration_items = []
-        for local_path, remote_filename, upload_info in zip(local_paths, remote_filenames, upload_results):
-            sha256_hash = compute_sha256(str(local_path))
-            registration_items.append({
-                "filename": f"{project_slug}/{remote_filename}",
-                "merkle_hash": upload_info.hash,
-                "sha256": sha256_hash,
-                "file_size": upload_info.file_size,
-                "content_type": "application/octet-stream"
-            })
-
-        headers = {
-            "Content-Type": "application/json",
-            "X-Project-Slug": project_slug,
-            "Authorization": f"Bearer {cas_token}"
-        }
+                progress.print("[cyan]Computing SHA-256 hashes and registering metadata for batch...[/cyan]")
+                registration_items = []
+                for local_path, remote_filename, (p, sha256_hash, upload_info) in zip(b_paths, b_remotes, upload_results):
+                    registration_items.append({
+                        "filename": f"{project_slug}/{remote_filename}",
+                        "merkle_hash": upload_info.xet_info.hash,
+                        "sha256": sha256_hash,
+                        "file_size": upload_info.xet_info.file_size,
+                        "content_type": "application/octet-stream"
+                    })
+
+                headers = {
+                    "Content-Type": "application/json",
+                    "X-Project-Slug": project_slug,
+                    "Authorization": f"Bearer {cas_token}"
+                }
 
-        try:
-            response = requests.post(
-                register_url,
-                json={"items": registration_items},
-                headers=headers,
-                verify=(not ACCLI_DEBUG)
-            )
-            response.raise_for_status()
-            print("[bold green][OK] Upload and bulk metadata registration completed successfully![/bold green]")
-        except requests.exceptions.HTTPError as e:
-            detail = None
-            try:
-                detail = e.response.json().get("detail")
-            except Exception:
-                pass
-            if detail:
-                print(f"[bold red]ERROR: Bulk metadata registration failed: {detail}[/bold red]")
-            else:
-                print(f"[bold red]ERROR: Bulk metadata registration failed: {e}[/bold red]")
-            raise typer.Exit(1)
-        except Exception as e:
-            print(f"[bold red]ERROR: Bulk metadata registration failed: {e}[/bold red]")
-            raise typer.Exit(1)
+                try:
+                    response = requests.post(
+                        register_url,
+                        json={"items": registration_items},
+                        headers=headers,
+                        verify=(not ACCLI_DEBUG)
+                    )
+                    response.raise_for_status()
+                    
+                    batch_size = sum(p.stat().st_size for p in b_paths)
+                    completed_in_previous_batches += batch_size
+                    progress.update(overall_task, completed=completed_in_previous_batches)
+                    
+                    progress.print(f"[bold green][OK] Batch {batch_index} registered successfully![/bold green]")
+                except requests.exceptions.HTTPError as e:
+                    detail = None
+                    try:
+                        detail = e.response.json().get("detail")
+                    except Exception:
+                        pass
+                    if detail:
+                        progress.print(f"[bold red]ERROR: Bulk metadata registration failed for batch {batch_index}: {detail}[/bold red]")
+                    else:
+                        progress.print(f"[bold red]ERROR: Bulk metadata registration failed for batch {batch_index}: {e}[/bold red]")
+                    raise typer.Exit(1)
+                except Exception as e:
+                    progress.print(f"[bold red]ERROR: Bulk metadata registration failed for batch {batch_index}: {e}[/bold red]")
+                    raise typer.Exit(1)
+
+        print("[bold green][OK] Upload completed successfully![/bold green]")
 
 
 # --- Mount commands group ---
@@ -1314,21 +1446,40 @@ def mount_status(
     args = [str(hf_mount_bin), "status"]
 
     import subprocess
+    active_mounts = []
+
+    # 1. Query hf-mount status directly
     try:
         result = subprocess.run(args, capture_output=True, text=True)
         if result.returncode == 0:
             stdout = result.stdout.strip()
-            if not stdout:
-                print("[cyan]No running mounts found.[/cyan]")
-            else:
-                print("[bold cyan]Active Mounts:[/bold cyan]")
-                print(stdout)
-        else:
-            print(f"[bold red][ERROR] Failed to retrieve mount status (exit code {result.returncode}):[/bold red]")
-            if result.stderr:
-                print(f"[red]{result.stderr}[/red]")
-            raise typer.Exit(result.returncode)
-    except Exception as e:
-        print(f"[bold red]ERROR: Failed to execute status process: {e}[/bold red]")
-        raise typer.Exit(1)
+            if stdout:
+                for line in stdout.splitlines():
+                    if line.strip():
+                        active_mounts.append(line.strip())
+    except Exception:
+        pass
+
+    # 2. Fallback: parse system mount table on Linux to find localhost NFS or FUSE hf-mounts
+    if not active_mounts and platform.system() == "Linux":
+        try:
+            if os.path.exists("/proc/mounts"):
+                with open("/proc/mounts", "r") as f:
+                    for line in f:
+                        parts = line.strip().split()
+                        if len(parts) >= 3:
+                            dev, mnt_point, fstype = parts[0], parts[1], parts[2]
+                            is_nfs_localhost = (fstype == "nfs" and dev.startswith("127.0.0.1:"))
+                            is_fuse_hf = ("hf-mount" in dev or "hf-mount" in fstype)
+                            if is_nfs_localhost or is_fuse_hf:
+                                active_mounts.append(f"{mnt_point}   ({fstype} via {dev})")
+        except Exception:
+            pass
+
+    if not active_mounts:
+        print("[cyan]No running mounts found.[/cyan]")
+    else:
+        print("[bold cyan]Active Mounts:[/bold cyan]")
+        for m in active_mounts:
+            print(m)
 

{accli-2.0.0rc1 → accli-2.0.0rc4}/accli/gui.py

@@ -113,7 +113,7 @@ class AccliGuiApp(tk.Tk):
         self.build_auth_tab()
         self.build_logs_tab()
 
-    def run_cli_async(self, args, on_done=None, log_to_viewer=True):
+    def run_cli_async(self, args, on_done=None, log_to_viewer=True, elevate=False):
         """Runs the CLI binary in a background thread to prevent UI freezing."""
         def worker():
             # Determine how to invoke accli
@@ -124,6 +124,13 @@ class AccliGuiApp(tk.Tk):
                 # Script mode
                 cmd = [sys.executable, "-m", "accli.cli"] + args
 
+            if elevate and platform.system() != "Windows":
+                import shutil
+                if shutil.which("pkexec"):
+                    cmd = ["pkexec"] + cmd
+                else:
+                    cmd = ["sudo"] + cmd
+
             try:
                 if log_to_viewer:
                     self.append_log(f"Executing: {' '.join(cmd)}\n")
@@ -491,7 +498,7 @@ class AccliGuiApp(tk.Tk):
                 self.status_bar_val.set("Mount Failed")
                 messagebox.showerror("Error", f"Failed to mount project '{slug}'. See logs for details.")
 
-        self.run_cli_async(args, on_done=on_done)
+        self.run_cli_async(args, on_done=on_done, elevate=True)
 
     def action_mount_stop(self):
         selected = self.mounts_list.get(tk.ACTIVE)
@@ -521,7 +528,7 @@ class AccliGuiApp(tk.Tk):
                 self.status_bar_val.set("Stop Failed")
                 messagebox.showerror("Error", f"Failed to stop mount at {mount_point}.")
 
-        self.run_cli_async(args, on_done=on_done)
+        self.run_cli_async(args, on_done=on_done, elevate=True)
 
     # ------------------ COPY TAB ------------------
     def build_copy_tab(self):

{accli-2.0.0rc1 → accli-2.0.0rc4}/accli/mount_downloader.py

@@ -9,7 +9,7 @@ import typer
 from rich import print
 from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn
 
-DEFAULT_VERSION = "v0.6.5-acc-pr140-p4"
+DEFAULT_VERSION = "v0.6.5-acc-pr140-p5"
 BINARY_DIR = Path.home() / ".accli" / "bin"
 
 # Regex to safely validate version strings to prevent URL manipulation or directory traversal

{accli-2.0.0rc1 → accli-2.0.0rc4}/accli/token.py

@@ -4,7 +4,21 @@ from tinydb import TinyDB
 
 
 def get_db_path():
-    home = os.path.expanduser("~")
+    sudo_user = os.environ.get("SUDO_USER")
+    pkexec_uid = os.environ.get("PKEXEC_UID")
+
+    if (sudo_user or pkexec_uid) and os.name != "nt":
+        try:
+            import pwd
+            if pkexec_uid:
+                home = pwd.getpwuid(int(pkexec_uid)).pw_dir
+            else:
+                home = pwd.getpwnam(sudo_user).pw_dir
+        except Exception:
+            home = os.path.expanduser("~")
+    else:
+        home = os.path.expanduser("~")
+
     token_directory = f"{home}/.accli"
 
     if not os.path.exists(token_directory):

{accli-2.0.0rc1 → accli-2.0.0rc4}/accli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: accli
-Version: 2.0.0rc1
+Version: 2.0.0rc4
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -39,7 +39,7 @@ Requires-Dist: requests>=2.25.1
 Requires-Dist: jsonschema>=4.5.1
 Requires-Dist: pydantic>=2.0
 Requires-Dist: GitPython
-Requires-Dist: hf-xet>=1.4.3
+Requires-Dist: hf-xet==1.5.2
 Dynamic: license-file
 
 # Accelerator Terminal Client and Python API
@@ -138,8 +138,8 @@ Usage: accli [OPTIONS] COMMAND [ARGS]...
 
 ### Handling self-signed certificates
 
-1. Copy and paste certificates from `<control_services_backend>/certs/public.crt` as
-   `./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/public.crt` as
+1. Copy and paste certificates from `<control_services_backend>/certs/ca.crt` as
+   `./certs/dev-backend.crt` and `<accelerator_service>/minio_certs/ca.crt` as
    `./certs/dev-minio-ca.crt`.
 2. Run
    ```

{accli-2.0.0rc1 → accli-2.0.0rc4}/accli.egg-info/requires.txt

@@ -5,4 +5,4 @@ requests>=2.25.1
 jsonschema>=4.5.1
 pydantic>=2.0
 GitPython
-hf-xet>=1.4.3
+hf-xet==1.5.2

{accli-2.0.0rc1 → accli-2.0.0rc4}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
     "jsonschema>=4.5.1",      # 4.5.1+ is safe across libraries
     "pydantic>=2.0",          # Major version 2 required if you're using v2 features
     "GitPython",
-    "hf-xet>=1.4.3",
+    "hf-xet==1.5.2",
 ]
 classifiers = [
     "Programming Language :: Python :: 3",