accli 2.0.12__tar.gz → 2.0.14__tar.gz

{accli-2.0.12 → accli-2.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: accli
-Version: 2.0.12
+Version: 2.0.14
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-2.0.12 → accli-2.0.14}/accli/AcceleratorTaskDispatcher.py

@@ -196,7 +196,7 @@ class JobDispatchModel(BaseModel):
     node_id: Optional[str] = None
 
     ignore_duplicate_job: bool = False
-    free_node: bool = False  # Only applies to immediate children jobs
+    free_node: bool = True
 
     children: List['JobDispatchModel'] = []
     callback: Optional['JobDispatchModel'] = None

{accli-2.0.12 → accli-2.0.14}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read a developer guide for git tag command with regex
-VERSION = "v2.0.12"
+VERSION = "v2.0.14"

{accli-2.0.12 → accli-2.0.14}/accli/cli.py

@@ -589,17 +589,30 @@ mount_app = typer.Typer(
 app.add_typer(mount_app, name="mount")
 
 
+def is_windows_drive_mapped(drive: str) -> bool:
+    """Non-blocking check if a Windows drive letter is currently mapped using Win32 GetLogicalDrives()."""
+    import ctypes
+    letter = drive.strip().rstrip(":\\/").upper()
+    if len(letter) != 1 or not letter.isalpha():
+        return os.path.exists(drive)
+    try:
+        bitmask = ctypes.windll.kernel32.GetLogicalDrives()
+        return bool(bitmask & (1 << (ord(letter) - ord("A"))))
+    except Exception:
+        return os.path.exists(f"{letter}:\\")
+
+
 def find_available_windows_drive(preferred: str) -> str:
     """Finds an available Windows drive letter, starting with preferred, then checking others."""
     import os
     preferred = preferred.upper()
-    if not os.path.exists(f"{preferred}:\\"):
+    if not is_windows_drive_mapped(preferred):
         return f"{preferred}:"
         
     # Search order: subsequent letters first, then previous descending to D
     search_order = ["X", "Y", "Z"] + [chr(x) for x in range(ord("V"), ord("D") - 1, -1)]
     for letter in search_order:
-        if not os.path.exists(f"{letter}:\\"):
+        if not is_windows_drive_mapped(letter):
             return f"{letter}:"
             
     raise RuntimeError("No available Windows drive letters found.")
@@ -672,6 +685,7 @@ def start_nfs_watcher(mount_point: str, interval: int):
         proc = subprocess.Popen(
             [executable, str(watcher_script), str(mount_point), str(interval)],
             creationflags=creationflags,
+            stdin=subprocess.DEVNULL,
             stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL,
             close_fds=True
@@ -722,6 +736,209 @@ def stop_nfs_watcher(mount_point: str):
             pass
 
 
+def cleanup_stuck_windows_mount_processes():
+    """Terminate any orphaned mount.exe / umount.exe instances that block Windows NFS client."""
+    import subprocess
+    try:
+        subprocess.run(["taskkill", "/f", "/im", "mount.exe"], capture_output=True)
+    except Exception:
+        pass
+    try:
+        subprocess.run(["taskkill", "/f", "/im", "umount.exe"], capture_output=True)
+        subprocess.run(["taskkill", "/f", "/im", "net.exe"], capture_output=True)
+    except Exception:
+        pass
+
+
+def cleanup_stale_rdp_namespace_entries():
+    """
+    Remove disconnected/stale RDP redirected drive entries (\\\\tsclient\\<drive>) from the user's
+    Explorer MyComputer Namespace registry key.
+    When Windows Client for NFS (NfsClnt) is active, any dead \\\\tsclient\\<drive> shortcut in
+    'This PC' fails in RDPNP and falls through Windows MUP to NfsClnt, which attempts to resolve
+    'tsclient' via DNS/NetBIOS/RPC and freezes new Explorer 'This PC' views for 21+ seconds.
+    """
+    import subprocess
+    ps_script = (
+        "$activeShares = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)\n"
+        "net use 2>$null | ForEach-Object {\n"
+        "    if ($_ -match '(\\\\\\\\tsclient\\\\[a-zA-Z0-9_$]+)') {\n"
+        "        [void]$activeShares.Add($matches[1])\n"
+        "    }\n"
+        "}\n"
+        "Get-ChildItem 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SessionInfo\\*\\MyComputer\\Namespace\\*' -ErrorAction SilentlyContinue | ForEach-Object {\n"
+        "    $clsid = $_.PSChildName\n"
+        "    $bag = Get-ItemProperty \"HKCU:\\Software\\Classes\\CLSID\\$clsid\\Instance\\InitPropertyBag\" -ErrorAction SilentlyContinue\n"
+        "    if ($bag -and $bag.Target -and $bag.Target -match '^\\\\\\\\tsclient\\\\') {\n"
+        "        if (-not $activeShares.Contains($bag.Target)) {\n"
+        "            Remove-Item -LiteralPath $_.PSPath -Recurse -Force -ErrorAction SilentlyContinue\n"
+        "        }\n"
+        "    }\n"
+        "}\n"
+    )
+    try:
+        subprocess.run(
+            ["powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", ps_script],
+            capture_output=True,
+            text=True,
+            errors="replace",
+            timeout=5,
+            creationflags=0x08000000  # CREATE_NO_WINDOW
+        )
+    except Exception:
+        pass
+
+
+def is_nfs_daemon_running() -> bool:
+    """Check if hf-mount-nfs.exe is currently running."""
+    import subprocess
+    try:
+        res = subprocess.run(["tasklist", "/FI", "IMAGENAME eq hf-mount-nfs.exe"], capture_output=True, text=True, errors="replace")
+        return res.stdout is not None and "hf-mount-nfs.exe" in res.stdout
+    except Exception:
+        return False
+
+
+def is_nfs_kernel_driver_active() -> bool:
+    """Check if the Windows Client for NFS kernel redirector driver (NfsRdr) is loaded and running."""
+    import subprocess
+    import shutil
+    if not shutil.which("mount.exe") and not os.path.isfile("C:\\Windows\\System32\\mount.exe"):
+        return False
+    try:
+        res = subprocess.run(["sc.exe", "query", "NfsRdr"], capture_output=True, text=True, errors="replace")
+        if res.stdout and "RUNNING" in res.stdout:
+            return True
+        # If stopped, try to start it
+        subprocess.run(["sc.exe", "start", "NfsRdr"], capture_output=True)
+        check = subprocess.run(["sc.exe", "query", "NfsRdr"], capture_output=True, text=True, errors="replace")
+        return check.stdout is not None and "RUNNING" in check.stdout
+    except Exception:
+        return False
+
+
+def wait_for_nfs_portmapper(host: str, port: int, timeout: float) -> bool:
+    """Poll until the NFS daemon's portmapper is accepting TCP connections and is stable."""
+    import socket
+    start = time.time()
+    consecutive_success = 0
+    while time.time() - start < timeout:
+        try:
+            with socket.create_connection((host, port), timeout=0.5):
+                consecutive_success += 1
+                if consecutive_success >= 2:
+                    time.sleep(1.0)
+                    return True
+        except (socket.error, OSError):
+            consecutive_success = 0
+            time.sleep(0.3)
+    return False
+
+
+def ensure_nfs_daemon_stable(timeout: float) -> bool:
+    """
+    Ensure the NFS daemon starts and survives any delayed UMNT packet flushed by Windows NFS client.
+    If the daemon exits due to UMNT, re-triggers the Task Scheduler task to start a fresh daemon.
+    """
+    import subprocess
+    for attempt in range(3):
+        if not wait_for_nfs_portmapper("127.0.0.1", 111, timeout):
+            return False
+
+        # Windows NFS client kernel redirector flushes delayed UMNT within 7-8 seconds of portmapper binding
+        exited = False
+        for _ in range(16):
+            time.sleep(0.5)
+            if not is_nfs_daemon_running():
+                exited = True
+                break
+
+        if not exited:
+            # Daemon remained running and stable without being killed by delayed UMNT
+            return True
+
+        # Daemon consumed the queued UMNT packet and terminated cleanly. Re-trigger task!
+        time.sleep(1.0)
+        subprocess.run(["schtasks", "/run", "/tn", "accli-mount-nfs"], capture_output=True)
+
+    return wait_for_nfs_portmapper("127.0.0.1", 111, timeout)
+
+
+def map_windows_network_drive(mount_point_abs, max_attempts: int) -> tuple[bool, str]:
+    """
+    Attempt to map the NFS share to the requested drive letter using mount.exe.
+    Retries with cleanup if Windows NFS client returns a transient error.
+    """
+    import subprocess
+    cleanup_stale_rdp_namespace_entries()
+    target_str = str(mount_point_abs).rstrip("\\/")
+    if re.match(r"^[a-zA-Z]$", target_str):
+        target_str += ":"
+
+    mount_cmd = [
+        "C:\\Windows\\System32\\mount.exe",
+        "-o", "mtype=soft",
+        "-o", "timeout=2",
+        "-o", "retry=2",
+        "-o", "rsize=32",
+        "-o", "wsize=32",
+        "-o", "anon",
+        "-o", "nolock",
+        "\\\\127.0.0.1\\!",
+        target_str
+    ]
+
+    last_err = ""
+    drive_root = str(mount_point_abs).rstrip("\\/") + "\\"
+    for attempt in range(1, max_attempts + 1):
+        cleanup_stuck_windows_mount_processes()
+
+        # If daemon died before or between attempts, revive it
+        if not is_nfs_daemon_running():
+            subprocess.run(["schtasks", "/run", "/tn", "accli-mount-nfs"], capture_output=True)
+            wait_for_nfs_portmapper("127.0.0.1", 111, 10.0)
+
+        try:
+            res = subprocess.run(mount_cmd, capture_output=True, text=True, errors="replace", timeout=15)
+            if res.returncode == 0:
+                # Confirm drive is actually mounted and directory listing is populated and queryable
+                drive_ready = False
+                for _ in range(30):
+                    try:
+                        if os.path.exists(drive_root):
+                            entries = os.listdir(drive_root)
+                            if len(entries) > 0:
+                                drive_ready = True
+                                return True, (res.stdout or "").strip()
+                    except OSError:
+                        pass
+                    time.sleep(0.5)
+                if not drive_ready:
+                    last_err = f"mount.exe reported success but drive {drive_root} was empty or not accessible"
+                    # Clean up the stale mount connection so next attempt can remount fresh
+                    subprocess.run(["net", "use", target_str, "/delete", "/y"], capture_output=True)
+            else:
+                last_err = (res.stderr or res.stdout or "").strip()
+                if "85" in last_err or "already in use" in last_err.lower():
+                    # Release stale network drive letter without unmounting the daemon UNC path
+                    subprocess.run(["net", "use", target_str, "/delete", "/y"], capture_output=True)
+                    time.sleep(1.0)
+        except subprocess.TimeoutExpired:
+            last_err = "mount.exe command timed out"
+            cleanup_stuck_windows_mount_processes()
+        except Exception as e:
+            last_err = str(e)
+
+        if attempt < max_attempts:
+            # Check if daemon needs recovery
+            if not is_nfs_daemon_running():
+                subprocess.run(["schtasks", "/run", "/tn", "accli-mount-nfs"], capture_output=True)
+            wait_for_nfs_portmapper("127.0.0.1", 111, 3.0)
+            time.sleep(1.0 + (attempt * 0.5))
+
+    return False, last_err
+
+
 def enable_windows_nfs_features():
     """Automatically check and enable Windows Client for NFS features, registry key, and Task Scheduler gateway if needed."""
     import subprocess
@@ -759,7 +976,17 @@ def enable_windows_nfs_features():
         "$taskEnabled = $false\n"
         "if (Get-Command Get-ScheduledTask -ErrorAction SilentlyContinue) {\n"
         "    $t = Get-ScheduledTask -TaskName 'accli-mount-nfs' -ErrorAction SilentlyContinue\n"
-        "    if ($t) { $taskEnabled = $true }\n"
+        "    if ($t) {\n"
+        "        $taskEnabled = $true\n"
+        "        if ($t.Settings.ExecutionTimeLimit -ne 'PT0S' -and $t.Settings.ExecutionTimeLimit -ne 'P0D' -and -not [string]::IsNullOrEmpty($t.Settings.ExecutionTimeLimit)) {\n"
+        "            try {\n"
+        "                $s = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -ExecutionTimeLimit ([TimeSpan]::Zero)\n"
+        "                Set-ScheduledTask -TaskName 'accli-mount-nfs' -Settings $s -ErrorAction Stop | Out-Null\n"
+        "            } catch {\n"
+        "                $taskEnabled = $false\n"
+        "            }\n"
+        "        }\n"
+        "    }\n"
         "} else {\n"
         "    $query = schtasks /query /tn 'accli-mount-nfs' 2> $null\n"
         "    if ($LASTEXITCODE -eq 0) { $taskEnabled = $true }\n"
@@ -780,8 +1007,8 @@ def enable_windows_nfs_features():
     task_ok = False
     
     try:
-        res = subprocess.run(check_cmd, capture_output=True, text=True, check=True)
-        output = res.stdout.strip()
+        res = subprocess.run(check_cmd, capture_output=True, text=True, errors="replace", check=True)
+        output = (res.stdout or "").strip()
         for part in output.split(';'):
             if part.startswith("NFS:"):
                 nfs_ok = (part.split(':')[1].lower() == 'true')
@@ -851,7 +1078,7 @@ def enable_windows_nfs_features():
             f"Start-Process powershell.exe -ArgumentList '-NoProfile -ExecutionPolicy Bypass -File \"{temp_file_path}\" {params_str}' -Verb RunAs -Wait"
         ]
         
-        result = subprocess.run(elevate_cmd, capture_output=True, text=True)
+        result = subprocess.run(elevate_cmd, capture_output=True, text=True, errors="replace")
         if result.returncode != 0:
             print("[bold red][ERROR] Administrative configuration failed or UAC prompt was denied.[/bold red]")
             if result.stderr:
@@ -860,13 +1087,16 @@ def enable_windows_nfs_features():
             
         print("[bold green][OK] Elevated administrative setup completed successfully![/bold green]")
         
-        # Reboot advisory for registry keys or NFS features
-        if not reg_ok or not nfs_ok:
-            print("")
-            print("[bold yellow][WARNING] IMPORTANT: A Windows system restart is required for the new NFS Client features and registry policies to take effect.[/bold yellow]")
-            print("[bold yellow]Please restart your computer, then run 'accli mount start' again to mount your drive passwordlessly![/bold yellow]")
-            print("")
-            raise typer.Exit(0)
+        # Verify if kernel driver is loaded; if active, restart is NOT required and we continue mounting
+        if is_nfs_kernel_driver_active():
+            print("[bold green][OK] Windows NFS kernel redirector is active. Continuing with mount...[/bold green]")
+            return
+
+        print("")
+        print("[bold yellow][WARNING] IMPORTANT: A Windows system restart is required for the new NFS Client features and registry policies to take effect.[/bold yellow]")
+        print("[bold yellow]Please restart your computer, then run 'accli mount start' again to mount your drive passwordlessly![/bold yellow]")
+        print("")
+        raise typer.Exit(0)
             
     except Exception as e:
         if isinstance(e, typer.Exit):
@@ -909,7 +1139,12 @@ def mount_start(
     try:
         token = get_token()
         server_url = get_server_url()
+        if not token:
+            print("[bold red]ERROR: No token found. Please run 'accli login' first.[/bold red]")
+            raise typer.Exit(1)
     except Exception as e:
+        if isinstance(e, typer.Exit):
+            raise e
         print(f"[bold red]ERROR: Could not retrieve login details. Please run 'accli login' first. Details: {e}[/bold red]")
         raise typer.Exit(1)
 
@@ -934,26 +1169,19 @@ def mount_start(
 
     # Standardize mount_point and handle defaults / auto-selection
     if sys_name == "Windows":
-        # Check if the daemon is already running to prevent double mount
-        import subprocess
-        try:
-            task_check = subprocess.run(["tasklist", "/FI", "IMAGENAME eq hf-mount-nfs.exe"], capture_output=True, text=True)
-            if "hf-mount-nfs.exe" in task_check.stdout:
-                config_file = Path("C:/ProgramData/accli/mount_config.json")
-                active_msg = ""
-                if config_file.is_file():
-                    try:
-                        import json
-                        cfg = json.loads(config_file.read_text(encoding="utf-8"))
-                        active_msg = f" for project '{cfg.get('project_slug')}' at '{cfg.get('mount_point')}'"
-                    except Exception:
-                        pass
-                print(f"[bold red]ERROR: A mount daemon is already running{active_msg}.[/bold red]")
-                print("[yellow]Please run 'accli mount stop' to unmount before starting a new mount.[/yellow]")
-                raise typer.Exit(1)
-        except Exception as e:
-            if isinstance(e, typer.Exit):
-                raise e
+        if is_nfs_daemon_running():
+            config_file = Path("C:/ProgramData/accli/mount_config.json")
+            active_msg = ""
+            if config_file.is_file():
+                try:
+                    import json
+                    cfg = json.loads(config_file.read_text(encoding="utf-8"))
+                    active_msg = f" for project '{cfg.get('project_slug')}' at '{cfg.get('mount_point')}'"
+                except Exception:
+                    pass
+            print(f"[bold red]ERROR: A mount daemon is already running{active_msg}.[/bold red]")
+            print("[yellow]Please run 'accli mount stop' to unmount before starting a new mount.[/yellow]")
+            raise typer.Exit(1)
 
         if mount_point is None:
             # Auto-detect available drive letter starting with W
@@ -965,10 +1193,10 @@ def mount_start(
                 print(f"[bold red]ERROR: {e}[/bold red]")
                 raise typer.Exit(1)
         else:
-            mount_point_str = str(mount_point)
-            if re.match(r"^[a-zA-Z]:?\\?$", mount_point_str) or re.match(r"^[a-zA-Z]:/?$", mount_point_str):
+            mount_point_str = str(mount_point).strip()
+            if re.match(r"^[a-zA-Z]:?[\\/]*$", mount_point_str):
                 drive_letter = mount_point_str[0].upper()
-                if os.path.exists(f"{drive_letter}:\\"):
+                if is_windows_drive_mapped(drive_letter):
                     print(f"[yellow]Warning: Drive '{drive_letter}:' is already in use.[/yellow]")
                     try:
                         fallback_drive = find_available_windows_drive("W")
@@ -997,6 +1225,12 @@ def mount_start(
     env["ACC_ENDPOINT"] = server_url
     env["ACC_CAS_ENDPOINT"] = f"{server_url.rstrip('/')}/api/xet-cas"
     env["ACC_TOKEN"] = token
+    cache_base = Path("C:/ProgramData/accli/cache") if sys_name == "Windows" else Path.home() / ".accli" / "cache"
+    (cache_base / "xet").mkdir(parents=True, exist_ok=True)
+    (cache_base / "hf-mount").mkdir(parents=True, exist_ok=True)
+    env["HF_XET_CACHE"] = str(cache_base / "xet")
+    env["HF_HOME"] = str(cache_base)
+    env["XDG_CACHE_HOME"] = str(cache_base)
 
     if sys_name == "Windows":
         # Enable Windows NFS Client features if needed
@@ -1018,6 +1252,7 @@ def mount_start(
             str(hf_mount_bin),
             "--token-file", db_path,
             "--hub-endpoint", server_url,
+            "--cache-dir", str(cache_base / "hf-mount"),
             mode,
             project_slug,
             str(mount_point_abs)
@@ -1049,7 +1284,7 @@ def mount_start(
         has_task = False
         if not is_admin:
             try:
-                task_check = subprocess.run(["schtasks", "/query", "/tn", "accli-mount-nfs"], capture_output=True, text=True)
+                task_check = subprocess.run(["schtasks", "/query", "/tn", "accli-mount-nfs"], capture_output=True, text=True, errors="replace")
                 if task_check.returncode == 0:
                     has_task = True
             except Exception:
@@ -1103,39 +1338,30 @@ def mount_start(
                 config_file = config_dir / "mount_config.json"
                 config_file.write_text(json.dumps(config_data, indent=2), encoding="utf-8")
                 
-                run_res = subprocess.run(["schtasks", "/run", "/tn", "accli-mount-nfs"], capture_output=True, text=True)
+                run_res = subprocess.run(["schtasks", "/run", "/tn", "accli-mount-nfs"], capture_output=True, text=True, errors="replace")
                 if run_res.returncode == 0:
                     print("[cyan]Waiting for elevated NFS daemon to initialize...[/cyan]")
-                    time.sleep(2.0)
+                    if not ensure_nfs_daemon_stable(15.0):
+                        print("[bold red][ERROR] Timed out waiting for NFS daemon to bind to 127.0.0.1:111.[/bold red]")
+                        raise typer.Exit(1)
                     
                     # Map the network drive in the CURRENT user session (so it is visible in Explorer)
-                    mount_cmd = [
-                        "C:\\Windows\\System32\\mount.exe",
-                        "-o", "mtype=hard",
-                        "-o", "timeout=10",
-                        "-o", "rsize=32",
-                        "-o", "wsize=32",
-                        "-o", "anon",
-                        "-o", "nolock",
-                        "\\\\127.0.0.1\\!",
-                        str(mount_point_abs)
-                    ]
-                    mount_res = subprocess.run(mount_cmd, capture_output=True, text=True)
-                    if mount_res.returncode == 0:
+                    success, err_msg = map_windows_network_drive(mount_point_abs, 5)
+                    if success:
                         start_nfs_watcher(str(mount_point_abs), interval=30)
                         print(f"[bold green][OK] NFS mount successfully mapped at [white]{mount_point_abs}[/white]![/bold green]")
                         print("[cyan]Use 'umount' or 'accli mount stop' to unmount the drive.[/cyan]")
                         return
                     else:
                         print(f"[bold red][ERROR] NFS daemon started, but failed to map drive {mount_point_abs} in user session.[/bold red]")
-                        if mount_res.stderr:
-                            print(f"[red]{mount_res.stderr.strip()}[/red]")
+                        if err_msg:
+                            print(f"[red]{err_msg}[/red]")
                         print("[yellow]Tip: Make sure Windows NFS client feature was fully enabled and the PC was restarted.[/yellow]")
                         raise typer.Exit(1)
                 else:
                     print("[bold red][ERROR] Failed to execute Scheduled Task 'accli-mount-nfs'.[/bold red]")
-                    if trigger_res.stderr:
-                        print(f"[red]{trigger_res.stderr.strip()}[/red]")
+                    if run_res.stderr:
+                        print(f"[red]{run_res.stderr.strip()}[/red]")
                     print("[yellow]Falling back to UAC elevation...[/yellow]")
                     has_task = False
             except Exception as e:
@@ -1199,7 +1425,7 @@ def mount_start(
                     ps_command
                 ]
                 
-                elevate_res = subprocess.run(ps_args, capture_output=True, text=True)
+                elevate_res = subprocess.run(ps_args, capture_output=True, text=True, errors="replace")
                 if elevate_res.returncode != 0:
                     print("[bold red][ERROR] Administrative elevation failed or was denied.[/bold red]")
                     print("[yellow]Tip: Please request your IT Administrator to run 'setup-nfs.ps1' to configure the passwordless Task Scheduler gateway.[/yellow]")
@@ -1208,29 +1434,20 @@ def mount_start(
                     raise typer.Exit(1)
                     
                 print("[cyan]Waiting for elevated NFS daemon to initialize...[/cyan]")
-                time.sleep(2.0)
+                if not wait_for_nfs_portmapper("127.0.0.1", 111, 15.0):
+                    print("[bold red][ERROR] Timed out waiting for NFS daemon to bind to 127.0.0.1:111.[/bold red]")
+                    raise typer.Exit(1)
                 
                 # Map the network drive in the CURRENT user session (so it is visible in Explorer)
-                mount_cmd = [
-                    "C:\\Windows\\System32\\mount.exe",
-                    "-o", "mtype=hard",
-                    "-o", "timeout=10",
-                    "-o", "rsize=32",
-                    "-o", "wsize=32",
-                    "-o", "anon",
-                    "-o", "nolock",
-                    "\\\\127.0.0.1\\!",
-                    str(mount_point_abs)
-                ]
-                mount_res = subprocess.run(mount_cmd, capture_output=True, text=True)
-                if mount_res.returncode == 0:
+                success, err_msg = map_windows_network_drive(mount_point_abs, 5)
+                if success:
                     start_nfs_watcher(str(mount_point_abs), interval=30)
                     print(f"[bold green][OK] NFS mount successfully mapped at [white]{mount_point_abs}[/white]![/bold green]")
                     print("[cyan]Use 'umount' or 'accli mount stop' to unmount the drive.[/cyan]")
                 else:
                     print(f"[bold red][ERROR] NFS daemon spawned elevated, but failed to map drive {mount_point_abs} in user session.[/bold red]")
-                    if mount_res.stderr:
-                        print(f"[red]{mount_res.stderr.strip()}[/red]")
+                    if err_msg:
+                        print(f"[red]{err_msg}[/red]")
                     raise typer.Exit(1)
         except Exception as e:
             if isinstance(e, typer.Exit):
@@ -1268,7 +1485,7 @@ def mount_start(
         print(f"[bold cyan]Starting mount for project '[white]{project_slug}[/white]' at '[white]{mount_point_abs}[/white]'...[/bold cyan]")
         import subprocess
         try:
-            result = subprocess.run(args, env=env, capture_output=True, text=True)
+            result = subprocess.run(args, env=env, capture_output=True, text=True, errors="replace")
             if result.returncode == 0:
                 print(f"[bold green][OK] Mount started successfully![/bold green]")
                 if result.stdout:
@@ -1349,10 +1566,17 @@ def mount_stop(
             mount_point_abs = Path.home() / "accelerator" / "mnt" / project_slug
             print(f"[cyan]No mount point specified. Selected default directory: [bold white]{mount_point_abs}[/bold white][/cyan]")
         else:
-            mount_point_abs = mount_point.resolve()
+            mount_point_str = str(mount_point).strip()
+            if sys_name == "Windows" and re.match(r"^[a-zA-Z]:?[\\/]*$", mount_point_str):
+                mount_point_abs = Path(f"{mount_point_str[0].upper()}:")
+            else:
+                mount_point_abs = mount_point.resolve()
 
     if sys_name == "Windows":
-        print(f"[bold cyan]Stopping mount at '[white]{mount_point_abs}[/white]' on Windows...[/bold cyan]")
+        target_str = str(mount_point_abs).rstrip("\\/")
+        if re.match(r"^[a-zA-Z]$", target_str):
+            target_str += ":"
+        print(f"[bold cyan]Stopping mount at '[white]{target_str}[/white]' on Windows...[/bold cyan]")
         import subprocess
         
         # Determine if current user is admin
@@ -1363,19 +1587,41 @@ def mount_stop(
             is_admin = False
 
         # 0. Stop background NFS watcher
-        stop_nfs_watcher(str(mount_point_abs))
+        stop_nfs_watcher(target_str)
+
+        # 1. Unmap the network drive in the user session using net use
+        # (Avoids sending RPC UMNT packet via umount.exe which causes kernel NFS redirector to queue stale UMNTs)
+        # If the drive is mapped in Windows but the local NFS daemon has died (e.g., 72h task timeout),
+        # Windows NFS kernel redirector (NfsRdr) with mtype=hard will block 'net use /delete' indefinitely
+        # waiting for 127.0.0.1:111. Briefly revive the daemon so the kernel can complete the unmount cleanly.
+        if is_windows_drive_mapped(target_str) and not is_nfs_daemon_running():
+            try:
+                subprocess.run(["schtasks", "/run", "/tn", "accli-mount-nfs"], capture_output=True, timeout=5)
+                wait_for_nfs_portmapper("127.0.0.1", 111, 5.0)
+            except Exception:
+                pass
+        res = None
+        try:
+            res = subprocess.run(["net", "use", target_str, "/delete", "/y"], capture_output=True, text=True, errors="replace", timeout=10)
+        except subprocess.TimeoutExpired:
+            cleanup_stuck_windows_mount_processes()
+        if res is not None and res.returncode != 0:
+            try:
+                res = subprocess.run(["C:\\Windows\\System32\\umount.exe", "-f", target_str], capture_output=True, text=True, errors="replace", timeout=10)
+            except subprocess.TimeoutExpired:
+                cleanup_stuck_windows_mount_processes()
+        try:
+            subprocess.run(["net", "use", "\\\\127.0.0.1\\!", "/delete", "/y"], capture_output=True, timeout=5)
+        except subprocess.TimeoutExpired:
+            cleanup_stuck_windows_mount_processes()
 
-        # 1. Unmap the network drive in the user session
-        res = subprocess.run(["C:\\Windows\\System32\\umount.exe", "-f", str(mount_point_abs)], capture_output=True, text=True)
-        if res.returncode != 0:
-            # Fallback to force-releasing using net use
-            subprocess.run(["net", "use", str(mount_point_abs), "/delete", "/y"], capture_output=True)
+        cleanup_stuck_windows_mount_processes()
 
         # 2. Terminate the background daemon
         has_umount_task = False
         if not is_admin:
             try:
-                task_check = subprocess.run(["schtasks", "/query", "/tn", "accli-umount-nfs"], capture_output=True, text=True)
+                task_check = subprocess.run(["schtasks", "/query", "/tn", "accli-umount-nfs"], capture_output=True, text=True, errors="replace")
                 if task_check.returncode == 0:
                     has_umount_task = True
             except Exception:
@@ -1387,9 +1633,20 @@ def mount_stop(
             # Poll for up to 5 seconds to ensure hf-mount-nfs.exe has fully terminated
             for _ in range(10):
                 time.sleep(0.5)
-                task_check = subprocess.run(["tasklist", "/FI", "IMAGENAME eq hf-mount-nfs.exe"], capture_output=True, text=True)
-                if "hf-mount-nfs.exe" not in task_check.stdout:
+                if not is_nfs_daemon_running():
+                    break
+            cleanup_stuck_windows_mount_processes()
+            for _ in range(10):
+                sched_check = subprocess.run(["schtasks", "/query", "/tn", "accli-mount-nfs"], capture_output=True, text=True, errors="replace")
+                if not sched_check.stdout or "Running" not in sched_check.stdout:
                     break
+                time.sleep(0.5)
+
+            # Wait until drive letter is released from Windows namespace
+            for _ in range(10):
+                if not is_windows_drive_mapped(target_str):
+                    break
+                time.sleep(0.3)
         else:
             # If admin or task scheduler gateway is not registered, stop/kill directly
             if is_admin:
@@ -1423,11 +1680,12 @@ def mount_stop(
                     ps_command
                 ], capture_output=True)
         
-        if res.returncode == 0 or "successfully" in res.stdout or "successfully" in res.stderr:
+        if not is_windows_drive_mapped(target_str) or (res is not None and (res.returncode == 0 or (res.stdout and "successfully" in res.stdout) or (res.stderr and "successfully" in res.stderr))):
             print("[bold green][OK] Mount stopped successfully.[/bold green]")
+            print("[cyan]Note: If mounting again, please allow a few seconds for the Windows NFS network cache to clear.[/cyan]")
         else:
             print("[yellow]Attempted to unmount. Please verify drive status manually using 'umount'.[/yellow]")
-            if res.stderr:
+            if res is not None and res.stderr:
                 print(f"[red]{res.stderr.strip()}[/red]")
         return
 
@@ -1454,7 +1712,7 @@ def mount_stop(
     
     import subprocess
     try:
-        result = subprocess.run(args, capture_output=True, text=True)
+        result = subprocess.run(args, capture_output=True, text=True, errors="replace")
         if result.returncode == 0:
             print(f"[bold green][OK] Mount stopped successfully.[/bold green]")
         else:
@@ -1484,13 +1742,17 @@ def mount_status(
         print("[bold cyan]Active NFS mounts on Windows:[/bold cyan]")
         try:
             # Windows 'mount' command lists active NFS mounts
-            result = subprocess.run(["mount"], capture_output=True, text=True)
-            if result.returncode == 0 and result.stdout.strip():
+            result = None
+            if is_nfs_daemon_running():
+                try:
+                    result = subprocess.run(["mount"], capture_output=True, text=True, errors="replace", timeout=5)
+                except subprocess.TimeoutExpired:
+                    cleanup_stuck_windows_mount_processes()
+            if result is not None and result.returncode == 0 and result.stdout and result.stdout.strip():
                 print(result.stdout)
             else:
                 # Check tasklist for active daemon process
-                task_res = subprocess.run(["tasklist", "/FI", "IMAGENAME eq hf-mount-nfs.exe"], capture_output=True, text=True)
-                if "hf-mount-nfs.exe" in task_res.stdout:
+                if is_nfs_daemon_running():
                     print("[cyan]hf-mount-nfs.exe process is running in the background.[/cyan]")
                 else:
                     print("[cyan]No running mounts found.[/cyan]")
@@ -1522,8 +1784,8 @@ def mount_status(
 
     # 1. Query hf-mount status directly
     try:
-        result = subprocess.run(args, capture_output=True, text=True)
-        if result.returncode == 0:
+        result = subprocess.run(args, capture_output=True, text=True, errors="replace")
+        if result.returncode == 0 and result.stdout:
             stdout = result.stdout.strip()
             if stdout:
                 for line in stdout.splitlines():

{accli-2.0.12 → accli-2.0.14}/accli/gui.py

@@ -150,6 +150,7 @@ class AccliGuiApp(tk.Tk):
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
                     text=True,
+                    errors="replace",
                     bufsize=1,
                     startupinfo=startupinfo
                 )
@@ -449,6 +450,9 @@ class AccliGuiApp(tk.Tk):
         
         mount_btn = ttk.Button(btn_frame, text="Start Mount", command=self.action_mount_start)
         mount_btn.pack(side=tk.LEFT, padx=5)
+
+        stop_form_btn = ttk.Button(btn_frame, text="Stop Mount", style="Stop.TButton", command=self.action_mount_stop)
+        stop_form_btn.pack(side=tk.LEFT, padx=5)
         
         # Right Panel (List of mounts)
         list_frame = ttk.LabelFrame(mounts_tab, text="Active Mounts", padding=10)
@@ -528,20 +532,33 @@ class AccliGuiApp(tk.Tk):
         self.run_cli_async(args, on_done=on_done, elevate=True)
 
     def action_mount_stop(self):
-        selected = self.mounts_list.get(tk.ACTIVE)
-        if not selected:
-            messagebox.showwarning("Warning", "Please select a mount to stop from the list.")
-            return
-            
-        # Parse the mount point from the status line
-        # E.g. "W:       \\\\127.0.0.1\\!" -> "W:"
-        # Or parse folder paths
-        parts = selected.split()
-        if not parts:
-            return
-            
-        mount_point = parts[0]
-        args = ["mount", "stop", mount_point]
+        # 1. Check if user selected an item from the active mounts list
+        mount_point = ""
+        selected_indices = self.mounts_list.curselection()
+        if selected_indices:
+            selected = self.mounts_list.get(selected_indices[0])
+            parts = selected.split()
+            if parts:
+                mount_point = parts[0]
+        elif self.mounts_list.size() > 0:
+            selected = self.mounts_list.get(tk.ACTIVE)
+            if selected:
+                parts = selected.split()
+                if parts:
+                    mount_point = parts[0]
+
+        # 2. If nothing selected or list is empty, fallback to the Mount Point / Drive Letter input field
+        if not mount_point:
+            entered_point = self.mount_point_val.get().strip()
+            if entered_point:
+                mount_point = entered_point
+
+        args = ["mount", "stop"]
+        if mount_point:
+            args.append(mount_point)
+            target_desc = mount_point
+        else:
+            target_desc = "active mount"
         
         self.status_bar_val.set("Stopping mount...")
         self.notebook.select(self.log_tab_idx)
@@ -550,10 +567,10 @@ class AccliGuiApp(tk.Tk):
             self.refresh_mounts_list()
             if code == 0:
                 self.status_bar_val.set("Mount Stopped")
-                messagebox.showinfo("Success", f"Successfully stopped mount at {mount_point}.")
+                messagebox.showinfo("Success", f"Successfully stopped mount ({target_desc}).")
             else:
                 self.status_bar_val.set("Stop Failed")
-                messagebox.showerror("Error", f"Failed to stop mount at {mount_point}.")
+                messagebox.showerror("Error", f"Failed to stop mount ({target_desc}). See logs for details.")
 
         self.run_cli_async(args, on_done=on_done, elevate=True)
 

{accli-2.0.12 → accli-2.0.14}/accli/nfs_watcher.py

@@ -17,7 +17,7 @@ WAIT_TIMEOUT = 0x00000102
 ERROR_ALREADY_EXISTS = 183
 EVENT_MODIFY_STATE = 0x0002
 
-kernel32 = ctypes.windll.kernel32
+kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
 
 
 def log(msg: str):
@@ -52,6 +52,7 @@ def get_open_explorer_paths(script_path: Path, mount_point: str) -> list[str]:
             ],
             capture_output=True,
             text=True,
+            errors="replace",
             creationflags=0x08000000  # CREATE_NO_WINDOW
         )
         if res.returncode == 0 and res.stdout:
@@ -83,6 +84,7 @@ def notify_explorer(script_path: Path, mount_point: str, path_str: str):
                 ],
                 capture_output=True,
                 text=True,
+                errors="replace",
                 creationflags=0x08000000  # CREATE_NO_WINDOW
             )
         except Exception as e:
@@ -142,8 +144,9 @@ def run_watcher(mount_point: str, interval: float):
     mutex_name, stop_event_name = get_sync_names(mount_point)
 
     # 1. Single-instance protection via Named Mutex
+    ctypes.set_last_error(0)
     mutex = kernel32.CreateMutexW(None, True, mutex_name)
-    if kernel32.GetLastError() == ERROR_ALREADY_EXISTS:
+    if ctypes.get_last_error() == ERROR_ALREADY_EXISTS:
         log(f"Another watcher is already active for {drive_root} ({mutex_name}). Exiting.")
         sys.exit(0)
 

{accli-2.0.12 → accli-2.0.14}/accli/setup-nfs.ps1

@@ -159,9 +159,21 @@ try {
     }
 } catch {}
 
+# Force unified, accessible cache under C:\ProgramData\accli\cache
+$cacheDir = "C:\ProgramData\accli\cache"
+$hfXetCache = "$cacheDir\xet"
+$hfMountCache = "$cacheDir\hf-mount"
+New-Item -ItemType Directory -Path $hfXetCache -Force | Out-Null
+New-Item -ItemType Directory -Path $hfMountCache -Force | Out-Null
+
+$env:HF_XET_CACHE = $hfXetCache
+$env:HF_HOME = $cacheDir
+$env:XDG_CACHE_HOME = $cacheDir
+
 $mountArgs = @(
     "--token-file", $cfg.db_path,
     "--hub-endpoint", $cfg.server_url,
+    "--cache-dir", $hfMountCache,
     $cfg.mode,
     $cfg.project_slug,
     $cfg.mount_point
@@ -224,7 +236,7 @@ if (Get-Command Register-ScheduledTask -ErrorAction SilentlyContinue) {
     try {
         $action = New-ScheduledTaskAction -Execute $psActionPath -Argument $psActionArgs
         $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -RunLevel Highest
-        $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
+        $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -ExecutionTimeLimit ([TimeSpan]::Zero)
         
         # Unregister task if already exists to update
         Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
@@ -253,7 +265,7 @@ if (Get-Command Register-ScheduledTask -ErrorAction SilentlyContinue) {
     try {
         $action = New-ScheduledTaskAction -Execute $psActionPath -Argument $psUmountActionArgs
         $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -RunLevel Highest
-        $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
+        $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -ExecutionTimeLimit ([TimeSpan]::Zero)
         
         Unregister-ScheduledTask -TaskName $umountTaskName -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
         Register-ScheduledTask -TaskName $umountTaskName -Action $action -Principal $principal -Settings $settings -Force | Out-Null

{accli-2.0.12 → accli-2.0.14}/accli/token.py

@@ -139,6 +139,13 @@ def exchange_refresh_token(project_slug: str) -> tuple[str, str, int]:
             _cached_expires_at = expires_at
 
             return cas_token, access_token, expires_at
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None and e.response.status_code in (400, 401, 403):
+                print("[bold red]ERROR: Your session has expired or the token is invalid.[/bold red]")
+                print("[yellow]Please run 'accli login' to authenticate again.[/yellow]")
+                raise typer.Exit(1)
+            print(f"[bold red]ERROR: Failed to authenticate/exchange refresh token: {e}[/bold red]")
+            raise typer.Exit(1)
         except Exception as e:
             print(f"[bold red]ERROR: Failed to authenticate/exchange refresh token: {e}[/bold red]")
             raise typer.Exit(1)

{accli-2.0.12 → accli-2.0.14}/accli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: accli
-Version: 2.0.12
+Version: 2.0.14
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-2.0.12 → accli-2.0.14}/accli.egg-info/SOURCES.txt

@@ -22,4 +22,5 @@ accli.egg-info/SOURCES.txt
 accli.egg-info/dependency_links.txt
 accli.egg-info/entry_points.txt
 accli.egg-info/requires.txt
-accli.egg-info/top_level.txt
+accli.egg-info/top_level.txt
+tests/test_mount_lifecycle.py

accli-2.0.14/tests/test_mount_lifecycle.py

@@ -0,0 +1,139 @@
+"""
+Test for mounting lifecycle:
+1. Ensure clean initial state (mount stop)
+2. accli mount start <project_slug>
+3. Capture directory structure (ensure not empty) & read any file
+4. accli mount stop
+5. accli mount start <project_slug> again
+6. Capture directory structure (must not be empty) & read any file
+"""
+import os
+import sys
+import time
+import platform
+import subprocess
+import unittest
+from pathlib import Path
+
+
+class TestMountLifecycle(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.project_slug = os.environ.get("ACCLI_TEST_PROJECT", "test")
+        cls.mount_drive = os.environ.get("ACCLI_TEST_DRIVE", "W:")
+        cls.mount_arg = cls.mount_drive.rstrip("\\/")
+        if platform.system() == "Windows" and len(cls.mount_arg) == 1 and cls.mount_arg.isalpha():
+            cls.mount_arg += ":"
+        cls.mount_path = Path(f"{cls.mount_arg}\\") if platform.system() == "Windows" else Path.home() / "accelerator" / "mnt" / cls.project_slug
+
+    def run_cli(self, args: list, timeout: int):
+        cmd = [sys.executable, "-m", "accli"] + args
+        print(f"\n[TEST EXEC] {' '.join(cmd)}")
+        result = subprocess.run(
+            cmd,
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            errors="replace",
+            timeout=timeout
+        )
+        print(f"[EXIT CODE] {result.returncode}")
+        if result.stdout:
+            print(f"[STDOUT]\n{result.stdout.strip()}")
+        if result.stderr:
+            print(f"[STDERR]\n{result.stderr.strip()}")
+        return result
+
+    def get_structure(self, target_path: Path):
+        """List files and folders at the mounted location and ensure it is not empty."""
+        entries = []
+        for _ in range(10):
+            try:
+                if target_path.exists():
+                    entries = [p.name for p in target_path.iterdir()]
+                    if entries:
+                        break
+            except OSError:
+                pass
+            time.sleep(0.5)
+
+        self.assertTrue(
+            target_path.exists(),
+            f"Mount path '{target_path}' does not exist or drive is not mapped."
+        )
+        print(f"[MOUNT STRUCTURE at {target_path}]: {entries}")
+        return entries
+
+    def read_any_file(self, target_path: Path):
+        """Locate any readable file under the mounted project path and read its content."""
+        search_dirs = [target_path / self.project_slug, target_path]
+        for sdir in search_dirs:
+            if not sdir.exists():
+                continue
+            for root, dirs, files in os.walk(sdir):
+                for file_name in files:
+                    candidate = Path(root) / file_name
+                    try:
+                        print(f"[TEST READ FILE] Attempting to read: {candidate}")
+                        with open(candidate, "rb") as fp:
+                            data = fp.read(1024)
+                        print(f"[TEST READ FILE] Successfully read {len(data)} bytes from '{candidate.name}'.")
+                        return candidate, len(data)
+                    except Exception as e:
+                        print(f"[TEST READ FILE] Skipping unreadable candidate {candidate}: {e}")
+
+        self.fail(f"Could not read any file under mounted path '{target_path}'.")
+
+    def tearDown(self):
+        try:
+            self.run_cli(["mount", "stop", self.mount_arg], 30)
+        except Exception:
+            pass
+
+    def test_mount_start_stop_start_lifecycle(self):
+        # Step 0: Ensure clean initial state
+        print("\n--- Step 0: Ensure clean initial state ---")
+        self.run_cli(["mount", "stop", self.mount_arg], 30)
+        time.sleep(2)
+
+        # Step 1: First mount start
+        print(f"\n--- Step 1: Starting mount for '{self.project_slug}' at '{self.mount_arg}' ---")
+        start1 = self.run_cli(["mount", "start", self.project_slug, self.mount_arg], 60)
+        self.assertEqual(start1.returncode, 0, f"First mount start failed: {start1.stderr or start1.stdout}")
+
+        # Step 2: Check structure is not empty and try to read any file
+        print("\n--- Step 2: Checking structure & reading file after 1st mount ---")
+        structure1 = self.get_structure(self.mount_path)
+        self.assertGreater(
+            len(structure1), 0,
+            f"Expected structure to be not empty after 1st mount, got: {structure1}"
+        )
+        file1, bytes_read1 = self.read_any_file(self.mount_path)
+        self.assertIsNotNone(file1)
+
+        # Step 3: Mount stop (clean)
+        print(f"\n--- Step 3: Stopping mount at '{self.mount_arg}' ---")
+        stop = self.run_cli(["mount", "stop", self.mount_arg], 30)
+        self.assertEqual(stop.returncode, 0, f"Mount stop failed: {stop.stderr or stop.stdout}")
+        time.sleep(3)
+
+        # Step 4: Second mount start
+        print(f"\n--- Step 4: Starting mount for '{self.project_slug}' at '{self.mount_arg}' (2nd time) ---")
+        start2 = self.run_cli(["mount", "start", self.project_slug, self.mount_arg], 120)
+        self.assertEqual(start2.returncode, 0, f"Second mount start failed: {start2.stderr or start2.stdout}")
+
+        # Step 5: Check structure is not empty and try to read any file after clean & remount
+        print("\n--- Step 5: Checking structure & reading file after 2nd mount ---")
+        structure2 = self.get_structure(self.mount_path)
+        self.assertGreater(
+            len(structure2), 0,
+            f"Expected structure to be not empty after 2nd mount, got: {structure2}"
+        )
+        file2, bytes_read2 = self.read_any_file(self.mount_path)
+        self.assertIsNotNone(file2)
+
+        print("\n[SUCCESS] Mount start -> read -> stop -> start -> read lifecycle test completed successfully!")
+
+
+if __name__ == "__main__":
+    unittest.main()