swm-gpu 0.1.0__py3-none-any.whl

swm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

swm/bootstrap.py

@@ -0,0 +1,337 @@
+"""Bootstrap scripts for setting up remote GPU instances with storage and tools."""
+
+from __future__ import annotations
+
+import subprocess
+
+from rich.console import Console
+
+from swm.remote.ssh import RemoteSession
+
+console = Console()
+
+S5CMD_VERSION = "2.3.0"
+S5CMD_URL = (
+    f"https://github.com/peak/s5cmd/releases/download/v{S5CMD_VERSION}/"
+    f"s5cmd_{S5CMD_VERSION}_Linux-64bit.tar.gz"
+)
+
+SAFETY_MARGIN = 0.90
+
+
+def _humanize(n: int | float) -> str:
+    v = float(n)
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if v < 1024:
+            return f"{v:.1f} {unit}" if unit != "B" else f"{int(v)} B"
+        v /= 1024
+    return f"{v:.1f} PB"
+
+
+def _step(session: RemoteSession, label: str, command: str) -> tuple[int, str, str]:
+    """Run a labelled step on the remote, streaming output to the terminal."""
+    console.print(f"\n[bold cyan]▸ {label}[/bold cyan]")
+    code, stdout, stderr = session.exec(command, stream=True)
+    if code != 0:
+        raise RuntimeError(f"Step failed (exit {code}): {label}")
+    return code, stdout, stderr
+
+
+# ── s5cmd ──────────────────────────────────────────────────────────
+
+
+def _s3_env(storage_slug: str) -> str:
+    """Build env-var prefix for s5cmd from swm config."""
+    from swm import config as cfg
+
+    if storage_slug == "b2":
+        endpoint = cfg.get("b2.s3_endpoint") or ""
+        ak = cfg.get("b2.key_id") or ""
+        sk = cfg.get("b2.app_key") or ""
+    elif storage_slug == "gcs":
+        endpoint = "https://storage.googleapis.com"
+        ak = cfg.get("gcs.hmac_access") or ""
+        sk = cfg.get("gcs.hmac_secret") or ""
+    elif storage_slug == "s3":
+        endpoint = ""
+        ak = cfg.get("s3.access_key") or ""
+        sk = cfg.get("s3.secret_key") or ""
+    else:
+        raise ValueError(f"Unknown storage slug: {storage_slug}")
+
+    parts = [
+        f"AWS_ACCESS_KEY_ID='{ak}'",
+        f"AWS_SECRET_ACCESS_KEY='{sk}'",
+    ]
+    if endpoint:
+        parts.append(f"S3_ENDPOINT_URL='{endpoint}'")
+    return " ".join(parts)
+
+
+def install_s5cmd(session: RemoteSession) -> None:
+    """Download the s5cmd static binary if not already present."""
+    _step(
+        session,
+        "Installing s5cmd",
+        f"command -v s5cmd >/dev/null 2>&1 && echo 's5cmd already installed' || "
+        f"(curl -sL '{S5CMD_URL}' | tar xz -C /usr/local/bin s5cmd "
+        f"&& chmod +x /usr/local/bin/s5cmd && s5cmd version)",
+    )
+
+
+def _install_inotify(session: RemoteSession) -> None:
+    """Install inotify-tools for the filesystem change watcher."""
+    _step(
+        session,
+        "Installing inotify-tools",
+        "command -v inotifywait >/dev/null 2>&1 && echo 'inotify-tools already installed' || "
+        "(apt-get update -qq && apt-get install -y -qq inotify-tools && echo 'installed')",
+    )
+
+
+def configure_storage(
+    session: RemoteSession, storage_slug: str, bucket: str = "",
+) -> None:
+    """Install s5cmd, inotify-tools, and verify the S3-compatible connection."""
+    install_s5cmd(session)
+    try:
+        _install_inotify(session)
+    except RuntimeError:
+        console.print("  [yellow]⚠ inotify-tools install failed — push will use find[/yellow]")
+    env = _s3_env(storage_slug)
+    target = f"s3://{bucket}/" if bucket else ""
+    _step(
+        session,
+        f"Verifying {storage_slug} connection",
+        f"{env} s5cmd ls {target} 2>&1 | head -5 || true",
+    )
+
+
+_WS_MARKER_NAMES = (
+    ".swm_last_push",
+    ".swm_changes.log",
+    ".swm_workspace.tar.gz",
+    ".swm_autosync.log",
+)
+
+
+def _ensure_workspace_empty_on_pod(session: RemoteSession) -> None:
+    """Raise if /workspace/ on the pod contains any non-marker files.
+
+    A blind ``touch PUSH_STAMP`` on a non-empty workspace would silently
+    declare those files synced even though they were never uploaded.
+    Refuse, and tell the user to push or clear before attaching.
+    """
+    excludes = " ".join(f"-not -name '{n}'" for n in _WS_MARKER_NAMES)
+    cmd = (
+        f"find /workspace -mindepth 1 -maxdepth 1 {excludes} 2>/dev/null "
+        "| head -5"
+    )
+    _, out, _ = session.exec(cmd, stream=False)
+    leftover = [line for line in out.splitlines() if line.strip()]
+    if leftover:
+        sample = ", ".join(p.rsplit("/", 1)[-1] for p in leftover[:3])
+        more = "" if len(leftover) <= 3 else f" (+ more)"
+        raise RuntimeError(
+            f"/workspace/ on the pod is not empty (e.g. {sample}{more}). "
+            "Refusing to mark this as a fresh workspace because those "
+            "files would not be uploaded. Either: (a) upload them first "
+            "with `swm sync push <pod> -b <provider:bucket> -d <name> "
+            "--force`, then re-run setup; or (b) clear /workspace/ on "
+            "the pod and re-run; or (c) re-run with an existing "
+            "workspace name to pull from storage."
+        )
+
+
+def bootstrap_workspace_on_pod(
+    session: RemoteSession,
+    storage_slug: str,
+    bucket: str,
+    workspace: str,
+    *,
+    qualified_id: str,
+    is_new: bool,
+    extra_excludes: list[str] | None = None,
+    autosync_interval: int = 60,
+    console_obj: Console | None = None,
+) -> list[tuple[str, str]]:
+    """Configure storage, pull (or watcher-init), and start auto-sync on a pod.
+
+    Returns a list of ``(label, recovery_command)`` tuples for any sub-step
+    that failed. An empty list means full success.
+    """
+    from swm.sync import start_watcher
+    from swm.sync.autosync import AutosyncUnsafeError, start_autosync
+    from swm.sync.paths import PUSH_STAMP, WATCH_LOG
+    from swm.sync.pull import workspace_pull
+
+    _con = console_obj or console
+    failed: list[tuple[str, str]] = []
+
+    def _fail(label: str, cmd: str, exc: Exception) -> None:
+        _con.print(f"  [yellow]⚠ {label} failed: {exc}[/yellow]")
+        failed.append((label, cmd))
+
+    storage_ok = False
+    try:
+        with _con.status(
+            "Installing s5cmd & configuring storage…", spinner="dots"
+        ):
+            configure_storage(session, storage_slug, bucket=bucket)
+        _con.print("[green]✓[/green] Storage configured")
+        storage_ok = True
+    except Exception as exc:
+        _fail("Storage configuration", f"swm setup storage {qualified_id}", exc)
+
+    pull_ok = False
+    if storage_ok:
+        try:
+            if is_new:
+                _ensure_workspace_empty_on_pod(session)
+                _con.print("  [dim]New workspace — skipping pull[/dim]")
+                session.exec(
+                    f": > {WATCH_LOG} 2>/dev/null; touch {PUSH_STAMP}",
+                    stream=False,
+                )
+                if start_watcher(session, "/workspace"):
+                    _con.print("  [dim]Watcher started for change tracking[/dim]")
+            else:
+                workspace_pull(
+                    session, storage_slug, bucket, workspace,
+                    extra_excludes=extra_excludes,
+                )
+            pull_ok = True
+        except Exception as exc:
+            _fail("Workspace pull", f"swm sync pull {qualified_id}", exc)
+    else:
+        failed.append(("Workspace pull", f"swm sync pull {qualified_id}"))
+
+    if pull_ok:
+        try:
+            if start_autosync(
+                session, storage_slug, bucket, workspace,
+                interval=autosync_interval,
+            ):
+                _con.print(
+                    "  [dim]Auto-sync started "
+                    f"(every {autosync_interval}s → "
+                    f"{storage_slug}:{bucket}/{workspace})[/dim]"
+                )
+        except (AutosyncUnsafeError, Exception) as exc:
+            _fail("Auto-sync start", f"swm sync auto {qualified_id}", exc)
+    else:
+        failed.append(("Auto-sync start", f"swm sync auto {qualified_id}"))
+
+    return failed
+
+
+_LOCK_FILE = "/tmp/.swm_transfer.lock"
+
+
+def _acquire_transfer_lock(session: RemoteSession, force: bool = False) -> None:
+    """Check for an existing transfer and acquire the lock.
+
+    If a lock exists with a live PID, raises unless *force* is True
+    (which kills the stale process first).
+    """
+    code, out, _ = session.exec(
+        f"cat {_LOCK_FILE} 2>/dev/null", stream=False,
+    )
+    old_pid = out.strip()
+
+    if old_pid:
+        _, alive, _ = session.exec(
+            f"kill -0 {old_pid} 2>/dev/null && echo alive || echo dead",
+            stream=False,
+        )
+        if "alive" in alive:
+            if not force:
+                raise RuntimeError(
+                    f"A transfer is already running (PID {old_pid}). "
+                    "Use --force to kill it and start a new one."
+                )
+            console.print(
+                f"  [yellow]⚠ Killing existing transfer (PID {old_pid})[/yellow]"
+            )
+            session.exec(f"kill -9 {old_pid} 2>/dev/null; sleep 1", stream=False)
+
+        # Stale lock — clean up temp files left behind
+        console.print("  [dim]Cleaning up stale temp files…[/dim]")
+        _, cleanup_out, _ = session.exec(
+            "find /workspace -maxdepth 5 -type f -regex '.*\\.[a-z]*[0-9]\\{9,\\}$' "
+            "-delete -print 2>/dev/null | wc -l",
+            stream=False,
+        )
+        n = cleanup_out.strip()
+        if n and n != "0":
+            console.print(f"  [dim]Removed {n} orphaned temp files[/dim]")
+
+    session.exec(f"echo $$ > {_LOCK_FILE}", stream=False)
+
+
+def _s5cmd_transfer(
+    session: RemoteSession,
+    label: str,
+    s5cmd_cmd: str,
+    force: bool = False,
+    total_bytes: int = 0,
+    total_files: int = 0,
+) -> int:
+    """Run an s5cmd transfer with output streamed directly to the terminal.
+
+    Acquires a lock file on the pod, wraps the command in a shell trap
+    for guaranteed cleanup (even on SSH disconnect), and streams
+    s5cmd's native ``--show-progress`` output to the terminal.
+
+    Returns the process exit code.
+    """
+    console.print(f"\n[bold cyan]▸ {label}[/bold cyan]")
+    _acquire_transfer_lock(session, force=force)
+
+    wrapped = (
+        f"trap 'rm -f {_LOCK_FILE}' EXIT; "
+        f"echo $$ > {_LOCK_FILE}; "
+        f"{s5cmd_cmd}"
+    )
+    cmd = session._ssh_cmd() + [wrapped]
+    code = subprocess.call(cmd)
+
+    if code != 0:
+        console.print(f"  [yellow]⚠ Transfer finished with warnings (exit {code})[/yellow]")
+    else:
+        console.print(f"  [green]✓ {label} — done[/green]")
+
+    return code
+
+
+# Backward-compatible re-exports (lazy to avoid circular imports)
+_RE_EXPORTS: dict[str, str] = {
+    "install_framework": "swm.bootstrap_frameworks",
+    "start_framework": "swm.bootstrap_frameworks",
+    "stop_framework": "swm.bootstrap_frameworks",
+    "install_comfyui": "swm.bootstrap_frameworks",
+    "install_swarmui": "swm.bootstrap_frameworks",
+    "link_models_to_comfyui": "swm.bootstrap_frameworks",
+    "wait_for_ssh": "swm.bootstrap_ssh",
+    "next_workspace_name": "swm.bootstrap_ssh",
+    "DiskCheck": "swm.sync",
+    "preflight_check": "swm.sync",
+    "start_watcher": "swm.sync",
+    "stop_watcher": "swm.sync",
+    "is_watcher_alive": "swm.sync",
+    "workspace_pull": "swm.sync",
+    "workspace_push": "swm.sync",
+    "tar_pull": "swm.sync",
+    "start_autosync": "swm.sync",
+    "stop_autosync": "swm.sync",
+    "is_autosync_alive": "swm.sync",
+    "autosync_status": "swm.sync",
+}
+
+
+def __getattr__(name: str):  # noqa: E302
+    if name in _RE_EXPORTS:
+        import importlib
+        mod = importlib.import_module(_RE_EXPORTS[name])
+        return getattr(mod, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

swm/bootstrap_frameworks.py

@@ -0,0 +1,177 @@
+"""Framework lifecycle management for remote GPU instances."""
+
+from __future__ import annotations
+
+import time
+
+from rich.console import Console
+
+from swm.bootstrap import _step
+from swm.remote.ssh import RemoteSession
+
+console = Console()
+
+
+def install_framework(
+    session: RemoteSession,
+    name: str,
+    console: Console | None = None,
+) -> None:
+    """Install a framework by name using its declarative step list."""
+    from swm.frameworks import Framework, get_framework
+
+    _con = console or globals()["console"]
+
+    fw: Framework = get_framework(name)
+    _con.print(f"\n[bold]Installing {fw.label}[/bold]")
+    env_prefix = f"{fw.env_setup} && " if fw.env_setup else ""
+
+    total = len(fw.steps) + len(fw.post_install)
+    for idx, step in enumerate(fw.steps, 1):
+        workdir = step.workdir or fw.install_dir
+        if step.check:
+            cmd = f"{step.check} && echo '{step.label}: already done' || ({env_prefix}cd {workdir} && {step.command})"
+        else:
+            cmd = f"{env_prefix}cd {workdir} && {step.command}"
+        _step(session, f"[{idx}/{total}] {step.label}", cmd)
+
+    for idx, step in enumerate(fw.post_install, len(fw.steps) + 1):
+        workdir = step.workdir or fw.install_dir
+        if step.check:
+            cmd = f"{step.check} && echo '{step.label}: already done' || ({env_prefix}cd {workdir} && {step.command})"
+        else:
+            cmd = f"{env_prefix}cd {workdir} && {step.command}"
+        _step(session, f"[{idx}/{total}] {step.label}", cmd)
+
+
+def start_framework(
+    session: RemoteSession,
+    name: str,
+    port: int | None = None,
+    console: Console | None = None,
+    qualified_id: str | None = None,
+) -> str | None:
+    """Launch a framework in the background. Returns proxy URL if applicable."""
+    from swm.frameworks import get_framework
+
+    _con = console or globals()["console"]
+    fw = get_framework(name)
+
+    if fw.process_pattern:
+        with _con.status(f"Checking if {fw.label} is running…", spinner="dots"):
+            _, out, _ = session.exec(
+                f"pgrep -fa '{fw.process_pattern}' | grep -v grep || true",
+                stream=False,
+            )
+        if out.strip():
+            pid = out.strip().split("\n")[0].split()[0]
+            _con.print(
+                f"  [yellow]{fw.label} is already running (PID {pid})[/yellow]"
+            )
+            return None
+
+    env_prefix = f"{fw.env_setup} && " if fw.env_setup else ""
+    if fw.pre_start:
+        _con.print(f"\n[bold cyan]▸ Preparing {fw.label}[/bold cyan]")
+        for step in fw.pre_start:
+            workdir = step.workdir or fw.install_dir
+            if step.check:
+                cmd = (
+                    f"{step.check} && echo '{step.label}: already done' "
+                    f"|| ({env_prefix}cd {workdir} && {step.command})"
+                )
+            else:
+                cmd = f"{env_prefix}cd {workdir} && {step.command}"
+            _step(session, step.label, cmd)
+
+    _con.print(f"\n[bold cyan]▸ Starting {fw.label}[/bold cyan]")
+    launch = fw.launch_cmd
+    if port and fw.ports:
+        default_port = str(next(iter(fw.ports)))
+        launch = launch.replace(default_port, str(port))
+
+    logfile = f"/tmp/{fw.name}.log"
+
+    with _con.status(f"Launching {fw.label}…", spinner="dots"):
+        session.exec_background(
+            launch,
+            logfile=logfile,
+            workdir=fw.launch_workdir,
+            env_setup=fw.env_setup,
+        )
+
+    max_checks = 5
+    alive = False
+    for i in range(max_checks):
+        time.sleep(3 if i == 0 else 2)
+        if not fw.process_pattern:
+            alive = True
+            break
+        _, out, _ = session.exec(
+            f"pgrep -fa '{fw.process_pattern}' | grep -v grep || true",
+            stream=False,
+        )
+        if out.strip():
+            alive = True
+            break
+
+    if alive:
+        _con.print(f"  [green]✓ {fw.label} started[/green]")
+        _pod_ref = qualified_id or "<pod>"
+        _con.print(f"  Logs: swm run {_pod_ref} 'tail -f {logfile}'")
+    else:
+        _con.print(f"  [red]✗ {fw.label} failed to start[/red]")
+        _con.print(f"  Last lines from {logfile}:")
+        _, tail, _ = session.exec(f"tail -15 {logfile} 2>/dev/null", stream=False)
+        if tail.strip():
+            for line in tail.strip().splitlines():
+                _con.print(f"    [dim]{line}[/dim]")
+        raise RuntimeError(f"{fw.label} exited immediately — check logs above")
+
+    return None
+
+
+def stop_framework(
+    session: RemoteSession,
+    name: str,
+    console: Console | None = None,
+) -> None:
+    """Stop a running framework."""
+    from swm.frameworks import get_framework
+
+    _con = console or globals()["console"]
+    fw = get_framework(name)
+    if not fw.stop_cmd:
+        _con.print(f"  [yellow]{fw.label} has no stop command defined[/yellow]")
+        return
+
+    _con.print(f"\n[bold cyan]▸ Stopping {fw.label}[/bold cyan]")
+    with _con.status(f"Stopping {fw.label}…", spinner="dots"):
+        session.exec(fw.stop_cmd, stream=False)
+    _con.print(f"  [green]✓ {fw.label} stopped[/green]")
+
+
+def install_comfyui(session: RemoteSession) -> None:
+    """Backward-compatible wrapper."""
+    install_framework(session, "comfyui")
+
+
+def install_swarmui(session: RemoteSession) -> None:
+    """Backward-compatible wrapper."""
+    install_framework(session, "swarmui")
+
+
+# ── symlinks ────────────────────────────────────────────────────────
+
+
+def link_models_to_comfyui(session: RemoteSession) -> None:
+    """Symlink /workspace/models into ComfyUI's model directory."""
+    dirs = ["checkpoints", "loras", "vae", "controlnet", "clip", "upscale_models", "unet"]
+    cmds = ["mkdir -p /workspace/models/{" + ",".join(dirs) + "}"]
+    for d in dirs:
+        cmds.append(
+            f"[ -L /workspace/ComfyUI/models/{d} ] || "
+            f"(rm -rf /workspace/ComfyUI/models/{d} "
+            f"&& ln -s /workspace/models/{d} /workspace/ComfyUI/models/{d})"
+        )
+    _step(session, "Symlinking models → ComfyUI", " && ".join(cmds))

swm/bootstrap_ssh.py

@@ -0,0 +1,175 @@
+"""SSH waiting utilities for remote GPU instances."""
+
+from __future__ import annotations
+
+import subprocess
+import time
+
+from rich.console import Console
+
+from swm.providers.base import Instance, InstanceStatus
+
+console = Console()
+
+
+def _has_direct_ssh(inst: Instance) -> bool:
+    """True when the instance exposes a public IP with a mapped SSH port."""
+    return bool(inst.ip_address and inst.ports.get(22))
+
+
+def _has_relay_ssh(inst: Instance) -> bool:
+    return bool(inst.ssh_host and inst.ssh_port)
+
+
+def _is_relay_host(inst: Instance) -> bool:
+    """True when ssh_host is a relay proxy, not the instance's own IP."""
+    if not inst.ssh_host:
+        return False
+    return inst.ssh_host != inst.ip_address
+
+
+def wait_for_ssh(
+    provider,
+    instance_id: str,
+    timeout: int = 600,
+    poll_interval: int = 10,
+    direct_grace: int = 30,
+) -> Instance:
+    """Poll until the instance is running and SSH is reachable.
+
+    Prefers a direct IP+port connection over the provider relay.  When
+    only a relay endpoint is found, keeps polling up to *direct_grace*
+    extra seconds for a direct endpoint before falling back.  Providers
+    that expose SSH directly (Lambda, GCP, AWS) skip the grace window.
+
+    Displays ``status_detail`` from the provider (e.g. Vast.ai Docker
+    build progress) when available to give the user visibility into
+    what the remote host is doing.
+    """
+    from swm import config as _cfg
+
+    start = time.time()
+    last_status = ""
+    last_detail = ""
+    inst = None
+    relay_seen_at: float | None = None
+
+    def _elapsed() -> str:
+        return f"{int(time.time() - start)}s"
+
+    # Phase 1: wait for the instance to be RUNNING with an SSH endpoint.
+    while time.time() - start < timeout:
+        try:
+            if hasattr(provider, "get_instance"):
+                inst = provider.get_instance(instance_id)
+            else:
+                instances = provider.list_instances()
+                inst = next((i for i in instances if i.id == instance_id), None)
+
+            if inst:
+                status = inst.status.value
+                if status != last_status:
+                    console.print(f"  Status: [bold]{status}[/bold]  ({_elapsed()})")
+                    last_status = status
+
+                detail = inst.status_detail or ""
+                if detail and detail != last_detail:
+                    truncated = (detail[:100] + "…") if len(detail) > 100 else detail
+                    console.print(f"  [dim]{truncated}[/dim]")
+                    last_detail = detail
+
+                if inst.status == InstanceStatus.RUNNING:
+                    if _has_direct_ssh(inst):
+                        break
+                    if _has_relay_ssh(inst):
+                        if not _is_relay_host(inst):
+                            break
+                        if relay_seen_at is None:
+                            relay_seen_at = time.time()
+                        elif time.time() - relay_seen_at >= direct_grace:
+                            break
+        except Exception:
+            pass
+
+        time.sleep(poll_interval)
+    else:
+        raise TimeoutError(
+            f"Pod not running after {timeout}s for instance {instance_id}. "
+            f"Last status: {last_status}"
+        )
+
+    # Phase 2: pick the best SSH path — direct mapped port always wins.
+    if _has_direct_ssh(inst):
+        ssh_target = inst.ip_address
+        port = inst.ports[22]
+        ssh_user = "root"
+        console.print(f"  Direct SSH: {ssh_target}:{port}")
+    elif _has_relay_ssh(inst):
+        ssh_target = inst.ssh_host
+        port = inst.ssh_port
+        ssh_user = inst.ssh_user or "root"
+        if _is_relay_host(inst):
+            console.print(f"  Relay SSH: {ssh_user}@{ssh_target}:{port}")
+        else:
+            console.print(f"  SSH: {ssh_user}@{ssh_target}:{port}")
+    else:
+        console.print("  [yellow]No SSH endpoint found — returning anyway[/yellow]")
+        return inst
+
+    # Phase 3: probe until SSH actually responds.
+    console.print(f"  Probing SSH…  ({_elapsed()})")
+    probe = [
+        "ssh",
+        "-o", "StrictHostKeyChecking=no",
+        "-o", "UserKnownHostsFile=/dev/null",
+        "-o", "ConnectTimeout=5",
+        "-o", "LogLevel=ERROR",
+        "-p", str(port),
+    ]
+    key = _cfg.get(f"{provider.slug}.ssh_key") or _cfg.get("ssh.key_path")
+    if key:
+        probe.extend(["-i", str(key)])
+    probe.extend([f"{ssh_user}@{ssh_target}", "echo __SWM_OK__"])
+
+    while time.time() - start < timeout:
+        try:
+            result = subprocess.run(probe, capture_output=True, timeout=15)
+            if b"__SWM_OK__" in result.stdout:
+                console.print(f"  [green]✓ SSH ready[/green]  ({_elapsed()})")
+                return inst
+        except (subprocess.TimeoutExpired, OSError):
+            pass
+        time.sleep(5)
+
+    raise TimeoutError(
+        f"SSH not reachable after {timeout}s for instance {instance_id}"
+    )
+
+
+def next_workspace_name(storage_provider, bucket: str) -> str:
+    """Find the next available workspace name in a bucket.
+
+    Existing: workspace/, workspace2/ → returns "workspace3".
+    Empty bucket → returns "workspace".
+    """
+    try:
+        objects = storage_provider.ls(bucket)
+    except Exception:
+        return "workspace"
+
+    existing: set[int] = set()
+    for obj in objects:
+        name = obj.key.rstrip("/")
+        if name == "workspace":
+            existing.add(1)
+        elif name.startswith("workspace"):
+            suffix = name.removeprefix("workspace")
+            try:
+                existing.add(int(suffix))
+            except ValueError:
+                pass
+
+    if not existing:
+        return "workspace"
+
+    return f"workspace{max(existing) + 1}"