flowmesh-cli-stack 0.1.8rc1__tar.gz → 0.1.10__tar.gz

{flowmesh_cli_stack-0.1.8rc1/src/flowmesh_cli_stack.egg-info → flowmesh_cli_stack-0.1.10}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: flowmesh-cli-stack
-Version: 0.1.8rc1
+Version: 0.1.10
 Summary: FlowMesh CLI stack commands
 License-Expression: Apache-2.0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flowmesh-cli==0.1.8rc1
-Requires-Dist: flowmesh-sdk-stack==0.1.8rc1
+Requires-Dist: flowmesh-cli==0.1.10
+Requires-Dist: flowmesh-sdk-stack==0.1.10
 Requires-Dist: docker>=7.1.0
 Requires-Dist: packaging>=24
 Requires-Dist: pyyaml>=6.0.2

{flowmesh_cli_stack-0.1.8rc1 → flowmesh_cli_stack-0.1.10}/pyproject.toml

@@ -4,15 +4,15 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "flowmesh-cli-stack"
-version = "0.1.8rc1"
+version = "0.1.10"
 description = "FlowMesh CLI stack commands"
 readme = "README.md"
 requires-python = ">=3.12"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
 dependencies = [
-    "flowmesh-cli==0.1.8rc1",
-    "flowmesh-sdk-stack==0.1.8rc1",
+    "flowmesh-cli==0.1.10",
+    "flowmesh-sdk-stack==0.1.10",
     "docker>=7.1.0",
     "packaging>=24",
     "pyyaml>=6.0.2",

{flowmesh_cli_stack-0.1.8rc1 → flowmesh_cli_stack-0.1.10}/src/flowmesh_cli_stack/assets/.env.example

@@ -43,6 +43,10 @@ SUPERVISOR_GRPC_KEEPALIVE_PERMIT_WITHOUT_CALLS=true
 SUPERVISOR_GRPC_MIN_RECV_PING_INTERVAL_MS=60000
 SUPERVISOR_GRPC_KEEPALIVE_TIME_MS=300000
 SUPERVISOR_GRPC_KEEPALIVE_TIMEOUT_MS=10000
+# Shared secret admitting `external` workers (empty disables them).
+EXTERNAL_WORKER_TOKEN=
+# File holding the secret; wins over EXTERNAL_WORKER_TOKEN.
+EXTERNAL_WORKER_TOKEN_FILE=
 
 # ==== Redis Connectivity ====
 REDIS_CONTROL_URL=redis://localhost:6379/0
@@ -85,7 +89,7 @@ SERVER_PORT_FORWARD_PORT_END=32100
 
 # ==== SSH Task Support ====
 ENABLE_SERVER_SSH_PROXY=true
-ENABLE_SERVER_SSH_CONNECTION_AUDIT=true
+ENABLE_SERVER_SSH_CONNECTION_REGISTRY=true
 
 # ==== Serve Task Support ====
 ENABLE_SERVER_SERVE_PROXY=true
@@ -105,7 +109,19 @@ SSH_MAX_PIDS=
 # Whether to apply requested GPU limits to SSH tasks.
 # If false, SSH tasks are allocated all available GPUs
 # regardless of their resource requests.
-ENABLE_SSH_GPU_LIMIT=false
+ENABLE_SSH_GPU_LIMIT=true
+# Whether a non-root worker may serve SSH sessions,
+# which run as the worker and can read its credentials.
+ENABLE_UNISOLATED_SSH_SESSION=false
+# Sandbox an SSH session runs in. 'docker' creates a
+# sibling container; 'process' runs sshd inside the
+# worker. 'auto' picks docker, falling back to process
+# on a root worker.
+SSH_SESSION_BACKEND=auto
+# Address a direct-mode session is advertised at.
+# Defaults to the worker's tailnet address, else its
+# FQDN.
+SSH_DIRECT_HOST=
 
 # ==== General Settings ====
 TZ=Asia/Singapore
@@ -128,6 +144,8 @@ ENABLE_WORKER_WATCHDOG=true
 WORKER_DEATH_CHECK_INTERVAL=30
 WORKER_DEATH_GRACE_SEC=60
 WORKER_REHYDRATION_GRACE_SEC=120
+ENABLE_WORKER_REAPER=true
+WORKER_REAP_GRACE_SEC=900
 
 # ==== Server Heartbeat ====
 SERVER_HEARTBEAT_INTERVAL=30
@@ -163,6 +181,14 @@ SERVER_CUDA_PROBE_IMAGE=nvidia/cuda:12.9.1-base-ubuntu24.04
 # Optional Docker runtime name for GPU containers.
 DOCKER_GPU_RUNTIME=nvidia
 CUDA_VISIBLE_DEVICES=all
+# Report a GPU as unavailable while a process outside FlowMesh uses it.
+WORKER_FOREIGN_GPU_GATE=true
+# Foreign GPU-memory threshold in MiB.
+WORKER_FOREIGN_GPU_MEM_MIB=1024
+# Consecutive occupancy readings required before a device changes availability.
+WORKER_FOREIGN_GPU_CONSECUTIVE=2
+# Delay occupancy checks after a worker task completes.
+WORKER_FOREIGN_GPU_GRACE_SEC=90
 WORKER_UPLOAD_RESULTS=false
 WORKER_EXECUTOR_IDLE_CLEANUP_SEC=60
 MODEL_CLEANUP_AFTER_UPLOAD=0

{flowmesh_cli_stack-0.1.8rc1 → flowmesh_cli_stack-0.1.10}/src/flowmesh_cli_stack/env_schema.py

@@ -1,6 +1,7 @@
 """Stack env schema."""
 
 from flowmesh.models.nodes import NodeRole
+from flowmesh_stack.env import parse_bool
 from flowmesh_stack.env_schema import (
     EnvSchema,
     EnvSection,
@@ -10,6 +11,24 @@ from flowmesh_stack.env_schema import (
     require_if_true,
 )
 
+
+def _warn_reaper_without_watchdog(
+    env: dict[str, str], errors: list[str], warnings: list[str]
+) -> None:
+    """Warn when the stale-worker reaper is enabled while the watchdog is off.
+
+    The reaper runs on the watchdog thread, so it never fires without it. An
+    unset or unparsable ENABLE_WORKER_WATCHDOG keeps its own default and is
+    left alone, so only an explicit false value warns.
+    """
+    if parse_bool(env.get("ENABLE_WORKER_REAPER", "")) and (
+        parse_bool(env.get("ENABLE_WORKER_WATCHDOG", "")) is False
+    ):
+        warnings.append(
+            "ENABLE_WORKER_REAPER has no effect while ENABLE_WORKER_WATCHDOG is false"
+        )
+
+
 STACK_ENV_SCHEMA = EnvSchema(
     name="stack",
     header=[
@@ -147,6 +166,17 @@ STACK_ENV_SCHEMA = EnvSchema(
                     var_type=EnvVarType.INT,
                     min_value=0,
                 ),
+                EnvVar(
+                    "EXTERNAL_WORKER_TOKEN",
+                    description="Shared secret admitting `external` workers "
+                    "(empty disables them).",
+                ),
+                EnvVar(
+                    "EXTERNAL_WORKER_TOKEN_FILE",
+                    description="File holding the secret; wins over "
+                    "EXTERNAL_WORKER_TOKEN.",
+                    var_type=EnvVarType.FILE_PATH,
+                ),
             ],
         ),
         EnvSection(
@@ -292,7 +322,7 @@ STACK_ENV_SCHEMA = EnvSchema(
             vars=[
                 EnvVar("ENABLE_SERVER_SSH_PROXY", "true", var_type=EnvVarType.BOOL),
                 EnvVar(
-                    "ENABLE_SERVER_SSH_CONNECTION_AUDIT",
+                    "ENABLE_SERVER_SSH_CONNECTION_REGISTRY",
                     "true",
                     var_type=EnvVarType.BOOL,
                 ),
@@ -325,7 +355,7 @@ STACK_ENV_SCHEMA = EnvSchema(
                 EnvVar("SSH_MAX_PIDS", var_type=EnvVarType.INT, min_value=1),
                 EnvVar(
                     "ENABLE_SSH_GPU_LIMIT",
-                    "false",
+                    "true",
                     var_type=EnvVarType.BOOL,
                     description=[
                         "Whether to apply requested GPU limits to SSH tasks.",
@@ -333,6 +363,35 @@ STACK_ENV_SCHEMA = EnvSchema(
                         "regardless of their resource requests.",
                     ],
                 ),
+                EnvVar(
+                    "ENABLE_UNISOLATED_SSH_SESSION",
+                    "false",
+                    var_type=EnvVarType.BOOL,
+                    description=[
+                        "Whether a non-root worker may serve SSH sessions,",
+                        "which run as the worker and can read its credentials.",
+                    ],
+                ),
+                EnvVar(
+                    "SSH_SESSION_BACKEND",
+                    "auto",
+                    var_type=EnvVarType.ENUM,
+                    choices={"auto", "docker", "process"},
+                    description=[
+                        "Sandbox an SSH session runs in. 'docker' creates a",
+                        "sibling container; 'process' runs sshd inside the",
+                        "worker. 'auto' picks docker, falling back to process",
+                        "on a root worker.",
+                    ],
+                ),
+                EnvVar(
+                    "SSH_DIRECT_HOST",
+                    description=[
+                        "Address a direct-mode session is advertised at.",
+                        "Defaults to the worker's tailnet address, else its",
+                        "FQDN.",
+                    ],
+                ),
             ],
         ),
         EnvSection(
@@ -427,6 +486,17 @@ STACK_ENV_SCHEMA = EnvSchema(
                     var_type=EnvVarType.INT,
                     min_value=0,
                 ),
+                EnvVar(
+                    "ENABLE_WORKER_REAPER",
+                    "true",
+                    var_type=EnvVarType.BOOL,
+                ),
+                EnvVar(
+                    "WORKER_REAP_GRACE_SEC",
+                    "900",
+                    var_type=EnvVarType.INT,
+                    min_value=0,
+                ),
             ],
         ),
         EnvSection(
@@ -523,6 +593,39 @@ STACK_ENV_SCHEMA = EnvSchema(
                 EnvVar(
                     "CUDA_VISIBLE_DEVICES", "all", var_type=EnvVarType.CSV_INTS_OR_ALL
                 ),
+                EnvVar(
+                    "WORKER_FOREIGN_GPU_GATE",
+                    "true",
+                    description=[
+                        "Report a GPU as unavailable while a process outside "
+                        "FlowMesh uses it.",
+                    ],
+                    var_type=EnvVarType.BOOL,
+                ),
+                EnvVar(
+                    "WORKER_FOREIGN_GPU_MEM_MIB",
+                    "1024",
+                    description="Foreign GPU-memory threshold in MiB.",
+                    var_type=EnvVarType.INT,
+                    min_value=1,
+                ),
+                EnvVar(
+                    "WORKER_FOREIGN_GPU_CONSECUTIVE",
+                    "2",
+                    description=[
+                        "Consecutive occupancy readings required before a device "
+                        "changes availability.",
+                    ],
+                    var_type=EnvVarType.INT,
+                    min_value=1,
+                ),
+                EnvVar(
+                    "WORKER_FOREIGN_GPU_GRACE_SEC",
+                    "90",
+                    description="Delay occupancy checks after a worker task completes.",
+                    var_type=EnvVarType.FLOAT,
+                    min_value=0,
+                ),
                 EnvVar("WORKER_UPLOAD_RESULTS", "false", var_type=EnvVarType.BOOL),
                 EnvVar(
                     "WORKER_EXECUTOR_IDLE_CLEANUP_SEC",
@@ -693,6 +796,7 @@ STACK_ENV_SCHEMA = EnvSchema(
             ],
             errors,
         ),
+        _warn_reaper_without_watchdog,
     ],
 )
 

{flowmesh_cli_stack-0.1.8rc1 → flowmesh_cli_stack-0.1.10}/src/flowmesh_cli_stack/worker.py

@@ -80,11 +80,11 @@ def worker_up(
             "Repeat --config-raw to provide multiple configs."
         ),
     ),
-    name_template: str | None = typer.Option(
+    alias_template: str | None = typer.Option(
         None,
-        "--name-template",
+        "--alias-template",
         help=(
-            "Worker name template for cpu/gpu presets. Placeholders: "
+            "Worker alias template for cpu/gpu presets. Placeholders: "
             "{slug}, {kind}, {idx}, {gpu}. "
             "Default: '{slug}_worker_{kind}_{idx|gpu}'."
         ),
@@ -113,21 +113,21 @@ def worker_up(
             targets=targets,
             config_paths=config,
             config_raw=config_raw,
-            name_template=name_template,
+            alias_template=alias_template,
             slug=slug,
         )
     except FlowMeshError as exc:
         logging.error(str(exc))
         raise typer.Exit(code=1)
     for label, worker_info in created:
-        worker_name = worker_info.get("name", "<unknown>")
-        logging.success(f"Created {label} '{worker_name}':")
+        worker_alias = worker_info.get("alias", "<unknown>")
+        logging.success(f"Created {label} '{worker_alias}':")
         logging.log(json.dumps(worker_info, indent=2))
 
 
 @app.command("start")
 def worker_start(
-    names: list[str] = typer.Argument(..., help="Worker name(s) or 'all'"),
+    aliases: list[str] = typer.Argument(..., help="Worker alias(es) or 'all'"),
     env_file: Path = typer.Option(
         DEFAULT_ENV_FILE, "--env-file", help="Env file to load defaults"
     ),
@@ -139,20 +139,20 @@ def worker_start(
     """Start a stopped worker container."""
     client = stack_node_client(env_file, base_url, token or None)
     try:
-        started = operate_workers(client, names, operation="start")
+        started = operate_workers(client, aliases, operation="start")
     except FlowMeshError as exc:
         logging.error(str(exc))
         raise typer.Exit(code=1)
     if not started:
         logging.warning("No workers found.")
         return
-    for name in started:
-        logging.success(f"Started worker {name}")
+    for alias in started:
+        logging.success(f"Started worker {alias}")
 
 
 @app.command("stop")
 def worker_stop(
-    names: list[str] = typer.Argument(..., help="Worker name(s) or 'all'"),
+    aliases: list[str] = typer.Argument(..., help="Worker alias(es) or 'all'"),
     env_file: Path = typer.Option(
         DEFAULT_ENV_FILE, "--env-file", help="Env file to load defaults"
     ),
@@ -164,20 +164,20 @@ def worker_stop(
     """Stop a running worker container without removing it."""
     client = stack_node_client(env_file, base_url, token or None)
     try:
-        stopped = operate_workers(client, names, operation="stop")
+        stopped = operate_workers(client, aliases, operation="stop")
     except FlowMeshError as exc:
         logging.error(str(exc))
         raise typer.Exit(code=1)
     if not stopped:
         logging.warning("No workers found.")
         return
-    for name in stopped:
-        logging.success(f"Stopped worker {name}")
+    for alias in stopped:
+        logging.success(f"Stopped worker {alias}")
 
 
 @app.command("down")
 def worker_down(
-    names: list[str] = typer.Argument(..., help="Worker name(s) or 'all'"),
+    aliases: list[str] = typer.Argument(..., help="Worker alias(es) or 'all'"),
     env_file: Path = typer.Option(
         DEFAULT_ENV_FILE, "--env-file", help="Env file to load defaults"
     ),
@@ -188,9 +188,9 @@ def worker_down(
 ) -> None:
     """Destroy a worker or all workers, removing containers and associated resources."""
     client = stack_node_client(env_file, base_url, token or None)
-    if "all" in names:
-        if len(names) != 1:
-            logging.error("Use either 'all' or worker names, not both.")
+    if "all" in aliases:
+        if len(aliases) != 1:
+            logging.error("Use either 'all' or worker aliases, not both.")
             raise typer.Exit(code=1)
         try:
             logging.info("Destroying all workers...")
@@ -202,15 +202,15 @@ def worker_down(
         return
 
     try:
-        destroyed = operate_workers(client, names, operation="destroy")
+        destroyed = operate_workers(client, aliases, operation="destroy")
     except FlowMeshError as exc:
         logging.error(str(exc))
         raise typer.Exit(code=1)
     if not destroyed:
         logging.warning("No workers found.")
         return
-    for name in destroyed:
-        logging.success(f"Destroyed worker {name}")
+    for alias in destroyed:
+        logging.success(f"Destroyed worker {alias}")
 
 
 @app.command("pull")

{flowmesh_cli_stack-0.1.8rc1 → flowmesh_cli_stack-0.1.10/src/flowmesh_cli_stack.egg-info}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: flowmesh-cli-stack
-Version: 0.1.8rc1
+Version: 0.1.10
 Summary: FlowMesh CLI stack commands
 License-Expression: Apache-2.0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: flowmesh-cli==0.1.8rc1
-Requires-Dist: flowmesh-sdk-stack==0.1.8rc1
+Requires-Dist: flowmesh-cli==0.1.10
+Requires-Dist: flowmesh-sdk-stack==0.1.10
 Requires-Dist: docker>=7.1.0
 Requires-Dist: packaging>=24
 Requires-Dist: pyyaml>=6.0.2

flowmesh_cli_stack-0.1.10/src/flowmesh_cli_stack.egg-info/requires.txt

@@ -0,0 +1,5 @@
+flowmesh-cli==0.1.10
+flowmesh-sdk-stack==0.1.10
+docker>=7.1.0
+packaging>=24
+pyyaml>=6.0.2

flowmesh_cli_stack-0.1.8rc1/src/flowmesh_cli_stack.egg-info/requires.txt

@@ -1,5 +0,0 @@
-flowmesh-cli==0.1.8rc1
-flowmesh-sdk-stack==0.1.8rc1
-docker>=7.1.0
-packaging>=24
-pyyaml>=6.0.2