docker-stack 2.2.5__py3-none-any.whl → 2.3.1__py3-none-any.whl

docker_stack/cli.py

@@ -8,7 +8,7 @@ import shutil
 import sys
 import textwrap
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Set, Tuple
 import os
 import yaml
 import json
@@ -36,11 +36,16 @@ from docker_stack.shell_auth import (
     run_managed_shell,
     shell_session_active,
 )
+from docker_stack.conflict_prompt import ForceDeployPrompt
 from docker_stack.manager_api import (
+    DEPLOY_WAIT_ENV,
     FEATURE_STACK_DEPLOY,
     FEATURE_STACK_QUERY,
     ManagerApiClient,
+    describe_active_deployment,
+    describe_run,
     discover_manager_client,
+    format_deploy_error_body,
 )
 from docker_stack.registry import DockerRegistry
 from .envsubst import LineCheckResult, SubstitutionError, envsubst, envsubst_load_file
@@ -698,6 +703,143 @@ def load_env_file(env_file: str, base_env: Optional[Dict[str, str]] = None, max_
     return _resolve_env_entries(entries, env_file, base_env=base_env, max_cycles=max_cycles)
 
 
+
+
+_ENV_REF_PATTERN = re.compile(r"\$\{([^}:\s]+)(?::-(.*?))?\}|\$([a-zA-Z_][a-zA-Z0-9_]*)")
+_ESCAPED_DOLLAR = "$_ESCAPED_DOLLAR_"
+
+
+def _escape_dollars(text: str) -> str:
+    """Marks text as a literal payload for the manager, which collapses `$$`."""
+    return text.replace("$", "$$")
+
+
+def _substitute_selected_vars(text: str, names: Set[str], replacements: Dict[str, str]) -> str:
+    """Expands only `names`, leaving every other `${VAR}` for the manager."""
+    if not names:
+        return text
+    guarded = text.replace("$$", _ESCAPED_DOLLAR)
+
+    def replace(match: "re.Match[str]") -> str:
+        var = match.group(1) if match.group(1) is not None else match.group(3)
+        if var not in names:
+            return match.group(0)
+        value = os.environ.get(var)
+        if value in (None, "") and match.group(2) is not None:
+            value = match.group(2)
+        if value in (None, ""):
+            return match.group(0)
+        for old, new in replacements.items():
+            value = value.replace(old, new)
+        return value
+
+    return _ENV_REF_PATTERN.sub(replace, guarded).replace(_ESCAPED_DOLLAR, "$$")
+
+
+def _substitute_selected_in_yaml_tree(value, names: Set[str], replacements: Dict[str, str]):
+    if isinstance(value, str):
+        return _substitute_selected_vars(value, names, replacements)
+    if isinstance(value, dict):
+        return {
+            _substitute_selected_in_yaml_tree(key, names, replacements): _substitute_selected_in_yaml_tree(item, names, replacements)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [_substitute_selected_in_yaml_tree(item, names, replacements) for item in value]
+    return value
+
+
+def _mark_inline_secret_content_literal(compose_data) -> None:
+    """Protects inlined secret payloads from the manager's interpolation.
+
+    `x-content` is a payload, not a template: the file it came from was already
+    rendered locally when it was declared as `x-template-file`, and a `file:` or
+    `environment:` secret is verbatim by definition. Without this a `$` inside a
+    secret is read as a variable reference and the secret is silently corrupted
+    or the deploy fails on a variable the payload never meant to name.
+    """
+    if not isinstance(compose_data, dict):
+        return
+    secrets = compose_data.get("secrets")
+    if not isinstance(secrets, dict):
+        return
+    for details in secrets.values():
+        if isinstance(details, dict) and isinstance(details.get("x-content"), str):
+            details["x-content"] = _escape_dollars(details["x-content"])
+
+
+def _render_manager_compose(compose_data, secret_env_vars: Set[str], replacements: Dict[str, str]) -> str:
+    """Renders the compose a manager-backed deploy sends.
+
+    The manager owns interpolation: it receives the authored compose and the
+    `.env` holding the values, and it substitutes in the parsed document. So
+    variables are left standing here rather than rendered twice, which is what
+    the `$` doubling in `replacements` exists to undo. Only the variables the
+    manager cannot see are resolved locally - a secret sourced from the
+    environment is deliberately kept out of `.env` so its value is not stored
+    with the stack.
+
+    References are still checked here so a missing variable is reported against
+    the authored file, with its line and surrounding context, instead of coming
+    back as a deploy failure.
+    """
+    _mark_inline_secret_content_literal(compose_data)
+    missing: List[str] = []
+    _substitute_in_yaml_tree(compose_data, replacements, missing)
+    if missing:
+        _report_missing_variables(compose_data, replacements, missing)
+    return yaml.dump(_substitute_selected_in_yaml_tree(compose_data, secret_env_vars, replacements), sort_keys=False)
+
+
+def _substitute_in_yaml_tree(value, replacements: Dict[str, str], missing: List[str]):
+    """Expands `${VAR}` inside the scalars of a parsed compose document.
+
+    Substituting in the serialized text instead would let a variable's value
+    change the structure of the document. `KEY: ${JSON}` holding `{"a":"b"}`
+    reparses as a nested mapping, which the manager rejects as a non-scalar
+    environment value, and a value containing `: ` makes the document fail to
+    parse at all. Walking the tree keeps every substituted value a scalar,
+    whatever characters it contains.
+    """
+    if isinstance(value, str):
+        try:
+            return envsubst(value, replacements=replacements, on_error="throw")
+        except SubstitutionError as error:
+            missing.extend(result.variable_name for result in error.results if result.has_error)
+            return value
+    if isinstance(value, dict):
+        return {
+            _substitute_in_yaml_tree(key, replacements, missing): _substitute_in_yaml_tree(item, replacements, missing)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [_substitute_in_yaml_tree(item, replacements, missing) for item in value]
+    return value
+
+
+def _render_compose_substitutions(compose_data, replacements: Dict[str, str]) -> str:
+    missing: List[str] = []
+    substituted = _substitute_in_yaml_tree(compose_data, replacements, missing)
+    if missing:
+        _report_missing_variables(compose_data, replacements, missing)
+    return yaml.dump(substituted, sort_keys=False)
+
+
+def _report_missing_variables(compose_data, replacements: Dict[str, str], missing: List[str]) -> None:
+    """Reports unresolved variables against the whole document.
+
+    Substitution happens scalar by scalar, which has no line numbers to report.
+    Re-running over the serialized document restores the line and context in the
+    error, and exits the same way it always has.
+    """
+    document = yaml.dump(compose_data, sort_keys=False)
+    envsubst(document, replacements=replacements)
+    raise SubstitutionError(
+        [LineCheckResult(line_no=1, line_content="", variable_name=name, start_index=0) for name in missing],
+        document,
+    )
+
+
 class Docker:
     def __init__(self, registries: List[str] = []):
         self.stack = DockerStack(self)
@@ -786,7 +928,27 @@ class DockerStack:
     def _relative_source_name(path: Path, base_dir: Path) -> str:
         return os.path.relpath(path, base_dir).replace(os.sep, "/")
 
-    def _build_env_xfile(self, sources: List[str], excluded_names: set) -> Optional[Dict[str, str]]:
+    @staticmethod
+    def _config_environment_vars(compose_data: dict) -> set:
+        """Variables named by `configs.<name>.environment`.
+
+        These are not `${VAR}` references, so scanning the compose text does not
+        find them, yet the manager resolves them from the stack `.env` like any
+        other variable. Secrets are deliberately absent: their value is inlined
+        locally so it is never stored alongside the stack.
+        """
+        config_env_vars = set()
+        configs = compose_data.get("configs", {})
+        if not isinstance(configs, dict):
+            return config_env_vars
+        for details in configs.values():
+            if isinstance(details, dict) and "environment" in details:
+                env_name = str(details["environment"]).strip()
+                if env_name:
+                    config_env_vars.add(env_name)
+        return config_env_vars
+
+    def _build_env_xfile(self, sources: List[str], excluded_names: set, extra_names: Optional[Set[str]] = None) -> Optional[Dict[str, str]]:
         seen = set()
         lines = []
         for source in sources:
@@ -800,6 +962,14 @@ class DockerStack:
                     continue
                 seen.add(name)
                 lines.append(f"{name}={value}")
+        for name in sorted(extra_names or ()):
+            if name in seen or name in excluded_names:
+                continue
+            value = os.environ.get(name)
+            if value in (None, ""):
+                continue
+            seen.add(name)
+            lines.append(f"{name}={value}")
         if not lines:
             return None
         content = ("\n".join(lines) + "\n").encode("utf-8")
@@ -907,12 +1077,19 @@ class DockerStack:
 
         # Define the replacements for '$' to '$$' for env variables in compose files
         replacements_map = {"$": "$$"}
-        clean_content = envsubst(yaml.dump(compose_data, sort_keys=False), replacements=replacements_map)
+        if manager_deploy:
+            clean_content = _render_manager_compose(compose_data, secret_env_vars, replacements_map)
+        else:
+            clean_content = _render_compose_substitutions(compose_data, replacements_map)
         enriched_data = yaml.safe_load(clean_content) or {}
         x_files = [
             {"path": "compose.yml", "content": self._b64_content(template_bytes)},
         ]
-        env_xfile = self._build_env_xfile(env_sources, secret_env_vars)
+        env_xfile = self._build_env_xfile(
+            env_sources,
+            secret_env_vars,
+            extra_names=self._config_environment_vars(compose_data) if manager_deploy else None,
+        )
         if env_xfile:
             x_files.append(env_xfile)
         x_files.extend(source_x_files)
@@ -1500,6 +1677,7 @@ class DockerStack:
                 options=options,
             )
         else:
+            prompt = ForceDeployPrompt.create()
             try:
                 payload = manager_client.deploy_stack_stream(
                     stack=stack_name,
@@ -1507,6 +1685,7 @@ class DockerStack:
                     compose=rendered_content,
                     options=options,
                     on_event=DockerStack._print_manager_deploy_event,
+                    wait_for_poll=prompt.wait if prompt else None,
                 )
             except RuntimeError as exc:
                 message = str(exc)
@@ -1571,6 +1750,51 @@ class DockerStack:
             message = data.get("message") or data.get("error") or ""
             if message:
                 print(f"[manager] {service}: {status}: {message}", flush=True)
+        elif event_name == "deployment":
+            deployment_id = data.get("deployment_id")
+            if deployment_id:
+                mode = data.get("mode") or "deployment"
+                print(f"[manager] {mode} {deployment_id}", flush=True)
+        elif event_name == "waiting":
+            DockerStack._print_manager_wait(data)
+
+    @staticmethod
+    def _print_manager_wait(active: Dict[str, object]) -> None:
+        """Narrate the wait for another run of this stack, and any attempt to force past it.
+
+        The manager will not queue an interactive deploy for us, so the CLI polls
+        and reports each phase: waiting, forcing (asking the manager to abort the
+        holder), and how that abort ended.
+        """
+        if not isinstance(active, dict):
+            active = {}
+        phase = str(active.get("phase") or "waiting")
+        run = describe_run(active)
+        if phase == "waiting":
+            budget = active.get("wait_budget_secs")
+            suffix = f" (waiting up to {budget}s, set {DEPLOY_WAIT_ENV} to change)" if isinstance(budget, int) else ""
+            message = f"waiting: {describe_active_deployment(active)}{suffix}"
+        elif phase == "forcing":
+            message = f"forcing: asking the manager to abort {run}"
+        elif phase == "aborted":
+            message = f"aborted {run}; deploying now"
+        elif phase == "aborted_other":
+            message = f"aborted a different run than shown: {run}; deploying now"
+        elif phase == "free":
+            message = "the stack is free; deploying now"
+        elif phase == "not_released":
+            message = f"asked the manager to abort {run}, but it has not released the stack yet; waiting for it to finish"
+        elif phase == "forbidden":
+            message = f"you do not have permission to abort {run}; still waiting"
+        elif phase == "still_held":
+            message = f"the stack is still held by {run}; waiting"
+        elif phase == "force_used":
+            message = "force was already used once in this run; waiting for the stack to free up"
+        elif phase == "interrupted":
+            message = "interrupted while asking the manager to abort; that request may still complete"
+        else:
+            message = f"{phase}: {run}"
+        print(f"[manager] {message}", file=sys.stderr, flush=True)
 
     @staticmethod
     def _rollback_via_manager(
@@ -1580,10 +1804,13 @@ class DockerStack:
         namespace: str,
         version: str,
     ) -> Optional[str]:
+        prompt = ForceDeployPrompt.create()
         payload = manager_client.rollback_stack(
             stack=stack_name,
             namespace=namespace,
             version=version,
+            on_wait=DockerStack._print_manager_wait,
+            wait_for_poll=prompt.wait if prompt else None,
         )
         warnings = payload.get("warnings") or []
         for warning in warnings:
@@ -1868,7 +2095,7 @@ def _humanize_manager_error(message: str) -> str:
         return message
     if not isinstance(body, dict):
         return message
-    detail = str(body.get("message", "")).strip()
+    detail = format_deploy_error_body(body) if body.get("code") else str(body.get("message", "")).strip()
     if not detail:
         return message
     incident = str(body.get("incident_id") or body.get("trace_id") or "").strip()
@@ -1896,6 +2123,17 @@ def _manager_deploy_suggestion(message: str) -> Optional[str]:
             "by this stack. Either allow that external network in the Docker-Manager deployment rule, attach the service to "
             "a stack-owned network, or relabel/recreate the existing network with the expected stack ownership before deploying."
         )
+    if "is still running" in lowered and ("gave up after waiting" in lowered or "http 409" in lowered):
+        return (
+            "Suggestion: another run holds this stack's deploy lock. Rerun and let it wait, raise "
+            f"{DEPLOY_WAIT_ENV} to queue longer, press Enter twice while waiting at a terminal to abort that run "
+            "and force your deploy, or abort it from the Docker-Manager stack page."
+        )
+    if "failed services:" in lowered:
+        return (
+            "Suggestion: every service operation has finished; the failed ones are listed above with their incident ids. "
+            "Fix the cause and redeploy, or use the rollback offered on the Docker-Manager stack page to restore the previous specs."
+        )
     if "stack not found" in lowered:
         return (
             "Suggestion: the manager was reached but has no stack by that name in that namespace. "
@@ -1918,9 +2156,10 @@ def _manager_deploy_suggestion(message: str) -> Optional[str]:
 def _format_called_process_error(exc: subprocess.CalledProcessError) -> str:
     output = _error_output(exc)
     lines = [f"docker-stack: command failed with exit code {exc.returncode}: {_format_command(exc.cmd)}"]
-    if output:
-        lines.append(_humanize_manager_error(output))
-    suggestion = _manager_deploy_suggestion(output)
+    humanized = _humanize_manager_error(output) if output else ""
+    if humanized:
+        lines.append(humanized)
+    suggestion = _manager_deploy_suggestion(humanized or output)
     if suggestion:
         lines.extend(["", suggestion])
     return "\n".join(lines)
@@ -1928,8 +2167,9 @@ def _format_called_process_error(exc: subprocess.CalledProcessError) -> str:
 
 def _format_runtime_error(exc: RuntimeError) -> str:
     message = str(exc).strip() or exc.__class__.__name__
-    lines = [f"docker-stack: {_humanize_manager_error(message)}"]
-    suggestion = _manager_deploy_suggestion(message)
+    humanized = _humanize_manager_error(message)
+    lines = [f"docker-stack: {humanized}"]
+    suggestion = _manager_deploy_suggestion(humanized)
     if suggestion:
         lines.extend(["", suggestion])
     return "\n".join(lines)
@@ -2031,14 +2271,14 @@ def _run(args: List[str] = None):
 
     checkout_parser = subparsers.add_parser("checkout", help="Deploy specific version of the stack")
     checkout_parser.add_argument("stack_name", help="Name of the stack")
-    checkout_parser.add_argument("version", help="Stack version to cat")
+    checkout_parser.add_argument("version", help="Stack version to deploy")
     _add_namespace_argument(checkout_parser)
 
     # version_parser = subparsers.add_parser("version",help="Deploy specific version of the stack")
     # version_parser.add_argument("stack_name", help="Name of the stack")
     # version_parser.add_argument("version","versions", help="Stack version to cat")
 
-    version_parser = subparsers.add_parser("version", aliases=["versions"], help="Deploy specific version of the stack")
+    version_parser = subparsers.add_parser("version", aliases=["versions"], help="List recorded versions of the stack")
     version_parser.add_argument("stack_name", help="Name of the stack")
     _add_namespace_argument(version_parser)
 

docker_stack/conflict_prompt.py

@@ -0,0 +1,134 @@
+"""Terminal prompt shown while a deploy waits for another run of the same stack.
+
+Docker-Manager refuses to queue interactive applies: it answers ``409
+deployment_in_progress`` and leaves the choice between waiting and aborting to
+whoever is on the other end. On a CLI that is the person at the terminal, so
+while the client polls for the stack to free up this prompt listens for two
+Enter presses in quick succession and reports ``"force"``; the caller then asks
+the manager to abort the running deployment. Ctrl+C is left alone and keeps
+exiting the CLI.
+
+Enter is line-oriented, so no terminal mode is changed and there is nothing to
+restore on the way out. The prompt is only created when both stdin and stderr
+are terminals and the process is in the foreground; anywhere else (CI, pipes,
+the GitHub Action's heredoc) the caller simply sleeps between polls.
+"""
+
+import io
+import os
+import select
+import sys
+import time
+from typing import Any, Optional
+
+DOUBLE_ENTER_SECS = 3.0
+WINDOWS_POLL_SECS = 0.1
+HINT = "[manager] press Enter twice to force your deploy (aborts that run; changes it already made to the daemon stay), Ctrl+C to quit"
+SECOND_ENTER_HINT = f"[manager] press Enter again within {int(DOUBLE_ENTER_SECS)}s to force the deploy"
+
+ENTER = "enter"
+OTHER = "other"
+TIMEOUT = "timeout"
+EOF = "eof"
+
+
+class _PosixEnterReader:
+    def __init__(self, stdin, fd: int):
+        self._stdin = stdin
+        self._fd = fd
+
+    def wait_for_enter(self, timeout: float) -> str:
+        try:
+            readable, _, _ = select.select([self._fd], [], [], max(0.0, timeout))
+        except (OSError, ValueError):
+            return EOF
+        if not readable:
+            return TIMEOUT
+        line = self._stdin.readline()
+        if line == "":
+            return EOF
+        return ENTER if line.strip() == "" else OTHER
+
+
+class _WindowsEnterReader:
+    def __init__(self, msvcrt_module):
+        self._msvcrt = msvcrt_module
+
+    def wait_for_enter(self, timeout: float) -> str:
+        deadline = time.monotonic() + max(0.0, timeout)
+        while True:
+            if self._msvcrt.kbhit():
+                char = self._msvcrt.getwch()
+                return ENTER if char in ("\r", "\n") else OTHER
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return TIMEOUT
+            time.sleep(min(WINDOWS_POLL_SECS, remaining))
+
+
+class ForceDeployPrompt:
+    """Listens for a double Enter between polls; see the module docstring."""
+
+    def __init__(self, stderr, reader):
+        self._stderr = stderr
+        self._reader = reader
+        self._hinted = False
+        self._dead = False
+        self._first_enter_at: Optional[float] = None
+
+    @classmethod
+    def create(cls) -> Optional["ForceDeployPrompt"]:
+        """Build a prompt when a person can see and answer it, else ``None``."""
+        stdin, stderr = sys.stdin, sys.stderr
+        if stdin is None or stderr is None:
+            return None
+        try:
+            if not stdin.isatty() or not stderr.isatty():
+                return None
+            fd = stdin.fileno()
+        except (AttributeError, ValueError, OSError, io.UnsupportedOperation):
+            return None
+        if os.name == "nt":
+            try:
+                import msvcrt  # type: ignore[import-not-found]
+            except ImportError:
+                return None
+            return cls(stderr, _WindowsEnterReader(msvcrt))
+        try:
+            # Reading a terminal from a background job stops the process (SIGTTIN).
+            if os.tcgetpgrp(fd) != os.getpgrp():
+                return None
+        except (AttributeError, OSError):
+            return None
+        return cls(stderr, _PosixEnterReader(stdin, fd))
+
+    def _say(self, message: str) -> None:
+        print(message, file=self._stderr, flush=True)
+
+    def wait(self, active: Any, seconds: float) -> Optional[str]:
+        """Wait up to ``seconds`` for the next poll; ``"force"`` if Enter was pressed twice."""
+        deadline = time.monotonic() + max(0.0, seconds)
+        if not self._hinted:
+            self._say(HINT)
+            self._hinted = True
+        while True:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return None
+            if self._dead:
+                time.sleep(remaining)
+                return None
+            result = self._reader.wait_for_enter(remaining)
+            if result == TIMEOUT:
+                return None
+            if result == EOF:
+                self._dead = True
+                continue
+            if result != ENTER:
+                continue
+            now = time.monotonic()
+            if self._first_enter_at is not None and now - self._first_enter_at <= DOUBLE_ENTER_SECS:
+                self._first_enter_at = None
+                return "force"
+            self._first_enter_at = now
+            self._say(SECOND_ENTER_HINT)

docker_stack/helpers.py

@@ -1,4 +1,4 @@
-import random
+import secrets as _secrets
 import string
 import sys
 from typing import Callable, Dict, List, Optional, Union
@@ -23,13 +23,17 @@ def generate_secret(
 
     Returns:
         A randomly generated secret string.
+
+    Note:
+        Uses the ``secrets`` module rather than ``random``, since these values
+        are used as passwords, API keys and tokens.
     """
     if length is None:
-        length = random.randint(12, 20)
+        length = 12 + _secrets.randbelow(9)  # 12..20 inclusive
 
     # Always start with a letter (uppercase or lowercase)
     first_char_pool = string.ascii_lowercase + string.ascii_uppercase
-    first_char = random.choice(first_char_pool)
+    first_char = _secrets.choice(first_char_pool)
 
     # Prepare the character pool for the rest of the secret
     characters = string.ascii_lowercase
@@ -47,7 +51,7 @@ def generate_secret(
         raise ValueError("No character types selected for secret generation.")
 
     # Generate the rest of the string
-    secret = [first_char] + [random.choice(characters) for _ in range(length - 1)]
+    secret = [first_char] + [_secrets.choice(characters) for _ in range(length - 1)]
 
     return "".join(secret)