procesio-cli 0.1.1__py3-none-any.whl

procesio_cli/tools/procesio/reliability.py

@@ -0,0 +1,193 @@
+"""Request-reliability primitives for the PROCESIO client.
+
+Encodes, in one place, the empirical rules learned driving the live Web API (see
+PROCESIO-API-NOTES.md "Empirical reliability profile" and the `procesio` agent's
+`guidance --topic reliability`):
+
+* **The client had no TOTAL deadline.** `requests`' scalar timeout is a *per-read
+  inactivity* guard, not a wall-clock ceiling: a connection that is accepted and
+  then dribbles bytes (or where the server holds it open) never trips it, so a
+  stalled call runs until the process is killed. `run_with_deadline` adds a real
+  total deadline on a daemon thread so a stall SURFACES as a structured error.
+* **Retries are safe for GET only.** Re-POSTing an execution double-runs a
+  process with real side effects; a timed-out PUT may already have applied. So the
+  retry policy is GET-only, on transient statuses + network errors.
+
+Everything here is stdlib-only (no new dependency) and provider-neutral.
+"""
+from __future__ import annotations
+
+import os
+import random
+import threading
+
+# -- knobs ------------------------------------------------------------------
+
+# Per-read (inactivity) timeout handed to `requests` — a socket that sends no
+# bytes for this long trips requests' own timeout. Kept as defense-in-depth
+# UNDER the total deadline below.
+READ_TIMEOUT = 60
+
+# Total wall-clock deadlines, by endpoint class. A definition read (~120-420KB
+# GET) should never legitimately take this long; a definition write (PUT) is
+# given more headroom. run-process is EXEMPT (deadline=None) — a synchronous run
+# of a many-action process legitimately takes minutes.
+READ_DEADLINE = 90.0
+WRITE_DEADLINE = 180.0
+
+# GET-only retry policy. 500 is deliberately EXCLUDED (may be a deterministic
+# server error, not transient). 429 is honoured via Retry-After when present.
+RETRYABLE_STATUSES = frozenset({429, 502, 503, 504})
+MAX_RETRIES = 2
+BACKOFF_BASE = 0.5   # seconds
+BACKOFF_CAP = 8.0    # seconds
+
+ENV_NO_DEADLINE = "AAT_PROCESIO_NO_DEADLINE"
+ENV_NO_RETRY = "AAT_PROCESIO_NO_RETRY"
+
+# Sentinel: "no explicit value passed — pick the class default". Distinct from
+# None, which for a deadline means "exempt" and for a read timeout means "wait
+# indefinitely" (used by synchronous run-process).
+DEFAULT = object()
+
+
+class DeadlineHit(Exception):
+    """Raised by run_with_deadline when the total wall-clock deadline elapses.
+    The client translates it into errors.DeadlineExceeded with full context."""
+
+
+# -- env opt-outs -----------------------------------------------------------
+
+def _truthy(val: str | None) -> bool:
+    return bool(val) and val.strip().lower() not in ("", "0", "false", "no", "off")
+
+
+def deadline_disabled() -> bool:
+    return _truthy(os.environ.get(ENV_NO_DEADLINE))
+
+
+def retry_disabled() -> bool:
+    return _truthy(os.environ.get(ENV_NO_RETRY))
+
+
+# -- deadline resolution ----------------------------------------------------
+
+def endpoint_class(method_upper: str) -> str:
+    """The deadline class of a verb: reads are cheap-and-bounded, writes get
+    more headroom."""
+    return "read" if method_upper == "GET" else "write"
+
+
+def resolve_deadline(method_upper: str, deadline) -> float | None:
+    """Turn the caller's `deadline` argument into a concrete seconds value or
+    None (exempt). `DEFAULT` -> the class default; None -> exempt; a number ->
+    that many seconds. The env opt-out forces exempt."""
+    if deadline is None:
+        return None
+    if deadline_disabled():
+        return None
+    if deadline is DEFAULT:
+        return READ_DEADLINE if method_upper == "GET" else WRITE_DEADLINE
+    return float(deadline)
+
+
+# -- retry policy -----------------------------------------------------------
+
+def is_retryable(method_upper: str, status: int, attempt: int) -> bool:
+    """A response is retryable only for GET, only on a transient status, and only
+    while retries remain. This is the SINGLE gate — a POST/PUT can never satisfy
+    it, so an execution is structurally un-retryable."""
+    if retry_disabled():
+        return False
+    return (
+        method_upper == "GET"
+        and status in RETRYABLE_STATUSES
+        and attempt < MAX_RETRIES
+    )
+
+
+def network_error_retryable(method_upper: str, attempt: int) -> bool:
+    """Connection/read errors are retryable for GET only (we cannot know whether
+    a POST/PUT reached the server)."""
+    if retry_disabled():
+        return False
+    return method_upper == "GET" and attempt < MAX_RETRIES
+
+
+def full_jitter(attempt: int, rng: random.Random | None = None) -> float:
+    """AWS-style full jitter: uniform(0, min(cap, base * 2**attempt)). `attempt`
+    is 1-based (the delay before the Nth retry)."""
+    ceiling = min(BACKOFF_CAP, BACKOFF_BASE * (2 ** attempt))
+    r = rng or random
+    return r.uniform(0.0, ceiling)
+
+
+def parse_retry_after(headers: dict | None) -> float | None:
+    """Retry-After in delta-seconds form (the only form PROCESIO/Kong emit).
+    Ignores HTTP-date form. Returns a capped seconds value or None."""
+    if not headers:
+        return None
+    # Header lookup is case-insensitive on real responses; be lenient.
+    raw = None
+    for k, v in headers.items():
+        if k.lower() == "retry-after":
+            raw = v
+            break
+    if raw is None:
+        return None
+    try:
+        secs = float(str(raw).strip())
+    except (TypeError, ValueError):
+        return None
+    if secs < 0:
+        return None
+    return min(secs, BACKOFF_CAP)
+
+
+# -- total deadline ---------------------------------------------------------
+
+def run_with_deadline(fn, timeout: float | None):
+    """Run `fn()` under a TOTAL wall-clock deadline.
+
+    `timeout` None -> run inline (no deadline). Otherwise run `fn` on a DAEMON
+    thread and wait at most `timeout` seconds. On expiry raise `DeadlineHit`; the
+    worker is abandoned (daemon => the interpreter never joins it, and the OS
+    reclaims the leaked socket when this short-lived CLI process exits). Any
+    exception raised by `fn` is re-raised in the caller.
+
+    Why a raw daemon thread and not `ThreadPoolExecutor`: the executor's worker
+    threads are joined by an interpreter atexit hook, which would HANG process
+    exit on a stuck HTTP read — exactly the failure we are eliminating.
+
+    CAVEAT — long-lived hosts (AAT_RUNNER worker pool). "The OS reclaims the
+    socket on process exit" holds for the normal short-lived CLI, but NOT under
+    the persistent tool-runner pool (`AAT_RUNNER=1`, tools/_lib/toolrunner),
+    whose worker process outlives the call. There an abandoned worker stays
+    parked on the stuck read, leaking one thread + one socket per deadline-hit
+    for the pool's lifetime. Deadline-hits are rare (a true stall), so this is a
+    slow leak, not a hot path — but a flapping endpoint under the pool will
+    accumulate. If that ever bites, bound it (recycle the pool worker after N
+    deadline-hits, or move the read onto a cancellable transport).
+    """
+    if timeout is None:
+        return fn()
+    if timeout <= 0:
+        raise DeadlineHit()
+    box: dict = {}
+    done = threading.Event()
+
+    def _worker():
+        try:
+            box["result"] = fn()
+        except BaseException as exc:  # noqa: BLE001 - re-raised below
+            box["exc"] = exc
+        finally:
+            done.set()
+
+    t = threading.Thread(target=_worker, name="procesio-http", daemon=True)
+    t.start()
+    if not done.wait(timeout):
+        raise DeadlineHit()
+    if "exc" in box:
+        raise box["exc"]
+    return box["result"]

procesio_cli/tools/procesio/sqlformat.py

@@ -0,0 +1,167 @@
+"""Lay a SQL statement out over several lines so a person can read it in the designer.
+
+WHY THIS EXISTS. A statement authored by concatenating strings arrives as ONE line. It runs
+perfectly and it is unreadable: the designer shows a single row that scrolls off the screen, and
+a reviewer cannot see a WHERE clause, a JOIN or a CASE without dragging sideways through three
+thousand characters. Readability is not a nicety here, because the SQL IS the logic: a Query
+Store node's SQL (one statement, or since PROCESIO v3.0.0 a `;`-separated script) is the whole
+node.
+
+WHAT IT GUARANTEES. Only whitespace OUTSIDE string literals ever changes:
+
+* text inside `'...'` is copied verbatim, quotes, keywords, doubled quotes and all;
+* `<%0%>` placeholders and `{{ds:Store}}` / `{{col:Store.Column}}` tokens are opaque words;
+* the result is checked against the input before it is returned - same tokens in the same order,
+  or the input is handed back unchanged. A formatter that can alter a statement is worse than no
+  formatter, so this one refuses to guess.
+
+It also leaves alone any statement that ALREADY contains a newline: an author who laid their own
+SQL out has made a decision, and reflowing it would overwrite that.
+"""
+from __future__ import annotations
+
+import re
+
+__all__ = ["format_sql", "is_unreadable", "SINGLE_LINE_LIMIT"]
+
+# Beyond this, a single-line statement is long enough that a reader has to scroll.
+SINGLE_LINE_LIMIT = 160
+
+# Clause heads, longest first so "GROUP BY" wins over "GROUP" and "LEFT JOIN" over "JOIN".
+_BREAK_BEFORE = [
+    "WITH", "SELECT", "FROM", "WHERE", "GROUP BY", "HAVING", "ORDER BY", "LIMIT", "OFFSET",
+    "UNION ALL", "UNION", "INSERT INTO", "VALUES", "UPDATE", "SET", "DELETE FROM",
+    "INNER JOIN", "LEFT OUTER JOIN", "RIGHT OUTER JOIN", "LEFT JOIN", "RIGHT JOIN",
+    "CROSS JOIN", "STRAIGHT_JOIN", "JOIN", "ON DUPLICATE KEY UPDATE",
+]
+# Broken only near the top level, or a nested function's arguments turn into a staircase.
+_BREAK_SHALLOW = ["AND", "OR"]
+
+_INDENT = "  "
+_WORD = re.compile(r"[A-Za-z_][A-Za-z_0-9]*")
+
+
+def _split(sql: str):
+    """Yield (kind, text) where kind is 'str' for a literal and 'sql' for everything else."""
+    i, n, start = 0, len(sql), 0
+    while i < n:
+        c = sql[i]
+        if c in "'\"":
+            if start != i:
+                yield "sql", sql[start:i]
+            quote, j = c, i + 1
+            while j < n:
+                if sql[j] == "\\":
+                    j += 2
+                    continue
+                if sql[j] == quote:
+                    if j + 1 < n and sql[j + 1] == quote:     # '' inside a literal
+                        j += 2
+                        continue
+                    j += 1
+                    break
+                j += 1
+            yield "str", sql[i:j]
+            i = start = j
+            continue
+        i += 1
+    if start < n:
+        yield "sql", sql[start:]
+
+
+def _tokens(sql: str):
+    """A comparable token stream: literals whole, everything else word/symbol by word/symbol."""
+    out = []
+    for kind, text in _split(sql):
+        if kind == "str":
+            out.append(text)
+        else:
+            out.extend(re.findall(r"[A-Za-z_][A-Za-z_0-9]*|[0-9]+|\S", text))
+    return out
+
+
+def _match_keyword(upper: str, pos: int, words):
+    """The longest keyword starting at pos, on word boundaries, or None."""
+    for kw in words:
+        end = pos + len(kw)
+        if not upper.startswith(kw, pos):
+            continue
+        before_ok = pos == 0 or not (upper[pos - 1].isalnum() or upper[pos - 1] == "_")
+        after_ok = end >= len(upper) or not (upper[end].isalnum() or upper[end] == "_")
+        if before_ok and after_ok:
+            return kw
+    return None
+
+
+def format_sql(sql: str, indent: str = _INDENT) -> str:
+    """Return `sql` laid out over lines, or `sql` unchanged when that cannot be done safely."""
+    if not isinstance(sql, str) or not sql.strip():
+        return sql
+    if "\n" in sql.strip():
+        return sql                      # the author already laid it out
+
+    pieces, depth, line_start = [], 0, True
+
+    def push(text):
+        nonlocal line_start
+        if text:
+            pieces.append(text)
+            line_start = text.endswith("\n")
+
+    def newline(extra=0):
+        while pieces and pieces[-1].endswith(" "):
+            pieces[-1] = pieces[-1][:-1]
+        if not pieces:
+            return
+        push("\n" + indent * max(0, depth + extra))
+
+    for kind, text in _split(sql):
+        if kind == "str":
+            push(text)
+            continue
+        upper, i, n = text.upper(), 0, len(text)
+        while i < n:
+            ch = text[i]
+            if ch == "(":
+                depth += 1
+                push(ch)
+                i += 1
+                continue
+            if ch == ")":
+                depth = max(0, depth - 1)
+                push(ch)
+                i += 1
+                continue
+            if ch.isspace():
+                if not line_start:
+                    push(" ")
+                while i < n and text[i].isspace():
+                    i += 1
+                continue
+            kw = _match_keyword(upper, i, _BREAK_BEFORE)
+            if kw is None and depth <= 1:
+                kw = _match_keyword(upper, i, _BREAK_SHALLOW)
+            if kw:
+                newline()
+                push(text[i:i + len(kw)])
+                i += len(kw)
+                continue
+            m = _WORD.match(text, i)
+            if m:
+                push(m.group(0))
+                i = m.end()
+                continue
+            push(ch)
+            i += 1
+
+    out = "".join(pieces).strip()
+    # The safety gate: same tokens in the same order, or give the caller their input back.
+    if _tokens(out) != _tokens(sql):
+        return sql
+    return out
+
+
+def is_unreadable(sql: str, limit: int = SINGLE_LINE_LIMIT) -> bool:
+    """True when a statement is one long line, which is the shape worth complaining about."""
+    return (isinstance(sql, str) and "\n" not in sql.strip()
+            and len(sql.strip()) > limit)

procesio_cli/tools/procesio/swagger.py

@@ -0,0 +1,51 @@
+"""Read the bundled PROCESIO endpoint index (data/endpoints.json).
+
+The index is generated from the live Web-API Swagger (v1.19) at build time and
+shipped with the tool so `list-endpoints` works offline and deterministically.
+Each entry: {method, path, tag, summary, params, body}.
+"""
+from __future__ import annotations
+
+import functools
+import json
+from pathlib import Path
+
+_DATA = Path(__file__).resolve().parent / "data" / "endpoints.json"
+
+
+@functools.lru_cache(maxsize=1)
+def _load() -> dict:
+    return json.loads(_DATA.read_text(encoding="utf-8"))
+
+
+def all_endpoints() -> list[dict]:
+    return list(_load().get("endpoints", []))
+
+
+def meta() -> dict:
+    d = _load()
+    return {k: d[k] for k in ("source", "openapi", "version", "count") if k in d}
+
+
+def tags() -> list[str]:
+    return sorted({e["tag"] for e in all_endpoints()})
+
+
+def find(filter_text: str | None = None, tag: str | None = None,
+         method: str | None = None) -> list[dict]:
+    out = all_endpoints()
+    if tag:
+        tl = tag.lower()
+        out = [e for e in out if e["tag"].lower() == tl]
+    if method:
+        ml = method.upper()
+        out = [e for e in out if e["method"] == ml]
+    if filter_text:
+        ft = filter_text.lower()
+        out = [
+            e for e in out
+            if ft in e["path"].lower()
+            or ft in e["tag"].lower()
+            or ft in (e.get("summary") or "").lower()
+        ]
+    return out

procesio_cli/tools/procesio/tests/conftest.py

@@ -0,0 +1,129 @@
+"""Shared fakes for the procesio test suite - zero live HTTP, zero real keyring.
+
+* FakeCreds   - in-memory stand-in for Windows Credential Manager.
+* FakeResp    - minimal response (status_code/.json()/.text).
+* FakeSession - records every .request(...) and returns canned/queued responses.
+"""
+from __future__ import annotations
+
+import pytest
+
+from tools.procesio import profiles
+
+
+class FakeCreds:
+    def __init__(self):
+        self.store: dict[str, str] = {}
+
+    @staticmethod
+    def _k(tool: str, name: str) -> str:
+        return f"{tool}:{name}"
+
+    def get_optional(self, tool, name):
+        return self.store.get(self._k(tool, name))
+
+    def get(self, tool, name):
+        v = self.get_optional(tool, name)
+        if v is None:
+            raise KeyError(name)
+        return v
+
+    def has_for_report(self, tool, name):
+        """The reporting probe. Real one never raises; here presence is the same
+        answer, so the stub stays honest about the contract rather than the
+        implementation."""
+        return self.get_optional(tool, name) is not None
+
+    def has(self, tool, name):
+        return self.get_optional(tool, name) is not None
+
+    def set_secret(self, tool, name, value):
+        self.store[self._k(tool, name)] = value
+
+    def delete(self, tool, name):
+        self.store.pop(self._k(tool, name), None)
+
+
+class FakeResp:
+    def __init__(self, status: int, body=None, text: str = "", cookies=None,
+                 headers=None, content: bytes = b""):
+        self.status_code = status
+        self._body = body
+        self.text = text
+        self.cookies = dict(cookies or {})      # name -> value (Set-Cookie)
+        self.headers = dict(headers or {})       # response headers
+        self.content = content                   # raw bytes (for request_bytes)
+
+    def json(self):
+        if self._body is None:
+            raise ValueError("no json body")
+        return self._body
+
+
+class FakeSession:
+    """responder(method, url, kwargs) -> FakeResp, OR a queue of FakeResp
+    consumed in order, OR a default 200 {}. Records every call (incl. form
+    `data` and `json` bodies)."""
+
+    def __init__(self, responder=None, queue=None):
+        self.calls: list[dict] = []
+        self.responder = responder
+        self.queue = list(queue or [])
+
+    def request(self, method, url, params=None, json=None, data=None,
+                headers=None, timeout=None, files=None):
+        self.calls.append({"method": method, "url": url, "params": params,
+                           "json": json, "data": data, "headers": headers,
+                           "timeout": timeout, "files": files})
+        if self.responder:
+            return self.responder(method, url, {"params": params, "json": json,
+                                                "data": data, "headers": headers,
+                                                "files": files})
+        if self.queue:
+            return self.queue.pop(0)
+        return FakeResp(200, {})
+
+
+@pytest.fixture(autouse=True)
+def _isolate_userdata(tmp_path, monkeypatch):
+    """Pin the user-data root to a per-test temp dir so the environments registry
+    (config/procesio/environments.json) is empty by default — every test resolves
+    the default environment to Internal-PROD (the historical production URLs) and no
+    test ever reads or writes the developer's real registry."""
+    monkeypatch.setenv("AAT_USERDATA_DIR", str(tmp_path / "userdata"))
+    yield
+
+
+@pytest.fixture(autouse=True)
+def _clear_cookie_mem_cache():
+    """The userpass in-process cookie cache is module-global — clear it around
+    every test so sessions don't leak between tests."""
+    from tools.procesio import auth
+    auth._MEM_COOKIES.clear()
+    yield
+    auth._MEM_COOKIES.clear()
+
+
+@pytest.fixture
+def store(monkeypatch):
+    """In-memory credential store wired into the profiles module."""
+    fake = FakeCreds()
+    monkeypatch.setattr(profiles, "creds", fake)
+    return fake
+
+
+@pytest.fixture
+def apikey_profile(store):
+    profiles.save_profile("k1", {
+        "type": "apikey", "key": "NAME1", "value": "SECRET1",
+        "workspace_id": "ws-guid-1", "workspace": "personal",
+    })
+    return "k1"
+
+
+@pytest.fixture
+def userpass_profile(store):
+    profiles.save_profile("u1", {
+        "type": "userpass", "username": "u@example.com", "password": "pw",
+    })
+    return "u1"