substratum-cli 0.2.2__tar.gz → 0.2.4__tar.gz

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: substratum-cli
-Version: 0.2.2
+Version: 0.2.4
 Summary: Substratum: verified-or-refused code, with provenance. The terminal over the comprehension engine.
 Author: Substratum
 License: LicenseRef-Proprietary

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/__init__.py

@@ -5,4 +5,4 @@ Import budget: this package is stdlib-only; the ONLY module reaching into backen
 """
 # SINGLE SOURCE OF TRUTH for the release version. pyproject reads this (dynamic version), the boot shows
 # it, and the update-check compares against it. To cut a release, bump ONLY this line, then publish.
-__version__ = "0.2.2"
+__version__ = "0.2.4"

substratum_cli-0.2.4/substratum_cli/_vendor/client_gate.py

@@ -0,0 +1,170 @@
+"""client_gate -- the Docker-less INSTANCE-GRAIN gate for client repos.
+
+The commercial unlock: the verified-or-refuse machine currently gates instance-grain
+emissions through benchmark Docker images. A client repo has no image -- it has ITS OWN
+tests. This gate runs them, with DIFFERENTIAL (fail_to_pass) semantics:
+
+    the named tests must FAIL on the UNPATCHED base and PASS with the candidate applied.
+
+The differential is load-bearing for zero-wrong: a candidate that "passes" tests which
+already passed at base has proven NOTHING about the fix (pass-because-it-tests-nothing);
+it is refused NAMED (GATE_TESTS_PASS_AT_BASE), never shipped on that warrant.
+
+Isolation: base_commit is materialized in a TEMP GIT WORKTREE (the proven toolchain_derive
+pattern); the client checkout is never touched; the worktree is always removed.
+
+Outcomes are a CLOSED set and only PASS ships:
+    PASS                  differential held (failed at base, passed with patch)
+    TEST_FAILED           patch applied, tests still fail
+    GATE_TESTS_PASS_AT_BASE  nothing to fix on this warrant -> refuse
+    GATE_BASE_ERROR       tests could not RUN at base (env/collection) -> refuse
+    GATE_ERROR            tests could not RUN with the patch -> refuse (never a pass)
+    GATE_TIMEOUT / GATE_WORKTREE_FAILED / GATE_NO_RUNNER   named infrastructure refusals
+
+Runner resolution (closed, explicit-first): caller-provided command > detected by repo
+shape (pytest for python repos, `go test` for go.mod repos, npm test for package.json with
+a test script). Detection failure refuses NAMED -- the gate never guesses how to test."""
+from __future__ import annotations
+
+import os
+import shlex
+import shutil
+import subprocess
+import tempfile
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Sequence, Tuple
+
+
+@dataclass(frozen=True)
+class ClientGateResult:
+    passed: bool
+    code: str                 # closed set above
+    detail: str = ""
+
+
+def _run(cmd: Sequence[str], cwd: str, timeout: int) -> Tuple[Optional[int], str]:
+    try:
+        # children get a REAL-fs tmpdir: pytest's capture holds unlinked-open tempfiles,
+        # which drvfs/9p mounts cannot represent (FileNotFoundError at truncate).
+        # PYTHONDONTWRITEBYTECODE is a CORRECTNESS requirement, not hygiene: the base run
+        # writes __pycache__; a patched source of the same size within the same second is
+        # SHADOWED by the stale pyc (timestamp validation is second-granular), and the
+        # with-patch run silently re-imports the buggy module -- the differential then
+        # reports TEST_FAILED for a genuine fix (or worse directions on other shapes).
+        # errors="replace": test output is arbitrary user bytes (a test may print binary or a
+        # non-UTF-8 traceback); strict decode would raise UnicodeDecodeError and crash the gate.
+        # The verdict is the return code, never the decoded text, so replacement is verdict-neutral.
+        r = subprocess.run(list(cmd), cwd=cwd, capture_output=True, text=True,
+                           errors="replace", timeout=timeout,
+                           env=dict(os.environ, PYTHONHASHSEED="0", TMPDIR="/tmp",
+                                    PYTHONDONTWRITEBYTECODE="1"))
+        return r.returncode, (r.stdout + "\n" + r.stderr)[-4000:]
+    except subprocess.TimeoutExpired:
+        return None, "timeout"
+    except FileNotFoundError as e:
+        return None, f"runner missing: {e}"
+
+
+def resolve_test_command(repo_dir: str, test_cmd: Optional[str],
+                         test_paths: Optional[Sequence[str]]) -> Optional[List[str]]:
+    """The command that runs the WARRANT tests. Explicit beats detection; detection is a
+    closed table keyed by repo shape; anything else -> None (refuse, never guess)."""
+    test_cmd = test_cmd or os.environ.get("SUBSTRATUM_CLIENT_TEST_CMD")
+    if test_cmd:
+        return shlex.split(test_cmd)
+    paths = list(test_paths or [])
+    if os.path.exists(os.path.join(repo_dir, "go.mod")):
+        go = os.path.expanduser("~/.local/go/bin/go")
+        go = go if os.path.exists(go) else shutil.which("go")
+        if not go:
+            return None
+        # go test takes package dirs + -run for test-name selectors (pkg::Test form)
+        pkgs, runs = [], []
+        for p in paths:
+            if "::" in p:
+                d, name = p.split("::", 1)
+                pkgs.append("./" + os.path.dirname(d) if os.path.dirname(d) else "./...")
+                runs.append(name)
+            else:
+                pkgs.append("./" + p if not p.startswith("./") else p)
+        cmd = [go, "test"] + (sorted(set(pkgs)) or ["./..."])
+        if runs:
+            cmd += ["-run", "|".join(dict.fromkeys(runs))]
+        return cmd
+    if any(os.path.exists(os.path.join(repo_dir, f))
+           for f in ("pytest.ini", "setup.py", "pyproject.toml", "setup.cfg", "tox.ini")) \
+            or os.path.isdir(os.path.join(repo_dir, "tests")) or any(
+                p.endswith(".py") or "::" in p for p in paths):
+        import sys
+        return [sys.executable, "-m", "pytest", "-x", "-q", "-p", "no:cacheprovider",
+                *paths]
+    pkg = os.path.join(repo_dir, "package.json")
+    if os.path.exists(pkg):
+        try:
+            import json
+            if json.load(open(pkg)).get("scripts", {}).get("test"):
+                return ["npm", "test", "--silent"]
+        except Exception:
+            return None
+    return None
+
+
+def run_client_gate(repo_dir: str, base_commit: str, edits: Dict[str, str],
+                    test_paths: Optional[Sequence[str]] = None,
+                    test_cmd: Optional[str] = None,
+                    differential: bool = True,
+                    timeout: int = 600,
+                    transcript_sink: Optional[dict] = None) -> ClientGateResult:
+    """THE gate. Materialize base_commit in a temp worktree; (differential) run the warrant
+    tests unpatched -- they must FAIL; write `edits`; run again -- they must PASS.
+
+    transcript_sink (out-param, additive; the proven refusal_sink precedent): when a dict is
+    passed, it is populated with {runner_command, base_rc, base_out, patched_rc, patched_out}
+    so the product surface can show the "base FAIL -> patched PASS" proof and store the gate
+    logs. Default None keeps every existing caller byte-identical."""
+    wt = tempfile.mkdtemp(prefix="cg_wt_", dir=os.environ.get("TMPDIR") or None)
+    try:
+        r = subprocess.run(["git", "-C", repo_dir, "worktree", "add", "--detach", "-f",
+                            wt, base_commit], capture_output=True, text=True,
+                           errors="replace", timeout=120)
+        if r.returncode != 0:
+            return ClientGateResult(False, "GATE_WORKTREE_FAILED", r.stderr[-300:])
+        cmd = resolve_test_command(wt, test_cmd, test_paths)
+        if cmd is None:
+            return ClientGateResult(False, "GATE_NO_RUNNER")
+        if transcript_sink is not None:
+            transcript_sink["runner_command"] = list(cmd)
+        if differential:
+            rc, out = _run(cmd, wt, timeout)
+            if transcript_sink is not None:
+                transcript_sink["base_rc"] = rc
+                transcript_sink["base_out"] = out
+            if rc is None:
+                return ClientGateResult(False, "GATE_TIMEOUT" if out == "timeout"
+                                        else "GATE_BASE_ERROR", out[-300:])
+            if rc == 0:
+                return ClientGateResult(False, "GATE_TESTS_PASS_AT_BASE")
+            # rc != 0 at base: the warrant tests fail before the fix, as fail_to_pass
+            # requires. (A collection error also exits non-zero; the with-patch run below
+            # is the arbiter -- a real fix makes the SAME invocation exit 0.)
+        for rel, content in edits.items():
+            p = os.path.join(wt, rel)
+            os.makedirs(os.path.dirname(p) or wt, exist_ok=True)
+            with open(p, "w", encoding="utf-8") as fh:
+                fh.write(content)
+        rc, out = _run(cmd, wt, timeout)
+        if transcript_sink is not None:
+            transcript_sink["patched_rc"] = rc
+            transcript_sink["patched_out"] = out
+        if rc is None:
+            return ClientGateResult(False, "GATE_TIMEOUT" if out == "timeout"
+                                    else "GATE_ERROR", out[-300:])
+        if rc == 0:
+            return ClientGateResult(True, "PASS")
+        return ClientGateResult(False, "TEST_FAILED", out[-400:])
+    except subprocess.TimeoutExpired:
+        return ClientGateResult(False, "GATE_WORKTREE_FAILED", "worktree add timeout")
+    finally:
+        subprocess.run(["git", "-C", repo_dir, "worktree", "remove", "-f", wt],
+                       capture_output=True)
+        shutil.rmtree(wt, ignore_errors=True)

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/commands/apply.py

@@ -35,10 +35,10 @@ def cmd_apply(repo: str, run_id: str) -> "_result.SubstratumResult":
         patch = fh.name
     try:
         r = subprocess.run(["git", "-C", repo, "apply", "--index", patch],
-                           capture_output=True, text=True)
+                           capture_output=True, text=True, errors="replace")
         if r.returncode != 0:
             r2 = subprocess.run(["git", "-C", repo, "apply", "--3way", patch],
-                                capture_output=True, text=True)
+                                capture_output=True, text=True, errors="replace")
             if r2.returncode != 0:
                 return _result.make(codes.REFUSED, run_id, "apply", code=codes.APPLY_STATE_DRIFT,
                                     refusal=refusals.build(codes.APPLY_STATE_DRIFT,

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/commands/failing.py

@@ -117,7 +117,8 @@ def cmd_failing(repo: str, path: Optional[str] = None, test_command: Optional[st
     if path:
         run_argv.append(path)
     try:
-        r = subprocess.run(run_argv, cwd=repo, capture_output=True, text=True, timeout=timeout,
+        r = subprocess.run(run_argv, cwd=repo, capture_output=True, text=True,
+                           errors="replace", timeout=timeout,
                            env={**os.environ, "TMPDIR": "/tmp", "CI": "1", "NO_COLOR": "1"})
     except subprocess.TimeoutExpired:
         return _refuse(rid, codes.GATE_PRECONDITION, f"the test suite did not finish within {timeout}s")

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/commands/pr.py

@@ -66,7 +66,7 @@ def cmd_pr(repo: str, run_id: str = None, title: str = None) -> "_result.Substra
     if gh:
         title = title or f"Substratum verified: {res.operation} {target}"[:100]
         r = subprocess.run([gh, "pr", "create", "--title", title, "--body", body],
-                           cwd=repo, capture_output=True, text=True, timeout=60)
+                           cwd=repo, capture_output=True, text=True, errors="replace", timeout=60)
         if r.returncode == 0:
             return _result.make(codes.PASS, run_id, "pr",
                                 data={"url": r.stdout.strip(), "branch": branch, "certified": run_id})

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/engine.py

@@ -26,10 +26,17 @@ _LEAF_CACHE: dict[str, ModuleType] = {}
 
 
 def _leaf(name: str) -> ModuleType:
-    """Load `<engine>/<name>.py` in isolation (no backend __init__), cached."""
+    """Load `<engine>/<name>.py` in isolation (no backend __init__), cached. On a THIN CLIENT
+    (pip install; no engine checkout on disk) fall back to the copy vendored INSIDE the wheel --
+    a pip user must never depend on a path that only exists on a dev machine. Dev machines still
+    get the live engine file first, so engine edits are picked up without a re-vendor."""
     if name in _LEAF_CACHE:
         return _LEAF_CACHE[name]
     path = os.path.join(_ENGINE_DIR, f"{name}.py")
+    if not os.path.isfile(path):
+        vendored = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_vendor", f"{name}.py")
+        if os.path.isfile(vendored):
+            path = vendored
     spec = importlib.util.spec_from_file_location(f"_substratum_leaf_{name}", path)
     if spec is None or spec.loader is None:
         raise ImportError(f"cannot leaf-load {path}")

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/gitio.py

@@ -7,8 +7,13 @@ from typing import List, Optional
 
 
 def _git(repo: str, *args: str, timeout: int = 60) -> subprocess.CompletedProcess:
+    # errors="replace": git output carries arbitrary working-tree bytes (a `diff` of a large file
+    # git treats as text but that holds a non-UTF-8 byte; a non-ASCII untracked filename). Strict
+    # decode would raise UnicodeDecodeError -- this runs on EVERY command via the run-id dirty-hash,
+    # so a single binary-ish file in the user's tree would crash the whole CLI. Replacement is
+    # deterministic (same bytes -> same text -> same hash), so the run-id story is preserved.
     return subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True,
-                          timeout=timeout)
+                          errors="replace", timeout=timeout)
 
 
 def is_git_repo(repo: str) -> bool:
@@ -23,11 +28,13 @@ def apply_reverse(repo: str, diff_text: str) -> tuple[bool, str]:
     if not diff_text.strip():
         return False, "empty diff"
     r = subprocess.run(["git", "-C", repo, "apply", "--reverse", "--3way", "-"],
-                       input=diff_text, capture_output=True, text=True, timeout=60)
+                       input=diff_text, capture_output=True, text=True,
+                       errors="replace", timeout=60)
     if r.returncode == 0:
         return True, "reverted"
     r2 = subprocess.run(["git", "-C", repo, "apply", "--reverse", "-"],
-                        input=diff_text, capture_output=True, text=True, timeout=60)
+                        input=diff_text, capture_output=True, text=True,
+                        errors="replace", timeout=60)
     if r2.returncode == 0:
         return True, "reverted"
     err = (r.stderr or r2.stderr or "").strip().splitlines()

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/product_ops.py

@@ -41,6 +41,15 @@ def op_fix_from_failing_test(repo: str, failing_tests: list[str], intent: str =
     stage("GROUND", f"reading {len(failing_tests)} failing test(s)")
 
     if not engine.available():        # client-only install: fix needs the engine (local or hosted)
+        # HOSTED: when logged in, the server SYNTHESIZES from focal snippets (only the focal module +
+        # test NAMES transit; test bodies and the wider repo never leave) and the candidate diff is
+        # gated HERE, differentially, on the user's own tests. The gate stays the arbiter, client-side.
+        from . import remote
+        if remote.enabled():
+            rr = _remote_fix(repo, base, rid, key, inputs, failing_tests, intent,
+                             test_command, target, stage)
+            if rr is not None:
+                return rr
         ref = refusals.build(codes.NO_ENGINE, engine_code="NO_ENGINE",
                              ctx={"target": target or "the failing test", "run_id": rid})
         return _store(repo, _result.make(codes.REFUSED, rid, "fix", code=codes.NO_ENGINE, refusal=ref),
@@ -155,6 +164,84 @@ def _route_test_derived(repo, base_commit, failing_tests, intent, test_command,
     return None
 
 
+def _focal_context(repo: str, failing_tests: list, intent: str):
+    """Build the focal payload for the hosted engine: parse the failing TEST locally (it never
+    transits), find the module it imports-and-calls, and send ONLY that module's content + names.
+    Returns the focal dict or None if no focal module is discoverable."""
+    test_files = sorted({t.split("::")[0] for t in failing_tests if "::" in t or t.endswith(".py")})
+    for tf in test_files:
+        tf_abs = os.path.join(repo, tf)
+        if not os.path.isfile(tf_abs):
+            continue
+        tsrc = open(tf_abs, encoding="utf-8", errors="replace").read()
+        try:
+            tree = ast.parse(tsrc)
+        except SyntaxError:
+            continue
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
+                mod_rel = node.module.replace(".", "/") + ".py"
+                mod_abs = os.path.join(repo, mod_rel)
+                if not os.path.isfile(mod_abs):
+                    continue
+                for a in node.names:
+                    if re.search(rf"\b{re.escape(a.name)}\s*\(", tsrc):
+                        content = open(mod_abs, encoding="utf-8", errors="replace").read()
+                        return {"files": {mod_rel: content}, "path": mod_rel,
+                                "function": a.name, "buggy_source": content,
+                                "repo": os.path.basename(repo), "intent": intent or ""}
+    return None
+
+
+def _remote_fix(repo, base, rid, key, inputs, failing_tests, intent, test_command, target, stage):
+    """fix over the hosted engine: POST focal -> receive a candidate diff -> GATE IT HERE
+    (differential, vendored client gate) -> VERIFIED or a named refusal. Returns None only when
+    the remote is unreachable AND no focal exists (falls through to the NO_ENGINE refusal)."""
+    from . import diffs, remote
+    focal = _focal_context(repo, failing_tests, intent)
+    if focal is None:
+        ref = refusals.from_engine("no_candidate", ctx={"target": target, "run_id": rid})
+        return _store(repo, _result.make(codes.REFUSED, rid, "fix", code=ref.code, refusal=ref),
+                      key, inputs)
+    stage("REMOTE", f"hosted engine synthesizes {focal['function']} (focal module only transits)")
+    try:
+        res = remote.candidate(list(failing_tests), focal)
+    except remote.RemoteError as e:
+        ref = refusals.build(codes.GATE_PRECONDITION, engine_code=str(e),
+                             ctx={"target": target, "run_id": rid})
+        return _store(repo, _result.make(codes.REFUSED, rid, "fix",
+                                         code=codes.GATE_PRECONDITION, refusal=ref), key, inputs)
+    if (res.get("status") or "") != "candidate" or not res.get("diff"):
+        reason = res.get("refuse_reason") or "no_candidate"
+        ref = refusals.from_engine(reason, ctx={"target": target, "run_id": rid})
+        return _store(repo, _result.make(codes.REFUSED, rid, "fix", code=ref.code, refusal=ref),
+                      key, inputs)
+    # the diff came back against the focal content we sent -- apply to that, then GATE locally
+    parsed = diffs.parse(res["diff"])
+    edits, stale = diffs.materialize({f.path: focal["files"].get(f.path, "") for f in parsed}, parsed)
+    if not edits:
+        ref = refusals.build(codes.HUNK_CONTEXT_STALE, engine_code="remote_diff_stale",
+                             ctx={"target": ", ".join(stale) or target, "run_id": rid})
+        return _store(repo, _result.make(codes.UNVERIFIABLE, rid, "fix",
+                                         code=codes.HUNK_CONTEXT_STALE, refusal=ref), key, inputs)
+    stage("GATE", f"differential gate on {', '.join(failing_tests)}")
+    sink: dict = {}
+    cg = engine.client_gate.run_client_gate(repo, base, edits, test_paths=list(failing_tests),
+                                            test_cmd=test_command, differential=True,
+                                            transcript_sink=sink)
+    if not cg.passed:
+        ref = refusals.from_engine("candidate_failed_execution_gate",
+                                   ctx={"target": target, "run_id": rid})
+        return _store(repo, _result.make(codes.REFUSED, rid, "fix", code=ref.code, refusal=ref,
+                                         gate=_gate_from_sink(sink, failing_tests)), key, inputs)
+    prov = [str(p) for p in (res.get("provenance") or [])][:8] + ["execution_gate:client:PASS"]
+    diff = "".join(_diff_for(repo, base, p, c) for p, c in sorted(edits.items()))
+    out = _result.make(codes.VERIFIED, rid, "fix", diff=diff, files=tuple(sorted(edits)),
+                       provenance=_result.Provenance(entries=tuple(prov)),
+                       gate=_gate_from_sink(sink, failing_tests), data={"target": target})
+    return _store(repo, out, key, inputs)
+
+
 def op_write_from_intent(repo: str, intent: str, signature: str,
                          into: Optional[str] = None, emit=None,
                          prefer_purpose: Optional[str] = None) -> "_result.SubstratumResult":

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/remote.py

@@ -85,8 +85,10 @@ def clarify(intent: str, signature: str) -> dict:
 def candidate(test_ids: list, focal: dict) -> dict:
     """POST /api/cli/candidate -> {status, diff, provenance, refuse_reason}. The server SYNTHESIZES;
     the client gates the returned diff locally, so the user's code + tests never leave the machine."""
+    # generous timeout: the first call on a cold server pays the numpy import chain + the lazy
+    # library's resident index load before it can synthesize
     return _post_json(account.server() + "/api/cli/candidate",
-                      {"test_ids": list(test_ids), "focal": focal}, _auth_headers())
+                      {"test_ids": list(test_ids), "focal": focal}, _auth_headers(), timeout=180)
 
 
 # ---- GitHub device-flow login --------------------------------------------------------------------

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli/session.py

@@ -592,7 +592,7 @@ class Session:
         head("MEMORY & PROOF")
         row("runs", "all you have done, and which refusals became fixes")
         row("show <run>", "see a run;  replay <run>  re-runs it byte-identical")
-        row("history", "re-run earlier commands  (!<n>  !!  !<prefix>)")
+        row("history", "the whole build trail;  history <word> searches;  run <n> replays a line")
         head("SETUP")
         row("doctor / init", "check the environment / what this repo can gate")
         row("login", "connect the hosted engine for fix and write")
@@ -638,8 +638,9 @@ class Session:
                 sys.stdout.write("\n")
                 self._say("  cancelled (session still warm)", _CYAN)
                 continue
-            except Exception as e:  # a command fault must not kill the session
-                self._say(f"  error: {e}", _CYAN)
+            except Exception as e:  # belt: the ops refuse, never raise -- but a fault must not kill the session
+                self._say(f"  unexpected internal error: {e}", _CYAN)
+                self._say("  the session is still warm; retry or try a different request", _DIM)
                 continue
             if not keep:
                 _leave()

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: substratum-cli
-Version: 0.2.2
+Version: 0.2.4
 Summary: Substratum: verified-or-refused code, with provenance. The terminal over the comprehension engine.
 Author: Substratum
 License: LicenseRef-Proprietary

{substratum_cli-0.2.2 → substratum_cli-0.2.4}/substratum_cli.egg-info/SOURCES.txt

@@ -32,6 +32,7 @@ substratum_cli.egg-info/requires.txt
 substratum_cli.egg-info/top_level.txt
 substratum_cli/_vendor/__init__.py
 substratum_cli/_vendor/_pro_test_derive.py
+substratum_cli/_vendor/client_gate.py
 substratum_cli/commands/__init__.py
 substratum_cli/commands/apply.py
 substratum_cli/commands/auth.py