vibexray 0.1.0__py3-none-any.whl

vibexray/__init__.py

@@ -0,0 +1,3 @@
+"""X-ray an AI-built prototype for the engineer who has to build it for real."""
+
+__version__ = "0.1.0"

vibexray/__main__.py

@@ -0,0 +1,3 @@
+from vibexray.cli import main
+
+raise SystemExit(main())

vibexray/apprun.py

@@ -0,0 +1,273 @@
+"""Start the prototype in a clean environment, crawl it, and always stop it again."""
+
+from __future__ import annotations
+
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from pathlib import Path
+
+from vibexray.apprun_boot import (
+    USUAL_PORTS,
+    Process,
+    clean_env,
+    detect_package_manager,
+    detect_script,
+    find_env_names,
+    free_port,
+    log_tail,
+    port_open,
+    python_entry,
+    read_package,
+    reap_strays,
+    url_from_log,
+    wait_for_port_free,
+)
+from vibexray.apprun_crawl import RUN_EXTRA_HINT, crawl
+from vibexray.model import AppRun
+from vibexray.rules.base import is_env_file
+
+BOOT_SECONDS = 90
+COPY_SKIP = (".git", "node_modules", ".next", "dist", "build", "vibexray-report")
+BUILD_MISSING = re.compile(r"(not found|ENOENT|Cannot find).*\b(dist|build|\.next)\b", re.I)
+INSTALL_SECONDS = 300
+INSTALL_ARGS = {
+    "npm": ["install", "--ignore-scripts", "--no-audit", "--no-fund"],
+    "pnpm": ["install", "--ignore-scripts"],
+    "yarn": ["install", "--ignore-scripts"],
+    "bun": ["install", "--ignore-scripts"],
+}
+
+
+def _not_attempted(reason: str) -> AppRun:
+    return AppRun(state="not_attempted", reason=reason)
+
+
+def run_app(root: Path, out_dir: Path, enabled: bool) -> AppRun:
+    if not enabled:
+        return _not_attempted("Skipped: the scan ran with --no-run.")
+    pkg = read_package(root)
+    entry = python_entry(root)
+    if pkg is None and entry is None:
+        return _not_attempted(
+            "There is no package.json or Python start file in the top folder, so vibexray "
+            "does not know how to start this app."
+        )
+    out_dir.mkdir(parents=True, exist_ok=True)
+    # The copy lives outside the user's folder, so a killed scan never leaves it there.
+    try:
+        run_dir = Path(tempfile.mkdtemp(prefix="vibexray-run-"))
+    except OSError as exc:
+        return AppRun(
+            state="could_not_boot",
+            reason=f"vibexray could not make a temporary folder for the app ({exc.strerror}).",
+        )
+    home = run_dir / "home"
+    home.mkdir()
+    previous = _trap_stop_signals()
+    try:
+        # Install and start in a copy, so the user's folder never changes.
+        try:
+            work = _make_copy(root, run_dir / "app", out_dir)
+        except (shutil.Error, OSError) as exc:
+            return AppRun(
+                state="could_not_boot",
+                reason="vibexray could not copy the app to start it. " + _copy_problem(exc),
+            )
+        if pkg is not None:
+            script = detect_script(pkg)
+            if script is None:
+                return _not_attempted(
+                    "package.json has no dev, start, or preview script, so vibexray cannot start it."
+                )
+            manager = detect_package_manager(root)
+            command = f"{manager} run {script}"
+            return _boot_and_crawl(work, out_dir, home, command, [manager, "run", script], manager)
+        cmd = ["python3", entry]
+        if entry == "manage.py":
+            cmd = ["python3", entry, "runserver"]
+        return _boot_and_crawl(work, out_dir, home, " ".join(cmd), cmd, None)
+    except PermissionError as exc:
+        return AppRun(
+            state="could_not_boot",
+            reason=(
+                "This computer blocked the app from starting "
+                f"({exc.strerror or exc}). A sandbox, such as the one in Codex, does this. "
+                "Run the scan in a normal terminal to see the app run."
+            ),
+        )
+    finally:
+        reap_strays(home)
+        shutil.rmtree(run_dir, ignore_errors=True)
+        _restore_signals(previous)
+
+
+def _trap_stop_signals() -> dict | None:
+    """Turn SIGTERM and SIGHUP into SystemExit, so the cleanup in run_app always runs."""
+    if threading.current_thread() is not threading.main_thread():
+        return None
+
+    def stop(signum, _frame):
+        raise SystemExit(128 + signum)
+
+    return {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGHUP)}
+
+
+def _restore_signals(previous: dict | None) -> None:
+    for sig, handler in (previous or {}).items():
+        signal.signal(sig, handler)
+
+
+def _copy_problem(exc: Exception) -> str:
+    # shutil.Error carries a list of (source, destination, reason) for each failed file.
+    if isinstance(exc, shutil.Error) and exc.args and isinstance(exc.args[0], list):
+        src, _, why = exc.args[0][0]
+        return f"It could not read {Path(src).name}: {str(why).split(': ')[0]}."
+    return f"{getattr(exc, 'strerror', None) or exc}."
+
+
+def _make_copy(root: Path, dest: Path, out_dir: Path) -> Path:
+    skip = set(COPY_SKIP)
+    out_resolved = out_dir.resolve()
+
+    def ignore(folder: str, names: list[str]) -> set[str]:
+        left_out = set()
+        for n in names:
+            path = Path(folder) / n
+            # .env files hold the user's keys. The app starts without them.
+            if (
+                n in skip
+                or is_env_file(n)
+                or path.resolve() == out_resolved
+                or not (path.is_symlink() or path.is_dir() or path.is_file())
+            ):
+                left_out.add(n)
+        return left_out
+
+    shutil.copytree(root, dest, ignore=ignore, symlinks=True)
+    if (root / "node_modules").is_dir():
+        _clone(root / "node_modules", dest / "node_modules")
+    return dest
+
+
+def _clone(src: Path, dest: Path) -> None:
+    """Copy node_modules copy-on-write, so caches the dev server writes stay in the copy.
+
+    APFS and Btrfs share the file blocks, so the clone costs almost no disk. If the clone
+    fails, the copy has no node_modules and the app run installs the packages instead.
+    """
+    if sys.platform == "darwin":
+        cmd = ["cp", "-cR", str(src), str(dest)]
+    else:
+        cmd = ["cp", "-R", "--reflink=auto", str(src), str(dest)]
+    if subprocess.run(cmd, capture_output=True).returncode != 0:
+        shutil.rmtree(dest, ignore_errors=True)
+
+
+def _boot_and_crawl(root, out_dir, home, command, cmd, manager) -> AppRun:
+    result = AppRun(command=command)
+    port = free_port()
+    env = clean_env(home, port)
+    tool = cmd[0]
+    if shutil.which(tool, path=env["PATH"]) is None:
+        return _failed(
+            result,
+            root,
+            f"{tool} is not installed on this computer, so the app did not start.",
+            None,
+        )
+    if manager and not (root / "node_modules").is_dir():
+        problem = _install(root, env, home, manager)
+        if problem:
+            return _failed(result, root, problem[0], problem[1])
+    # Ports that were already busy before the start belong to someone else.
+    fallback = [p for p in USUAL_PORTS if not port_open(p)]
+    server = Process(cmd, root, env, home / "server.log")
+    try:
+        url, why = _wait_for_url(server, port, fallback)
+        if url is None:
+            return _failed(result, root, why, server.log)
+        result.url = url
+        try:
+            result.pages = crawl(url, root, out_dir)
+        except ImportError:
+            return _failed(result, root, RUN_EXTRA_HINT, None, keep_url=True)
+        except Exception as exc:  # noqa: BLE001 - browser launch problems must not leak a traceback
+            return _failed(
+                result,
+                root,
+                "The browser could not open the app: "
+                + str(exc).splitlines()[0][:160]
+                + " Run: playwright install chromium",
+                None,
+                keep_url=True,
+            )
+        result.state = "ran"
+        n = len(result.pages)
+        result.reason = f"vibexray started the app and visited {n} page{'' if n == 1 else 's'}."
+        return result
+    finally:
+        server.stop()
+        wait_for_port_free(port, 5.0)
+        if result.url:
+            wait_for_port_free(int(result.url.rsplit(":", 1)[1]), 5.0)
+
+
+def _install(root: Path, env: dict, home: Path, manager: str) -> tuple[str, Path] | None:
+    log = home / "install.log"
+    proc = Process([manager, *INSTALL_ARGS[manager]], root, env, log)
+    try:
+        code = proc.wait(INSTALL_SECONDS)
+    finally:
+        proc.stop()
+    if code is None:
+        return (
+            "Installing the app's packages took longer than 10 minutes, so vibexray stopped.",
+            log,
+        )
+    if code != 0:
+        return ("Installing the app's packages failed, so the app did not start.", log)
+    return None
+
+
+def _wait_for_url(server: Process, port: int, fallback: list[int]) -> tuple[str | None, str]:
+    deadline = time.monotonic() + BOOT_SECONDS
+    while time.monotonic() < deadline:
+        if not server.alive():
+            return None, "The app started and then stopped on its own, so it cannot run here."
+        url = url_from_log(server.log)
+        if url:
+            port_in_url = int(url.rsplit(":", 1)[1])
+            if port_open(port_in_url):
+                return url, ""
+        elif port_open(port):
+            return f"http://localhost:{port}", ""
+        else:
+            for p in fallback:
+                if port_open(p):
+                    return f"http://localhost:{p}", ""
+        time.sleep(0.5)
+    return None, f"The app did not answer within {BOOT_SECONDS} seconds."
+
+
+def _failed(
+    result: AppRun, root: Path, reason: str, log: Path | None, keep_url: bool = False
+) -> AppRun:
+    result.state = "could_not_boot"
+    missing = find_env_names(root)
+    result.missing_env = missing
+    tail = log_tail(log) if log is not None else []
+    if any(BUILD_MISSING.search(line) for line in tail):
+        reason += " A build step looks missing: the start file does not exist yet."
+    elif missing and log is not None:
+        reason += " It may need settings or keys that vibexray does not pass on."
+    result.reason = reason
+    result.log_tail = tail
+    if not keep_url:
+        result.url = None
+    return result

vibexray/apprun_boot.py

@@ -0,0 +1,273 @@
+"""Pure helpers and process control for the app run: detection, URL parsing, safe start and stop."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import re
+import signal
+import socket
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+from vibexray.rules.base import redact
+
+ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]")
+URL = re.compile(r"https?://(localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1?\]):(\d{2,5})")
+ENV_CODE = re.compile(r"(?:process\.env|import\.meta\.env)\.([A-Za-z_][A-Za-z0-9_]*)")
+ENV_KEY = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=", re.M)
+SOURCE_SUFFIXES = {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".vue", ".svelte"}
+SKIP_DIRS = {"node_modules", ".git", "dist", "build", ".next", ".nuxt", ".svelte-kit", "tmp"}
+# Set by the runtime or by the clean environment itself, so never "missing".
+BUILTIN_ENV = {"NODE_ENV", "PORT", "PATH", "HOME", "MODE", "DEV", "PROD", "SSR", "BASE_URL", "CI"}
+USUAL_PORTS = (3000, 5173, 4173, 8080, 8000, 4321, 3001)
+PYTHON_ENTRIES = ("app.py", "main.py", "server.py", "manage.py")
+SCRIPT_ORDER = ("dev", "start", "preview")
+
+
+def parse_url(line: str) -> str | None:
+    """Return the local URL a dev server prints, or None. Network lines are skipped."""
+    text = ANSI.sub("", line)
+    if "Network" in text:
+        return None
+    m = URL.search(text)
+    if not m:
+        return None
+    host = "127.0.0.1" if m.group(1) == "127.0.0.1" else "localhost"
+    return f"http://{host}:{m.group(2)}"
+
+
+def detect_package_manager(root: Path) -> str:
+    # Bun first: a repo with both bun.lock and package-lock.json usually runs its scripts with bun.
+    if (root / "bun.lock").exists() or (root / "bun.lockb").exists():
+        return "bun"
+    if (root / "pnpm-lock.yaml").exists():
+        return "pnpm"
+    if (root / "yarn.lock").exists():
+        return "yarn"
+    return "npm"
+
+
+def detect_script(pkg: dict) -> str | None:
+    scripts = pkg.get("scripts")
+    if not isinstance(scripts, dict):
+        return None
+    return next((s for s in SCRIPT_ORDER if s in scripts), None)
+
+
+def read_package(root: Path) -> dict | None:
+    try:
+        data = json.loads((root / "package.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def python_entry(root: Path) -> str | None:
+    return next((e for e in PYTHON_ENTRIES if (root / e).is_file()), None)
+
+
+def iter_source(root: Path):
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
+        for name in sorted(filenames):
+            path = Path(dirpath) / name
+            if path.suffix in SOURCE_SUFFIXES:
+                yield path
+
+
+def find_env_names(root: Path) -> list[str]:
+    names: set[str] = set()
+    for path in iter_source(root):
+        try:
+            names.update(ENV_CODE.findall(path.read_text(encoding="utf-8", errors="ignore")))
+        except OSError:
+            continue
+    for example in (".env.example", ".env.sample", ".env.template"):
+        try:
+            names.update(ENV_KEY.findall((root / example).read_text(encoding="utf-8")))
+        except OSError:
+            continue
+    return sorted(names - BUILTIN_ENV)
+
+
+def clean_env(home: Path, port: int) -> dict[str, str]:
+    """The only variables a scanned app sees. The user's own keys never pass through."""
+    return {"PATH": os.environ.get("PATH", ""), "HOME": str(home), "PORT": str(port)}
+
+
+def port_open(port: int, host: str = "127.0.0.1") -> bool:
+    with socket.socket() as s:
+        s.settimeout(0.5)
+        return s.connect_ex((host, port)) == 0
+
+
+def free_port() -> int:
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+def log_tail(path: Path, n: int = 20) -> list[str]:
+    try:
+        lines = ANSI.sub("", path.read_text(encoding="utf-8", errors="replace")).splitlines()
+    except OSError:
+        return []
+    return [redact(line) for line in lines if line.strip()][-n:]
+
+
+def url_from_log(path: Path) -> str | None:
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return None
+    for line in text.splitlines():
+        url = parse_url(line)
+        if url:
+            return url
+    return None
+
+
+class Process:
+    """A child in its own process group, so one call stops it and everything it spawned."""
+
+    def __init__(self, cmd: list[str], cwd: Path, env: dict[str, str], log: Path):
+        log.parent.mkdir(parents=True, exist_ok=True)
+        self.log = log
+        self._fh = open(log, "wb")  # noqa: SIM115 - closed in stop()
+        try:
+            self.proc = subprocess.Popen(
+                cmd,
+                cwd=cwd,
+                env=env,
+                stdin=subprocess.DEVNULL,
+                stdout=self._fh,
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+            )
+        except OSError:
+            self._fh.close()
+            raise
+
+    def alive(self) -> bool:
+        return self.proc.poll() is None
+
+    def wait(self, timeout: float) -> int | None:
+        try:
+            return self.proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            return None
+
+    def stop(self) -> None:
+        pgid = self.proc.pid
+        # Some tools start helpers in their own session (Next.js telemetry does). The group
+        # kill misses those, so list every descendant first and stop them by pid too.
+        kids = _descendants(pgid)
+        for sig in (signal.SIGTERM, signal.SIGKILL):
+            _signal_all(pgid, kids, sig)
+            deadline = time.monotonic() + 5.0
+            while time.monotonic() < deadline and (_group_alive(pgid) or _any_alive(kids)):
+                time.sleep(0.1)
+            if not _group_alive(pgid) and not _any_alive(kids):
+                break
+        with contextlib.suppress(subprocess.TimeoutExpired):
+            self.proc.wait(timeout=1)
+        self._fh.close()
+
+
+def _ps(args: list[str]) -> str:
+    # A sandbox, such as the one Codex uses, can block ps. The process group kill still works.
+    try:
+        return subprocess.run(["ps", *args], capture_output=True, text=True).stdout
+    except OSError:
+        return ""
+
+
+def _descendants(pid: int) -> list[int]:
+    out = _ps(["-axo", "pid=,ppid="])
+    children: dict[int, list[int]] = {}
+    for line in out.splitlines():
+        parts = line.split()
+        if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit():
+            children.setdefault(int(parts[1]), []).append(int(parts[0]))
+    found, todo = [], [pid]
+    while todo:
+        for kid in children.get(todo.pop(), []):
+            found.append(kid)
+            todo.append(kid)
+    return found
+
+
+def _signal_all(pgid: int, pids: list[int], sig: int) -> None:
+    if pgid > 0:
+        with contextlib.suppress(ProcessLookupError, PermissionError):
+            os.killpg(pgid, sig)
+    for pid in pids:
+        with contextlib.suppress(ProcessLookupError, PermissionError):
+            os.kill(pid, sig)
+
+
+def _any_alive(pids: list[int]) -> bool:
+    for pid in pids:
+        try:
+            os.kill(pid, 0)
+        except (ProcessLookupError, PermissionError):
+            continue
+        return True
+    return False
+
+
+def _group_alive(pgid: int) -> bool:
+    try:
+        os.killpg(pgid, 0)
+    except (ProcessLookupError, PermissionError):
+        return False
+    return True
+
+
+def wait_for_port_free(port: int, seconds: float = 10.0) -> bool:
+    deadline = time.monotonic() + seconds
+    while time.monotonic() < deadline:
+        if not port_open(port):
+            return True
+        time.sleep(0.2)
+    return not port_open(port)
+
+
+def reap_strays(home: Path, seconds: float = 10.0) -> None:
+    """Stop leftovers that escaped the process group, such as the detached telemetry flush
+    that Next.js starts at exit. Only processes that carry our unique HOME in their
+    environment count, so the user's own processes are never touched."""
+    marker = f"HOME={home.resolve()}"
+    start = time.monotonic()
+    for _ in range(int(seconds / 0.2)):
+        pids = _pids_with_env(marker)
+        if not pids:
+            return
+        if time.monotonic() - start > 2.0:
+            _signal_all(-1, pids, signal.SIGKILL)
+        time.sleep(0.2)
+
+
+def _pids_with_env(marker: str) -> list[int]:
+    me = os.getpid()
+    if sys.platform == "darwin":
+        # -E appends each process's environment to its command line.
+        found = []
+        for line in _ps(["-axE", "-o", "pid=,command="]).splitlines():
+            pid, _, rest = line.strip().partition(" ")
+            if pid.isdigit() and int(pid) != me and f" {marker}" in f" {rest} ":
+                found.append(int(pid))
+        return found
+    found = []
+    for entry in Path("/proc").glob("[0-9]*"):
+        try:
+            env = (entry / "environ").read_bytes().split(b"\0")
+        except OSError:
+            continue
+        if marker.encode() in env and int(entry.name) != me:
+            found.append(int(entry.name))
+    return found