vigil-codeintel 0.1.0__py3-none-any.whl

vigil_mapper/runtime_tracer.py

@@ -0,0 +1,243 @@
+"""Runtime tracer orchestrator -- Map 2 subprocess-based startup capture.
+
+Launches a target Python module in an isolated subprocess with sys.settrace
+and __import__ hooks installed INSIDE that subprocess only (never in parent).
+
+This module is the PARENT-side orchestrator only. It:
+  - Sanitises environment (strips known API keys and secrets).
+  - Spawns runtime_tracer_entry as a subprocess.
+  - Reads the JSON output file written by the subprocess.
+  - Returns structured trace results for merging into RuntimeNode map.
+
+Security guarantees (per plan sec.7.2):
+  - BLOCKED_ENV vars are never passed to subprocess.
+  - shell=False always -- no shell injection possible.
+  - Timeout enforced via subprocess.run(timeout=...).
+
+Public API:
+    capture_startup_trace(target_module, target_argv, project_dir, timeout_s)
+        -> dict with keys: events, import_events, exit_code, duration_s, stderr
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Sequence
+
+from .map_errors import RuntimeTracerError, RuntimeTracerTimeoutError
+
+__all__ = ["capture_startup_trace", "BLOCKED_ENV"]
+
+_log = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Security: blocked environment variable names (plan §7.2)
+# ---------------------------------------------------------------------------
+
+BLOCKED_ENV: frozenset[str] = frozenset({
+    "ANTHROPIC_API_KEY",
+    "GEMINI_API_KEY",
+    "OPENAI_API_KEY",
+    "GITHUB_TOKEN",
+    "AWS_SECRET_ACCESS_KEY",
+    "AWS_ACCESS_KEY_ID",
+    "SSH_AUTH_SOCK",
+    "PERPLEXITY_API_KEY",
+})
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _build_sanitised_env(project_dir: Path | None) -> dict[str, str]:
+    """Return os.environ copy with BLOCKED_ENV keys removed + tracer marker set."""
+    env = {k: v for k, v in os.environ.items() if k not in BLOCKED_ENV}
+    env["VIGIL_MAPPER_TRACE"] = "1"
+
+    if project_dir is not None:
+        existing_pp = env.get("PYTHONPATH", "")
+        project_str = str(project_dir.resolve())
+        if existing_pp:
+            env["PYTHONPATH"] = project_str + os.pathsep + existing_pp
+        else:
+            env["PYTHONPATH"] = project_str
+
+    return env
+
+
+def _build_argv(
+    target_module: str,
+    temp_path: str,
+    timeout_s: float,
+    target_argv: Sequence[str],
+) -> list[str]:
+    """Build subprocess argv list (shell=False safe, no injection possible)."""
+    argv = [
+        sys.executable,
+        "-m",
+        "vigil_mapper.runtime_tracer_entry",
+        "--target", target_module,
+        "--out", temp_path,
+        "--timeout-s", str(timeout_s),
+    ]
+    if target_argv:
+        argv += ["--", *target_argv]
+    return argv
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def capture_startup_trace(
+    target_module: str,
+    target_argv: Sequence[str] = (),
+    project_dir: Path | None = None,
+    timeout_s: float = 30.0,
+) -> dict:
+    """Capture startup trace of target_module by running it in a subprocess.
+
+    The subprocess installs sys.settrace and a __import__ hook, runs the
+    target, then writes a JSON file with all captured events.
+
+    Args:
+        target_module: Dotted Python module name to run (e.g. "json" or
+            "mypackage.app").
+        target_argv: Arguments forwarded to the target module's sys.argv.
+        project_dir: If provided, prepended to subprocess PYTHONPATH so the
+            target module can be imported.
+        timeout_s: Maximum seconds to allow the subprocess to run. Hard kill
+            at timeout_s + 5 seconds via subprocess.run timeout param.
+
+    Returns:
+        dict with keys:
+            events         - list[dict]: call-level trace events
+            import_events  - list[dict]: import hook events
+            exit_code      - int: target exit code (0 = normal)
+            duration_s     - float: elapsed time inside subprocess
+            stderr         - str: subprocess stderr output
+
+    Raises:
+        RuntimeTracerTimeoutError: If subprocess does not complete within
+            timeout_s + 5 seconds.
+        RuntimeTracerError: If the tracer entry itself fails to produce valid
+            JSON output on a zero exit.
+    """
+    import subprocess
+
+    if project_dir is not None:
+        project_dir = project_dir.resolve()
+
+    env = _build_sanitised_env(project_dir)
+
+    # Create temp file. We close the fd immediately and pass the path to the
+    # subprocess. The subprocess writes JSON there. We read it AFTER the
+    # subprocess finishes (before we unlink).
+    tmp_fd, tmp_path = tempfile.mkstemp(suffix=".json", prefix="vigil_trace_")
+    os.close(tmp_fd)
+
+    argv = _build_argv(target_module, tmp_path, timeout_s, target_argv)
+
+    _log.info(
+        "capture_startup_trace: spawning subprocess for target=%r timeout=%.1fs",
+        target_module,
+        timeout_s,
+    )
+    t_wall = time.perf_counter()
+
+    timed_out = False
+    proc = None
+    try:
+        proc = subprocess.run(
+            argv,
+            env=env,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=timeout_s + 5.0,
+            shell=False,
+        )
+    except subprocess.TimeoutExpired as exc:
+        timed_out = True
+        _log.error(
+            "capture_startup_trace: subprocess timed out after %.1fs for target=%r",
+            timeout_s + 5.0,
+            target_module,
+        )
+        raise RuntimeTracerTimeoutError(
+            "runtime tracer timed out after %.1fs for target %r" % (timeout_s + 5.0, target_module)
+        ) from exc
+    finally:
+        if timed_out:
+            # On timeout we can't read a partial file — just clean up.
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+
+    wall_elapsed = time.perf_counter() - t_wall
+
+    # Read output JSON from temp file, then clean up.
+    payload: dict = {}
+    read_error: str | None = None
+    try:
+        raw = Path(tmp_path).read_text(encoding="utf-8")
+        if raw.strip():
+            payload = json.loads(raw)
+    except (OSError, json.JSONDecodeError) as exc:
+        read_error = str(exc)
+    finally:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+
+    if proc.returncode != 0:
+        _log.error(
+            "capture_startup_trace: subprocess exited with code %d for target=%r stderr=%r",
+            proc.returncode,
+            target_module,
+            proc.stderr[:500] if proc.stderr else "",
+        )
+        # Return partial result (events captured so far) — caller decides degraded mode.
+        return {
+            "events": payload.get("events", []),
+            "import_events": payload.get("import_events", []),
+            "exit_code": proc.returncode,
+            "duration_s": payload.get("duration_s", wall_elapsed),
+            "stderr": proc.stderr or "",
+        }
+
+    if read_error is not None:
+        raise RuntimeTracerError(
+            "runtime tracer for target %r exited 0 but output file unreadable: %s"
+            % (target_module, read_error)
+        )
+
+    if not payload:
+        raise RuntimeTracerError(
+            "runtime tracer for target %r exited 0 but produced empty JSON output"
+            % target_module
+        )
+
+    _log.debug(
+        "capture_startup_trace: done exit=0 wall=%.2fs events=%d imports=%d",
+        wall_elapsed,
+        len(payload.get("events", [])),
+        len(payload.get("import_events", [])),
+    )
+
+    return {
+        "events": payload.get("events", []),
+        "import_events": payload.get("import_events", []),
+        "exit_code": payload.get("exit_code", 0),
+        "duration_s": payload.get("duration_s", wall_elapsed),
+        "stderr": proc.stderr or "",
+    }

vigil_mapper/runtime_tracer_entry.py

@@ -0,0 +1,199 @@
+"""Subprocess entrypoint for runtime startup tracing.
+
+CRITICAL: This file must NEVER be imported by the parent process or any
+other module. It is designed to run only as a __main__ subprocess via:
+    python -m vigil_mapper.runtime_tracer_entry
+
+It installs sys.settrace and a __import__ hook, runs the target module,
+captures all call events and import events, and writes a JSON file to
+the path given by --out.
+
+Safety net: refuses to run unless VIGIL_MAPPER_TRACE=1 is set
+in the environment, preventing accidental in-process execution.
+
+Do NOT import this module. Do NOT add it to __all__ in __init__.py.
+"""
+from __future__ import annotations
+
+import argparse
+import builtins
+import json
+import os
+import runpy
+import sys
+import time
+from typing import Any
+import logging
+_log = logging.getLogger(__name__)
+
+
+def main() -> int:
+    # ------------------------------------------------------------------
+    # Safety net: require VIGIL_MAPPER_TRACE=1.
+    # This prevents accidental execution in-process by parent code that
+    # accidentally imports this module.
+    # ------------------------------------------------------------------
+    if os.environ.get("VIGIL_MAPPER_TRACE") != "1":
+        print(
+            "ERROR: runtime_tracer_entry must only run as a subprocess with "
+            "VIGIL_MAPPER_TRACE=1 set. Refusing to execute.",
+            file=sys.stderr,
+        )
+        return 2
+
+    # ------------------------------------------------------------------
+    # Argument parsing.
+    # We must handle `-- target_argv` manually: argparse stops at `--`.
+    # ------------------------------------------------------------------
+    parser = argparse.ArgumentParser(
+        description="Subprocess entrypoint for cortex runtime tracer.",
+        add_help=True,
+    )
+    parser.add_argument("--target", required=True, help="Dotted module name to run.")
+    parser.add_argument("--out", required=True, help="Path to write JSON output.")
+    parser.add_argument(
+        "--timeout-s",
+        type=float,
+        default=30.0,
+        help="Soft time budget for the target (informational only here).",
+    )
+
+    # Split off target argv (everything after `--`).
+    raw_args = sys.argv[1:]
+    target_argv_start: list[str] = []
+    if "--" in raw_args:
+        sep_idx = raw_args.index("--")
+        target_argv_start = raw_args[sep_idx + 1:]
+        raw_args = raw_args[:sep_idx]
+
+    args = parser.parse_args(raw_args)
+    target_module: str = args.target
+    out_path: str = args.out
+
+    # ------------------------------------------------------------------
+    # Local accumulators (NOT module-level globals — plan ban).
+    # ------------------------------------------------------------------
+    events: list[dict[str, Any]] = []
+    import_events: list[dict[str, Any]] = []
+    t0 = time.perf_counter()
+
+    # ------------------------------------------------------------------
+    # Save original trace/import hooks so we can restore them in finally.
+    # ------------------------------------------------------------------
+    orig_settrace = sys.gettrace()
+    orig_import = builtins.__import__
+
+    exit_code: int = 0
+    exception_info: dict[str, str] | None = None
+
+    # ------------------------------------------------------------------
+    # Skip-list for trace events: exclude non-user code.
+    # ------------------------------------------------------------------
+    _skip_fragments = (
+        os.sep + "libs" + os.sep,
+        "__pycache__",
+        "frozen importlib",
+        "importlib",
+    )
+
+    def _should_skip_filename(fname: str) -> bool:
+        if not fname:
+            return True
+        if fname.startswith("<"):
+            return True
+        for frag in _skip_fragments:
+            if frag in fname:
+                return True
+        return False
+
+    # ------------------------------------------------------------------
+    # Install sys.settrace call tracer.
+    # ------------------------------------------------------------------
+    def tracefunc(frame: Any, event: str, arg: Any) -> Any:
+        if event == "call":
+            fname = frame.f_code.co_filename
+            if not _should_skip_filename(fname):
+                qualname = frame.f_code.co_qualname if hasattr(frame.f_code, "co_qualname") else frame.f_code.co_name
+                events.append({
+                    "event": "call",
+                    "qualname": qualname,
+                    "filename": fname,
+                    "lineno": frame.f_lineno,
+                    "ts": time.perf_counter() - t0,
+                })
+        return tracefunc
+
+    # ------------------------------------------------------------------
+    # Install __import__ hook to capture import events.
+    # ------------------------------------------------------------------
+    def traced_import(name: str, *import_args: Any, **import_kwargs: Any) -> Any:
+        import_events.append({
+            "event": "import",
+            "module": name,
+            "ts": time.perf_counter() - t0,
+        })
+        return orig_import(name, *import_args, **import_kwargs)
+
+    # ------------------------------------------------------------------
+    # Run the target module with hooks installed.
+    # ------------------------------------------------------------------
+    sys.settrace(tracefunc)
+    builtins.__import__ = traced_import
+
+    try:
+        # Set sys.argv so the target module sees appropriate argv.
+        sys.argv = [target_module] + list(target_argv_start)
+        runpy.run_module(target_module, run_name="__main__", alter_sys=True)
+        exit_code = 0
+
+    except SystemExit as exc:
+        # Normal CLI exit — not a failure.
+        code = exc.code
+        if code is None:
+            exit_code = 0
+        elif isinstance(code, int):
+            exit_code = code
+        else:
+            # SystemExit with a string message → treat as error (code 1).
+            exit_code = 1
+
+    except Exception as exc:  # noqa: BLE001 -- intentional broad catch for target
+        exit_code = 2
+        exception_info = {
+            "type": type(exc).__name__,
+            "message": str(exc),
+        }
+
+    finally:
+        # Always restore hooks — plan §4b critical requirement.
+        sys.settrace(None)
+        builtins.__import__ = orig_import
+
+    # ------------------------------------------------------------------
+    # Write JSON output to the file specified by --out.
+    # ------------------------------------------------------------------
+    duration_s = time.perf_counter() - t0
+    output: dict[str, Any] = {
+        "events": events,
+        "import_events": import_events,
+        "exit_code": exit_code,
+        "duration_s": duration_s,
+    }
+    if exception_info is not None:
+        output["exception"] = exception_info
+
+    try:
+        out_text = json.dumps(output, ensure_ascii=False)
+        with open(out_path, "w", encoding="utf-8") as fh:
+            fh.write(out_text)
+    except OSError as exc:
+        print("ERROR: failed to write output file %r: %s" % (out_path, exc), file=sys.stderr)
+        return 3
+
+    # Return 0 if target exited normally (SystemExit 0 or clean return),
+    # 2 if target raised an unhandled exception.
+    return 0 if exit_code in (0,) else 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

vigil_mapper/semantic_diff.py

@@ -0,0 +1,71 @@
+"""Semantic diff for map JSON files, ignoring timestamp-like fields.
+
+Used by rebuild worker (Phase E1) to decide whether to promote temp-dir
+rebuild output to canonical location -- skip write if content unchanged
+modulo ignored timestamps.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+__all__ = ["semantic_map_diff"]
+
+_log = logging.getLogger(__name__)
+
+_IGNORED_FIELDS: frozenset[str] = frozenset({
+    # Build timestamps / envelope — never semantic content
+    "built_at",
+    "freshness",
+    "produced_by",
+    "build_duration_s",
+    "generated_at",
+    "map_name",
+    # Build-environment git metadata (hotspot map only — see cli_entry.py
+    # `_hotspot_churn_meta`).  `git_head_sha` leaks into the payload when any
+    # git activity occurs between the two rebuild runs (commit, branch move,
+    # rebase).  `churn_source` toggles between "git_log_numstat",
+    # "git_log_numstat_empty", and "skipped" depending on whether the index
+    # refresh picked up a git hiccup.  `since_window` is stable for a given
+    # build config but belongs to the same metadata class and is therefore
+    # excluded for consistency.  Excluding these three fields keeps the
+    # semantic-diff focused on map *content*, not build environment.
+    "git_head_sha",
+    "churn_source",
+    "since_window",
+    # Derived size of the serialized map file — jitters ±1 byte whenever
+    # build_duration_s crosses a decimal-digit boundary (e.g. "0.42" → "0.395").
+    # Recorded in 00_map_index.json per-map entry; not semantic payload.
+    "file_bytes",
+})
+
+
+def semantic_map_diff(new_path: Path, old_path: Path) -> bool:
+    """Return True if two map JSONs are semantically identical (ignoring timestamp fields).
+
+    Returns True (identical) if both parse and stripped structures equal.
+    Returns False if either file unreadable/invalid JSON or content differs.
+    Fail-safe: any error -> False (treat as "changed", triggers write).
+    """
+    try:
+        new_data = json.loads(new_path.read_text(encoding="utf-8"))
+        old_data = json.loads(old_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        _log.debug(
+            "semantic_map_diff: read/parse failed (%s): %s",
+            type(exc).__name__,
+            new_path,
+        )
+        return False  # treat as changed on any error
+    return _strip_ignored(new_data) == _strip_ignored(old_data)
+
+
+def _strip_ignored(obj: Any) -> Any:
+    """Recursively strip ignored fields from nested dict/list structure."""
+    if isinstance(obj, dict):
+        return {k: _strip_ignored(v) for k, v in obj.items() if k not in _IGNORED_FIELDS}
+    if isinstance(obj, list):
+        return [_strip_ignored(x) for x in obj]
+    return obj

vigil_mapper/source_adapters/__init__.py

@@ -0,0 +1,109 @@
+"""Source adapter registry and dispatch helpers.
+
+Public API:
+    ADAPTERS              -- dict mapping file extension -> SourceAdapter instance.
+    get_adapter_for_file  -- return adapter for a given Path (by extension).
+    supported_extensions  -- tuple of all currently registered extensions.
+    SourceAdapter         -- Protocol for type-checking adapter compliance.
+    RegexAdapterBase      -- Base class for regex-based adapters (L2+ languages).
+    IR signal classes     -- ImportEdge, SymbolDef, ContractCandidate,
+                            RuntimeSignal, AuthorityWriteCandidate.
+
+Registry population (current state — all 5 languages registered):
+    Python (.py), TypeScript (.ts, .tsx), JavaScript (.js, .jsx),
+    Go (.go), Java (.java).
+    Historical note: L1 shipped Python only; L2 added TS/JS; L5 added Go/Java.
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from ._base import RegexAdapterBase, SourceAdapter
+from ._ir import (
+    AuthorityWriteCandidate,
+    ContractCandidate,
+    ImportEdge,
+    RuntimeSignal,
+    TSRuntimeSignal,
+    SymbolDef,
+)
+from .go import GoAdapter
+from .java import JavaAdapter
+from .javascript import JavascriptAdapter
+from .python import PythonAdapter
+from .typescript import TypescriptAdapter
+
+__all__ = [
+    "ADAPTERS",
+    "get_adapter_for_file",
+    "supported_extensions",
+    "SourceAdapter",
+    "RegexAdapterBase",
+    "ImportEdge",
+    "SymbolDef",
+    "ContractCandidate",
+    "RuntimeSignal",
+    "TSRuntimeSignal",
+    "AuthorityWriteCandidate",
+    "PythonAdapter",
+    "TypescriptAdapter",
+    "JavascriptAdapter",
+    "GoAdapter",
+    "JavaAdapter",
+]
+
+_log = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Static adapter registry — keyed by lowercase file extension
+# ---------------------------------------------------------------------------
+
+# Populated at import time via _register(). Extensions must be lowercase.
+ADAPTERS: dict[str, SourceAdapter] = {}
+
+
+def _register(adapter: SourceAdapter) -> None:
+    """Register *adapter* for each extension it declares.
+
+    Extensions are stored in lowercase. Raises ValueError if an extension is
+    already registered (prevents silent override during development).
+    """
+    for ext in adapter.file_extensions:
+        key = ext.lower()
+        if key in ADAPTERS:
+            raise ValueError(
+                f"Duplicate adapter registration for extension {key!r}: "
+                f"existing={ADAPTERS[key].__class__.__name__!r}, "
+                f"new={adapter.__class__.__name__!r}"
+            )
+        ADAPTERS[key] = adapter
+        _log.debug("source_adapters: registered %s for %r", adapter.__class__.__name__, key)
+
+
+# All 5 adapters registered: Python, TypeScript, JavaScript, Go, Java.
+_register(PythonAdapter())
+_register(TypescriptAdapter())
+_register(JavascriptAdapter())
+_register(GoAdapter())
+_register(JavaAdapter())
+
+
+# ---------------------------------------------------------------------------
+# Dispatch helpers
+# ---------------------------------------------------------------------------
+
+def get_adapter_for_file(path: Path) -> SourceAdapter | None:
+    """Return the registered adapter for *path*'s extension, or None.
+
+    Extension lookup is case-insensitive: ``Path("FOO.PY")`` resolves to the
+    same adapter as ``Path("foo.py")``.
+
+    Returns None for extensions with no registered adapter (e.g. ``.ts`` in L1).
+    """
+    return ADAPTERS.get(path.suffix.lower())
+
+
+def supported_extensions() -> tuple[str, ...]:
+    """Return a sorted tuple of all currently registered file extensions."""
+    return tuple(sorted(ADAPTERS.keys()))