stackthread 0.1.0.dev2__py3-none-any.whl

stackthread/link.py

@@ -0,0 +1,130 @@
+"""Frontend requests -> backend endpoints (design §9)."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import cast
+
+from stackthread.config import Config, HttpLink
+from stackthread.lang.js.build import Request
+from stackthread.model import CONFIDENCE_RANK, Confidence, Edge, Finding, Graph, Node, Site
+
+_PARAM = re.compile(r"^(\{.*\}|\$.+|:.+|\*)$")
+
+
+def segments(path: str) -> list[str]:
+    """`/api/items/{id}/` -> ['api', 'items', '{}']. Parameters in any style become `{}`."""
+    out = []
+    for seg in path.split("?")[0].split("/"):
+        if not seg:
+            continue
+        if _PARAM.match(seg) or "{}" in seg or ("{" in seg and "}" in seg):
+            out.append("{}")
+        else:
+            out.append(seg)
+    return out
+
+
+@dataclass(frozen=True, slots=True)
+class Target:
+    node: str
+    method: str
+    segs: tuple[str, ...]
+    source: str
+
+
+def match_score(request: list[str], endpoint: tuple[str, ...]) -> int | None:
+    """None when they cannot match; otherwise how many segments matched literally (higher is better)."""
+    if len(request) != len(endpoint):
+        return None
+    score = 0
+    for a, b in zip(request, endpoint, strict=True):
+        if a == b:
+            score += 2 if a != "{}" else 1
+        elif a == "{}" or b == "{}":
+            continue
+        else:
+            return None
+    return score
+
+
+def _apply(url: str, link: HttpLink) -> str:
+    if link.client_prefix and (url == link.client_prefix or url.startswith(link.client_prefix.rstrip("/") + "/")):
+        return link.server_prefix.rstrip("/") + "/" + url[len(link.client_prefix) :].lstrip("/")
+    return url
+
+
+def link_requests(
+    graph: Graph, config: Config, requests: Iterable[Request], proxies: dict[str, list[tuple[str, str]]]
+) -> None:
+    targets_by_source: dict[str, list[Target]] = {}
+    for node in graph.of_kind("endpoint"):
+        targets_by_source.setdefault(node.source, []).append(
+            Target(node.id, node.attrs.get("method", "ANY"), tuple(segments(node.attrs.get("path", ""))), node.source)
+        )
+    backends = [s.name for s in config.sources if s.role == "backend"]
+    for req in requests:
+        links = [lk for lk in config.http.links if lk.frm == req.source]
+        if not links:
+            proxy = proxies.get(req.source, [])
+            links = [HttpLink(req.source, b, c, s) for b in backends for c, s in proxy] or [
+                HttpLink(req.source, b) for b in backends
+            ]
+            if proxy:
+                links += [HttpLink(req.source, b) for b in backends]
+        best: list[tuple[int, Target]] = []
+        for link in links:
+            url = _apply(req.url, link)
+            req_segs = segments(url)
+            for target in targets_by_source.get(link.to, []):
+                if req.method not in ("?", "ANY") and target.method not in ("ANY", req.method):
+                    continue
+                score = match_score(req_segs, target.segs)
+                if score is None:
+                    continue
+                if not best or score > best[0][0]:
+                    best = [(score, target)]
+                elif score == best[0][0] and target not in [t for _, t in best]:
+                    best.append((score, target))
+        site = Site(req.file, req.line)
+        if not best:
+            if req.url.startswith("/") or req.url.startswith("{}"):
+                graph.finding(
+                    Finding(
+                        "unmatched-request",
+                        "warning",
+                        f"{req.method} {req.url} has no matching backend endpoint",
+                        req.file,
+                        req.line,
+                        req.node,
+                    )
+                )
+            if req.node in graph.nodes:
+                graph.nodes[req.node].attrs.setdefault("requests", []).append(
+                    {"method": req.method, "url": req.url, "line": req.line, "matched": False}
+                )
+            continue
+        for _, target in best:
+            confidence = cast(Confidence, req.confidence)
+            if len(best) > 1 and CONFIDENCE_RANK[confidence] < CONFIDENCE_RANK["inferred"]:
+                confidence = "inferred"
+            graph.add_edge(
+                Edge(
+                    req.node,
+                    target.node,
+                    "http",
+                    confidence,
+                    site=site,
+                    label=f"{req.method} {req.url}",
+                )
+            )
+        if req.node in graph.nodes:
+            graph.nodes[req.node].attrs.setdefault("requests", []).append(
+                {"method": req.method, "url": req.url, "line": req.line, "matched": True}
+            )
+
+
+def endpoint_callers(graph: Graph, node: Node) -> list[Edge]:
+    return graph.into(node.id, "http")

stackthread/logs.py

@@ -0,0 +1,314 @@
+"""Logging: where messages go, what they look like, and how to read them back.
+
+Every part of stackthread logs through the standard `logging` module, under named loggers:
+
+    stackthread.cli       commands and start-up
+    stackthread.index     discovery, extraction, graph builds, the facts cache
+    stackthread.worker    the Node process that reads JavaScript and TypeScript
+    stackthread.server    the HTTP server, sessions, the Vite plugin announcing itself
+    stackthread.http      one line per request (debug), refusals (warning), failures (error)
+    stackthread.edit      saves from the code pane
+    stackthread.adapters  project adapters
+    stackthread.client    errors reported by the browser: the UI, the in-app overlay, the Vite plugin
+
+Two destinations:
+
+- the log file, `.stackthread/logs/stackthread.log`: one JSON object per line, rotated by size, kept on this
+  computer only. It records everything at the file level (debug by default) so a problem can be understood
+  after the fact.
+- the console (stderr): short human lines at the console level (info for `serve`, warning otherwise).
+
+The session token is masked wherever it would appear. File contents are never logged; paths are project-relative.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import logging.handlers
+import os
+import sys
+import threading
+import time
+import uuid
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+ROOT = "stackthread"
+FILE_NAME = "stackthread.log"
+LEVELS = {"debug": logging.DEBUG, "info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR}
+RUN = uuid.uuid4().hex[:8]  # ties together every line written by one process
+MASK = "[hidden]"
+
+
+def get(area: str) -> logging.Logger:
+    """The logger for one part of stackthread, such as `get("index")`."""
+    return logging.getLogger(f"{ROOT}.{area}")
+
+
+def ctx(**fields: Any) -> dict[str, Any]:
+    """Structured fields for one log line: `log.info("indexed", extra=ctx(files=12, ms=40))`."""
+    return {"ctx": fields}
+
+
+# ---------------------------------------------------------------- formatting
+
+
+def _iso(created: float) -> str:
+    local = time.localtime(created)
+    offset = time.strftime("%z", local)
+    return time.strftime("%Y-%m-%dT%H:%M:%S", local) + f".{int((created % 1) * 1000):03d}{offset[:3]}:{offset[3:]}"
+
+
+class JsonFormatter(logging.Formatter):
+    """One JSON object per line: easy to read back, filter and attach to a report."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        data: dict[str, Any] = {
+            "ts": _iso(record.created),
+            "level": record.levelname.lower(),
+            "area": record.name.removeprefix(f"{ROOT}.") if record.name != ROOT else "main",
+            "msg": record.getMessage(),
+            "run": RUN,
+        }
+        fields = getattr(record, "ctx", None)
+        if isinstance(fields, dict):
+            for key, value in fields.items():
+                if key not in data:
+                    data[key] = value
+        if record.exc_info or record.exc_text:
+            data["error"] = record.exc_text or self.formatException(record.exc_info)  # type: ignore[arg-type]
+        return json.dumps(data, ensure_ascii=False, default=str)
+
+
+class ConsoleFormatter(logging.Formatter):
+    """Short lines for a terminal. Warnings and errors say so; information does not."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        text = record.getMessage()
+        if record.levelno >= logging.ERROR:
+            text = f"error: {text}"
+        elif record.levelno >= logging.WARNING:
+            text = f"warning: {text}"
+        elif record.levelno <= logging.DEBUG:
+            text = f"debug [{record.name.removeprefix(ROOT + '.')}] {text}"
+        if (record.exc_info or record.exc_text) and record.levelno >= logging.ERROR:
+            text += "\n" + (record.exc_text or self.formatException(record.exc_info))  # type: ignore[arg-type]
+        return text
+
+
+class Redact(logging.Filter):
+    """Masks secrets (the session token) in messages, fields and tracebacks before any handler sees them."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.secrets: set[str] = set()
+
+    def scrub(self, value: Any) -> Any:
+        if isinstance(value, str):
+            for secret in self.secrets:
+                value = value.replace(secret, MASK)
+            return value
+        if isinstance(value, dict):
+            return {k: self.scrub(v) for k, v in value.items()}
+        if isinstance(value, (list, tuple)):
+            return [self.scrub(v) for v in value]
+        return value
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not self.secrets:
+            return True
+        record.msg = self.scrub(record.getMessage())
+        record.args = None
+        fields = getattr(record, "ctx", None)
+        if isinstance(fields, dict):
+            record.ctx = self.scrub(fields)
+        if record.exc_info and not record.exc_text:
+            record.exc_text = self.scrub(logging.Formatter().formatException(record.exc_info))
+        elif record.exc_text:
+            record.exc_text = self.scrub(record.exc_text)
+        return True
+
+
+_redact = Redact()
+
+
+def hide(secret: str) -> None:
+    """Never write this value to any log. Short values are refused: masking them would garble every message."""
+    if len(secret) >= 12:
+        _redact.secrets.add(secret)
+
+
+# ---------------------------------------------------------------- set-up
+
+
+def level_of(name: str | None, default: int) -> int:
+    return LEVELS.get((name or "").strip().lower(), default)
+
+
+def log_dir(state_dir: Path) -> Path:
+    return state_dir / "logs"
+
+
+def setup(
+    state_dir: Path | None,
+    console: str | int = "warning",
+    file_level: str | int = "debug",
+    size_kb: int = 1024,
+    keep: int = 5,
+) -> Path | None:
+    """Configure logging for this process. Safe to call again: earlier stackthread handlers are replaced.
+
+    Returns the log file path, or None when only the console is used.
+    """
+    root = logging.getLogger(ROOT)
+    root.setLevel(logging.DEBUG)
+    root.propagate = False
+    for handler in list(root.handlers):
+        root.removeHandler(handler)
+        handler.close()
+
+    con = logging.StreamHandler(sys.stderr)
+    con.setLevel(console if isinstance(console, int) else level_of(console, logging.WARNING))
+    con.setFormatter(ConsoleFormatter())
+    con.addFilter(_redact)
+    root.addHandler(con)
+
+    path: Path | None = None
+    if state_dir is not None:
+        try:
+            folder = log_dir(state_dir)
+            folder.mkdir(parents=True, exist_ok=True)
+            ignore = state_dir / ".gitignore"
+            if not ignore.exists():
+                ignore.write_text("*\n", encoding="utf-8")
+            path = folder / FILE_NAME
+            fh = logging.handlers.RotatingFileHandler(
+                path, maxBytes=max(64, size_kb) * 1024, backupCount=max(1, keep), encoding="utf-8", delay=True
+            )
+            fh.setLevel(file_level if isinstance(file_level, int) else level_of(file_level, logging.DEBUG))
+            fh.setFormatter(JsonFormatter())
+            fh.addFilter(_redact)
+            root.addHandler(fh)
+        except OSError as e:
+            path = None
+            get("cli").warning("cannot write logs in %s: %s", state_dir, e)
+    _install_hooks()
+    return path
+
+
+_hooked = False
+
+
+def _install_hooks() -> None:
+    """Uncaught errors, in the main thread or any other, end up in the log with their traceback."""
+    global _hooked
+    if _hooked:
+        return
+    _hooked = True
+    previous = sys.excepthook
+
+    def on_error(kind: type[BaseException], value: BaseException, tb: Any) -> None:
+        if not issubclass(kind, KeyboardInterrupt):
+            get("main").critical("unexpected error: %s", value, exc_info=(kind, value, tb))
+        previous(kind, value, tb)
+
+    def on_thread_error(args: threading.ExceptHookArgs) -> None:
+        if args.exc_type is SystemExit or args.exc_value is None:
+            return
+        name = args.thread.name if args.thread else "?"
+        get("main").critical(
+            "unexpected error in thread %s: %s",
+            name,
+            args.exc_value,
+            exc_info=(args.exc_type, args.exc_value, args.exc_traceback),
+        )
+
+    previous_thread = threading.excepthook
+
+    def on_thread_error_and_pass_on(args: threading.ExceptHookArgs) -> None:
+        on_thread_error(args)
+        previous_thread(args)
+
+    sys.excepthook = on_error
+    threading.excepthook = on_thread_error_and_pass_on
+
+
+# ---------------------------------------------------------------- reading back
+
+
+def files(state_dir: Path) -> list[Path]:
+    """The log file and its rotated copies, oldest first."""
+    folder = log_dir(state_dir)
+    if not folder.is_dir():
+        return []
+    rotated = sorted(
+        (p for p in folder.glob(f"{FILE_NAME}.*") if p.suffix.lstrip(".").isdigit()),
+        key=lambda p: -int(p.suffix.lstrip(".")),
+    )
+    current = folder / FILE_NAME
+    return rotated + ([current] if current.is_file() else [])
+
+
+def entries(state_dir: Path) -> Iterator[dict[str, Any]]:
+    for path in files(state_dir):
+        try:
+            with path.open(encoding="utf-8", errors="replace") as fh:
+                for line in fh:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        item = json.loads(line)
+                    except ValueError:
+                        item = {"level": "info", "area": "?", "msg": line}
+                    if isinstance(item, dict):
+                        yield item
+        except OSError:
+            continue
+
+
+def recent(state_dir: Path, level: str = "debug", limit: int = 200, area: str | None = None) -> list[dict[str, Any]]:
+    floor = level_of(level, logging.DEBUG)
+    out = [
+        e
+        for e in entries(state_dir)
+        if level_of(str(e.get("level")), logging.INFO) >= floor and (area is None or e.get("area") == area)
+    ]
+    return out[-limit:] if limit > 0 else out
+
+
+def render(entry: dict[str, Any]) -> str:
+    """One entry as a terminal line: time, level, area, message, then any fields."""
+    ts = str(entry.get("ts", ""))[11:19]
+    level = str(entry.get("level", ""))
+    skip = {"ts", "level", "area", "msg", "run", "error"}
+    extra = " ".join(
+        f"{k}={json.dumps(v, ensure_ascii=False) if not isinstance(v, str) else v}"
+        for k, v in entry.items()
+        if k not in skip
+    )
+    line = f"{ts} {level:<8} {entry.get('area', '')!s:<9} {entry.get('msg', '')}"
+    if extra:
+        line += f"  ({extra})"
+    if entry.get("error"):
+        line += "\n" + "\n".join("    " + x for x in str(entry["error"]).splitlines())
+    return line
+
+
+def environment() -> dict[str, Any]:
+    """What a support request needs to know about this machine, without anything personal."""
+    import platform
+
+    from stackthread import __version__
+
+    return {
+        "stackthread": __version__,
+        "python": platform.python_version(),
+        "implementation": platform.python_implementation(),
+        "os": f"{platform.system()} {platform.release()}",
+        "machine": platform.machine(),
+        "pid": os.getpid(),
+        "run": RUN,
+    }

stackthread/model.py

@@ -0,0 +1,161 @@
+"""The graph: nodes and edges built from every source's facts. One model behind every view (design §8)."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Iterable, Iterator
+from dataclasses import asdict, dataclass, field
+from typing import Any, Literal
+
+NodeKind = Literal[
+    "file", "module", "class", "function", "method", "component", "hook", "page", "endpoint", "model", "store"
+]
+EdgeKind = Literal["contains", "imports", "calls", "renders", "routes", "http", "reads", "writes", "uses", "inherits"]
+Layer = Literal["screen", "client", "network", "server", "data", "none"]
+Confidence = Literal["exact", "inferred", "heuristic"]
+
+LAYERS: tuple[Layer, ...] = ("screen", "client", "network", "server", "data")
+CONFIDENCE_RANK = {"exact": 0, "inferred": 1, "heuristic": 2}
+
+
+@dataclass(slots=True)
+class Node:
+    id: str
+    kind: NodeKind
+    name: str
+    lang: str = ""
+    source: str = ""
+    file: str = ""
+    line: int = 0
+    end: int = 0
+    col: int = 0
+    qualname: str = ""
+    layer: Layer = "none"
+    doc: str = ""
+    tags: list[str] = field(default_factory=list)
+    attrs: dict[str, Any] = field(default_factory=dict)
+
+    def to_json(self) -> dict[str, Any]:
+        data = asdict(self)
+        return {k: v for k, v in data.items() if v not in ("", 0, [], {}, None) or k in ("id", "kind", "name")}
+
+
+@dataclass(slots=True, frozen=True)
+class Site:
+    file: str
+    line: int
+
+
+@dataclass(slots=True)
+class Edge:
+    src: str
+    dst: str
+    kind: EdgeKind
+    confidence: Confidence = "exact"
+    site: Site | None = None
+    label: str = ""
+
+    def key(self) -> tuple[str, str, str, str]:
+        return (self.src, self.dst, self.kind, self.label)
+
+    def to_json(self) -> dict[str, Any]:
+        data: dict[str, Any] = {"from": self.src, "to": self.dst, "kind": self.kind}
+        if self.confidence != "exact":
+            data["confidence"] = self.confidence
+        if self.site:
+            data["site"] = f"{self.site.file}:{self.site.line}"
+        if self.label:
+            data["label"] = self.label
+        return data
+
+
+@dataclass(slots=True)
+class Finding:
+    rule: str
+    severity: Literal["error", "warning", "info"]
+    message: str
+    file: str = ""
+    line: int = 0
+    node: str = ""
+
+    def to_json(self) -> dict[str, Any]:
+        return {k: v for k, v in asdict(self).items() if v not in ("", 0)}
+
+
+class Graph:
+    """Nodes and edges with indexes. Built once per revision and then only read, so it is safe to share."""
+
+    def __init__(self) -> None:
+        self.nodes: dict[str, Node] = {}
+        self.edges: list[Edge] = []
+        self.findings: list[Finding] = []
+        self._out: dict[str, list[Edge]] = defaultdict(list)
+        self._in: dict[str, list[Edge]] = defaultdict(list)
+        self._edge_keys: set[tuple[str, str, str, str]] = set()
+        self._by_file: dict[str, list[Node]] = defaultdict(list)
+
+    # ---- building ----
+
+    def add_node(self, node: Node) -> Node:
+        existing = self.nodes.get(node.id)
+        if existing is not None:
+            return existing
+        self.nodes[node.id] = node
+        if node.file:
+            self._by_file[node.file].append(node)
+        return node
+
+    def add_edge(self, edge: Edge) -> bool:
+        """Add an edge once. A repeated edge keeps the most confident version."""
+        if edge.src == edge.dst and edge.kind not in ("calls",):
+            return False
+        key = edge.key()
+        if key in self._edge_keys:
+            for existing in self._out[edge.src]:
+                if existing.key() == key and CONFIDENCE_RANK[edge.confidence] < CONFIDENCE_RANK[existing.confidence]:
+                    existing.confidence = edge.confidence
+            return False
+        self._edge_keys.add(key)
+        self.edges.append(edge)
+        self._out[edge.src].append(edge)
+        self._in[edge.dst].append(edge)
+        return True
+
+    def finding(self, finding: Finding) -> None:
+        self.findings.append(finding)
+
+    # ---- reading ----
+
+    def out(self, node_id: str, *kinds: EdgeKind) -> list[Edge]:
+        edges = self._out.get(node_id, [])
+        return [e for e in edges if e.kind in kinds] if kinds else list(edges)
+
+    def into(self, node_id: str, *kinds: EdgeKind) -> list[Edge]:
+        edges = self._in.get(node_id, [])
+        return [e for e in edges if e.kind in kinds] if kinds else list(edges)
+
+    def in_file(self, file: str) -> list[Node]:
+        return list(self._by_file.get(file, []))
+
+    def files(self) -> Iterable[str]:
+        return self._by_file.keys()
+
+    def at(self, file: str, line: int) -> Node | None:
+        """The innermost definition in a file that spans a line."""
+        best: Node | None = None
+        for node in self._by_file.get(file, []):
+            if node.kind in ("file", "module", "page", "endpoint"):
+                continue
+            if node.line <= line <= max(node.end, node.line) and (best is None or node.line >= best.line):
+                best = node
+        return best
+
+    def of_kind(self, *kinds: NodeKind) -> Iterator[Node]:
+        return (n for n in self.nodes.values() if n.kind in kinds)
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "nodes": [n.to_json() for n in self.nodes.values()],
+            "edges": [e.to_json() for e in self.edges],
+            "findings": [f.to_json() for f in self.findings],
+        }