stackthread 0.1.0.dev2__py3-none-any.whl

stackthread/paths.py

@@ -0,0 +1,173 @@
+"""Glob matching, .gitignore rules and file discovery. Paths in the model are POSIX and relative to the project root."""
+
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Iterator
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+#: Folders never worth reading, whatever the config says.
+ALWAYS_SKIP = frozenset(
+    {
+        ".git",
+        ".hg",
+        ".svn",
+        "node_modules",
+        ".venv",
+        "venv",
+        "env",
+        "__pycache__",
+        "dist",
+        "build",
+        ".next",
+        ".nuxt",
+        ".output",
+        "coverage",
+        "htmlcov",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".pytest_cache",
+        ".tox",
+        ".stackthread",
+        ".turbo",
+        ".cache",
+        ".idea",
+        ".vscode",
+        "site-packages",
+        ".eggs",
+    }
+)
+
+PYTHON_EXTS = (".py",)
+SCRIPT_EXTS = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts")
+
+
+@lru_cache(maxsize=1024)
+def glob_regex(pattern: str) -> re.Pattern[str]:
+    """Compile a glob with `**` (any depth), `*` (within a segment), `?` and `[...]` to a regex."""
+    i, out = 0, []
+    while i < len(pattern):
+        c = pattern[i]
+        if c == "*":
+            if pattern[i : i + 2] == "**":
+                i += 2
+                if pattern[i : i + 1] == "/":
+                    i += 1
+                    out.append("(?:.*/)?")
+                else:
+                    out.append(".*")
+                continue
+            out.append("[^/]*")
+        elif c == "?":
+            out.append("[^/]")
+        elif c == "[":
+            end = pattern.find("]", i + 1)
+            if end == -1:
+                out.append(re.escape(c))
+            else:
+                body = pattern[i + 1 : end].replace("\\", "\\\\")
+                if body.startswith("!"):
+                    body = "^" + body[1:]
+                out.append(f"[{body}]")
+                i = end
+        else:
+            out.append(re.escape(c))
+        i += 1
+    return re.compile("".join(out) + r"\Z")
+
+
+def matches(path: str, patterns: list[str] | tuple[str, ...]) -> bool:
+    return any(glob_regex(p).match(path) for p in patterns)
+
+
+@dataclass(frozen=True, slots=True)
+class _Rule:
+    regex: re.Pattern[str]
+    negate: bool
+    dir_only: bool
+
+
+class GitIgnore:
+    """The subset of .gitignore that matters here: comments, negation, anchoring, directory-only rules, `**`."""
+
+    def __init__(self, rules: list[tuple[str, _Rule]]) -> None:
+        self._rules = rules  # (base dir relative to project root, rule)
+
+    @classmethod
+    def load(cls, root: Path, dirs: list[Path]) -> GitIgnore:
+        rules: list[tuple[str, _Rule]] = []
+        for d in dirs:
+            file = d / ".gitignore"
+            if not file.is_file():
+                continue
+            base = "" if d == root else d.relative_to(root).as_posix()
+            for raw in file.read_text(encoding="utf-8", errors="replace").splitlines():
+                line = raw.strip()
+                if not line or line.startswith("#"):
+                    continue
+                negate = line.startswith("!")
+                line = line[1:] if negate else line
+                dir_only = line.endswith("/")
+                line = line.rstrip("/")
+                anchored = "/" in line
+                line = line.lstrip("/")
+                pattern = line if anchored else f"**/{line}"
+                rules.append((base, _Rule(glob_regex(pattern), negate, dir_only)))
+        return cls(rules)
+
+    def ignored(self, rel: str, is_dir: bool) -> bool:
+        result = False
+        for base, rule in self._rules:
+            if base:
+                if not rel.startswith(base + "/"):
+                    continue
+                sub = rel[len(base) + 1 :]
+            else:
+                sub = rel
+            if rule.dir_only and not is_dir:
+                continue
+            if rule.regex.match(sub):
+                result = not rule.negate
+        return result
+
+
+def walk(root: Path, base: Path, include: list[str], exclude: list[str], exts: tuple[str, ...]) -> Iterator[Path]:
+    """Files under `base` with one of `exts`, matching include and not exclude (globs relative to `base`),
+    skipping ALWAYS_SKIP folders and anything the project's .gitignore files ignore."""
+    chain = [base]
+    while chain[-1] != root and root in chain[-1].parents:
+        chain.append(chain[-1].parent)
+    gitignore = GitIgnore.load(root, list(reversed(chain)))  # root first, so deeper files take precedence
+    base_resolved = base.resolve()
+    for dirpath, dirnames, filenames in os.walk(base):
+        current = Path(dirpath)
+        rel_dir = current.relative_to(root).as_posix() if current != root else ""
+        keep = []
+        for d in dirnames:
+            if d in ALWAYS_SKIP or d.endswith(".egg-info"):
+                continue
+            rel = f"{rel_dir}/{d}" if rel_dir else d
+            if gitignore.ignored(rel, True):
+                continue
+            keep.append(d)
+        dirnames[:] = sorted(keep)
+        for name in sorted(filenames):
+            if not name.endswith(exts) or name.endswith((".d.ts", ".d.mts", ".d.cts")):
+                continue
+            path = current / name
+            rel_root = path.relative_to(root).as_posix()
+            if gitignore.ignored(rel_root, False):
+                continue
+            rel_base = path.resolve().relative_to(base_resolved).as_posix()
+            if include and not matches(rel_base, include):
+                continue
+            if exclude and matches(rel_base, exclude):
+                continue
+            yield path
+
+
+def posix_rel(path: Path, root: Path) -> str:
+    return path.resolve().relative_to(root.resolve()).as_posix()

stackthread/project.py

@@ -0,0 +1,414 @@
+"""A project: discovery, extraction with a cache, graph builds, revisions and change polling (design §5, §15)."""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from stackthread import SCHEMA_VERSION, __version__, logs
+from stackthread.config import Config, SourceConfig
+from stackthread.lang.js import aliases as js_aliases
+from stackthread.lang.js.build import JsBuild
+from stackthread.lang.js.build import build as build_js
+from stackthread.lang.js.worker import JsWorker, NodeUnavailable
+from stackthread.lang.python.build import PyBuild
+from stackthread.lang.python.build import build as build_py
+from stackthread.lang.python.extract import extract as extract_py
+from stackthread.link import link_requests
+from stackthread.model import Finding, Graph
+from stackthread.paths import PYTHON_EXTS, SCRIPT_EXTS, walk
+
+Stat = tuple[int, int]
+JS_BATCH = 40  # files per request to the Node worker, so progress moves while a large frontend is read
+_log = logs.get("index")
+
+
+def _fingerprint() -> str:
+    """Changes whenever either extractor changes, so facts from an older extractor are never reused."""
+    import hashlib
+
+    from stackthread.lang.js.worker import extractor_path
+    from stackthread.lang.python import extract as py_extract
+
+    h = hashlib.sha256(__version__.encode())
+    h.update(Path(py_extract.__file__).read_bytes())
+    script = extractor_path()
+    if script is not None and script.is_file():
+        h.update(script.read_bytes())
+    return h.hexdigest()[:16]
+
+
+FINGERPRINT = _fingerprint()
+
+
+@dataclass
+class Snapshot:
+    """One immutable revision of everything the server shows. Swapped atomically on rebuild."""
+
+    rev: int
+    graph: Graph
+    py: dict[str, PyBuild]
+    js: dict[str, JsBuild]
+    facts: dict[str, dict[str, dict[str, Any]]]
+    built_ms: float
+    changed: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def element(self, loc: str) -> tuple[JsBuild, dict[str, Any]] | None:
+        for build in self.js.values():
+            el = build.elements.get(loc)
+            if el is not None:
+                return build, el
+        return None
+
+    def source_of_file(self, file: str) -> str | None:
+        for source, files in self.facts.items():
+            if file in files:
+                return source
+        return None
+
+
+def default_includes(source: SourceConfig, root: Path) -> list[str]:
+    if source.include:
+        return source.include
+    if not source.is_python and (root / "src").is_dir():
+        return ["src/**/*"]
+    return []
+
+
+class Project:
+    def __init__(self, config: Config, log: Callable[[str], None] | None = None) -> None:
+        self.config = config
+        self.log = log or _log.warning  # for adapters and older callers; everything else logs directly
+        self.stats: dict[str, dict[str, Stat]] = {}
+        self.facts: dict[str, dict[str, dict[str, Any]]] = {}
+        self.worker = JsWorker()
+        self.snapshot: Snapshot | None = None
+        self.activity: dict[str, Any] = {"state": "starting"}
+        self._rev = 0
+        self._lock = threading.RLock()
+        self._listeners: list[Callable[[Snapshot], None]] = []
+        self._activity_listeners: list[Callable[[dict[str, Any]], None]] = []
+        self._cache_file = config.state_dir / "cache" / "facts.json"
+        self._node_problem: str | None = None
+
+    # ---- listeners ----
+
+    def subscribe(self, fn: Callable[[Snapshot], None]) -> None:
+        self._listeners.append(fn)
+
+    def unsubscribe(self, fn: Callable[[Snapshot], None]) -> None:
+        if fn in self._listeners:
+            self._listeners.remove(fn)
+
+    def on_activity(self, fn: Callable[[dict[str, Any]], None]) -> None:
+        self._activity_listeners.append(fn)
+
+    def _set_activity(self, state: str, **data: Any) -> None:
+        """What the project is doing: starting, indexing (done of total), updating (files), idle, or failed."""
+        self.activity = {"state": state, "at": time.time(), **data}
+        for listener in list(self._activity_listeners):
+            try:
+                listener(self.activity)
+            except Exception:  # a listener must never break indexing
+                _log.exception("activity listener failed")
+
+    # ---- discovery ----
+
+    def _discover(self, source: SourceConfig) -> dict[str, tuple[Path, Stat]]:
+        base = self.config.source_root(source)
+        exts = PYTHON_EXTS if source.is_python else SCRIPT_EXTS
+        out: dict[str, tuple[Path, Stat]] = {}
+        for path in walk(self.config.root, base, default_includes(source, base), source.exclude, exts):
+            try:
+                st = path.stat()
+            except OSError:
+                continue
+            out[path.relative_to(self.config.root).as_posix()] = (path, (st.st_mtime_ns, st.st_size))
+        return out
+
+    # ---- extraction ----
+
+    def _extract(
+        self, source: SourceConfig, items: dict[str, Path], progress: Callable[[int], None] | None = None
+    ) -> dict[str, dict[str, Any]]:
+        if not items:
+            return {}
+        started = time.perf_counter()
+        out: dict[str, dict[str, Any]] = {}
+        if source.is_python:
+            base = self.config.source_root(source)
+            for n, (file, path) in enumerate(items.items(), 1):
+                try:
+                    text = path.read_text(encoding="utf-8")
+                except (OSError, UnicodeDecodeError) as e:
+                    text = ""
+                    _log.warning("cannot read %s: %s", file, e, extra=logs.ctx(file=file))
+                out[file] = extract_py(text, path.resolve().relative_to(base).as_posix(), file)
+                if progress and n % 20 == 0:
+                    progress(20)
+            if progress:
+                progress(len(items) % 20)
+        else:
+            pairs = [(p, f) for f, p in items.items()]
+            try:
+                for i in range(0, len(pairs), JS_BATCH):
+                    batch = pairs[i : i + JS_BATCH]
+                    out.update(self.worker.extract(batch, {"clients": self.config.http.clients}))
+                    if progress:
+                        progress(len(batch))
+                self._node_problem = None
+            except NodeUnavailable as e:
+                if str(e) != self._node_problem:
+                    _log.error("cannot read the %s frontend: %s", source.name, e, extra=logs.ctx(source=source.name))
+                self._node_problem = str(e)
+                return out
+        _log.debug(
+            "read %d %s files in %.0f ms",
+            len(items),
+            source.name,
+            (time.perf_counter() - started) * 1000,
+            extra=logs.ctx(source=source.name, files=len(items)),
+        )
+        return out
+
+    # ---- cache ----
+
+    def _load_cache(self) -> dict[str, dict[str, Any]]:
+        try:
+            data = json.loads(self._cache_file.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            return {}
+        except (OSError, ValueError) as e:
+            _log.warning("ignoring an unreadable facts cache: %s", e)
+            return {}
+        if data.get("schema") != SCHEMA_VERSION or data.get("extractors") != FINGERPRINT:
+            _log.info("the facts cache is from another version of stackthread; reading every file again")
+            return {}
+        sources: dict[str, dict[str, Any]] = data.get("sources", {})
+        return sources
+
+    def _save_cache(self) -> None:
+        data = {
+            "schema": SCHEMA_VERSION,
+            "version": __version__,
+            "extractors": FINGERPRINT,
+            "sources": {
+                name: {
+                    file: {"stat": list(self.stats[name][file]), "facts": facts}
+                    for file, facts in files.items()
+                    if file in self.stats.get(name, {})
+                }
+                for name, files in self.facts.items()
+            },
+        }
+        try:
+            self._cache_file.parent.mkdir(parents=True, exist_ok=True)
+            state = self.config.state_dir
+            if state in self._cache_file.parents and not (state / ".gitignore").exists():
+                (state / ".gitignore").write_text("*\n", encoding="utf-8")
+            tmp = self._cache_file.with_suffix(".tmp")
+            tmp.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8")
+            tmp.replace(self._cache_file)
+        except OSError as e:
+            _log.warning("cannot write the facts cache: %s", e)
+
+    # ---- scanning ----
+
+    def scan(self, use_cache: bool = True) -> Snapshot:
+        """Index everything. Files whose size and mtime match the cache are not read again."""
+        with self._lock:
+            started = time.perf_counter()
+            cached = self._load_cache() if use_cache else {}
+            plan: list[tuple[SourceConfig, dict[str, tuple[Path, Stat]], dict[str, Any], dict[str, Path]]] = []
+            for source in self.config.sources:
+                found = self._discover(source)
+                previous = cached.get(source.name, {})
+                keep = {
+                    f: entry["facts"]
+                    for f, entry in previous.items()
+                    if f in found and tuple(entry["stat"]) == found[f][1]
+                }
+                todo = {f: p for f, (p, _) in found.items() if f not in keep}
+                plan.append((source, found, keep, todo))
+                _log.debug(
+                    "%s: %d files, %d from the cache, %d to read",
+                    source.name,
+                    len(found),
+                    len(keep),
+                    len(todo),
+                    extra=logs.ctx(source=source.name, found=len(found), cached=len(keep), read=len(todo)),
+                )
+            total = sum(len(todo) for *_, todo in plan)
+            done = 0
+            self._set_activity("indexing", done=0, total=total)
+
+            def advance(n: int) -> None:
+                nonlocal done
+                done += n
+                self._set_activity("indexing", done=min(done, total), total=total)
+
+            for source, found, keep, todo in plan:
+                fresh = self._extract(source, todo, advance)
+                self.facts[source.name] = {**keep, **fresh}
+                self.stats[source.name] = {f: s for f, (_, s) in found.items() if f in self.facts[source.name]}
+            snapshot = self._build(changed=[])
+            self._save_cache()
+            files = sum(len(f) for f in self.facts.values())
+            ms = (time.perf_counter() - started) * 1000
+            _log.info(
+                "indexed %d files in %.0f ms (%d read, %d from the cache); revision %d",
+                files,
+                ms,
+                total,
+                files - total,
+                snapshot.rev,
+                extra=logs.ctx(
+                    files=files,
+                    read=total,
+                    ms=round(ms),
+                    rev=snapshot.rev,
+                    nodes=len(snapshot.graph.nodes),
+                    edges=len(snapshot.graph.edges),
+                ),
+            )
+            self._set_activity("idle", rev=snapshot.rev, ms=round(ms), files=files)
+            return snapshot
+
+    def poll(self) -> Snapshot | None:
+        """Re-index what changed on disk since the last scan. Returns the new snapshot, or None if nothing did."""
+        with self._lock:
+            started = time.perf_counter()
+            work: list[tuple[SourceConfig, dict[str, tuple[Path, Stat]], list[str], dict[str, Path]]] = []
+            for source in self.config.sources:
+                found = self._discover(source)
+                known = self.stats.setdefault(source.name, {})
+                removed = [f for f in known if f not in found]
+                todo = {f: p for f, (p, s) in found.items() if known.get(f) != s}
+                if removed or todo:
+                    work.append((source, found, removed, todo))
+            if not work:
+                return None
+            changed = sorted(f for _, _, removed, todo in work for f in [*removed, *todo])
+            self._set_activity("updating", files=changed[:20], count=len(changed))
+            for source, found, removed, todo in work:
+                known = self.stats[source.name]
+                files = self.facts.setdefault(source.name, {})
+                for f in removed:
+                    known.pop(f, None)
+                    files.pop(f, None)
+                if todo:
+                    files.update(self._extract(source, todo))
+                    for f in todo:
+                        known[f] = found[f][1]
+            snapshot = self._build(changed=changed)
+            threading.Thread(target=self._save_cache, name="stackthread-cache", daemon=True).start()
+            ms = (time.perf_counter() - started) * 1000
+            _log.info(
+                "%s changed; updated in %.0f ms; revision %d",
+                _short(changed),
+                ms,
+                snapshot.rev,
+                extra=logs.ctx(changed=changed[:50], count=len(changed), ms=round(ms), rev=snapshot.rev),
+            )
+            self._set_activity("idle", rev=snapshot.rev, ms=round(ms), files=changed[:20])
+            return snapshot
+
+    def invalidate(self, files: list[str]) -> None:
+        """Forget files so the next poll reads them again (after an edit through the server)."""
+        with self._lock:
+            for source in self.stats.values():
+                for f in files:
+                    source.pop(f, None)
+
+    # ---- building ----
+
+    def _build(self, changed: list[str]) -> Snapshot:
+        started = time.perf_counter()
+        graph = Graph()
+        py: dict[str, PyBuild] = {}
+        js: dict[str, JsBuild] = {}
+        warnings: list[str] = []
+        for source in self.config.sources:
+            facts = self.facts.get(source.name, {})
+            if source.is_python:
+                py[source.name] = build_py(graph, source, list(facts.values()))
+        for source in self.config.sources:
+            if not source.is_python:
+                js[source.name] = build_js(graph, source, self.config.root, self.facts.get(source.name, {}))
+        if self._node_problem and any(not s.is_python for s in self.config.sources):
+            warnings.append(self._node_problem)
+            graph.finding(Finding("node-unavailable", "error", self._node_problem))
+        proxies = {
+            name: js_aliases.proxy_prefixes(self.config.root / self.config.source(name).root)  # type: ignore[union-attr]
+            for name in js
+        }
+        link_requests(graph, self.config, [r for b in js.values() for r in b.requests], proxies)
+        from stackthread import adapters, rules
+
+        rules.run(graph, self.config)
+        adapters.extend_all(self, graph)
+        unreadable = sorted({f.file for f in graph.findings if f.rule == "parse-error" and f.file})
+        if unreadable:
+            _log.warning(
+                "%d file(s) could not be parsed: %s",
+                len(unreadable),
+                _short(unreadable),
+                extra=logs.ctx(files=unreadable[:50]),
+            )
+        self._rev += 1
+        snapshot = Snapshot(
+            self._rev,
+            graph,
+            py,
+            js,
+            {k: dict(v) for k, v in self.facts.items()},
+            (time.perf_counter() - started) * 1000,
+            changed,
+            warnings,
+        )
+        self.snapshot = snapshot
+        for listener in list(self._listeners):
+            try:
+                listener(snapshot)
+            except Exception:  # a listener must never break indexing
+                _log.exception("snapshot listener failed")
+        return snapshot
+
+    def close(self) -> None:
+        self.worker.close()
+
+
+def _short(files: list[str], n: int = 3) -> str:
+    names = [f.rsplit("/", 1)[-1] for f in files[:n]]
+    more = len(files) - n
+    return ", ".join(names) + (f" and {more} more" if more > 0 else "")
+
+
+class Poller:
+    """Polls the project for changes in a background thread (stdlib only; about one second of latency)."""
+
+    def __init__(self, project: Project, interval: float = 0.75) -> None:
+        self.project = project
+        self.interval = interval
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, name="stackthread-poll", daemon=True)
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def _run(self) -> None:
+        while not self._stop.wait(self.interval):
+            try:
+                self.project.poll()
+            except Exception as e:  # keep watching whatever happens in one rebuild
+                _log.exception("rebuild failed: %s", e)
+                self.project._set_activity("failed", error=f"{type(e).__name__}: {e}")