codeterrain 0.3.0__py3-none-any.whl

codeterrain/languages.py

@@ -0,0 +1,293 @@
+"""Language packs: installed distributions (``codeterrain-terraform`` and the like) that add the
+definitions and references of another language to ``terrain def`` and ``terrain refs``.
+
+A pack registers one entry point in the ``codeterrain.languages`` group, named after its language,
+pointing at an object with these attributes:
+
+- ``interface``: the version of this contract it implements (``INTERFACE``);
+- ``suffixes`` and ``read(source: str) -> dict``, for ``def`` and ``refs``: the file suffixes it
+  reads (``(".tf",)``) and the facts of one file's text, ``{"defs": [...], "refs": [...]}``, each
+  definition ``{"name", "kind", "line", "doc"}`` and each reference ``{"name", "line", "scope"}``
+  (``scope`` is the name of the definition the reference sits in, or ``""``);
+- ``call_suffixes`` and ``read_calls(source: str, suffix: str) -> dict``, for ``calls`` and
+  ``callers``: the front-end file suffixes it reads in place of the built-in reader, and the HTTP
+  calls of one file, ``{"calls": [...], "error": str | None}`` in the shape
+  ``calls_static.CALL_FACT_TYPES`` describes.
+
+A pack offers either pair or both. ``read`` and ``read_calls`` are pure functions of the text:
+they read no other file and run nothing. The core walks
+the repository, caches the facts of each file by content, checks every field the pack returns and
+answers the queries, so a pack never sees the cache or the repository. A pack is code that runs in
+this process, so the import path is first cleared of every entry a repository being read controls
+(``executables.drop_distrusted_imports``), and a pack whose distribution or module still lies inside
+one is refused before it is imported.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import importlib.metadata
+import importlib.util
+import re
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from codeterrain import code_static
+from codeterrain.cache import Cache, ViewRecord
+from codeterrain.executables import distrusted, drop_distrusted_imports
+
+#: The version of the pack contract this core implements.
+INTERFACE = 1
+GROUP = "codeterrain.languages"
+_NAME = re.compile(r"[a-z][a-z0-9_]{0,31}\Z")
+_SUFFIX = re.compile(r"\.[A-Za-z0-9_.-]{1,16}\Z")
+
+MAX_FILE_BYTES = code_static.MAX_FILE_BYTES
+MAX_FILE_FACTS = code_static.MAX_FILE_FACTS
+#: Longest name, kind, scope or doc text kept from a pack.
+MAX_TEXT = 1_000
+MAX_MATCHES = code_static.MAX_MATCHES
+
+DEF_TYPES = {"name": str, "kind": str, "line": "int", "doc": str}
+REF_TYPES = {"name": str, "line": "int", "scope": str}
+
+
+@dataclass(frozen=True)
+class Language:
+    name: str
+    version: str
+    suffixes: tuple[str, ...]
+    read: Callable[[str], Any] | None
+    call_suffixes: tuple[str, ...] = ()
+    read_calls: Callable[[str, str], Any] | None = None
+
+    @property
+    def view(self) -> str:
+        return f"lang-{self.name}"
+
+    @property
+    def object_kind(self) -> str:
+        """Facts are cached per pack version: a new version of a pack re-reads every file."""
+        tag = hashlib.sha256(f"{INTERFACE}\0{self.version}".encode()).hexdigest()[:12]
+        return f"lang-{self.name}-{tag}"
+
+
+def _origins(module: str) -> list[Path] | None:
+    """Every place the top-level package of ``module`` would be imported from (all the parts of
+    a namespace package), found without importing it; None when it cannot be found."""
+    try:
+        spec = importlib.util.find_spec(module.partition(".")[0])
+    except (ImportError, ValueError):
+        return None
+    if spec is None:
+        return None
+    places = [Path(p) for p in spec.submodule_search_locations or []]
+    if spec.origin and spec.origin not in ("built-in", "frozen", "namespace"):
+        places.append(Path(spec.origin))
+    return places or None
+
+
+def _load(entry: importlib.metadata.EntryPoint, dist: importlib.metadata.Distribution) -> Any:
+    """The object ``entry`` names; raises ``ValueError`` for a pack the repository could have
+    planted (its distribution, or the module the name would import, inside it)."""
+    located = Path(str(dist.locate_file("")))
+    if not located.is_absolute() or distrusted(located):
+        raise ValueError("installed inside the repository; not loaded")
+    origins = _origins(entry.module)
+    if origins is None:
+        raise ValueError(f"module {entry.module} not found")
+    if any(distrusted(origin) for origin in origins):
+        raise ValueError(
+            f"module {entry.module} resolves inside the repository; not loaded"
+        )
+    return entry.load()
+
+
+def _language(name: str, version: str, found: Any) -> Language:
+    interface = getattr(found, "interface", None)
+    if interface != INTERFACE:
+        raise ValueError(f"implements interface {interface!r}; this codeterrain reads {INTERFACE}")
+    pairs = []
+    for suffixes_name, read_name in (("suffixes", "read"), ("call_suffixes", "read_calls")):
+        suffixes = getattr(found, suffixes_name, None)
+        read = getattr(found, read_name, None)
+        if suffixes is None and read is None:
+            pairs.append(((), None))
+            continue
+        if (
+            not isinstance(suffixes, tuple | list) or not suffixes
+            or not all(isinstance(s, str) and _SUFFIX.match(s) for s in suffixes)
+        ):
+            raise ValueError(
+                f"{suffixes_name} must be a non-empty list of file suffixes such as '.tf'"
+            )
+        if not callable(read):
+            raise ValueError(f"has {suffixes_name} but no {read_name} function")
+        pairs.append((tuple(suffixes), read))
+    if all(read is None for _, read in pairs):
+        raise ValueError("has no read function")
+    (suffixes, read), (call_suffixes, read_calls) = pairs
+    return Language(name, version, suffixes, read, call_suffixes, read_calls)
+
+
+def discover(
+    path: list[str] | None = None,
+) -> tuple[list[Language], list[str]]:
+    """The installed language packs, and a line for each one that was refused. The import path
+    is first cleared of what the repositories being read control, so neither a pack nor anything
+    it imports later can be resolved inside one."""
+    drop_distrusted_imports()
+    found: dict[str, Language] = {}
+    problems: list[str] = []
+    for dist in importlib.metadata.distributions(**({"path": path} if path is not None else {})):
+        for entry in dist.entry_points:
+            if entry.group != GROUP or entry.name in found:
+                continue
+            label = f"language pack {entry.name!r} ({dist.metadata['Name']})"
+            if not _NAME.match(entry.name):
+                problems.append(f"{label}: the name must be lowercase letters, digits or _")
+                continue
+            try:
+                found[entry.name] = _language(
+                    entry.name, dist.version or "", _load(entry, dist)
+                )
+            except Exception as exc:  # a broken pack must not stop the other answers
+                problems.append(f"{label}: {exc}")
+    return sorted(found.values(), key=lambda lang: lang.name), problems
+
+
+# --- facts ---------------------------------------------------------------------------------------
+
+
+def _typed(row: Any, types: dict[str, Any]) -> bool:
+    if not isinstance(row, dict) or set(row) != set(types):
+        return False
+    for key, kind in types.items():
+        value = row[key]
+        if kind == "int":
+            if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+                return False
+        elif not isinstance(value, kind) or len(value) > MAX_TEXT:
+            return False
+    return True
+
+
+def _valid_facts(facts: Any) -> bool:
+    return (
+        isinstance(facts, dict)
+        and set(facts) == {"defs", "refs", "error"}
+        and (facts["error"] is None or isinstance(facts["error"], str))
+        and isinstance(facts["defs"], list)
+        and isinstance(facts["refs"], list)
+        and len(facts["defs"]) + len(facts["refs"]) <= MAX_FILE_FACTS
+        and all(_typed(d, DEF_TYPES) for d in facts["defs"])
+        and all(_typed(r, REF_TYPES) for r in facts["refs"])
+    )
+
+
+def _failed(error: str) -> dict[str, Any]:
+    return {"defs": [], "refs": [], "error": error}
+
+
+def read_file(language: Language, data: bytes) -> dict[str, Any]:
+    """The checked facts of one file: a pack that raises or returns anything malformed gives a
+    file error, never a crash or an unchecked field."""
+    if len(data) > MAX_FILE_BYTES:
+        return _failed(f"file larger than {MAX_FILE_BYTES} bytes; not indexed")
+    assert language.read is not None
+    try:
+        got = language.read(data.decode("utf-8", errors="replace"))
+    except Exception as exc:  # the pack's own failure on one file
+        return _failed(f"{type(exc).__name__}: {str(exc)[:MAX_TEXT]}")
+    facts = {
+        "defs": got.get("defs") if isinstance(got, dict) else None,
+        "refs": got.get("refs") if isinstance(got, dict) else None,
+        "error": None,
+    }
+    if not _valid_facts(facts):
+        return _failed(f"language pack {language.name} returned malformed facts")
+    return facts
+
+
+def _valid_view(data: Any, language: Language) -> bool:
+    return (
+        isinstance(data, dict)
+        and data.get("format") == language.object_kind
+        and isinstance(data.get("defs"), list)
+        and isinstance(data.get("refs"), list)
+        and isinstance(data.get("errors"), list)
+        and all(_located(d, DEF_TYPES) for d in data["defs"])
+        and all(_located(r, REF_TYPES) for r in data["refs"])
+    )
+
+
+def _located(row: Any, types: dict[str, Any]) -> bool:
+    """A view row: the fact with the ``file`` it was read from."""
+    return (
+        isinstance(row, dict) and isinstance(row.get("file"), str)
+        and _typed({k: v for k, v in row.items() if k != "file"}, types)
+    )
+
+
+def build(cache: Cache, language: Language, force: bool = False) -> ViewRecord:
+    """The view of one language: its definitions and references in every file it reads,
+    rebuilt incrementally when a file changed."""
+    fingerprint = cache.scan(None, language.suffixes)
+    record = cache.read_view(language.view)
+    if (
+        record is not None and not force and record.fingerprint == fingerprint
+        and (
+            record := cache.served(
+                record, language.object_kind, lambda d: _valid_view(d, language)
+            )
+        ) is not None
+    ):
+        cache.flush()
+        return record
+    objects = cache.load_objects(language.object_kind)
+    added = False
+    for rel, sha in fingerprint.items():
+        if not _valid_facts(objects.get(sha)):
+            with open(cache.root / rel, "rb") as f:
+                # never more than the cap, whatever the file's size
+                objects[sha] = read_file(language, f.read(MAX_FILE_BYTES + 1))
+            added = True
+    if added:
+        cache.save_objects(language.object_kind, objects)
+    defs: list[dict[str, Any]] = []
+    refs: list[dict[str, Any]] = []
+    errors: list[dict[str, str]] = []
+    for rel in sorted(fingerprint):
+        facts = objects[fingerprint[rel]]
+        if facts["error"]:
+            errors.append({"file": rel, "error": facts["error"]})
+        defs.extend({**d, "file": rel} for d in facts["defs"])
+        refs.extend({**r, "file": rel} for r in facts["refs"])
+    record = cache.write_view(
+        language.view,
+        {"format": language.object_kind, "defs": defs, "refs": refs, "errors": errors},
+        fingerprint,
+        sign=True,  # built from checked facts: a warm read need not check every row again
+    )
+    cache.flush()
+    return record
+
+
+# --- queries -------------------------------------------------------------------------------------
+
+
+def find(data: dict[str, Any], name: str, kind: str | None = None) -> list[dict[str, Any]]:
+    """The definitions named ``name``, or whose last dotted part is ``name`` (``logs`` finds
+    ``aws_s3_bucket.logs``), of ``kind`` if given; at most ``MAX_MATCHES``."""
+    defs = [d for d in data["defs"] if kind is None or d["kind"] == kind]
+    exact = [d for d in defs if d["name"] == name]
+    found = exact or [d for d in defs if d["name"].endswith(f".{name}")]
+    return found[:MAX_MATCHES]
+
+
+def references(data: dict[str, Any], names: Iterable[str]) -> list[dict[str, Any]]:
+    wanted = set(names)
+    return [r for r in data["refs"] if r["name"] in wanted]

codeterrain/plugins/fastapi_routes.py

@@ -0,0 +1,193 @@
+"""FastAPI route plugin: the ``iter_routes`` hook, run inside the target's interpreter.
+
+FastAPI 0.137 stopped flattening included routers into ``app.routes``; they stay as nested
+containers and ``fastapi.routing._iter_routes_with_context`` yields each concrete route together
+with its effective (prefix- and dependency-merged) context. That helper is private, so it is read
+in exactly one function (``iter_route_contexts``) with the flat ``app.routes`` walk as the fallback.
+
+A context carries the handler only for ``APIRoute``; websockets, plain Starlette routes and mounts
+sit in ``context.starlette_route`` (already prefixed). Mounts are walked into their sub-app;
+``Host`` routes are not walked and are reported through ``host_routes_seen``.
+"""
+
+from __future__ import annotations
+
+import inspect
+import sys
+import types
+import typing
+from collections.abc import Iterable, Iterator, Sequence
+from typing import Any
+
+from codeterrain.hookspecs import hookimpl
+
+_HOSTS: list[str] = []
+
+
+def forget_host_routes() -> None:
+    _HOSTS.clear()
+
+
+def host_routes_seen() -> list[str]:
+    """Host patterns met (and skipped) by the last ``iter_routes`` call."""
+    return list(_HOSTS)
+
+
+def iter_route_contexts(routes: Sequence[Any]) -> Iterator[tuple[Any, Any | None]]:
+    """(route, effective context or None) for every concrete route in ``routes``.
+
+    Private API, isolated here: ``fastapi.routing._iter_routes_with_context`` (FastAPI >= 0.137).
+    Older versions already hold the merged routes flat in ``app.routes``.
+    """
+    try:
+        from fastapi import routing
+
+        walker = getattr(routing, "_iter_routes_with_context", None)
+    except ImportError:
+        walker = None
+    if walker is None:
+        for route in routes:
+            yield route, None
+        return
+    yield from walker(routes)
+
+
+def _callable_name(func: Any) -> str:
+    target = inspect.unwrap(func) if callable(func) else func
+    module = getattr(target, "__module__", "") or ""
+    name = getattr(target, "__qualname__", None) or type(target).__name__
+    return f"{module}.{name}" if module else name
+
+
+def _class_location(cls: type) -> tuple[str, int]:
+    try:
+        file = inspect.getsourcefile(cls) or inspect.getfile(cls)
+        return file, inspect.getsourcelines(cls)[1]
+    except (OSError, TypeError):
+        module = sys.modules.get(getattr(cls, "__module__", ""), None)
+        return getattr(module, "__file__", None) or "", 1
+
+
+def _location(endpoint: Any) -> tuple[str, int]:
+    """Absolute file and first line of the handler, decorators and wrappers looked through.
+
+    A class endpoint (``HTTPEndpoint``) reports the class statement."""
+    target = inspect.unwrap(endpoint)
+    if inspect.isclass(target):
+        return _class_location(target)
+    for candidate in (target, type(target).__call__):
+        code = getattr(candidate, "__code__", None)
+        if code is not None:
+            return code.co_filename, code.co_firstlineno
+    return "", 0
+
+
+def _is_websocket(source: Any) -> bool:
+    from starlette.routing import WebSocketRoute
+
+    return isinstance(source, WebSocketRoute)
+
+
+def _methods(source: Any) -> list[str]:
+    """HTTP methods sorted, ``WEBSOCKET`` for websockets, ``ANY`` when Starlette matches any method
+    (``methods`` None, or empty: FastAPI < 0.137 turns a class endpoint's None into ``[]``)."""
+    if not hasattr(source, "methods") or _is_websocket(source):
+        return ["WEBSOCKET"]
+    if not source.methods:
+        return ["ANY"]
+    found = sorted(source.methods)
+    if "GET" in found and "HEAD" in found:
+        found.remove("HEAD")
+    return found
+
+
+def _also_head(source: Any) -> bool:
+    """True when the route answers HEAD next to GET (a Starlette ``Route`` adds it; a FastAPI
+    ``APIRoute`` does not), which ``_methods`` folds into the GET record."""
+    methods = getattr(source, "methods", None) or ()
+    return "GET" in methods and "HEAD" in methods
+
+
+def _type_name(tp: Any) -> str:
+    """``Item``, ``list[Item]``, ``Optional[Item]``: how the annotation reads in source."""
+    if tp is None or tp is type(None):
+        return "None"
+    if tp is Ellipsis:
+        return "..."
+    origin, args = typing.get_origin(tp), typing.get_args(tp)
+    if origin is None:
+        return getattr(tp, "__name__", None) or str(tp)
+    if origin is typing.Annotated:
+        return _type_name(args[0])
+    # types.UnionType (X | Y) is 3.10+; the plugin also runs on a 3.9 target.
+    if origin is typing.Union or origin is getattr(types, "UnionType", typing.Union):
+        members = [a for a in args if a is not type(None)]
+        inner = _type_name(members[0]) if len(members) == 1 else (
+            "Union[" + ", ".join(_type_name(a) for a in members) + "]"
+        )
+        return f"Optional[{inner}]" if len(members) < len(args) else inner
+    if origin is typing.Literal:
+        return "Literal[" + ", ".join(repr(a) for a in args) + "]"
+    name = getattr(origin, "__name__", None) or str(origin)
+    return f"{name}[{', '.join(_type_name(a) for a in args)}]" if args else name
+
+
+def _response_model(source: Any) -> str | None:
+    model = getattr(source, "response_model", None)
+    if model is None:
+        return None
+    return _type_name(model)
+
+
+def _dependencies(source: Any) -> list[str]:
+    dependant = getattr(source, "dependant", None)
+    names: list[str] = []
+    for dep in getattr(dependant, "dependencies", None) or []:
+        call = getattr(dep, "call", None)
+        if call is not None:
+            names.append(_callable_name(call))
+    return names
+
+
+def _records(routes: Iterable[Any], prefix: str) -> Iterator[dict[str, Any]]:
+    from starlette.routing import Host, Mount
+
+    for route, context in iter_route_contexts(list(routes)):
+        effective = getattr(context, "starlette_route", None)
+        source = context if getattr(context, "endpoint", None) is not None else None
+        source = source or effective or route
+        if isinstance(source, Host) or isinstance(route, Host):
+            _HOSTS.append(str(getattr(source, "host", "")))
+            continue
+        if isinstance(source, Mount):
+            yield from _records(source.routes, prefix + source.path)
+            continue
+        endpoint = getattr(source, "endpoint", None)
+        path = getattr(source, "path", None)
+        if endpoint is None or path is None:
+            continue
+        file, line = _location(endpoint)
+        methods_of = source if source is context else route
+        for method in _methods(methods_of):
+            yield {
+                "method": method,
+                "also_head": method == "GET" and _also_head(methods_of),
+                "path": prefix + path,
+                "handler": _callable_name(endpoint),
+                "file": file,
+                "line": line,
+                "dependencies": _dependencies(source),
+                "response_model": _response_model(source),
+            }
+
+
+@hookimpl
+def iter_routes(app: Any) -> list[dict[str, Any]] | None:
+    try:
+        from starlette.applications import Starlette
+    except ImportError:
+        return None
+    if not isinstance(app, Starlette):
+        return None
+    forget_host_routes()
+    return list(_records(app.routes, ""))

codeterrain/plugins/manifest_join.py

@@ -0,0 +1,107 @@
+"""Join extracted routes to an access manifest (a JSON file the repository already keeps).
+
+Host side: reads the manifest, never imports the target. Rows are matched on (method, path) with
+path converters ignored; the first matching row wins. A row that lists ``apps`` applies only to the
+routes whose app module equals one of them (an empty list applies to none; a bare string is one
+module); a row without ``apps`` applies to every app.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+class ManifestError(ValueError):
+    """The route manifest is missing or does not have the configured shape."""
+
+
+@dataclass
+class JoinResult:
+    rows: list[dict[str, Any]]
+    routes_without_row: list[dict[str, Any]] = field(default_factory=list)
+    rows_without_route: list[dict[str, Any]] = field(default_factory=list)
+
+
+def load_rows(path: Path, keys: dict[str, str]) -> list[dict[str, Any]]:
+    try:
+        document = json.loads(path.read_bytes())
+    except OSError as exc:
+        raise ManifestError(f"route manifest not readable: {path.as_posix()} ({exc})") from exc
+    except ValueError as exc:
+        raise ManifestError(f"route manifest is not JSON: {path.as_posix()} ({exc})") from exc
+    if isinstance(document, list):
+        rows = document
+    elif isinstance(document, dict):
+        rows = document.get(keys["rows"])
+    else:
+        raise ManifestError("route manifest must be a JSON object or a list of rows")
+    if not isinstance(rows, list):
+        raise ManifestError(f"route manifest has no list under {keys['rows']!r}")
+    for key in ("method", "path", "access"):
+        if any(not isinstance(r, dict) or keys[key] not in r for r in rows):
+            raise ManifestError(f"route manifest rows must all carry {keys[key]!r} ({key})")
+    for key in ("method", "path"):
+        if any(not isinstance(r[keys[key]], str) for r in rows):
+            raise ManifestError(f"manifest {keys[key]!r} ({key}) must be a string in every row")
+    if any(not isinstance(r[keys["access"]], str) for r in rows):
+        raise ManifestError(f"manifest {keys['access']!r} (access) must be a string in every row")
+    for row in rows:
+        if keys["apps"] in row and not _app_list(row[keys["apps"]]):
+            raise ManifestError(
+                f"manifest {keys['apps']!r} (apps) must be a string or a list of strings"
+            )
+    return rows
+
+
+def _app_list(value: Any) -> bool:
+    return isinstance(value, str) or (
+        isinstance(value, list) and all(isinstance(v, str) for v in value)
+    )
+
+
+_CONVERTER = re.compile(r"\{([^}:]+):[^}]*\}")
+
+
+def route_key(method: str, path: str) -> tuple[str, str]:
+    """(METHOD, path) with Starlette converters dropped: ``{slug:path}`` and ``{slug}`` are one."""
+    return method.upper(), _CONVERTER.sub(lambda m: "{" + m.group(1) + "}", path)
+
+
+def _app_module(app: str) -> str:
+    return app.partition(":")[0]
+
+
+def _applies(row_apps: Any, app: str) -> bool:
+    if row_apps is None:
+        return True
+    modules = [row_apps] if isinstance(row_apps, str) else row_apps
+    return _app_module(app) in modules
+
+
+def join(
+    routes: list[dict[str, Any]], manifest_rows: list[dict[str, Any]], keys: dict[str, str]
+) -> JoinResult:
+    index: dict[tuple[str, str], list[int]] = {}
+    for position, row in enumerate(manifest_rows):
+        key = route_key(str(row[keys["method"]]), row[keys["path"]])
+        index.setdefault(key, []).append(position)
+    used: set[int] = set()
+    result = JoinResult(rows=[])
+    for route in routes:
+        chosen = None
+        for position in index.get(route_key(route["method"], route["path"]), []):
+            if _applies(manifest_rows[position].get(keys["apps"]), route["app"]):
+                chosen = position
+                break
+        if chosen is None:
+            result.routes_without_row.append(route)
+            result.rows.append({**route, "access": None})
+            continue
+        used.add(chosen)
+        result.rows.append({**route, "access": manifest_rows[chosen][keys["access"]]})
+    result.rows_without_route = [r for i, r in enumerate(manifest_rows) if i not in used]
+    return result