ground-control-cli 0.2.0__py3-none-any.whl

client/probes/code/__init__.py

@@ -0,0 +1,435 @@
+"""The code readers and the code probes: one interface over the source, whatever its language.
+
+`symbols(path)` returns the classes, functions and methods a file defines, each a Symbol:
+kind, qualified name, file, signature as written (on one line), first and last line.
+`calls(path)` returns who calls what in a file, for information only.
+
+A reader is chosen by file extension; a file with no reader for its extension whose first line
+is a bash or sh shebang is a Bash script, read by the Bash reader and counted as `.sh`. READERS
+is the closed list of readers; SOURCE is the closed list of extensions that are source code.
+`languages(root)` groups the source files git tracks by extension and lists the extensions no
+reader reads.
+
+The probes code.symbols, code.callers and code.languages return what the readers found as facts
+(app/method/facts.py): a name travels as `name_hash`, a signature as `signature_hash`, never as
+text. These hash as the service's facts do, by the contract's signature vectors.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import NamedTuple
+
+from .. import errors
+from . import bash, javascript, python
+
+KINDS = ("class", "function", "method")
+
+# extension -> the module that reads it; each has symbols(path, name), calls(path, name),
+# missing(), what to install for it to run ("" when nothing), and tools_missing(), the names of
+# its tools that are missing.
+READERS = {
+    ".py": python,
+    ".js": javascript,
+    ".mjs": javascript,
+    ".cjs": javascript,
+    ".jsx": javascript,
+    ".ts": javascript,
+    ".tsx": javascript,
+    ".html": javascript,
+    ".sh": bash,
+}
+
+# The extensions that are source code. HTML counts only for its inline <script> blocks.
+SOURCE = (
+    ".py",
+    ".js",
+    ".mjs",
+    ".cjs",
+    ".ts",
+    ".tsx",
+    ".jsx",
+    ".sh",
+    ".rb",
+    ".go",
+    ".java",
+    ".kt",
+    ".swift",
+    ".php",
+    ".cs",
+    ".rs",
+    ".c",
+    ".h",
+    ".cpp",
+    ".html",
+)
+HTML = ".html"
+SCRIPT = ".sh"
+INLINE_SCRIPT_RE = re.compile(r"<script\b(?![^>]*\bsrc\s*=)[^>]*>\s*\S", re.IGNORECASE)
+
+
+class Symbol(NamedTuple):
+    """A class, function or method a source file defines.
+
+    Attributes:
+        kind: One of KINDS: "class", "function" or "method".
+        name: The qualified name.
+        file: The file, as it is shown.
+        signature: The signature as written, on one line.
+        first: The symbol's first line.
+        last: The symbol's last line.
+
+    Company: assumes none.
+    """
+
+    kind: str
+    name: str
+    file: str
+    signature: str
+    first: int
+    last: int
+
+
+class Call(NamedTuple):
+    """One call in a source file, matched by name, not resolved.
+
+    Attributes:
+        caller: The enclosing symbol's name, or `<module>`.
+        callee: The called name.
+        file: The file, as it is shown.
+        line: The line of the call.
+
+    Company: assumes none.
+    """
+
+    caller: str
+    callee: str
+    file: str
+    line: int
+
+
+class Languages(NamedTuple):
+    """The tracked source files of a checkout, by extension.
+
+    Attributes:
+        files: Extension -> the tracked source files with it, in git's order.
+        unread: The extensions in `files` that no reader reads.
+
+    Company: assumes none.
+    """
+
+    files: dict  # extension -> the tracked source files with it, in git's order
+    unread: list  # the extensions in `files` that no reader reads
+
+
+def reader(path):
+    """The reader module for this file, or None: by its extension, else by its shebang."""
+    found = READERS.get(Path(path).suffix.lower())
+    if found is None and bash.is_script(path):
+        return bash
+    return found
+
+
+def symbols(path, name: str = "") -> list:
+    """The file's symbols; `name` is the file as it is shown (default: the path)."""
+    found = reader(path)
+    if found is None:
+        raise errors.Refused(
+            f"{name or path}: no reader for `{Path(path).suffix}`; the readers "
+            f"are {', '.join(READERS)}",
+            errors.NO_READER,
+        )
+    return found.symbols(Path(path), name or str(path))
+
+
+def missing(extensions) -> dict:
+    """Say what to install for the readers of these extensions whose tooling is missing.
+
+    Args:
+        extensions: The extensions to look at; one with no reader is skipped.
+
+    Returns:
+        What to install -> the extensions it reads; empty when nothing is missing.
+
+    Company: assumes none.
+    """
+    needs = {}
+    for extension in extensions:
+        found = READERS.get(extension)
+        line = found.missing() if found else ""
+        if line:
+            needs.setdefault(line, []).append(extension)
+    return needs
+
+
+def calls(path, name: str = "") -> list:
+    """Return who calls what in a file, for information only.
+
+    Args:
+        path: The file to read.
+        name: The file as it is shown (default: the path).
+
+    Returns:
+        The file's Calls, or an empty list when no reader reads it.
+
+    Raises:
+        errors.Refused: If the file's reader cannot parse it or its tooling is missing.
+
+    Company: assumes none.
+    """
+    found = reader(path)
+    return found.calls(Path(path), name or str(path)) if found else []
+
+
+def tracked(root) -> list:
+    """The files git tracks under root, relative to it."""
+    out = subprocess.run(["git", "ls-files", "-z"], cwd=root, capture_output=True, text=True)
+    if out.returncode != 0:
+        raise errors.Refused(f"git ls-files: {out.stderr.strip()}", errors.failed(out.stderr))
+    return [f for f in out.stdout.split("\0") if f]
+
+
+def _language(root: Path, name: str) -> str:
+    """The extension the file counts under as source code, or "" when it is not source code."""
+    suffix = Path(name).suffix.lower()
+    if suffix not in SOURCE:
+        return SCRIPT if bash.is_script(root / name) else ""
+    return suffix if _is_source(root, name) else ""
+
+
+def _is_source(root: Path, name: str) -> bool:
+    suffix = Path(name).suffix.lower()
+    if suffix != HTML:
+        return True
+    try:
+        return bool(
+            INLINE_SCRIPT_RE.search((root / name).read_text(encoding="utf-8", errors="replace"))
+        )
+    except OSError:
+        return False
+
+
+def languages(root) -> Languages:
+    """Group the source files git tracks by extension, and list the extensions no reader reads.
+
+    A file with no source extension counts as `.sh` when its first line is a bash or sh
+    shebang; an HTML file counts only when it has an inline `<script>` block.
+
+    Args:
+        root: The checkout to read.
+
+    Returns:
+        The Languages of the checkout.
+
+    Raises:
+        errors.Refused: If `git ls-files` fails.
+
+    Company: assumes none.
+    """
+    root = Path(root)
+    files = {}
+    for name in tracked(root):
+        language = _language(root, name)
+        if language:
+            files.setdefault(language, []).append(name)
+    return Languages(files, [e for e in files if e not in READERS])
+
+
+def callers(root, names) -> dict:
+    """Find, for each short name, the places that call it.
+
+    Only the tracked files a reader reads are looked at. Information only: a call is matched by
+    name, not resolved.
+
+    Args:
+        root: The checkout to read.
+        names: The short names to look for.
+
+    Returns:
+        Each name -> its callers, as `file:line caller` lines.
+
+    Raises:
+        errors.Refused: If `git ls-files` fails, or a reader cannot read a file.
+
+    Company: assumes none.
+    """
+    root = Path(root)
+    wanted = {n: [] for n in names}
+    for name in tracked(root):
+        if reader(root / name) is None or not (root / name).is_file():
+            continue
+        for call in calls(root / name, name):
+            if call.callee in wanted:
+                wanted[call.callee].append(f"{call.file}:{call.line} {call.caller}")
+    return wanted
+
+
+# --- the facts ---------------------------------------------------------------------------------
+
+
+def name_hash(name: str) -> str:
+    """The fingerprint of a name in code: sha256 of its UTF-8, in hex, as the service's facts."""
+    return hashlib.sha256(name.encode("utf-8")).hexdigest()
+
+
+def bare(signature: str) -> str:
+    """A signature without whitespace and without a leading def, class or function keyword."""
+    text = re.sub(r"^\s*(async\s+def|def|class|function)\s+", "", signature or "")
+    return re.sub(r"\s+", "", text)
+
+
+def signature_hash(signature: str) -> str:
+    """The fingerprint of a signature: name_hash of its bare form, so equal ones match."""
+    return name_hash(bare(signature))
+
+
+def _fact(symbol: Symbol) -> dict:
+    """One symbol as a fact: its kind, its lines, and its names and signature hashed."""
+    return {
+        "kind": symbol.kind,
+        "name_hash": name_hash(symbol.name),
+        "short_hash": name_hash(symbol.name.split(".")[-1]),
+        "signature_hash": signature_hash(symbol.signature),
+        "first": symbol.first,
+        "last": symbol.last,
+    }
+
+
+def _file_symbols(target: Path, name: str) -> dict:
+    """The FileSymbols fields of a file on disk, shown as `name`."""
+    if reader(target) is None:
+        return {"path": name, "error": errors.NO_READER}
+    try:
+        return {"path": name, "symbols": [_fact(s) for s in symbols(target, name)]}
+    except errors.Refused as e:
+        return {"path": name, "error": e.error}
+
+
+def _at_commit(root: Path, at: str, name: str) -> dict:
+    """The FileSymbols fields of a file as a commit holds it, copied out of git to be read."""
+    out = subprocess.run(["git", "show", f"{at}:{name}"], cwd=root, capture_output=True)
+    if out.returncode != 0:
+        return {"path": name, "present": False}
+    with tempfile.TemporaryDirectory(prefix="method-code-") as folder:
+        target = Path(folder) / name
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(out.stdout)
+        return _file_symbols(target, name)
+
+
+def symbols_probe(params: dict, root) -> dict:
+    """code.symbols: the classes, functions and methods some files define, hashed.
+
+    HEAD reads the working tree; any other ref reads each file as git holds it there, whatever
+    is on disk. A file that is not there is not present; a ref that names no commit gives every
+    file not_found.
+
+    Args:
+        params: {"files", "at"}.
+        root: The repo.
+
+    Returns:
+        The SymbolsFacts fields.
+
+    Company: assumes none.
+    """
+    root, at = Path(root), params.get("at") or "HEAD"
+    names = list(params.get("files") or [])
+    if at != "HEAD":
+        try:
+            known = subprocess.run(
+                ["git", "rev-parse", "--verify", "-q", f"{at}^{{commit}}"],
+                cwd=root,
+                capture_output=True,
+            )
+        except FileNotFoundError:
+            return {"files": [{"path": n, "error": errors.TOOL_MISSING} for n in names]}
+        if known.returncode != 0:
+            return {
+                "files": [{"path": n, "present": False, "error": errors.NOT_FOUND} for n in names]
+            }
+    found = []
+    for name in names:
+        if at != "HEAD":
+            found.append(_at_commit(root, at, name))
+        elif not (root / name).is_file():
+            found.append({"path": name, "present": False})
+        else:
+            found.append(_file_symbols(root / name, name))
+    return {"files": found}
+
+
+def callers_probe(params: dict, root) -> dict:
+    """code.callers: the calls of some short names in the tracked files a reader reads.
+
+    A call is matched by name, not resolved. A file a reader cannot read is passed over; the
+    fact's error then says which error stopped the first one.
+
+    Args:
+        params: {"short_hashes"}.
+        root: The repo.
+
+    Returns:
+        The CallersFacts fields.
+
+    Company: assumes none.
+    """
+    root = Path(root)
+    wanted = set(params.get("short_hashes") or [])
+    try:
+        names = tracked(root)
+    except errors.Refused as e:
+        return {"error": e.error}
+    except FileNotFoundError:
+        return {"error": errors.TOOL_MISSING}
+    found, error = [], None
+    for name in names:
+        if reader(root / name) is None or not (root / name).is_file():
+            continue
+        try:
+            read = calls(root / name, name)
+        except errors.Refused as e:
+            error = error or e.error
+            continue
+        found += [
+            {"short_hash": h, "path": name, "line": c.line}
+            for c in read
+            if (h := name_hash(c.callee)) in wanted
+        ]
+    found.sort(key=lambda c: (c["path"], c["line"]))
+    return dict({"calls": found}, **({"error": error} if error else {}))
+
+
+def languages_probe(params: dict, root) -> dict:
+    """code.languages: the repo's tracked source files by extension, and which a reader reads.
+
+    Args:
+        params: Nothing.
+        root: The repo.
+
+    Returns:
+        The LanguagesFacts fields.
+
+    Company: assumes none.
+    """
+    try:
+        found = languages(root)
+    except errors.Refused as e:
+        return {"error": e.error}
+    except FileNotFoundError:
+        return {"error": errors.TOOL_MISSING}
+    return {
+        "languages": [
+            {
+                "extension": extension,
+                "files": len(found.files[extension]),
+                "read": extension in READERS,
+                "tool_missing": READERS[extension].tools_missing() if extension in READERS else [],
+            }
+            for extension in sorted(found.files)
+        ]
+    }

client/probes/code/bash.py

@@ -0,0 +1,163 @@
+"""The Bash reader, with `shfmt --tojson`.
+
+It reads .sh files, and files whose first line is a bash or sh shebang (`#!/bin/bash`,
+`#!/usr/bin/env sh` and the like), whatever their extension. Every file is parsed as Bash, the
+superset of sh. Symbols are the script's functions, in both forms, `name() {` and
+`function name {`, wherever they are defined: at the top, or inside an `if`, a loop or a block.
+A function inside a function is part of the outer one, as in the other readers. A Bash
+function's signature is its name. The first line is the function's own line, the last its
+closing line.
+
+shfmt is v3.14.1, the method's own private copy: one binary in `<data folder>/bin`, downloaded
+from the shfmt release for this system by the install line, and run from there, never from a
+global install. The data folder is METHOD_DATA if set, else ~/.method. `missing()` says what to
+install when that copy is not there.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from pathlib import Path
+
+from .. import errors
+from .javascript import data_dir
+
+VERSION = "v3.14.1"
+SHEBANG_RE = re.compile(r"#!\s*(?:\S*/)?(?:env\s+(?:-\S+\s+)*)?(?:\S*/)?(?:bash|sh)(?:\s|$)")
+RELEASE = f"https://github.com/mvdan/sh/releases/download/{VERSION}/shfmt_{VERSION}"
+SYSTEM = (
+    "$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')"
+)
+ERROR_RE = re.compile(r"(?:^|:)(\d+):(\d+): (.*)")
+
+
+def shfmt() -> Path:
+    """Where the method's copy of shfmt is (or is to be) installed."""
+    return data_dir() / "bin" / "shfmt"
+
+
+def install_line() -> str:
+    """Return the shell line that installs the method's copy of shfmt for this system.
+
+    Returns:
+        A curl line that downloads shfmt into `shfmt()` and makes it executable.
+
+    Company: assumes none.
+    """
+    target = shfmt()
+    return f"curl -fsSL --create-dirs -o {target} {RELEASE}_{SYSTEM} && chmod +x {target}"
+
+
+def is_script(path) -> bool:
+    """True when the file's first line is a bash or sh shebang."""
+    try:
+        with open(path, "rb") as f:
+            first = f.readline(256)
+    except OSError:
+        return False
+    return bool(SHEBANG_RE.match(first.decode("utf-8", errors="replace").strip()))
+
+
+def _installed() -> bool:
+    try:
+        out = subprocess.run([str(shfmt()), "--version"], capture_output=True, text=True)
+    except OSError:
+        return False
+    return out.returncode == 0 and out.stdout.strip() == VERSION
+
+
+def missing() -> str:
+    """What to install for this reader to run, or "" when the method's shfmt is there."""
+    return "" if _installed() else f"install shfmt: {install_line()}"
+
+
+def tools_missing() -> list:
+    """The names of its missing tools: shfmt, when the method's copy is not there."""
+    return [] if _installed() else ["shfmt"]
+
+
+def _tree(path: Path, name: str) -> dict:
+    need = missing()
+    if need:
+        raise errors.Refused(f"{name}: reading it needs shfmt: {need}", errors.TOOL_MISSING)
+    out = subprocess.run(
+        [str(shfmt()), "-ln", "bash", "--tojson"],
+        input=path.read_text(encoding="utf-8", errors="replace"),
+        capture_output=True,
+        text=True,
+    )
+    if out.returncode != 0:
+        error = ERROR_RE.search(out.stderr.strip().splitlines()[0] if out.stderr.strip() else "")
+        where = f"line {error.group(1)}: {error.group(3)}" if error else out.stderr.strip()
+        raise errors.Refused(f"{name}: shfmt cannot parse it: {where}")
+    return json.loads(out.stdout)
+
+
+def _walk(node, caller: str, found: list):
+    """Collect each function and each call with its enclosing function, in written order.
+
+    Found as ("function", node, name) and ("call", callee, line, caller).
+    """
+    if isinstance(node, list):
+        for item in node:
+            _walk(item, caller, found)
+        return
+    if not isinstance(node, dict):
+        return
+    if node.get("Type") == "FuncDecl":
+        name = node["Name"]["Value"]
+        if not caller:
+            found.append(("function", node, name))
+        _walk(node.get("Body"), caller or name, found)
+        return
+    if node.get("Type") == "CallExpr" and node.get("Args"):
+        parts = node["Args"][0].get("Parts") or []
+        if len(parts) == 1 and parts[0].get("Type") == "Lit":
+            found.append(("call", parts[0]["Value"], parts[0]["ValuePos"]["Line"], caller))
+    for value in node.values():
+        if isinstance(value, (dict, list)):
+            _walk(value, caller, found)
+
+
+def _found(path: Path, name: str) -> list:
+    found = []
+    _walk(_tree(path, name), "", found)
+    return found
+
+
+def symbols(path: Path, name: str) -> list:
+    """Return the script's top-level functions, wherever they are defined.
+
+    Args:
+        path: The file to read.
+        name: The file as it is shown.
+
+    Returns:
+        A Symbol of kind "function" for each function, its signature its name.
+
+    Raises:
+        errors.Refused: If shfmt is missing or cannot parse the file.
+
+    Company: assumes none.
+    """
+    from . import Symbol
+
+    return [
+        Symbol("function", fname, name, fname, node["Pos"]["Line"], node["End"]["Line"])
+        for kind, node, fname in (f for f in _found(path, name) if f[0] == "function")
+    ]
+
+
+def calls(path: Path, name: str) -> list:
+    """Each command run by name: the enclosing function (or `<module>`), the name, the line."""
+    from . import Call
+
+    return sorted(
+        (
+            Call(caller or "<module>", callee, name, line)
+            for _, callee, line, caller in (f for f in _found(path, name) if f[0] == "call")
+        ),
+        key=lambda c: c.line,
+    )