livery-extensions-python 0.0.0__py3-none-any.whl

livery/extensions/python/_parse.py

@@ -0,0 +1,324 @@
+"""Parse each python source once, and say what it references.
+
+The layering phase reads every python source through this module: the
+python extension's ``references`` answer and its layering steps,
+private-reaches, runner-terminal and declared-imports. ``parsed_source``
+parses a file once for a given content, memoised by the file's path,
+size and modification time, so the steps, which run in one process,
+read one tree, and an unchanged file is never parsed twice.
+``parsed_modules`` hands a rule the sources in its scope, every one or
+the files a narrowed run selects, `RuleContext` is what a rule judges,
+and ``graph_changed`` is what each step's inputs widen on.
+"""
+
+from __future__ import annotations
+
+import ast
+from collections.abc import Iterator
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import TYPE_CHECKING, cast
+
+from livery.workshop import ReferencedName
+
+if TYPE_CHECKING:
+    from livery.workshop import Changes, Package
+
+
+@dataclass(frozen=True)
+class Parsed:
+    """One parsed source.
+
+    Attributes:
+        tree: The module's syntax tree.
+        references: Every dotted name it imports, once for each place
+            it imports it from: at import time or lazily, and under
+            which guard.
+    """
+
+    tree: ast.Module
+    references: tuple[ReferencedName, ...]
+
+
+_PARSED: dict[tuple[Path, int, int], Parsed] = {}
+
+
+def parsed_source(path: Path) -> Parsed | None:
+    """*path* parsed once for its content; None when it does not parse.
+
+    The memo key is the path with its size and modification time, so
+    an edited file parses again and an unchanged one never does.
+    """
+    try:
+        stat = path.stat()
+    except OSError:
+        return None
+    key = (path, stat.st_size, stat.st_mtime_ns)
+    found = _PARSED.get(key)
+    if found is None:
+        found = _parse(path)
+        if found is None:
+            return None
+        _PARSED[key] = found
+    return found
+
+
+def _parse(path: Path) -> Parsed | None:
+    """Read and parse *path*: the one call site, which a test may count."""
+    try:
+        tree = ast.parse(path.read_text("utf-8"), filename=str(path))
+    except SyntaxError:
+        return None  # the syntax gate names it; the rules read what parses
+    return Parsed(tree, references_of(tree))
+
+
+@dataclass(frozen=True)
+class _Where:
+    """Where an import statement sits.
+
+    Attributes:
+        lazy: Inside a function or a method body.
+        type_checking: In the body of ``if TYPE_CHECKING:``.
+        optional: In the body of a ``try`` that handles the module's
+            absence.
+    """
+
+    lazy: bool = False
+    type_checking: bool = False
+    optional: bool = False
+
+
+def references_of(tree: ast.Module) -> tuple[ReferencedName, ...]:
+    """Every dotted name *tree* imports, in source order, once for each way.
+
+    A ``from`` import references the module and each name imported
+    from it: under a PEP 420 namespace the module alone can be the
+    namespace, and the package that owns the code is one segment
+    further down. A name imported twice the same way is listed once. A
+    relative import names the source's own package and is left out.
+
+    An import inside a function or a method body is lazy: importing the
+    module does not run it. One in the body of ``if TYPE_CHECKING:`` is
+    made by a type checker alone. One in the body of a ``try`` with a
+    handler for ``ImportError``, ``ModuleNotFoundError`` or every
+    exception is optional: the source runs without the module.
+
+    Only a statement imports, so the walk reads the statements and
+    never descends into an expression.
+    """
+    found: dict[ReferencedName, None] = {}
+    pending = [(statement, _Where()) for statement in reversed(tree.body)]
+    while pending:
+        statement, where = pending.pop()
+        if isinstance(statement, ast.Import):
+            names = [alias.name for alias in statement.names]
+        elif isinstance(statement, ast.ImportFrom):
+            module = statement.module
+            names = (
+                [module, *(f"{module}.{alias.name}" for alias in statement.names)]
+                if module and statement.level == 0
+                else []
+            )
+        else:
+            pending.extend(reversed(list(_inside(statement, where))))
+            continue
+        for name in names:
+            found.setdefault(
+                ReferencedName(
+                    name,
+                    lazy=where.lazy,
+                    type_checking=where.type_checking,
+                    optional=where.optional,
+                )
+            )
+    return tuple(found)
+
+
+def _inside(statement: ast.stmt, where: _Where) -> Iterator[tuple[ast.stmt, _Where]]:
+    """The statements *statement* holds, in source order, each with where it sits."""
+    if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
+        where = replace(where, lazy=True)
+    for field, value in ast.iter_fields(statement):
+        if not isinstance(value, list):
+            continue
+        held = where
+        if field == "body":
+            held = replace(
+                where,
+                type_checking=where.type_checking or _only_for_type_checking(statement),
+                optional=where.optional or _handles_absence(statement),
+            )
+        for item in cast("list[object]", value):
+            if isinstance(item, ast.stmt):
+                yield item, held
+            elif isinstance(item, (ast.ExceptHandler, ast.match_case)):
+                # A handler's body and a case's body sit where the
+                # statement does: a try's guard covers its own body alone.
+                for inner in item.body:
+                    yield inner, where
+
+
+def _only_for_type_checking(statement: ast.stmt) -> bool:
+    """Whether *statement* is ``if TYPE_CHECKING:``, the name bare or qualified."""
+    if not isinstance(statement, ast.If):
+        return False
+    test = statement.test
+    return (isinstance(test, ast.Name) and test.id == "TYPE_CHECKING") or (
+        isinstance(test, ast.Attribute) and test.attr == "TYPE_CHECKING"
+    )
+
+
+def _handles_absence(statement: ast.stmt) -> bool:
+    """Whether *statement* is a ``try`` with a handler for an absent module.
+
+    A handler for ``ImportError``, ``ModuleNotFoundError`` or every
+    exception says the code runs without what the body imports, which
+    is a declaration of its own and needs no other.
+    """
+    if not isinstance(statement, ast.Try):
+        return False
+    return any(
+        handler.type is None
+        or (
+            isinstance(handler.type, ast.Name)
+            and handler.type.id in ("ImportError", "ModuleNotFoundError")
+        )
+        for handler in statement.handlers
+    )
+
+
+@dataclass(frozen=True)
+class RuleContext:
+    """What a layering rule over the python sources judges: the workspace and a scope.
+
+    Attributes:
+        root: The workspace root.
+        packages: Every package the workspace carries.
+        files: The root-relative sources the run judges; None for every
+            one.
+    """
+
+    root: Path
+    packages: tuple[Package, ...]
+    files: frozenset[str] | None = None
+
+    def packages_in_scope(self) -> tuple[Package, ...]:
+        """The packages holding a source this run judges, every one without a scope."""
+        if self.files is None:
+            return self.packages
+        files = self.files
+        return tuple(
+            package
+            for package in self.packages
+            if any(path.startswith(f"{package.path}/") for path in files)
+        )
+
+
+@dataclass(frozen=True)
+class ParsedModule:
+    """One python source as a rule sees it.
+
+    Attributes:
+        package: The package the file belongs to; None for the root's
+            own ``tasks.py``.
+        path: The file, absolute.
+        relative: The file, relative to the workspace root, posix.
+        area: ``src``, ``tests``, or ``root``.
+        dotted: The module's dotted path under ``src``; empty elsewhere.
+        tree: The syntax tree.
+    """
+
+    package: Package | None
+    path: Path
+    relative: str
+    area: str
+    dotted: str
+    tree: ast.Module
+
+
+def parsed_modules(
+    root: Path, packages: tuple[Package, ...], files: frozenset[str] | None = None
+) -> tuple[ParsedModule, ...]:
+    """Every python source the workspace holds, or the named ones, parsed once.
+
+    The root's own ``tasks.py``, then each package's ``src`` and
+    ``tests`` trees in package order. *files* keeps the root-relative
+    sources it names, and a named file that no longer exists is left
+    out. A file that does not parse is left out: the syntax gate names
+    it.
+    """
+
+    def wanted(source: Path) -> bool:
+        return files is None or source.relative_to(root).as_posix() in files
+
+    found: list[ParsedModule] = []
+    tasks = root / "tasks.py"
+    if tasks.is_file() and wanted(tasks):
+        _collect(found, None, tasks, root, "root", tasks.parent)
+    for package in packages:
+        for area in ("src", "tests"):
+            base = package.directory / area
+            if not base.is_dir():
+                continue
+            if files is not None:
+                prefix = f"{package.path}/{area}/"
+                named = sorted(
+                    root / path
+                    for path in files
+                    if path.startswith(prefix)
+                    and path.endswith(".py")
+                    and (root / path).is_file()
+                )
+                for source in named:
+                    _collect(found, package, source, root, area, base)
+                continue
+            for source in sorted(base.rglob("*.py")):
+                _collect(found, package, source, root, area, base)
+    return tuple(found)
+
+
+def dotted_name(source: Path, src: Path) -> str:
+    """The dotted module path of *source*, a file under the ``src`` tree *src*.
+
+    ``livery/forge/_dev/__init__.py`` under ``src`` is ``livery.forge._dev``.
+    """
+    dotted = ".".join(source.relative_to(src).parts)
+    return dotted.removesuffix(".py").removesuffix(".__init__")
+
+
+def _collect(
+    found: list[ParsedModule],
+    package: Package | None,
+    source: Path,
+    root: Path,
+    area: str,
+    base: Path,
+) -> None:
+    parsed = parsed_source(source)
+    if parsed is None:
+        return
+    found.append(
+        ParsedModule(
+            package=package,
+            path=source,
+            relative=source.relative_to(root).as_posix(),
+            area=area,
+            dotted=dotted_name(source, base) if area == "src" else "",
+            tree=parsed.tree,
+        )
+    )
+
+
+def graph_changed(changes: Changes) -> bool:
+    """Whether a contract, a native manifest or a root ``src/`` changed.
+
+    The layering steps' inputs widen on them: the distributions' roots
+    and declarations come from the manifests, and the allowances from
+    the root contract, so a change there judges every source, as the
+    base's layering steps do. The workshop answers it
+    ([livery.workshop.graph_changed][]); a step's ``widen`` reference
+    names the extension's own code, so this function hands it on.
+    """
+    from livery.workshop import graph_changed as workspace_graph_changed
+
+    return workspace_graph_changed(changes)

livery/extensions/python/_reaches.py

@@ -0,0 +1,165 @@
+"""The private-reaches rule: a source reaching another distribution's privates.
+
+A private is a module or a name with a leading underscore under another
+distribution's root, imported (guarded or under ``TYPE_CHECKING``
+alike) or read as an attribute of something imported from there. A
+test may reach one; a package's source may not, except through the
+root contract's ``[[python.private-reaches]]``, each entry with its
+reason. An entry no source uses any more refuses, so the list only
+shrinks. Each distribution's roots are what its build ships, so the
+roots one wheel ships are one distribution's.
+
+The rule is a step of the layering phase,
+``[phases.layering.private-reaches]``, judged over the one parse of
+the python sources (``parsed_modules``): the files a narrowed run
+reaches, every source otherwise.
+
+The scan reads names, not types: a private read through a value
+(``current()._x``) or named in a string (an entry point,
+``import_module``) is not seen.
+"""
+
+from __future__ import annotations
+
+import ast
+from typing import TYPE_CHECKING
+
+from livery.footman import fail
+from livery.workshop import selected_files
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from livery.extensions.python._parse import ParsedModule, RuleContext
+    from livery.workshop import Package, PhaseContext
+
+#: Where the allowance lives in the root contract.
+ALLOWANCE = "[[python.private-reaches]]"
+
+
+def private_reaches(ctx: PhaseContext) -> None:
+    """The step: refuse each reach outside the allowance, and each unused entry.
+
+    The files the run reaches ([livery.workshop.selected_files][]) are
+    judged over the one parse; an unused entry is judged on a run over
+    every source alone.
+    """
+    from livery.extensions.python._parse import RuleContext, parsed_modules
+
+    files = selected_files(ctx)
+    context = RuleContext(root=ctx.root, packages=ctx.packages, files=files)
+    found = problems(parsed_modules(ctx.root, ctx.packages, files), context)
+    if found:
+        fail("the sources break the private-reaches rule:\n  " + "\n  ".join(found))
+
+
+def problems(modules: tuple[ParsedModule, ...], context: RuleContext) -> list[str]:
+    """Each reach outside the allowance, by its place; each allowance entry unused.
+
+    An unused entry is judged only when the run reads every source: a
+    narrowed run cannot tell an entry nothing uses from one its scope
+    leaves out.
+    """
+    reaches = scan(modules, context.packages)
+    allowed = allowance(context.root)
+    problems = [
+        f"{place}: {source} reaches {name}, another distribution's private; make"
+        f" the name public where it lives, or allow the reach in the root"
+        f" contract's {ALLOWANCE} with its reason"
+        for (source, name), places in sorted(reaches.items())
+        if (source, name) not in allowed
+        for place in places
+    ]
+    if context.files is None:
+        problems += [
+            f"workshop.toml: {ALLOWANCE} allows {source} to reach {name}, and no"
+            " source reaches it any more; delete the entry"
+            for source, name in sorted(allowed)
+            if (source, name) not in reaches
+        ]
+    return problems
+
+
+def scan(
+    modules: tuple[ParsedModule, ...], packages: tuple[Package, ...]
+) -> dict[tuple[str, str], list[str]]:
+    """Every reach in *modules*' sources, by distribution and private, to its places."""
+    owners = _owners(packages)
+    reaches: dict[tuple[str, str], list[str]] = {}
+    for module in modules:
+        if module.package is None or module.area != "src":
+            continue
+        source = module.package.name
+        for name, line in _named(module.tree):
+            roots = [
+                root for root in owners if name == root or name.startswith(root + ".")
+            ]
+            if not roots:
+                continue
+            owner = max(roots, key=len)
+            below = name[len(owner) :].split(".")
+            if owners[owner] == source or not any(map(_private, below)):
+                continue
+            reaches.setdefault((source, name), []).append(f"{module.relative}:{line}")
+    return reaches
+
+
+def allowance(root: Path) -> dict[tuple[str, str], str]:
+    """The root contract's allowed reaches: each distribution and private, to why."""
+    from livery.workshop import read_contract
+
+    table = read_contract(root).get("python", {})
+    entries = table.get("private-reaches", []) if isinstance(table, dict) else []
+    return {
+        (str(entry["from"]), str(entry["reaches"])): str(entry.get("reason", ""))
+        for entry in entries
+        if isinstance(entry, dict)
+    }
+
+
+def _owners(packages: tuple[Package, ...]) -> dict[str, str]:
+    """Each module root a distribution ships, to the distribution."""
+    from livery.workshop import MODULE_ROOTS, answer
+
+    return {
+        root: package.name
+        for package in packages
+        for root in answer(package, MODULE_ROOTS)
+    }
+
+
+def _private(part: str) -> bool:
+    return part.startswith("_") and not (part.startswith("__") and part.endswith("__"))
+
+
+def _dotted(node: ast.expr, imported: dict[str, str]) -> str | None:
+    """The dotted path an attribute chain names, when it starts at an import."""
+    if isinstance(node, ast.Name):
+        return imported.get(node.id)
+    if isinstance(node, ast.Attribute):
+        base = _dotted(node.value, imported)
+        return f"{base}.{node.attr}" if base else None
+    return None
+
+
+def _named(tree: ast.Module) -> list[tuple[str, int]]:
+    """Every dotted path *tree* imports or reads through an import, with its line."""
+    named: list[tuple[str, int]] = []
+    imported: dict[str, str] = {}
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                named.append((alias.name, node.lineno))
+                local = alias.asname or alias.name.partition(".")[0]
+                imported[local] = alias.name if alias.asname else local
+        elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
+            for alias in node.names:
+                path = f"{node.module}.{alias.name}"
+                named.append((path, node.lineno))
+                imported[alias.asname or alias.name] = path
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Attribute) and _private(node.attr):
+            read = _dotted(node, imported)
+            if read is not None:
+                named.append((read, node.lineno))
+    return named

livery/extensions/python/_steps.py

@@ -0,0 +1,71 @@
+"""The python extension's steps: a package's lifecycle phases and its isolated test."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, cast
+
+from livery.extensions.python import _backend
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from livery.workshop import PhaseContext
+
+
+def stamp(ctx: PhaseContext) -> None:
+    """Write the engine's version into ``pyproject.toml`` and every ``__version__``."""
+    version = cast("str", ctx.read("version"))
+    ctx.provide("changed", _backend.stamp_version(ctx.member).stamp(version))
+
+
+def build(ctx: PhaseContext) -> None:
+    """Build the package's wheel and sdist into its ``dist/``, at the engine's epoch."""
+    epoch = cast("int", ctx.read("epoch"))
+    ctx.provide("dist", _backend.build(ctx.member, ctx.root, epoch=epoch))
+
+
+def publish(ctx: PhaseContext) -> None:
+    """Upload the built distributions to the registry target the engine resolved."""
+    from livery.workshop import RegistryTarget
+
+    version = cast("str", ctx.read("version"))
+    target = RegistryTarget.from_table(
+        cast("Mapping[str, object]", ctx.read("registry-target"))
+    )
+    published = _backend.publish_artifact(
+        ctx.member, ctx.root, version=version, target=target
+    )
+    ctx.provide("published", published)
+
+
+def isolated_test(ctx: PhaseContext) -> None:
+    """Test the built wheel in a fresh venv at the engine's resolution; what it got."""
+    from pathlib import Path
+
+    resolution = cast("str", ctx.read("resolution"))
+    dirs = tuple(Path(path) for path in cast("list[Path]", ctx.read("release-dirs")))
+    resolved = _backend.isolated_test(
+        ctx.member, ctx.root, release_dirs=dirs, resolution=resolution
+    )
+    ctx.provide("resolved", dict(resolved))
+
+
+def run(ctx: PhaseContext) -> None:
+    """Start the script the engine names with the words after the dashes; its code."""
+    name = cast("str", ctx.read("executable"))
+    ctx.exit(_backend.run_script(ctx.member, ctx.root, name, ctx.arguments))
+
+
+def verify(ctx: PhaseContext) -> None:
+    """Install the released version alone and run its tests at the engine's tree."""
+    from pathlib import Path
+
+    code = _backend.verify(
+        ctx.member,
+        tree=Path(cast("Path", ctx.read("tree"))),
+        version=cast("str", ctx.read("version")),
+        python=cast("str", ctx.read("interpreter")),
+        index=cast("str", ctx.read("index")),
+        extras=cast("str", ctx.read("extras")),
+    )
+    ctx.provide("exit-code", code)

livery/extensions/python/_terminal.py

@@ -0,0 +1,118 @@
+"""The runner-terminal rule: a module that imports the runner never asks the terminal.
+
+Whether a person is there has one answer, [livery.footman.attended][],
+which knows ``--no-input`` and ``--dry-run`` as well as the terminal. A
+module that imports the runner has that answer in reach, and asking
+the terminal itself (``sys.stdin.isatty()`` and its relatives) ignores
+both flags in silence. The runner's own sources implement the answer
+and are exempt, and a module that never imports the runner is the
+workspace's own business.
+
+The rule is a step of the layering phase,
+``[phases.layering.runner-terminal]``, over the one parse of the python
+sources: the root's ``tasks.py`` and each package's ``src``, the files
+a narrowed run reaches or every one. It runs while the python
+extension is mounted, which a package whose set holds python mounts.
+"""
+
+from __future__ import annotations
+
+import ast
+from typing import TYPE_CHECKING
+
+from livery.footman import fail
+from livery.workshop import selected_files
+
+if TYPE_CHECKING:
+    from livery.extensions.python._parse import ParsedModule
+    from livery.workshop import PhaseContext
+
+#: What a module that imports the runner must never call: a terminal
+#: says nothing about ``--no-input`` or ``--dry-run``.
+TERMINAL_CALLS = (
+    "sys.stdin.isatty",
+    "sys.stdout.isatty",
+    "sys.stderr.isatty",
+    "os.isatty",
+)
+
+
+def runner_terminal(ctx: PhaseContext) -> None:
+    """The step: refuse each terminal question in a module that imports the runner.
+
+    The sources the run reaches ([livery.workshop.selected_files][])
+    are judged over the one parse; every source when the run knows
+    nothing of what changed.
+    """
+    from livery.extensions.python._parse import parsed_modules
+
+    found = problems(parsed_modules(ctx.root, ctx.packages, selected_files(ctx)))
+    if found:
+        fail("the sources break the runner-terminal rule:\n  " + "\n  ".join(found))
+
+
+def problems(modules: tuple[ParsedModule, ...]) -> list[str]:
+    """Each terminal question asked by a module of *modules* that imports the runner.
+
+    The root's ``tasks.py`` and the sources under each package's
+    ``src`` are judged, apart from the runner's own.
+    """
+    from livery.extensions.python._backend import RUNNER_DISTRIBUTION
+
+    found: list[str] = []
+    for module in modules:
+        if module.area not in ("root", "src"):
+            continue
+        if module.package is not None and module.package.name == RUNNER_DISTRIBUTION:
+            continue
+        if not imports_the_runner(module.tree):
+            continue
+        found.extend(
+            f"{module.relative} asks the terminal with {spelling}(): a module"
+            " that imports the runner asks livery.footman.attended() instead,"
+            " which knows --no-input and --dry-run as well"
+            for spelling in terminal_calls(module.tree)
+        )
+    return found
+
+
+def imports_the_runner(tree: ast.Module) -> bool:
+    """Whether the module *tree* parses imports the runner, in any spelling."""
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            if any(
+                alias.name == "livery.footman"
+                or alias.name.startswith("livery.footman.")
+                for alias in node.names
+            ):
+                return True
+        elif isinstance(node, ast.ImportFrom) and node.module:
+            if node.module == "livery.footman" or node.module.startswith(
+                "livery.footman."
+            ):
+                return True
+            if node.module == "livery" and any(
+                alias.name == "footman" for alias in node.names
+            ):
+                return True
+    return False
+
+
+def terminal_calls(tree: ast.Module) -> list[str]:
+    """Every call in the module *tree* parses that asks a terminal directly."""
+    bare = {
+        alias.asname or alias.name
+        for node in ast.walk(tree)
+        if isinstance(node, ast.ImportFrom) and node.module == "sys"
+        for alias in node.names
+        if alias.name in ("stdin", "stdout", "stderr")
+    }
+    found: list[str] = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        spelling = ast.unparse(node.func)
+        named = spelling.endswith(".isatty") and spelling.split(".")[0] in bare
+        if spelling in TERMINAL_CALLS or named:
+            found.append(spelling)
+    return found