livery-extensions-python 0.0.0__py3-none-any.whl

livery/extensions/python/_declared.py

@@ -0,0 +1,252 @@
+"""The declared-imports rule: a package's source imports what its manifest declares.
+
+At import time, a module under a package's ``src`` imports only the
+standard library, the workspace's own members, and the distributions
+the package's ``pyproject.toml`` declares in ``[project.dependencies]``.
+Inside a function or a method it may also import a distribution an
+optional extra declares, since importing the module does not run that
+import. An import under ``TYPE_CHECKING``, or in a ``try`` that handles
+``ImportError``, is not judged: no run makes the first, and the source
+runs without the second.
+
+A module a program loads as a plugin, named in one of the program's
+entry-point groups (``footman.tasks`` and ``footman.builtin`` for the
+runner, ``pytest11`` for pytest), may also import what that program
+brings: its distribution and the distributions it requires. The
+program is present whenever such a module runs. A name another member
+owns is left to the sibling-references step, which judges the edges
+between members.
+
+Which distribution provides a module is read from the metadata of the
+distributions the workspace's environment installed, so a module that
+no installed distribution provides refuses, naming it.
+
+The rule is a step of the layering phase,
+``[phases.layering.declared-imports]``, over each python package's
+``references`` answer: the sources a narrowed run reaches, every one
+otherwise.
+"""
+
+from __future__ import annotations
+
+import sys
+import tomllib
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, cast
+
+from livery.footman import fail, prog
+from livery.workshop import selected_files
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from livery.extensions.python._parse import RuleContext
+    from livery.workshop import (
+        Neighbours,
+        Package,
+        PhaseContext,
+        ReferencedName,
+        SourceReferences,
+    )
+
+#: Where a package declares the distributions its sources import at
+#: import time.
+DEPENDENCIES = "[project.dependencies]"
+
+
+def declared_imports(ctx: PhaseContext) -> None:
+    """The step: refuse each import that no declaration of its package accounts for.
+
+    The sources the run reaches ([livery.workshop.selected_files][])
+    are judged; every source when the run knows nothing of what changed.
+    """
+    from livery.extensions.python._parse import RuleContext
+
+    context = RuleContext(
+        root=ctx.root, packages=ctx.packages, files=selected_files(ctx)
+    )
+    found = problems(context)
+    if found:
+        fail("the sources break the declared-imports rule:\n  " + "\n  ".join(found))
+
+
+@dataclass(frozen=True)
+class Manifest:
+    """What a package's ``pyproject.toml`` declares, as the rule reads it.
+
+    Attributes:
+        path: The file, relative to the workspace root, posix.
+        dependencies: The distributions ``[project.dependencies]``
+            names, normalized.
+        extras: Each optional extra, to the distributions it names,
+            normalized.
+        plugins: Each module the package names in a plugin entry-point
+            group, to the distributions of the programs that load it.
+    """
+
+    path: str
+    dependencies: frozenset[str]
+    extras: Mapping[str, frozenset[str]]
+    plugins: Mapping[str, frozenset[str]]
+
+    def loaded_by(self, module: str) -> frozenset[str]:
+        """The programs that load *module* as a plugin, or a package above it."""
+        found: set[str] = set()
+        for plugin, programs in self.plugins.items():
+            if module == plugin or module.startswith(plugin + "."):
+                found |= programs
+        return frozenset(found)
+
+    def extra_naming(self, distributions: frozenset[str]) -> str:
+        """The first optional extra naming one of *distributions*; empty for none."""
+        return next(
+            (extra for extra, names in self.extras.items() if names & distributions),
+            "",
+        )
+
+
+def manifest(package: Package) -> Manifest:
+    """What *package*'s ``pyproject.toml`` declares that the rule reads."""
+    from livery.extensions.python._backend import plugin_hosts
+    from livery.extensions.python._installed import requirement_name
+
+    pyproject = package.directory / "pyproject.toml"
+    data = tomllib.loads(pyproject.read_text("utf-8")) if pyproject.is_file() else {}
+    project = cast("dict[str, Any]", data.get("project", {}))
+    dependencies = cast("list[str]", project.get("dependencies", []))
+    extras = cast("dict[str, list[str]]", project.get("optional-dependencies", {}))
+    return Manifest(
+        path=f"{package.path}/pyproject.toml",
+        dependencies=frozenset(map(requirement_name, dependencies)),
+        extras={
+            extra: frozenset(map(requirement_name, requirements))
+            for extra, requirements in extras.items()
+        },
+        plugins=plugin_hosts(package),
+    )
+
+
+def problems(context: RuleContext) -> list[str]:
+    """Each import no declaration accounts for, in package and source order.
+
+    A package that holds the python extension is judged, each of its
+    sources under ``src`` that the run reaches, as this extension's
+    ``references`` answer records them.
+    """
+    from livery.extensions.python._backend import references
+    from livery.workshop import extension_set, neighbours
+
+    around = neighbours(context.packages)
+    found: list[str] = []
+    for package in context.packages_in_scope():
+        if "python" not in extension_set(package):
+            continue
+        declared = manifest(package)
+        shipped = f"{package.path}/src/"
+        for source in references(package):
+            if not source.path.startswith(shipped):
+                continue
+            if context.files is not None and source.path not in context.files:
+                continue
+            found += judged(source, declared, around)
+    return found
+
+
+def judged(
+    source: SourceReferences, declared: Manifest, around: Neighbours
+) -> list[str]:
+    """What *source* imports that *declared* does not account for, one line each.
+
+    Import-time references are judged before lazy ones, and each
+    distribution is named once per source, at the first reference that
+    needs it. A namespace several distributions share is judged through
+    the names under it that the source references, since those decide
+    which distribution the source needs.
+    """
+    from livery.extensions.python._installed import installed
+
+    judgeable = [
+        reference
+        for reference in source.references
+        if not reference.type_checking and not reference.optional
+    ]
+    ordered = [
+        *(reference for reference in judgeable if not reference.lazy),
+        *(reference for reference in judgeable if reference.lazy),
+    ]
+    names = {reference.name for reference in judgeable}
+    programs = declared.loaded_by(source.module)
+    reported: set[object] = set()
+    found: list[str] = []
+    for reference in ordered:
+        name = reference.name
+        if _standard(name) or _member(name, around):
+            continue
+        environment = installed()
+        providers = environment.providers(name)
+        if not providers:
+            top = name.partition(".")[0]
+            if top not in reported:
+                reported.add(top)
+                found.append(
+                    f"{source.path} imports {name}, and no distribution installed"
+                    " in the workspace's environment provides it: declare the"
+                    f" distribution that does in {declared.path}, then"
+                    f" `{prog()} sync` installs it; one an optional extra declares"
+                    " installs once the package's dev-extras lists that extra"
+                )
+            continue
+        if len(providers) > 1 and any(other.startswith(name + ".") for other in names):
+            continue
+        allowed = set(declared.dependencies)
+        for program in programs:
+            allowed |= environment.brings(program)
+        if reference.lazy:
+            for extra in declared.extras.values():
+                allowed |= extra
+        if providers & allowed or providers in reported:
+            continue
+        reported.add(providers)
+        found.append(_refusal(source, reference, providers, declared))
+    return found
+
+
+def _refusal(
+    source: SourceReferences,
+    reference: ReferencedName,
+    providers: frozenset[str],
+    declared: Manifest,
+) -> str:
+    """The line that names *reference*'s module, its distribution and the manifest."""
+    named = " or ".join(sorted(providers))
+    if reference.lazy:
+        return (
+            f"{source.path} imports {reference.name} inside a function, and"
+            f" {named} provides it, which {declared.path} declares neither in"
+            f" {DEPENDENCIES} nor in an optional extra: declare it in one of them"
+        )
+    extra = declared.extra_naming(providers)
+    if extra:
+        return (
+            f"{source.path} imports {reference.name} at import time, and {named}"
+            f" provides it, which {declared.path} declares in its optional extra"
+            f" {extra!r} alone: import it inside the function that uses it, or"
+            f" declare it in {DEPENDENCIES}"
+        )
+    return (
+        f"{source.path} imports {reference.name} at import time, and {named}"
+        f" provides it, which {declared.path} does not declare in {DEPENDENCIES}:"
+        " declare it there"
+    )
+
+
+def _standard(name: str) -> bool:
+    """Whether *name* is in the standard library of the running interpreter."""
+    return name.partition(".")[0] in sys.stdlib_module_names
+
+
+def _member(name: str, around: Neighbours) -> bool:
+    """Whether a workspace member owns *name*, or *name* is a namespace above one."""
+    return bool(around.owner_of(name)) or any(
+        root.startswith(name + ".") for root in around.owners
+    )

livery/extensions/python/_docs/index.md

@@ -0,0 +1,165 @@
+# livery-extensions-python
+
+Python distributions: each package's version stamped, its wheel and
+sdist built, published to a python index and verified, its suite run
+against the installed wheel for a release, and the layering rules over
+the python sources. A package turns it on by listing the extension in
+its own `workshop.toml`:
+
+```toml
+extensions = ["python"]
+name = "acme-core"
+```
+
+The package's `pyproject.toml` is its manifest: it declares the
+version, the requirements and the scripts. `fm new.project` lists
+python at the root, since the workspace's own tests and `tasks.py` are
+python. A workspace that does not list it has no python steps, no
+`python-library` template, and no way to create a `workshop-extension`
+package, whose template extends `python-library`. The nanobind
+extension requires python, since its packages are python
+distributions, and its distribution requires this one.
+
+## A new package
+
+`fm new.package.python-library <name>` creates a python package while
+the workspace lists python: a typed module under `src/`, its tests, and
+a `pyproject.toml` uv builds. `--command=<script>` adds the script
+`<script>` to `[project.scripts]`, which runs `main()` in the package's
+`__main__.py`, and a test of it; without it the package has no script.
+A workspace that does not list python has no such verb: list the
+extension in `[workspace] extensions` and run `fm sync`, which installs
+`livery-extensions-python`, and the verb exists. Until then
+`fm new.package.workshop-extension` refuses, naming python and its
+distribution.
+
+## A package's files
+
+python gives the paths of each package whose set holds it these
+categories:
+
+| Category | Patterns |
+| --- | --- |
+| `test` | `tests/**/test_*.py`, `tests/**/*_test.py` |
+| `test-support` | `tests/**` |
+| `source` | `src/**` |
+
+The most specific pattern wins, so a test module under `tests/` is a
+test and every other file there, `conftest.py` among them, supports
+the tests.
+
+## A release
+
+python adds a step to each phase of a python package's release:
+
+- `stamp.python` writes the version into `pyproject.toml` and into
+  each `__init__.py` under `src/` that carries `__version__`.
+- `build.python` builds the wheel and the sdist with `uv build` into
+  the package's `dist/`, from an empty `dist/` each time. The wheel
+  carries the package's `docs/` as the module's `_docs`, copied for
+  the build and removed after it. The build refuses a wheel tagged for
+  a platform: python's build ships a pure wheel, tagged `none-any`, so
+  a package with compiled code lists the extension that builds its
+  platform wheels, such as nanobind.
+- `test.python` installs the built wheel into a fresh environment and
+  runs the package's tests against the installed copy, once per
+  resolution the release proves: `floor` installs every direct
+  dependency at its declared floor, `latest` the newest, and `head` the
+  versions the workspace lock pins. The wheel resolves from the
+  co-released set's `dist/` directories and the workspace's indexes
+  alone, and installs with its `test` extra when the package declares
+  one. The test toolchain installs after the wheel, at the lock's
+  pins, and a toolchain install that moves a direct dependency the leg
+  resolved refuses, naming it.
+- `publish.python` uploads the wheel and the sdist with `uv publish` to
+  the registry target the release resolves. An empty upload address
+  refuses, since uv's own default is PyPI. An empty
+  `UV_PUBLISH_TOKEN` never reaches uv, so trusted publishing still
+  runs, and a version the index already holds is walked past.
+- `verify.python`, which `fm release.verify` runs, installs the
+  released version alone into a plain environment and runs the
+  package's tests at the released tree against it.
+
+The release train probes the python index the registry ladder resolves
+for the package, through the simple API, before it uploads.
+
+## fm run
+
+python answers `executables` with the scripts a package's
+`pyproject.toml` names under `[project.scripts]`. `fm run` starts the
+one it names from the workspace's environment, which `fm sync`
+installs, with the words after the dashes, and exits with its code. A
+script the environment lacks refuses, naming `fm sync`.
+
+## The layering rules
+
+python answers the `references` query for python sources from one
+parse per gate, memoised by the file's path, size and modification
+time. Its three steps of the layering phase read the same parse, over
+the files a narrowed run reaches or every source:
+
+- `private-reaches`: no package's source reaches another
+  distribution's private name, apart from the reaches the root
+  contract's `[[python.private-reaches]]` allows, each with its reason.
+  An entry no source uses any more refuses.
+- `runner-terminal`: a module that imports the runner asks
+  `livery.footman.attended()` whether a person is there, never the
+  terminal (`sys.stdin.isatty()` and its relatives), since the
+  terminal knows nothing of `--no-input` or `--dry-run`. The runner's
+  own sources, and a module that never imports the runner, are exempt.
+  The step judges the root's `tasks.py` with the packages' sources.
+- `declared-imports`: at import time, a module under a package's `src`
+  imports the standard library, the workspace's own members, and the
+  distributions its `pyproject.toml` declares in
+  `[project.dependencies]`. Inside a function it may also import a
+  distribution an optional extra declares. An import under
+  `TYPE_CHECKING`, or in a `try` that handles `ImportError`, is not
+  judged. A module a program loads as a plugin, named under the
+  `footman.tasks` or `footman.builtin` entry points for the runner or
+  `pytest11` for pytest, may also import what that program brings: its
+  distribution and the distributions it requires. A refusal names the
+  source, the module, the distribution and the `pyproject.toml` to
+  declare it in.
+
+An allowed reach is a table of the root contract:
+
+```toml
+[[python.private-reaches]]
+from = "acme-tools"
+reaches = "acme.core._registry.purge"
+reason = "the loop deletes published versions until the admin protocol exists"
+```
+
+## Requirements
+
+python's requirement writers take a sibling that holds python alone. A
+build or runtime edge writes the sibling's requirement into the
+package's `[project.dependencies]` at the edge's floor; a test edge
+writes it into the `test` extra of `[project.optional-dependencies]`,
+which is python's answer to `test-requirements`, and which the
+release's isolated test installs the wheel with. A manifest with no
+`[project]` table refuses, and so does a test extra in a form the
+edit does not take, naming the line to write by hand.
+
+## The queries it declares
+
+- `python.test.runner` and `python.examples.runner`: who runs the
+  python packages' suites, several in one call, and one package's
+  documentation examples. python answers both while pytest is listed,
+  and pytest's test and examples steps ask them.
+- `python.test.preview`: who prints a machine's coverage preview after
+  its suites ran. The coverage extension answers it while python is
+  listed.
+
+## Its public names
+
+`livery.extensions.python` serves the names an extension that builds a
+python package's wheels with a tool of its own reads, as nanobind
+does:
+
+- `module_docs(package)` copies the package's `docs/` into its source
+  tree as the wheel's embedded `_docs` for one build, and removes the
+  copy after it.
+- `run_isolated_test(package, root, ...)` installs a built wheel into a
+  fresh environment and runs the package's tests against the installed
+  copy.

livery/extensions/python/_docs/nav.toml

@@ -0,0 +1,14 @@
+# This package's section of the workspace site, hand-authored.
+# Labels map to pages relative to this docs/ tree, or to nested
+# lists. Every authored page must appear here, and every entry must
+# name a real page. A generator owns a block between `# nav:begin
+# <name>` and `# nav:end <name>` markers (tasks, tools); the emitter's
+# own sections, changelog, coverage and api, are blocks too: place a
+# pair where one belongs, and a section without one lands after the
+# authored entries, changelog and coverage ahead of the tasks block,
+# api last.
+nav = [
+    { "Index" = "index.md" },
+    # nav:begin tasks
+    # nav:end tasks
+]

livery/extensions/python/_installed.py

@@ -0,0 +1,219 @@
+"""Read what the environment's installed distributions ship and require.
+
+The declared-imports rule asks the environment two questions: which
+installed distribution ships the module a source imports, and what a
+distribution brings with it. Both answers come from the distributions'
+own metadata (``importlib.metadata``) beside the running interpreter,
+which is the workspace's environment when the gate runs, read once per
+process. The answer reads the same files on every platform: a regular
+install's ``RECORD``, and the path an editable install's ``.pth`` file
+adds, so a workspace member installed for development and the same
+distribution installed from its wheel ship the same modules.
+"""
+
+from __future__ import annotations
+
+import csv
+import functools
+import re
+from collections.abc import Iterable, Iterator, Mapping
+from dataclasses import dataclass
+from importlib import machinery, metadata
+from pathlib import Path
+
+#: The name at the start of a requirement, before its extras, version,
+#: marker or URL.
+_NAME = re.compile(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
+
+#: A marker that makes a requirement part of an optional extra.
+_EXTRA = re.compile(r"\bextra\b")
+
+#: The suffixes a module's file ends with on this interpreter, longest
+#: first, so ``.cpython-314-darwin.so`` wins over ``.so``.
+_SUFFIXES = tuple(sorted(machinery.all_suffixes(), key=len, reverse=True))
+
+
+def normalized(name: str) -> str:
+    """*name* as distributions are compared: lower case, each run of ``-_.`` a dash."""
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def requirement_name(requirement: str) -> str:
+    """The normalized distribution name a requirement string names; empty for none."""
+    match = _NAME.match(requirement)
+    return normalized(match.group(1)) if match else ""
+
+
+@dataclass(frozen=True)
+class Installed:
+    """What the environment's distributions ship, and what each requires.
+
+    Attributes:
+        modules: Each module path a distribution ships, and each package
+            path above it, to the distributions shipping it, by
+            normalized name.
+        requires: Each installed distribution, by normalized name, to
+            the distributions it requires outside its optional extras.
+    """
+
+    modules: Mapping[str, frozenset[str]]
+    requires: Mapping[str, frozenset[str]]
+
+    def providers(self, name: str) -> frozenset[str]:
+        """The distributions shipping the module that *name* names; empty for none.
+
+        The longest dotted prefix of *name* that is a module decides,
+        so ``yaml.safe_load`` resolves to the distribution that ships
+        ``yaml``, and a namespace several distributions share to all
+        of them.
+        """
+        parts = name.split(".")
+        for end in range(len(parts), 0, -1):
+            found = self.modules.get(".".join(parts[:end]))
+            if found:
+                return found
+        return frozenset()
+
+    def brings(self, distribution: str) -> frozenset[str]:
+        """*distribution* and every distribution it requires, transitively.
+
+        A requirement under any marker but an extra's counts, whatever
+        the platform, so the answer is the same on every machine.
+        """
+        found: set[str] = set()
+        pending = [normalized(distribution)]
+        while pending:
+            name = pending.pop()
+            if name not in found:
+                found.add(name)
+                pending.extend(self.requires.get(name, ()))
+        return frozenset(found)
+
+
+@functools.cache
+def installed() -> Installed:
+    """The environment's distributions, read from their metadata once per process."""
+    return read_installed(metadata.distributions())
+
+
+def read_installed(distributions: Iterable[metadata.Distribution]) -> Installed:
+    """What *distributions* ship and require, read from their metadata."""
+    modules: dict[str, set[str]] = {}
+    requires: dict[str, frozenset[str]] = {}
+    for distribution in distributions:
+        about = distribution.metadata
+        name = normalized(about["Name"]) if "Name" in about else ""
+        if not name:
+            continue
+        requires[name] = requires.get(name, frozenset()) | _required(distribution)
+        for module in _shipped(distribution):
+            for end in range(1, len(module) + 1):
+                modules.setdefault(".".join(module[:end]), set()).add(name)
+    return Installed(
+        modules={module: frozenset(names) for module, names in modules.items()},
+        requires=requires,
+    )
+
+
+def _required(distribution: metadata.Distribution) -> frozenset[str]:
+    """The distributions *distribution* requires outside its optional extras."""
+    found: set[str] = set()
+    for requirement in distribution.requires or ():
+        spec, _, marker = requirement.partition(";")
+        if _EXTRA.search(marker):
+            continue
+        name = requirement_name(spec)
+        if name:
+            found.add(name)
+    return frozenset(found)
+
+
+def _shipped(distribution: metadata.Distribution) -> Iterator[tuple[str, ...]]:
+    """Each module *distribution* ships, as its dotted parts.
+
+    A regular install lists every file in its ``RECORD``. An editable
+    install lists a ``.pth`` file at the top of the environment instead,
+    whose lines add directories to the import path: what those hold is
+    the distribution's.
+    """
+    for file in _files(distribution):
+        parts = file.split("/")
+        if len(parts) == 1 and file.endswith(".pth"):
+            pth = Path(str(distribution.locate_file(file)))
+            for directory in _pth_directories(pth):
+                yield from _modules_under(directory, ())
+            continue
+        module = _module_name(parts[-1])
+        if module is None or not all(part.isidentifier() for part in parts[:-1]):
+            continue
+        yield tuple(parts[:-1] if module == "__init__" else [*parts[:-1], module])
+
+
+def _files(distribution: metadata.Distribution) -> list[str]:
+    """The files *distribution* installed, each relative to its environment, posix.
+
+    Read from ``RECORD`` as text, which costs a fraction of building
+    a path object for each of the tens of thousands of lines an
+    environment lists; an install without one answers through
+    ``importlib.metadata``.
+    """
+    record = distribution.read_text("RECORD")
+    if record is None:
+        return [str(file) for file in distribution.files or ()]
+    return [row[0] for row in csv.reader(record.splitlines()) if row and row[0]]
+
+
+def _pth_directories(pth: Path) -> Iterator[Path]:
+    """The directories the ``.pth`` file *pth* adds to the import path.
+
+    A line that is empty, a comment, or code (``import ...``) adds
+    none; a relative line is read against the file's own directory, as
+    the interpreter reads it.
+    """
+    try:
+        lines = pth.read_text("utf-8").splitlines()
+    except (OSError, UnicodeDecodeError):
+        return
+    for line in lines:
+        entry = line.strip()
+        if not entry or entry.startswith(("#", "import ", "import\t")):
+            continue
+        directory = pth.parent / entry
+        if directory.is_dir():
+            yield directory
+
+
+def _modules_under(
+    directory: Path, prefix: tuple[str, ...]
+) -> Iterator[tuple[str, ...]]:
+    """What *directory* on the import path makes importable, under *prefix*.
+
+    A regular package (a directory holding ``__init__.py``) is named
+    whole, since everything under it is the package's. A directory
+    without one is a namespace portion, read through to what it holds.
+    """
+    try:
+        entries = sorted(directory.iterdir())
+    except OSError:
+        return
+    for entry in entries:
+        if entry.is_dir():
+            if entry.name == "__pycache__" or not entry.name.isidentifier():
+                continue
+            if (entry / "__init__.py").is_file():
+                yield (*prefix, entry.name)
+            else:
+                yield from _modules_under(entry, (*prefix, entry.name))
+            continue
+        module = _module_name(entry.name)
+        if module is not None and module != "__init__":
+            yield (*prefix, module)
+
+
+def _module_name(file: str) -> str | None:
+    """The module a file named *file* is, by its suffix; None for no module."""
+    for suffix in _SUFFIXES:
+        if file.endswith(suffix):
+            module = file[: -len(suffix)]
+            return module if module.isidentifier() else None
+    return None