pants-backend-generate 0.1.0rc0__tar.gz

pants_backend_generate-0.1.0rc0/LICENSE-MIT.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Tom Solberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pants_backend_generate-0.1.0rc0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.1
+Name: pants_backend_generate
+Version: 0.1.0rc0
+Summary: A Pants backend for code generators whose outputs are committed: typed codegen targets, a `generate` goal that writes the outputs and a linter that checks them.
+Keywords: pantsbuild,pants,backend,codegen,generate
+Author-Email: Tom Solberg <me@sbg.dev>
+License: MIT License
+         
+         Copyright (c) 2022 Tom Solberg
+         
+         Permission is hereby granted, free of charge, to any person obtaining a copy
+         of this software and associated documentation files (the "Software"), to deal
+         in the Software without restriction, including without limitation the rights
+         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+         copies of the Software, and to permit persons to whom the Software is
+         furnished to do so, subject to the following conditions:
+         
+         The above copyright notice and this permission notice shall be included in all
+         copies or substantial portions of the Software.
+         
+         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+         SOFTWARE.
+         
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Build Tools
+Project-URL: Repository, https://github.com/tgolsson/pants-backends
+Project-URL: Code, https://github.com/tgolsson/pants-backends/tree/main/pants-plugins/generate
+Project-URL: Changelog, https://github.com/tgolsson/pants-backends/tree/main/pants-plugins/generate/CHANGELOG.md
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# pants-backend-generate
+
+A [Pants](https://www.pantsbuild.org) backend for code generators whose outputs are committed
+because something outside Pants reads them from git. Inside Pants a generator is ordinary
+codegen, so consumers get fresh outputs in their sandboxes. `pants generate` writes the
+committed copies and `pants lint` fails when one is stale. Both use the same cached codegen
+rule, so the committed bytes are the bytes Pants consumers see.
+
+This release targets Pants **2.32.x**. Install it with `pants-backend-generate==0.1.0rc0` in
+`[GLOBAL].plugins` in `pants.toml`, and enable `pants_backend_generate` in
+`[GLOBAL].backend_packages`.
+
+**Status: alpha.**
+
+## `file_generate`
+
+Generated files (`FileSourceField`), kept as written. Language backends ship their own
+specializations with a formatter, for example `cpp_generate` in pants-backend-cmake. Every
+`*_generate` target has these fields:
+
+- `runnable` (required): the generator, a target `pants run` can run in a sandbox (such as a
+  `pex_binary`), with `args`.
+- `sources`: files the generator reads, relative to the BUILD file. Optional.
+- `outputs` (required): every file the generator writes, as exact, normalised paths relative
+  to the build root.
+- `execution_dependencies`: targets whose files are present in the sandbox.
+- `runnable_dependencies`: runnable targets put on `PATH`.
+- `extra_env_vars`, `timeout`, `environment`.
+
+The generator runs at the build root with its `sources` and `execution_dependencies` at their
+repo-relative paths, and the parent directory of every output already exists. The outputs are
+exact. Each of these is an error:
+
+- a declared file that is not written
+- an undeclared file written next to a declared one
+- an input in an output directory that the generator overwrites
+- an output that is also an input (exclude the committed copy with a `!` glob)
+
+Put outputs in a directory: files at the build root are not checked for extras.
+
+The host environment is not inherited, `PATH` included. A generator that runs host tools needs
+`extra_env_vars=["PATH"]` or the tools as `runnable_dependencies`.
+
+The committed outputs belong to no target:
+
+- If another target's `sources` glob matches a committed output, a consumer of both targets
+  gets two copies. Once the copy is stale, that fails with an opaque `MergeDigests` error.
+  Exclude the output from that glob with `!`.
+- File-address specs and `--changed-since` do not reach the generating target, so run the
+  check as `pants lint ::`.
+
+## `pants generate`
+
+Runs the generators of the given `*_generate` targets and writes their outputs into the
+workspace. It prints one `<address>: wrote <files>` line per target. Two targets writing the
+same file is an error, and then nothing is written.
+
+## `pants lint`
+
+The `generate` linter reports each committed output that differs from the generated one
+(`stale`) or does not exist (`missing`), followed by the `pants generate` command that
+refreshes them. `--generate-skip` turns it off.
+
+## Specializing
+
+Codegen dispatches on the class of the target's sources field, so each output type needs its
+own target type. `GenerateSpecialization` makes one. In the sketch below, `MySourceField` is the
+sources field class your consumers ask for:
+
+```python
+from pants.engine.rules import collect_rules, rule
+from pants.engine.unions import UnionRule
+
+from pants_backend_generate.specialization import GenerateSpecialization
+from pants_backend_generate.util_rules.formatting import FormatOutputsRequest, FormattedOutputs
+
+
+class MyFormatOutputsRequest(FormatOutputsRequest):
+    pass
+
+
+@rule
+async def format_my_outputs(request: MyFormatOutputsRequest) -> FormattedOutputs:
+    ...  # format request.snapshot; return exactly the same files
+
+
+my_generate = GenerateSpecialization(
+    "my_generate",
+    output=MySourceField,
+    formatter=MyFormatOutputsRequest,  # or None
+    help="Sources written by a generator.",
+)
+
+
+def target_types():
+    return my_generate.target_types()
+
+
+def rules():
+    return [
+        *collect_rules(),
+        *my_generate.rules(),
+        UnionRule(FormatOutputsRequest, MyFormatOutputsRequest),
+    ]
+```
+
+A specialization's `rules()` carries everything codegen needs, so another backend can register
+it without `pants_backend_generate` in `backend_packages`. Codegen then works, and a one-time
+warning says that `pants generate` and the linter need the backend enabled.
+
+A formatter runs inside the codegen rule. It receives `snapshot` and `address` with
+`EnvironmentName` in scope, and it must return the same set of files. Aliases must be unique.
+Several specializations may share an `output`.

pants_backend_generate-0.1.0rc0/README.md

@@ -0,0 +1,113 @@
+# pants-backend-generate
+
+A [Pants](https://www.pantsbuild.org) backend for code generators whose outputs are committed
+because something outside Pants reads them from git. Inside Pants a generator is ordinary
+codegen, so consumers get fresh outputs in their sandboxes. `pants generate` writes the
+committed copies and `pants lint` fails when one is stale. Both use the same cached codegen
+rule, so the committed bytes are the bytes Pants consumers see.
+
+This release targets Pants **2.32.x**. Install it with `pants-backend-generate==0.1.0rc0` in
+`[GLOBAL].plugins` in `pants.toml`, and enable `pants_backend_generate` in
+`[GLOBAL].backend_packages`.
+
+**Status: alpha.**
+
+## `file_generate`
+
+Generated files (`FileSourceField`), kept as written. Language backends ship their own
+specializations with a formatter, for example `cpp_generate` in pants-backend-cmake. Every
+`*_generate` target has these fields:
+
+- `runnable` (required): the generator, a target `pants run` can run in a sandbox (such as a
+  `pex_binary`), with `args`.
+- `sources`: files the generator reads, relative to the BUILD file. Optional.
+- `outputs` (required): every file the generator writes, as exact, normalised paths relative
+  to the build root.
+- `execution_dependencies`: targets whose files are present in the sandbox.
+- `runnable_dependencies`: runnable targets put on `PATH`.
+- `extra_env_vars`, `timeout`, `environment`.
+
+The generator runs at the build root with its `sources` and `execution_dependencies` at their
+repo-relative paths, and the parent directory of every output already exists. The outputs are
+exact. Each of these is an error:
+
+- a declared file that is not written
+- an undeclared file written next to a declared one
+- an input in an output directory that the generator overwrites
+- an output that is also an input (exclude the committed copy with a `!` glob)
+
+Put outputs in a directory: files at the build root are not checked for extras.
+
+The host environment is not inherited, `PATH` included. A generator that runs host tools needs
+`extra_env_vars=["PATH"]` or the tools as `runnable_dependencies`.
+
+The committed outputs belong to no target:
+
+- If another target's `sources` glob matches a committed output, a consumer of both targets
+  gets two copies. Once the copy is stale, that fails with an opaque `MergeDigests` error.
+  Exclude the output from that glob with `!`.
+- File-address specs and `--changed-since` do not reach the generating target, so run the
+  check as `pants lint ::`.
+
+## `pants generate`
+
+Runs the generators of the given `*_generate` targets and writes their outputs into the
+workspace. It prints one `<address>: wrote <files>` line per target. Two targets writing the
+same file is an error, and then nothing is written.
+
+## `pants lint`
+
+The `generate` linter reports each committed output that differs from the generated one
+(`stale`) or does not exist (`missing`), followed by the `pants generate` command that
+refreshes them. `--generate-skip` turns it off.
+
+## Specializing
+
+Codegen dispatches on the class of the target's sources field, so each output type needs its
+own target type. `GenerateSpecialization` makes one. In the sketch below, `MySourceField` is the
+sources field class your consumers ask for:
+
+```python
+from pants.engine.rules import collect_rules, rule
+from pants.engine.unions import UnionRule
+
+from pants_backend_generate.specialization import GenerateSpecialization
+from pants_backend_generate.util_rules.formatting import FormatOutputsRequest, FormattedOutputs
+
+
+class MyFormatOutputsRequest(FormatOutputsRequest):
+    pass
+
+
+@rule
+async def format_my_outputs(request: MyFormatOutputsRequest) -> FormattedOutputs:
+    ...  # format request.snapshot; return exactly the same files
+
+
+my_generate = GenerateSpecialization(
+    "my_generate",
+    output=MySourceField,
+    formatter=MyFormatOutputsRequest,  # or None
+    help="Sources written by a generator.",
+)
+
+
+def target_types():
+    return my_generate.target_types()
+
+
+def rules():
+    return [
+        *collect_rules(),
+        *my_generate.rules(),
+        UnionRule(FormatOutputsRequest, MyFormatOutputsRequest),
+    ]
+```
+
+A specialization's `rules()` carries everything codegen needs, so another backend can register
+it without `pants_backend_generate` in `backend_packages`. Codegen then works, and a one-time
+warning says that `pants generate` and the linter need the backend enabled.
+
+A formatter runs inside the codegen rule. It receives `snapshot` and `address` with
+`EnvironmentName` in scope, and it must return the same set of files. Aliases must be unique.
+Several specializations may share an `output`.

pants_backend_generate-0.1.0rc0/pants_backend_generate/file_generate.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from pants.core.target_types import FileSourceField
+
+from pants_backend_generate.specialization import GenerateSpecialization
+
+file_generate = GenerateSpecialization(
+    "file_generate",
+    output=FileSourceField,
+    help="Files written by a generator, kept as written.",
+)

pants_backend_generate-0.1.0rc0/pants_backend_generate/goals/generate.py

@@ -0,0 +1,102 @@
+"""`pants generate` writes the codegen outputs into the workspace: `fix` cannot create files and
+`export-codegen` writes only under `dist/`."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+
+from pants.engine.addresses import Address
+from pants.engine.console import Console
+from pants.engine.fs import MergeDigests, Snapshot, Workspace
+from pants.engine.goal import Goal, GoalSubsystem
+from pants.engine.internals.graph import hydrate_sources
+from pants.engine.internals.specs_rules import find_valid_field_sets_for_target_roots
+from pants.engine.intrinsics import merge_digests
+from pants.engine.rules import collect_rules, concurrently, goal_rule, implicitly
+from pants.engine.target import (
+    HydrateSourcesRequest,
+    NoApplicableTargetsBehavior,
+    TargetRootsToFieldSetsRequest,
+)
+from pants.engine.unions import UnionMembership
+from pants.option.option_types import SkipOption
+from pants.util.strutil import bullet_list
+
+from pants_backend_generate.target_types import GenerateFieldSet
+
+
+class GenerateSubsystem(GoalSubsystem):
+    name = "generate"
+    help = "Run the generators of `*_generate` targets and write their outputs into the workspace."
+
+    # Also the linter's tool subsystem: two subsystems cannot share the `generate` scope.
+    skip = SkipOption("lint")
+
+    @classmethod
+    def activated(cls, union_membership: UnionMembership) -> bool:
+        return GenerateFieldSet in union_membership
+
+
+class Generate(Goal):
+    subsystem_cls = GenerateSubsystem
+    environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY
+
+
+@dataclass(frozen=True)
+class GeneratedOutputs:
+    address: Address
+    snapshot: Snapshot
+
+
+async def generate_outputs(field_set: GenerateFieldSet) -> GeneratedOutputs:
+    hydrated = await hydrate_sources(
+        HydrateSourcesRequest(
+            field_set.sources,
+            for_sources_types=(field_set.output_type,),
+            enable_codegen=True,
+        ),
+        **implicitly(),
+    )
+    return GeneratedOutputs(field_set.address, hydrated.snapshot)
+
+
+@goal_rule
+async def generate(console: Console, workspace: Workspace) -> Generate:
+    targets_to_field_sets = await find_valid_field_sets_for_target_roots(
+        TargetRootsToFieldSetsRequest(
+            GenerateFieldSet,
+            goal_description=f"the `{GenerateSubsystem.name}` goal",
+            no_applicable_targets_behavior=NoApplicableTargetsBehavior.warn,
+        ),
+        **implicitly(),
+    )
+    results = await concurrently(
+        generate_outputs(field_set) for field_set in targets_to_field_sets.field_sets
+    )
+    results = sorted(results, key=lambda r: r.address)
+
+    writers: defaultdict[str, list[Address]] = defaultdict(list)
+    for result in results:
+        for path in result.snapshot.files:
+            writers[path].append(result.address)
+    clashes = [
+        f"{path}: {', '.join(str(a) for a in addresses)}"
+        for path, addresses in sorted(writers.items())
+        if len(addresses) > 1
+    ]
+    if clashes:
+        console.print_stderr(
+            f"More than one target writes the same file; nothing was written:\n\n{bullet_list(clashes)}"
+        )
+        return Generate(exit_code=1)
+
+    digest = await merge_digests(MergeDigests(result.snapshot.digest for result in results))
+    workspace.write_digest(digest)
+    for result in results:
+        console.print_stdout(f"{result.address}: wrote {', '.join(result.snapshot.files)}")
+    return Generate(exit_code=0)
+
+
+def rules():
+    return collect_rules()

pants_backend_generate-0.1.0rc0/pants_backend_generate/goals/lint.py

@@ -0,0 +1,60 @@
+"""The `generate` linter: committed outputs must match what their generator writes now."""
+
+from __future__ import annotations
+
+from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
+from pants.core.goals.lint import LintResult, LintTargetsRequest
+from pants.core.util_rules.partitions import PartitionerType
+from pants.engine.fs import PathGlobs
+from pants.engine.intrinsics import get_digest_contents, path_globs_to_digest
+from pants.engine.rules import collect_rules, rule
+from pants.util.logging import LogLevel
+
+from pants_backend_generate.goals.generate import GenerateSubsystem, generate_outputs
+from pants_backend_generate.target_types import GenerateFieldSet
+
+
+class GenerateLintRequest(LintTargetsRequest):
+    field_set_type = GenerateFieldSet
+    tool_subsystem = GenerateSubsystem  # type: ignore[assignment]
+    partitioner_type = PartitionerType.DEFAULT_ONE_PARTITION_PER_INPUT
+
+
+async def _stale_outputs(field_set: GenerateFieldSet) -> list[str]:
+    generated = await generate_outputs(field_set)
+    committed_digest = await path_globs_to_digest(
+        PathGlobs(
+            field_set.outputs.value,
+            glob_match_error_behavior=GlobMatchErrorBehavior.ignore,
+        )
+    )
+    generated_contents = await get_digest_contents(generated.snapshot.digest)
+    committed_contents = await get_digest_contents(committed_digest)
+    expected = {fc.path: fc.content for fc in generated_contents}
+    committed = {fc.path: fc.content for fc in committed_contents}
+    return [
+        f"{field_set.address}: {path}: {'missing' if path not in committed else 'stale'}"
+        for path in sorted(expected)
+        if committed.get(path) != expected[path]
+    ]
+
+
+@rule(desc="Check generated files are current", level=LogLevel.DEBUG)
+async def lint_generated(request: GenerateLintRequest.Batch) -> LintResult:
+    lines: list[str] = []
+    for field_set in request.elements:
+        lines.extend(await _stale_outputs(field_set))
+    stderr = ""
+    if lines:
+        addresses = " ".join(sorted({str(field_set.address) for field_set in request.elements}))
+        stderr = "\n".join(lines) + f"\n\nRun `pants generate {addresses}` to refresh."
+    return LintResult(
+        exit_code=1 if lines else 0,
+        stdout="",
+        stderr=stderr,
+        linter_name=request.tool_name,
+    )
+
+
+def rules():
+    return [*collect_rules(), *GenerateLintRequest.rules()]

pants_backend_generate-0.1.0rc0/pants_backend_generate/register.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from pants_backend_generate.file_generate import file_generate
+from pants_backend_generate.goals import generate, lint
+from pants_backend_generate.util_rules import formatting, generator
+
+
+def target_types():
+    return file_generate.target_types()
+
+
+def rules():
+    return [
+        *generator.rules(),
+        *formatting.rules(),
+        *generate.rules(),
+        *lint.rules(),
+        *file_generate.rules(),
+    ]

pants_backend_generate-0.1.0rc0/pants_backend_generate/specialization.py

@@ -0,0 +1,127 @@
+"""Codegen dispatches on the class of the target's sources field, so each output type needs its
+own sources field, target type, request and field set; `GenerateSpecialization` builds all four."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field, make_dataclass
+
+from pants.core.environments.target_types import EnvironmentField
+from pants.engine.rules import collect_rules, rule
+from pants.engine.target import (
+    COMMON_TARGET_FIELDS,
+    GeneratedSources,
+    GenerateSourcesRequest,
+    SourcesField,
+    Target,
+)
+from pants.engine.unions import UnionRule
+from pants.option.global_options import GlobalOptions
+from pants.util.memo import memoized_method
+
+from pants_backend_generate.target_types import (
+    GENERATE_FIELDS,
+    GenerateFieldSet,
+    GenerateOutputsField,
+    GenerateSourcesField,
+)
+from pants_backend_generate.util_rules import formatting, generator
+from pants_backend_generate.util_rules.formatting import FormatOutputsRequest
+from pants_backend_generate.util_rules.generator import run_generator
+
+logger = logging.getLogger(__name__)
+
+_ALIASES: set[str] = set()
+_warned_without_goal_backend = False
+
+
+def _warn_without_goal_backend(global_options: GlobalOptions) -> None:
+    global _warned_without_goal_backend
+    if _warned_without_goal_backend or "pants_backend_generate" in global_options.backend_packages:
+        return
+    _warned_without_goal_backend = True
+    logger.warning(
+        "`pants generate` and the `generate` linter are unavailable until `pants_backend_generate` "
+        "is added to `[GLOBAL].backend_packages`."
+    )
+
+
+@dataclass(frozen=True)
+class GenerateSpecialization:
+    """A `*_generate` target type whose outputs are `output` sources, formatted by `formatter`."""
+
+    alias: str
+    output: type[SourcesField]
+    formatter: type[FormatOutputsRequest] | None = None
+    help: str = ""
+
+    sources_field_type: type[GenerateSourcesField] = field(init=False, repr=False)
+    target_type: type[Target] = field(init=False, repr=False)
+    request_type: type[GenerateSourcesRequest] = field(init=False, repr=False)
+    field_set_type: type[GenerateFieldSet] = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        # Otherwise the second target type silently shadows the first.
+        if self.alias in _ALIASES:
+            raise ValueError(f"A `GenerateSpecialization` with alias {self.alias!r} already exists.")
+        _ALIASES.add(self.alias)
+
+        # As Pants' `wrap_source_rule_and_target` does: one class of each per specialization.
+        class SourcesField(GenerateSourcesField):
+            output_type = self.output
+            formatter_type = self.formatter
+
+        class Request(GenerateSourcesRequest):
+            input = SourcesField
+            output = self.output
+
+        class SpecializedTarget(Target):
+            alias = self.alias
+            core_fields = (*COMMON_TARGET_FIELDS, SourcesField, *GENERATE_FIELDS, EnvironmentField)
+            help = self.help
+
+        # `FieldSet.fields` uses `get_type_hints`: the annotation must be the class, not a name.
+        field_set_type = make_dataclass(
+            "FieldSet",
+            [("sources", SourcesField)],
+            bases=(GenerateFieldSet,),
+            frozen=True,
+            namespace={"required_fields": (SourcesField, GenerateOutputsField)},
+        )
+        camel = "".join(part.capitalize() for part in self.alias.split("_"))
+        for cls, suffix in (
+            (SourcesField, "SourcesField"),
+            (Request, "Request"),
+            (SpecializedTarget, "Target"),
+            (field_set_type, "FieldSet"),
+        ):
+            cls.__name__ = cls.__qualname__ = f"{camel}{suffix}"
+        object.__setattr__(self, "sources_field_type", SourcesField)
+        object.__setattr__(self, "target_type", SpecializedTarget)
+        object.__setattr__(self, "request_type", Request)
+        object.__setattr__(self, "field_set_type", field_set_type)
+
+    def target_types(self) -> tuple[type[Target], ...]:
+        return (self.target_type,)
+
+    # A second rule with the same canonical name would clash.
+    @memoized_method
+    def rules(self) -> tuple:
+        @rule(
+            canonical_name_suffix=self.alias,
+            _param_type_overrides={"request": self.request_type},
+        )
+        async def generate(
+            request: GenerateSourcesRequest, global_options: GlobalOptions
+        ) -> GeneratedSources:
+            _warn_without_goal_backend(global_options)
+            return await run_generator(request)
+
+        # The codegen path works without the goal backend enabled; the engine dedupes these.
+        return (
+            *generator.rules(),
+            *formatting.rules(),
+            *collect_rules({"generate": generate}),
+            UnionRule(GenerateSourcesRequest, self.request_type),
+            UnionRule(GenerateFieldSet, self.field_set_type),
+        )

pants_backend_generate-0.1.0rc0/pants_backend_generate/target_types.py

@@ -0,0 +1,173 @@
+"""Fields shared by every `*_generate` target; only `sources` is subclassed per specialization."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import ClassVar
+
+from pants.engine.addresses import Address
+from pants.engine.env_vars import EXTRA_ENV_VARS_USAGE_HELP
+from pants.engine.target import (
+    FieldSet,
+    IntField,
+    InvalidFieldException,
+    MultipleSourcesField,
+    SourcesField,
+    SpecialCasedDependencies,
+    StringField,
+    StringSequenceField,
+    ValidNumbers,
+)
+from pants.engine.unions import union
+from pants.util.strutil import help_text, softwrap
+
+_GLOB_CHARACTERS = frozenset("*?[]{}!")
+
+
+class GenerateRunnableField(StringField):
+    alias: ClassVar[str] = "runnable"
+    required = True
+    help = help_text(
+        """
+        Address of the generator, a target `pants run` can run in a sandbox (such as a
+        `pex_binary`). It runs at the build root.
+        """
+    )
+
+
+class GenerateArgsField(StringSequenceField):
+    alias: ClassVar[str] = "args"
+    default = ()
+    help = "Arguments passed to the `runnable`."
+
+
+class GenerateSourcesField(MultipleSourcesField):
+    alias: ClassVar[str] = "sources"
+    required = False
+    default = ()
+    help = help_text(
+        """
+        Files the generator reads, relative to the BUILD file; present in the sandbox at their
+        repo-relative paths.
+        """
+    )
+
+    output_type: ClassVar[type[SourcesField]]
+    formatter_type: ClassVar[type | None] = None
+
+
+class GenerateOutputsField(StringSequenceField):
+    alias: ClassVar[str] = "outputs"
+    required = True
+    help = help_text(
+        """
+        Every file the generator writes, exactly: build-root-relative, normalised paths, each
+        once, no globs. A missing or an extra file is an error.
+        """
+    )
+
+    @classmethod
+    def compute_value(cls, raw_value: Iterable[str] | None, address: Address) -> tuple[str, ...]:
+        value = super().compute_value(raw_value, address)
+        if not value:
+            raise InvalidFieldException(f"The `{cls.alias}` field of {address} must name at least one file.")
+        seen: set[str] = set()
+        for path in value:
+            problem = _output_path_problem(path)
+            if problem:
+                raise InvalidFieldException(f"The `{cls.alias}` field of {address} has {path!r}: {problem}.")
+            if path in seen:
+                raise InvalidFieldException(
+                    f"The `{cls.alias}` field of {address} lists {path!r} more than once."
+                )
+            seen.add(path)
+        return value
+
+
+def _output_path_problem(path: str) -> str | None:
+    if not path:
+        return "outputs must not be empty"
+    if os.path.basename(path) in ("", "."):
+        return "outputs are files, not directories"
+    if os.path.isabs(path):
+        return "outputs are paths relative to the build root"
+    if _GLOB_CHARACTERS & set(path):
+        return "outputs are exact paths, not globs"
+    normalised = os.path.normpath(path)
+    if normalised == ".":
+        return "outputs are files, not directories"
+    if normalised.split("/")[0] == "..":
+        return "outputs must stay inside the build root"
+    if normalised != path:
+        return softwrap(
+            f"""
+            write it as {normalised!r} (outputs are compared as strings)
+            """
+        )
+    return None
+
+
+class GenerateExecutionDependenciesField(SpecialCasedDependencies):
+    alias: ClassVar[str] = "execution_dependencies"
+    help = help_text(
+        """
+        Targets whose transitive files are present in the sandbox at their repo-relative
+        paths.
+        """
+    )
+
+
+class GenerateRunnableDependenciesField(SpecialCasedDependencies):
+    alias: ClassVar[str] = "runnable_dependencies"
+    help = help_text(
+        """
+        Runnable targets put on the generator's `PATH` under their target name.
+        """
+    )
+
+
+class GenerateExtraEnvVarsField(StringSequenceField):
+    alias: ClassVar[str] = "extra_env_vars"
+    help = help_text(
+        f"""
+        Additional environment variables for the generator.
+
+        {EXTRA_ENV_VARS_USAGE_HELP}
+        """
+    )
+
+
+class GenerateTimeoutField(IntField):
+    alias: ClassVar[str] = "timeout"
+    default = None
+    valid_numbers = ValidNumbers.positive_only
+    help = "Seconds the generator may run before it is killed. No limit by default."
+
+
+GENERATE_FIELDS = (
+    GenerateRunnableField,
+    GenerateArgsField,
+    GenerateOutputsField,
+    GenerateExecutionDependenciesField,
+    GenerateRunnableDependenciesField,
+    GenerateExtraEnvVarsField,
+    GenerateTimeoutField,
+)
+
+
+@union
+@dataclass(frozen=True)
+class GenerateFieldSet(FieldSet):
+    """The `lint` goal instantiates this base directly, so `output_type` reads the field
+    instance, not the annotation."""
+
+    required_fields = (GenerateSourcesField, GenerateOutputsField)
+
+    sources: GenerateSourcesField
+    outputs: GenerateOutputsField
+
+    @property
+    def output_type(self) -> type[SourcesField]:
+        return self.sources.output_type

pants_backend_generate-0.1.0rc0/pants_backend_generate/util_rules/formatting.py

@@ -0,0 +1,35 @@
+"""Formatting runs inside the codegen rule, because `fmt` never sees codegen output and `lint`
+compares against the formatted bytes."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from pants.engine.addresses import Address
+from pants.engine.environment import EnvironmentName
+from pants.engine.fs import Snapshot
+from pants.engine.rules import collect_rules, rule
+from pants.engine.unions import union
+
+
+@union(in_scope_types=[EnvironmentName])
+@dataclass(frozen=True)
+class FormatOutputsRequest:
+    """Union base: a member's rule returns `FormattedOutputs` with exactly the files given."""
+
+    snapshot: Snapshot
+    address: Address
+
+
+@dataclass(frozen=True)
+class FormattedOutputs:
+    snapshot: Snapshot
+
+
+@rule(polymorphic=True)
+async def format_outputs(request: FormatOutputsRequest, env_name: EnvironmentName) -> FormattedOutputs:
+    raise NotImplementedError()
+
+
+def rules():
+    return collect_rules()

pants_backend_generate-0.1.0rc0/pants_backend_generate/util_rules/generator.py

@@ -0,0 +1,176 @@
+"""The codegen rule body. Output directories are captured whole so that undeclared and overwritten
+files are seen; outputs at the build root therefore get neither check."""
+
+from __future__ import annotations
+
+import os
+
+from pants.core.environments.rules import (
+    EnvironmentNameRequest,
+    get_target_for_environment_name,
+    resolve_environment_name,
+)
+from pants.core.util_rules import adhoc_process_support
+from pants.core.util_rules.adhoc_process_support import (
+    AdhocProcessRequest,
+    ToolRunnerRequest,
+    convert_fallible_adhoc_process_result,
+    create_tool_runner,
+    prepare_env_vars,
+)
+from pants.engine.environment import EnvironmentName
+from pants.engine.fs import (
+    CreateDigest,
+    DigestSubset,
+    Directory,
+    FileEntry,
+    MergeDigests,
+    PathGlobs,
+)
+from pants.engine.intrinsics import (
+    create_digest,
+    digest_subset_to_digest,
+    digest_to_snapshot,
+    get_digest_entries,
+    merge_digests,
+)
+from pants.engine.rules import concurrently, implicitly
+from pants.engine.target import GeneratedSources, GenerateSourcesRequest
+from pants.util.frozendict import FrozenDict
+from pants.util.strutil import bullet_list
+
+from pants_backend_generate.target_types import (
+    GenerateArgsField,
+    GenerateExecutionDependenciesField,
+    GenerateExtraEnvVarsField,
+    GenerateOutputsField,
+    GenerateRunnableDependenciesField,
+    GenerateRunnableField,
+    GenerateTimeoutField,
+)
+from pants_backend_generate.util_rules.formatting import FormatOutputsRequest, format_outputs
+
+
+class GenerateOutputsError(Exception):
+    pass
+
+
+async def run_generator(request: GenerateSourcesRequest) -> GeneratedSources:
+    target = request.protocol_target
+    address = target.address
+    outputs = target[GenerateOutputsField].value
+    description = f"the `{target.alias}` at {address}"
+
+    environment_name = await resolve_environment_name(
+        EnvironmentNameRequest.from_target(target), **implicitly()
+    )
+    environment_target = await get_target_for_environment_name(environment_name, **implicitly())
+    tool_runner = await create_tool_runner(
+        ToolRunnerRequest(
+            runnable_address_str=target[GenerateRunnableField].value,
+            args=target[GenerateArgsField].value or (),
+            execution_dependencies=target[GenerateExecutionDependenciesField].value or (),
+            runnable_dependencies=target[GenerateRunnableDependenciesField].value or (),
+            target=target,
+            runnable_address_field_alias=GenerateRunnableField.alias,
+        )
+    )
+
+    output_directories = tuple(sorted({os.path.dirname(o) for o in outputs} - {""}))
+    directories = await create_digest(CreateDigest(Directory(d) for d in output_directories))
+    input_digest = await merge_digests(
+        MergeDigests([tool_runner.digest, request.protocol_sources.digest, directories])
+    )
+    input_snapshot = await digest_to_snapshot(input_digest)
+    overlap = sorted(set(outputs) & set(input_snapshot.files))
+    if overlap:
+        raise GenerateOutputsError(
+            f"{description} declares outputs that are also inputs (in `sources`, or from "
+            f"`execution_dependencies` or the runnable):\n\n{bullet_list(overlap)}\n\n"
+            "Exclude the committed copy from those inputs, for example with a `!` glob."
+        )
+
+    env_vars = await prepare_env_vars(
+        tool_runner.extra_env,
+        target[GenerateExtraEnvVarsField].value or (),
+        extra_paths=tool_runner.extra_paths,
+        description_of_origin=f"`{GenerateExtraEnvVarsField.alias}` of {description}",
+    )
+    process_request = AdhocProcessRequest(
+        description=description,
+        address=address,
+        working_directory="",
+        root_output_directory="",
+        argv=tool_runner.args,
+        timeout=target[GenerateTimeoutField].value,
+        input_digest=input_digest,
+        immutable_input_digests=FrozenDict.frozen(tool_runner.immutable_input_digests),
+        append_only_caches=FrozenDict(tool_runner.append_only_caches),
+        output_files=outputs,
+        output_directories=output_directories,
+        env_vars=env_vars,
+        log_on_process_errors=None,
+        log_output=False,
+        capture_stdout_file=None,
+        capture_stderr_file=None,
+        workspace_invalidation_globs=None,
+        cache_scope=environment_target.default_cache_scope,
+        # The exact-set check below replaces adhoc_tool's glob check.
+        use_working_directory_as_base_for_output_captures=False,
+        outputs_match_conjunction=None,
+    )
+    result = await convert_fallible_adhoc_process_result(
+        **implicitly({environment_name: EnvironmentName, process_request: AdhocProcessRequest})
+    )
+
+    input_entries, produced_entries = await concurrently(
+        get_digest_entries(input_digest), get_digest_entries(result.adjusted_digest)
+    )
+    inputs = {e.path: e.file_digest for e in input_entries if isinstance(e, FileEntry)}
+    produced = {e.path: e.file_digest for e in produced_entries if isinstance(e, FileEntry)}
+    missing = sorted(set(outputs) - set(produced))
+    # Unchanged inputs inside a captured output directory are not extras.
+    extra = sorted(set(produced) - set(outputs) - set(inputs))
+    overwritten = sorted(
+        path for path, digest in produced.items() if path in inputs and inputs[path] != digest
+    )
+    if missing or extra or overwritten:
+        problems = []
+        if missing:
+            problems.append(f"Declared in `outputs` but not written:\n\n{bullet_list(missing)}")
+        if extra:
+            problems.append(f"Written but not declared in `outputs`:\n\n{bullet_list(extra)}")
+        if overwritten:
+            problems.append(
+                "Written but not declared in `outputs` (these were inputs, which the generator "
+                f"overwrote):\n\n{bullet_list(overwritten)}"
+            )
+        raise GenerateOutputsError(
+            f"{description} did not write exactly its `outputs`.\n\n" + "\n\n".join(problems)
+        )
+
+    exact_digest = await digest_subset_to_digest(DigestSubset(result.adjusted_digest, PathGlobs(outputs)))
+    exact = await digest_to_snapshot(exact_digest)
+
+    formatter_type = request.input.formatter_type
+    if formatter_type is None:
+        return GeneratedSources(exact)
+
+    formatted = await format_outputs(
+        **implicitly(
+            {
+                formatter_type(snapshot=exact, address=address): FormatOutputsRequest,
+                environment_name: EnvironmentName,
+            }
+        )
+    )
+    if set(formatted.snapshot.files) != set(exact.files):
+        raise GenerateOutputsError(
+            f"The formatter {formatter_type.__name__} changed the files of {description}: "
+            f"it was given {sorted(exact.files)} and returned {sorted(formatted.snapshot.files)}."
+        )
+    return GeneratedSources(formatted.snapshot)
+
+
+def rules():
+    return [*adhoc_process_support.rules()]

pants_backend_generate-0.1.0rc0/pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "pants_backend_generate"
+description = "A Pants backend for code generators whose outputs are committed: typed codegen targets, a `generate` goal that writes the outputs and a linter that checks them."
+version = "0.1.0rc0"
+requires-python = ">=3.11"
+authors = [
+    { name = "Tom Solberg", email = "me@sbg.dev" },
+]
+dependencies = []
+readme = "README.md"
+keywords = [
+    "pantsbuild",
+    "pants",
+    "backend",
+    "codegen",
+    "generate",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Programming Language :: Python",
+    "Topic :: Software Development :: Build Tools",
+]
+
+[project.license]
+file = "LICENSE-MIT.txt"
+
+[project.urls]
+Repository = "https://github.com/tgolsson/pants-backends"
+Code = "https://github.com/tgolsson/pants-backends/tree/main/pants-plugins/generate"
+Changelog = "https://github.com/tgolsson/pants-backends/tree/main/pants-plugins/generate/CHANGELOG.md"
+
+[tool.pdm.build]
+includes = [
+    "pants_backend_generate",
+]
+
+[tool.isort]
+profile = "black"
+known_first_party = [
+    "pants_backend_generate",
+]
+line_length = 100
+include_trailing_comma = true
+
+[tool.black]
+line-length = 100
+
+[build-system]
+requires = [
+    "pdm-backend>2.0",
+]
+build-backend = "pdm.backend"