vendoring 1.3.0__py3-none-any.whl

vendoring/__init__.py

@@ -0,0 +1,6 @@
+"""A command line tool, to simplify vendoring pure Python dependencies."""
+
+# WARNING: vendoring is a CLI-only tool. DO NOT `import vendoring` or anything
+#          within this namespace.
+
+__version__ = "1.3.0"

vendoring/__main__.py

@@ -0,0 +1,4 @@
+from . import cli
+
+if __name__ == "__main__":
+    cli.main()

vendoring/cli.py

@@ -0,0 +1,94 @@
+import sys
+from pathlib import Path
+from typing import Callable, List, NamedTuple, Optional
+
+import click
+
+from vendoring.configuration import load_configuration
+from vendoring.errors import VendoringError
+from vendoring.interactive import interactive_updates
+from vendoring.sync import run_sync
+from vendoring.tasks.update import update_requirements
+from vendoring.ui import UI
+
+_EntryPoint = Callable[..., None]
+_Param = Callable[[_EntryPoint], _EntryPoint]
+
+
+class _Template(NamedTuple):
+    # Arguments
+    package: _Param
+    # Options
+    verbose: _Param
+
+
+template = _Template(
+    package=click.argument("package", default=None, required=False, type=str),
+    verbose=click.option("-v", "--verbose", is_flag=True),
+)
+
+
+@click.group()
+def main() -> None:
+    pass
+
+
+@main.command()
+@template.verbose
+def sync(verbose: bool) -> None:
+    """Vendor libraries as described in lockfile"""
+    UI.verbose = verbose
+    project_path = Path()
+
+    print(f"Working in {project_path}")
+
+    try:
+        with UI.task("Load configuration"):
+            config = load_configuration(project_path)
+        run_sync(config)
+    except VendoringError as e:
+        UI.show_error(e)
+        sys.exit(1)
+
+
+@main.command()
+@template.verbose
+@template.package
+def update(verbose: bool, package: Optional[str]) -> None:
+    """Update a single package version"""
+    UI.verbose = verbose
+    project_path = Path()
+
+    try:
+        with UI.task("Load configuration"):
+            config = load_configuration(project_path)
+        with UI.task("Updating requirements"):
+            update_requirements(config, package)
+    except VendoringError as e:
+        UI.show_error(e)
+        sys.exit(1)
+
+
+@main.command("update-interactive")
+@click.option("--skip", help="Skip named packages", multiple=True)
+@click.option("--only", help="Only update named packages", multiple=True)
+@click.option(
+    "--from-start",
+    help="Start from the beginning, ignoring existing markers",
+    is_flag=True,
+)
+@template.verbose
+def update_interactive(
+    verbose: bool, skip: List[str], only: List[str], from_start: bool
+) -> None:
+    """Update all package versions, interactively"""
+    UI.verbose = verbose
+    project_path = Path()
+
+    try:
+        with UI.task("Load configuration"):
+            config = load_configuration(project_path)
+        interactive_updates(config, skip=skip, only=only, from_start=from_start)
+    except VendoringError as e:
+        UI.show_error(e)
+        sys.exit(1)

vendoring/configuration.py

@@ -0,0 +1,178 @@
+"""Loads configuration from pyproject.toml"""
+
+# mypy: allow-any-generics, allow-any-explicit
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from jsonschema import ValidationError, validate
+from tomli import TOMLDecodeError
+from tomli import loads as parse_toml
+
+from vendoring.errors import ConfigurationError
+from vendoring.ui import UI
+
+
+@dataclass
+class Configuration:
+    # Base directory for all of the operation of this project
+    base_directory: Path
+
+    # Location to unpack into
+    destination: Path
+    # Final namespace to rewrite imports to originate from
+    namespace: str
+    # Path to a pip-style requirement files
+    requirements: Path
+    # Filenames to ignore in target directory
+    protected_files: List[str]
+    # Location to ``.patch` files to apply after vendoring
+    patches_dir: Optional[Path]
+
+    # Additional substitutions, done in addition to import rewriting
+    substitute: List[Dict[str, str]]
+    # Drop
+    drop_paths: List[str]
+
+    # Fallbacks for licenses that can't be found
+    license_fallback_urls: Dict[str, str]
+    # Alternate directory name, when distribution name differs from the package name
+    license_directories: Dict[str, str]
+
+    # Overrides for which stub files are generated
+    typing_stubs: Dict[str, List[str]]
+
+    # SBOM file
+    sbom_file: Optional[Path]
+
+    @classmethod
+    def load_from_dict(
+        cls, dictionary: Dict[str, Any], *, location: Path
+    ) -> "Configuration":
+        """Constructs a Configuration object from dictionary.
+
+        It also performs validation of the values in `dictionary`,
+        expecting paths to be within `location`.
+        """
+
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "required": ["destination", "namespace", "requirements"],
+            "properties": {
+                "destination": {"type": "string"},
+                "namespace": {"type": "string"},
+                "requirements": {"type": "string"},
+                "protected-files": {"type": "array", "items": {"type": "string"}},
+                "patches-dir": {"type": "string"},
+                "sbom-file": {"type": "string"},
+                "transformations": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "properties": {
+                        "substitute": {
+                            "type": "array",
+                            "items": {
+                                "type": "object",
+                                "additionalProperties": False,
+                                "required": ["match", "replace"],
+                                "properties": {
+                                    "match": {"type": "string"},
+                                    "replace": {"type": "string"},
+                                },
+                            },
+                        },
+                        "drop": {"type": "array", "items": {"type": "string"}},
+                    },
+                },
+                "license": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "properties": {
+                        "directories": {
+                            "type": "object",
+                            "patternProperties": {"^.*$": {"type": "string"}},
+                        },
+                        "fallback-urls": {
+                            "type": "object",
+                            "patternProperties": {"^.*$": {"type": "string"}},
+                        },
+                    },
+                },
+                "typing-stubs": {
+                    "type": "object",
+                    "patternProperties": {
+                        "^.*$": {"type": "array", "items": {"type": "string"}},
+                    },
+                },
+            },
+        }
+
+        try:
+            validate(dictionary, schema)
+        except ValidationError as e:
+            raise ConfigurationError(str(e))
+
+        def path_or_none(key: str) -> Optional[Path]:
+            if key in dictionary:
+                return Path(dictionary[key])
+            return None
+
+        return Configuration(
+            base_directory=location,
+            destination=Path(dictionary["destination"]),
+            namespace=dictionary["namespace"],
+            requirements=Path(dictionary["requirements"]),
+            protected_files=dictionary.get("protected-files", []),
+            patches_dir=path_or_none("patches-dir"),
+            sbom_file=path_or_none("sbom-file"),
+            substitute=dictionary.get("transformations", {}).get("substitute", {}),
+            drop_paths=dictionary.get("transformations", {}).get("drop", []),
+            license_fallback_urls=dictionary.get("license", {}).get(
+                "fallback-urls", {}
+            ),
+            license_directories=dictionary.get("license", {}).get("directories", {}),
+            typing_stubs=dictionary.get("typing-stubs", {}),
+        )
+
+
+def load_configuration(directory: Path) -> Configuration:
+    # Read the contents of the file.
+    file = directory / "pyproject.toml"
+    UI.log(f"Will attempt to load {file}.")
+
+    try:
+        file_contents = file.read_text(encoding="utf8")
+    except IOError as read_error:
+        raise ConfigurationError("Could not read pyproject.toml.") from read_error
+    else:
+        UI.log("Read configuration file.")
+
+    try:
+        parsed_contents = parse_toml(file_contents)
+    except TOMLDecodeError as toml_error:
+        raise ConfigurationError("Could not parse pyproject.toml.") from toml_error
+    else:
+        UI.log("Parsed configuration file.")
+
+    if (
+        "tool" not in parsed_contents
+        or not isinstance(parsed_contents["tool"], dict)
+        or "vendoring" not in parsed_contents["tool"]
+        or not isinstance(parsed_contents["tool"]["vendoring"], dict)
+    ):
+        raise ConfigurationError("Can not load `tool.vendoring` from pyproject.toml")
+
+    tool_config = parsed_contents["tool"]["vendoring"]
+
+    try:
+        retval = Configuration.load_from_dict(tool_config, location=directory)
+    except ConfigurationError as e:
+        raise ConfigurationError(
+            "Could not load values from [tool.vendoring] in pyproject.toml.\n"
+            f"  REASON: {e}"
+        )
+    else:
+        UI.log("Validated configuration.")
+        return retval

vendoring/errors.py

@@ -0,0 +1,10 @@
+class VendoringError(Exception):
+    """Errors originating from this package."""
+
+
+class ConfigurationError(VendoringError):
+    """Errors related to configuration handling."""
+
+
+class RequirementsError(VendoringError):
+    """Errors related to requirements.txt handling."""

vendoring/interactive.py

@@ -0,0 +1,197 @@
+"""Interactive mode, for updating all packages."""
+
+from __future__ import annotations
+
+import subprocess
+from typing import Literal
+
+import click as _click
+from packaging.version import Version
+
+from vendoring.configuration import Configuration
+from vendoring.errors import VendoringError
+from vendoring.sync import run_sync
+from vendoring.tasks.update import (
+    PinnedPackageInfo,
+    determine_latest_release,
+    parse_pinned_packages,
+)
+from vendoring.ui import UI
+
+Action = Literal["skip", "incomplete", "done"]
+
+
+class InteractionState:
+    """Manages the state of the interactive session.
+
+    This handles the following:
+    - Reading the requirements file
+    - Writing the requirements file
+    - Reading the "state" file
+    - Keeping track of the current package, in memory and in the "state" file
+    - Writing the "state" file
+    - Cleaning up the "state" file
+    """
+
+    def __init__(self, config: Configuration) -> None:
+        self._requirements = config.requirements
+        self._state_file = (
+            config.base_directory
+            / ".vendoring_cache"
+            / "do-not-commit.interactive.current-package"
+        )
+
+        packages = parse_pinned_packages(self._requirements)
+        self._packages = {p.name: p for p in packages}
+        self._package_order = [p.name for p in packages]
+
+        self._current_package: str | None = None
+        if self._state_file.exists():
+            self._current_package = self._state_file.read_text()
+
+    @property
+    def packages(self) -> tuple[tuple[str, str], ...]:
+        """Return the list of packages."""
+        return tuple((p.name, p.version) for p in self._packages.values())
+
+    @property
+    def resuming_from(self) -> PinnedPackageInfo | None:
+        """Return the package we are resuming from."""
+        if self._current_package is None:
+            return None
+
+        if self._current_package not in self._packages:
+            raise VendoringError(
+                "The package to 'resume from' is not in requirements file\n"
+                f"package_name: {self._current_package}\n"
+                f"packages: {list(self._packages)}\n"
+                "Please run with `--from-start` to reset the state."
+            )
+        return self._packages[self._current_package]
+
+    def get_info(self, package: list[str]) -> list[PinnedPackageInfo]:
+        return [self._packages[p] for p in package]
+
+    def update(self, package_name: str, version: str) -> None:
+        """Update the state of the session."""
+        self._packages[package_name].version = version
+        with self._requirements.open("w", encoding="utf-8") as f:
+            f.writelines(f"{self._packages[p]}\n" for p in self._package_order)
+
+        self._current_package = package_name
+        if not self._state_file.parent.exists():
+            self._state_file.parent.mkdir()
+        self._state_file.write_text(package_name)
+
+    def cleanup(self) -> None:
+        if not self._state_file.exists():
+            return
+
+        self._state_file.unlink()
+        self._state_file.parent.rmdir()
+
+
+def git(*args: str) -> None:
+    """Run a git command."""
+    cmd = ["git", *args]
+
+    UI.log(" ".join(map(str, cmd)))
+    subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL)
+
+
+def do_one_update(config: Configuration, *, name: str, version: str) -> None:
+    """Perform the update of a single package."""
+    run_sync(config)
+
+    # Determine the correct message
+    message = f"Upgrade {name} to {version}"
+    # Write our news fragment
+    news_file = config.base_directory / "news" / (name + ".vendor.rst")
+    news_file.write_text(message + "\n")  # "\n" appeases end-of-line-fixer
+
+    # Commit the changes
+    git("add", str(news_file))
+    git("commit", "-m", message)
+
+
+def determine_packages(
+    packages: tuple[tuple[str, str], ...],
+    *,
+    resuming_from: PinnedPackageInfo | None,
+    skip: list[str],
+    only: list[str],
+) -> list[str]:
+    """Determine the packages to update."""
+    if resuming_from is not None:
+        if resuming_from.name in skip:
+            raise VendoringError(
+                f"Cannot resume from {resuming_from!r} as it is in the skip list"
+            )
+        if only and resuming_from.name not in only:
+            raise VendoringError(
+                f"Cannot resume from {resuming_from!r} as it is not in the only list"
+            )
+
+    if only:
+        known = {package[0] for package in packages}
+        for name in only:
+            if name not in known:
+                raise VendoringError(
+                    f"Package {name!r} is not in the requirements file"
+                )
+        return only
+
+    if skip:
+        return [package[0] for package in packages if package[0] not in skip]
+
+    return [package[0] for package in packages]
+
+
+def interactive_updates(
+    config: Configuration, *, skip: list[str], only: list[str], from_start: bool
+) -> None:
+    """Interactive mode, for updating all packages."""
+    with UI.task("Read incoming requirements"):
+        state = InteractionState(config)
+
+    if from_start:
+        state.cleanup()
+
+    resuming_from = state.resuming_from
+    package_names = determine_packages(
+        state.packages, resuming_from=resuming_from, skip=skip, only=only
+    )
+
+    def _present(package_info: PinnedPackageInfo, *, prefix: str | None = None) -> None:
+        real_prefix = f"{prefix} " if prefix else ""
+        UI.log(
+            f"{real_prefix}"
+            f"{_click.style(package_info.name, fg='green')} "
+            f"{_click.style('==', fg='blue')} "
+            f"{_click.style(package_info.version, fg='magenta')}"
+        )
+
+    if resuming_from is not None:
+        _present(resuming_from, prefix="Resuming from")
+        with UI.indent():
+            do_one_update(
+                config, name=resuming_from.name, version=resuming_from.version
+            )
+        UI.log(f"Processing remaining {len(package_names)} package(s)")
+    else:
+        UI.log(f"Processing {len(package_names)} package(s)")
+
+    with UI.indent():
+        for package_info in state.get_info(package_names):
+            _present(package_info)
+            with UI.indent():
+                latest_version = determine_latest_release(package_info.name)
+                if Version(latest_version) <= Version(package_info.version):
+                    UI.log("Already up-to-date")
+                    continue
+
+                state.update(package_info.name, latest_version)
+                do_one_update(config, name=package_info.name, version=latest_version)
+
+    UI.log("All done, removing marker file")
+    state.cleanup()

vendoring/sbom.py

@@ -0,0 +1,53 @@
+"""Code for generating a Software Bill-of-Materials (SBOM)
+from vendored libraries.
+"""
+
+import json
+from pathlib import Path
+from typing import Any, List
+from urllib.parse import quote
+
+from vendoring import __version__ as _vendoring_version
+from vendoring.tasks.update import parse_pinned_packages as _parse_pinned_packages
+
+
+def create_sbom_file(namespace: str, requirements: Path, sbom_file: Path) -> None:
+    # The top-most name in the module namespace is the
+    # most likely to be a recognizable name.
+    top_level = namespace.split(".", 1)[0]
+    top_level_bom_ref = f"bom-ref:{top_level}"
+    components: List[Any] = [
+        {"bom-ref": top_level_bom_ref, "name": top_level, "type": "library"}
+    ]
+    dependencies: List[Any] = [{"ref": top_level_bom_ref, "dependsOn": []}]
+    sbom = {
+        "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json",
+        "bomFormat": "CycloneDX",
+        "specVersion": "1.4",
+        "version": 1,
+        "metadata": {
+            "tools": [{"name": "vendoring", "version": _vendoring_version}],
+            "component": components[0],
+        },
+        "components": components,
+        "dependencies": dependencies,
+    }
+
+    pkgs = sorted(
+        _parse_pinned_packages(requirements), key=lambda item: (item.name, item.version)
+    )
+    for pkg in pkgs:
+        purl = f"pkg:pypi/{quote(pkg.name, safe='')}@{quote(pkg.version, safe='')}"
+        components.append(
+            {
+                "name": pkg.name,
+                "version": pkg.version,
+                "purl": purl,
+                "type": "library",
+                "bom-ref": purl,
+            }
+        )
+        dependencies[0]["dependsOn"].append(purl)
+        dependencies.append({"ref": purl})
+
+    sbom_file.write_text(json.dumps(sbom, indent=2, sort_keys=True))

vendoring/sync.py

@@ -0,0 +1,19 @@
+"""Core logic of the sync task."""
+
+from vendoring.configuration import Configuration
+from vendoring.tasks.cleanup import cleanup_existing_vendored
+from vendoring.tasks.license import fetch_licenses
+from vendoring.tasks.stubs import generate_stubs
+from vendoring.tasks.vendor import vendor_libraries
+from vendoring.ui import UI
+
+
+def run_sync(config: Configuration) -> None:
+    with UI.task("Clean existing libraries"):
+        cleanup_existing_vendored(config)
+    with UI.task("Add vendored libraries"):
+        libraries = vendor_libraries(config)
+    with UI.task("Fetch licenses"):
+        fetch_licenses(config)
+    with UI.task("Generate static-typing stubs"):
+        generate_stubs(config, libraries)

vendoring/tasks/cleanup.py

@@ -0,0 +1,32 @@
+"""Logic for cleaning up already vendored files."""
+
+from pathlib import Path
+from typing import Iterable, List
+
+from vendoring.configuration import Configuration
+from vendoring.utils import remove_all
+
+
+def determine_items_to_remove(
+    destination: Path, *, files_to_skip: List[str]
+) -> Iterable[Path]:
+    if not destination.exists():
+        # Folder does not exist, nothing to cleanup.
+        return
+
+    for item in destination.iterdir():
+        if item.is_dir():
+            # Directory
+            yield item
+        elif item.name not in files_to_skip:
+            # File, not in files_to_skip
+            yield item
+
+
+def cleanup_existing_vendored(config: Configuration) -> None:
+    """Cleans up existing vendored files in `destination` directory."""
+    destination = config.destination
+    items = determine_items_to_remove(destination, files_to_skip=config.protected_files)
+
+    # TODO: log how many items were removed.
+    remove_all(items, protected=[config.requirements])