gha-utils 4.24.0__py3-none-any.whl

gha_utils/__init__.py

@@ -0,0 +1,20 @@
+# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+"""Expose package-wide elements."""
+
+from __future__ import annotations
+
+__version__ = "4.24.0"

gha_utils/__main__.py

@@ -0,0 +1,49 @@
+# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+"""Allow the module to be run as a CLI. I.e.:
+
+.. code-block:: shell-session
+
+    $ python -m gha_utils
+"""
+
+from __future__ import annotations
+
+
+def main():
+    """Execute the CLI but force its name to not let Click defaults to:
+
+    .. code-block:: shell-session
+        $ python -m gha_utils --version
+        python -m gha_utils, version 4.0.0
+
+    Indirection via this ``main()`` method was `required to reconcile
+    <https://github.com/python-poetry/poetry/issues/5981>`_:
+
+        - plain inline package call: ``python -m gha_utils``,
+        - Poetry's script entry point: ``gha-utils = 'gha_utils.__main__:main``,
+        - Nuitka's main module invocation requirement:
+          ``python -m nuitka (...) gha_utils/__main__.py``
+
+    That way we can deduce all three cases from the entry point.
+    """
+    from gha_utils.cli import gha_utils
+
+    gha_utils(prog_name=gha_utils.name)
+
+
+if __name__ == "__main__":
+    main()

gha_utils/changelog.py

@@ -0,0 +1,146 @@
+# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+from __future__ import annotations
+
+import logging
+import re
+from functools import cached_property
+from pathlib import Path
+from textwrap import indent
+
+import tomllib
+
+
+class Changelog:
+    """Helpers to manipulate changelog files written in Markdown."""
+
+    def __init__(self, initial_changelog: str | None = None) -> None:
+        if not initial_changelog:
+            self.content = "# Changelog\n"
+        else:
+            self.content = initial_changelog
+        logging.debug(f"Initial content set to:\n{self.content}")
+
+    @cached_property
+    def current_version(self) -> str | None:
+        # Extract current version as defined by bump-my-version.
+        config_file = Path("./pyproject.toml").resolve()
+        logging.info(f"Open {config_file}")
+        config = tomllib.loads(config_file.read_text(encoding="UTF-8"))
+        current_version = config["tool"]["bumpversion"]["current_version"]
+        logging.info(f"Current version: {current_version}")
+        return current_version if current_version else None
+
+    def update(self) -> str:
+        r"""Adds a new empty entry at the top of the changelog.
+
+        Will return the same content as the current changelog if it has already been updated.
+
+        This is designed to be used just after a new release has been tagged. And before a
+        post-release version increment is applied with a call to:
+
+        ```shell-session
+        $ bump-my-version bump --verbose patch
+        Starting BumpVersion 0.5.1.dev6
+        Reading config file pyproject.toml:
+        Specified version (2.17.5) does not match last tagged version (2.17.4)
+        Parsing version '2.17.5' using '(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)'
+        Parsed the following values: major=2, minor=17, patch=5
+        Attempting to increment part 'patch'
+        Values are now: major=2, minor=17, patch=6
+        New version will be '2.17.6'
+        Dry run active, won't touch any files.
+        Asserting files ./changelog.md contain the version string...
+        Found '[2.17.5 (unreleased)](' in ./changelog.md at line 2:
+        ## [2.17.5 (unreleased)](https://github.com/kdeldycke/workflows/compare/v2.17.4...main)
+        Would change file ./changelog.md:
+        *** before ./changelog.md
+        --- after ./changelog.md
+        ***************
+        *** 1,6 ****
+          # Changelog
+
+        ! ## [2.17.5 (unreleased)](https://github.com/kdeldycke/workflows/compare/v2.17.4...main)
+
+          > [!IMPORTANT]
+          > This version is not released yet and is under active development.
+        --- 1,6 ----
+          # Changelog
+
+        ! ## [2.17.6 (unreleased)](https://github.com/kdeldycke/workflows/compare/v2.17.4...main)
+
+          > [!IMPORTANT]
+          > This version is not released yet and is under active development.
+        Would write to config file pyproject.toml:
+        *** before pyproject.toml
+        --- after pyproject.toml
+        ***************
+        *** 1,5 ****
+          [tool.bumpversion]
+        ! current_version = "2.17.5"
+          allow_dirty = true
+
+          [[tool.bumpversion.files]]
+        --- 1,5 ----
+          [tool.bumpversion]
+        ! current_version = "2.17.6"
+          allow_dirty = true
+
+          [[tool.bumpversion.files]]
+        Would not commit
+        Would not tag since we are not committing
+        ```
+        """
+        # Extract parts of the changelog or set default values.
+        SECTION_START = "##"
+        sections = self.content.split(SECTION_START, 2)
+        changelog_header = sections[0] if len(sections) > 0 else "# Changelog\n\n"
+        current_entry = f"{SECTION_START}{sections[1]}" if len(sections) > 1 else ""
+        past_entries = f"{SECTION_START}{sections[2]}" if len(sections) > 2 else ""
+
+        # Derive the release template from the last entry.
+        DATE_REGEX = r"\d{4}\-\d{2}\-\d{2}"
+        VERSION_REGEX = r"\d+\.\d+\.\d+"
+
+        # Replace the release date with the unreleased tag.
+        new_entry = re.sub(DATE_REGEX, "unreleased", current_entry, count=1)
+
+        # Update GitHub's comparison URL to target the main branch.
+        new_entry = re.sub(
+            rf"v{VERSION_REGEX}\.\.\.v{VERSION_REGEX}",
+            f"v{self.current_version}...main",
+            new_entry,
+            count=1,
+        )
+
+        # Replace the whole paragraph of changes by a notice message. The paragraph is
+        # identified as starting by a blank line, at which point everything gets
+        # replaced.
+        new_entry = re.sub(
+            r"\n\n.*",
+            "\n\n"
+            "> [!IMPORTANT]\n"
+            "> This version is not released yet and is under active development.\n\n",
+            new_entry,
+            flags=re.MULTILINE | re.DOTALL,
+        )
+        logging.info("New generated section:\n" + indent(new_entry, " " * 2))
+
+        history = current_entry + past_entries
+        if new_entry not in history:
+            history = new_entry + history
+        return (changelog_header + history).rstrip()

gha_utils/cli.py

@@ -0,0 +1,452 @@
+# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import sys
+from collections import Counter
+from datetime import datetime
+from pathlib import Path
+
+from boltons.iterutils import unique
+from click_extra import (
+    Choice,
+    Context,
+    EnumChoice,
+    FloatRange,
+    IntRange,
+    argument,
+    echo,
+    file_path,
+    group,
+    option,
+    pass_context,
+)
+from click_extra.envvar import merge_envvar_ids
+from extra_platforms import ALL_IDS, is_github_ci
+
+from . import __version__
+from .changelog import Changelog
+from .mailmap import Mailmap
+from .metadata import NUITKA_BUILD_TARGETS, Dialect, Metadata
+from .test_plan import DEFAULT_TEST_PLAN, SkippedTest, parse_test_plan
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import IO
+
+
+def is_stdout(filepath: Path) -> bool:
+    """Check if a file path is set to stdout.
+
+    Prevents the creation of a ``-`` file in the current directory.
+    """
+    return str(filepath) == "-"
+
+
+def prep_path(filepath: Path) -> IO | None:
+    """Prepare the output file parameter for Click's echo function."""
+    if is_stdout(filepath):
+        return None
+    return filepath.open("w", encoding="UTF-8")
+
+
+def generate_header(ctx: Context) -> str:
+    """Generate metadata to be left as comments to the top of a file generated by
+    this CLI.
+    """
+    header = (
+        f"# Generated by {ctx.command_path} v{__version__}"
+        " - https://github.com/kdeldycke/workflows\n"
+        f"# Timestamp: {datetime.now().isoformat()}\n"
+    )
+    logging.debug(f"Generated header:\n{header}")
+    return header
+
+
+def remove_header(content: str) -> str:
+    """Return the same content provided, but without the blank lines and header metadata generated by the function above."""
+    logging.debug(f"Removing header from:\n{content}")
+    lines = []
+    still_in_header = True
+    for line in content.splitlines():
+        if still_in_header:
+            # We are still in the header as long as we have blank lines or we have
+            # comment lines matching the format produced by the method above.
+            if not line.strip() or line.startswith((
+                "# Generated by ",
+                "# Timestamp: ",
+            )):
+                continue
+            else:
+                still_in_header = False
+        # We are past the header, so keep all the lines: we have nothing left to remove.
+        lines.append(line)
+
+    headerless_content = "\n".join(lines)
+    logging.debug(f"Result of header removal:\n{headerless_content}")
+    return headerless_content
+
+
+@group
+def gha_utils():
+    pass
+
+
+@gha_utils.command(short_help="Output project metadata")
+@option(
+    "-u",
+    "--unstable-targets",
+    help="Build targets for which Nuitka is allowed to fail without comprimising the "
+    " release workflow. This option accepts a mangled string with multiple targets "
+    "separated by arbitrary separators. Recognized targets are: "
+    f"{', '.join(NUITKA_BUILD_TARGETS)}.",
+)
+@option(
+    "--format",
+    type=EnumChoice(Dialect),
+    default=Dialect.github,
+    help="Rendering format of the metadata.",
+)
+@option(
+    "--overwrite",
+    "--force",
+    "--replace",
+    is_flag=True,
+    default=False,
+    help="Allow output target file to be silently wiped out if it already exists.",
+)
+@argument(
+    "output_path",
+    type=file_path(writable=True, resolve_path=True, allow_dash=True),
+    default="-",
+)
+@pass_context
+def metadata(ctx, unstable_targets, format, overwrite, output_path):
+    """Dump project metadata to a file.
+
+    By default the metadata produced are displayed directly to the console output.
+    So `gha-utils metadata` is the same as a call to `gha-utils metadata -`. To have
+    the results written in a file on disk, specify the output file like so:
+    `gha-utils metadata dump.txt`.
+
+    For GitHub you want to output to the standard environment file pointed to by the
+    `$GITHUB_OUTPUT` variable. I.e.:
+
+        $ gha-utils metadata --format github "$GITHUB_OUTPUT"
+    """
+    if is_stdout(output_path):
+        if overwrite:
+            logging.warning("Ignore the --overwrite/--force/--replace option.")
+        logging.info(f"Print metadata to {sys.stdout.name}")
+    else:
+        logging.info(f"Dump all metadata to {output_path}")
+
+        if output_path.exists():
+            msg = "Target file exist and will be overwritten."
+            if overwrite:
+                logging.warning(msg)
+            else:
+                logging.critical(msg)
+                ctx.exit(2)
+
+    # Extract targets from the raw string provided by the user.
+    valid_targets = set()
+    if unstable_targets:
+        for target in re.split(r"[^a-z0-9\-]", unstable_targets.lower()):
+            if target:
+                if target not in NUITKA_BUILD_TARGETS:
+                    logging.fatal(
+                        f"Unrecognized {target!r} target. "
+                        f"Must be one of {', '.join(NUITKA_BUILD_TARGETS)}."
+                    )
+                    sys.exit(1)
+                valid_targets.add(target)
+        logging.debug(
+            f"Parsed {unstable_targets!r} string into {valid_targets} targets."
+        )
+
+    metadata = Metadata(valid_targets)
+
+    # Output a warning in GitHub runners if metadata are not saved to $GITHUB_OUTPUT.
+    if is_github_ci():
+        env_file = os.getenv("GITHUB_OUTPUT")
+        if env_file and Path(env_file) != output_path:
+            logging.warning(
+                "Output path is not the same as $GITHUB_OUTPUT environment variable,"
+                " which is generally what we're looking to do in GitHub CI runners for"
+                " other jobs to consume the produced metadata."
+            )
+
+    dialect = Dialect(format)
+    content = metadata.dump(dialect=dialect)
+    echo(content, file=prep_path(output_path))
+
+
+@gha_utils.command(short_help="Maintain a Markdown-formatted changelog")
+@option(
+    "--source",
+    type=file_path(exists=True, readable=True, resolve_path=True),
+    default="changelog.md",
+    help="Changelog source file in Markdown format.",
+)
+@argument(
+    "changelog_path",
+    type=file_path(writable=True, resolve_path=True, allow_dash=True),
+    default="-",
+)
+@pass_context
+def changelog(ctx, source, changelog_path):
+    initial_content = None
+    if source:
+        logging.info(f"Read initial changelog from {source}")
+        initial_content = source.read_text(encoding="UTF-8")
+
+    changelog = Changelog(initial_content)
+    content = changelog.update()
+    if content == initial_content:
+        logging.warning("Changelog already up to date. Do nothing.")
+        ctx.exit()
+
+    if is_stdout(changelog_path):
+        logging.info(f"Print updated results to {sys.stdout.name}")
+    else:
+        logging.info(f"Save updated results to {changelog_path}")
+    echo(content, file=prep_path(changelog_path))
+
+
+@gha_utils.command(short_help="Update Git's .mailmap file with missing contributors")
+@option(
+    "--source",
+    type=file_path(readable=True, resolve_path=True),
+    default=".mailmap",
+    help="Mailmap source file to use as reference for contributors identities that "
+    "are already grouped.",
+)
+@option(
+    "--create-if-missing/--skip-if-missing",
+    is_flag=True,
+    default=True,
+    help="If not found, either create the missing destination mailmap file, or skip "
+    "the update process entirely. This option is ignored if the destination is to print "
+    f"the result to {sys.stdout.name}.",
+)
+@argument(
+    "destination_mailmap",
+    type=file_path(writable=True, resolve_path=True, allow_dash=True),
+    default="-",
+)
+@pass_context
+def mailmap_sync(ctx, source, create_if_missing, destination_mailmap):
+    """Update a ``.mailmap`` file with all missing contributors found in Git commit
+    history.
+
+    By default the ``.mailmap`` at the root of the repository is read and its content
+    is reused as reference, so identities already aliased in there are preserved and
+    used as initial mapping. Only missing contributors not found in this initial mapping
+    are added.
+
+    The resulting updated mapping is printed to the console output. So a bare call to
+    `gha-utils mailmap-sync` is the same as a call to
+    `gha-utils mailmap-sync --source .mailmap -`.
+
+    To have the updated mapping written to a file, specify the output file like so:
+    `gha-utils mailmap-sync .mailmap`.
+
+    The updated results are sorted. But no attempts are made at regrouping new
+    contributors. SO you have to edit entries by hand to regroup them
+    """
+    mailmap = Mailmap()
+
+    if source.exists():
+        logging.info(f"Read initial mapping from {source}")
+        content = remove_header(source.read_text(encoding="UTF-8"))
+        mailmap.parse(content)
+    else:
+        logging.debug(f"Mailmap source file {source} does not exists.")
+
+    mailmap.update_from_git()
+    new_content = mailmap.render()
+
+    if is_stdout(destination_mailmap):
+        logging.info(f"Print updated results to {sys.stdout.name}.")
+        logging.debug(
+            "Ignore the "
+            + ("--create-if-missing" if create_if_missing else "--skip-if-missing")
+            + " option."
+        )
+    else:
+        logging.info(f"Save updated results to {destination_mailmap}")
+        if not create_if_missing and not destination_mailmap.exists():
+            logging.warning(
+                f"{destination_mailmap} does not exists, stop the sync process."
+            )
+            ctx.exit()
+        if content == new_content:
+            logging.warning("Nothing to update, stop the sync process.")
+            ctx.exit()
+
+    echo(generate_header(ctx) + new_content, file=prep_path(destination_mailmap))
+
+
+@gha_utils.command(short_help="Run a test plan from a file against a binary")
+@option(
+    "--command",
+    "--binary",
+    required=True,
+    metavar="COMMAND",
+    help="Path to the binary file to test, or a command line to be executed.",
+)
+@option(
+    "-F",
+    "--plan-file",
+    type=file_path(exists=True, readable=True, resolve_path=True),
+    multiple=True,
+    metavar="FILE_PATH",
+    help="Path to a test plan file in YAML. This option can be repeated to run "
+    "multiple test plans in sequence. If not provided, a default test plan will be "
+    "executed.",
+)
+@option(
+    "-E",
+    "--plan-envvar",
+    multiple=True,
+    metavar="ENVVAR_NAME",
+    help="Name of an environment variable containing a test plan in YAML. This "
+    "option can be repeated to collect multiple test plans.",
+)
+@option(
+    "-t",
+    "--select-test",
+    type=IntRange(min=1),
+    multiple=True,
+    metavar="INTEGER",
+    help="Only run the tests matching the provided test case numbers. This option can "
+    "be repeated to run multiple test cases. If not provided, all test cases will be "
+    "run.",
+)
+@option(
+    "-s",
+    "--skip-platform",
+    type=Choice(sorted(ALL_IDS), case_sensitive=False),
+    multiple=True,
+    help="Skip tests for the specified platforms. This option can be repeated to "
+    "skip multiple platforms.",
+)
+@option(
+    "-x",
+    "--exit-on-error",
+    is_flag=True,
+    default=False,
+    help="Exit instantly on first failed test.",
+)
+@option(
+    "-T",
+    "--timeout",
+    # Timeout passed to subprocess.run() is a float that is silently clamped to
+    # 0.0 is negative values are provided, so we mimic this behavior here:
+    # https://github.com/python/cpython/blob/5740b95076b57feb6293cda4f5504f706a7d622d/Lib/subprocess.py#L1596-L1597
+    type=FloatRange(min=0, clamp=True),
+    metavar="SECONDS",
+    help="Set the default timeout for each CLI call, if not specified in the "
+    "test plan.",
+)
+@option(
+    "--show-trace-on-error/--hide-trace-on-error",
+    default=True,
+    help="Show execution trace of failed tests.",
+)
+@option(
+    "--stats/--no-stats",
+    is_flag=True,
+    default=True,
+    help="Print per-manager package statistics.",
+)
+def test_plan(
+    command: str,
+    plan_file: tuple[Path, ...] | None,
+    plan_envvar: tuple[str, ...] | None,
+    select_test: tuple[int, ...] | None,
+    skip_platform: tuple[str, ...] | None,
+    exit_on_error: bool,
+    timeout: float | None,
+    show_trace_on_error: bool,
+    stats: bool,
+) -> None:
+    # Load test plan from workflow input, or use a default one.
+    test_list = []
+    if plan_file or plan_envvar:
+        for file in unique(plan_file):
+            logging.info(f"Get test plan from {file} file")
+            tests = list(parse_test_plan(file.read_text(encoding="UTF-8")))
+            logging.info(f"{len(tests)} test cases found.")
+            test_list.extend(tests)
+        for envvar_id in merge_envvar_ids(plan_envvar):
+            logging.info(f"Get test plan from {envvar_id!r} environment variable")
+            tests = list(parse_test_plan(os.getenv(envvar_id)))
+            logging.info(f"{len(tests)} test cases found.")
+            test_list.extend(tests)
+
+    else:
+        logging.warning(
+            "No test plan provided through --plan-file/-F or --plan-envvar/-E options:"
+            " use default test plan."
+        )
+        test_list = DEFAULT_TEST_PLAN
+    logging.debug(f"Test plan: {test_list}")
+
+    counter = Counter(total=len(test_list), skipped=0, failed=0)
+
+    for index, test_case in enumerate(test_list):
+        test_number = index + 1
+        test_name = f"#{test_number}"
+        logging.info(f"Run test {test_name}...")
+
+        if select_test and test_number not in select_test:
+            logging.warning(f"Test {test_name} skipped by user request.")
+            counter["skipped"] += 1
+            continue
+
+        try:
+            logging.debug(f"Test case parameters: {test_case}")
+            test_case.run_cli_test(
+                command,
+                additional_skip_platforms=skip_platform,
+                default_timeout=timeout,
+            )
+        except SkippedTest as ex:
+            counter["skipped"] += 1
+            logging.warning(f"Test {test_name} skipped: {ex}")
+        except Exception as ex:
+            counter["failed"] += 1
+            logging.error(f"Test {test_name} failed: {ex}")
+            if show_trace_on_error and test_case.execution_trace:
+                echo(test_case.execution_trace)
+            if exit_on_error:
+                logging.debug("Don't continue testing, a failed test was found.")
+                sys.exit(1)
+
+    if stats:
+        echo(
+            "Test plan results - "
+            + ", ".join((f"{k.title()}: {v}" for k, v in counter.items()))
+        )
+
+    if counter["failed"]:
+        sys.exit(1)