antsibull-nox 0.0.1__py3-none-any.whl → 0.1.0__py3-none-any.whl

antsibull_nox/data/action-groups.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Make sure all modules that should show up in the action group."""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+import typing as t
+
+import yaml
+
+from antsibull_nox.data_util import setup
+from antsibull_nox.sessions import ActionGroup
+
+
+def compile_patterns(
+    config: list[ActionGroup], errors: list[str]
+) -> dict[str, re.Pattern] | None:
+    patterns: dict[str, re.Pattern] = {}
+    for action_group in config:
+        if action_group.name in config:
+            errors.append(
+                f"noxfile.py: Action group {action_group.name!r} defined multiple times"
+            )
+            return None
+        patterns[action_group.name] = re.compile(action_group.pattern)
+    return patterns
+
+
+def load_redirects(
+    config: list[ActionGroup], errors: list[str], meta_runtime: str
+) -> dict[str, list[str]]:
+    # Load redirects
+    try:
+        with open(meta_runtime, "rb") as f:
+            data = yaml.safe_load(f)
+        action_groups = data["action_groups"]
+    except Exception as exc:
+        errors.append(f"{meta_runtime}: cannot load action groups: {exc}")
+        return {}
+
+    if not isinstance(action_groups, dict):
+        errors.append(f"{meta_runtime}: action_groups is not a dictionary")
+        return {}
+    if not all(
+        isinstance(k, str) and isinstance(v, list) for k, v in action_groups.items()
+    ):
+        errors.append(
+            f"{meta_runtime}: action_groups is not a dictionary mapping strings to list of strings"
+        )
+        return {}
+
+    # Compare meta/runtime.yml content with config
+    config_groups = {cfg.name for cfg in config}
+    for action_group in action_groups:
+        if action_group not in config_groups:
+            errors.append(
+                f"{meta_runtime}: found unknown action group"
+                f" {action_group!r}; likely noxfile needs updating"
+            )
+    for action_group in config:
+        if action_group.name not in action_groups:
+            errors.append(
+                f"{meta_runtime}: cannot find action group"
+                f" {action_group.name!r}; likely noxfile needs updating"
+            )
+
+    return action_groups
+
+
+def load_docs(path: str, errors: list[str]) -> dict[str, t.Any] | None:
+    documentation = []
+    in_docs = False
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("DOCUMENTATION ="):
+                in_docs = True
+            elif line.startswith(("'''", '"""')) and in_docs:
+                in_docs = False
+            elif in_docs:
+                documentation.append(line)
+    if in_docs:
+        errors.append(f"{path}: cannot find DOCUMENTATION end")
+    if not documentation:
+        errors.append(f"{path}: cannot find DOCUMENTATION")
+        return None
+
+    try:
+        docs = yaml.safe_load("\n".join(documentation))
+        if not isinstance(docs, dict):
+            raise Exception("is not a top-level dictionary")
+        return docs
+    except Exception as exc:
+        errors.append(f"{path}: cannot load DOCUMENTATION as YAML: {exc}")
+        return None
+
+
+def scan(config: list[ActionGroup], errors: list[str]) -> None:
+    patterns = compile_patterns(config, errors)
+    if patterns is None:
+        return
+
+    meta_runtime = "meta/runtime.yml"
+    action_groups = load_redirects(config, errors, meta_runtime)
+
+    modules_directory = "plugins/modules/"
+    modules_suffix = ".py"
+
+    errors = []
+    for file in os.listdir(modules_directory):
+        if not file.endswith(modules_suffix):
+            continue
+        module_name = file[: -len(modules_suffix)]
+
+        for action_group in config:
+            action_group_content = action_groups.get(action_group.name) or []
+            path = os.path.join(modules_directory, file)
+
+            if not patterns[action_group.name].match(module_name):
+                if module_name in action_group_content:
+                    errors.append(
+                        f"{path}: module is in action group {action_group.name!r}"
+                        " despite not matching its pattern as defined in noxfile"
+                    )
+                continue
+
+            should_be_in_action_group = (
+                module_name not in action_group.exclusions
+                if action_group.exclusions
+                else True
+            )
+
+            if should_be_in_action_group:
+                if module_name not in action_group_content:
+                    errors.append(
+                        f"{meta_runtime}: module {module_name!r} is not part"
+                        f" of {action_group.name!r} action group"
+                    )
+                else:
+                    action_group_content.remove(module_name)
+
+            docs = load_docs(path, errors)
+            if docs is None:
+                continue
+
+            docs_fragments = docs.get("extends_documentation_fragment") or []
+            is_in_action_group = action_group.doc_fragment in docs_fragments
+
+            if should_be_in_action_group != is_in_action_group:
+                if should_be_in_action_group:
+                    errors.append(
+                        f"{path}: module does not document itself as part of"
+                        f" action group {action_group.name!r}, but it should;"
+                        f" you need to add {action_group.doc_fragment} to"
+                        f' "extends_documentation_fragment" in DOCUMENTATION'
+                    )
+                else:
+                    errors.append(
+                        f"{path}: module documents itself as part of"
+                        f" action group {action_group.name!r}, but it should not be"
+                    )
+
+    for action_group in config:
+        action_group_content = action_groups.get(action_group.name) or []
+        for module_name in action_group_content:
+            errors.append(
+                f"{meta_runtime}: module {module_name} mentioned"
+                f" in {action_group.name!r} action group does not exist"
+                " or does not match pattern defined in noxfile"
+            )
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+
+    if not isinstance(extra_data.get("config"), list):
+        raise ValueError("config is not a list")
+    if not all(isinstance(cfg, dict) for cfg in extra_data["config"]):
+        raise ValueError("config is not a list of dictionaries")
+    config = [ActionGroup(**cfg) for cfg in extra_data["config"]]
+
+    errors: list[str] = []
+    scan(config, errors)
+
+    for error in sorted(errors):
+        print(error)
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data/license-check.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Prevent files without a correct license identifier from being added to the source tree."""
+
+from __future__ import annotations
+
+import glob
+import os
+import sys
+
+from antsibull_nox.data_util import get_list_of_strings, setup
+
+
+def format_license_list(licenses: list[str]) -> str:
+    if not licenses:
+        return "(empty)"
+    return ", ".join([f'"{license}"' for license in licenses])
+
+
+def find_licenses(errors: list[str], filename: str, relax: bool = False) -> list[str]:
+    spdx_license_identifiers: list[str] = []
+    other_license_identifiers: list[str] = []
+    has_copyright = False
+    try:
+        with open(filename, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.rstrip()
+                if "Copyright " in line:
+                    has_copyright = True
+                if "Copyright: " in line:
+                    errors.append(
+                        f'{filename}: found copyright line with "Copyright:".'
+                        " Please remove the colon."
+                    )
+                if "SPDX-FileCopyrightText: " in line:
+                    has_copyright = True
+                idx = line.find("SPDX-License-Identifier: ")
+                if idx >= 0:
+                    lic_id = line[idx + len("SPDX-License-Identifier: ") :]
+                    spdx_license_identifiers.extend(lic_id.split(" OR "))
+                if "GNU General Public License" in line:
+                    if "v3.0+" in line:
+                        other_license_identifiers.append("GPL-3.0-or-later")
+                    if "version 3 or later" in line:
+                        other_license_identifiers.append("GPL-3.0-or-later")
+                if "Simplified BSD License" in line:
+                    other_license_identifiers.append("BSD-2-Clause")
+                if "Apache License 2.0" in line:
+                    other_license_identifiers.append("Apache-2.0")
+                if "PSF License" in line or "Python-2.0" in line:
+                    other_license_identifiers.append("PSF-2.0")
+                if "MIT License" in line:
+                    other_license_identifiers.append("MIT")
+    except Exception as exc:
+        errors.append(f"{filename}: error while processing file: {exc}")
+    if len(set(spdx_license_identifiers)) < len(spdx_license_identifiers):
+        errors.append(f"{filename}: found identical SPDX-License-Identifier values")
+    if other_license_identifiers and set(other_license_identifiers) != set(
+        spdx_license_identifiers
+    ):
+        errors.append(
+            f"{filename}: SPDX-License-Identifier yielded the license list"
+            f" {format_license_list(spdx_license_identifiers)}, while manual guessing"
+            f" yielded the license list {format_license_list(other_license_identifiers)}"
+        )
+    if not has_copyright and not relax:
+        errors.append(f"{filename}: found no copyright notice")
+    return sorted(spdx_license_identifiers)
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+
+    # The following paths are allowed to have no license identifier
+    no_comments_allowed = [
+        "changelogs/fragments/*.yml",
+        "changelogs/fragments/*.yaml",
+    ]
+
+    # These files are completely ignored
+    ignore_paths = [
+        ".ansible-test-timeout.json",
+        ".reuse/dep5",
+        "LICENSES/*.txt",
+        "COPYING",
+    ] + get_list_of_strings(extra_data, "extra_ignore_paths", default=[])
+
+    no_comments_allowed = [
+        fn for pattern in no_comments_allowed for fn in glob.glob(pattern)
+    ]
+    ignore_paths = [fn for pattern in ignore_paths for fn in glob.glob(pattern)]
+
+    valid_licenses = [
+        license_file[len("LICENSES/") : -len(".txt")]
+        for license_file in glob.glob("LICENSES/*.txt")
+    ]
+
+    errors: list[str] = []
+
+    for path in paths:
+        if path.startswith("./"):
+            path = path[2:]
+        if path in ignore_paths or path.startswith("tests/output/"):
+            continue
+        if os.stat(path).st_size == 0:
+            continue
+        if not path.endswith(".license") and os.path.exists(path + ".license"):
+            path = path + ".license"
+        valid_licenses_for_path = valid_licenses
+        if (
+            path.startswith("plugins/")
+            and not path.startswith(
+                ("plugins/modules/", "plugins/module_utils/", "plugins/doc_fragments/")
+            )
+            and path.endswith((".py", ".py.license"))
+        ):
+            valid_licenses_for_path = [
+                license for license in valid_licenses if license == "GPL-3.0-or-later"
+            ]
+        licenses = find_licenses(errors, path, relax=path in no_comments_allowed)
+        if not licenses:
+            if path not in no_comments_allowed:
+                errors.append(f"{path}: must have at least one license")
+        else:
+            for license in licenses:
+                if license not in valid_licenses_for_path:
+                    errors.append(
+                        f"{path}: found not allowed license {license!r}, must be one of"
+                        f" {format_license_list(valid_licenses_for_path)}"
+                    )
+
+    for error in sorted(errors):
+        print(error)
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data/license-check.py.license

@@ -0,0 +1,3 @@
+GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+SPDX-License-Identifier: GPL-3.0-or-later
+SPDX-FileCopyrightText: 2022, Felix Fontein <felix@fontein.de>

antsibull_nox/data/no-unwanted-files.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Prevent unwanted files from being added to the source tree."""
+
+from __future__ import annotations
+
+import os
+import sys
+
+from antsibull_nox.data_util import get_bool, get_list_of_strings, setup
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+
+    module_extensions = tuple(
+        sorted(
+            get_list_of_strings(
+                extra_data,
+                "module_extensions",
+                default=[
+                    ".cs",
+                    ".ps1",
+                    ".psm1",
+                    ".py",
+                ],
+            )
+        )
+    )
+
+    other_extensions = tuple(
+        sorted(
+            get_list_of_strings(
+                extra_data,
+                "other_extensions",
+                default=[
+                    ".py",
+                    ".pyi",
+                ],
+            )
+        )
+    )
+
+    yaml_extensions = set(
+        get_list_of_strings(
+            extra_data,
+            "yaml_extensions",
+            default=[
+                ".yml",
+                ".yaml",
+            ],
+        )
+    )
+
+    skip_paths = set(get_list_of_strings(extra_data, "skip_paths", default=[]))
+
+    skip_directories = tuple(
+        get_list_of_strings(extra_data, "skip_prefixes", default=[])
+    )
+
+    yaml_directories = tuple(
+        get_list_of_strings(
+            extra_data,
+            "yaml_directories",
+            default=[
+                "plugins/test/",
+                "plugins/filter/",
+            ],
+        )
+    )
+
+    allow_symlinks = get_bool(extra_data, "allow_symlinks")
+
+    errors: list[str] = []
+    for path in paths:
+        if not path.startswith("plugins/"):
+            continue
+
+        if path in skip_paths:
+            continue
+
+        if any(path.startswith(skip_directory) for skip_directory in skip_directories):
+            continue
+
+        if os.path.islink(path):
+            if not allow_symlinks:
+                errors.append("%s: is a symbolic link" % (path,))
+        elif not os.path.isfile(path):
+            errors.append("%s: is not a regular file" % (path,))
+
+        ext = os.path.splitext(path)[1]
+
+        if ext in yaml_extensions and any(
+            path.startswith(yaml_directory) for yaml_directory in yaml_directories
+        ):
+            continue
+
+        extensions = (
+            module_extensions
+            if path.startswith("plugins/modules/")
+            else other_extensions
+        )
+
+        if ext not in extensions:
+            errors.append(f"{path}: extension must be one of: {', '.join(extensions)}")
+
+    for error in sorted(errors):
+        print(error)
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data_util.py

@@ -0,0 +1,115 @@
+# Author: Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025, Ansible Project
+
+"""
+Utility code for scripts in data.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import typing as t
+from pathlib import Path
+
+import nox
+
+
+def setup() -> tuple[list[str], dict[str, t.Any]]:
+    """
+    Fetch list of paths and potential extra configuration.
+
+    First thing to call in an extra sanity check script in data/.
+    """
+    if len(sys.argv) == 3 and sys.argv[1] == "--data":
+        # Preferred way: load information from JSON file
+        path = sys.argv[2]
+        try:
+            with open(path, "rb") as f:
+                data = json.load(f)
+        except Exception as exc:
+            raise ValueError(f"Error while reading JSON from {path}") from exc
+        try:
+            paths = get_list_of_strings(data, "paths")
+        except ValueError as exc:
+            raise ValueError(f"Invalid JSON content in {path}: {exc}") from exc
+        data.pop("paths")
+        return paths, data
+    if len(sys.argv) >= 2:
+        # It's also possible to pass a list of paths on the command line, to simplify
+        # testing these scripts.
+        return sys.argv[1:], {}
+    # Alternatively one can pass a list of files from stdin, for example by piping
+    # the output of 'git ls-files' into this script. This is also for testing these
+    # scripts.
+    return sys.stdin.read().splitlines(), {}
+
+
+def get_list_of_strings(
+    data: dict[str, t.Any],
+    key: str,
+    *,
+    default: list[str] | None = None,
+) -> list[str]:
+    """
+    Retrieves a list of strings from key ``key`` of the JSON object ``data``.
+
+    If ``default`` is set to a list, a missing key results in this value being returned.
+    """
+    sentinel = object()
+    value = data.get(key, sentinel)
+    if value is sentinel:
+        if default is not None:
+            return default
+        raise ValueError(f"{key!r} is not a present")
+    if not isinstance(value, list):
+        raise ValueError(f"{key!r} is not a list, but {type(key)}")
+    if not all(isinstance(entry, str) for entry in value):
+        raise ValueError(f"{key!r} is not a list of strings")
+    return t.cast(list[str], value)
+
+
+def get_bool(
+    data: dict[str, t.Any],
+    key: str,
+    *,
+    default: bool | None = None,
+) -> bool:
+    """
+    Retrieves a boolean from key ``key`` of the JSON object ``data``.
+
+    If ``default`` is set to a boolean, a missing key results in this value being returned.
+    """
+    sentinel = object()
+    value = data.get(key, sentinel)
+    if value is sentinel:
+        if default is not None:
+            return default
+        raise ValueError(f"{key!r} is not a present")
+    if not isinstance(value, bool):
+        raise ValueError(f"{key!r} is not a bool, but {type(key)}")
+    return value
+
+
+def prepare_data_script(
+    session: nox.Session,
+    *,
+    base_name: str,
+    paths: list[Path],
+    extra_data: dict[str, t.Any] | None = None,
+) -> Path:
+    """
+    Prepare a data JSON file for the extra sanity check scripts.
+    """
+    cwd = Path.cwd()
+    data = {}
+    data["paths"] = [str(path.relative_to(cwd)) for path in paths]
+    if extra_data:
+        data.update(extra_data)
+    file = Path(session.create_tmp()) / f"{base_name}-data.json"
+    with open(file, "w", encoding="utf-8") as f:
+        json.dump(data, f)
+    return file