antsibull-nox 0.0.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

antsibull_nox/data/license-check.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Prevent files without a correct license identifier from being added to the source tree."""
+
+from __future__ import annotations
+
+import glob
+import os
+import sys
+
+from antsibull_nox.data.antsibull_nox_data_util import get_list_of_strings, setup
+
+
+def format_license_list(licenses: list[str]) -> str:
+    if not licenses:
+        return "(empty)"
+    return ", ".join([f'"{license}"' for license in licenses])
+
+
+def find_licenses(errors: list[str], filename: str, relax: bool = False) -> list[str]:
+    spdx_license_identifiers: list[str] = []
+    other_license_identifiers: list[str] = []
+    has_copyright = False
+    try:
+        with open(filename, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.rstrip()
+                if "Copyright " in line:
+                    has_copyright = True
+                if "Copyright: " in line:
+                    errors.append(
+                        f'{filename}: found copyright line with "Copyright:".'
+                        " Please remove the colon."
+                    )
+                if "SPDX-FileCopyrightText: " in line:
+                    has_copyright = True
+                idx = line.find("SPDX-License-Identifier: ")
+                if idx >= 0:
+                    lic_id = line[idx + len("SPDX-License-Identifier: ") :]
+                    spdx_license_identifiers.extend(lic_id.split(" OR "))
+                if "GNU General Public License" in line:
+                    if "v3.0+" in line:
+                        other_license_identifiers.append("GPL-3.0-or-later")
+                    if "version 3 or later" in line:
+                        other_license_identifiers.append("GPL-3.0-or-later")
+                if "Simplified BSD License" in line:
+                    other_license_identifiers.append("BSD-2-Clause")
+                if "Apache License 2.0" in line:
+                    other_license_identifiers.append("Apache-2.0")
+                if "PSF License" in line or "Python-2.0" in line:
+                    other_license_identifiers.append("PSF-2.0")
+                if "MIT License" in line:
+                    other_license_identifiers.append("MIT")
+    except Exception as exc:
+        errors.append(f"{filename}: error while processing file: {exc}")
+    if len(set(spdx_license_identifiers)) < len(spdx_license_identifiers):
+        errors.append(f"{filename}: found identical SPDX-License-Identifier values")
+    if other_license_identifiers and set(other_license_identifiers) != set(
+        spdx_license_identifiers
+    ):
+        errors.append(
+            f"{filename}: SPDX-License-Identifier yielded the license list"
+            f" {format_license_list(spdx_license_identifiers)}, while manual guessing"
+            f" yielded the license list {format_license_list(other_license_identifiers)}"
+        )
+    if not has_copyright and not relax:
+        errors.append(f"{filename}: found no copyright notice")
+    return sorted(spdx_license_identifiers)
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+
+    # The following paths are allowed to have no license identifier
+    no_comments_allowed = [
+        "changelogs/fragments/*.yml",
+        "changelogs/fragments/*.yaml",
+    ]
+
+    # These files are completely ignored
+    ignore_paths = [
+        ".ansible-test-timeout.json",
+        ".reuse/dep5",
+        "LICENSES/*.txt",
+        "COPYING",
+    ] + get_list_of_strings(extra_data, "extra_ignore_paths", default=[])
+
+    no_comments_allowed = [
+        fn for pattern in no_comments_allowed for fn in glob.glob(pattern)
+    ]
+    ignore_paths = [fn for pattern in ignore_paths for fn in glob.glob(pattern)]
+
+    valid_licenses = [
+        license_file[len("LICENSES/") : -len(".txt")]
+        for license_file in glob.glob("LICENSES/*.txt")
+    ]
+
+    errors: list[str] = []
+
+    for path in paths:
+        if path.startswith("./"):
+            path = path[2:]
+        if path in ignore_paths or path.startswith("tests/output/"):
+            continue
+        if os.stat(path).st_size == 0:
+            continue
+        if not path.endswith(".license") and os.path.exists(path + ".license"):
+            path = path + ".license"
+        valid_licenses_for_path = valid_licenses
+        if (
+            path.startswith("plugins/")
+            and not path.startswith(
+                ("plugins/modules/", "plugins/module_utils/", "plugins/doc_fragments/")
+            )
+            and path.endswith((".py", ".py.license"))
+        ):
+            valid_licenses_for_path = [
+                license for license in valid_licenses if license == "GPL-3.0-or-later"
+            ]
+        licenses = find_licenses(errors, path, relax=path in no_comments_allowed)
+        if not licenses:
+            if path not in no_comments_allowed:
+                errors.append(f"{path}: must have at least one license")
+        else:
+            for license in licenses:
+                if license not in valid_licenses_for_path:
+                    errors.append(
+                        f"{path}: found not allowed license {license!r}, must be one of"
+                        f" {format_license_list(valid_licenses_for_path)}"
+                    )
+
+    for error in sorted(errors):
+        print(error)
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data/license-check.py.license

@@ -0,0 +1,3 @@
+GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+SPDX-License-Identifier: GPL-3.0-or-later
+SPDX-FileCopyrightText: 2022, Felix Fontein <felix@fontein.de>

antsibull_nox/data/no-unwanted-files.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Prevent unwanted files from being added to the source tree."""
+
+from __future__ import annotations
+
+import os
+import sys
+
+from antsibull_nox.data.antsibull_nox_data_util import (
+    get_bool,
+    get_list_of_strings,
+    setup,
+)
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+
+    module_extensions = tuple(
+        sorted(
+            get_list_of_strings(
+                extra_data,
+                "module_extensions",
+                default=[
+                    ".cs",
+                    ".ps1",
+                    ".psm1",
+                    ".py",
+                ],
+            )
+        )
+    )
+
+    other_extensions = tuple(
+        sorted(
+            get_list_of_strings(
+                extra_data,
+                "other_extensions",
+                default=[
+                    ".py",
+                    ".pyi",
+                ],
+            )
+        )
+    )
+
+    yaml_extensions = set(
+        get_list_of_strings(
+            extra_data,
+            "yaml_extensions",
+            default=[
+                ".yml",
+                ".yaml",
+            ],
+        )
+    )
+
+    skip_paths = set(get_list_of_strings(extra_data, "skip_paths", default=[]))
+
+    skip_directories = tuple(
+        get_list_of_strings(extra_data, "skip_prefixes", default=[])
+    )
+
+    yaml_directories = tuple(
+        get_list_of_strings(
+            extra_data,
+            "yaml_directories",
+            default=[
+                "plugins/test/",
+                "plugins/filter/",
+            ],
+        )
+    )
+
+    allow_symlinks = get_bool(extra_data, "allow_symlinks")
+
+    errors: list[str] = []
+    for path in paths:
+        if not path.startswith("plugins/"):
+            continue
+
+        if path in skip_paths:
+            continue
+
+        if any(path.startswith(skip_directory) for skip_directory in skip_directories):
+            continue
+
+        if os.path.islink(path):
+            if not allow_symlinks:
+                errors.append("%s: is a symbolic link" % (path,))
+        elif not os.path.isfile(path):
+            errors.append("%s: is not a regular file" % (path,))
+
+        ext = os.path.splitext(path)[1]
+
+        if ext in yaml_extensions and any(
+            path.startswith(yaml_directory) for yaml_directory in yaml_directories
+        ):
+            continue
+
+        extensions = (
+            module_extensions
+            if path.startswith("plugins/modules/")
+            else other_extensions
+        )
+
+        if ext not in extensions:
+            errors.append(f"{path}: extension must be one of: {', '.join(extensions)}")
+
+    for error in sorted(errors):
+        print(error)
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data/plugin-yamllint.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Make sure all plugin and module documentation adheres to yamllint."""
+
+from __future__ import annotations
+
+import ast
+import io
+import re
+import sys
+import traceback
+import typing as t
+
+import yaml
+from antsibull_nox_data_util import setup  # type: ignore
+from yamllint import linter
+from yamllint.config import YamlLintConfig
+from yamllint.linter import PROBLEM_LEVELS
+
+REPORT_LEVELS: set[PROBLEM_LEVELS] = {
+    "warning",
+    "error",
+}
+
+EXAMPLES_FMT_RE = re.compile(r"^# fmt:\s+(\S+)")
+
+EXAMPLES_SECTION = "EXAMPLES"
+
+
+def lint(
+    *,
+    errors: list[dict[str, t.Any]],
+    path: str,
+    data: str,
+    row_offset: int,
+    col_offset: int,
+    section: str,
+    config: YamlLintConfig,
+    extra_for_errors: dict[str, t.Any] | None = None,
+) -> None:
+    try:
+        problems = linter.run(
+            io.StringIO(data),
+            config,
+            path,
+        )
+        for problem in problems:
+            if problem.level not in REPORT_LEVELS:
+                continue
+            msg = f"{section}: {problem.level}: {problem.desc}"
+            if problem.rule:
+                msg += f"  ({problem.rule})"
+            errors.append(
+                {
+                    "path": path,
+                    "line": row_offset + problem.line,
+                    # The col_offset is only valid for line 1; otherwise the offset is 0
+                    "col": (col_offset if problem.line == 1 else 0) + problem.column,
+                    "message": msg,
+                }
+            )
+            if extra_for_errors:
+                errors[-1].update(extra_for_errors)
+    except Exception as exc:
+        error = str(exc).replace("\n", " / ")
+        errors.append(
+            {
+                "path": path,
+                "line": row_offset + 1,
+                "col": col_offset + 1,
+                "message": (
+                    f"{section}: Internal error while linting YAML: exception {type(exc)}:"
+                    f" {error}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+        if extra_for_errors:
+            errors[-1].update(extra_for_errors)
+
+
+def process_python_file(
+    errors: list[dict[str, t.Any]],
+    path: str,
+    config: YamlLintConfig,
+    config_examples: YamlLintConfig,
+) -> None:
+    try:
+        with open(path, "rt", encoding="utf-8") as f:
+            root = ast.parse(f.read(), filename=path)
+    except Exception as exc:
+        errors.append(
+            {
+                "path": path,
+                "line": 1,
+                "col": 1,
+                "message": (
+                    f"Error while parsing Python code: exception {type(exc)}:"
+                    f" {exc}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+        return
+
+    # We look for top-level assignments
+    for child in root.body:
+        if not isinstance(child, ast.Assign):
+            continue
+        if not isinstance(child.value, ast.Constant):
+            continue
+        if not isinstance(child.value.value, str):
+            continue
+        for target in child.targets:
+            try:
+                section = target.id  # type: ignore
+            except AttributeError:
+                continue
+            if section not in ("DOCUMENTATION", "EXAMPLES", "RETURN"):
+                continue
+
+            # Extract value and offsets
+            data = child.value.value
+            row_offset = child.value.lineno - 1
+            col_offset = child.value.col_offset - 1
+
+            # If the string start with optional whitespace + linebreak, skip that line
+            idx = data.find("\n")
+            if idx >= 0 and (idx == 0 or data[:idx].isspace()):
+                data = data[idx + 1 :]
+                row_offset += 1
+                col_offset = 0
+
+            # Check for non-YAML examples
+            if section == EXAMPLES_SECTION:
+                fmt_match = EXAMPLES_FMT_RE.match(data.lstrip())
+                if fmt_match and fmt_match.group(1) != "yaml":
+                    continue
+
+            # Parse the (remaining) string content
+            lint(
+                errors=errors,
+                path=path,
+                data=data,
+                row_offset=row_offset,
+                col_offset=col_offset,
+                section=section,
+                config=config_examples if section == EXAMPLES_SECTION else config,
+            )
+
+
+def process_sidecar_docs_file(
+    errors: list[dict[str, t.Any]],
+    path: str,
+    config_examples: YamlLintConfig,
+) -> None:
+    try:
+        # TODO: get hold of YAML structure so we also get correct line/col numbers
+        #       inside EXAMPLES!
+        with open(path, "rb") as stream:
+            root = yaml.load(stream, Loader=yaml.SafeLoader)
+    except Exception as exc:
+        errors.append(
+            {
+                "path": path,
+                "line": 1,
+                "col": 1,
+                "message": (
+                    f"Error while parsing Python code: exception {type(exc)}:"
+                    f" {exc}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+        return
+
+    if not isinstance(root, dict):
+        return
+    examples = root.get(EXAMPLES_SECTION)
+    if not isinstance(examples, str):
+        return
+
+    # Check for non-YAML examples
+    fmt_match = EXAMPLES_FMT_RE.match(examples.lstrip())
+    if fmt_match and fmt_match.group(1) != "yaml":
+        return
+
+    lint(
+        errors=errors,
+        path=path,
+        data=examples,
+        row_offset=0,  # TODO
+        col_offset=0,  # TODO
+        section=EXAMPLES_SECTION,
+        config=config_examples,
+        extra_for_errors={
+            "note": "Line/column are relative to EXAMPLES string contents"
+        },
+    )
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+    config: str | None = extra_data.get("config")
+    config_examples: str | None = extra_data.get("config_examples")
+
+    if config:
+        yamllint_config = YamlLintConfig(file=config)
+    else:
+        yamllint_config = YamlLintConfig(content="extends: default")
+
+    if config_examples:
+        yamllint_config_examples = YamlLintConfig(file=config_examples)
+    else:
+        yamllint_config_examples = yamllint_config
+
+    errors: list[dict[str, t.Any]] = []
+    for path in paths:
+        if path.endswith(".py"):
+            process_python_file(errors, path, yamllint_config, yamllint_config_examples)
+        if path.endswith((".yml", ".yaml")):
+            process_sidecar_docs_file(errors, path, yamllint_config_examples)
+
+    errors.sort(
+        key=lambda error: (error["path"], error["line"], error["col"], error["message"])
+    )
+    for error in errors:
+        prefix = f"{error['path']}:{error['line']}:{error['col']}: "
+        msg = error["message"]
+        if "note" in error:
+            msg = f"{msg}\nNote: {error['note']}"
+        for i, line in enumerate(msg.splitlines()):
+            print(f"{prefix}{line}")
+            if i == 0:
+                prefix = " " * len(prefix)
+
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data_util.py

@@ -0,0 +1,38 @@
+# Author: Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025, Ansible Project
+
+"""
+Utility code for scripts in data.
+"""
+
+from __future__ import annotations
+
+import json
+import typing as t
+from pathlib import Path
+
+import nox
+
+
+def prepare_data_script(
+    session: nox.Session,
+    *,
+    base_name: str,
+    paths: list[Path],
+    extra_data: dict[str, t.Any] | None = None,
+) -> Path:
+    """
+    Prepare a data JSON file for the extra sanity check scripts.
+    """
+    cwd = Path.cwd()
+    data = {}
+    data["paths"] = [str(path.relative_to(cwd)) for path in paths]
+    if extra_data:
+        data.update(extra_data)
+    file = Path(session.create_tmp()) / f"{base_name}-data.json"
+    with open(file, "w", encoding="utf-8") as f:
+        json.dump(data, f)
+    return file