antsibull-nox 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

antsibull_nox/config.py

@@ -0,0 +1,332 @@
+# Author: Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025, Ansible Project
+
+"""
+Config file schema.
+"""
+
+from __future__ import annotations
+
+import os
+import typing as t
+
+import pydantic as p
+
+from .ansible import AnsibleCoreVersion
+from .utils import Version
+
+try:
+    from tomllib import load as _load_toml
+except ImportError:
+    from tomli import load as _load_toml  # type: ignore
+
+
+def _parse_version(value: t.Any) -> Version:
+    if isinstance(value, Version):
+        return value
+    if isinstance(value, str) and "." in value:
+        return Version.parse(value)
+    raise ValueError("Must be version string")
+
+
+def _parse_ansible_core_version(value: t.Any) -> AnsibleCoreVersion:
+    if isinstance(value, Version):
+        return value
+    if isinstance(value, str):
+        if value == "devel":
+            return "devel"
+        if value == "milestone":
+            return "milestone"
+        if "." in value:
+            return Version.parse(value)
+    raise ValueError("Must be ansible-core version string")
+
+
+PVersion = t.Annotated[Version, p.BeforeValidator(_parse_version)]
+PAnsibleCoreVersion = t.Annotated[
+    AnsibleCoreVersion, p.BeforeValidator(_parse_ansible_core_version)
+]
+
+
+class _BaseModel(p.BaseModel):
+    model_config = p.ConfigDict(frozen=True, extra="allow", validate_default=True)
+
+
+class SessionLint(_BaseModel):
+    """
+    Lint session config.
+    """
+
+    default: bool = True
+    extra_code_files: list[str] = []
+
+    # isort:
+    run_isort: bool = True
+    isort_config: t.Optional[p.FilePath] = None
+    isort_package: str = "isort"
+
+    # black:
+    run_black: bool = True
+    run_black_modules: t.Optional[bool] = None
+    black_config: t.Optional[p.FilePath] = None
+    black_package: str = "black"
+
+    # flake8:
+    run_flake8: bool = True
+    flake8_config: t.Optional[p.FilePath] = None
+    flake8_package: str = "flake8"
+
+    # pylint:
+    run_pylint: bool = True
+    pylint_rcfile: t.Optional[p.FilePath] = None
+    pylint_modules_rcfile: t.Optional[p.FilePath] = None
+    pylint_package: str = "pylint"
+    pylint_ansible_core_package: t.Optional[str] = "ansible-core"
+    pylint_extra_deps: list[str] = []
+
+    # yamllint:
+    run_yamllint: bool = True
+    yamllint_config: t.Optional[p.FilePath] = None
+    yamllint_config_plugins: t.Optional[p.FilePath] = None
+    yamllint_config_plugins_examples: t.Optional[p.FilePath] = None
+    yamllint_package: str = "yamllint"
+
+    # mypy:
+    run_mypy: bool = True
+    mypy_config: t.Optional[p.FilePath] = None
+    mypy_package: str = "mypy"
+    mypy_ansible_core_package: t.Optional[str] = "ansible-core"
+    mypy_extra_deps: list[str] = []
+
+
+class SessionDocsCheck(_BaseModel):
+    """
+    Docs check session config.
+    """
+
+    default: bool = True
+
+    antsibull_docs_package: str = "antsibull-docs"
+    ansible_core_package: str = "ansible-core"
+    validate_collection_refs: t.Optional[t.Literal["self", "dependent", "all"]] = None
+    extra_collections: list[str] = []
+
+
+class SessionLicenseCheck(_BaseModel):
+    """
+    License check session config.
+    """
+
+    default: bool = True
+
+    run_reuse: bool = True
+    reuse_package: str = "reuse"
+    run_license_check: bool = True
+    license_check_extra_ignore_paths: list[str] = []
+
+
+class ActionGroup(_BaseModel):
+    """
+    Information about an action group.
+    """
+
+    # Name of the action group.
+    name: str
+    # Regex pattern to match modules that could belong to this action group.
+    pattern: str
+    # Doc fragment that members of the action group must have, but no other module
+    # must have
+    doc_fragment: str
+    # Exclusion list of modules that match the regex, but should not be part of the
+    # action group. All other modules matching the regex are assumed to be part of
+    # the action group.
+    exclusions: list[str] = []
+
+
+class SessionExtraChecks(_BaseModel):
+    """
+    Extra checks session config.
+    """
+
+    default: bool = True
+
+    # no-unwanted-files:
+    run_no_unwanted_files: bool = True
+    no_unwanted_files_module_extensions: list[str] = [".cs", ".ps1", ".psm1", ".py"]
+    no_unwanted_files_other_extensions: list[str] = [".py", ".pyi"]
+    no_unwanted_files_yaml_extensions: list[str] = [".yml", ".yaml"]
+    no_unwanted_files_skip_paths: list[str] = []
+    no_unwanted_files_skip_directories: t.Optional[list[str]] = []
+    no_unwanted_files_yaml_directories: t.Optional[list[str]] = [
+        "plugins/test/",
+        "plugins/filter/",
+    ]
+    no_unwanted_files_allow_symlinks: bool = False
+
+    # action-groups:
+    run_action_groups: bool = False
+    action_groups_config: list[ActionGroup] = []
+
+
+class SessionBuildImportCheck(_BaseModel):
+    """
+    Collection build and Galaxy import session config.
+    """
+
+    default: bool = True
+
+    ansible_core_package: str = "ansible-core"
+    run_galaxy_importer: bool = True
+    galaxy_importer_package: str = "galaxy-importer"
+    # https://github.com/ansible/galaxy-importer#configuration
+    galaxy_importer_config_path: t.Optional[p.FilePath] = None
+
+
+class DevelLikeBranch(_BaseModel):
+    """
+    A Git repository + branch for a devel-like branch of ansible-core.
+    """
+
+    repository: t.Optional[str] = None
+    branch: str
+
+    @p.model_validator(mode="before")
+    @classmethod
+    def _pre_validate(cls, values: t.Any) -> t.Any:
+        if isinstance(values, str):
+            return {"branch": values}
+        if (
+            isinstance(values, list)
+            and len(values) == 2
+            and all(isinstance(v, str) for v in values)
+        ):
+            return {"repository": values[0], "branch": values[1]}
+        return values
+
+
+class SessionAnsibleTestSanity(_BaseModel):
+    """
+    Ansible-test sanity tests session config.
+    """
+
+    default: bool = False
+
+    include_devel: bool = False
+    include_milestone: bool = False
+    add_devel_like_branches: list[DevelLikeBranch] = []
+    min_version: t.Optional[PVersion] = None
+    max_version: t.Optional[PVersion] = None
+    except_versions: list[PAnsibleCoreVersion] = []
+
+
+class SessionAnsibleTestUnits(_BaseModel):
+    """
+    Ansible-test unit tests session config.
+    """
+
+    default: bool = False
+
+    include_devel: bool = False
+    include_milestone: bool = False
+    add_devel_like_branches: list[DevelLikeBranch] = []
+    min_version: t.Optional[PVersion] = None
+    max_version: t.Optional[PVersion] = None
+    except_versions: list[PAnsibleCoreVersion] = []
+
+
+class SessionAnsibleTestIntegrationWDefaultContainer(_BaseModel):
+    """
+    Ansible-test integration tests with default container session config.
+    """
+
+    default: bool = False
+
+    include_devel: bool = False
+    include_milestone: bool = False
+    add_devel_like_branches: list[DevelLikeBranch] = []
+    min_version: t.Optional[PVersion] = None
+    max_version: t.Optional[PVersion] = None
+    except_versions: list[PAnsibleCoreVersion] = []
+    core_python_versions: dict[t.Union[PAnsibleCoreVersion, str], list[PVersion]] = {}
+    controller_python_versions_only: bool = False
+
+    @p.model_validator(mode="after")
+    def _validate_core_keys(self) -> t.Self:
+        branch_names = [dlb.branch for dlb in self.add_devel_like_branches]
+        for key in self.core_python_versions:
+            if isinstance(key, Version) or key in {"devel", "milestone"}:
+                continue
+            if key in branch_names:
+                continue
+            raise ValueError(
+                f"Unknown ansible-core version or branch name {key!r} in core_python_versions"
+            )
+        return self
+
+
+class SessionAnsibleLint(_BaseModel):
+    """
+    Ansible-lint session config.
+    """
+
+    default: bool = True
+
+    ansible_lint_package: str = "ansible-lint"
+    strict: bool = False
+
+
+class Sessions(_BaseModel):
+    """
+    Configuration of nox sessions to add.
+    """
+
+    lint: t.Optional[SessionLint] = None
+    docs_check: t.Optional[SessionDocsCheck] = None
+    license_check: t.Optional[SessionLicenseCheck] = None
+    extra_checks: t.Optional[SessionExtraChecks] = None
+    build_import_check: t.Optional[SessionBuildImportCheck] = None
+    ansible_test_sanity: t.Optional[SessionAnsibleTestSanity] = None
+    ansible_test_units: t.Optional[SessionAnsibleTestUnits] = None
+    ansible_test_integration_w_default_container: t.Optional[
+        SessionAnsibleTestIntegrationWDefaultContainer
+    ] = None
+    ansible_lint: t.Optional[SessionAnsibleLint] = None
+
+
+class CollectionSource(_BaseModel):
+    """
+    Source from which to install a collection.
+    """
+
+    source: str
+
+    @p.model_validator(mode="before")
+    @classmethod
+    def _pre_validate(cls, values):
+        if isinstance(values, str):
+            return {"source": values}
+        return values
+
+
+class Config(_BaseModel):
+    """
+    The contents of a antsibull-nox config file.
+    """
+
+    collection_sources: dict[str, CollectionSource] = {}
+    sessions: Sessions = Sessions()
+
+
+def load_config_from_toml(path: str | os.PathLike) -> Config:
+    """
+    Load a config TOML file.
+    """
+    with open(path, "rb") as f:
+        try:
+            data = _load_toml(f)
+        except ValueError as exc:
+            raise ValueError(f"Error while reading {path}: {exc}") from exc
+    return Config.model_validate(data)

antsibull_nox/data/action-groups.py

@@ -16,7 +16,7 @@ import typing as t
 
 import yaml
 
-from antsibull_nox.data_util import setup
+from antsibull_nox.data.antsibull_nox_data_util import setup
 from antsibull_nox.sessions import ActionGroup
 
 

antsibull_nox/data/antsibull_nox_data_util.py

@@ -0,0 +1,91 @@
+# Author: Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025, Ansible Project
+
+"""
+Utility code for scripts in data.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import typing as t
+
+
+def setup() -> tuple[list[str], dict[str, t.Any]]:
+    """
+    Fetch list of paths and potential extra configuration.
+
+    First thing to call in an extra sanity check script in data/.
+    """
+    if len(sys.argv) == 3 and sys.argv[1] == "--data":
+        # Preferred way: load information from JSON file
+        path = sys.argv[2]
+        try:
+            with open(path, "rb") as f:
+                data = json.load(f)
+        except Exception as exc:
+            raise ValueError(f"Error while reading JSON from {path}") from exc
+        try:
+            paths = get_list_of_strings(data, "paths")
+        except ValueError as exc:
+            raise ValueError(f"Invalid JSON content in {path}: {exc}") from exc
+        data.pop("paths")
+        return paths, data
+    if len(sys.argv) >= 2:
+        # It's also possible to pass a list of paths on the command line, to simplify
+        # testing these scripts.
+        return sys.argv[1:], {}
+    # Alternatively one can pass a list of files from stdin, for example by piping
+    # the output of 'git ls-files' into this script. This is also for testing these
+    # scripts.
+    return sys.stdin.read().splitlines(), {}
+
+
+def get_list_of_strings(
+    data: dict[str, t.Any],
+    key: str,
+    *,
+    default: list[str] | None = None,
+) -> list[str]:
+    """
+    Retrieves a list of strings from key ``key`` of the JSON object ``data``.
+
+    If ``default`` is set to a list, a missing key results in this value being returned.
+    """
+    sentinel = object()
+    value = data.get(key, sentinel)
+    if value is sentinel:
+        if default is not None:
+            return default
+        raise ValueError(f"{key!r} is not a present")
+    if not isinstance(value, list):
+        raise ValueError(f"{key!r} is not a list, but {type(key)}")
+    if not all(isinstance(entry, str) for entry in value):
+        raise ValueError(f"{key!r} is not a list of strings")
+    return t.cast(list[str], value)
+
+
+def get_bool(
+    data: dict[str, t.Any],
+    key: str,
+    *,
+    default: bool | None = None,
+) -> bool:
+    """
+    Retrieves a boolean from key ``key`` of the JSON object ``data``.
+
+    If ``default`` is set to a boolean, a missing key results in this value being returned.
+    """
+    sentinel = object()
+    value = data.get(key, sentinel)
+    if value is sentinel:
+        if default is not None:
+            return default
+        raise ValueError(f"{key!r} is not a present")
+    if not isinstance(value, bool):
+        raise ValueError(f"{key!r} is not a bool, but {type(key)}")
+    return value

antsibull_nox/data/license-check.py

@@ -13,7 +13,7 @@ import glob
 import os
 import sys
 
-from antsibull_nox.data_util import get_list_of_strings, setup
+from antsibull_nox.data.antsibull_nox_data_util import get_list_of_strings, setup
 
 
 def format_license_list(licenses: list[str]) -> str:

antsibull_nox/data/no-unwanted-files.py

@@ -12,7 +12,11 @@ from __future__ import annotations
 import os
 import sys
 
-from antsibull_nox.data_util import get_bool, get_list_of_strings, setup
+from antsibull_nox.data.antsibull_nox_data_util import (
+    get_bool,
+    get_list_of_strings,
+    setup,
+)
 
 
 def main() -> int:

antsibull_nox/data/plugin-yamllint.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Make sure all plugin and module documentation adheres to yamllint."""
+
+from __future__ import annotations
+
+import ast
+import io
+import re
+import sys
+import traceback
+import typing as t
+
+import yaml
+from antsibull_nox_data_util import setup  # type: ignore
+from yamllint import linter
+from yamllint.config import YamlLintConfig
+from yamllint.linter import PROBLEM_LEVELS
+
+REPORT_LEVELS: set[PROBLEM_LEVELS] = {
+    "warning",
+    "error",
+}
+
+EXAMPLES_FMT_RE = re.compile(r"^# fmt:\s+(\S+)")
+
+EXAMPLES_SECTION = "EXAMPLES"
+
+
+def lint(
+    *,
+    errors: list[dict[str, t.Any]],
+    path: str,
+    data: str,
+    row_offset: int,
+    col_offset: int,
+    section: str,
+    config: YamlLintConfig,
+    extra_for_errors: dict[str, t.Any] | None = None,
+) -> None:
+    try:
+        problems = linter.run(
+            io.StringIO(data),
+            config,
+            path,
+        )
+        for problem in problems:
+            if problem.level not in REPORT_LEVELS:
+                continue
+            msg = f"{section}: {problem.level}: {problem.desc}"
+            if problem.rule:
+                msg += f"  ({problem.rule})"
+            errors.append(
+                {
+                    "path": path,
+                    "line": row_offset + problem.line,
+                    # The col_offset is only valid for line 1; otherwise the offset is 0
+                    "col": (col_offset if problem.line == 1 else 0) + problem.column,
+                    "message": msg,
+                }
+            )
+            if extra_for_errors:
+                errors[-1].update(extra_for_errors)
+    except Exception as exc:
+        error = str(exc).replace("\n", " / ")
+        errors.append(
+            {
+                "path": path,
+                "line": row_offset + 1,
+                "col": col_offset + 1,
+                "message": (
+                    f"{section}: Internal error while linting YAML: exception {type(exc)}:"
+                    f" {error}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+        if extra_for_errors:
+            errors[-1].update(extra_for_errors)
+
+
+def process_python_file(
+    errors: list[dict[str, t.Any]],
+    path: str,
+    config: YamlLintConfig,
+    config_examples: YamlLintConfig,
+) -> None:
+    try:
+        with open(path, "rt", encoding="utf-8") as f:
+            root = ast.parse(f.read(), filename=path)
+    except Exception as exc:
+        errors.append(
+            {
+                "path": path,
+                "line": 1,
+                "col": 1,
+                "message": (
+                    f"Error while parsing Python code: exception {type(exc)}:"
+                    f" {exc}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+        return
+
+    # We look for top-level assignments
+    for child in root.body:
+        if not isinstance(child, ast.Assign):
+            continue
+        if not isinstance(child.value, ast.Constant):
+            continue
+        if not isinstance(child.value.value, str):
+            continue
+        for target in child.targets:
+            try:
+                section = target.id  # type: ignore
+            except AttributeError:
+                continue
+            if section not in ("DOCUMENTATION", "EXAMPLES", "RETURN"):
+                continue
+
+            # Extract value and offsets
+            data = child.value.value
+            row_offset = child.value.lineno - 1
+            col_offset = child.value.col_offset - 1
+
+            # If the string start with optional whitespace + linebreak, skip that line
+            idx = data.find("\n")
+            if idx >= 0 and (idx == 0 or data[:idx].isspace()):
+                data = data[idx + 1 :]
+                row_offset += 1
+                col_offset = 0
+
+            # Check for non-YAML examples
+            if section == EXAMPLES_SECTION:
+                fmt_match = EXAMPLES_FMT_RE.match(data.lstrip())
+                if fmt_match and fmt_match.group(1) != "yaml":
+                    continue
+
+            # Parse the (remaining) string content
+            lint(
+                errors=errors,
+                path=path,
+                data=data,
+                row_offset=row_offset,
+                col_offset=col_offset,
+                section=section,
+                config=config_examples if section == EXAMPLES_SECTION else config,
+            )
+
+
+def process_sidecar_docs_file(
+    errors: list[dict[str, t.Any]],
+    path: str,
+    config_examples: YamlLintConfig,
+) -> None:
+    try:
+        # TODO: get hold of YAML structure so we also get correct line/col numbers
+        #       inside EXAMPLES!
+        with open(path, "rb") as stream:
+            root = yaml.load(stream, Loader=yaml.SafeLoader)
+    except Exception as exc:
+        errors.append(
+            {
+                "path": path,
+                "line": 1,
+                "col": 1,
+                "message": (
+                    f"Error while parsing Python code: exception {type(exc)}:"
+                    f" {exc}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+        return
+
+    if not isinstance(root, dict):
+        return
+    examples = root.get(EXAMPLES_SECTION)
+    if not isinstance(examples, str):
+        return
+
+    # Check for non-YAML examples
+    fmt_match = EXAMPLES_FMT_RE.match(examples.lstrip())
+    if fmt_match and fmt_match.group(1) != "yaml":
+        return
+
+    lint(
+        errors=errors,
+        path=path,
+        data=examples,
+        row_offset=0,  # TODO
+        col_offset=0,  # TODO
+        section=EXAMPLES_SECTION,
+        config=config_examples,
+        extra_for_errors={
+            "note": "Line/column are relative to EXAMPLES string contents"
+        },
+    )
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+    config: str | None = extra_data.get("config")
+    config_examples: str | None = extra_data.get("config_examples")
+
+    if config:
+        yamllint_config = YamlLintConfig(file=config)
+    else:
+        yamllint_config = YamlLintConfig(content="extends: default")
+
+    if config_examples:
+        yamllint_config_examples = YamlLintConfig(file=config_examples)
+    else:
+        yamllint_config_examples = yamllint_config
+
+    errors: list[dict[str, t.Any]] = []
+    for path in paths:
+        if path.endswith(".py"):
+            process_python_file(errors, path, yamllint_config, yamllint_config_examples)
+        if path.endswith((".yml", ".yaml")):
+            process_sidecar_docs_file(errors, path, yamllint_config_examples)
+
+    errors.sort(
+        key=lambda error: (error["path"], error["line"], error["col"], error["message"])
+    )
+    for error in errors:
+        prefix = f"{error['path']}:{error['line']}:{error['col']}: "
+        msg = error["message"]
+        if "note" in error:
+            msg = f"{msg}\nNote: {error['note']}"
+        for i, line in enumerate(msg.splitlines()):
+            print(f"{prefix}{line}")
+            if i == 0:
+                prefix = " " * len(prefix)
+
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())