antsibull-nox 0.0.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

antsibull_nox/config.py

@@ -0,0 +1,332 @@
+# Author: Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025, Ansible Project
+
+"""
+Config file schema.
+"""
+
+from __future__ import annotations
+
+import os
+import typing as t
+
+import pydantic as p
+
+from .ansible import AnsibleCoreVersion
+from .utils import Version
+
+try:
+    from tomllib import load as _load_toml
+except ImportError:
+    from tomli import load as _load_toml  # type: ignore
+
+
+def _parse_version(value: t.Any) -> Version:
+    if isinstance(value, Version):
+        return value
+    if isinstance(value, str) and "." in value:
+        return Version.parse(value)
+    raise ValueError("Must be version string")
+
+
+def _parse_ansible_core_version(value: t.Any) -> AnsibleCoreVersion:
+    if isinstance(value, Version):
+        return value
+    if isinstance(value, str):
+        if value == "devel":
+            return "devel"
+        if value == "milestone":
+            return "milestone"
+        if "." in value:
+            return Version.parse(value)
+    raise ValueError("Must be ansible-core version string")
+
+
+PVersion = t.Annotated[Version, p.BeforeValidator(_parse_version)]
+PAnsibleCoreVersion = t.Annotated[
+    AnsibleCoreVersion, p.BeforeValidator(_parse_ansible_core_version)
+]
+
+
+class _BaseModel(p.BaseModel):
+    model_config = p.ConfigDict(frozen=True, extra="allow", validate_default=True)
+
+
+class SessionLint(_BaseModel):
+    """
+    Lint session config.
+    """
+
+    default: bool = True
+    extra_code_files: list[str] = []
+
+    # isort:
+    run_isort: bool = True
+    isort_config: t.Optional[p.FilePath] = None
+    isort_package: str = "isort"
+
+    # black:
+    run_black: bool = True
+    run_black_modules: t.Optional[bool] = None
+    black_config: t.Optional[p.FilePath] = None
+    black_package: str = "black"
+
+    # flake8:
+    run_flake8: bool = True
+    flake8_config: t.Optional[p.FilePath] = None
+    flake8_package: str = "flake8"
+
+    # pylint:
+    run_pylint: bool = True
+    pylint_rcfile: t.Optional[p.FilePath] = None
+    pylint_modules_rcfile: t.Optional[p.FilePath] = None
+    pylint_package: str = "pylint"
+    pylint_ansible_core_package: t.Optional[str] = "ansible-core"
+    pylint_extra_deps: list[str] = []
+
+    # yamllint:
+    run_yamllint: bool = True
+    yamllint_config: t.Optional[p.FilePath] = None
+    yamllint_config_plugins: t.Optional[p.FilePath] = None
+    yamllint_config_plugins_examples: t.Optional[p.FilePath] = None
+    yamllint_package: str = "yamllint"
+
+    # mypy:
+    run_mypy: bool = True
+    mypy_config: t.Optional[p.FilePath] = None
+    mypy_package: str = "mypy"
+    mypy_ansible_core_package: t.Optional[str] = "ansible-core"
+    mypy_extra_deps: list[str] = []
+
+
+class SessionDocsCheck(_BaseModel):
+    """
+    Docs check session config.
+    """
+
+    default: bool = True
+
+    antsibull_docs_package: str = "antsibull-docs"
+    ansible_core_package: str = "ansible-core"
+    validate_collection_refs: t.Optional[t.Literal["self", "dependent", "all"]] = None
+    extra_collections: list[str] = []
+
+
+class SessionLicenseCheck(_BaseModel):
+    """
+    License check session config.
+    """
+
+    default: bool = True
+
+    run_reuse: bool = True
+    reuse_package: str = "reuse"
+    run_license_check: bool = True
+    license_check_extra_ignore_paths: list[str] = []
+
+
+class ActionGroup(_BaseModel):
+    """
+    Information about an action group.
+    """
+
+    # Name of the action group.
+    name: str
+    # Regex pattern to match modules that could belong to this action group.
+    pattern: str
+    # Doc fragment that members of the action group must have, but no other module
+    # must have
+    doc_fragment: str
+    # Exclusion list of modules that match the regex, but should not be part of the
+    # action group. All other modules matching the regex are assumed to be part of
+    # the action group.
+    exclusions: list[str] = []
+
+
+class SessionExtraChecks(_BaseModel):
+    """
+    Extra checks session config.
+    """
+
+    default: bool = True
+
+    # no-unwanted-files:
+    run_no_unwanted_files: bool = True
+    no_unwanted_files_module_extensions: list[str] = [".cs", ".ps1", ".psm1", ".py"]
+    no_unwanted_files_other_extensions: list[str] = [".py", ".pyi"]
+    no_unwanted_files_yaml_extensions: list[str] = [".yml", ".yaml"]
+    no_unwanted_files_skip_paths: list[str] = []
+    no_unwanted_files_skip_directories: t.Optional[list[str]] = []
+    no_unwanted_files_yaml_directories: t.Optional[list[str]] = [
+        "plugins/test/",
+        "plugins/filter/",
+    ]
+    no_unwanted_files_allow_symlinks: bool = False
+
+    # action-groups:
+    run_action_groups: bool = False
+    action_groups_config: list[ActionGroup] = []
+
+
+class SessionBuildImportCheck(_BaseModel):
+    """
+    Collection build and Galaxy import session config.
+    """
+
+    default: bool = True
+
+    ansible_core_package: str = "ansible-core"
+    run_galaxy_importer: bool = True
+    galaxy_importer_package: str = "galaxy-importer"
+    # https://github.com/ansible/galaxy-importer#configuration
+    galaxy_importer_config_path: t.Optional[p.FilePath] = None
+
+
+class DevelLikeBranch(_BaseModel):
+    """
+    A Git repository + branch for a devel-like branch of ansible-core.
+    """
+
+    repository: t.Optional[str] = None
+    branch: str
+
+    @p.model_validator(mode="before")
+    @classmethod
+    def _pre_validate(cls, values: t.Any) -> t.Any:
+        if isinstance(values, str):
+            return {"branch": values}
+        if (
+            isinstance(values, list)
+            and len(values) == 2
+            and all(isinstance(v, str) for v in values)
+        ):
+            return {"repository": values[0], "branch": values[1]}
+        return values
+
+
+class SessionAnsibleTestSanity(_BaseModel):
+    """
+    Ansible-test sanity tests session config.
+    """
+
+    default: bool = False
+
+    include_devel: bool = False
+    include_milestone: bool = False
+    add_devel_like_branches: list[DevelLikeBranch] = []
+    min_version: t.Optional[PVersion] = None
+    max_version: t.Optional[PVersion] = None
+    except_versions: list[PAnsibleCoreVersion] = []
+
+
+class SessionAnsibleTestUnits(_BaseModel):
+    """
+    Ansible-test unit tests session config.
+    """
+
+    default: bool = False
+
+    include_devel: bool = False
+    include_milestone: bool = False
+    add_devel_like_branches: list[DevelLikeBranch] = []
+    min_version: t.Optional[PVersion] = None
+    max_version: t.Optional[PVersion] = None
+    except_versions: list[PAnsibleCoreVersion] = []
+
+
+class SessionAnsibleTestIntegrationWDefaultContainer(_BaseModel):
+    """
+    Ansible-test integration tests with default container session config.
+    """
+
+    default: bool = False
+
+    include_devel: bool = False
+    include_milestone: bool = False
+    add_devel_like_branches: list[DevelLikeBranch] = []
+    min_version: t.Optional[PVersion] = None
+    max_version: t.Optional[PVersion] = None
+    except_versions: list[PAnsibleCoreVersion] = []
+    core_python_versions: dict[t.Union[PAnsibleCoreVersion, str], list[PVersion]] = {}
+    controller_python_versions_only: bool = False
+
+    @p.model_validator(mode="after")
+    def _validate_core_keys(self) -> t.Self:
+        branch_names = [dlb.branch for dlb in self.add_devel_like_branches]
+        for key in self.core_python_versions:
+            if isinstance(key, Version) or key in {"devel", "milestone"}:
+                continue
+            if key in branch_names:
+                continue
+            raise ValueError(
+                f"Unknown ansible-core version or branch name {key!r} in core_python_versions"
+            )
+        return self
+
+
+class SessionAnsibleLint(_BaseModel):
+    """
+    Ansible-lint session config.
+    """
+
+    default: bool = True
+
+    ansible_lint_package: str = "ansible-lint"
+    strict: bool = False
+
+
+class Sessions(_BaseModel):
+    """
+    Configuration of nox sessions to add.
+    """
+
+    lint: t.Optional[SessionLint] = None
+    docs_check: t.Optional[SessionDocsCheck] = None
+    license_check: t.Optional[SessionLicenseCheck] = None
+    extra_checks: t.Optional[SessionExtraChecks] = None
+    build_import_check: t.Optional[SessionBuildImportCheck] = None
+    ansible_test_sanity: t.Optional[SessionAnsibleTestSanity] = None
+    ansible_test_units: t.Optional[SessionAnsibleTestUnits] = None
+    ansible_test_integration_w_default_container: t.Optional[
+        SessionAnsibleTestIntegrationWDefaultContainer
+    ] = None
+    ansible_lint: t.Optional[SessionAnsibleLint] = None
+
+
+class CollectionSource(_BaseModel):
+    """
+    Source from which to install a collection.
+    """
+
+    source: str
+
+    @p.model_validator(mode="before")
+    @classmethod
+    def _pre_validate(cls, values):
+        if isinstance(values, str):
+            return {"source": values}
+        return values
+
+
+class Config(_BaseModel):
+    """
+    The contents of a antsibull-nox config file.
+    """
+
+    collection_sources: dict[str, CollectionSource] = {}
+    sessions: Sessions = Sessions()
+
+
+def load_config_from_toml(path: str | os.PathLike) -> Config:
+    """
+    Load a config TOML file.
+    """
+    with open(path, "rb") as f:
+        try:
+            data = _load_toml(f)
+        except ValueError as exc:
+            raise ValueError(f"Error while reading {path}: {exc}") from exc
+    return Config.model_validate(data)

antsibull_nox/data/action-groups.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Make sure all modules that should show up in the action group."""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+import typing as t
+
+import yaml
+
+from antsibull_nox.data.antsibull_nox_data_util import setup
+from antsibull_nox.sessions import ActionGroup
+
+
+def compile_patterns(
+    config: list[ActionGroup], errors: list[str]
+) -> dict[str, re.Pattern] | None:
+    patterns: dict[str, re.Pattern] = {}
+    for action_group in config:
+        if action_group.name in config:
+            errors.append(
+                f"noxfile.py: Action group {action_group.name!r} defined multiple times"
+            )
+            return None
+        patterns[action_group.name] = re.compile(action_group.pattern)
+    return patterns
+
+
+def load_redirects(
+    config: list[ActionGroup], errors: list[str], meta_runtime: str
+) -> dict[str, list[str]]:
+    # Load redirects
+    try:
+        with open(meta_runtime, "rb") as f:
+            data = yaml.safe_load(f)
+        action_groups = data["action_groups"]
+    except Exception as exc:
+        errors.append(f"{meta_runtime}: cannot load action groups: {exc}")
+        return {}
+
+    if not isinstance(action_groups, dict):
+        errors.append(f"{meta_runtime}: action_groups is not a dictionary")
+        return {}
+    if not all(
+        isinstance(k, str) and isinstance(v, list) for k, v in action_groups.items()
+    ):
+        errors.append(
+            f"{meta_runtime}: action_groups is not a dictionary mapping strings to list of strings"
+        )
+        return {}
+
+    # Compare meta/runtime.yml content with config
+    config_groups = {cfg.name for cfg in config}
+    for action_group in action_groups:
+        if action_group not in config_groups:
+            errors.append(
+                f"{meta_runtime}: found unknown action group"
+                f" {action_group!r}; likely noxfile needs updating"
+            )
+    for action_group in config:
+        if action_group.name not in action_groups:
+            errors.append(
+                f"{meta_runtime}: cannot find action group"
+                f" {action_group.name!r}; likely noxfile needs updating"
+            )
+
+    return action_groups
+
+
+def load_docs(path: str, errors: list[str]) -> dict[str, t.Any] | None:
+    documentation = []
+    in_docs = False
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("DOCUMENTATION ="):
+                in_docs = True
+            elif line.startswith(("'''", '"""')) and in_docs:
+                in_docs = False
+            elif in_docs:
+                documentation.append(line)
+    if in_docs:
+        errors.append(f"{path}: cannot find DOCUMENTATION end")
+    if not documentation:
+        errors.append(f"{path}: cannot find DOCUMENTATION")
+        return None
+
+    try:
+        docs = yaml.safe_load("\n".join(documentation))
+        if not isinstance(docs, dict):
+            raise Exception("is not a top-level dictionary")
+        return docs
+    except Exception as exc:
+        errors.append(f"{path}: cannot load DOCUMENTATION as YAML: {exc}")
+        return None
+
+
+def scan(config: list[ActionGroup], errors: list[str]) -> None:
+    patterns = compile_patterns(config, errors)
+    if patterns is None:
+        return
+
+    meta_runtime = "meta/runtime.yml"
+    action_groups = load_redirects(config, errors, meta_runtime)
+
+    modules_directory = "plugins/modules/"
+    modules_suffix = ".py"
+
+    errors = []
+    for file in os.listdir(modules_directory):
+        if not file.endswith(modules_suffix):
+            continue
+        module_name = file[: -len(modules_suffix)]
+
+        for action_group in config:
+            action_group_content = action_groups.get(action_group.name) or []
+            path = os.path.join(modules_directory, file)
+
+            if not patterns[action_group.name].match(module_name):
+                if module_name in action_group_content:
+                    errors.append(
+                        f"{path}: module is in action group {action_group.name!r}"
+                        " despite not matching its pattern as defined in noxfile"
+                    )
+                continue
+
+            should_be_in_action_group = (
+                module_name not in action_group.exclusions
+                if action_group.exclusions
+                else True
+            )
+
+            if should_be_in_action_group:
+                if module_name not in action_group_content:
+                    errors.append(
+                        f"{meta_runtime}: module {module_name!r} is not part"
+                        f" of {action_group.name!r} action group"
+                    )
+                else:
+                    action_group_content.remove(module_name)
+
+            docs = load_docs(path, errors)
+            if docs is None:
+                continue
+
+            docs_fragments = docs.get("extends_documentation_fragment") or []
+            is_in_action_group = action_group.doc_fragment in docs_fragments
+
+            if should_be_in_action_group != is_in_action_group:
+                if should_be_in_action_group:
+                    errors.append(
+                        f"{path}: module does not document itself as part of"
+                        f" action group {action_group.name!r}, but it should;"
+                        f" you need to add {action_group.doc_fragment} to"
+                        f' "extends_documentation_fragment" in DOCUMENTATION'
+                    )
+                else:
+                    errors.append(
+                        f"{path}: module documents itself as part of"
+                        f" action group {action_group.name!r}, but it should not be"
+                    )
+
+    for action_group in config:
+        action_group_content = action_groups.get(action_group.name) or []
+        for module_name in action_group_content:
+            errors.append(
+                f"{meta_runtime}: module {module_name} mentioned"
+                f" in {action_group.name!r} action group does not exist"
+                " or does not match pattern defined in noxfile"
+            )
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+
+    if not isinstance(extra_data.get("config"), list):
+        raise ValueError("config is not a list")
+    if not all(isinstance(cfg, dict) for cfg in extra_data["config"]):
+        raise ValueError("config is not a list of dictionaries")
+    config = [ActionGroup(**cfg) for cfg in extra_data["config"]]
+
+    errors: list[str] = []
+    scan(config, errors)
+
+    for error in sorted(errors):
+        print(error)
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data/antsibull_nox_data_util.py

@@ -0,0 +1,91 @@
+# Author: Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025, Ansible Project
+
+"""
+Utility code for scripts in data.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import typing as t
+
+
+def setup() -> tuple[list[str], dict[str, t.Any]]:
+    """
+    Fetch list of paths and potential extra configuration.
+
+    First thing to call in an extra sanity check script in data/.
+    """
+    if len(sys.argv) == 3 and sys.argv[1] == "--data":
+        # Preferred way: load information from JSON file
+        path = sys.argv[2]
+        try:
+            with open(path, "rb") as f:
+                data = json.load(f)
+        except Exception as exc:
+            raise ValueError(f"Error while reading JSON from {path}") from exc
+        try:
+            paths = get_list_of_strings(data, "paths")
+        except ValueError as exc:
+            raise ValueError(f"Invalid JSON content in {path}: {exc}") from exc
+        data.pop("paths")
+        return paths, data
+    if len(sys.argv) >= 2:
+        # It's also possible to pass a list of paths on the command line, to simplify
+        # testing these scripts.
+        return sys.argv[1:], {}
+    # Alternatively one can pass a list of files from stdin, for example by piping
+    # the output of 'git ls-files' into this script. This is also for testing these
+    # scripts.
+    return sys.stdin.read().splitlines(), {}
+
+
+def get_list_of_strings(
+    data: dict[str, t.Any],
+    key: str,
+    *,
+    default: list[str] | None = None,
+) -> list[str]:
+    """
+    Retrieves a list of strings from key ``key`` of the JSON object ``data``.
+
+    If ``default`` is set to a list, a missing key results in this value being returned.
+    """
+    sentinel = object()
+    value = data.get(key, sentinel)
+    if value is sentinel:
+        if default is not None:
+            return default
+        raise ValueError(f"{key!r} is not a present")
+    if not isinstance(value, list):
+        raise ValueError(f"{key!r} is not a list, but {type(key)}")
+    if not all(isinstance(entry, str) for entry in value):
+        raise ValueError(f"{key!r} is not a list of strings")
+    return t.cast(list[str], value)
+
+
+def get_bool(
+    data: dict[str, t.Any],
+    key: str,
+    *,
+    default: bool | None = None,
+) -> bool:
+    """
+    Retrieves a boolean from key ``key`` of the JSON object ``data``.
+
+    If ``default`` is set to a boolean, a missing key results in this value being returned.
+    """
+    sentinel = object()
+    value = data.get(key, sentinel)
+    if value is sentinel:
+        if default is not None:
+            return default
+        raise ValueError(f"{key!r} is not a present")
+    if not isinstance(value, bool):
+        raise ValueError(f"{key!r} is not a bool, but {type(key)}")
+    return value