antsibull-nox 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

antsibull_nox/config.py

@@ -275,6 +275,7 @@ class SessionAnsibleTestIntegrationWDefaultContainer(_BaseModel):
     except_versions: list[PAnsibleCoreVersion] = []
     core_python_versions: dict[t.Union[PAnsibleCoreVersion, str], list[PVersion]] = {}
     controller_python_versions_only: bool = False
+    ansible_vars_from_env_vars: dict[str, str] = {}
 
     @p.model_validator(mode="after")
     def _validate_core_keys(self) -> t.Self:
@@ -340,6 +341,9 @@ class Config(_BaseModel):
     """
 
     collection_sources: dict[CollectionName, CollectionSource] = {}
+    collection_sources_per_ansible: dict[
+        PAnsibleCoreVersion, dict[CollectionName, CollectionSource]
+    ] = {}
     sessions: Sessions = Sessions()
 
 

antsibull_nox/data/action-groups.py

@@ -113,7 +113,6 @@ def scan(config: list[ActionGroup], errors: list[str]) -> None:
     modules_directory = "plugins/modules/"
     modules_suffix = ".py"
 
-    errors = []
     for file in os.listdir(modules_directory):
         if not file.endswith(modules_suffix):
             continue

antsibull_nox/data/file-yamllint.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2025, Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
+# or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Make sure all plugin and module documentation adheres to yamllint."""
+
+from __future__ import annotations
+
+import io
+import os
+import sys
+import traceback
+import typing as t
+
+from antsibull_nox_data_util import setup  # type: ignore
+from yamllint import linter
+from yamllint.cli import find_project_config_filepath
+from yamllint.config import YamlLintConfig
+from yamllint.linter import PROBLEM_LEVELS
+
+REPORT_LEVELS: set[PROBLEM_LEVELS] = {
+    "warning",
+    "error",
+}
+
+
+def lint(
+    *,
+    errors: list[dict[str, t.Any]],
+    path: str,
+    data: str,
+    config: YamlLintConfig,
+) -> None:
+    try:
+        problems = linter.run(
+            io.StringIO(data),
+            config,
+            path,
+        )
+        for problem in problems:
+            if problem.level not in REPORT_LEVELS:
+                continue
+            msg = f"{problem.level}: {problem.desc}"
+            if problem.rule:
+                msg += f"  ({problem.rule})"
+            errors.append(
+                {
+                    "path": path,
+                    "line": problem.line,
+                    "col": problem.column,
+                    "message": msg,
+                }
+            )
+    except Exception as exc:
+        error = str(exc).replace("\n", " / ")
+        errors.append(
+            {
+                "path": path,
+                "line": 1,
+                "col": 1,
+                "message": (
+                    f"Internal error while linting YAML: exception {type(exc)}:"
+                    f" {error}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+
+
+def process_yaml_file(
+    errors: list[dict[str, t.Any]],
+    path: str,
+    config: YamlLintConfig,
+) -> None:
+    try:
+        with open(path, "rt", encoding="utf-8") as stream:
+            data = stream.read()
+    except Exception as exc:
+        errors.append(
+            {
+                "path": path,
+                "line": 1,
+                "col": 1,
+                "message": (
+                    f"Error while parsing Python code: exception {type(exc)}:"
+                    f" {exc}; traceback: {traceback.format_exc()!r}"
+                ),
+            }
+        )
+        return
+
+    lint(
+        errors=errors,
+        path=path,
+        data=data,
+        config=config,
+    )
+
+
+def main() -> int:
+    """Main entry point."""
+    paths, extra_data = setup()
+    config: str | None = extra_data.get("config")
+
+    if config is None:
+        config = find_project_config_filepath()
+
+    if config:
+        yamllint_config = YamlLintConfig(file=config)
+    else:
+        yamllint_config = YamlLintConfig(content="extends: default")
+
+    errors: list[dict[str, t.Any]] = []
+    for path in paths:
+        if not os.path.isfile(path):
+            continue
+        process_yaml_file(errors, path, yamllint_config)
+
+    errors.sort(
+        key=lambda error: (error["path"], error["line"], error["col"], error["message"])
+    )
+    for error in errors:
+        prefix = f"{error['path']}:{error['line']}:{error['col']}: "
+        msg = error["message"]
+        if "note" in error:
+            msg = f"{msg}\nNote: {error['note']}"
+        for i, line in enumerate(msg.splitlines()):
+            print(f"{prefix}{line}")
+            if i == 0:
+                prefix = " " * len(prefix)
+
+    return len(errors) > 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

antsibull_nox/data/plugin-yamllint.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2024, Felix Fontein <felix@fontein.de>
+# Copyright (c) 2025, Felix Fontein <felix@fontein.de>
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
 # or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
@@ -44,6 +44,13 @@ def lint(
     config: YamlLintConfig,
     extra_for_errors: dict[str, t.Any] | None = None,
 ) -> None:
+    # If the string start with optional whitespace + linebreak, skip that line
+    idx = data.find("\n")
+    if idx >= 0 and (idx == 0 or data[:idx].isspace()):
+        data = data[idx + 1 :]
+        row_offset += 1
+        col_offset = 0
+
     try:
         problems = linter.run(
             io.StringIO(data),
@@ -84,6 +91,20 @@ def lint(
             errors[-1].update(extra_for_errors)
 
 
+def iterate_targets(
+    assignment: ast.Assign,
+) -> t.Iterable[tuple[ast.Constant, str, str]]:
+    if not isinstance(assignment.value, ast.Constant):
+        return
+    if not isinstance(assignment.value.value, str):
+        return
+    for target in assignment.targets:
+        try:
+            yield assignment.value, assignment.value.value, target.id  # type: ignore
+        except AttributeError:
+            continue
+
+
 def process_python_file(
     errors: list[dict[str, t.Any]],
     path: str,
@@ -107,33 +128,39 @@ def process_python_file(
         )
         return
 
-    # We look for top-level assignments
+    is_doc_fragment = path.startswith("plugins/doc_fragments/")
+
+    # We look for top-level assignments and classes
     for child in root.body:
+        if (
+            is_doc_fragment
+            and isinstance(child, ast.ClassDef)
+            and child.name == "ModuleDocFragment"
+        ):
+            for fragment in child.body:
+                if not isinstance(fragment, ast.Assign):
+                    continue
+                for constant, data, fragment_name in iterate_targets(fragment):
+                    lint(
+                        errors=errors,
+                        path=path,
+                        data=data,
+                        row_offset=constant.lineno - 1,
+                        col_offset=constant.col_offset - 1,
+                        section=fragment_name,
+                        config=config,
+                    )
         if not isinstance(child, ast.Assign):
             continue
-        if not isinstance(child.value, ast.Constant):
-            continue
-        if not isinstance(child.value.value, str):
-            continue
-        for target in child.targets:
-            try:
-                section = target.id  # type: ignore
-            except AttributeError:
-                continue
+        for constant, data, section in iterate_targets(child):
             if section not in ("DOCUMENTATION", "EXAMPLES", "RETURN"):
                 continue
 
-            # Extract value and offsets
-            data = child.value.value
-            row_offset = child.value.lineno - 1
-            col_offset = child.value.col_offset - 1
-
-            # If the string start with optional whitespace + linebreak, skip that line
-            idx = data.find("\n")
-            if idx >= 0 and (idx == 0 or data[:idx].isspace()):
-                data = data[idx + 1 :]
-                row_offset += 1
-                col_offset = 0
+            # Handle special values
+            if data in ("#", " # ") and section == "RETURN":
+                # Not skipping it here could result in all kind of linting errors,
+                # like no document start, or trailing space.
+                continue
 
             # Check for non-YAML examples
             if section == EXAMPLES_SECTION:
@@ -146,8 +173,8 @@ def process_python_file(
                 errors=errors,
                 path=path,
                 data=data,
-                row_offset=row_offset,
-                col_offset=col_offset,
+                row_offset=constant.lineno - 1,
+                col_offset=constant.col_offset - 1,
                 section=section,
                 config=config_examples if section == EXAMPLES_SECTION else config,
             )

antsibull_nox/interpret_config.py

@@ -46,6 +46,18 @@ def _interpret_config(config: Config) -> None:
                 for name, source in config.collection_sources.items()
             }
         )
+    if config.collection_sources_per_ansible:
+        for (
+            ansible_core_version,
+            collection_sources,
+        ) in config.collection_sources_per_ansible.items():
+            setup_collection_sources(
+                {
+                    name: CollectionSource(name=name, source=source.source)
+                    for name, source in collection_sources.items()
+                },
+                ansible_core_version=ansible_core_version,
+            )
 
 
 def _convert_action_groups(
@@ -226,6 +238,7 @@ def _add_sessions(sessions: Sessions) -> None:
                 cfg.core_python_versions
             ),
             controller_python_versions_only=cfg.controller_python_versions_only,
+            ansible_vars_from_env_vars=cfg.ansible_vars_from_env_vars,
         )
     if sessions.ansible_lint:
         add_ansible_lint(

antsibull_nox/sessions/ansible_lint.py

@@ -44,6 +44,8 @@ def add_ansible_lint(
         command = ["ansible-lint", "--offline"]
         if strict:
             command.append("--strict")
+        if session.posargs:
+            command.extend(session.posargs)
         session.run(*command, env=env)
 
     ansible_lint.__doc__ = "Run ansible-lint."

antsibull_nox/sessions/ansible_test.py

@@ -16,12 +16,14 @@ from collections.abc import Callable
 from pathlib import Path
 
 import nox
+from antsibull_fileutils.yaml import store_yaml_file
 
 from ..ansible import (
     AnsibleCoreVersion,
     get_ansible_core_info,
     get_ansible_core_package_name,
     get_supported_core_versions,
+    parse_ansible_core_version,
 )
 from ..paths import copy_directory_tree_into
 from ..python import get_installed_python_versions
@@ -33,17 +35,6 @@ from .utils import (
 )
 
 
-def _parse_ansible_core_version(
-    version: str | AnsibleCoreVersion,
-) -> AnsibleCoreVersion:
-    if version in ("devel", "milestone"):
-        # For some reason mypy doesn't notice that
-        return t.cast(AnsibleCoreVersion, version)
-    if isinstance(version, Version):
-        return version
-    return Version.parse(version)
-
-
 def add_ansible_test_session(
     *,
     name: str,
@@ -67,7 +58,7 @@ def add_ansible_test_session(
 
     Returns a list of Python versions set for this session.
     """
-    parsed_ansible_core_version = _parse_ansible_core_version(ansible_core_version)
+    parsed_ansible_core_version = parse_ansible_core_version(ansible_core_version)
 
     def compose_dependencies() -> list[str]:
         deps = [
@@ -84,6 +75,7 @@ def add_ansible_test_session(
         install(session, *compose_dependencies())
         prepared_collections = prepare_collections(
             session,
+            ansible_core_version=parsed_ansible_core_version,
             install_in_site_packages=False,
             extra_deps_files=extra_deps_files,
             install_out_of_tree=True,
@@ -177,7 +169,7 @@ def add_ansible_test_sanity_test_session(
     """
     Add generic ansible-test sanity test session.
     """
-    command = ["sanity", "--docker", "-v", "--color"]
+    command = ["sanity", "--color", "-v", "--docker"]
     if skip_tests:
         for test in skip_tests:
             command.extend(["--skip", test])
@@ -210,7 +202,7 @@ def _parse_min_max_except(
         max_version = Version.parse(max_version)
     if except_versions is None:
         return min_version, max_version, None
-    evs = tuple(_parse_ansible_core_version(version) for version in except_versions)
+    evs = tuple(parse_ansible_core_version(version) for version in except_versions)
     return min_version, max_version, evs
 
 
@@ -311,7 +303,7 @@ def add_ansible_test_unit_test_session(
     add_ansible_test_session(
         name=name,
         description=description,
-        ansible_test_params=["units", "--docker", "-v", "--color"],
+        ansible_test_params=["units", "--color", "-v", "--docker"],
         extra_deps_files=["tests/unit/requirements.yml"],
         default=default,
         ansible_core_version=ansible_core_version,
@@ -406,6 +398,7 @@ def add_ansible_test_integration_sessions_default_container(
         dict[str | AnsibleCoreVersion, list[str | Version]] | None
     ) = None,
     controller_python_versions_only: bool = False,
+    ansible_vars_from_env_vars: dict[str, str] | None = None,
     default: bool = False,
 ) -> None:
     """
@@ -418,6 +411,18 @@ def add_ansible_test_integration_sessions_default_container(
     controller Python versions.
     """
 
+    def callback_before() -> None:
+        if not ansible_vars_from_env_vars:
+            return
+
+        path = Path("tests", "integration", "integration_config.yml")
+        content: dict[str, t.Any] = {}
+        for ans_var, env_var in ansible_vars_from_env_vars.items():
+            value = os.environ.get(env_var)
+            if value is not None:
+                content[ans_var] = env_var
+        store_yaml_file(path, content, nice=True, sort_keys=True)
+
     def add_integration_tests(
         ansible_core_version: AnsibleCoreVersion,
         repo_name: str | None = None,
@@ -468,10 +473,10 @@ def add_ansible_test_integration_sessions_default_container(
                 description=description,
                 ansible_test_params=[
                     "integration",
+                    "--color",
+                    "-v",
                     "--docker",
                     "default",
-                    "-v",
-                    "--color",
                     "--python",
                     str(py_version),
                 ],
@@ -479,6 +484,7 @@ def add_ansible_test_integration_sessions_default_container(
                 ansible_core_version=ansible_core_version,
                 ansible_core_repo_name=repo_name,
                 ansible_core_branch_name=branch_name,
+                callback_before=callback_before,
                 default=False,
                 register_name="integration",
                 register_extra_data={

antsibull_nox/sessions/collections.py

@@ -18,6 +18,7 @@ from pathlib import Path
 
 import nox
 
+from ..ansible import AnsibleCoreVersion, parse_ansible_core_version
 from ..collection import (
     CollectionData,
     setup_collections,
@@ -76,6 +77,7 @@ def _run_subprocess(args: list[str]) -> tuple[bytes, bytes]:
 def prepare_collections(
     session: nox.Session,
     *,
+    ansible_core_version: AnsibleCoreVersion | str | None = None,
     install_in_site_packages: bool,
     extra_deps_files: list[str | os.PathLike] | None = None,
     extra_collections: list[str] | None = None,
@@ -84,6 +86,11 @@ def prepare_collections(
     """
     Install collections in site-packages.
     """
+    parsed_ansible_core_version = (
+        parse_ansible_core_version(ansible_core_version)
+        if ansible_core_version is not None
+        else "devel"
+    )
     if install_out_of_tree and install_in_site_packages:
         raise ValueError(
             "install_out_of_tree=True cannot be combined with install_in_site_packages=True"
@@ -116,6 +123,7 @@ def prepare_collections(
     setup = setup_collections(
         place,
         _run_subprocess,
+        ansible_core_version=parsed_ansible_core_version,
         extra_deps_files=extra_deps_files,
         extra_collections=extra_collections,
         with_current=False,

antsibull_nox/sessions/lint.py

@@ -10,6 +10,7 @@ Create nox lint sessions.
 
 from __future__ import annotations
 
+import json
 import os
 import shlex
 from pathlib import Path
@@ -29,6 +30,7 @@ from .utils import (
     compose_description,
     install,
     run_bare_script,
+    silence_run_verbosity,
 )
 
 CODE_FILES = [
@@ -189,6 +191,28 @@ def add_formatters(
     nox.session(name="formatters", default=False)(formatters)
 
 
+def process_pylint_errors(
+    session: nox.Session,
+    prepared_collections: CollectionSetup,
+    output: str,
+) -> None:
+    """
+    Process errors reported by pylint in 'json2' format.
+    """
+    data = json.loads(output)
+    found_error = False
+    if data["messages"]:
+        for message in data["messages"]:
+            path = os.path.relpath(
+                message["absolutePath"], prepared_collections.current_path
+            )
+            prefix = f"{path}:{message['line']}:{message['column']}: [{message['messageId']}]"
+            print(f"{prefix} {message['message']} [{message['symbol']}]")
+            found_error = True
+    if found_error:
+        session.error("Pylint failed")
+
+
 def add_codeqa(  # noqa: C901
     *,
     extra_code_files: list[str],
@@ -249,9 +273,17 @@ def add_codeqa(  # noqa: C901
                 ]
             )
         command.extend(["--source-roots", "."])
+        command.extend(["--output-format", "json2"])
         command.extend(session.posargs)
         command.extend(prepared_collections.prefix_current_paths(paths))
-        session.run(*command)
+        with silence_run_verbosity():
+            # Exit code is OR of some of 1, 2, 4, 8, 16
+            output = session.run(
+                *command, silent=True, success_codes=list(range(0, 32))
+            )
+
+        if output:
+            process_pylint_errors(session, prepared_collections, output)
 
     def execute_pylint(
         session: nox.Session, prepared_collections: CollectionSetup
@@ -335,29 +367,22 @@ def add_yamllint(
     def execute_yamllint(session: nox.Session) -> None:
         # Run yamllint
         all_files = list_all_files()
-        cwd = Path.cwd()
         all_yaml_filenames = [
-            str(file.relative_to(cwd))
-            for file in all_files
-            if file.name.lower().endswith((".yml", ".yaml"))
+            file for file in all_files if file.name.lower().endswith((".yml", ".yaml"))
         ]
         if not all_yaml_filenames:
             session.warn("Skipping yamllint since no YAML file was found...")
             return
 
-        command = ["yamllint"]
-        if yamllint_config is not None:
-            command.extend(
-                [
-                    "-c",
-                    str(yamllint_config),
-                ]
-            )
-        command.append("--strict")
-        command.append("--")
-        command.extend(all_yaml_filenames)
-        command.extend(session.posargs)
-        session.run(*command)
+        run_bare_script(
+            session,
+            "file-yamllint",
+            use_session_python=True,
+            files=all_yaml_filenames,
+            extra_data={
+                "config": to_str(yamllint_config),
+            },
+        )
 
     def execute_plugin_yamllint(session: nox.Session) -> None:
         # Run yamllint
@@ -415,6 +440,40 @@ def add_yamllint(
     nox.session(name="yamllint", default=False)(yamllint)
 
 
+def process_mypy_errors(
+    session: nox.Session,
+    prepared_collections: CollectionSetup,
+    output: str,
+) -> None:
+    """
+    Process errors reported by mypy in 'json' format.
+    """
+    found_error = False
+    for line in output.splitlines():
+        if not line.strip():
+            continue
+        try:
+            data = json.loads(line)
+            path = os.path.relpath(
+                prepared_collections.current_place / data["file"],
+                prepared_collections.current_path,
+            )
+            prefix = f"{path}:{data['line']}:{data['column']}: [{data['severity']}]"
+            if data["code"]:
+                print(f"{prefix} {data['message']} [{data['code']}]")
+            else:
+                print(f"{prefix} {data['message']}")
+            if data["hint"]:
+                prefix = " " * len(prefix)
+                for hint in data["hint"].splitlines():
+                    print(f"{prefix} {hint}")
+        except Exception:  # pylint: disable=broad-exception-caught
+            session.warn(f"Cannot parse mypy output: {line}")
+        found_error = True
+    if found_error:
+        session.error("Type checking failed")
+
+
 def add_typing(
     *,
     extra_code_files: list[str],
@@ -459,13 +518,21 @@ def add_typing(
                 )
             command.append("--namespace-packages")
             command.append("--explicit-package-bases")
+            command.extend(["--output", "json"])
             command.extend(session.posargs)
             command.extend(
                 prepared_collections.prefix_current_paths(CODE_FILES + extra_code_files)
             )
-            session.run(
-                *command, env={"MYPYPATH": str(prepared_collections.current_place)}
-            )
+            with silence_run_verbosity():
+                output = session.run(
+                    *command,
+                    env={"MYPYPATH": str(prepared_collections.current_place)},
+                    silent=True,
+                    success_codes=(0, 1, 2),
+                )
+
+            if output:
+                process_mypy_errors(session, prepared_collections, output)
 
     def typing(session: nox.Session) -> None:
         install(session, *compose_dependencies())

{antsibull_nox-0.3.0.dist-info → antsibull_nox-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: antsibull-nox
-Version: 0.3.0
+Version: 0.5.0
 Summary: Changelog tool for Ansible-core and Ansible collections
 Project-URL: Documentation, https://ansible.readthedocs.io/projects/antsibull-nox/
 Project-URL: Source code, https://github.com/ansible-community/antsibull-nox/