antsibull-nox 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- antsibull_nox/__init__.py +1 -1
- antsibull_nox/ansible.py +15 -0
- antsibull_nox/collection/data.py +12 -0
- antsibull_nox/collection/install.py +194 -79
- antsibull_nox/collection/search.py +128 -30
- antsibull_nox/config.py +3 -0
- antsibull_nox/data/file-yamllint.py +138 -0
- antsibull_nox/data/plugin-yamllint.py +51 -24
- antsibull_nox/interpret_config.py +12 -0
- antsibull_nox/sessions/ansible_lint.py +2 -0
- antsibull_nox/sessions/ansible_test.py +4 -13
- antsibull_nox/sessions/collections.py +8 -0
- antsibull_nox/sessions/lint.py +88 -21
- {antsibull_nox-0.3.0.dist-info → antsibull_nox-0.4.0.dist-info}/METADATA +1 -1
- {antsibull_nox-0.3.0.dist-info → antsibull_nox-0.4.0.dist-info}/RECORD +18 -17
- {antsibull_nox-0.3.0.dist-info → antsibull_nox-0.4.0.dist-info}/WHEEL +0 -0
- {antsibull_nox-0.3.0.dist-info → antsibull_nox-0.4.0.dist-info}/entry_points.txt +0 -0
- {antsibull_nox-0.3.0.dist-info → antsibull_nox-0.4.0.dist-info}/licenses/LICENSES/GPL-3.0-or-later.txt +0 -0
antsibull_nox/__init__.py
CHANGED
antsibull_nox/ansible.py
CHANGED
@@ -253,8 +253,23 @@ def get_supported_core_versions(
|
|
253
253
|
return result
|
254
254
|
|
255
255
|
|
256
|
+
def parse_ansible_core_version(
|
257
|
+
version: str | AnsibleCoreVersion,
|
258
|
+
) -> AnsibleCoreVersion:
|
259
|
+
"""
|
260
|
+
Coerce a string or a AnsibleCoreVersion to a AnsibleCoreVersion.
|
261
|
+
"""
|
262
|
+
if version in ("devel", "milestone"):
|
263
|
+
# For some reason mypy doesn't notice that
|
264
|
+
return t.cast(AnsibleCoreVersion, version)
|
265
|
+
if isinstance(version, Version):
|
266
|
+
return version
|
267
|
+
return Version.parse(version)
|
268
|
+
|
269
|
+
|
256
270
|
__all__ = [
|
257
271
|
"AnsibleCoreInfo",
|
258
272
|
"get_ansible_core_info",
|
259
273
|
"get_ansible_core_package_name",
|
274
|
+
"parse_ansible_core_version",
|
260
275
|
]
|
antsibull_nox/collection/data.py
CHANGED
@@ -10,6 +10,8 @@ Data types for collections.
|
|
10
10
|
|
11
11
|
from __future__ import annotations
|
12
12
|
|
13
|
+
import base64
|
14
|
+
import hashlib
|
13
15
|
from dataclasses import dataclass
|
14
16
|
from pathlib import Path
|
15
17
|
|
@@ -98,6 +100,16 @@ class CollectionSource:
|
|
98
100
|
)
|
99
101
|
return source
|
100
102
|
|
103
|
+
def identifier(self) -> str:
|
104
|
+
"""
|
105
|
+
Compute a source identifier.
|
106
|
+
"""
|
107
|
+
hasher = hashlib.sha256()
|
108
|
+
hasher.update(self.name.encode("utf-8"))
|
109
|
+
hasher.update(b"::")
|
110
|
+
hasher.update(self.source.encode("utf-8"))
|
111
|
+
return base64.b32encode(hasher.digest())[:16].decode("ascii")
|
112
|
+
|
101
113
|
|
102
114
|
__all__ = [
|
103
115
|
"CollectionData",
|
@@ -20,6 +20,7 @@ from pathlib import Path
|
|
20
20
|
|
21
21
|
from antsibull_fileutils.yaml import load_yaml_file
|
22
22
|
|
23
|
+
from ..ansible import AnsibleCoreVersion
|
23
24
|
from ..paths import copy_collection as _paths_copy_collection
|
24
25
|
from ..paths import remove_path as _remove
|
25
26
|
from .data import CollectionData, CollectionSource, SetupResult
|
@@ -47,84 +48,170 @@ class _CollectionSources:
|
|
47
48
|
"""
|
48
49
|
self.sources[name] = source
|
49
50
|
|
50
|
-
|
51
|
+
@t.overload
|
52
|
+
def get_source(
|
53
|
+
self, name: str, *, create_default: t.Literal[True] = True
|
54
|
+
) -> CollectionSource: ...
|
55
|
+
|
56
|
+
@t.overload
|
57
|
+
def get_source(
|
58
|
+
self, name: str, *, create_default: t.Literal[False]
|
59
|
+
) -> CollectionSource | None: ...
|
60
|
+
|
61
|
+
def get_source(
|
62
|
+
self, name: str, *, create_default: bool = True
|
63
|
+
) -> CollectionSource | None:
|
51
64
|
"""
|
52
65
|
Get source for collection.
|
53
66
|
"""
|
54
67
|
source = self.sources.get(name)
|
55
|
-
if source is None:
|
68
|
+
if source is None and create_default:
|
56
69
|
source = CollectionSource(name, name)
|
57
70
|
return source
|
58
71
|
|
59
72
|
|
73
|
+
class _CollectionDownloadCache:
|
74
|
+
@staticmethod
|
75
|
+
def _parse_galaxy_filename(file: Path) -> tuple[str, str, str]:
|
76
|
+
"""
|
77
|
+
Split filename into (namespace, name, version) tuple.
|
78
|
+
"""
|
79
|
+
if not file.name.endswith(_TARBALL_EXTENSION):
|
80
|
+
raise ValueError(
|
81
|
+
f"Filename {file.name!r} does not end with {_TARBALL_EXTENSION}"
|
82
|
+
)
|
83
|
+
parts = file.name[: -len(_TARBALL_EXTENSION)].split("-", 2)
|
84
|
+
if len(parts) != 3:
|
85
|
+
raise ValueError(
|
86
|
+
f"Filename {file.name!r} does not belong to a Galaxy tarball"
|
87
|
+
)
|
88
|
+
return parts[0], parts[1], parts[2]
|
89
|
+
|
90
|
+
@staticmethod
|
91
|
+
def _parse_cache_filename(file: Path) -> tuple[str, str, str, str] | None:
|
92
|
+
"""
|
93
|
+
Split cache filename into (namespace, name, source_id, version) tuple.
|
94
|
+
"""
|
95
|
+
if not file.name.endswith(_TARBALL_EXTENSION):
|
96
|
+
return None
|
97
|
+
parts = file.name[: -len(_TARBALL_EXTENSION)].split("-", 3)
|
98
|
+
if len(parts) != 4:
|
99
|
+
return None
|
100
|
+
return parts[0], parts[1], parts[2], parts[3]
|
101
|
+
|
102
|
+
@staticmethod
|
103
|
+
def _encode_cache_filename(
|
104
|
+
namespace: str, name: str, version: str, source: CollectionSource
|
105
|
+
) -> str:
|
106
|
+
return f"{namespace}-{name}-{source.identifier()}-{version}{_TARBALL_EXTENSION}"
|
107
|
+
|
108
|
+
def download_collections(
|
109
|
+
self, *, destination: Path, sources: list[CollectionSource], runner: Runner
|
110
|
+
) -> None:
|
111
|
+
"""
|
112
|
+
Given a set of collection sources, downloads these and stores them in destination.
|
113
|
+
"""
|
114
|
+
destination.mkdir(exist_ok=True)
|
115
|
+
names = ", ".join(sorted(source.name for source in sources))
|
116
|
+
print(f"Downloading {names} to {destination}...")
|
117
|
+
sources_by_name = {}
|
118
|
+
for source in sources:
|
119
|
+
sources_by_name[source.name] = source
|
120
|
+
if source.name != source.source:
|
121
|
+
print(f" Installing {source.name} via {source.source}...")
|
122
|
+
with tempfile.TemporaryDirectory(
|
123
|
+
prefix="antsibull-nox-galaxy-download"
|
124
|
+
) as dest:
|
125
|
+
tempdir = Path(dest)
|
126
|
+
command = [
|
127
|
+
"ansible-galaxy",
|
128
|
+
"collection",
|
129
|
+
"download",
|
130
|
+
"--no-deps",
|
131
|
+
"--download-path",
|
132
|
+
str(tempdir),
|
133
|
+
"--",
|
134
|
+
*(source.source for source in sources),
|
135
|
+
]
|
136
|
+
runner(command)
|
137
|
+
for file in tempdir.iterdir():
|
138
|
+
if file.name.endswith(_TARBALL_EXTENSION) and file.is_file():
|
139
|
+
namespace, name, version = self._parse_galaxy_filename(file)
|
140
|
+
source_opt = sources_by_name.get(f"{namespace}.{name}")
|
141
|
+
if source_opt is None:
|
142
|
+
print(
|
143
|
+
f"Found unknown collection artifact {file.name!r}, ignoring..."
|
144
|
+
)
|
145
|
+
continue
|
146
|
+
destfile = destination / self._encode_cache_filename(
|
147
|
+
namespace, name, version, source_opt
|
148
|
+
)
|
149
|
+
_remove(destfile)
|
150
|
+
shutil.move(file, destfile)
|
151
|
+
|
152
|
+
def list_downloaded_dir(
|
153
|
+
self, *, path: Path
|
154
|
+
) -> dict[tuple[str, str], tuple[Path, str]]:
|
155
|
+
"""
|
156
|
+
List contents of download cache.
|
157
|
+
|
158
|
+
Returns a dictionary mapping (collection_name, source_id) tuples to
|
159
|
+
(tarball_path, version) tuples.
|
160
|
+
"""
|
161
|
+
if not path.is_dir():
|
162
|
+
return {}
|
163
|
+
result: dict[tuple[str, str], tuple[Path, str]] = {}
|
164
|
+
for file in path.iterdir():
|
165
|
+
if not file.is_file():
|
166
|
+
continue
|
167
|
+
parsed = self._parse_cache_filename(file)
|
168
|
+
if parsed is None:
|
169
|
+
continue
|
170
|
+
namespace, name, source_id, version = parsed
|
171
|
+
collection_name = f"{namespace}.{name}"
|
172
|
+
key = collection_name, source_id
|
173
|
+
if key in result:
|
174
|
+
# Determine older entry
|
175
|
+
old_file = result[key][0]
|
176
|
+
old_stat = old_file.stat()
|
177
|
+
new_stat = file.stat()
|
178
|
+
if new_stat.st_mtime > old_stat.st_mtime:
|
179
|
+
older_file = old_file
|
180
|
+
result[key] = file, version
|
181
|
+
else:
|
182
|
+
older_file = file
|
183
|
+
# Clean up older entry
|
184
|
+
_remove(older_file)
|
185
|
+
else:
|
186
|
+
result[key] = file, version
|
187
|
+
return result
|
188
|
+
|
189
|
+
|
60
190
|
_COLLECTION_SOURCES = _CollectionSources()
|
191
|
+
_COLLECTION_SOURCES_PER_CORE_VERSION: dict[AnsibleCoreVersion, _CollectionSources] = {}
|
192
|
+
_COLLECTION_DOWNLOAD_CACHE = _CollectionDownloadCache()
|
61
193
|
_TARBALL_EXTENSION = ".tar.gz"
|
62
194
|
_INSTALLATION_CONFIG_ENV_VAR = "ANTSIBULL_NOX_INSTALL_COLLECTIONS"
|
63
195
|
|
64
196
|
|
65
|
-
def setup_collection_sources(
|
197
|
+
def setup_collection_sources(
|
198
|
+
collection_sources: dict[str, CollectionSource],
|
199
|
+
*,
|
200
|
+
ansible_core_version: AnsibleCoreVersion | None = None,
|
201
|
+
) -> None:
|
66
202
|
"""
|
67
203
|
Setup collection sources.
|
68
204
|
"""
|
205
|
+
sources = _COLLECTION_SOURCES
|
206
|
+
if ansible_core_version is not None:
|
207
|
+
sources = _COLLECTION_SOURCES_PER_CORE_VERSION.get(
|
208
|
+
ansible_core_version, _COLLECTION_SOURCES
|
209
|
+
)
|
210
|
+
if sources is _COLLECTION_SOURCES:
|
211
|
+
sources = _CollectionSources()
|
212
|
+
_COLLECTION_SOURCES_PER_CORE_VERSION[ansible_core_version] = sources
|
69
213
|
for name, source in collection_sources.items():
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
def _download_collections(
|
74
|
-
*, destination: Path, sources: list[CollectionSource], runner: Runner
|
75
|
-
) -> None:
|
76
|
-
destination.mkdir(exist_ok=True)
|
77
|
-
names = ", ".join(sorted(source.name for source in sources))
|
78
|
-
print(f"Downloading {names} to {destination}...")
|
79
|
-
for source in sources:
|
80
|
-
if source.name != source.source:
|
81
|
-
print(f" Installing {source.name} via {source.source}...")
|
82
|
-
with tempfile.TemporaryDirectory(prefix="antsibull-nox-galaxy-download") as dest:
|
83
|
-
tempdir = Path(dest)
|
84
|
-
command = [
|
85
|
-
"ansible-galaxy",
|
86
|
-
"collection",
|
87
|
-
"download",
|
88
|
-
"--no-deps",
|
89
|
-
"--download-path",
|
90
|
-
str(tempdir),
|
91
|
-
"--",
|
92
|
-
*(source.source for source in sources),
|
93
|
-
]
|
94
|
-
runner(command)
|
95
|
-
for file in tempdir.iterdir():
|
96
|
-
if file.name.endswith(_TARBALL_EXTENSION) and file.is_file():
|
97
|
-
destfile = destination / file.name
|
98
|
-
_remove(destfile)
|
99
|
-
shutil.move(file, destfile)
|
100
|
-
|
101
|
-
|
102
|
-
def _list_downloaded_dir(*, path: Path) -> dict[str, Path]:
|
103
|
-
if not path.is_dir():
|
104
|
-
return {}
|
105
|
-
result: dict[str, Path] = {}
|
106
|
-
for file in path.iterdir():
|
107
|
-
if not file.name.endswith(_TARBALL_EXTENSION) or not file.is_file():
|
108
|
-
continue
|
109
|
-
basename = file.name[: -len(_TARBALL_EXTENSION)]
|
110
|
-
# Format: community-internal_test_tools-0.15.0, community-aws-10.0.0-dev0
|
111
|
-
parts = basename.split("-", 2)
|
112
|
-
if len(parts) != 3:
|
113
|
-
continue
|
114
|
-
full_name = ".".join(parts[:2])
|
115
|
-
if full_name in result:
|
116
|
-
old_stat = result[full_name].stat()
|
117
|
-
new_stat = file.stat()
|
118
|
-
if new_stat.st_mtime > old_stat.st_mtime:
|
119
|
-
older_file = result[full_name]
|
120
|
-
result[full_name] = file
|
121
|
-
else:
|
122
|
-
older_file = file
|
123
|
-
# Clean up older entry
|
124
|
-
_remove(older_file)
|
125
|
-
else:
|
126
|
-
result[full_name] = file
|
127
|
-
return result
|
214
|
+
sources.set_source(name, source)
|
128
215
|
|
129
216
|
|
130
217
|
def _install_from_download_cache(
|
@@ -137,9 +224,21 @@ def _install_from_download_cache(
|
|
137
224
|
return destination_dir
|
138
225
|
|
139
226
|
|
227
|
+
def _get_source(
|
228
|
+
name: str, *, ansible_core_version: AnsibleCoreVersion
|
229
|
+
) -> CollectionSource:
|
230
|
+
sources_per_version = _COLLECTION_SOURCES_PER_CORE_VERSION.get(ansible_core_version)
|
231
|
+
if sources_per_version:
|
232
|
+
result = sources_per_version.get_source(name, create_default=False)
|
233
|
+
if result is not None:
|
234
|
+
return result
|
235
|
+
return _COLLECTION_SOURCES.get_source(name)
|
236
|
+
|
237
|
+
|
140
238
|
def _install_missing(
|
141
239
|
collections: list[str],
|
142
240
|
*,
|
241
|
+
ansible_core_version: AnsibleCoreVersion,
|
143
242
|
runner: Runner,
|
144
243
|
) -> list[CollectionData]:
|
145
244
|
config = os.environ.get(_INSTALLATION_CONFIG_ENV_VAR)
|
@@ -151,38 +250,49 @@ def _install_missing(
|
|
151
250
|
f" thus cannot install missing exception{plural_s} {names}..."
|
152
251
|
)
|
153
252
|
return []
|
154
|
-
sources = [
|
253
|
+
sources = [
|
254
|
+
_get_source(name, ansible_core_version=ansible_core_version)
|
255
|
+
for name in collections
|
256
|
+
]
|
155
257
|
result: list[CollectionData] = []
|
156
|
-
with _update_collection_list() as updater:
|
258
|
+
with _update_collection_list(ansible_core_version=ansible_core_version) as updater:
|
157
259
|
global_cache = updater.get_global_cache()
|
158
|
-
install: list[
|
260
|
+
install: list[CollectionSource] = []
|
159
261
|
download: list[CollectionSource] = []
|
160
|
-
download_cache =
|
262
|
+
download_cache = _COLLECTION_DOWNLOAD_CACHE.list_downloaded_dir(
|
263
|
+
path=global_cache.download_cache
|
264
|
+
)
|
161
265
|
for source in sources:
|
162
266
|
if cd := updater.find(source.name):
|
163
267
|
result.append(cd)
|
164
268
|
else:
|
165
|
-
install.append(source
|
166
|
-
if not download_cache.get(source.name):
|
269
|
+
install.append(source)
|
270
|
+
if not download_cache.get((source.name, source.identifier())):
|
167
271
|
download.append(source)
|
168
272
|
if download:
|
169
|
-
|
273
|
+
_COLLECTION_DOWNLOAD_CACHE.download_collections(
|
170
274
|
destination=global_cache.download_cache, sources=download, runner=runner
|
171
275
|
)
|
172
|
-
download_cache =
|
276
|
+
download_cache = _COLLECTION_DOWNLOAD_CACHE.list_downloaded_dir(
|
277
|
+
path=global_cache.download_cache
|
278
|
+
)
|
173
279
|
if install:
|
174
|
-
for
|
175
|
-
|
280
|
+
for source in install:
|
281
|
+
key = source.name, source.identifier()
|
282
|
+
if key not in download_cache:
|
176
283
|
raise ValueError(
|
177
|
-
f"Error: cannot find {name}
|
178
|
-
f" {global_cache.download_cache}
|
284
|
+
f"Error: cannot find {source.name} (source ID {source.identifier()})"
|
285
|
+
f" in download cache {global_cache.download_cache}"
|
286
|
+
" after successful download!"
|
179
287
|
)
|
180
288
|
c_dir = _install_from_download_cache(
|
181
|
-
full_name=name,
|
182
|
-
tarball=download_cache[
|
183
|
-
destination=global_cache.
|
289
|
+
full_name=source.name,
|
290
|
+
tarball=download_cache[key][0],
|
291
|
+
destination=global_cache.get_extracted_path(
|
292
|
+
ansible_core_version=ansible_core_version
|
293
|
+
),
|
184
294
|
)
|
185
|
-
c_namespace, c_name = name.split(".", 1)
|
295
|
+
c_namespace, c_name = source.name.split(".", 1)
|
186
296
|
result.append(
|
187
297
|
updater.add_collection(
|
188
298
|
directory=c_dir, namespace=c_namespace, name=c_name
|
@@ -414,6 +524,7 @@ def setup_collections(
|
|
414
524
|
destination: str | os.PathLike,
|
415
525
|
runner: Runner,
|
416
526
|
*,
|
527
|
+
ansible_core_version: AnsibleCoreVersion,
|
417
528
|
extra_collections: list[str] | None = None,
|
418
529
|
extra_deps_files: list[str | os.PathLike] | None = None,
|
419
530
|
global_cache_dir: Path,
|
@@ -423,7 +534,9 @@ def setup_collections(
|
|
423
534
|
Setup all collections in a tree structure inside the destination directory.
|
424
535
|
"""
|
425
536
|
all_collections = get_collection_list(
|
426
|
-
runner=runner,
|
537
|
+
runner=runner,
|
538
|
+
global_cache_dir=global_cache_dir,
|
539
|
+
ansible_core_version=ansible_core_version,
|
427
540
|
)
|
428
541
|
destination_root = Path(destination) / "ansible_collections"
|
429
542
|
destination_root.mkdir(exist_ok=True)
|
@@ -451,7 +564,9 @@ def setup_collections(
|
|
451
564
|
if missing.is_empty():
|
452
565
|
break
|
453
566
|
for collection_data in _install_missing(
|
454
|
-
missing.get_missing_names(),
|
567
|
+
missing.get_missing_names(),
|
568
|
+
ansible_core_version=ansible_core_version,
|
569
|
+
runner=runner,
|
455
570
|
):
|
456
571
|
collections_to_install[collection_data.full_name] = collection_data
|
457
572
|
missing.remove(collection_data.full_name)
|
@@ -21,6 +21,7 @@ from pathlib import Path
|
|
21
21
|
|
22
22
|
from antsibull_fileutils.yaml import load_yaml_file
|
23
23
|
|
24
|
+
from ..ansible import AnsibleCoreVersion
|
24
25
|
from .data import CollectionData
|
25
26
|
|
26
27
|
# Function that runs a command (and fails on non-zero return code)
|
@@ -49,6 +50,12 @@ class _GlobalCache:
|
|
49
50
|
extracted_cache=root / "extracted",
|
50
51
|
)
|
51
52
|
|
53
|
+
def get_extracted_path(self, *, ansible_core_version: AnsibleCoreVersion) -> Path:
|
54
|
+
"""
|
55
|
+
Given an ansible-core version, returns its extracted collection cache directory.
|
56
|
+
"""
|
57
|
+
return self.extracted_cache / str(ansible_core_version)
|
58
|
+
|
52
59
|
|
53
60
|
def _load_galaxy_yml(galaxy_yml: Path) -> dict[str, t.Any]:
|
54
61
|
try:
|
@@ -279,9 +286,9 @@ class CollectionList:
|
|
279
286
|
)
|
280
287
|
|
281
288
|
@classmethod
|
282
|
-
def
|
289
|
+
def collect_global(cls, *, runner: Runner) -> CollectionList:
|
283
290
|
"""
|
284
|
-
Search for a list of collections. The result is not cached.
|
291
|
+
Search for a global list of collections. The result is not cached.
|
285
292
|
"""
|
286
293
|
found_collections = {}
|
287
294
|
for collection_data in _fs_list_local_collections():
|
@@ -291,7 +298,25 @@ class CollectionList:
|
|
291
298
|
# Similar to Ansible, we use the first match
|
292
299
|
if collection_data.full_name not in found_collections:
|
293
300
|
found_collections[collection_data.full_name] = collection_data
|
294
|
-
|
301
|
+
return cls.create(found_collections)
|
302
|
+
|
303
|
+
@classmethod
|
304
|
+
def collect_local(
|
305
|
+
cls,
|
306
|
+
*,
|
307
|
+
ansible_core_version: AnsibleCoreVersion,
|
308
|
+
global_cache: _GlobalCache,
|
309
|
+
current: CollectionData,
|
310
|
+
) -> CollectionList:
|
311
|
+
"""
|
312
|
+
Search for a list of collections from a local cache path. The result is not cached.
|
313
|
+
"""
|
314
|
+
found_collections = {
|
315
|
+
current.full_name: current,
|
316
|
+
}
|
317
|
+
for collection_data in _fs_list_global_cache(
|
318
|
+
global_cache.get_extracted_path(ansible_core_version=ansible_core_version)
|
319
|
+
):
|
295
320
|
# Similar to Ansible, we use the first match
|
296
321
|
if collection_data.full_name not in found_collections:
|
297
322
|
found_collections[collection_data.full_name] = collection_data
|
@@ -313,6 +338,18 @@ class CollectionList:
|
|
313
338
|
current=self.current,
|
314
339
|
)
|
315
340
|
|
341
|
+
def merge_with(self, other: CollectionList) -> CollectionList:
|
342
|
+
"""
|
343
|
+
Merge this collection list with another (local) one.
|
344
|
+
"""
|
345
|
+
result = dict(self.collection_map)
|
346
|
+
for collection in other.collections:
|
347
|
+
# Similar to Ansible, we use the first match.
|
348
|
+
# For merge, this means we prefer self over other.
|
349
|
+
if collection.full_name not in result:
|
350
|
+
result[collection.full_name] = collection
|
351
|
+
return CollectionList.create(result)
|
352
|
+
|
316
353
|
def _add(self, collection: CollectionData, *, force: bool = True) -> bool:
|
317
354
|
if not force and collection.full_name in self.collection_map:
|
318
355
|
return False
|
@@ -323,16 +360,23 @@ class CollectionList:
|
|
323
360
|
|
324
361
|
class _CollectionListUpdater:
|
325
362
|
def __init__(
|
326
|
-
self,
|
363
|
+
self,
|
364
|
+
*,
|
365
|
+
owner: "_CollectionListSingleton",
|
366
|
+
merged_collection_list: CollectionList,
|
367
|
+
local_collection_list: CollectionList,
|
368
|
+
ansible_core_version: AnsibleCoreVersion,
|
327
369
|
) -> None:
|
328
370
|
self._owner = owner
|
329
|
-
self.
|
371
|
+
self._merged_collection_list = merged_collection_list
|
372
|
+
self._local_collection_list = local_collection_list
|
373
|
+
self._ansible_core_version = ansible_core_version
|
330
374
|
|
331
375
|
def find(self, name: str) -> CollectionData | None:
|
332
376
|
"""
|
333
377
|
Find a collection for a given name.
|
334
378
|
"""
|
335
|
-
return self.
|
379
|
+
return self._merged_collection_list.find(name)
|
336
380
|
|
337
381
|
def add_collection(
|
338
382
|
self, *, directory: Path, namespace: str, name: str
|
@@ -341,9 +385,15 @@ class _CollectionListUpdater:
|
|
341
385
|
Add a new collection to the cache.
|
342
386
|
"""
|
343
387
|
# pylint: disable-next=protected-access
|
344
|
-
|
345
|
-
directory=directory,
|
388
|
+
result = self._owner._add_collection(
|
389
|
+
directory=directory,
|
390
|
+
namespace=namespace,
|
391
|
+
name=name,
|
392
|
+
ansible_core_version=self._ansible_core_version,
|
346
393
|
)
|
394
|
+
self._merged_collection_list._add(result) # pylint: disable=protected-access
|
395
|
+
self._local_collection_list._add(result) # pylint: disable=protected-access
|
396
|
+
return result
|
347
397
|
|
348
398
|
def get_global_cache(self) -> _GlobalCache:
|
349
399
|
"""
|
@@ -356,7 +406,10 @@ class _CollectionListSingleton:
|
|
356
406
|
_lock = threading.Lock()
|
357
407
|
|
358
408
|
_global_cache_dir: Path | None = None
|
359
|
-
|
409
|
+
_global_collection_list: CollectionList | None = None
|
410
|
+
_global_collection_list_per_ansible_core_version: dict[
|
411
|
+
AnsibleCoreVersion, CollectionList
|
412
|
+
] = {}
|
360
413
|
|
361
414
|
def setup(self, *, global_cache_dir: Path) -> None:
|
362
415
|
"""
|
@@ -378,30 +431,48 @@ class _CollectionListSingleton:
|
|
378
431
|
Clear collection cache.
|
379
432
|
"""
|
380
433
|
with self._lock:
|
381
|
-
self.
|
434
|
+
self._global_collection_list = None
|
435
|
+
self._global_collection_list_per_ansible_core_version.clear()
|
382
436
|
|
383
|
-
def get_cached(
|
437
|
+
def get_cached(
|
438
|
+
self, *, ansible_core_version: AnsibleCoreVersion | None = None
|
439
|
+
) -> CollectionList | None:
|
384
440
|
"""
|
385
441
|
Return cached list of collections, if present.
|
386
442
|
Do not modify the result!
|
387
443
|
"""
|
388
|
-
|
444
|
+
if ansible_core_version is None:
|
445
|
+
return self._global_collection_list
|
446
|
+
return self._global_collection_list_per_ansible_core_version.get(
|
447
|
+
ansible_core_version
|
448
|
+
)
|
389
449
|
|
390
|
-
def get(
|
450
|
+
def get(
|
451
|
+
self, *, ansible_core_version: AnsibleCoreVersion, runner: Runner
|
452
|
+
) -> CollectionList:
|
391
453
|
"""
|
392
454
|
Search for a list of collections. The result is cached.
|
393
455
|
"""
|
394
456
|
with self._lock:
|
395
457
|
if self._global_cache_dir is None:
|
396
458
|
raise ValueError("Internal error: global cache dir not setup")
|
397
|
-
|
398
|
-
if
|
399
|
-
|
400
|
-
|
459
|
+
global_list = self._global_collection_list
|
460
|
+
if global_list is None:
|
461
|
+
global_list = CollectionList.collect_global(runner=runner)
|
462
|
+
self._global_collection_list = global_list
|
463
|
+
local_list = self._global_collection_list_per_ansible_core_version.get(
|
464
|
+
ansible_core_version
|
465
|
+
)
|
466
|
+
if local_list is None:
|
467
|
+
local_list = CollectionList.collect_local(
|
401
468
|
global_cache=_GlobalCache.create(root=self._global_cache_dir),
|
469
|
+
ansible_core_version=ansible_core_version,
|
470
|
+
current=global_list.current,
|
402
471
|
)
|
403
|
-
self.
|
404
|
-
|
472
|
+
self._global_collection_list_per_ansible_core_version[
|
473
|
+
ansible_core_version
|
474
|
+
] = local_list
|
475
|
+
return global_list.merge_with(local_list)
|
405
476
|
|
406
477
|
def _get_global_cache(self) -> _GlobalCache:
|
407
478
|
"""
|
@@ -412,26 +483,45 @@ class _CollectionListSingleton:
|
|
412
483
|
return _GlobalCache.create(root=self._global_cache_dir)
|
413
484
|
|
414
485
|
def _add_collection(
|
415
|
-
self,
|
486
|
+
self,
|
487
|
+
*,
|
488
|
+
directory: Path,
|
489
|
+
namespace: str,
|
490
|
+
name: str,
|
491
|
+
ansible_core_version: AnsibleCoreVersion,
|
416
492
|
) -> CollectionData:
|
417
493
|
"""
|
418
494
|
Add collection in directory if the collection list has been cached.
|
419
495
|
"""
|
420
|
-
|
421
|
-
|
496
|
+
local_list = self._global_collection_list_per_ansible_core_version.get(
|
497
|
+
ansible_core_version
|
498
|
+
)
|
499
|
+
if not local_list:
|
500
|
+
raise ValueError(
|
501
|
+
f"Internal error: collections not listed for {ansible_core_version}"
|
502
|
+
)
|
422
503
|
data = load_collection_data_from_disk(directory, namespace=namespace, name=name)
|
423
|
-
|
504
|
+
local_list._add(data) # pylint: disable=protected-access
|
424
505
|
return data
|
425
506
|
|
426
507
|
@contextmanager
|
427
|
-
def _update_collection_list(
|
508
|
+
def _update_collection_list(
|
509
|
+
self, *, ansible_core_version: AnsibleCoreVersion
|
510
|
+
) -> t.Iterator[_CollectionListUpdater]:
|
428
511
|
with self._lock:
|
429
|
-
|
512
|
+
global_list = self._global_collection_list
|
513
|
+
local_list = self._global_collection_list_per_ansible_core_version.get(
|
514
|
+
ansible_core_version
|
515
|
+
)
|
516
|
+
if not global_list or self._global_cache_dir is None or local_list is None:
|
430
517
|
raise ValueError(
|
431
518
|
"Internal error: collections not listed or global cache not setup"
|
432
519
|
)
|
433
520
|
yield _CollectionListUpdater(
|
434
|
-
owner=self,
|
521
|
+
owner=self,
|
522
|
+
merged_collection_list=global_list.merge_with(local_list),
|
523
|
+
local_collection_list=local_list,
|
524
|
+
ansible_core_version=ansible_core_version,
|
435
525
|
)
|
436
526
|
|
437
527
|
|
@@ -439,18 +529,26 @@ _COLLECTION_LIST = _CollectionListSingleton()
|
|
439
529
|
|
440
530
|
|
441
531
|
@contextmanager
|
442
|
-
def _update_collection_list(
|
532
|
+
def _update_collection_list(
|
533
|
+
*, ansible_core_version: AnsibleCoreVersion
|
534
|
+
) -> t.Iterator[_CollectionListUpdater]:
|
443
535
|
# pylint: disable-next=protected-access
|
444
|
-
with _COLLECTION_LIST._update_collection_list(
|
536
|
+
with _COLLECTION_LIST._update_collection_list(
|
537
|
+
ansible_core_version=ansible_core_version
|
538
|
+
) as result:
|
445
539
|
yield result
|
446
540
|
|
447
541
|
|
448
|
-
def get_collection_list(
|
542
|
+
def get_collection_list(
|
543
|
+
*, runner: Runner, global_cache_dir: Path, ansible_core_version: AnsibleCoreVersion
|
544
|
+
) -> CollectionList:
|
449
545
|
"""
|
450
546
|
Search for a list of collections. The result is cached.
|
451
547
|
"""
|
452
548
|
_COLLECTION_LIST.setup(global_cache_dir=global_cache_dir)
|
453
|
-
return _COLLECTION_LIST.get(
|
549
|
+
return _COLLECTION_LIST.get(
|
550
|
+
runner=runner, ansible_core_version=ansible_core_version
|
551
|
+
)
|
454
552
|
|
455
553
|
|
456
554
|
__all__ = [
|
antsibull_nox/config.py
CHANGED
@@ -340,6 +340,9 @@ class Config(_BaseModel):
|
|
340
340
|
"""
|
341
341
|
|
342
342
|
collection_sources: dict[CollectionName, CollectionSource] = {}
|
343
|
+
collection_sources_per_ansible: dict[
|
344
|
+
PAnsibleCoreVersion, dict[CollectionName, CollectionSource]
|
345
|
+
] = {}
|
343
346
|
sessions: Sessions = Sessions()
|
344
347
|
|
345
348
|
|
@@ -0,0 +1,138 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
# Copyright (c) 2025, Felix Fontein <felix@fontein.de>
|
4
|
+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
|
5
|
+
# or https://www.gnu.org/licenses/gpl-3.0.txt)
|
6
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
7
|
+
|
8
|
+
"""Make sure all plugin and module documentation adheres to yamllint."""
|
9
|
+
|
10
|
+
from __future__ import annotations
|
11
|
+
|
12
|
+
import io
|
13
|
+
import os
|
14
|
+
import sys
|
15
|
+
import traceback
|
16
|
+
import typing as t
|
17
|
+
|
18
|
+
from antsibull_nox_data_util import setup # type: ignore
|
19
|
+
from yamllint import linter
|
20
|
+
from yamllint.cli import find_project_config_filepath
|
21
|
+
from yamllint.config import YamlLintConfig
|
22
|
+
from yamllint.linter import PROBLEM_LEVELS
|
23
|
+
|
24
|
+
REPORT_LEVELS: set[PROBLEM_LEVELS] = {
|
25
|
+
"warning",
|
26
|
+
"error",
|
27
|
+
}
|
28
|
+
|
29
|
+
|
30
|
+
def lint(
|
31
|
+
*,
|
32
|
+
errors: list[dict[str, t.Any]],
|
33
|
+
path: str,
|
34
|
+
data: str,
|
35
|
+
config: YamlLintConfig,
|
36
|
+
) -> None:
|
37
|
+
try:
|
38
|
+
problems = linter.run(
|
39
|
+
io.StringIO(data),
|
40
|
+
config,
|
41
|
+
path,
|
42
|
+
)
|
43
|
+
for problem in problems:
|
44
|
+
if problem.level not in REPORT_LEVELS:
|
45
|
+
continue
|
46
|
+
msg = f"{problem.level}: {problem.desc}"
|
47
|
+
if problem.rule:
|
48
|
+
msg += f" ({problem.rule})"
|
49
|
+
errors.append(
|
50
|
+
{
|
51
|
+
"path": path,
|
52
|
+
"line": problem.line,
|
53
|
+
"col": problem.column,
|
54
|
+
"message": msg,
|
55
|
+
}
|
56
|
+
)
|
57
|
+
except Exception as exc:
|
58
|
+
error = str(exc).replace("\n", " / ")
|
59
|
+
errors.append(
|
60
|
+
{
|
61
|
+
"path": path,
|
62
|
+
"line": 1,
|
63
|
+
"col": 1,
|
64
|
+
"message": (
|
65
|
+
f"Internal error while linting YAML: exception {type(exc)}:"
|
66
|
+
f" {error}; traceback: {traceback.format_exc()!r}"
|
67
|
+
),
|
68
|
+
}
|
69
|
+
)
|
70
|
+
|
71
|
+
|
72
|
+
def process_yaml_file(
|
73
|
+
errors: list[dict[str, t.Any]],
|
74
|
+
path: str,
|
75
|
+
config: YamlLintConfig,
|
76
|
+
) -> None:
|
77
|
+
try:
|
78
|
+
with open(path, "rt", encoding="utf-8") as stream:
|
79
|
+
data = stream.read()
|
80
|
+
except Exception as exc:
|
81
|
+
errors.append(
|
82
|
+
{
|
83
|
+
"path": path,
|
84
|
+
"line": 1,
|
85
|
+
"col": 1,
|
86
|
+
"message": (
|
87
|
+
f"Error while parsing Python code: exception {type(exc)}:"
|
88
|
+
f" {exc}; traceback: {traceback.format_exc()!r}"
|
89
|
+
),
|
90
|
+
}
|
91
|
+
)
|
92
|
+
return
|
93
|
+
|
94
|
+
lint(
|
95
|
+
errors=errors,
|
96
|
+
path=path,
|
97
|
+
data=data,
|
98
|
+
config=config,
|
99
|
+
)
|
100
|
+
|
101
|
+
|
102
|
+
def main() -> int:
|
103
|
+
"""Main entry point."""
|
104
|
+
paths, extra_data = setup()
|
105
|
+
config: str | None = extra_data.get("config")
|
106
|
+
|
107
|
+
if config is None:
|
108
|
+
config = find_project_config_filepath()
|
109
|
+
|
110
|
+
if config:
|
111
|
+
yamllint_config = YamlLintConfig(file=config)
|
112
|
+
else:
|
113
|
+
yamllint_config = YamlLintConfig(content="extends: default")
|
114
|
+
|
115
|
+
errors: list[dict[str, t.Any]] = []
|
116
|
+
for path in paths:
|
117
|
+
if not os.path.isfile(path):
|
118
|
+
continue
|
119
|
+
process_yaml_file(errors, path, yamllint_config)
|
120
|
+
|
121
|
+
errors.sort(
|
122
|
+
key=lambda error: (error["path"], error["line"], error["col"], error["message"])
|
123
|
+
)
|
124
|
+
for error in errors:
|
125
|
+
prefix = f"{error['path']}:{error['line']}:{error['col']}: "
|
126
|
+
msg = error["message"]
|
127
|
+
if "note" in error:
|
128
|
+
msg = f"{msg}\nNote: {error['note']}"
|
129
|
+
for i, line in enumerate(msg.splitlines()):
|
130
|
+
print(f"{prefix}{line}")
|
131
|
+
if i == 0:
|
132
|
+
prefix = " " * len(prefix)
|
133
|
+
|
134
|
+
return len(errors) > 0
|
135
|
+
|
136
|
+
|
137
|
+
if __name__ == "__main__":
|
138
|
+
sys.exit(main())
|
@@ -1,6 +1,6 @@
|
|
1
1
|
#!/usr/bin/env python
|
2
2
|
|
3
|
-
# Copyright (c)
|
3
|
+
# Copyright (c) 2025, Felix Fontein <felix@fontein.de>
|
4
4
|
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt
|
5
5
|
# or https://www.gnu.org/licenses/gpl-3.0.txt)
|
6
6
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
@@ -44,6 +44,13 @@ def lint(
|
|
44
44
|
config: YamlLintConfig,
|
45
45
|
extra_for_errors: dict[str, t.Any] | None = None,
|
46
46
|
) -> None:
|
47
|
+
# If the string start with optional whitespace + linebreak, skip that line
|
48
|
+
idx = data.find("\n")
|
49
|
+
if idx >= 0 and (idx == 0 or data[:idx].isspace()):
|
50
|
+
data = data[idx + 1 :]
|
51
|
+
row_offset += 1
|
52
|
+
col_offset = 0
|
53
|
+
|
47
54
|
try:
|
48
55
|
problems = linter.run(
|
49
56
|
io.StringIO(data),
|
@@ -84,6 +91,20 @@ def lint(
|
|
84
91
|
errors[-1].update(extra_for_errors)
|
85
92
|
|
86
93
|
|
94
|
+
def iterate_targets(
|
95
|
+
assignment: ast.Assign,
|
96
|
+
) -> t.Iterable[tuple[ast.Constant, str, str]]:
|
97
|
+
if not isinstance(assignment.value, ast.Constant):
|
98
|
+
return
|
99
|
+
if not isinstance(assignment.value.value, str):
|
100
|
+
return
|
101
|
+
for target in assignment.targets:
|
102
|
+
try:
|
103
|
+
yield assignment.value, assignment.value.value, target.id # type: ignore
|
104
|
+
except AttributeError:
|
105
|
+
continue
|
106
|
+
|
107
|
+
|
87
108
|
def process_python_file(
|
88
109
|
errors: list[dict[str, t.Any]],
|
89
110
|
path: str,
|
@@ -107,33 +128,39 @@ def process_python_file(
|
|
107
128
|
)
|
108
129
|
return
|
109
130
|
|
110
|
-
|
131
|
+
is_doc_fragment = path.startswith("plugins/doc_fragments/")
|
132
|
+
|
133
|
+
# We look for top-level assignments and classes
|
111
134
|
for child in root.body:
|
135
|
+
if (
|
136
|
+
is_doc_fragment
|
137
|
+
and isinstance(child, ast.ClassDef)
|
138
|
+
and child.name == "ModuleDocFragment"
|
139
|
+
):
|
140
|
+
for fragment in child.body:
|
141
|
+
if not isinstance(fragment, ast.Assign):
|
142
|
+
continue
|
143
|
+
for constant, data, fragment_name in iterate_targets(fragment):
|
144
|
+
lint(
|
145
|
+
errors=errors,
|
146
|
+
path=path,
|
147
|
+
data=data,
|
148
|
+
row_offset=constant.lineno - 1,
|
149
|
+
col_offset=constant.col_offset - 1,
|
150
|
+
section=fragment_name,
|
151
|
+
config=config,
|
152
|
+
)
|
112
153
|
if not isinstance(child, ast.Assign):
|
113
154
|
continue
|
114
|
-
|
115
|
-
continue
|
116
|
-
if not isinstance(child.value.value, str):
|
117
|
-
continue
|
118
|
-
for target in child.targets:
|
119
|
-
try:
|
120
|
-
section = target.id # type: ignore
|
121
|
-
except AttributeError:
|
122
|
-
continue
|
155
|
+
for constant, data, section in iterate_targets(child):
|
123
156
|
if section not in ("DOCUMENTATION", "EXAMPLES", "RETURN"):
|
124
157
|
continue
|
125
158
|
|
126
|
-
#
|
127
|
-
data
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
# If the string start with optional whitespace + linebreak, skip that line
|
132
|
-
idx = data.find("\n")
|
133
|
-
if idx >= 0 and (idx == 0 or data[:idx].isspace()):
|
134
|
-
data = data[idx + 1 :]
|
135
|
-
row_offset += 1
|
136
|
-
col_offset = 0
|
159
|
+
# Handle special values
|
160
|
+
if data in ("#", " # ") and section == "RETURN":
|
161
|
+
# Not skipping it here could result in all kind of linting errors,
|
162
|
+
# like no document start, or trailing space.
|
163
|
+
continue
|
137
164
|
|
138
165
|
# Check for non-YAML examples
|
139
166
|
if section == EXAMPLES_SECTION:
|
@@ -146,8 +173,8 @@ def process_python_file(
|
|
146
173
|
errors=errors,
|
147
174
|
path=path,
|
148
175
|
data=data,
|
149
|
-
row_offset=
|
150
|
-
col_offset=col_offset,
|
176
|
+
row_offset=constant.lineno - 1,
|
177
|
+
col_offset=constant.col_offset - 1,
|
151
178
|
section=section,
|
152
179
|
config=config_examples if section == EXAMPLES_SECTION else config,
|
153
180
|
)
|
@@ -46,6 +46,18 @@ def _interpret_config(config: Config) -> None:
|
|
46
46
|
for name, source in config.collection_sources.items()
|
47
47
|
}
|
48
48
|
)
|
49
|
+
if config.collection_sources_per_ansible:
|
50
|
+
for (
|
51
|
+
ansible_core_version,
|
52
|
+
collection_sources,
|
53
|
+
) in config.collection_sources_per_ansible.items():
|
54
|
+
setup_collection_sources(
|
55
|
+
{
|
56
|
+
name: CollectionSource(name=name, source=source.source)
|
57
|
+
for name, source in collection_sources.items()
|
58
|
+
},
|
59
|
+
ansible_core_version=ansible_core_version,
|
60
|
+
)
|
49
61
|
|
50
62
|
|
51
63
|
def _convert_action_groups(
|
@@ -22,6 +22,7 @@ from ..ansible import (
|
|
22
22
|
get_ansible_core_info,
|
23
23
|
get_ansible_core_package_name,
|
24
24
|
get_supported_core_versions,
|
25
|
+
parse_ansible_core_version,
|
25
26
|
)
|
26
27
|
from ..paths import copy_directory_tree_into
|
27
28
|
from ..python import get_installed_python_versions
|
@@ -33,17 +34,6 @@ from .utils import (
|
|
33
34
|
)
|
34
35
|
|
35
36
|
|
36
|
-
def _parse_ansible_core_version(
|
37
|
-
version: str | AnsibleCoreVersion,
|
38
|
-
) -> AnsibleCoreVersion:
|
39
|
-
if version in ("devel", "milestone"):
|
40
|
-
# For some reason mypy doesn't notice that
|
41
|
-
return t.cast(AnsibleCoreVersion, version)
|
42
|
-
if isinstance(version, Version):
|
43
|
-
return version
|
44
|
-
return Version.parse(version)
|
45
|
-
|
46
|
-
|
47
37
|
def add_ansible_test_session(
|
48
38
|
*,
|
49
39
|
name: str,
|
@@ -67,7 +57,7 @@ def add_ansible_test_session(
|
|
67
57
|
|
68
58
|
Returns a list of Python versions set for this session.
|
69
59
|
"""
|
70
|
-
parsed_ansible_core_version =
|
60
|
+
parsed_ansible_core_version = parse_ansible_core_version(ansible_core_version)
|
71
61
|
|
72
62
|
def compose_dependencies() -> list[str]:
|
73
63
|
deps = [
|
@@ -84,6 +74,7 @@ def add_ansible_test_session(
|
|
84
74
|
install(session, *compose_dependencies())
|
85
75
|
prepared_collections = prepare_collections(
|
86
76
|
session,
|
77
|
+
ansible_core_version=parsed_ansible_core_version,
|
87
78
|
install_in_site_packages=False,
|
88
79
|
extra_deps_files=extra_deps_files,
|
89
80
|
install_out_of_tree=True,
|
@@ -210,7 +201,7 @@ def _parse_min_max_except(
|
|
210
201
|
max_version = Version.parse(max_version)
|
211
202
|
if except_versions is None:
|
212
203
|
return min_version, max_version, None
|
213
|
-
evs = tuple(
|
204
|
+
evs = tuple(parse_ansible_core_version(version) for version in except_versions)
|
214
205
|
return min_version, max_version, evs
|
215
206
|
|
216
207
|
|
@@ -18,6 +18,7 @@ from pathlib import Path
|
|
18
18
|
|
19
19
|
import nox
|
20
20
|
|
21
|
+
from ..ansible import AnsibleCoreVersion, parse_ansible_core_version
|
21
22
|
from ..collection import (
|
22
23
|
CollectionData,
|
23
24
|
setup_collections,
|
@@ -76,6 +77,7 @@ def _run_subprocess(args: list[str]) -> tuple[bytes, bytes]:
|
|
76
77
|
def prepare_collections(
|
77
78
|
session: nox.Session,
|
78
79
|
*,
|
80
|
+
ansible_core_version: AnsibleCoreVersion | str | None = None,
|
79
81
|
install_in_site_packages: bool,
|
80
82
|
extra_deps_files: list[str | os.PathLike] | None = None,
|
81
83
|
extra_collections: list[str] | None = None,
|
@@ -84,6 +86,11 @@ def prepare_collections(
|
|
84
86
|
"""
|
85
87
|
Install collections in site-packages.
|
86
88
|
"""
|
89
|
+
parsed_ansible_core_version = (
|
90
|
+
parse_ansible_core_version(ansible_core_version)
|
91
|
+
if ansible_core_version is not None
|
92
|
+
else "devel"
|
93
|
+
)
|
87
94
|
if install_out_of_tree and install_in_site_packages:
|
88
95
|
raise ValueError(
|
89
96
|
"install_out_of_tree=True cannot be combined with install_in_site_packages=True"
|
@@ -116,6 +123,7 @@ def prepare_collections(
|
|
116
123
|
setup = setup_collections(
|
117
124
|
place,
|
118
125
|
_run_subprocess,
|
126
|
+
ansible_core_version=parsed_ansible_core_version,
|
119
127
|
extra_deps_files=extra_deps_files,
|
120
128
|
extra_collections=extra_collections,
|
121
129
|
with_current=False,
|
antsibull_nox/sessions/lint.py
CHANGED
@@ -10,6 +10,7 @@ Create nox lint sessions.
|
|
10
10
|
|
11
11
|
from __future__ import annotations
|
12
12
|
|
13
|
+
import json
|
13
14
|
import os
|
14
15
|
import shlex
|
15
16
|
from pathlib import Path
|
@@ -29,6 +30,7 @@ from .utils import (
|
|
29
30
|
compose_description,
|
30
31
|
install,
|
31
32
|
run_bare_script,
|
33
|
+
silence_run_verbosity,
|
32
34
|
)
|
33
35
|
|
34
36
|
CODE_FILES = [
|
@@ -189,6 +191,28 @@ def add_formatters(
|
|
189
191
|
nox.session(name="formatters", default=False)(formatters)
|
190
192
|
|
191
193
|
|
194
|
+
def process_pylint_errors(
|
195
|
+
session: nox.Session,
|
196
|
+
prepared_collections: CollectionSetup,
|
197
|
+
output: str,
|
198
|
+
) -> None:
|
199
|
+
"""
|
200
|
+
Process errors reported by pylint in 'json2' format.
|
201
|
+
"""
|
202
|
+
data = json.loads(output)
|
203
|
+
found_error = False
|
204
|
+
if data["messages"]:
|
205
|
+
for message in data["messages"]:
|
206
|
+
path = os.path.relpath(
|
207
|
+
message["absolutePath"], prepared_collections.current_path
|
208
|
+
)
|
209
|
+
prefix = f"{path}:{message['line']}:{message['column']}: [{message['messageId']}]"
|
210
|
+
print(f"{prefix} {message['message']} [{message['symbol']}]")
|
211
|
+
found_error = True
|
212
|
+
if found_error:
|
213
|
+
session.error("Pylint failed")
|
214
|
+
|
215
|
+
|
192
216
|
def add_codeqa( # noqa: C901
|
193
217
|
*,
|
194
218
|
extra_code_files: list[str],
|
@@ -249,9 +273,17 @@ def add_codeqa( # noqa: C901
|
|
249
273
|
]
|
250
274
|
)
|
251
275
|
command.extend(["--source-roots", "."])
|
276
|
+
command.extend(["--output-format", "json2"])
|
252
277
|
command.extend(session.posargs)
|
253
278
|
command.extend(prepared_collections.prefix_current_paths(paths))
|
254
|
-
|
279
|
+
with silence_run_verbosity():
|
280
|
+
# Exit code is OR of some of 1, 2, 4, 8, 16
|
281
|
+
output = session.run(
|
282
|
+
*command, silent=True, success_codes=list(range(0, 32))
|
283
|
+
)
|
284
|
+
|
285
|
+
if output:
|
286
|
+
process_pylint_errors(session, prepared_collections, output)
|
255
287
|
|
256
288
|
def execute_pylint(
|
257
289
|
session: nox.Session, prepared_collections: CollectionSetup
|
@@ -335,29 +367,22 @@ def add_yamllint(
|
|
335
367
|
def execute_yamllint(session: nox.Session) -> None:
|
336
368
|
# Run yamllint
|
337
369
|
all_files = list_all_files()
|
338
|
-
cwd = Path.cwd()
|
339
370
|
all_yaml_filenames = [
|
340
|
-
|
341
|
-
for file in all_files
|
342
|
-
if file.name.lower().endswith((".yml", ".yaml"))
|
371
|
+
file for file in all_files if file.name.lower().endswith((".yml", ".yaml"))
|
343
372
|
]
|
344
373
|
if not all_yaml_filenames:
|
345
374
|
session.warn("Skipping yamllint since no YAML file was found...")
|
346
375
|
return
|
347
376
|
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
command.append("--")
|
358
|
-
command.extend(all_yaml_filenames)
|
359
|
-
command.extend(session.posargs)
|
360
|
-
session.run(*command)
|
377
|
+
run_bare_script(
|
378
|
+
session,
|
379
|
+
"file-yamllint",
|
380
|
+
use_session_python=True,
|
381
|
+
files=all_yaml_filenames,
|
382
|
+
extra_data={
|
383
|
+
"config": to_str(yamllint_config),
|
384
|
+
},
|
385
|
+
)
|
361
386
|
|
362
387
|
def execute_plugin_yamllint(session: nox.Session) -> None:
|
363
388
|
# Run yamllint
|
@@ -415,6 +440,40 @@ def add_yamllint(
|
|
415
440
|
nox.session(name="yamllint", default=False)(yamllint)
|
416
441
|
|
417
442
|
|
443
|
+
def process_mypy_errors(
|
444
|
+
session: nox.Session,
|
445
|
+
prepared_collections: CollectionSetup,
|
446
|
+
output: str,
|
447
|
+
) -> None:
|
448
|
+
"""
|
449
|
+
Process errors reported by mypy in 'json' format.
|
450
|
+
"""
|
451
|
+
found_error = False
|
452
|
+
for line in output.splitlines():
|
453
|
+
if not line.strip():
|
454
|
+
continue
|
455
|
+
try:
|
456
|
+
data = json.loads(line)
|
457
|
+
path = os.path.relpath(
|
458
|
+
prepared_collections.current_place / data["file"],
|
459
|
+
prepared_collections.current_path,
|
460
|
+
)
|
461
|
+
prefix = f"{path}:{data['line']}:{data['column']}: [{data['severity']}]"
|
462
|
+
if data["code"]:
|
463
|
+
print(f"{prefix} {data['message']} [{data['code']}]")
|
464
|
+
else:
|
465
|
+
print(f"{prefix} {data['message']}")
|
466
|
+
if data["hint"]:
|
467
|
+
prefix = " " * len(prefix)
|
468
|
+
for hint in data["hint"].splitlines():
|
469
|
+
print(f"{prefix} {hint}")
|
470
|
+
except Exception: # pylint: disable=broad-exception-caught
|
471
|
+
session.warn(f"Cannot parse mypy output: {line}")
|
472
|
+
found_error = True
|
473
|
+
if found_error:
|
474
|
+
session.error("Type checking failed")
|
475
|
+
|
476
|
+
|
418
477
|
def add_typing(
|
419
478
|
*,
|
420
479
|
extra_code_files: list[str],
|
@@ -459,13 +518,21 @@ def add_typing(
|
|
459
518
|
)
|
460
519
|
command.append("--namespace-packages")
|
461
520
|
command.append("--explicit-package-bases")
|
521
|
+
command.extend(["--output", "json"])
|
462
522
|
command.extend(session.posargs)
|
463
523
|
command.extend(
|
464
524
|
prepared_collections.prefix_current_paths(CODE_FILES + extra_code_files)
|
465
525
|
)
|
466
|
-
|
467
|
-
|
468
|
-
|
526
|
+
with silence_run_verbosity():
|
527
|
+
output = session.run(
|
528
|
+
*command,
|
529
|
+
env={"MYPYPATH": str(prepared_collections.current_place)},
|
530
|
+
silent=True,
|
531
|
+
success_codes=(0, 1, 2),
|
532
|
+
)
|
533
|
+
|
534
|
+
if output:
|
535
|
+
process_mypy_errors(session, prepared_collections, output)
|
469
536
|
|
470
537
|
def typing(session: nox.Session) -> None:
|
471
538
|
install(session, *compose_dependencies())
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: antsibull-nox
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.4.0
|
4
4
|
Summary: Changelog tool for Ansible-core and Ansible collections
|
5
5
|
Project-URL: Documentation, https://ansible.readthedocs.io/projects/antsibull-nox/
|
6
6
|
Project-URL: Source code, https://github.com/ansible-community/antsibull-nox/
|
@@ -1,40 +1,41 @@
|
|
1
|
-
antsibull_nox/__init__.py,sha256=
|
1
|
+
antsibull_nox/__init__.py,sha256=D9rbPsKuklFL2MNeCAGCo6Qrm1C71Gaje9Om-vviX4k,845
|
2
2
|
antsibull_nox/_pydantic.py,sha256=VTIh-u0TpWvnY6Dhe4ba8nAmR2tYmz4PXNp2_8Sr9pw,3203
|
3
|
-
antsibull_nox/ansible.py,sha256=
|
3
|
+
antsibull_nox/ansible.py,sha256=2EgaPK9ckS_KYTgpvb5HU9UOb6q5dpP3Az8sr1S7MU4,9346
|
4
4
|
antsibull_nox/cli.py,sha256=NKeRlWc_0taNRcZcdMv9LHZ5nCz8-DEJxBLPxJ9vFYQ,3358
|
5
|
-
antsibull_nox/config.py,sha256=
|
5
|
+
antsibull_nox/config.py,sha256=SNiqRhu5P6XS-Uve0faLTEG0THaMEoiYti-n9-f_Kec,11037
|
6
6
|
antsibull_nox/data_util.py,sha256=7FVoqESEc-_QdqrQ16K1AHRVHEglNbRCH_mNaYDJ7a4,953
|
7
7
|
antsibull_nox/init.py,sha256=eRltIrS3AcHqEHk2yNAqJXv7kR6m_ysvFxIHpowd-2M,2259
|
8
|
-
antsibull_nox/interpret_config.py,sha256=
|
8
|
+
antsibull_nox/interpret_config.py,sha256=tEG6Hr7kQVpZrSAbprSqrlsVa7oPfUUcKd7WxiC5t84,10844
|
9
9
|
antsibull_nox/lint_config.py,sha256=ZnsUbX6IdQK_IP9nvs8Kk6jb5lPdiREFSHAUuEGGceU,3848
|
10
10
|
antsibull_nox/paths.py,sha256=86HOynhCMTVop3ml_77JI06vM9nyK7PHMzLP_4M0V88,6317
|
11
11
|
antsibull_nox/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
12
12
|
antsibull_nox/python.py,sha256=0pyGqgwsuyc0BhzoXNZLTLbjaFA4kAYEHrFD6A1o1-o,2113
|
13
13
|
antsibull_nox/utils.py,sha256=lgBNuJ67Agl9YpFNWCjr6TBUcbC1LroZrJkv6S5VuxA,2437
|
14
14
|
antsibull_nox/collection/__init__.py,sha256=2k6uYdiPjAgJgmz5oviYpu2Sh5ZSKSJEJCR01iz-29A,1494
|
15
|
-
antsibull_nox/collection/data.py,sha256=
|
15
|
+
antsibull_nox/collection/data.py,sha256=HmnASNuv97xFM08gYuQcAF3Nz6Oc6eXd5NN-wbMtYUU,2966
|
16
16
|
antsibull_nox/collection/extract.py,sha256=qNVknQRtRrCt10aawuU1Z6NTs16CA1bUEX3WDGBw68g,684
|
17
|
-
antsibull_nox/collection/install.py,sha256=
|
18
|
-
antsibull_nox/collection/search.py,sha256=
|
17
|
+
antsibull_nox/collection/install.py,sha256=xRXJ0kY8J1eOMmtHBZC6DqG38BF-RhjhvLF-GRT-Bb0,21622
|
18
|
+
antsibull_nox/collection/search.py,sha256=WZkcgNldl-feAqtKvn9fMMFxJ46kLeS0mqU-5R6yt7o,19296
|
19
19
|
antsibull_nox/data/action-groups.py,sha256=qLuh2YqIpc6i2jpWFzjtFhZhx6_SDbbvJe6VH7EXrqg,6934
|
20
20
|
antsibull_nox/data/antsibull-nox-lint-config.py,sha256=tXkKd9AqgfDs5w7S6OaBIt9HnT0KSbiQIU9tFxtYE2U,657
|
21
21
|
antsibull_nox/data/antsibull_nox_data_util.py,sha256=D4i_sKxjAeZuDV-z9Ibow0YYIqhXo2V_YC0LONLcEXM,2931
|
22
|
+
antsibull_nox/data/file-yamllint.py,sha256=hlS9tULwQSUMkdbYFfGtQGcPSj2scxEay6IalQfjSFE,3625
|
22
23
|
antsibull_nox/data/license-check.py,sha256=or3GyQC0WWYMxMqL-369krGsHaySH1vX-2fwpRyJGp0,5665
|
23
24
|
antsibull_nox/data/license-check.py.license,sha256=iPdFtdkeE3E2nCB-M5KHevbz4d5I-6ymOnKNTc954Dw,218
|
24
25
|
antsibull_nox/data/no-unwanted-files.py,sha256=_B3m-XWvWpdzGN-XAP8rLIS_5RMJGntFWL-Jy0WY9Mc,2959
|
25
|
-
antsibull_nox/data/plugin-yamllint.py,sha256=
|
26
|
+
antsibull_nox/data/plugin-yamllint.py,sha256=bPIFmNwuTznaUjUhebccFa0IF_joyjIO7d3uF2HqjZQ,8466
|
26
27
|
antsibull_nox/sessions/__init__.py,sha256=4wTTO1E6rdCz4pVMuGUeuXi_vqFaH4whAL9qcjfOqto,2022
|
27
|
-
antsibull_nox/sessions/ansible_lint.py,sha256=
|
28
|
-
antsibull_nox/sessions/ansible_test.py,sha256=
|
28
|
+
antsibull_nox/sessions/ansible_lint.py,sha256=ik2heGsvpRwYm_4XGwlm53UvWQ_7FHDWaBt7ttvUYbU,1661
|
29
|
+
antsibull_nox/sessions/ansible_test.py,sha256=uZ5fa9fBY-D4BBWqgl1bI-GfpKmXAu9YKC5G4ajum18,20379
|
29
30
|
antsibull_nox/sessions/build_import_check.py,sha256=kdr_Cqc0jb8XQQ-2QL-g_X7wgezE04oMVFCPr7a34iA,4719
|
30
|
-
antsibull_nox/sessions/collections.py,sha256=
|
31
|
+
antsibull_nox/sessions/collections.py,sha256=nhj_W2tbnsVJw6p7NkyP1xvmr3ZUmSJzwVuK0HE3oxw,4681
|
31
32
|
antsibull_nox/sessions/docs_check.py,sha256=mVYt278xy5AVwo5rCf6FLZlhqBiEYgJ3mmWsVBShKD0,2344
|
32
33
|
antsibull_nox/sessions/extra_checks.py,sha256=sBn0YFD8cM3OqeQ4UgYwD0NAcjKnYoy2zd8q-z4Xl2I,4110
|
33
34
|
antsibull_nox/sessions/license_check.py,sha256=t5ut4ZluhFfk-qE6kcU8VNdvIGvzND81N7WCsbA4jLc,1824
|
34
|
-
antsibull_nox/sessions/lint.py,sha256=
|
35
|
+
antsibull_nox/sessions/lint.py,sha256=Idl3g3CB_zqlDbrCvItkjvYMw4TzbBcQWQnYLt5AL5Y,21639
|
35
36
|
antsibull_nox/sessions/utils.py,sha256=rrQdzmjdrLQula8t-BCTKbO-tAmzfOKHNn1pN2B1QVc,5632
|
36
|
-
antsibull_nox-0.
|
37
|
-
antsibull_nox-0.
|
38
|
-
antsibull_nox-0.
|
39
|
-
antsibull_nox-0.
|
40
|
-
antsibull_nox-0.
|
37
|
+
antsibull_nox-0.4.0.dist-info/METADATA,sha256=h--FXM-iETEiwgpBeIGAQajtcGMWAkwjKmyuFoOCZFg,7670
|
38
|
+
antsibull_nox-0.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
39
|
+
antsibull_nox-0.4.0.dist-info/entry_points.txt,sha256=solWA9TCB37UlaGk8sHXxJg-k1HWckfKdncHDBsVSsI,57
|
40
|
+
antsibull_nox-0.4.0.dist-info/licenses/LICENSES/GPL-3.0-or-later.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
41
|
+
antsibull_nox-0.4.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|