antsibull-fileutils 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- antsibull_fileutils/__init__.py +14 -0
- antsibull_fileutils/copier.py +150 -0
- antsibull_fileutils/hashing.py +86 -0
- antsibull_fileutils/io.py +119 -0
- antsibull_fileutils/py.typed +0 -0
- antsibull_fileutils/vcs.py +102 -0
- antsibull_fileutils/yaml.py +81 -0
- antsibull_fileutils-1.0.0.dist-info/METADATA +129 -0
- antsibull_fileutils-1.0.0.dist-info/RECORD +11 -0
- antsibull_fileutils-1.0.0.dist-info/WHEEL +4 -0
- antsibull_fileutils-1.0.0.dist-info/licenses/LICENSES/GPL-3.0-or-later.txt +674 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
|
|
2
|
+
# https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
3
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
4
|
+
# SPDX-FileCopyrightText: 2020, Ansible Project
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
Shared code used by tools for building the Ansible distribution
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
__version__ = "1.0.0"
|
|
13
|
+
|
|
14
|
+
__all__ = ("__version__",)
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# Author: Felix Fontein <felix@fontein.de>
|
|
2
|
+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
|
|
3
|
+
# https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
4
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
5
|
+
# SPDX-FileCopyrightText: 2024, Ansible Project
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
Directory and collection copying helpers.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import shutil
|
|
15
|
+
import tempfile
|
|
16
|
+
import typing as t
|
|
17
|
+
|
|
18
|
+
from antsibull_fileutils.vcs import list_git_files
|
|
19
|
+
|
|
20
|
+
if t.TYPE_CHECKING:
|
|
21
|
+
from _typeshed import StrPath
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class CopierError(Exception):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Copier:
|
|
29
|
+
"""
|
|
30
|
+
Allows to copy directories.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, *, log_debug: t.Callable[[str], None] | None = None):
|
|
34
|
+
self._log_debug = log_debug
|
|
35
|
+
|
|
36
|
+
def _do_log_debug(self, msg: str, *args: t.Any) -> None:
|
|
37
|
+
if self._log_debug:
|
|
38
|
+
self._log_debug(msg, *args)
|
|
39
|
+
|
|
40
|
+
def copy(self, from_path: StrPath, to_path: StrPath) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Copy a directory ``from_path`` to a destination ``to_path``.
|
|
43
|
+
|
|
44
|
+
``to_path`` must not exist, but its parent directory must exist.
|
|
45
|
+
"""
|
|
46
|
+
self._do_log_debug(
|
|
47
|
+
"Copying complete directory from {!r} to {!r}", from_path, to_path
|
|
48
|
+
)
|
|
49
|
+
shutil.copytree(from_path, to_path, symlinks=True)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class GitCopier(Copier):
|
|
53
|
+
"""
|
|
54
|
+
Allows to copy directories that are part of a Git repository.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(
|
|
58
|
+
self,
|
|
59
|
+
*,
|
|
60
|
+
git_bin_path: StrPath = "git",
|
|
61
|
+
log_debug: t.Callable[[str], None] | None = None,
|
|
62
|
+
):
|
|
63
|
+
super().__init__(log_debug=log_debug)
|
|
64
|
+
self.git_bin_path = git_bin_path
|
|
65
|
+
|
|
66
|
+
def copy(self, from_path: StrPath, to_path: StrPath) -> None:
|
|
67
|
+
self._do_log_debug("Identifying files not ignored by Git in {!r}", from_path)
|
|
68
|
+
try:
|
|
69
|
+
files = list_git_files(
|
|
70
|
+
from_path, git_bin_path=self.git_bin_path, log_debug=self._log_debug
|
|
71
|
+
)
|
|
72
|
+
except ValueError as exc:
|
|
73
|
+
raise CopierError(
|
|
74
|
+
f"Error while listing files not ignored by Git in {from_path}: {exc}"
|
|
75
|
+
) from exc
|
|
76
|
+
|
|
77
|
+
self._do_log_debug(
|
|
78
|
+
"Copying {} file(s) from {!r} to {!r}", len(files), from_path, to_path
|
|
79
|
+
)
|
|
80
|
+
os.mkdir(to_path, mode=0o700)
|
|
81
|
+
created_directories = set()
|
|
82
|
+
for file in files:
|
|
83
|
+
# Decode filename and check whether the file still exists
|
|
84
|
+
# (deleted files are part of the output)
|
|
85
|
+
file_decoded = file.decode("utf-8")
|
|
86
|
+
src_path = os.path.join(from_path, file_decoded)
|
|
87
|
+
if not os.path.exists(src_path):
|
|
88
|
+
continue
|
|
89
|
+
|
|
90
|
+
# Check whether the directory for this file exists
|
|
91
|
+
directory, _ = os.path.split(file_decoded)
|
|
92
|
+
if directory not in created_directories:
|
|
93
|
+
os.makedirs(os.path.join(to_path, directory), mode=0o700, exist_ok=True)
|
|
94
|
+
created_directories.add(directory)
|
|
95
|
+
|
|
96
|
+
# Copy the file
|
|
97
|
+
dst_path = os.path.join(to_path, file_decoded)
|
|
98
|
+
shutil.copyfile(src_path, dst_path)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class CollectionCopier:
|
|
102
|
+
"""
|
|
103
|
+
Creates a copy of a collection to a place where ``--playbook-dir`` can be used
|
|
104
|
+
to prefer this copy of the collection over any installed ones.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
def __init__(
|
|
108
|
+
self,
|
|
109
|
+
*,
|
|
110
|
+
source_directory: str,
|
|
111
|
+
namespace: str,
|
|
112
|
+
name: str,
|
|
113
|
+
copier: Copier,
|
|
114
|
+
log_debug: t.Callable[[str], None] | None = None,
|
|
115
|
+
):
|
|
116
|
+
self.source_directory = source_directory
|
|
117
|
+
self.namespace = namespace
|
|
118
|
+
self.name = name
|
|
119
|
+
self.copier = copier
|
|
120
|
+
self._log_debug = log_debug
|
|
121
|
+
|
|
122
|
+
self.dir = os.path.realpath(tempfile.mkdtemp(prefix="antsibull-fileutils"))
|
|
123
|
+
|
|
124
|
+
def _do_log_debug(self, msg: str, *args: t.Any) -> None:
|
|
125
|
+
if self._log_debug:
|
|
126
|
+
self._log_debug(msg, *args)
|
|
127
|
+
|
|
128
|
+
def __enter__(self) -> tuple[str, str]:
|
|
129
|
+
try:
|
|
130
|
+
collection_container_dir = os.path.join(
|
|
131
|
+
self.dir, "collections", "ansible_collections", self.namespace
|
|
132
|
+
)
|
|
133
|
+
os.makedirs(collection_container_dir)
|
|
134
|
+
|
|
135
|
+
collection_dir = os.path.join(collection_container_dir, self.name)
|
|
136
|
+
self._do_log_debug("Temporary collection directory: {!r}", collection_dir)
|
|
137
|
+
|
|
138
|
+
self.copier.copy(self.source_directory, collection_dir)
|
|
139
|
+
|
|
140
|
+
self._do_log_debug("Temporary collection directory has been populated")
|
|
141
|
+
return (
|
|
142
|
+
self.dir,
|
|
143
|
+
collection_dir,
|
|
144
|
+
)
|
|
145
|
+
except Exception:
|
|
146
|
+
shutil.rmtree(self.dir, ignore_errors=True)
|
|
147
|
+
raise
|
|
148
|
+
|
|
149
|
+
def __exit__(self, type_, value, traceback_):
|
|
150
|
+
shutil.rmtree(self.dir, ignore_errors=True)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Author: Toshio Kuratomi <tkuratom@redhat.com>
|
|
2
|
+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
|
|
3
|
+
# https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
4
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
5
|
+
# SPDX-FileCopyrightText: 2020, Ansible Project
|
|
6
|
+
"""Functions to help with hashing."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import dataclasses
|
|
11
|
+
import hashlib
|
|
12
|
+
import typing as t
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
|
|
15
|
+
import aiofiles
|
|
16
|
+
|
|
17
|
+
if t.TYPE_CHECKING:
|
|
18
|
+
from _typeshed import StrOrBytesPath
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclasses.dataclass(frozen=True)
|
|
22
|
+
class _AlgorithmData:
|
|
23
|
+
name: str
|
|
24
|
+
algorithm: str
|
|
25
|
+
kwargs: dict[str, t.Any]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
_PREFERRED_HASHES: tuple[_AlgorithmData, ...] = (
|
|
29
|
+
# https://pypi.org/help/#verify-hashes, https://github.com/pypi/warehouse/issues/9628
|
|
30
|
+
_AlgorithmData(name="sha256", algorithm="sha256", kwargs={}),
|
|
31
|
+
_AlgorithmData(name="blake2b_256", algorithm="blake2b", kwargs={"digest_size": 32}),
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def verify_hash(
|
|
36
|
+
filename: StrOrBytesPath,
|
|
37
|
+
hash_digest: str,
|
|
38
|
+
*,
|
|
39
|
+
algorithm: str = "sha256",
|
|
40
|
+
algorithm_kwargs: dict[str, t.Any] | None = None,
|
|
41
|
+
chunksize: int,
|
|
42
|
+
) -> bool:
|
|
43
|
+
"""
|
|
44
|
+
Verify whether a file has a given sha256sum.
|
|
45
|
+
|
|
46
|
+
:arg filename: The file to verify the sha256sum of.
|
|
47
|
+
:arg hash_digest: The hash that is expected.
|
|
48
|
+
:kwarg algorithm: The hash algorithm to use. This must be present in hashlib on this
|
|
49
|
+
system. The default is 'sha256'
|
|
50
|
+
:kwarg algorithm_kwargs: Parameters to provide to the hash algorithm's constructor.
|
|
51
|
+
:returns: True if the hash matches, otherwise False.
|
|
52
|
+
"""
|
|
53
|
+
hasher = getattr(hashlib, algorithm)(**(algorithm_kwargs or {}))
|
|
54
|
+
async with aiofiles.open(filename, "rb") as f:
|
|
55
|
+
while chunk := await f.read(chunksize):
|
|
56
|
+
hasher.update(chunk)
|
|
57
|
+
if hasher.hexdigest() != hash_digest:
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def verify_a_hash(
|
|
64
|
+
filename: StrOrBytesPath,
|
|
65
|
+
hash_digests: Mapping[str, str],
|
|
66
|
+
*,
|
|
67
|
+
chunksize: int,
|
|
68
|
+
) -> bool:
|
|
69
|
+
"""
|
|
70
|
+
Verify whether a file has a given hash, given a set of digests with different algorithms.
|
|
71
|
+
Will only test trustworthy hashes and return ``False`` if none matches.
|
|
72
|
+
|
|
73
|
+
:arg filename: The file to verify the hash of.
|
|
74
|
+
:arg hash_digest: A mapping of hash types to digests.
|
|
75
|
+
:returns: True if the hash matches, otherwise False.
|
|
76
|
+
"""
|
|
77
|
+
for algorithm_data in _PREFERRED_HASHES:
|
|
78
|
+
if algorithm_data.name in hash_digests:
|
|
79
|
+
return await verify_hash(
|
|
80
|
+
filename,
|
|
81
|
+
hash_digests[algorithm_data.name],
|
|
82
|
+
algorithm=algorithm_data.algorithm,
|
|
83
|
+
algorithm_kwargs=algorithm_data.kwargs,
|
|
84
|
+
chunksize=chunksize,
|
|
85
|
+
)
|
|
86
|
+
return False
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Author: Toshio Kuratomi <tkuratom@redhat.com>
|
|
2
|
+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
|
|
3
|
+
# https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
4
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
5
|
+
# SPDX-FileCopyrightText: 2021, Ansible Project
|
|
6
|
+
"""I/O helper functions."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import os.path
|
|
12
|
+
import typing as t
|
|
13
|
+
|
|
14
|
+
import aiofiles
|
|
15
|
+
|
|
16
|
+
if t.TYPE_CHECKING:
|
|
17
|
+
from _typeshed import StrOrBytesPath
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
async def copy_file(
|
|
21
|
+
source_path: StrOrBytesPath,
|
|
22
|
+
dest_path: StrOrBytesPath,
|
|
23
|
+
*,
|
|
24
|
+
check_content: bool = True,
|
|
25
|
+
file_check_content: int = 0,
|
|
26
|
+
chunksize: int,
|
|
27
|
+
) -> bool:
|
|
28
|
+
"""
|
|
29
|
+
Copy content from one file to another.
|
|
30
|
+
|
|
31
|
+
:arg source_path: Source path. Must be a file.
|
|
32
|
+
:arg dest_path: Destination path.
|
|
33
|
+
:kwarg check_content: If ``True`` (default) and ``file_check_content > 0`` and the
|
|
34
|
+
destination file exists, first check whether source and destination are potentially equal
|
|
35
|
+
before actually copying,
|
|
36
|
+
:return: ``True`` if the file was actually copied.
|
|
37
|
+
"""
|
|
38
|
+
if check_content and file_check_content > 0:
|
|
39
|
+
# Check whether the destination file exists and has the same content as the source file,
|
|
40
|
+
# in which case we won't overwrite the destination file
|
|
41
|
+
try:
|
|
42
|
+
stat_d = os.stat(dest_path)
|
|
43
|
+
if stat_d.st_size <= file_check_content:
|
|
44
|
+
stat_s = os.stat(source_path)
|
|
45
|
+
if stat_d.st_size == stat_s.st_size:
|
|
46
|
+
# Read both files and compare
|
|
47
|
+
async with aiofiles.open(source_path, "rb") as f_in:
|
|
48
|
+
content_to_copy = await f_in.read()
|
|
49
|
+
async with aiofiles.open(dest_path, "rb") as f_in:
|
|
50
|
+
existing_content = await f_in.read()
|
|
51
|
+
if content_to_copy == existing_content:
|
|
52
|
+
return False
|
|
53
|
+
# Since we already read the contents of the file to copy, simply write it to
|
|
54
|
+
# the destination instead of reading it again
|
|
55
|
+
async with aiofiles.open(dest_path, "wb") as f_out:
|
|
56
|
+
await f_out.write(content_to_copy)
|
|
57
|
+
return True
|
|
58
|
+
except FileNotFoundError:
|
|
59
|
+
# Destination (or source) file does not exist
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
async with aiofiles.open(source_path, "rb") as f_in:
|
|
63
|
+
async with aiofiles.open(dest_path, "wb") as f_out:
|
|
64
|
+
while chunk := await f_in.read(chunksize):
|
|
65
|
+
await f_out.write(chunk)
|
|
66
|
+
return True
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
async def write_file(
|
|
70
|
+
filename: StrOrBytesPath,
|
|
71
|
+
content: str,
|
|
72
|
+
*,
|
|
73
|
+
file_check_content: int = 0,
|
|
74
|
+
encoding: str = "utf-8",
|
|
75
|
+
) -> bool:
|
|
76
|
+
"""
|
|
77
|
+
Write encoded content to file.
|
|
78
|
+
|
|
79
|
+
:arg filename: The filename to write to.
|
|
80
|
+
:arg content: The content to write to the file.
|
|
81
|
+
:kwarg file_check_content: If > 0 and the file exists and its size in bytes does not exceed this
|
|
82
|
+
value, will read the file and compare it to the encoded content before overwriting.
|
|
83
|
+
:return: ``True`` if the file was actually written.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
content_bytes = content.encode(encoding)
|
|
87
|
+
|
|
88
|
+
print(file_check_content, len(content_bytes))
|
|
89
|
+
if file_check_content > 0 and len(content_bytes) <= file_check_content:
|
|
90
|
+
# Check whether the destination file exists and has the same content as the one we want to
|
|
91
|
+
# write, in which case we won't overwrite the file
|
|
92
|
+
try:
|
|
93
|
+
stat = os.stat(filename)
|
|
94
|
+
if stat.st_size == len(content_bytes):
|
|
95
|
+
# Read file and compare
|
|
96
|
+
async with aiofiles.open(filename, "rb") as f:
|
|
97
|
+
existing_content = await f.read()
|
|
98
|
+
if existing_content == content_bytes:
|
|
99
|
+
return False
|
|
100
|
+
except FileNotFoundError:
|
|
101
|
+
# Destination file does not exist
|
|
102
|
+
pass
|
|
103
|
+
|
|
104
|
+
async with aiofiles.open(filename, "wb") as f:
|
|
105
|
+
await f.write(content_bytes)
|
|
106
|
+
return True
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
async def read_file(filename: StrOrBytesPath, *, encoding: str = "utf-8") -> str:
|
|
110
|
+
"""
|
|
111
|
+
Read the file and decode its contents with the given encoding.
|
|
112
|
+
|
|
113
|
+
:arg filename: The filename to read from.
|
|
114
|
+
:kwarg encoding: The encoding to use.
|
|
115
|
+
"""
|
|
116
|
+
async with aiofiles.open(filename, "r", encoding=encoding) as f:
|
|
117
|
+
content = await f.read()
|
|
118
|
+
|
|
119
|
+
return content
|
|
File without changes
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Author: Felix Fontein <felix@fontein.de>
|
|
2
|
+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
|
|
3
|
+
# https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
4
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
5
|
+
# SPDX-FileCopyrightText: 2024, Ansible Project
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
Git functions.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import subprocess
|
|
14
|
+
import typing as t
|
|
15
|
+
|
|
16
|
+
if t.TYPE_CHECKING:
|
|
17
|
+
from _typeshed import StrPath
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def detect_vcs(
|
|
21
|
+
path: StrPath,
|
|
22
|
+
*,
|
|
23
|
+
git_bin_path: StrPath = "git",
|
|
24
|
+
log_debug: t.Callable[[str], None] | None = None,
|
|
25
|
+
log_info: t.Callable[[str], None] | None = None,
|
|
26
|
+
) -> t.Literal["none", "git"]:
|
|
27
|
+
"""
|
|
28
|
+
Try to detect whether the given ``path`` is part of a VCS repository.
|
|
29
|
+
|
|
30
|
+
NOTE: The return type might be extended in the future. To be on the safe
|
|
31
|
+
side, test for the types you support, and use a fallback for unknown
|
|
32
|
+
values (treat them like ``"none"``).
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def do_log_debug(msg: str, *args: t.Any) -> None:
|
|
36
|
+
if log_debug:
|
|
37
|
+
log_debug(msg, *args)
|
|
38
|
+
|
|
39
|
+
def do_log_info(msg: str, *args: t.Any) -> None:
|
|
40
|
+
if log_info:
|
|
41
|
+
log_info(msg, *args)
|
|
42
|
+
|
|
43
|
+
do_log_debug("Trying to determine whether {!r} is a Git repository", path)
|
|
44
|
+
try:
|
|
45
|
+
result = subprocess.check_output(
|
|
46
|
+
[str(git_bin_path), "-C", path, "rev-parse", "--is-inside-work-tree"],
|
|
47
|
+
text=True,
|
|
48
|
+
encoding="utf-8",
|
|
49
|
+
).strip()
|
|
50
|
+
do_log_debug("Git output: {}", result)
|
|
51
|
+
if result == "true":
|
|
52
|
+
do_log_info("Identified {!r} as a Git repository", path)
|
|
53
|
+
return "git"
|
|
54
|
+
except subprocess.CalledProcessError as exc:
|
|
55
|
+
# This is likely not inside a work tree
|
|
56
|
+
do_log_debug("Git failed: {}", exc)
|
|
57
|
+
except FileNotFoundError as exc:
|
|
58
|
+
# Cannot find git executable
|
|
59
|
+
do_log_debug("Cannot find git: {}", exc)
|
|
60
|
+
|
|
61
|
+
# Fallback: no VCS detected
|
|
62
|
+
do_log_debug("Cannot identify VCS")
|
|
63
|
+
return "none"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def list_git_files(
|
|
67
|
+
directory: StrPath,
|
|
68
|
+
*,
|
|
69
|
+
git_bin_path: StrPath = "git",
|
|
70
|
+
log_debug: t.Callable[[str], None] | None = None,
|
|
71
|
+
) -> list[bytes]:
|
|
72
|
+
"""
|
|
73
|
+
List all files not ignored by git in a directory and subdirectories.
|
|
74
|
+
|
|
75
|
+
Raises ``ValueError`` in case of errors.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def do_log_debug(msg: str, *args) -> None:
|
|
79
|
+
if log_debug:
|
|
80
|
+
log_debug(msg, *args)
|
|
81
|
+
|
|
82
|
+
do_log_debug("Identifying files not ignored by Git in {!r}", directory)
|
|
83
|
+
try:
|
|
84
|
+
result = subprocess.check_output(
|
|
85
|
+
[
|
|
86
|
+
str(git_bin_path),
|
|
87
|
+
"ls-files",
|
|
88
|
+
"-z",
|
|
89
|
+
"--cached",
|
|
90
|
+
"--others",
|
|
91
|
+
"--exclude-standard",
|
|
92
|
+
"--deduplicate",
|
|
93
|
+
],
|
|
94
|
+
cwd=directory,
|
|
95
|
+
).strip(b"\x00")
|
|
96
|
+
if result == b"":
|
|
97
|
+
return []
|
|
98
|
+
return result.split(b"\x00")
|
|
99
|
+
except subprocess.CalledProcessError as exc:
|
|
100
|
+
raise ValueError("Error while running git") from exc
|
|
101
|
+
except FileNotFoundError as exc:
|
|
102
|
+
raise ValueError("Cannot find git executable") from exc
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Author: Felix Fontein <felix@fontein.de>
|
|
2
|
+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
|
|
3
|
+
# https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
4
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
5
|
+
# SPDX-FileCopyrightText: 2021, Ansible Project
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
YAML handling.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import typing as t
|
|
14
|
+
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
_SafeLoader: t.Any
|
|
18
|
+
_SafeDumper: t.Any
|
|
19
|
+
try:
|
|
20
|
+
# use C version if possible for speedup
|
|
21
|
+
from yaml import CSafeDumper as _SafeDumper
|
|
22
|
+
from yaml import CSafeLoader as _SafeLoader
|
|
23
|
+
except ImportError: # pragma: no cover
|
|
24
|
+
from yaml import SafeDumper as _SafeDumper
|
|
25
|
+
from yaml import SafeLoader as _SafeLoader
|
|
26
|
+
|
|
27
|
+
if t.TYPE_CHECKING:
|
|
28
|
+
from _typeshed import StrOrBytesPath, SupportsWrite
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _IndentedDumper(yaml.SafeDumper):
|
|
32
|
+
"""
|
|
33
|
+
Extend YAML dumper to increase indent of list items.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def increase_indent(self, flow=False, indentless=False):
|
|
37
|
+
return super().increase_indent(flow, False)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_yaml_bytes(data: bytes) -> t.Any:
|
|
41
|
+
"""
|
|
42
|
+
Load and parse YAML from given bytes.
|
|
43
|
+
"""
|
|
44
|
+
return yaml.load(data, Loader=_SafeLoader)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def load_yaml_file(path: StrOrBytesPath) -> t.Any:
|
|
48
|
+
"""
|
|
49
|
+
Load and parse YAML file ``path``.
|
|
50
|
+
"""
|
|
51
|
+
with open(path, "rb") as stream:
|
|
52
|
+
return yaml.load(stream, Loader=_SafeLoader)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def store_yaml_file(
|
|
56
|
+
path: StrOrBytesPath, content: t.Any, *, nice: bool = False, sort_keys: bool = True
|
|
57
|
+
) -> None:
|
|
58
|
+
"""
|
|
59
|
+
Store ``content`` as YAML file under ``path``.
|
|
60
|
+
"""
|
|
61
|
+
with open(path, "wb") as stream:
|
|
62
|
+
store_yaml_stream(stream, content, nice=nice, sort_keys=sort_keys)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def store_yaml_stream(
|
|
66
|
+
stream: SupportsWrite, content: t.Any, *, nice: bool = False, sort_keys: bool = True
|
|
67
|
+
) -> None:
|
|
68
|
+
"""
|
|
69
|
+
Dump ``content`` as YAML to an IO ``stream``.
|
|
70
|
+
"""
|
|
71
|
+
dumper = _IndentedDumper if nice else _SafeDumper
|
|
72
|
+
dumper.ignore_aliases = lambda *args: True
|
|
73
|
+
yaml.dump(
|
|
74
|
+
content,
|
|
75
|
+
stream,
|
|
76
|
+
default_flow_style=False,
|
|
77
|
+
encoding="utf-8",
|
|
78
|
+
Dumper=dumper,
|
|
79
|
+
explicit_start=nice,
|
|
80
|
+
sort_keys=sort_keys,
|
|
81
|
+
)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: antsibull-fileutils
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Tools for building the Ansible Distribution
|
|
5
|
+
Project-URL: Source code, https://github.com/ansible-community/antsibull-fileutils
|
|
6
|
+
Project-URL: Code of Conduct, https://docs.ansible.com/ansible/latest/community/code_of_conduct.html
|
|
7
|
+
Project-URL: Bug tracker, https://github.com/ansible-community/antsibull-fileutils/issues
|
|
8
|
+
Author-email: Felix Fontein <felix@fontein.de>
|
|
9
|
+
Maintainer-email: Felix Fontein <felix@fontein.de>, Maxwell G <maxwell@gtmx.me>
|
|
10
|
+
License-Expression: GPL-3.0-or-later AND BSD-2-Clause AND MIT AND PSF-2.0
|
|
11
|
+
License-File: LICENSES/GPL-3.0-or-later.txt
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Framework :: Ansible
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Requires-Dist: aiofiles
|
|
23
|
+
Requires-Dist: pyyaml
|
|
24
|
+
Provides-Extra: codeqa
|
|
25
|
+
Requires-Dist: antsibull-changelog; extra == 'codeqa'
|
|
26
|
+
Requires-Dist: flake8>=6.0.0; extra == 'codeqa'
|
|
27
|
+
Requires-Dist: pylint>=2.15.7; extra == 'codeqa'
|
|
28
|
+
Requires-Dist: reuse; extra == 'codeqa'
|
|
29
|
+
Provides-Extra: coverage
|
|
30
|
+
Requires-Dist: coverage[toml]; extra == 'coverage'
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: antsibull-changelog; extra == 'dev'
|
|
33
|
+
Requires-Dist: black>=24; extra == 'dev'
|
|
34
|
+
Requires-Dist: coverage[toml]; extra == 'dev'
|
|
35
|
+
Requires-Dist: flake8>=6.0.0; extra == 'dev'
|
|
36
|
+
Requires-Dist: isort; extra == 'dev'
|
|
37
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
38
|
+
Requires-Dist: nox; extra == 'dev'
|
|
39
|
+
Requires-Dist: pylint>=2.15.7; extra == 'dev'
|
|
40
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
41
|
+
Requires-Dist: pytest-asyncio>=0.20; extra == 'dev'
|
|
42
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
43
|
+
Requires-Dist: pytest-error-for-skips; extra == 'dev'
|
|
44
|
+
Requires-Dist: reuse; extra == 'dev'
|
|
45
|
+
Requires-Dist: types-aiofiles; extra == 'dev'
|
|
46
|
+
Requires-Dist: types-pyyaml; extra == 'dev'
|
|
47
|
+
Requires-Dist: typing-extensions; extra == 'dev'
|
|
48
|
+
Provides-Extra: formatters
|
|
49
|
+
Requires-Dist: black>=24; extra == 'formatters'
|
|
50
|
+
Requires-Dist: isort; extra == 'formatters'
|
|
51
|
+
Provides-Extra: test
|
|
52
|
+
Requires-Dist: pytest; extra == 'test'
|
|
53
|
+
Requires-Dist: pytest-asyncio>=0.20; extra == 'test'
|
|
54
|
+
Requires-Dist: pytest-cov; extra == 'test'
|
|
55
|
+
Requires-Dist: pytest-error-for-skips; extra == 'test'
|
|
56
|
+
Provides-Extra: typing
|
|
57
|
+
Requires-Dist: mypy; extra == 'typing'
|
|
58
|
+
Requires-Dist: types-aiofiles; extra == 'typing'
|
|
59
|
+
Requires-Dist: types-pyyaml; extra == 'typing'
|
|
60
|
+
Requires-Dist: typing-extensions; extra == 'typing'
|
|
61
|
+
Description-Content-Type: text/markdown
|
|
62
|
+
|
|
63
|
+
<!--
|
|
64
|
+
Copyright (c) Ansible Project
|
|
65
|
+
GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
66
|
+
SPDX-License-Identifier: GPL-3.0-or-later
|
|
67
|
+
-->
|
|
68
|
+
|
|
69
|
+
# antsibull-fileutils -- File Utility Library for Community Ansible Tools
|
|
70
|
+
[](https://matrix.to/#/#antsibull:ansible.com)
|
|
71
|
+
[](https://github.com/ansible-community/antsibull-fileutils/actions/workflows/nox.yml)
|
|
72
|
+
[](https://codecov.io/gh/ansible-community/antsibull-fileutils)
|
|
73
|
+
[](https://api.reuse.software/info/github.com/ansible-community/antsibull-fileutils)
|
|
74
|
+
|
|
75
|
+
This library provides file utils needed by community Ansible tooling.
|
|
76
|
+
|
|
77
|
+
You can find a list of changes in [the antsibull-fileutils changelog](./CHANGELOG.rst).
|
|
78
|
+
|
|
79
|
+
Unless otherwise noted in the code, it is licensed under the terms of the GNU
|
|
80
|
+
General Public License v3 or, at your option, later.
|
|
81
|
+
|
|
82
|
+
antsibull-fileutils is covered by the [Ansible Code of Conduct](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html).
|
|
83
|
+
|
|
84
|
+
## Versioning and compatibility
|
|
85
|
+
|
|
86
|
+
From version 1.0.0 on, antsibull-fileutils sticks to semantic versioning and aims at providing no backwards compatibility breaking changes during a major release cycle. We might make exceptions from this in case of security fixes for vulnerabilities that are severe enough.
|
|
87
|
+
|
|
88
|
+
The current development version is 1.x.y. 1.x.y is developed on the `main` branch.
|
|
89
|
+
|
|
90
|
+
## Development
|
|
91
|
+
|
|
92
|
+
Install and run `nox` to run all tests. That's it for simple contributions!
|
|
93
|
+
`nox` will create virtual environments in `.nox` inside the checked out project
|
|
94
|
+
and install the requirements needed to run the tests there.
|
|
95
|
+
|
|
96
|
+
To run specific tests:
|
|
97
|
+
|
|
98
|
+
1. `nox -e test` to only run unit tests;
|
|
99
|
+
2. `nox -e coverage` to display combined coverage results after running `nox -e
|
|
100
|
+
test`;
|
|
101
|
+
3. `nox -e lint` to run all linters and formatters at once;
|
|
102
|
+
4. `nox -e formatters` to run `isort` and `black`;
|
|
103
|
+
3. `nox -e codeqa` to run `flake8`, `pylint`, `reuse lint`, and `antsibull-changelog lint`;
|
|
104
|
+
6. `nox -e typing` to run `mypy` and `pyre`
|
|
105
|
+
|
|
106
|
+
## Creating a new release:
|
|
107
|
+
|
|
108
|
+
1. Run `nox -e bump -- <version> <release_summary_message>`. This:
|
|
109
|
+
* Bumps the package version in `src/antsibull_fileutils/__init__.py`.
|
|
110
|
+
* Creates `changelogs/fragments/<version>.yml` with a `release_summary` section.
|
|
111
|
+
* Runs `antsibull-changelog release` and adds the changed files to git.
|
|
112
|
+
* Commits with message `Release <version>.` and runs `git tag -a -m 'antsibull-fileutils <version>' <version>`.
|
|
113
|
+
* Runs `hatch build`.
|
|
114
|
+
2. Run `git push` to the appropriate remotes.
|
|
115
|
+
3. Once CI passes on GitHub, run `nox -e publish`. This:
|
|
116
|
+
* Runs `hatch publish`;
|
|
117
|
+
* Bumps the version to `<version>.post0`;
|
|
118
|
+
* Adds the changed file to git and run `git commit -m 'Post-release version bump.'`;
|
|
119
|
+
4. Run `git push --follow-tags` to the appropriate remotes and create a GitHub release.
|
|
120
|
+
|
|
121
|
+
## License
|
|
122
|
+
|
|
123
|
+
Unless otherwise noted in the code, it is licensed under the terms of the GNU
|
|
124
|
+
General Public License v3 or, at your option, later. See
|
|
125
|
+
[LICENSES/GPL-3.0-or-later.txt](https://github.com/ansible-community/antsibull-changelog/tree/main/LICENSE)
|
|
126
|
+
for a copy of the license.
|
|
127
|
+
|
|
128
|
+
The repository follows the [REUSE Specification](https://reuse.software/spec/) for declaring copyright and
|
|
129
|
+
licensing information. The only exception are changelog fragments in ``changelog/fragments/``.
|