file-groups 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- file_groups/__init__.py +3 -1
- file_groups/config_files.py +110 -85
- file_groups/groups.py +49 -45
- file_groups/handler.py +17 -23
- file_groups/handler_compare.py +2 -2
- file_groups/types.py +1 -2
- {file_groups-0.3.0.dist-info → file_groups-0.4.0.dist-info}/METADATA +8 -8
- file_groups-0.4.0.dist-info/RECORD +14 -0
- {file_groups-0.3.0.dist-info → file_groups-0.4.0.dist-info}/WHEEL +1 -1
- file_groups-0.3.0.dist-info/RECORD +0 -14
- {file_groups-0.3.0.dist-info → file_groups-0.4.0.dist-info}/LICENSE.txt +0 -0
- {file_groups-0.3.0.dist-info → file_groups-0.4.0.dist-info}/top_level.txt +0 -0
- {file_groups-0.3.0.dist-info → file_groups-0.4.0.dist-info}/zip-safe +0 -0
file_groups/__init__.py
CHANGED
file_groups/config_files.py
CHANGED
@@ -7,7 +7,7 @@ import itertools
|
|
7
7
|
from pprint import pformat
|
8
8
|
import logging
|
9
9
|
from dataclasses import dataclass
|
10
|
-
from typing import Tuple, Sequence,
|
10
|
+
from typing import Tuple, Sequence, Any
|
11
11
|
|
12
12
|
from appdirs import AppDirs # type: ignore
|
13
13
|
|
@@ -22,31 +22,56 @@ class ConfigException(Exception):
|
|
22
22
|
|
23
23
|
|
24
24
|
@dataclass
|
25
|
-
class
|
26
|
-
"""Hold
|
27
|
-
|
25
|
+
class ProtectConfig():
|
26
|
+
"""Hold global (site or user) protect config."""
|
27
|
+
protect_recursive: set[re.Pattern]
|
28
|
+
|
29
|
+
def __json__(self) -> dict[str, Any]:
|
30
|
+
return {
|
31
|
+
ProtectConfig.__name__: {
|
32
|
+
"protect_recursive": [str(pat) for pat in self.protect_recursive],
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
36
|
+
|
37
|
+
@dataclass
|
38
|
+
class DirConfig(ProtectConfig):
|
39
|
+
"""Hold directory specific protect config."""
|
40
|
+
protect_local: set[re.Pattern]
|
28
41
|
config_dir: Path|None
|
29
|
-
config_files:
|
42
|
+
config_files: list[str]
|
30
43
|
|
31
|
-
def is_protected(self, ff: FsPath):
|
44
|
+
def is_protected(self, ff: FsPath) -> re.Pattern|None:
|
32
45
|
"""If ff id protected by a regex pattern then return the pattern, otherwise return None."""
|
33
46
|
|
34
|
-
|
47
|
+
# _LOG.debug("ff '%s'", ff)
|
48
|
+
for pattern in itertools.chain(self.protect_local, self.protect_recursive):
|
35
49
|
if os.sep in str(pattern):
|
36
|
-
#
|
50
|
+
# _LOG.debug("re.Pattern '%s' has path sep", pattern)
|
37
51
|
assert os.path.isabs(ff), f"Expected absolute path, got '{ff}'"
|
52
|
+
|
53
|
+
# Search against full path
|
38
54
|
if pattern.search(os.fspath(ff)):
|
39
55
|
return pattern
|
40
56
|
|
57
|
+
# Attempt exact match against path relative to , i.e. if pattern starts with '^'.
|
58
|
+
# This makes sense for patterns specified on commandline
|
59
|
+
cwd = os.getcwd()
|
60
|
+
ff_relative = str(Path(ff).relative_to(cwd))
|
61
|
+
# _LOG.debug("ff '%s' relative to start dir'%s'", ff_relative, cwd)
|
62
|
+
|
63
|
+
if pattern.match(ff_relative):
|
64
|
+
return pattern
|
65
|
+
|
41
66
|
elif pattern.search(ff.name):
|
42
67
|
return pattern
|
43
68
|
|
44
69
|
return None
|
45
70
|
|
46
|
-
def __json__(self):
|
71
|
+
def __json__(self) -> dict[str, Any]:
|
47
72
|
return {
|
48
|
-
|
49
|
-
"
|
73
|
+
DirConfig.__name__: super().__json__()[ProtectConfig.__name__] | {
|
74
|
+
"protect_local": [str(pat) for pat in self.protect_local],
|
50
75
|
"config_dir": str(self.config_dir),
|
51
76
|
"config_files": self.config_files,
|
52
77
|
}
|
@@ -113,15 +138,16 @@ class ConfigFiles():
|
|
113
138
|
protect: An optional sequence of regexes to be added to protect[recursive] for all directories.
|
114
139
|
ignore_config_dirs_config_files: Ignore config files in standard config directories.
|
115
140
|
ignore_per_directory_config_files: Ignore config files in collected directories.
|
116
|
-
remember_configs: Store loaded and merged configs in `
|
141
|
+
remember_configs: Store loaded and merged configs in `per_dir_configs` member variable.
|
117
142
|
app_dirs: Provide your own instance of AppDirs in addition to or as a replacement of the default to add config file names and path.
|
118
143
|
Configuration from later entries have higher precedence.
|
119
144
|
Note that if no AppDirs are specified, no config files will be loaded, neither from config dirs, nor from collected directories.
|
120
145
|
See: https://pypi.org/project/appdirs/
|
121
146
|
|
122
147
|
Members:
|
123
|
-
|
124
|
-
|
148
|
+
conf_file_names: File names which are config files.
|
149
|
+
remember_configs: Whether per directory resolved/merged configs are stored in `per_dir_configs`.
|
150
|
+
per_dir_configs: Mapping from dir name to directory specific config dict. Only if remember_configs is True.
|
125
151
|
"""
|
126
152
|
|
127
153
|
default_appdirs: AppDirs = AppDirs("file_groups", "Hupfeldt_IT")
|
@@ -131,47 +157,43 @@ class ConfigFiles():
|
|
131
157
|
_valid_dir_protect_scopes = ("local", "recursive")
|
132
158
|
_valid_config_dir_protect_scopes = ("local", "recursive", "global")
|
133
159
|
|
134
|
-
def __init__(
|
160
|
+
def __init__( # pylint: disable=too-many-positional-arguments,too-many-arguments
|
135
161
|
self, protect: Sequence[re.Pattern] = (),
|
136
|
-
ignore_config_dirs_config_files=False, ignore_per_directory_config_files=False, remember_configs=True,
|
162
|
+
ignore_config_dirs_config_files: bool = False, ignore_per_directory_config_files: bool =False, remember_configs: bool =True,
|
137
163
|
app_dirs: Sequence[AppDirs]|None = None,
|
138
164
|
*,
|
139
165
|
config_file: Path|None = None,
|
140
166
|
):
|
141
167
|
super().__init__()
|
142
168
|
|
143
|
-
self._global_config =
|
144
|
-
"local": set(),
|
145
|
-
"recursive": set(protect),
|
146
|
-
}, None, ())
|
147
|
-
|
169
|
+
self._global_config = ProtectConfig(set(protect))
|
148
170
|
self.remember_configs = remember_configs
|
149
|
-
self.per_dir_configs: dict[
|
171
|
+
self.per_dir_configs: dict[Path, DirConfig] = {} # key is abs_dir_path
|
150
172
|
self.ignore_per_directory_config_files = ignore_per_directory_config_files
|
151
173
|
|
152
174
|
app_dirs = app_dirs or (ConfigFiles.default_appdirs,)
|
153
|
-
self.
|
154
|
-
_LOG.debug("Conf file names: %s", self.
|
175
|
+
self.conf_file_name_pairs = tuple((apd.appname + ".conf", "." + apd.appname + ".conf") for apd in app_dirs)
|
176
|
+
_LOG.debug("Conf file names: %s", self.conf_file_name_pairs)
|
177
|
+
self.conf_file_names = list(itertools.chain.from_iterable(self.conf_file_name_pairs))
|
178
|
+
|
179
|
+
self.ignore_config_dirs_config_files = ignore_config_dirs_config_files
|
155
180
|
self.config_dirs = []
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
self.config_dirs.append(appd.user_config_dir)
|
181
|
+
for appd in app_dirs:
|
182
|
+
self.config_dirs.extend(appd.site_config_dir.split(':'))
|
183
|
+
for appd in app_dirs:
|
184
|
+
self.config_dirs.append(appd.user_config_dir)
|
161
185
|
|
162
186
|
self.config_file = config_file
|
163
187
|
|
164
188
|
# self.default_config_file_example = self.default_config_file.with_suffix('.example.py')
|
165
189
|
|
166
|
-
def
|
167
|
-
self, conf_dir: Path, conf_file_name_pair: Sequence[str],
|
168
|
-
) -> Tuple[dict[str,
|
169
|
-
"""Read config file
|
190
|
+
def _read_and_eval_config_file_for_one_appname(
|
191
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], ignore_config_files: bool
|
192
|
+
) -> Tuple[dict[str, Any], Path|None]:
|
193
|
+
"""Read config file.
|
170
194
|
|
171
195
|
Error if config files are found both with and withput '.' prefix.
|
172
|
-
|
173
|
-
|
174
|
-
Return: merged config dict with compiled regexes, config file name. If no config files is found, then return inherited parent conf and None.
|
196
|
+
Return: Config dict, config file name, or if no config files is found, None, None.
|
175
197
|
"""
|
176
198
|
|
177
199
|
assert conf_dir.is_absolute()
|
@@ -180,27 +202,41 @@ class ConfigFiles():
|
|
180
202
|
match [conf_dir/cfn for cfn in conf_file_name_pair if (conf_dir/cfn).exists()]:
|
181
203
|
case []:
|
182
204
|
_LOG.debug("No config file in directory %s", conf_dir)
|
183
|
-
|
184
|
-
"local": set(),
|
185
|
-
"recursive": parent_conf.protect["recursive"]
|
186
|
-
}
|
187
|
-
return no_conf_file, None
|
205
|
+
return {}, None
|
188
206
|
|
189
207
|
case [conf_file]:
|
190
208
|
if ignore_config_files:
|
191
209
|
_LOG.debug("Ignoring config file: %s", conf_file)
|
192
|
-
return
|
210
|
+
return {}, None
|
193
211
|
|
194
212
|
_LOG.debug("Read config file: %s", conf_file)
|
195
213
|
with open(conf_file, encoding="utf-8") as fh:
|
196
214
|
new_config = ast.literal_eval(fh.read())
|
197
215
|
_LOG.debug("%s", pformat(new_config))
|
216
|
+
return new_config, conf_file
|
198
217
|
|
199
218
|
case config_files:
|
200
219
|
msg = f"More than one config file in dir '{conf_dir}': {[cf.name for cf in config_files]}."
|
201
220
|
_LOG.debug("%s", msg)
|
202
221
|
raise ConfigException(msg)
|
203
222
|
|
223
|
+
def _read_and_validate_config_file_for_one_appname(
|
224
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], valid_protect_scopes: Tuple[str, ...], ignore_config_files: bool
|
225
|
+
) -> Tuple[dict[str, set[re.Pattern]], Path|None]:
|
226
|
+
"""Read config file, validate keys and compile regexes.
|
227
|
+
|
228
|
+
Error if config files are found both with and withput '.' prefix.
|
229
|
+
|
230
|
+
Return: merged config dict with compiled regexes, config file name. If no config files is found, then return empty sets and None.
|
231
|
+
"""
|
232
|
+
|
233
|
+
new_config, conf_file = self._read_and_eval_config_file_for_one_appname(conf_dir, conf_file_name_pair, ignore_config_files)
|
234
|
+
if not new_config:
|
235
|
+
return {
|
236
|
+
"local": set(),
|
237
|
+
"recursive": set(),
|
238
|
+
}, None
|
239
|
+
|
204
240
|
try:
|
205
241
|
protect_conf: dict[str, set[re.Pattern]] = new_config[self._fg_key][self._protect_key]
|
206
242
|
except KeyError as ex:
|
@@ -213,8 +249,6 @@ class ConfigFiles():
|
|
213
249
|
raise ConfigException(msg)
|
214
250
|
|
215
251
|
protect_conf[key] = set(re.compile(pattern) for pattern in val)
|
216
|
-
if key == "recursive":
|
217
|
-
protect_conf[key].update(parent_conf.protect[key])
|
218
252
|
|
219
253
|
for key in self._valid_dir_protect_scopes: # Do NOT use the 'valid_protect_scopes' argument here
|
220
254
|
protect_conf.setdefault(key, set())
|
@@ -223,55 +257,34 @@ class ConfigFiles():
|
|
223
257
|
if _LOG.isEnabledFor(lvl):
|
224
258
|
_LOG.log(lvl, "Merged directory config:\n%s", pformat(new_config))
|
225
259
|
|
226
|
-
return protect_conf, conf_file
|
227
|
-
|
228
|
-
def _read_and_validate_config_files(
|
229
|
-
self, conf_dir: Path, parent_conf: DirConfig, valid_protect_scopes: Tuple[str, ...], ignore_config_files: bool) -> DirConfig:
|
230
|
-
cfg_merge: dict[str, set[re.Pattern]] = {}
|
231
|
-
cfg_files: list[str] = []
|
232
|
-
for conf_file_name_pair in self.conf_file_names:
|
233
|
-
cfg, cfg_file = self._read_and_validate_config_file_for_one_appname(
|
234
|
-
conf_dir, conf_file_name_pair, parent_conf, valid_protect_scopes, ignore_config_files)
|
235
|
-
for key, val in cfg.items():
|
236
|
-
cfg_merge.setdefault(key, set()).update(val)
|
237
|
-
if cfg_file:
|
238
|
-
cfg_files.append(cfg_file)
|
239
|
-
|
240
|
-
return DirConfig(cfg_merge, conf_dir, cfg_files)
|
260
|
+
return protect_conf, conf_file
|
241
261
|
|
242
262
|
def load_config_dir_files(self) -> None:
|
243
263
|
"""Load config files from platform standard directories and specified config file, if any."""
|
244
264
|
|
245
|
-
|
246
|
-
|
247
|
-
|
265
|
+
if not self.ignore_config_dirs_config_files:
|
266
|
+
_LOG.debug("config_dirs: %s", self.config_dirs)
|
267
|
+
for conf_dir in self.config_dirs:
|
268
|
+
conf_dir = Path(conf_dir)
|
269
|
+
if not conf_dir.exists():
|
270
|
+
continue
|
248
271
|
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
del new_config.protect['global']
|
254
|
-
except KeyError:
|
255
|
-
pass
|
256
|
-
|
257
|
-
_LOG.debug("config_dirs: %s", self.config_dirs)
|
258
|
-
for conf_dir in self.config_dirs:
|
259
|
-
conf_dir = Path(conf_dir)
|
260
|
-
if not conf_dir.exists():
|
261
|
-
continue
|
262
|
-
|
263
|
-
new_config = self._read_and_validate_config_files(conf_dir, self._global_config, self._valid_config_dir_protect_scopes, False)
|
264
|
-
merge_one_config_to_global(conf_dir, new_config)
|
272
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
273
|
+
cfg, _ = self._read_and_validate_config_file_for_one_appname(
|
274
|
+
conf_dir, conf_file_name_pair, self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
275
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
265
276
|
|
266
277
|
if self.config_file:
|
278
|
+
_LOG.debug("specified config_file: %s", self.config_file)
|
267
279
|
conf_dir = self.config_file.parent.absolute()
|
268
280
|
conf_name = self.config_file.name
|
269
|
-
cfg,
|
270
|
-
conf_dir, (conf_name,), self.
|
271
|
-
if not
|
281
|
+
cfg, fpath = self._read_and_validate_config_file_for_one_appname(
|
282
|
+
conf_dir, (conf_name,), self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
283
|
+
if not fpath:
|
272
284
|
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(self.config_file))
|
285
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
273
286
|
|
274
|
-
|
287
|
+
_LOG.debug("Merged global config:\n %s", self._global_config)
|
275
288
|
|
276
289
|
def dir_config(self, conf_dir: Path, parent_conf: DirConfig|None) -> DirConfig:
|
277
290
|
"""Read and merge config file from directory 'conf_dir' with 'parent_conf'.
|
@@ -279,12 +292,24 @@ class ConfigFiles():
|
|
279
292
|
If directory has no parent in the file_groups included dirs, then None should be supplied as parent_conf.
|
280
293
|
"""
|
281
294
|
|
282
|
-
|
283
|
-
|
295
|
+
cfg_merge_local: set[re.Pattern] = set()
|
296
|
+
cfg_merge_recursive: set[re.Pattern] = set()
|
297
|
+
cfg_files: list[str] = []
|
298
|
+
|
299
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
300
|
+
cfg, cfg_file = self._read_and_validate_config_file_for_one_appname(
|
301
|
+
conf_dir, conf_file_name_pair, self._valid_dir_protect_scopes, self.ignore_per_directory_config_files)
|
302
|
+
cfg_merge_local.update(cfg.get("local", set()))
|
303
|
+
cfg_merge_recursive.update(cfg.get("recursive", set()))
|
304
|
+
if cfg_file:
|
305
|
+
cfg_files.append(cfg_file.name)
|
306
|
+
|
307
|
+
parent_protect_recursive = parent_conf.protect_recursive if parent_conf else self._global_config.protect_recursive
|
308
|
+
new_config = DirConfig(cfg_merge_recursive | parent_protect_recursive, cfg_merge_local, conf_dir, cfg_files)
|
284
309
|
_LOG.debug("new_config:\n %s", new_config)
|
285
310
|
|
286
311
|
if self.remember_configs:
|
287
|
-
self.per_dir_configs[
|
312
|
+
self.per_dir_configs[conf_dir] = new_config
|
288
313
|
# _LOG.debug("per_dir_configs:\n %s", self.per_dir_configs)
|
289
314
|
|
290
315
|
return new_config
|
file_groups/groups.py
CHANGED
@@ -7,7 +7,7 @@ from dataclasses import dataclass
|
|
7
7
|
from itertools import chain
|
8
8
|
from enum import Enum
|
9
9
|
import logging
|
10
|
-
from typing import Sequence
|
10
|
+
from typing import Sequence, cast
|
11
11
|
|
12
12
|
from .config_files import DirConfig, ConfigFiles
|
13
13
|
|
@@ -35,14 +35,14 @@ class _Group():
|
|
35
35
|
num_directories: int = 0
|
36
36
|
num_directory_symlinks: int = 0
|
37
37
|
|
38
|
-
def add_entry_match(self, entry: DirEntry):
|
38
|
+
def add_entry_match(self, entry: DirEntry) -> None:
|
39
39
|
"""Abstract, but abstract and dataclass does not work with mypy. https://github.com/python/mypy/issues/500"""
|
40
40
|
|
41
41
|
@dataclass
|
42
42
|
class _IncludeMatchGroup(_Group):
|
43
43
|
include: re.Pattern|None = None
|
44
44
|
|
45
|
-
def add_entry_match(self, entry: DirEntry):
|
45
|
+
def add_entry_match(self, entry: DirEntry) -> None:
|
46
46
|
if not self.include:
|
47
47
|
self.files[entry.path] = entry
|
48
48
|
return
|
@@ -58,7 +58,7 @@ class _IncludeMatchGroup(_Group):
|
|
58
58
|
class _ExcludeMatchGroup(_Group):
|
59
59
|
exclude: re.Pattern|None = None
|
60
60
|
|
61
|
-
def add_entry_match(self, entry: DirEntry):
|
61
|
+
def add_entry_match(self, entry: DirEntry) -> None:
|
62
62
|
if not self.exclude:
|
63
63
|
self.files[entry.path] = entry
|
64
64
|
return
|
@@ -89,7 +89,7 @@ class FileGroups():
|
|
89
89
|
Note: Since these files are excluded from protection, it means they er NOT protected!
|
90
90
|
work_include: ONLY include files matching regex in the may_work_on files (does not apply to symlinks). Default: Include ALL.
|
91
91
|
|
92
|
-
config_files: Load config files. See config_files.ConfigFiles.
|
92
|
+
config_files: Load config files. See config_files.ConfigFiles. Note that the default 'None' means use the `config_files.ConfigFiles` class with default arguments.
|
93
93
|
"""
|
94
94
|
|
95
95
|
def __init__(
|
@@ -169,8 +169,46 @@ class FileGroups():
|
|
169
169
|
|
170
170
|
checked_dirs: set[str] = set()
|
171
171
|
|
172
|
-
def
|
173
|
-
"""
|
172
|
+
def handle_entry(abs_dir_path: str, group: _Group, other_group: _Group, dir_config: DirConfig, entry: DirEntry) -> None:
|
173
|
+
"""Put entry in the correct group. Call 'find_group' if entry is a directory."""
|
174
|
+
if group.typ is GroupType.MAY_WORK_ON:
|
175
|
+
# Check for match against configured protect patterns, if match, then the file must got to protect group instead
|
176
|
+
pattern = dir_config.is_protected(entry)
|
177
|
+
if pattern:
|
178
|
+
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
179
|
+
group, other_group = other_group, group
|
180
|
+
|
181
|
+
if entry.is_dir(follow_symlinks=False):
|
182
|
+
if entry.path in other_group.dirs:
|
183
|
+
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
184
|
+
find_group(entry.path, other_group, group, dir_config)
|
185
|
+
return
|
186
|
+
|
187
|
+
find_group(entry.path, group, other_group, dir_config)
|
188
|
+
return
|
189
|
+
|
190
|
+
if entry.name in self.config_files.conf_file_names:
|
191
|
+
return
|
192
|
+
|
193
|
+
if entry.is_symlink():
|
194
|
+
# cast: https://github.com/python/mypy/issues/11964
|
195
|
+
points_to = os.readlink(cast(str, entry))
|
196
|
+
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
197
|
+
|
198
|
+
if entry.is_dir(follow_symlinks=True):
|
199
|
+
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", group.typ.name, entry.path, points_to)
|
200
|
+
group.num_directory_symlinks += 1
|
201
|
+
return
|
202
|
+
|
203
|
+
group.symlinks[entry.path] = entry
|
204
|
+
group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
205
|
+
return
|
206
|
+
|
207
|
+
_LOG.debug("find %s - entry name: %s", group.typ.name, entry.name)
|
208
|
+
group.add_entry_match(entry)
|
209
|
+
|
210
|
+
def find_group(abs_dir_path: str, group: _Group, other_group: _Group, parent_conf: DirConfig|None) -> None:
|
211
|
+
"""Recursively find all files belonging to 'group'"""
|
174
212
|
_LOG.debug("find %s: %s", group.typ.name, abs_dir_path)
|
175
213
|
if abs_dir_path in checked_dirs:
|
176
214
|
_LOG.debug("directory already checked")
|
@@ -180,48 +218,14 @@ class FileGroups():
|
|
180
218
|
dir_config = self.config_files.dir_config(Path(abs_dir_path), parent_conf)
|
181
219
|
|
182
220
|
for entry in os.scandir(abs_dir_path):
|
183
|
-
|
184
|
-
if entry.path in other_group.dirs:
|
185
|
-
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
186
|
-
find_group(entry.path, other_group, group, dir_config)
|
187
|
-
continue
|
188
|
-
|
189
|
-
find_group(entry.path, group, other_group, dir_config)
|
190
|
-
continue
|
191
|
-
|
192
|
-
if entry.name in dir_config.config_files:
|
193
|
-
continue
|
194
|
-
|
195
|
-
current_group = group
|
196
|
-
if group.typ is GroupType.MAY_WORK_ON:
|
197
|
-
# We need to check for match against configured protect patterns, if match, then the file must got to protect group instead
|
198
|
-
pattern = dir_config.is_protected(entry)
|
199
|
-
if pattern:
|
200
|
-
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
201
|
-
current_group = other_group
|
202
|
-
|
203
|
-
if entry.is_symlink():
|
204
|
-
points_to = os.readlink(entry)
|
205
|
-
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
206
|
-
|
207
|
-
if entry.is_dir(follow_symlinks=True):
|
208
|
-
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", current_group.typ.name, entry.path, points_to)
|
209
|
-
current_group.num_directory_symlinks += 1
|
210
|
-
continue
|
211
|
-
|
212
|
-
current_group.symlinks[entry.path] = entry
|
213
|
-
current_group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
214
|
-
continue
|
215
|
-
|
216
|
-
_LOG.debug("find %s - entry name: %s", current_group.typ.name, entry.name)
|
217
|
-
current_group.add_entry_match(entry)
|
221
|
+
handle_entry(abs_dir_path, group, other_group, dir_config, entry)
|
218
222
|
|
219
223
|
checked_dirs.add(abs_dir_path)
|
220
224
|
|
221
225
|
for any_dir in sorted(chain(self.must_protect.dirs, self.may_work_on.dirs), key=lambda dd: len(Path(dd).parts)):
|
222
226
|
parent_dir = Path(any_dir)
|
223
227
|
while len(parent_dir.parts) > 1:
|
224
|
-
parent_conf = self.config_files.per_dir_configs.get(
|
228
|
+
parent_conf = self.config_files.per_dir_configs.get(parent_dir)
|
225
229
|
if parent_conf:
|
226
230
|
break
|
227
231
|
|
@@ -234,7 +238,7 @@ class FileGroups():
|
|
234
238
|
else:
|
235
239
|
find_group(any_dir, self.may_work_on, self.must_protect, parent_conf)
|
236
240
|
|
237
|
-
def dump(self):
|
241
|
+
def dump(self) -> None:
|
238
242
|
"""Log collected files. This may be A LOT of output for large directories."""
|
239
243
|
|
240
244
|
log = _LOG.getChild("dump")
|
@@ -276,7 +280,7 @@ class FileGroups():
|
|
276
280
|
|
277
281
|
log.log(lvl, "")
|
278
282
|
|
279
|
-
def stats(self):
|
283
|
+
def stats(self) -> None:
|
280
284
|
"""Log collection numbers."""
|
281
285
|
log = _LOG.getChild("stats")
|
282
286
|
lvl = logging.INFO
|
file_groups/handler.py
CHANGED
@@ -2,13 +2,12 @@ import os
|
|
2
2
|
from pathlib import Path
|
3
3
|
import shutil
|
4
4
|
import re
|
5
|
-
from contextlib import contextmanager
|
6
5
|
import logging
|
7
6
|
from typing import Sequence
|
8
7
|
|
9
8
|
from .groups import FileGroups
|
10
9
|
from .config_files import ConfigFiles
|
11
|
-
|
10
|
+
from .types import FsPath
|
12
11
|
|
13
12
|
_LOG = logging.getLogger(__name__)
|
14
13
|
|
@@ -28,14 +27,14 @@ class FileHandler(FileGroups):
|
|
28
27
|
they could have logically been made to point to a file in a protect dir.
|
29
28
|
"""
|
30
29
|
|
31
|
-
def __init__(
|
30
|
+
def __init__( # pylint: disable=too-many-arguments
|
32
31
|
self,
|
33
32
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path],
|
34
33
|
*,
|
35
34
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
36
35
|
config_files: ConfigFiles|None = None,
|
37
36
|
dry_run: bool,
|
38
|
-
delete_symlinks_instead_of_relinking=False):
|
37
|
+
delete_symlinks_instead_of_relinking: bool =False):
|
39
38
|
super().__init__(
|
40
39
|
protect_dirs_seq=protect_dirs_seq, work_dirs_seq=work_dirs_seq,
|
41
40
|
protect_exclude=protect_exclude, work_include=work_include,
|
@@ -55,7 +54,7 @@ class FileHandler(FileGroups):
|
|
55
54
|
self.num_moved = 0
|
56
55
|
self.num_relinked = 0
|
57
56
|
|
58
|
-
def reset(self):
|
57
|
+
def reset(self) -> None:
|
59
58
|
"""Reset internal housekeeping of deleted/renamed/moved files.
|
60
59
|
|
61
60
|
This makes it possible to do a 'dry_run' and an actual run without collecting files again.
|
@@ -69,7 +68,7 @@ class FileHandler(FileGroups):
|
|
69
68
|
self.num_moved = 0
|
70
69
|
self.num_relinked = 0
|
71
70
|
|
72
|
-
def _no_symlink_check_registered_delete(self, delete_path: str):
|
71
|
+
def _no_symlink_check_registered_delete(self, delete_path: str) -> None:
|
73
72
|
"""Does a registered delete without checking for symlinks, so that we can use this in the symlink handling."""
|
74
73
|
assert isinstance(delete_path, str)
|
75
74
|
assert os.path.isabs(delete_path), f"Expected absolute path, got '{delete_path}'"
|
@@ -84,7 +83,7 @@ class FileHandler(FileGroups):
|
|
84
83
|
if delete_path in self.may_work_on.symlinks:
|
85
84
|
self.deleted_symlinks.add(delete_path)
|
86
85
|
|
87
|
-
def _handle_single_symlink_chain(self, symlnk_path: str, keep_path):
|
86
|
+
def _handle_single_symlink_chain(self, symlnk_path: str, keep_path: str|FsPath|None) -> None:
|
88
87
|
"""TODO doc - Symlink will only be deleted if it is in self.may_work_on.files."""
|
89
88
|
|
90
89
|
assert os.path.isabs(symlnk_path), f"Expected an absolute path, got '{symlnk_path}'"
|
@@ -141,8 +140,12 @@ class FileHandler(FileGroups):
|
|
141
140
|
|
142
141
|
self.num_relinked += 1
|
143
142
|
|
144
|
-
def _fix_symlinks_to_deleted_or_moved_files(self, from_path: str, to_path):
|
145
|
-
"""Any symlinks pointing to 'from_path' will be change to point to 'to_path'
|
143
|
+
def _fix_symlinks_to_deleted_or_moved_files(self, from_path: str, to_path: str|FsPath|None) -> None:
|
144
|
+
"""Any symlinks pointing to 'from_path' will be change to point to 'to_path' or deleted.
|
145
|
+
|
146
|
+
If 'to_path' is None then delete full chains of symlinks in 'may_work_on' dirs.
|
147
|
+
"""
|
148
|
+
|
146
149
|
_LOG.debug("_fix_symlinks_to_deleted_or_moved_files(self, %s, %s)", from_path, to_path)
|
147
150
|
|
148
151
|
for symlnk in self.must_protect.symlinks_by_abs_points_to.get(from_path, ()):
|
@@ -153,13 +156,13 @@ class FileHandler(FileGroups):
|
|
153
156
|
_LOG.debug("_fix_symlinks_to_deleted_or_moved_files, may_work_on symlink: '%s'.", symlnk)
|
154
157
|
self._handle_single_symlink_chain(os.fspath(symlnk), to_path)
|
155
158
|
|
156
|
-
def registered_delete(self, delete_path: str, corresponding_keep_path) -> Path|None:
|
159
|
+
def registered_delete(self, delete_path: str, corresponding_keep_path: str|FsPath|None) -> Path|None:
|
157
160
|
"""Return `corresponding_keep_path` as absolute Path"""
|
158
161
|
self._no_symlink_check_registered_delete(delete_path)
|
159
162
|
self._fix_symlinks_to_deleted_or_moved_files(delete_path, corresponding_keep_path)
|
160
163
|
return Path(corresponding_keep_path).absolute() if corresponding_keep_path else None
|
161
164
|
|
162
|
-
def _registered_move_or_rename(self, from_path: str, to_path, *, is_move) -> Path:
|
165
|
+
def _registered_move_or_rename(self, from_path: str, to_path: str|FsPath, *, is_move: bool) -> Path:
|
163
166
|
"""Return `to_path` as absolute Path"""
|
164
167
|
assert isinstance(from_path, str)
|
165
168
|
assert os.path.isabs(from_path), f"Expected absolute path, got '{from_path}'"
|
@@ -189,20 +192,18 @@ class FileHandler(FileGroups):
|
|
189
192
|
self._fix_symlinks_to_deleted_or_moved_files(from_path, to_path)
|
190
193
|
return res
|
191
194
|
|
192
|
-
def registered_move(self, from_path: str, to_path) -> Path:
|
195
|
+
def registered_move(self, from_path: str, to_path: str|FsPath) -> Path:
|
193
196
|
"""Return `to_path` as absolute Path"""
|
194
197
|
return self._registered_move_or_rename(from_path, to_path, is_move=True)
|
195
198
|
|
196
|
-
def registered_rename(self, from_path: str, to_path) -> Path:
|
199
|
+
def registered_rename(self, from_path: str, to_path: str|FsPath) -> Path:
|
197
200
|
"""Return `to_path` as absolute Path"""
|
198
201
|
return self._registered_move_or_rename(from_path, to_path, is_move=False)
|
199
202
|
|
200
|
-
|
201
|
-
def stats(self):
|
203
|
+
def stats(self) -> None:
|
202
204
|
log = _LOG.getChild("stats")
|
203
205
|
lvl = logging.INFO
|
204
206
|
if not log.isEnabledFor(lvl):
|
205
|
-
yield
|
206
207
|
return
|
207
208
|
|
208
209
|
prefix = ''
|
@@ -218,10 +219,3 @@ class FileHandler(FileGroups):
|
|
218
219
|
log.log(lvl, "%srenamed: %s", prefix, self.num_renamed)
|
219
220
|
log.log(lvl, "%smoved: %s", prefix, self.num_moved)
|
220
221
|
log.log(lvl, "%srelinked: %s", prefix, self.num_relinked)
|
221
|
-
|
222
|
-
try:
|
223
|
-
yield
|
224
|
-
|
225
|
-
finally:
|
226
|
-
if self.dry_run:
|
227
|
-
log.log(lvl, "*** DRY RUN ***")
|
file_groups/handler_compare.py
CHANGED
@@ -22,14 +22,14 @@ class FileHandlerCompare(FileHandler):
|
|
22
22
|
fcmp: Object providing compare function.
|
23
23
|
"""
|
24
24
|
|
25
|
-
def __init__(
|
25
|
+
def __init__( # pylint: disable=too-many-arguments
|
26
26
|
self,
|
27
27
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path], fcmp: CompareFiles,
|
28
28
|
*,
|
29
29
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
30
30
|
config_files: ConfigFiles|None = None,
|
31
31
|
dry_run: bool,
|
32
|
-
delete_symlinks_instead_of_relinking=False):
|
32
|
+
delete_symlinks_instead_of_relinking: bool = False):
|
33
33
|
super().__init__(
|
34
34
|
protect_dirs_seq=protect_dirs_seq, work_dirs_seq=work_dirs_seq,
|
35
35
|
protect_exclude=protect_exclude, work_include=work_include,
|
file_groups/types.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
2
|
-
Name:
|
3
|
-
Version: 0.
|
1
|
+
Metadata-Version: 2.2
|
2
|
+
Name: file_groups
|
3
|
+
Version: 0.4.0
|
4
4
|
Summary: Group files into 'protect' and 'work_on' and provide operations for safe delete/move and symlink handling.
|
5
5
|
Home-page: https://github.com/lhupfeldt/file_groups.git
|
6
6
|
Author: Lars Hupfeldt Nielsen
|
@@ -11,16 +11,16 @@ Classifier: Intended Audience :: Developers
|
|
11
11
|
Classifier: License :: OSI Approved :: BSD License
|
12
12
|
Classifier: Natural Language :: English
|
13
13
|
Classifier: Operating System :: OS Independent
|
14
|
-
Classifier: Programming Language :: Python :: 3.
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
17
17
|
Classifier: Topic :: Software Development :: Libraries
|
18
|
-
Requires-Python: >=3.
|
18
|
+
Requires-Python: >=3.11
|
19
19
|
Description-Content-Type: text/x-rst
|
20
20
|
License-File: LICENSE.txt
|
21
|
-
Requires-Dist: appdirs
|
21
|
+
Requires-Dist: appdirs>=1.4.4
|
22
22
|
Provides-Extra: dev
|
23
|
-
Requires-Dist: nox
|
23
|
+
Requires-Dist: nox; extra == "dev"
|
24
24
|
|
25
25
|
Library for grouping files into sets of 'work-on' and 'protect' based on arbitrarily nested directories.
|
26
26
|
|
@@ -0,0 +1,14 @@
|
|
1
|
+
file_groups/__init__.py,sha256=FAl1D2-Fm2nQgXdM1rttTymLby24pN3ogmbS5dri770,111
|
2
|
+
file_groups/compare_files.py,sha256=38EfZzpEpj0wH7SUruPxgyF_W8eqCbIjG6yUOx8hcFA,434
|
3
|
+
file_groups/config_files.py,sha256=-sEyPkcds9gZEVvIJVnhWqXk9H4X20w_TT8BQg3_xzM,14101
|
4
|
+
file_groups/groups.py,sha256=fMn85U2QPF39sgYzma806XgtUOozOsA-CUNRjB2mC5I,11433
|
5
|
+
file_groups/handler.py,sha256=QerrcMtLEzl5NUOpwX2QbbngBUl8ssfYemcbGbaV208,10156
|
6
|
+
file_groups/handler_compare.py,sha256=zVRHdqP-2Z7u3D2jQ_6_HLF3SOUk0mSyJ7xiS5Y0Hig,2182
|
7
|
+
file_groups/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
file_groups/types.py,sha256=IEDtGNxhQhQaWn8sTHn5-jbpM-dbba7wiIfaud1XSXc,74
|
9
|
+
file_groups-0.4.0.dist-info/LICENSE.txt,sha256=NZc9ictLEwfoL4ywpAbLbd-n02KETGHLYQU820TJD_Q,1494
|
10
|
+
file_groups-0.4.0.dist-info/METADATA,sha256=hCMNdysRiHy14tnW7TKa4vxO0BKBRO_RlMHdISRDJe4,1510
|
11
|
+
file_groups-0.4.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
12
|
+
file_groups-0.4.0.dist-info/top_level.txt,sha256=mAdRf0R2TA8tpH7ftHzXP7VMRhw07p7B2mI8QKpDnjU,12
|
13
|
+
file_groups-0.4.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
14
|
+
file_groups-0.4.0.dist-info/RECORD,,
|
@@ -1,14 +0,0 @@
|
|
1
|
-
file_groups/__init__.py,sha256=1OO3tLnRnIe51VfZSQz0sqUaUP-xS7oHxiSexnKwpEY,94
|
2
|
-
file_groups/compare_files.py,sha256=38EfZzpEpj0wH7SUruPxgyF_W8eqCbIjG6yUOx8hcFA,434
|
3
|
-
file_groups/config_files.py,sha256=kU1wTANXjk5H95x1TpxIBFZrobJFjTMbQ_mjpnewh-U,13011
|
4
|
-
file_groups/groups.py,sha256=R3wF8DV3rhMckjgz6wuycxJ9UxAB2p56zzYcdbh72M0,11072
|
5
|
-
file_groups/handler.py,sha256=X_uFNAMtuiBro_8WG6XFEq_Ys4L8175tSP_gajA01fo,10045
|
6
|
-
file_groups/handler_compare.py,sha256=J6Jev9ev9oECzYCD9M6FUlkKMMMkaP2YlygG11lKqPI,2136
|
7
|
-
file_groups/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
-
file_groups/types.py,sha256=giq3H58RuKpS6b7sjxnzP-sN1Xr4VTQ-rG4brLP5qMI,107
|
9
|
-
file_groups-0.3.0.dist-info/LICENSE.txt,sha256=NZc9ictLEwfoL4ywpAbLbd-n02KETGHLYQU820TJD_Q,1494
|
10
|
-
file_groups-0.3.0.dist-info/METADATA,sha256=BseS9ayqXIwuDBBUb95mPQUcLmrHbmHLO7mSGOr9IEQ,1512
|
11
|
-
file_groups-0.3.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
12
|
-
file_groups-0.3.0.dist-info/top_level.txt,sha256=mAdRf0R2TA8tpH7ftHzXP7VMRhw07p7B2mI8QKpDnjU,12
|
13
|
-
file_groups-0.3.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
14
|
-
file_groups-0.3.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|