file-groups 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- file_groups/__init__.py +2 -0
- file_groups/config_files.py +194 -123
- file_groups/groups.py +53 -55
- file_groups/handler.py +8 -8
- file_groups/handler_compare.py +8 -9
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/METADATA +7 -6
- file_groups-0.3.1.dist-info/RECORD +14 -0
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/WHEEL +1 -1
- file_groups-0.2.1.dist-info/RECORD +0 -14
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/LICENSE.txt +0 -0
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/top_level.txt +0 -0
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/zip-safe +0 -0
file_groups/__init__.py
CHANGED
file_groups/config_files.py
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
import ast
|
2
2
|
import os
|
3
|
+
import errno
|
3
4
|
import re
|
4
5
|
from pathlib import Path
|
5
6
|
import itertools
|
6
7
|
from pprint import pformat
|
7
8
|
import logging
|
8
|
-
from
|
9
|
+
from dataclasses import dataclass
|
10
|
+
from typing import Tuple, Sequence, Any
|
9
11
|
|
10
12
|
from appdirs import AppDirs # type: ignore
|
11
13
|
|
@@ -19,11 +21,74 @@ class ConfigException(Exception):
|
|
19
21
|
"""Invalid configuration"""
|
20
22
|
|
21
23
|
|
24
|
+
@dataclass
|
25
|
+
class ProtectConfig():
|
26
|
+
"""Hold global (site or user) protect config."""
|
27
|
+
protect_recursive: set[re.Pattern]
|
28
|
+
|
29
|
+
def __json__(self):
|
30
|
+
return {
|
31
|
+
ProtectConfig.__name__: {
|
32
|
+
"protect_recursive": [str(pat) for pat in self.protect_recursive],
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
36
|
+
|
37
|
+
@dataclass
|
38
|
+
class DirConfig(ProtectConfig):
|
39
|
+
"""Hold directory specific protect config."""
|
40
|
+
protect_local: set[re.Pattern]
|
41
|
+
config_dir: Path|None
|
42
|
+
config_files: list[str]
|
43
|
+
|
44
|
+
def is_protected(self, ff: FsPath):
|
45
|
+
"""If ff id protected by a regex pattern then return the pattern, otherwise return None."""
|
46
|
+
|
47
|
+
# _LOG.debug("ff '%s'", ff)
|
48
|
+
for pattern in itertools.chain(self.protect_local, self.protect_recursive):
|
49
|
+
if os.sep in str(pattern):
|
50
|
+
# _LOG.debug("Pattern '%s' has path sep", pattern)
|
51
|
+
assert os.path.isabs(ff), f"Expected absolute path, got '{ff}'"
|
52
|
+
|
53
|
+
# Search against full path
|
54
|
+
if pattern.search(os.fspath(ff)):
|
55
|
+
return pattern
|
56
|
+
|
57
|
+
# Attempt exact match against path relative to , i.e. if pattern starts with '^'.
|
58
|
+
# This makes sense for patterns specified on commandline
|
59
|
+
cwd = os.getcwd()
|
60
|
+
ff_relative = str(Path(ff).relative_to(cwd))
|
61
|
+
# _LOG.debug("ff '%s' relative to start dir'%s'", ff_relative, cwd)
|
62
|
+
|
63
|
+
if pattern.match(ff_relative):
|
64
|
+
return pattern
|
65
|
+
|
66
|
+
elif pattern.search(ff.name):
|
67
|
+
return pattern
|
68
|
+
|
69
|
+
return None
|
70
|
+
|
71
|
+
def __json__(self):
|
72
|
+
return {
|
73
|
+
DirConfig.__name__: super().__json__()[ProtectConfig.__name__] | {
|
74
|
+
"protect_local": [str(pat) for pat in self.protect_local],
|
75
|
+
"config_dir": str(self.config_dir),
|
76
|
+
"config_files": self.config_files,
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
|
22
81
|
class ConfigFiles():
|
23
82
|
r"""Handle config files.
|
24
83
|
|
25
84
|
Config files are searched for in the standard config directories on the platform AND in any collected directory.
|
26
|
-
|
85
|
+
|
86
|
+
The 'app_dirs' default sets default config dirs and config-file names.
|
87
|
+
It is also possible to specify additional or alternative config files specific to the application using this library.
|
88
|
+
Config files must be named after the AppDirs.appname (first argument) as <appname>.conf or .<appname>.conf.
|
89
|
+
The defaults are 'file_groups.conf' and '.file_groups.conf'.
|
90
|
+
You should consider carefully before disabling loading of the default files, as an end user likely wants the protection rules to apply for any
|
91
|
+
application using this library.
|
27
92
|
|
28
93
|
The content of a conf file is a Python dict with the following structure.
|
29
94
|
|
@@ -56,14 +121,14 @@ class ConfigFiles():
|
|
56
121
|
}
|
57
122
|
}
|
58
123
|
|
59
|
-
The level one
|
60
|
-
Applications are free to add entries at this level.
|
124
|
+
The level one key is 'file_groups'.
|
125
|
+
Applications are free to add entries at this level, but not underneath. This is protect against ignored misspelled keys.
|
61
126
|
|
62
127
|
The 'file_groups' entry is a dict with a single 'protect' entry.
|
63
128
|
The 'protect' entry is a dict with at most three entries: 'local', 'recursive' and 'global'. These specify whether a directory specific
|
64
129
|
configuration will inherit and extend the parent (and global) config, or whether it is local to current directory only.
|
65
130
|
The 'local', 'recursive' and 'global' entries are lists of regex patterns to match against collected 'work_on' files.
|
66
|
-
Regexes are checked against the simple
|
131
|
+
Regexes are checked against the simple file name (i.e. not the full path) unless they contain at least one path separator (os.sep), in
|
67
132
|
which case they are checked against the absolute path.
|
68
133
|
All checks are done as regex *search* (better to protect too much than too little). Write the regex to match the full name or path if needed.
|
69
134
|
|
@@ -73,126 +138,107 @@ class ConfigFiles():
|
|
73
138
|
protect: An optional sequence of regexes to be added to protect[recursive] for all directories.
|
74
139
|
ignore_config_dirs_config_files: Ignore config files in standard config directories.
|
75
140
|
ignore_per_directory_config_files: Ignore config files in collected directories.
|
76
|
-
remember_configs: Store loaded and merged configs in `
|
77
|
-
app_dirs:
|
141
|
+
remember_configs: Store loaded and merged configs in `per_dir_configs` member variable.
|
142
|
+
app_dirs: Provide your own instance of AppDirs in addition to or as a replacement of the default to add config file names and path.
|
143
|
+
Configuration from later entries have higher precedence.
|
144
|
+
Note that if no AppDirs are specified, no config files will be loaded, neither from config dirs, nor from collected directories.
|
78
145
|
See: https://pypi.org/project/appdirs/
|
79
146
|
|
80
147
|
Members:
|
81
|
-
|
82
|
-
remember_configs: Whether per directory resolved/merged configs are stored in `
|
83
|
-
|
148
|
+
conf_file_names: File names which are config files.
|
149
|
+
remember_configs: Whether per directory resolved/merged configs are stored in `per_dir_configs`.
|
150
|
+
per_dir_configs: Mapping from dir name to directory specific config dict. Only if remember_configs is True.
|
84
151
|
"""
|
85
152
|
|
86
|
-
|
153
|
+
default_appdirs: AppDirs = AppDirs("file_groups", "Hupfeldt_IT")
|
87
154
|
|
88
155
|
_fg_key = "file_groups"
|
89
156
|
_protect_key = "protect"
|
90
157
|
_valid_dir_protect_scopes = ("local", "recursive")
|
91
158
|
_valid_config_dir_protect_scopes = ("local", "recursive", "global")
|
92
159
|
|
93
|
-
def __init__(
|
160
|
+
def __init__( # pylint: disable=too-many-positional-arguments,too-many-arguments
|
94
161
|
self, protect: Sequence[re.Pattern] = (),
|
95
|
-
ignore_config_dirs_config_files=False, ignore_per_directory_config_files=False, remember_configs=
|
96
|
-
app_dirs=None
|
162
|
+
ignore_config_dirs_config_files=False, ignore_per_directory_config_files=False, remember_configs=True,
|
163
|
+
app_dirs: Sequence[AppDirs]|None = None,
|
164
|
+
*,
|
165
|
+
config_file: Path|None = None,
|
166
|
+
):
|
97
167
|
super().__init__()
|
168
|
+
|
169
|
+
self._global_config = ProtectConfig(set(protect))
|
98
170
|
self.remember_configs = remember_configs
|
171
|
+
self.per_dir_configs: dict[Path, DirConfig] = {} # key is abs_dir_path
|
172
|
+
self.ignore_per_directory_config_files = ignore_per_directory_config_files
|
99
173
|
|
100
|
-
|
101
|
-
self.
|
102
|
-
|
103
|
-
|
104
|
-
"local": set(),
|
105
|
-
"recursive": set(protect),
|
106
|
-
}
|
107
|
-
}
|
108
|
-
}
|
174
|
+
app_dirs = app_dirs or (ConfigFiles.default_appdirs,)
|
175
|
+
self.conf_file_name_pairs = tuple((apd.appname + ".conf", "." + apd.appname + ".conf") for apd in app_dirs)
|
176
|
+
_LOG.debug("Conf file names: %s", self.conf_file_name_pairs)
|
177
|
+
self.conf_file_names = list(itertools.chain.from_iterable(self.conf_file_name_pairs))
|
109
178
|
|
110
|
-
self.
|
179
|
+
self.ignore_config_dirs_config_files = ignore_config_dirs_config_files
|
180
|
+
self.config_dirs = []
|
181
|
+
for appd in app_dirs:
|
182
|
+
self.config_dirs.extend(appd.site_config_dir.split(':'))
|
183
|
+
for appd in app_dirs:
|
184
|
+
self.config_dirs.append(appd.user_config_dir)
|
185
|
+
|
186
|
+
self.config_file = config_file
|
111
187
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
if not conf_dir.exists():
|
122
|
-
continue
|
123
|
-
|
124
|
-
new_config, _ = self._read_and_validate_config_file(conf_dir, self.global_config, self._valid_config_dir_protect_scopes, False)
|
125
|
-
if self.remember_configs:
|
126
|
-
self.per_dir_configs[str(conf_dir)] = new_config
|
127
|
-
|
128
|
-
fpt = new_config["file_groups"]["protect"]
|
129
|
-
cast(set, gfpt["recursive"]).update(fpt.get("global", ()))
|
130
|
-
_LOG.debug("Merged global config:\n %s", pformat(new_config))
|
131
|
-
|
132
|
-
try:
|
133
|
-
del fpt['global']
|
134
|
-
except KeyError:
|
135
|
-
pass
|
136
|
-
|
137
|
-
# self.default_config_file_example = self.default_config_file.with_suffix('.example.py')
|
138
|
-
|
139
|
-
def _get_single_conf_file(self, conf_dir: Path, ignore_config_files: bool) -> Tuple[dict|None, Path|None]:
|
140
|
-
"""Return the config file content and path if any config file is found in conf_dir. Error if two are found."""
|
141
|
-
_LOG.debug("Checking for config file in directory: %s", conf_dir)
|
142
|
-
|
143
|
-
num_files = 0
|
144
|
-
for cfn in self.conf_file_names:
|
145
|
-
tmp_conf_file = conf_dir/cfn
|
146
|
-
if tmp_conf_file.exists():
|
147
|
-
conf_file = tmp_conf_file
|
148
|
-
num_files += 1
|
149
|
-
|
150
|
-
if num_files == 1:
|
151
|
-
if ignore_config_files:
|
152
|
-
_LOG.debug("Ignoring config file: %s", conf_file)
|
153
|
-
return None, None
|
154
|
-
|
155
|
-
_LOG.debug("Read config file: %s", conf_file)
|
156
|
-
with open(conf_file, encoding="utf-8") as fh:
|
157
|
-
new_config = ast.literal_eval(fh.read())
|
158
|
-
_LOG.debug("%s", pformat(new_config))
|
159
|
-
return new_config, conf_file
|
160
|
-
|
161
|
-
if num_files == 0:
|
162
|
-
_LOG.debug("No config file in directory %s", conf_dir)
|
163
|
-
return None, None
|
164
|
-
|
165
|
-
msg = f"More than one config file in dir '{conf_dir}': {self.conf_file_names}."
|
166
|
-
_LOG.debug("%s", msg)
|
167
|
-
raise ConfigException(msg)
|
168
|
-
|
169
|
-
def _read_and_validate_config_file(
|
170
|
-
self, conf_dir: Path, parent_conf: dict, valid_protect_scopes: Tuple[str, ...], ignore_config_files: bool
|
171
|
-
) -> Tuple[dict, Path|None]:
|
172
|
-
"""Read config file, validate keys and compile regexes and merge with parent.
|
173
|
-
|
174
|
-
Merge parent conf into conf_dir conf (if any) and return the merged dict. The parent conf is not modified.
|
175
|
-
|
176
|
-
Return: merged config dict with compiled regexes.
|
188
|
+
# self.default_config_file_example = self.default_config_file.with_suffix('.example.py')
|
189
|
+
|
190
|
+
def _read_and_eval_config_file_for_one_appname(
|
191
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], ignore_config_files: bool
|
192
|
+
) -> Tuple[dict[str, Any], Path|None]:
|
193
|
+
"""Read config file.
|
194
|
+
|
195
|
+
Error if config files are found both with and withput '.' prefix.
|
196
|
+
Return: Config dict, config file name, or if no config files is found, None, None.
|
177
197
|
"""
|
178
198
|
|
179
199
|
assert conf_dir.is_absolute()
|
200
|
+
_LOG.debug("Checking for config files %s in directory: %s", conf_file_name_pair, conf_dir)
|
201
|
+
|
202
|
+
match [conf_dir/cfn for cfn in conf_file_name_pair if (conf_dir/cfn).exists()]:
|
203
|
+
case []:
|
204
|
+
_LOG.debug("No config file in directory %s", conf_dir)
|
205
|
+
return {}, None
|
206
|
+
|
207
|
+
case [conf_file]:
|
208
|
+
if ignore_config_files:
|
209
|
+
_LOG.debug("Ignoring config file: %s", conf_file)
|
210
|
+
return {}, None
|
211
|
+
|
212
|
+
_LOG.debug("Read config file: %s", conf_file)
|
213
|
+
with open(conf_file, encoding="utf-8") as fh:
|
214
|
+
new_config = ast.literal_eval(fh.read())
|
215
|
+
_LOG.debug("%s", pformat(new_config))
|
216
|
+
return new_config, conf_file
|
217
|
+
|
218
|
+
case config_files:
|
219
|
+
msg = f"More than one config file in dir '{conf_dir}': {[cf.name for cf in config_files]}."
|
220
|
+
_LOG.debug("%s", msg)
|
221
|
+
raise ConfigException(msg)
|
180
222
|
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
223
|
+
def _read_and_validate_config_file_for_one_appname(
|
224
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], valid_protect_scopes: Tuple[str, ...], ignore_config_files: bool
|
225
|
+
) -> Tuple[dict[str, set[re.Pattern]], Path|None]:
|
226
|
+
"""Read config file, validate keys and compile regexes.
|
227
|
+
|
228
|
+
Error if config files are found both with and withput '.' prefix.
|
229
|
+
|
230
|
+
Return: merged config dict with compiled regexes, config file name. If no config files is found, then return empty sets and None.
|
231
|
+
"""
|
189
232
|
|
190
|
-
new_config, conf_file = self.
|
191
|
-
if not new_config
|
192
|
-
return
|
233
|
+
new_config, conf_file = self._read_and_eval_config_file_for_one_appname(conf_dir, conf_file_name_pair, ignore_config_files)
|
234
|
+
if not new_config:
|
235
|
+
return {
|
236
|
+
"local": set(),
|
237
|
+
"recursive": set(),
|
238
|
+
}, None
|
193
239
|
|
194
240
|
try:
|
195
|
-
protect_conf = new_config[self._fg_key][self._protect_key]
|
241
|
+
protect_conf: dict[str, set[re.Pattern]] = new_config[self._fg_key][self._protect_key]
|
196
242
|
except KeyError as ex:
|
197
243
|
raise ConfigException(f"Config file '{conf_file}' is missing mandatory configuration '{self._fg_key}[{self._protect_key}]'.") from ex
|
198
244
|
|
@@ -203,8 +249,6 @@ class ConfigFiles():
|
|
203
249
|
raise ConfigException(msg)
|
204
250
|
|
205
251
|
protect_conf[key] = set(re.compile(pattern) for pattern in val)
|
206
|
-
if key == "recursive":
|
207
|
-
protect_conf[key].update(parent_conf[self._fg_key][self._protect_key][key])
|
208
252
|
|
209
253
|
for key in self._valid_dir_protect_scopes: # Do NOT use the 'valid_protect_scopes' argument here
|
210
254
|
protect_conf.setdefault(key, set())
|
@@ -213,32 +257,59 @@ class ConfigFiles():
|
|
213
257
|
if _LOG.isEnabledFor(lvl):
|
214
258
|
_LOG.log(lvl, "Merged directory config:\n%s", pformat(new_config))
|
215
259
|
|
216
|
-
return
|
260
|
+
return protect_conf, conf_file
|
261
|
+
|
262
|
+
def load_config_dir_files(self) -> None:
|
263
|
+
"""Load config files from platform standard directories and specified config file, if any."""
|
264
|
+
|
265
|
+
if not self.ignore_config_dirs_config_files:
|
266
|
+
_LOG.debug("config_dirs: %s", self.config_dirs)
|
267
|
+
for conf_dir in self.config_dirs:
|
268
|
+
conf_dir = Path(conf_dir)
|
269
|
+
if not conf_dir.exists():
|
270
|
+
continue
|
271
|
+
|
272
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
273
|
+
cfg, _ = self._read_and_validate_config_file_for_one_appname(
|
274
|
+
conf_dir, conf_file_name_pair, self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
275
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
217
276
|
|
218
|
-
|
277
|
+
if self.config_file:
|
278
|
+
_LOG.debug("specified config_file: %s", self.config_file)
|
279
|
+
conf_dir = self.config_file.parent.absolute()
|
280
|
+
conf_name = self.config_file.name
|
281
|
+
cfg, fpath = self._read_and_validate_config_file_for_one_appname(
|
282
|
+
conf_dir, (conf_name,), self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
283
|
+
if not fpath:
|
284
|
+
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(self.config_file))
|
285
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
286
|
+
|
287
|
+
_LOG.debug("Merged global config:\n %s", self._global_config)
|
288
|
+
|
289
|
+
def dir_config(self, conf_dir: Path, parent_conf: DirConfig|None) -> DirConfig:
|
219
290
|
"""Read and merge config file from directory 'conf_dir' with 'parent_conf'.
|
220
291
|
|
221
|
-
If directory has no parent in the file_groups included dirs, then
|
292
|
+
If directory has no parent in the file_groups included dirs, then None should be supplied as parent_conf.
|
222
293
|
"""
|
223
294
|
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
self.per_dir_configs[str(conf_dir)] = new_config
|
228
|
-
return new_config, conf_file
|
295
|
+
cfg_merge_local: set[re.Pattern] = set()
|
296
|
+
cfg_merge_recursive: set[re.Pattern] = set()
|
297
|
+
cfg_files: list[str] = []
|
229
298
|
|
230
|
-
|
231
|
-
|
299
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
300
|
+
cfg, cfg_file = self._read_and_validate_config_file_for_one_appname(
|
301
|
+
conf_dir, conf_file_name_pair, self._valid_dir_protect_scopes, self.ignore_per_directory_config_files)
|
302
|
+
cfg_merge_local.update(cfg.get("local", set()))
|
303
|
+
cfg_merge_recursive.update(cfg.get("recursive", set()))
|
304
|
+
if cfg_file:
|
305
|
+
cfg_files.append(cfg_file.name)
|
232
306
|
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
# Match against full path
|
237
|
-
assert os.path.isabs(ff), f"Expected absolute path, got '{ff}'"
|
238
|
-
if pattern.search(os.fspath(ff)):
|
239
|
-
return pattern
|
307
|
+
parent_protect_recursive = parent_conf.protect_recursive if parent_conf else self._global_config.protect_recursive
|
308
|
+
new_config = DirConfig(cfg_merge_recursive | parent_protect_recursive, cfg_merge_local, conf_dir, cfg_files)
|
309
|
+
_LOG.debug("new_config:\n %s", new_config)
|
240
310
|
|
241
|
-
|
242
|
-
|
311
|
+
if self.remember_configs:
|
312
|
+
self.per_dir_configs[conf_dir] = new_config
|
313
|
+
# _LOG.debug("per_dir_configs:\n %s", self.per_dir_configs)
|
243
314
|
|
244
|
-
return
|
315
|
+
return new_config
|
file_groups/groups.py
CHANGED
@@ -7,9 +7,9 @@ from dataclasses import dataclass
|
|
7
7
|
from itertools import chain
|
8
8
|
from enum import Enum
|
9
9
|
import logging
|
10
|
-
from typing import Sequence
|
10
|
+
from typing import Sequence, cast
|
11
11
|
|
12
|
-
from .config_files import ConfigFiles
|
12
|
+
from .config_files import DirConfig, ConfigFiles
|
13
13
|
|
14
14
|
|
15
15
|
_LOG = logging.getLogger(__name__)
|
@@ -87,27 +87,21 @@ class FileGroups():
|
|
87
87
|
|
88
88
|
protect_exclude: Exclude files matching regex in the protected files (does not apply to symlinks). Default: Include ALL.
|
89
89
|
Note: Since these files are excluded from protection, it means they er NOT protected!
|
90
|
-
work_include:
|
90
|
+
work_include: ONLY include files matching regex in the may_work_on files (does not apply to symlinks). Default: Include ALL.
|
91
91
|
|
92
|
-
|
93
|
-
ignore_per_directory_config_files: Ignore config files in collected directories.
|
92
|
+
config_files: Load config files. See config_files.ConfigFiles. Note that the default 'None' means use the `config_files.ConfigFiles` class with default arguments.
|
94
93
|
"""
|
95
94
|
|
96
95
|
def __init__(
|
97
96
|
self,
|
98
97
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path],
|
99
98
|
*,
|
100
|
-
protect: Sequence[re.Pattern] = (),
|
101
99
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
102
|
-
|
103
|
-
remember_configs=True):
|
100
|
+
config_files: ConfigFiles|None = None):
|
104
101
|
super().__init__()
|
105
102
|
|
106
|
-
self.config_files = ConfigFiles(
|
107
|
-
|
108
|
-
ignore_config_dirs_config_files=ignore_config_dirs_config_files,
|
109
|
-
ignore_per_directory_config_files=ignore_per_directory_config_files,
|
110
|
-
remember_configs=remember_configs)
|
103
|
+
self.config_files = config_files or ConfigFiles()
|
104
|
+
self.config_files.load_config_dir_files()
|
111
105
|
|
112
106
|
# Turn all paths into absolute paths with symlinks resolved, keep referrence to original argument for messages
|
113
107
|
protect_dirs: dict[str, Path] = {os.path.abspath(os.path.realpath(kp)): kp for kp in protect_dirs_seq}
|
@@ -153,8 +147,8 @@ class FileGroups():
|
|
153
147
|
top/d2/d1
|
154
148
|
top/d2/d1/f1.jpg
|
155
149
|
|
156
|
-
When:
|
157
|
-
And:
|
150
|
+
When: work_dirs_seq is [top, top/d1/d1]
|
151
|
+
And: protect_dirs_seq is [top/d1]
|
158
152
|
|
159
153
|
Then:
|
160
154
|
|
@@ -175,65 +169,69 @@ class FileGroups():
|
|
175
169
|
|
176
170
|
checked_dirs: set[str] = set()
|
177
171
|
|
178
|
-
def
|
179
|
-
"""
|
172
|
+
def handle_entry(abs_dir_path: str, group: _Group, other_group: _Group, dir_config: DirConfig, entry: DirEntry):
|
173
|
+
"""Put entry in the correct group. Call 'find_group' if entry is a directory."""
|
174
|
+
if group.typ is GroupType.MAY_WORK_ON:
|
175
|
+
# Check for match against configured protect patterns, if match, then the file must got to protect group instead
|
176
|
+
pattern = dir_config.is_protected(entry)
|
177
|
+
if pattern:
|
178
|
+
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
179
|
+
group, other_group = other_group, group
|
180
|
+
|
181
|
+
if entry.is_dir(follow_symlinks=False):
|
182
|
+
if entry.path in other_group.dirs:
|
183
|
+
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
184
|
+
find_group(entry.path, other_group, group, dir_config)
|
185
|
+
return
|
186
|
+
|
187
|
+
find_group(entry.path, group, other_group, dir_config)
|
188
|
+
return
|
189
|
+
|
190
|
+
if entry.name in self.config_files.conf_file_names:
|
191
|
+
return
|
192
|
+
|
193
|
+
if entry.is_symlink():
|
194
|
+
# cast: https://github.com/python/mypy/issues/11964
|
195
|
+
points_to = os.readlink(cast(str, entry))
|
196
|
+
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
197
|
+
|
198
|
+
if entry.is_dir(follow_symlinks=True):
|
199
|
+
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", group.typ.name, entry.path, points_to)
|
200
|
+
group.num_directory_symlinks += 1
|
201
|
+
return
|
202
|
+
|
203
|
+
group.symlinks[entry.path] = entry
|
204
|
+
group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
205
|
+
return
|
206
|
+
|
207
|
+
_LOG.debug("find %s - entry name: %s", group.typ.name, entry.name)
|
208
|
+
group.add_entry_match(entry)
|
209
|
+
|
210
|
+
def find_group(abs_dir_path: str, group: _Group, other_group: _Group, parent_conf: DirConfig|None):
|
211
|
+
"""Recursively find all files belonging to 'group'"""
|
180
212
|
_LOG.debug("find %s: %s", group.typ.name, abs_dir_path)
|
181
213
|
if abs_dir_path in checked_dirs:
|
182
214
|
_LOG.debug("directory already checked")
|
183
215
|
return
|
184
216
|
|
185
217
|
group.num_directories += 1
|
186
|
-
dir_config
|
218
|
+
dir_config = self.config_files.dir_config(Path(abs_dir_path), parent_conf)
|
187
219
|
|
188
220
|
for entry in os.scandir(abs_dir_path):
|
189
|
-
|
190
|
-
if entry.path in other_group.dirs:
|
191
|
-
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
192
|
-
find_group(entry.path, other_group, group, dir_config)
|
193
|
-
continue
|
194
|
-
|
195
|
-
find_group(entry.path, group, other_group, dir_config)
|
196
|
-
continue
|
197
|
-
|
198
|
-
if config_file and entry.name == config_file.name:
|
199
|
-
continue
|
200
|
-
|
201
|
-
current_group = group
|
202
|
-
if group.typ is GroupType.MAY_WORK_ON:
|
203
|
-
# We need to check for match against configured protect patterns, if match, then the file must got to protect group instead
|
204
|
-
pattern = self.config_files.is_protected(entry, dir_config)
|
205
|
-
if pattern:
|
206
|
-
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
207
|
-
current_group = other_group
|
208
|
-
|
209
|
-
if entry.is_symlink():
|
210
|
-
points_to = os.readlink(entry)
|
211
|
-
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
212
|
-
|
213
|
-
if entry.is_dir(follow_symlinks=True):
|
214
|
-
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", current_group.typ.name, entry.path, points_to)
|
215
|
-
current_group.num_directory_symlinks += 1
|
216
|
-
continue
|
217
|
-
|
218
|
-
current_group.symlinks[entry.path] = entry
|
219
|
-
current_group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
220
|
-
continue
|
221
|
-
|
222
|
-
_LOG.debug("find %s - entry name: %s", current_group.typ.name, entry.name)
|
223
|
-
current_group.add_entry_match(entry)
|
221
|
+
handle_entry(abs_dir_path, group, other_group, dir_config, entry)
|
224
222
|
|
225
223
|
checked_dirs.add(abs_dir_path)
|
226
224
|
|
227
225
|
for any_dir in sorted(chain(self.must_protect.dirs, self.may_work_on.dirs), key=lambda dd: len(Path(dd).parts)):
|
228
226
|
parent_dir = Path(any_dir)
|
229
227
|
while len(parent_dir.parts) > 1:
|
230
|
-
parent_conf = self.config_files.per_dir_configs.get(
|
228
|
+
parent_conf = self.config_files.per_dir_configs.get(parent_dir)
|
231
229
|
if parent_conf:
|
232
230
|
break
|
233
231
|
|
234
232
|
parent_dir = parent_dir.parent
|
235
233
|
else:
|
236
|
-
parent_conf =
|
234
|
+
parent_conf = None
|
237
235
|
|
238
236
|
if any_dir in self.must_protect.dirs:
|
239
237
|
find_group(any_dir, self.must_protect, self.may_work_on, parent_conf)
|
file_groups/handler.py
CHANGED
@@ -7,6 +7,7 @@ import logging
|
|
7
7
|
from typing import Sequence
|
8
8
|
|
9
9
|
from .groups import FileGroups
|
10
|
+
from .config_files import ConfigFiles
|
10
11
|
|
11
12
|
|
12
13
|
_LOG = logging.getLogger(__name__)
|
@@ -20,26 +21,25 @@ class FileHandler(FileGroups):
|
|
20
21
|
Re-link symlinks when a file being deleted has a corresponding file.
|
21
22
|
|
22
23
|
Arguments:
|
23
|
-
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include: See `FileGroups` class.
|
24
|
-
dry_run: Don't
|
25
|
-
protected_regexes: Protect files matching this from being deleted or moved.
|
24
|
+
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include, config_files: See `FileGroups` class.
|
25
|
+
dry_run: Don't change any files.
|
26
26
|
delete_symlinks_instead_of_relinking: Normal operation is to re-link to a 'corresponding' or renamed file when renaming or deleting a file.
|
27
27
|
If delete_symlinks_instead_of_relinking is true, then symlinks in work_on dirs pointing to renamed/deletes files will be deleted even if
|
28
28
|
they could have logically been made to point to a file in a protect dir.
|
29
29
|
"""
|
30
30
|
|
31
|
-
def __init__(
|
31
|
+
def __init__( # pylint: disable=too-many-arguments
|
32
32
|
self,
|
33
33
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path],
|
34
34
|
*,
|
35
|
-
dry_run: bool,
|
36
|
-
protected_regexes: Sequence[re.Pattern],
|
37
35
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
36
|
+
config_files: ConfigFiles|None = None,
|
37
|
+
dry_run: bool,
|
38
38
|
delete_symlinks_instead_of_relinking=False):
|
39
39
|
super().__init__(
|
40
|
-
protect=protected_regexes,
|
41
40
|
protect_dirs_seq=protect_dirs_seq, work_dirs_seq=work_dirs_seq,
|
42
|
-
protect_exclude=protect_exclude, work_include=work_include
|
41
|
+
protect_exclude=protect_exclude, work_include=work_include,
|
42
|
+
config_files=config_files)
|
43
43
|
|
44
44
|
self.dry_run = dry_run
|
45
45
|
self.delete_symlinks_instead_of_relinking = delete_symlinks_instead_of_relinking
|
file_groups/handler_compare.py
CHANGED
@@ -7,6 +7,7 @@ from typing import Sequence
|
|
7
7
|
from .compare_files import CompareFiles
|
8
8
|
from .types import FsPath
|
9
9
|
from .handler import FileHandler
|
10
|
+
from .config_files import ConfigFiles
|
10
11
|
|
11
12
|
|
12
13
|
_LOG = logging.getLogger(__name__)
|
@@ -16,26 +17,24 @@ class FileHandlerCompare(FileHandler):
|
|
16
17
|
"""Extend `FileHandler` with a compare method
|
17
18
|
|
18
19
|
Arguments:
|
19
|
-
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include: See `FileGroups` class.
|
20
|
+
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include, config_files: See `FileGroups` class.
|
20
21
|
dry_run, protected_regexes, delete_symlinks_instead_of_relinking: See `FileHandler` class.
|
21
22
|
fcmp: Object providing compare function.
|
22
23
|
"""
|
23
24
|
|
24
|
-
def __init__(
|
25
|
+
def __init__( # pylint: disable=too-many-arguments
|
25
26
|
self,
|
26
27
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path], fcmp: CompareFiles,
|
27
28
|
*,
|
28
|
-
dry_run: bool,
|
29
|
-
protected_regexes: Sequence[re.Pattern],
|
30
29
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
30
|
+
config_files: ConfigFiles|None = None,
|
31
|
+
dry_run: bool,
|
31
32
|
delete_symlinks_instead_of_relinking=False):
|
32
33
|
super().__init__(
|
33
|
-
protect_dirs_seq=protect_dirs_seq,
|
34
|
-
|
34
|
+
protect_dirs_seq=protect_dirs_seq, work_dirs_seq=work_dirs_seq,
|
35
|
+
protect_exclude=protect_exclude, work_include=work_include,
|
36
|
+
config_files=config_files,
|
35
37
|
dry_run=dry_run,
|
36
|
-
protected_regexes=protected_regexes,
|
37
|
-
protect_exclude=protect_exclude,
|
38
|
-
work_include=work_include,
|
39
38
|
delete_symlinks_instead_of_relinking=delete_symlinks_instead_of_relinking)
|
40
39
|
|
41
40
|
self._fcmp = fcmp
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
|
-
Name:
|
3
|
-
Version: 0.
|
2
|
+
Name: file_groups
|
3
|
+
Version: 0.3.1
|
4
4
|
Summary: Group files into 'protect' and 'work_on' and provide operations for safe delete/move and symlink handling.
|
5
5
|
Home-page: https://github.com/lhupfeldt/file_groups.git
|
6
6
|
Author: Lars Hupfeldt Nielsen
|
@@ -11,16 +11,17 @@ Classifier: Intended Audience :: Developers
|
|
11
11
|
Classifier: License :: OSI Approved :: BSD License
|
12
12
|
Classifier: Natural Language :: English
|
13
13
|
Classifier: Operating System :: OS Independent
|
14
|
-
Classifier: Programming Language :: Python :: 3.
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
17
18
|
Classifier: Topic :: Software Development :: Libraries
|
18
19
|
Requires-Python: >=3.10
|
19
20
|
Description-Content-Type: text/x-rst
|
20
21
|
License-File: LICENSE.txt
|
21
|
-
Requires-Dist: appdirs
|
22
|
+
Requires-Dist: appdirs>=1.4.4
|
22
23
|
Provides-Extra: dev
|
23
|
-
Requires-Dist: nox
|
24
|
+
Requires-Dist: nox; extra == "dev"
|
24
25
|
|
25
26
|
Library for grouping files into sets of 'work-on' and 'protect' based on arbitrarily nested directories.
|
26
27
|
|
@@ -0,0 +1,14 @@
|
|
1
|
+
file_groups/__init__.py,sha256=XBp5563xhTer1vQY-DpKRvViAvlHaZqx9GrINiGqbeI,127
|
2
|
+
file_groups/compare_files.py,sha256=38EfZzpEpj0wH7SUruPxgyF_W8eqCbIjG6yUOx8hcFA,434
|
3
|
+
file_groups/config_files.py,sha256=VwRmZYEemKQawYI43qV9XhvPyI7ouaDjjadhc74wqNs,14021
|
4
|
+
file_groups/groups.py,sha256=Lt8QlAET8032t9OXcq9cmzLy27tZsYTnLAaGilzEtcM,11377
|
5
|
+
file_groups/handler.py,sha256=NmYfmPLGXgPsGxM9uR2r8aS3cI-8S7D5-8884EoGA1M,10083
|
6
|
+
file_groups/handler_compare.py,sha256=qbGHwT0FSvfnWtAnjYi4px6V5xIM2cStm0soflwYAi4,2174
|
7
|
+
file_groups/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
file_groups/types.py,sha256=giq3H58RuKpS6b7sjxnzP-sN1Xr4VTQ-rG4brLP5qMI,107
|
9
|
+
file_groups-0.3.1.dist-info/LICENSE.txt,sha256=NZc9ictLEwfoL4ywpAbLbd-n02KETGHLYQU820TJD_Q,1494
|
10
|
+
file_groups-0.3.1.dist-info/METADATA,sha256=wSlWiFxs8yMgFvY6jN3pWIBGBVbbneRhyMQM5tNcoPw,1561
|
11
|
+
file_groups-0.3.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
12
|
+
file_groups-0.3.1.dist-info/top_level.txt,sha256=mAdRf0R2TA8tpH7ftHzXP7VMRhw07p7B2mI8QKpDnjU,12
|
13
|
+
file_groups-0.3.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
14
|
+
file_groups-0.3.1.dist-info/RECORD,,
|
@@ -1,14 +0,0 @@
|
|
1
|
-
file_groups/__init__.py,sha256=1OO3tLnRnIe51VfZSQz0sqUaUP-xS7oHxiSexnKwpEY,94
|
2
|
-
file_groups/compare_files.py,sha256=38EfZzpEpj0wH7SUruPxgyF_W8eqCbIjG6yUOx8hcFA,434
|
3
|
-
file_groups/config_files.py,sha256=IDwhznEM8wpMBp0Fl69r1CbPxfgYCu82iE3b_mxkk6g,10211
|
4
|
-
file_groups/groups.py,sha256=6tVIYBC5nCVPI4Ig1EuijT12accw2DZ08TEIMSJecLo,11531
|
5
|
-
file_groups/handler.py,sha256=06_8_fBEaHndrfUdxMsFw6KZw8PRcRj4MsH4FY3I4sU,10083
|
6
|
-
file_groups/handler_compare.py,sha256=78TXl9QFMIL5gZQhiLYvbzF5OidS7kcaRz6zCCNVG6I,2120
|
7
|
-
file_groups/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
-
file_groups/types.py,sha256=giq3H58RuKpS6b7sjxnzP-sN1Xr4VTQ-rG4brLP5qMI,107
|
9
|
-
file_groups-0.2.1.dist-info/LICENSE.txt,sha256=NZc9ictLEwfoL4ywpAbLbd-n02KETGHLYQU820TJD_Q,1494
|
10
|
-
file_groups-0.2.1.dist-info/METADATA,sha256=hnoCAPIfqQ_bjodGN2qVzLhGWCBdg1-PCmYpf5G0I1U,1512
|
11
|
-
file_groups-0.2.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
|
12
|
-
file_groups-0.2.1.dist-info/top_level.txt,sha256=mAdRf0R2TA8tpH7ftHzXP7VMRhw07p7B2mI8QKpDnjU,12
|
13
|
-
file_groups-0.2.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
14
|
-
file_groups-0.2.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|