file-groups 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- file_groups/__init__.py +2 -0
- file_groups/config_files.py +194 -123
- file_groups/groups.py +53 -55
- file_groups/handler.py +8 -8
- file_groups/handler_compare.py +8 -9
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/METADATA +7 -6
- file_groups-0.3.1.dist-info/RECORD +14 -0
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/WHEEL +1 -1
- file_groups-0.2.1.dist-info/RECORD +0 -14
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/LICENSE.txt +0 -0
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/top_level.txt +0 -0
- {file_groups-0.2.1.dist-info → file_groups-0.3.1.dist-info}/zip-safe +0 -0
file_groups/__init__.py
CHANGED
file_groups/config_files.py
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
import ast
|
2
2
|
import os
|
3
|
+
import errno
|
3
4
|
import re
|
4
5
|
from pathlib import Path
|
5
6
|
import itertools
|
6
7
|
from pprint import pformat
|
7
8
|
import logging
|
8
|
-
from
|
9
|
+
from dataclasses import dataclass
|
10
|
+
from typing import Tuple, Sequence, Any
|
9
11
|
|
10
12
|
from appdirs import AppDirs # type: ignore
|
11
13
|
|
@@ -19,11 +21,74 @@ class ConfigException(Exception):
|
|
19
21
|
"""Invalid configuration"""
|
20
22
|
|
21
23
|
|
24
|
+
@dataclass
|
25
|
+
class ProtectConfig():
|
26
|
+
"""Hold global (site or user) protect config."""
|
27
|
+
protect_recursive: set[re.Pattern]
|
28
|
+
|
29
|
+
def __json__(self):
|
30
|
+
return {
|
31
|
+
ProtectConfig.__name__: {
|
32
|
+
"protect_recursive": [str(pat) for pat in self.protect_recursive],
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
36
|
+
|
37
|
+
@dataclass
|
38
|
+
class DirConfig(ProtectConfig):
|
39
|
+
"""Hold directory specific protect config."""
|
40
|
+
protect_local: set[re.Pattern]
|
41
|
+
config_dir: Path|None
|
42
|
+
config_files: list[str]
|
43
|
+
|
44
|
+
def is_protected(self, ff: FsPath):
|
45
|
+
"""If ff id protected by a regex pattern then return the pattern, otherwise return None."""
|
46
|
+
|
47
|
+
# _LOG.debug("ff '%s'", ff)
|
48
|
+
for pattern in itertools.chain(self.protect_local, self.protect_recursive):
|
49
|
+
if os.sep in str(pattern):
|
50
|
+
# _LOG.debug("Pattern '%s' has path sep", pattern)
|
51
|
+
assert os.path.isabs(ff), f"Expected absolute path, got '{ff}'"
|
52
|
+
|
53
|
+
# Search against full path
|
54
|
+
if pattern.search(os.fspath(ff)):
|
55
|
+
return pattern
|
56
|
+
|
57
|
+
# Attempt exact match against path relative to , i.e. if pattern starts with '^'.
|
58
|
+
# This makes sense for patterns specified on commandline
|
59
|
+
cwd = os.getcwd()
|
60
|
+
ff_relative = str(Path(ff).relative_to(cwd))
|
61
|
+
# _LOG.debug("ff '%s' relative to start dir'%s'", ff_relative, cwd)
|
62
|
+
|
63
|
+
if pattern.match(ff_relative):
|
64
|
+
return pattern
|
65
|
+
|
66
|
+
elif pattern.search(ff.name):
|
67
|
+
return pattern
|
68
|
+
|
69
|
+
return None
|
70
|
+
|
71
|
+
def __json__(self):
|
72
|
+
return {
|
73
|
+
DirConfig.__name__: super().__json__()[ProtectConfig.__name__] | {
|
74
|
+
"protect_local": [str(pat) for pat in self.protect_local],
|
75
|
+
"config_dir": str(self.config_dir),
|
76
|
+
"config_files": self.config_files,
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
|
22
81
|
class ConfigFiles():
|
23
82
|
r"""Handle config files.
|
24
83
|
|
25
84
|
Config files are searched for in the standard config directories on the platform AND in any collected directory.
|
26
|
-
|
85
|
+
|
86
|
+
The 'app_dirs' default sets default config dirs and config-file names.
|
87
|
+
It is also possible to specify additional or alternative config files specific to the application using this library.
|
88
|
+
Config files must be named after the AppDirs.appname (first argument) as <appname>.conf or .<appname>.conf.
|
89
|
+
The defaults are 'file_groups.conf' and '.file_groups.conf'.
|
90
|
+
You should consider carefully before disabling loading of the default files, as an end user likely wants the protection rules to apply for any
|
91
|
+
application using this library.
|
27
92
|
|
28
93
|
The content of a conf file is a Python dict with the following structure.
|
29
94
|
|
@@ -56,14 +121,14 @@ class ConfigFiles():
|
|
56
121
|
}
|
57
122
|
}
|
58
123
|
|
59
|
-
The level one
|
60
|
-
Applications are free to add entries at this level.
|
124
|
+
The level one key is 'file_groups'.
|
125
|
+
Applications are free to add entries at this level, but not underneath. This is protect against ignored misspelled keys.
|
61
126
|
|
62
127
|
The 'file_groups' entry is a dict with a single 'protect' entry.
|
63
128
|
The 'protect' entry is a dict with at most three entries: 'local', 'recursive' and 'global'. These specify whether a directory specific
|
64
129
|
configuration will inherit and extend the parent (and global) config, or whether it is local to current directory only.
|
65
130
|
The 'local', 'recursive' and 'global' entries are lists of regex patterns to match against collected 'work_on' files.
|
66
|
-
Regexes are checked against the simple
|
131
|
+
Regexes are checked against the simple file name (i.e. not the full path) unless they contain at least one path separator (os.sep), in
|
67
132
|
which case they are checked against the absolute path.
|
68
133
|
All checks are done as regex *search* (better to protect too much than too little). Write the regex to match the full name or path if needed.
|
69
134
|
|
@@ -73,126 +138,107 @@ class ConfigFiles():
|
|
73
138
|
protect: An optional sequence of regexes to be added to protect[recursive] for all directories.
|
74
139
|
ignore_config_dirs_config_files: Ignore config files in standard config directories.
|
75
140
|
ignore_per_directory_config_files: Ignore config files in collected directories.
|
76
|
-
remember_configs: Store loaded and merged configs in `
|
77
|
-
app_dirs:
|
141
|
+
remember_configs: Store loaded and merged configs in `per_dir_configs` member variable.
|
142
|
+
app_dirs: Provide your own instance of AppDirs in addition to or as a replacement of the default to add config file names and path.
|
143
|
+
Configuration from later entries have higher precedence.
|
144
|
+
Note that if no AppDirs are specified, no config files will be loaded, neither from config dirs, nor from collected directories.
|
78
145
|
See: https://pypi.org/project/appdirs/
|
79
146
|
|
80
147
|
Members:
|
81
|
-
|
82
|
-
remember_configs: Whether per directory resolved/merged configs are stored in `
|
83
|
-
|
148
|
+
conf_file_names: File names which are config files.
|
149
|
+
remember_configs: Whether per directory resolved/merged configs are stored in `per_dir_configs`.
|
150
|
+
per_dir_configs: Mapping from dir name to directory specific config dict. Only if remember_configs is True.
|
84
151
|
"""
|
85
152
|
|
86
|
-
|
153
|
+
default_appdirs: AppDirs = AppDirs("file_groups", "Hupfeldt_IT")
|
87
154
|
|
88
155
|
_fg_key = "file_groups"
|
89
156
|
_protect_key = "protect"
|
90
157
|
_valid_dir_protect_scopes = ("local", "recursive")
|
91
158
|
_valid_config_dir_protect_scopes = ("local", "recursive", "global")
|
92
159
|
|
93
|
-
def __init__(
|
160
|
+
def __init__( # pylint: disable=too-many-positional-arguments,too-many-arguments
|
94
161
|
self, protect: Sequence[re.Pattern] = (),
|
95
|
-
ignore_config_dirs_config_files=False, ignore_per_directory_config_files=False, remember_configs=
|
96
|
-
app_dirs=None
|
162
|
+
ignore_config_dirs_config_files=False, ignore_per_directory_config_files=False, remember_configs=True,
|
163
|
+
app_dirs: Sequence[AppDirs]|None = None,
|
164
|
+
*,
|
165
|
+
config_file: Path|None = None,
|
166
|
+
):
|
97
167
|
super().__init__()
|
168
|
+
|
169
|
+
self._global_config = ProtectConfig(set(protect))
|
98
170
|
self.remember_configs = remember_configs
|
171
|
+
self.per_dir_configs: dict[Path, DirConfig] = {} # key is abs_dir_path
|
172
|
+
self.ignore_per_directory_config_files = ignore_per_directory_config_files
|
99
173
|
|
100
|
-
|
101
|
-
self.
|
102
|
-
|
103
|
-
|
104
|
-
"local": set(),
|
105
|
-
"recursive": set(protect),
|
106
|
-
}
|
107
|
-
}
|
108
|
-
}
|
174
|
+
app_dirs = app_dirs or (ConfigFiles.default_appdirs,)
|
175
|
+
self.conf_file_name_pairs = tuple((apd.appname + ".conf", "." + apd.appname + ".conf") for apd in app_dirs)
|
176
|
+
_LOG.debug("Conf file names: %s", self.conf_file_name_pairs)
|
177
|
+
self.conf_file_names = list(itertools.chain.from_iterable(self.conf_file_name_pairs))
|
109
178
|
|
110
|
-
self.
|
179
|
+
self.ignore_config_dirs_config_files = ignore_config_dirs_config_files
|
180
|
+
self.config_dirs = []
|
181
|
+
for appd in app_dirs:
|
182
|
+
self.config_dirs.extend(appd.site_config_dir.split(':'))
|
183
|
+
for appd in app_dirs:
|
184
|
+
self.config_dirs.append(appd.user_config_dir)
|
185
|
+
|
186
|
+
self.config_file = config_file
|
111
187
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
if not conf_dir.exists():
|
122
|
-
continue
|
123
|
-
|
124
|
-
new_config, _ = self._read_and_validate_config_file(conf_dir, self.global_config, self._valid_config_dir_protect_scopes, False)
|
125
|
-
if self.remember_configs:
|
126
|
-
self.per_dir_configs[str(conf_dir)] = new_config
|
127
|
-
|
128
|
-
fpt = new_config["file_groups"]["protect"]
|
129
|
-
cast(set, gfpt["recursive"]).update(fpt.get("global", ()))
|
130
|
-
_LOG.debug("Merged global config:\n %s", pformat(new_config))
|
131
|
-
|
132
|
-
try:
|
133
|
-
del fpt['global']
|
134
|
-
except KeyError:
|
135
|
-
pass
|
136
|
-
|
137
|
-
# self.default_config_file_example = self.default_config_file.with_suffix('.example.py')
|
138
|
-
|
139
|
-
def _get_single_conf_file(self, conf_dir: Path, ignore_config_files: bool) -> Tuple[dict|None, Path|None]:
|
140
|
-
"""Return the config file content and path if any config file is found in conf_dir. Error if two are found."""
|
141
|
-
_LOG.debug("Checking for config file in directory: %s", conf_dir)
|
142
|
-
|
143
|
-
num_files = 0
|
144
|
-
for cfn in self.conf_file_names:
|
145
|
-
tmp_conf_file = conf_dir/cfn
|
146
|
-
if tmp_conf_file.exists():
|
147
|
-
conf_file = tmp_conf_file
|
148
|
-
num_files += 1
|
149
|
-
|
150
|
-
if num_files == 1:
|
151
|
-
if ignore_config_files:
|
152
|
-
_LOG.debug("Ignoring config file: %s", conf_file)
|
153
|
-
return None, None
|
154
|
-
|
155
|
-
_LOG.debug("Read config file: %s", conf_file)
|
156
|
-
with open(conf_file, encoding="utf-8") as fh:
|
157
|
-
new_config = ast.literal_eval(fh.read())
|
158
|
-
_LOG.debug("%s", pformat(new_config))
|
159
|
-
return new_config, conf_file
|
160
|
-
|
161
|
-
if num_files == 0:
|
162
|
-
_LOG.debug("No config file in directory %s", conf_dir)
|
163
|
-
return None, None
|
164
|
-
|
165
|
-
msg = f"More than one config file in dir '{conf_dir}': {self.conf_file_names}."
|
166
|
-
_LOG.debug("%s", msg)
|
167
|
-
raise ConfigException(msg)
|
168
|
-
|
169
|
-
def _read_and_validate_config_file(
|
170
|
-
self, conf_dir: Path, parent_conf: dict, valid_protect_scopes: Tuple[str, ...], ignore_config_files: bool
|
171
|
-
) -> Tuple[dict, Path|None]:
|
172
|
-
"""Read config file, validate keys and compile regexes and merge with parent.
|
173
|
-
|
174
|
-
Merge parent conf into conf_dir conf (if any) and return the merged dict. The parent conf is not modified.
|
175
|
-
|
176
|
-
Return: merged config dict with compiled regexes.
|
188
|
+
# self.default_config_file_example = self.default_config_file.with_suffix('.example.py')
|
189
|
+
|
190
|
+
def _read_and_eval_config_file_for_one_appname(
|
191
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], ignore_config_files: bool
|
192
|
+
) -> Tuple[dict[str, Any], Path|None]:
|
193
|
+
"""Read config file.
|
194
|
+
|
195
|
+
Error if config files are found both with and withput '.' prefix.
|
196
|
+
Return: Config dict, config file name, or if no config files is found, None, None.
|
177
197
|
"""
|
178
198
|
|
179
199
|
assert conf_dir.is_absolute()
|
200
|
+
_LOG.debug("Checking for config files %s in directory: %s", conf_file_name_pair, conf_dir)
|
201
|
+
|
202
|
+
match [conf_dir/cfn for cfn in conf_file_name_pair if (conf_dir/cfn).exists()]:
|
203
|
+
case []:
|
204
|
+
_LOG.debug("No config file in directory %s", conf_dir)
|
205
|
+
return {}, None
|
206
|
+
|
207
|
+
case [conf_file]:
|
208
|
+
if ignore_config_files:
|
209
|
+
_LOG.debug("Ignoring config file: %s", conf_file)
|
210
|
+
return {}, None
|
211
|
+
|
212
|
+
_LOG.debug("Read config file: %s", conf_file)
|
213
|
+
with open(conf_file, encoding="utf-8") as fh:
|
214
|
+
new_config = ast.literal_eval(fh.read())
|
215
|
+
_LOG.debug("%s", pformat(new_config))
|
216
|
+
return new_config, conf_file
|
217
|
+
|
218
|
+
case config_files:
|
219
|
+
msg = f"More than one config file in dir '{conf_dir}': {[cf.name for cf in config_files]}."
|
220
|
+
_LOG.debug("%s", msg)
|
221
|
+
raise ConfigException(msg)
|
180
222
|
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
223
|
+
def _read_and_validate_config_file_for_one_appname(
|
224
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], valid_protect_scopes: Tuple[str, ...], ignore_config_files: bool
|
225
|
+
) -> Tuple[dict[str, set[re.Pattern]], Path|None]:
|
226
|
+
"""Read config file, validate keys and compile regexes.
|
227
|
+
|
228
|
+
Error if config files are found both with and withput '.' prefix.
|
229
|
+
|
230
|
+
Return: merged config dict with compiled regexes, config file name. If no config files is found, then return empty sets and None.
|
231
|
+
"""
|
189
232
|
|
190
|
-
new_config, conf_file = self.
|
191
|
-
if not new_config
|
192
|
-
return
|
233
|
+
new_config, conf_file = self._read_and_eval_config_file_for_one_appname(conf_dir, conf_file_name_pair, ignore_config_files)
|
234
|
+
if not new_config:
|
235
|
+
return {
|
236
|
+
"local": set(),
|
237
|
+
"recursive": set(),
|
238
|
+
}, None
|
193
239
|
|
194
240
|
try:
|
195
|
-
protect_conf = new_config[self._fg_key][self._protect_key]
|
241
|
+
protect_conf: dict[str, set[re.Pattern]] = new_config[self._fg_key][self._protect_key]
|
196
242
|
except KeyError as ex:
|
197
243
|
raise ConfigException(f"Config file '{conf_file}' is missing mandatory configuration '{self._fg_key}[{self._protect_key}]'.") from ex
|
198
244
|
|
@@ -203,8 +249,6 @@ class ConfigFiles():
|
|
203
249
|
raise ConfigException(msg)
|
204
250
|
|
205
251
|
protect_conf[key] = set(re.compile(pattern) for pattern in val)
|
206
|
-
if key == "recursive":
|
207
|
-
protect_conf[key].update(parent_conf[self._fg_key][self._protect_key][key])
|
208
252
|
|
209
253
|
for key in self._valid_dir_protect_scopes: # Do NOT use the 'valid_protect_scopes' argument here
|
210
254
|
protect_conf.setdefault(key, set())
|
@@ -213,32 +257,59 @@ class ConfigFiles():
|
|
213
257
|
if _LOG.isEnabledFor(lvl):
|
214
258
|
_LOG.log(lvl, "Merged directory config:\n%s", pformat(new_config))
|
215
259
|
|
216
|
-
return
|
260
|
+
return protect_conf, conf_file
|
261
|
+
|
262
|
+
def load_config_dir_files(self) -> None:
|
263
|
+
"""Load config files from platform standard directories and specified config file, if any."""
|
264
|
+
|
265
|
+
if not self.ignore_config_dirs_config_files:
|
266
|
+
_LOG.debug("config_dirs: %s", self.config_dirs)
|
267
|
+
for conf_dir in self.config_dirs:
|
268
|
+
conf_dir = Path(conf_dir)
|
269
|
+
if not conf_dir.exists():
|
270
|
+
continue
|
271
|
+
|
272
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
273
|
+
cfg, _ = self._read_and_validate_config_file_for_one_appname(
|
274
|
+
conf_dir, conf_file_name_pair, self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
275
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
217
276
|
|
218
|
-
|
277
|
+
if self.config_file:
|
278
|
+
_LOG.debug("specified config_file: %s", self.config_file)
|
279
|
+
conf_dir = self.config_file.parent.absolute()
|
280
|
+
conf_name = self.config_file.name
|
281
|
+
cfg, fpath = self._read_and_validate_config_file_for_one_appname(
|
282
|
+
conf_dir, (conf_name,), self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
283
|
+
if not fpath:
|
284
|
+
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(self.config_file))
|
285
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
286
|
+
|
287
|
+
_LOG.debug("Merged global config:\n %s", self._global_config)
|
288
|
+
|
289
|
+
def dir_config(self, conf_dir: Path, parent_conf: DirConfig|None) -> DirConfig:
|
219
290
|
"""Read and merge config file from directory 'conf_dir' with 'parent_conf'.
|
220
291
|
|
221
|
-
If directory has no parent in the file_groups included dirs, then
|
292
|
+
If directory has no parent in the file_groups included dirs, then None should be supplied as parent_conf.
|
222
293
|
"""
|
223
294
|
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
self.per_dir_configs[str(conf_dir)] = new_config
|
228
|
-
return new_config, conf_file
|
295
|
+
cfg_merge_local: set[re.Pattern] = set()
|
296
|
+
cfg_merge_recursive: set[re.Pattern] = set()
|
297
|
+
cfg_files: list[str] = []
|
229
298
|
|
230
|
-
|
231
|
-
|
299
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
300
|
+
cfg, cfg_file = self._read_and_validate_config_file_for_one_appname(
|
301
|
+
conf_dir, conf_file_name_pair, self._valid_dir_protect_scopes, self.ignore_per_directory_config_files)
|
302
|
+
cfg_merge_local.update(cfg.get("local", set()))
|
303
|
+
cfg_merge_recursive.update(cfg.get("recursive", set()))
|
304
|
+
if cfg_file:
|
305
|
+
cfg_files.append(cfg_file.name)
|
232
306
|
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
# Match against full path
|
237
|
-
assert os.path.isabs(ff), f"Expected absolute path, got '{ff}'"
|
238
|
-
if pattern.search(os.fspath(ff)):
|
239
|
-
return pattern
|
307
|
+
parent_protect_recursive = parent_conf.protect_recursive if parent_conf else self._global_config.protect_recursive
|
308
|
+
new_config = DirConfig(cfg_merge_recursive | parent_protect_recursive, cfg_merge_local, conf_dir, cfg_files)
|
309
|
+
_LOG.debug("new_config:\n %s", new_config)
|
240
310
|
|
241
|
-
|
242
|
-
|
311
|
+
if self.remember_configs:
|
312
|
+
self.per_dir_configs[conf_dir] = new_config
|
313
|
+
# _LOG.debug("per_dir_configs:\n %s", self.per_dir_configs)
|
243
314
|
|
244
|
-
return
|
315
|
+
return new_config
|
file_groups/groups.py
CHANGED
@@ -7,9 +7,9 @@ from dataclasses import dataclass
|
|
7
7
|
from itertools import chain
|
8
8
|
from enum import Enum
|
9
9
|
import logging
|
10
|
-
from typing import Sequence
|
10
|
+
from typing import Sequence, cast
|
11
11
|
|
12
|
-
from .config_files import ConfigFiles
|
12
|
+
from .config_files import DirConfig, ConfigFiles
|
13
13
|
|
14
14
|
|
15
15
|
_LOG = logging.getLogger(__name__)
|
@@ -87,27 +87,21 @@ class FileGroups():
|
|
87
87
|
|
88
88
|
protect_exclude: Exclude files matching regex in the protected files (does not apply to symlinks). Default: Include ALL.
|
89
89
|
Note: Since these files are excluded from protection, it means they er NOT protected!
|
90
|
-
work_include:
|
90
|
+
work_include: ONLY include files matching regex in the may_work_on files (does not apply to symlinks). Default: Include ALL.
|
91
91
|
|
92
|
-
|
93
|
-
ignore_per_directory_config_files: Ignore config files in collected directories.
|
92
|
+
config_files: Load config files. See config_files.ConfigFiles. Note that the default 'None' means use the `config_files.ConfigFiles` class with default arguments.
|
94
93
|
"""
|
95
94
|
|
96
95
|
def __init__(
|
97
96
|
self,
|
98
97
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path],
|
99
98
|
*,
|
100
|
-
protect: Sequence[re.Pattern] = (),
|
101
99
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
102
|
-
|
103
|
-
remember_configs=True):
|
100
|
+
config_files: ConfigFiles|None = None):
|
104
101
|
super().__init__()
|
105
102
|
|
106
|
-
self.config_files = ConfigFiles(
|
107
|
-
|
108
|
-
ignore_config_dirs_config_files=ignore_config_dirs_config_files,
|
109
|
-
ignore_per_directory_config_files=ignore_per_directory_config_files,
|
110
|
-
remember_configs=remember_configs)
|
103
|
+
self.config_files = config_files or ConfigFiles()
|
104
|
+
self.config_files.load_config_dir_files()
|
111
105
|
|
112
106
|
# Turn all paths into absolute paths with symlinks resolved, keep referrence to original argument for messages
|
113
107
|
protect_dirs: dict[str, Path] = {os.path.abspath(os.path.realpath(kp)): kp for kp in protect_dirs_seq}
|
@@ -153,8 +147,8 @@ class FileGroups():
|
|
153
147
|
top/d2/d1
|
154
148
|
top/d2/d1/f1.jpg
|
155
149
|
|
156
|
-
When:
|
157
|
-
And:
|
150
|
+
When: work_dirs_seq is [top, top/d1/d1]
|
151
|
+
And: protect_dirs_seq is [top/d1]
|
158
152
|
|
159
153
|
Then:
|
160
154
|
|
@@ -175,65 +169,69 @@ class FileGroups():
|
|
175
169
|
|
176
170
|
checked_dirs: set[str] = set()
|
177
171
|
|
178
|
-
def
|
179
|
-
"""
|
172
|
+
def handle_entry(abs_dir_path: str, group: _Group, other_group: _Group, dir_config: DirConfig, entry: DirEntry):
|
173
|
+
"""Put entry in the correct group. Call 'find_group' if entry is a directory."""
|
174
|
+
if group.typ is GroupType.MAY_WORK_ON:
|
175
|
+
# Check for match against configured protect patterns, if match, then the file must got to protect group instead
|
176
|
+
pattern = dir_config.is_protected(entry)
|
177
|
+
if pattern:
|
178
|
+
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
179
|
+
group, other_group = other_group, group
|
180
|
+
|
181
|
+
if entry.is_dir(follow_symlinks=False):
|
182
|
+
if entry.path in other_group.dirs:
|
183
|
+
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
184
|
+
find_group(entry.path, other_group, group, dir_config)
|
185
|
+
return
|
186
|
+
|
187
|
+
find_group(entry.path, group, other_group, dir_config)
|
188
|
+
return
|
189
|
+
|
190
|
+
if entry.name in self.config_files.conf_file_names:
|
191
|
+
return
|
192
|
+
|
193
|
+
if entry.is_symlink():
|
194
|
+
# cast: https://github.com/python/mypy/issues/11964
|
195
|
+
points_to = os.readlink(cast(str, entry))
|
196
|
+
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
197
|
+
|
198
|
+
if entry.is_dir(follow_symlinks=True):
|
199
|
+
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", group.typ.name, entry.path, points_to)
|
200
|
+
group.num_directory_symlinks += 1
|
201
|
+
return
|
202
|
+
|
203
|
+
group.symlinks[entry.path] = entry
|
204
|
+
group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
205
|
+
return
|
206
|
+
|
207
|
+
_LOG.debug("find %s - entry name: %s", group.typ.name, entry.name)
|
208
|
+
group.add_entry_match(entry)
|
209
|
+
|
210
|
+
def find_group(abs_dir_path: str, group: _Group, other_group: _Group, parent_conf: DirConfig|None):
|
211
|
+
"""Recursively find all files belonging to 'group'"""
|
180
212
|
_LOG.debug("find %s: %s", group.typ.name, abs_dir_path)
|
181
213
|
if abs_dir_path in checked_dirs:
|
182
214
|
_LOG.debug("directory already checked")
|
183
215
|
return
|
184
216
|
|
185
217
|
group.num_directories += 1
|
186
|
-
dir_config
|
218
|
+
dir_config = self.config_files.dir_config(Path(abs_dir_path), parent_conf)
|
187
219
|
|
188
220
|
for entry in os.scandir(abs_dir_path):
|
189
|
-
|
190
|
-
if entry.path in other_group.dirs:
|
191
|
-
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
192
|
-
find_group(entry.path, other_group, group, dir_config)
|
193
|
-
continue
|
194
|
-
|
195
|
-
find_group(entry.path, group, other_group, dir_config)
|
196
|
-
continue
|
197
|
-
|
198
|
-
if config_file and entry.name == config_file.name:
|
199
|
-
continue
|
200
|
-
|
201
|
-
current_group = group
|
202
|
-
if group.typ is GroupType.MAY_WORK_ON:
|
203
|
-
# We need to check for match against configured protect patterns, if match, then the file must got to protect group instead
|
204
|
-
pattern = self.config_files.is_protected(entry, dir_config)
|
205
|
-
if pattern:
|
206
|
-
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
207
|
-
current_group = other_group
|
208
|
-
|
209
|
-
if entry.is_symlink():
|
210
|
-
points_to = os.readlink(entry)
|
211
|
-
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
212
|
-
|
213
|
-
if entry.is_dir(follow_symlinks=True):
|
214
|
-
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", current_group.typ.name, entry.path, points_to)
|
215
|
-
current_group.num_directory_symlinks += 1
|
216
|
-
continue
|
217
|
-
|
218
|
-
current_group.symlinks[entry.path] = entry
|
219
|
-
current_group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
220
|
-
continue
|
221
|
-
|
222
|
-
_LOG.debug("find %s - entry name: %s", current_group.typ.name, entry.name)
|
223
|
-
current_group.add_entry_match(entry)
|
221
|
+
handle_entry(abs_dir_path, group, other_group, dir_config, entry)
|
224
222
|
|
225
223
|
checked_dirs.add(abs_dir_path)
|
226
224
|
|
227
225
|
for any_dir in sorted(chain(self.must_protect.dirs, self.may_work_on.dirs), key=lambda dd: len(Path(dd).parts)):
|
228
226
|
parent_dir = Path(any_dir)
|
229
227
|
while len(parent_dir.parts) > 1:
|
230
|
-
parent_conf = self.config_files.per_dir_configs.get(
|
228
|
+
parent_conf = self.config_files.per_dir_configs.get(parent_dir)
|
231
229
|
if parent_conf:
|
232
230
|
break
|
233
231
|
|
234
232
|
parent_dir = parent_dir.parent
|
235
233
|
else:
|
236
|
-
parent_conf =
|
234
|
+
parent_conf = None
|
237
235
|
|
238
236
|
if any_dir in self.must_protect.dirs:
|
239
237
|
find_group(any_dir, self.must_protect, self.may_work_on, parent_conf)
|
file_groups/handler.py
CHANGED
@@ -7,6 +7,7 @@ import logging
|
|
7
7
|
from typing import Sequence
|
8
8
|
|
9
9
|
from .groups import FileGroups
|
10
|
+
from .config_files import ConfigFiles
|
10
11
|
|
11
12
|
|
12
13
|
_LOG = logging.getLogger(__name__)
|
@@ -20,26 +21,25 @@ class FileHandler(FileGroups):
|
|
20
21
|
Re-link symlinks when a file being deleted has a corresponding file.
|
21
22
|
|
22
23
|
Arguments:
|
23
|
-
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include: See `FileGroups` class.
|
24
|
-
dry_run: Don't
|
25
|
-
protected_regexes: Protect files matching this from being deleted or moved.
|
24
|
+
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include, config_files: See `FileGroups` class.
|
25
|
+
dry_run: Don't change any files.
|
26
26
|
delete_symlinks_instead_of_relinking: Normal operation is to re-link to a 'corresponding' or renamed file when renaming or deleting a file.
|
27
27
|
If delete_symlinks_instead_of_relinking is true, then symlinks in work_on dirs pointing to renamed/deletes files will be deleted even if
|
28
28
|
they could have logically been made to point to a file in a protect dir.
|
29
29
|
"""
|
30
30
|
|
31
|
-
def __init__(
|
31
|
+
def __init__( # pylint: disable=too-many-arguments
|
32
32
|
self,
|
33
33
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path],
|
34
34
|
*,
|
35
|
-
dry_run: bool,
|
36
|
-
protected_regexes: Sequence[re.Pattern],
|
37
35
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
36
|
+
config_files: ConfigFiles|None = None,
|
37
|
+
dry_run: bool,
|
38
38
|
delete_symlinks_instead_of_relinking=False):
|
39
39
|
super().__init__(
|
40
|
-
protect=protected_regexes,
|
41
40
|
protect_dirs_seq=protect_dirs_seq, work_dirs_seq=work_dirs_seq,
|
42
|
-
protect_exclude=protect_exclude, work_include=work_include
|
41
|
+
protect_exclude=protect_exclude, work_include=work_include,
|
42
|
+
config_files=config_files)
|
43
43
|
|
44
44
|
self.dry_run = dry_run
|
45
45
|
self.delete_symlinks_instead_of_relinking = delete_symlinks_instead_of_relinking
|
file_groups/handler_compare.py
CHANGED
@@ -7,6 +7,7 @@ from typing import Sequence
|
|
7
7
|
from .compare_files import CompareFiles
|
8
8
|
from .types import FsPath
|
9
9
|
from .handler import FileHandler
|
10
|
+
from .config_files import ConfigFiles
|
10
11
|
|
11
12
|
|
12
13
|
_LOG = logging.getLogger(__name__)
|
@@ -16,26 +17,24 @@ class FileHandlerCompare(FileHandler):
|
|
16
17
|
"""Extend `FileHandler` with a compare method
|
17
18
|
|
18
19
|
Arguments:
|
19
|
-
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include: See `FileGroups` class.
|
20
|
+
protect_dirs_seq, work_dirs_seq, protect_exclude, work_include, config_files: See `FileGroups` class.
|
20
21
|
dry_run, protected_regexes, delete_symlinks_instead_of_relinking: See `FileHandler` class.
|
21
22
|
fcmp: Object providing compare function.
|
22
23
|
"""
|
23
24
|
|
24
|
-
def __init__(
|
25
|
+
def __init__( # pylint: disable=too-many-arguments
|
25
26
|
self,
|
26
27
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path], fcmp: CompareFiles,
|
27
28
|
*,
|
28
|
-
dry_run: bool,
|
29
|
-
protected_regexes: Sequence[re.Pattern],
|
30
29
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
30
|
+
config_files: ConfigFiles|None = None,
|
31
|
+
dry_run: bool,
|
31
32
|
delete_symlinks_instead_of_relinking=False):
|
32
33
|
super().__init__(
|
33
|
-
protect_dirs_seq=protect_dirs_seq,
|
34
|
-
|
34
|
+
protect_dirs_seq=protect_dirs_seq, work_dirs_seq=work_dirs_seq,
|
35
|
+
protect_exclude=protect_exclude, work_include=work_include,
|
36
|
+
config_files=config_files,
|
35
37
|
dry_run=dry_run,
|
36
|
-
protected_regexes=protected_regexes,
|
37
|
-
protect_exclude=protect_exclude,
|
38
|
-
work_include=work_include,
|
39
38
|
delete_symlinks_instead_of_relinking=delete_symlinks_instead_of_relinking)
|
40
39
|
|
41
40
|
self._fcmp = fcmp
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
|
-
Name:
|
3
|
-
Version: 0.
|
2
|
+
Name: file_groups
|
3
|
+
Version: 0.3.1
|
4
4
|
Summary: Group files into 'protect' and 'work_on' and provide operations for safe delete/move and symlink handling.
|
5
5
|
Home-page: https://github.com/lhupfeldt/file_groups.git
|
6
6
|
Author: Lars Hupfeldt Nielsen
|
@@ -11,16 +11,17 @@ Classifier: Intended Audience :: Developers
|
|
11
11
|
Classifier: License :: OSI Approved :: BSD License
|
12
12
|
Classifier: Natural Language :: English
|
13
13
|
Classifier: Operating System :: OS Independent
|
14
|
-
Classifier: Programming Language :: Python :: 3.
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
17
18
|
Classifier: Topic :: Software Development :: Libraries
|
18
19
|
Requires-Python: >=3.10
|
19
20
|
Description-Content-Type: text/x-rst
|
20
21
|
License-File: LICENSE.txt
|
21
|
-
Requires-Dist: appdirs
|
22
|
+
Requires-Dist: appdirs>=1.4.4
|
22
23
|
Provides-Extra: dev
|
23
|
-
Requires-Dist: nox
|
24
|
+
Requires-Dist: nox; extra == "dev"
|
24
25
|
|
25
26
|
Library for grouping files into sets of 'work-on' and 'protect' based on arbitrarily nested directories.
|
26
27
|
|
@@ -0,0 +1,14 @@
|
|
1
|
+
file_groups/__init__.py,sha256=XBp5563xhTer1vQY-DpKRvViAvlHaZqx9GrINiGqbeI,127
|
2
|
+
file_groups/compare_files.py,sha256=38EfZzpEpj0wH7SUruPxgyF_W8eqCbIjG6yUOx8hcFA,434
|
3
|
+
file_groups/config_files.py,sha256=VwRmZYEemKQawYI43qV9XhvPyI7ouaDjjadhc74wqNs,14021
|
4
|
+
file_groups/groups.py,sha256=Lt8QlAET8032t9OXcq9cmzLy27tZsYTnLAaGilzEtcM,11377
|
5
|
+
file_groups/handler.py,sha256=NmYfmPLGXgPsGxM9uR2r8aS3cI-8S7D5-8884EoGA1M,10083
|
6
|
+
file_groups/handler_compare.py,sha256=qbGHwT0FSvfnWtAnjYi4px6V5xIM2cStm0soflwYAi4,2174
|
7
|
+
file_groups/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
file_groups/types.py,sha256=giq3H58RuKpS6b7sjxnzP-sN1Xr4VTQ-rG4brLP5qMI,107
|
9
|
+
file_groups-0.3.1.dist-info/LICENSE.txt,sha256=NZc9ictLEwfoL4ywpAbLbd-n02KETGHLYQU820TJD_Q,1494
|
10
|
+
file_groups-0.3.1.dist-info/METADATA,sha256=wSlWiFxs8yMgFvY6jN3pWIBGBVbbneRhyMQM5tNcoPw,1561
|
11
|
+
file_groups-0.3.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
12
|
+
file_groups-0.3.1.dist-info/top_level.txt,sha256=mAdRf0R2TA8tpH7ftHzXP7VMRhw07p7B2mI8QKpDnjU,12
|
13
|
+
file_groups-0.3.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
14
|
+
file_groups-0.3.1.dist-info/RECORD,,
|
@@ -1,14 +0,0 @@
|
|
1
|
-
file_groups/__init__.py,sha256=1OO3tLnRnIe51VfZSQz0sqUaUP-xS7oHxiSexnKwpEY,94
|
2
|
-
file_groups/compare_files.py,sha256=38EfZzpEpj0wH7SUruPxgyF_W8eqCbIjG6yUOx8hcFA,434
|
3
|
-
file_groups/config_files.py,sha256=IDwhznEM8wpMBp0Fl69r1CbPxfgYCu82iE3b_mxkk6g,10211
|
4
|
-
file_groups/groups.py,sha256=6tVIYBC5nCVPI4Ig1EuijT12accw2DZ08TEIMSJecLo,11531
|
5
|
-
file_groups/handler.py,sha256=06_8_fBEaHndrfUdxMsFw6KZw8PRcRj4MsH4FY3I4sU,10083
|
6
|
-
file_groups/handler_compare.py,sha256=78TXl9QFMIL5gZQhiLYvbzF5OidS7kcaRz6zCCNVG6I,2120
|
7
|
-
file_groups/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
-
file_groups/types.py,sha256=giq3H58RuKpS6b7sjxnzP-sN1Xr4VTQ-rG4brLP5qMI,107
|
9
|
-
file_groups-0.2.1.dist-info/LICENSE.txt,sha256=NZc9ictLEwfoL4ywpAbLbd-n02KETGHLYQU820TJD_Q,1494
|
10
|
-
file_groups-0.2.1.dist-info/METADATA,sha256=hnoCAPIfqQ_bjodGN2qVzLhGWCBdg1-PCmYpf5G0I1U,1512
|
11
|
-
file_groups-0.2.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
|
12
|
-
file_groups-0.2.1.dist-info/top_level.txt,sha256=mAdRf0R2TA8tpH7ftHzXP7VMRhw07p7B2mI8QKpDnjU,12
|
13
|
-
file_groups-0.2.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
14
|
-
file_groups-0.2.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|