file-groups 0.2.1__tar.gz → 0.3.1__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {file_groups-0.2.1 → file_groups-0.3.1}/.copier-answers.yml +2 -1
- {file_groups-0.2.1 → file_groups-0.3.1}/.gitignore +4 -2
- {file_groups-0.2.1 → file_groups-0.3.1}/PKG-INFO +4 -3
- {file_groups-0.2.1 → file_groups-0.3.1}/file_groups.egg-info/PKG-INFO +5 -4
- {file_groups-0.2.1 → file_groups-0.3.1}/file_groups.egg-info/SOURCES.txt +6 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/noxfile.py +16 -5
- {file_groups-0.2.1 → file_groups-0.3.1}/pytest.ini +1 -1
- {file_groups-0.2.1 → file_groups-0.3.1}/setup.cfg +3 -2
- {file_groups-0.2.1 → file_groups-0.3.1}/src/__init__.py +2 -0
- file_groups-0.3.1/src/config_files.py +315 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/src/groups.py +53 -55
- {file_groups-0.2.1 → file_groups-0.3.1}/src/handler.py +8 -8
- {file_groups-0.2.1 → file_groups-0.3.1}/src/handler_compare.py +8 -9
- {file_groups-0.2.1 → file_groups-0.3.1}/test/.coveragerc +3 -1
- {file_groups-0.2.1 → file_groups-0.3.1}/test/config_files_test.py +178 -201
- {file_groups-0.2.1 → file_groups-0.3.1}/test/conftest.py +50 -21
- file_groups-0.3.1/test/groups/config_files_test.py +33 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/groups/directory_symlinks_test.py +4 -4
- {file_groups-0.2.1 → file_groups-0.3.1}/test/groups/re_protect_test.py +17 -1
- {file_groups-0.2.1 → file_groups-0.3.1}/test/handler/core_test.py +32 -32
- {file_groups-0.2.1 → file_groups-0.3.1}/test/handler/private_test.py +7 -6
- {file_groups-0.2.1 → file_groups-0.3.1}/test/handler/regex_protection_test.py +9 -8
- {file_groups-0.2.1 → file_groups-0.3.1}/test/handler_compare_test.py +4 -4
- file_groups-0.3.1/test/in/configs/config_files_sys_user_config_files_additional_appdirs/home/an_app.conf +15 -0
- file_groups-0.3.1/test/in/configs/config_files_sys_user_config_files_additional_appdirs/home/file_groups.conf +14 -0
- file_groups-0.3.1/test/in/configs/config_files_sys_user_config_files_additional_appdirs/sys/file_groups.conf +16 -0
- file_groups-0.3.1/test/in/configs/file_groups_group_dirs_by_config_file_protect/home/file_groups.conf +11 -0
- file_groups-0.3.1/test/in/configs/file_groups_group_files_by_config_protect/home/file_groups.conf +17 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/version_test.py +1 -1
- file_groups-0.2.1/src/config_files.py +0 -244
- file_groups-0.2.1/test/groups/config_files_test.py +0 -68
- {file_groups-0.2.1 → file_groups-0.3.1}/.pylintrc +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/LICENSE.txt +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/README.rst +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/file_groups.egg-info/dependency_links.txt +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/file_groups.egg-info/requires.txt +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/file_groups.egg-info/top_level.txt +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/file_groups.egg-info/zip-safe +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/pyproject.toml +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/setup.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/src/compare_files.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/src/py.typed +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/src/types.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/__init__.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/compare_files_test.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/groups/__init__.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/groups/core_test.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/groups/utils.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/handler/__init__.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/handler/utils.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_inherit_global_recursive/home/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_inherit_global_recursive/sys/file_groups.conf +0 -0
- /file_groups-0.2.1/test/in/configs/file_groups_group_files_by_config_protect/home/file_groups.conf → /file_groups-0.3.1/test/in/configs/config_files_specified/direct.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_sys_config_file_no_global/sys/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_sys_user_config_files_no_global/home/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_sys_user_config_files_no_global/sys/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_two_in_same_config_dir/home/.file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_two_in_same_config_dir/home/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_unknown_protect_sub_key_config_dir/sys/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/config_files_user_config_file_no_global/home/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/in/configs/file_groups_group_files_by_config_protect/sys/file_groups.conf +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/mypy_requirements.txt +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/perf/protect_home_work_on_pictures.py +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/perf/requirements.txt +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/pylint_requirements.txt +0 -0
- {file_groups-0.2.1 → file_groups-0.3.1}/test/requirements.txt +0 -0
@@ -1,5 +1,5 @@
|
|
1
1
|
# Changes here will be overwritten by Copier
|
2
|
-
_commit:
|
2
|
+
_commit: 1ddef8a
|
3
3
|
_src_path: git@github.com:lhupfeldt/copier_python_package_template.git
|
4
4
|
author_email: lhn@hupfeldtit.dk
|
5
5
|
author_name: Lars Hupfeldt Nielsen
|
@@ -11,5 +11,6 @@ description: Group files into 'protect' and 'work_on' and provide operations for
|
|
11
11
|
git_repo_url: https://github.com/lhupfeldt/file_groups.git
|
12
12
|
github_organization: lhupfeldt
|
13
13
|
legal_entity: ApS
|
14
|
+
local_devel_dependency: ''
|
14
15
|
module_name: file_groups
|
15
16
|
package_name: file_groups
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: file_groups
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.3.1
|
4
4
|
Summary: Group files into 'protect' and 'work_on' and provide operations for safe delete/move and symlink handling.
|
5
5
|
Home-page: https://github.com/lhupfeldt/file_groups.git
|
6
6
|
Author: Lars Hupfeldt Nielsen
|
@@ -11,9 +11,10 @@ Classifier: Intended Audience :: Developers
|
|
11
11
|
Classifier: License :: OSI Approved :: BSD License
|
12
12
|
Classifier: Natural Language :: English
|
13
13
|
Classifier: Operating System :: OS Independent
|
14
|
-
Classifier: Programming Language :: Python :: 3.
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
17
18
|
Classifier: Topic :: Software Development :: Libraries
|
18
19
|
Requires-Python: >=3.10
|
19
20
|
Description-Content-Type: text/x-rst
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
|
-
Name:
|
3
|
-
Version: 0.
|
2
|
+
Name: file_groups
|
3
|
+
Version: 0.3.1
|
4
4
|
Summary: Group files into 'protect' and 'work_on' and provide operations for safe delete/move and symlink handling.
|
5
5
|
Home-page: https://github.com/lhupfeldt/file_groups.git
|
6
6
|
Author: Lars Hupfeldt Nielsen
|
@@ -11,9 +11,10 @@ Classifier: Intended Audience :: Developers
|
|
11
11
|
Classifier: License :: OSI Approved :: BSD License
|
12
12
|
Classifier: Natural Language :: English
|
13
13
|
Classifier: Operating System :: OS Independent
|
14
|
-
Classifier: Programming Language :: Python :: 3.
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
17
18
|
Classifier: Topic :: Software Development :: Libraries
|
18
19
|
Requires-Python: >=3.10
|
19
20
|
Description-Content-Type: text/x-rst
|
@@ -47,16 +47,22 @@ test/in/configs/config_files_inherit_global_recursive_ignore_other
|
|
47
47
|
test/in/configs/config_files_inherit_global_recursive_no_other
|
48
48
|
test/in/configs/config_files_inherit_ignore_global_recursive
|
49
49
|
test/in/configs/config_files_sys_user_and_and_other_dir_config_files_no_global_no_other_recursive
|
50
|
+
test/in/configs/config_files_sys_user_config_files_replaced_appdirs
|
50
51
|
test/in/configs/file_groups_sys_user_config_files_no_global
|
51
52
|
test/in/configs/config_files_inherit_global_recursive/home/file_groups.conf
|
52
53
|
test/in/configs/config_files_inherit_global_recursive/sys/file_groups.conf
|
54
|
+
test/in/configs/config_files_specified/direct.conf
|
53
55
|
test/in/configs/config_files_sys_config_file_no_global/sys/file_groups.conf
|
56
|
+
test/in/configs/config_files_sys_user_config_files_additional_appdirs/home/an_app.conf
|
57
|
+
test/in/configs/config_files_sys_user_config_files_additional_appdirs/home/file_groups.conf
|
58
|
+
test/in/configs/config_files_sys_user_config_files_additional_appdirs/sys/file_groups.conf
|
54
59
|
test/in/configs/config_files_sys_user_config_files_no_global/home/file_groups.conf
|
55
60
|
test/in/configs/config_files_sys_user_config_files_no_global/sys/file_groups.conf
|
56
61
|
test/in/configs/config_files_two_in_same_config_dir/home/.file_groups.conf
|
57
62
|
test/in/configs/config_files_two_in_same_config_dir/home/file_groups.conf
|
58
63
|
test/in/configs/config_files_unknown_protect_sub_key_config_dir/sys/file_groups.conf
|
59
64
|
test/in/configs/config_files_user_config_file_no_global/home/file_groups.conf
|
65
|
+
test/in/configs/file_groups_group_dirs_by_config_file_protect/home/file_groups.conf
|
60
66
|
test/in/configs/file_groups_group_files_by_config_protect/home/file_groups.conf
|
61
67
|
test/in/configs/file_groups_group_files_by_config_protect/sys/file_groups.conf
|
62
68
|
test/perf/protect_home_work_on_pictures.py
|
@@ -2,14 +2,16 @@
|
|
2
2
|
|
3
3
|
# Use nox >= 2023.4.22
|
4
4
|
|
5
|
+
import os
|
5
6
|
from pathlib import Path
|
7
|
+
import shutil
|
6
8
|
|
7
9
|
import nox
|
8
10
|
|
9
11
|
|
10
12
|
_HERE = Path(__file__).absolute().parent
|
11
13
|
_TEST_DIR = _HERE/"test"
|
12
|
-
_PY_VERSIONS = [
|
14
|
+
_PY_VERSIONS = ['3.13', '3.12', '3.11', '3.10']
|
13
15
|
|
14
16
|
nox.options.error_on_missing_interpreters = True
|
15
17
|
|
@@ -17,17 +19,17 @@ nox.options.error_on_missing_interpreters = True
|
|
17
19
|
@nox.session(python=_PY_VERSIONS, reuse_venv=True)
|
18
20
|
def typecheck(session):
|
19
21
|
session.install("-e", ".", "mypy>=1.5.1")
|
20
|
-
session.run("mypy",
|
22
|
+
session.run("mypy", str(_HERE/"src"))
|
21
23
|
|
22
24
|
|
23
|
-
|
24
|
-
@nox.session(python="3.11", reuse_venv=True)
|
25
|
+
@nox.session(python=_PY_VERSIONS[0], reuse_venv=True)
|
25
26
|
def pylint(session):
|
26
|
-
session.install(".", "pylint>=
|
27
|
+
session.install(".", "pylint>=3.3.1", "pylint-pytest>=1.1.8")
|
27
28
|
|
28
29
|
print("\nPylint src")
|
29
30
|
disable_checks = "missing-module-docstring"
|
30
31
|
session.run("pylint", "--fail-under", "10", "--disable", disable_checks, str(_HERE/"src"))
|
32
|
+
|
31
33
|
print("\nPylint test sources")
|
32
34
|
disable_checks += ",missing-class-docstring,missing-function-docstring,multiple-imports,invalid-name,duplicate-code"
|
33
35
|
session.run("pylint", "--fail-under", "9.94", "--variable-rgx", r"[a-z_][a-z0-9_]{1,30}$", "--disable", disable_checks, str(_HERE/"test"))
|
@@ -37,3 +39,12 @@ def pylint(session):
|
|
37
39
|
def unit(session):
|
38
40
|
session.install(".", "pytest>=7.4.1", "coverage>=7.3.1", "pytest-cov>=4.1.0")
|
39
41
|
session.run("pytest", "--import-mode=append", "--cov", "--cov-report=term-missing", f"--cov-config={_TEST_DIR}/.coveragerc", *session.posargs)
|
42
|
+
|
43
|
+
|
44
|
+
@nox.session(python=_PY_VERSIONS[0], reuse_venv=True)
|
45
|
+
def build(session):
|
46
|
+
if Path("dist").is_dir():
|
47
|
+
shutil.rmtree("dist")
|
48
|
+
session.install("build>=1.0.3", "twine>=4.0.2")
|
49
|
+
session.run("python", "-m", "build")
|
50
|
+
session.run("python", "-m", "twine", "check", "dist/*")
|
@@ -13,9 +13,10 @@ classifiers =
|
|
13
13
|
License :: OSI Approved :: BSD License
|
14
14
|
Natural Language :: English
|
15
15
|
Operating System :: OS Independent
|
16
|
-
Programming Language :: Python :: 3.
|
17
|
-
Programming Language :: Python :: 3.11
|
16
|
+
Programming Language :: Python :: 3.13
|
18
17
|
Programming Language :: Python :: 3.12
|
18
|
+
Programming Language :: Python :: 3.11
|
19
|
+
Programming Language :: Python :: 3.10
|
19
20
|
Topic :: Software Development :: Libraries
|
20
21
|
|
21
22
|
[options]
|
@@ -0,0 +1,315 @@
|
|
1
|
+
import ast
|
2
|
+
import os
|
3
|
+
import errno
|
4
|
+
import re
|
5
|
+
from pathlib import Path
|
6
|
+
import itertools
|
7
|
+
from pprint import pformat
|
8
|
+
import logging
|
9
|
+
from dataclasses import dataclass
|
10
|
+
from typing import Tuple, Sequence, Any
|
11
|
+
|
12
|
+
from appdirs import AppDirs # type: ignore
|
13
|
+
|
14
|
+
from .types import FsPath
|
15
|
+
|
16
|
+
|
17
|
+
_LOG = logging.getLogger(__name__)
|
18
|
+
|
19
|
+
|
20
|
+
class ConfigException(Exception):
|
21
|
+
"""Invalid configuration"""
|
22
|
+
|
23
|
+
|
24
|
+
@dataclass
|
25
|
+
class ProtectConfig():
|
26
|
+
"""Hold global (site or user) protect config."""
|
27
|
+
protect_recursive: set[re.Pattern]
|
28
|
+
|
29
|
+
def __json__(self):
|
30
|
+
return {
|
31
|
+
ProtectConfig.__name__: {
|
32
|
+
"protect_recursive": [str(pat) for pat in self.protect_recursive],
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
36
|
+
|
37
|
+
@dataclass
|
38
|
+
class DirConfig(ProtectConfig):
|
39
|
+
"""Hold directory specific protect config."""
|
40
|
+
protect_local: set[re.Pattern]
|
41
|
+
config_dir: Path|None
|
42
|
+
config_files: list[str]
|
43
|
+
|
44
|
+
def is_protected(self, ff: FsPath):
|
45
|
+
"""If ff id protected by a regex pattern then return the pattern, otherwise return None."""
|
46
|
+
|
47
|
+
# _LOG.debug("ff '%s'", ff)
|
48
|
+
for pattern in itertools.chain(self.protect_local, self.protect_recursive):
|
49
|
+
if os.sep in str(pattern):
|
50
|
+
# _LOG.debug("Pattern '%s' has path sep", pattern)
|
51
|
+
assert os.path.isabs(ff), f"Expected absolute path, got '{ff}'"
|
52
|
+
|
53
|
+
# Search against full path
|
54
|
+
if pattern.search(os.fspath(ff)):
|
55
|
+
return pattern
|
56
|
+
|
57
|
+
# Attempt exact match against path relative to , i.e. if pattern starts with '^'.
|
58
|
+
# This makes sense for patterns specified on commandline
|
59
|
+
cwd = os.getcwd()
|
60
|
+
ff_relative = str(Path(ff).relative_to(cwd))
|
61
|
+
# _LOG.debug("ff '%s' relative to start dir'%s'", ff_relative, cwd)
|
62
|
+
|
63
|
+
if pattern.match(ff_relative):
|
64
|
+
return pattern
|
65
|
+
|
66
|
+
elif pattern.search(ff.name):
|
67
|
+
return pattern
|
68
|
+
|
69
|
+
return None
|
70
|
+
|
71
|
+
def __json__(self):
|
72
|
+
return {
|
73
|
+
DirConfig.__name__: super().__json__()[ProtectConfig.__name__] | {
|
74
|
+
"protect_local": [str(pat) for pat in self.protect_local],
|
75
|
+
"config_dir": str(self.config_dir),
|
76
|
+
"config_files": self.config_files,
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
|
81
|
+
class ConfigFiles():
|
82
|
+
r"""Handle config files.
|
83
|
+
|
84
|
+
Config files are searched for in the standard config directories on the platform AND in any collected directory.
|
85
|
+
|
86
|
+
The 'app_dirs' default sets default config dirs and config-file names.
|
87
|
+
It is also possible to specify additional or alternative config files specific to the application using this library.
|
88
|
+
Config files must be named after the AppDirs.appname (first argument) as <appname>.conf or .<appname>.conf.
|
89
|
+
The defaults are 'file_groups.conf' and '.file_groups.conf'.
|
90
|
+
You should consider carefully before disabling loading of the default files, as an end user likely wants the protection rules to apply for any
|
91
|
+
application using this library.
|
92
|
+
|
93
|
+
The content of a conf file is a Python dict with the following structure.
|
94
|
+
|
95
|
+
{
|
96
|
+
"file_groups": { # Required
|
97
|
+
"protect": { # Optional
|
98
|
+
"local": [ # Optional
|
99
|
+
... # Regex patterns
|
100
|
+
],
|
101
|
+
"recursive": [ # Optional, merged with parent config dir property
|
102
|
+
... # Regex patterns
|
103
|
+
]
|
104
|
+
"global": [ # Optional. Only allowed in config directory files. Merged into collect dir configs 'recursive' property.
|
105
|
+
... # Regex patterns
|
106
|
+
],
|
107
|
+
},
|
108
|
+
}
|
109
|
+
...
|
110
|
+
}
|
111
|
+
|
112
|
+
E.g.:
|
113
|
+
|
114
|
+
{
|
115
|
+
"file_groups": {
|
116
|
+
"protect": {
|
117
|
+
"recursive": [
|
118
|
+
r"PP.*\.jpg", # Don't mess with JPEG files starting with 'PP'.
|
119
|
+
]
|
120
|
+
}
|
121
|
+
}
|
122
|
+
}
|
123
|
+
|
124
|
+
The level one key is 'file_groups'.
|
125
|
+
Applications are free to add entries at this level, but not underneath. This is protect against ignored misspelled keys.
|
126
|
+
|
127
|
+
The 'file_groups' entry is a dict with a single 'protect' entry.
|
128
|
+
The 'protect' entry is a dict with at most three entries: 'local', 'recursive' and 'global'. These specify whether a directory specific
|
129
|
+
configuration will inherit and extend the parent (and global) config, or whether it is local to current directory only.
|
130
|
+
The 'local', 'recursive' and 'global' entries are lists of regex patterns to match against collected 'work_on' files.
|
131
|
+
Regexes are checked against the simple file name (i.e. not the full path) unless they contain at least one path separator (os.sep), in
|
132
|
+
which case they are checked against the absolute path.
|
133
|
+
All checks are done as regex *search* (better to protect too much than too little). Write the regex to match the full name or path if needed.
|
134
|
+
|
135
|
+
Note that for security ast.literal_eval is used to interpret the config, so no code is allowed.
|
136
|
+
|
137
|
+
Arguments:
|
138
|
+
protect: An optional sequence of regexes to be added to protect[recursive] for all directories.
|
139
|
+
ignore_config_dirs_config_files: Ignore config files in standard config directories.
|
140
|
+
ignore_per_directory_config_files: Ignore config files in collected directories.
|
141
|
+
remember_configs: Store loaded and merged configs in `per_dir_configs` member variable.
|
142
|
+
app_dirs: Provide your own instance of AppDirs in addition to or as a replacement of the default to add config file names and path.
|
143
|
+
Configuration from later entries have higher precedence.
|
144
|
+
Note that if no AppDirs are specified, no config files will be loaded, neither from config dirs, nor from collected directories.
|
145
|
+
See: https://pypi.org/project/appdirs/
|
146
|
+
|
147
|
+
Members:
|
148
|
+
conf_file_names: File names which are config files.
|
149
|
+
remember_configs: Whether per directory resolved/merged configs are stored in `per_dir_configs`.
|
150
|
+
per_dir_configs: Mapping from dir name to directory specific config dict. Only if remember_configs is True.
|
151
|
+
"""
|
152
|
+
|
153
|
+
default_appdirs: AppDirs = AppDirs("file_groups", "Hupfeldt_IT")
|
154
|
+
|
155
|
+
_fg_key = "file_groups"
|
156
|
+
_protect_key = "protect"
|
157
|
+
_valid_dir_protect_scopes = ("local", "recursive")
|
158
|
+
_valid_config_dir_protect_scopes = ("local", "recursive", "global")
|
159
|
+
|
160
|
+
def __init__( # pylint: disable=too-many-positional-arguments,too-many-arguments
|
161
|
+
self, protect: Sequence[re.Pattern] = (),
|
162
|
+
ignore_config_dirs_config_files=False, ignore_per_directory_config_files=False, remember_configs=True,
|
163
|
+
app_dirs: Sequence[AppDirs]|None = None,
|
164
|
+
*,
|
165
|
+
config_file: Path|None = None,
|
166
|
+
):
|
167
|
+
super().__init__()
|
168
|
+
|
169
|
+
self._global_config = ProtectConfig(set(protect))
|
170
|
+
self.remember_configs = remember_configs
|
171
|
+
self.per_dir_configs: dict[Path, DirConfig] = {} # key is abs_dir_path
|
172
|
+
self.ignore_per_directory_config_files = ignore_per_directory_config_files
|
173
|
+
|
174
|
+
app_dirs = app_dirs or (ConfigFiles.default_appdirs,)
|
175
|
+
self.conf_file_name_pairs = tuple((apd.appname + ".conf", "." + apd.appname + ".conf") for apd in app_dirs)
|
176
|
+
_LOG.debug("Conf file names: %s", self.conf_file_name_pairs)
|
177
|
+
self.conf_file_names = list(itertools.chain.from_iterable(self.conf_file_name_pairs))
|
178
|
+
|
179
|
+
self.ignore_config_dirs_config_files = ignore_config_dirs_config_files
|
180
|
+
self.config_dirs = []
|
181
|
+
for appd in app_dirs:
|
182
|
+
self.config_dirs.extend(appd.site_config_dir.split(':'))
|
183
|
+
for appd in app_dirs:
|
184
|
+
self.config_dirs.append(appd.user_config_dir)
|
185
|
+
|
186
|
+
self.config_file = config_file
|
187
|
+
|
188
|
+
# self.default_config_file_example = self.default_config_file.with_suffix('.example.py')
|
189
|
+
|
190
|
+
def _read_and_eval_config_file_for_one_appname(
|
191
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], ignore_config_files: bool
|
192
|
+
) -> Tuple[dict[str, Any], Path|None]:
|
193
|
+
"""Read config file.
|
194
|
+
|
195
|
+
Error if config files are found both with and withput '.' prefix.
|
196
|
+
Return: Config dict, config file name, or if no config files is found, None, None.
|
197
|
+
"""
|
198
|
+
|
199
|
+
assert conf_dir.is_absolute()
|
200
|
+
_LOG.debug("Checking for config files %s in directory: %s", conf_file_name_pair, conf_dir)
|
201
|
+
|
202
|
+
match [conf_dir/cfn for cfn in conf_file_name_pair if (conf_dir/cfn).exists()]:
|
203
|
+
case []:
|
204
|
+
_LOG.debug("No config file in directory %s", conf_dir)
|
205
|
+
return {}, None
|
206
|
+
|
207
|
+
case [conf_file]:
|
208
|
+
if ignore_config_files:
|
209
|
+
_LOG.debug("Ignoring config file: %s", conf_file)
|
210
|
+
return {}, None
|
211
|
+
|
212
|
+
_LOG.debug("Read config file: %s", conf_file)
|
213
|
+
with open(conf_file, encoding="utf-8") as fh:
|
214
|
+
new_config = ast.literal_eval(fh.read())
|
215
|
+
_LOG.debug("%s", pformat(new_config))
|
216
|
+
return new_config, conf_file
|
217
|
+
|
218
|
+
case config_files:
|
219
|
+
msg = f"More than one config file in dir '{conf_dir}': {[cf.name for cf in config_files]}."
|
220
|
+
_LOG.debug("%s", msg)
|
221
|
+
raise ConfigException(msg)
|
222
|
+
|
223
|
+
def _read_and_validate_config_file_for_one_appname(
|
224
|
+
self, conf_dir: Path, conf_file_name_pair: Sequence[str], valid_protect_scopes: Tuple[str, ...], ignore_config_files: bool
|
225
|
+
) -> Tuple[dict[str, set[re.Pattern]], Path|None]:
|
226
|
+
"""Read config file, validate keys and compile regexes.
|
227
|
+
|
228
|
+
Error if config files are found both with and withput '.' prefix.
|
229
|
+
|
230
|
+
Return: merged config dict with compiled regexes, config file name. If no config files is found, then return empty sets and None.
|
231
|
+
"""
|
232
|
+
|
233
|
+
new_config, conf_file = self._read_and_eval_config_file_for_one_appname(conf_dir, conf_file_name_pair, ignore_config_files)
|
234
|
+
if not new_config:
|
235
|
+
return {
|
236
|
+
"local": set(),
|
237
|
+
"recursive": set(),
|
238
|
+
}, None
|
239
|
+
|
240
|
+
try:
|
241
|
+
protect_conf: dict[str, set[re.Pattern]] = new_config[self._fg_key][self._protect_key]
|
242
|
+
except KeyError as ex:
|
243
|
+
raise ConfigException(f"Config file '{conf_file}' is missing mandatory configuration '{self._fg_key}[{self._protect_key}]'.") from ex
|
244
|
+
|
245
|
+
for key, val in protect_conf.items():
|
246
|
+
if key not in valid_protect_scopes:
|
247
|
+
msg = f"The only keys allowed in '{self._fg_key}[{self._protect_key}]' section in the config file '{conf_file}' are: {valid_protect_scopes}. Got: '{key}'."
|
248
|
+
_LOG.debug("%s", msg)
|
249
|
+
raise ConfigException(msg)
|
250
|
+
|
251
|
+
protect_conf[key] = set(re.compile(pattern) for pattern in val)
|
252
|
+
|
253
|
+
for key in self._valid_dir_protect_scopes: # Do NOT use the 'valid_protect_scopes' argument here
|
254
|
+
protect_conf.setdefault(key, set())
|
255
|
+
|
256
|
+
lvl = logging.DEBUG
|
257
|
+
if _LOG.isEnabledFor(lvl):
|
258
|
+
_LOG.log(lvl, "Merged directory config:\n%s", pformat(new_config))
|
259
|
+
|
260
|
+
return protect_conf, conf_file
|
261
|
+
|
262
|
+
def load_config_dir_files(self) -> None:
|
263
|
+
"""Load config files from platform standard directories and specified config file, if any."""
|
264
|
+
|
265
|
+
if not self.ignore_config_dirs_config_files:
|
266
|
+
_LOG.debug("config_dirs: %s", self.config_dirs)
|
267
|
+
for conf_dir in self.config_dirs:
|
268
|
+
conf_dir = Path(conf_dir)
|
269
|
+
if not conf_dir.exists():
|
270
|
+
continue
|
271
|
+
|
272
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
273
|
+
cfg, _ = self._read_and_validate_config_file_for_one_appname(
|
274
|
+
conf_dir, conf_file_name_pair, self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
275
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
276
|
+
|
277
|
+
if self.config_file:
|
278
|
+
_LOG.debug("specified config_file: %s", self.config_file)
|
279
|
+
conf_dir = self.config_file.parent.absolute()
|
280
|
+
conf_name = self.config_file.name
|
281
|
+
cfg, fpath = self._read_and_validate_config_file_for_one_appname(
|
282
|
+
conf_dir, (conf_name,), self._valid_config_dir_protect_scopes, self.ignore_config_dirs_config_files)
|
283
|
+
if not fpath:
|
284
|
+
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(self.config_file))
|
285
|
+
self._global_config.protect_recursive |= cfg.get("global", set())
|
286
|
+
|
287
|
+
_LOG.debug("Merged global config:\n %s", self._global_config)
|
288
|
+
|
289
|
+
def dir_config(self, conf_dir: Path, parent_conf: DirConfig|None) -> DirConfig:
|
290
|
+
"""Read and merge config file from directory 'conf_dir' with 'parent_conf'.
|
291
|
+
|
292
|
+
If directory has no parent in the file_groups included dirs, then None should be supplied as parent_conf.
|
293
|
+
"""
|
294
|
+
|
295
|
+
cfg_merge_local: set[re.Pattern] = set()
|
296
|
+
cfg_merge_recursive: set[re.Pattern] = set()
|
297
|
+
cfg_files: list[str] = []
|
298
|
+
|
299
|
+
for conf_file_name_pair in self.conf_file_name_pairs:
|
300
|
+
cfg, cfg_file = self._read_and_validate_config_file_for_one_appname(
|
301
|
+
conf_dir, conf_file_name_pair, self._valid_dir_protect_scopes, self.ignore_per_directory_config_files)
|
302
|
+
cfg_merge_local.update(cfg.get("local", set()))
|
303
|
+
cfg_merge_recursive.update(cfg.get("recursive", set()))
|
304
|
+
if cfg_file:
|
305
|
+
cfg_files.append(cfg_file.name)
|
306
|
+
|
307
|
+
parent_protect_recursive = parent_conf.protect_recursive if parent_conf else self._global_config.protect_recursive
|
308
|
+
new_config = DirConfig(cfg_merge_recursive | parent_protect_recursive, cfg_merge_local, conf_dir, cfg_files)
|
309
|
+
_LOG.debug("new_config:\n %s", new_config)
|
310
|
+
|
311
|
+
if self.remember_configs:
|
312
|
+
self.per_dir_configs[conf_dir] = new_config
|
313
|
+
# _LOG.debug("per_dir_configs:\n %s", self.per_dir_configs)
|
314
|
+
|
315
|
+
return new_config
|
@@ -7,9 +7,9 @@ from dataclasses import dataclass
|
|
7
7
|
from itertools import chain
|
8
8
|
from enum import Enum
|
9
9
|
import logging
|
10
|
-
from typing import Sequence
|
10
|
+
from typing import Sequence, cast
|
11
11
|
|
12
|
-
from .config_files import ConfigFiles
|
12
|
+
from .config_files import DirConfig, ConfigFiles
|
13
13
|
|
14
14
|
|
15
15
|
_LOG = logging.getLogger(__name__)
|
@@ -87,27 +87,21 @@ class FileGroups():
|
|
87
87
|
|
88
88
|
protect_exclude: Exclude files matching regex in the protected files (does not apply to symlinks). Default: Include ALL.
|
89
89
|
Note: Since these files are excluded from protection, it means they er NOT protected!
|
90
|
-
work_include:
|
90
|
+
work_include: ONLY include files matching regex in the may_work_on files (does not apply to symlinks). Default: Include ALL.
|
91
91
|
|
92
|
-
|
93
|
-
ignore_per_directory_config_files: Ignore config files in collected directories.
|
92
|
+
config_files: Load config files. See config_files.ConfigFiles. Note that the default 'None' means use the `config_files.ConfigFiles` class with default arguments.
|
94
93
|
"""
|
95
94
|
|
96
95
|
def __init__(
|
97
96
|
self,
|
98
97
|
protect_dirs_seq: Sequence[Path], work_dirs_seq: Sequence[Path],
|
99
98
|
*,
|
100
|
-
protect: Sequence[re.Pattern] = (),
|
101
99
|
protect_exclude: re.Pattern|None = None, work_include: re.Pattern|None = None,
|
102
|
-
|
103
|
-
remember_configs=True):
|
100
|
+
config_files: ConfigFiles|None = None):
|
104
101
|
super().__init__()
|
105
102
|
|
106
|
-
self.config_files = ConfigFiles(
|
107
|
-
|
108
|
-
ignore_config_dirs_config_files=ignore_config_dirs_config_files,
|
109
|
-
ignore_per_directory_config_files=ignore_per_directory_config_files,
|
110
|
-
remember_configs=remember_configs)
|
103
|
+
self.config_files = config_files or ConfigFiles()
|
104
|
+
self.config_files.load_config_dir_files()
|
111
105
|
|
112
106
|
# Turn all paths into absolute paths with symlinks resolved, keep referrence to original argument for messages
|
113
107
|
protect_dirs: dict[str, Path] = {os.path.abspath(os.path.realpath(kp)): kp for kp in protect_dirs_seq}
|
@@ -153,8 +147,8 @@ class FileGroups():
|
|
153
147
|
top/d2/d1
|
154
148
|
top/d2/d1/f1.jpg
|
155
149
|
|
156
|
-
When:
|
157
|
-
And:
|
150
|
+
When: work_dirs_seq is [top, top/d1/d1]
|
151
|
+
And: protect_dirs_seq is [top/d1]
|
158
152
|
|
159
153
|
Then:
|
160
154
|
|
@@ -175,65 +169,69 @@ class FileGroups():
|
|
175
169
|
|
176
170
|
checked_dirs: set[str] = set()
|
177
171
|
|
178
|
-
def
|
179
|
-
"""
|
172
|
+
def handle_entry(abs_dir_path: str, group: _Group, other_group: _Group, dir_config: DirConfig, entry: DirEntry):
|
173
|
+
"""Put entry in the correct group. Call 'find_group' if entry is a directory."""
|
174
|
+
if group.typ is GroupType.MAY_WORK_ON:
|
175
|
+
# Check for match against configured protect patterns, if match, then the file must got to protect group instead
|
176
|
+
pattern = dir_config.is_protected(entry)
|
177
|
+
if pattern:
|
178
|
+
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
179
|
+
group, other_group = other_group, group
|
180
|
+
|
181
|
+
if entry.is_dir(follow_symlinks=False):
|
182
|
+
if entry.path in other_group.dirs:
|
183
|
+
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
184
|
+
find_group(entry.path, other_group, group, dir_config)
|
185
|
+
return
|
186
|
+
|
187
|
+
find_group(entry.path, group, other_group, dir_config)
|
188
|
+
return
|
189
|
+
|
190
|
+
if entry.name in self.config_files.conf_file_names:
|
191
|
+
return
|
192
|
+
|
193
|
+
if entry.is_symlink():
|
194
|
+
# cast: https://github.com/python/mypy/issues/11964
|
195
|
+
points_to = os.readlink(cast(str, entry))
|
196
|
+
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
197
|
+
|
198
|
+
if entry.is_dir(follow_symlinks=True):
|
199
|
+
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", group.typ.name, entry.path, points_to)
|
200
|
+
group.num_directory_symlinks += 1
|
201
|
+
return
|
202
|
+
|
203
|
+
group.symlinks[entry.path] = entry
|
204
|
+
group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
205
|
+
return
|
206
|
+
|
207
|
+
_LOG.debug("find %s - entry name: %s", group.typ.name, entry.name)
|
208
|
+
group.add_entry_match(entry)
|
209
|
+
|
210
|
+
def find_group(abs_dir_path: str, group: _Group, other_group: _Group, parent_conf: DirConfig|None):
|
211
|
+
"""Recursively find all files belonging to 'group'"""
|
180
212
|
_LOG.debug("find %s: %s", group.typ.name, abs_dir_path)
|
181
213
|
if abs_dir_path in checked_dirs:
|
182
214
|
_LOG.debug("directory already checked")
|
183
215
|
return
|
184
216
|
|
185
217
|
group.num_directories += 1
|
186
|
-
dir_config
|
218
|
+
dir_config = self.config_files.dir_config(Path(abs_dir_path), parent_conf)
|
187
219
|
|
188
220
|
for entry in os.scandir(abs_dir_path):
|
189
|
-
|
190
|
-
if entry.path in other_group.dirs:
|
191
|
-
_LOG.debug("find %s - '%s' is in '%s' dir list and not in '%s' dir list", group.typ.name, entry.path, other_group.typ.name, group.typ.name)
|
192
|
-
find_group(entry.path, other_group, group, dir_config)
|
193
|
-
continue
|
194
|
-
|
195
|
-
find_group(entry.path, group, other_group, dir_config)
|
196
|
-
continue
|
197
|
-
|
198
|
-
if config_file and entry.name == config_file.name:
|
199
|
-
continue
|
200
|
-
|
201
|
-
current_group = group
|
202
|
-
if group.typ is GroupType.MAY_WORK_ON:
|
203
|
-
# We need to check for match against configured protect patterns, if match, then the file must got to protect group instead
|
204
|
-
pattern = self.config_files.is_protected(entry, dir_config)
|
205
|
-
if pattern:
|
206
|
-
_LOG.debug("find %s - '%s' is protected by regex %s, assigning to group %s instead.", group.typ.name, entry.path, pattern, other_group.typ.name)
|
207
|
-
current_group = other_group
|
208
|
-
|
209
|
-
if entry.is_symlink():
|
210
|
-
points_to = os.readlink(entry)
|
211
|
-
abs_points_to = os.path.normpath(os.path.join(abs_dir_path, points_to))
|
212
|
-
|
213
|
-
if entry.is_dir(follow_symlinks=True):
|
214
|
-
_LOG.debug("find %s - '%s' -> '%s' is a symlink to a directory - ignoring", current_group.typ.name, entry.path, points_to)
|
215
|
-
current_group.num_directory_symlinks += 1
|
216
|
-
continue
|
217
|
-
|
218
|
-
current_group.symlinks[entry.path] = entry
|
219
|
-
current_group.symlinks_by_abs_points_to[abs_points_to].append(entry)
|
220
|
-
continue
|
221
|
-
|
222
|
-
_LOG.debug("find %s - entry name: %s", current_group.typ.name, entry.name)
|
223
|
-
current_group.add_entry_match(entry)
|
221
|
+
handle_entry(abs_dir_path, group, other_group, dir_config, entry)
|
224
222
|
|
225
223
|
checked_dirs.add(abs_dir_path)
|
226
224
|
|
227
225
|
for any_dir in sorted(chain(self.must_protect.dirs, self.may_work_on.dirs), key=lambda dd: len(Path(dd).parts)):
|
228
226
|
parent_dir = Path(any_dir)
|
229
227
|
while len(parent_dir.parts) > 1:
|
230
|
-
parent_conf = self.config_files.per_dir_configs.get(
|
228
|
+
parent_conf = self.config_files.per_dir_configs.get(parent_dir)
|
231
229
|
if parent_conf:
|
232
230
|
break
|
233
231
|
|
234
232
|
parent_dir = parent_dir.parent
|
235
233
|
else:
|
236
|
-
parent_conf =
|
234
|
+
parent_conf = None
|
237
235
|
|
238
236
|
if any_dir in self.must_protect.dirs:
|
239
237
|
find_group(any_dir, self.must_protect, self.may_work_on, parent_conf)
|