dir-content-diff 1.12.1__tar.gz → 1.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/workflows/commitlint.yml +2 -2
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/workflows/publish-sdist.yml +2 -2
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/workflows/run-tox.yml +2 -2
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.gitignore +1 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/CHANGELOG.md +17 -0
- {dir_content_diff-1.12.1/dir_content_diff.egg-info → dir_content_diff-1.13.0}/PKG-INFO +32 -4
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/README.md +30 -3
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/__init__.py +267 -77
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0/dir_content_diff.egg-info}/PKG-INFO +32 -4
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff.egg-info/SOURCES.txt +1 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff.egg-info/requires.txt +1 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/docs/Makefile +1 -1
- dir_content_diff-1.13.0/docs/source/conf.py +209 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/docs/source/index.rst +1 -1
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/pyproject.toml +1 -0
- dir_content_diff-1.13.0/tests/test_attrs_validation.py +218 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/test_base.py +81 -1
- dir_content_diff-1.12.1/docs/source/conf.py +0 -107
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.auto-changelog +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.auto-changelog-template.hbs +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.codespellignorelines +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.codespellrc +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.copier-answers.yml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.copyright.tmpl +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.coveragerc +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.coveragerc_min_versions +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.gitattributes +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/ISSUE_TEMPLATE/bug_report.yaml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/ISSUE_TEMPLATE/feature_request.yaml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/ISSUE_TEMPLATE/how_to_use.yaml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/dependabot.yml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.github/pull_request_template.md +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.gitreview +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.pre-commit-config.yaml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.pylintrc +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/.readthedocs.yml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/AUTHORS.md +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/CONTRIBUTING.md +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/LICENSE.txt +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/MANIFEST.in +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/codecov.yml +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/commitlint.config.js +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/base_comparators.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/cli/__init__.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/comparators/__init__.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/comparators/dependencies.json +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/comparators/morphio.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/comparators/pandas.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/comparators/voxcell.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/pytest_plugin.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff/util.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff.egg-info/dependency_links.txt +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff.egg-info/entry_points.txt +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/dir_content_diff.egg-info/top_level.txt +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/docs/source/.gitignore +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/docs/source/api_ref.rst +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/docs/source/changelog.rst +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/docs/source/contributing.rst +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/package.json +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/setup.cfg +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/setup.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/__init__.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/comparators/__init__.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/comparators/test_morphio.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/comparators/test_pandas.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/comparators/test_voxcell.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/conftest.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/generate_test_files.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/test_cli.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/test_missing_deps.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/test_pytest_plugin.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/test_util.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tests/test_version.py +0 -0
- {dir_content_diff-1.12.1 → dir_content_diff-1.13.0}/tox.ini +0 -0
|
@@ -11,8 +11,8 @@ jobs:
|
|
|
11
11
|
env:
|
|
12
12
|
PR_TITLE: ${{ github.event.pull_request.title }}
|
|
13
13
|
steps:
|
|
14
|
-
- uses: actions/checkout@
|
|
15
|
-
- uses: actions/setup-node@
|
|
14
|
+
- uses: actions/checkout@v5
|
|
15
|
+
- uses: actions/setup-node@v5
|
|
16
16
|
with:
|
|
17
17
|
node-version: '20'
|
|
18
18
|
- run: npm install -g --force commitlint @commitlint/cli commitlint-plugin-cleanfeet
|
|
@@ -10,9 +10,9 @@ jobs:
|
|
|
10
10
|
name: Build and publish on PyPI
|
|
11
11
|
runs-on: ubuntu-latest
|
|
12
12
|
steps:
|
|
13
|
-
- uses: actions/checkout@
|
|
13
|
+
- uses: actions/checkout@v5
|
|
14
14
|
- name: Set up Python 3.10
|
|
15
|
-
uses: actions/setup-python@
|
|
15
|
+
uses: actions/setup-python@v6
|
|
16
16
|
with:
|
|
17
17
|
python-version: "3.10"
|
|
18
18
|
- name: Build a wheel and a source tarball
|
|
@@ -24,9 +24,9 @@ jobs:
|
|
|
24
24
|
min_versions: "min_versions"
|
|
25
25
|
|
|
26
26
|
steps:
|
|
27
|
-
- uses: actions/checkout@
|
|
27
|
+
- uses: actions/checkout@v5
|
|
28
28
|
- name: Set up Python ${{ matrix.python-version }}
|
|
29
|
-
uses: actions/setup-python@
|
|
29
|
+
uses: actions/setup-python@v6
|
|
30
30
|
with:
|
|
31
31
|
python-version: ${{ matrix.python-version }}
|
|
32
32
|
- name: Get current month
|
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.13.0](https://github.com/BlueBrain/dir-content-diff/compare/1.12.1..1.13.0)
|
|
4
|
+
|
|
5
|
+
> 12 September 2025
|
|
6
|
+
|
|
7
|
+
### New Features
|
|
8
|
+
|
|
9
|
+
- Add patterns to ignore files in the reference directory (Adrien Berchet - [#86](https://github.com/BlueBrain/dir-content-diff/pull/86))
|
|
10
|
+
|
|
11
|
+
### Documentation Changes
|
|
12
|
+
|
|
13
|
+
- Fix format in Sphinx (Adrien Berchet - [#85](https://github.com/BlueBrain/dir-content-diff/pull/85))
|
|
14
|
+
- Clarify how reference and compared directories are processed (Adrien Berchet - [#83](https://github.com/BlueBrain/dir-content-diff/pull/83))
|
|
15
|
+
|
|
16
|
+
### Performance Improvements
|
|
17
|
+
|
|
18
|
+
- Build comparator registry only once before iterating over all files (Adrien Berchet - [#87](https://github.com/BlueBrain/dir-content-diff/pull/87))
|
|
19
|
+
|
|
3
20
|
## [1.12.1](https://github.com/BlueBrain/dir-content-diff/compare/1.12.0..1.12.1)
|
|
4
21
|
|
|
5
22
|
> 3 July 2025
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dir-content-diff
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.13.0
|
|
4
4
|
Summary: Simple tool to compare directory contents.
|
|
5
5
|
Author: Blue Brain Project, EPFL
|
|
6
6
|
License: Apache License 2.0
|
|
@@ -21,6 +21,7 @@ Requires-Python: >=3.9
|
|
|
21
21
|
Description-Content-Type: text/markdown
|
|
22
22
|
License-File: LICENSE.txt
|
|
23
23
|
License-File: AUTHORS.md
|
|
24
|
+
Requires-Dist: attrs>=21.3.0
|
|
24
25
|
Requires-Dist: click>=8
|
|
25
26
|
Requires-Dist: dictdiffer>=0.8
|
|
26
27
|
Requires-Dist: dicttoxml>=1.7.12
|
|
@@ -101,7 +102,7 @@ the files, so it is not able to report which values are different.
|
|
|
101
102
|
|
|
102
103
|
### Compare two directories
|
|
103
104
|
|
|
104
|
-
|
|
105
|
+
Let's compare two directories with the following structures:
|
|
105
106
|
|
|
106
107
|
```bash
|
|
107
108
|
└── reference_dir
|
|
@@ -120,6 +121,9 @@ If one wants to compare two directories with the following structures:
|
|
|
120
121
|
└── file_1.c
|
|
121
122
|
```
|
|
122
123
|
|
|
124
|
+
The `reference` directory contains all the files that should be checked in the `compared` directory,
|
|
125
|
+
which means that extraneous files in the `compared` directory are just ignored.
|
|
126
|
+
|
|
123
127
|
These two directories can be compared with the following code:
|
|
124
128
|
|
|
125
129
|
```python
|
|
@@ -128,7 +132,15 @@ import dir_content_diff
|
|
|
128
132
|
dir_content_diff.compare_trees("reference_dir", "compared_dir")
|
|
129
133
|
```
|
|
130
134
|
|
|
131
|
-
|
|
135
|
+
> [!WARNING]
|
|
136
|
+
> The order of the parameters is important: the first path is considered as the `reference`
|
|
137
|
+
> directory while the second one is the `compared` directory. Inverting the parameters may return
|
|
138
|
+
> a different result (in this example it would return that the file `sub_file_3.b` is missing).
|
|
139
|
+
|
|
140
|
+
If all the files are identical, this code will return an empty dictionary because no difference
|
|
141
|
+
was detected. As mentioned previously, this is because `dir-content-diff` is only looking for files
|
|
142
|
+
in the compared directory that are also present in the reference directory, so the file
|
|
143
|
+
`sub_file_3.b` is just ignored in this case.
|
|
132
144
|
|
|
133
145
|
If ``reference_dir/file_1.c`` is the following JSON-like file:
|
|
134
146
|
|
|
@@ -228,7 +240,7 @@ specific_args = {
|
|
|
228
240
|
}
|
|
229
241
|
```
|
|
230
242
|
|
|
231
|
-
|
|
243
|
+
Another possibility is to use regular expressions to associate specific arguments to
|
|
232
244
|
a set of files:
|
|
233
245
|
|
|
234
246
|
```python
|
|
@@ -240,6 +252,22 @@ specific_args = {
|
|
|
240
252
|
}
|
|
241
253
|
```
|
|
242
254
|
|
|
255
|
+
And last but not least, it's possible to filter files from the reference directory (for example
|
|
256
|
+
because the reference directory contains temporary files that should not be compared). For
|
|
257
|
+
example, the following code will ignore all files whose name does not start with `file_` and does
|
|
258
|
+
not ends with `_tmp.yaml`:
|
|
259
|
+
|
|
260
|
+
```python
|
|
261
|
+
import dir_content_diff
|
|
262
|
+
|
|
263
|
+
dir_content_diff.compare_trees(
|
|
264
|
+
"reference_dir",
|
|
265
|
+
"compared_dir",
|
|
266
|
+
include_patterns=[r"file_.*"],
|
|
267
|
+
exclude_patterns=[r".*_tmp\.yaml"],
|
|
268
|
+
)
|
|
269
|
+
```
|
|
270
|
+
|
|
243
271
|
|
|
244
272
|
### Export formatted data
|
|
245
273
|
|
|
@@ -36,7 +36,7 @@ the files, so it is not able to report which values are different.
|
|
|
36
36
|
|
|
37
37
|
### Compare two directories
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
Let's compare two directories with the following structures:
|
|
40
40
|
|
|
41
41
|
```bash
|
|
42
42
|
└── reference_dir
|
|
@@ -55,6 +55,9 @@ If one wants to compare two directories with the following structures:
|
|
|
55
55
|
└── file_1.c
|
|
56
56
|
```
|
|
57
57
|
|
|
58
|
+
The `reference` directory contains all the files that should be checked in the `compared` directory,
|
|
59
|
+
which means that extraneous files in the `compared` directory are just ignored.
|
|
60
|
+
|
|
58
61
|
These two directories can be compared with the following code:
|
|
59
62
|
|
|
60
63
|
```python
|
|
@@ -63,7 +66,15 @@ import dir_content_diff
|
|
|
63
66
|
dir_content_diff.compare_trees("reference_dir", "compared_dir")
|
|
64
67
|
```
|
|
65
68
|
|
|
66
|
-
|
|
69
|
+
> [!WARNING]
|
|
70
|
+
> The order of the parameters is important: the first path is considered as the `reference`
|
|
71
|
+
> directory while the second one is the `compared` directory. Inverting the parameters may return
|
|
72
|
+
> a different result (in this example it would return that the file `sub_file_3.b` is missing).
|
|
73
|
+
|
|
74
|
+
If all the files are identical, this code will return an empty dictionary because no difference
|
|
75
|
+
was detected. As mentioned previously, this is because `dir-content-diff` is only looking for files
|
|
76
|
+
in the compared directory that are also present in the reference directory, so the file
|
|
77
|
+
`sub_file_3.b` is just ignored in this case.
|
|
67
78
|
|
|
68
79
|
If ``reference_dir/file_1.c`` is the following JSON-like file:
|
|
69
80
|
|
|
@@ -163,7 +174,7 @@ specific_args = {
|
|
|
163
174
|
}
|
|
164
175
|
```
|
|
165
176
|
|
|
166
|
-
|
|
177
|
+
Another possibility is to use regular expressions to associate specific arguments to
|
|
167
178
|
a set of files:
|
|
168
179
|
|
|
169
180
|
```python
|
|
@@ -175,6 +186,22 @@ specific_args = {
|
|
|
175
186
|
}
|
|
176
187
|
```
|
|
177
188
|
|
|
189
|
+
And last but not least, it's possible to filter files from the reference directory (for example
|
|
190
|
+
because the reference directory contains temporary files that should not be compared). For
|
|
191
|
+
example, the following code will ignore all files whose name does not start with `file_` and does
|
|
192
|
+
not ends with `_tmp.yaml`:
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
import dir_content_diff
|
|
196
|
+
|
|
197
|
+
dir_content_diff.compare_trees(
|
|
198
|
+
"reference_dir",
|
|
199
|
+
"compared_dir",
|
|
200
|
+
include_patterns=[r"file_.*"],
|
|
201
|
+
exclude_patterns=[r".*_tmp\.yaml"],
|
|
202
|
+
)
|
|
203
|
+
```
|
|
204
|
+
|
|
178
205
|
|
|
179
206
|
### Export formatted data
|
|
180
207
|
|
|
@@ -15,7 +15,17 @@ Simple tool to compare directory contents.
|
|
|
15
15
|
import copy
|
|
16
16
|
import importlib.metadata
|
|
17
17
|
import re
|
|
18
|
+
from collections.abc import Callable
|
|
18
19
|
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
from typing import Dict
|
|
22
|
+
from typing import Iterable
|
|
23
|
+
from typing import Optional
|
|
24
|
+
from typing import Pattern
|
|
25
|
+
from typing import Tuple
|
|
26
|
+
from typing import Union
|
|
27
|
+
|
|
28
|
+
import attrs
|
|
19
29
|
|
|
20
30
|
from dir_content_diff.base_comparators import BaseComparator
|
|
21
31
|
from dir_content_diff.base_comparators import DefaultComparator
|
|
@@ -28,6 +38,9 @@ from dir_content_diff.util import LOGGER
|
|
|
28
38
|
from dir_content_diff.util import diff_msg_formatter
|
|
29
39
|
from dir_content_diff.util import format_ext
|
|
30
40
|
|
|
41
|
+
# Type alias for comparators
|
|
42
|
+
ComparatorType = Union[BaseComparator, Callable]
|
|
43
|
+
|
|
31
44
|
__version__ = importlib.metadata.version("dir-content-diff")
|
|
32
45
|
|
|
33
46
|
_DEFAULT_COMPARATORS = {
|
|
@@ -61,13 +74,15 @@ def get_comparators():
|
|
|
61
74
|
return copy.deepcopy(_COMPARATORS)
|
|
62
75
|
|
|
63
76
|
|
|
64
|
-
def register_comparator(
|
|
77
|
+
def register_comparator(
|
|
78
|
+
ext: str, comparator: ComparatorType, force: bool = False
|
|
79
|
+
) -> None:
|
|
65
80
|
"""Add a comparator to the registry.
|
|
66
81
|
|
|
67
82
|
Args:
|
|
68
|
-
ext
|
|
69
|
-
comparator
|
|
70
|
-
force
|
|
83
|
+
ext: The extension to register.
|
|
84
|
+
comparator: The comparator that should be associated with the given extension.
|
|
85
|
+
force: If set to ``True``, no exception is raised if the given ``ext`` is already
|
|
71
86
|
registered and the comparator is replaced.
|
|
72
87
|
|
|
73
88
|
.. note::
|
|
@@ -98,12 +113,12 @@ def register_comparator(ext, comparator, force=False):
|
|
|
98
113
|
_COMPARATORS[ext] = comparator
|
|
99
114
|
|
|
100
115
|
|
|
101
|
-
def unregister_comparator(ext, quiet=False):
|
|
116
|
+
def unregister_comparator(ext: str, quiet: bool = False):
|
|
102
117
|
"""Remove a comparator from the registry.
|
|
103
118
|
|
|
104
119
|
Args:
|
|
105
|
-
ext
|
|
106
|
-
quiet
|
|
120
|
+
ext: The extension to unregister.
|
|
121
|
+
quiet: If set to ``True``, no exception is raised if the given ``ext`` is not
|
|
107
122
|
registered.
|
|
108
123
|
|
|
109
124
|
Returns:
|
|
@@ -115,23 +130,212 @@ def unregister_comparator(ext, quiet=False):
|
|
|
115
130
|
return _COMPARATORS.pop(ext, None)
|
|
116
131
|
|
|
117
132
|
|
|
133
|
+
def _convert_iterable_to_tuple(
|
|
134
|
+
x: Optional[Iterable[str]],
|
|
135
|
+
) -> Optional[Tuple[str, ...]]:
|
|
136
|
+
"""Convert an iterable to a tuple, or return None."""
|
|
137
|
+
if x is None:
|
|
138
|
+
return None
|
|
139
|
+
return tuple(x)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _validate_specific_args(instance, attribute, value): # pylint: disable=unused-argument
|
|
143
|
+
"""Validate specific_args structure."""
|
|
144
|
+
for file_path, args in value.items():
|
|
145
|
+
if not isinstance(args, dict):
|
|
146
|
+
raise ValueError(f"specific_args['{file_path}'] must be a dictionary")
|
|
147
|
+
# Note: regex patterns in specific_args will be validated during compilation
|
|
148
|
+
# in __attrs_post_init__, so no need to validate them here
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _validate_export_formatted_files(instance, attribute, value): # pylint: disable=unused-argument
|
|
152
|
+
"""Validate export_formatted_files is either bool or non-empty string."""
|
|
153
|
+
if isinstance(value, str) and len(value.strip()) == 0:
|
|
154
|
+
raise ValueError(
|
|
155
|
+
"export_formatted_files must be a non-empty string when provided as string"
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _validate_comparators(instance, attribute, value): # pylint: disable=unused-argument
|
|
160
|
+
"""Validate comparators are either BaseComparator instances or callable."""
|
|
161
|
+
for ext, comparator in value.items():
|
|
162
|
+
if not (isinstance(comparator, BaseComparator) or callable(comparator)):
|
|
163
|
+
raise ValueError(
|
|
164
|
+
f"Comparator for extension '{ext}' must be a BaseComparator instance "
|
|
165
|
+
"or callable"
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@attrs.frozen
|
|
170
|
+
class ComparisonConfig:
|
|
171
|
+
"""Configuration class to store comparison settings.
|
|
172
|
+
|
|
173
|
+
Attributes:
|
|
174
|
+
include_patterns: A list of regular expression patterns. If the relative path of a
|
|
175
|
+
file does not match any of these patterns, it is ignored during the comparison. Note
|
|
176
|
+
that this means that any specific arguments for that file will also be ignored.
|
|
177
|
+
exclude_patterns: A list of regular expression patterns. If the relative path of a
|
|
178
|
+
file matches any of these patterns, it is ignored during the comparison. Note that
|
|
179
|
+
this means that any specific arguments for that file will also be ignored.
|
|
180
|
+
comparators: A ``dict`` to override the registered comparators.
|
|
181
|
+
specific_args: A ``dict`` with the args/kwargs that should be given to the
|
|
182
|
+
comparator for a given file. This ``dict`` should be like the following:
|
|
183
|
+
|
|
184
|
+
.. code-block:: Python
|
|
185
|
+
|
|
186
|
+
{
|
|
187
|
+
<relative_file_path>: {
|
|
188
|
+
comparator: ComparatorInstance,
|
|
189
|
+
args: [arg1, arg2, ...],
|
|
190
|
+
kwargs: {
|
|
191
|
+
kwarg_name_1: kwarg_value_1,
|
|
192
|
+
kwarg_name_2: kwarg_value_2,
|
|
193
|
+
}
|
|
194
|
+
},
|
|
195
|
+
<another_file_path>: {...},
|
|
196
|
+
<a name for this category>: {
|
|
197
|
+
"patterns": ["regex1", "regex2", ...],
|
|
198
|
+
... (other arguments)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
If the "patterns" entry is present, then the name is not considered and is only used as
|
|
203
|
+
a helper for the user. When a "patterns" entry is detected, the other arguments are
|
|
204
|
+
applied to all files whose relative name matches one of the given regular expression
|
|
205
|
+
patterns. If a file could match multiple patterns of different groups, only the first
|
|
206
|
+
one is considered.
|
|
207
|
+
|
|
208
|
+
Note that all entries in this ``dict`` are optional.
|
|
209
|
+
return_raw_diffs: If set to ``True``, only the raw differences are returned instead
|
|
210
|
+
of a formatted report.
|
|
211
|
+
export_formatted_files: If set to ``True`` or a not empty string, create a
|
|
212
|
+
new directory with formatted compared data files. If a string is passed, this string is
|
|
213
|
+
used as suffix for the new directory. If `True` is passed, the suffix is
|
|
214
|
+
``_FORMATTED``.
|
|
215
|
+
"""
|
|
216
|
+
|
|
217
|
+
include_patterns: Optional[Iterable[str]] = attrs.field(
|
|
218
|
+
default=None, converter=_convert_iterable_to_tuple
|
|
219
|
+
)
|
|
220
|
+
exclude_patterns: Optional[Iterable[str]] = attrs.field(
|
|
221
|
+
default=None, converter=_convert_iterable_to_tuple
|
|
222
|
+
)
|
|
223
|
+
comparators: Optional[Dict[Optional[str], ComparatorType]] = attrs.field(
|
|
224
|
+
default=None, validator=attrs.validators.optional(_validate_comparators)
|
|
225
|
+
)
|
|
226
|
+
specific_args: Optional[Dict[str, Dict[str, Any]]] = attrs.field(
|
|
227
|
+
default=None, validator=attrs.validators.optional(_validate_specific_args)
|
|
228
|
+
)
|
|
229
|
+
return_raw_diffs: bool = attrs.field(default=False)
|
|
230
|
+
export_formatted_files: Union[bool, str] = attrs.field(
|
|
231
|
+
default=False, validator=_validate_export_formatted_files
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
# Compiled patterns - computed once, no caching complexity needed
|
|
235
|
+
compiled_include_patterns: Tuple[Pattern[str], ...] = attrs.field(init=False)
|
|
236
|
+
compiled_exclude_patterns: Tuple[Pattern[str], ...] = attrs.field(init=False)
|
|
237
|
+
pattern_specific_args: Dict[Pattern[str], Dict[str, Any]] = attrs.field(
|
|
238
|
+
init=False, repr=False
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
def __attrs_post_init__(self):
|
|
242
|
+
"""Initialize computed fields after attrs initialization."""
|
|
243
|
+
# Validate and compile patterns - with frozen, we compile once and store directly
|
|
244
|
+
try:
|
|
245
|
+
compiled_include = self._compile_patterns(self.include_patterns)
|
|
246
|
+
object.__setattr__(self, "compiled_include_patterns", compiled_include)
|
|
247
|
+
except ValueError as e:
|
|
248
|
+
raise ValueError(f"Error in include_patterns: {e}") from e
|
|
249
|
+
|
|
250
|
+
try:
|
|
251
|
+
compiled_exclude = self._compile_patterns(self.exclude_patterns)
|
|
252
|
+
object.__setattr__(self, "compiled_exclude_patterns", compiled_exclude)
|
|
253
|
+
except ValueError as e:
|
|
254
|
+
raise ValueError(f"Error in exclude_patterns: {e}") from e
|
|
255
|
+
|
|
256
|
+
# Setup specific args and pattern specific args
|
|
257
|
+
if self.specific_args is None:
|
|
258
|
+
# Use object.__setattr__ to modify the field even if it's frozen
|
|
259
|
+
object.__setattr__(self, "specific_args", {})
|
|
260
|
+
|
|
261
|
+
# Setup pattern specific args
|
|
262
|
+
pattern_specific_args = {}
|
|
263
|
+
if self.specific_args: # Check if it's not None
|
|
264
|
+
for file_path, v in self.specific_args.items():
|
|
265
|
+
if "patterns" in v:
|
|
266
|
+
patterns = v.pop("patterns", [])
|
|
267
|
+
for pattern in patterns:
|
|
268
|
+
try:
|
|
269
|
+
compiled_pattern = self._compile_pattern(pattern)
|
|
270
|
+
pattern_specific_args[compiled_pattern] = v
|
|
271
|
+
except ValueError as e:
|
|
272
|
+
raise ValueError(
|
|
273
|
+
f"Error in specific_args['{file_path}']['patterns']: {e}"
|
|
274
|
+
) from e
|
|
275
|
+
|
|
276
|
+
object.__setattr__(self, "pattern_specific_args", pattern_specific_args)
|
|
277
|
+
|
|
278
|
+
# Setup comparators
|
|
279
|
+
if self.comparators is None:
|
|
280
|
+
object.__setattr__(self, "comparators", get_comparators())
|
|
281
|
+
|
|
282
|
+
def _compile_pattern(self, pattern: str) -> Pattern[str]:
|
|
283
|
+
"""Compile a regex pattern."""
|
|
284
|
+
try:
|
|
285
|
+
return re.compile(pattern)
|
|
286
|
+
except re.error as e:
|
|
287
|
+
raise ValueError(f"Invalid regex pattern: '{pattern}'") from e
|
|
288
|
+
|
|
289
|
+
def _compile_patterns(
|
|
290
|
+
self, patterns: Optional[Iterable[str]]
|
|
291
|
+
) -> Tuple[Pattern[str], ...]:
|
|
292
|
+
"""Compile regex patterns from any iterable to tuple."""
|
|
293
|
+
if patterns is None:
|
|
294
|
+
return ()
|
|
295
|
+
return tuple(self._compile_pattern(pattern) for pattern in patterns)
|
|
296
|
+
|
|
297
|
+
# Note: compiled_include_patterns, compiled_exclude_patterns, and pattern_specific_args
|
|
298
|
+
# are now direct attributes set in __attrs_post_init__, no properties needed!
|
|
299
|
+
|
|
300
|
+
def should_ignore_file(self, relative_path: str) -> bool:
|
|
301
|
+
"""Check if a file should be ignored."""
|
|
302
|
+
# Check inclusion patterns first
|
|
303
|
+
if self.compiled_include_patterns:
|
|
304
|
+
included = any(
|
|
305
|
+
pattern.match(relative_path)
|
|
306
|
+
for pattern in self.compiled_include_patterns
|
|
307
|
+
)
|
|
308
|
+
if not included:
|
|
309
|
+
return True
|
|
310
|
+
|
|
311
|
+
# Check exclusion patterns
|
|
312
|
+
return any(
|
|
313
|
+
pattern.match(relative_path) for pattern in self.compiled_exclude_patterns
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
|
|
118
317
|
def compare_files(
|
|
119
|
-
ref_file
|
|
120
|
-
|
|
318
|
+
ref_file: str,
|
|
319
|
+
comp_file: str,
|
|
320
|
+
comparator: ComparatorType,
|
|
321
|
+
*args,
|
|
322
|
+
return_raw_diffs: bool = False,
|
|
323
|
+
**kwargs,
|
|
324
|
+
) -> Union[bool, str]:
|
|
121
325
|
"""Compare 2 files and return the difference.
|
|
122
326
|
|
|
123
327
|
Args:
|
|
124
|
-
ref_file
|
|
125
|
-
comp_file
|
|
126
|
-
comparator
|
|
328
|
+
ref_file: Path to the reference file.
|
|
329
|
+
comp_file: Path to the compared file.
|
|
330
|
+
comparator: The comparator to use (see in :func:`register_comparator` for the
|
|
127
331
|
comparator signature).
|
|
128
|
-
return_raw_diffs
|
|
332
|
+
return_raw_diffs: If set to ``True``, only the raw differences are returned instead
|
|
129
333
|
of a formatted report.
|
|
130
334
|
*args: passed to the comparator.
|
|
131
335
|
**kwargs: passed to the comparator.
|
|
132
336
|
|
|
133
337
|
Returns:
|
|
134
|
-
|
|
338
|
+
``False`` if the files are equal or a string with a message explaining the
|
|
135
339
|
differences if they are different.
|
|
136
340
|
"""
|
|
137
341
|
# Get the compared file
|
|
@@ -172,15 +376,20 @@ def compare_files(
|
|
|
172
376
|
)
|
|
173
377
|
|
|
174
378
|
|
|
175
|
-
def export_formatted_file(
|
|
379
|
+
def export_formatted_file(
|
|
380
|
+
file: str,
|
|
381
|
+
formatted_file: str,
|
|
382
|
+
comparator: ComparatorType,
|
|
383
|
+
**kwargs,
|
|
384
|
+
) -> None:
|
|
176
385
|
"""Format a data file and export it.
|
|
177
386
|
|
|
178
387
|
.. note:: A new file is created only if the corresponding comparator has saving capability.
|
|
179
388
|
|
|
180
389
|
Args:
|
|
181
|
-
file
|
|
182
|
-
formatted_file
|
|
183
|
-
comparator
|
|
390
|
+
file: Path to the compared file.
|
|
391
|
+
formatted_file: Path to the formatted file.
|
|
392
|
+
comparator: The comparator to use (see in :func:`register_comparator` for the
|
|
184
393
|
comparator signature).
|
|
185
394
|
**kwargs: Can contain the following dictionaries: 'load_kwargs', 'format_data_kwargs' and
|
|
186
395
|
'save_kwargs'.
|
|
@@ -252,13 +461,24 @@ def pick_comparator(comparator=None, suffix=None, comparators=None):
|
|
|
252
461
|
return _COMPARATORS.get(None)
|
|
253
462
|
|
|
254
463
|
|
|
464
|
+
def _check_config(config=None, **kwargs):
|
|
465
|
+
if config is not None:
|
|
466
|
+
if kwargs:
|
|
467
|
+
# Override config attributes with kwargs
|
|
468
|
+
config = attrs.evolve(config, **kwargs)
|
|
469
|
+
else:
|
|
470
|
+
config = ComparisonConfig(
|
|
471
|
+
**kwargs,
|
|
472
|
+
)
|
|
473
|
+
return config
|
|
474
|
+
|
|
475
|
+
|
|
255
476
|
def compare_trees(
|
|
256
|
-
ref_path,
|
|
257
|
-
comp_path,
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
export_formatted_files=False,
|
|
477
|
+
ref_path: Union[str, Path],
|
|
478
|
+
comp_path: Union[str, Path],
|
|
479
|
+
*,
|
|
480
|
+
config: ComparisonConfig = None,
|
|
481
|
+
**kwargs,
|
|
262
482
|
):
|
|
263
483
|
"""Compare all files from 2 different directory trees and return the differences.
|
|
264
484
|
|
|
@@ -269,70 +489,36 @@ def compare_trees(
|
|
|
269
489
|
ignored.
|
|
270
490
|
|
|
271
491
|
Args:
|
|
272
|
-
ref_path
|
|
273
|
-
comp_path
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
comparator for a given file. This ``dict`` should be like the following:
|
|
492
|
+
ref_path: Path to the reference directory.
|
|
493
|
+
comp_path: Path to the directory that must be compared against the reference.
|
|
494
|
+
config (ComparisonConfig): A config object. If given, all other configuration parameters
|
|
495
|
+
should be set to default values.
|
|
277
496
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
<relative_file_path>: {
|
|
282
|
-
comparator: ComparatorInstance,
|
|
283
|
-
args: [arg1, arg2, ...],
|
|
284
|
-
kwargs: {
|
|
285
|
-
kwarg_name_1: kwarg_value_1,
|
|
286
|
-
kwarg_name_2: kwarg_value_2,
|
|
287
|
-
}
|
|
288
|
-
},
|
|
289
|
-
<another_file_path>: {...},
|
|
290
|
-
<a name for this category>: {
|
|
291
|
-
"patterns": ["regex1", "regex2", ...],
|
|
292
|
-
... (other arguments)
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
If the "patterns" entry is present, then the name is not considered and is only used as
|
|
297
|
-
a helper for the user. When a "patterns" entry is detected, the other arguments are
|
|
298
|
-
applied to all files whose relative name matches one of the given regular expression
|
|
299
|
-
patterns. If a file could match multiple patterns of different groups, only the first
|
|
300
|
-
one is considered.
|
|
301
|
-
|
|
302
|
-
Note that all entries in this ``dict`` are optional.
|
|
303
|
-
return_raw_diffs (bool): If set to ``True``, only the raw differences are returned instead
|
|
304
|
-
of a formatted report.
|
|
305
|
-
export_formatted_files (bool or str): If set to ``True`` or a not empty string, create a
|
|
306
|
-
new directory with formatted compared data files. If a string is passed, this string is
|
|
307
|
-
used as suffix for the new directory. If `True` is passed, the suffix is
|
|
308
|
-
``_FORMATTED``.
|
|
497
|
+
Keyword Args:
|
|
498
|
+
**kwargs (dict): Additional keyword arguments are used to build a ComparisonConfig object
|
|
499
|
+
and will override the values of the given `config` argument.
|
|
309
500
|
|
|
310
501
|
Returns:
|
|
311
502
|
dict: A ``dict`` in which the keys are the relative file paths and the values are the
|
|
312
503
|
difference messages. If the directories are considered as equal, an empty ``dict`` is
|
|
313
504
|
returned.
|
|
314
505
|
"""
|
|
506
|
+
config = _check_config(config, **kwargs)
|
|
507
|
+
|
|
315
508
|
ref_path = Path(ref_path)
|
|
316
509
|
comp_path = Path(comp_path)
|
|
317
510
|
formatted_data_path = comp_path.with_name(
|
|
318
511
|
comp_path.name
|
|
319
512
|
+ (
|
|
320
|
-
export_formatted_files
|
|
321
|
-
if (
|
|
513
|
+
config.export_formatted_files
|
|
514
|
+
if (
|
|
515
|
+
config.export_formatted_files is not True
|
|
516
|
+
and config.export_formatted_files
|
|
517
|
+
)
|
|
322
518
|
else _DEFAULT_EXPORT_SUFFIX
|
|
323
519
|
)
|
|
324
520
|
)
|
|
325
521
|
|
|
326
|
-
if specific_args is None:
|
|
327
|
-
specific_args = {}
|
|
328
|
-
else:
|
|
329
|
-
specific_args = copy.deepcopy(specific_args)
|
|
330
|
-
|
|
331
|
-
pattern_specific_args = {}
|
|
332
|
-
for v in specific_args.values():
|
|
333
|
-
for pattern in v.pop("patterns", []):
|
|
334
|
-
pattern_specific_args[re.compile(pattern)] = v
|
|
335
|
-
|
|
336
522
|
# Loop over all files and call the correct comparator
|
|
337
523
|
different_files = {}
|
|
338
524
|
for ref_file in ref_path.glob("**/*"):
|
|
@@ -342,10 +528,14 @@ def compare_trees(
|
|
|
342
528
|
relative_path = ref_file.relative_to(ref_path).as_posix()
|
|
343
529
|
comp_file = comp_path / relative_path
|
|
344
530
|
|
|
531
|
+
if config.should_ignore_file(relative_path):
|
|
532
|
+
LOGGER.debug("Ignore file: %s", relative_path)
|
|
533
|
+
continue
|
|
534
|
+
|
|
345
535
|
if comp_file.exists():
|
|
346
|
-
specific_file_args = specific_args.get(relative_path, None)
|
|
536
|
+
specific_file_args = (config.specific_args or {}).get(relative_path, None)
|
|
347
537
|
if specific_file_args is None:
|
|
348
|
-
for pattern, pattern_args in pattern_specific_args.items():
|
|
538
|
+
for pattern, pattern_args in config.pattern_specific_args.items():
|
|
349
539
|
if pattern.match(relative_path):
|
|
350
540
|
specific_file_args = copy.deepcopy(pattern_args)
|
|
351
541
|
break
|
|
@@ -354,7 +544,7 @@ def compare_trees(
|
|
|
354
544
|
comparator = pick_comparator(
|
|
355
545
|
comparator=specific_file_args.pop("comparator", None),
|
|
356
546
|
suffix=ref_file.suffix,
|
|
357
|
-
comparators=comparators,
|
|
547
|
+
comparators=config.comparators,
|
|
358
548
|
)
|
|
359
549
|
comparator_args = specific_file_args.pop("args", [])
|
|
360
550
|
res = compare_files(
|
|
@@ -362,12 +552,12 @@ def compare_trees(
|
|
|
362
552
|
comp_file,
|
|
363
553
|
comparator,
|
|
364
554
|
*comparator_args,
|
|
365
|
-
return_raw_diffs=return_raw_diffs,
|
|
555
|
+
return_raw_diffs=config.return_raw_diffs,
|
|
366
556
|
**specific_file_args,
|
|
367
557
|
)
|
|
368
558
|
if res is not False:
|
|
369
559
|
different_files[relative_path] = res
|
|
370
|
-
if export_formatted_files is not False:
|
|
560
|
+
if config.export_formatted_files is not False:
|
|
371
561
|
export_formatted_file(
|
|
372
562
|
comp_file,
|
|
373
563
|
formatted_data_path / relative_path,
|