dir-content-diff 1.6.0__tar.gz → 1.8.0__tar.gz

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/.copier-answers.yml

@@ -1,6 +1,6 @@
 # Changes here will be overwritten by Copier
 
-_commit: 0.1.53
+_commit: 0.1.54
 _src_path: git@bbpgitlab.epfl.ch:neuromath/python-template.git
 author_email: ''
 author_name: Blue Brain Project, EPFL

dir-content-diff-1.8.0/.coveragerc_py38

@@ -0,0 +1,11 @@
+[paths]
+source_paths =
+	dir_content_diff
+	*/site-packages/dir_content_diff
+	*/dir_content_diff/dir_content_diff
+	*/dir-content-diff/dir_content_diff
+
+[report]
+exclude_also =
+    return pd\.read_feather\(path, \*\*kwargs\)
+    data\.to_feather\(path, \*\*kwargs\)

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/.github/workflows/commitlint.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: '16'
+          node-version: '20'
       - run: npm install -g --force commitlint @commitlint/cli commitlint-plugin-cleanfeet
       - run: npm install conventional-changelog-conventionalcommits
       - run: touch .git/COMMIT_EDITMSG

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/.github/workflows/run-tox.yml

@@ -64,12 +64,13 @@ jobs:
       with:
         report_paths: 'reports/pytest-*.xml'
     - name: Upload to codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         fail_ci_if_error: false
         directory: ./reports
         flags: pytest
         name: "dir-content-diff-py${{ matrix.python-version }}-${{ matrix.min_versions }}"
+        token: ${{ secrets.CODECOV_TOKEN }}
     - name: Upload test artifacts
       uses: actions/upload-artifact@v4
       if: always()

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/.pylintrc

@@ -2,6 +2,7 @@
 disable=
 	arguments-differ,
 	invalid-name,
+	too-many-lines,
 
 [FORMAT]
 # Regexp for a line that is allowed to be longer than the limit.

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## [1.8.0](https://github.com/BlueBrain/dir-content-diff/compare/1.7.0..1.8.0)
+
+> 8 April 2024
+
+### New Features
+
+- Add new comparators for Feather, Parquet and Stata files (Adrien Berchet - [#56](https://github.com/BlueBrain/dir-content-diff/pull/56))
+
+## [1.7.0](https://github.com/BlueBrain/dir-content-diff/compare/1.6.0..1.7.0)
+
+> 5 February 2024
+
+### New Features
+
+- Add 'replace_pattern' entry for DictComparator and several related save capabilities (Adrien Berchet - [#53](https://github.com/BlueBrain/dir-content-diff/pull/53))
+- Add 'patterns' entry in specific_args (Adrien Berchet - [#52](https://github.com/BlueBrain/dir-content-diff/pull/52))
+
 ## [1.6.0](https://github.com/BlueBrain/dir-content-diff/compare/1.5.0..1.6.0)
 
 > 29 January 2024

{dir-content-diff-1.6.0/dir_content_diff.egg-info → dir-content-diff-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dir-content-diff
-Version: 1.6.0
+Version: 1.8.0
 Summary: Simple tool to compare directory contents.
 Home-page: https://dir-content-diff.readthedocs.io
 Author: Blue Brain Project, EPFL
@@ -22,25 +22,30 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 License-File: AUTHORS.md
 Requires-Dist: dictdiffer>=0.8
+Requires-Dist: dicttoxml>=1.7.12
 Requires-Dist: diff_pdf_visually>=1.7
+Requires-Dist: jsonpath-ng>=1.5
 Requires-Dist: PyYaml>=6
 Provides-Extra: pandas
 Requires-Dist: pandas>=1.4; extra == "pandas"
+Requires-Dist: pyarrow>=11; extra == "pandas"
 Requires-Dist: tables>=3.7; extra == "pandas"
 Provides-Extra: docs
 Requires-Dist: m2r2; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-bluebrain-theme; extra == "docs"
 Requires-Dist: pandas>=1.4; extra == "docs"
+Requires-Dist: pyarrow>=11; extra == "docs"
 Requires-Dist: tables>=3.7; extra == "docs"
 Provides-Extra: test
-Requires-Dist: coverage>=6; extra == "test"
+Requires-Dist: coverage>=7.2; extra == "test"
 Requires-Dist: dicttoxml>=1.7.16; extra == "test"
 Requires-Dist: matplotlib>=3; extra == "test"
 Requires-Dist: rst2pdf>=0.99; extra == "test"
 Requires-Dist: pytest>=6.2; extra == "test"
 Requires-Dist: pytest-html<4,>=2; extra == "test"
 Requires-Dist: pandas>=1.4; extra == "test"
+Requires-Dist: pyarrow>=11; extra == "test"
 Requires-Dist: tables>=3.7; extra == "test"
 
 [![Version](https://img.shields.io/pypi/v/dir-content-diff)](https://github.com/BlueBrain/dir-content-diff/releases)
@@ -208,6 +213,18 @@ specific_args = {
 }
 ```
 
+And last but not least, it's possible to use regular expressions to associate specific arguments to
+a set of files:
+
+```python
+specific_args = {
+    "all files with *.a of *.b extensions": {
+        "patterns": [r".*\.[a,b]$"],
+        "comparator": dir_content_diff.BaseComparator(),
+    }
+}
+```
+
 
 ### Export formatted data
 

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/README.md

@@ -163,6 +163,18 @@ specific_args = {
 }
 ```
 
+And last but not least, it's possible to use regular expressions to associate specific arguments to
+a set of files:
+
+```python
+specific_args = {
+    "all files with *.a of *.b extensions": {
+        "patterns": [r".*\.[a,b]$"],
+        "comparator": dir_content_diff.BaseComparator(),
+    }
+}
+```
+
 
 ### Export formatted data
 

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/dir_content_diff/__init__.py

@@ -5,6 +5,7 @@ Simple tool to compare directory contents.
 import copy
 import importlib.metadata
 import logging
+import re
 from pathlib import Path
 
 from dir_content_diff.base_comparators import DefaultComparator
@@ -240,9 +241,19 @@ def compare_trees(
                             kwarg_name_2: kwarg_value_2,
                         }
                     },
-                    <another_file_path>: {...}
+                    <another_file_path>: {...},
+                    <a name for this category>: {
+                        "patterns": ["regex1", "regex2", ...],
+                        ... (other arguments)
+                    }
                 }
 
+            If the "patterns" entry is present, then the name is not considered and is only used as
+            a helper for the user. When a "patterns" entry is detected, the other arguments are
+            applied to all files whose relative name matches one of the given regular expression
+            patterns. If a file could match multiple patterns of different groups, only the first
+            one is considered.
+
             Note that all entries in this ``dict`` are optional.
         return_raw_diffs (bool): If set to ``True``, only the raw differences are returned instead
             of a formatted report.
@@ -275,6 +286,11 @@ def compare_trees(
     else:
         specific_args = copy.deepcopy(specific_args)
 
+    pattern_specific_args = {}
+    for v in specific_args.values():
+        for pattern in v.pop("patterns", []):
+            pattern_specific_args[re.compile(pattern)] = v
+
     # Loop over all files and call the correct comparator
     different_files = {}
     for ref_file in ref_path.glob("**/*"):
@@ -285,7 +301,14 @@ def compare_trees(
         comp_file = comp_path / relative_path
 
         if comp_file.exists():
-            specific_file_args = specific_args.get(relative_path, {})
+            specific_file_args = specific_args.get(relative_path, None)
+            if specific_file_args is None:
+                for pattern, pattern_args in pattern_specific_args.items():
+                    if pattern.match(relative_path):
+                        specific_file_args = copy.deepcopy(pattern_args)
+                        break
+            if specific_file_args is None:
+                specific_file_args = {}
             comparator = specific_file_args.pop(
                 "comparator",
                 comparators.get(

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/dir_content_diff/base_comparators.py

@@ -2,12 +2,15 @@
 import configparser
 import filecmp
 import json
+import re
 from abc import ABC
 from abc import abstractmethod
 from xml.etree import ElementTree
 
 import dictdiffer
+import jsonpath_ng
 import yaml
+from dicttoxml import dicttoxml
 from diff_pdf_visually import pdf_similar
 
 from dir_content_diff.util import diff_msg_formatter
@@ -282,6 +285,14 @@ class DictComparator(BaseComparator):
         "add": "Added the value(s) '{value}' in the '{key}' key.",
         "change": "Changed the value of '{key}' from {value[0]} to {value[1]}.",
         "remove": "Removed the value(s) '{value}' from '{key}' key.",
+        "missing_ref_entry": (
+            "The path '{key}' is missing in the reference dictionary, please fix the "
+            "'replace_pattern' argument."
+        ),
+        "missing_comp_entry": (
+            "The path '{key}' is missing in the compared dictionary, please fix the "
+            "'replace_pattern' argument."
+        ),
     }
 
     def __init__(self, *args, **kwargs):
@@ -318,6 +329,43 @@ class DictComparator(BaseComparator):
                 value[num] = str(i)
         return value
 
+    def format_data(self, data, ref=None, replace_pattern=None, **kwargs):
+        """Format the loaded data."""
+        # pylint: disable=too-many-nested-blocks
+        self.current_state["format_errors"] = errors = []
+
+        if replace_pattern is not None:
+            for pat, paths in replace_pattern.items():
+                pattern = pat[0]
+                new_value = pat[1]
+                count = pat[2] if len(pat) > 2 else 0
+                flags = pat[3] if len(pat) > 3 else 0
+                for raw_path in paths:
+                    path = jsonpath_ng.parse(raw_path)
+                    if ref is not None and len(path.find(ref)) == 0:
+                        errors.append(
+                            (
+                                "missing_ref_entry",
+                                raw_path,
+                                None,
+                            )
+                        )
+                    elif len(path.find(data)) == 0:
+                        errors.append(
+                            (
+                                "missing_comp_entry",
+                                raw_path,
+                                None,
+                            )
+                        )
+                    else:
+                        for i in path.find(data):
+                            if isinstance(i.value, str):
+                                i.full_path.update(
+                                    data, re.sub(pattern, new_value, i.value, count, flags)
+                                )
+        return data
+
     def diff(self, ref, comp, *args, **kwargs):
         """Compare 2 dictionaries.
 
@@ -332,13 +380,16 @@ class DictComparator(BaseComparator):
             path_limit (list[str]): List of path limit tuples or :class:`dictdiffer.utils.PathLimit`
                 object to limit the diff recursion depth.
         """
+        errors = self.current_state.get("format_errors", [])
+
         if len(args) > 5:
             dot_notation = args[5]
             args = args[:5] + args[6:]
         else:
             dot_notation = kwargs.pop("dot_notation", False)
         kwargs["dot_notation"] = dot_notation
-        return list(dictdiffer.diff(ref, comp, *args, **kwargs))
+        errors.extend(list(dictdiffer.diff(ref, comp, *args, **kwargs)))
+        return errors
 
     def format_diff(self, difference):
         """Format one element difference."""
@@ -361,6 +412,11 @@ class JsonComparator(DictComparator):
             data = json.load(file)
         return data
 
+    def save(self, data, path):
+        """Save formatted data into a JSON file."""
+        with open(path, "w", encoding="utf-8") as file:
+            json.dump(data, file)
+
 
 class YamlComparator(DictComparator):
     """Comparator for YAML files.
@@ -374,6 +430,11 @@ class YamlComparator(DictComparator):
             data = yaml.full_load(file)
         return data
 
+    def save(self, data, path):
+        """Save formatted data into a YAML file."""
+        with open(path, "w", encoding="utf-8") as file:
+            yaml.dump(data, file)
+
 
 class XmlComparator(DictComparator):
     """Comparator for XML files.
@@ -407,9 +468,14 @@ class XmlComparator(DictComparator):
             data = self.xmltodict(file.read())
         return data
 
+    def save(self, data, path):
+        """Save formatted data into a XML file."""
+        with open(path, "w", encoding="utf-8") as file:
+            file.write(dicttoxml(data["root"]).decode())
+
     @staticmethod
     def _cast_from_attribute(text, attr):
-        """Converts XML text into a Python data format based on the tag attribute."""
+        """Convert XML text into a Python data format based on the tag attribute."""
         if "type" not in attr:
             return text
         value_type = attr.get("type", "").lower()
@@ -453,7 +519,7 @@ class XmlComparator(DictComparator):
 
     @staticmethod
     def xmltodict(obj):
-        """Converts an XML string into a Python object based on each tag's attribute."""
+        """Convert an XML string into a Python object based on each tag's attribute."""
         root = ElementTree.fromstring(obj)
         output = {}
 
@@ -473,11 +539,16 @@ class IniComparator(DictComparator):
     """
 
     def load(self, path, **kwargs):  # pylint: disable=arguments-differ
-        """Open a XML file."""
+        """Open a INI file."""
         data = configparser.ConfigParser(**kwargs)
         data.read(path)
         return self.configparser_to_dict(data)
 
+    def save(self, data, path):
+        """Save formatted data into a INI file."""
+        with open(path, "w", encoding="utf-8") as file:
+            self.dict_to_configparser(data).write(file)
+
     @staticmethod
     def configparser_to_dict(config):
         """Transform a ConfigParser object into a dict."""
@@ -494,6 +565,16 @@ class IniComparator(DictComparator):
                 dict_config[section][option] = val
         return dict_config
 
+    @staticmethod
+    def dict_to_configparser(data, **kwargs):
+        """Transform a dict object into a ConfigParser."""
+        config = configparser.ConfigParser(**kwargs)
+        for k, v in data.items():
+            config.add_section(k)
+            for opt, val in v.items():
+                config[k][opt] = json.dumps(val)
+        return config
+
 
 class PdfComparator(BaseComparator):
     """Comparator for PDF files."""

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/dir_content_diff/pandas.py

@@ -147,6 +147,42 @@ class HdfComparator(DataframeComparator):
         data.to_hdf(path, index=index, key=key, **kwargs)
 
 
+class FeatherComparator(DataframeComparator):
+    """Comparator for Feather files."""
+
+    def load(self, path, **kwargs):
+        """Load a Feather file into a :class:`pandas.DataFrame` object."""
+        return pd.read_feather(path, **kwargs)
+
+    def save(self, data, path, **kwargs):
+        """Save data to a Feather file."""
+        data.to_feather(path, **kwargs)
+
+
+class ParquetComparator(DataframeComparator):
+    """Comparator for Parquet files."""
+
+    def load(self, path, **kwargs):
+        """Load a Parquet file into a :class:`pandas.DataFrame` object."""
+        return pd.read_parquet(path, **kwargs)
+
+    def save(self, data, path, **kwargs):
+        """Save data to a Parquet file."""
+        data.to_parquet(path, **kwargs)
+
+
+class StataComparator(DataframeComparator):
+    """Comparator for Stata files."""
+
+    def load(self, path, **kwargs):
+        """Load a Stata file into a :class:`pandas.DataFrame` object."""
+        return pd.read_stata(path, **kwargs)
+
+    def save(self, data, path, **kwargs):
+        """Save data to a Stata file."""
+        data.to_stata(path, **kwargs)
+
+
 def register():
     """Register Pandas extensions."""
     register_comparator(".csv", CsvComparator())
@@ -156,3 +192,6 @@ def register():
     register_comparator(".hdf", HdfComparator())
     register_comparator(".hdf4", HdfComparator())
     register_comparator(".hdf5", HdfComparator())
+    register_comparator(".feather", FeatherComparator())
+    register_comparator(".parquet", ParquetComparator())
+    register_comparator(".dta", StataComparator())

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0/dir_content_diff.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dir-content-diff
-Version: 1.6.0
+Version: 1.8.0
 Summary: Simple tool to compare directory contents.
 Home-page: https://dir-content-diff.readthedocs.io
 Author: Blue Brain Project, EPFL
@@ -22,25 +22,30 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 License-File: AUTHORS.md
 Requires-Dist: dictdiffer>=0.8
+Requires-Dist: dicttoxml>=1.7.12
 Requires-Dist: diff_pdf_visually>=1.7
+Requires-Dist: jsonpath-ng>=1.5
 Requires-Dist: PyYaml>=6
 Provides-Extra: pandas
 Requires-Dist: pandas>=1.4; extra == "pandas"
+Requires-Dist: pyarrow>=11; extra == "pandas"
 Requires-Dist: tables>=3.7; extra == "pandas"
 Provides-Extra: docs
 Requires-Dist: m2r2; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-bluebrain-theme; extra == "docs"
 Requires-Dist: pandas>=1.4; extra == "docs"
+Requires-Dist: pyarrow>=11; extra == "docs"
 Requires-Dist: tables>=3.7; extra == "docs"
 Provides-Extra: test
-Requires-Dist: coverage>=6; extra == "test"
+Requires-Dist: coverage>=7.2; extra == "test"
 Requires-Dist: dicttoxml>=1.7.16; extra == "test"
 Requires-Dist: matplotlib>=3; extra == "test"
 Requires-Dist: rst2pdf>=0.99; extra == "test"
 Requires-Dist: pytest>=6.2; extra == "test"
 Requires-Dist: pytest-html<4,>=2; extra == "test"
 Requires-Dist: pandas>=1.4; extra == "test"
+Requires-Dist: pyarrow>=11; extra == "test"
 Requires-Dist: tables>=3.7; extra == "test"
 
 [![Version](https://img.shields.io/pypi/v/dir-content-diff)](https://github.com/BlueBrain/dir-content-diff/releases)
@@ -208,6 +213,18 @@ specific_args = {
 }
 ```
 
+And last but not least, it's possible to use regular expressions to associate specific arguments to
+a set of files:
+
+```python
+specific_args = {
+    "all files with *.a of *.b extensions": {
+        "patterns": [r".*\.[a,b]$"],
+        "comparator": dir_content_diff.BaseComparator(),
+    }
+}
+```
+
 
 ### Export formatted data
 

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/dir_content_diff.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@
 .codespellrc
 .copier-answers.yml
 .coveragerc
+.coveragerc_py38
 .flake8
 .gitattributes
 .gitignore

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/dir_content_diff.egg-info/requires.txt

@@ -1,5 +1,7 @@
 dictdiffer>=0.8
+dicttoxml>=1.7.12
 diff_pdf_visually>=1.7
+jsonpath-ng>=1.5
 PyYaml>=6
 
 [docs]
@@ -7,18 +9,21 @@ m2r2
 sphinx
 sphinx-bluebrain-theme
 pandas>=1.4
+pyarrow>=11
 tables>=3.7
 
 [pandas]
 pandas>=1.4
+pyarrow>=11
 tables>=3.7
 
 [test]
-coverage>=6
+coverage>=7.2
 dicttoxml>=1.7.16
 matplotlib>=3
 rst2pdf>=0.99
 pytest>=6.2
 pytest-html<4,>=2
 pandas>=1.4
+pyarrow>=11
 tables>=3.7

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/setup.py

@@ -6,7 +6,9 @@ from setuptools import setup
 
 reqs = [
     "dictdiffer>=0.8",
+    "dicttoxml>=1.7.12",
     "diff_pdf_visually>=1.7",
+    "jsonpath-ng>=1.5",
     "PyYaml>=6",
 ]
 
@@ -18,11 +20,12 @@ doc_reqs = [
 
 pandas_reqs = [
     "pandas>=1.4",
+    "pyarrow>=11",
     "tables>=3.7",
 ]
 
 test_reqs = [
-    "coverage>=6",
+    "coverage>=7.2",
     "dicttoxml>=1.7.16",
     "matplotlib>=3",
     "rst2pdf>=0.99",

{dir-content-diff-1.6.0 → dir-content-diff-1.8.0}/tests/conftest.py

@@ -125,6 +125,12 @@ def dict_diff():
     return diff
 
 
+@pytest.fixture
+def base_diff():
+    """The diff that should be reported for the XML files."""
+    return r"The files '\S*/file\..{3,4}' and '\S*/file\..{3,4}' are different\."
+
+
 @pytest.fixture
 def xml_diff(dict_diff):
     """The diff that should be reported for the XML files."""