dir-content-diff 1.7.0__tar.gz → 1.8.0__tar.gz

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/.copier-answers.yml

@@ -1,6 +1,6 @@
 # Changes here will be overwritten by Copier
 
-_commit: 0.1.53
+_commit: 0.1.54
 _src_path: git@bbpgitlab.epfl.ch:neuromath/python-template.git
 author_email: ''
 author_name: Blue Brain Project, EPFL

dir-content-diff-1.8.0/.coveragerc_py38

@@ -0,0 +1,11 @@
+[paths]
+source_paths =
+	dir_content_diff
+	*/site-packages/dir_content_diff
+	*/dir_content_diff/dir_content_diff
+	*/dir-content-diff/dir_content_diff
+
+[report]
+exclude_also =
+    return pd\.read_feather\(path, \*\*kwargs\)
+    data\.to_feather\(path, \*\*kwargs\)

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/.github/workflows/commitlint.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: '16'
+          node-version: '20'
       - run: npm install -g --force commitlint @commitlint/cli commitlint-plugin-cleanfeet
       - run: npm install conventional-changelog-conventionalcommits
       - run: touch .git/COMMIT_EDITMSG

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/.github/workflows/run-tox.yml

@@ -70,6 +70,7 @@ jobs:
         directory: ./reports
         flags: pytest
         name: "dir-content-diff-py${{ matrix.python-version }}-${{ matrix.min_versions }}"
+        token: ${{ secrets.CODECOV_TOKEN }}
     - name: Upload test artifacts
       uses: actions/upload-artifact@v4
       if: always()

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [1.8.0](https://github.com/BlueBrain/dir-content-diff/compare/1.7.0..1.8.0)
+
+> 8 April 2024
+
+### New Features
+
+- Add new comparators for Feather, Parquet and Stata files (Adrien Berchet - [#56](https://github.com/BlueBrain/dir-content-diff/pull/56))
+
 ## [1.7.0](https://github.com/BlueBrain/dir-content-diff/compare/1.6.0..1.7.0)
 
 > 5 February 2024

{dir-content-diff-1.7.0/dir_content_diff.egg-info → dir-content-diff-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dir-content-diff
-Version: 1.7.0
+Version: 1.8.0
 Summary: Simple tool to compare directory contents.
 Home-page: https://dir-content-diff.readthedocs.io
 Author: Blue Brain Project, EPFL
@@ -28,21 +28,24 @@ Requires-Dist: jsonpath-ng>=1.5
 Requires-Dist: PyYaml>=6
 Provides-Extra: pandas
 Requires-Dist: pandas>=1.4; extra == "pandas"
+Requires-Dist: pyarrow>=11; extra == "pandas"
 Requires-Dist: tables>=3.7; extra == "pandas"
 Provides-Extra: docs
 Requires-Dist: m2r2; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-bluebrain-theme; extra == "docs"
 Requires-Dist: pandas>=1.4; extra == "docs"
+Requires-Dist: pyarrow>=11; extra == "docs"
 Requires-Dist: tables>=3.7; extra == "docs"
 Provides-Extra: test
-Requires-Dist: coverage>=6; extra == "test"
+Requires-Dist: coverage>=7.2; extra == "test"
 Requires-Dist: dicttoxml>=1.7.16; extra == "test"
 Requires-Dist: matplotlib>=3; extra == "test"
 Requires-Dist: rst2pdf>=0.99; extra == "test"
 Requires-Dist: pytest>=6.2; extra == "test"
 Requires-Dist: pytest-html<4,>=2; extra == "test"
 Requires-Dist: pandas>=1.4; extra == "test"
+Requires-Dist: pyarrow>=11; extra == "test"
 Requires-Dist: tables>=3.7; extra == "test"
 
 [![Version](https://img.shields.io/pypi/v/dir-content-diff)](https://github.com/BlueBrain/dir-content-diff/releases)

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/dir_content_diff/pandas.py

@@ -147,6 +147,42 @@ class HdfComparator(DataframeComparator):
         data.to_hdf(path, index=index, key=key, **kwargs)
 
 
+class FeatherComparator(DataframeComparator):
+    """Comparator for Feather files."""
+
+    def load(self, path, **kwargs):
+        """Load a Feather file into a :class:`pandas.DataFrame` object."""
+        return pd.read_feather(path, **kwargs)
+
+    def save(self, data, path, **kwargs):
+        """Save data to a Feather file."""
+        data.to_feather(path, **kwargs)
+
+
+class ParquetComparator(DataframeComparator):
+    """Comparator for Parquet files."""
+
+    def load(self, path, **kwargs):
+        """Load a Parquet file into a :class:`pandas.DataFrame` object."""
+        return pd.read_parquet(path, **kwargs)
+
+    def save(self, data, path, **kwargs):
+        """Save data to a Parquet file."""
+        data.to_parquet(path, **kwargs)
+
+
+class StataComparator(DataframeComparator):
+    """Comparator for Stata files."""
+
+    def load(self, path, **kwargs):
+        """Load a Stata file into a :class:`pandas.DataFrame` object."""
+        return pd.read_stata(path, **kwargs)
+
+    def save(self, data, path, **kwargs):
+        """Save data to a Stata file."""
+        data.to_stata(path, **kwargs)
+
+
 def register():
     """Register Pandas extensions."""
     register_comparator(".csv", CsvComparator())
@@ -156,3 +192,6 @@ def register():
     register_comparator(".hdf", HdfComparator())
     register_comparator(".hdf4", HdfComparator())
     register_comparator(".hdf5", HdfComparator())
+    register_comparator(".feather", FeatherComparator())
+    register_comparator(".parquet", ParquetComparator())
+    register_comparator(".dta", StataComparator())

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0/dir_content_diff.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dir-content-diff
-Version: 1.7.0
+Version: 1.8.0
 Summary: Simple tool to compare directory contents.
 Home-page: https://dir-content-diff.readthedocs.io
 Author: Blue Brain Project, EPFL
@@ -28,21 +28,24 @@ Requires-Dist: jsonpath-ng>=1.5
 Requires-Dist: PyYaml>=6
 Provides-Extra: pandas
 Requires-Dist: pandas>=1.4; extra == "pandas"
+Requires-Dist: pyarrow>=11; extra == "pandas"
 Requires-Dist: tables>=3.7; extra == "pandas"
 Provides-Extra: docs
 Requires-Dist: m2r2; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-bluebrain-theme; extra == "docs"
 Requires-Dist: pandas>=1.4; extra == "docs"
+Requires-Dist: pyarrow>=11; extra == "docs"
 Requires-Dist: tables>=3.7; extra == "docs"
 Provides-Extra: test
-Requires-Dist: coverage>=6; extra == "test"
+Requires-Dist: coverage>=7.2; extra == "test"
 Requires-Dist: dicttoxml>=1.7.16; extra == "test"
 Requires-Dist: matplotlib>=3; extra == "test"
 Requires-Dist: rst2pdf>=0.99; extra == "test"
 Requires-Dist: pytest>=6.2; extra == "test"
 Requires-Dist: pytest-html<4,>=2; extra == "test"
 Requires-Dist: pandas>=1.4; extra == "test"
+Requires-Dist: pyarrow>=11; extra == "test"
 Requires-Dist: tables>=3.7; extra == "test"
 
 [![Version](https://img.shields.io/pypi/v/dir-content-diff)](https://github.com/BlueBrain/dir-content-diff/releases)

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/dir_content_diff.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@
 .codespellrc
 .copier-answers.yml
 .coveragerc
+.coveragerc_py38
 .flake8
 .gitattributes
 .gitignore

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/dir_content_diff.egg-info/requires.txt

@@ -9,18 +9,21 @@ m2r2
 sphinx
 sphinx-bluebrain-theme
 pandas>=1.4
+pyarrow>=11
 tables>=3.7
 
 [pandas]
 pandas>=1.4
+pyarrow>=11
 tables>=3.7
 
 [test]
-coverage>=6
+coverage>=7.2
 dicttoxml>=1.7.16
 matplotlib>=3
 rst2pdf>=0.99
 pytest>=6.2
 pytest-html<4,>=2
 pandas>=1.4
+pyarrow>=11
 tables>=3.7

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/setup.py

@@ -20,11 +20,12 @@ doc_reqs = [
 
 pandas_reqs = [
     "pandas>=1.4",
+    "pyarrow>=11",
     "tables>=3.7",
 ]
 
 test_reqs = [
-    "coverage>=6",
+    "coverage>=7.2",
     "dicttoxml>=1.7.16",
     "matplotlib>=3",
     "rst2pdf>=0.99",

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/tests/test_pandas.py

@@ -4,6 +4,7 @@
 # pylint: disable=unused-argument
 # pylint: disable=use-implicit-booleaness-not-comparison
 import re
+import sys
 
 import pandas as pd
 import pytest
@@ -49,6 +50,9 @@ class TestRegistry:
             ".hdf": dir_content_diff.pandas.HdfComparator(),
             ".hdf4": dir_content_diff.pandas.HdfComparator(),
             ".hdf5": dir_content_diff.pandas.HdfComparator(),
+            ".feather": dir_content_diff.pandas.FeatherComparator(),
+            ".parquet": dir_content_diff.pandas.ParquetComparator(),
+            ".dta": dir_content_diff.pandas.StataComparator(),
         }
 
 
@@ -59,19 +63,71 @@ def pandas_registry_reseter(registry_reseter):
 
 
 @pytest.fixture
-def ref_hdf5(empty_ref_tree):
-    """The reference HDF5 file."""
+def ref_data(empty_ref_tree):
     ref_data = {
         "col_a": [1, 2, 3],
         "col_b": ["a", "b", "c"],
         "col_c": [4, 5, 6],
     }
     df = pd.DataFrame(ref_data, index=["idx1", "idx2", "idx3"])
+    return df
+
+
+@pytest.fixture
+def ref_hdf5(ref_data, empty_ref_tree):
+    """The reference HDF5 file."""
     filename = empty_ref_tree / "file.h5"
-    df.to_hdf(filename, key="data", index=True)
+    ref_data.to_hdf(filename, key="data", index=True)
+    return filename
+
+
+@pytest.fixture
+def ref_feather(ref_data, empty_ref_tree):
+    """The reference Feather file."""
+    filename = empty_ref_tree / "file.feather"
+    ref_data.to_feather(filename)
     return filename
 
 
+@pytest.fixture
+def ref_parquet(ref_data, empty_ref_tree):
+    """The reference Parquet file."""
+    filename = empty_ref_tree / "file.parquet"
+    ref_data.to_parquet(filename)
+    return filename
+
+
+@pytest.fixture
+def ref_stata(ref_data, empty_ref_tree):
+    """The reference Stata file."""
+    filename = empty_ref_tree / "file.dta"
+    ref_data.to_stata(filename)
+    return filename
+
+
+def _update_df(df):
+    df.loc["idx1", "col_a"] *= 10
+    df.loc["idx2", "col_b"] += "_new"
+
+
+@pytest.fixture
+def res_diff_checker():
+    """The regex used to check the diff result."""
+    return (
+        r"The files '\S*/ref/file\.\S*' and '\S*/res/file\.\S*' are different:\n\n"
+        r"Column 'col_a': Series are different\n\n"
+        r"Series values are different \(33\.33333 %\)\n"
+        r"\[index\]: \[idx1, idx2, idx3\]\n"
+        r"\[left\]:  \[1, 2, 3\]\n\[right\]: \[10, 2, 3\]\n"
+        r"""(At positional index 0, first diff: 1 != 10\n)?\n"""
+        r"Column 'col_b': Series are different\n\n"
+        r"Series values are different \(33\.33333 %\)\n"
+        r"\[index\]: \[idx1, idx2, idx3\]\n"
+        r"\[left\]:  \[a, b, c\]\n\[right\]: \[a, b_new, c\]"
+        r"""(\nAt positional index 1, first diff: b != b_new)?"""
+    )
+
+
 @pytest.fixture
 def res_hdf5_equal(ref_hdf5, empty_res_tree):
     """The result hdf5 file equal to the reference."""
@@ -85,13 +141,69 @@ def res_hdf5_equal(ref_hdf5, empty_res_tree):
 def res_hdf5_diff(ref_hdf5, empty_res_tree):
     """The result hdf5 file different from the reference."""
     df = pd.read_hdf(ref_hdf5, index_col="index")
-    df.loc["idx1", "col_a"] *= 10
-    df.loc["idx2", "col_b"] += "_new"
+    _update_df(df)
     filename = empty_res_tree / "file.h5"
     df.to_hdf(filename, key="data", index=True)
     return filename
 
 
+@pytest.fixture
+def res_feather_equal(ref_feather, empty_res_tree):
+    """The result feather file equal to the reference."""
+    df = pd.read_feather(ref_feather)
+    filename = empty_res_tree / "file.feather"
+    df.to_feather(filename)
+    return filename
+
+
+@pytest.fixture
+def res_feather_diff(ref_feather, empty_res_tree):
+    """The result feather file different from the reference."""
+    df = pd.read_feather(ref_feather)
+    _update_df(df)
+    filename = empty_res_tree / "file.feather"
+    df.to_feather(filename)
+    return filename
+
+
+@pytest.fixture
+def res_parquet_equal(ref_parquet, empty_res_tree):
+    """The result parquet file equal to the reference."""
+    df = pd.read_parquet(ref_parquet)
+    filename = empty_res_tree / "file.parquet"
+    df.to_parquet(filename)
+    return filename
+
+
+@pytest.fixture
+def res_parquet_diff(ref_parquet, empty_res_tree):
+    """The result parquet file different from the reference."""
+    df = pd.read_parquet(ref_parquet)
+    _update_df(df)
+    filename = empty_res_tree / "file.parquet"
+    df.to_parquet(filename)
+    return filename
+
+
+@pytest.fixture
+def res_stata_equal(ref_stata, empty_res_tree):
+    """The result stata file equal to the reference."""
+    df = pd.read_stata(ref_stata, index_col="index")
+    filename = empty_res_tree / "file.dta"
+    df.to_stata(filename)
+    return filename
+
+
+@pytest.fixture
+def res_stata_diff(ref_stata, empty_res_tree):
+    """The result stata file different from the reference."""
+    df = pd.read_stata(ref_stata, index_col="index")
+    _update_df(df)
+    filename = empty_res_tree / "file.dta"
+    df.to_stata(filename)
+    return filename
+
+
 class TestEqualTrees:
     """Tests that should return no difference."""
 
@@ -222,17 +334,46 @@ class TestEqualTrees:
         )
         assert match_res is not None
 
-    def test_hdf5_comparator(
-        self, empty_ref_tree, empty_res_tree, res_hdf5_equal, pandas_registry_reseter
-    ):
-        """Test the comparator for HDF5 files."""
+    def _check_equal(self, empty_ref_tree, empty_res_tree):
         assert_equal_trees(empty_ref_tree, empty_res_tree, export_formatted_files=True)
         ref_files = list(empty_ref_tree.rglob("*"))
         res_files = list(empty_res_tree.rglob("*"))
         assert len(ref_files) == len(res_files)
+        return ref_files, res_files
+
+    def test_hdf5_comparator(
+        self, empty_ref_tree, empty_res_tree, res_hdf5_equal, pandas_registry_reseter
+    ):
+        """Test the comparator for HDF5 files."""
+        ref_files, res_files = self._check_equal(empty_ref_tree, empty_res_tree)
         for i, j in zip(ref_files, res_files):
             assert pd.read_hdf(i).equals(pd.read_hdf(j))
 
+    @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or higher")
+    def test_feather_comparator(
+        self, empty_ref_tree, empty_res_tree, res_feather_equal, pandas_registry_reseter
+    ):
+        """Test the comparator for Feather files."""
+        ref_files, res_files = self._check_equal(empty_ref_tree, empty_res_tree)
+        for i, j in zip(ref_files, res_files):
+            assert pd.read_feather(i).equals(pd.read_feather(j))
+
+    def test_parquet_comparator(
+        self, empty_ref_tree, empty_res_tree, res_parquet_equal, pandas_registry_reseter
+    ):
+        """Test the comparator for Parquet files."""
+        ref_files, res_files = self._check_equal(empty_ref_tree, empty_res_tree)
+        for i, j in zip(ref_files, res_files):
+            assert pd.read_parquet(i).equals(pd.read_parquet(j))
+
+    def test_stata_comparator(
+        self, empty_ref_tree, empty_res_tree, res_stata_equal, pandas_registry_reseter
+    ):
+        """Test the comparator for Stata files."""
+        ref_files, res_files = self._check_equal(empty_ref_tree, empty_res_tree)
+        for i, j in zip(ref_files, res_files):
+            assert pd.read_stata(i).equals(pd.read_stata(j))
+
 
 class TestDiffTrees:
     """Tests that should return differences."""
@@ -287,26 +428,70 @@ class TestDiffTrees:
         )
         assert match_res is not None
 
-    def test_hdf5_comparator(
-        self, empty_ref_tree, empty_res_tree, res_hdf5_diff, pandas_registry_reseter
+    def _check_diff_comparator(
+        self, empty_ref_tree, empty_res_tree, res_diff_checker, ext, specific_args=None
     ):
-        """Test the comparator for HDF5 files."""
-        res = compare_trees(empty_ref_tree, empty_res_tree)
+        res = compare_trees(empty_ref_tree, empty_res_tree, specific_args=specific_args)
 
         assert len(res) == 1
-        res_hdf = res["file.h5"]
+        filename = f"file.{ext}"
+        res_ext = res[filename]
+        if specific_args is not None and filename in specific_args:
+            res_diff_checker = res_diff_checker.replace(
+                "' are different:",
+                "' are different:\nKwargs used for loading data: "
+                f"{specific_args[filename]['load_kwargs']}",
+            )
         match_res = re.match(
-            r"The files '\S*/ref/file\.h5' and '\S*/res/file\.h5' are different:\n\n"
-            r"Column 'col_a': Series are different\n\n"
-            r"Series values are different \(33\.33333 %\)\n"
-            r"\[index\]: \[idx1, idx2, idx3\]\n"
-            r"\[left\]:  \[1, 2, 3\]\n\[right\]: \[10, 2, 3\]\n"
-            r"""(At positional index 0, first diff: 1 != 10\n)?\n"""
-            r"Column 'col_b': Series are different\n\n"
-            r"Series values are different \(33\.33333 %\)\n"
-            r"\[index\]: \[idx1, idx2, idx3\]\n"
-            r"\[left\]:  \[a, b, c\]\n\[right\]: \[a, b_new, c\]"
-            r"""(\nAt positional index 1, first diff: b != b_new)?""",
-            res_hdf,
+            res_diff_checker,
+            res_ext,
         )
         assert match_res is not None
+
+    def test_hdf5_comparator(
+        self,
+        empty_ref_tree,
+        empty_res_tree,
+        res_hdf5_diff,
+        res_diff_checker,
+        pandas_registry_reseter,
+    ):
+        """Test the comparator for HDF5 files."""
+        self._check_diff_comparator(empty_ref_tree, empty_res_tree, res_diff_checker, "h5")
+
+    @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or higher")
+    def test_feather_comparator(
+        self,
+        empty_ref_tree,
+        empty_res_tree,
+        res_feather_diff,
+        res_diff_checker,
+        pandas_registry_reseter,
+    ):
+        """Test the comparator for feather files."""
+        self._check_diff_comparator(empty_ref_tree, empty_res_tree, res_diff_checker, "feather")
+
+    def test_parquet_comparator(
+        self,
+        empty_ref_tree,
+        empty_res_tree,
+        res_parquet_diff,
+        res_diff_checker,
+        pandas_registry_reseter,
+    ):
+        """Test the comparator for parquet files."""
+        self._check_diff_comparator(empty_ref_tree, empty_res_tree, res_diff_checker, "parquet")
+
+    def test_stata_comparator(
+        self,
+        empty_ref_tree,
+        empty_res_tree,
+        res_stata_diff,
+        res_diff_checker,
+        pandas_registry_reseter,
+    ):
+        """Test the comparator for stata files."""
+        specific_args = {"file.dta": {"load_kwargs": {"index_col": "index"}}}
+        self._check_diff_comparator(
+            empty_ref_tree, empty_res_tree, res_diff_checker, "dta", specific_args=specific_args
+        )

{dir-content-diff-1.7.0 → dir-content-diff-1.8.0}/tox.ini

@@ -19,6 +19,7 @@ extras =
     test
 setenv =
     COVERAGE_FILE = {env:COVERAGE_FILE:.coverage-{envname}}
+    py38,min_versions: COVERAGE_RCFILE=.coveragerc_py38
 deps =
     min_versions: Requirements-Builder
 commands =