csvsmith 0.7.3__tar.gz → 0.8.0__tar.gz

{csvsmith-0.7.3/src/csvsmith.egg-info → csvsmith-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvsmith
-Version: 0.7.3
+Version: 0.8.0
 Summary: Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers.
 Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
 License: MIT License
@@ -87,6 +87,7 @@ Features
 - Convert Excel workbooks to CSV
 - Move files by suffix
 - Find matching values inside CSV files
+- Concatenate CSV files with identical headers
 - Use the tools either from Python or from the command line
 
 Installation
@@ -182,6 +183,12 @@ Find matches in a CSV:
 
    csvsmith find-matches input.csv target --ignore-case --ignore-whitespace
 
+Concatenate CSV files:
+
+.. code-block:: console
+
+   csvsmith strict-concat file1.csv file2.csv -o combined.csv
+
 Find matches in a CSV
 ---------------------
 
@@ -241,7 +248,7 @@ File and conversion helpers:
 
 .. code-block:: python
 
-   from csvsmith import excel_to_csv, move_by_suffix
+   from csvsmith import excel_to_csv, move_by_suffix, strict_concat_rows, save_csv
 
 String comparison utilities:
 

{csvsmith-0.7.3 → csvsmith-0.8.0}/README.rst

@@ -41,6 +41,7 @@ Features
 - Convert Excel workbooks to CSV
 - Move files by suffix
 - Find matching values inside CSV files
+- Concatenate CSV files with identical headers
 - Use the tools either from Python or from the command line
 
 Installation
@@ -136,6 +137,12 @@ Find matches in a CSV:
 
    csvsmith find-matches input.csv target --ignore-case --ignore-whitespace
 
+Concatenate CSV files:
+
+.. code-block:: console
+
+   csvsmith strict-concat file1.csv file2.csv -o combined.csv
+
 Find matches in a CSV
 ---------------------
 
@@ -195,7 +202,7 @@ File and conversion helpers:
 
 .. code-block:: python
 
-   from csvsmith import excel_to_csv, move_by_suffix
+   from csvsmith import excel_to_csv, move_by_suffix, strict_concat_rows, save_csv
 
 String comparison utilities:
 

{csvsmith-0.7.3 → csvsmith-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "csvsmith"
-version = "0.7.3"
+version = "0.8.0"
 description = "Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers."
 readme = "README.rst"
 requires-python = ">=3.10"

{csvsmith-0.7.3 → csvsmith-0.8.0}/src/csvsmith/__init__.py

@@ -16,6 +16,8 @@ Public API:
 - Relation
 - Result
 - analyze_pair
+- strict_concat_rows
+- save_csv
 
 Compatibility aliases:
 - CSVCleaner
@@ -28,10 +30,11 @@ Submodules:
 - csvsmith.filter_rows
 - csvsmith.excel2csv
 - csvsmith.move_files
+- csvsmith.strict_concat
 - csvsmith.cli (CLI entrypoint)
 """
 
-__version__ = "0.7.3"
+__version__ = "0.8.0"
 
 from .tools.classify import CSVClassifier
 from .tools.excel2csv import excel_to_csv
@@ -43,6 +46,7 @@ from .tools.row_dedup import (
     find_duplicate_rows,
     dedupe_with_report,
 )
+from .tools.strict_concat import save_csv, strict_concat_rows
 from .utils.clean_numeric import clean_numeric
 from .utils.distance import StringDistance, Relation, Result, analyze_pair
 from .utils.io import (
@@ -68,4 +72,6 @@ __all__ = [
     "Result",
     "analyze_pair",
     "clean_numeric",
-]
+    "strict_concat_rows",
+    "save_csv",
+]

{csvsmith-0.7.3 → csvsmith-0.8.0}/src/csvsmith/cli.py

@@ -16,6 +16,7 @@ from .tools.row_dedup import (
     dedupe_with_report,
     find_duplicate_rows,
 )
+from .tools.strict_concat import save_csv, strict_concat_rows
 from .tools.find_matches_in_csv import find_matches_in_csv
 from .utils.distance import analyze_pair
 from .utils.io import read_csv_rows, write_csv_rows
@@ -180,6 +181,23 @@ def cmd_find_matches(args: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_strict_concat(args: argparse.Namespace) -> int:
+    input_dir = Path(args.input_dir)
+    if not input_dir.is_dir():
+        print(f"Error: input directory not found: {input_dir}", file=sys.stderr)
+        return 1
+
+    try:
+        rows = strict_concat_rows(input_dir)
+        save_csv(rows, args.output)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    print(f"Wrote concatenated CSV to: {args.output}")
+    return 0
+
+
 def _add_find_matches_parser(subparsers) -> None:
     parser = subparsers.add_parser("find-matches", help="Find matches in a CSV file.")
     parser.add_argument("input", help="Input CSV file.")
@@ -190,6 +208,16 @@ def _add_find_matches_parser(subparsers) -> None:
     parser.set_defaults(func=cmd_find_matches)
 
 
+def _add_strict_concat_parser(subparsers) -> None:
+    parser = subparsers.add_parser(
+        "strict-concat",
+        help="Concatenate CSVs in a directory only when all headers match exactly.",
+    )
+    parser.add_argument("input_dir", help="Directory containing CSV files to concatenate.")
+    parser.add_argument("-o", "--output", required=True, help="Output CSV file path.")
+    parser.set_defaults(func=cmd_strict_concat)
+
+
 def _add_row_duplicates_parser(subparsers) -> None:
     parser = subparsers.add_parser("row-duplicates", help="Find duplicate rows in a CSV.")
     parser.add_argument("input", help="Input CSV file.")
@@ -317,6 +345,7 @@ def build_parser() -> argparse.ArgumentParser:
 
     _add_row_duplicates_parser(subparsers)
     _add_find_matches_parser(subparsers)
+    _add_strict_concat_parser(subparsers)
     _add_dedupe_parser(subparsers)
     _add_classify_parser(subparsers)
     _add_move_files_parser(subparsers)

csvsmith-0.8.0/src/csvsmith/tools/strict_concat.py

@@ -0,0 +1,101 @@
+import csv
+from collections.abc import Iterable
+from pathlib import Path
+
+
+def find_csvs(csv_dir: Path | str) -> list[Path]:
+    """
+    Find all CSV files in the specified directory.
+
+    This function searches for all files with a ``.csv`` extension in the given
+    directory and returns a sorted list of their paths.
+
+    :param csv_dir: The directory to search for CSV files. This can be provided
+        as either a ``Path`` object or a string representing the path to the
+        directory.
+    :type csv_dir: Path | str
+    :return: Sorted list of paths to all ``.csv`` files found in the specified
+        directory.
+    :rtype: list[Path]
+    """
+    return sorted(Path(csv_dir).glob("*.csv"))
+
+
+def read_header(csv_path: Path) -> list[str]:
+    """
+    Reads the header row of a given CSV file.
+
+    This function opens a CSV file located at the specified path, reads its first
+    row, and returns it as a list of strings. The file is assumed to be encoded
+    in UTF-8 with optional BOM (Byte Order Mark). If the CSV file is empty, a
+    ValueError is raised indicating the problem. The CSV file is expected to be
+    opened in read mode with no newline translation.
+
+    :param csv_path: The path to the CSV file to read the header from.
+    :type csv_path: Path
+    :return: A list of strings representing the header row of the CSV file.
+    :rtype: list[str]
+    :raises ValueError: If the CSV file is empty and no header can be retrieved.
+    """
+    with csv_path.open(encoding="utf-8-sig", newline="") as f:
+        reader = csv.reader(f)
+        try:
+            return next(reader)
+        except StopIteration as e:
+            raise ValueError(f"Empty CSV: {csv_path}") from e
+
+
+def _validate_headers_match(csv_paths: list[Path]) -> list[str]:
+    """
+    Validates that the headers of all provided CSV files match each other. The function compares the
+    header of each CSV file in the input list against the header of the first file. If a mismatch
+    is encountered, a ValueError is raised indicating the problematic file. The function returns
+    the header of the first file if all headers are consistent.
+
+    :param csv_paths: A list of Path objects representing the file paths to the CSV files to validate.
+    :type csv_paths: list[Path]
+    :return: A list of strings representing the matched header of the first CSV file.
+    :rtype: list[str]
+    """
+    expected_header = read_header(csv_paths[0])
+    for csv_path in csv_paths[1:]:
+        if read_header(csv_path) != expected_header:
+            raise ValueError(f"Header mismatch: {csv_path}")
+    return expected_header
+
+
+def strict_concat_rows(csv_dir: Path | str) -> list[list[str]]:
+    """
+    Concatenates rows from multiple CSV files into a list of lists of strings, ensuring
+    the headers across all CSV files match. The output includes a new column indicating
+    the file stem.
+
+    :param csv_dir: Directory containing the CSV files or a specific path to a CSV file.
+    :type csv_dir: Path | str
+    :return: A list of lists, where each inner list represents a row from the concatenated
+        CSV files. The first row contains the headers, including a "file_stem" column.
+    :rtype: list[list[str]]
+    :raises FileNotFoundError: If no CSV files are found in the provided directory.
+    """
+    csv_paths = find_csvs(csv_dir)
+    if not csv_paths:
+        raise FileNotFoundError(f"No CSV files found in: {csv_dir}")
+    
+    expected_header = _validate_headers_match(csv_paths)
+    out_rows: list[list[str]] = [["file_stem", *expected_header]]
+    
+    for csv_path in csv_paths:
+        with csv_path.open(encoding="utf-8-sig", newline="") as f:
+            reader = csv.reader(f)
+            next(reader)  # skip header
+            for row in reader:
+                out_rows.append([csv_path.stem, *row])
+    return out_rows
+
+
+def save_csv(rows: Iterable[list[str]], out_path: Path | str) -> None:
+    """Write rows to out_path."""
+    out_path = Path(out_path)
+    with out_path.open("w", encoding="utf-8", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerows(rows)

{csvsmith-0.7.3 → csvsmith-0.8.0}/src/csvsmith/utils/clean_numeric.py

@@ -5,7 +5,7 @@ NON_BREAKING_SPACE = "\xa0"
 SEPARATOR_PATTERN = re.compile(r"[ _\xa0]")
 NUMBER_PATTERN = re.compile(r"^-?(?:\d+|\d*\.\d+)$")
 INVALID_NUMBER_MESSAGE = "Could not convert {value!r} to a valid number."
-CURRENCY_PREFIX_PATTERN = re.compile(r"^[\$€£¥]")
+CURRENCY_PREFIX_PATTERN = re.compile(r"^[\$€£¥₹]")
 
 
 def strip_currency_prefix(value: Any) -> Any:

{csvsmith-0.7.3 → csvsmith-0.8.0/src/csvsmith.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvsmith
-Version: 0.7.3
+Version: 0.8.0
 Summary: Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers.
 Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
 License: MIT License
@@ -87,6 +87,7 @@ Features
 - Convert Excel workbooks to CSV
 - Move files by suffix
 - Find matching values inside CSV files
+- Concatenate CSV files with identical headers
 - Use the tools either from Python or from the command line
 
 Installation
@@ -182,6 +183,12 @@ Find matches in a CSV:
 
    csvsmith find-matches input.csv target --ignore-case --ignore-whitespace
 
+Concatenate CSV files:
+
+.. code-block:: console
+
+   csvsmith strict-concat file1.csv file2.csv -o combined.csv
+
 Find matches in a CSV
 ---------------------
 
@@ -241,7 +248,7 @@ File and conversion helpers:
 
 .. code-block:: python
 
-   from csvsmith import excel_to_csv, move_by_suffix
+   from csvsmith import excel_to_csv, move_by_suffix, strict_concat_rows, save_csv
 
 String comparison utilities:
 

{csvsmith-0.7.3 → csvsmith-0.8.0}/src/csvsmith.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/csvsmith/tools/filter_rows.py
 src/csvsmith/tools/find_matches_in_csv.py
 src/csvsmith/tools/move_files.py
 src/csvsmith/tools/row_dedup.py
+src/csvsmith/tools/strict_concat.py
 src/csvsmith/utils/__init__.py
 src/csvsmith/utils/clean_numeric.py
 src/csvsmith/utils/distance.py
@@ -30,4 +31,5 @@ tests/test_find_matches_in_csv.py
 tests/test_move_files.py
 tests/test_normalize.py
 tests/test_row_dedup.py
+tests/test_strict_concat.py
 tests/test_string_distance.py

{csvsmith-0.7.3 → csvsmith-0.8.0}/tests/test_cli.py

@@ -187,4 +187,13 @@ def test_cli_parses_find_matches_command():
     assert args.target == "target"
     assert args.ignore_case is True
     assert args.ignore_whitespace is True
-    assert args.no_nfkc is True
+    assert args.no_nfkc is True
+
+
+def test_cli_parses_strict_concat_command():
+    parser = build_parser()
+    args = parser.parse_args(["strict-concat", "input_dir", "-o", "output.csv"])
+
+    assert args.command == "strict-concat"
+    assert args.input_dir == "input_dir"
+    assert args.output == "output.csv"

csvsmith-0.8.0/tests/test_strict_concat.py

@@ -0,0 +1,102 @@
+import csv
+from pathlib import Path
+
+import pytest
+
+from csvsmith.tools.strict_concat import strict_concat_rows
+
+
+def write_csv(path: Path, rows: list[list[str]]):
+    with path.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerows(rows)
+
+
+def test_strict_concat_happy(tmp_path):
+    f1 = tmp_path / "a.csv"
+    f2 = tmp_path / "b.csv"
+
+    write_csv(f1, [["id", "name"], ["1", "Alice"]])
+    write_csv(f2, [["id", "name"], ["2", "Bob"]])
+
+    rows = strict_concat_rows(tmp_path)
+
+    assert rows == [
+        ["file_stem", "id", "name"],
+        ["a", "1", "Alice"],
+        ["b", "2", "Bob"],
+    ]
+
+
+def test_strict_concat_header_mismatch(tmp_path):
+    write_csv(tmp_path / "a.csv", [["id", "name"], ["1", "Alice"]])
+    write_csv(tmp_path / "b.csv", [["id", "age"], ["2", "30"]])
+
+    with pytest.raises(ValueError, match="Header mismatch"):
+        strict_concat_rows(tmp_path)
+
+
+def test_strict_concat_empty_dir(tmp_path):
+    with pytest.raises(FileNotFoundError):
+        strict_concat_rows(tmp_path)
+
+    def test_strict_concat_with_non_csv_files(tmp_path):
+        # Create sample CSV files.
+        write_csv(tmp_path / "a.csv", [["id", "name"], ["1", "Alice"]])
+        write_csv(tmp_path / "b.csv", [["id", "name"], ["2", "Bob"]])
+
+        # Create some non-CSV files in the same directory.
+        (tmp_path / "file1.txt").write_text("This is a text file.", encoding="utf-8")
+        (tmp_path / "file2.doc").write_text("This is a Word document.", encoding="utf-8")
+
+        # Call function and check results ignore non-CSV files.
+        rows = strict_concat_rows(tmp_path)
+        assert rows == [
+            ["file_stem", "id", "name"],
+            ["a", "1", "Alice"],
+            ["b", "2", "Bob"],
+        ]
+
+
+def test_strict_concat_empty_csv(tmp_path):
+    (tmp_path / "a.csv").write_text("", encoding="utf-8")
+
+    with pytest.raises(ValueError, match="Empty CSV"):
+        strict_concat_rows(tmp_path)
+
+
+def test_header_only_files(tmp_path):
+    write_csv(tmp_path / "a.csv", [["id", "name"]])
+    write_csv(tmp_path / "b.csv", [["id", "name"]])
+
+    rows = strict_concat_rows(tmp_path)
+
+    assert rows == [["file_stem", "id", "name"]]
+
+
+def test_strict_concat_cli(tmp_path, capsys):
+    from csvsmith.cli import main
+    f1 = tmp_path / "a.csv"
+    f2 = tmp_path / "b.csv"
+    out = tmp_path / "out.csv"
+
+    write_csv(f1, [["id", "name"], ["1", "Alice"]])
+    write_csv(f2, [["id", "name"], ["2", "Bob"]])
+
+    exit_code = main(["strict-concat", str(tmp_path), "-o", str(out)])
+
+    assert exit_code == 0
+    assert out.exists()
+    
+    with out.open(encoding="utf-8") as f:
+        reader = csv.reader(f)
+        rows = list(reader)
+    
+    assert rows == [
+        ["file_stem", "id", "name"],
+        ["a", "1", "Alice"],
+        ["b", "2", "Bob"],
+    ]
+    
+    captured = capsys.readouterr()
+    assert "Wrote concatenated CSV to:" in captured.out