csvsmith 0.7.3__tar.gz → 0.9.0__tar.gz

{csvsmith-0.7.3/src/csvsmith.egg-info → csvsmith-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csvsmith
-Version: 0.7.3
+Version: 0.9.0
 Summary: Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers.
 Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
 License: MIT License
@@ -87,6 +87,8 @@ Features
 - Convert Excel workbooks to CSV
 - Move files by suffix
 - Find matching values inside CSV files
+- Concatenate CSV files with identical headers
+- Tokenize repeated CSV values and restore them with a versioned map
 - Use the tools either from Python or from the command line
 
 Installation
@@ -168,7 +170,7 @@ Convert Excel to CSV:
 
 .. code-block:: console
 
-   csvsmith excel2csv input.xlsx
+   csvsmith excel-to-csv input.xlsx
 
 Move files by suffix:
 
@@ -182,6 +184,54 @@ Find matches in a CSV:
 
    csvsmith find-matches input.csv target --ignore-case --ignore-whitespace
 
+Concatenate CSV files:
+
+.. code-block:: console
+
+   csvsmith strict-concat incoming/ -o combined.csv
+
+Concentrate repeated values and restore them later:
+
+.. code-block:: console
+
+   csvsmith concentrate input.csv
+   csvsmith rehydrate input.dense.csv -m input.dense-map.json -o restored.csv
+
+Dense CSV scope and performance
+-------------------------------
+
+The dense CSV workflow is intended for spreadsheet-oriented and medium-sized
+CSV files, roughly in the 100 MB class. Concentration makes two passes over the
+input and keeps value counts in memory, so memory use depends on the number and
+size of distinct values in the selected columns.
+
+For substantially larger datasets, especially gigabyte-scale pipelines, a
+flat CSV plus a separate JSON map is usually the wrong interchange format.
+Consider a binary columnar format such as Apache Parquet instead.
+
+Dense CSV replaces repeated values with tokens containing a 64-character
+SHA-256 digest. The complete token includes the ``csvsmith:sha256:`` prefix and
+is therefore 79 ASCII characters. The JSON map also stores each original value.
+
+The workflow can still reduce expensive downstream work even when the files
+become larger. A consumer can process each mapped value once, store the result
+against its token, and apply that result to every repeated occurrence during
+rehydration. The CLI reports this potential repeated-operation reduction among
+mapped cells.
+
+For example, 937 mapped cell occurrences backed by 82 unique map values imply
+855 avoidable repeated operations, or about 91.2%. This is a deduplication
+ratio for mapped work, not a file-compression ratio or a guarantee of total
+pipeline savings.
+
+.. warning::
+
+   Concentration does not guarantee a smaller combined output. Replacing short
+   values such as ``True``, ``M``, or ``NY`` with a 79-character token will
+   increase the CSV size. Select columns containing sufficiently long,
+   repeated values and consider both the concentrated CSV and its JSON map
+   when evaluating storage savings.
+
 Find matches in a CSV
 ---------------------
 
@@ -235,13 +285,20 @@ CSV classification and filtering:
 
 .. code-block:: python
 
-   from csvsmith import CSVClassifier, DropRowsBySubstring, CSVCleaner
+   from csvsmith import CSVClassifier, DropRowsBySubstring
 
 File and conversion helpers:
 
 .. code-block:: python
 
-   from csvsmith import excel_to_csv, move_by_suffix
+   from csvsmith import (
+       concentrate_csv,
+       excel_to_csv,
+       move_by_suffix,
+       rehydrate_csv,
+       save_csv,
+       strict_concat_rows,
+   )
 
 String comparison utilities:
 

{csvsmith-0.7.3 → csvsmith-0.9.0}/README.rst

@@ -41,6 +41,8 @@ Features
 - Convert Excel workbooks to CSV
 - Move files by suffix
 - Find matching values inside CSV files
+- Concatenate CSV files with identical headers
+- Tokenize repeated CSV values and restore them with a versioned map
 - Use the tools either from Python or from the command line
 
 Installation
@@ -122,7 +124,7 @@ Convert Excel to CSV:
 
 .. code-block:: console
 
-   csvsmith excel2csv input.xlsx
+   csvsmith excel-to-csv input.xlsx
 
 Move files by suffix:
 
@@ -136,6 +138,54 @@ Find matches in a CSV:
 
    csvsmith find-matches input.csv target --ignore-case --ignore-whitespace
 
+Concatenate CSV files:
+
+.. code-block:: console
+
+   csvsmith strict-concat incoming/ -o combined.csv
+
+Concentrate repeated values and restore them later:
+
+.. code-block:: console
+
+   csvsmith concentrate input.csv
+   csvsmith rehydrate input.dense.csv -m input.dense-map.json -o restored.csv
+
+Dense CSV scope and performance
+-------------------------------
+
+The dense CSV workflow is intended for spreadsheet-oriented and medium-sized
+CSV files, roughly in the 100 MB class. Concentration makes two passes over the
+input and keeps value counts in memory, so memory use depends on the number and
+size of distinct values in the selected columns.
+
+For substantially larger datasets, especially gigabyte-scale pipelines, a
+flat CSV plus a separate JSON map is usually the wrong interchange format.
+Consider a binary columnar format such as Apache Parquet instead.
+
+Dense CSV replaces repeated values with tokens containing a 64-character
+SHA-256 digest. The complete token includes the ``csvsmith:sha256:`` prefix and
+is therefore 79 ASCII characters. The JSON map also stores each original value.
+
+The workflow can still reduce expensive downstream work even when the files
+become larger. A consumer can process each mapped value once, store the result
+against its token, and apply that result to every repeated occurrence during
+rehydration. The CLI reports this potential repeated-operation reduction among
+mapped cells.
+
+For example, 937 mapped cell occurrences backed by 82 unique map values imply
+855 avoidable repeated operations, or about 91.2%. This is a deduplication
+ratio for mapped work, not a file-compression ratio or a guarantee of total
+pipeline savings.
+
+.. warning::
+
+   Concentration does not guarantee a smaller combined output. Replacing short
+   values such as ``True``, ``M``, or ``NY`` with a 79-character token will
+   increase the CSV size. Select columns containing sufficiently long,
+   repeated values and consider both the concentrated CSV and its JSON map
+   when evaluating storage savings.
+
 Find matches in a CSV
 ---------------------
 
@@ -189,13 +239,20 @@ CSV classification and filtering:
 
 .. code-block:: python
 
-   from csvsmith import CSVClassifier, DropRowsBySubstring, CSVCleaner
+   from csvsmith import CSVClassifier, DropRowsBySubstring
 
 File and conversion helpers:
 
 .. code-block:: python
 
-   from csvsmith import excel_to_csv, move_by_suffix
+   from csvsmith import (
+       concentrate_csv,
+       excel_to_csv,
+       move_by_suffix,
+       rehydrate_csv,
+       save_csv,
+       strict_concat_rows,
+   )
 
 String comparison utilities:
 
@@ -225,4 +282,4 @@ Example:
 License
 -------
 
-See the project license for details.
+See the project license for details.

{csvsmith-0.7.3 → csvsmith-0.9.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "csvsmith"
-version = "0.7.3"
+version = "0.9.0"
 description = "Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers."
 readme = "README.rst"
 requires-python = ">=3.10"
@@ -50,4 +50,12 @@ Documentation = "https://csvsmith.readthedocs.io/en/latest/"
 csvsmith = "csvsmith.cli:main"
 
 [tool.setuptools.packages.find]
-where = ["src"]
+where = ["src"]
+
+[tool.semantic_release]
+allow_zero_version = true
+tag_format = "v{version}"
+version_toml = ["pyproject.toml:project.version"]
+
+[tool.semantic_release.changelog.default_templates]
+changelog_file = "CHANGELOG.rst"

{csvsmith-0.7.3 → csvsmith-0.9.0}/src/csvsmith/__init__.py

@@ -16,6 +16,10 @@ Public API:
 - Relation
 - Result
 - analyze_pair
+- strict_concat_rows
+- save_csv
+- concentrate_csv
+- rehydrate_csv
 
 Compatibility aliases:
 - CSVCleaner
@@ -28,12 +32,19 @@ Submodules:
 - csvsmith.filter_rows
 - csvsmith.excel2csv
 - csvsmith.move_files
+- csvsmith.strict_concat
 - csvsmith.cli (CLI entrypoint)
 """
 
-__version__ = "0.7.3"
+__version__ = "0.8.0"
 
 from .tools.classify import CSVClassifier
+from .tools.dense_csv import (
+    ConcentrateResult,
+    RehydrateResult,
+    concentrate_csv,
+    rehydrate_csv,
+)
 from .tools.excel2csv import excel_to_csv
 from .tools.filter_rows import DropRowsBySubstring
 from .tools.find_matches_in_csv import find_matches_in_csv
@@ -43,6 +54,7 @@ from .tools.row_dedup import (
     find_duplicate_rows,
     dedupe_with_report,
 )
+from .tools.strict_concat import save_csv, strict_concat_rows
 from .utils.clean_numeric import clean_numeric
 from .utils.distance import StringDistance, Relation, Result, analyze_pair
 from .utils.io import (
@@ -68,4 +80,10 @@ __all__ = [
     "Result",
     "analyze_pair",
     "clean_numeric",
+    "strict_concat_rows",
+    "save_csv",
+    "ConcentrateResult",
+    "RehydrateResult",
+    "concentrate_csv",
+    "rehydrate_csv",
 ]

{csvsmith-0.7.3 → csvsmith-0.9.0}/src/csvsmith/cli.py

@@ -8,6 +8,7 @@ from typing import Optional, Sequence
 
 from . import __version__
 from .tools.classify import CSVClassifier
+from .tools.dense_csv import concentrate_csv, rehydrate_csv
 from .tools.excel2csv import excel_to_csv
 from .utils.clean_numeric import clean_currency_numeric, clean_numeric
 from .tools.filter_rows import DropRowsBySubstring
@@ -16,6 +17,7 @@ from .tools.row_dedup import (
     dedupe_with_report,
     find_duplicate_rows,
 )
+from .tools.strict_concat import save_csv, strict_concat_rows
 from .tools.find_matches_in_csv import find_matches_in_csv
 from .utils.distance import analyze_pair
 from .utils.io import read_csv_rows, write_csv_rows
@@ -180,6 +182,87 @@ def cmd_find_matches(args: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_strict_concat(args: argparse.Namespace) -> int:
+    input_dir = Path(args.input_dir)
+    if not input_dir.is_dir():
+        print(f"Error: input directory not found: {input_dir}", file=sys.stderr)
+        return 1
+
+    try:
+        rows = strict_concat_rows(input_dir)
+        save_csv(rows, args.output)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    print(f"Wrote concatenated CSV to: {args.output}")
+    return 0
+
+
+def cmd_concentrate(args: argparse.Namespace) -> int:
+    input_path = Path(args.input)
+    output_path = (
+        Path(args.output)
+        if args.output
+        else input_path.with_name(f"{input_path.stem}.dense.csv")
+    )
+    map_path = (
+        Path(args.map)
+        if args.map
+        else input_path.with_name(f"{input_path.stem}.dense-map.json")
+    )
+    columns = (
+        [column.strip() for column in args.columns.split(",") if column.strip()]
+        if args.columns
+        else None
+    )
+    try:
+        result = concentrate_csv(
+            input_path,
+            output_path,
+            map_path,
+            columns=columns,
+            min_occurrences=args.min_occurrences,
+        )
+    except (OSError, ValueError, csv.Error, json.JSONDecodeError) as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    print(
+        f"Wrote concentrated CSV to: {result.output_csv_path} "
+        f"({result.transformed_cell_count} cells, "
+        f"{result.mapped_value_count} mapped values)"
+    )
+    if result.transformed_cell_count:
+        avoided_operations = (
+            result.transformed_cell_count - result.mapped_value_count
+        )
+        reduction_percentage = (
+            avoided_operations / result.transformed_cell_count * 100
+        )
+        print(
+            "Potential repeated-operation reduction for mapped values: "
+            f"{avoided_operations}/{result.transformed_cell_count} "
+            f"({reduction_percentage:.1f}%)"
+        )
+    print(f"Wrote dense CSV map to: {result.output_map_path}")
+    return 0
+
+
+def cmd_rehydrate(args: argparse.Namespace) -> int:
+    try:
+        result = rehydrate_csv(args.input, args.map, args.output)
+    except (OSError, ValueError, csv.Error, json.JSONDecodeError) as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    print(
+        f"Wrote rehydrated CSV to: {result.output_csv_path} "
+        f"({result.restored_cell_count} cells restored)"
+    )
+    return 0
+
+
 def _add_find_matches_parser(subparsers) -> None:
     parser = subparsers.add_parser("find-matches", help="Find matches in a CSV file.")
     parser.add_argument("input", help="Input CSV file.")
@@ -190,6 +273,56 @@ def _add_find_matches_parser(subparsers) -> None:
     parser.set_defaults(func=cmd_find_matches)
 
 
+def _add_strict_concat_parser(subparsers) -> None:
+    parser = subparsers.add_parser(
+        "strict-concat",
+        help="Concatenate CSVs in a directory only when all headers match exactly.",
+    )
+    parser.add_argument("input_dir", help="Directory containing CSV files to concatenate.")
+    parser.add_argument("-o", "--output", required=True, help="Output CSV file path.")
+    parser.set_defaults(func=cmd_strict_concat)
+
+
+def _add_concentrate_parser(subparsers) -> None:
+    parser = subparsers.add_parser(
+        "concentrate",
+        help="Replace repeated CSV values with deterministic tokens.",
+    )
+    parser.add_argument("input", help="Input CSV file.")
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output CSV file (default: <input-stem>.dense.csv).",
+    )
+    parser.add_argument(
+        "-m",
+        "--map",
+        help="Output JSON map file (default: <input-stem>.dense-map.json).",
+    )
+    parser.add_argument(
+        "--columns",
+        help="Comma-separated column names to concentrate (default: all columns).",
+    )
+    parser.add_argument(
+        "--min-occurrences",
+        type=int,
+        default=2,
+        help="Minimum repetitions required before replacing a value (default: 2).",
+    )
+    parser.set_defaults(func=cmd_concentrate)
+
+
+def _add_rehydrate_parser(subparsers) -> None:
+    parser = subparsers.add_parser(
+        "rehydrate",
+        help="Restore a concentrated CSV using its JSON map.",
+    )
+    parser.add_argument("input", help="Concentrated CSV file.")
+    parser.add_argument("-m", "--map", required=True, help="Dense CSV JSON map file.")
+    parser.add_argument("-o", "--output", required=True, help="Output CSV file.")
+    parser.set_defaults(func=cmd_rehydrate)
+
+
 def _add_row_duplicates_parser(subparsers) -> None:
     parser = subparsers.add_parser("row-duplicates", help="Find duplicate rows in a CSV.")
     parser.add_argument("input", help="Input CSV file.")
@@ -317,6 +450,9 @@ def build_parser() -> argparse.ArgumentParser:
 
     _add_row_duplicates_parser(subparsers)
     _add_find_matches_parser(subparsers)
+    _add_strict_concat_parser(subparsers)
+    _add_concentrate_parser(subparsers)
+    _add_rehydrate_parser(subparsers)
     _add_dedupe_parser(subparsers)
     _add_classify_parser(subparsers)
     _add_move_files_parser(subparsers)
@@ -342,4 +478,3 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
 
 if __name__ == "__main__":
     sys.exit(main())
-