csvsmith 0.1.0__py3-none-any.whl

csvsmith/__init__.py

@@ -0,0 +1,21 @@
+"""
+csvsmith: small, focused CSV utilities.
+
+Current submodules:
+- csvsmith.duplicates
+- csvsmith.cli (CLI entrypoint)
+"""
+
+from .duplicates import (
+    count_duplicates_sorted,
+    add_row_digest,
+    find_duplicate_rows,
+    dedupe_with_report,
+)
+
+__all__ = [
+    "count_duplicates_sorted",
+    "add_row_digest",
+    "find_duplicate_rows",
+    "dedupe_with_report",
+]

csvsmith/cli.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""
+csvsmith CLI
+
+Duplicate-related helpers on CSV files.
+
+Subcommands:
+  - row-duplicates: show only rows that are duplicated
+  - dedupe: drop duplicates and write both deduped CSV and a report CSV
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Sequence, Optional, List
+
+import pandas as pd
+
+from .duplicates import find_duplicate_rows, dedupe_with_report
+
+
+def _parse_cols(cols: Optional[Sequence[str]]) -> Optional[List[str]]:
+    """
+    Normalize column list arguments from CLI.
+
+    We accept:
+      --subset col1 col2 col3
+      --exclude colA colB
+    or omit entirely.
+    """
+    if cols is None:
+        return None
+    if len(cols) == 0:
+        return None
+    return list(cols)
+
+
+def _effective_subset(
+    df: pd.DataFrame,
+    subset: Optional[Sequence[str]],
+    exclude: Optional[Sequence[str]],
+) -> Optional[List[str]]:
+    """
+    Compute the effective subset of columns to use for duplicate detection,
+    given a requested subset and/or exclude list.
+
+    Logic:
+      - if subset is None: start from all columns
+      - else: start from subset
+      - then remove any columns in exclude
+    """
+    if subset is None:
+        cols = list(df.columns)
+    else:
+        cols = list(subset)
+
+    if exclude:
+        exclude_set = set(exclude)
+        cols = [c for c in cols if c not in exclude_set]
+
+    if not cols:
+        return None
+
+    return cols
+
+
+def cmd_row_duplicates(args: argparse.Namespace) -> int:
+    input_path = Path(args.input)
+    if not input_path.is_file():
+        print(f"Error: input file not found: {input_path}", file=sys.stderr)
+        return 1
+
+    df = pd.read_csv(input_path)
+    subset = _parse_cols(args.subset)
+    exclude = _parse_cols(args.exclude)
+
+    eff_subset = _effective_subset(df, subset=subset, exclude=exclude)
+
+    dup_df = find_duplicate_rows(df, subset=eff_subset)
+
+    if args.output:
+        output_path = Path(args.output)
+        dup_df.to_csv(output_path, index=False)
+    else:
+        dup_df.to_csv(sys.stdout, index=False)
+
+    return 0
+
+
+def cmd_dedupe(args: argparse.Namespace) -> int:
+    input_path = Path(args.input)
+    if not input_path.is_file():
+        print(f"Error: input file not found: {input_path}", file=sys.stderr)
+        return 1
+
+    df = pd.read_csv(input_path)
+    subset = _parse_cols(args.subset)
+    exclude = _parse_cols(args.exclude)
+
+    deduped, report = dedupe_with_report(
+        df,
+        subset=subset,
+        exclude=exclude,
+        keep=args.keep,
+        digest_col=args.digest_col,
+    )
+
+    deduped_path = Path(args.deduped)
+    report_path = Path(args.report)
+
+    deduped_path.parent.mkdir(parents=True, exist_ok=True)
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+
+    deduped.to_csv(deduped_path, index=False)
+    report.to_csv(report_path, index=False)
+
+    print(f"Wrote deduped CSV to: {deduped_path}")
+    print(f"Wrote duplicate report to: {report_path}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="csvsmith",
+        description="Small CSV utilities (duplicates-focused, first iteration).",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # row-duplicates
+    p_row = subparsers.add_parser(
+        "row-duplicates",
+        help="Print only rows that have duplicates.",
+    )
+    p_row.add_argument("input", help="Input CSV file.")
+    p_row.add_argument(
+        "--subset",
+        nargs="*",
+        help="Column names to consider when detecting duplicates. "
+             "If omitted, all columns are used.",
+    )
+    p_row.add_argument(
+        "--exclude",
+        nargs="*",
+        help="Column names to exclude from duplicate detection. "
+             "Useful for ID columns, timestamps, etc.",
+    )
+    p_row.add_argument(
+        "-o",
+        "--output",
+        help="Output CSV file for duplicate rows. If omitted, writes to stdout.",
+    )
+    p_row.set_defaults(func=cmd_row_duplicates)
+
+    # dedupe
+    p_dedupe = subparsers.add_parser(
+        "dedupe",
+        help="Drop duplicates and generate a duplicate-report CSV.",
+    )
+    p_dedupe.add_argument("input", help="Input CSV file.")
+    p_dedupe.add_argument(
+        "--subset",
+        nargs="*",
+        help="Column names to consider when detecting duplicates. "
+             "If omitted, all columns are used.",
+    )
+    p_dedupe.add_argument(
+        "--exclude",
+        nargs="*",
+        help="Column names to exclude from duplicate detection. "
+             "Useful for ID columns, timestamps, etc.",
+    )
+    p_dedupe.add_argument(
+        "--keep",
+        choices=["first", "last", "False"],
+        default="first",
+        help='Which duplicate to keep (same as pandas.drop_duplicates). '
+             '"False" = drop all occurrences. Default: "first".',
+    )
+    p_dedupe.add_argument(
+        "--digest-col",
+        default="row_digest",
+        help='Name of digest column used in the report. Default: "row_digest".',
+    )
+    p_dedupe.add_argument(
+        "--deduped",
+        required=True,
+        help="Path to write the deduplicated CSV.",
+    )
+    p_dedupe.add_argument(
+        "--report",
+        required=True,
+        help="Path to write the duplicate-report CSV.",
+    )
+    p_dedupe.set_defaults(func=cmd_dedupe)
+
+    return parser
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        func = args.func
+    except AttributeError:
+        parser.print_help()
+        return 1
+
+    return func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

csvsmith/duplicates.py

@@ -0,0 +1,221 @@
+"""
+Duplicate-related helpers for csvsmith.
+
+Includes:
+- count_duplicates_sorted: generic iterable duplicate counter
+- add_row_digest: add a SHA-256 digest per row to a DataFrame
+- find_duplicate_rows: return only rows that have duplicates
+- dedupe_with_report: drop duplicates and report duplicate groups
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from hashlib import sha256
+from typing import Iterable, List, Tuple, Hashable, Sequence, Optional
+
+import pandas as pd
+
+
+def count_duplicates_sorted(
+    items: Iterable[Hashable],
+    threshold: int = 2,
+    reverse: bool = True,
+) -> List[Tuple[Hashable, int]]:
+    """
+    Count occurrences in an iterable and return items whose frequency
+    is at or above `threshold`, sorted by count.
+
+    Args:
+        items:
+            Any iterable of hashable items (str, int, tuple, etc.)
+        threshold:
+            Minimum count to include in output (default: 2).
+        reverse:
+            Whether to sort in descending order (default: True).
+
+    Returns:
+        A list of (item, count) tuples sorted by frequency.
+    """
+    counter = Counter(items)
+    duplicates = [(k, v) for k, v in counter.items() if v >= threshold]
+    duplicates.sort(key=lambda x: x[1], reverse=reverse)
+    return duplicates
+
+
+def add_row_digest(
+    df: pd.DataFrame,
+    *,
+    subset: Optional[Sequence[Hashable]] = None,
+    exclude: Optional[Sequence[Hashable]] = None,
+    colname: str = "row_digest",
+    inplace: bool = False,
+) -> pd.DataFrame:
+    """
+    Add a SHA-256 digest for each row of a DataFrame.
+
+    Args:
+        df:
+            Input DataFrame.
+        subset:
+            Optional list/sequence of column labels to use for the digest.
+            If None, all columns are used.
+        exclude:
+            Optional list/sequence of column labels to exclude from the digest,
+            after `subset` is applied. This is useful for excluding ID columns,
+            timestamps, etc.
+        colname:
+            Name of the digest column to add (default: "row_digest").
+        inplace:
+            If True, modify the original DataFrame and return it.
+            If False (default), return a copy.
+
+    Returns:
+        DataFrame with an extra column containing hex digests.
+
+    Notes:
+        We use the ASCII "Unit Separator" (0x1F, "\\x1f") as the internal
+        delimiter when concatenating row values before hashing. It is a
+        non-printable control character that almost never appears in normal
+        CSV data, which helps avoid accidental collisions like:
+
+            ["ab", "c"] vs ["a", "bc"]
+
+        Credo #5 of csvsmith: "Love thy \\x1f."
+    """
+    # Determine columns to include
+    if subset is None:
+        cols = list(df.columns)
+    else:
+        cols = list(subset)
+
+    if exclude:
+        exclude_set = set(exclude)
+        cols = [c for c in cols if c not in exclude_set]
+
+    # Convert to string, fill NaNs, and join with a non-printable separator
+    concatted = df[cols].astype("string").fillna("").agg("\x1f".join, axis=1)
+    digests = concatted.map(lambda s: sha256(s.encode("utf-8")).hexdigest())
+
+    if inplace:
+        df[colname] = digests
+        return df
+    else:
+        df2 = df.copy()
+        df2[colname] = digests
+        return df2
+
+
+def find_duplicate_rows(
+    df: pd.DataFrame,
+    *,
+    subset: Optional[Sequence[Hashable]] = None,
+) -> pd.DataFrame:
+    """
+    Return only rows that participate in duplicates.
+
+    This is a convenience wrapper around `df.duplicated(keep=False)`.
+
+    Args:
+        df:
+            Input DataFrame.
+        subset:
+            Columns to consider when identifying duplicates. If None,
+            all columns are used.
+
+    Returns:
+        A DataFrame containing only rows that have at least one duplicate,
+        preserving the original order and index.
+    """
+    mask = df.duplicated(subset=subset, keep=False)
+    return df[mask]
+
+
+def dedupe_with_report(
+    df: pd.DataFrame,
+    *,
+    subset: Optional[Sequence[Hashable]] = None,
+    exclude: Optional[Sequence[Hashable]] = None,
+    keep: str = "first",
+    digest_col: str = "row_digest",
+) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    """
+    Drop duplicate rows *and* return a report of what was duplicated.
+
+    Args:
+        df:
+            Input DataFrame.
+        subset:
+            Columns to consider when identifying duplicates. If None,
+            all columns are used.
+        exclude:
+            Columns to exclude from the duplicate check and digest,
+            after `subset` is applied. Useful for ID columns, timestamps, etc.
+        keep:
+            Which duplicate to keep. Same semantics as pandas:
+            "first", "last", or "False" (string). Default: "first".
+        digest_col:
+            Name of the temporary digest column used for grouping in the
+            report (default: "row_digest").
+
+    Returns:
+        (df_deduped, report)
+
+        df_deduped:
+            DataFrame with duplicates dropped according to the effective
+            subset (subset minus exclude) and `keep`.
+
+        report:
+            DataFrame with one row per duplicate group, columns:
+              - digest_col: the SHA-256 row digest
+              - count: number of rows in this group
+              - indices: list of original DataFrame indices in this group
+
+            Only groups with count > 1 are included, sorted by `count`
+            descending.
+    """
+    # Determine effective subset for both hashing and drop_duplicates
+    if subset is None:
+        cols = list(df.columns)
+    else:
+        cols = list(subset)
+
+    if exclude:
+        exclude_set = set(exclude)
+        cols = [c for c in cols if c not in exclude_set]
+
+    subset_for_dupes: Optional[Sequence[Hashable]]
+    if cols:
+        subset_for_dupes = cols
+    else:
+        subset_for_dupes = None
+
+    # Work on a copy with a digest column, using the effective subset
+    work = add_row_digest(
+        df,
+        subset=subset_for_dupes,
+        exclude=None,
+        colname=digest_col,
+        inplace=False,
+    )
+
+    grouped = work.groupby(digest_col, dropna=False)
+
+    sizes = grouped.size().rename("count")
+    indices_map = {k: list(v) for k, v in grouped.indices.items()}
+    indices = pd.Series(indices_map, name="indices")
+
+    report = (
+        pd.concat([sizes, indices], axis=1)
+        .reset_index()
+        .rename(columns={"index": digest_col})
+    )
+
+    report = (
+        report[report["count"] > 1]
+        .sort_values("count", ascending=False)
+        .reset_index(drop=True)
+    )
+
+    df_deduped = df.drop_duplicates(subset=subset_for_dupes, keep=keep)
+    return df_deduped, report

csvsmith-0.1.0.dist-info/METADATA

@@ -0,0 +1,198 @@
+Metadata-Version: 2.4
+Name: csvsmith
+Version: 0.1.0
+Summary: Small CSV utilities: duplicates, row digests, and CLI helpers.
+Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
+License: MIT License
+        
+        Copyright (c) 2025 YOUR NAME
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the “Software”), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/yeiichi/csvsmith
+Project-URL: Repository, https://github.com/yeiichi/csvsmith
+Keywords: csv,pandas,duplicates,data-cleaning
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=2.0
+Dynamic: license-file
+
+# csvsmith
+
+[![PyPI version](https://img.shields.io/pypi/v/csvsmith.svg)](https://pypi.org/project/csvsmith/)
+![Python versions](https://img.shields.io/pypi/pyversions/csvsmith.svg)
+[![License](https://img.shields.io/pypi/l/csvsmith.svg)](https://pypi.org/project/csvsmith/)
+
+`csvsmith` is a small collection of CSV utilities.
+
+---
+
+Current focus:
+
+- Duplicate value counting (`count_duplicates_sorted`)
+- Row-level digest creation (`add_row_digest`)
+- Duplicate-row detection (`find_duplicate_rows`)
+- Deduplication with full duplicate report (`dedupe_with_report`)
+- Command-line interface (CLI) for quick operations
+
+---
+
+## Installation
+
+From PyPI (future):
+
+```bash
+pip install csvsmith
+```
+
+For local development:
+
+```bash
+git clone https://github.com/YOUR_GITHUB_USERNAME/csvsmith.git
+cd csvsmith
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .[dev]
+```
+
+---
+
+## Python API Usage
+
+### Count duplicate values
+
+```python
+from csvsmith import count_duplicates_sorted
+
+items = ["a", "b", "a", "c", "a", "b"]
+print(count_duplicates_sorted(items))
+# [('a', 3), ('b', 2)]
+```
+
+### Find duplicate rows in a DataFrame
+
+```python
+import pandas as pd
+from csvsmith import find_duplicate_rows
+
+df = pd.read_csv("input.csv")
+dup_rows = find_duplicate_rows(df)
+print(dup_rows)
+```
+
+### Deduplicate with report
+
+```python
+import pandas as pd
+from csvsmith import dedupe_with_report
+
+df = pd.read_csv("input.csv")
+
+# Use all columns
+deduped, report = dedupe_with_report(df)
+deduped.to_csv("deduped.csv", index=False)
+report.to_csv("duplicate_report.csv", index=False)
+
+# Use all columns except an ID column
+deduped_no_id, report_no_id = dedupe_with_report(df, exclude=["id"])
+```
+
+---
+
+## CLI Usage
+
+`csvsmith` includes a small command-line interface for duplicate detection
+and CSV deduplication.
+
+### Show duplicate rows
+
+```bash
+csvsmith row-duplicates input.csv
+```
+
+Save only duplicate rows to a file:
+
+```bash
+csvsmith row-duplicates input.csv -o duplicates_only.csv
+```
+
+Use only a subset of columns to determine duplicates:
+
+```bash
+csvsmith row-duplicates input.csv --subset col1 col2 -o dup_rows_subset.csv
+```
+
+Exclude ID column(s) when looking for duplicates:
+
+```bash
+csvsmith row-duplicates input.csv --exclude id -o dup_rows_no_id.csv
+```
+
+### Deduplicate and generate a duplicate report
+
+```bash
+csvsmith dedupe input.csv   --deduped deduped.csv   --report duplicate_report.csv
+```
+
+### Deduplicate using selected columns
+
+```bash
+csvsmith dedupe input.csv   --subset col1 col2   --deduped deduped_subset.csv   --report duplicate_report_subset.csv
+```
+
+### Remove *all* occurrences of duplicated rows
+
+```bash
+csvsmith dedupe input.csv   --subset col1   --keep False   --deduped deduped_no_dups.csv   --report duplicate_report_col1.csv
+```
+
+Exclude “id” from duplicate logic:
+
+```bash
+csvsmith dedupe input.csv   --exclude id   --deduped deduped_no_id.csv   --report duplicate_report_no_id.csv
+```
+
+---
+
+## Philosophy (“csvsmith Manifesto”)
+
+1. CSVs deserve tools that are simple, predictable, and transparent.
+2. A row has meaning only when its identity is stable and hashable.
+3. Collisions are sin; determinism is virtue.
+4. Let no delimiter sow ambiguity among fields.
+5. **Love thy `\x1f`.**  
+   The unseen separator, the quiet guardian of clean hashes.  
+   Chosen not for aesthetics, but for truth.
+6. The pipeline should be silent unless something is wrong.
+7. Your data deserves respect — and your tools should help you give it.
+
+For more, see `MANIFESTO.md`.
+
+---
+
+## License
+
+MIT License.

csvsmith-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+csvsmith/__init__.py,sha256=t0vhxiJFalD-oLnNt4ud06Qcv4lBtubM8-kNKwhQT3c,371
+csvsmith/cli.py,sha256=k46U-cGjydclQ4vZ1_tsfV6xRrVgWvh_KHPmNQCDi5Y,5692
+csvsmith/duplicates.py,sha256=PzYzb6KnEf2E6HchXJO4b4PT8URV1FUPIhI0GbupeQ4,6655
+csvsmith-0.1.0.dist-info/licenses/LICENSE,sha256=A8UVDOPo1679c7fl5k7o7t5m7g-5TqpHo8RPzDbAUHs,1074
+csvsmith-0.1.0.dist-info/METADATA,sha256=pMNyw-NuYq-f18Beb_rYw7UV8_Ee-pih3clrWGSu44k,5546
+csvsmith-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+csvsmith-0.1.0.dist-info/entry_points.txt,sha256=9Vmom48a5jBhWflrspYBPHD_HCYP5nzzxcHCPz7tagI,47
+csvsmith-0.1.0.dist-info/top_level.txt,sha256=BF99nykjiyBPqYqvbTuS8WS-zaVvQW-uD7uVHNTw_3Y,9
+csvsmith-0.1.0.dist-info/RECORD,,

csvsmith-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

csvsmith-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+csvsmith = csvsmith.cli:main

csvsmith-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 YOUR NAME
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

csvsmith-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+csvsmith