kryptorious-csvclean 1.0.0__py3-none-any.whl

csvclean/__init__.py

@@ -0,0 +1,2 @@
+"""CSVClean — CSV file cleaner and validator."""
+__version__ = "1.0.0"

csvclean/cleaner.py

@@ -0,0 +1,260 @@
+"""CSVClean — CSV analysis and cleaning engine."""
+
+import csv
+import io
+import os
+from collections import Counter
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+
+
+def analyze_csv(path: str) -> Dict[str, Any]:
+    """Analyze a CSV file and return diagnostic info.
+
+    Detects: encoding, delimiter, quote char, header issues,
+    empty rows, inconsistent column counts, type patterns, duplicates.
+    """
+    filepath = Path(path)
+    if not filepath.exists():
+        raise FileNotFoundError(f"File not found: {path}")
+
+    size = filepath.stat().st_size
+    raw = filepath.read_bytes()
+
+    # Detect encoding
+    encoding = _detect_encoding(raw)
+
+    # Read with detected encoding
+    text = raw.decode(encoding, errors="replace")
+    lines = text.splitlines()
+
+    if not lines:
+        return {"error": "File is empty", "rows": 0, "columns": 0}
+
+    # Detect delimiter
+    delimiter = _detect_delimiter(lines)
+
+    # Parse with csv module
+    reader = csv.reader(io.StringIO(text), delimiter=delimiter)
+    rows = list(reader)
+
+    if not rows:
+        return {"error": "No data rows found", "rows": 0, "columns": 0}
+
+    header = rows[0]
+    data_rows = rows[1:]
+
+    # Build result
+    result = {
+        "path": str(filepath),
+        "size_bytes": size,
+        "size_human": _format_size(size),
+        "encoding": encoding,
+        "delimiter": delimiter if delimiter != "," else "comma",
+        "total_rows": len(data_rows),
+        "total_columns": len(header),
+        "headers": header,
+        "issues": [],
+    }
+
+    # Check for empty headers
+    empty_headers = [i for i, h in enumerate(header) if not h.strip()]
+    if empty_headers:
+        result["issues"].append({
+            "severity": "error",
+            "message": f"Empty header at column(s): {', '.join(str(i+1) for i in empty_headers)}"
+        })
+
+    # Check for duplicate headers
+    seen = set()
+    dupes = []
+    for h in header:
+        clean = h.strip().lower()
+        if clean in seen:
+            dupes.append(h)
+        seen.add(clean)
+    if dupes:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"Duplicate headers: {', '.join(dupes)}"
+        })
+
+    # Check rows
+    empty_rows = 0
+    short_rows = 0
+    long_rows = 0
+    type_patterns = {i: Counter() for i in range(len(header))}
+
+    for i, row in enumerate(data_rows, start=2):
+        # Empty row
+        if all(not cell.strip() for cell in row):
+            empty_rows += 1
+            continue
+
+        # Column count mismatch
+        if len(row) < len(header):
+            short_rows += 1
+        elif len(row) > len(header):
+            long_rows += 1
+
+        # Type inference per column
+        for j, cell in enumerate(row):
+            if j >= len(header):
+                continue
+            val = cell.strip()
+            if not val:
+                type_patterns[j]["empty"] += 1
+            elif val.replace(".", "").replace("-", "").isdigit():
+                type_patterns[j]["int" if "." not in val and "-" not in val else "float"] += 1
+            elif val.lower() in ("true", "false", "yes", "no"):
+                type_patterns[j]["bool"] += 1
+            else:
+                type_patterns[j]["string"] += 1
+
+    result["empty_rows"] = empty_rows
+    result["column_count_issues"] = short_rows + long_rows
+
+    if empty_rows:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"{empty_rows} empty row(s)"
+        })
+    if short_rows:
+        result["issues"].append({
+            "severity": "error",
+            "message": f"{short_rows} row(s) with fewer columns than header"
+        })
+    if long_rows:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"{long_rows} row(s) with more columns than header"
+        })
+
+    # Infer types
+    result["column_types"] = {}
+    for j, counter in type_patterns.items():
+        if j < len(header):
+            col_name = header[j].strip() or f"column_{j}"
+            dominant = counter.most_common(1)[0][0] if counter else "empty"
+            result["column_types"][col_name] = {
+                "dominant_type": dominant,
+                "distribution": dict(counter.most_common()),
+                "empty_count": counter.get("empty", 0),
+                "empty_pct": round(counter.get("empty", 0) / max(len(data_rows), 1) * 100, 1),
+            }
+
+    # Duplicate detection
+    seen_rows = set()
+    duplicate_count = 0
+    for row in data_rows:
+        row_key = tuple(cell.strip() for cell in row)
+        if row_key in seen_rows:
+            duplicate_count += 1
+        seen_rows.add(row_key)
+
+    result["duplicate_rows"] = duplicate_count
+    if duplicate_count:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"{duplicate_count} duplicate row(s)"
+        })
+
+    # Health score
+    max_score = 100
+    penalties = 0
+    for issue in result["issues"]:
+        if issue["severity"] == "error":
+            penalties += 15
+        else:
+            penalties += 5
+    penalties += duplicate_count * 2
+    result["health_score"] = max(0, max_score - penalties)
+
+    return result
+
+
+def clean_csv(path: str, output: str, fix_encoding: bool = True,
+              remove_empty: bool = True, normalize_delimiter: bool = True) -> Dict:
+    """Clean a CSV file (premium feature stub)."""
+    result = analyze_csv(path)
+
+    # This is a stub — full cleaning is premium
+    issues = []
+    if fix_encoding:
+        issues.append("Would fix encoding consistency")
+    if remove_empty:
+        issues.append(f"Would remove {result.get('empty_rows', 0)} empty rows")
+    if normalize_delimiter:
+        issues.append("Would normalize delimiter")
+
+    return {
+        "source": path,
+        "output": output,
+        "would_fix": issues,
+        "premium_feature": True,
+    }
+
+
+def _detect_encoding(raw: bytes) -> str:
+    """Detect file encoding from BOM or content."""
+    if raw.startswith(b"\xff\xfe"):
+        return "utf-16-le"
+    if raw.startswith(b"\xfe\xff"):
+        return "utf-16-be"
+    if raw.startswith(b"\xef\xbb\xbf"):
+        return "utf-8-sig"
+    try:
+        raw.decode("utf-8")
+        return "utf-8"
+    except UnicodeDecodeError:
+        try:
+            raw.decode("latin-1")
+            return "latin-1"
+        except UnicodeDecodeError:
+            return "utf-8"
+
+
+def _detect_delimiter(lines: List[str]) -> str:
+    """Detect CSV delimiter from content."""
+    if not lines:
+        return ","
+
+    # Check first 5 non-empty lines
+    sample = [l for l in lines[:20] if l.strip()][:5]
+    if not sample:
+        return ","
+
+    candidates = [",", "\t", ";", "|"]
+    header_line = sample[0]
+
+    # Count occurrences
+    counts = {}
+    for delim in candidates:
+        counts[delim] = header_line.count(delim)
+
+    # Find the delimiter that appears consistently across lines
+    best_delim = ","
+    best_score = 0
+
+    for delim in candidates:
+        line_counts = [line.count(delim) for line in sample]
+        if len(set(line_counts)) <= 1 and line_counts[0] > 0:
+            # Consistent count across lines
+            score = line_counts[0] * 10 + (5 if delim != "," else 0)
+            if score > best_score:
+                best_score = score
+                best_delim = delim
+        elif line_counts[0] > counts.get(best_delim, 0):
+            best_delim = delim
+
+    return best_delim
+
+
+def _format_size(size: int) -> str:
+    if size < 1024:
+        return f"{size} B"
+    elif size < 1024 * 1024:
+        return f"{size / 1024:.1f} KB"
+    elif size < 1024 * 1024 * 1024:
+        return f"{size / (1024 * 1024):.1f} MB"
+    return f"{size / (1024 * 1024 * 1024):.2f} GB"

csvclean/cli.py

@@ -0,0 +1,174 @@
+"""CSVClean CLI — Clean and validate CSV files."""
+
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version="1.0.0", prog_name="csvclean")
+def main():
+    """CSVClean — Find and fix CSV file problems.
+
+    Detect encoding issues, inconsistent delimiters, missing headers,
+    duplicate rows, type mismatches — all in one command.
+    """
+    pass
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
+def check(path, output_json):
+    """Analyze a CSV file and report all issues.
+
+    \b
+    Examples:
+        csvclean check data.csv
+        csvclean check data.csv --json
+    """
+    from .cleaner import analyze_csv
+
+    console.print()
+    console.print(Panel(
+        f"[bold]CSVClean Check[/bold] — [cyan]{path}[/cyan]",
+        border_style="blue"
+    ))
+
+    try:
+        result = analyze_csv(path)
+    except Exception as e:
+        console.print(f"[red]Error:[/red] {e}")
+        return
+
+    if output_json:
+        import json as _json
+        console.print(_json.dumps(result, indent=2, default=str))
+        return
+
+    if "error" in result:
+        console.print(f"[red]{result['error']}[/red]")
+        return
+
+    # Overview
+    score = result["health_score"]
+    color = "green" if score >= 80 else "yellow" if score >= 50 else "red"
+    console.print(f"\n[bold {color}]Health: {score}/100[/bold {color}]")
+    console.print(f"  Size: {result['size_human']}")
+    console.print(f"  Encoding: [cyan]{result['encoding']}[/cyan]")
+    console.print(f"  Delimiter: [cyan]{result['delimiter']}[/cyan]")
+    console.print(f"  Rows: [bold]{result['total_rows']:,}[/bold] × {result['total_columns']} columns")
+    console.print(f"  Empty rows: {result.get('empty_rows', 0)}")
+    console.print(f"  Duplicates: {result.get('duplicate_rows', 0)}")
+
+    # Issues
+    if result["issues"]:
+        console.print()
+        console.print("[bold]Issues found:[/bold]")
+        for issue in result["issues"]:
+            icon = "[red]✗[/red]" if issue["severity"] == "error" else "[yellow]![/yellow]"
+            console.print(f"  {icon} {issue['message']}")
+    else:
+        console.print()
+        console.print("[green]No issues found. File is clean![/green]")
+
+    # Column types
+    if result.get("column_types"):
+        console.print()
+        console.print("[bold]Column Analysis:[/bold]")
+        col_table = Table()
+        col_table.add_column("Column")
+        col_table.add_column("Type", style="cyan")
+        col_table.add_column("Empty %", justify="right")
+        col_table.add_column("Distribution")
+
+        for col_name, info in result["column_types"].items():
+            dist = ", ".join(f"{t}:{c}" for t, c in info["distribution"].items() if t != "empty")
+            col_table.add_row(
+                col_name[:25],
+                info["dominant_type"],
+                f"{info['empty_pct']}%",
+                dist[:40]
+            )
+
+        console.print(col_table)
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.argument("output")
+@click.option("--remove-empty/--keep-empty", default=True, help="Remove empty rows")
+@click.option("--normalize/--preserve", default=True, help="Normalize delimiter to comma")
+def clean(path, output, remove_empty, normalize):
+    """Clean a CSV file and write fixed output (premium).
+
+    \b
+    Example:
+        csvclean clean messy.csv clean.csv
+    """
+    console.print()
+    console.print("[yellow]CSV cleaning is a premium feature.[/yellow]")
+    console.print("Upgrade at https://kryptorious.gumroad.com/l/jbvet")
+    console.print()
+
+    from .cleaner import clean_csv
+    result = clean_csv(path, output, remove_empty=remove_empty, normalize_delimiter=normalize)
+    for fix in result["would_fix"]:
+        console.print(f"  [dim]• {fix}[/dim]")
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+def dedupe(path):
+    """Remove duplicate rows (premium).
+
+    \b
+    Example:
+        csvclean dedupe data.csv
+    """
+    console.print()
+    console.print("[yellow]Deduplication is a premium feature.[/yellow]")
+    console.print("Upgrade at https://kryptorious.gumroad.com/l/jbvet")
+    console.print()
+
+    from .cleaner import analyze_csv
+    result = analyze_csv(path)
+    dups = result.get("duplicate_rows", 0)
+    console.print(f"Would remove [bold]{dups}[/bold] duplicate row(s)")
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+def stats(path):
+    """Show column statistics.
+
+    \b
+    Example:
+        csvclean stats data.csv
+    """
+    from .cleaner import analyze_csv
+
+    result = analyze_csv(path)
+    console.print()
+    console.print(Panel(
+        f"[bold]CSVClean Stats[/bold] — [cyan]{path}[/cyan]",
+        border_style="blue"
+    ))
+
+    ct = result.get("column_types", {})
+    for col_name, info in ct.items():
+        console.print(f"\n[bold cyan]{col_name}[/bold cyan]")
+        console.print(f"  Type: {info['dominant_type']}")
+        console.print(f"  Empty: {info['empty_count']} ({info['empty_pct']}%)")
+        dist = info.get("distribution", {})
+        if len(dist) > 1:
+            console.print(f"  Distribution: {dist}")
+
+
+if __name__ == "__main__":
+    main()

kryptorious_csvclean-1.0.0.dist-info/METADATA

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: kryptorious-csvclean
+Version: 1.0.0
+Summary: CSV file cleaner and validator — detect encoding, inconsistent delimiters, missing headers, duplicates, type mismatches.
+Author: Kryptorious Quantum Biosciences, Inc.
+License: MIT
+Project-URL: Homepage, https://devflow.sh
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.0
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"

kryptorious_csvclean-1.0.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+csvclean/__init__.py,sha256=YecLNa4q0BWGMG6D_ozVn8_XBFwhcj75dNXptxXSPV4,73
+csvclean/cleaner.py,sha256=0YxIjH7DO5Ve96liakep8WHqOX-6mLJzc1qVJ3QdkPk,7742
+csvclean/cli.py,sha256=laCKJItuWXkA-lHfddT_iLsu2kkvtHjESFJqAFyWgro,5395
+kryptorious_csvclean-1.0.0.dist-info/METADATA,sha256=7hnxsNzofu732Z1oRCiPPPEoTmxbe-h9vPaqde5Q950,488
+kryptorious_csvclean-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+kryptorious_csvclean-1.0.0.dist-info/entry_points.txt,sha256=8rVSFSPXR9AIbo2mmnm9BtdEltwQy953PiMDI-5R1uQ,47
+kryptorious_csvclean-1.0.0.dist-info/top_level.txt,sha256=YhFNS9J01BBcnWCmEJdv_grCykCt_GUBC1NK8MeYyro,9
+kryptorious_csvclean-1.0.0.dist-info/RECORD,,

kryptorious_csvclean-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

kryptorious_csvclean-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+csvclean = csvclean.cli:main

kryptorious_csvclean-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+csvclean