kryptorious-csvclean 1.1.0__tar.gz

kryptorious_csvclean-1.1.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: kryptorious-csvclean
+Version: 1.1.0
+Summary: CSV file cleaner and validator — detect encoding, inconsistent delimiters, missing headers, duplicates, type mismatches.
+Author: Kryptorious Quantum Biosciences, Inc.
+License: MIT
+Project-URL: Homepage, https://devflow.sh
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.0
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"

kryptorious_csvclean-1.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kryptorious-csvclean"
+version = "1.1.0"
+description = "CSV file cleaner and validator — detect encoding, inconsistent delimiters, missing headers, duplicates, type mismatches."
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+authors = [{name = "Kryptorious Quantum Biosciences, Inc."}]
+dependencies = ["click>=8.0", "rich>=13.0"]
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0"]
+
+[project.scripts]
+csvclean = "csvclean.cli:main"
+
+[project.urls]
+Homepage = "https://devflow.sh"
+
+[tool.setuptools.packages.find]
+where = ["src"]

kryptorious_csvclean-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

kryptorious_csvclean-1.1.0/src/csvclean/__init__.py

@@ -0,0 +1,2 @@
+"""CSVClean — CSV file cleaner and validator."""
+__version__ = "1.0.0"

kryptorious_csvclean-1.1.0/src/csvclean/cleaner.py

@@ -0,0 +1,328 @@
+"""CSVClean — CSV analysis and cleaning engine."""
+
+import csv
+import io
+import os
+from collections import Counter
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+
+
+def analyze_csv(path: str) -> Dict[str, Any]:
+    """Analyze a CSV file and return diagnostic info.
+
+    Detects: encoding, delimiter, quote char, header issues,
+    empty rows, inconsistent column counts, type patterns, duplicates.
+    """
+    filepath = Path(path)
+    if not filepath.exists():
+        raise FileNotFoundError(f"File not found: {path}")
+
+    size = filepath.stat().st_size
+    raw = filepath.read_bytes()
+
+    # Detect encoding
+    encoding = _detect_encoding(raw)
+
+    # Read with detected encoding
+    text = raw.decode(encoding, errors="replace")
+    lines = text.splitlines()
+
+    if not lines:
+        return {"error": "File is empty", "rows": 0, "columns": 0}
+
+    # Detect delimiter
+    delimiter = _detect_delimiter(lines)
+
+    # Parse with csv module
+    reader = csv.reader(io.StringIO(text), delimiter=delimiter)
+    rows = list(reader)
+
+    if not rows:
+        return {"error": "No data rows found", "rows": 0, "columns": 0}
+
+    header = rows[0]
+    data_rows = rows[1:]
+
+    # Build result
+    result = {
+        "path": str(filepath),
+        "size_bytes": size,
+        "size_human": _format_size(size),
+        "encoding": encoding,
+        "delimiter": delimiter if delimiter != "," else "comma",
+        "total_rows": len(data_rows),
+        "total_columns": len(header),
+        "headers": header,
+        "issues": [],
+    }
+
+    # Check for empty headers
+    empty_headers = [i for i, h in enumerate(header) if not h.strip()]
+    if empty_headers:
+        result["issues"].append({
+            "severity": "error",
+            "message": f"Empty header at column(s): {', '.join(str(i+1) for i in empty_headers)}"
+        })
+
+    # Check for duplicate headers
+    seen = set()
+    dupes = []
+    for h in header:
+        clean = h.strip().lower()
+        if clean in seen:
+            dupes.append(h)
+        seen.add(clean)
+    if dupes:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"Duplicate headers: {', '.join(dupes)}"
+        })
+
+    # Check rows
+    empty_rows = 0
+    short_rows = 0
+    long_rows = 0
+    type_patterns = {i: Counter() for i in range(len(header))}
+
+    for i, row in enumerate(data_rows, start=2):
+        # Empty row
+        if all(not cell.strip() for cell in row):
+            empty_rows += 1
+            continue
+
+        # Column count mismatch
+        if len(row) < len(header):
+            short_rows += 1
+        elif len(row) > len(header):
+            long_rows += 1
+
+        # Type inference per column
+        for j, cell in enumerate(row):
+            if j >= len(header):
+                continue
+            val = cell.strip()
+            if not val:
+                type_patterns[j]["empty"] += 1
+            elif val.replace(".", "").replace("-", "").isdigit():
+                type_patterns[j]["int" if "." not in val and "-" not in val else "float"] += 1
+            elif val.lower() in ("true", "false", "yes", "no"):
+                type_patterns[j]["bool"] += 1
+            else:
+                type_patterns[j]["string"] += 1
+
+    result["empty_rows"] = empty_rows
+    result["column_count_issues"] = short_rows + long_rows
+
+    if empty_rows:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"{empty_rows} empty row(s)"
+        })
+    if short_rows:
+        result["issues"].append({
+            "severity": "error",
+            "message": f"{short_rows} row(s) with fewer columns than header"
+        })
+    if long_rows:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"{long_rows} row(s) with more columns than header"
+        })
+
+    # Infer types
+    result["column_types"] = {}
+    for j, counter in type_patterns.items():
+        if j < len(header):
+            col_name = header[j].strip() or f"column_{j}"
+            dominant = counter.most_common(1)[0][0] if counter else "empty"
+            result["column_types"][col_name] = {
+                "dominant_type": dominant,
+                "distribution": dict(counter.most_common()),
+                "empty_count": counter.get("empty", 0),
+                "empty_pct": round(counter.get("empty", 0) / max(len(data_rows), 1) * 100, 1),
+            }
+
+    # Duplicate detection
+    seen_rows = set()
+    duplicate_count = 0
+    for row in data_rows:
+        row_key = tuple(cell.strip() for cell in row)
+        if row_key in seen_rows:
+            duplicate_count += 1
+        seen_rows.add(row_key)
+
+    result["duplicate_rows"] = duplicate_count
+    if duplicate_count:
+        result["issues"].append({
+            "severity": "warning",
+            "message": f"{duplicate_count} duplicate row(s)"
+        })
+
+    # Health score
+    max_score = 100
+    penalties = 0
+    for issue in result["issues"]:
+        if issue["severity"] == "error":
+            penalties += 15
+        else:
+            penalties += 5
+    penalties += duplicate_count * 2
+    result["health_score"] = max(0, max_score - penalties)
+
+    return result
+
+
+def clean_csv(path: str, output: str, fix_encoding: bool = True,
+              remove_empty: bool = True, normalize_delimiter: bool = True) -> Dict:
+    """Clean a CSV file and write the result to OUTPUT.
+
+    Real, free operation: decodes with detected encoding, optionally
+    drops fully-empty rows and normalizes the delimiter to comma.
+    Returns a summary of what was changed.
+    """
+    filepath = Path(path)
+    if not filepath.exists():
+        raise FileNotFoundError(f"File not found: {path}")
+
+    raw = filepath.read_bytes()
+    encoding = _detect_encoding(raw)
+    text = raw.decode(encoding, errors="replace")
+    lines = text.splitlines()
+
+    if not lines:
+        return {"source": path, "output": output, "removed_empty": 0,
+                "normalized_delimiter": False, "wrote": False}
+
+    delimiter = _detect_delimiter(lines)
+    target_delim = "," if normalize_delimiter else (delimiter or ",")
+    reader = csv.reader(io.StringIO(text), delimiter=delimiter)
+    rows = list(reader)
+
+    header = rows[0] if rows else []
+    data_rows = rows[1:]
+
+    seen = set()
+    kept = []
+    removed_empty = 0
+    removed_dupes = 0
+    for row in data_rows:
+        if all(not cell.strip() for cell in row):
+            removed_empty += 1
+            continue
+        key = tuple(cell.strip() for cell in row)
+        if key in seen:
+            removed_dupes += 1
+            continue
+        seen.add(key)
+        kept.append(row)
+
+    out_rows = [header] + kept if header else kept
+    with open(output, "w", encoding="utf-8", newline="") as fh:
+        writer = csv.writer(fh, delimiter=target_delim)
+        writer.writerows(out_rows)
+
+    return {
+        "source": path,
+        "output": output,
+        "removed_empty": removed_empty,
+        "removed_duplicates": removed_dupes,
+        "normalized_delimiter": bool(normalize_delimiter and delimiter not in (",", "")),
+        "encoding_used": encoding,
+        "wrote": True,
+    }
+
+
+def dedupe_csv(path: str, output: str | None = None) -> Dict:
+    """Remove duplicate data rows from a CSV, writing to OUTPUT (or stdout summary)."""
+    filepath = Path(path)
+    raw = filepath.read_bytes()
+    encoding = _detect_encoding(raw)
+    text = raw.decode(encoding, errors="replace")
+    lines = text.splitlines()
+    if not lines:
+        return {"source": path, "removed": 0, "wrote": False}
+    delimiter = _detect_delimiter(lines)
+    rows = list(csv.reader(io.StringIO(text), delimiter=delimiter))
+    header = rows[0] if rows else []
+    data_rows = rows[1:]
+    seen = set()
+    kept = []
+    removed = 0
+    for row in data_rows:
+        key = tuple(cell.strip() for cell in row)
+        if key in seen:
+            removed += 1
+            continue
+        seen.add(key)
+        kept.append(row)
+    if output:
+        with open(output, "w", encoding="utf-8", newline="") as fh:
+            csv.writer(fh, delimiter=delimiter).writerows([header] + kept if header else kept)
+        return {"source": path, "output": output, "removed": removed, "wrote": True}
+    return {"source": path, "removed": removed, "wrote": False}
+
+
+def _detect_encoding(raw: bytes) -> str:
+    """Detect file encoding from BOM or content."""
+    if raw.startswith(b"\xff\xfe"):
+        return "utf-16-le"
+    if raw.startswith(b"\xfe\xff"):
+        return "utf-16-be"
+    if raw.startswith(b"\xef\xbb\xbf"):
+        return "utf-8-sig"
+    try:
+        raw.decode("utf-8")
+        return "utf-8"
+    except UnicodeDecodeError:
+        try:
+            raw.decode("latin-1")
+            return "latin-1"
+        except UnicodeDecodeError:
+            return "utf-8"
+
+
+def _detect_delimiter(lines: List[str]) -> str:
+    """Detect CSV delimiter from content."""
+    if not lines:
+        return ","
+
+    # Check first 5 non-empty lines
+    sample = [l for l in lines[:20] if l.strip()][:5]
+    if not sample:
+        return ","
+
+    candidates = [",", "\t", ";", "|"]
+    header_line = sample[0]
+
+    # Count occurrences
+    counts = {}
+    for delim in candidates:
+        counts[delim] = header_line.count(delim)
+
+    # Find the delimiter that appears consistently across lines
+    best_delim = ","
+    best_score = 0
+
+    for delim in candidates:
+        line_counts = [line.count(delim) for line in sample]
+        if len(set(line_counts)) <= 1 and line_counts[0] > 0:
+            # Consistent count across lines
+            score = line_counts[0] * 10 + (5 if delim != "," else 0)
+            if score > best_score:
+                best_score = score
+                best_delim = delim
+        elif line_counts[0] > counts.get(best_delim, 0):
+            best_delim = delim
+
+    return best_delim
+
+
+def _format_size(size: int) -> str:
+    if size < 1024:
+        return f"{size} B"
+    elif size < 1024 * 1024:
+        return f"{size / 1024:.1f} KB"
+    elif size < 1024 * 1024 * 1024:
+        return f"{size / (1024 * 1024):.1f} MB"
+    return f"{size / (1024 * 1024 * 1024):.2f} GB"

kryptorious_csvclean-1.1.0/src/csvclean/cli.py

@@ -0,0 +1,191 @@
+"""CSVClean CLI — Clean and validate CSV files."""
+
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version="1.0.0", prog_name="csvclean")
+def main():
+    """CSVClean — Find and fix CSV file problems.
+
+    Detect encoding issues, inconsistent delimiters, missing headers,
+    duplicate rows, type mismatches — all in one command.
+    """
+    pass
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
+def check(path, output_json):
+    """Analyze a CSV file and report all issues.
+
+    \b
+    Examples:
+        csvclean check data.csv
+        csvclean check data.csv --json
+    """
+    from .cleaner import analyze_csv
+
+    console.print()
+    console.print(Panel(
+        f"[bold]CSVClean Check[/bold] — [cyan]{path}[/cyan]",
+        border_style="blue"
+    ))
+
+    try:
+        result = analyze_csv(path)
+    except Exception as e:
+        console.print(f"[red]Error:[/red] {e}")
+        return
+
+    if output_json:
+        import json as _json
+        console.print(_json.dumps(result, indent=2, default=str))
+        return
+
+    if "error" in result:
+        console.print(f"[red]{result['error']}[/red]")
+        return
+
+    # Overview
+    score = result["health_score"]
+    color = "green" if score >= 80 else "yellow" if score >= 50 else "red"
+    console.print(f"\n[bold {color}]Health: {score}/100[/bold {color}]")
+    console.print(f"  Size: {result['size_human']}")
+    console.print(f"  Encoding: [cyan]{result['encoding']}[/cyan]")
+    console.print(f"  Delimiter: [cyan]{result['delimiter']}[/cyan]")
+    console.print(f"  Rows: [bold]{result['total_rows']:,}[/bold] × {result['total_columns']} columns")
+    console.print(f"  Empty rows: {result.get('empty_rows', 0)}")
+    console.print(f"  Duplicates: {result.get('duplicate_rows', 0)}")
+
+    # Issues
+    if result["issues"]:
+        console.print()
+        console.print("[bold]Issues found:[/bold]")
+        for issue in result["issues"]:
+            icon = "[red]✗[/red]" if issue["severity"] == "error" else "[yellow]![/yellow]"
+            console.print(f"  {icon} {issue['message']}")
+    else:
+        console.print()
+        console.print("[green]No issues found. File is clean![/green]")
+
+    # Column types
+    if result.get("column_types"):
+        console.print()
+        console.print("[bold]Column Analysis:[/bold]")
+        col_table = Table()
+        col_table.add_column("Column")
+        col_table.add_column("Type", style="cyan")
+        col_table.add_column("Empty %", justify="right")
+        col_table.add_column("Distribution")
+
+        for col_name, info in result["column_types"].items():
+            dist = ", ".join(f"{t}:{c}" for t, c in info["distribution"].items() if t != "empty")
+            col_table.add_row(
+                col_name[:25],
+                info["dominant_type"],
+                f"{info['empty_pct']}%",
+                dist[:40]
+            )
+
+        console.print(col_table)
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.argument("output")
+@click.option("--remove-empty/--keep-empty", default=True, help="Remove empty rows")
+@click.option("--normalize/--preserve", default=True, help="Normalize delimiter to comma")
+def clean(path, output, remove_empty, normalize):
+    """Clean a CSV file and write the fixed output.
+
+    \b
+    Example:
+        csvclean clean messy.csv clean.csv
+    """
+    console.print()
+    console.print(Panel(
+        f"[bold]CSVClean Clean[/bold] — [cyan]{path}[/cyan] → [green]{output}[/green]",
+        border_style="blue"))
+
+    from .cleaner import clean_csv
+    result = clean_csv(path, output, remove_empty=remove_empty,
+                      normalize_delimiter=normalize)
+    console.print()
+    if result.get("removed_empty"):
+        console.print(f"  [green]✓[/green] Removed {result['removed_empty']} empty row(s)")
+    if result.get("removed_duplicates"):
+        console.print(f"  [green]✓[/green] Removed {result['removed_duplicates']} duplicate row(s)")
+    if result.get("normalized_delimiter"):
+        console.print(f"  [green]✓[/green] Normalized delimiter → comma")
+    console.print(f"  [bold]Wrote[/bold] {result['output']} (encoding: {result['encoding_used']})")
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.argument("output", required=False)
+@click.option("--in-place/--new-file", "inplace", default=False,
+              help="Overwrite the input file instead of writing to OUTPUT")
+def dedupe(path, output, inplace):
+    """Remove duplicate data rows from a CSV.
+
+    \b
+    Examples:
+        csvclean dedupe data.csv deduped.csv
+        csvclean dedupe data.csv --in-place
+    """
+    console.print()
+    console.print(Panel(
+        f"[bold]CSVClean Dedupe[/bold] — [cyan]{path}[/cyan]",
+        border_style="blue"))
+
+    from .cleaner import dedupe_csv
+    target = path if inplace else (output or "deduped.csv")
+    result = dedupe_csv(path, target)
+    console.print()
+    if result["removed"]:
+        console.print(f"  [green]✓[/green] Removed {result['removed']} duplicate row(s)")
+    else:
+        console.print("  [green]No duplicate rows found.[/green]")
+    if result.get("wrote"):
+        console.print(f"  [bold]Wrote[/bold] {result['output']}")
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+def stats(path):
+    """Show column statistics.
+
+    \b
+    Example:
+        csvclean stats data.csv
+    """
+    from .cleaner import analyze_csv
+
+    result = analyze_csv(path)
+    console.print()
+    console.print(Panel(
+        f"[bold]CSVClean Stats[/bold] — [cyan]{path}[/cyan]",
+        border_style="blue"
+    ))
+
+    ct = result.get("column_types", {})
+    for col_name, info in ct.items():
+        console.print(f"\n[bold cyan]{col_name}[/bold cyan]")
+        console.print(f"  Type: {info['dominant_type']}")
+        console.print(f"  Empty: {info['empty_count']} ({info['empty_pct']}%)")
+        dist = info.get("distribution", {})
+        if len(dist) > 1:
+            console.print(f"  Distribution: {dist}")
+
+
+if __name__ == "__main__":
+    main()

kryptorious_csvclean-1.1.0/src/kryptorious_csvclean.egg-info/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: kryptorious-csvclean
+Version: 1.1.0
+Summary: CSV file cleaner and validator — detect encoding, inconsistent delimiters, missing headers, duplicates, type mismatches.
+Author: Kryptorious Quantum Biosciences, Inc.
+License: MIT
+Project-URL: Homepage, https://devflow.sh
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.0
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"

kryptorious_csvclean-1.1.0/src/kryptorious_csvclean.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+pyproject.toml
+src/csvclean/__init__.py
+src/csvclean/cleaner.py
+src/csvclean/cli.py
+src/kryptorious_csvclean.egg-info/PKG-INFO
+src/kryptorious_csvclean.egg-info/SOURCES.txt
+src/kryptorious_csvclean.egg-info/dependency_links.txt
+src/kryptorious_csvclean.egg-info/entry_points.txt
+src/kryptorious_csvclean.egg-info/requires.txt
+src/kryptorious_csvclean.egg-info/top_level.txt

kryptorious_csvclean-1.1.0/src/kryptorious_csvclean.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

kryptorious_csvclean-1.1.0/src/kryptorious_csvclean.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+csvclean = csvclean.cli:main

kryptorious_csvclean-1.1.0/src/kryptorious_csvclean.egg-info/requires.txt

@@ -0,0 +1,5 @@
+click>=8.0
+rich>=13.0
+
+[dev]
+pytest>=7.0

kryptorious_csvclean-1.1.0/src/kryptorious_csvclean.egg-info/top_level.txt

@@ -0,0 +1 @@
+csvclean