lethe-cli 0.1.1__py3-none-any.whl

lethe/__init__.py

@@ -0,0 +1,3 @@
+"""Lethe: data anonymization CLI tool."""
+
+__version__ = "0.1.1"

lethe/cli.py

@@ -0,0 +1,138 @@
+"""Lethe CLI: data anonymization for structured files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from lethe.config import DEFAULT_CHUNK_SIZE, DEFAULT_LOCALE, DEFAULT_THRESHOLD, LetheConfig
+from lethe.multiply_pipeline import run_multiply
+from lethe.pipeline import run_pipeline
+
+app = typer.Typer(
+    name="lethe",
+    help="Anonymize PII in structured data files.",
+    no_args_is_help=True,
+    invoke_without_command=True,
+)
+
+
+@app.callback()
+def main() -> None:
+    """Lethe: data anonymization for structured files."""
+
+
+@app.command()
+def anonymize(
+    input_file: Annotated[
+        Path,
+        typer.Argument(help="Input data file to anonymize (CSV, TSV, or TXT).", exists=True, readable=True),
+    ],
+    output: Annotated[
+        Optional[Path],
+        typer.Option("-o", "--output", help="Output file path. Defaults to <input>_anonymized.<ext>."),
+    ] = None,
+    model: Annotated[
+        str,
+        typer.Option("--model", "-m", help="spaCy model: 'trf' (transformer, default) or 'sm' (small, fast)."),
+    ] = "trf",
+    threshold: Annotated[
+        float,
+        typer.Option("--threshold", "-t", help="Minimum confidence score to treat a cell as PII."),
+    ] = DEFAULT_THRESHOLD,
+    chunk_size: Annotated[
+        int,
+        typer.Option("--chunk-size", help="Rows per chunk for streaming."),
+    ] = DEFAULT_CHUNK_SIZE,
+    locale: Annotated[
+        str,
+        typer.Option("--locale", help="Faker locale for generated replacement values."),
+    ] = DEFAULT_LOCALE,
+    seed: Annotated[
+        Optional[int],
+        typer.Option("--seed", help="Random seed for reproducible fake data."),
+    ] = None,
+    clean: Annotated[
+        bool,
+        typer.Option("--clean", help="Delete the original input file after successful anonymization."),
+    ] = False,
+    confirm_clean: Annotated[
+        bool,
+        typer.Option("--confirm-clean", help="Required confirmation for --clean. Destructive actions must be explicitly confirmed."),
+    ] = False,
+) -> None:
+    """Anonymize PII in a data file (CSV, TSV, or TXT)."""
+    if model not in ("trf", "sm"):
+        typer.echo(f"Error: model must be 'trf' or 'sm', got '{model}'", err=True)
+        raise typer.Exit(1)
+
+    if clean and not confirm_clean:
+        typer.echo("Error: --clean requires --confirm-clean to confirm deletion of the input file.", err=True)
+        raise typer.Exit(1)
+
+    config = LetheConfig(
+        model=model,
+        threshold=threshold,
+        chunk_size=chunk_size,
+        locale=locale,
+        seed=seed,
+    )
+
+    run_pipeline(input_file, output, config)
+
+    if clean:
+        input_file.unlink()
+        typer.echo(f"Deleted original file: {input_file}")
+
+
+@app.command()
+def multiply(
+    input_file: Annotated[
+        Path,
+        typer.Argument(help="Input data file to multiply (CSV or TSV).", exists=True, readable=True),
+    ],
+    factor: Annotated[
+        int,
+        typer.Option("--factor", "-f", help="Multiplication factor (output rows = input rows * factor)."),
+    ] = 3,
+    output: Annotated[
+        Optional[Path],
+        typer.Option("-o", "--output", help="Output file path. Defaults to <input>_multiplied.<ext>."),
+    ] = None,
+    locale: Annotated[
+        str,
+        typer.Option("--locale", help="Faker locale for generated replacement values."),
+    ] = DEFAULT_LOCALE,
+    seed: Annotated[
+        Optional[int],
+        typer.Option("--seed", help="Random seed for reproducible fake data."),
+    ] = None,
+    sanitize: Annotated[
+        bool,
+        typer.Option("--sanitize", help="Validate and fix emails and URLs to be RFC-compliant ASCII."),
+    ] = False,
+    clean: Annotated[
+        bool,
+        typer.Option("--clean", help="Delete the original input file after successful multiplication."),
+    ] = False,
+    confirm_clean: Annotated[
+        bool,
+        typer.Option("--confirm-clean", help="Required confirmation for --clean. Destructive actions must be explicitly confirmed."),
+    ] = False,
+) -> None:
+    """Multiply a dataset with synthetic rows (CSV or TSV)."""
+    if factor < 1:
+        typer.echo("Error: factor must be >= 1", err=True)
+        raise typer.Exit(1)
+
+    if clean and not confirm_clean:
+        typer.echo("Error: --clean requires --confirm-clean to confirm deletion of the input file.", err=True)
+        raise typer.Exit(1)
+
+    run_multiply(input_file, output, factor, locale=locale, seed=seed, sanitize=sanitize)
+
+    if clean:
+        input_file.unlink()
+        typer.echo(f"Deleted original file: {input_file}")

lethe/config.py

@@ -0,0 +1,27 @@
+"""Configuration for Lethe anonymization runs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+SPACY_MODELS = {
+    "trf": "en_core_web_trf",
+    "sm": "en_core_web_sm",
+}
+
+DEFAULT_THRESHOLD = 0.35
+DEFAULT_CHUNK_SIZE = 5000
+DEFAULT_LOCALE = "en_US"
+
+
+@dataclass(frozen=True)
+class LetheConfig:
+    model: str = "trf"
+    threshold: float = DEFAULT_THRESHOLD
+    chunk_size: int = DEFAULT_CHUNK_SIZE
+    locale: str = DEFAULT_LOCALE
+    seed: int | None = None
+
+    @property
+    def spacy_model(self) -> str:
+        return SPACY_MODELS[self.model]

lethe/mapping/session_index.py

@@ -0,0 +1,65 @@
+"""Session-scoped mapping index: original value -> consistent fake replacement."""
+
+from __future__ import annotations
+
+from faker import Faker
+
+
+FAKER_GENERATORS: dict[str, str] = {
+    "PERSON": "name",
+    "EMAIL_ADDRESS": "email",
+    "PHONE_NUMBER": "phone_number",
+    "LOCATION": "address",
+    "US_SSN": "ssn",
+    "CREDIT_CARD": "credit_card_number",
+    "DATE_TIME": "date",
+    "IP_ADDRESS": "ipv4",
+    "IBAN_CODE": "iban",
+    "US_DRIVER_LICENSE": "bothify",
+    "US_PASSPORT": "bothify",
+    "UK_NINO": "bothify",
+    "NL_BSN": "numerify",
+    "URL": "url",
+    "CRYPTO": "sha1",
+}
+
+FAKER_GENERATOR_KWARGS: dict[str, dict] = {
+    "US_DRIVER_LICENSE": {"text": "??#######"},
+    "US_PASSPORT": {"text": "#########"},
+    "UK_NINO": {"text": "??######?"},
+    "NL_BSN": {"text": "#########"},
+}
+
+
+class SessionIndex:
+    """Maintains a dict mapping (entity_type, original) -> fake value.
+
+    Ensures cross-table consistency: the same original value always maps
+    to the same fake replacement within a session.
+    """
+
+    def __init__(self, locale: str = "en_US", seed: int | None = None) -> None:
+        self._faker = Faker(locale)
+        if seed is not None:
+            Faker.seed(seed)
+        self._index: dict[tuple[str, str], str] = {}
+
+    def get_or_create(self, entity_type: str, original: str) -> str:
+        key = (entity_type, original)
+        if key in self._index:
+            return self._index[key]
+
+        fake_value = self._generate(entity_type)
+        self._index[key] = fake_value
+        return fake_value
+
+    def _generate(self, entity_type: str) -> str:
+        method_name = FAKER_GENERATORS.get(entity_type, "text")
+        method = getattr(self._faker, method_name)
+        kwargs = FAKER_GENERATOR_KWARGS.get(entity_type, {})
+        result = method(**kwargs) if kwargs else method()
+        return str(result)
+
+    @property
+    def mapping_count(self) -> int:
+        return len(self._index)

lethe/multiplier.py

@@ -0,0 +1,102 @@
+"""Multiply datasets: produce N * factor rows with synthetic data."""
+
+from __future__ import annotations
+
+import random
+
+import pandas as pd
+
+from lethe.mapping.session_index import SessionIndex
+from lethe.scanner.column_heuristics import infer_pii_type
+
+
+class Multiplier:
+    """Takes an input DataFrame and generates a multiplied version.
+
+    Columns are classified using heuristics:
+    - PII columns: fresh Faker values for every row
+    - ID columns: sequential integers continuing from max existing value
+    - Sample columns (non-PII / unknown): sampled from existing values
+    """
+
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        locale: str = "en_US",
+        seed: int | None = None,
+    ) -> None:
+        self._df = df
+        self._session = SessionIndex(locale=locale, seed=seed)
+        if seed is not None:
+            random.seed(seed)
+        self._pii_columns: dict[str, str] = {}
+        self._id_columns: list[str] = []
+        self._sample_columns: list[str] = []
+        self.classify_columns()
+
+    def classify_columns(self) -> None:
+        for col in self._df.columns:
+            hint = infer_pii_type(col)
+            if hint is None:
+                self._sample_columns.append(col)
+            elif hint == "SKIP":
+                if self._is_id_column(col):
+                    self._id_columns.append(col)
+                else:
+                    self._sample_columns.append(col)
+            else:
+                self._pii_columns[col] = hint
+
+    def _is_id_column(self, col: str) -> bool:
+        """Check if a SKIP column is specifically an ID/primary-key column."""
+        series = self._df[col]
+        if not pd.api.types.is_numeric_dtype(series):
+            return False
+        clean = series.dropna()
+        if clean.empty:
+            return False
+        as_int = clean.astype(int)
+        return as_int.is_unique and as_int.is_monotonic_increasing
+
+    def generate(self, factor: int) -> pd.DataFrame:
+        """Return a DataFrame with original_rows * factor total rows.
+
+        Rows 0..N-1 are the originals with PII replaced (anonymized).
+        Rows N..N*factor-1 are fully synthetic.
+        """
+        n_original = len(self._df)
+        n_synthetic = n_original * (factor - 1)
+        rows: dict[str, list] = {col: [] for col in self._df.columns}
+
+        # Phase 1: anonymize original rows
+        for _, row in self._df.iterrows():
+            for col in self._df.columns:
+                val = row[col]
+                if col in self._pii_columns:
+                    rows[col].append(self._session._generate(self._pii_columns[col]))
+                elif col in self._id_columns:
+                    rows[col].append(val)
+                else:
+                    rows[col].append(val)
+
+        # Phase 2: generate synthetic rows
+        id_counters: dict[str, int] = {}
+        for col in self._id_columns:
+            id_counters[col] = int(self._df[col].max()) + 1
+
+        col_values: dict[str, list] = {}
+        for col in self._sample_columns:
+            col_values[col] = self._df[col].dropna().tolist()
+
+        for _ in range(n_synthetic):
+            for col in self._df.columns:
+                if col in self._pii_columns:
+                    rows[col].append(self._session._generate(self._pii_columns[col]))
+                elif col in self._id_columns:
+                    rows[col].append(id_counters[col])
+                    id_counters[col] += 1
+                else:
+                    pool = col_values.get(col, [])
+                    rows[col].append(random.choice(pool) if pool else None)
+
+        return pd.DataFrame(rows)

lethe/multiply_pipeline.py

@@ -0,0 +1,69 @@
+"""Pipeline for the multiply command: read -> multiply -> write."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+from rich.console import Console
+
+from lethe.multiplier import Multiplier
+from lethe.parsers import TextFormat, detect_format, make_writer
+from lethe.sanitizer import sanitize_dataframe
+
+console = Console()
+
+
+def default_multiply_output(input_path: Path) -> Path:
+    return input_path.with_stem(input_path.stem + "_multiplied")
+
+
+def run_multiply(
+    input_path: Path,
+    output_path: Path | None,
+    factor: int,
+    locale: str = "en_US",
+    seed: int | None = None,
+    sanitize: bool = False,
+) -> Path:
+    if output_path is None:
+        output_path = default_multiply_output(input_path)
+
+    fmt = detect_format(input_path)
+
+    if fmt in (TextFormat.FREEFORM, TextFormat.ONE_VALUE_PER_LINE):
+        console.print(
+            f"[red]Error:[/red] Cannot multiply {fmt.value} files. "
+            "Multiply only supports CSV and TSV formats with columnar data.",
+            highlight=False,
+        )
+        raise SystemExit(1)
+
+    console.print(f"[bold]Lethe[/bold] multiplying [cyan]{input_path.name}[/cyan]")
+    console.print(f"  Format: [yellow]{fmt.value}[/yellow]")
+    console.print(f"  Factor: [yellow]{factor}x[/yellow]")
+    console.print(f"  Output: [cyan]{output_path}[/cyan]")
+
+    sep = "\t" if fmt == TextFormat.TSV else ","
+    df = pd.read_csv(input_path, sep=sep)
+    n_input = len(df)
+
+    console.print(f"  Input rows: [yellow]{n_input}[/yellow]")
+
+    multiplier = Multiplier(df, locale=locale, seed=seed)
+    result = multiplier.generate(factor)
+
+    if sanitize:
+        console.print("  Sanitizing emails and URLs...", style="dim")
+        result = sanitize_dataframe(result, multiplier._pii_columns)
+
+    writer = make_writer(output_path, fmt)
+    writer.write_chunk(result)
+    writer.close()
+
+    n_output = len(result)
+    console.print(
+        f"\n  [green]Done.[/green] {n_input} input rows -> {n_output} output rows "
+        f"({factor}x multiplication)."
+    )
+    return output_path

lethe/parsers/__init__.py

@@ -0,0 +1,96 @@
+"""Format detection and parser factory for supported file types."""
+
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+
+from lethe.parsers.csv_parser import CsvReader, CsvWriter
+from lethe.parsers.txt_parser import (
+    FreeformReader,
+    FreeformWriter,
+    LineReader,
+    LineWriter,
+    TsvReader,
+    TsvWriter,
+)
+
+
+class TextFormat(Enum):
+    CSV = "csv"
+    TSV = "tsv"
+    ONE_VALUE_PER_LINE = "one_value_per_line"
+    FREEFORM = "freeform"
+
+
+def detect_format(path: Path) -> TextFormat:
+    """Detect file format from extension and content heuristics."""
+    suffix = path.suffix.lower()
+
+    if suffix == ".csv":
+        return TextFormat.CSV
+    if suffix == ".tsv":
+        return TextFormat.TSV
+
+    if suffix == ".txt":
+        return _detect_txt_format(path)
+
+    return TextFormat.CSV
+
+
+def _detect_txt_format(path: Path) -> TextFormat:
+    """Inspect .txt file content to determine the sub-format."""
+    lines: list[str] = []
+    with open(path, encoding="utf-8") as f:
+        for i, line in enumerate(f):
+            if i >= 20:
+                break
+            lines.append(line.rstrip("\n\r"))
+
+    if not lines:
+        return TextFormat.FREEFORM
+
+    non_empty = [l for l in lines if l.strip()]
+    if not non_empty:
+        return TextFormat.FREEFORM
+
+    tab_counts = [l.count("\t") for l in non_empty]
+    if tab_counts and min(tab_counts) >= 1:
+        consistent = all(c == tab_counts[0] for c in tab_counts)
+        if consistent:
+            return TextFormat.TSV
+
+    total_chars = sum(len(l) for l in non_empty)
+    avg_chars = total_chars / len(non_empty) if non_empty else 0
+    max_words = max((len(l.split()) for l in non_empty), default=0)
+
+    if avg_chars < 60 and max_words <= 4:
+        return TextFormat.ONE_VALUE_PER_LINE
+
+    return TextFormat.FREEFORM
+
+
+def make_reader(path: Path, fmt: TextFormat, chunk_size: int = 5000):
+    """Create the appropriate reader for a given format."""
+    if fmt == TextFormat.CSV:
+        return CsvReader(path, chunk_size=chunk_size)
+    if fmt == TextFormat.TSV:
+        return TsvReader(path, chunk_size=chunk_size)
+    if fmt == TextFormat.ONE_VALUE_PER_LINE:
+        return LineReader(path, chunk_size=chunk_size)
+    if fmt == TextFormat.FREEFORM:
+        return FreeformReader(path, chunk_size=chunk_size)
+    return CsvReader(path, chunk_size=chunk_size)
+
+
+def make_writer(path: Path, fmt: TextFormat):
+    """Create the appropriate writer for a given format."""
+    if fmt == TextFormat.CSV:
+        return CsvWriter(path)
+    if fmt == TextFormat.TSV:
+        return TsvWriter(path)
+    if fmt == TextFormat.ONE_VALUE_PER_LINE:
+        return LineWriter(path)
+    if fmt == TextFormat.FREEFORM:
+        return FreeformWriter(path)
+    return CsvWriter(path)

lethe/parsers/base.py

@@ -0,0 +1,17 @@
+"""Protocols for chunked file I/O."""
+
+from __future__ import annotations
+
+from typing import Iterator, Protocol
+
+import pandas as pd
+
+
+class ChunkedReader(Protocol):
+    def read_chunks(self) -> Iterator[pd.DataFrame]: ...
+
+
+class ChunkedWriter(Protocol):
+    def write_chunk(self, chunk: pd.DataFrame, *, header: bool = False) -> None: ...
+
+    def close(self) -> None: ...

lethe/parsers/csv_parser.py

@@ -0,0 +1,39 @@
+"""Chunked CSV reader and writer using pandas."""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Iterator
+
+import pandas as pd
+
+
+class CsvReader:
+    def __init__(self, path: Path, chunk_size: int = 5000) -> None:
+        self.path = path
+        self.chunk_size = chunk_size
+
+    def read_chunks(self) -> Iterator[pd.DataFrame]:
+        with pd.read_csv(self.path, chunksize=self.chunk_size) as reader:
+            yield from reader
+
+
+class CsvWriter:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._first = True
+
+    def write_chunk(self, chunk: pd.DataFrame, *, header: bool = False) -> None:
+        write_header = header or self._first
+        chunk.to_csv(
+            self.path,
+            mode="a" if not self._first else "w",
+            header=write_header,
+            index=False,
+            quoting=csv.QUOTE_ALL,
+        )
+        self._first = False
+
+    def close(self) -> None:
+        pass

lethe/parsers/txt_parser.py

@@ -0,0 +1,108 @@
+"""Parsers for text-based file formats: TSV, one-value-per-line, free-form."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterator
+
+import pandas as pd
+
+
+class TsvReader:
+    def __init__(self, path: Path, chunk_size: int = 5000) -> None:
+        self.path = path
+        self.chunk_size = chunk_size
+
+    def read_chunks(self) -> Iterator[pd.DataFrame]:
+        with pd.read_csv(self.path, sep="\t", chunksize=self.chunk_size) as reader:
+            yield from reader
+
+
+class TsvWriter:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._first = True
+
+    def write_chunk(self, chunk: pd.DataFrame, *, header: bool = False) -> None:
+        write_header = header or self._first
+        chunk.to_csv(
+            self.path,
+            mode="a" if not self._first else "w",
+            header=write_header,
+            index=False,
+            sep="\t",
+        )
+        self._first = False
+
+    def close(self) -> None:
+        pass
+
+
+class LineReader:
+    def __init__(self, path: Path, chunk_size: int = 5000) -> None:
+        self.path = path
+        self.chunk_size = chunk_size
+
+    def read_chunks(self) -> Iterator[pd.DataFrame]:
+        buf: list[str] = []
+        with open(self.path, encoding="utf-8") as f:
+            for line in f:
+                stripped = line.rstrip("\n\r")
+                if not stripped:
+                    continue
+                buf.append(stripped)
+                if len(buf) >= self.chunk_size:
+                    yield pd.DataFrame({"value": buf})
+                    buf = []
+        if buf:
+            yield pd.DataFrame({"value": buf})
+
+
+class LineWriter:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._first = True
+
+    def write_chunk(self, chunk: pd.DataFrame, *, header: bool = False) -> None:
+        mode = "w" if self._first else "a"
+        with open(self.path, mode, encoding="utf-8") as f:
+            for value in chunk["value"]:
+                f.write(str(value) + "\n")
+        self._first = False
+
+    def close(self) -> None:
+        pass
+
+
+class FreeformReader:
+    def __init__(self, path: Path, chunk_size: int = 5000) -> None:
+        self.path = path
+        self.chunk_size = chunk_size
+
+    def read_chunks(self) -> Iterator[pd.DataFrame]:
+        buf: list[str] = []
+        with open(self.path, encoding="utf-8") as f:
+            for line in f:
+                text = line.rstrip("\n\r")
+                buf.append(text)
+                if len(buf) >= self.chunk_size:
+                    yield pd.DataFrame({"text": buf})
+                    buf = []
+        if buf:
+            yield pd.DataFrame({"text": buf})
+
+
+class FreeformWriter:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._first = True
+
+    def write_chunk(self, chunk: pd.DataFrame, *, header: bool = False) -> None:
+        mode = "w" if self._first else "a"
+        with open(self.path, mode, encoding="utf-8") as f:
+            for text in chunk["text"]:
+                f.write(str(text) + "\n")
+        self._first = False
+
+    def close(self) -> None:
+        pass