normalize-tabular-data 0.1.0__py3-none-any.whl

normalize_tabular_data/io.py

@@ -0,0 +1,187 @@
+"""Format dispatch: suffix -> polars read/write. No Textual imports."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import polars as pl
+
+READ_SUFFIXES: dict[str, str] = {
+    ".csv": "csv",
+    ".tsv": "tsv",
+    ".txt": "tsv",
+    ".jsonl": "jsonl",
+    ".ndjson": "jsonl",
+    ".parquet": "parquet",
+    ".pq": "parquet",
+    ".parq": "parquet",
+    ".xlsx": "xlsx",
+    ".xls": "xlsx",
+}
+
+WRITE_SUFFIXES: dict[str, str] = {k: v for k, v in READ_SUFFIXES.items() if k != ".xls"}
+
+FORMAT_SUFFIX: dict[str, str] = {
+    "csv": ".csv",
+    "tsv": ".tsv",
+    "jsonl": ".jsonl",
+    "parquet": ".parquet",
+    "xlsx": ".xlsx",
+}
+
+
+def detect_format(path: Path) -> str:
+    suffix = path.suffix.lower()
+    fmt = READ_SUFFIXES.get(suffix)
+    if fmt is None:
+        known = ", ".join(sorted(READ_SUFFIXES))
+        raise ValueError(f"Unsupported file type {suffix!r} (known: {known})")
+    return fmt
+
+
+def _read_tabular(path: Path, separator: str = ",") -> pl.DataFrame:
+    """CSV/TSV read with a retry fallback.
+
+    Inference looks at up to 10,000 rows; dtype-incompatible values can
+    still appear later in a large file and fail parsing. When that happens,
+    re-read with zero-length inference so every column stays String instead
+    of erroring out."""
+    try:
+        return pl.read_csv(path, separator=separator, infer_schema_length=10_000)
+    except pl.exceptions.PolarsError:
+        return pl.read_csv(path, separator=separator, infer_schema_length=0)
+
+
+def read_table(path: Path, fmt: str, sheet: str | None = None) -> pl.DataFrame:
+    """Read one table. For xlsx, `sheet` picks a sheet; default is the first."""
+    if fmt == "csv":
+        return _read_tabular(path)
+    if fmt == "tsv":
+        return _read_tabular(path, separator="\t")
+    if fmt == "jsonl":
+        return pl.read_ndjson(path)
+    if fmt == "parquet":
+        return pl.read_parquet(path)
+    if fmt == "xlsx":
+        if sheet is None:
+            sheets = excel_sheets(path)
+            sheet = sheets[0] if sheets else "Sheet1"
+        return pl.read_excel(path, sheet_name=sheet)
+    raise ValueError(f"Unknown format {fmt!r}")
+
+
+def excel_sheets(path: Path) -> list[str]:
+    """Sheet names of a workbook, in order (fastexcel/calamine)."""
+    import fastexcel
+
+    return list(fastexcel.read_excel(path).sheet_names)
+
+
+def write_table(
+    df: pl.DataFrame,
+    path: Path,
+    fmt: str,
+    sheet_name: str = "data",
+) -> None:
+    if fmt == "csv":
+        df.write_csv(path)
+    elif fmt == "tsv":
+        df.write_csv(path, separator="\t")
+    elif fmt == "jsonl":
+        df.write_ndjson(path)
+    elif fmt == "parquet":
+        df.write_parquet(path)
+    elif fmt == "xlsx":
+        df.write_excel(path, worksheet=sheet_name, autofit=True, freeze_panes="B2")
+    else:
+        raise ValueError(f"Unknown format {fmt!r}")
+
+
+# --- operation scripts --------------------------------------------------------
+
+SCRIPT_SUFFIX = ".ntd"
+
+
+def script_text(steps: list[tuple[str, dict]], header_fields: dict[str, str]) -> str:
+    """Human-readable ASCII text, one operation description per line.
+
+    Every operation is `<key>(<param>=<json value>, ...)` so the lines stay
+    readable while remaining mechanically parseable for a future
+    "apply a script" capability. `header_fields` become leading `#`
+    comment lines (source file, save time, ...)"""
+    lines = ["# normalize-tabular-data script"]
+    lines += [f"# {text}" for text in header_fields.values() if text]
+    lines += [
+        f"{key}("
+        + ", ".join(f"{param}={json.dumps(value)}" for param, value in params.items())
+        + ")"
+        for key, params in steps
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def write_script(
+    path: Path,
+    steps: list[tuple[str, dict]],
+    header_fields: dict[str, str],
+) -> None:
+    """Write the script as plain ASCII (non-ASCII input becomes '?')."""
+    path.write_text(
+        script_text(steps, header_fields), encoding="ascii", errors="replace"
+    )
+
+
+_SCRIPT_LINE_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)\((.*)\)")
+_PARAM_NAME_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)\s*=\s*")
+
+
+def parse_script_line(line: str) -> tuple[str, dict]:
+    """Parse one `key(param=<json value>, ...)` operation description.
+
+    Values are plain JSON literals, so each parameter is read with a
+    JSON decoder (no evaluation). Raises ValueError on anything else."""
+    match = _SCRIPT_LINE_RE.fullmatch(line.strip())
+    if match is None:
+        raise ValueError(f"not an operation line: {line!r}")
+    key, args = match.group(1), match.group(2)
+    decoder = json.JSONDecoder()
+    params: dict = {}
+    pos = 0
+    n = len(args)
+    while pos < n:
+        name_match = _PARAM_NAME_RE.match(args, pos)
+        if name_match is None:
+            raise ValueError(f"missing parameter value in: {line!r}")
+        name = name_match.group(1)
+        pos = name_match.end()
+        try:
+            params[name], pos = decoder.raw_decode(args, pos)
+        except json.JSONDecodeError:
+            raise ValueError(f"bad value for {name!r} in: {line!r}") from None
+        while pos < n and args[pos] in " \t":
+            pos += 1
+        if pos < n and args[pos] == ",":
+            pos += 1
+            while pos < n and args[pos] in " \t":
+                pos += 1
+        elif pos < n:
+            raise ValueError(f"expected ',' between parameters in: {line!r}")
+    return key, params
+
+
+def read_script(path: Path) -> list[tuple[str, dict]]:
+    """Read an operation script: one (key, params) step per non-comment line."""
+    steps: list[tuple[str, dict]] = []
+    for lineno, raw in enumerate(
+        path.read_text(encoding="ascii", errors="replace").splitlines(), 1
+    ):
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        try:
+            steps.append(parse_script_line(line))
+        except ValueError as exc:
+            raise ValueError(f"{path.name} line {lineno}: {exc}") from exc
+    return steps

normalize_tabular_data/ops.py

@@ -0,0 +1,333 @@
+"""Normalization engine: operation registry and re-computing pipeline.
+
+Pure polars — no Textual imports here, so the whole engine is testable
+without a terminal.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+import date_parser
+import polars as pl
+
+ParamKind = Literal["column_multi", "column", "text", "number", "choice", "bool"]
+
+
+@dataclass(frozen=True)
+class ParamSpec:
+    """One parameter of an operation, consumed by the UI to build a dialog."""
+
+    name: str
+    kind: ParamKind
+    title: str
+    default: Any = None
+    choices: tuple[str, ...] | None = None
+    min_value: int | None = None
+    help: str = ""
+
+
+@dataclass(frozen=True)
+class Operation:
+    key: str
+    title: str
+    params: tuple[ParamSpec, ...]
+    apply: Callable[[pl.DataFrame, dict[str, Any]], pl.DataFrame]
+    # designated chooser hotkey letter; "" lets the chooser pick the first
+    # unused letter in the title
+    hotkey: str = ""
+
+
+# --- apply functions -------------------------------------------------------
+
+
+def _apply_date_normalize(df: pl.DataFrame, p: dict[str, Any]) -> pl.DataFrame:
+    col: str = p["column"]
+    series = df.get_column(col)
+    if series.dtype != pl.String:
+        series = series.cast(pl.String)
+    # Datetime("ns"), unparseable -> null
+    return df.with_columns(date_parser.parse_series(series).alias(col))
+
+
+def _apply_trim_collapse(df: pl.DataFrame, p: dict[str, Any]) -> pl.DataFrame:
+    cols: list[str] = p["columns"]
+    return df.with_columns(
+        pl.col(c).cast(pl.String).str.strip_chars().str.replace_all(r"\s+", " ")
+        for c in cols
+    )
+
+
+def _apply_dedup_rows(df: pl.DataFrame, p: dict[str, Any]) -> pl.DataFrame:
+    cols: list[str] = p["columns"]
+    keep: str = p["keep"]
+    return df.unique(
+        subset=cols or None,
+        keep="first" if keep == "first" else "last",
+    )
+
+
+def _apply_drop_columns(df: pl.DataFrame, p: dict[str, Any]) -> pl.DataFrame:
+    return df.drop(p["columns"])
+
+
+def _apply_combine_columns(df: pl.DataFrame, p: dict[str, Any]) -> pl.DataFrame:
+    cols: list[str] = p["columns"]
+    sep: str = p["separator"]
+    name: str = p["new_name"]
+    return df.with_columns(
+        pl.concat_str(
+            [pl.col(c).cast(pl.String) for c in cols],
+            separator=sep,
+            ignore_nulls=True,
+        ).alias(name)
+    )
+
+
+def _apply_split_column(df: pl.DataFrame, p: dict[str, Any]) -> pl.DataFrame:
+    col: str = p["column"]
+    delim: str = p["delimiter"]
+    rest = df.drop(col)
+    # Always strip edge whitespace first; a space delimiter additionally
+    # collapses internal whitespace runs so one-or-many spaces/tabs split
+    # the same way.
+    cleaned = pl.col(col).cast(pl.String).str.strip_chars()
+    if delim.strip() == "" or set(delim) <= set(" \t\r\n\f\v"):
+        cleaned = cleaned.str.replace_all(r"\s+", " ").replace("", None)
+        delim = " "
+    split = df.select(cleaned.str.split(delim).alias("__split"))
+    # part count comes from the data: the longest actual split for this
+    # delimiter; short rows pad with null
+    width = split.select(pl.col("__split").list.len().max()).item() or 1
+    names = [f"{col}_{i + 1}" for i in range(width)]
+    parts = split.select(
+        pl.col("__split").list.to_struct(
+            fields=names,
+            upper_bound=width,
+        )
+    )
+    pieces = parts.get_column("__split").struct.unnest()
+    return rest.hstack(pieces) if rest.width else pieces
+
+
+# --- registry ---------------------------------------------------------------
+
+OPS: tuple[Operation, ...] = (
+    Operation(
+        key="date_normalize",
+        title="Normalize dates",
+        hotkey="d",
+        params=(
+            ParamSpec(
+                "column",
+                "column",
+                "Date column",
+                help="Any input format; unparseable -> null",
+            ),
+        ),
+        apply=_apply_date_normalize,
+    ),
+    Operation(
+        key="trim_collapse",
+        title="Trim whitespace",
+        hotkey="w",
+        params=(
+            ParamSpec(
+                "columns",
+                "column_multi",
+                "Columns",
+                help="Strip edges, collapse internal runs to one space",
+            ),
+        ),
+        apply=_apply_trim_collapse,
+    ),
+    Operation(
+        key="dedup_rows",
+        title="Deduplicate rows",
+        hotkey="p",
+        params=(
+            ParamSpec("columns", "column_multi", "Key columns (empty = all)"),
+            ParamSpec(
+                "keep",
+                "choice",
+                "Keep",
+                default="first",
+                choices=("first", "last"),
+            ),
+        ),
+        apply=_apply_dedup_rows,
+    ),
+    Operation(
+        key="combine_columns",
+        title="Combine columns",
+        hotkey="c",
+        params=(
+            ParamSpec("columns", "column_multi", "Columns (>= 2)"),
+            ParamSpec("separator", "text", "Separator", default=" "),
+            ParamSpec("new_name", "text", "New column name"),
+        ),
+        apply=_apply_combine_columns,
+    ),
+    Operation(
+        key="split_column",
+        title="Split column",
+        hotkey="s",
+        params=(
+            ParamSpec("column", "column", "Column to split"),
+            ParamSpec(
+                "delimiter",
+                "text",
+                "Delimiter",
+                default="",
+                help=(
+                    "Default split is on whitespace. "
+                    "If specified, split on exact characters given."
+                ),
+            ),
+        ),
+        apply=_apply_split_column,
+    ),
+    Operation(
+        key="remove_columns",
+        title="Remove columns",
+        hotkey="r",
+        params=(
+            ParamSpec(
+                "columns",
+                "column_multi",
+                "Columns to remove",
+                help="Remove the selected columns entirely",
+            ),
+        ),
+        apply=_apply_drop_columns,
+    ),
+)
+
+OP_REGISTRY: dict[str, Operation] = {op.key: op for op in OPS}
+
+# internal op (not in the chooser): renames one column via sample header clicks
+RENAME_OP = Operation(
+    key="rename_single",
+    title="Rename column",
+    params=(
+        ParamSpec("column", "column", "Column"),
+        ParamSpec("new_name", "text", "New name"),
+    ),
+    apply=lambda df, p: df.rename({p["column"]: p["new_name"]}),
+)
+
+# The script player resolves against this: every chooser op plus the internal
+# rename op (header-click renames are logged under the key "rename_single",
+# so saved scripts contain those lines and have to play back).
+PLAY_REGISTRY: dict[str, Operation] = {**OP_REGISTRY, RENAME_OP.key: RENAME_OP}
+
+
+# --- pipeline ---------------------------------------------------------------
+
+
+@dataclass
+class AppliedOp:
+    op: Operation
+    params: dict[str, Any]
+
+
+@dataclass
+class Pipeline:
+    """Original DataFrame + ordered applied ops; every change re-folds from source."""
+
+    source: pl.DataFrame
+    applied: list[AppliedOp] = field(default_factory=list)
+    redo_stack: list[AppliedOp] = field(default_factory=list)
+
+    def current(self) -> pl.DataFrame:
+        df = self.source
+        for step in self.applied:
+            df = step.op.apply(df, step.params)
+        return df
+
+    def validate(self, op: Operation, params: dict[str, Any]) -> str | None:
+        cols = list(self.current().columns)
+        for spec in op.params:
+            val = params.get(spec.name)
+            if spec.kind == "column":
+                if not isinstance(val, str) or val not in cols:
+                    return f"{op.title}: pick a column"
+            elif spec.kind == "column_multi":
+                picked = [c for c in (val or []) if c in cols]
+                if not picked:
+                    return f"{op.title}: pick at least one column"
+                need = 2 if op.key == "combine_columns" else 1
+                if len(set(picked)) < need:
+                    return f"{op.title}: pick at least {need} columns"
+            elif spec.kind == "text":
+                # split_column's blank delimiter is allowed: it means
+                # "split on runs of whitespace"
+                if op.key == "combine_columns" and spec.name == "new_name" and not val:
+                    return f"{op.title}: new name required"
+                if (
+                    op.key == "combine_columns"
+                    and spec.name == "new_name"
+                    and val in cols
+                ):
+                    return f"{op.title}: {val!r} already exists"
+            elif spec.kind == "number":
+                if val is None:
+                    return f"{op.title}: set a value for {spec.title}"
+                if spec.min_value is not None and val < spec.min_value:
+                    return f"{op.title}: {spec.title} must be >= {spec.min_value}"
+        return None
+
+    def apply(self, op: Operation, params: dict[str, Any]) -> None:
+        if (err := self.validate(op, params)) is not None:
+            raise ValueError(err)
+        self.applied.append(AppliedOp(op, dict(params)))
+        self.redo_stack.clear()
+
+    def undo(self) -> bool:
+        if not self.applied:
+            return False
+        self.redo_stack.append(self.applied.pop())
+        return True
+
+    def redo(self) -> bool:
+        if not self.redo_stack:
+            return False
+        self.applied.append(self.redo_stack.pop())
+        return True
+
+    def step_summary(self) -> list[str]:
+        out = []
+        for step in self.applied:
+            bits = " ".join(
+                f"{k}={v}" for k, v in step.params.items() if v not in (None, ())
+            )
+            out.append(f"{step.op.key}({bits})" if bits else step.op.key)
+        return out
+
+
+# --- column analysis ---------------------------------------------------------
+
+
+@dataclass
+class ColumnInfo:
+    name: str
+    dtype: str
+    null_count: int
+    date_candidate: bool = False
+
+
+def analyze_column(df: pl.DataFrame, col: str) -> ColumnInfo:
+    """dtype, null count, and a date-candidate flag from a 200-row sample."""
+    s = df.get_column(col)
+    info = ColumnInfo(name=col, dtype=str(s.dtype), null_count=int(s.null_count()))
+    if s.dtype == pl.String:
+        sample = (
+            s.drop_nulls().str.strip_chars().replace("", None).drop_nulls().head(200)
+        )
+        if sample.len() >= 3:
+            parsed = date_parser.parse_series(sample)
+            ratio = 1.0 - (parsed.null_count() / sample.len())
+            info.date_candidate = ratio >= 0.5
+    return info