normalize-tabular-data 0.1.0__py3-none-any.whl

normalize_tabular_data/widgets.py

@@ -0,0 +1,227 @@
+"""Custom widgets: column sidebar with dtype/date-candidate markers, steps bar,
+and a menu-style footer."""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from itertools import groupby
+
+from rich.text import Text
+from textual.app import ComposeResult
+from textual.containers import VerticalScroll
+from textual.css.query import NoMatches
+from textual.widget import Widget
+from textual.widgets import Footer, Static
+from textual.widgets._footer import FooterKey, FooterLabel, KeyGroup
+
+from normalize_tabular_data.ops import ColumnInfo
+
+
+class ColumnSidebar(Widget):
+    """Read-only panel listing columns with dtype, nulls, date-candidate flag.
+
+    Column selection for operations happens inside the operation parameter
+    dialogs (SelectionList); the sidebar is informational.
+
+    The list lives in a VerticalScroll child because a plain Widget never
+    registers as scrollable: a list longer than the panel scrolls instead
+    of clipping."""
+
+    DEFAULT_CSS = """
+    ColumnSidebar {
+        width: 39;
+        height: 1fr;
+        border: round $accent;
+        padding: 1 1;
+        layout: vertical;
+    }
+    ColumnSidebar > VerticalScroll {
+        height: 1fr;
+        width: 1fr;
+    }
+    ColumnSidebar > VerticalScroll > Static {
+        width: 100%;
+    }
+    """
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.infos: list[ColumnInfo] = []
+        self.content_text = Text("Columns (0)")
+
+    def compose(self) -> ComposeResult:
+        with VerticalScroll():
+            yield Static(self.content_text, id="sidebar_text")
+
+    def set_infos(self, infos: list[ColumnInfo]) -> None:
+        self.infos = infos
+        self.content_text = self._build_text()
+        try:
+            self.query_one("#sidebar_text", Static).update(self.content_text)
+        except NoMatches:
+            pass  # not unmounted yet: compose will show content_text
+
+    def _build_text(self) -> Text:
+        text = Text(f"Columns ({len(self.infos)})")
+        # usable width (self.size already excludes border and padding); keep
+        # every line within it so long column names truncate instead of wrapping
+        avail = max(self.size.width, 9)
+        # when the list outgrows the panel the scrollbar claims two columns
+        # of the inner area: build the lines for the reduced width so they
+        # never wrap inside the scrolling region
+        if len(self.infos) + 1 > self.size.height:
+            avail = max(avail - 2, 9)
+        # the null counts left-align as one column, as wide as the widest count
+        null_w = max(
+            (len(f"nulls={info.null_count}") for info in self.infos),
+            default=6,
+        )
+        name_w = max(avail - 6 - null_w - 1, 1)
+        for info in self.infos:
+            text.append("\n")
+            text.append("~" if info.date_candidate else " ")
+            text.append(" ")
+            # the dtype mark in the highlight color
+            text.append(_dtype_mark(info.dtype), style="cyan")
+            name = info.name[:name_w]
+            # pad the (possibly cut) name so every "nulls=..." starts together
+            text.append(" " + name.ljust(name_w))
+            text.append(f" nulls={info.null_count}")
+        return text
+
+
+def _dtype_mark(dtype: str) -> str:
+    """Compact 3-wide dtype mark for the sidebar: dt (datetime/date),
+    str (string), int (integer), dec (float/decimal), t/f (boolean);
+    everything else oth. Each mark is padded to 3 columns so column
+    names left-align."""
+    for prefix, mark in (
+        ("Datetime", "dt "),
+        ("Date", "dt "),
+        ("Time", "dt "),
+        ("String", "str"),
+        ("Categorical", "str"),
+        ("Enum", "str"),
+        ("Int", "int"),
+        ("UInt", "int"),
+        ("Float", "dec"),
+        ("Decimal", "dec"),
+        ("Boolean", "t/f"),
+    ):
+        if dtype.startswith(prefix):
+            return mark
+    return "oth"
+
+
+class StepsBar(Static):
+    """One-line summary of the applied operation pipeline."""
+
+    DEFAULT_CSS = """
+    StepsBar {
+        height: 1;
+        dock: bottom;
+        background: $panel;
+        color: $text;
+        padding: 0 1;
+    }
+    """
+
+    def set_steps(self, steps: list[str]) -> None:
+        text = "Steps: " + ("  ->  ".join(steps) if steps else "(none)")
+        self.update(text)
+
+
+_HOTKEY_RE = re.compile(r"^\((.)\)(.*)$")
+
+
+class MenuKey(FooterKey):
+    """Footer key shown menu-style: no key chip; the hotkey letter inside
+    parentheses (e.g. "(F)ile") is drawn in the key-highlight style.
+    Descriptions that don't follow the pattern fall back to the default
+    chip rendering."""
+
+    DEFAULT_CSS = """
+    MenuKey {
+        margin-right: 2;
+    }
+    """
+
+    def render(self) -> Text:
+        if _HOTKEY_RE.match(self.description) is None:
+            return super().render()
+        key_style = self.get_component_rich_style("footer-key--key")
+        description_style = self.get_component_rich_style("footer-key--description")
+        text = Text(self.description, description_style)
+        # the hotkey letter is the single character between the parentheses
+        text.stylize(key_style, 1, 2)
+        text.stylize_before(self.rich_style)
+        return text
+
+
+class MenuFooter(Footer):
+    """Footer whose keys render as menus. Mirrors Footer.compose but
+    yields MenuKey widgets."""
+
+    def compose(self) -> "ComposeResult":
+        if not self._bindings_ready:
+            return
+        active_bindings = self.screen.active_bindings
+        bindings = [
+            (binding, enabled, tooltip)
+            for (_, binding, enabled, tooltip) in active_bindings.values()
+            if binding.show
+        ]
+        action_to_bindings: defaultdict[str, list] = defaultdict(list)
+        for binding, enabled, tooltip in bindings:
+            action_to_bindings[binding.action].append((binding, enabled, tooltip))
+
+        self.styles.grid_size_columns = len(action_to_bindings)
+
+        for group, multi_bindings_iterable in groupby(
+            action_to_bindings.values(),
+            lambda multi_bindings_: multi_bindings_[0][0].group,
+        ):
+            multi_bindings = list(multi_bindings_iterable)
+            if group is not None and len(multi_bindings) > 1:
+                with KeyGroup(classes="-compact" if group.compact else ""):
+                    for multi_bindings in multi_bindings:
+                        binding, enabled, tooltip = multi_bindings[0]
+                        yield MenuKey(
+                            binding.key,
+                            self.app.get_key_display(binding),
+                            "",
+                            binding.action,
+                            disabled=not enabled,
+                            tooltip=tooltip or binding.description,
+                            classes="-grouped",
+                        ).data_bind(compact=Footer.compact)
+                yield FooterLabel(group.description)
+            else:
+                for multi_bindings in multi_bindings:
+                    binding, enabled, tooltip = multi_bindings[0]
+                    yield MenuKey(
+                        binding.key,
+                        self.app.get_key_display(binding),
+                        binding.description,
+                        binding.action,
+                        disabled=not enabled,
+                        tooltip=tooltip,
+                    ).data_bind(compact=Footer.compact)
+        if self.show_command_palette and self.app.ENABLE_COMMAND_PALETTE:
+            try:
+                _node, binding, enabled, tooltip = active_bindings[
+                    self.app.COMMAND_PALETTE_BINDING
+                ]
+            except KeyError:
+                pass
+            else:
+                yield MenuKey(
+                    binding.key,
+                    self.app.get_key_display(binding),
+                    binding.description,
+                    binding.action,
+                    classes="-command-palette",
+                    disabled=not enabled,
+                    tooltip=binding.tooltip or binding.description,
+                )

normalize_tabular_data-0.1.0.dist-info/METADATA

@@ -0,0 +1,164 @@
+Metadata-Version: 2.4
+Name: normalize-tabular-data
+Version: 0.1.0
+Summary: TUI for normalizing tabular data with polars
+Author: David Mertz, Ph.D.
+Author-email: David Mertz, Ph.D. <mertz@gnosis.cx>
+License-Expression: BSD-2-Clause
+License-File: LICENSE
+Requires-Dist: gnosis-date-parser>=1.0.4
+Requires-Dist: textual>=8.2.8
+Requires-Dist: polars>=1.44,<2
+Requires-Dist: fastexcel>=0.10
+Requires-Dist: xlsxwriter>=0.9
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+
+# normalize-tabular-data
+
+A terminal UI (Textual) for interactively normalizing tabular data, backed by
+[polars](https://pola.rs) for speed and
+[gnosis-date-parser](https://pypi.org/project/gnosis-date-parser/) for fuzzy
+date parsing at Rust speed.
+
+Load a file, see a preview, build up a pipeline of normalization operations
+(normalize messy dates, trim whitespace, rename a column, deduplicate,
+combine/split columns, drop columns), then save the
+cleaned result.
+
+## Running
+
+### Persistent install
+
+```bash
+uv tool install normalize-tabular-data
+# upgrade later with:
+uv tool upgrade normalize-tabular-data
+```
+
+### Ephemeral run (no install)
+
+`uvx` fetches into a throwaway environment each time:
+
+```bash
+uvx normalize-tabular-data
+# pin a specific version:
+uvx normalize-tabular-data==0.1.0
+```
+
+### Straight from a checkout
+
+```bash
+uv run normalize-tabular-data
+```
+
+### The TestPyPI sandbox
+
+The test build (uploaded via `make testpypi`) is not on PyPI; point uv
+at TestPyPI first and PyPI as the fallback index — the tool itself is
+found on TestPyPI, its dependencies (`textual`, `polars`, …) on PyPI:
+
+```bash
+uvx \
+  --index-url https://test.pypi.org/simple/ \
+  --extra-index-url https://pypi.org/simple/ \
+  normalize-tabular-data==0.1.0
+
+# or installed persistently:
+uv tool install \
+  --index-url https://test.pypi.org/simple/ \
+  --extra-index-url https://pypi.org/simple/ \
+  normalize-tabular-data==0.1.0
+```
+
+uv caches resolved packages aggressively: after a TestPyPI re-upload of
+the same version, add `--refresh-package normalize-tabular-data` so it
+re-consults the index rather than reusing the cached artifact.
+
+## Usage
+
+Launch with `normalize-tabular-data`. Keys:
+
+| Key | Action                      |
+|-----|-----------------------------|
+| `f` | (F)ile — open a file        |
+| `o` | (O)peration — choose an op  |
+| `p` | (P)lay script — reapply a saved `.ntd` sequence |
+| `u` | (U)ndo last applied step    |
+| `r` | (R)edo                      |
+| `s` | (S)ave the data in a format |
+| `q` | (Q)uit                      |
+
+## Operations
+
+- **Normalize dates** — parse a messy date column of *any* input format into
+  canonical UTC datetimes; unparseable values become null.
+- **Trim whitespace** — strip edges and collapse internal whitespace runs,
+  per selected columns.
+- Rename a column — click its header in the preview and type the new name.
+- **Deduplicate rows** — on all or selected columns, keeping first or last.
+- **Combine columns** — concatenate two or more columns with a separator.
+- **Split column** — break one column into `{col}_1..{col}_k`; a blank
+  delimiter splits on runs of whitespace.
+- **Remove columns** — drop selected columns entirely.
+
+## Formats
+
+Reads: CSV, TSV, JSON lines, Parquet, Excel (`.xlsx`/`.xls` — first sheet).
+Writes: CSV, TSV, JSON lines, Parquet, Excel (`.xlsx`).
+
+Large files are previewed with a random sample of 250 rows; every operation
+still runs on the full data.
+
+## Operation scripts (`.ntd`)
+
+Every open file keeps an internal log of the operations performed on it. When
+saving, the dialog offers a **Save sequence of operations?** checkbox
+(unticked by default); with it ticked, a second dialog asks where to write
+the script — suggested `<table>.ntd`, any other extension you type is kept.
+
+The script is plain ASCII text, one operation per line, e.g.:
+
+```
+# normalize-tabular-data script
+# source: employees.csv
+# table: /data/employees-normalized.csv
+# saved: 2026-10-05T12:30:11
+trim_collapse(columns=["Dept", "Name"])
+date_normalize(column="Hired Date")
+```
+
+## Playing a script back
+
+Press `p` (available while a file is loaded) to pick a script file: the
+dialog previews the highlighted `.ntd` file (syntax-highlighted, first 40
+lines) before you confirm. Its operations are applied, in order, to the table you have open. If any step
+cannot be performed against the currently loaded file — a column it
+renames, trims or splits is missing, the operation is unknown — playing
+stops with an alert naming the failing step, and every step the script had
+already applied is rolled back, so the table is left exactly as it was.
+
+![TUI preview](docs/screenshot.png)
+
+## Publishing
+
+PyPI credentials live in `$HOME/.pypirc` (sections `[pypi]` and
+`[testpypi]`); uploads go through `twine`, which honors that file —
+`uv publish` does not read `.pypirc`, so it is not used here.
+
+```bash
+make dist       # build sdist+wheel into dist/ and run `twine check`
+make testpypi   # upload to the TestPyPI sandbox (login section [testpypi])
+make pypi       # upload to PyPI (login section [pypi])
+twine upload    # equivalent of `make pypi`, minus a fresh build
+```
+
+The targets fail fast if the package metadata does not pass `twine check`,
+and `twine` runs with `--non-interactive` so a missing or wrong
+credential aborts the upload instead of prompting mid-run.
+
+## License
+
+BSD 2-Clause. Copyright (c) 2026, Service Employees International Union (SEIU). 
+
+See [LICENSE](LICENSE).

normalize_tabular_data-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+normalize_tabular_data/__init__.py,sha256=E0YV8ua_YINsIO2p7pzcUBekzqYfoEC9cAkgMSU8j_M,563
+normalize_tabular_data/__main__.py,sha256=siufNzKOY-aQWdyRvDPMMoL3shnNpIDLQAeYsOEHdgs,79
+normalize_tabular_data/app.py,sha256=QzeUyE0difnR-KOFGxlH5J5frvm0dc-XcF0yYr4EJJM,23359
+normalize_tabular_data/io.py,sha256=3gggwTUA1-Pht7nRd4aNWsWLyos8s6RC010KY96wOrs,6012
+normalize_tabular_data/ops.py,sha256=ZCSS5Lt2XnJUE9etdOtyzVvcu5x0-XlmIjyr4NISxS0,10626
+normalize_tabular_data/screens.py,sha256=4MfEnkTSpa6gbf5u4O6RB5c4TE_xPh-Ij9WNT14WYiE,19241
+normalize_tabular_data/widgets.py,sha256=xy9hsPTUEDVWsWhoenLmRaLzrS0guXTAmFQ20KA1F9k,8140
+normalize_tabular_data-0.1.0.dist-info/licenses/LICENSE,sha256=wiXKxH5DofILxnKHLXy8Qxl9WCXTOMGyBKqUHnP3lOA,1343
+normalize_tabular_data-0.1.0.dist-info/WHEEL,sha256=EmLkUISDECbcUx3FMCYOqokNOJqNp2r0d4mJzjErvvs,80
+normalize_tabular_data-0.1.0.dist-info/entry_points.txt,sha256=3odd6xXTXdVQiq_G8vDtu26BeWBoeVR0MCIOcEwZ1HY,72
+normalize_tabular_data-0.1.0.dist-info/METADATA,sha256=HbGZc-0rlv-JpeNbVyr4paC8GtkT602r71UgIFlY3Ys,5478
+normalize_tabular_data-0.1.0.dist-info/RECORD,,

normalize_tabular_data-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.12.3
+Root-Is-Purelib: true
+Tag: py3-none-any

normalize_tabular_data-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+normalize-tabular-data = normalize_tabular_data:main
+

normalize_tabular_data-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2026, Members of Service Employees International Union (SEIU)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.