normalize-tabular-data 0.1.5__tar.gz → 0.2.0__tar.gz

{normalize_tabular_data-0.1.5 → normalize_tabular_data-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: normalize-tabular-data
-Version: 0.1.5
+Version: 0.2.0
 Summary: TUI for normalizing tabular data with Polars
 Author: David Mertz, Ph.D.
 Author-email: David Mertz, Ph.D. <mertz@gnosis.cx>
@@ -12,6 +12,7 @@ Requires-Dist: polars>=1.44,<2
 Requires-Dist: fastexcel>=0.10
 Requires-Dist: xlsxwriter>=0.9
 Requires-Dist: platformdirs>=3.0
+Requires-Dist: chardet>=7.0.0
 Requires-Python: >=3.14
 Description-Content-Type: text/markdown
 
@@ -35,6 +36,11 @@ combine/split columns, drop columns), then save the cleaned result.
 Reads: CSV, TSV, JSON lines, Parquet, Excel (`.xlsx`/`.xls` — first sheet).
 Writes: CSV, TSV, JSON lines, Parquet, Excel (`.xlsx`).
 
+Text formats are parsed as UTF-8; a file that is not (ISO 8859-3,
+Windows-1256, Shift_JIS-2004, UTF-16, ...) has its encoding sniffed with
+[chardet](https://pypi.org/project/chardet/) and is transcoded to UTF-8
+before parsing.
+
 Large files are previewed with a random sample of 250 rows; every operation
 still runs on the full data.
 

{normalize_tabular_data-0.1.5 → normalize_tabular_data-0.2.0}/README.md

@@ -18,6 +18,11 @@ combine/split columns, drop columns), then save the cleaned result.
 Reads: CSV, TSV, JSON lines, Parquet, Excel (`.xlsx`/`.xls` — first sheet).
 Writes: CSV, TSV, JSON lines, Parquet, Excel (`.xlsx`).
 
+Text formats are parsed as UTF-8; a file that is not (ISO 8859-3,
+Windows-1256, Shift_JIS-2004, UTF-16, ...) has its encoding sniffed with
+[chardet](https://pypi.org/project/chardet/) and is transcoded to UTF-8
+before parsing.
+
 Large files are previewed with a random sample of 250 rows; every operation
 still runs on the full data.
 

{normalize_tabular_data-0.1.5 → normalize_tabular_data-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "normalize-tabular-data"
-version = "0.1.5"
+version = "0.2.0"
 description = "TUI for normalizing tabular data with Polars"
 readme = "README.md"
 license = "BSD-2-Clause"
@@ -13,6 +13,7 @@ dependencies = [
     "fastexcel>=0.10",
     "xlsxwriter>=0.9",
     "platformdirs>=3.0",
+    "chardet>=7.0.0",
 ]
 
 [[project.authors]]

{normalize_tabular_data-0.1.5 → normalize_tabular_data-0.2.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "normalize-tabular-data"
-version = "0.1.5"
+version = "0.2.0"
 description = "TUI for normalizing tabular data with Polars"
 readme = "README.md"
 license = "BSD-2-Clause"
@@ -16,6 +16,7 @@ dependencies = [
     "fastexcel>=0.10",
     "xlsxwriter>=0.9",
     "platformdirs>=3.0",
+    "chardet>=7.0.0",
 ]
 
 [dependency-groups]

{normalize_tabular_data-0.1.5 → normalize_tabular_data-0.2.0}/src/normalize_tabular_data/__init__.py

@@ -6,7 +6,7 @@ import argparse
 import sys
 from pathlib import Path
 
-__version__ = "0.1.5"
+__version__ = "0.2.0"
 
 
 def _run_script(path: Path, script_path: Path, output: Path | None) -> int:

{normalize_tabular_data-0.1.5 → normalize_tabular_data-0.2.0}/src/normalize_tabular_data/app.py

@@ -21,6 +21,7 @@ from textual.screen import Screen
 from textual.widgets import DataTable, Header, Label, ListItem, ListView
 
 from normalize_tabular_data import io
+from normalize_tabular_data import __version__
 from normalize_tabular_data.io import SCRIPT_SUFFIX
 from normalize_tabular_data.ops import (
     OP_REGISTRY,
@@ -207,7 +208,7 @@ class AppHeader(Header):
 
     def compose(self) -> ComposeResult:
         yield from super().compose()
-        yield Label("normalize-tabular-data", classes="app_name")
+        yield Label(f"normalize-tabular-data {__version__}", classes="app_name")
 
 
 class MainScreen(Screen[None]):

{normalize_tabular_data-0.1.5 → normalize_tabular_data-0.2.0}/src/normalize_tabular_data/io.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+import chardet
 import json
 import re
+from io import BytesIO
 from pathlib import Path
 
 import polars as pl
@@ -41,17 +43,55 @@ def detect_format(path: Path) -> str:
     return fmt
 
 
+# bytes probed for encoding detection; large files stay fast
+_SNIFF_BYTES = 100_000
+
+
+def _decoded_text(data: bytes) -> bytes:
+    """`data`, transcoded to UTF-8 (polars parses text formats as UTF-8 only).
+
+    Already-valid UTF-8 (the common case) passes through untouched. Anything
+    else is decoded with the encoding chardet sniffs from the leading bytes
+    -- ISO 8859-3, Windows-1256, Shift_JIS-2004, UTF-16, etc. -- and
+    re-encoded. When sniffer or decodng fails, an errors=replace decode
+    keeps the load alive instead of erroring out."""
+    if not data:
+        return data
+    if data.startswith(b"\xef\xbb\xbf"):  # UTF-8 BOM
+        data = data[3:]
+    try:
+        data.decode("utf-8")
+        return data
+    except UnicodeDecodeError:
+        pass
+    encoding = chardet.detect(data[:_SNIFF_BYTES])["encoding"]
+    try:
+        decoded = data.decode(encoding)
+    except (LookupError, UnicodeDecodeError):
+        decoded = data.decode("utf-8", errors="replace")
+    return decoded.removeprefix("").encode()
+
+
 def _read_tabular(path: Path, separator: str = ",") -> pl.DataFrame:
     """CSV/TSV read with a retry fallback.
 
     Inference looks at up to 10,000 rows; dtype-incompatible values can
     still appear later in a large file and fail parsing. When that happens,
     re-read with zero-length inference so every column stays String instead
-    of erroring out."""
+    of erroring out. The same retry also covers a non-UTF-8 encoding: the
+    sniffer (`_decoded_text`) transcoded the bytes to UTF-8."""
     try:
         return pl.read_csv(path, separator=separator, infer_schema_length=10_000)
     except pl.exceptions.PolarsError:
-        return pl.read_csv(path, separator=separator, infer_schema_length=0)
+        data = _decoded_text(path.read_bytes())
+        try:
+            return pl.read_csv(
+                BytesIO(data), separator=separator, infer_schema_length=10_000
+            )
+        except pl.exceptions.PolarsError:
+            return pl.read_csv(
+                BytesIO(data), separator=separator, infer_schema_length=0
+            )
 
 
 def read_table(path: Path, fmt: str, sheet: str | None = None) -> pl.DataFrame:
@@ -61,7 +101,10 @@ def read_table(path: Path, fmt: str, sheet: str | None = None) -> pl.DataFrame:
     if fmt == "tsv":
         return _read_tabular(path, separator="\t")
     if fmt == "jsonl":
-        return pl.read_ndjson(path)
+        try:
+            return pl.read_ndjson(path)
+        except pl.exceptions.PolarsError:
+            return pl.read_ndjson(BytesIO(_decoded_text(path.read_bytes())))
     if fmt == "parquet":
         return pl.read_parquet(path)
     if fmt == "xlsx":