dataframe-toolkit 0.7.2__tar.gz → 0.8.0__tar.gz

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.7.2
+Version: 0.8.0
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson
@@ -109,20 +109,21 @@ dftk dataset tips -o \
 
 ### Data transformation
 
-| Command     | Description                                                                          |
-|-------------|---------------------------------------------------------------------------------------|
-| `eval`      | Add or modify columns: eval expressions, string/path functions, statistical ops      |
-| `query`     | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
-| `merge`     | Join two tables on key columns (inner/left/right/outer)                              |
-| `concat`    | Concatenate two or more tables row-wise                                              |
-| `melt`      | Reshape wide-to-long (`pd.melt`)                                                     |
-| `pivot`     | Reshape long-to-wide with per-cell aggregation                                       |
-| `func`      | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
-| `scale`     | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
-| `interp`    | Interpolate values from a reference curve into a table (1-D lookup)                  |
-| `binx`      | Assign bin indices to a column based on explicit or generated edges                  |
-| `transpose` | Flip rows and columns; original column names land in a new `--keycol` column         |
-| `segid`     | Assign a segment ID that increments on each value change in a column                 |
+| Command      | Description                                                                          |
+|--------------|---------------------------------------------------------------------------------------|
+| `eval`       | Add or modify columns: eval expressions, string/path functions, statistical ops      |
+| `query`      | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
+| `merge`      | Join two tables on key columns (inner/left/right/outer)                              |
+| `concat`     | Concatenate two or more tables row-wise                                              |
+| `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
+| `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
+| `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
+| `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
+| `interp`     | Interpolate values from a reference curve into a table (1-D lookup)                  |
+| `binx`       | Assign bin indices to a column based on explicit or generated edges                  |
+| `transpose`  | Flip rows and columns; original column names land in a new `--keycol` column         |
+| `segid`      | Assign a segment ID that increments on each value change in a column                 |
+| `cleannames` | Rename columns to lowercase identifiers (`Sample ID` → `sample_id`), e.g. for SQL    |
 
 ### Statistics
 
@@ -173,6 +174,18 @@ dftk dataset iris -o \
   | dftk fit ... -f "sepal_length_scaled ~ petal_length" -g species
 ```
 
+### Clean up spreadsheet headers
+
+Headers such as `Country Name` or `1960` need quoting in SQL. `cleannames` turns them into lowercase identifiers (`country_name`, `x1960`) and keeps the original names in the parquet metadata (`dftk annotate FILE`):
+
+```bash
+dftk dataset fertility -o \
+  | dftk cleannames ... -o \
+  | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
+```
+
+For your own export, `dftk cleannames data.tsv --backend duckdb -o data.parquet` also keeps integer columns with empty cells as integers.
+
 ### Group summary then plot
 
 ```bash

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/README.md

@@ -80,20 +80,21 @@ dftk dataset tips -o \
 
 ### Data transformation
 
-| Command     | Description                                                                          |
-|-------------|---------------------------------------------------------------------------------------|
-| `eval`      | Add or modify columns: eval expressions, string/path functions, statistical ops      |
-| `query`     | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
-| `merge`     | Join two tables on key columns (inner/left/right/outer)                              |
-| `concat`    | Concatenate two or more tables row-wise                                              |
-| `melt`      | Reshape wide-to-long (`pd.melt`)                                                     |
-| `pivot`     | Reshape long-to-wide with per-cell aggregation                                       |
-| `func`      | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
-| `scale`     | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
-| `interp`    | Interpolate values from a reference curve into a table (1-D lookup)                  |
-| `binx`      | Assign bin indices to a column based on explicit or generated edges                  |
-| `transpose` | Flip rows and columns; original column names land in a new `--keycol` column         |
-| `segid`     | Assign a segment ID that increments on each value change in a column                 |
+| Command      | Description                                                                          |
+|--------------|---------------------------------------------------------------------------------------|
+| `eval`       | Add or modify columns: eval expressions, string/path functions, statistical ops      |
+| `query`      | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
+| `merge`      | Join two tables on key columns (inner/left/right/outer)                              |
+| `concat`     | Concatenate two or more tables row-wise                                              |
+| `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
+| `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
+| `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
+| `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
+| `interp`     | Interpolate values from a reference curve into a table (1-D lookup)                  |
+| `binx`       | Assign bin indices to a column based on explicit or generated edges                  |
+| `transpose`  | Flip rows and columns; original column names land in a new `--keycol` column         |
+| `segid`      | Assign a segment ID that increments on each value change in a column                 |
+| `cleannames` | Rename columns to lowercase identifiers (`Sample ID` → `sample_id`), e.g. for SQL    |
 
 ### Statistics
 
@@ -144,6 +145,18 @@ dftk dataset iris -o \
   | dftk fit ... -f "sepal_length_scaled ~ petal_length" -g species
 ```
 
+### Clean up spreadsheet headers
+
+Headers such as `Country Name` or `1960` need quoting in SQL. `cleannames` turns them into lowercase identifiers (`country_name`, `x1960`) and keeps the original names in the parquet metadata (`dftk annotate FILE`):
+
+```bash
+dftk dataset fertility -o \
+  | dftk cleannames ... -o \
+  | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
+```
+
+For your own export, `dftk cleannames data.tsv --backend duckdb -o data.parquet` also keeps integer columns with empty cells as integers.
+
 ### Group summary then plot
 
 ```bash

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.2"
+version = "0.8.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 keywords = [

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.2"
+version = "0.8.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/src/dftk/commands/__init__.py

@@ -37,6 +37,7 @@ COMMAND_GROUPS: list[tuple[str, list[tuple[str, str]]]] = [
             ("binx", "binx_cmd:BinxCommand"),
             ("transpose", "transpose_cmd:TransposeCommand"),
             ("segid", "segid_cmd:SegidCommand"),
+            ("cleannames", "cleannames_cmd:CleannamesCommand"),
         ],
     ),
     (

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/src/dftk/commands/base.py

@@ -71,6 +71,7 @@ class BaseCommand(ABC):
         tabular data and produces tabular output.
         """
         from dftk.common.io import io
+
         io.parser_read(parser)
         io.parser_output(parser)
 
@@ -82,4 +83,5 @@ class BaseCommand(ABC):
         exposed.
         """
         from dftk.common.io import io
+
         io.parser_read(parser)

dataframe_toolkit-0.8.0/src/dftk/commands/cleannames_cmd.py

@@ -0,0 +1,184 @@
+"""
+dftk.commands.cleannames_cmd — dftk cleannames subcommand.
+
+Rename columns to lowercase ASCII identifiers (snake_case), so they can be
+used unquoted in query --sql, eval -f and fit formulas.  Typical input is a
+spreadsheet export with headers such as "Sample ID", "GC %" or "Hæð (cm)".
+
+Unlike R janitor's clean_names, camelCase is not split ("sampleID" becomes
+"sampleid", not "sample_id"): splitting would also turn "HbA1c" into
+"hb_a1c" and "pH" into "p_h".  "%" and "#" become "pct" and "n".
+
+The original names are kept in the parquet metadata (key original_names, a
+JSON object cleaned → original; see dftk annotate) and, with --mapping, in a
+TSV file.
+"""
+
+import argparse
+import json
+import logging
+import re
+import unicodedata
+
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import io
+
+_EPILOG = """\
+RULES
+-----
+  Sample ID       ->  sample_id    lowercase; spaces and symbols become _
+   Age (years)    ->  age_years    trimmed; one _ per run, none at the ends
+  GC %            ->  gc_pct       % becomes pct
+  Count #         ->  count_n      # becomes n
+  1960            ->  x1960        a leading digit gets an x
+  Hæð (µg)        ->  haed_ug      letters become plain ASCII
+  sampleID        ->  sampleid     camelCase is not split
+  ---             ->  colN         nothing left: col + its position (1-based)
+  GC % , gc_pct   ->  gc_pct, gc_pct_2   clashes get _2, _3, ...
+
+Duplicate and empty headers are already renamed when the file is read
+(Count #.1, Unnamed: 7), so they come out as count_n_1 and unnamed_7.
+Names that are SQL reserved words (group, order, end, ...) are kept, with a
+warning: quote them in query --sql ("group").
+
+EXAMPLES
+--------
+  dftk cleannames data.tsv
+
+Convert a spreadsheet export to parquet with clean names, keeping integer
+columns with empty cells as integers:
+
+  dftk cleannames data.tsv --backend duckdb -o data.parquet
+
+The World Bank fertility table has headers such as "Country Name" and one
+column per year (1960, 1961, ...); after cleaning they can be used
+unquoted in SQL:
+
+  dftk dataset fertility -o \\
+      | dftk cleannames ... -o \\
+      | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
+
+Write the old and new names to a TSV file, e.g. for a data dictionary:
+
+  dftk dataset fertility -o \\
+      | dftk cleannames ... --mapping names.tsv -o fertility.parquet
+"""
+
+# Letters that Unicode does not decompose into ASCII letter + accent
+_TRANSLIT = str.maketrans(
+    {
+        "æ": "ae",
+        "ð": "d",
+        "đ": "d",
+        "þ": "th",
+        "ø": "o",
+        "œ": "oe",
+        "ß": "ss",
+        "ł": "l",
+        "ı": "i",
+        "µ": "u",  # micro sign, as in µg
+        "μ": "u",  # Greek mu
+    }
+)
+_SYMBOLS = {"%": " pct ", "#": " n "}
+
+
+def clean_name(name) -> str:
+    """Return *name* as a lowercase ASCII identifier ("" if nothing is left)."""
+    s = str(name).strip()
+    for symbol, word in _SYMBOLS.items():
+        s = s.replace(symbol, word)
+    s = s.lower().translate(_TRANSLIT)
+    s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode()
+    s = re.sub(r"[^a-z0-9]+", "_", s).strip("_")
+    if s[:1].isdigit():
+        s = "x" + s
+    return s
+
+
+def clean_names(names: list) -> list[str]:
+    """Clean every name; empty results become colN, clashes get _2, _3, …"""
+    cleaned = [clean_name(n) or f"col{i + 1}" for i, n in enumerate(names)]
+    taken = set(cleaned)  # a new name must not clash with any cleaned name
+    out: list[str] = []
+    for name in cleaned:
+        new, k = name, 1
+        while new in out or (new != name and new in taken):
+            k += 1
+            new = f"{name}_{k}"
+        out.append(new)
+    return out
+
+
+def _sql_keywords(names: list[str]) -> list[str]:
+    """Return the names that DuckDB SQL only accepts quoted."""
+    try:
+        import duckdb
+    except ImportError:
+        return []
+    rows = duckdb.sql(
+        "SELECT keyword_name FROM duckdb_keywords() "
+        "WHERE keyword_category IN ('reserved', 'type_function')"
+    ).fetchall()
+    keywords = {r[0] for r in rows}
+    return [n for n in names if n in keywords]
+
+
+class CleannamesCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "cleannames"
+
+    @property
+    def help(self) -> str:
+        return "Rename columns to lowercase identifiers (sample_id, gc_pct)"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("cleannames options")
+        g.add_argument(
+            "--mapping",
+            metavar="FILE",
+            default=None,
+            help="Also write the names to FILE as TSV (columns original, cleaned).",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+        original = [str(c) for c in df.columns]
+        cleaned = clean_names(original)
+        df.columns = cleaned
+
+        # Carry names from an earlier cleannames run, so the first originals
+        # (e.g. spreadsheet headers) are kept when a file is cleaned again
+        meta = dict(df.attrs.get("_parquet_meta", {}))
+        earlier = json.loads(meta.get("original_names", "{}"))
+        renamed = {}
+        for old, new in zip(original, cleaned, strict=True):
+            first = earlier.get(old, old)
+            if first != new:
+                renamed[new] = first
+        if renamed or earlier:
+            meta["original_names"] = json.dumps(renamed, ensure_ascii=False)
+            df.attrs["_parquet_meta"] = meta
+
+        if args.mapping is not None:
+            pd.DataFrame({"original": original, "cleaned": cleaned}).to_csv(
+                args.mapping, sep="\t", index=False
+            )
+
+        keywords = _sql_keywords(cleaned)
+        if keywords:
+            quoted = ", ".join(f'"{k}"' for k in keywords)
+            logging.warning(
+                f"cleannames: {', '.join(keywords)} are SQL reserved words; "
+                f"quote them in query --sql ({quoted})"
+            )
+
+        io.printdf(df, args)

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/src/dftk/commands/fit_cmd.py

@@ -73,12 +73,33 @@ def weighted_model(df: pd.DataFrame, args: argparse.Namespace):
     )
 
 
+def _nullable_to_float(df: pd.DataFrame) -> pd.DataFrame:
+    """Return *df* with nullable numeric columns (e.g. Int64) as float64.
+
+    patsy finds missing values with np.isnan, which fails on pd.NA
+    ("boolean value of NA is ambiguous").  As float64 they become NaN, and
+    statsmodels drops incomplete rows as it does for any other float column.
+    Nullable booleans are left alone: patsy treats them as categorical.
+    """
+    cols = [
+        c
+        for c in df.columns
+        if pd.api.types.is_extension_array_dtype(df[c])
+        and pd.api.types.is_numeric_dtype(df[c])
+        and not pd.api.types.is_bool_dtype(df[c])
+    ]
+    if not cols:
+        return df
+    return df.astype({c: "float64" for c in cols})
+
+
 def regress_it(df: pd.DataFrame, args: argparse.Namespace):
     """Fit the appropriate model and return the fitted result object.
 
     A formula that cannot be evaluated (unknown column or function) raises
     ValueError, which the CLI reports as an input error.
     """
+    df = _nullable_to_float(df)
     try:
         if getattr(args, "weights", None) is not None:
             mod = weighted_model(df, args)

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/src/dftk/commands/query_cmd.py

@@ -91,9 +91,29 @@ def _sql_query(args: argparse.Namespace, sql: str, table: str) -> pd.DataFrame:
         conn.register(table, df)
 
     try:
-        return conn.execute(sql).df()
+        result = conn.execute(sql)
+        description = result.description
+        df = result.df()
     except Exception as exc:
         raise ValueError(f"DuckDB SQL failed: {exc}") from exc
+    return _hugeint_to_int64(df, description)
+
+
+def _hugeint_to_int64(df: pd.DataFrame, description) -> pd.DataFrame:
+    """Return 128-bit integer result columns as Int64 instead of float64.
+
+    DuckDB's SUM of an integer column is a HUGEINT, which pandas has no type
+    for, so .df() gives float64 (19.0).  Below 2**53 that float is exact and
+    is converted back; from 2**53 on the float may already be rounded, so it
+    stays float.
+    """
+    for i, (_, type_code, *_) in enumerate(description or []):
+        if str(type_code) not in ("HUGEINT", "UHUGEINT"):
+            continue
+        col = df.iloc[:, i]
+        if (col.dropna().abs() < 2**53).all():
+            df.isetitem(i, col.astype("Int64"))
+    return df
 
 
 # ---------------------------------------------------------------------------

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.8.0}/src/dftk/common/io.py

@@ -48,6 +48,17 @@ logging.getLogger("matplotlib.font_manager").setLevel(logging.WARNING)
 # Parquet-pipe temp directory
 # ---------------------------------------------------------------------------
 
+# --cast TYPE names → dtypes
+_CAST_TYPES: dict[str, object] = {
+    "object": object,
+    "int64": "int64",
+    "Int64": "Int64",
+    "float64": "float64",
+    "bool": "bool",
+    "datetime64": "datetime64[ns]",
+    "category": "category",
+}
+
 DFTK_TMPDIR: str = os.environ.get(
     "DFTK_TMPDIR",
     os.path.join(tempfile.gettempdir(), "dftk"),
@@ -116,6 +127,90 @@ def _write_pipe_parquet(df: pd.DataFrame) -> str:
     return path
 
 
+def _warn_digits_exponent(df: pd.DataFrame, digits: int) -> None:
+    """Warn if ``--digits`` would print whole numbers in exponent form.
+
+    An integer column with empty cells is read as float, and ``%.Ng`` then
+    writes e.g. 248956422 as 2.49e+08.  The output is left as it is; the
+    warning suggests --cast COL:Int64 or --round instead.
+    """
+    import shlex
+
+    cols: list[str] = []
+    example = None
+    for c in df.columns:
+        if not pd.api.types.is_float_dtype(df[c]):
+            continue
+        s = df[c].dropna()
+        s = s[np.isfinite(s)]
+        hit = s[(s == s.round()) & (s.abs() >= 10**digits)]
+        if len(hit):
+            cols.append(str(c))
+            if example is None:
+                example = hit.iloc[0]
+    if not cols:
+        return
+
+    shown = cols[:3]
+    names = ", ".join(repr(c) for c in shown)
+    if len(cols) > len(shown):
+        names += f" and {len(cols) - len(shown)} more"
+    casts = " ".join(shlex.quote(f"{c}:Int64") for c in shown)
+    if len(cols) > len(shown):
+        casts += " ..."
+    logging.warning(
+        f"--digits {digits} writes whole numbers in column(s) {names} in "
+        f"exponent form (e.g. {int(example)} -> {example:.{digits}g}).\n"
+        f"  If these columns hold integers:  --cast {casts}\n"
+        f"  To round without exponent form: --round {digits} (decimal places) "
+        f"instead of --digits {digits}"
+    )
+
+
+def _unique_column_names(names: list) -> list:
+    """Name empty and duplicate headers as pandas' default reader does.
+
+    The pyarrow reader keeps ``""`` and repeated names as they are, which
+    breaks commands that look columns up by name.  An empty header becomes
+    ``Unnamed: N`` (0-based position) and a repeat ``name.1``, ``name.2``, …
+    """
+    names = [f"Unnamed: {i}" if n == "" else n for i, n in enumerate(names)]
+    if len(set(names)) == len(names):
+        return names
+    taken = set(names)  # a new name must not clash with any header, later ones too
+    out: list = []
+    for n in names:
+        new, k = n, 0
+        while new in out or (new != n and new in taken):
+            k += 1
+            new = f"{n}.{k}"
+        out.append(new)
+    return out
+
+
+def _write_tsv(df: pd.DataFrame, dest, **to_csv_kwargs) -> None:
+    """``df.to_csv(dest, ...)``, also for one column with empty cells.
+
+    With quoting off, Python's csv module refuses to write a row that is a
+    single empty field ("single empty field record must be quoted").  For a
+    one-column table the row is written with a constant second column, which
+    is then cut off, leaving an empty line: TSV for one missing value.
+    """
+    if len(df.columns) != 1:
+        df.to_csv(dest, **to_csv_kwargs)
+        return
+    pad = "_pad" if df.columns[0] != "_pad" else "_pad2"
+    text = df.assign(**{pad: "x"}).to_csv(None, lineterminator="\n", **to_csv_kwargs)
+    # Values cannot contain a tab (unescaped, the csv module refuses them), so
+    # the last tab on each line is the one before the padding column
+    text = "".join(line.rsplit("\t", 1)[0] + "\n" for line in text.splitlines())
+    if isinstance(dest, str):
+        with open(dest, "w", encoding="utf-8", newline="") as fh:
+            fh.write(text)
+    else:
+        dest.write(text)
+
+
 # ---------------------------------------------------------------------------
 # Utility helpers
 # ---------------------------------------------------------------------------
@@ -242,30 +337,48 @@ class io:
 
         # -- helpers -------------------------------------------------------
 
-        def _rename_if_noheader(df: pd.DataFrame) -> pd.DataFrame:
+        def _name_columns(df: pd.DataFrame) -> pd.DataFrame:
             if noheader:
                 df.columns = [f"V{i + 1}" for i in range(len(df.columns))]
+            else:
+                df.columns = _unique_column_names(list(df.columns))
             return df
 
+        def _pandas_csv(source, **extra) -> pd.DataFrame:
+            return pd.read_csv(
+                source, sep=sep, header=header, dtype=dtype, engine="pyarrow", **extra
+            )
+
+        def _with_blank_lines(df: pd.DataFrame, reread) -> pd.DataFrame:
+            # In a one-column file an empty line is a missing value, not a
+            # line to skip (dftk writes such files; DuckDB reads them this way)
+            if len(df.columns) != 1:
+                return df
+            full = reread()
+            return full if list(full.columns) == list(df.columns) else df
+
         def _csv_from_bytes(data: bytes) -> pd.DataFrame:
-            df = pd.read_csv(
-                _io.BytesIO(data),
-                sep=sep,
-                header=header,
-                dtype=dtype,
-                engine="pyarrow",
+            df = _pandas_csv(_io.BytesIO(data))
+            df = _with_blank_lines(
+                df, lambda: _pandas_csv(_io.BytesIO(data), skip_blank_lines=False)
             )
-            return _rename_if_noheader(df)
+            return _name_columns(df)
+
+        def _csv_with_duckdb(path: str) -> pd.DataFrame:
+            # DuckDB keeps integer columns with blanks as Int64 (pandas: float64)
+            try:
+                df = duckdb.read_csv(path, sep=sep, header=not noheader).df()
+            except Exception as exc:
+                logging.error(f"DuckDB failed to read {path!r}: {exc}")
+                raise
+            return _name_columns(df)
 
         def _csv_from_file(path: str) -> pd.DataFrame:
-            df = pd.read_csv(
-                path,
-                sep=sep,
-                header=header,
-                dtype=dtype,
-                engine="pyarrow",
+            df = _pandas_csv(path)
+            df = _with_blank_lines(
+                df, lambda: _pandas_csv(path, skip_blank_lines=False)
             )
-            return _rename_if_noheader(df)
+            return _name_columns(df)
 
         # -- read ----------------------------------------------------------
 
@@ -290,15 +403,22 @@ class io:
                 df = pd.read_parquet(filename)
                 df.attrs["_parquet_meta"] = _read_parquet_meta(filename)
             elif backend == "duckdb" and duckdb is not None:
-                try:
-                    df = duckdb.read_csv(filename, sep=sep, header=not noheader).df()
-                    df = _rename_if_noheader(df)
-                except Exception as exc:
-                    logging.error(f"DuckDB failed to read {filename!r}: {exc}")
-                    raise
+                df = _csv_with_duckdb(filename)
             else:
                 df = _csv_from_file(filename)
 
+        elif backend == "duckdb" and duckdb is not None:
+            # stdin with DuckDB: it reads files, not buffers (that needs fsspec),
+            # so spool stdin to a temp file and read it like a named file
+            os.makedirs(DFTK_TMPDIR, exist_ok=True)
+            fd, path = tempfile.mkstemp(suffix=".tsv", dir=DFTK_TMPDIR)
+            try:
+                with os.fdopen(fd, "wb") as fh:
+                    fh.write(sys.stdin.buffer.read())
+                df = _csv_with_duckdb(path)
+            finally:
+                os.unlink(path)
+
         else:
             # stdin — read TSV directly
             df = _csv_from_bytes(sys.stdin.buffer.read())
@@ -385,7 +505,12 @@ class io:
         )
         g.add_argument(
             "--cast",
-            help="Cast column to dtype (COL:TYPE).  Glob patterns supported.",
+            help=(
+                "Cast column to dtype (COL:TYPE).  TYPE is one of object, int64, "
+                "Int64, float64, bool, datetime64, category (a prefix is enough, "
+                "e.g. int, Int, cat).  Int64 is an integer column that allows "
+                "empty cells.  Glob patterns supported."
+            ),
             nargs="*",
             default=None,
             metavar="COL:TYPE",
@@ -471,7 +596,11 @@ class io:
         )
         na_group.add_argument(
             "--dropna",
-            help="Drop rows containing any NA value before output.",
+            help=(
+                "Drop rows containing any NA value from the output.  Applied "
+                "after the command has run, so it does not change what the "
+                "command sees (use --prequery for that)."
+            ),
             action="store_true",
         )
 
@@ -530,17 +659,18 @@ class io:
                 raise
 
         # Cast columns
-        _DTYPES = ["object", "int64", "float64", "bool", "datetime64", "category"]
         for item in getattr(args, "cast", None) or []:
             try:
                 col, coltype = item.split(":")
-                coltype = fnmatch.filter(_DTYPES, f"{coltype}*")[0]
-                np_type = getattr(np, coltype)
+                # Case-sensitive: "int" is int64, "Int" is the nullable Int64
+                pattern = f"{coltype}*"
+                matches = [t for t in _CAST_TYPES if fnmatch.fnmatchcase(t, pattern)]
+                dtype = _CAST_TYPES[matches[0]]
                 for c in fnmatch.filter(df.columns, col):
                     try:
-                        df[c] = df[c].astype(np_type)
+                        df[c] = df[c].astype(dtype)
                     except Exception:
-                        df[c] = df[c].astype(float).astype(np_type)
+                        df[c] = df[c].astype(float).astype(dtype)
             except Exception:
                 logging.warning(f"--cast: failed for {item!r}")
 
@@ -665,8 +795,11 @@ class io:
 
         else:
             # TSV — stdout or named file
+            if digits is not None:
+                _warn_digits_exponent(df, digits)
             dest = output if output is not None else sys.stdout
-            df.to_csv(
+            _write_tsv(
+                df,
                 dest,
                 sep="\t",
                 index=False,