dataframe-toolkit 0.7.1__tar.gz → 0.7.3__tar.gz

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.7.1
+Version: 0.7.3
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson
@@ -17,6 +17,7 @@ Requires-Dist: numpy>=1.24
 Requires-Dist: pyarrow>=12.0
 Requires-Dist: scipy>=1.10
 Requires-Dist: statsmodels>=0.14
+Requires-Dist: patsy>=0.5.6
 Requires-Dist: duckdb>=0.8
 Requires-Dist: matplotlib>=3.7
 Requires-Dist: seaborn>=0.12
@@ -73,6 +74,20 @@ cd dataframe-toolkit
 uv tool install .
 ```
 
+### Skill for coding agents
+
+`skills/dftk/` is an agent skill: a short guide that coding agents such as
+Codex and Claude Code load when a task involves dftk. It covers the piping
+conventions, which command to use for what, and patterns for Snakemake and
+Make. Copy it into your agent's skills directory, for example:
+
+```bash
+cp -r skills/dftk ~/.codex/skills/     # Codex
+cp -r skills/dftk ~/.claude/skills/    # Claude Code
+```
+
+Copy it again after upgrading dftk so it matches the installed version.
+
 ## Quick example
 
 ```bash

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/README.md

@@ -45,6 +45,20 @@ cd dataframe-toolkit
 uv tool install .
 ```
 
+### Skill for coding agents
+
+`skills/dftk/` is an agent skill: a short guide that coding agents such as
+Codex and Claude Code load when a task involves dftk. It covers the piping
+conventions, which command to use for what, and patterns for Snakemake and
+Make. Copy it into your agent's skills directory, for example:
+
+```bash
+cp -r skills/dftk ~/.codex/skills/     # Codex
+cp -r skills/dftk ~/.claude/skills/    # Claude Code
+```
+
+Copy it again after upgrading dftk so it matches the installed version.
+
 ## Quick example
 
 ```bash

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.1"
+version = "0.7.3"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 keywords = [
@@ -26,6 +26,7 @@ dependencies = [
     "pyarrow>=12.0",
     "scipy>=1.10",
     "statsmodels>=0.14",
+    "patsy>=0.5.6",
     "duckdb>=0.8",
     "matplotlib>=3.7",
     "seaborn>=0.12",

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.1"
+version = "0.7.3"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [
@@ -23,6 +23,7 @@ dependencies = [
     "pyarrow>=12.0",
     "scipy>=1.10",
     "statsmodels>=0.14",
+    "patsy>=0.5.6",
     "duckdb>=0.8",
     "matplotlib>=3.7",
     "seaborn>=0.12",

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/src/dftk/commands/concat_cmd.py

@@ -40,6 +40,8 @@ COLUMN ALIGNMENT
 ----------------
   Columns present in all files are aligned by name.
   Columns missing from some files are filled with NaN (or --fill value).
+  --fill applies only to those missing columns: a cell that is empty within
+  a file stays empty.
   Use --select to enforce a fixed output column set.
 
 EXAMPLES
@@ -79,7 +81,8 @@ Mismatched columns are outer-joined and filled (NaN by default, or --fill):
             metavar="VALUE",
             help=(
                 "Fill value for columns absent in some files "
-                "(default: NaN).  Use 0 or '' for numeric/string defaults."
+                "(default: NaN).  Use 0 or '' for numeric/string defaults.  "
+                "Cells that are empty within a file stay empty."
             ),
         )
 
@@ -152,8 +155,6 @@ Mismatched columns are outer-joined and filled (NaN by default, or --fill):
         if not dfs:
             raise ValueError("concat: no data read from any input file")
 
-        result = pd.concat(dfs, ignore_index=True)
-
         if args.fill is not None:
             fill: int | float | str
             try:
@@ -163,6 +164,14 @@ Mismatched columns are outer-joined and filled (NaN by default, or --fill):
                     fill = float(args.fill)
                 except ValueError:
                     fill = args.fill
-            result = result.fillna(fill)
+            # Add each file's missing columns before stacking, so --fill
+            # touches only those, not cells that were already empty in a file.
+            columns = list(dict.fromkeys(c for df in dfs for c in df.columns))
+            dfs = [
+                df.assign(**{c: fill for c in columns if c not in df.columns})[columns]
+                for df in dfs
+            ]
+
+        result = pd.concat(dfs, ignore_index=True)
 
         io.printdf(result, args)

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/src/dftk/commands/fit_cmd.py

@@ -34,10 +34,12 @@ tidy coefficient tables into one output dataframe.
 import argparse
 import logging
 
+import numpy as np  # noqa: F401 — for formulas such as "np.log(y) ~ x", see below
 import pandas as pd
 import statsmodels.api as sm
 import statsmodels.formula.api as smf
 import statsmodels.stats.anova as sa
+from patsy import PatsyError
 
 from dftk.commands.base import BaseCommand
 from dftk.common.io import check_cols, io
@@ -50,6 +52,11 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 
+# Formulas are evaluated in the namespace of the function that builds the
+# model (statsmodels' default eval_env), i.e. this module's globals — which
+# is why numpy is imported above as np.
+
+
 def standard_model(df: pd.DataFrame, args: argparse.Namespace):
     """Return an OLS or RLM (robust) model, not yet fitted."""
     if getattr(args, "robust", False):
@@ -66,12 +73,40 @@ def weighted_model(df: pd.DataFrame, args: argparse.Namespace):
     )
 
 
+def _nullable_to_float(df: pd.DataFrame) -> pd.DataFrame:
+    """Return *df* with nullable numeric columns (e.g. Int64) as float64.
+
+    patsy finds missing values with np.isnan, which fails on pd.NA
+    ("boolean value of NA is ambiguous").  As float64 they become NaN, and
+    statsmodels drops incomplete rows as it does for any other float column.
+    Nullable booleans are left alone: patsy treats them as categorical.
+    """
+    cols = [
+        c
+        for c in df.columns
+        if pd.api.types.is_extension_array_dtype(df[c])
+        and pd.api.types.is_numeric_dtype(df[c])
+        and not pd.api.types.is_bool_dtype(df[c])
+    ]
+    if not cols:
+        return df
+    return df.astype({c: "float64" for c in cols})
+
+
 def regress_it(df: pd.DataFrame, args: argparse.Namespace):
-    """Fit the appropriate model and return the fitted result object."""
-    if getattr(args, "weights", None) is not None:
-        mod = weighted_model(df, args)
-    else:
-        mod = standard_model(df, args)
+    """Fit the appropriate model and return the fitted result object.
+
+    A formula that cannot be evaluated (unknown column or function) raises
+    ValueError, which the CLI reports as an input error.
+    """
+    df = _nullable_to_float(df)
+    try:
+        if getattr(args, "weights", None) is not None:
+            mod = weighted_model(df, args)
+        else:
+            mod = standard_model(df, args)
+    except PatsyError as exc:
+        raise ValueError(f"cannot evaluate formula {args.formula!r}: {exc}") from None
     return mod.fit()
 
 

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/src/dftk/commands/query_cmd.py

@@ -91,9 +91,29 @@ def _sql_query(args: argparse.Namespace, sql: str, table: str) -> pd.DataFrame:
         conn.register(table, df)
 
     try:
-        return conn.execute(sql).df()
+        result = conn.execute(sql)
+        description = result.description
+        df = result.df()
     except Exception as exc:
         raise ValueError(f"DuckDB SQL failed: {exc}") from exc
+    return _hugeint_to_int64(df, description)
+
+
+def _hugeint_to_int64(df: pd.DataFrame, description) -> pd.DataFrame:
+    """Return 128-bit integer result columns as Int64 instead of float64.
+
+    DuckDB's SUM of an integer column is a HUGEINT, which pandas has no type
+    for, so .df() gives float64 (19.0).  Below 2**53 that float is exact and
+    is converted back; from 2**53 on the float may already be rounded, so it
+    stays float.
+    """
+    for i, (_, type_code, *_) in enumerate(description or []):
+        if str(type_code) not in ("HUGEINT", "UHUGEINT"):
+            continue
+        col = df.iloc[:, i]
+        if (col.dropna().abs() < 2**53).all():
+            df.isetitem(i, col.astype("Int64"))
+    return df
 
 
 # ---------------------------------------------------------------------------

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/src/dftk/commands/stat_cmd.py

@@ -132,6 +132,9 @@ def _compute_stats(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
         stats = (
             df[cols].agg(agg_funcs).T.reset_index().rename(columns={"index": "name"})
         )
+        # Before the transpose each column held count and the float stats
+        # together, so count came out as float; a count is never missing.
+        stats["count"] = stats["count"].astype("int64")
         return stats[["name"] + _STAT_NAMES]
 
     else:

{dataframe_toolkit-0.7.1 → dataframe_toolkit-0.7.3}/src/dftk/common/io.py

@@ -48,6 +48,17 @@ logging.getLogger("matplotlib.font_manager").setLevel(logging.WARNING)
 # Parquet-pipe temp directory
 # ---------------------------------------------------------------------------
 
+# --cast TYPE names → dtypes
+_CAST_TYPES: dict[str, object] = {
+    "object": object,
+    "int64": "int64",
+    "Int64": "Int64",
+    "float64": "float64",
+    "bool": "bool",
+    "datetime64": "datetime64[ns]",
+    "category": "category",
+}
+
 DFTK_TMPDIR: str = os.environ.get(
     "DFTK_TMPDIR",
     os.path.join(tempfile.gettempdir(), "dftk"),
@@ -116,6 +127,69 @@ def _write_pipe_parquet(df: pd.DataFrame) -> str:
     return path
 
 
+def _warn_digits_exponent(df: pd.DataFrame, digits: int) -> None:
+    """Warn if ``--digits`` would print whole numbers in exponent form.
+
+    An integer column with empty cells is read as float, and ``%.Ng`` then
+    writes e.g. 248956422 as 2.49e+08.  The output is left as it is; the
+    warning suggests --cast COL:Int64 or --round instead.
+    """
+    import shlex
+
+    cols: list[str] = []
+    example = None
+    for c in df.columns:
+        if not pd.api.types.is_float_dtype(df[c]):
+            continue
+        s = df[c].dropna()
+        s = s[np.isfinite(s)]
+        hit = s[(s == s.round()) & (s.abs() >= 10**digits)]
+        if len(hit):
+            cols.append(str(c))
+            if example is None:
+                example = hit.iloc[0]
+    if not cols:
+        return
+
+    shown = cols[:3]
+    names = ", ".join(repr(c) for c in shown)
+    if len(cols) > len(shown):
+        names += f" and {len(cols) - len(shown)} more"
+    casts = " ".join(shlex.quote(f"{c}:Int64") for c in shown)
+    if len(cols) > len(shown):
+        casts += " ..."
+    logging.warning(
+        f"--digits {digits} writes whole numbers in column(s) {names} in "
+        f"exponent form (e.g. {int(example)} -> {example:.{digits}g}).\n"
+        f"  If these columns hold integers:  --cast {casts}\n"
+        f"  To round without exponent form: --round {digits} (decimal places) "
+        f"instead of --digits {digits}"
+    )
+
+
+def _write_tsv(df: pd.DataFrame, dest, **to_csv_kwargs) -> None:
+    """``df.to_csv(dest, ...)``, also for one column with empty cells.
+
+    With quoting off, Python's csv module refuses to write a row that is a
+    single empty field ("single empty field record must be quoted").  For a
+    one-column table the row is written with a constant second column, which
+    is then cut off, leaving an empty line: TSV for one missing value.
+    """
+    if len(df.columns) != 1:
+        df.to_csv(dest, **to_csv_kwargs)
+        return
+    pad = "_pad" if df.columns[0] != "_pad" else "_pad2"
+    text = df.assign(**{pad: "x"}).to_csv(None, lineterminator="\n", **to_csv_kwargs)
+    # Values cannot contain a tab (unescaped, the csv module refuses them), so
+    # the last tab on each line is the one before the padding column
+    text = "".join(line.rsplit("\t", 1)[0] + "\n" for line in text.splitlines())
+    if isinstance(dest, str):
+        with open(dest, "w", encoding="utf-8", newline="") as fh:
+            fh.write(text)
+    else:
+        dest.write(text)
+
+
 # ---------------------------------------------------------------------------
 # Utility helpers
 # ---------------------------------------------------------------------------
@@ -247,23 +321,39 @@ class io:
                 df.columns = [f"V{i + 1}" for i in range(len(df.columns))]
             return df
 
+        def _pandas_csv(source, **extra) -> pd.DataFrame:
+            return pd.read_csv(
+                source, sep=sep, header=header, dtype=dtype, engine="pyarrow", **extra
+            )
+
+        def _with_blank_lines(df: pd.DataFrame, reread) -> pd.DataFrame:
+            # In a one-column file an empty line is a missing value, not a
+            # line to skip (dftk writes such files; DuckDB reads them this way)
+            if len(df.columns) != 1:
+                return df
+            full = reread()
+            return full if list(full.columns) == list(df.columns) else df
+
         def _csv_from_bytes(data: bytes) -> pd.DataFrame:
-            df = pd.read_csv(
-                _io.BytesIO(data),
-                sep=sep,
-                header=header,
-                dtype=dtype,
-                engine="pyarrow",
+            df = _pandas_csv(_io.BytesIO(data))
+            df = _with_blank_lines(
+                df, lambda: _pandas_csv(_io.BytesIO(data), skip_blank_lines=False)
             )
             return _rename_if_noheader(df)
 
+        def _csv_with_duckdb(path: str) -> pd.DataFrame:
+            # DuckDB keeps integer columns with blanks as Int64 (pandas: float64)
+            try:
+                df = duckdb.read_csv(path, sep=sep, header=not noheader).df()
+            except Exception as exc:
+                logging.error(f"DuckDB failed to read {path!r}: {exc}")
+                raise
+            return _rename_if_noheader(df)
+
         def _csv_from_file(path: str) -> pd.DataFrame:
-            df = pd.read_csv(
-                path,
-                sep=sep,
-                header=header,
-                dtype=dtype,
-                engine="pyarrow",
+            df = _pandas_csv(path)
+            df = _with_blank_lines(
+                df, lambda: _pandas_csv(path, skip_blank_lines=False)
             )
             return _rename_if_noheader(df)
 
@@ -290,15 +380,22 @@ class io:
                 df = pd.read_parquet(filename)
                 df.attrs["_parquet_meta"] = _read_parquet_meta(filename)
             elif backend == "duckdb" and duckdb is not None:
-                try:
-                    df = duckdb.read_csv(filename, sep=sep, header=not noheader).df()
-                    df = _rename_if_noheader(df)
-                except Exception as exc:
-                    logging.error(f"DuckDB failed to read {filename!r}: {exc}")
-                    raise
+                df = _csv_with_duckdb(filename)
             else:
                 df = _csv_from_file(filename)
 
+        elif backend == "duckdb" and duckdb is not None:
+            # stdin with DuckDB: it reads files, not buffers (that needs fsspec),
+            # so spool stdin to a temp file and read it like a named file
+            os.makedirs(DFTK_TMPDIR, exist_ok=True)
+            fd, path = tempfile.mkstemp(suffix=".tsv", dir=DFTK_TMPDIR)
+            try:
+                with os.fdopen(fd, "wb") as fh:
+                    fh.write(sys.stdin.buffer.read())
+                df = _csv_with_duckdb(path)
+            finally:
+                os.unlink(path)
+
         else:
             # stdin — read TSV directly
             df = _csv_from_bytes(sys.stdin.buffer.read())
@@ -385,7 +482,12 @@ class io:
         )
         g.add_argument(
             "--cast",
-            help="Cast column to dtype (COL:TYPE).  Glob patterns supported.",
+            help=(
+                "Cast column to dtype (COL:TYPE).  TYPE is one of object, int64, "
+                "Int64, float64, bool, datetime64, category (a prefix is enough, "
+                "e.g. int, Int, cat).  Int64 is an integer column that allows "
+                "empty cells.  Glob patterns supported."
+            ),
             nargs="*",
             default=None,
             metavar="COL:TYPE",
@@ -471,7 +573,11 @@ class io:
         )
         na_group.add_argument(
             "--dropna",
-            help="Drop rows containing any NA value before output.",
+            help=(
+                "Drop rows containing any NA value from the output.  Applied "
+                "after the command has run, so it does not change what the "
+                "command sees (use --prequery for that)."
+            ),
             action="store_true",
         )
 
@@ -530,17 +636,18 @@ class io:
                 raise
 
         # Cast columns
-        _DTYPES = ["object", "int64", "float64", "bool", "datetime64", "category"]
         for item in getattr(args, "cast", None) or []:
             try:
                 col, coltype = item.split(":")
-                coltype = fnmatch.filter(_DTYPES, f"{coltype}*")[0]
-                np_type = getattr(np, coltype)
+                # Case-sensitive: "int" is int64, "Int" is the nullable Int64
+                pattern = f"{coltype}*"
+                matches = [t for t in _CAST_TYPES if fnmatch.fnmatchcase(t, pattern)]
+                dtype = _CAST_TYPES[matches[0]]
                 for c in fnmatch.filter(df.columns, col):
                     try:
-                        df[c] = df[c].astype(np_type)
+                        df[c] = df[c].astype(dtype)
                     except Exception:
-                        df[c] = df[c].astype(float).astype(np_type)
+                        df[c] = df[c].astype(float).astype(dtype)
             except Exception:
                 logging.warning(f"--cast: failed for {item!r}")
 
@@ -665,8 +772,11 @@ class io:
 
         else:
             # TSV — stdout or named file
+            if digits is not None:
+                _warn_digits_exponent(df, digits)
             dest = output if output is not None else sys.stdout
-            df.to_csv(
+            _write_tsv(
+                df,
                 dest,
                 sep="\t",
                 index=False,