dataframe-toolkit 0.7.2__tar.gz → 0.7.3__tar.gz

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.7.2
+Version: 0.7.3
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.2"
+version = "0.7.3"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 keywords = [

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.7.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.2"
+version = "0.7.3"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.7.3}/src/dftk/commands/fit_cmd.py

@@ -73,12 +73,33 @@ def weighted_model(df: pd.DataFrame, args: argparse.Namespace):
     )
 
 
+def _nullable_to_float(df: pd.DataFrame) -> pd.DataFrame:
+    """Return *df* with nullable numeric columns (e.g. Int64) as float64.
+
+    patsy finds missing values with np.isnan, which fails on pd.NA
+    ("boolean value of NA is ambiguous").  As float64 they become NaN, and
+    statsmodels drops incomplete rows as it does for any other float column.
+    Nullable booleans are left alone: patsy treats them as categorical.
+    """
+    cols = [
+        c
+        for c in df.columns
+        if pd.api.types.is_extension_array_dtype(df[c])
+        and pd.api.types.is_numeric_dtype(df[c])
+        and not pd.api.types.is_bool_dtype(df[c])
+    ]
+    if not cols:
+        return df
+    return df.astype({c: "float64" for c in cols})
+
+
 def regress_it(df: pd.DataFrame, args: argparse.Namespace):
     """Fit the appropriate model and return the fitted result object.
 
     A formula that cannot be evaluated (unknown column or function) raises
     ValueError, which the CLI reports as an input error.
     """
+    df = _nullable_to_float(df)
     try:
         if getattr(args, "weights", None) is not None:
             mod = weighted_model(df, args)

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.7.3}/src/dftk/commands/query_cmd.py

@@ -91,9 +91,29 @@ def _sql_query(args: argparse.Namespace, sql: str, table: str) -> pd.DataFrame:
         conn.register(table, df)
 
     try:
-        return conn.execute(sql).df()
+        result = conn.execute(sql)
+        description = result.description
+        df = result.df()
     except Exception as exc:
         raise ValueError(f"DuckDB SQL failed: {exc}") from exc
+    return _hugeint_to_int64(df, description)
+
+
+def _hugeint_to_int64(df: pd.DataFrame, description) -> pd.DataFrame:
+    """Return 128-bit integer result columns as Int64 instead of float64.
+
+    DuckDB's SUM of an integer column is a HUGEINT, which pandas has no type
+    for, so .df() gives float64 (19.0).  Below 2**53 that float is exact and
+    is converted back; from 2**53 on the float may already be rounded, so it
+    stays float.
+    """
+    for i, (_, type_code, *_) in enumerate(description or []):
+        if str(type_code) not in ("HUGEINT", "UHUGEINT"):
+            continue
+        col = df.iloc[:, i]
+        if (col.dropna().abs() < 2**53).all():
+            df.isetitem(i, col.astype("Int64"))
+    return df
 
 
 # ---------------------------------------------------------------------------

{dataframe_toolkit-0.7.2 → dataframe_toolkit-0.7.3}/src/dftk/common/io.py

@@ -48,6 +48,17 @@ logging.getLogger("matplotlib.font_manager").setLevel(logging.WARNING)
 # Parquet-pipe temp directory
 # ---------------------------------------------------------------------------
 
+# --cast TYPE names → dtypes
+_CAST_TYPES: dict[str, object] = {
+    "object": object,
+    "int64": "int64",
+    "Int64": "Int64",
+    "float64": "float64",
+    "bool": "bool",
+    "datetime64": "datetime64[ns]",
+    "category": "category",
+}
+
 DFTK_TMPDIR: str = os.environ.get(
     "DFTK_TMPDIR",
     os.path.join(tempfile.gettempdir(), "dftk"),
@@ -116,6 +127,69 @@ def _write_pipe_parquet(df: pd.DataFrame) -> str:
     return path
 
 
+def _warn_digits_exponent(df: pd.DataFrame, digits: int) -> None:
+    """Warn if ``--digits`` would print whole numbers in exponent form.
+
+    An integer column with empty cells is read as float, and ``%.Ng`` then
+    writes e.g. 248956422 as 2.49e+08.  The output is left as it is; the
+    warning suggests --cast COL:Int64 or --round instead.
+    """
+    import shlex
+
+    cols: list[str] = []
+    example = None
+    for c in df.columns:
+        if not pd.api.types.is_float_dtype(df[c]):
+            continue
+        s = df[c].dropna()
+        s = s[np.isfinite(s)]
+        hit = s[(s == s.round()) & (s.abs() >= 10**digits)]
+        if len(hit):
+            cols.append(str(c))
+            if example is None:
+                example = hit.iloc[0]
+    if not cols:
+        return
+
+    shown = cols[:3]
+    names = ", ".join(repr(c) for c in shown)
+    if len(cols) > len(shown):
+        names += f" and {len(cols) - len(shown)} more"
+    casts = " ".join(shlex.quote(f"{c}:Int64") for c in shown)
+    if len(cols) > len(shown):
+        casts += " ..."
+    logging.warning(
+        f"--digits {digits} writes whole numbers in column(s) {names} in "
+        f"exponent form (e.g. {int(example)} -> {example:.{digits}g}).\n"
+        f"  If these columns hold integers:  --cast {casts}\n"
+        f"  To round without exponent form: --round {digits} (decimal places) "
+        f"instead of --digits {digits}"
+    )
+
+
+def _write_tsv(df: pd.DataFrame, dest, **to_csv_kwargs) -> None:
+    """``df.to_csv(dest, ...)``, also for one column with empty cells.
+
+    With quoting off, Python's csv module refuses to write a row that is a
+    single empty field ("single empty field record must be quoted").  For a
+    one-column table the row is written with a constant second column, which
+    is then cut off, leaving an empty line: TSV for one missing value.
+    """
+    if len(df.columns) != 1:
+        df.to_csv(dest, **to_csv_kwargs)
+        return
+    pad = "_pad" if df.columns[0] != "_pad" else "_pad2"
+    text = df.assign(**{pad: "x"}).to_csv(None, lineterminator="\n", **to_csv_kwargs)
+    # Values cannot contain a tab (unescaped, the csv module refuses them), so
+    # the last tab on each line is the one before the padding column
+    text = "".join(line.rsplit("\t", 1)[0] + "\n" for line in text.splitlines())
+    if isinstance(dest, str):
+        with open(dest, "w", encoding="utf-8", newline="") as fh:
+            fh.write(text)
+    else:
+        dest.write(text)
+
+
 # ---------------------------------------------------------------------------
 # Utility helpers
 # ---------------------------------------------------------------------------
@@ -247,23 +321,39 @@ class io:
                 df.columns = [f"V{i + 1}" for i in range(len(df.columns))]
             return df
 
+        def _pandas_csv(source, **extra) -> pd.DataFrame:
+            return pd.read_csv(
+                source, sep=sep, header=header, dtype=dtype, engine="pyarrow", **extra
+            )
+
+        def _with_blank_lines(df: pd.DataFrame, reread) -> pd.DataFrame:
+            # In a one-column file an empty line is a missing value, not a
+            # line to skip (dftk writes such files; DuckDB reads them this way)
+            if len(df.columns) != 1:
+                return df
+            full = reread()
+            return full if list(full.columns) == list(df.columns) else df
+
         def _csv_from_bytes(data: bytes) -> pd.DataFrame:
-            df = pd.read_csv(
-                _io.BytesIO(data),
-                sep=sep,
-                header=header,
-                dtype=dtype,
-                engine="pyarrow",
+            df = _pandas_csv(_io.BytesIO(data))
+            df = _with_blank_lines(
+                df, lambda: _pandas_csv(_io.BytesIO(data), skip_blank_lines=False)
             )
             return _rename_if_noheader(df)
 
+        def _csv_with_duckdb(path: str) -> pd.DataFrame:
+            # DuckDB keeps integer columns with blanks as Int64 (pandas: float64)
+            try:
+                df = duckdb.read_csv(path, sep=sep, header=not noheader).df()
+            except Exception as exc:
+                logging.error(f"DuckDB failed to read {path!r}: {exc}")
+                raise
+            return _rename_if_noheader(df)
+
         def _csv_from_file(path: str) -> pd.DataFrame:
-            df = pd.read_csv(
-                path,
-                sep=sep,
-                header=header,
-                dtype=dtype,
-                engine="pyarrow",
+            df = _pandas_csv(path)
+            df = _with_blank_lines(
+                df, lambda: _pandas_csv(path, skip_blank_lines=False)
             )
             return _rename_if_noheader(df)
 
@@ -290,15 +380,22 @@ class io:
                 df = pd.read_parquet(filename)
                 df.attrs["_parquet_meta"] = _read_parquet_meta(filename)
             elif backend == "duckdb" and duckdb is not None:
-                try:
-                    df = duckdb.read_csv(filename, sep=sep, header=not noheader).df()
-                    df = _rename_if_noheader(df)
-                except Exception as exc:
-                    logging.error(f"DuckDB failed to read {filename!r}: {exc}")
-                    raise
+                df = _csv_with_duckdb(filename)
             else:
                 df = _csv_from_file(filename)
 
+        elif backend == "duckdb" and duckdb is not None:
+            # stdin with DuckDB: it reads files, not buffers (that needs fsspec),
+            # so spool stdin to a temp file and read it like a named file
+            os.makedirs(DFTK_TMPDIR, exist_ok=True)
+            fd, path = tempfile.mkstemp(suffix=".tsv", dir=DFTK_TMPDIR)
+            try:
+                with os.fdopen(fd, "wb") as fh:
+                    fh.write(sys.stdin.buffer.read())
+                df = _csv_with_duckdb(path)
+            finally:
+                os.unlink(path)
+
         else:
             # stdin — read TSV directly
             df = _csv_from_bytes(sys.stdin.buffer.read())
@@ -385,7 +482,12 @@ class io:
         )
         g.add_argument(
             "--cast",
-            help="Cast column to dtype (COL:TYPE).  Glob patterns supported.",
+            help=(
+                "Cast column to dtype (COL:TYPE).  TYPE is one of object, int64, "
+                "Int64, float64, bool, datetime64, category (a prefix is enough, "
+                "e.g. int, Int, cat).  Int64 is an integer column that allows "
+                "empty cells.  Glob patterns supported."
+            ),
             nargs="*",
             default=None,
             metavar="COL:TYPE",
@@ -471,7 +573,11 @@ class io:
         )
         na_group.add_argument(
             "--dropna",
-            help="Drop rows containing any NA value before output.",
+            help=(
+                "Drop rows containing any NA value from the output.  Applied "
+                "after the command has run, so it does not change what the "
+                "command sees (use --prequery for that)."
+            ),
             action="store_true",
         )
 
@@ -530,17 +636,18 @@ class io:
                 raise
 
         # Cast columns
-        _DTYPES = ["object", "int64", "float64", "bool", "datetime64", "category"]
         for item in getattr(args, "cast", None) or []:
             try:
                 col, coltype = item.split(":")
-                coltype = fnmatch.filter(_DTYPES, f"{coltype}*")[0]
-                np_type = getattr(np, coltype)
+                # Case-sensitive: "int" is int64, "Int" is the nullable Int64
+                pattern = f"{coltype}*"
+                matches = [t for t in _CAST_TYPES if fnmatch.fnmatchcase(t, pattern)]
+                dtype = _CAST_TYPES[matches[0]]
                 for c in fnmatch.filter(df.columns, col):
                     try:
-                        df[c] = df[c].astype(np_type)
+                        df[c] = df[c].astype(dtype)
                     except Exception:
-                        df[c] = df[c].astype(float).astype(np_type)
+                        df[c] = df[c].astype(float).astype(dtype)
             except Exception:
                 logging.warning(f"--cast: failed for {item!r}")
 
@@ -665,8 +772,11 @@ class io:
 
         else:
             # TSV — stdout or named file
+            if digits is not None:
+                _warn_digits_exponent(df, digits)
             dest = output if output is not None else sys.stdout
-            df.to_csv(
+            _write_tsv(
+                df,
                 dest,
                 sep="\t",
                 index=False,