dataframe-toolkit 0.9.0__tar.gz → 0.9.1__tar.gz

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.9.0
+Version: 0.9.1
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson
@@ -251,6 +251,21 @@ All commands accept:
 > parquet and prints *its path* to stdout — `-` would try to parse that path
 > as TSV and fail.
 
+### Comment lines and preambles
+
+dftk has no comment or skip-rows option: `#` is ordinary data, so headers
+like `Count #` and values like `A #1` are read as they are. Drop comment
+lines or a preamble in the shell instead:
+
+```bash
+grep -v '^#' data.tsv | dftk print           # drop every line starting with #
+tail -n +3 data.tsv | dftk print             # skip a 2-line preamble
+dftk print <(grep -v '^#' data.tsv)          # the same, as a filename
+```
+
+With `--backend duckdb`, a whole-line `#` comment below the header is an
+error, and `#` lines above the header are skipped.
+
 Standard output options (available on all tabular commands):
 
 - `-o` / `--output` — controls where output goes; takes an *optional* value:

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.9.1}/README.md

@@ -222,6 +222,21 @@ All commands accept:
 > parquet and prints *its path* to stdout — `-` would try to parse that path
 > as TSV and fail.
 
+### Comment lines and preambles
+
+dftk has no comment or skip-rows option: `#` is ordinary data, so headers
+like `Count #` and values like `A #1` are read as they are. Drop comment
+lines or a preamble in the shell instead:
+
+```bash
+grep -v '^#' data.tsv | dftk print           # drop every line starting with #
+tail -n +3 data.tsv | dftk print             # skip a 2-line preamble
+dftk print <(grep -v '^#' data.tsv)          # the same, as a filename
+```
+
+With `--backend duckdb`, a whole-line `#` comment below the header is an
+error, and `#` lines above the header are skipped.
+
 Standard output options (available on all tabular commands):
 
 - `-o` / `--output` — controls where output goes; takes an *optional* value:

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.9.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.9.0"
+version = "0.9.1"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 keywords = [

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.9.1}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.9.0"
+version = "0.9.1"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.9.1}/src/dftk/commands/merge_cmd.py

@@ -17,6 +17,8 @@ Key features
 * --select [:right:]  expands to all right-file columns, suppressing ``_l``.
 * Both input files can be TSV or Parquet (.parquet extension auto-detected).
   The left file defaults to stdin, making it parquet-pipe friendly.
+* Integer columns that a left, right or outer join leaves gaps in become
+  nullable (Int64) instead of float64.
 """
 
 import argparse
@@ -107,6 +109,35 @@ def _add_cross_col(
     args.drop = drop
 
 
+def _gap_sides(args: argparse.Namespace) -> tuple[bool, bool]:
+    """Return (left, right): which side's columns can get missing values.
+
+    --only runs an outer join before filtering, so both sides count.
+    """
+    return args.type in ("right", "outer"), args.type in ("left", "outer")
+
+
+def _nullable_ints(df: pd.DataFrame, keep: list[str]) -> pd.DataFrame:
+    """Return *df* with numpy integer columns (except *keep*) as Int64 etc.
+
+    A plain int64 column cannot hold a missing value, so pandas turns it into
+    float64 when a join leaves rows unmatched; the nullable type keeps the
+    integers.
+    """
+    cols = [
+        c
+        for c in df.columns
+        if c not in keep
+        and pd.api.types.is_integer_dtype(df[c])
+        and not isinstance(df[c].dtype, pd.api.extensions.ExtensionDtype)
+    ]
+    if not cols:
+        return df
+    # int32 -> Int32, uint8 -> UInt8, ...
+    dtypes = {c: df[c].dtype.name.capitalize().replace("Uint", "UInt") for c in cols}
+    return df.astype(dtypes)
+
+
 def _filter_only(
     df: pd.DataFrame,
     df_left: pd.DataFrame,
@@ -152,6 +183,15 @@ def _do_merge(
 
     suffixes = _expand_select(list(df_left.columns), list(df_right.columns), args)
 
+    # Keep integer columns integers when the join leaves gaps in them.  The
+    # -k keys are filled from both sides, so they never get gaps.
+    keys = args.keys or []
+    left_gaps, right_gaps = _gap_sides(args)
+    if left_gaps:
+        df_left = _nullable_ints(df_left, keys)
+    if right_gaps:
+        df_right = _nullable_ints(df_right, keys)
+
     if args.keys is not None:
         df_merged = pd.merge(
             df_left,
@@ -278,6 +318,14 @@ would be read as if it were the data.
 Named-file self-merge (no stdin required):
 
   dftk merge penguins.tsv -r penguins_adelie.tsv -k species --only left
+
+INTEGER COLUMNS
+---------------
+A left, right or outer join leaves the other side's columns empty in
+unmatched rows.  Integer columns there are made nullable (Int64), so they
+stay integers (100, not 100.0); the -k key columns are not changed.  Read
+TSV input with --backend duckdb to keep integer columns that already have
+empty cells as integers too.
 """
 
 

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.9.1}/src/dftk/common/io.py

@@ -27,6 +27,7 @@ import fnmatch
 import io as _io
 import logging
 import os
+import shutil
 import sys
 import tempfile
 
@@ -365,14 +366,30 @@ class io:
             return _name_columns(df)
 
         def _csv_with_duckdb(path: str) -> pd.DataFrame:
-            # DuckDB keeps integer columns with blanks as Int64 (pandas: float64)
+            # DuckDB keeps integer columns with blanks as Int64 (pandas: float64).
+            # comment="" stops it guessing "#" as a comment character: after one
+            # whole-line "#" comment it cuts every line at "#", headers included
             try:
-                df = duckdb.read_csv(path, sep=sep, header=not noheader).df()
+                df = duckdb.read_csv(
+                    path, sep=sep, header=not noheader, comment=""
+                ).df()
             except Exception as exc:
                 logging.error(f"DuckDB failed to read {path!r}: {exc}")
                 raise
             return _name_columns(df)
 
+        def _spooled_csv_with_duckdb(stream) -> pd.DataFrame:
+            # DuckDB reads its input twice (type sniffing, then data), so a
+            # stream must be copied to a temp file first
+            os.makedirs(DFTK_TMPDIR, exist_ok=True)
+            fd, path = tempfile.mkstemp(suffix=".tsv", dir=DFTK_TMPDIR)
+            try:
+                with os.fdopen(fd, "wb") as fh:
+                    shutil.copyfileobj(stream, fh)
+                return _csv_with_duckdb(path)
+            finally:
+                os.unlink(path)
+
         def _csv_from_file(path: str) -> pd.DataFrame:
             df = _pandas_csv(path)
             df = _with_blank_lines(
@@ -403,21 +420,18 @@ class io:
                 df = pd.read_parquet(filename)
                 df.attrs["_parquet_meta"] = _read_parquet_meta(filename)
             elif backend == "duckdb" and duckdb is not None:
-                df = _csv_with_duckdb(filename)
+                if os.path.exists(filename) and not os.path.isfile(filename):
+                    # A pipe, e.g. <(grep -v '^#' file): one read would use it up
+                    with open(filename, "rb") as fh:
+                        df = _spooled_csv_with_duckdb(fh)
+                else:
+                    df = _csv_with_duckdb(filename)
             else:
                 df = _csv_from_file(filename)
 
         elif backend == "duckdb" and duckdb is not None:
-            # stdin with DuckDB: it reads files, not buffers (that needs fsspec),
-            # so spool stdin to a temp file and read it like a named file
-            os.makedirs(DFTK_TMPDIR, exist_ok=True)
-            fd, path = tempfile.mkstemp(suffix=".tsv", dir=DFTK_TMPDIR)
-            try:
-                with os.fdopen(fd, "wb") as fh:
-                    fh.write(sys.stdin.buffer.read())
-                df = _csv_with_duckdb(path)
-            finally:
-                os.unlink(path)
+            # stdin with DuckDB: it reads files, not buffers (that needs fsspec)
+            df = _spooled_csv_with_duckdb(sys.stdin.buffer)
 
         else:
             # stdin — read TSV directly