dataframe-toolkit 0.9.0__tar.gz → 0.10.0__tar.gz

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.9.0
+Version: 0.10.0
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson
@@ -116,6 +116,7 @@ dftk dataset tips -o \
 | `merge`      | Join two tables on key columns (inner/left/right/outer)                              |
 | `concat`     | Concatenate two or more tables row-wise                                              |
 | `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
+| `explode`    | Split separated values in a cell (`a,b`) into one row per value                      |
 | `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
 | `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
 | `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
@@ -251,6 +252,21 @@ All commands accept:
 > parquet and prints *its path* to stdout — `-` would try to parse that path
 > as TSV and fail.
 
+### Comment lines and preambles
+
+dftk has no comment or skip-rows option: `#` is ordinary data, so headers
+like `Count #` and values like `A #1` are read as they are. Drop comment
+lines or a preamble in the shell instead:
+
+```bash
+grep -v '^#' data.tsv | dftk print           # drop every line starting with #
+tail -n +3 data.tsv | dftk print             # skip a 2-line preamble
+dftk print <(grep -v '^#' data.tsv)          # the same, as a filename
+```
+
+With `--backend duckdb`, a whole-line `#` comment below the header is an
+error, and `#` lines above the header are skipped.
+
 Standard output options (available on all tabular commands):
 
 - `-o` / `--output` — controls where output goes; takes an *optional* value:

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.10.0}/README.md

@@ -87,6 +87,7 @@ dftk dataset tips -o \
 | `merge`      | Join two tables on key columns (inner/left/right/outer)                              |
 | `concat`     | Concatenate two or more tables row-wise                                              |
 | `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
+| `explode`    | Split separated values in a cell (`a,b`) into one row per value                      |
 | `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
 | `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
 | `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
@@ -222,6 +223,21 @@ All commands accept:
 > parquet and prints *its path* to stdout — `-` would try to parse that path
 > as TSV and fail.
 
+### Comment lines and preambles
+
+dftk has no comment or skip-rows option: `#` is ordinary data, so headers
+like `Count #` and values like `A #1` are read as they are. Drop comment
+lines or a preamble in the shell instead:
+
+```bash
+grep -v '^#' data.tsv | dftk print           # drop every line starting with #
+tail -n +3 data.tsv | dftk print             # skip a 2-line preamble
+dftk print <(grep -v '^#' data.tsv)          # the same, as a filename
+```
+
+With `--backend duckdb`, a whole-line `#` comment below the header is an
+error, and `#` lines above the header are skipped.
+
 Standard output options (available on all tabular commands):
 
 - `-o` / `--output` — controls where output goes; takes an *optional* value:

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.9.0"
+version = "0.10.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 keywords = [

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.10.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.9.0"
+version = "0.10.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.10.0}/src/dftk/commands/__init__.py

@@ -30,6 +30,7 @@ COMMAND_GROUPS: list[tuple[str, list[tuple[str, str]]]] = [
             ("merge", "merge_cmd:MergeCommand"),
             ("concat", "concat_cmd:ConcatCommand"),
             ("melt", "melt_cmd:MeltCommand"),
+            ("explode", "explode_cmd:ExplodeCommand"),
             ("pivot", "pivot_cmd:PivotCommand"),
             ("func", "func_cmd:FuncCommand"),
             ("scale", "scale_cmd:ScaleCommand"),

dataframe_toolkit-0.10.0/src/dftk/commands/explode_cmd.py

@@ -0,0 +1,180 @@
+"""
+dftk.commands.explode_cmd — dftk explode subcommand.
+
+Split a column of separated values (e.g. "GO:1, GO:2") into one row per
+value, repeating the other columns on each new row (pandas str.split +
+explode). Several columns are split in parallel: their n-th items end up
+on the same row, so every row must have the same number of items in them.
+"""
+
+import argparse
+
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import check_cols, io
+
+_EPILOG = """\
+EXAMPLE
+-------
+  gene   go          tissue
+  BRCA1  GO:1, GO:2  breast
+  TP53   GO:3        colon
+
+  dftk explode data.tsv -c go
+
+  gene   go    tissue
+  BRCA1  GO:1  breast
+  BRCA1  GO:2  breast
+  TP53   GO:3  colon
+
+Spaces around each item are removed. An empty cell stays as one row with
+an empty value; --dropempty removes such rows. If all the new values in a
+split column are numbers, the column becomes numeric (integers stay
+integers); columns named in --readasobject stay text.
+
+Several columns are split in parallel, item by item. For every
+combination instead, run explode once per column:
+
+  id  a    b
+  1   x,y  10,20
+
+  dftk explode data.tsv -c a b       ->  (1, x, 10), (1, y, 20)
+
+Runnable version: collect each species' islands into one comma-separated
+cell with SQL, then split them back into rows, numbering each item:
+
+  dftk dataset penguins -o \\
+      | dftk query ... -o --sql "SELECT species,
+            string_agg(DISTINCT island, ',' ORDER BY island) AS islands
+            FROM data GROUP BY species ORDER BY species" \\
+      | dftk explode ... -c islands --poscol n
+
+Another separator, keeping only some columns:
+
+  dftk dataset tips -o \\
+      | dftk query ... -o --sql "SELECT day,
+            string_agg(DISTINCT size::VARCHAR, '|' ORDER BY size::VARCHAR) AS sizes
+            FROM data GROUP BY day ORDER BY day" \\
+      | dftk explode ... -c sizes --sep '|' --select sizes day
+"""
+
+
+def _split_cell(value, sep: str):
+    if pd.isna(value):
+        return value
+    return [item.strip() for item in str(value).split(sep)]
+
+
+def _to_numeric(s: pd.Series) -> pd.Series:
+    """Return *s* as integers or floats if every non-empty value is one."""
+    values = s.dropna()
+    if values.empty:
+        return s
+    text = values.astype(str)
+    if text.str.fullmatch(r"[+-]?\d+").all():
+        return s.astype("Int64") if s.isna().any() else s.astype("int64")
+    try:
+        return pd.to_numeric(s)
+    except (ValueError, TypeError):
+        return s
+
+
+def _explode(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
+    cols = args.cols
+    if not args.sep:
+        raise ValueError("--sep must not be empty")
+    if len(set(cols)) != len(cols):
+        raise ValueError(f"-c/--cols lists a column more than once: {cols}")
+    if args.poscol is not None and args.poscol in df.columns:
+        raise ValueError(f"--poscol {args.poscol!r} is already a column")
+
+    df = df.reset_index(drop=True)
+    for col in cols:
+        df[col] = df[col].map(lambda v: _split_cell(v, args.sep)).astype(object)
+
+    if len(cols) > 1:
+        counts = pd.DataFrame(
+            {c: df[c].map(lambda v: len(v) if isinstance(v, list) else 1) for c in cols}
+        )
+        bad = counts.index[counts.nunique(axis=1) > 1]
+        if len(bad):
+            shown = "; ".join(
+                f"row {i + 1} ("
+                + ", ".join(f"{c}: {counts.at[i, c]}" for c in cols)
+                + ")"
+                for i in bad[:5]
+            )
+            more = f" and {len(bad) - 5} more" if len(bad) > 5 else ""
+            raise ValueError(
+                f"Rows with different numbers of items in -c/--cols: {shown}{more}. "
+                "Run explode once per column for every combination."
+            )
+
+    result = df.explode(cols)
+    if args.poscol is not None:
+        result[args.poscol] = result.groupby(level=0).cumcount() + 1
+    result = result.reset_index(drop=True)
+
+    for col in cols:
+        result[col] = result[col].replace("", pd.NA)
+    if args.dropempty:
+        result = result[result[cols].notna().any(axis=1)].reset_index(drop=True)
+
+    readasobject = getattr(args, "readasobject", None)
+    for col in cols:
+        if readasobject is not None and (not readasobject or col in readasobject):
+            continue
+        result[col] = _to_numeric(result[col])
+    return result
+
+
+class ExplodeCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "explode"
+
+    @property
+    def help(self) -> str:
+        return "Split separated values in a column into one row per value"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("explode options")
+        g.add_argument(
+            "-c",
+            "--cols",
+            nargs="+",
+            required=True,
+            metavar="COL",
+            help="Column(s) to split. Several columns are split in parallel "
+            "and must have the same number of items on each row.",
+        )
+        g.add_argument(
+            "--sep",
+            default=",",
+            metavar="SEP",
+            help="Separator between items, a plain string, not a regex (default: ,).",
+        )
+        g.add_argument(
+            "--poscol",
+            default=None,
+            metavar="NAME",
+            help="Add a column NAME with each item's position in its cell (1, 2, ...).",
+        )
+        g.add_argument(
+            "--dropempty",
+            action="store_true",
+            help="Drop rows whose split columns are all empty "
+            "(default: keep them, so no input row is lost).",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+        check_cols(df, args.cols, "-c/--cols")
+        result = _explode(df, args)
+        io.printdf(result, args)

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.10.0}/src/dftk/commands/merge_cmd.py

@@ -17,6 +17,8 @@ Key features
 * --select [:right:]  expands to all right-file columns, suppressing ``_l``.
 * Both input files can be TSV or Parquet (.parquet extension auto-detected).
   The left file defaults to stdin, making it parquet-pipe friendly.
+* Integer columns that a left, right or outer join leaves gaps in become
+  nullable (Int64) instead of float64.
 """
 
 import argparse
@@ -107,6 +109,35 @@ def _add_cross_col(
     args.drop = drop
 
 
+def _gap_sides(args: argparse.Namespace) -> tuple[bool, bool]:
+    """Return (left, right): which side's columns can get missing values.
+
+    --only runs an outer join before filtering, so both sides count.
+    """
+    return args.type in ("right", "outer"), args.type in ("left", "outer")
+
+
+def _nullable_ints(df: pd.DataFrame, keep: list[str]) -> pd.DataFrame:
+    """Return *df* with numpy integer columns (except *keep*) as Int64 etc.
+
+    A plain int64 column cannot hold a missing value, so pandas turns it into
+    float64 when a join leaves rows unmatched; the nullable type keeps the
+    integers.
+    """
+    cols = [
+        c
+        for c in df.columns
+        if c not in keep
+        and pd.api.types.is_integer_dtype(df[c])
+        and not isinstance(df[c].dtype, pd.api.extensions.ExtensionDtype)
+    ]
+    if not cols:
+        return df
+    # int32 -> Int32, uint8 -> UInt8, ...
+    dtypes = {c: df[c].dtype.name.capitalize().replace("Uint", "UInt") for c in cols}
+    return df.astype(dtypes)
+
+
 def _filter_only(
     df: pd.DataFrame,
     df_left: pd.DataFrame,
@@ -152,6 +183,15 @@ def _do_merge(
 
     suffixes = _expand_select(list(df_left.columns), list(df_right.columns), args)
 
+    # Keep integer columns integers when the join leaves gaps in them.  The
+    # -k keys are filled from both sides, so they never get gaps.
+    keys = args.keys or []
+    left_gaps, right_gaps = _gap_sides(args)
+    if left_gaps:
+        df_left = _nullable_ints(df_left, keys)
+    if right_gaps:
+        df_right = _nullable_ints(df_right, keys)
+
     if args.keys is not None:
         df_merged = pd.merge(
             df_left,
@@ -278,6 +318,14 @@ would be read as if it were the data.
 Named-file self-merge (no stdin required):
 
   dftk merge penguins.tsv -r penguins_adelie.tsv -k species --only left
+
+INTEGER COLUMNS
+---------------
+A left, right or outer join leaves the other side's columns empty in
+unmatched rows.  Integer columns there are made nullable (Int64), so they
+stay integers (100, not 100.0); the -k key columns are not changed.  Read
+TSV input with --backend duckdb to keep integer columns that already have
+empty cells as integers too.
 """
 
 

{dataframe_toolkit-0.9.0 → dataframe_toolkit-0.10.0}/src/dftk/common/io.py

@@ -27,6 +27,7 @@ import fnmatch
 import io as _io
 import logging
 import os
+import shutil
 import sys
 import tempfile
 
@@ -365,14 +366,30 @@ class io:
             return _name_columns(df)
 
         def _csv_with_duckdb(path: str) -> pd.DataFrame:
-            # DuckDB keeps integer columns with blanks as Int64 (pandas: float64)
+            # DuckDB keeps integer columns with blanks as Int64 (pandas: float64).
+            # comment="" stops it guessing "#" as a comment character: after one
+            # whole-line "#" comment it cuts every line at "#", headers included
             try:
-                df = duckdb.read_csv(path, sep=sep, header=not noheader).df()
+                df = duckdb.read_csv(
+                    path, sep=sep, header=not noheader, comment=""
+                ).df()
             except Exception as exc:
                 logging.error(f"DuckDB failed to read {path!r}: {exc}")
                 raise
             return _name_columns(df)
 
+        def _spooled_csv_with_duckdb(stream) -> pd.DataFrame:
+            # DuckDB reads its input twice (type sniffing, then data), so a
+            # stream must be copied to a temp file first
+            os.makedirs(DFTK_TMPDIR, exist_ok=True)
+            fd, path = tempfile.mkstemp(suffix=".tsv", dir=DFTK_TMPDIR)
+            try:
+                with os.fdopen(fd, "wb") as fh:
+                    shutil.copyfileobj(stream, fh)
+                return _csv_with_duckdb(path)
+            finally:
+                os.unlink(path)
+
         def _csv_from_file(path: str) -> pd.DataFrame:
             df = _pandas_csv(path)
             df = _with_blank_lines(
@@ -403,21 +420,18 @@ class io:
                 df = pd.read_parquet(filename)
                 df.attrs["_parquet_meta"] = _read_parquet_meta(filename)
             elif backend == "duckdb" and duckdb is not None:
-                df = _csv_with_duckdb(filename)
+                if os.path.exists(filename) and not os.path.isfile(filename):
+                    # A pipe, e.g. <(grep -v '^#' file): one read would use it up
+                    with open(filename, "rb") as fh:
+                        df = _spooled_csv_with_duckdb(fh)
+                else:
+                    df = _csv_with_duckdb(filename)
             else:
                 df = _csv_from_file(filename)
 
         elif backend == "duckdb" and duckdb is not None:
-            # stdin with DuckDB: it reads files, not buffers (that needs fsspec),
-            # so spool stdin to a temp file and read it like a named file
-            os.makedirs(DFTK_TMPDIR, exist_ok=True)
-            fd, path = tempfile.mkstemp(suffix=".tsv", dir=DFTK_TMPDIR)
-            try:
-                with os.fdopen(fd, "wb") as fh:
-                    fh.write(sys.stdin.buffer.read())
-                df = _csv_with_duckdb(path)
-            finally:
-                os.unlink(path)
+            # stdin with DuckDB: it reads files, not buffers (that needs fsspec)
+            df = _spooled_csv_with_duckdb(sys.stdin.buffer)
 
         else:
             # stdin — read TSV directly