dataframe-toolkit 0.10.0__tar.gz → 0.11.0__tar.gz

{dataframe_toolkit-0.10.0 → dataframe_toolkit-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.10.0
+Version: 0.11.0
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson
@@ -117,6 +117,7 @@ dftk dataset tips -o \
 | `concat`     | Concatenate two or more tables row-wise                                              |
 | `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
 | `explode`    | Split separated values in a cell (`a,b`) into one row per value                      |
+| `implode`    | Join a column's values within groups into one cell, the inverse of `explode`         |
 | `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
 | `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
 | `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
@@ -223,6 +224,38 @@ printf "sample\tfluor\ns1\t60\ns2\t110\n" \
   | dftk interp - --ref stdcurve.tsv -x fluor --refx fluor -v conc -d conc_ng_ul
 ```
 
+### Filling missing values
+
+`eval -f` takes pandas column methods (`COL.method(...)`), and `fillna`,
+`ffill`, `bfill` and `interpolate` fill empty cells. Defaults from another
+table, after a merge (a column in both files gets the suffix `_r`):
+
+```bash
+dftk merge data.tsv -r defaults.tsv -k type -t left -o \
+  | dftk eval ... -f "depth = depth.fillna(depth_r)" --drop depth_r
+```
+
+The group mean, a fixed value, or the last value above (`ffill`) within
+each group:
+
+```bash
+dftk dataset penguins -o | dftk eval ... \
+  -f "mean = body_mass_g.groupby(species).transform('mean')" \
+  -f "body_mass_g = body_mass_g.fillna(mean)" --drop mean \
+  -f "sex = sex.fillna('unknown')"
+
+dftk dataset healthexp -o \
+  | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" -o \
+  | dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
+```
+
+`ffill`, `bfill` and `interpolate` use the rows in the order they are in:
+sort in a step of its own first (`dftk eval data.tsv --sortasc site day -o
+| dftk eval ... -f ...`), and use `groupby(...)` when the table holds
+several series, or values are taken from the neighbouring group.
+`interpolate()` has no grouped form and works by row position. `dftk eval
+-h` covers these cases, with examples.
+
 ### Bootstrap confidence intervals
 
 Bootstrap mode repeats the full stat computation N times on resampled data

{dataframe_toolkit-0.10.0 → dataframe_toolkit-0.11.0}/README.md

@@ -88,6 +88,7 @@ dftk dataset tips -o \
 | `concat`     | Concatenate two or more tables row-wise                                              |
 | `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
 | `explode`    | Split separated values in a cell (`a,b`) into one row per value                      |
+| `implode`    | Join a column's values within groups into one cell, the inverse of `explode`         |
 | `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
 | `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
 | `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
@@ -194,6 +195,38 @@ printf "sample\tfluor\ns1\t60\ns2\t110\n" \
   | dftk interp - --ref stdcurve.tsv -x fluor --refx fluor -v conc -d conc_ng_ul
 ```
 
+### Filling missing values
+
+`eval -f` takes pandas column methods (`COL.method(...)`), and `fillna`,
+`ffill`, `bfill` and `interpolate` fill empty cells. Defaults from another
+table, after a merge (a column in both files gets the suffix `_r`):
+
+```bash
+dftk merge data.tsv -r defaults.tsv -k type -t left -o \
+  | dftk eval ... -f "depth = depth.fillna(depth_r)" --drop depth_r
+```
+
+The group mean, a fixed value, or the last value above (`ffill`) within
+each group:
+
+```bash
+dftk dataset penguins -o | dftk eval ... \
+  -f "mean = body_mass_g.groupby(species).transform('mean')" \
+  -f "body_mass_g = body_mass_g.fillna(mean)" --drop mean \
+  -f "sex = sex.fillna('unknown')"
+
+dftk dataset healthexp -o \
+  | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" -o \
+  | dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
+```
+
+`ffill`, `bfill` and `interpolate` use the rows in the order they are in:
+sort in a step of its own first (`dftk eval data.tsv --sortasc site day -o
+| dftk eval ... -f ...`), and use `groupby(...)` when the table holds
+several series, or values are taken from the neighbouring group.
+`interpolate()` has no grouped form and works by row position. `dftk eval
+-h` covers these cases, with examples.
+
 ### Bootstrap confidence intervals
 
 Bootstrap mode repeats the full stat computation N times on resampled data

{dataframe_toolkit-0.10.0 → dataframe_toolkit-0.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.10.0"
+version = "0.11.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 keywords = [

{dataframe_toolkit-0.10.0 → dataframe_toolkit-0.11.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.10.0"
+version = "0.11.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [

{dataframe_toolkit-0.10.0 → dataframe_toolkit-0.11.0}/src/dftk/commands/__init__.py

@@ -31,6 +31,7 @@ COMMAND_GROUPS: list[tuple[str, list[tuple[str, str]]]] = [
             ("concat", "concat_cmd:ConcatCommand"),
             ("melt", "melt_cmd:MeltCommand"),
             ("explode", "explode_cmd:ExplodeCommand"),
+            ("implode", "implode_cmd:ImplodeCommand"),
             ("pivot", "pivot_cmd:PivotCommand"),
             ("func", "func_cmd:FuncCommand"),
             ("scale", "scale_cmd:ScaleCommand"),

{dataframe_toolkit-0.10.0 → dataframe_toolkit-0.11.0}/src/dftk/commands/eval_cmd.py

@@ -14,6 +14,7 @@ import fnmatch
 import hashlib
 import logging
 import os
+import re
 from functools import partial
 from pathlib import Path
 
@@ -44,6 +45,88 @@ arithmetic and boolean expressions directly on column names:
   dftk dataset tips -o | dftk eval ... -f "bill_and_tip = total_bill + tip * 2"
   dftk dataset tips -o | dftk eval ... -f "good_tip = (tip > 3) & (total_bill < 30)"
 
+Column methods are pandas Series methods, written COL.method(...); text
+methods are under .str (name.str.lower(), not name.lower() or tolower()):
+
+  x = x.abs()   x = x.round(1)   x = x.clip(lower=0)   y = name.str.upper()
+
+
+FILLING MISSING VALUES (-f)
+---------------------------
+fillna() fills the empty cells of a column; the other cells are kept:
+
+  dftk eval data.tsv -f "x = x.fillna(y)"          from another column
+  -f "x = x.fillna(0)"   -f "sex = sex.fillna('unknown')"     a fixed value
+  -f "x = x.fillna(y).fillna(0)"                   y first, then 0
+  -f "x_filled = x.fillna(y)"                      keep x, add a new column
+  -f "missing = x.isna()"                          mark the empty cells
+
+Defaults from another table: a column that is in both files gets the
+suffix _r from merge, so fill from it and drop it. Here the defaults are
+each species' median body mass:
+
+  dftk dataset penguins -o | dftk pivot ... -i species -v body_mass_g \\
+      -f median --move median_body_mass_g:body_mass_g -o defaults.parquet
+  dftk dataset penguins -o \\
+      | dftk merge ... -r defaults.parquet -k species -t left -o \\
+      | dftk eval ... -f "body_mass_g = body_mass_g.fillna(body_mass_g_r)" \\
+          --drop body_mass_g_r
+
+The group mean (or 'median', 'min', ...), in one step:
+
+  dftk dataset penguins -o | dftk eval ... \\
+      -f "mean = body_mass_g.groupby(species).transform('mean')" \\
+      -f "body_mass_g = body_mass_g.fillna(mean)" --drop mean
+
+From neighbouring rows (the co2 dataset is a weekly series with gaps):
+
+  dftk dataset co2 -o | dftk eval ... -f "ffill = co2.ffill()" \\
+      -f "bfill = co2.bfill()" -f "linear = co2.interpolate()"
+
+  ffill()          the last value above; the first rows stay empty
+  bfill()          the next value below; the last rows stay empty
+  ffill(limit=1)   fill at most one row after each value
+  interpolate()    a straight line between the values above and below, by
+                   row position: rows must be evenly spaced (for x-based
+                   lookup from a reference curve, see dftk interp)
+
+Row order matters: these methods use the rows in the order they are in.
+Sort first, in a step of its own (--sortasc is applied to the output, so
+in the same command it comes after the fill):
+
+  dftk eval data.tsv --sortasc site day -o | dftk eval ... -f "..."
+
+Several series in one table (sites, countries, samples) must be filled
+within each group, or values are taken from the neighbouring group:
+
+  -f "x = x.groupby(site).ffill()"
+  -f "x = x.groupby(site).bfill()"
+  -f "x = x.groupby(site).ffill().fillna(x.groupby(site).bfill())"
+
+A group's rows don't have to be next to each other, but within each group
+they must be in order. In healthexp, rows are sorted by Year with the
+countries mixed together, so a grouped ffill is right as it is:
+
+  dftk dataset healthexp -o \\
+      | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" -o \\
+      | dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
+
+interpolate() has no grouped form. Sort so that each group's rows are
+together; then gaps inside a group are right, but a gap at the start or
+end of a group is filled across the boundary. To leave those empty, keep
+only values with something before and after them in the same group:
+
+  dftk dataset healthexp -o \\
+      | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \\
+          --sortasc Country Year -o \\
+      | dftk eval ... -f "before = le.groupby(Country).ffill().notna()" \\
+          -f "after = le.groupby(Country).bfill().notna()" \\
+          -f "le_linear = le.interpolate().where(before & after)" \\
+          --drop before after
+
+Only empty cells are filled; missing rows (a year with no row at all) are
+not added.
+
 If df.eval() fails (e.g. for function calls), the expression is retried
 as a special function call using the syntax:
 
@@ -516,6 +599,68 @@ def _string_function(formula: str, joinsep: str):
 # ---------------------------------------------------------------------------
 
 
+def _formula_error(formula: str, exc: Exception) -> str:
+    """Return a short message for a mistake pandas found in an -f formula."""
+    msg = str(exc)
+    m = re.search(r"'Series' object has no attribute '(\w+)'", msg)
+    if m:
+        name = m.group(1)
+        col = re.search(rf"(\w+)\.{name}\b", formula)
+        col = col.group(1) if col else "COL"
+        if hasattr(pd.Series.str, name):
+            hint = f"Text methods are under .str: {col}.str.{name}()."
+        else:
+            hint = f"Text methods are under .str, e.g. {col}.str.lower()."
+        return (
+            f"-f {formula!r}: column has no method {name!r}. {hint}  See dftk eval -h."
+        )
+    return f"-f {formula!r}: {msg}"
+
+
+def _apply_special_function(df: pd.DataFrame, formula: str) -> None:
+    """Evaluate a special-function formula and store its column in *df*."""
+    dest, func, cols = _special_function(formula, list(df.columns))
+    if len(df) == 0:
+        df[dest] = True
+    elif isinstance(func, _WhereFunc):
+        df[dest] = func(df)
+    elif isinstance(func, _FrameFunc):
+        # Frame-level functions (idxmax/idxmin) need column labels
+        df[dest] = func(df[cols])
+    elif len(cols) == 1:
+        # Element-wise on a single column (e.g. np.sign, os.path.*)
+        df[dest] = df[cols[0]].apply(func)
+    else:
+        # Multi-column row function: pass numpy array so positional
+        # indexing (row[0], row[1], ...) works in pandas 2.x
+        df[dest] = df[cols].apply(lambda row, f=func: f(row.to_numpy()), axis=1)
+
+
+def _eval_formula(df: pd.DataFrame, formula: str) -> pd.DataFrame:
+    """Evaluate one -f formula: pandas df.eval(), else a special function."""
+    try:
+        # Not inplace: that writes into the existing column, which must keep
+        # its dtype, so x = x.fillna(y) or x = x.isna() failed whenever the
+        # result had another dtype
+        return df.eval(formula)
+    except (AttributeError, TypeError, NameError, KeyError) as exc:
+        # pandas ran the expression: a mistake in it, not a dftk bug
+        raise ValueError(_formula_error(formula, exc)) from exc
+    except (ValueError, SyntaxError) as eval_exc:
+        if re.match(r"\s*\w+\s*=\s*\w+\.", formula):
+            # dest = col.method(...) is never a special function
+            raise ValueError(_formula_error(formula, eval_exc)) from eval_exc
+        logger.debug(f"df.eval failed for {formula!r}, trying special_function")
+        try:
+            _apply_special_function(df, formula)
+        except ValueError as exc:
+            raise ValueError(f"{exc}  (pandas df.eval: {eval_exc})") from exc
+        except Exception:
+            logger.error(f"Could not evaluate formula: {formula!r}")
+            raise
+        return df
+
+
 def _eval(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
     joinsep: str = getattr(args, "joinsep", "")
     intbool: bool = getattr(args, "intbool", False)
@@ -536,31 +681,11 @@ def _eval(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
     # -f: pandas eval with special-function fallback
     for formula in getattr(args, "formula", []):
         try:
-            df.eval(formula, inplace=True)
-        except (ValueError, SyntaxError):
-            logger.debug(f"df.eval failed for {formula!r}, trying special_function")
-            dest, func, cols = _special_function(formula, list(df.columns))
-            if len(df) == 0:
-                df[dest] = True
-            elif isinstance(func, _WhereFunc):
-                df[dest] = func(df)
-            elif isinstance(func, _FrameFunc):
-                # Frame-level functions (idxmax/idxmin) need column labels
-                df[dest] = func(df[cols])
-            elif len(cols) == 1:
-                # Element-wise on a single column (e.g. np.sign, os.path.*)
-                df[dest] = df[cols[0]].apply(func)
-            else:
-                # Multi-column row function: pass numpy array so positional
-                # indexing (row[0], row[1], ...) works in pandas 2.x
-                df[dest] = df[cols].apply(lambda row, f=func: f(row.to_numpy()), axis=1)
+            df = _eval_formula(df, formula)
         except Exception:
-            if len(df) == 0:
-                dest = formula.split("=", 1)[0].strip()
-                df[dest] = True
-            else:
-                logger.error(f"Could not evaluate formula: {formula!r}")
+            if len(df) != 0:
                 raise
+            df[formula.split("=", 1)[0].strip()] = True
 
     # -s: string / row functions
     for strfunc in getattr(args, "strfunc", []):

dataframe_toolkit-0.11.0/src/dftk/commands/implode_cmd.py

@@ -0,0 +1,189 @@
+"""
+dftk.commands.implode_cmd — dftk implode subcommand.
+
+The inverse of explode: join the values of a column within each group of
+rows into one cell ("GO:1,GO:2"). Several columns are joined in the same
+row order, so explode -c a b followed by implode -c a b gives back the
+original rows.
+"""
+
+import argparse
+
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import check_cols, io
+
+_EPILOG = """\
+EXAMPLE
+-------
+  gene   go    tissue
+  BRCA1  GO:1  breast
+  BRCA1  GO:2  breast
+  TP53   GO:3  colon
+
+  dftk implode data.tsv -c go
+
+  gene   go         tissue
+  BRCA1  GO:1,GO:2  breast
+  TP53   GO:3       colon
+
+By default rows are grouped by all the other columns, so rows are only
+joined when everything else in them is the same. With -i, rows are grouped
+by those columns only and the other columns are dropped. Rows with an
+empty key form a group of their own.
+
+Values are joined in the order of the rows; --sortvalues sorts them and
+--unique drops repeats. Empty values are left out, and a group with none
+gets an empty cell. Whole numbers are written without ".0" when every
+value in the column is one (pandas reads integers with gaps as floats).
+
+Several columns are joined row by row, so their n-th items still belong
+together; --unique and --sortvalues then work on whole rows.
+
+Runnable version: each species' islands, one cell per species:
+
+  dftk dataset penguins -o \\
+      | dftk implode ... -c island -i species --unique --sortvalues
+
+Numbers, with another separator: the party sizes seen on each day:
+
+  dftk dataset tips -o \\
+      | dftk implode ... -c size -i day --unique --sortvalues --sep '|'
+
+Undoing explode: --poscol puts the items back in their original order,
+here after shuffling the rows, and is dropped from the output:
+
+  dftk dataset penguins -o \\
+      | dftk implode ... -c island -i species --unique -o \\
+      | dftk explode ... -c island --poscol n -o \\
+      | dftk sample ... -f 1 --randomseed 1 -o \\
+      | dftk implode ... -c island --poscol n
+"""
+
+
+def _as_text(s: pd.Series) -> pd.Series:
+    """Return *s* as strings, keeping missing values missing."""
+    values = s.dropna()
+    if (
+        pd.api.types.is_float_dtype(s)
+        and len(values)
+        and (values == values.round()).all()
+    ):
+        # Integers with gaps are read as floats: write 10, not 10.0
+        s = s.astype("Int64")
+    text = s.astype(object).where(s.notna())
+    text = text.map(lambda v: v if pd.isna(v) else str(v))
+    return text.where(text != "")
+
+
+def _implode(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
+    cols = args.cols
+    poscol = args.poscol
+    if len(set(cols)) != len(cols):
+        raise ValueError(f"-c/--cols lists a column more than once: {cols}")
+    if poscol is not None and poscol in cols:
+        raise ValueError(f"--poscol {poscol!r} is also in -c/--cols")
+
+    if args.indexcols is not None:
+        keys = list(args.indexcols)
+        overlap = [c for c in keys if c in cols or c == poscol]
+        if overlap:
+            raise ValueError(
+                f"-i/--indexcols overlaps -c/--cols or --poscol: {overlap}"
+            )
+    else:
+        keys = [c for c in df.columns if c not in cols and c != poscol]
+
+    order = [c for c in df.columns if c in keys or c in cols]
+    work = df[keys + cols + ([poscol] if poscol else [])].reset_index(drop=True)
+    if keys:
+        work["_gid"] = work.groupby(keys, sort=False, dropna=False).ngroup()
+    else:
+        work["_gid"] = 0
+
+    if args.sortvalues:
+        work = work.sort_values(["_gid", *cols], kind="stable", na_position="last")
+    elif poscol:
+        work = work.sort_values(["_gid", poscol], kind="stable")
+    else:
+        work = work.sort_values("_gid", kind="stable")
+    if args.unique:
+        work = work.drop_duplicates(subset=["_gid", *cols])
+
+    for col in cols:
+        work[col] = _as_text(work[col])
+    grouped = work.groupby("_gid", sort=True)
+    joined = grouped[cols].agg(lambda s: args.sep.join(s.dropna()))
+    joined = joined.where(joined != "")
+    result = grouped[keys].first().join(joined) if keys else joined
+    return result.reset_index(drop=True)[order]
+
+
+class ImplodeCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "implode"
+
+    @property
+    def help(self) -> str:
+        return "Join a column's values within groups of rows into one cell"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("implode options")
+        g.add_argument(
+            "-c",
+            "--cols",
+            nargs="+",
+            required=True,
+            metavar="COL",
+            help="Column(s) whose values are joined. Several columns are "
+            "joined in the same row order.",
+        )
+        g.add_argument(
+            "-i",
+            "--indexcols",
+            nargs="+",
+            default=None,
+            metavar="COL",
+            help="Group by these columns and drop the others "
+            "(default: group by all columns not in -c or --poscol).",
+        )
+        g.add_argument(
+            "--sep",
+            default=",",
+            metavar="SEP",
+            help="Separator between joined values (default: ,).",
+        )
+        g.add_argument(
+            "--poscol",
+            default=None,
+            metavar="NAME",
+            help="Join the values in the order of column NAME (e.g. the "
+            "positions from explode --poscol) and drop it.",
+        )
+        g.add_argument(
+            "--unique",
+            action="store_true",
+            help="Drop repeated values (with several -c columns, repeated rows).",
+        )
+        g.add_argument(
+            "--sortvalues",
+            action="store_true",
+            help="Sort the values in each cell (with several -c columns, "
+            "sort whole rows by them).",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+        check_cols(df, args.cols, "-c/--cols")
+        check_cols(df, args.indexcols, "-i/--indexcols")
+        if args.poscol is not None:
+            check_cols(df, [args.poscol], "--poscol")
+        result = _implode(df, args)
+        io.printdf(result, args)