dataframe-toolkit 0.4.1__py3-none-any.whl

dftk/commands/pivot_cmd.py

@@ -0,0 +1,438 @@
+"""
+dftk.commands.pivot_cmd — dftk pivot subcommand.
+
+Wraps pandas pivot_table with optional bootstrap resampling and three
+sampling strategies for hierarchical data.
+"""
+
+import argparse
+import logging
+
+import numpy as np
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import check_cols, io
+from dftk.common.seed import normalize_seed
+
+logger = logging.getLogger(__name__)
+
+_EPILOG = """\
+AGGREGATION FUNCTIONS (-f / --aggfunc)
+---------------------------------------
+Pass one or more names from this list:
+
+  count  min  max  mean  median  mode  sum  std  var  skew  kurt  prod
+  cilo   cihi   (confidence-interval bounds, level set by --confidencelevel)
+  bitwise_and  bitwise_or  bitwise_xor
+
+Bare names apply the same function(s) to every value column:
+
+  dftk pivot data.tsv -v sales cost -i region -f mean std
+
+Per-column functions use COL:FUNC syntax — all items must use this form if
+any do:
+
+  dftk pivot data.tsv -v sales cost -i region -f sales:mean cost:sum
+
+When -f is omitted, pandas default aggregation (mean) is used.
+
+
+BOOTSTRAP SAMPLING MODES (--bootstrap N)
+-----------------------------------------
+Three sampling strategies are available, determined by which flags are given.
+
+  1. Full resampling (--fullsampling)
+     Sample every row in the dataset with replacement, ignoring group
+     structure.  Use this when there is no meaningful grouping to preserve.
+
+       dftk pivot data.tsv -v value -i subject -f mean \\
+         --bootstrap 500 --fullsampling
+
+  2. Group-based row sampling (--samplinggroup COLS)
+     Resample rows with replacement *within* each group defined by
+     COLS.  Group sizes are preserved across samples.
+
+       dftk pivot data.tsv -v value -i timepoint -g condition -f mean \\
+         --bootstrap 500 --samplinggroup subject
+
+     If neither --fullsampling nor --samplinggroup nor --samplingcols is
+     given, the sampling groups default to the union of --index and
+     --groupcols (the pivot dimensions themselves).
+
+  3. Hierarchical (parent/child) sampling (--samplingcols COLS)
+     Use when each logical unit has multiple data rows (e.g. a subject has
+     many measurements).  Sampling parents and pulling all their children
+     keeps the within-unit structure intact.
+
+     Steps:
+       a. Build a sampling set of unique combinations of --samplingcols
+          (the parent identifiers).
+       b. Bootstrap-resample that set with replacement — optionally within
+          groups defined by --samplinggroup.
+       c. Merge the resampled set back against the full DataFrame, pulling
+          in all child rows for each selected parent.
+
+     If parent P is drawn twice, all of P's rows appear twice in the
+     bootstrap sample; if Q is not drawn, none of Q's rows appear.
+
+       # Sample subjects (parents) within each sex group;
+       # each subject has multiple visit rows (children).
+       dftk pivot data.tsv -v score -i visit -g condition -f mean \\
+         --bootstrap 500 \\
+         --samplingcols subject_id \\
+         --samplinggroup sex
+
+     Note: --samplingcols values are combined with --samplinggroup values
+     to form the merge key, so the group column must also be in the data.
+
+
+PIPE MODE
+---------
+Use -o alone to pass data to the next dftk command without writing to a
+file:
+
+  cat data.tsv \\
+    | dftk pivot - -v value -i subject -g condition -f mean -o \\
+    | dftk stat - -c condition_A condition_B
+"""
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _ci(name: str, level: float, method: str):
+    """Return a named percentile function for use as a pandas agg function."""
+
+    def cifunction(a):
+        return np.percentile(a, q=level, method=method)
+
+    cifunction.__name__ = name
+    return cifunction
+
+
+def _bitwise(op: str):
+    """Return a named bitwise-reduce function."""
+
+    def bitfunction(a):
+        if op == "and":
+            return np.bitwise_and.reduce(a)
+        if op == "or":
+            return np.bitwise_or.reduce(a)
+        if op == "xor":
+            return np.bitwise_xor.reduce(a)
+
+    bitfunction.__name__ = op
+    return bitfunction
+
+
+def _prepare_funcs(
+    aggfuncs: list[str] | None, confidencelevel: float, confidencemethod: str
+):
+    """Convert aggfunc name strings into a pandas-compatible aggfunc argument.
+
+    Returns:
+        None          → pandas default (mean)
+        list          → same function(s) applied to all value columns
+        dict          → per-column functions, keyed by column name
+    """
+    if aggfuncs is None:
+        return None
+
+    funclist: list = []
+    funcdict: dict = {}
+
+    for item in aggfuncs:
+        colname = None
+        funcname = item
+        if ":" in item:
+            colname, funcname = item.split(":", 1)
+            funcdict.setdefault(colname, [])
+
+        if funcname in {
+            "count",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "mode",
+            "sum",
+            "std",
+            "var",
+            "skew",
+            "kurt",
+            "prod",
+        }:
+            func = funcname
+        elif funcname == "cilo":
+            lo = (100.0 - confidencelevel) / 2.0
+            func = _ci("cilo", lo, confidencemethod)
+        elif funcname == "cihi":
+            hi = 100.0 - (100.0 - confidencelevel) / 2.0
+            func = _ci("cihi", hi, confidencemethod)
+        elif funcname in {"bitwise_and", "bitwise_or", "bitwise_xor"}:
+            func = _bitwise(funcname.split("_", 1)[1])
+        else:
+            try:
+                func = getattr(np, funcname)
+            except AttributeError:
+                raise ValueError(  # noqa: E501
+                    f"Unknown aggregation function: {funcname!r}"
+                ) from None
+
+        if funcdict and colname is None:
+            raise ValueError(
+                "Mixed aggfunc spec: either all items must use COL:FUNC "
+                "syntax or none must."
+            )
+
+        if colname is not None:
+            funcdict[colname].append(func)
+        else:
+            funclist.append(func)
+
+    return funcdict if funcdict else funclist
+
+
+def _flatten_columns(cols) -> list[str]:
+    """Flatten MultiIndex column tuples to underscore-joined strings."""
+    result = []
+    for col in cols:
+        if isinstance(col, str):
+            result.append(col)
+        else:
+            result.append("_".join(str(x) for x in col if str(x) != ""))
+    return result
+
+
+def _do_pivot(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
+    """Run pivot_table and flatten column names."""
+    funcs = _prepare_funcs(
+        getattr(args, "aggfunc", None),
+        getattr(args, "confidencelevel", 95.0),
+        getattr(args, "confidencemethod", "linear"),
+    )
+
+    index = args.index or None
+    columns = args.groupcols or None
+
+    pivoted = df.pivot_table(
+        values=args.values,
+        index=index,
+        columns=columns,
+        aggfunc=funcs if funcs is not None else "mean",
+    )
+
+    cols = pivoted.columns
+    pivoted.columns = _flatten_columns(cols.values if hasattr(cols, "values") else cols)
+
+    if getattr(args, "fillzero", False) and hasattr(pivoted.index, "levels"):
+        pidx = pd.MultiIndex.from_product(
+            pivoted.index.levels, names=pivoted.index.names
+        )
+        pivoted = pivoted.reindex(pidx, fill_value=0, copy=False)
+
+    return pivoted.reset_index()
+
+
+# ---------------------------------------------------------------------------
+# Command
+# ---------------------------------------------------------------------------
+
+
+class PivotCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "pivot"
+
+    @property
+    def help(self) -> str:
+        return "Pivot / aggregate a DataFrame (pivot_table)"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        import argparse as ap
+
+        parser.formatter_class = ap.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        io.parser_read(parser)
+
+        g = parser.add_argument_group("pivot options")
+        g.add_argument(
+            "-v",
+            "--values",
+            help="Value column(s) to aggregate.",
+            nargs="+",
+            required=True,
+            metavar="COL",
+        )
+        g.add_argument(
+            "-i",
+            "--index",
+            help="Column(s) that become row labels in the pivot table.",
+            nargs="*",
+            default=[],
+            metavar="COL",
+        )
+        g.add_argument(
+            "-g",
+            "--groupcols",
+            help="Column(s) whose unique values are spread into new column headers.",
+            nargs="*",
+            default=[],
+            metavar="COL",
+        )
+        g.add_argument(
+            "-f",
+            "--aggfunc",
+            help=(
+                "Aggregation function(s).  "
+                "Use bare names (mean std) or COL:FUNC pairs (sales:mean cost:sum).  "
+                "See --help for full list.  Default: mean."
+            ),
+            nargs="*",
+            default=None,
+            metavar="FUNC",
+        )
+        g.add_argument(
+            "-z",
+            "--fillzero",
+            help="Reindex to all (index × columns) combinations, fill missing with 0.",
+            action="store_true",
+        )
+        g.add_argument(
+            "--confidencelevel",
+            help="Confidence level for cilo/cihi aggfuncs, in percent (default: 95.0).",
+            type=float,
+            default=95.0,
+            metavar="PCT",
+        )
+        g.add_argument(
+            "--confidencemethod",
+            help="Interpolation method for cilo/cihi (default: linear).",
+            default="linear",
+            metavar="METHOD",
+        )
+
+        bs = parser.add_argument_group("bootstrap options")
+        bs.add_argument(
+            "--bootstrap",
+            help=(
+                "Number of bootstrap samples.  "
+                "Produces a single DataFrame with a samplenum column."
+            ),
+            type=int,
+            default=None,
+            metavar="N",
+        )
+        bs.add_argument(
+            "--randomseed",
+            help="Random seed.  Accepts an integer or a string (hashed to uint32).",
+            default=None,
+            metavar="SEED",
+        )
+        bs.add_argument(
+            "--fullsampling",
+            help="Resample all rows with replacement, ignoring group structure.",
+            action="store_true",
+        )
+        bs.add_argument(
+            "--samplinggroup",
+            help=(
+                "Column(s) defining groups within which rows are resampled.  "
+                "Also used as the within-group dimension when --samplingcols is given."
+            ),
+            nargs="*",
+            default=[],
+            metavar="COL",
+        )
+        bs.add_argument(
+            "--samplingcols",
+            help=(
+                "Column(s) identifying the sampling unit (e.g. a subject ID).  "
+                "Enables hierarchical bootstrap: unique units are resampled and "
+                "all their rows are pulled in via merge.  "
+                "See --help for a detailed example."
+            ),
+            nargs="*",
+            default=[],
+            metavar="COL",
+        )
+
+        io.parser_output(parser)
+
+    def execute(self, args: argparse.Namespace) -> None:
+        args.randomseed = normalize_seed(args.randomseed)
+        np.random.seed(args.randomseed)
+
+        df = io.read(args)
+
+        check_cols(df, args.values, "-v/--values")
+        check_cols(df, args.index, "-i/--index")
+        check_cols(df, args.groupcols, "-g/--groupcols")
+        check_cols(df, args.samplingcols, "--samplingcols")
+        check_cols(df, args.samplinggroup, "--samplinggroup")
+
+        if args.bootstrap is None:
+            result = _do_pivot(df, args)
+            io.printdf(result, args)
+            return
+
+        # ------------------------------------------------------------------
+        # Bootstrap
+        # ------------------------------------------------------------------
+
+        logger.info(
+            f"Bootstrap pivot: {args.bootstrap} samples, seed={args.randomseed}"
+        )
+
+        # Determine sampling mode and build sampling set if needed.
+        if args.fullsampling:
+            mode = "full"
+            s = None
+            samplinggroup: list[str] = []
+        elif args.samplingcols:
+            mode = "hierarchical"
+            # The merge key is samplingcols + samplinggroup so the group
+            # column is preserved through the merge.
+            merge_cols = args.samplingcols + args.samplinggroup
+            s = df[merge_cols].drop_duplicates().reset_index(drop=True)
+            samplinggroup = args.samplinggroup
+        else:
+            mode = "grouped"
+            s = None
+            samplinggroup = args.samplinggroup or (args.index + args.groupcols)
+
+        logger.info(f"Sampling mode: {mode}")
+
+        frames: list[pd.DataFrame] = []
+
+        for ns in range(1, args.bootstrap + 1):
+            if mode == "full":
+                bf = df.sample(frac=1, replace=True)
+
+            elif mode == "hierarchical":
+                if samplinggroup:
+                    ss = (
+                        s.groupby(samplinggroup, group_keys=False)[s.columns.tolist()]
+                        .apply(lambda x: x.sample(frac=1.0, replace=True))
+                        .reset_index(drop=True)
+                    )
+                else:
+                    ss = s.sample(frac=1.0, replace=True).reset_index(drop=True)
+                bf = pd.merge(ss, df)
+
+            else:  # grouped
+                bf = (
+                    df.groupby(samplinggroup, group_keys=False)[df.columns.tolist()]
+                    .apply(lambda x: x.sample(frac=1.0, replace=True))
+                    .reset_index(drop=True)
+                )
+
+            result = _do_pivot(bf, args)
+            result["samplenum"] = ns
+            frames.append(result)
+
+        io.printdf(pd.concat(frames, ignore_index=True), args)

dftk/commands/print_cmd.py

@@ -0,0 +1,36 @@
+"""
+dftk.commands.print_cmd — the ``dftk print`` subcommand.
+
+Reads any dftk-compatible input (TSV file, TSV on stdin, or a parquet pipe
+path on stdin) and writes TSV to stdout.  Acts as the terminal stage of a
+dftk pipeline:
+
+    dftk stat -g grp -s val -o \\
+      | dftk pivot -i grp -v val_mean -o \\
+      | dftk print
+"""
+
+import argparse
+import sys
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import io
+
+
+class PrintCommand(BaseCommand):
+    """Read any dftk input and write TSV to stdout."""
+
+    name = "print"
+    help = "Convert any input (TSV or parquet pipe) to TSV on stdout"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        # Only the read-side arguments — no -o/--output, since the whole
+        # point of this command is to force TSV to stdout.
+        io.parser_read(parser)
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+        try:
+            df.to_csv(sys.stdout, sep="\t", index=False)
+        except (OSError, BrokenPipeError):
+            sys.stderr.close()

dftk/commands/query_cmd.py

@@ -0,0 +1,230 @@
+"""
+dftk.commands.query_cmd — dftk query subcommand.
+
+Filter (and optionally reshape) a dataframe using either pandas query
+expressions or full DuckDB SQL.
+
+Two modes
+---------
+pandas mode  (-q / --query, repeatable)
+    Applies pandas DataFrame.query() expressions in sequence.  Each
+    expression is a boolean row filter; all expressions are ANDed together.
+
+      dftk query data.tsv -q "x > 0" -q "group == 'A'"
+
+sql mode  (--sql)
+    Passes a full SQL statement to DuckDB.  The input data is available as
+    a virtual table named "data" (override with --table).  Any SQL that
+    DuckDB supports is valid: SELECT, aggregation, window functions, CTEs.
+
+      dftk query data.tsv --sql "SELECT grp, AVG(x) AS mean FROM data GROUP BY grp"
+
+    When the input is a named .parquet file, DuckDB queries it natively
+    without loading the full file into memory — suitable for large datasets.
+
+    Requires: pip install duckdb
+"""
+
+import argparse
+import logging
+import os
+
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import io
+
+try:
+    import duckdb
+
+    _HAS_DUCKDB = True
+except ImportError:
+    duckdb = None
+    _HAS_DUCKDB = False
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Core functions
+# ---------------------------------------------------------------------------
+
+
+def _pandas_query(df: pd.DataFrame, expressions: list[str]) -> pd.DataFrame:
+    """Apply each expression as a pandas boolean row filter (AND logic)."""
+    for expr in expressions:
+        try:
+            df = df.query(expr)
+        except Exception as exc:
+            raise ValueError(f"pandas query failed ({expr!r}): {exc}") from exc
+    return df
+
+
+def _sql_query(args: argparse.Namespace, sql: str, table: str) -> pd.DataFrame:
+    """Run *sql* against the input data registered as a DuckDB virtual table.
+
+    For named .parquet inputs the file is queried as a lazy DuckDB relation —
+    DuckDB pushes down filters and projections so the full dataset need not
+    fit in memory.  All other inputs (TSV files, stdin, parquet-pipe) are
+    read into a pandas DataFrame first and then registered with DuckDB.
+    """
+    if not _HAS_DUCKDB:
+        raise ImportError("--sql requires the duckdb package: pip install duckdb")
+
+    conn = duckdb.connect()
+    filename: str | None = getattr(args, "DATAFILE", None)
+
+    # Named parquet: let DuckDB read it lazily (potentially out-of-core).
+    direct_parquet = (
+        filename
+        and filename not in (None, "-")
+        and filename.endswith(".parquet")
+        and os.path.isfile(filename)
+    )
+
+    if direct_parquet:
+        rel = conn.read_parquet(filename)
+        conn.register(table, rel)
+    else:
+        # stdin, TSV files, or parquet-pipe: read into pandas first.
+        df = io.read(args)
+        conn.register(table, df)
+
+    try:
+        return conn.execute(sql).df()
+    except Exception as exc:
+        raise ValueError(f"DuckDB SQL failed: {exc}") from exc
+
+
+# ---------------------------------------------------------------------------
+# Command
+# ---------------------------------------------------------------------------
+
+_EPILOG = """\
+PANDAS QUERY MODE  (-q / --query)
+----------------------------------
+Each expression is applied as a boolean row filter in sequence; multiple
+-q flags are ANDed together.  Syntax mirrors Python comparisons on column
+names:
+
+  dftk query data.tsv -q "x > 0"
+  dftk query data.tsv -q "group == 'A'" -q "score >= 10"
+  dftk query data.tsv -q "x > 0 and y < 100"
+  dftk query data.tsv -q "label.str.startswith('foo')"
+
+Backtick-quoting handles column names with spaces or special characters:
+
+  dftk query data.tsv -q "`my col` > 0"
+
+This is a thin wrapper around pandas DataFrame.query().
+
+
+SQL MODE  (--sql)
+------------------
+A full SQL statement is passed to DuckDB.  The input data is available as
+a virtual table named "data" (or the name given by --table).
+
+Simple filtering:
+
+  dftk query data.tsv --sql "SELECT * FROM data WHERE x > 0"
+  cat data.tsv | dftk query --sql "SELECT * FROM data WHERE x > 0" -o
+
+Aggregation (replaces dftk pivot for simple cases):
+
+  dftk query data.tsv --sql \\
+    "SELECT group, AVG(x) AS mean, STDDEV(x) AS std, COUNT(*) AS n
+     FROM data GROUP BY group"
+
+Window functions:
+
+  dftk query data.tsv --sql \\
+    "SELECT *, ROW_NUMBER() OVER (PARTITION BY group ORDER BY x DESC) AS rank
+     FROM data"
+
+CTEs:
+
+  dftk query data.tsv --sql \\
+    "WITH ranked AS (
+       SELECT *, RANK() OVER (PARTITION BY group ORDER BY score DESC) AS rank
+       FROM data
+     )
+     SELECT * FROM ranked WHERE rank <= 3"
+
+OUT-OF-CORE LARGE FILES
+------------------------
+When the input is a named .parquet file, DuckDB queries it natively without
+loading the full dataset into memory.  Filter pushdown and column pruning
+mean that only the rows and columns needed by the query are read:
+
+  dftk query huge.parquet --sql "SELECT * FROM data WHERE x > 0"
+  dftk query huge.parquet --sql \\
+    "SELECT group, AVG(x) AS mean FROM data GROUP BY group"
+
+For piped input (stdin or parquet-pipe), the data is loaded into pandas
+first and then registered with DuckDB — in-memory, same as pandas mode.
+Use a named checkpoint file (-o huge.parquet) when you need out-of-core
+queries on a large intermediate result:
+
+  dftk eval "y = x * 2" huge.tsv -o huge.parquet
+  dftk query huge.parquet --sql "SELECT * FROM data WHERE y > 1000"
+
+Requires: pip install duckdb
+"""
+
+
+class QueryCommand(BaseCommand):
+    name = "query"
+    help = "Filter rows with pandas expressions (-q) or full DuckDB SQL (--sql)."
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("query options")
+        g.add_argument(
+            "-q",
+            "--query",
+            action="append",
+            default=[],
+            metavar="EXPR",
+            help=(
+                "Pandas boolean expression for row filtering.  "
+                "Repeatable; all expressions are ANDed together.  "
+                "Cannot be combined with --sql."
+            ),
+        )
+        g.add_argument(
+            "--sql",
+            default=None,
+            metavar="SQL",
+            help=(
+                "Full DuckDB SQL query.  The input is registered as a virtual "
+                "table named 'data' (see --table).  Named .parquet inputs are "
+                "queried natively without full load into memory.  "
+                "Cannot be combined with -q.  Requires duckdb."
+            ),
+        )
+        g.add_argument(
+            "--table",
+            default="data",
+            metavar="NAME",
+            help=(
+                "Name of the virtual table in --sql queries (default: data).  "
+                "Change this if your data has a column named 'data'."
+            ),
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        if args.sql is not None and args.query:
+            raise ValueError("Cannot combine -q/--query and --sql.")
+
+        if args.sql is not None:
+            df = _sql_query(args, args.sql, args.table)
+        else:
+            df = io.read(args)
+            if args.query:
+                df = _pandas_query(df, args.query)
+
+        io.printdf(df, args)