dataframe-toolkit 0.4.1__py3-none-any.whl

dftk/commands/base.py

@@ -0,0 +1,85 @@
+"""
+dftk.commands.base — abstract base class for all dftk subcommands.
+
+Every subcommand must:
+
+  1. Inherit from BaseCommand.
+  2. Define the ``name`` and ``help`` class attributes (or properties).
+  3. Implement ``add_arguments(parser)`` to declare its CLI arguments.
+  4. Implement ``execute(args)`` to run the command.
+
+Most commands also call ``self.add_io_arguments(parser)`` inside
+``add_arguments`` to attach the standard dftk read + output argument groups.
+
+Example skeleton::
+
+    from dftk.commands.base import BaseCommand
+    from dftk.common.io import io
+
+    class MyCommand(BaseCommand):
+        name = "mycommand"
+        help = "Does something useful"
+
+        def add_arguments(self, parser):
+            self.add_io_arguments(parser)           # adds DATAFILE, -o, etc.
+            g = parser.add_argument_group("mycommand options")
+            g.add_argument("-x", "--xcol", ...)
+
+        def execute(self, args):
+            df = io.read(args)
+            # ... transform df ...
+            io.printdf(df, args)
+"""
+
+import argparse
+from abc import ABC, abstractmethod
+
+
+class BaseCommand(ABC):
+    """Abstract base class for all dftk subcommands."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Command name as it appears on the CLI (e.g. ``"stat"``)."""
+
+    @property
+    @abstractmethod
+    def help(self) -> str:
+        """One-line description shown in ``dftk --help``."""
+
+    @abstractmethod
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        """Declare all CLI arguments for this command on *parser*."""
+
+    @abstractmethod
+    def execute(self, args: argparse.Namespace) -> None:
+        """Run the command using the parsed *args*."""
+
+    # ------------------------------------------------------------------ #
+    # Shared helpers                                                        #
+    # ------------------------------------------------------------------ #
+
+    def add_io_arguments(self, parser: argparse.ArgumentParser) -> None:
+        """Attach the standard dftk read + output argument groups to *parser*.
+
+        Adds:
+          * All ``io.parser_read`` arguments  (DATAFILE, --backend, --noheader, …)
+          * All ``io.parser_output`` arguments (-o/--output, --select, --drop, …)
+
+        Call this at the top of ``add_arguments`` for any command that reads
+        tabular data and produces tabular output.
+        """
+        from dftk.common.io import io
+        io.parser_read(parser)
+        io.parser_output(parser)
+
+    def add_read_arguments(self, parser: argparse.ArgumentParser) -> None:
+        """Attach only the read argument group to *parser* (no output args).
+
+        Use this for commands that read tabular data but produce non-tabular
+        output (e.g. plots), so that --select, --drop, --round, etc. are not
+        exposed.
+        """
+        from dftk.common.io import io
+        io.parser_read(parser)

dftk/commands/binx_cmd.py

@@ -0,0 +1,130 @@
+"""
+dftk.commands.binx_cmd — dftk binx subcommand.
+
+Port of dfbinx.py: assign each row a bin index based on a column value and
+a set of bin edges, with an optional replacement by the lower, middle, or
+upper edge value of that bin.
+"""
+
+import argparse
+
+import numpy as np
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import check_cols, io
+
+
+def _parse_binspec(binspec: str) -> list[float]:
+    """Parse 'min:max:step' or 'e1,e2,...,eN' into a list of bin edges."""
+    if "," in binspec:
+        return [float(x) for x in binspec.split(",")]
+    parts = binspec.split(":")
+    if len(parts) != 3:
+        raise ValueError(
+            f"binspec must be 'min:max:step' or comma-separated edges, got: {binspec!r}"
+        )
+    bmin, bmax, bstep = float(parts[0]), float(parts[1]), float(parts[2])
+    return list(np.arange(bmin, bmax, bstep))
+
+
+def _binx(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
+    bins = _parse_binspec(args.binspec)
+    if len(bins) < 2:
+        raise ValueError(
+            f"binspec must produce at least 2 edges; got {len(bins)}: {bins}"
+        )
+
+    dest = args.destcol or f"{args.col}_bin"
+    df[dest] = pd.cut(df[args.col], bins, labels=False, include_lowest=True)
+
+    if args.usevalue is not None:
+        frac = {"l": 0.0, "m": 0.5, "u": 1.0}[args.usevalue]
+        edges = bins
+
+        def _to_edge(idx: float) -> float:
+            if pd.isna(idx):
+                return float("nan")
+            i = int(idx)
+            return (1.0 - frac) * edges[i] + frac * edges[i + 1]
+
+        df[dest] = df[dest].apply(_to_edge)
+
+    return df
+
+
+_EPILOG = """\
+BIN SPECIFICATION (--binspec)
+------------------------------
+  min:max:step   Generate edges with np.arange(min, max, step).
+                 Note: max is exclusive, same as Python range().
+  e1,e2,...,eN   Explicit comma-separated edge values.
+
+  Examples:
+    0:100:10      edges [0, 10, 20, ..., 90]  → 9 bins
+    0,5,10,50     3 bins: [0,5), [5,10), [10,50]
+
+USEVALUE (--usevalue)
+---------------------
+  Replaces the integer bin index with the actual edge value:
+    l   lower edge of the bin
+    m   midpoint between lower and upper edges
+    u   upper edge of the bin
+
+EXAMPLES
+--------
+  dftk binx data.tsv -c age -b 0:100:10
+  dftk binx data.tsv -c score -b 0,25,50,75,100 -d quartile --usevalue m
+  dftk binx data.tsv -c x -b -3:4:1 -d x_bin -o | dftk stat -g x_bin -s y
+"""
+
+
+class BinxCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "binx"
+
+    @property
+    def help(self) -> str:
+        return "Assign bin indices to a column based on explicit or generated edges."
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("binx options")
+        g.add_argument(
+            "-c",
+            "--col",
+            required=True,
+            metavar="COL",
+            help="Column to bin.",
+        )
+        g.add_argument(
+            "-d",
+            "--destcol",
+            default=None,
+            metavar="DESTCOL",
+            help="Name of the output bin column (default: <col>_bin).",
+        )
+        g.add_argument(
+            "-b",
+            "--binspec",
+            required=True,
+            metavar="SPEC",
+            help="Bin edges: 'min:max:step' or 'e1,e2,...,eN'.",
+        )
+        g.add_argument(
+            "--usevalue",
+            choices=["l", "m", "u"],
+            default=None,
+            help="Replace bin index with bin edge value: l=lower, m=mid, u=upper.",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+        check_cols(df, [args.col], "-c/--col")
+        result = _binx(df, args)
+        io.printdf(result, args)

dftk/commands/clean_cmd.py

@@ -0,0 +1,51 @@
+"""
+dftk.commands.clean_cmd — the ``dftk clean`` subcommand.
+
+Removes all temp parquet pipe files left in DFTK_TMPDIR.  Useful after a
+pipeline is interrupted before the final ``dftk print`` stage has had a
+chance to consume (and therefore delete) them.
+
+    dftk clean
+"""
+
+import argparse
+import os
+import sys
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import DFTK_TMPDIR
+
+
+class CleanCommand(BaseCommand):
+    """Remove leftover temp parquet pipe files."""
+
+    name = "clean"
+    help = "Remove temp parquet pipe files from the dftk temp directory"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        # No arguments needed — always cleans DFTK_TMPDIR.
+        pass
+
+    def execute(self, args: argparse.Namespace) -> None:
+        if not os.path.isdir(DFTK_TMPDIR):
+            print(
+                f"Nothing to clean (directory does not exist: {DFTK_TMPDIR})",
+                file=sys.stderr,
+            )
+            return
+
+        files = [f for f in os.listdir(DFTK_TMPDIR) if f.endswith(".parquet")]
+        errors = 0
+        for f in files:
+            try:
+                os.unlink(os.path.join(DFTK_TMPDIR, f))
+            except OSError as exc:
+                print(f"Warning: could not remove {f}: {exc}", file=sys.stderr)
+                errors += 1
+
+        removed = len(files) - errors
+        print(
+            f"Removed {removed} file(s) from {DFTK_TMPDIR}"
+            + (f" ({errors} error(s))" if errors else ""),
+            file=sys.stderr,
+        )

dftk/commands/concat_cmd.py

@@ -0,0 +1,165 @@
+"""
+dftk.commands.concat_cmd — dftk concat subcommand.
+
+Concatenate two or more input files (TSV or parquet) row-wise into a single
+DataFrame.  Column alignment follows pandas outer-join semantics: columns
+present in some but not all files are filled with NaN (or --fill value).
+
+Examples
+--------
+    dftk concat a.tsv b.tsv c.tsv
+    dftk concat a.tsv b.tsv --sourcecol source
+    dftk concat a.parquet b.parquet -o
+    dftk eval data.tsv -f "x=1" -o | dftk concat ... extra.tsv
+"""
+
+import argparse
+import copy
+
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import io
+
+
+class ConcatCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "concat"
+
+    @property
+    def help(self) -> str:
+        return "Concatenate multiple input files row-wise"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = """\
+COLUMN ALIGNMENT
+----------------
+  Columns present in all files are aligned by name.
+  Columns missing from some files are filled with NaN (or --fill value).
+  Use --select to enforce a fixed output column set.
+
+EXAMPLES
+--------
+  dftk concat a.tsv b.tsv
+  dftk concat a.tsv b.tsv c.tsv --sourcecol source
+  dftk concat a.parquet b.parquet -o result.parquet
+  dftk eval data.tsv -f "x=1" -o | dftk concat ... extra.tsv
+"""
+        g = parser.add_argument_group("concat options")
+        g.add_argument(
+            "DATAFILES",
+            nargs="+",
+            metavar="FILE",
+            help=(
+                "Input files to concatenate (TSV or .parquet).  "
+                "Use - for TSV stdin or ... for a parquet pipe path on stdin.  "
+                "At most one stdin source is allowed."
+            ),
+        )
+        g.add_argument(
+            "--sourcecol",
+            default=None,
+            metavar="COL",
+            help="Add a column with this name containing the source filename.",
+        )
+        g.add_argument(
+            "--fill",
+            default=None,
+            metavar="VALUE",
+            help=(
+                "Fill value for columns absent in some files "
+                "(default: NaN).  Use 0 or '' for numeric/string defaults."
+            ),
+        )
+
+        r = parser.add_argument_group("data input")
+        r.add_argument(
+            "--backend",
+            help="Backend for reading data (default: pandas)",
+            choices=["pandas", "duckdb"],
+            default="pandas",
+        )
+        r.add_argument(
+            "--noheader",
+            help="Input files have no header row; columns are named V1, V2, …",
+            action="store_true",
+        )
+        r.add_argument(
+            "--nrows",
+            help="Maximum number of rows to read per file",
+            type=int,
+            default=None,
+        )
+        r.add_argument(
+            "--delimiter",
+            help="Column delimiter for TSV inputs (default: tab)",
+            default=None,
+        )
+        r.add_argument(
+            "--readasobject",
+            help=(
+                "Read named columns as object dtype.  "
+                "Omit column names to apply to all columns."
+            ),
+            nargs="*",
+            default=None,
+            metavar="COL",
+        )
+        r.add_argument(
+            "--prequery",
+            help="Pandas query expression after reading each file.  Repeatable.",
+            default=[],
+            action="append",
+            metavar="EXPR",
+        )
+
+        io.parser_output(parser)
+
+    def execute(self, args: argparse.Namespace) -> None:
+        stdin_used = False
+        dfs: list[pd.DataFrame] = []
+
+        for path in args.DATAFILES:
+            if path in ("-", "..."):
+                if stdin_used:
+                    raise ValueError(
+                        "stdin (-/...) can only appear once in a concat file list"
+                    )
+                stdin_used = True
+
+            # Build a per-file args with DATAFILE set to this path.
+            file_args = copy.copy(args)
+            file_args.DATAFILE = path
+
+            try:
+                df = io.read(file_args)
+            except pd.errors.EmptyDataError:
+                import logging
+
+                logging.warning(f"concat: skipping empty file {path!r}")
+                continue
+
+            if args.sourcecol:
+                df[args.sourcecol] = path
+
+            dfs.append(df)
+
+        if not dfs:
+            raise ValueError("concat: no data read from any input file")
+
+        result = pd.concat(dfs, ignore_index=True)
+
+        if args.fill is not None:
+            fill: int | float | str
+            try:
+                fill = int(args.fill)
+            except ValueError:
+                try:
+                    fill = float(args.fill)
+                except ValueError:
+                    fill = args.fill
+            result = result.fillna(fill)
+
+        io.printdf(result, args)