dataframe-toolkit 0.5.1__tar.gz → 0.7.1__tar.gz

{dataframe_toolkit-0.5.1 → dataframe_toolkit-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.5.1
+Version: 0.7.1
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson

dataframe_toolkit-0.7.1/pyproject.toml

@@ -0,0 +1,82 @@
+[project]
+name = "dataframe-toolkit"
+version = "0.7.1"
+description = "A command-line toolkit for DataFrame analysis and manipulation"
+readme = "README.md"
+keywords = [
+    "dataframe",
+    "statistics",
+    "data analysis",
+    "CLI",
+    "TSV",
+    "pandas",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Utilities",
+    "Programming Language :: Python :: 3",
+]
+requires-python = ">=3.12"
+dependencies = [
+    "pandas>=2.1",
+    "numpy>=1.24",
+    "pyarrow>=12.0",
+    "scipy>=1.10",
+    "statsmodels>=0.14",
+    "duckdb>=0.8",
+    "matplotlib>=3.7",
+    "seaborn>=0.12",
+    "tqdm>=4.65",
+]
+
+[[project.authors]]
+name = "Gunnar Pálsson"
+email = "hemingur@gmail.com"
+
+[project.license]
+text = "MIT"
+
+[project.urls]
+Homepage = "https://github.com/hemingur/dataframe-toolkit"
+Repository = "https://github.com/hemingur/dataframe-toolkit"
+
+[project.scripts]
+dftk = "dftk.cli:main"
+
+[build-system]
+requires = ["uv_build>=0.10.4,<0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "dftk"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 88
+target-version = "py312"
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "UP",
+    "B",
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["dftk"]
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.6.0",
+    "pytest>=8.0",
+    "pytest-cov>=7.1.0",
+    "ruff>=0.15.12",
+]

dataframe_toolkit-0.5.1/pyproject.toml → dataframe_toolkit-0.7.1/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.5.1"
+version = "0.7.1"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [

{dataframe_toolkit-0.5.1 → dataframe_toolkit-0.7.1}/src/dftk/cli.py

@@ -1,23 +1,44 @@
 """
 dftk.cli — entry point for the ``dftk`` command.
 
-All subcommands are discovered through the COMMANDS list in
+All subcommands are discovered through the registry in
 dftk/commands/__init__.py.  This file contains no command-specific
 logic and never needs to change when a new subcommand is added.
+
+Only the command being run is imported, so e.g. ``dftk print`` does not pay
+for importing scipy, statsmodels or matplotlib.  The full set is imported only
+when it is needed: ``dftk --help``, no command, or an unknown command name.
 """
 
 import argparse
 import logging
+import os
+import signal
 import sys
 
 from dftk import __version__
-from dftk.commands import COMMANDS
+from dftk.commands import COMMAND_NAMES, load_command, load_command_groups
+from dftk.commands.base import BaseCommand
 from dftk.common.io import DFTK_TMPDIR
 
 logger = logging.getLogger("dftk")
 logging.basicConfig(format="%(asctime)s %(module)s %(message)s", level=logging.INFO)
 
 
+def _commands_for(argv: list[str]) -> list[BaseCommand]:
+    """Return the commands whose subparsers *argv* needs.
+
+    The top-level parser has no options that take a value, so a valid
+    invocation always has the command name first.
+    """
+    if argv and argv[0] in COMMAND_NAMES:
+        return [load_command(argv[0])]
+    # `dftk --help`, no command, or an unknown name: load everything so the
+    # command listing and argparse's "invalid choice" message are complete.
+    grouped = [cmd for _, cmds in load_command_groups() for cmd in cmds]
+    return grouped + [load_command("help")]
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         prog="dftk",
@@ -38,7 +59,7 @@ def main() -> None:
     subparsers = parser.add_subparsers(dest="command", metavar="COMMAND")
     subparsers.required = True
 
-    for cmd in COMMANDS:
+    for cmd in _commands_for(sys.argv[1:]):
         cmd_parser = subparsers.add_parser(cmd.name, help=cmd.help)
         cmd.add_arguments(cmd_parser)
         cmd_parser.set_defaults(command_instance=cmd)
@@ -47,6 +68,13 @@ def main() -> None:
 
     try:
         args.command_instance.execute(args)
+        sys.stdout.flush()
+    except BrokenPipeError:
+        # The reader closed the pipe early (e.g. `dftk ... | head`).  Exit
+        # quietly with the SIGPIPE status like cat/grep, and point stdout at
+        # devnull so the interpreter's final flush doesn't raise again.
+        os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
+        sys.exit(128 + signal.SIGPIPE)
     except FileNotFoundError as exc:
         print(f"File error: {exc}", file=sys.stderr)
         sys.exit(1)

dataframe_toolkit-0.7.1/src/dftk/commands/__init__.py

@@ -0,0 +1,98 @@
+# dftk.commands — registry of all dftk subcommands.
+#
+# Commands are registered by name and import path instead of being imported
+# here, so that running one command imports only that command's module and its
+# dependencies (scipy, statsmodels, matplotlib, …), not every command's.
+#
+# To add a new subcommand:
+#   1. Create src/dftk/commands/<name>_cmd.py with a class that inherits
+#      from BaseCommand (see base.py for the contract and an example skeleton).
+#   2. Add a ("<name>", "<name>_cmd:<ClassName>") entry to COMMAND_GROUPS below.
+#   That's it — cli.py discovers commands solely through this registry.
+#   tests/commands/test_registry.py checks that every entry imports and that
+#   every *_cmd.py module is registered.
+
+import importlib
+
+from dftk.commands.base import BaseCommand
+
+# Ported subcommands — add as each one is implemented:
+#   ("color", "color_cmd:ColorCommand")
+#   ("wavelet", "wavelet_cmd:WaveletCommand")
+#   ("fisher", ...)  — superseded by eval's fisher_test/fisher_OR
+
+COMMAND_GROUPS: list[tuple[str, list[tuple[str, str]]]] = [
+    (
+        "Data transformation",
+        [
+            ("eval", "eval_cmd:EvalCommand"),
+            ("query", "query_cmd:QueryCommand"),
+            ("merge", "merge_cmd:MergeCommand"),
+            ("concat", "concat_cmd:ConcatCommand"),
+            ("melt", "melt_cmd:MeltCommand"),
+            ("pivot", "pivot_cmd:PivotCommand"),
+            ("func", "func_cmd:FuncCommand"),
+            ("scale", "scale_cmd:ScaleCommand"),
+            ("interp", "interp_cmd:InterpCommand"),
+            ("binx", "binx_cmd:BinxCommand"),
+            ("transpose", "transpose_cmd:TransposeCommand"),
+            ("segid", "segid_cmd:SegidCommand"),
+        ],
+    ),
+    (
+        "Statistics",
+        [
+            ("stat", "stat_cmd:StatCommand"),
+            ("wstat", "wstat_cmd:WstatCommand"),
+            ("fit", "fit_cmd:FitCommand"),
+            ("test", "test_cmd:TestCommand"),
+            ("corr", "corr_cmd:CorrCommand"),
+            ("describe", "describe_cmd:DescribeCommand"),
+            ("info", "info_cmd:InfoCommand"),
+            ("randvar", "randvar_cmd:RandvarCommand"),
+        ],
+    ),
+    (
+        "Plots",
+        [
+            ("scat", "scat_cmd:ScatCommand"),
+            ("line", "line_cmd:LineCommand"),
+            ("hist", "hist_cmd:HistCommand"),
+            ("chi", "chi_cmd:ChiCommand"),
+        ],
+    ),
+    (
+        "Utilities",
+        [
+            ("dataset", "dataset_cmd:DatasetCommand"),
+            ("sample", "sample_cmd:SampleCommand"),
+            ("split", "split_cmd:SplitCommand"),
+            ("annotate", "annotate_cmd:AnnotateCommand"),
+            ("print", "print_cmd:PrintCommand"),
+            ("clean", "clean_cmd:CleanCommand"),
+        ],
+    ),
+]
+
+# `help` is a command like the others but is not listed under a group heading.
+_REGISTRY: dict[str, str] = dict(
+    [entry for _, entries in COMMAND_GROUPS for entry in entries]
+    + [("help", "help:HelpCommand")]
+)
+
+COMMAND_NAMES: list[str] = list(_REGISTRY)
+
+
+def load_command(name: str) -> BaseCommand:
+    """Import and instantiate the command registered as *name*."""
+    module_name, class_name = _REGISTRY[name].split(":")
+    module = importlib.import_module(f"dftk.commands.{module_name}")
+    return getattr(module, class_name)()
+
+
+def load_command_groups() -> list[tuple[str, list[BaseCommand]]]:
+    """Import and instantiate every grouped command (slow: imports everything)."""
+    return [
+        (header, [load_command(name) for name, _ in entries])
+        for header, entries in COMMAND_GROUPS
+    ]

{dataframe_toolkit-0.5.1 → dataframe_toolkit-0.7.1}/src/dftk/commands/annotate_cmd.py

@@ -25,6 +25,10 @@ Usage examples:
   dftk eval data.tsv -f "z = x + y" -o result.parquet \\
       --meta genome=hg38 --meta source=my_pipeline
 
+  # Same, built from a bundled dataset:
+  dftk dataset iris -o | dftk eval ... -f "z = sepal_length + petal_length" \\
+      -o result.parquet --meta genome=hg38 --meta source=my_pipeline
+
 Annotations propagate through the parquet pipe: a file annotated at
 creation is re-annotated on every subsequent -o write unless explicitly
 overridden.

{dataframe_toolkit-0.5.1 → dataframe_toolkit-0.7.1}/src/dftk/commands/binx_cmd.py

@@ -74,8 +74,10 @@ USEVALUE (--usevalue)
 EXAMPLES
 --------
   dftk binx data.tsv -c age -b 0:100:10
-  dftk binx data.tsv -c score -b 0,25,50,75,100 -d quartile --usevalue m
-  dftk binx data.tsv -c x -b -3:4:1 -d x_bin -o | dftk stat -g x_bin -s y
+  dftk dataset tips -o | dftk binx ... -c total_bill -b 0:60:10
+  dftk dataset tips -o | dftk binx ... -c tip -b 0,2,4,6,10 -d tip_bracket --usevalue m
+  dftk dataset tips -o | dftk binx ... -c total_bill -b 0:60:10 -d bill_bin -o \\
+      | dftk stat ... -c tip -g bill_bin
 """
 
 

dataframe_toolkit-0.7.1/src/dftk/commands/chi_cmd.py

@@ -0,0 +1,193 @@
+"""
+dftk.commands.chi_cmd — dftk chi subcommand.
+
+Corner-resolved chi-plot: a visual diagnostic for which direction (if any)
+two variables have tail dependence in, complementing the numeric
+lambda_lower/lambda_upper columns from `dftk corr --tail`.
+
+Follows the structure of scat_cmd.py -- same --groupcol / --subgraphcol /
+--size / --fontsize figure conventions as the rest of the Plots group.
+"""
+
+import argparse
+
+import numpy as np
+
+from dftk.commands.base import BaseCommand
+from dftk.common.copula import corner_chi, pseudo_observations
+from dftk.common.io import check_cols, io
+from dftk.common.plot import (
+    add_figure_arguments,
+    add_group_arguments,
+    add_legend_arguments,
+    add_xy_arguments,
+    apply_labels,
+    apply_style,
+    make_figure,
+    make_grouplabel,
+    save_or_show,
+    subgraph_groups,
+    subgraph_layout,
+)
+
+# ---------------------------------------------------------------------------
+# Single-axes chi-plot (reused per subplot)
+# ---------------------------------------------------------------------------
+
+
+def _chi_ax(ax, df, args, title_suffix: str = ""):
+    xcol, ycol = args.xcol, args.ycol
+    legendloc = tuple(args.legendloc) if getattr(args, "legendloc", None) else "best"
+
+    def _draw(sub_df, label=None, **scatter_kw):
+        a, b = sub_df[[xcol, ycol]].dropna().values.T
+        if len(a) < 3:
+            return
+        u, v = pseudo_observations(a, b)
+        x, chi = corner_chi(u, v)
+        mask = np.abs(x) < 0.8  # trim the noisiest extreme points, standard practice
+        ax.scatter(x[mask], chi[mask], s=8, alpha=0.4, label=label, **scatter_kw)
+        ci = 1.78 / np.sqrt(len(u))
+        return ci
+
+    if args.groupcol is not None:
+        # Each group gets its OWN chi statistic computed within its own subset
+        # (mixing groups into one leave-one-out CDF would misrepresent both) --
+        # overlaid here the same way scat/hist overlay per-group series.
+        cis = []
+        for groupname, gdf in subgraph_groups(
+            df, args.groupcol, getattr(args, "groupcolorder", None)
+        ):
+            label = make_grouplabel(
+                groupname, args.groupcol, getattr(args, "groupcolformat", None)
+            )
+            ci = _draw(gdf, label=label)
+            if ci is not None:
+                cis.append(ci)
+        ax.legend(loc=legendloc, title=getattr(args, "legendtitle", None))
+        ci = max(cis) if cis else 0.0
+    else:
+        ci = _draw(df) or 0.0
+
+    ax.axhline(0, color="black", lw=0.8)
+    ax.axvline(0, color="gray", lw=0.6, ls=":")
+    if ci:
+        ax.axhline(ci, color="crimson", lw=1, ls="--", alpha=0.7)
+        ax.axhline(-ci, color="crimson", lw=1, ls="--", alpha=0.7)
+    ax.set_xlim(-1, 1)
+    ax.set_ylim(-0.2, 1)
+
+    apply_labels(
+        ax,
+        args,
+        default_xlabel="<- lower corner    |    upper corner ->",
+        default_ylabel=r"$\chi$ (local dependence)",
+    )
+    if title_suffix:
+        existing = ax.get_title()
+        ax.set_title(f"{existing}  [{title_suffix}]" if existing else title_suffix)
+
+    ax.grid(True, linewidth=0.4, alpha=0.5)
+
+
+# ---------------------------------------------------------------------------
+# Command
+# ---------------------------------------------------------------------------
+
+_EPILOG = """\
+Reads as: chi elevated (and rising) on the LEFT (negative x) only => lower-
+tail dependence -- the two variables' worst values tend to co-occur, but
+their best values don't (e.g. joint crashes but not joint rallies). Elevated
+on the RIGHT only => upper-tail dependence (joint highs, not joint lows).
+Elevated similarly on BOTH sides, decaying back toward 0 well before the
+edges => generic correlation with no real tail dependence. Flat at 0
+everywhere => independence. Dashed red lines mark an approximate 95%
+control limit for "indistinguishable from independence".
+
+This is a visual complement to `dftk corr --tail`'s numeric
+lambda_lower/lambda_upper columns -- same underlying question, this command
+answers it by eye and per-corner in one plot instead of at a chosen set of
+thresholds. Needs a reasonable amount of data to be legible: a few hundred
+points is usually enough to see the shape, but a strong one-sided asymmetry
+is easiest to trust with several thousand.
+
+EXAMPLES
+--------
+Basic chi-plot:
+
+  dftk chi data.tsv -x recomb_rate -y gc_content
+  dftk dataset tips -o | dftk chi ... -x total_bill -y tip
+
+Split into a subplot grid by category, to check whether tail dependence is
+consistent across the whole dataset or driven by a few subgroups (e.g.
+chromosome, sector, region — here, diamond cut quality):
+
+  dftk dataset diamonds -o | dftk chi ... -x carat -y price --subgraphcol cut
+
+Colour-code by group instead of splitting into subplots:
+
+  dftk dataset diamonds -o | dftk chi ... -x carat -y price -g cut
+
+Publication figure (Nature single column, PDF):
+
+  dftk dataset tips -o | dftk chi ... -x total_bill -y tip \\
+      --size single --fontsize publication -f fig.pdf
+"""
+
+
+class ChiCommand(BaseCommand):
+    name = "chi"
+    help = "Corner-resolved chi-plot: visualize lower vs upper tail dependence."
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_read_arguments(parser)
+        add_xy_arguments(parser)
+        add_figure_arguments(parser)
+        add_group_arguments(parser)
+        add_legend_arguments(parser)
+
+    def execute(self, args: argparse.Namespace) -> None:
+        if not args.xcol or not args.ycol:
+            raise ValueError("--xcol (-x) and --ycol (-y) are required.")
+
+        df = io.read(args)
+        check_cols(df, [args.xcol, args.ycol], "-x/-y")
+        check_cols(df, args.groupcol, "-g/--groupcol")
+        check_cols(df, args.subgraphcol, "--subgraphcol")
+
+        if args.file:
+            import matplotlib
+
+            matplotlib.use("Agg")
+
+        import matplotlib.pyplot as plt
+
+        if getattr(args, "usetex", False):
+            plt.rc("text", usetex=True)
+
+        apply_style(args)
+
+        if args.subgraphcol is not None:
+            groups = subgraph_groups(
+                df, args.subgraphcol, getattr(args, "subgraphorder", None)
+            )
+            nrows, ncols = subgraph_layout(len(groups), args.ncols)
+            fig, axes = make_figure(args, nrows=nrows, ncols=ncols)
+            for idx, (groupname, gdf) in enumerate(groups):
+                ax = axes[idx // ncols][idx % ncols]
+                suffix = make_grouplabel(
+                    groupname, args.subgraphcol, getattr(args, "subgraphformat", None)
+                )
+                _chi_ax(ax, gdf, args, title_suffix=suffix)
+            for idx in range(len(groups), nrows * ncols):
+                axes[idx // ncols][idx % ncols].set_visible(False)
+            fig.tight_layout()
+        else:
+            fig, axes = make_figure(args)
+            _chi_ax(axes[0][0], df, args)
+            fig.tight_layout()
+
+        save_or_show(fig, args)

{dataframe_toolkit-0.5.1 → dataframe_toolkit-0.7.1}/src/dftk/commands/concat_cmd.py

@@ -8,9 +8,11 @@ present in some but not all files are filled with NaN (or --fill value).
 Examples
 --------
     dftk concat a.tsv b.tsv c.tsv
-    dftk concat a.tsv b.tsv --sourcecol source
-    dftk concat a.parquet b.parquet -o
-    dftk eval data.tsv -f "x=1" -o | dftk concat ... extra.tsv
+    dftk dataset penguins -o penguins.tsv
+    dftk concat penguins.tsv penguins.tsv --sourcecol source
+    dftk dataset penguins -o penguins.parquet
+    dftk concat penguins.parquet penguins.parquet -o
+    dftk dataset penguins -o | dftk eval ... -f "x=1" -o | dftk concat ... penguins.tsv
 """
 
 import argparse
@@ -43,9 +45,16 @@ COLUMN ALIGNMENT
 EXAMPLES
 --------
   dftk concat a.tsv b.tsv
-  dftk concat a.tsv b.tsv c.tsv --sourcecol source
-  dftk concat a.parquet b.parquet -o result.parquet
-  dftk eval data.tsv -f "x=1" -o | dftk concat ... extra.tsv
+  dftk dataset penguins -o penguins.tsv
+  dftk concat penguins.tsv penguins.tsv --sourcecol source
+  dftk dataset penguins -o penguins.parquet
+  dftk concat penguins.parquet penguins.parquet -o result.parquet
+  dftk dataset penguins -o | dftk eval ... -f "x=1" -o | dftk concat ... penguins.tsv
+
+Mismatched columns are outer-joined and filled (NaN by default, or --fill):
+
+  dftk dataset iris -o iris.tsv
+  dftk concat iris.tsv penguins.tsv --fill 0
 """
         g = parser.add_argument_group("concat options")
         g.add_argument(
@@ -86,12 +95,6 @@ EXAMPLES
             help="Input files have no header row; columns are named V1, V2, …",
             action="store_true",
         )
-        r.add_argument(
-            "--nrows",
-            help="Maximum number of rows to read per file",
-            type=int,
-            default=None,
-        )
         r.add_argument(
             "--delimiter",
             help="Column delimiter for TSV inputs (default: tab)",