dataframe-toolkit 0.4.1__py3-none-any.whl

dftk/commands/fit_cmd.py

@@ -0,0 +1,265 @@
+"""
+dftk.commands.fit_cmd — dftk fit subcommand.
+
+Port of dfsmfit.py: OLS / robust (RLM) / weighted (WLS) linear regression
+using statsmodels with R-style formulas.
+
+TODO: variable selection — add --select forward/backward/both to perform
+  stepwise AIC/BIC-driven predictor selection on top of OLS.  The selected
+  formula would be logged to stderr and the output table would be identical
+  to a normal fit run.  Lasso / Ridge / Elastic Net regularisation paths are
+  better handled as a separate command (e.g. dftk regpath) using sklearn,
+  since the output and cross-validation workflow differ fundamentally.
+
+TODO: add support for additional model types via a --model flag:
+  --model logit / glm (+ --family binomial/poisson/gamma) — logistic and
+      generalised linear models via smf.logit() / smf.glm().
+  --model mixedlm (+ --groups COL) — mixed linear models with random effects
+      via smf.mixedlm(); output table would need extra columns for random
+      effects covariance.  res2df() would need a branch for each new type
+      since result attributes (rsquared, condnum, etc.) differ.
+
+Output modes
+------------
+Default           Tidy coefficient table (one row per predictor).
+--summary         Print the full statsmodels summary text (no grouping).
+--anova           Print the ANOVA table (no grouping, OLS only).
+
+Grouping (-g)
+-------------
+Runs the regression independently for each group and concatenates the
+tidy coefficient tables into one output dataframe.
+"""
+
+import argparse
+import logging
+
+import pandas as pd
+import statsmodels.api as sm
+import statsmodels.formula.api as smf
+import statsmodels.stats.anova as sa
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import check_cols, io
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Core regression helpers  (also imported by scale_cmd)
+# ---------------------------------------------------------------------------
+
+
+def standard_model(df: pd.DataFrame, args: argparse.Namespace):
+    """Return an OLS or RLM (robust) model, not yet fitted."""
+    if getattr(args, "robust", False):
+        return sm.robust.robust_linear_model.RLM.from_formula(
+            formula=args.formula, data=df, M=sm.robust.norms.HuberT()
+        )
+    return smf.ols(formula=args.formula, data=df)
+
+
+def weighted_model(df: pd.DataFrame, args: argparse.Namespace):
+    """Return a WLS model, not yet fitted."""
+    return sm.regression.linear_model.WLS.from_formula(
+        args.formula, df, weights=df[args.weights]
+    )
+
+
+def regress_it(df: pd.DataFrame, args: argparse.Namespace):
+    """Fit the appropriate model and return the fitted result object."""
+    if getattr(args, "weights", None) is not None:
+        mod = weighted_model(df, args)
+    else:
+        mod = standard_model(df, args)
+    return mod.fit()
+
+
+# ---------------------------------------------------------------------------
+# Result → DataFrame
+# ---------------------------------------------------------------------------
+
+
+def res2df(
+    res,
+    args: argparse.Namespace,
+    groupname=None,
+) -> pd.DataFrame:
+    """Convert a fitted statsmodels result to a tidy coefficient DataFrame."""
+    is_robust = getattr(args, "robust", False)
+
+    ci = res.conf_int()
+    out_dict = {
+        "tag": args.tag,
+        "variable": res.model.data.ynames,
+        "marker": res.model.data.xnames,
+        "effect": res.params,
+        "stdev": res.bse,
+        "pval": res.pvalues,
+        "nobs": res.nobs,
+        "cilo": ci[0],
+        "cihi": ci[1],
+    }
+    stat_cols = [
+        "tag",
+        "variable",
+        "marker",
+        "effect",
+        "stdev",
+        "pval",
+        "nobs",
+        "cilo",
+        "cihi",
+    ]
+
+    if not is_robust:
+        out_dict["rsquared"] = res.rsquared
+        out_dict["condnum"] = res.condition_number
+        stat_cols += ["rsquared", "condnum"]
+
+    if groupname is None:
+        return pd.DataFrame(out_dict)[stat_cols]
+
+    # Grouped: prepend group-column values
+    vals = list(groupname) if isinstance(groupname, tuple) else [groupname]
+    grpcols = args.groupcol[:]
+    n = len(res.params)
+    for grpcol, val in zip(grpcols, vals, strict=False):
+        out_dict[grpcol] = [val] * n
+    return pd.DataFrame(out_dict)[grpcols + stat_cols]
+
+
+# ---------------------------------------------------------------------------
+# Command
+# ---------------------------------------------------------------------------
+
+
+_EPILOG = """\
+OUTPUT COLUMNS
+--------------
+  tag       Value of --tag (label for the result set).
+  variable  Dependent variable name (left-hand side of formula).
+  marker    Predictor name (Intercept plus each right-hand side term).
+  effect    Estimated coefficient.
+  stdev     Standard error of the coefficient.
+  pval      Two-sided p-value.
+  nobs      Number of observations used.
+  cilo      Lower bound of the 95 % confidence interval.
+  cihi      Upper bound of the 95 % confidence interval.
+  rsquared  R² (OLS / WLS only).
+  condnum   Condition number (OLS / WLS only).
+
+
+EXAMPLES
+--------
+Simple OLS:
+
+  dftk fit data.tsv -f "y ~ x"
+
+Grouped OLS (one regression per group):
+
+  dftk fit data.tsv -f "y ~ x" -g group
+
+Weighted least squares:
+
+  dftk fit data.tsv -f "y ~ x" -w weight_col
+
+Robust regression (HuberT):
+
+  dftk fit data.tsv -f "y ~ x" --robust
+
+Print detailed summary (no grouping):
+
+  dftk fit data.tsv -f "y ~ x" --summary
+
+ANOVA table (OLS, no grouping):
+
+  dftk fit data.tsv -f "y ~ x + z" --anova
+"""
+
+
+class FitCommand(BaseCommand):
+    name = "fit"
+    help = "OLS / robust / weighted linear regression (R-style formula)."
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("regression options")
+        g.add_argument(
+            "-f",
+            "--formula",
+            required=True,
+            metavar="FORMULA",
+            help="R-style formula, e.g. 'y ~ x + z'.",
+        )
+        g.add_argument(
+            "-w",
+            "--weights",
+            default=None,
+            metavar="COL",
+            help="Column name to use as observation weights (WLS).",
+        )
+        g.add_argument(
+            "--tag",
+            default="tag",
+            metavar="LABEL",
+            help="Label added to the 'tag' column in every output row (default: tag).",
+        )
+        g.add_argument(
+            "-r",
+            "--robust",
+            action="store_true",
+            help="Use robust linear regression (RLM with HuberT norm) instead of OLS.",
+        )
+        g.add_argument(
+            "-g",
+            "--groupcol",
+            nargs="+",
+            default=None,
+            metavar="COL",
+            help="Run a separate regression for each combination of these columns.",
+        )
+        g.add_argument(
+            "-s",
+            "--summary",
+            action="store_true",
+            help="Print the full statsmodels summary (overrides tabular output; no grouping).",  # noqa: E501
+        )
+        g.add_argument(
+            "--anova",
+            action="store_true",
+            help="Print the ANOVA table (OLS only, no grouping).",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+        check_cols(df, args.groupcol, "-g/--groupcol")
+        if args.weights is not None:
+            check_cols(df, [args.weights], "-w/--weights")
+
+        if args.groupcol is None:
+            res = regress_it(df, args)
+            if args.summary:
+                print(res.summary())
+                return
+            if args.anova:
+                print(sa.anova_lm(res))
+                return
+            out = res2df(res, args)
+        else:
+            grouped = df.groupby(args.groupcol)
+            parts = []
+            for groupname, groupdf in grouped:
+                try:
+                    res = regress_it(groupdf, args)
+                except Exception as exc:
+                    logger.error("Regression failed for group %s: %s", groupname, exc)
+                    raise
+                parts.append(res2df(res, args, groupname))
+            out = pd.concat(parts, ignore_index=True)
+
+        io.printdf(out, args)

dftk/commands/func_cmd.py

@@ -0,0 +1,186 @@
+"""
+dftk.commands.func_cmd — dftk func subcommand.
+
+Applies column transforms, optionally within groups.  The main use case is
+adding derived columns to a DataFrame: running totals, quantile bins, group-
+level aggregates broadcast back to the original row count, etc.
+
+TRANSFORMS (-t / --transform)
+------------------------------
+  sum       Group sum broadcast to each row (or simple column sum).
+  mean      Group mean broadcast to each row.
+  min       Group min broadcast to each row.
+  max       Group max broadcast to each row.
+  count     Non-null count per group broadcast to each row.
+  median    Group median broadcast to each row.
+  std       Group standard deviation broadcast to each row.
+  cumsum    Cumulative sum (within group when -g is given).
+  rank      Rank within group (average method, ascending).
+  pct_rank  Percentile rank within group (0–1).
+  qcut:N    Assign each value to one of N equal-frequency quantile bins.
+            The bin label is an integer 1..N.
+
+EXAMPLES
+--------
+Cumulative sum of "value":
+
+  dftk func data.tsv -c value -t cumsum
+
+Group mean broadcast (adds "value_mean" unless -d is given):
+
+  dftk func data.tsv -c value -g group -t mean
+
+Quantile bins (quartiles):
+
+  dftk func data.tsv -c score -t qcut:4 -d score_quartile
+
+Multiple transforms in one call:
+
+  dftk func data.tsv -c expr -t cumsum -d expr_cumsum \\
+      | dftk func - -c expr -t mean -g condition -d expr_grpmean
+"""
+
+import argparse
+import re
+
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import check_cols, io
+
+# Transforms that use groupby().transform()
+_GROUPBY_TRANSFORMS = {"sum", "mean", "min", "max", "count", "median", "std"}
+
+# Transforms that use cumsum / rank within an optional group
+_CUMULATIVE_TRANSFORMS = {"cumsum", "rank", "pct_rank"}
+
+
+def _apply_transform(
+    df: pd.DataFrame,
+    col: str,
+    transform: str,
+    groupcols: list[str] | None,
+    destcol: str,
+) -> pd.DataFrame:
+    """Apply *transform* on *col* within optional *groupcols*, storing result in *destcol*."""  # noqa: E501
+
+    # --- qcut:N ---
+    m = re.fullmatch(r"qcut:(\d+)", transform)
+    if m:
+        n = int(m.group(1))
+        if n < 2:
+            raise ValueError("qcut requires N >= 2")
+        if groupcols:
+
+            def _qcut_group(s: pd.Series) -> pd.Series:
+                return (
+                    pd.qcut(s, n, labels=False, duplicates="drop").astype("Int64") + 1
+                )  # noqa: E501
+
+            df[destcol] = df.groupby(groupcols)[col].transform(_qcut_group)
+        else:
+            df[destcol] = (
+                pd.qcut(df[col], n, labels=False, duplicates="drop").astype("Int64") + 1
+            )
+        return df
+
+    # --- groupby().transform() aggregates ---
+    if transform in _GROUPBY_TRANSFORMS:
+        if groupcols:
+            df[destcol] = df.groupby(groupcols)[col].transform(transform)
+        else:
+            agg_val = getattr(df[col], transform)()
+            df[destcol] = agg_val
+        return df
+
+    # --- cumsum ---
+    if transform == "cumsum":
+        if groupcols:
+            df[destcol] = df.groupby(groupcols)[col].transform("cumsum")
+        else:
+            df[destcol] = df[col].cumsum()
+        return df
+
+    # --- rank ---
+    if transform == "rank":
+        if groupcols:
+            df[destcol] = df.groupby(groupcols)[col].rank(method="average")
+        else:
+            df[destcol] = df[col].rank(method="average")
+        return df
+
+    # --- pct_rank ---
+    if transform == "pct_rank":
+        if groupcols:
+            df[destcol] = df.groupby(groupcols)[col].rank(method="average", pct=True)
+        else:
+            df[destcol] = df[col].rank(method="average", pct=True)
+        return df
+
+    raise ValueError(
+        f"Unknown transform {transform!r}.  "
+        "Valid: sum, mean, min, max, count, median, std, cumsum, rank, pct_rank, qcut:N"
+    )
+
+
+def _default_destcol(col: str, transform: str) -> str:
+    """Generate a destination column name from source column + transform."""
+    safe = re.sub(r"[^a-zA-Z0-9_]", "_", transform)
+    return f"{col}_{safe}"
+
+
+class FuncCommand(BaseCommand):
+    name = "func"
+    help = "Apply column transforms (cumsum, group mean/sum, qcut, …)."
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = __doc__
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("transform options")
+        g.add_argument(
+            "-c",
+            "--col",
+            required=True,
+            metavar="COL",
+            help="Source column to transform.",
+        )
+        g.add_argument(
+            "-t",
+            "--transform",
+            required=True,
+            metavar="TRANSFORM",
+            help=(
+                "Transform to apply: sum, mean, min, max, count, median, std, "
+                "cumsum, rank, pct_rank, or qcut:N."
+            ),
+        )
+        g.add_argument(
+            "-d",
+            "--destcol",
+            default=None,
+            metavar="NAME",
+            help="Name of the output column (default: <col>_<transform>).",
+        )
+        g.add_argument(
+            "-g",
+            "--groupcol",
+            nargs="+",
+            default=None,
+            metavar="COL",
+            help="Group by these column(s) before applying the transform.",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+
+        check_cols(df, [args.col], "-c/--col")
+        check_cols(df, args.groupcol, "-g/--groupcol")
+
+        destcol = args.destcol or _default_destcol(args.col, args.transform)
+
+        df = _apply_transform(df, args.col, args.transform, args.groupcol, destcol)
+
+        io.printdf(df, args)

dftk/commands/help.py

@@ -0,0 +1,63 @@
+"""
+dftk.commands.help — the ``dftk help`` subcommand.
+
+  dftk help              — list all available subcommands
+  dftk help <command>    — show full argparse help for one subcommand
+"""
+
+import argparse
+
+from dftk.commands.base import BaseCommand
+
+
+class HelpCommand(BaseCommand):
+    """Show help for dftk subcommands."""
+
+    name = "help"
+    help = "Show detailed help for a subcommand"
+
+    def __init__(
+        self,
+        available_commands: list[BaseCommand] | None = None,
+        command_groups: list[tuple[str, list[BaseCommand]]] | None = None,
+    ) -> None:
+        self.available_commands: list[BaseCommand] = available_commands or []
+        self.command_groups: list[tuple[str, list[BaseCommand]]] = command_groups or []
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "command_name",
+            nargs="?",
+            metavar="COMMAND",
+            help="Name of the subcommand to get help for",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        if args.command_name is None:
+            if self.command_groups:
+                for header, cmds in self.command_groups:
+                    print(f"\n{header}:")
+                    for cmd in cmds:
+                        print(f"  {cmd.name:<16} {cmd.help}")
+            else:
+                print("Available commands:")
+                for cmd in self.available_commands:
+                    print(f"  {cmd.name:<16} {cmd.help}")
+            print("\nUse 'dftk help <command>' for detailed help on a command.")
+            return
+
+        cmd = next(
+            (c for c in self.available_commands if c.name == args.command_name),
+            None,
+        )
+        if cmd is None:
+            raise ValueError(f"Unknown command: {args.command_name!r}")
+
+        # Build a temporary parser for this command so argparse formats the
+        # help text exactly as it would appear during normal use.
+        temp_parser = argparse.ArgumentParser(
+            prog=f"dftk {cmd.name}",
+            description=cmd.help,
+        )
+        cmd.add_arguments(temp_parser)
+        temp_parser.print_help()