dataframe-toolkit 0.4.1__py3-none-any.whl

dftk/commands/randvar_cmd.py

@@ -0,0 +1,199 @@
+"""
+dftk.commands.randvar_cmd — dftk randvar subcommand.
+
+Port of dfrvs.py: append a column of random variates drawn from any
+scipy.stats distribution, or generate a stand-alone sample without an
+input file.
+
+Usage
+-----
+  # Append a column to an existing file
+  dftk randvar data.tsv -d x --dist norm --parameters loc:0,scale:1
+
+  # Generate a stand-alone sample (no input file)
+  dftk randvar -n 1000 -d x --dist norm --parameters loc:0,scale:1
+  dftk randvar -n 500  -d x --dist beta --parameters a:2,b:5 --randomseed 42
+
+  # List all available distributions and their parameters
+  dftk randvar --list
+"""
+
+import argparse
+import logging
+import sys
+
+import numpy as np
+import pandas as pd
+import scipy.stats as ss
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import io
+from dftk.common.seed import normalize_seed
+
+logger = logging.getLogger(__name__)
+
+_SCIPY_DOCS = "https://docs.scipy.org/doc/scipy/reference/stats.html"
+
+_EPILOG = f"""\
+PARAMETERS
+----------
+Distribution parameters are passed as key:val pairs separated by commas:
+
+  --parameters loc:0,scale:1
+  --parameters a:2,b:5,loc:0,scale:1
+
+The parameter names are distribution-specific.  Use --list to see the
+accepted parameters for every distribution, or consult the scipy.stats
+documentation:
+
+  {_SCIPY_DOCS}
+
+RANDOM SEED
+-----------
+--randomseed accepts an integer or any string.  A non-numeric string is
+hashed (MD5) to a reproducible uint32 seed, which is printed to stderr.
+
+EXAMPLES
+--------
+  dftk randvar data.tsv -d noise --dist norm
+  dftk randvar data.tsv -d noise --dist norm --parameters loc:5,scale:2
+  dftk randvar -n 1000 -d u --dist uniform --randomseed 99
+  dftk randvar -n 500  -d k --dist poisson --parameters mu:3
+  dftk randvar --list
+  dftk randvar --list | dftk query - --query "type == 'continuous'"
+"""
+
+
+def _parse_parameters(paramstring: str | None) -> dict:
+    """Parse 'key:val,key:val' into {key: float} dict."""
+    if not paramstring:
+        return {}
+    pairs = [x.split(":") for x in paramstring.split(",")]
+    return {k: float(v) for k, v in pairs}
+
+
+def _list_distributions() -> pd.DataFrame:
+    """Return a DataFrame of all scipy.stats distributions and their parameters."""
+    rows = []
+    for name in sorted(ss._continuous_distns._distn_names):
+        dist = getattr(ss, name)
+        shapes = dist.shapes or ""
+        params = (shapes + ", " if shapes else "") + "loc, scale"
+        rows.append({"name": name, "type": "continuous", "parameters": params})
+    for name in sorted(ss._discrete_distns._distn_names):
+        dist = getattr(ss, name)
+        shapes = dist.shapes or ""
+        params = (shapes + ", " if shapes else "") + "loc"
+        rows.append({"name": name, "type": "discrete", "parameters": params})
+    return pd.DataFrame(rows)
+
+
+class RandvarCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "randvar"
+
+    @property
+    def help(self) -> str:
+        return "Append a column of random variates from a scipy.stats distribution."
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("distribution")
+        g.add_argument(
+            "-d",
+            "--destcol",
+            default=None,
+            metavar="COL",
+            help="Name of the new column to append.",
+        )
+        g.add_argument(
+            "--dist",
+            default=None,
+            metavar="DIST",
+            help="scipy.stats distribution name (e.g. norm, uniform, beta, poisson).",
+        )
+        g.add_argument(
+            "--parameters",
+            default=None,
+            metavar="KEY:VAL,...",
+            help="Distribution parameters as key:val pairs, e.g. loc:0,scale:1.",
+        )
+        g.add_argument(
+            "--randomseed",
+            default=None,
+            metavar="SEED",
+            help="Random seed (integer or string; strings are hashed to uint32).",
+        )
+        g.add_argument(
+            "-n",
+            "--nsamples",
+            type=int,
+            default=None,
+            metavar="N",
+            help=(
+                "Number of samples to generate.  Required when no DATAFILE is "
+                "provided and stdin is not piped; ignored otherwise."
+            ),
+        )
+        g.add_argument(
+            "--list",
+            action="store_true",
+            help="List all available scipy.stats distributions and their parameters.",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        # --list short-circuits everything
+        if args.list:
+            df = _list_distributions()
+            io.printdf(df, args)
+            return
+
+        # Validate required args (not enforced at parse time to allow --list)
+        if not args.destcol:
+            logger.error("-d/--destcol is required.")
+            sys.exit(1)
+        if not args.dist:
+            logger.error("--dist is required.")
+            sys.exit(1)
+
+        # Resolve distribution
+        try:
+            dist_cls = getattr(ss, args.dist)
+        except AttributeError:
+            logger.error(
+                "Distribution %r not found in scipy.stats. Use --list.",
+                args.dist,
+            )
+            sys.exit(1)
+
+        kwargs = _parse_parameters(args.parameters)
+        try:
+            rv = dist_cls(**kwargs)
+        except TypeError as exc:
+            logger.error("Invalid parameters for %r: %s", args.dist, exc)
+            sys.exit(1)
+
+        seed = normalize_seed(args.randomseed)
+        rng = np.random.default_rng(seed)
+
+        # -n given → standalone mode (no input file needed)
+        # -n absent → read DATAFILE or stdin, use len(df) as sample count
+        if args.nsamples is not None:
+            df = pd.DataFrame(
+                {args.destcol: rv.rvs(size=args.nsamples, random_state=rng)}
+            )
+        else:
+            if args.DATAFILE is None and sys.stdin.isatty():
+                logger.error(
+                    "No input: provide a DATAFILE, pipe to stdin, or use -n/--nsamples."
+                )
+                sys.exit(1)
+            df = io.read(args)
+            df[args.destcol] = rv.rvs(size=len(df), random_state=rng)
+
+        io.printdf(df, args)

dftk/commands/sample_cmd.py

@@ -0,0 +1,126 @@
+"""
+dftk.commands.sample_cmd — dftk sample subcommand.
+
+Draw a random sample of rows from a DataFrame, optionally within groups.
+
+Sampling size (one required)
+-----------------------------
+  -n N      draw exactly N rows
+  -f FRAC   draw a fraction of rows (0.0 – 1.0); use 1.0 to shuffle
+
+Options
+-------
+  -g COL …     sample independently within each group
+  --randomseed random seed — integer or string (strings are MD5-hashed)
+  --replace    sample with replacement
+
+Examples
+--------
+    dftk sample data.tsv -n 100
+    dftk sample data.tsv -f 0.8
+    dftk sample data.tsv -n 50 -g treatment --randomseed 42
+    dftk sample data.tsv -f 1.0 --randomseed shuffle
+"""
+
+import argparse
+import sys
+
+import numpy as np
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import io
+from dftk.common.seed import normalize_seed
+
+
+class SampleCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "sample"
+
+    @property
+    def help(self) -> str:
+        return "Draw a random sample of rows, optionally within groups"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        import argparse as ap
+
+        parser.formatter_class = ap.RawDescriptionHelpFormatter
+        parser.epilog = """\
+EXAMPLES
+--------
+  dftk sample data.tsv -n 100
+  dftk sample data.tsv -f 0.8
+  dftk sample data.tsv -n 50 -g treatment --randomseed 42
+  dftk sample data.tsv -f 1.0 --randomseed shuffle   # reproducible shuffle
+  dftk sample data.tsv -n 10 -g group --replace
+"""
+        io.parser_read(parser)
+
+        g = parser.add_argument_group("sampling")
+        size = g.add_mutually_exclusive_group(required=True)
+        size.add_argument(
+            "-n",
+            dest="samplesize",
+            type=int,
+            default=None,
+            metavar="N",
+            help="Number of rows to sample.",
+        )
+        size.add_argument(
+            "-f",
+            dest="samplefrac",
+            type=float,
+            default=None,
+            metavar="FRAC",
+            help="Fraction of rows to sample (0.0 – 1.0).  Use 1.0 to shuffle.",
+        )
+        g.add_argument(
+            "-g",
+            "--groupcol",
+            nargs="+",
+            default=None,
+            metavar="COL",
+            help="Sample independently within each group defined by these columns.",
+        )
+        g.add_argument(
+            "--randomseed",
+            default=None,
+            metavar="SEED",
+            help=(
+                "Random seed for reproducibility.  "
+                "Accepts an integer or any string (hashed via MD5 to a uint32)."
+            ),
+        )
+        g.add_argument(
+            "--replace",
+            action="store_true",
+            help="Sample with replacement.",
+        )
+
+        io.parser_output(parser)
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+
+        seed = normalize_seed(args.randomseed)
+        if seed is not None:
+            print(f"randomseed = {seed}", file=sys.stderr)
+
+        rng = np.random.default_rng(seed)
+
+        sample_kwargs: dict = {"replace": args.replace, "random_state": rng}
+        if args.samplesize is not None:
+            sample_kwargs["n"] = args.samplesize
+        else:
+            sample_kwargs["frac"] = args.samplefrac
+
+        if args.groupcol is not None:
+            pieces = [
+                grp.sample(**sample_kwargs) for _, grp in df.groupby(args.groupcol)
+            ]
+            result = pd.concat(pieces).reset_index(drop=True)
+        else:
+            result = df.sample(**sample_kwargs).reset_index(drop=True)
+
+        io.printdf(result, args)

dftk/commands/scale_cmd.py

@@ -0,0 +1,299 @@
+"""
+dftk.commands.scale_cmd — dftk scale subcommand.
+
+Port of dfscale.py: column-wise normalization / rank transformation /
+regression detrending.
+
+Three modes (mutually exclusive flags)
+---------------------------------------
+Default (scale)   Linear shift + scale.  New options --shift / --scale
+                  replace the old --noshift / --minshift / --inishift /
+                  --noscale / --sumscale / --maxscale / --meanscale flags.
+--rank            Rank normalization.
+--resid           Regression detrending (residuals); requires -f/--formula.
+
+Output
+------
+Each processed column gets a companion column named <col>_scaled appended
+to the output dataframe.  All other columns are preserved.
+
+Grouping (-g)
+-------------
+Any mode can be applied per group.  Results are concatenated into a single
+dataframe and written once (cleaner than the old loop-with-removeheader
+approach).
+"""
+
+import argparse
+import logging
+import sys
+
+import numpy as np
+import pandas as pd
+from scipy.stats import norm
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import check_cols, io
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Mode implementations
+# ---------------------------------------------------------------------------
+
+
+def scalecols(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
+    """Linear shift + scale normalization."""
+    for col in args.cols:
+        newcol = f"{col}_scaled"
+        x = df[col].astype(float)
+
+        shift_val = {
+            "mean": x.mean(),
+            "min": x.min(),
+            "first": float(x.iloc[0]),
+            "none": 0.0,
+        }[args.shift]
+
+        shifted = x - shift_val
+
+        if args.scale_by == "none":
+            df[newcol] = shifted
+            continue
+
+        scale_val = {
+            "std": shifted.std(),
+            "sum": shifted.sum(),
+            "max": shifted.abs().max(),
+            "mean": shifted.mean(),
+        }[args.scale_by]
+
+        if args.verbose:
+            print(
+                f"{newcol}: shift={shift_val:.6g}  scale={scale_val:.6g}",
+                file=sys.stderr,
+            )
+
+        df[newcol] = shifted / scale_val if scale_val != 0 else shifted
+
+    return df
+
+
+def rankcols(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
+    """Rank normalization to uniform [0,1] or normal scores."""
+    n = len(df)
+    for col in args.cols:
+        newcol = f"{col}_scaled"
+        x = df[col].astype(float)
+        rnk = x.rank(method="average").values
+
+        if args.rankdist == "uniform":
+            df[newcol] = (rnk - 0.5) / n
+        else:  # normal (Blom transform)
+            p = (rnk - 3 / 8.0) / (n + 1 / 4.0)
+            z = norm.ppf(p)
+            z = z - z.mean()
+            s = z.std(ddof=0)
+            df[newcol] = z / s if s > 0 else np.zeros_like(z)
+
+    return df
+
+
+def residcols(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
+    """Regression detrending: replace column with (intercept +) residuals."""
+    # Import here to avoid circular imports at module level
+    from dftk.commands.fit_cmd import regress_it
+
+    # regress_it needs these attrs; set safe defaults if absent
+    if not hasattr(args, "robust"):
+        args.robust = False
+    if not hasattr(args, "weights"):
+        args.weights = None
+
+    res = regress_it(df, args)
+    col = res.model.endog_names
+    newcol = f"{col}_scaled"
+
+    intercept = 0.0
+    if not args.nointercept and "Intercept" in res.params:
+        intercept = res.params["Intercept"]
+
+    if args.verbose:
+        print(res.summary(), file=sys.stderr)
+
+    df[newcol] = intercept + res.resid
+    return df
+
+
+# ---------------------------------------------------------------------------
+# Command
+# ---------------------------------------------------------------------------
+
+_EPILOG = """\
+MODES
+-----
+Default (scale):
+
+  dftk scale data.tsv -c y                        z-score (mean shift, std scale)
+  dftk scale data.tsv -c y --shift min            shift so min=0, scale by std
+  dftk scale data.tsv -c y --shift min --scale max shift so min=0, scale so max=1
+  dftk scale data.tsv -c y --shift none --scale sum no shift, scale so sum=1
+  dftk scale data.tsv -c y --scale none           shift to zero mean, no scaling
+
+Rank normalization (--rank):
+
+  dftk scale data.tsv -c y --rank                 normal scores (default)
+  dftk scale data.tsv -c y --rank --rankdist uniform  uniform [0, 1]
+
+Regression detrending (--resid):
+
+  dftk scale data.tsv -c y --resid -f "y ~ x"
+  dftk scale data.tsv -c y --resid -f "y ~ x" --nointercept
+
+Grouping:
+
+  dftk scale data.tsv -c y -g group               z-score within each group
+  dftk scale data.tsv -c y --rank -g group         rank within each group
+
+SHIFT OPTIONS (--shift)
+-----------------------
+  mean    Subtract the column mean (default).  Results in zero mean.
+  min     Subtract the column minimum.  Lowest value becomes 0.
+  first   Subtract the first observation.  First value becomes 0.
+  none    No shift applied.
+
+SCALE OPTIONS (--scale)
+-----------------------
+  std     Divide by standard deviation (default).  Results in unit std.
+  sum     Divide by sum.  Values sum to 1.
+  max     Divide by absolute maximum.  Largest absolute value becomes 1.
+  mean    Divide by mean.  Mean becomes 1.
+  none    No scaling applied.
+"""
+
+
+class ScaleCommand(BaseCommand):
+    name = "scale"
+    help = "Column-wise normalization, rank transformation, or regression detrending."
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("column selection")
+        g.add_argument(
+            "-c",
+            "--cols",
+            nargs="+",
+            metavar="COL",
+            default=None,
+            help="Column(s) to process.  Required for scale and rank modes.",
+        )
+        g.add_argument(
+            "-g",
+            "--groupcol",
+            nargs="+",
+            default=None,
+            metavar="COL",
+            help="Apply transformation independently within each group.",
+        )
+
+        mode = parser.add_argument_group("mode (mutually exclusive)")
+        mx = mode.add_mutually_exclusive_group()
+        mx.add_argument(
+            "--rank",
+            action="store_true",
+            help="Rank normalization (normal scores by default; see --rankdist).",
+        )
+        mx.add_argument(
+            "--resid",
+            action="store_true",
+            help="Regression detrending: replace column with (intercept +) residuals. "
+            "Requires -f/--formula.",
+        )
+
+        sp = parser.add_argument_group("scale mode options")
+        sp.add_argument(
+            "--shift",
+            choices=["mean", "min", "first", "none"],
+            default="mean",
+            help="What to subtract before scaling (default: mean).",
+        )
+        sp.add_argument(
+            "--scale",
+            dest="scale_by",
+            choices=["std", "sum", "max", "mean", "none"],
+            default="std",
+            help="What to divide by after shifting (default: std).",
+        )
+
+        rp = parser.add_argument_group("rank mode options")
+        rp.add_argument(
+            "--rankdist",
+            choices=["normal", "uniform"],
+            default="normal",
+            help="Distribution for rank normalization (default: normal = Blom scores).",
+        )
+
+        dp = parser.add_argument_group("resid mode options")
+        dp.add_argument(
+            "-f",
+            "--formula",
+            default=None,
+            metavar="FORMULA",
+            help="R-style formula for regression detrending, e.g. 'y ~ x'.",
+        )
+        dp.add_argument(
+            "--nointercept",
+            action="store_true",
+            help="Return raw residuals without adding back the intercept.",
+        )
+
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            action="store_true",
+            help="Print shift/scale values (scale mode) or full summary (resid mode) "
+            "to stderr.",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        # Validate mode / required args
+        if args.resid:
+            if not args.formula:
+                raise ValueError("--resid requires -f/--formula.")
+            fn = residcols
+        elif args.rank:
+            if not args.cols:
+                raise ValueError("--rank requires -c/--cols.")
+            fn = rankcols
+        else:
+            if not args.cols:
+                raise ValueError("-c/--cols is required.")
+            fn = scalecols
+
+        df = io.read(args)
+        check_cols(df, args.cols, "-c/--cols")
+        check_cols(df, args.groupcol, "-g/--groupcol")
+
+        if args.groupcol is None:
+            result = fn(df, args)
+            io.printdf(result, args)
+        else:
+            parts = []
+            for groupname, groupdf in df.groupby(args.groupcol):
+                if args.verbose:
+                    groupstr = (
+                        ", ".join(
+                            f"{k}={v}"
+                            for k, v in zip(args.groupcol, groupname, strict=False)
+                            if isinstance(groupname, tuple)
+                        )
+                        or f"{args.groupcol[0]}={groupname}"
+                    )
+                    print(f"Group: {groupstr}", file=sys.stderr)
+                parts.append(fn(groupdf.copy(), args))
+            io.printdf(pd.concat(parts, ignore_index=True), args)