dataframe-toolkit 0.7.3__tar.gz → 0.8.0__tar.gz

{dataframe_toolkit-0.7.3 → dataframe_toolkit-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dataframe-toolkit
-Version: 0.7.3
+Version: 0.8.0
 Summary: A command-line toolkit for DataFrame analysis and manipulation
 Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
 Author: Gunnar Pálsson
@@ -109,20 +109,21 @@ dftk dataset tips -o \
 
 ### Data transformation
 
-| Command     | Description                                                                          |
-|-------------|---------------------------------------------------------------------------------------|
-| `eval`      | Add or modify columns: eval expressions, string/path functions, statistical ops      |
-| `query`     | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
-| `merge`     | Join two tables on key columns (inner/left/right/outer)                              |
-| `concat`    | Concatenate two or more tables row-wise                                              |
-| `melt`      | Reshape wide-to-long (`pd.melt`)                                                     |
-| `pivot`     | Reshape long-to-wide with per-cell aggregation                                       |
-| `func`      | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
-| `scale`     | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
-| `interp`    | Interpolate values from a reference curve into a table (1-D lookup)                  |
-| `binx`      | Assign bin indices to a column based on explicit or generated edges                  |
-| `transpose` | Flip rows and columns; original column names land in a new `--keycol` column         |
-| `segid`     | Assign a segment ID that increments on each value change in a column                 |
+| Command      | Description                                                                          |
+|--------------|---------------------------------------------------------------------------------------|
+| `eval`       | Add or modify columns: eval expressions, string/path functions, statistical ops      |
+| `query`      | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
+| `merge`      | Join two tables on key columns (inner/left/right/outer)                              |
+| `concat`     | Concatenate two or more tables row-wise                                              |
+| `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
+| `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
+| `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
+| `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
+| `interp`     | Interpolate values from a reference curve into a table (1-D lookup)                  |
+| `binx`       | Assign bin indices to a column based on explicit or generated edges                  |
+| `transpose`  | Flip rows and columns; original column names land in a new `--keycol` column         |
+| `segid`      | Assign a segment ID that increments on each value change in a column                 |
+| `cleannames` | Rename columns to lowercase identifiers (`Sample ID` → `sample_id`), e.g. for SQL    |
 
 ### Statistics
 
@@ -173,6 +174,18 @@ dftk dataset iris -o \
   | dftk fit ... -f "sepal_length_scaled ~ petal_length" -g species
 ```
 
+### Clean up spreadsheet headers
+
+Headers such as `Country Name` or `1960` need quoting in SQL. `cleannames` turns them into lowercase identifiers (`country_name`, `x1960`) and keeps the original names in the parquet metadata (`dftk annotate FILE`):
+
+```bash
+dftk dataset fertility -o \
+  | dftk cleannames ... -o \
+  | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
+```
+
+For your own export, `dftk cleannames data.tsv --backend duckdb -o data.parquet` also keeps integer columns with empty cells as integers.
+
 ### Group summary then plot
 
 ```bash

{dataframe_toolkit-0.7.3 → dataframe_toolkit-0.8.0}/README.md

@@ -80,20 +80,21 @@ dftk dataset tips -o \
 
 ### Data transformation
 
-| Command     | Description                                                                          |
-|-------------|---------------------------------------------------------------------------------------|
-| `eval`      | Add or modify columns: eval expressions, string/path functions, statistical ops      |
-| `query`     | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
-| `merge`     | Join two tables on key columns (inner/left/right/outer)                              |
-| `concat`    | Concatenate two or more tables row-wise                                              |
-| `melt`      | Reshape wide-to-long (`pd.melt`)                                                     |
-| `pivot`     | Reshape long-to-wide with per-cell aggregation                                       |
-| `func`      | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
-| `scale`     | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
-| `interp`    | Interpolate values from a reference curve into a table (1-D lookup)                  |
-| `binx`      | Assign bin indices to a column based on explicit or generated edges                  |
-| `transpose` | Flip rows and columns; original column names land in a new `--keycol` column         |
-| `segid`     | Assign a segment ID that increments on each value change in a column                 |
+| Command      | Description                                                                          |
+|--------------|---------------------------------------------------------------------------------------|
+| `eval`       | Add or modify columns: eval expressions, string/path functions, statistical ops      |
+| `query`      | Filter rows with pandas query expressions or SQL (`--sql` via DuckDB)                |
+| `merge`      | Join two tables on key columns (inner/left/right/outer)                              |
+| `concat`     | Concatenate two or more tables row-wise                                              |
+| `melt`       | Reshape wide-to-long (`pd.melt`)                                                     |
+| `pivot`      | Reshape long-to-wide with per-cell aggregation                                       |
+| `func`       | Column transforms: cumsum, group mean/sum/min/max/count/median/std, rank, qcut:N     |
+| `scale`      | Normalise: z-score, min-shift, sum/max/mean scaling, Blom rank, regression residuals |
+| `interp`     | Interpolate values from a reference curve into a table (1-D lookup)                  |
+| `binx`       | Assign bin indices to a column based on explicit or generated edges                  |
+| `transpose`  | Flip rows and columns; original column names land in a new `--keycol` column         |
+| `segid`      | Assign a segment ID that increments on each value change in a column                 |
+| `cleannames` | Rename columns to lowercase identifiers (`Sample ID` → `sample_id`), e.g. for SQL    |
 
 ### Statistics
 
@@ -144,6 +145,18 @@ dftk dataset iris -o \
   | dftk fit ... -f "sepal_length_scaled ~ petal_length" -g species
 ```
 
+### Clean up spreadsheet headers
+
+Headers such as `Country Name` or `1960` need quoting in SQL. `cleannames` turns them into lowercase identifiers (`country_name`, `x1960`) and keeps the original names in the parquet metadata (`dftk annotate FILE`):
+
+```bash
+dftk dataset fertility -o \
+  | dftk cleannames ... -o \
+  | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
+```
+
+For your own export, `dftk cleannames data.tsv --backend duckdb -o data.parquet` also keeps integer columns with empty cells as integers.
+
 ### Group summary then plot
 
 ```bash

{dataframe_toolkit-0.7.3 → dataframe_toolkit-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.3"
+version = "0.8.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 keywords = [

{dataframe_toolkit-0.7.3 → dataframe_toolkit-0.8.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dataframe-toolkit"
-version = "0.7.3"
+version = "0.8.0"
 description = "A command-line toolkit for DataFrame analysis and manipulation"
 readme = "README.md"
 authors = [

{dataframe_toolkit-0.7.3 → dataframe_toolkit-0.8.0}/src/dftk/commands/__init__.py

@@ -37,6 +37,7 @@ COMMAND_GROUPS: list[tuple[str, list[tuple[str, str]]]] = [
             ("binx", "binx_cmd:BinxCommand"),
             ("transpose", "transpose_cmd:TransposeCommand"),
             ("segid", "segid_cmd:SegidCommand"),
+            ("cleannames", "cleannames_cmd:CleannamesCommand"),
         ],
     ),
     (

{dataframe_toolkit-0.7.3 → dataframe_toolkit-0.8.0}/src/dftk/commands/base.py

@@ -71,6 +71,7 @@ class BaseCommand(ABC):
         tabular data and produces tabular output.
         """
         from dftk.common.io import io
+
         io.parser_read(parser)
         io.parser_output(parser)
 
@@ -82,4 +83,5 @@ class BaseCommand(ABC):
         exposed.
         """
         from dftk.common.io import io
+
         io.parser_read(parser)

dataframe_toolkit-0.8.0/src/dftk/commands/cleannames_cmd.py

@@ -0,0 +1,184 @@
+"""
+dftk.commands.cleannames_cmd — dftk cleannames subcommand.
+
+Rename columns to lowercase ASCII identifiers (snake_case), so they can be
+used unquoted in query --sql, eval -f and fit formulas.  Typical input is a
+spreadsheet export with headers such as "Sample ID", "GC %" or "Hæð (cm)".
+
+Unlike R janitor's clean_names, camelCase is not split ("sampleID" becomes
+"sampleid", not "sample_id"): splitting would also turn "HbA1c" into
+"hb_a1c" and "pH" into "p_h".  "%" and "#" become "pct" and "n".
+
+The original names are kept in the parquet metadata (key original_names, a
+JSON object cleaned → original; see dftk annotate) and, with --mapping, in a
+TSV file.
+"""
+
+import argparse
+import json
+import logging
+import re
+import unicodedata
+
+import pandas as pd
+
+from dftk.commands.base import BaseCommand
+from dftk.common.io import io
+
+_EPILOG = """\
+RULES
+-----
+  Sample ID       ->  sample_id    lowercase; spaces and symbols become _
+   Age (years)    ->  age_years    trimmed; one _ per run, none at the ends
+  GC %            ->  gc_pct       % becomes pct
+  Count #         ->  count_n      # becomes n
+  1960            ->  x1960        a leading digit gets an x
+  Hæð (µg)        ->  haed_ug      letters become plain ASCII
+  sampleID        ->  sampleid     camelCase is not split
+  ---             ->  colN         nothing left: col + its position (1-based)
+  GC % , gc_pct   ->  gc_pct, gc_pct_2   clashes get _2, _3, ...
+
+Duplicate and empty headers are already renamed when the file is read
+(Count #.1, Unnamed: 7), so they come out as count_n_1 and unnamed_7.
+Names that are SQL reserved words (group, order, end, ...) are kept, with a
+warning: quote them in query --sql ("group").
+
+EXAMPLES
+--------
+  dftk cleannames data.tsv
+
+Convert a spreadsheet export to parquet with clean names, keeping integer
+columns with empty cells as integers:
+
+  dftk cleannames data.tsv --backend duckdb -o data.parquet
+
+The World Bank fertility table has headers such as "Country Name" and one
+column per year (1960, 1961, ...); after cleaning they can be used
+unquoted in SQL:
+
+  dftk dataset fertility -o \\
+      | dftk cleannames ... -o \\
+      | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
+
+Write the old and new names to a TSV file, e.g. for a data dictionary:
+
+  dftk dataset fertility -o \\
+      | dftk cleannames ... --mapping names.tsv -o fertility.parquet
+"""
+
+# Letters that Unicode does not decompose into ASCII letter + accent
+_TRANSLIT = str.maketrans(
+    {
+        "æ": "ae",
+        "ð": "d",
+        "đ": "d",
+        "þ": "th",
+        "ø": "o",
+        "œ": "oe",
+        "ß": "ss",
+        "ł": "l",
+        "ı": "i",
+        "µ": "u",  # micro sign, as in µg
+        "μ": "u",  # Greek mu
+    }
+)
+_SYMBOLS = {"%": " pct ", "#": " n "}
+
+
+def clean_name(name) -> str:
+    """Return *name* as a lowercase ASCII identifier ("" if nothing is left)."""
+    s = str(name).strip()
+    for symbol, word in _SYMBOLS.items():
+        s = s.replace(symbol, word)
+    s = s.lower().translate(_TRANSLIT)
+    s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode()
+    s = re.sub(r"[^a-z0-9]+", "_", s).strip("_")
+    if s[:1].isdigit():
+        s = "x" + s
+    return s
+
+
+def clean_names(names: list) -> list[str]:
+    """Clean every name; empty results become colN, clashes get _2, _3, …"""
+    cleaned = [clean_name(n) or f"col{i + 1}" for i, n in enumerate(names)]
+    taken = set(cleaned)  # a new name must not clash with any cleaned name
+    out: list[str] = []
+    for name in cleaned:
+        new, k = name, 1
+        while new in out or (new != name and new in taken):
+            k += 1
+            new = f"{name}_{k}"
+        out.append(new)
+    return out
+
+
+def _sql_keywords(names: list[str]) -> list[str]:
+    """Return the names that DuckDB SQL only accepts quoted."""
+    try:
+        import duckdb
+    except ImportError:
+        return []
+    rows = duckdb.sql(
+        "SELECT keyword_name FROM duckdb_keywords() "
+        "WHERE keyword_category IN ('reserved', 'type_function')"
+    ).fetchall()
+    keywords = {r[0] for r in rows}
+    return [n for n in names if n in keywords]
+
+
+class CleannamesCommand(BaseCommand):
+    @property
+    def name(self) -> str:
+        return "cleannames"
+
+    @property
+    def help(self) -> str:
+        return "Rename columns to lowercase identifiers (sample_id, gc_pct)"
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.epilog = _EPILOG
+
+        self.add_io_arguments(parser)
+
+        g = parser.add_argument_group("cleannames options")
+        g.add_argument(
+            "--mapping",
+            metavar="FILE",
+            default=None,
+            help="Also write the names to FILE as TSV (columns original, cleaned).",
+        )
+
+    def execute(self, args: argparse.Namespace) -> None:
+        df = io.read(args)
+        original = [str(c) for c in df.columns]
+        cleaned = clean_names(original)
+        df.columns = cleaned
+
+        # Carry names from an earlier cleannames run, so the first originals
+        # (e.g. spreadsheet headers) are kept when a file is cleaned again
+        meta = dict(df.attrs.get("_parquet_meta", {}))
+        earlier = json.loads(meta.get("original_names", "{}"))
+        renamed = {}
+        for old, new in zip(original, cleaned, strict=True):
+            first = earlier.get(old, old)
+            if first != new:
+                renamed[new] = first
+        if renamed or earlier:
+            meta["original_names"] = json.dumps(renamed, ensure_ascii=False)
+            df.attrs["_parquet_meta"] = meta
+
+        if args.mapping is not None:
+            pd.DataFrame({"original": original, "cleaned": cleaned}).to_csv(
+                args.mapping, sep="\t", index=False
+            )
+
+        keywords = _sql_keywords(cleaned)
+        if keywords:
+            quoted = ", ".join(f'"{k}"' for k in keywords)
+            logging.warning(
+                f"cleannames: {', '.join(keywords)} are SQL reserved words; "
+                f"quote them in query --sql ({quoted})"
+            )
+
+        io.printdf(df, args)

{dataframe_toolkit-0.7.3 → dataframe_toolkit-0.8.0}/src/dftk/common/io.py

@@ -167,6 +167,27 @@ def _warn_digits_exponent(df: pd.DataFrame, digits: int) -> None:
     )
 
 
+def _unique_column_names(names: list) -> list:
+    """Name empty and duplicate headers as pandas' default reader does.
+
+    The pyarrow reader keeps ``""`` and repeated names as they are, which
+    breaks commands that look columns up by name.  An empty header becomes
+    ``Unnamed: N`` (0-based position) and a repeat ``name.1``, ``name.2``, …
+    """
+    names = [f"Unnamed: {i}" if n == "" else n for i, n in enumerate(names)]
+    if len(set(names)) == len(names):
+        return names
+    taken = set(names)  # a new name must not clash with any header, later ones too
+    out: list = []
+    for n in names:
+        new, k = n, 0
+        while new in out or (new != n and new in taken):
+            k += 1
+            new = f"{n}.{k}"
+        out.append(new)
+    return out
+
+
 def _write_tsv(df: pd.DataFrame, dest, **to_csv_kwargs) -> None:
     """``df.to_csv(dest, ...)``, also for one column with empty cells.
 
@@ -316,9 +337,11 @@ class io:
 
         # -- helpers -------------------------------------------------------
 
-        def _rename_if_noheader(df: pd.DataFrame) -> pd.DataFrame:
+        def _name_columns(df: pd.DataFrame) -> pd.DataFrame:
             if noheader:
                 df.columns = [f"V{i + 1}" for i in range(len(df.columns))]
+            else:
+                df.columns = _unique_column_names(list(df.columns))
             return df
 
         def _pandas_csv(source, **extra) -> pd.DataFrame:
@@ -339,7 +362,7 @@ class io:
             df = _with_blank_lines(
                 df, lambda: _pandas_csv(_io.BytesIO(data), skip_blank_lines=False)
             )
-            return _rename_if_noheader(df)
+            return _name_columns(df)
 
         def _csv_with_duckdb(path: str) -> pd.DataFrame:
             # DuckDB keeps integer columns with blanks as Int64 (pandas: float64)
@@ -348,14 +371,14 @@ class io:
             except Exception as exc:
                 logging.error(f"DuckDB failed to read {path!r}: {exc}")
                 raise
-            return _rename_if_noheader(df)
+            return _name_columns(df)
 
         def _csv_from_file(path: str) -> pd.DataFrame:
             df = _pandas_csv(path)
             df = _with_blank_lines(
                 df, lambda: _pandas_csv(path, skip_blank_lines=False)
             )
-            return _rename_if_noheader(df)
+            return _name_columns(df)
 
         # -- read ----------------------------------------------------------