dataframe-toolkit 0.11.3__tar.gz → 0.13.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/PKG-INFO +66 -41
  2. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/README.md +65 -40
  3. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/pyproject.toml +1 -1
  4. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/pyproject.toml.orig +1 -1
  5. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/__init__.py +1 -0
  6. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/annotate_cmd.py +1 -1
  7. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/binx_cmd.py +4 -4
  8. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/chi_cmd.py +4 -4
  9. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/cleannames_cmd.py +3 -3
  10. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/concat_cmd.py +3 -3
  11. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/corr_cmd.py +6 -6
  12. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/describe_cmd.py +4 -4
  13. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/eval_cmd.py +15 -9
  14. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/explode_cmd.py +4 -4
  15. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/fit_cmd.py +6 -6
  16. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/func_cmd.py +5 -5
  17. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/hist_cmd.py +10 -10
  18. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/implode_cmd.py +6 -6
  19. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/info_cmd.py +2 -2
  20. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/interp_cmd.py +6 -6
  21. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/line_cmd.py +14 -14
  22. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/melt_cmd.py +1 -1
  23. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/merge_cmd.py +32 -35
  24. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/pivot_cmd.py +8 -8
  25. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/print_cmd.py +8 -6
  26. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/query_cmd.py +12 -12
  27. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/randvar_cmd.py +3 -3
  28. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/sample_cmd.py +9 -9
  29. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/scale_cmd.py +11 -11
  30. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/scat_cmd.py +12 -12
  31. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/segid_cmd.py +2 -2
  32. dataframe_toolkit-0.13.0/src/dftk/commands/skill_cmd.py +146 -0
  33. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/split_cmd.py +10 -7
  34. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/stat_cmd.py +4 -4
  35. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/test_cmd.py +3 -3
  36. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/transpose_cmd.py +2 -2
  37. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/wstat_cmd.py +4 -4
  38. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/io.py +85 -35
  39. dataframe_toolkit-0.13.0/src/dftk/skills/dftk/SKILL.md +197 -0
  40. dataframe_toolkit-0.13.0/src/dftk/skills/dftk/references/snakemake.md +167 -0
  41. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/__init__.py +0 -0
  42. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/cli.py +0 -0
  43. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/base.py +0 -0
  44. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/clean_cmd.py +0 -0
  45. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/dataset_cmd.py +0 -0
  46. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/commands/help.py +0 -0
  47. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/__init__.py +0 -0
  48. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/copula.py +0 -0
  49. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/evalfuncs.py +0 -0
  50. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/plot.py +0 -0
  51. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/seed.py +0 -0
  52. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/seq.py +0 -0
  53. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/common/stats.py +0 -0
  54. {dataframe_toolkit-0.11.3 → dataframe_toolkit-0.13.0}/src/dftk/io.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: dataframe-toolkit
3
- Version: 0.11.3
3
+ Version: 0.13.0
4
4
  Summary: A command-line toolkit for DataFrame analysis and manipulation
5
5
  Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
6
6
  Author: Gunnar Pálsson
@@ -77,17 +77,20 @@ uv tool install .
77
77
 
78
78
  ### Skill for coding agents
79
79
 
80
- `skills/dftk/` is an agent skill: a short guide that coding agents such as
81
- Codex and Claude Code load when a task involves dftk. It covers the piping
80
+ dftk ships an agent skill: a short guide that coding agents such as Codex
81
+ and Claude Code load when a task involves dftk. It covers the piping
82
82
  conventions, which command to use for what, and patterns for Snakemake and
83
- Make. Copy it into your agent's skills directory, for example:
83
+ Make. Install it with:
84
84
 
85
85
  ```bash
86
- cp -r skills/dftk ~/.codex/skills/ # Codex
87
- cp -r skills/dftk ~/.claude/skills/ # Claude Code
86
+ dftk skill install # every agent found: ~/.claude and/or ~/.codex
87
+ dftk skill install --codex # only one (--claude or --codex; creates the folder)
88
+ dftk skill install --dir DIR # another agent's skills folder
88
89
  ```
89
90
 
90
- Copy it again after upgrading dftk so it matches the installed version.
91
+ It prints where it installed the skill. Run it again after upgrading dftk
92
+ so the skill matches the installed version; it replaces only the earlier
93
+ dftk skill. The source is in `src/dftk/skills/dftk/`.
91
94
 
92
95
  ## Quick example
93
96
 
@@ -95,15 +98,15 @@ Copy it again after upgrading dftk so it matches the installed version.
95
98
  dftk dataset tips | dftk stat - -c tip total_bill -g day
96
99
  ```
97
100
 
98
- Chain commands with `-o` (write to a temp parquet, print its path) and `...`
99
- (read a parquet path from stdin) instead of piping TSV text directly — faster
100
- for large data, and the recommended pattern once a pipeline has more than one
101
- step:
101
+ Chain commands with a bare `-o` (write a temp parquet, print its path)
102
+ instead of piping TSV text: the next command sees the path on stdin and reads
103
+ that file. It's faster for large data, keeps column types exact, and is the
104
+ recommended pattern once a pipeline has more than one step:
102
105
 
103
106
  ```bash
104
107
  dftk dataset tips -o \
105
- | dftk pivot ... -i day -v tip -f mean -o \
106
- | dftk print ...
108
+ | dftk pivot -i day -v tip -f mean -o \
109
+ | dftk print
107
110
  ```
108
111
 
109
112
  ## Subcommands
@@ -163,7 +166,7 @@ publication-quality output.
163
166
  | `sample` | Random row sampling (with or without replacement, by count or fraction) |
164
167
  | `split` | Split a dataframe into one file per group |
165
168
  | `annotate` | Read and write provenance metadata (genome, source, …) in parquet files |
166
- | `print` | Read any dftk input (TSV, stdin, `...` parquet pipe) and write TSV |
169
+ | `print` | Read any dftk input (TSV, parquet, stdin) and write TSV |
167
170
  | `clean` | Remove leftover temp parquet pipe files from interrupted pipelines |
168
171
  | `help` | List all subcommands or show full help for one: `dftk help stat` |
169
172
 
@@ -174,8 +177,8 @@ publication-quality output.
174
177
  ```bash
175
178
  # z-score sepal_length within species, then fit against petal_length
176
179
  dftk dataset iris -o \
177
- | dftk scale ... -c sepal_length -g species -o \
178
- | dftk fit ... -f "sepal_length_scaled ~ petal_length" -g species
180
+ | dftk scale -c sepal_length -g species -o \
181
+ | dftk fit -f "sepal_length_scaled ~ petal_length" -g species
179
182
  ```
180
183
 
181
184
  ### Clean up spreadsheet headers
@@ -184,8 +187,8 @@ Headers such as `Country Name` or `1960` need quoting in SQL. `cleannames` turns
184
187
 
185
188
  ```bash
186
189
  dftk dataset fertility -o \
187
- | dftk cleannames ... -o \
188
- | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
190
+ | dftk cleannames -o \
191
+ | dftk query --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
189
192
  ```
190
193
 
191
194
  For your own export, `dftk cleannames data.tsv --backend duckdb -o data.parquet` also keeps integer columns with empty cells as integers.
@@ -196,7 +199,7 @@ With tens of thousands of points a scatter plot is a solid blob. `--hexbin` coun
196
199
 
197
200
  ```bash
198
201
  dftk dataset diamonds -o \
199
- | dftk scat ... -x carat -y price --hexbin --hexlog --logx --logy -f diamonds.png
202
+ | dftk scat -x carat -y price --hexbin --hexlog --logx --logy -f diamonds.png
200
203
  ```
201
204
 
202
205
  `--colorcol COL` colours each hexagon by the mean of `COL` instead of the count; with `--subgraphcol` all panels share one colour scale.
@@ -205,8 +208,8 @@ dftk dataset diamonds -o \
205
208
 
206
209
  ```bash
207
210
  dftk dataset tips -o \
208
- | dftk stat ... -c tip -g day -o \
209
- | dftk line ... -x day -y mean --yerr sem -f fig.png
211
+ | dftk stat -c tip -g day -o \
212
+ | dftk line -x day -y mean --yerr sem -f fig.png
210
213
  ```
211
214
 
212
215
  ### Wide-to-long then plot overlaid histograms
@@ -250,18 +253,18 @@ suffix `_r` from `merge`; fill from it, then drop it. Here the defaults are
250
253
  each species' median body mass:
251
254
 
252
255
  ```bash
253
- dftk dataset penguins -o | dftk pivot ... -i species -v body_mass_g \
256
+ dftk dataset penguins -o | dftk pivot -i species -v body_mass_g \
254
257
  -f median --move median_body_mass_g:body_mass_g -o defaults.parquet
255
258
  dftk dataset penguins -o \
256
- | dftk merge ... -r defaults.parquet -k species -t left -o \
257
- | dftk eval ... -f "body_mass_g = body_mass_g.fillna(body_mass_g_r)" \
259
+ | dftk merge -r defaults.parquet -k species -t left -o \
260
+ | dftk eval -f "body_mass_g = body_mass_g.fillna(body_mass_g_r)" \
258
261
  --drop body_mass_g_r
259
262
  ```
260
263
 
261
264
  **From neighbouring rows** (`co2` is a weekly series with gaps):
262
265
 
263
266
  ```bash
264
- dftk dataset co2 -o | dftk eval ... -f "ffill = co2.ffill()" \
267
+ dftk dataset co2 -o | dftk eval -f "ffill = co2.ffill()" \
265
268
  -f "bfill = co2.bfill()" -f "linear = co2.interpolate()"
266
269
  ```
267
270
 
@@ -279,9 +282,9 @@ to be next to each other, but within each group they must be in order:
279
282
 
280
283
  ```bash
281
284
  dftk dataset healthexp -o \
282
- | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
285
+ | dftk eval -f "le = Life_Expectancy.where(Year % 10 != 5)" \
283
286
  --sortasc Country Year -o \
284
- | dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
287
+ | dftk eval -f "le_filled = le.groupby(Country).ffill()"
285
288
  ```
286
289
 
287
290
  `interpolate()` has no grouped form. Once each group's rows are together,
@@ -291,9 +294,9 @@ with something before and after them in the same group:
291
294
 
292
295
  ```bash
293
296
  dftk dataset healthexp -o \
294
- | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
297
+ | dftk eval -f "le = Life_Expectancy.where(Year % 10 != 5)" \
295
298
  --sortasc Country Year -o \
296
- | dftk eval ... -f "before = le.groupby(Country).ffill().notna()" \
299
+ | dftk eval -f "before = le.groupby(Country).ffill().notna()" \
297
300
  -f "after = le.groupby(Country).bfill().notna()" \
298
301
  -f "le_linear = le.interpolate().where(before & after)" \
299
302
  --drop before after
@@ -306,8 +309,8 @@ pandas `diff()` and `pct_change()`, also within groups. They use the rows
306
309
  in their order, so sort first, in a step of its own:
307
310
 
308
311
  ```bash
309
- dftk dataset healthexp -o | dftk eval ... --sortasc Country Year -o \
310
- | dftk eval ... -f "change = Spending_USD.groupby(Country).diff().round(1)" \
312
+ dftk dataset healthexp -o | dftk eval --sortasc Country Year -o \
313
+ | dftk eval -f "change = Spending_USD.groupby(Country).diff().round(1)" \
311
314
  -f "pct = (Spending_USD.groupby(Country).pct_change() * 100).round(1)" \
312
315
  -f "years = Year.groupby(Country).diff()"
313
316
  ```
@@ -372,9 +375,9 @@ confidence interval:
372
375
 
373
376
  ```bash
374
377
  dftk dataset tips -o \
375
- | dftk stat ... -c tip -g day --bootstrap 1000 --randomseed 42 -o \
376
- | dftk pivot ... -i day -v mean -f mean cilo cihi -o \
377
- | dftk print ...
378
+ | dftk stat -c tip -g day --bootstrap 1000 --randomseed 42 -o \
379
+ | dftk pivot -i day -v mean -f mean cilo cihi -o \
380
+ | dftk print
378
381
  ```
379
382
 
380
383
  ## Input/output
@@ -382,15 +385,21 @@ dftk dataset tips -o \
382
385
  All commands accept:
383
386
 
384
387
  - A TSV filename as a positional argument
385
- - `-` to read TSV from stdin
386
- - `...` to receive a parquet path from stdin (written by a previous `-o` command)
387
388
  - A `.parquet` filename to read a named parquet file directly
389
+ - Nothing, or `-`, to read stdin: either TSV, or the parquet path printed by
390
+ a previous command's bare `-o` (that temp file is then read and deleted)
391
+ - A pipe as the filename, e.g. `<(grep -v '^#' data.tsv)` or `<(dftk … -o)`,
392
+ read the same way as stdin
388
393
 
389
- > **`-` vs `...`** — these are not interchangeable. Use `-` when the previous
390
- > command in the pipe wrote plain TSV to stdout (the default). Use `...` when
391
- > the previous command used bare `-o` (pipe mode), which writes a temp
392
- > parquet and prints *its path* to stdout — `-` would try to parse that path
393
- > as TSV and fail.
394
+ ```bash
395
+ dftk dataset tips -o | dftk stat -c tip -g day # parquet path on stdin
396
+ dftk dataset tips | dftk stat -c tip -g day # TSV on stdin
397
+ dftk merge -l tips.tsv -r <(dftk pivot tips.tsv -i day -v tip -f mean -o) -k day
398
+ ```
399
+
400
+ Older scripts give `...` as the file to read a parquet path from stdin; that
401
+ still works. In zsh with Oh My Zsh an unquoted `...` becomes `../..`, so
402
+ leave it out (or use `-`) in scripts you share.
394
403
 
395
404
  ### Comment lines and preambles
396
405
 
@@ -407,6 +416,20 @@ dftk print <(grep -v '^#' data.tsv) # the same, as a filename
407
416
  With `--backend duckdb`, a whole-line `#` comment below the header is an
408
417
  error, and `#` lines above the header are skipped.
409
418
 
419
+ ### Files without a header
420
+
421
+ `--noheader` reads the first line as data and names the columns `V1`,
422
+ `V2`, …. It only describes the input: the output gets a header (`V1 V2 …`)
423
+ unless you add `--removeheader`. Name the columns with `--move`:
424
+
425
+ ```bash
426
+ dftk print data.txt --noheader --move V1:chr V2:begin V3:end -o named.tsv
427
+ dftk eval data.txt --noheader -f "len = V3 - V2" --removeheader # no header out
428
+ ```
429
+
430
+ Parquet always has column names, so `--removeheader` with `-o` and
431
+ `--noheader` on parquet input do nothing (dftk warns).
432
+
410
433
  Standard output options (available on all tabular commands):
411
434
 
412
435
  - `-o` / `--output` — controls where output goes; takes an *optional* value:
@@ -418,6 +441,8 @@ Standard output options (available on all tabular commands):
418
441
  - `--drop col1 col2 …` — remove these columns
419
442
  - `--round N` — round numeric output
420
443
  - `--postquery EXPR` — filter output rows after processing
444
+ - `--move OLD:NEW …` — rename columns
445
+ - `--removeheader` — leave out the header row in TSV output
421
446
  - `--meta KEY=VALUE` — embed provenance metadata in parquet output (repeatable)
422
447
 
423
448
  ### Provenance annotations
@@ -47,17 +47,20 @@ uv tool install .
47
47
 
48
48
  ### Skill for coding agents
49
49
 
50
- `skills/dftk/` is an agent skill: a short guide that coding agents such as
51
- Codex and Claude Code load when a task involves dftk. It covers the piping
50
+ dftk ships an agent skill: a short guide that coding agents such as Codex
51
+ and Claude Code load when a task involves dftk. It covers the piping
52
52
  conventions, which command to use for what, and patterns for Snakemake and
53
- Make. Copy it into your agent's skills directory, for example:
53
+ Make. Install it with:
54
54
 
55
55
  ```bash
56
- cp -r skills/dftk ~/.codex/skills/ # Codex
57
- cp -r skills/dftk ~/.claude/skills/ # Claude Code
56
+ dftk skill install # every agent found: ~/.claude and/or ~/.codex
57
+ dftk skill install --codex # only one (--claude or --codex; creates the folder)
58
+ dftk skill install --dir DIR # another agent's skills folder
58
59
  ```
59
60
 
60
- Copy it again after upgrading dftk so it matches the installed version.
61
+ It prints where it installed the skill. Run it again after upgrading dftk
62
+ so the skill matches the installed version; it replaces only the earlier
63
+ dftk skill. The source is in `src/dftk/skills/dftk/`.
61
64
 
62
65
  ## Quick example
63
66
 
@@ -65,15 +68,15 @@ Copy it again after upgrading dftk so it matches the installed version.
65
68
  dftk dataset tips | dftk stat - -c tip total_bill -g day
66
69
  ```
67
70
 
68
- Chain commands with `-o` (write to a temp parquet, print its path) and `...`
69
- (read a parquet path from stdin) instead of piping TSV text directly — faster
70
- for large data, and the recommended pattern once a pipeline has more than one
71
- step:
71
+ Chain commands with a bare `-o` (write a temp parquet, print its path)
72
+ instead of piping TSV text: the next command sees the path on stdin and reads
73
+ that file. It's faster for large data, keeps column types exact, and is the
74
+ recommended pattern once a pipeline has more than one step:
72
75
 
73
76
  ```bash
74
77
  dftk dataset tips -o \
75
- | dftk pivot ... -i day -v tip -f mean -o \
76
- | dftk print ...
78
+ | dftk pivot -i day -v tip -f mean -o \
79
+ | dftk print
77
80
  ```
78
81
 
79
82
  ## Subcommands
@@ -133,7 +136,7 @@ publication-quality output.
133
136
  | `sample` | Random row sampling (with or without replacement, by count or fraction) |
134
137
  | `split` | Split a dataframe into one file per group |
135
138
  | `annotate` | Read and write provenance metadata (genome, source, …) in parquet files |
136
- | `print` | Read any dftk input (TSV, stdin, `...` parquet pipe) and write TSV |
139
+ | `print` | Read any dftk input (TSV, parquet, stdin) and write TSV |
137
140
  | `clean` | Remove leftover temp parquet pipe files from interrupted pipelines |
138
141
  | `help` | List all subcommands or show full help for one: `dftk help stat` |
139
142
 
@@ -144,8 +147,8 @@ publication-quality output.
144
147
  ```bash
145
148
  # z-score sepal_length within species, then fit against petal_length
146
149
  dftk dataset iris -o \
147
- | dftk scale ... -c sepal_length -g species -o \
148
- | dftk fit ... -f "sepal_length_scaled ~ petal_length" -g species
150
+ | dftk scale -c sepal_length -g species -o \
151
+ | dftk fit -f "sepal_length_scaled ~ petal_length" -g species
149
152
  ```
150
153
 
151
154
  ### Clean up spreadsheet headers
@@ -154,8 +157,8 @@ Headers such as `Country Name` or `1960` need quoting in SQL. `cleannames` turns
154
157
 
155
158
  ```bash
156
159
  dftk dataset fertility -o \
157
- | dftk cleannames ... -o \
158
- | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
160
+ | dftk cleannames -o \
161
+ | dftk query --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
159
162
  ```
160
163
 
161
164
  For your own export, `dftk cleannames data.tsv --backend duckdb -o data.parquet` also keeps integer columns with empty cells as integers.
@@ -166,7 +169,7 @@ With tens of thousands of points a scatter plot is a solid blob. `--hexbin` coun
166
169
 
167
170
  ```bash
168
171
  dftk dataset diamonds -o \
169
- | dftk scat ... -x carat -y price --hexbin --hexlog --logx --logy -f diamonds.png
172
+ | dftk scat -x carat -y price --hexbin --hexlog --logx --logy -f diamonds.png
170
173
  ```
171
174
 
172
175
  `--colorcol COL` colours each hexagon by the mean of `COL` instead of the count; with `--subgraphcol` all panels share one colour scale.
@@ -175,8 +178,8 @@ dftk dataset diamonds -o \
175
178
 
176
179
  ```bash
177
180
  dftk dataset tips -o \
178
- | dftk stat ... -c tip -g day -o \
179
- | dftk line ... -x day -y mean --yerr sem -f fig.png
181
+ | dftk stat -c tip -g day -o \
182
+ | dftk line -x day -y mean --yerr sem -f fig.png
180
183
  ```
181
184
 
182
185
  ### Wide-to-long then plot overlaid histograms
@@ -220,18 +223,18 @@ suffix `_r` from `merge`; fill from it, then drop it. Here the defaults are
220
223
  each species' median body mass:
221
224
 
222
225
  ```bash
223
- dftk dataset penguins -o | dftk pivot ... -i species -v body_mass_g \
226
+ dftk dataset penguins -o | dftk pivot -i species -v body_mass_g \
224
227
  -f median --move median_body_mass_g:body_mass_g -o defaults.parquet
225
228
  dftk dataset penguins -o \
226
- | dftk merge ... -r defaults.parquet -k species -t left -o \
227
- | dftk eval ... -f "body_mass_g = body_mass_g.fillna(body_mass_g_r)" \
229
+ | dftk merge -r defaults.parquet -k species -t left -o \
230
+ | dftk eval -f "body_mass_g = body_mass_g.fillna(body_mass_g_r)" \
228
231
  --drop body_mass_g_r
229
232
  ```
230
233
 
231
234
  **From neighbouring rows** (`co2` is a weekly series with gaps):
232
235
 
233
236
  ```bash
234
- dftk dataset co2 -o | dftk eval ... -f "ffill = co2.ffill()" \
237
+ dftk dataset co2 -o | dftk eval -f "ffill = co2.ffill()" \
235
238
  -f "bfill = co2.bfill()" -f "linear = co2.interpolate()"
236
239
  ```
237
240
 
@@ -249,9 +252,9 @@ to be next to each other, but within each group they must be in order:
249
252
 
250
253
  ```bash
251
254
  dftk dataset healthexp -o \
252
- | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
255
+ | dftk eval -f "le = Life_Expectancy.where(Year % 10 != 5)" \
253
256
  --sortasc Country Year -o \
254
- | dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
257
+ | dftk eval -f "le_filled = le.groupby(Country).ffill()"
255
258
  ```
256
259
 
257
260
  `interpolate()` has no grouped form. Once each group's rows are together,
@@ -261,9 +264,9 @@ with something before and after them in the same group:
261
264
 
262
265
  ```bash
263
266
  dftk dataset healthexp -o \
264
- | dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
267
+ | dftk eval -f "le = Life_Expectancy.where(Year % 10 != 5)" \
265
268
  --sortasc Country Year -o \
266
- | dftk eval ... -f "before = le.groupby(Country).ffill().notna()" \
269
+ | dftk eval -f "before = le.groupby(Country).ffill().notna()" \
267
270
  -f "after = le.groupby(Country).bfill().notna()" \
268
271
  -f "le_linear = le.interpolate().where(before & after)" \
269
272
  --drop before after
@@ -276,8 +279,8 @@ pandas `diff()` and `pct_change()`, also within groups. They use the rows
276
279
  in their order, so sort first, in a step of its own:
277
280
 
278
281
  ```bash
279
- dftk dataset healthexp -o | dftk eval ... --sortasc Country Year -o \
280
- | dftk eval ... -f "change = Spending_USD.groupby(Country).diff().round(1)" \
282
+ dftk dataset healthexp -o | dftk eval --sortasc Country Year -o \
283
+ | dftk eval -f "change = Spending_USD.groupby(Country).diff().round(1)" \
281
284
  -f "pct = (Spending_USD.groupby(Country).pct_change() * 100).round(1)" \
282
285
  -f "years = Year.groupby(Country).diff()"
283
286
  ```
@@ -342,9 +345,9 @@ confidence interval:
342
345
 
343
346
  ```bash
344
347
  dftk dataset tips -o \
345
- | dftk stat ... -c tip -g day --bootstrap 1000 --randomseed 42 -o \
346
- | dftk pivot ... -i day -v mean -f mean cilo cihi -o \
347
- | dftk print ...
348
+ | dftk stat -c tip -g day --bootstrap 1000 --randomseed 42 -o \
349
+ | dftk pivot -i day -v mean -f mean cilo cihi -o \
350
+ | dftk print
348
351
  ```
349
352
 
350
353
  ## Input/output
@@ -352,15 +355,21 @@ dftk dataset tips -o \
352
355
  All commands accept:
353
356
 
354
357
  - A TSV filename as a positional argument
355
- - `-` to read TSV from stdin
356
- - `...` to receive a parquet path from stdin (written by a previous `-o` command)
357
358
  - A `.parquet` filename to read a named parquet file directly
359
+ - Nothing, or `-`, to read stdin: either TSV, or the parquet path printed by
360
+ a previous command's bare `-o` (that temp file is then read and deleted)
361
+ - A pipe as the filename, e.g. `<(grep -v '^#' data.tsv)` or `<(dftk … -o)`,
362
+ read the same way as stdin
358
363
 
359
- > **`-` vs `...`** — these are not interchangeable. Use `-` when the previous
360
- > command in the pipe wrote plain TSV to stdout (the default). Use `...` when
361
- > the previous command used bare `-o` (pipe mode), which writes a temp
362
- > parquet and prints *its path* to stdout — `-` would try to parse that path
363
- > as TSV and fail.
364
+ ```bash
365
+ dftk dataset tips -o | dftk stat -c tip -g day # parquet path on stdin
366
+ dftk dataset tips | dftk stat -c tip -g day # TSV on stdin
367
+ dftk merge -l tips.tsv -r <(dftk pivot tips.tsv -i day -v tip -f mean -o) -k day
368
+ ```
369
+
370
+ Older scripts give `...` as the file to read a parquet path from stdin; that
371
+ still works. In zsh with Oh My Zsh an unquoted `...` becomes `../..`, so
372
+ leave it out (or use `-`) in scripts you share.
364
373
 
365
374
  ### Comment lines and preambles
366
375
 
@@ -377,6 +386,20 @@ dftk print <(grep -v '^#' data.tsv) # the same, as a filename
377
386
  With `--backend duckdb`, a whole-line `#` comment below the header is an
378
387
  error, and `#` lines above the header are skipped.
379
388
 
389
+ ### Files without a header
390
+
391
+ `--noheader` reads the first line as data and names the columns `V1`,
392
+ `V2`, …. It only describes the input: the output gets a header (`V1 V2 …`)
393
+ unless you add `--removeheader`. Name the columns with `--move`:
394
+
395
+ ```bash
396
+ dftk print data.txt --noheader --move V1:chr V2:begin V3:end -o named.tsv
397
+ dftk eval data.txt --noheader -f "len = V3 - V2" --removeheader # no header out
398
+ ```
399
+
400
+ Parquet always has column names, so `--removeheader` with `-o` and
401
+ `--noheader` on parquet input do nothing (dftk warns).
402
+
380
403
  Standard output options (available on all tabular commands):
381
404
 
382
405
  - `-o` / `--output` — controls where output goes; takes an *optional* value:
@@ -388,6 +411,8 @@ Standard output options (available on all tabular commands):
388
411
  - `--drop col1 col2 …` — remove these columns
389
412
  - `--round N` — round numeric output
390
413
  - `--postquery EXPR` — filter output rows after processing
414
+ - `--move OLD:NEW …` — rename columns
415
+ - `--removeheader` — leave out the header row in TSV output
391
416
  - `--meta KEY=VALUE` — embed provenance metadata in parquet output (repeatable)
392
417
 
393
418
  ### Provenance annotations
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dataframe-toolkit"
3
- version = "0.11.3"
3
+ version = "0.13.0"
4
4
  description = "A command-line toolkit for DataFrame analysis and manipulation"
5
5
  readme = "README.md"
6
6
  keywords = [
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dataframe-toolkit"
3
- version = "0.11.3"
3
+ version = "0.13.0"
4
4
  description = "A command-line toolkit for DataFrame analysis and manipulation"
5
5
  readme = "README.md"
6
6
  authors = [
@@ -73,6 +73,7 @@ COMMAND_GROUPS: list[tuple[str, list[tuple[str, str]]]] = [
73
73
  ("annotate", "annotate_cmd:AnnotateCommand"),
74
74
  ("print", "print_cmd:PrintCommand"),
75
75
  ("clean", "clean_cmd:CleanCommand"),
76
+ ("skill", "skill_cmd:SkillCommand"),
76
77
  ],
77
78
  ),
78
79
  ]
@@ -26,7 +26,7 @@ Usage examples:
26
26
  --meta genome=hg38 --meta source=my_pipeline
27
27
 
28
28
  # Same, built from a bundled dataset:
29
- dftk dataset iris -o | dftk eval ... -f "z = sepal_length + petal_length" \\
29
+ dftk dataset iris -o | dftk eval -f "z = sepal_length + petal_length" \\
30
30
  -o result.parquet --meta genome=hg38 --meta source=my_pipeline
31
31
 
32
32
  Annotations propagate through the parquet pipe: a file annotated at
@@ -74,10 +74,10 @@ USEVALUE (--usevalue)
74
74
  EXAMPLES
75
75
  --------
76
76
  dftk binx data.tsv -c age -b 0:100:10
77
- dftk dataset tips -o | dftk binx ... -c total_bill -b 0:60:10
78
- dftk dataset tips -o | dftk binx ... -c tip -b 0,2,4,6,10 -d tip_bracket --usevalue m
79
- dftk dataset tips -o | dftk binx ... -c total_bill -b 0:60:10 -d bill_bin -o \\
80
- | dftk stat ... -c tip -g bill_bin
77
+ dftk dataset tips -o | dftk binx -c total_bill -b 0:60:10
78
+ dftk dataset tips -o | dftk binx -c tip -b 0,2,4,6,10 -d tip_bracket --usevalue m
79
+ dftk dataset tips -o | dftk binx -c total_bill -b 0:60:10 -d bill_bin -o \\
80
+ | dftk stat -c tip -g bill_bin
81
81
  """
82
82
 
83
83
 
@@ -116,21 +116,21 @@ EXAMPLES
116
116
  Basic chi-plot:
117
117
 
118
118
  dftk chi data.tsv -x recomb_rate -y gc_content
119
- dftk dataset tips -o | dftk chi ... -x total_bill -y tip
119
+ dftk dataset tips -o | dftk chi -x total_bill -y tip
120
120
 
121
121
  Split into a subplot grid by category, to check whether tail dependence is
122
122
  consistent across the whole dataset or driven by a few subgroups (e.g.
123
123
  chromosome, sector, region — here, diamond cut quality):
124
124
 
125
- dftk dataset diamonds -o | dftk chi ... -x carat -y price --subgraphcol cut
125
+ dftk dataset diamonds -o | dftk chi -x carat -y price --subgraphcol cut
126
126
 
127
127
  Colour-code by group instead of splitting into subplots:
128
128
 
129
- dftk dataset diamonds -o | dftk chi ... -x carat -y price -g cut
129
+ dftk dataset diamonds -o | dftk chi -x carat -y price -g cut
130
130
 
131
131
  Publication figure (Nature single column, PDF):
132
132
 
133
- dftk dataset tips -o | dftk chi ... -x total_bill -y tip \\
133
+ dftk dataset tips -o | dftk chi -x total_bill -y tip \\
134
134
  --size single --fontsize publication -f fig.pdf
135
135
  """
136
136
 
@@ -57,13 +57,13 @@ column per year (1960, 1961, ...); after cleaning they can be used
57
57
  unquoted in SQL:
58
58
 
59
59
  dftk dataset fertility -o \\
60
- | dftk cleannames ... -o \\
61
- | dftk query ... --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
60
+ | dftk cleannames -o \\
61
+ | dftk query --sql "SELECT country_name, x1960, x2000 FROM data LIMIT 3"
62
62
 
63
63
  Write the old and new names to a TSV file, e.g. for a data dictionary:
64
64
 
65
65
  dftk dataset fertility -o \\
66
- | dftk cleannames ... --mapping names.tsv -o fertility.parquet
66
+ | dftk cleannames --mapping names.tsv -o fertility.parquet
67
67
  """
68
68
 
69
69
  # Letters that Unicode does not decompose into ASCII letter + accent
@@ -12,7 +12,7 @@ Examples
12
12
  dftk concat penguins.tsv penguins.tsv --sourcecol source
13
13
  dftk dataset penguins -o penguins.parquet
14
14
  dftk concat penguins.parquet penguins.parquet -o
15
- dftk dataset penguins -o | dftk eval ... -f "x=1" -o | dftk concat ... penguins.tsv
15
+ dftk dataset penguins -o | dftk eval -f "x=1" -o | dftk concat - penguins.tsv
16
16
  """
17
17
 
18
18
  import argparse
@@ -51,7 +51,7 @@ EXAMPLES
51
51
  dftk concat penguins.tsv penguins.tsv --sourcecol source
52
52
  dftk dataset penguins -o penguins.parquet
53
53
  dftk concat penguins.parquet penguins.parquet -o result.parquet
54
- dftk dataset penguins -o | dftk eval ... -f "x=1" -o | dftk concat ... penguins.tsv
54
+ dftk dataset penguins -o | dftk eval -f "x=1" -o | dftk concat - penguins.tsv
55
55
 
56
56
  Mismatched columns are outer-joined and filled (NaN by default, or --fill):
57
57
 
@@ -65,7 +65,7 @@ Mismatched columns are outer-joined and filled (NaN by default, or --fill):
65
65
  metavar="FILE",
66
66
  help=(
67
67
  "Input files to concatenate (TSV or .parquet). "
68
- "Use - for TSV stdin or ... for a parquet pipe path on stdin. "
68
+ "Use - for stdin (TSV, or a parquet path from -o). "
69
69
  "At most one stdin source is allowed."
70
70
  ),
71
71
  )
@@ -195,7 +195,7 @@ COLUMN PAIRS (-c)
195
195
  Pairs are specified as col1:col2. Multiple pairs may be given:
196
196
 
197
197
  dftk corr data.tsv -c x:y
198
- dftk dataset tips -o | dftk corr ... -c total_bill:tip tip:size
198
+ dftk dataset tips -o | dftk corr -c total_bill:tip tip:size
199
199
 
200
200
  METHODS (--method)
201
201
  ------------------
@@ -261,13 +261,13 @@ TAIL DEPENDENCE (--tail / --tail-q)
261
261
  EXAMPLES
262
262
  --------
263
263
  dftk corr data.tsv -c x:y
264
- dftk dataset tips -o | dftk corr ... -c total_bill:tip
265
- dftk dataset tips -o | dftk corr ... -c total_bill:tip -g day --method spearman
266
- dftk dataset tips -o | dftk corr ... -c total_bill:tip --ci
264
+ dftk dataset tips -o | dftk corr -c total_bill:tip
265
+ dftk dataset tips -o | dftk corr -c total_bill:tip -g day --method spearman
266
+ dftk dataset tips -o | dftk corr -c total_bill:tip --ci
267
267
  dftk dataset tips -o \\
268
- | dftk corr ... -c total_bill:tip --method spearman \\
268
+ | dftk corr -c total_bill:tip --method spearman \\
269
269
  --bootstrap 2000 --randomseed 42
270
- dftk dataset diamonds -o | dftk corr ... -c carat:price --tail --tail-q 0.02 0.05 0.1
270
+ dftk dataset diamonds -o | dftk corr -c carat:price --tail --tail-q 0.02 0.05 0.1
271
271
  """
272
272
 
273
273
 
@@ -37,7 +37,7 @@ Optional flags
37
37
  Example
38
38
  -------
39
39
  dftk describe data.tsv
40
- dftk dataset iris -o | dftk describe ... --summary --correlations
40
+ dftk dataset iris -o | dftk describe --summary --correlations
41
41
  """
42
42
 
43
43
  import argparse
@@ -314,9 +314,9 @@ NOTES FLAGS
314
314
  EXAMPLES
315
315
  --------
316
316
  dftk describe data.tsv
317
- dftk dataset iris -o | dftk describe ...
318
- dftk dataset iris -o | dftk describe ... --summary --correlations
319
- dftk dataset iris -o | dftk describe ... --summary --correlations --corr-threshold 0.5
317
+ dftk dataset iris -o | dftk describe
318
+ dftk dataset iris -o | dftk describe --summary --correlations
319
+ dftk dataset iris -o | dftk describe --summary --correlations --corr-threshold 0.5
320
320
  """
321
321
  io.parser_read(parser)
322
322