dataframe-toolkit 0.11.1__tar.gz → 0.11.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/PKG-INFO +61 -19
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/README.md +60 -18
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/pyproject.toml +1 -1
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/pyproject.toml.orig +1 -1
- dataframe_toolkit-0.11.2/src/dftk/commands/eval_cmd.py +391 -0
- dataframe_toolkit-0.11.2/src/dftk/common/evalfuncs.py +439 -0
- dataframe_toolkit-0.11.1/src/dftk/commands/eval_cmd.py +0 -831
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/__init__.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/cli.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/__init__.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/annotate_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/base.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/binx_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/chi_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/clean_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/cleannames_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/concat_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/corr_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/dataset_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/describe_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/explode_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/fit_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/func_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/help.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/hist_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/implode_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/info_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/interp_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/line_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/melt_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/merge_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/pivot_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/print_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/query_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/randvar_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/sample_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/scale_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/scat_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/segid_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/split_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/stat_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/test_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/transpose_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/commands/wstat_cmd.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/common/__init__.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/common/copula.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/common/io.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/common/plot.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/common/seed.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/common/seq.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/common/stats.py +0 -0
- {dataframe_toolkit-0.11.1 → dataframe_toolkit-0.11.2}/src/dftk/io.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: dataframe-toolkit
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.2
|
|
4
4
|
Summary: A command-line toolkit for DataFrame analysis and manipulation
|
|
5
5
|
Keywords: dataframe,statistics,data analysis,CLI,TSV,pandas
|
|
6
6
|
Author: Gunnar Pálsson
|
|
@@ -226,35 +226,77 @@ printf "sample\tfluor\ns1\t60\ns2\t110\n" \
|
|
|
226
226
|
|
|
227
227
|
### Filling missing values
|
|
228
228
|
|
|
229
|
-
`eval -f` takes pandas column methods (`COL.method(...)`)
|
|
230
|
-
`ffill`, `bfill` and `interpolate` fill empty cells
|
|
231
|
-
|
|
229
|
+
`eval -f` takes pandas column methods (`COL.method(...)`); `fillna`,
|
|
230
|
+
`ffill`, `bfill` and `interpolate` fill empty cells and keep the others.
|
|
231
|
+
|
|
232
|
+
| Formula | Fills with |
|
|
233
|
+
|---|---|
|
|
234
|
+
| `x = x.fillna(y)` | the value in column `y` |
|
|
235
|
+
| `x = x.fillna(0)`, `sex = sex.fillna('unknown')` | a fixed value |
|
|
236
|
+
| `x = x.fillna(y).fillna(0)` | `y` first, then 0 |
|
|
237
|
+
| `x = x.fillna(x.groupby(g).transform('mean'))` | the mean of the row's group (or `'median'`, ...) |
|
|
238
|
+
| `x = x.ffill()` / `x = x.bfill()` | the value above / below (`ffill(limit=1)`: at most one row) |
|
|
239
|
+
| `x = x.groupby(g).ffill()` | the value above, within each group of `g` |
|
|
240
|
+
| `x = x.interpolate()` | a straight line between the values above and below, by row position |
|
|
241
|
+
|
|
242
|
+
Write to a new column (`x_filled = x.fillna(y)`) to keep the original, and
|
|
243
|
+
mark the gaps first (`missing = x.isna()`) to tell filled values apart.
|
|
244
|
+
Only empty cells are filled: a missing row (a year with no row at all) is
|
|
245
|
+
not added.
|
|
246
|
+
|
|
247
|
+
**Defaults from another table.** A column that is in both files gets the
|
|
248
|
+
suffix `_r` from `merge`; fill from it, then drop it. Here the defaults are
|
|
249
|
+
each species' median body mass:
|
|
232
250
|
|
|
233
251
|
```bash
|
|
234
|
-
dftk
|
|
235
|
-
|
|
252
|
+
dftk dataset penguins -o | dftk pivot ... -i species -v body_mass_g \
|
|
253
|
+
-f median --move median_body_mass_g:body_mass_g -o defaults.parquet
|
|
254
|
+
dftk dataset penguins -o \
|
|
255
|
+
| dftk merge ... -r defaults.parquet -k species -t left -o \
|
|
256
|
+
| dftk eval ... -f "body_mass_g = body_mass_g.fillna(body_mass_g_r)" \
|
|
257
|
+
--drop body_mass_g_r
|
|
236
258
|
```
|
|
237
259
|
|
|
238
|
-
|
|
239
|
-
each group:
|
|
260
|
+
**From neighbouring rows** (`co2` is a weekly series with gaps):
|
|
240
261
|
|
|
241
262
|
```bash
|
|
242
|
-
dftk dataset
|
|
243
|
-
-f "
|
|
244
|
-
|
|
245
|
-
|
|
263
|
+
dftk dataset co2 -o | dftk eval ... -f "ffill = co2.ffill()" \
|
|
264
|
+
-f "bfill = co2.bfill()" -f "linear = co2.interpolate()"
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
`ffill` leaves the first rows empty and `bfill` the last ones;
|
|
268
|
+
`x.ffill().fillna(x.bfill())` fills both. `interpolate()` counts one step
|
|
269
|
+
per row, so the rows must be evenly spaced; for an x-based lookup from a
|
|
270
|
+
reference curve, see `dftk interp`.
|
|
271
|
+
|
|
272
|
+
**Row order and groups.** These methods use the rows in the order they are
|
|
273
|
+
in. Sort first, in a step of its own: `--sortasc` applies to a command's
|
|
274
|
+
output, so in the same command it would come after the fill. When a table
|
|
275
|
+
holds several series (sites, countries, samples), fill within each group,
|
|
276
|
+
or values are taken from the neighbouring group. Rows of a group don't need
|
|
277
|
+
to be next to each other, but within each group they must be in order:
|
|
246
278
|
|
|
279
|
+
```bash
|
|
247
280
|
dftk dataset healthexp -o \
|
|
248
|
-
| dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)"
|
|
281
|
+
| dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
|
|
282
|
+
--sortasc Country Year -o \
|
|
249
283
|
| dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
|
|
250
284
|
```
|
|
251
285
|
|
|
252
|
-
`
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
286
|
+
`interpolate()` has no grouped form. Once each group's rows are together,
|
|
287
|
+
gaps inside a group are right, but a gap at a group's start or end is
|
|
288
|
+
filled from the neighbouring group. To leave those empty, keep only values
|
|
289
|
+
with something before and after them in the same group:
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
dftk dataset healthexp -o \
|
|
293
|
+
| dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
|
|
294
|
+
--sortasc Country Year -o \
|
|
295
|
+
| dftk eval ... -f "before = le.groupby(Country).ffill().notna()" \
|
|
296
|
+
-f "after = le.groupby(Country).bfill().notna()" \
|
|
297
|
+
-f "le_linear = le.interpolate().where(before & after)" \
|
|
298
|
+
--drop before after
|
|
299
|
+
```
|
|
258
300
|
|
|
259
301
|
### Bootstrap confidence intervals
|
|
260
302
|
|
|
@@ -197,35 +197,77 @@ printf "sample\tfluor\ns1\t60\ns2\t110\n" \
|
|
|
197
197
|
|
|
198
198
|
### Filling missing values
|
|
199
199
|
|
|
200
|
-
`eval -f` takes pandas column methods (`COL.method(...)`)
|
|
201
|
-
`ffill`, `bfill` and `interpolate` fill empty cells
|
|
202
|
-
|
|
200
|
+
`eval -f` takes pandas column methods (`COL.method(...)`); `fillna`,
|
|
201
|
+
`ffill`, `bfill` and `interpolate` fill empty cells and keep the others.
|
|
202
|
+
|
|
203
|
+
| Formula | Fills with |
|
|
204
|
+
|---|---|
|
|
205
|
+
| `x = x.fillna(y)` | the value in column `y` |
|
|
206
|
+
| `x = x.fillna(0)`, `sex = sex.fillna('unknown')` | a fixed value |
|
|
207
|
+
| `x = x.fillna(y).fillna(0)` | `y` first, then 0 |
|
|
208
|
+
| `x = x.fillna(x.groupby(g).transform('mean'))` | the mean of the row's group (or `'median'`, ...) |
|
|
209
|
+
| `x = x.ffill()` / `x = x.bfill()` | the value above / below (`ffill(limit=1)`: at most one row) |
|
|
210
|
+
| `x = x.groupby(g).ffill()` | the value above, within each group of `g` |
|
|
211
|
+
| `x = x.interpolate()` | a straight line between the values above and below, by row position |
|
|
212
|
+
|
|
213
|
+
Write to a new column (`x_filled = x.fillna(y)`) to keep the original, and
|
|
214
|
+
mark the gaps first (`missing = x.isna()`) to tell filled values apart.
|
|
215
|
+
Only empty cells are filled: a missing row (a year with no row at all) is
|
|
216
|
+
not added.
|
|
217
|
+
|
|
218
|
+
**Defaults from another table.** A column that is in both files gets the
|
|
219
|
+
suffix `_r` from `merge`; fill from it, then drop it. Here the defaults are
|
|
220
|
+
each species' median body mass:
|
|
203
221
|
|
|
204
222
|
```bash
|
|
205
|
-
dftk
|
|
206
|
-
|
|
223
|
+
dftk dataset penguins -o | dftk pivot ... -i species -v body_mass_g \
|
|
224
|
+
-f median --move median_body_mass_g:body_mass_g -o defaults.parquet
|
|
225
|
+
dftk dataset penguins -o \
|
|
226
|
+
| dftk merge ... -r defaults.parquet -k species -t left -o \
|
|
227
|
+
| dftk eval ... -f "body_mass_g = body_mass_g.fillna(body_mass_g_r)" \
|
|
228
|
+
--drop body_mass_g_r
|
|
207
229
|
```
|
|
208
230
|
|
|
209
|
-
|
|
210
|
-
each group:
|
|
231
|
+
**From neighbouring rows** (`co2` is a weekly series with gaps):
|
|
211
232
|
|
|
212
233
|
```bash
|
|
213
|
-
dftk dataset
|
|
214
|
-
-f "
|
|
215
|
-
|
|
216
|
-
|
|
234
|
+
dftk dataset co2 -o | dftk eval ... -f "ffill = co2.ffill()" \
|
|
235
|
+
-f "bfill = co2.bfill()" -f "linear = co2.interpolate()"
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
`ffill` leaves the first rows empty and `bfill` the last ones;
|
|
239
|
+
`x.ffill().fillna(x.bfill())` fills both. `interpolate()` counts one step
|
|
240
|
+
per row, so the rows must be evenly spaced; for an x-based lookup from a
|
|
241
|
+
reference curve, see `dftk interp`.
|
|
217
242
|
|
|
243
|
+
**Row order and groups.** These methods use the rows in the order they are
|
|
244
|
+
in. Sort first, in a step of its own: `--sortasc` applies to a command's
|
|
245
|
+
output, so in the same command it would come after the fill. When a table
|
|
246
|
+
holds several series (sites, countries, samples), fill within each group,
|
|
247
|
+
or values are taken from the neighbouring group. Rows of a group don't need
|
|
248
|
+
to be next to each other, but within each group they must be in order:
|
|
249
|
+
|
|
250
|
+
```bash
|
|
218
251
|
dftk dataset healthexp -o \
|
|
219
|
-
| dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)"
|
|
252
|
+
| dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
|
|
253
|
+
--sortasc Country Year -o \
|
|
220
254
|
| dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
|
|
221
255
|
```
|
|
222
256
|
|
|
223
|
-
`
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
257
|
+
`interpolate()` has no grouped form. Once each group's rows are together,
|
|
258
|
+
gaps inside a group are right, but a gap at a group's start or end is
|
|
259
|
+
filled from the neighbouring group. To leave those empty, keep only values
|
|
260
|
+
with something before and after them in the same group:
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
dftk dataset healthexp -o \
|
|
264
|
+
| dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \
|
|
265
|
+
--sortasc Country Year -o \
|
|
266
|
+
| dftk eval ... -f "before = le.groupby(Country).ffill().notna()" \
|
|
267
|
+
-f "after = le.groupby(Country).bfill().notna()" \
|
|
268
|
+
-f "le_linear = le.interpolate().where(before & after)" \
|
|
269
|
+
--drop before after
|
|
270
|
+
```
|
|
229
271
|
|
|
230
272
|
### Bootstrap confidence intervals
|
|
231
273
|
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
"""
|
|
2
|
+
dftk.commands.eval_cmd — dftk eval subcommand.
|
|
3
|
+
|
|
4
|
+
Evaluates formulas, constants, and string/row functions against a DataFrame.
|
|
5
|
+
Three formula types are supported, each with its own flag:
|
|
6
|
+
|
|
7
|
+
-f pandas eval expressions (df.eval, falls back to special_function)
|
|
8
|
+
-c constant column assignments
|
|
9
|
+
-s string / row functions
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import logging
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
from dftk.commands.base import BaseCommand
|
|
20
|
+
from dftk.common.evalfuncs import (
|
|
21
|
+
SPECIAL_FUNCTIONS,
|
|
22
|
+
FormulaMistake,
|
|
23
|
+
apply_special_function,
|
|
24
|
+
is_special_call,
|
|
25
|
+
string_function,
|
|
26
|
+
)
|
|
27
|
+
from dftk.common.io import io
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
_EPILOG = """\
|
|
33
|
+
FORMULA EXPRESSIONS (-f)
|
|
34
|
+
-------------------------
|
|
35
|
+
Each -f argument is first passed to pandas df.eval(). This handles
|
|
36
|
+
arithmetic and boolean expressions directly on column names:
|
|
37
|
+
|
|
38
|
+
dftk eval data.tsv -f "z = x + y * 2"
|
|
39
|
+
dftk dataset tips -o | dftk eval ... -f "bill_and_tip = total_bill + tip * 2"
|
|
40
|
+
dftk dataset tips -o | dftk eval ... -f "good_tip = (tip > 3) & (total_bill < 30)"
|
|
41
|
+
|
|
42
|
+
Column methods are pandas Series methods, written COL.method(...); text
|
|
43
|
+
methods are under .str (name.str.lower(), not name.lower() or tolower()):
|
|
44
|
+
|
|
45
|
+
x = x.abs() x = x.round(1) x = x.clip(lower=0) y = name.str.upper()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
FILLING MISSING VALUES (-f)
|
|
49
|
+
---------------------------
|
|
50
|
+
-f "x = x.fillna(y)" from another column (or a value: fillna(0))
|
|
51
|
+
-f "x = x.fillna(y).fillna(0)" y first, then 0
|
|
52
|
+
-f "x = x.ffill()" the value above (bfill(): the value below)
|
|
53
|
+
-f "x = x.interpolate()" a straight line, by row position
|
|
54
|
+
-f "x = x.groupby(g).ffill()" within each group of g (also bfill)
|
|
55
|
+
-f "x = x.fillna(x.groupby(g).transform('mean'))" the group mean
|
|
56
|
+
|
|
57
|
+
ffill, bfill and interpolate use the rows in their order: sort first, in a
|
|
58
|
+
step of its own (--sortasc applies to the output), and fill within groups
|
|
59
|
+
when the table holds several series, or values come from the next group:
|
|
60
|
+
|
|
61
|
+
dftk dataset healthexp -o \\
|
|
62
|
+
| dftk eval ... -f "le = Life_Expectancy.where(Year % 10 != 5)" \\
|
|
63
|
+
--sortasc Country Year -o \\
|
|
64
|
+
| dftk eval ... -f "le_filled = le.groupby(Country).ffill()"
|
|
65
|
+
|
|
66
|
+
interpolate() has no grouped form. The README ("Filling missing values")
|
|
67
|
+
covers defaults from a merge, interpolating within groups, and more.
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
DFTK FUNCTIONS (-f)
|
|
71
|
+
-------------------
|
|
72
|
+
A formula whose right-hand side is one call to a function below runs that
|
|
73
|
+
function; any other formula goes to pandas df.eval():
|
|
74
|
+
|
|
75
|
+
dest = func(col1, col2, ...)
|
|
76
|
+
|
|
77
|
+
The call must be the whole right-hand side: for z = sum(a, b) * 2, write
|
|
78
|
+
-f "tmp = sum(a, b)" -f "z = tmp * 2".
|
|
79
|
+
|
|
80
|
+
Available special functions:
|
|
81
|
+
|
|
82
|
+
Row aggregation (applied across named columns):
|
|
83
|
+
sum mean std min max median
|
|
84
|
+
|
|
85
|
+
Conditional assignment:
|
|
86
|
+
where(cond_col, true_val, false_val)
|
|
87
|
+
— np.where-style conditional; cond_col must be a boolean column.
|
|
88
|
+
true_val and false_val can each be: a column name, a quoted
|
|
89
|
+
string literal ("foo"), or a numeric literal.
|
|
90
|
+
Example:
|
|
91
|
+
dftk dataset tips -o | dftk eval ... \
|
|
92
|
+
-f "big = total_bill > 30" \
|
|
93
|
+
-f "label = where(big, \"large\", \"small\")"
|
|
94
|
+
|
|
95
|
+
Bitwise:
|
|
96
|
+
applymask(col, mask_col) — col & mask_col
|
|
97
|
+
overlap(lo1, hi1, lo2, hi2) — length of interval overlap
|
|
98
|
+
|
|
99
|
+
Column aggregation (glob pattern):
|
|
100
|
+
colsum(col*) — sum of all columns matching the glob pattern
|
|
101
|
+
|
|
102
|
+
NumPy math (element-wise):
|
|
103
|
+
sign(col) cbrt(col) log1p(col) expm1(col)
|
|
104
|
+
|
|
105
|
+
scipy.special — log-gamma / digamma:
|
|
106
|
+
gamma(col) lgamma(col) digamma(col)
|
|
107
|
+
beta(a, b) betaln(a, b)
|
|
108
|
+
|
|
109
|
+
scipy.special — error functions:
|
|
110
|
+
erf(col) erfc(col) erfinv(col)
|
|
111
|
+
|
|
112
|
+
scipy.special — sigmoid / logit:
|
|
113
|
+
expit(col) logit(col)
|
|
114
|
+
|
|
115
|
+
scipy.special — Bessel functions (fixed order):
|
|
116
|
+
j0(col) j1(col) — Bessel J, order 0 / 1
|
|
117
|
+
y0(col) y1(col) — Bessel Y, order 0 / 1
|
|
118
|
+
i0(col) i1(col) — modified Bessel I, order 0 / 1
|
|
119
|
+
k0(col) k1(col) — modified Bessel K, order 0 / 1
|
|
120
|
+
|
|
121
|
+
scipy.special — Bessel functions (parametric order):
|
|
122
|
+
jn(n, col) — Bessel J, integer order n
|
|
123
|
+
yn(n, col) — Bessel Y, integer order n
|
|
124
|
+
iv(v, col) — modified Bessel I, real order v
|
|
125
|
+
kv(v, col) — modified Bessel K, real order v
|
|
126
|
+
(n/v is a numeric literal, e.g. result = jn(3, x))
|
|
127
|
+
|
|
128
|
+
Path functions (applied element-wise):
|
|
129
|
+
basename dirname exists getsize realpath
|
|
130
|
+
|
|
131
|
+
Statistical functions (see common/stats.py):
|
|
132
|
+
binom_test(k, n, p)
|
|
133
|
+
fisher_test(a, b, c, d) — p-value
|
|
134
|
+
fisher_OR(a, b, c, d) — odds ratio
|
|
135
|
+
boschloo_test(a, b, c, d) — p-value
|
|
136
|
+
boschloo_OR(a, b, c, d) — odds ratio
|
|
137
|
+
pval2se(effect, pval)
|
|
138
|
+
t2pval(t, df)
|
|
139
|
+
chi2_to_neglogp(chi2, df)
|
|
140
|
+
neglogp_to_chi2(neglogp, df)
|
|
141
|
+
pval_to_chi2(pval, df)
|
|
142
|
+
generalized_poisson_nll(x, lam, theta)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
CONSTANT COLUMNS (-c)
|
|
146
|
+
----------------------
|
|
147
|
+
Add a column with a fixed value:
|
|
148
|
+
|
|
149
|
+
dftk dataset tips -o | dftk eval ... -c "batch = 3"
|
|
150
|
+
dftk dataset tips -o | dftk eval ... -c "region = northeast"
|
|
151
|
+
|
|
152
|
+
The value is coerced to numeric when possible.
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
STRING / ROW FUNCTIONS (-s)
|
|
156
|
+
----------------------------
|
|
157
|
+
Apply a string or row-level function using the same dest = func(cols) syntax:
|
|
158
|
+
|
|
159
|
+
String operations:
|
|
160
|
+
translate(col, from, to)
|
|
161
|
+
len(col)
|
|
162
|
+
join(col1, col2, ...) — joined with --joinsep (default: "")
|
|
163
|
+
lower(col) upper(col)
|
|
164
|
+
zfill(col, width)
|
|
165
|
+
find(col, substr)
|
|
166
|
+
match(col, pattern_col)
|
|
167
|
+
substr(col, start, length)
|
|
168
|
+
substring(col, start_col, length)
|
|
169
|
+
replace(col, old, new)
|
|
170
|
+
leftsplit(col, sep, idx) — split on first sep; idx=0 left part, 1 right
|
|
171
|
+
rightsplit(col, sep, idx) — split on last sep
|
|
172
|
+
sort(col) — sort characters in string
|
|
173
|
+
ord(col) — minimum ord value of characters
|
|
174
|
+
template(...) — Python str.format_map across all columns
|
|
175
|
+
|
|
176
|
+
Path operations:
|
|
177
|
+
basename(col) stem(col) realpath(col) isfile(col) exists(col)
|
|
178
|
+
|
|
179
|
+
Crypto / hash:
|
|
180
|
+
md5(col)
|
|
181
|
+
|
|
182
|
+
DNA / sequence (see common/seq.py):
|
|
183
|
+
gccontent(col)
|
|
184
|
+
basecount(col)
|
|
185
|
+
motifcount(pattern, seq_col)
|
|
186
|
+
readoffset(read_pos, ref_pos, cigar, seq, count)
|
|
187
|
+
|
|
188
|
+
Edit distance (requires python-Levenshtein):
|
|
189
|
+
levenshtein(col1, col2)
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
# ---------------------------------------------------------------------------
|
|
193
|
+
# Core eval logic
|
|
194
|
+
# ---------------------------------------------------------------------------
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# A message pandas gives for a method that no column has, whatever its dtype
|
|
198
|
+
_NO_METHOD = re.compile(r"'Series' object has no attribute '(\w+)'")
|
|
199
|
+
|
|
200
|
+
# pandas' message for a function it doesn't know
|
|
201
|
+
_NO_FUNCTION = re.compile(r'"(\w+)" is not a supported function')
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _formula_error(formula: str, exc: Exception) -> str:
|
|
205
|
+
"""Return a short message for a mistake pandas found in an -f formula."""
|
|
206
|
+
msg = str(exc)
|
|
207
|
+
m = _NO_METHOD.search(msg)
|
|
208
|
+
if m:
|
|
209
|
+
name = m.group(1)
|
|
210
|
+
col = re.search(rf"([A-Za-z_]\w*)\.{name}\b", formula)
|
|
211
|
+
col = col.group(1) if col else "COL"
|
|
212
|
+
if hasattr(pd.Series.str, name):
|
|
213
|
+
hint = f"Text methods are under .str: {col}.str.{name}()."
|
|
214
|
+
else:
|
|
215
|
+
hint = f"Text methods are under .str, e.g. {col}.str.lower()."
|
|
216
|
+
return (
|
|
217
|
+
f"-f {formula!r}: column has no method {name!r}. {hint} See dftk eval -h."
|
|
218
|
+
)
|
|
219
|
+
m = _NO_FUNCTION.search(msg)
|
|
220
|
+
if m:
|
|
221
|
+
name = m.group(1)
|
|
222
|
+
if name in SPECIAL_FUNCTIONS:
|
|
223
|
+
return (
|
|
224
|
+
f"-f {formula!r}: {name}() must be the whole right-hand side, "
|
|
225
|
+
f"e.g. -f 'tmp = {name}(...)' -f 'z = tmp * 2'."
|
|
226
|
+
)
|
|
227
|
+
return f"-f {formula!r}: unknown function {name!r}. See dftk eval -h."
|
|
228
|
+
return f"-f {formula!r}: {msg}"
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _eval_formula(df: pd.DataFrame, formula: str) -> pd.DataFrame:
|
|
232
|
+
"""Evaluate one -f formula: a special function, or else pandas df.eval()."""
|
|
233
|
+
if is_special_call(formula):
|
|
234
|
+
apply_special_function(df, formula)
|
|
235
|
+
return df
|
|
236
|
+
try:
|
|
237
|
+
# Not inplace: that writes into the existing column, which must keep
|
|
238
|
+
# its dtype, so x = x.fillna(y) or x = x.isna() failed whenever the
|
|
239
|
+
# result had another dtype
|
|
240
|
+
result = df.eval(formula)
|
|
241
|
+
except (NameError, SyntaxError) as exc:
|
|
242
|
+
# An unknown column (pandas' UndefinedVariableError) or bad syntax
|
|
243
|
+
raise FormulaMistake(_formula_error(formula, exc)) from exc
|
|
244
|
+
except (AttributeError, TypeError, KeyError, ValueError) as exc:
|
|
245
|
+
# Raised while pandas evaluated the user's expression, so most likely
|
|
246
|
+
# a mistake in it (wrong method, arguments, or .str on numbers)
|
|
247
|
+
msg = str(exc)
|
|
248
|
+
mistake = _NO_METHOD.search(msg) or _NO_FUNCTION.search(msg)
|
|
249
|
+
cls = FormulaMistake if mistake else ValueError
|
|
250
|
+
raise cls(_formula_error(formula, exc)) from exc
|
|
251
|
+
if not isinstance(result, pd.DataFrame):
|
|
252
|
+
# No "dest =": df.eval returned the values instead of a table
|
|
253
|
+
raise FormulaMistake(
|
|
254
|
+
f"-f {formula!r} has no assignment; write it as 'name = {formula}'"
|
|
255
|
+
)
|
|
256
|
+
return result
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _eval(df: pd.DataFrame, args: argparse.Namespace) -> pd.DataFrame:
|
|
260
|
+
joinsep: str = getattr(args, "joinsep", "")
|
|
261
|
+
intbool: bool = getattr(args, "intbool", False)
|
|
262
|
+
existing = set(df.columns)
|
|
263
|
+
|
|
264
|
+
# -c: constant columns
|
|
265
|
+
for expr in getattr(args, "constant", []):
|
|
266
|
+
colname, constval_str = (x.strip() for x in expr.split("=", 1))
|
|
267
|
+
try:
|
|
268
|
+
constval: int | float | str = int(constval_str)
|
|
269
|
+
except ValueError:
|
|
270
|
+
try:
|
|
271
|
+
constval = float(constval_str)
|
|
272
|
+
except ValueError:
|
|
273
|
+
constval = constval_str
|
|
274
|
+
df[colname] = constval
|
|
275
|
+
|
|
276
|
+
# -f: special functions, or else pandas df.eval
|
|
277
|
+
for formula in getattr(args, "formula", []):
|
|
278
|
+
try:
|
|
279
|
+
df = _eval_formula(df, formula)
|
|
280
|
+
except Exception as exc:
|
|
281
|
+
# An empty table has no column types (pandas reads them as
|
|
282
|
+
# float64), so on one only mistakes that don't depend on types
|
|
283
|
+
# are reported; for the rest the column is just added
|
|
284
|
+
if len(df) != 0 or isinstance(exc, FormulaMistake):
|
|
285
|
+
raise
|
|
286
|
+
df[formula.split("=", 1)[0].strip()] = True
|
|
287
|
+
|
|
288
|
+
# -s: string / row functions
|
|
289
|
+
for strfunc in getattr(args, "strfunc", []):
|
|
290
|
+
try:
|
|
291
|
+
dest, func, cols = string_function(strfunc, joinsep)
|
|
292
|
+
if len(df) == 0:
|
|
293
|
+
df[dest] = True
|
|
294
|
+
elif len(cols) == 0:
|
|
295
|
+
df[dest] = df.apply(func, axis=1)
|
|
296
|
+
elif len(cols) == 1:
|
|
297
|
+
df[dest] = df[cols[0]].apply(func)
|
|
298
|
+
else:
|
|
299
|
+
df[dest] = df[cols].apply(func, axis=1)
|
|
300
|
+
except Exception:
|
|
301
|
+
if len(df) == 0:
|
|
302
|
+
dest = strfunc.split("=", 1)[0].strip()
|
|
303
|
+
df[dest] = True
|
|
304
|
+
else:
|
|
305
|
+
logger.error(f"Could not evaluate string function: {strfunc!r}")
|
|
306
|
+
raise
|
|
307
|
+
|
|
308
|
+
# --intbool: convert newly created boolean columns to int
|
|
309
|
+
if intbool:
|
|
310
|
+
new_cols = set(df.columns) - existing
|
|
311
|
+
for col in new_cols:
|
|
312
|
+
if df[col].dtype == np.bool_:
|
|
313
|
+
df[col] = df[col].astype(np.int64)
|
|
314
|
+
|
|
315
|
+
return df
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
# ---------------------------------------------------------------------------
|
|
319
|
+
# Command
|
|
320
|
+
# ---------------------------------------------------------------------------
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class EvalCommand(BaseCommand):
|
|
324
|
+
@property
|
|
325
|
+
def name(self) -> str:
|
|
326
|
+
return "eval"
|
|
327
|
+
|
|
328
|
+
@property
|
|
329
|
+
def help(self) -> str:
|
|
330
|
+
return "Evaluate formulas, constants, and string functions on a DataFrame"
|
|
331
|
+
|
|
332
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
333
|
+
import argparse as ap
|
|
334
|
+
|
|
335
|
+
parser.formatter_class = ap.RawDescriptionHelpFormatter
|
|
336
|
+
parser.epilog = _EPILOG
|
|
337
|
+
|
|
338
|
+
io.parser_read(parser)
|
|
339
|
+
|
|
340
|
+
g = parser.add_argument_group("formula evaluation")
|
|
341
|
+
g.add_argument(
|
|
342
|
+
"-f",
|
|
343
|
+
"--formula",
|
|
344
|
+
help=(
|
|
345
|
+
"pandas eval expression, or special function call "
|
|
346
|
+
"(dest = func(col1, col2, ...)). Repeatable."
|
|
347
|
+
),
|
|
348
|
+
default=[],
|
|
349
|
+
action="append",
|
|
350
|
+
metavar="EXPR",
|
|
351
|
+
)
|
|
352
|
+
g.add_argument(
|
|
353
|
+
"-c",
|
|
354
|
+
"--constant",
|
|
355
|
+
help=(
|
|
356
|
+
"Add a constant column (dest = value). "
|
|
357
|
+
"Value is coerced to numeric when possible. Repeatable."
|
|
358
|
+
),
|
|
359
|
+
default=[],
|
|
360
|
+
action="append",
|
|
361
|
+
metavar="EXPR",
|
|
362
|
+
)
|
|
363
|
+
g.add_argument(
|
|
364
|
+
"-s",
|
|
365
|
+
"--strfunc",
|
|
366
|
+
help=(
|
|
367
|
+
"String or row function (dest = func(col1, col2, ...)). "
|
|
368
|
+
"Repeatable. See --help for full function list."
|
|
369
|
+
),
|
|
370
|
+
default=[],
|
|
371
|
+
action="append",
|
|
372
|
+
metavar="EXPR",
|
|
373
|
+
)
|
|
374
|
+
g.add_argument(
|
|
375
|
+
"--joinsep",
|
|
376
|
+
help="Separator used by the join() string function (default: empty).",
|
|
377
|
+
default="",
|
|
378
|
+
metavar="SEP",
|
|
379
|
+
)
|
|
380
|
+
g.add_argument(
|
|
381
|
+
"--intbool",
|
|
382
|
+
help="Convert newly created boolean columns to int (False=0, True=1).",
|
|
383
|
+
action="store_true",
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
io.parser_output(parser)
|
|
387
|
+
|
|
388
|
+
def execute(self, args: argparse.Namespace) -> None:
|
|
389
|
+
df = io.read(args)
|
|
390
|
+
df = _eval(df, args)
|
|
391
|
+
io.printdf(df, args)
|