pytae 3.4.2__tar.gz → 3.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytae-3.4.2 → pytae-3.4.3}/CHANGELOG.md +12 -0
- {pytae-3.4.2 → pytae-3.4.3}/PKG-INFO +1 -1
- {pytae-3.4.2 → pytae-3.4.3}/docs/CLI.md +39 -5
- {pytae-3.4.2 → pytae-3.4.3}/pyproject.toml +1 -1
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/cli.py +16 -1
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/cli_parsing.py +20 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/cli_pipeline.py +24 -2
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_cli_order.py +169 -0
- {pytae-3.4.2 → pytae-3.4.3}/.githooks/pre-commit +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/.github/workflows/ci.yml +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/.gitignore +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/CONTRIBUTING.md +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/LICENSE +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/README.md +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/CLI_MULTI_FILE.md +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/LIBRARY.md +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/PLOTTING.md +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/images/plotting_bar.png +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/images/plotting_dashboard.png +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/images/plotting_dashboard_enhanced.png +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/images/plotting_facet.png +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/images/plotting_pie.png +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/images/plotting_scatter.png +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/docs/images/plotting_secondary_axis.png +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/agg_df.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/more_plots.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/mutate.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/other_utilities.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/plotter.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/qry.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/select.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/notebooks/shape.ipynb +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/scripts/run_notebooks.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/__init__.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/agg_df.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/anagrams.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/anscombe.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/attention.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/brain_networks.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/car_crashes.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/diamonds.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/dots.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/dowjones.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/exercise.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/flights.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/fmri.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/geyser.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/glue.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/healthexp.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/iris.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/mpg.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/penguins.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/planets.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/seaice.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/taxis.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/tips.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/datasets/titanic.parquet +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/mutate.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/other_utilities.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/plotting.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/qry.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/readers.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/select.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/src/pytae/shape.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/conftest.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_agg_df.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_cli_sample_datasets.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_import.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_mutate.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_plotter.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_qry.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_readers.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_select.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_shape.py +0 -0
- {pytae-3.4.2 → pytae-3.4.3}/tests/test_utilities.py +0 -0
|
@@ -4,6 +4,18 @@ All notable changes to this project are documented in this file.
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [3.4.3] - 2026-09-20
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- CLI `-drop`: subtract exact column names at that point in the pipeline (`df.drop(columns=...)`).
|
|
11
|
+
Names only — no `dtype=` / `contains=` / `regex=` / slices (those stay on `-select`). Remaining
|
|
12
|
+
columns keep their order. Repeatable; missing names error with a typo hint; dropping every
|
|
13
|
+
remaining column is an error.
|
|
14
|
+
|
|
15
|
+
### Documentation
|
|
16
|
+
- `docs/CLI.md`: `-mutate` example for string concat via `species.str.cat(island, sep='_')`
|
|
17
|
+
(pandas `eval()` does not support `+` for strings).
|
|
18
|
+
|
|
7
19
|
## [3.4.2] - 2026-09-20
|
|
8
20
|
|
|
9
21
|
### Changed
|
|
@@ -9,6 +9,7 @@ Inspect and convert tabular files (`.parquet`, `.csv`, `.txt`, `.dat`, `.sas7bda
|
|
|
9
9
|
- [Sample datasets](#sample-datasets)
|
|
10
10
|
- [Pytae-specific operations](#pytae-specific-operations)
|
|
11
11
|
- [Column selection — `-select`](#select)
|
|
12
|
+
- [Drop columns — `-drop`](#drop)
|
|
12
13
|
- [Row filtering — `-qry`](#qry)
|
|
13
14
|
- [Mutated columns — `-mutate`](#mutate)
|
|
14
15
|
- [SQL — `-sql`](#sql)
|
|
@@ -47,7 +48,7 @@ How the CLI pipeline works, and how to grab a real dataset to try it on.
|
|
|
47
48
|
<a id="basics"></a>
|
|
48
49
|
### Basics
|
|
49
50
|
|
|
50
|
-
Flag **order is the pipeline**, the same as a pandas/pytae method chain. `-select … -agg_df … -select … -shape` is `df.select(…).agg_df(…).select(…).shape`. Put `-qry` / `-query` first yourself if you need a column you later drop. **Only the last operation prints.** Earlier flags still run.
|
|
51
|
+
Flag **order is the pipeline**, the same as a pandas/pytae method chain. `-select … -agg_df … -select … -shape` is `df.select(…).agg_df(…).select(…).shape`. Put `-qry` / `-query` first yourself if you need a column you later `-drop` / `-select` away. **Only the last operation prints.** Earlier flags still run.
|
|
51
52
|
|
|
52
53
|
```bash
|
|
53
54
|
# first 3 rows, then shape of that 3-row frame → prints (3, n)
|
|
@@ -65,6 +66,9 @@ pytae penguins.parquet -select "species,body_mass_g" -agg_df mean
|
|
|
65
66
|
# heaviest 5 penguins, name + weight only
|
|
66
67
|
pytae penguins.parquet -select "species,body_mass_g" -sort_by "body_mass_g desc" -head 5
|
|
67
68
|
|
|
69
|
+
# drop columns; the rest keep file order
|
|
70
|
+
pytae penguins.parquet -drop "sex,island" -head 5
|
|
71
|
+
|
|
68
72
|
# ad-hoc SQL over the same file
|
|
69
73
|
pytae penguins.parquet -sql "select species, avg(body_mass_g) from df group by species"
|
|
70
74
|
```
|
|
@@ -178,10 +182,34 @@ Exact names must exist on the **current** columns; missing names error with a ty
|
|
|
178
182
|
|
|
179
183
|
---
|
|
180
184
|
|
|
185
|
+
<a id="drop"></a>
|
|
186
|
+
### Drop columns — `-drop`
|
|
187
|
+
|
|
188
|
+
Subtract **exact column names**. Complements `-select`: `-select` picks (and may reorder); `-drop` removes and leaves the rest in their current order. No `dtype=` / `contains=` / `regex=` / slices — those stay on `-select`.
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
df.drop(columns=["sex", "island"])
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
# yank known columns
|
|
196
|
+
pytae penguins.parquet -drop "sex,island" -head 5
|
|
197
|
+
|
|
198
|
+
# keep a shape, then pull one column out of it
|
|
199
|
+
pytae penguins.parquet -select "dtype=numeric,species" -drop "body_mass_g" -cols
|
|
200
|
+
|
|
201
|
+
# filter on a column, then remove it
|
|
202
|
+
pytae penguins.parquet -qry "species: 'Adelie'" -drop "species" -head
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Same quoting as `-select` names (a space is not a separator; quote a name only to protect a comma). Missing names error with a typo suggestion. Repeat `-drop` to subtract more. Dropping every remaining column is an error.
|
|
206
|
+
|
|
207
|
+
---
|
|
208
|
+
|
|
181
209
|
<a id="qry"></a>
|
|
182
210
|
### Row filtering — `-qry`
|
|
183
211
|
|
|
184
|
-
pytae's dict-based filter, `df.qry()` — safer than `-query` for odd strings (values with spaces or special characters). **Narrows rows** at this point in the pipeline. Put it **before** `-select` if you need a column you then drop. Can be combined with [`-query`](#query) (stacks sequentially — AND on the remaining rows).
|
|
212
|
+
pytae's dict-based filter, `df.qry()` — safer than `-query` for odd strings (values with spaces or special characters). **Narrows rows** at this point in the pipeline. Put it **before** `-select` / `-drop` if you need a column you then drop. Can be combined with [`-query`](#query) (stacks sequentially — AND on the remaining rows).
|
|
185
213
|
|
|
186
214
|
```python
|
|
187
215
|
df.qry({"species": "Adelie", "body_mass_g": (">", 3500)})
|
|
@@ -193,8 +221,8 @@ df.qry({"species": "Adelie"}).select("species", "body_mass_g")
|
|
|
193
221
|
# vs 3500 vs ('>', 3500)). That is not the same as -select contains=bill.
|
|
194
222
|
pytae penguins.parquet -qry "species: 'Adelie', body_mass_g: ('>', 3500)"
|
|
195
223
|
|
|
196
|
-
# filter first, then drop the filter column — same as df.qry(...).
|
|
197
|
-
pytae penguins.parquet -qry "species: 'Adelie'" -
|
|
224
|
+
# filter first, then drop the filter column — same as df.qry(...).drop(columns=...)
|
|
225
|
+
pytae penguins.parquet -qry "species: 'Adelie'" -drop "species" -head
|
|
198
226
|
|
|
199
227
|
# string-matching operators: startswith / endswith / contains / regex (search-anywhere,
|
|
200
228
|
# like re.search — 'regex' is 'contains' with regex=True); missing values never match (na=False)
|
|
@@ -236,6 +264,9 @@ pytae penguins.parquet -mutate "mass_kg: body_mass_g / 1000, mass_lb: mass_kg *
|
|
|
236
264
|
# key quoting optional (matches -qry); string literals in the expression still need quotes
|
|
237
265
|
pytae penguins.parquet -mutate "is_adelie: species == 'Adelie'" -select "species,is_adelie" -head
|
|
238
266
|
|
|
267
|
+
# string concat: pandas eval does not support + for strings; use .str.cat
|
|
268
|
+
pytae penguins.parquet -mutate "dummy: species.str.cat(island, sep='_')" -select "species,island,dummy" -head
|
|
269
|
+
|
|
239
270
|
# chains into -qry, filtering on a column just mutated
|
|
240
271
|
pytae penguins.parquet -mutate "bmi: body_mass_g / bill_length_mm ** 2" -qry "bmi: ('>', 2)" -select "species,bmi" -head
|
|
241
272
|
|
|
@@ -682,7 +713,7 @@ Every structured flag is one of two families. Wrap the **whole spec** in `""` wh
|
|
|
682
713
|
|
|
683
714
|
**kwargs** — `key=value,key=value`. Used by `-select` kwargs, `-agg`, `-group_x`, `-crosstab`, `-long`/`-wide`, `-merge`, `-concat`, `-replace_values`, `-clean_columns`, `-file` extras. Quote a value only when it contains a comma. `contains=bill`, `index=species`, and `dtype=numeric` stay unquoted — they are tokens, not Python literals (`contains='bill'` is accepted, not the style).
|
|
684
715
|
|
|
685
|
-
**names and mappings** — `a,b` lists, or `name: payload`. Used by `-select` names/slices, `-group_by`, `-sort_by`, `-qry`, `-mutate`, `-rename`, and nested maps like `v=` / `on=` pairs. `-qry` is the exception that *requires* quotes on string values: they are typed literals (`'Adelie'` vs `3500` vs `('>', 3500)`), so unquoted `Adelie` is not valid. `-mutate` column names inside an expression must stay unquoted.
|
|
716
|
+
**names and mappings** — `a,b` lists, or `name: payload`. Used by `-select` names/slices, `-drop`, `-group_by`, `-sort_by`, `-qry`, `-mutate`, `-rename`, and nested maps like `v=` / `on=` pairs. `-qry` is the exception that *requires* quotes on string values: they are typed literals (`'Adelie'` vs `3500` vs `('>', 3500)`), so unquoted `Adelie` is not valid. `-mutate` column names inside an expression must stay unquoted.
|
|
686
717
|
|
|
687
718
|
Simple flags (`-head 5`, `-sql "..."`, `-query "..."`, `-pretty`, `-dropna false`) are neither family.
|
|
688
719
|
|
|
@@ -702,6 +733,7 @@ pytae -file "a.csv=df1;b.csv=df2" -concat "frames='df1,df2'"
|
|
|
702
733
|
# names: comma list; quote a name only to protect a comma
|
|
703
734
|
pytae data.parquet -select "bill length mm,body mass g"
|
|
704
735
|
pytae data.parquet -select "'city, state',other_col"
|
|
736
|
+
pytae penguins.parquet -drop "sex,island"
|
|
705
737
|
pytae penguins.parquet -sort_by "species,body_mass_g desc"
|
|
706
738
|
|
|
707
739
|
# mappings: name: payload. -qry strings are typed literals, so they need quotes
|
|
@@ -724,6 +756,7 @@ Some flags/keys are thin passthroughs to standard pandas methods and parameter n
|
|
|
724
756
|
| Flag / key | Pandas equivalent |
|
|
725
757
|
|---|---|
|
|
726
758
|
| `-query` | `df.query()` |
|
|
759
|
+
| `-drop` | `df.drop(columns=...)` — exact names only; patterns/dtypes/slices stay on `-select` |
|
|
727
760
|
| `-describe` / `-info` / `-shape` / `-cols` / `-dtype` / `-nulls` | `df.describe()` / `df.info()` / `df.shape` / `df.columns` / `df.dtypes` / `df.isna().sum()` |
|
|
728
761
|
| `-sort_by` | `df.sort_values()` |
|
|
729
762
|
| `-crosstab`'s `values=` / `aggfunc=` / `normalize=` / `margins=` | same keyword names as `pd.crosstab()` |
|
|
@@ -883,6 +916,7 @@ pytae penguins.parquet -qry "island: 'Dream'" -select "species,island,body_mass_
|
|
|
883
916
|
| Flag | Description |
|
|
884
917
|
|---|---|
|
|
885
918
|
| `-select SPEC` | Restrict columns at this point in the pipeline (union in one spec; repeat to filter remaining) |
|
|
919
|
+
| `-drop COLUMNS` | Drop columns by exact name at this point (comma-separated names only; remaining keep their order) |
|
|
886
920
|
| `-qry CONDITIONS` | Filter rows at this point (`df.qry()`); surrounding `{}` and column-name quotes optional |
|
|
887
921
|
| `-mutate SPEC` | Create/overwrite columns at this point (`df.mutate()`); `"new_col: expression"` entries, same tokenizer/key-quoting rules as `-qry` (no surrounding `{}`), but the value is a pandas `eval()` expression, not a literal |
|
|
888
922
|
| `-query EXPR` | Filter rows at this point (`df.query()`) |
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pytae"
|
|
7
|
-
version = "3.4.
|
|
7
|
+
version = "3.4.3"
|
|
8
8
|
description = "A package to enhance common tasks in data and plotting"
|
|
9
9
|
authors = [{ name = "Madhukar Kumar Jha", email = "madhukar.k.jha@gmail.com" }]
|
|
10
10
|
readme = "README.md"
|
|
@@ -20,6 +20,7 @@ from pytae.cli_parsing import (
|
|
|
20
20
|
parse_columns,
|
|
21
21
|
parse_concat_arg,
|
|
22
22
|
parse_crosstab_arg,
|
|
23
|
+
parse_drop_spec,
|
|
23
24
|
parse_file_arg,
|
|
24
25
|
parse_fraction,
|
|
25
26
|
parse_group_agg,
|
|
@@ -194,6 +195,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
194
195
|
"names, start:end slices, and key=value (dtype, contains, startswith, endswith, "
|
|
195
196
|
"regex, exclude_dtype); repeat to filter remaining columns, including after "
|
|
196
197
|
'-agg_df/-long/-wide, e.g. -select "dtype=numeric" -select "contains=bill"')
|
|
198
|
+
parser.add_argument("--drop", "-drop", dest="drop", action=_OrderedAppend, default=None, metavar="COLUMNS",
|
|
199
|
+
help="drop columns by exact name at this point in the pipeline (comma-separated names "
|
|
200
|
+
"only; use -select for dtype=/contains=/regex=/slices); remaining columns keep "
|
|
201
|
+
'their order, e.g. -drop "sex,island"')
|
|
197
202
|
parser.add_argument("-convert", "--convert", dest="convert",
|
|
198
203
|
action=_OrderedFlag,
|
|
199
204
|
help="convert to another format (.parquet/.csv/.txt/.dat, inferred from -o's extension, "
|
|
@@ -356,6 +361,7 @@ def _process_path(
|
|
|
356
361
|
*,
|
|
357
362
|
show_all: bool,
|
|
358
363
|
select_specs: list[tuple[list[str], dict]],
|
|
364
|
+
drop_specs: list[list[str]],
|
|
359
365
|
qry_specs: list[dict],
|
|
360
366
|
mutate_specs: list[str],
|
|
361
367
|
query_specs: list[str],
|
|
@@ -397,6 +403,7 @@ def _process_path(
|
|
|
397
403
|
op_order = getattr(args, "op_order", [])
|
|
398
404
|
last_idx = len(op_order) - 1
|
|
399
405
|
select_iter = iter(select_specs)
|
|
406
|
+
drop_iter = iter(drop_specs)
|
|
400
407
|
qry_iter = iter(qry_specs)
|
|
401
408
|
mutate_iter = iter(mutate_specs)
|
|
402
409
|
query_iter = iter(query_specs)
|
|
@@ -425,6 +432,11 @@ def _process_path(
|
|
|
425
432
|
if err:
|
|
426
433
|
return _fail(parser, batch, err)
|
|
427
434
|
emit_frame(idx)
|
|
435
|
+
elif op == "drop":
|
|
436
|
+
err = pipeline.apply_drop(next(drop_iter))
|
|
437
|
+
if err:
|
|
438
|
+
return _fail(parser, batch, err)
|
|
439
|
+
emit_frame(idx)
|
|
428
440
|
elif op == "qry":
|
|
429
441
|
err = pipeline.apply_qry(next(qry_iter))
|
|
430
442
|
if err:
|
|
@@ -766,7 +778,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
766
778
|
args.convert, args.agg_df is not None, args.agg is not None,
|
|
767
779
|
args.group_x is not None, args.handle_missing is not None,
|
|
768
780
|
args.long is not None, args.wide is not None, args.crosstab is not None,
|
|
769
|
-
args.select, args.qry, args.query, args.sql, args.replace_values,
|
|
781
|
+
args.select, args.drop, args.qry, args.query, args.sql, args.replace_values,
|
|
770
782
|
args.clean_columns is not None, args.merge, args.concat])
|
|
771
783
|
|
|
772
784
|
wants_df = any([args.cols, args.dtype, args.nulls, args.describe, show_all,
|
|
@@ -790,6 +802,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
790
802
|
|
|
791
803
|
rename_map = parse_rename(args.rename) if args.rename else None
|
|
792
804
|
select_specs = [parse_select_spec(raw) for raw in (args.select or [])]
|
|
805
|
+
drop_specs = [parse_drop_spec(raw) for raw in (args.drop or [])]
|
|
793
806
|
qry_specs = [parse_qry(raw) for raw in (args.qry or [])]
|
|
794
807
|
mutate_specs = list(args.mutate or [])
|
|
795
808
|
query_specs = list(args.query or [])
|
|
@@ -813,6 +826,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
813
826
|
except UnicodeError as exc:
|
|
814
827
|
parser.error(_encoding_error_message(entry_path, entry["encoding"], exc))
|
|
815
828
|
failed = _process_path(None, args, parser, False, show_all=show_all, select_specs=select_specs,
|
|
829
|
+
drop_specs=drop_specs,
|
|
816
830
|
qry_specs=qry_specs, mutate_specs=mutate_specs, query_specs=query_specs, sql_specs=sql_specs,
|
|
817
831
|
replace_specs=replace_specs, rename_map=rename_map, frames=frames,
|
|
818
832
|
merge_specs=merge_specs, concat_specs=concat_specs)
|
|
@@ -830,6 +844,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
830
844
|
print(f"== {path} ==")
|
|
831
845
|
try:
|
|
832
846
|
failed = _process_path(path, args, parser, batch, show_all=show_all, select_specs=select_specs,
|
|
847
|
+
drop_specs=drop_specs,
|
|
833
848
|
qry_specs=qry_specs, mutate_specs=mutate_specs, query_specs=query_specs, sql_specs=sql_specs,
|
|
834
849
|
replace_specs=replace_specs, rename_map=rename_map)
|
|
835
850
|
except UnicodeError as exc:
|
|
@@ -90,6 +90,26 @@ def _split_groups(raw: str, sep: str = ";") -> list[str]:
|
|
|
90
90
|
return [t for t in _tokenize(raw, sep, keep_quotes=True) if t]
|
|
91
91
|
|
|
92
92
|
|
|
93
|
+
def parse_drop_spec(raw: str) -> list[str]:
|
|
94
|
+
"""Parse -drop into exact column names.
|
|
95
|
+
|
|
96
|
+
Names only: comma-separated, same quoting as -select names (a space is not
|
|
97
|
+
a separator). key=value tokens and select matchers (dtype=/contains=/regex=/
|
|
98
|
+
slices) are rejected — those stay on -select.
|
|
99
|
+
"""
|
|
100
|
+
tokens = _split_tokens(raw)
|
|
101
|
+
if not tokens:
|
|
102
|
+
raise SystemExit("-drop: expected column names")
|
|
103
|
+
names: list[str] = []
|
|
104
|
+
for token in tokens:
|
|
105
|
+
if "=" in token:
|
|
106
|
+
raise SystemExit(
|
|
107
|
+
"-drop: only column names; use -select for dtype=/contains=/regex=/exclude_dtype="
|
|
108
|
+
)
|
|
109
|
+
names.append(token)
|
|
110
|
+
return names
|
|
111
|
+
|
|
112
|
+
|
|
93
113
|
def parse_select_spec(raw: str) -> tuple[list[str], dict]:
|
|
94
114
|
"""Parse -select into positional names/slices and select() kwargs.
|
|
95
115
|
|
|
@@ -36,8 +36,8 @@ def _mask_quoted(spec: str) -> str:
|
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
class _Pipeline:
|
|
39
|
-
"""Flag order is the method chain. Each -select/-qry/-query/-head/… call
|
|
40
|
-
runs on the current view, same as df.select().qry().head().
|
|
39
|
+
"""Flag order is the method chain. Each -select/-drop/-qry/-query/-head/… call
|
|
40
|
+
runs on the current view, same as df.select().drop(columns=…).qry().head().
|
|
41
41
|
Only the last flag prints. Schema-only ops (-cols/-dtype/-shape) avoid loading
|
|
42
42
|
row data until something actually requires it. -shape/-cols/-dtype/-nulls/-info
|
|
43
43
|
don't return a DataFrame/Series in pandas either, so (like main()'s validation)
|
|
@@ -82,6 +82,28 @@ class _Pipeline:
|
|
|
82
82
|
return f"-select: {exc}"
|
|
83
83
|
return None
|
|
84
84
|
|
|
85
|
+
def apply_drop(self, names: list[str]) -> str | None:
|
|
86
|
+
"""Apply one -drop spec (exact column names) to the current view.
|
|
87
|
+
|
|
88
|
+
Remaining columns keep their existing order. Returns an error message or None.
|
|
89
|
+
"""
|
|
90
|
+
available = self._available_columns()
|
|
91
|
+
unknown = [c for c in names if c not in available]
|
|
92
|
+
if unknown:
|
|
93
|
+
msg = unknown_columns_message("-drop", unknown, available)
|
|
94
|
+
if any(":" in c for c in unknown):
|
|
95
|
+
msg += " (-drop does not accept slices; use -select)"
|
|
96
|
+
return msg
|
|
97
|
+
drop_set = set(names)
|
|
98
|
+
remaining = [c for c in available if c not in drop_set]
|
|
99
|
+
if not remaining:
|
|
100
|
+
return "-drop: no columns left"
|
|
101
|
+
if self._df is not None:
|
|
102
|
+
self._df = self._df.drop(columns=list(dict.fromkeys(names)))
|
|
103
|
+
else:
|
|
104
|
+
self._pending_exact = remaining
|
|
105
|
+
return None
|
|
106
|
+
|
|
85
107
|
def apply_qry(self, conditions: dict) -> str | None:
|
|
86
108
|
"""Apply one -qry spec to the current view. Returns an error message or None."""
|
|
87
109
|
df = self.dataframe()
|
|
@@ -797,6 +797,21 @@ def test_parse_select_spec_quoted_name_and_repeated_key():
|
|
|
797
797
|
assert kwargs == {"contains": ["bill", "body"]}
|
|
798
798
|
|
|
799
799
|
|
|
800
|
+
def test_parse_drop_spec_names_and_quoted():
|
|
801
|
+
assert cli.parse_drop_spec("sex,year") == ["sex", "year"]
|
|
802
|
+
assert cli.parse_drop_spec("'bill length mm',body_mass_g") == ["bill length mm", "body_mass_g"]
|
|
803
|
+
assert cli.parse_drop_spec("a:b") == ["a:b"] # exact name, not a slice
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def test_parse_drop_spec_rejects_select_kwargs_and_empty():
|
|
807
|
+
with pytest.raises(SystemExit, match="only column names"):
|
|
808
|
+
cli.parse_drop_spec("dtype=numeric")
|
|
809
|
+
with pytest.raises(SystemExit, match="only column names"):
|
|
810
|
+
cli.parse_drop_spec("sex,contains=bill")
|
|
811
|
+
with pytest.raises(SystemExit, match="expected column names"):
|
|
812
|
+
cli.parse_drop_spec("")
|
|
813
|
+
|
|
814
|
+
|
|
800
815
|
def test_parse_agg_name_list_and_mapping():
|
|
801
816
|
from pytae.cli_parsing import parse_agg
|
|
802
817
|
assert parse_agg("mean") == "mean"
|
|
@@ -918,6 +933,160 @@ def test_select_unknown_exact_name_errors(tmp_path, capsys):
|
|
|
918
933
|
assert "'d'" in err
|
|
919
934
|
|
|
920
935
|
|
|
936
|
+
def test_drop_does_not_collide_with_dropna():
|
|
937
|
+
parser = cli.build_parser()
|
|
938
|
+
args, extras = parser.parse_known_args(["file.csv", "-drop", "sex", "-dropna", "false"])
|
|
939
|
+
assert extras == []
|
|
940
|
+
assert args.drop == ["sex"]
|
|
941
|
+
assert args.dropna is False
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def test_drop_removes_names_and_keeps_order(tmp_path, capsys):
|
|
945
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "c": [2], "b": [3], "d": [4]}))
|
|
946
|
+
|
|
947
|
+
exit_code = cli.main([path, "-drop", "c,d", "-cols"])
|
|
948
|
+
|
|
949
|
+
assert exit_code == 0
|
|
950
|
+
assert capsys.readouterr().out.strip().splitlines() == ["a", "b"]
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def test_drop_quoted_name_with_space(tmp_path, capsys):
|
|
954
|
+
path = _write_csv(tmp_path, pd.DataFrame({"bill length mm": [1], "body mass g": [2], "keep": [3]}))
|
|
955
|
+
|
|
956
|
+
exit_code = cli.main([path, "-drop", "bill length mm", "-cols"])
|
|
957
|
+
|
|
958
|
+
assert exit_code == 0
|
|
959
|
+
assert capsys.readouterr().out.strip().splitlines() == ["body mass g", "keep"]
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
def test_drop_after_select_subtracts_from_kept(tmp_path, capsys):
|
|
963
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "b": [2], "c": [3], "d": [4]}))
|
|
964
|
+
|
|
965
|
+
exit_code = cli.main([path, "-select", "a,b,c", "-drop", "b", "-cols"])
|
|
966
|
+
|
|
967
|
+
assert exit_code == 0
|
|
968
|
+
assert capsys.readouterr().out.strip().splitlines() == ["a", "c"]
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def test_drop_rejects_name_not_kept_by_select(tmp_path, capsys):
|
|
972
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "b": [2], "c": [3]}))
|
|
973
|
+
|
|
974
|
+
with pytest.raises(SystemExit) as exc_info:
|
|
975
|
+
cli.main([path, "-select", "a,b", "-drop", "c", "-cols"])
|
|
976
|
+
assert exc_info.value.code == 2
|
|
977
|
+
err = capsys.readouterr().err
|
|
978
|
+
assert "unknown column" in err
|
|
979
|
+
assert "'c'" in err
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
def test_select_then_drop_from_dtype_union(tmp_path, capsys):
|
|
983
|
+
path = _write_csv(
|
|
984
|
+
tmp_path,
|
|
985
|
+
pd.DataFrame({"species": ["A"], "n": [1], "x": [2.0], "label": ["z"]}),
|
|
986
|
+
)
|
|
987
|
+
|
|
988
|
+
exit_code = cli.main([path, "-select", "dtype=numeric,species", "-drop", "x", "-cols"])
|
|
989
|
+
|
|
990
|
+
assert exit_code == 0
|
|
991
|
+
# -select puts explicit names first, then dtype matches; -drop leaves that order
|
|
992
|
+
assert capsys.readouterr().out.strip().splitlines() == ["species", "n"]
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def test_repeated_drop_subtracts_sequentially(tmp_path, capsys):
|
|
996
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "b": [2], "c": [3], "d": [4]}))
|
|
997
|
+
|
|
998
|
+
exit_code = cli.main([path, "-drop", "b", "-drop", "d", "-cols"])
|
|
999
|
+
|
|
1000
|
+
assert exit_code == 0
|
|
1001
|
+
assert capsys.readouterr().out.strip().splitlines() == ["a", "c"]
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
def test_drop_unknown_name_errors_with_typo_hint(tmp_path, capsys):
|
|
1005
|
+
path = _write_csv(tmp_path, pd.DataFrame({"species": [1], "island": [2]}))
|
|
1006
|
+
|
|
1007
|
+
with pytest.raises(SystemExit) as exc_info:
|
|
1008
|
+
cli.main([path, "-drop", "speces", "-cols"])
|
|
1009
|
+
assert exc_info.value.code == 2
|
|
1010
|
+
err = capsys.readouterr().err
|
|
1011
|
+
assert "unknown column" in err
|
|
1012
|
+
assert "'speces'" in err
|
|
1013
|
+
assert "species" in err
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def test_drop_slice_token_is_not_a_slice(tmp_path, capsys):
|
|
1017
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "b": [2], "c": [3]}))
|
|
1018
|
+
|
|
1019
|
+
with pytest.raises(SystemExit) as exc_info:
|
|
1020
|
+
cli.main([path, "-drop", "a:c", "-cols"])
|
|
1021
|
+
assert exc_info.value.code == 2
|
|
1022
|
+
err = capsys.readouterr().err
|
|
1023
|
+
assert "unknown column" in err
|
|
1024
|
+
assert "does not accept slices" in err
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def test_drop_colon_column_name_is_exact(tmp_path, capsys):
|
|
1028
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a:b": [1], "c": [2]}))
|
|
1029
|
+
|
|
1030
|
+
exit_code = cli.main([path, "-drop", "a:b", "-cols"])
|
|
1031
|
+
|
|
1032
|
+
assert exit_code == 0
|
|
1033
|
+
assert capsys.readouterr().out.strip().splitlines() == ["c"]
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
def test_drop_rejects_select_kwargs(tmp_path, capsys):
|
|
1037
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "b": [2]}))
|
|
1038
|
+
|
|
1039
|
+
with pytest.raises(SystemExit, match="only column names"):
|
|
1040
|
+
cli.main([path, "-drop", "dtype=numeric", "-cols"])
|
|
1041
|
+
with pytest.raises(SystemExit, match="only column names"):
|
|
1042
|
+
cli.main([path, "-drop", "contains=a", "-cols"])
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
def test_drop_all_columns_errors(tmp_path, capsys):
|
|
1046
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "b": [2]}))
|
|
1047
|
+
|
|
1048
|
+
with pytest.raises(SystemExit) as exc_info:
|
|
1049
|
+
cli.main([path, "-drop", "a,b", "-cols"])
|
|
1050
|
+
assert exc_info.value.code == 2
|
|
1051
|
+
assert "no columns left" in capsys.readouterr().err
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def test_drop_only_prints_remaining_table(tmp_path, capsys):
|
|
1055
|
+
path = _write_csv(tmp_path, pd.DataFrame({"a": [1], "b": [2], "c": [3]}))
|
|
1056
|
+
|
|
1057
|
+
exit_code = cli.main([path, "-drop", "c"])
|
|
1058
|
+
|
|
1059
|
+
out = capsys.readouterr().out
|
|
1060
|
+
assert exit_code == 0
|
|
1061
|
+
assert "a" in out and "b" in out
|
|
1062
|
+
assert "c" not in out.split()
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
def test_drop_after_agg_df_sees_agg_columns(tmp_path, capsys):
|
|
1066
|
+
path = _write_csv(
|
|
1067
|
+
tmp_path,
|
|
1068
|
+
pd.DataFrame({"grp": ["x", "x", "y"], "val": [1, 2, 3], "z": [9, 8, 7]}),
|
|
1069
|
+
)
|
|
1070
|
+
|
|
1071
|
+
exit_code = cli.main(
|
|
1072
|
+
[path, "-agg_df", "val: sum, n: n", "-drop", "val", "-cols"]
|
|
1073
|
+
)
|
|
1074
|
+
assert exit_code == 0
|
|
1075
|
+
assert capsys.readouterr().out.strip().splitlines() == ["grp", "n"]
|
|
1076
|
+
|
|
1077
|
+
|
|
1078
|
+
def test_qry_then_drop_filter_column(tmp_path, capsys):
|
|
1079
|
+
path = _write_csv(
|
|
1080
|
+
tmp_path,
|
|
1081
|
+
pd.DataFrame({"keep": [1, 2], "flt": ["A", "B"], "val": [10, 20]}),
|
|
1082
|
+
)
|
|
1083
|
+
|
|
1084
|
+
exit_code = cli.main([path, "-qry", "flt: 'A'", "-drop", "flt", "-cols"])
|
|
1085
|
+
|
|
1086
|
+
assert exit_code == 0
|
|
1087
|
+
assert capsys.readouterr().out.strip().splitlines() == ["keep", "val"]
|
|
1088
|
+
|
|
1089
|
+
|
|
921
1090
|
def test_select_after_agg_df_sees_agg_columns(tmp_path, capsys):
|
|
922
1091
|
path = _write_csv(
|
|
923
1092
|
tmp_path,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|