parq-cli 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parq_cli-0.2.2/.claude/settings.json +6 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.github/workflows/publish.yml +12 -7
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.github/workflows/test.yml +1 -1
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.gitignore +3 -0
- parq_cli-0.2.2/AGENTS.md +15 -0
- parq_cli-0.2.2/CHANGELOG.md +26 -0
- parq_cli-0.2.2/CLAUDE.md +24 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/PKG-INFO +27 -17
- {parq_cli-0.2.0 → parq_cli-0.2.2}/README.md +20 -11
- {parq_cli-0.2.0 → parq_cli-0.2.2}/README_CN.md +36 -27
- {parq_cli-0.2.0 → parq_cli-0.2.2}/docs/QUICKSTART.md +4 -7
- {parq_cli-0.2.0 → parq_cli-0.2.2}/docs/RELEASE.md +6 -2
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/__init__.py +1 -1
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/cli.py +172 -35
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/__init__.py +1 -2
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_chunk_writers.py +15 -11
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_common.py +12 -62
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_parquet.py +10 -0
- parq_cli-0.2.2/parq/formats/_xlsx.py +337 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/output.py +40 -9
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/plain_output.py +43 -6
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/reader.py +429 -77
- {parq_cli-0.2.0 → parq_cli-0.2.2}/pyproject.toml +7 -6
- {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/README.md +1 -1
- {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/performance_benchmark.py +3 -1
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_cli.py +223 -9
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_output.py +33 -0
- parq_cli-0.2.2/tests/test_output_not_input.py +26 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_plain_output.py +19 -3
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_reader.py +203 -16
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_release_scripts.py +1 -1
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_workflows.py +11 -0
- parq_cli-0.2.2/tests/test_xlsx_defused.py +31 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/uv.lock +226 -347
- parq_cli-0.2.0/AGENTS.md +0 -46
- parq_cli-0.2.0/parq/formats/_xlsx.py +0 -205
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.python-version +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/csv-large.csv +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/csv-small.csv +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/parquet-large.parquet +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/parquet-small.parquet +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/LICENSE +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/data/test.parquet +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/examples/README.md +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/examples/create_sample_data.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/__main__.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_csv.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/bump_version.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/check_version.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/publish.sh +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/__init__.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/conftest.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_main_entry.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_package_meta.py +0 -0
- {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_performance.py +0 -0
|
@@ -35,8 +35,6 @@ jobs:
|
|
|
35
35
|
build-and-publish:
|
|
36
36
|
needs: pre-publish-tests
|
|
37
37
|
runs-on: ubuntu-latest
|
|
38
|
-
permissions:
|
|
39
|
-
id-token: write
|
|
40
38
|
|
|
41
39
|
steps:
|
|
42
40
|
- name: 📥 Checkout code
|
|
@@ -68,10 +66,17 @@ jobs:
|
|
|
68
66
|
path: dist/*
|
|
69
67
|
|
|
70
68
|
- name: 🚀 Publish to PyPI
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
69
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
70
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
71
|
+
with:
|
|
72
|
+
user: __token__
|
|
73
|
+
password: ${{ secrets.PYPI_API_TOKEN }}
|
|
74
|
+
|
|
75
|
+
- name: ⛔ Refuse publish outside a version tag
|
|
76
|
+
if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
|
|
77
|
+
run: |
|
|
78
|
+
echo "Refusing to publish: PyPI uploads only run from a v* tag."
|
|
79
|
+
exit 1
|
|
75
80
|
|
|
76
81
|
create-github-release:
|
|
77
82
|
needs: build-and-publish
|
|
@@ -87,7 +92,7 @@ jobs:
|
|
|
87
92
|
path: dist
|
|
88
93
|
|
|
89
94
|
- name: 📝 Create GitHub Release
|
|
90
|
-
uses: softprops/action-gh-release@
|
|
95
|
+
uses: softprops/action-gh-release@v2
|
|
91
96
|
with:
|
|
92
97
|
files: dist/*
|
|
93
98
|
generate_release_notes: true
|
parq_cli-0.2.2/AGENTS.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Project Agent Entrypoint
|
|
2
|
+
|
|
3
|
+
This file is the cross-tool entrypoint for parq-cli.
|
|
4
|
+
|
|
5
|
+
## Working Rules
|
|
6
|
+
|
|
7
|
+
- parq-cli is a pure Python CLI package, not a full-stack or frontend project.
|
|
8
|
+
- Keep Python on `>=3.10`.
|
|
9
|
+
- Preserve Typer command contracts, rich/plain/json output modes, and streaming behavior for large CSV/XLSX workflows.
|
|
10
|
+
|
|
11
|
+
## Execution Style
|
|
12
|
+
|
|
13
|
+
- Read `parq/cli.py`, `parq/reader.py`, relevant `parq/formats/*`, output formatters, and tests before editing.
|
|
14
|
+
- Keep changes small and covered by CLI/reader/output tests.
|
|
15
|
+
- Run focused tests first, then the repository verification gate: `pytest -m "not performance"` and `ruff check parq tests`.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.2.2
|
|
4
|
+
|
|
5
|
+
- `merge --force` and `convert` now refuse an output path that is one of the inputs. Previously `--force` deleted that input before reading it.
|
|
6
|
+
- XLSX workbook XML is parsed with `defusedxml` (added to the `xlsx` extra).
|
|
7
|
+
- Table output escapes cell markup and summarizes binary values (#4).
|
|
8
|
+
- Errors include the file path (#5).
|
|
9
|
+
- Dependency security updates: certifi, urllib3, idna, pygments, cryptography, pyarrow.
|
|
10
|
+
|
|
11
|
+
## 0.2.1
|
|
12
|
+
|
|
13
|
+
- `diff` and `merge` honor the global `--delimiter` and `--sheet` options.
|
|
14
|
+
- `.tsv` is a supported output format for `convert`, `split`, and `merge`.
|
|
15
|
+
- XLSX `schema` still samples the first 1000 rows. Full reads widen a column when a later row does not fit that sample.
|
|
16
|
+
- String cardinality and top values stay exact past 10,000 distinct values. Nulls are not counted as empty strings.
|
|
17
|
+
- `diff` spills inputs larger than 100,000 rows per side into temporary files.
|
|
18
|
+
- `split --record-count` progress stays indeterminate while the total is still unknown.
|
|
19
|
+
- PyPI uploads run from a `v*` tag and use the repository `PYPI_API_TOKEN`.
|
|
20
|
+
- XLSX reads use `python-calamine`. A full read parses the workbook once and reuses those rows.
|
|
21
|
+
- CSV `convert` no longer counts every row before writing when a progress bar is shown.
|
|
22
|
+
- `--output`, `--delimiter`, and `--sheet` work before a subcommand and on the subcommand. The value on the subcommand wins when both are set.
|
|
23
|
+
|
|
24
|
+
## 0.2.0
|
|
25
|
+
|
|
26
|
+
- Inspect, split, convert, diff, and merge Parquet, CSV, TSV, and XLSX files.
|
parq_cli-0.2.2/CLAUDE.md
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Claude Project Entrypoint
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
## Guardrails
|
|
5
|
+
|
|
6
|
+
- parq-cli is a Python command-line tool for Parquet, CSV, TSV, and XLSX files.
|
|
7
|
+
- There is no frontend layer.
|
|
8
|
+
- Package baseline is Python `>=3.10`.
|
|
9
|
+
- Default tests exclude performance scenarios; run performance tests only when relevant.
|
|
10
|
+
- Keep `parq` CLI output stable for automation, especially `plain` and `json` output modes.
|
|
11
|
+
|
|
12
|
+
## Commands
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pytest -m "not performance"
|
|
16
|
+
ruff check parq tests
|
|
17
|
+
pytest tests/test_performance.py -m performance -q -s
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Gotchas
|
|
21
|
+
|
|
22
|
+
- `parq schema` on XLSX samples the first 1000 rows. Full reads parse the sheet once with python-calamine and widen a column when a later value does not fit. Writing XLSX still uses openpyxl.
|
|
23
|
+
- `diff` and `merge` use the global `--delimiter` and `--sheet`. `diff` keeps at most 100,000 rows per side in memory, then spills to temporary files.
|
|
24
|
+
- `.tsv` is valid for both input and output. Output uses a tab delimiter.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: parq-cli
|
|
3
|
-
Version: 0.2.
|
|
4
|
-
Summary: A
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: A command-line tool for inspecting tabular files: Parquet, CSV, TSV, and XLSX
|
|
5
5
|
Project-URL: Homepage, https://github.com/Tendo33/parq-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/Tendo33/parq-cli
|
|
7
7
|
Project-URL: Issues, https://github.com/Tendo33/parq-cli/issues
|
|
@@ -14,12 +14,11 @@ Classifier: Development Status :: 3 - Alpha
|
|
|
14
14
|
Classifier: Intended Audience :: Developers
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Classifier: Programming Language :: Python :: 3
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
-
Requires-Python: >=3.
|
|
21
|
+
Requires-Python: >=3.10
|
|
23
22
|
Requires-Dist: pyarrow>=18.0.0
|
|
24
23
|
Requires-Dist: rich>=13.0.0
|
|
25
24
|
Requires-Dist: typer>=0.15.0
|
|
@@ -27,16 +26,18 @@ Provides-Extra: dev
|
|
|
27
26
|
Requires-Dist: build>=1.2.0; extra == 'dev'
|
|
28
27
|
Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
|
|
29
28
|
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
30
|
-
Requires-Dist: ruff
|
|
29
|
+
Requires-Dist: ruff==0.14.0; extra == 'dev'
|
|
31
30
|
Requires-Dist: twine>=5.1.0; extra == 'dev'
|
|
32
31
|
Requires-Dist: vulture>=2.14; extra == 'dev'
|
|
33
32
|
Provides-Extra: xlsx
|
|
33
|
+
Requires-Dist: defusedxml>=0.7.1; extra == 'xlsx'
|
|
34
34
|
Requires-Dist: openpyxl>=3.1.0; extra == 'xlsx'
|
|
35
|
+
Requires-Dist: python-calamine>=0.8.0; extra == 'xlsx'
|
|
35
36
|
Description-Content-Type: text/markdown
|
|
36
37
|
|
|
37
38
|
# parq-cli
|
|
38
39
|
|
|
39
|
-
[](https://www.python.org/downloads/)
|
|
40
41
|
[](LICENSE)
|
|
41
42
|
|
|
42
43
|
A command-line tool for inspecting, transforming, and comparing tabular files.
|
|
@@ -133,7 +134,7 @@ parq merge chunks/*.parquet merged.parquet --force # overwrite if exists
|
|
|
133
134
|
| `diff` | yes | yes | yes | no, convert first |
|
|
134
135
|
| `merge` | yes | yes | yes | yes |
|
|
135
136
|
|
|
136
|
-
`XLSX`
|
|
137
|
+
`XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
|
|
137
138
|
|
|
138
139
|
## Command Reference
|
|
139
140
|
|
|
@@ -200,8 +201,8 @@ Rules:
|
|
|
200
201
|
- specify exactly one of `--file-count` or `--record-count`
|
|
201
202
|
- output format is inferred from `--name-format`
|
|
202
203
|
- by default, existing target files raise an error; use `--force` / `-F` to overwrite
|
|
203
|
-
- in `--record-count` mode, CSV
|
|
204
|
-
-
|
|
204
|
+
- in `--record-count` mode, CSV streams in one pass and the progress bar stays indeterminate
|
|
205
|
+
- XLSX is parsed once before writing; the record-count progress bar stays indeterminate
|
|
205
206
|
|
|
206
207
|
### `stats`
|
|
207
208
|
|
|
@@ -218,6 +219,8 @@ Computes simple per-column statistics.
|
|
|
218
219
|
- string, boolean, and date columns additionally include `cardinality` and `top_values` (top N most frequent values with their occurrence counts)
|
|
219
220
|
- default `--top-n` is `5`; set to `0` to suppress top-values output entirely
|
|
220
221
|
- default `--limit` is `50` to avoid flooding the terminal on very wide tables
|
|
222
|
+
- cardinality and top values are exact, including columns with more than 10,000 distinct values
|
|
223
|
+
- nulls stay in `null_count` and are not treated as empty strings
|
|
221
224
|
|
|
222
225
|
### `convert`
|
|
223
226
|
|
|
@@ -233,7 +236,7 @@ Notes:
|
|
|
233
236
|
|
|
234
237
|
- current targets are `.parquet`, `.csv`, `.tsv`, and `.xlsx`
|
|
235
238
|
- conversion is streaming-based where possible
|
|
236
|
-
- a live progress bar is shown during the conversion
|
|
239
|
+
- a live progress bar is shown during the conversion; CSV stays indeterminate because the row count is not known ahead of time
|
|
237
240
|
- by default, existing output files raise an error; use `--force` / `-F` to overwrite
|
|
238
241
|
|
|
239
242
|
### `diff`
|
|
@@ -255,9 +258,11 @@ Compares two datasets by key and reports:
|
|
|
255
258
|
Notes:
|
|
256
259
|
|
|
257
260
|
- `--key` is required
|
|
258
|
-
- `diff` currently supports Parquet and
|
|
261
|
+
- `diff` currently supports Parquet, CSV, and TSV inputs
|
|
262
|
+
- the global `--delimiter` applies to both CSV inputs
|
|
259
263
|
- XLSX files should be converted first
|
|
260
264
|
- duplicate keys on either side are treated as an error
|
|
265
|
+
- inputs larger than 100,000 rows on a side spill into temporary files instead of holding every key in memory
|
|
261
266
|
- `--summary-only` keeps the counts and omits sample payloads
|
|
262
267
|
|
|
263
268
|
### `merge`
|
|
@@ -274,19 +279,22 @@ Notes:
|
|
|
274
279
|
|
|
275
280
|
- schemas must be identical or safely unifiable by Arrow
|
|
276
281
|
- by default, existing output files raise an error; use `--force` / `-F` to overwrite
|
|
277
|
-
- output format is inferred from the output suffix
|
|
282
|
+
- output format is inferred from the output suffix, including `.tsv`
|
|
283
|
+
- the global `--delimiter` and `--sheet` apply to every input
|
|
278
284
|
- a live progress bar is shown during the merge
|
|
279
285
|
|
|
280
286
|
## Output Modes
|
|
281
287
|
|
|
282
288
|
Global options:
|
|
283
289
|
|
|
284
|
-
- `--version`, `-v`: show version information
|
|
290
|
+
- `--version`, `-v`: show version information. Place this before the subcommand: `parq --version`
|
|
285
291
|
- `--output`, `-o`: select output format (`rich` | `plain` | `json`)
|
|
286
292
|
- `--delimiter`, `-d`: field delimiter for CSV/TSV input (default: `,`); `.tsv` files default to `\t` automatically
|
|
287
|
-
- `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet)
|
|
293
|
+
- `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet). A value made only of digits is an index, including a sheet whose name is `1` or `2`
|
|
288
294
|
- `--help`: show command help
|
|
289
295
|
|
|
296
|
+
`--output`, `--delimiter`, and `--sheet` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
|
|
297
|
+
|
|
290
298
|
Available output modes:
|
|
291
299
|
|
|
292
300
|
- `rich`: human-friendly terminal rendering
|
|
@@ -309,9 +317,11 @@ On Windows terminals that cannot safely render emoji or extended characters, Ric
|
|
|
309
317
|
|
|
310
318
|
- Parquet metadata, row counts, and previews use Arrow metadata and row-group shortcuts where available.
|
|
311
319
|
- CSV `tail` uses a fixed-size column window instead of materializing every row as Python dicts.
|
|
312
|
-
- CSV
|
|
320
|
+
- CSV `split --record-count` and CSV `convert` stream in one pass. Their progress bars stay indeterminate because the row count is not known ahead of time.
|
|
313
321
|
- `meta --fast` is the best option when you need quick metadata from large CSV/XLSX inputs.
|
|
314
|
-
-
|
|
322
|
+
- `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, and `merge` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
|
|
323
|
+
- `split --record-count` keeps the progress bar indeterminate while it writes.
|
|
324
|
+
- `diff` indexes at most 100,000 rows per side in memory, then hash-partitions larger inputs into temporary Parquet files.
|
|
315
325
|
|
|
316
326
|
For repeated heavy workflows, converting large CSV/XLSX files to Parquet is still the best path for throughput.
|
|
317
327
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# parq-cli
|
|
2
2
|
|
|
3
|
-
[](https://www.python.org/downloads/)
|
|
4
4
|
[](LICENSE)
|
|
5
5
|
|
|
6
6
|
A command-line tool for inspecting, transforming, and comparing tabular files.
|
|
@@ -97,7 +97,7 @@ parq merge chunks/*.parquet merged.parquet --force # overwrite if exists
|
|
|
97
97
|
| `diff` | yes | yes | yes | no, convert first |
|
|
98
98
|
| `merge` | yes | yes | yes | yes |
|
|
99
99
|
|
|
100
|
-
`XLSX`
|
|
100
|
+
`XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
|
|
101
101
|
|
|
102
102
|
## Command Reference
|
|
103
103
|
|
|
@@ -164,8 +164,8 @@ Rules:
|
|
|
164
164
|
- specify exactly one of `--file-count` or `--record-count`
|
|
165
165
|
- output format is inferred from `--name-format`
|
|
166
166
|
- by default, existing target files raise an error; use `--force` / `-F` to overwrite
|
|
167
|
-
- in `--record-count` mode, CSV
|
|
168
|
-
-
|
|
167
|
+
- in `--record-count` mode, CSV streams in one pass and the progress bar stays indeterminate
|
|
168
|
+
- XLSX is parsed once before writing; the record-count progress bar stays indeterminate
|
|
169
169
|
|
|
170
170
|
### `stats`
|
|
171
171
|
|
|
@@ -182,6 +182,8 @@ Computes simple per-column statistics.
|
|
|
182
182
|
- string, boolean, and date columns additionally include `cardinality` and `top_values` (top N most frequent values with their occurrence counts)
|
|
183
183
|
- default `--top-n` is `5`; set to `0` to suppress top-values output entirely
|
|
184
184
|
- default `--limit` is `50` to avoid flooding the terminal on very wide tables
|
|
185
|
+
- cardinality and top values are exact, including columns with more than 10,000 distinct values
|
|
186
|
+
- nulls stay in `null_count` and are not treated as empty strings
|
|
185
187
|
|
|
186
188
|
### `convert`
|
|
187
189
|
|
|
@@ -197,7 +199,7 @@ Notes:
|
|
|
197
199
|
|
|
198
200
|
- current targets are `.parquet`, `.csv`, `.tsv`, and `.xlsx`
|
|
199
201
|
- conversion is streaming-based where possible
|
|
200
|
-
- a live progress bar is shown during the conversion
|
|
202
|
+
- a live progress bar is shown during the conversion; CSV stays indeterminate because the row count is not known ahead of time
|
|
201
203
|
- by default, existing output files raise an error; use `--force` / `-F` to overwrite
|
|
202
204
|
|
|
203
205
|
### `diff`
|
|
@@ -219,9 +221,11 @@ Compares two datasets by key and reports:
|
|
|
219
221
|
Notes:
|
|
220
222
|
|
|
221
223
|
- `--key` is required
|
|
222
|
-
- `diff` currently supports Parquet and
|
|
224
|
+
- `diff` currently supports Parquet, CSV, and TSV inputs
|
|
225
|
+
- the global `--delimiter` applies to both CSV inputs
|
|
223
226
|
- XLSX files should be converted first
|
|
224
227
|
- duplicate keys on either side are treated as an error
|
|
228
|
+
- inputs larger than 100,000 rows on a side spill into temporary files instead of holding every key in memory
|
|
225
229
|
- `--summary-only` keeps the counts and omits sample payloads
|
|
226
230
|
|
|
227
231
|
### `merge`
|
|
@@ -238,19 +242,22 @@ Notes:
|
|
|
238
242
|
|
|
239
243
|
- schemas must be identical or safely unifiable by Arrow
|
|
240
244
|
- by default, existing output files raise an error; use `--force` / `-F` to overwrite
|
|
241
|
-
- output format is inferred from the output suffix
|
|
245
|
+
- output format is inferred from the output suffix, including `.tsv`
|
|
246
|
+
- the global `--delimiter` and `--sheet` apply to every input
|
|
242
247
|
- a live progress bar is shown during the merge
|
|
243
248
|
|
|
244
249
|
## Output Modes
|
|
245
250
|
|
|
246
251
|
Global options:
|
|
247
252
|
|
|
248
|
-
- `--version`, `-v`: show version information
|
|
253
|
+
- `--version`, `-v`: show version information. Place this before the subcommand: `parq --version`
|
|
249
254
|
- `--output`, `-o`: select output format (`rich` | `plain` | `json`)
|
|
250
255
|
- `--delimiter`, `-d`: field delimiter for CSV/TSV input (default: `,`); `.tsv` files default to `\t` automatically
|
|
251
|
-
- `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet)
|
|
256
|
+
- `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet). A value made only of digits is an index, including a sheet whose name is `1` or `2`
|
|
252
257
|
- `--help`: show command help
|
|
253
258
|
|
|
259
|
+
`--output`, `--delimiter`, and `--sheet` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
|
|
260
|
+
|
|
254
261
|
Available output modes:
|
|
255
262
|
|
|
256
263
|
- `rich`: human-friendly terminal rendering
|
|
@@ -273,9 +280,11 @@ On Windows terminals that cannot safely render emoji or extended characters, Ric
|
|
|
273
280
|
|
|
274
281
|
- Parquet metadata, row counts, and previews use Arrow metadata and row-group shortcuts where available.
|
|
275
282
|
- CSV `tail` uses a fixed-size column window instead of materializing every row as Python dicts.
|
|
276
|
-
- CSV
|
|
283
|
+
- CSV `split --record-count` and CSV `convert` stream in one pass. Their progress bars stay indeterminate because the row count is not known ahead of time.
|
|
277
284
|
- `meta --fast` is the best option when you need quick metadata from large CSV/XLSX inputs.
|
|
278
|
-
-
|
|
285
|
+
- `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, and `merge` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
|
|
286
|
+
- `split --record-count` keeps the progress bar indeterminate while it writes.
|
|
287
|
+
- `diff` indexes at most 100,000 rows per side in memory, then hash-partitions larger inputs into temporary Parquet files.
|
|
279
288
|
|
|
280
289
|
For repeated heavy workflows, converting large CSV/XLSX files to Parquet is still the best path for throughput.
|
|
281
290
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# parq-cli
|
|
2
2
|
|
|
3
|
-
[](https://www.python.org/downloads/)
|
|
4
4
|
[](LICENSE)
|
|
5
5
|
|
|
6
6
|
一个用于查看、转换、对比和处理表格文件的命令行工具。
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
## 简介
|
|
11
11
|
|
|
12
|
-
`parq` 面向日常最常见的表格文件操作,支持 `.parquet`、`.csv`、`.xlsx`:
|
|
12
|
+
`parq` 面向日常最常见的表格文件操作,支持 `.parquet`、`.csv`、`.tsv`、`.xlsx`:
|
|
13
13
|
|
|
14
14
|
- 查看元数据和 schema
|
|
15
15
|
- 预览前几行或后几行
|
|
@@ -73,19 +73,19 @@ parq merge part-001.parquet part-002.parquet merged.parquet
|
|
|
73
73
|
|
|
74
74
|
## 支持的格式
|
|
75
75
|
|
|
76
|
-
| 命令 | Parquet | CSV | XLSX |
|
|
77
|
-
| --- | --- | --- | --- |
|
|
78
|
-
| `meta` | 支持 | 支持 | 支持 |
|
|
79
|
-
| `schema` | 支持 | 支持 | 支持 |
|
|
80
|
-
| `head` / `tail` | 支持 | 支持 | 支持 |
|
|
81
|
-
| `count` | 支持 | 支持 | 支持 |
|
|
82
|
-
| `split` | 支持 | 支持 | 支持 |
|
|
83
|
-
| `stats` | 支持 | 支持 | 支持 |
|
|
84
|
-
| `convert` | 支持 | 支持 | 支持 |
|
|
85
|
-
| `diff` | 支持 | 支持 | 先转换再对比 |
|
|
86
|
-
| `merge` | 支持 | 支持 | 支持 |
|
|
76
|
+
| 命令 | Parquet | CSV | TSV | XLSX |
|
|
77
|
+
| --- | --- | --- | --- | --- |
|
|
78
|
+
| `meta` | 支持 | 支持 | 支持 | 支持 |
|
|
79
|
+
| `schema` | 支持 | 支持 | 支持 | 支持 |
|
|
80
|
+
| `head` / `tail` | 支持 | 支持 | 支持 | 支持 |
|
|
81
|
+
| `count` | 支持 | 支持 | 支持 | 支持 |
|
|
82
|
+
| `split` | 支持 | 支持 | 支持 | 支持 |
|
|
83
|
+
| `stats` | 支持 | 支持 | 支持 | 支持 |
|
|
84
|
+
| `convert` | 支持 | 支持 | 支持 | 支持 |
|
|
85
|
+
| `diff` | 支持 | 支持 | 支持 | 先转换再对比 |
|
|
86
|
+
| `merge` | 支持 | 支持 | 支持 | 支持 |
|
|
87
87
|
|
|
88
|
-
`XLSX`
|
|
88
|
+
`XLSX` 的读取使用 `python-calamine`,写出仍使用 `openpyxl`,两者都随 `parq-cli[xlsx]` 安装。公式单元格读取的是 Excel 已经算出的缓存值,没有缓存的公式会是空值。`.tsv` 按扩展名自动使用制表符;其它分隔符用 `--delimiter`。`--sheet` 选择 XLSX 工作表,对 `merge` 同样生效。
|
|
89
89
|
|
|
90
90
|
## 命令说明
|
|
91
91
|
|
|
@@ -149,9 +149,9 @@ parq split FILE --record-count 100000 -n "chunks/part-%03d.parquet"
|
|
|
149
149
|
规则:
|
|
150
150
|
|
|
151
151
|
- `--file-count` 与 `--record-count` 二选一
|
|
152
|
-
- 输出格式由 `--name-format`
|
|
153
|
-
-
|
|
154
|
-
- 在 `--record-count` 模式下,CSV
|
|
152
|
+
- 输出格式由 `--name-format` 的后缀决定,包括 `.tsv`
|
|
153
|
+
- 默认遇到已存在的目标文件会报错;`--force` / `-F` 会覆盖
|
|
154
|
+
- 在 `--record-count` 模式下,CSV 单遍流式写出,进度条保持不确定状态。XLSX 会先把工作表解析进内存,写出时进度条同样保持不确定状态
|
|
155
155
|
|
|
156
156
|
### `stats`
|
|
157
157
|
|
|
@@ -164,7 +164,9 @@ parq stats FILE --limit 20
|
|
|
164
164
|
计算按列汇总的轻量统计:
|
|
165
165
|
|
|
166
166
|
- 数值列输出 `count`、`null_count`、`min`、`max`、`mean`
|
|
167
|
-
-
|
|
167
|
+
- 字符串、布尔和时间列另外输出 `cardinality` 和 `top_values`
|
|
168
|
+
- 默认 `--top-n` 是 `5`;设为 `0` 则不输出 top values
|
|
169
|
+
- 不同值超过 1 万个时,基数和 top values 仍然是精确结果;空值只计入 `null_count`
|
|
168
170
|
- 默认 `--limit 50`,避免宽表直接刷满终端
|
|
169
171
|
|
|
170
172
|
### `convert`
|
|
@@ -178,9 +180,9 @@ parq convert SOURCE OUTPUT --columns id,name,status
|
|
|
178
180
|
|
|
179
181
|
说明:
|
|
180
182
|
|
|
181
|
-
- 当前目标格式为 `.parquet`、`.csv`、`.xlsx`
|
|
183
|
+
- 当前目标格式为 `.parquet`、`.csv`、`.tsv`、`.xlsx`
|
|
182
184
|
- 能流式处理的路径会尽量流式处理
|
|
183
|
-
-
|
|
185
|
+
- 默认遇到已存在的输出文件会报错;`--force` / `-F` 会覆盖
|
|
184
186
|
|
|
185
187
|
### `diff`
|
|
186
188
|
|
|
@@ -201,9 +203,11 @@ parq diff LEFT RIGHT --key id --summary-only
|
|
|
201
203
|
说明:
|
|
202
204
|
|
|
203
205
|
- `--key` 必填
|
|
204
|
-
- `diff`
|
|
206
|
+
- `diff` 当前支持 Parquet、CSV 和 TSV
|
|
207
|
+
- 全局 `--delimiter` 会同时用于左右两个 CSV 输入
|
|
205
208
|
- XLSX 需要先转换
|
|
206
209
|
- 任一侧若出现重复 key,会直接报错
|
|
210
|
+
- 单侧超过 10 万行时,会把数据按主键散列到临时文件,而不是把全部 key 放进内存
|
|
207
211
|
- `--summary-only` 只保留计数,不输出样本内容
|
|
208
212
|
|
|
209
213
|
### `merge`
|
|
@@ -218,17 +222,22 @@ parq merge chunks/*.parquet merged.parquet
|
|
|
218
222
|
说明:
|
|
219
223
|
|
|
220
224
|
- schema 必须一致,或能被 Arrow 安全统一
|
|
221
|
-
-
|
|
222
|
-
-
|
|
225
|
+
- 默认遇到已存在的输出文件会报错;`--force` / `-F` 会覆盖
|
|
226
|
+
- 输出格式由输出文件后缀决定,包括 `.tsv`
|
|
227
|
+
- 全局 `--delimiter` 和 `--sheet` 会作用于每一个输入
|
|
223
228
|
|
|
224
229
|
## 输出模式
|
|
225
230
|
|
|
226
231
|
全局选项:
|
|
227
232
|
|
|
228
|
-
- `--version`, `-v
|
|
229
|
-
- `--output`, `-o
|
|
233
|
+
- `--version`, `-v`:显示版本号。放在子命令前面:`parq --version`
|
|
234
|
+
- `--output`, `-o`:选择输出格式(`rich` | `plain` | `json`)
|
|
235
|
+
- `--delimiter`, `-d`:CSV/TSV 输入分隔符,默认 `,`;`.tsv` 会自动使用制表符
|
|
236
|
+
- `--sheet`:XLSX 工作表名称或从 0 开始的序号,默认活动工作表。纯数字会按序号处理,名叫 `1` 或 `2` 的工作表也一样
|
|
230
237
|
- `--help`:显示帮助
|
|
231
238
|
|
|
239
|
+
`--output`、`--delimiter`、`--sheet` 可以写在子命令前面,也可以写在子命令上面。同一个选项两处都写时,子命令上的值生效。`convert` 的 `--output` / `-o` 仍然是显示格式,目标文件是位置参数 `OUTPUT`。
|
|
240
|
+
|
|
232
241
|
可选输出模式:
|
|
233
242
|
|
|
234
243
|
- `rich`:适合人在终端中阅读
|
|
@@ -249,9 +258,9 @@ parq diff left.parquet right.parquet --key id --summary-only --output json
|
|
|
249
258
|
|
|
250
259
|
- Parquet 的元数据、行数和预览会优先利用 Arrow metadata 与 row group 能力。
|
|
251
260
|
- CSV `tail` 使用固定大小列窗口,而不是把整份数据都转成 Python dict。
|
|
252
|
-
- CSV
|
|
261
|
+
- CSV 的 `split --record-count` 和 `convert` 单遍流式处理,进度条保持不确定状态,因为行数要等这一遍结束才知道。
|
|
253
262
|
- 对大 CSV/XLSX,`meta --fast` 是最快的元数据探查入口。
|
|
254
|
-
-
|
|
263
|
+
- `parq schema` 和较短的 `head` 对 XLSX 只采样前 1000 行。`count`、`tail`、`stats`、`convert`、`split`、`merge` 把选中的工作表解析一次,后续值放不进样本类型时放宽该列,并把这些行留在内存里复用。
|
|
255
264
|
|
|
256
265
|
如果你会反复处理大文件,把 CSV/XLSX 先转成 Parquet 依然是吞吐量最好的路径。
|
|
257
266
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
### 1. 克隆项目
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
git clone
|
|
8
|
+
git clone https://github.com/Tendo33/parq-cli.git
|
|
9
9
|
cd parq-cli
|
|
10
10
|
```
|
|
11
11
|
|
|
@@ -118,18 +118,15 @@ ruff check parq tests
|
|
|
118
118
|
ruff check --fix parq tests
|
|
119
119
|
```
|
|
120
120
|
|
|
121
|
-
###
|
|
121
|
+
### 检查门
|
|
122
122
|
|
|
123
|
-
|
|
124
|
-
pip install mypy
|
|
125
|
-
mypy parq
|
|
126
|
-
```
|
|
123
|
+
仓库的检查门是 Ruff 和 pytest。当前没有配置 mypy,CI 也不运行它。
|
|
127
124
|
|
|
128
125
|
## 📋 常见问题
|
|
129
126
|
|
|
130
127
|
### Q: 如何处理大文件?
|
|
131
128
|
|
|
132
|
-
A: Parquet
|
|
129
|
+
A: Parquet 用文件元数据和 row group,不必先读完全部数据。CSV 按批次流式处理。XLSX 的 `schema` 和较短的 `head` 只采样前 1000 行;`count`、`tail`、`stats`、`convert`、`split`、`merge` 会把选中的工作表解析一次并放在内存里复用。反复处理大文件时,先转成 Parquet 吞吐更好。
|
|
133
130
|
|
|
134
131
|
### Q: 空文件会怎么处理?
|
|
135
132
|
|
|
@@ -83,7 +83,7 @@ python scripts/bump_version.py --version 0.1.1 --yes
|
|
|
83
83
|
|
|
84
84
|
**检查步骤:**
|
|
85
85
|
1. 查看 Actions 日志: https://github.com/Tendo33/parq-cli/actions
|
|
86
|
-
2.
|
|
86
|
+
2. 确认仓库 Secret `PYPI_API_TOKEN` 仍然有效
|
|
87
87
|
3. 运行本地检查: `python scripts/check_version.py`
|
|
88
88
|
|
|
89
89
|
## 🎯 手动发布 (不推荐)
|
|
@@ -114,6 +114,10 @@ git push origin v0.1.2
|
|
|
114
114
|
- **PyPI 页面:** https://pypi.org/project/parq-cli/
|
|
115
115
|
- **GitHub Releases:** https://github.com/Tendo33/parq-cli/releases
|
|
116
116
|
|
|
117
|
+
## PyPI 上传
|
|
118
|
+
|
|
119
|
+
发布 job 在 `v*` tag 上使用仓库 Secret `PYPI_API_TOKEN` 上传。从普通分支手动运行会在上传前失败。GitHub 上已经有一个名为 `pypi` 的 environment,当前 workflow 不依赖它。
|
|
120
|
+
|
|
117
121
|
## 💡 最佳实践
|
|
118
122
|
|
|
119
123
|
1. **发布前测试:** 始终运行 `./.venv/bin/pytest -m "not performance"` 确保测试通过
|
|
@@ -124,5 +128,5 @@ git push origin v0.1.2
|
|
|
124
128
|
|
|
125
129
|
---
|
|
126
130
|
|
|
127
|
-
**上次更新:**
|
|
131
|
+
**上次更新:** 2026-10-10
|
|
128
132
|
**维护者:** SimonSun (@Tendo33)
|