parq-cli 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. parq_cli-0.2.2/.claude/settings.json +6 -0
  2. {parq_cli-0.2.0 → parq_cli-0.2.2}/.github/workflows/publish.yml +12 -7
  3. {parq_cli-0.2.0 → parq_cli-0.2.2}/.github/workflows/test.yml +1 -1
  4. {parq_cli-0.2.0 → parq_cli-0.2.2}/.gitignore +3 -0
  5. parq_cli-0.2.2/AGENTS.md +15 -0
  6. parq_cli-0.2.2/CHANGELOG.md +26 -0
  7. parq_cli-0.2.2/CLAUDE.md +24 -0
  8. {parq_cli-0.2.0 → parq_cli-0.2.2}/PKG-INFO +27 -17
  9. {parq_cli-0.2.0 → parq_cli-0.2.2}/README.md +20 -11
  10. {parq_cli-0.2.0 → parq_cli-0.2.2}/README_CN.md +36 -27
  11. {parq_cli-0.2.0 → parq_cli-0.2.2}/docs/QUICKSTART.md +4 -7
  12. {parq_cli-0.2.0 → parq_cli-0.2.2}/docs/RELEASE.md +6 -2
  13. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/__init__.py +1 -1
  14. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/cli.py +172 -35
  15. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/__init__.py +1 -2
  16. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_chunk_writers.py +15 -11
  17. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_common.py +12 -62
  18. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_parquet.py +10 -0
  19. parq_cli-0.2.2/parq/formats/_xlsx.py +337 -0
  20. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/output.py +40 -9
  21. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/plain_output.py +43 -6
  22. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/reader.py +429 -77
  23. {parq_cli-0.2.0 → parq_cli-0.2.2}/pyproject.toml +7 -6
  24. {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/README.md +1 -1
  25. {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/performance_benchmark.py +3 -1
  26. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_cli.py +223 -9
  27. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_output.py +33 -0
  28. parq_cli-0.2.2/tests/test_output_not_input.py +26 -0
  29. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_plain_output.py +19 -3
  30. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_reader.py +203 -16
  31. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_release_scripts.py +1 -1
  32. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_workflows.py +11 -0
  33. parq_cli-0.2.2/tests/test_xlsx_defused.py +31 -0
  34. {parq_cli-0.2.0 → parq_cli-0.2.2}/uv.lock +226 -347
  35. parq_cli-0.2.0/AGENTS.md +0 -46
  36. parq_cli-0.2.0/parq/formats/_xlsx.py +0 -205
  37. {parq_cli-0.2.0 → parq_cli-0.2.2}/.python-version +0 -0
  38. {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/csv-large.csv +0 -0
  39. {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/csv-small.csv +0 -0
  40. {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/parquet-large.parquet +0 -0
  41. {parq_cli-0.2.0 → parq_cli-0.2.2}/.tmp-bench/parquet-small.parquet +0 -0
  42. {parq_cli-0.2.0 → parq_cli-0.2.2}/LICENSE +0 -0
  43. {parq_cli-0.2.0 → parq_cli-0.2.2}/data/test.parquet +0 -0
  44. {parq_cli-0.2.0 → parq_cli-0.2.2}/examples/README.md +0 -0
  45. {parq_cli-0.2.0 → parq_cli-0.2.2}/examples/create_sample_data.py +0 -0
  46. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/__main__.py +0 -0
  47. {parq_cli-0.2.0 → parq_cli-0.2.2}/parq/formats/_csv.py +0 -0
  48. {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/bump_version.py +0 -0
  49. {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/check_version.py +0 -0
  50. {parq_cli-0.2.0 → parq_cli-0.2.2}/scripts/publish.sh +0 -0
  51. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/__init__.py +0 -0
  52. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/conftest.py +0 -0
  53. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_main_entry.py +0 -0
  54. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_package_meta.py +0 -0
  55. {parq_cli-0.2.0 → parq_cli-0.2.2}/tests/test_performance.py +0 -0
@@ -0,0 +1,6 @@
1
+ {
2
+ "env": {
3
+ "CLAUDE_BASH_MAINTAIN_PROJECT_WORKING_DIR": "1"
4
+ },
5
+ "enabledPlugins": {}
6
+ }
@@ -35,8 +35,6 @@ jobs:
35
35
  build-and-publish:
36
36
  needs: pre-publish-tests
37
37
  runs-on: ubuntu-latest
38
- permissions:
39
- id-token: write
40
38
 
41
39
  steps:
42
40
  - name: 📥 Checkout code
@@ -68,10 +66,17 @@ jobs:
68
66
  path: dist/*
69
67
 
70
68
  - name: 🚀 Publish to PyPI
71
- env:
72
- TWINE_USERNAME: __token__
73
- TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
74
- run: twine upload dist/*
69
+ if: startsWith(github.ref, 'refs/tags/v')
70
+ uses: pypa/gh-action-pypi-publish@release/v1
71
+ with:
72
+ user: __token__
73
+ password: ${{ secrets.PYPI_API_TOKEN }}
74
+
75
+ - name: ⛔ Refuse publish outside a version tag
76
+ if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
77
+ run: |
78
+ echo "Refusing to publish: PyPI uploads only run from a v* tag."
79
+ exit 1
75
80
 
76
81
  create-github-release:
77
82
  needs: build-and-publish
@@ -87,7 +92,7 @@ jobs:
87
92
  path: dist
88
93
 
89
94
  - name: 📝 Create GitHub Release
90
- uses: softprops/action-gh-release@v1
95
+ uses: softprops/action-gh-release@v2
91
96
  with:
92
97
  files: dist/*
93
98
  generate_release_notes: true
@@ -12,7 +12,7 @@ jobs:
12
12
  strategy:
13
13
  matrix:
14
14
  os: [ubuntu-latest, windows-latest, macos-latest]
15
- python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
15
+ python-version: ['3.10', '3.11', '3.12', '3.13']
16
16
 
17
17
  steps:
18
18
  - name: 📥 Checkout code
@@ -210,3 +210,6 @@ project_document/
210
210
  .history/
211
211
  .worktrees/
212
212
  worktrees/
213
+
214
+ # macOS
215
+ .DS_Store
@@ -0,0 +1,15 @@
1
+ # Project Agent Entrypoint
2
+
3
+ This file is the cross-tool entrypoint for parq-cli.
4
+
5
+ ## Working Rules
6
+
7
+ - parq-cli is a pure Python CLI package, not a full-stack or frontend project.
8
+ - Keep Python on `>=3.10`.
9
+ - Preserve Typer command contracts, rich/plain/json output modes, and streaming behavior for large CSV/XLSX workflows.
10
+
11
+ ## Execution Style
12
+
13
+ - Read `parq/cli.py`, `parq/reader.py`, relevant `parq/formats/*`, output formatters, and tests before editing.
14
+ - Keep changes small and covered by CLI/reader/output tests.
15
+ - Run focused tests first, then the repository verification gate: `pytest -m "not performance"` and `ruff check parq tests`.
@@ -0,0 +1,26 @@
1
+ # Changelog
2
+
3
+ ## 0.2.2
4
+
5
+ - `merge --force` and `convert` now refuse an output path that is one of the inputs. Previously `--force` deleted that input before reading it.
6
+ - XLSX workbook XML is parsed with `defusedxml` (added to the `xlsx` extra).
7
+ - Table output escapes cell markup and summarizes binary values (#4).
8
+ - Errors include the file path (#5).
9
+ - Dependency security updates: certifi, urllib3, idna, pygments, cryptography, pyarrow.
10
+
11
+ ## 0.2.1
12
+
13
+ - `diff` and `merge` honor the global `--delimiter` and `--sheet` options.
14
+ - `.tsv` is a supported output format for `convert`, `split`, and `merge`.
15
+ - XLSX `schema` still samples the first 1000 rows. Full reads widen a column when a later row does not fit that sample.
16
+ - String cardinality and top values stay exact past 10,000 distinct values. Nulls are not counted as empty strings.
17
+ - `diff` spills inputs larger than 100,000 rows per side into temporary files.
18
+ - `split --record-count` progress stays indeterminate while the total is still unknown.
19
+ - PyPI uploads run from a `v*` tag and use the repository `PYPI_API_TOKEN`.
20
+ - XLSX reads use `python-calamine`. A full read parses the workbook once and reuses those rows.
21
+ - CSV `convert` no longer counts every row before writing when a progress bar is shown.
22
+ - `--output`, `--delimiter`, and `--sheet` work before a subcommand and on the subcommand. The value on the subcommand wins when both are set.
23
+
24
+ ## 0.2.0
25
+
26
+ - Inspect, split, convert, diff, and merge Parquet, CSV, TSV, and XLSX files.
@@ -0,0 +1,24 @@
1
+ # Claude Project Entrypoint
2
+
3
+
4
+ ## Guardrails
5
+
6
+ - parq-cli is a Python command-line tool for Parquet, CSV, TSV, and XLSX files.
7
+ - There is no frontend layer.
8
+ - Package baseline is Python `>=3.10`.
9
+ - Default tests exclude performance scenarios; run performance tests only when relevant.
10
+ - Keep `parq` CLI output stable for automation, especially `plain` and `json` output modes.
11
+
12
+ ## Commands
13
+
14
+ ```bash
15
+ pytest -m "not performance"
16
+ ruff check parq tests
17
+ pytest tests/test_performance.py -m performance -q -s
18
+ ```
19
+
20
+ ## Gotchas
21
+
22
+ - `parq schema` on XLSX samples the first 1000 rows. Full reads parse the sheet once with python-calamine and widen a column when a later value does not fit. Writing XLSX still uses openpyxl.
23
+ - `diff` and `merge` use the global `--delimiter` and `--sheet`. `diff` keeps at most 100,000 rows per side in memory, then spills to temporary files.
24
+ - `.tsv` is valid for both input and output. Output uses a tab delimiter.
@@ -1,7 +1,7 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: parq-cli
3
- Version: 0.2.0
4
- Summary: A powerful command-line tool for inspecting tabular files like Parquet, CSV, and XLSX
3
+ Version: 0.2.2
4
+ Summary: A command-line tool for inspecting tabular files: Parquet, CSV, TSV, and XLSX
5
5
  Project-URL: Homepage, https://github.com/Tendo33/parq-cli
6
6
  Project-URL: Repository, https://github.com/Tendo33/parq-cli
7
7
  Project-URL: Issues, https://github.com/Tendo33/parq-cli/issues
@@ -14,12 +14,11 @@ Classifier: Development Status :: 3 - Alpha
14
14
  Classifier: Intended Audience :: Developers
15
15
  Classifier: License :: OSI Approved :: MIT License
16
16
  Classifier: Programming Language :: Python :: 3
17
- Classifier: Programming Language :: Python :: 3.9
18
17
  Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
20
19
  Classifier: Programming Language :: Python :: 3.12
21
20
  Classifier: Programming Language :: Python :: 3.13
22
- Requires-Python: >=3.9
21
+ Requires-Python: >=3.10
23
22
  Requires-Dist: pyarrow>=18.0.0
24
23
  Requires-Dist: rich>=13.0.0
25
24
  Requires-Dist: typer>=0.15.0
@@ -27,16 +26,18 @@ Provides-Extra: dev
27
26
  Requires-Dist: build>=1.2.0; extra == 'dev'
28
27
  Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
29
28
  Requires-Dist: pytest>=8.0.0; extra == 'dev'
30
- Requires-Dist: ruff>=0.7.0; extra == 'dev'
29
+ Requires-Dist: ruff==0.14.0; extra == 'dev'
31
30
  Requires-Dist: twine>=5.1.0; extra == 'dev'
32
31
  Requires-Dist: vulture>=2.14; extra == 'dev'
33
32
  Provides-Extra: xlsx
33
+ Requires-Dist: defusedxml>=0.7.1; extra == 'xlsx'
34
34
  Requires-Dist: openpyxl>=3.1.0; extra == 'xlsx'
35
+ Requires-Dist: python-calamine>=0.8.0; extra == 'xlsx'
35
36
  Description-Content-Type: text/markdown
36
37
 
37
38
  # parq-cli
38
39
 
39
- [![Python Version](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/downloads/)
40
+ [![Python Version](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
40
41
  [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
41
42
 
42
43
  A command-line tool for inspecting, transforming, and comparing tabular files.
@@ -133,7 +134,7 @@ parq merge chunks/*.parquet merged.parquet --force # overwrite if exists
133
134
  | `diff` | yes | yes | yes | no, convert first |
134
135
  | `merge` | yes | yes | yes | yes |
135
136
 
136
- `XLSX` support requires `openpyxl`. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
137
+ `XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
137
138
 
138
139
  ## Command Reference
139
140
 
@@ -200,8 +201,8 @@ Rules:
200
201
  - specify exactly one of `--file-count` or `--record-count`
201
202
  - output format is inferred from `--name-format`
202
203
  - by default, existing target files raise an error; use `--force` / `-F` to overwrite
203
- - in `--record-count` mode, CSV/XLSX now stream in a single pass instead of pre-counting the entire file
204
- - a live progress bar is shown during the split
204
+ - in `--record-count` mode, CSV streams in one pass and the progress bar stays indeterminate
205
+ - XLSX is parsed once before writing; the record-count progress bar stays indeterminate
205
206
 
206
207
  ### `stats`
207
208
 
@@ -218,6 +219,8 @@ Computes simple per-column statistics.
218
219
  - string, boolean, and date columns additionally include `cardinality` and `top_values` (top N most frequent values with their occurrence counts)
219
220
  - default `--top-n` is `5`; set to `0` to suppress top-values output entirely
220
221
  - default `--limit` is `50` to avoid flooding the terminal on very wide tables
222
+ - cardinality and top values are exact, including columns with more than 10,000 distinct values
223
+ - nulls stay in `null_count` and are not treated as empty strings
221
224
 
222
225
  ### `convert`
223
226
 
@@ -233,7 +236,7 @@ Notes:
233
236
 
234
237
  - current targets are `.parquet`, `.csv`, `.tsv`, and `.xlsx`
235
238
  - conversion is streaming-based where possible
236
- - a live progress bar is shown during the conversion
239
+ - a live progress bar is shown during the conversion; CSV stays indeterminate because the row count is not known ahead of time
237
240
  - by default, existing output files raise an error; use `--force` / `-F` to overwrite
238
241
 
239
242
  ### `diff`
@@ -255,9 +258,11 @@ Compares two datasets by key and reports:
255
258
  Notes:
256
259
 
257
260
  - `--key` is required
258
- - `diff` currently supports Parquet and CSV inputs
261
+ - `diff` currently supports Parquet, CSV, and TSV inputs
262
+ - the global `--delimiter` applies to both CSV inputs
259
263
  - XLSX files should be converted first
260
264
  - duplicate keys on either side are treated as an error
265
+ - inputs larger than 100,000 rows on a side spill into temporary files instead of holding every key in memory
261
266
  - `--summary-only` keeps the counts and omits sample payloads
262
267
 
263
268
  ### `merge`
@@ -274,19 +279,22 @@ Notes:
274
279
 
275
280
  - schemas must be identical or safely unifiable by Arrow
276
281
  - by default, existing output files raise an error; use `--force` / `-F` to overwrite
277
- - output format is inferred from the output suffix
282
+ - output format is inferred from the output suffix, including `.tsv`
283
+ - the global `--delimiter` and `--sheet` apply to every input
278
284
  - a live progress bar is shown during the merge
279
285
 
280
286
  ## Output Modes
281
287
 
282
288
  Global options:
283
289
 
284
- - `--version`, `-v`: show version information
290
+ - `--version`, `-v`: show version information. Place this before the subcommand: `parq --version`
285
291
  - `--output`, `-o`: select output format (`rich` | `plain` | `json`)
286
292
  - `--delimiter`, `-d`: field delimiter for CSV/TSV input (default: `,`); `.tsv` files default to `\t` automatically
287
- - `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet)
293
+ - `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet). A value made only of digits is an index, including a sheet whose name is `1` or `2`
288
294
  - `--help`: show command help
289
295
 
296
+ `--output`, `--delimiter`, and `--sheet` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
297
+
290
298
  Available output modes:
291
299
 
292
300
  - `rich`: human-friendly terminal rendering
@@ -309,9 +317,11 @@ On Windows terminals that cannot safely render emoji or extended characters, Ric
309
317
 
310
318
  - Parquet metadata, row counts, and previews use Arrow metadata and row-group shortcuts where available.
311
319
  - CSV `tail` uses a fixed-size column window instead of materializing every row as Python dicts.
312
- - CSV/XLSX `split --record-count` streams in one pass.
320
+ - CSV `split --record-count` and CSV `convert` stream in one pass. Their progress bars stay indeterminate because the row count is not known ahead of time.
313
321
  - `meta --fast` is the best option when you need quick metadata from large CSV/XLSX inputs.
314
- - XLSX schema inference samples the first 1000 rows instead of scanning the entire sheet up front.
322
+ - `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, and `merge` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
323
+ - `split --record-count` keeps the progress bar indeterminate while it writes.
324
+ - `diff` indexes at most 100,000 rows per side in memory, then hash-partitions larger inputs into temporary Parquet files.
315
325
 
316
326
  For repeated heavy workflows, converting large CSV/XLSX files to Parquet is still the best path for throughput.
317
327
 
@@ -1,6 +1,6 @@
1
1
  # parq-cli
2
2
 
3
- [![Python Version](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/downloads/)
3
+ [![Python Version](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
4
4
  [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
5
5
 
6
6
  A command-line tool for inspecting, transforming, and comparing tabular files.
@@ -97,7 +97,7 @@ parq merge chunks/*.parquet merged.parquet --force # overwrite if exists
97
97
  | `diff` | yes | yes | yes | no, convert first |
98
98
  | `merge` | yes | yes | yes | yes |
99
99
 
100
- `XLSX` support requires `openpyxl`. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
100
+ `XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
101
101
 
102
102
  ## Command Reference
103
103
 
@@ -164,8 +164,8 @@ Rules:
164
164
  - specify exactly one of `--file-count` or `--record-count`
165
165
  - output format is inferred from `--name-format`
166
166
  - by default, existing target files raise an error; use `--force` / `-F` to overwrite
167
- - in `--record-count` mode, CSV/XLSX now stream in a single pass instead of pre-counting the entire file
168
- - a live progress bar is shown during the split
167
+ - in `--record-count` mode, CSV streams in one pass and the progress bar stays indeterminate
168
+ - XLSX is parsed once before writing; the record-count progress bar stays indeterminate
169
169
 
170
170
  ### `stats`
171
171
 
@@ -182,6 +182,8 @@ Computes simple per-column statistics.
182
182
  - string, boolean, and date columns additionally include `cardinality` and `top_values` (top N most frequent values with their occurrence counts)
183
183
  - default `--top-n` is `5`; set to `0` to suppress top-values output entirely
184
184
  - default `--limit` is `50` to avoid flooding the terminal on very wide tables
185
+ - cardinality and top values are exact, including columns with more than 10,000 distinct values
186
+ - nulls stay in `null_count` and are not treated as empty strings
185
187
 
186
188
  ### `convert`
187
189
 
@@ -197,7 +199,7 @@ Notes:
197
199
 
198
200
  - current targets are `.parquet`, `.csv`, `.tsv`, and `.xlsx`
199
201
  - conversion is streaming-based where possible
200
- - a live progress bar is shown during the conversion
202
+ - a live progress bar is shown during the conversion; CSV stays indeterminate because the row count is not known ahead of time
201
203
  - by default, existing output files raise an error; use `--force` / `-F` to overwrite
202
204
 
203
205
  ### `diff`
@@ -219,9 +221,11 @@ Compares two datasets by key and reports:
219
221
  Notes:
220
222
 
221
223
  - `--key` is required
222
- - `diff` currently supports Parquet and CSV inputs
224
+ - `diff` currently supports Parquet, CSV, and TSV inputs
225
+ - the global `--delimiter` applies to both CSV inputs
223
226
  - XLSX files should be converted first
224
227
  - duplicate keys on either side are treated as an error
228
+ - inputs larger than 100,000 rows on a side spill into temporary files instead of holding every key in memory
225
229
  - `--summary-only` keeps the counts and omits sample payloads
226
230
 
227
231
  ### `merge`
@@ -238,19 +242,22 @@ Notes:
238
242
 
239
243
  - schemas must be identical or safely unifiable by Arrow
240
244
  - by default, existing output files raise an error; use `--force` / `-F` to overwrite
241
- - output format is inferred from the output suffix
245
+ - output format is inferred from the output suffix, including `.tsv`
246
+ - the global `--delimiter` and `--sheet` apply to every input
242
247
  - a live progress bar is shown during the merge
243
248
 
244
249
  ## Output Modes
245
250
 
246
251
  Global options:
247
252
 
248
- - `--version`, `-v`: show version information
253
+ - `--version`, `-v`: show version information. Place this before the subcommand: `parq --version`
249
254
  - `--output`, `-o`: select output format (`rich` | `plain` | `json`)
250
255
  - `--delimiter`, `-d`: field delimiter for CSV/TSV input (default: `,`); `.tsv` files default to `\t` automatically
251
- - `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet)
256
+ - `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet). A value made only of digits is an index, including a sheet whose name is `1` or `2`
252
257
  - `--help`: show command help
253
258
 
259
+ `--output`, `--delimiter`, and `--sheet` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
260
+
254
261
  Available output modes:
255
262
 
256
263
  - `rich`: human-friendly terminal rendering
@@ -273,9 +280,11 @@ On Windows terminals that cannot safely render emoji or extended characters, Ric
273
280
 
274
281
  - Parquet metadata, row counts, and previews use Arrow metadata and row-group shortcuts where available.
275
282
  - CSV `tail` uses a fixed-size column window instead of materializing every row as Python dicts.
276
- - CSV/XLSX `split --record-count` streams in one pass.
283
+ - CSV `split --record-count` and CSV `convert` stream in one pass. Their progress bars stay indeterminate because the row count is not known ahead of time.
277
284
  - `meta --fast` is the best option when you need quick metadata from large CSV/XLSX inputs.
278
- - XLSX schema inference samples the first 1000 rows instead of scanning the entire sheet up front.
285
+ - `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, and `merge` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
286
+ - `split --record-count` keeps the progress bar indeterminate while it writes.
287
+ - `diff` indexes at most 100,000 rows per side in memory, then hash-partitions larger inputs into temporary Parquet files.
279
288
 
280
289
  For repeated heavy workflows, converting large CSV/XLSX files to Parquet is still the best path for throughput.
281
290
 
@@ -1,6 +1,6 @@
1
1
  # parq-cli
2
2
 
3
- [![Python Version](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/downloads/)
3
+ [![Python Version](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
4
4
  [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
5
5
 
6
6
  一个用于查看、转换、对比和处理表格文件的命令行工具。
@@ -9,7 +9,7 @@
9
9
 
10
10
  ## 简介
11
11
 
12
- `parq` 面向日常最常见的表格文件操作,支持 `.parquet`、`.csv`、`.xlsx`:
12
+ `parq` 面向日常最常见的表格文件操作,支持 `.parquet`、`.csv`、`.tsv`、`.xlsx`:
13
13
 
14
14
  - 查看元数据和 schema
15
15
  - 预览前几行或后几行
@@ -73,19 +73,19 @@ parq merge part-001.parquet part-002.parquet merged.parquet
73
73
 
74
74
  ## 支持的格式
75
75
 
76
- | 命令 | Parquet | CSV | XLSX |
77
- | --- | --- | --- | --- |
78
- | `meta` | 支持 | 支持 | 支持 |
79
- | `schema` | 支持 | 支持 | 支持 |
80
- | `head` / `tail` | 支持 | 支持 | 支持 |
81
- | `count` | 支持 | 支持 | 支持 |
82
- | `split` | 支持 | 支持 | 支持 |
83
- | `stats` | 支持 | 支持 | 支持 |
84
- | `convert` | 支持 | 支持 | 支持 |
85
- | `diff` | 支持 | 支持 | 先转换再对比 |
86
- | `merge` | 支持 | 支持 | 支持 |
76
+ | 命令 | Parquet | CSV | TSV | XLSX |
77
+ | --- | --- | --- | --- | --- |
78
+ | `meta` | 支持 | 支持 | 支持 | 支持 |
79
+ | `schema` | 支持 | 支持 | 支持 | 支持 |
80
+ | `head` / `tail` | 支持 | 支持 | 支持 | 支持 |
81
+ | `count` | 支持 | 支持 | 支持 | 支持 |
82
+ | `split` | 支持 | 支持 | 支持 | 支持 |
83
+ | `stats` | 支持 | 支持 | 支持 | 支持 |
84
+ | `convert` | 支持 | 支持 | 支持 | 支持 |
85
+ | `diff` | 支持 | 支持 | 支持 | 先转换再对比 |
86
+ | `merge` | 支持 | 支持 | 支持 | 支持 |
87
87
 
88
- `XLSX` 依赖 `openpyxl`。
88
+ `XLSX` 的读取使用 `python-calamine`,写出仍使用 `openpyxl`,两者都随 `parq-cli[xlsx]` 安装。公式单元格读取的是 Excel 已经算出的缓存值,没有缓存的公式会是空值。`.tsv` 按扩展名自动使用制表符;其它分隔符用 `--delimiter`。`--sheet` 选择 XLSX 工作表,对 `merge` 同样生效。
89
89
 
90
90
  ## 命令说明
91
91
 
@@ -149,9 +149,9 @@ parq split FILE --record-count 100000 -n "chunks/part-%03d.parquet"
149
149
  规则:
150
150
 
151
151
  - `--file-count` 与 `--record-count` 二选一
152
- - 输出格式由 `--name-format` 的后缀决定
153
- - 不会覆盖已存在的目标文件
154
- - 在 `--record-count` 模式下,CSV/XLSX 现在走单遍流式拆分,不再先整表计数
152
+ - 输出格式由 `--name-format` 的后缀决定,包括 `.tsv`
153
+ - 默认遇到已存在的目标文件会报错;`--force` / `-F` 会覆盖
154
+ - 在 `--record-count` 模式下,CSV 单遍流式写出,进度条保持不确定状态。XLSX 会先把工作表解析进内存,写出时进度条同样保持不确定状态
155
155
 
156
156
  ### `stats`
157
157
 
@@ -164,7 +164,9 @@ parq stats FILE --limit 20
164
164
  计算按列汇总的轻量统计:
165
165
 
166
166
  - 数值列输出 `count`、`null_count`、`min`、`max`、`mean`
167
- - 非数值列输出 `count` 和 `null_count`
167
+ - 字符串、布尔和时间列另外输出 `cardinality` 和 `top_values`
168
+ - 默认 `--top-n` 是 `5`;设为 `0` 则不输出 top values
169
+ - 不同值超过 1 万个时,基数和 top values 仍然是精确结果;空值只计入 `null_count`
168
170
  - 默认 `--limit 50`,避免宽表直接刷满终端
169
171
 
170
172
  ### `convert`
@@ -178,9 +180,9 @@ parq convert SOURCE OUTPUT --columns id,name,status
178
180
 
179
181
  说明:
180
182
 
181
- - 当前目标格式为 `.parquet`、`.csv`、`.xlsx`
183
+ - 当前目标格式为 `.parquet`、`.csv`、`.tsv`、`.xlsx`
182
184
  - 能流式处理的路径会尽量流式处理
183
- - 若输出文件已存在,会直接报错,不会覆盖
185
+ - 默认遇到已存在的输出文件会报错;`--force` / `-F` 会覆盖
184
186
 
185
187
  ### `diff`
186
188
 
@@ -201,9 +203,11 @@ parq diff LEFT RIGHT --key id --summary-only
201
203
  说明:
202
204
 
203
205
  - `--key` 必填
204
- - `diff` 当前仅支持 Parquet 和 CSV
206
+ - `diff` 当前支持 Parquet、CSV 和 TSV
207
+ - 全局 `--delimiter` 会同时用于左右两个 CSV 输入
205
208
  - XLSX 需要先转换
206
209
  - 任一侧若出现重复 key,会直接报错
210
+ - 单侧超过 10 万行时,会把数据按主键散列到临时文件,而不是把全部 key 放进内存
207
211
  - `--summary-only` 只保留计数,不输出样本内容
208
212
 
209
213
  ### `merge`
@@ -218,17 +222,22 @@ parq merge chunks/*.parquet merged.parquet
218
222
  说明:
219
223
 
220
224
  - schema 必须一致,或能被 Arrow 安全统一
221
- - 不会覆盖已存在的输出文件
222
- - 输出格式由输出文件后缀决定
225
+ - 默认遇到已存在的输出文件会报错;`--force` / `-F` 会覆盖
226
+ - 输出格式由输出文件后缀决定,包括 `.tsv`
227
+ - 全局 `--delimiter` 和 `--sheet` 会作用于每一个输入
223
228
 
224
229
  ## 输出模式
225
230
 
226
231
  全局选项:
227
232
 
228
- - `--version`, `-v`:显示版本号
229
- - `--output`, `-o`:选择输出格式
233
+ - `--version`, `-v`:显示版本号。放在子命令前面:`parq --version`
234
+ - `--output`, `-o`:选择输出格式(`rich` | `plain` | `json`)
235
+ - `--delimiter`, `-d`:CSV/TSV 输入分隔符,默认 `,`;`.tsv` 会自动使用制表符
236
+ - `--sheet`:XLSX 工作表名称或从 0 开始的序号,默认活动工作表。纯数字会按序号处理,名叫 `1` 或 `2` 的工作表也一样
230
237
  - `--help`:显示帮助
231
238
 
239
+ `--output`、`--delimiter`、`--sheet` 可以写在子命令前面,也可以写在子命令上面。同一个选项两处都写时,子命令上的值生效。`convert` 的 `--output` / `-o` 仍然是显示格式,目标文件是位置参数 `OUTPUT`。
240
+
232
241
  可选输出模式:
233
242
 
234
243
  - `rich`:适合人在终端中阅读
@@ -249,9 +258,9 @@ parq diff left.parquet right.parquet --key id --summary-only --output json
249
258
 
250
259
  - Parquet 的元数据、行数和预览会优先利用 Arrow metadata 与 row group 能力。
251
260
  - CSV `tail` 使用固定大小列窗口,而不是把整份数据都转成 Python dict。
252
- - CSV/XLSX 的 `split --record-count` 走单遍流式处理。
261
+ - CSV 的 `split --record-count` 和 `convert` 单遍流式处理,进度条保持不确定状态,因为行数要等这一遍结束才知道。
253
262
  - 对大 CSV/XLSX,`meta --fast` 是最快的元数据探查入口。
254
- - XLSX 的 schema 推断只采样前 1000 行,不再默认整表扫描。
263
+ - `parq schema` 和较短的 `head` 对 XLSX 只采样前 1000 行。`count`、`tail`、`stats`、`convert`、`split`、`merge` 把选中的工作表解析一次,后续值放不进样本类型时放宽该列,并把这些行留在内存里复用。
255
264
 
256
265
  如果你会反复处理大文件,把 CSV/XLSX 先转成 Parquet 依然是吞吐量最好的路径。
257
266
 
@@ -5,7 +5,7 @@
5
5
  ### 1. 克隆项目
6
6
 
7
7
  ```bash
8
- git clone <your-repo-url>
8
+ git clone https://github.com/Tendo33/parq-cli.git
9
9
  cd parq-cli
10
10
  ```
11
11
 
@@ -118,18 +118,15 @@ ruff check parq tests
118
118
  ruff check --fix parq tests
119
119
  ```
120
120
 
121
- ### 类型检查(可选)
121
+ ### 检查门
122
122
 
123
- ```bash
124
- pip install mypy
125
- mypy parq
126
- ```
123
+ 仓库的检查门是 Ruff 和 pytest。当前没有配置 mypy,CI 也不运行它。
127
124
 
128
125
  ## 📋 常见问题
129
126
 
130
127
  ### Q: 如何处理大文件?
131
128
 
132
- A: Parquet 文件会尽量利用元数据和 row group 做更高效的读取;CSV 会按批次流式处理,XLSX 会按行增量处理,因此预览、计数和分割都不再需要先整表载入内存。对于超大数据集,Parquet 仍然会有更好的整体吞吐。
129
+ A: Parquet 用文件元数据和 row group,不必先读完全部数据。CSV 按批次流式处理。XLSX 的 `schema` 和较短的 `head` 只采样前 1000 行;`count`、`tail`、`stats`、`convert`、`split`、`merge` 会把选中的工作表解析一次并放在内存里复用。反复处理大文件时,先转成 Parquet 吞吐更好。
133
130
 
134
131
  ### Q: 空文件会怎么处理?
135
132
 
@@ -83,7 +83,7 @@ python scripts/bump_version.py --version 0.1.1 --yes
83
83
 
84
84
  **检查步骤:**
85
85
  1. 查看 Actions 日志: https://github.com/Tendo33/parq-cli/actions
86
- 2. 确认 PyPI token 配置正确 (Repository Settings → Secrets)
86
+ 2. 确认仓库 Secret `PYPI_API_TOKEN` 仍然有效
87
87
  3. 运行本地检查: `python scripts/check_version.py`
88
88
 
89
89
  ## 🎯 手动发布 (不推荐)
@@ -114,6 +114,10 @@ git push origin v0.1.2
114
114
  - **PyPI 页面:** https://pypi.org/project/parq-cli/
115
115
  - **GitHub Releases:** https://github.com/Tendo33/parq-cli/releases
116
116
 
117
+ ## PyPI 上传
118
+
119
+ 发布 job 在 `v*` tag 上使用仓库 Secret `PYPI_API_TOKEN` 上传。从普通分支手动运行会在上传前失败。GitHub 上已经有一个名为 `pypi` 的 environment,当前 workflow 不依赖它。
120
+
117
121
  ## 💡 最佳实践
118
122
 
119
123
  1. **发布前测试:** 始终运行 `./.venv/bin/pytest -m "not performance"` 确保测试通过
@@ -124,5 +128,5 @@ git push origin v0.1.2
124
128
 
125
129
  ---
126
130
 
127
- **上次更新:** 2025-11-17
131
+ **上次更新:** 2026-10-10
128
132
  **维护者:** SimonSun (@Tendo33)
@@ -1,5 +1,5 @@
1
1
  """
2
- parq-cli: A powerful command-line tool for inspecting and analyzing Apache Parquet files.
2
+ parq-cli: A command-line tool for inspecting Parquet, CSV, TSV, and XLSX files.
3
3
  """
4
4
 
5
5
  __author__ = "SimonSun"