parq-cli 0.2.3__tar.gz → 0.2.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. parq_cli-0.2.5/CHANGELOG.md +60 -0
  2. {parq_cli-0.2.3 → parq_cli-0.2.5}/CLAUDE.md +2 -1
  3. {parq_cli-0.2.3 → parq_cli-0.2.5}/PKG-INFO +15 -12
  4. {parq_cli-0.2.3 → parq_cli-0.2.5}/README.md +14 -11
  5. {parq_cli-0.2.3 → parq_cli-0.2.5}/README_CN.md +14 -11
  6. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/cli.py +57 -15
  7. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/formats/_common.py +39 -9
  8. parq_cli-0.2.5/parq/formats/_csv.py +182 -0
  9. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/formats/_parquet.py +40 -10
  10. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/formats/_xlsx.py +11 -3
  11. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/plain_output.py +9 -0
  12. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/reader.py +295 -58
  13. {parq_cli-0.2.3 → parq_cli-0.2.5}/pyproject.toml +1 -1
  14. {parq_cli-0.2.3 → parq_cli-0.2.5}/scripts/check_version.py +0 -3
  15. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_cli.py +110 -0
  16. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_reader.py +387 -0
  17. {parq_cli-0.2.3 → parq_cli-0.2.5}/uv.lock +12 -1
  18. parq_cli-0.2.3/CHANGELOG.md +0 -41
  19. parq_cli-0.2.3/parq/formats/_csv.py +0 -39
  20. {parq_cli-0.2.3 → parq_cli-0.2.5}/.claude/settings.json +0 -0
  21. {parq_cli-0.2.3 → parq_cli-0.2.5}/.github/workflows/publish.yml +0 -0
  22. {parq_cli-0.2.3 → parq_cli-0.2.5}/.github/workflows/test.yml +0 -0
  23. {parq_cli-0.2.3 → parq_cli-0.2.5}/.gitignore +0 -0
  24. {parq_cli-0.2.3 → parq_cli-0.2.5}/.python-version +0 -0
  25. {parq_cli-0.2.3 → parq_cli-0.2.5}/.tmp-bench/csv-large.csv +0 -0
  26. {parq_cli-0.2.3 → parq_cli-0.2.5}/.tmp-bench/csv-small.csv +0 -0
  27. {parq_cli-0.2.3 → parq_cli-0.2.5}/.tmp-bench/parquet-large.parquet +0 -0
  28. {parq_cli-0.2.3 → parq_cli-0.2.5}/.tmp-bench/parquet-small.parquet +0 -0
  29. {parq_cli-0.2.3 → parq_cli-0.2.5}/AGENTS.md +0 -0
  30. {parq_cli-0.2.3 → parq_cli-0.2.5}/LICENSE +0 -0
  31. {parq_cli-0.2.3 → parq_cli-0.2.5}/data/test.parquet +0 -0
  32. {parq_cli-0.2.3 → parq_cli-0.2.5}/docs/QUICKSTART.md +0 -0
  33. {parq_cli-0.2.3 → parq_cli-0.2.5}/docs/RELEASE.md +0 -0
  34. {parq_cli-0.2.3 → parq_cli-0.2.5}/examples/README.md +0 -0
  35. {parq_cli-0.2.3 → parq_cli-0.2.5}/examples/create_sample_data.py +0 -0
  36. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/__init__.py +0 -0
  37. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/__main__.py +0 -0
  38. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/formats/__init__.py +0 -0
  39. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/formats/_chunk_writers.py +0 -0
  40. {parq_cli-0.2.3 → parq_cli-0.2.5}/parq/output.py +0 -0
  41. {parq_cli-0.2.3 → parq_cli-0.2.5}/scripts/README.md +0 -0
  42. {parq_cli-0.2.3 → parq_cli-0.2.5}/scripts/bump_version.py +0 -0
  43. {parq_cli-0.2.3 → parq_cli-0.2.5}/scripts/performance_benchmark.py +0 -0
  44. {parq_cli-0.2.3 → parq_cli-0.2.5}/scripts/publish.sh +0 -0
  45. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/__init__.py +0 -0
  46. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/conftest.py +0 -0
  47. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_main_entry.py +0 -0
  48. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_output.py +0 -0
  49. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_output_not_input.py +0 -0
  50. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_package_meta.py +0 -0
  51. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_performance.py +0 -0
  52. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_plain_output.py +0 -0
  53. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_release_scripts.py +0 -0
  54. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_workflows.py +0 -0
  55. {parq_cli-0.2.3 → parq_cli-0.2.5}/tests/test_xlsx_defused.py +0 -0
@@ -0,0 +1,60 @@
1
+ # Changelog
2
+
3
+ ## Unreleased
4
+
5
+ ## 0.2.5
6
+
7
+ - `diff` on inputs large enough to spill samples the smallest differing keys, matching what a smaller input reports. Each partition used to keep its own smallest keys and the merge kept whichever partition came first, so a spilled diff could report keys that were not the smallest in the file. Counts were never affected, only the `only_left` / `only_right` samples in `plain` and `json` output.
8
+ - `diff` partitions into half the row limit per bucket instead of aiming at the limit itself. Hash partitioning is uneven, so most buckets landed just over the threshold and partitioned a second time, rewriting the rows into temporary files two to three times. A 1,000,000-row diff against a 1,000,000-row file now takes about 5.1 s instead of 8.4 s on the same machine.
9
+ - `diff` reads rows as tuples rather than building a dict per row, and compares comparable values with plain tuple equality before falling back to the per-cell NaN-aware comparison. A key part is normalized once instead of twice.
10
+ - A partition still writes at most 32 buckets, so the smaller target only changes inputs below roughly 3.2 million rows: those split into twice as many, half-sized temporary files. At or above that size both limits clamp to 32 buckets and behave identically.
11
+
12
+ ## 0.2.4
13
+
14
+ - CSV and TSV reads accept `--encoding` (default `utf8`). The flag works before a subcommand and on it, and the subcommand value wins. An unknown name fails with `Unknown encoding`. Writes stay UTF-8.
15
+ - `--encoding` is only validated for CSV and TSV input. A script that passes it globally no longer fails on a parquet or xlsx file.
16
+ - A CSV or TSV column that cannot be decoded with `--encoding` fails the read, including a header whose name is not valid in that encoding. A bad byte after the first block names that column. UTF-16 read as UTF-8 reports the NUL bytes instead of a column-count error. PyArrow would otherwise return an undecoded data column as raw bytes.
17
+ - `--encoding punycode` is rejected. PyArrow passes a buffer to that decoder and it raises AttributeError.
18
+ - `diff` reads `.xlsx` files. It parses the selected sheet in full so a value past the first 1000 rows can widen a column before types are compared. Rows are kept only up to 100,000 per side; a larger sheet is parsed again in batches instead of held in memory.
19
+ - A type difference stays in `schema_type_mismatches`, including with `--summary-only`. The text produced by `str(value)` when a number, date, or boolean is widened is not a cell edit, in any input format. Any other text is. Keys stay exact, so `1` and `"1"` do not match.
20
+ - `diff` can sample rows whose keys are null next to numbers or strings. A Decimal NaN key matches another Decimal NaN. A list or struct column used as a key is rejected with the column name.
21
+ - Parquet `head` stops after the requested rows, including inside one large row group. A preview no longer reads that whole group.
22
+ - Streamed `tail` keeps the trailing Arrow batches and slices them. CSV `tail` no longer converts every cell to Python.
23
+
24
+ ## 0.2.3
25
+
26
+ - `merge --force` and `convert --force` replace an existing output file only after the new file is fully written. A schema mismatch or a write error leaves the previous file in place, and the temporary file is removed, including on Windows.
27
+ - `diff` treats NaN as equal to NaN, including when NaN is the key.
28
+ - `diff` and `merge` close Parquet file handles when they finish.
29
+ - JSON table output summarizes binary values, matching rich and plain output.
30
+ - `split --force` replaces existing part files only after every part is written. A later write error leaves the previous parts in place.
31
+ - Column stats ignore NaN when computing min, max, and mean. Nulls are still counted separately, and NaN stays in the non-null count.
32
+ - An empty, multi-character, or non-ASCII `--delimiter` is rejected with `delimiter must be a single ASCII character` before CSV parsing or writing.
33
+ - `stats --limit`, `stats --top-n`, and `diff --limit` reject negative values instead of slicing from the end. Zero still means an empty result.
34
+ - Selecting a duplicated column name reports that the name cannot be selected, instead of claiming the column does not exist. A unique column in the same file can still be selected.
35
+ - `diff` compares repeated column names by position. A diff that would compare those names by default asks for `--columns` instead of claiming the column does not exist.
36
+
37
+ ## 0.2.2
38
+
39
+ - `merge --force` and `convert` now refuse an output path that is one of the inputs. Previously `--force` deleted that input before reading it.
40
+ - XLSX workbook XML is parsed with `defusedxml` (added to the `xlsx` extra).
41
+ - Table output escapes cell markup and summarizes binary values (#4).
42
+ - Errors include the file path (#5).
43
+ - Dependency security updates: certifi, urllib3, idna, pygments, cryptography, pyarrow.
44
+
45
+ ## 0.2.1
46
+
47
+ - `diff` and `merge` honor the global `--delimiter` and `--sheet` options.
48
+ - `.tsv` is a supported output format for `convert`, `split`, and `merge`.
49
+ - XLSX `schema` still samples the first 1000 rows. Full reads widen a column when a later row does not fit that sample.
50
+ - String cardinality and top values stay exact past 10,000 distinct values. Nulls are not counted as empty strings.
51
+ - `diff` spills inputs larger than 100,000 rows per side into temporary files.
52
+ - `split --record-count` progress stays indeterminate while the total is still unknown.
53
+ - PyPI uploads run from a `v*` tag and use the repository `PYPI_API_TOKEN`.
54
+ - XLSX reads use `python-calamine`. A full read parses the workbook once and reuses those rows.
55
+ - CSV `convert` no longer counts every row before writing when a progress bar is shown.
56
+ - `--output`, `--delimiter`, and `--sheet` work before a subcommand and on the subcommand. The value on the subcommand wins when both are set.
57
+
58
+ ## 0.2.0
59
+
60
+ - Inspect, split, convert, diff, and merge Parquet, CSV, TSV, and XLSX files.
@@ -20,5 +20,6 @@ pytest tests/test_performance.py -m performance -q -s
20
20
  ## Gotchas
21
21
 
22
22
  - `parq schema` on XLSX samples the first 1000 rows. Full reads parse the sheet once with python-calamine and widen a column when a later value does not fit. Writing XLSX still uses openpyxl.
23
- - `diff` and `merge` use the global `--delimiter` and `--sheet`. `diff` keeps at most 100,000 rows per side in memory, then spills to temporary files.
23
+ - Parquet `head` stops inside a large row group. `tail` still reads the whole group when the last rows share one group. CSV type inference still uses PyArrow's 1 MiB block.
24
+ - `diff` and `merge` use the global `--delimiter`, `--sheet`, and `--encoding`. `diff` reads XLSX after a full sheet parse and keeps at most 100,000 rows per side; a larger sheet is parsed again in batches. Both sides spill to temporary files past that limit. CSV/TSV `--encoding` applies to reads; a column or header that does not decode fails the read. It is not validated for parquet or xlsx input. Writes stay UTF-8.
24
25
  - `.tsv` is valid for both input and output. Output uses a tab delimiter.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: parq-cli
3
- Version: 0.2.3
3
+ Version: 0.2.5
4
4
  Summary: A command-line tool for inspecting tabular files: Parquet, CSV, TSV, and XLSX
5
5
  Project-URL: Homepage, https://github.com/Tendo33/parq-cli
6
6
  Project-URL: Repository, https://github.com/Tendo33/parq-cli
@@ -131,10 +131,10 @@ parq merge chunks/*.parquet merged.parquet --force # overwrite if exists
131
131
  | `split` | yes | yes | yes | yes |
132
132
  | `stats` | yes | yes | yes | yes |
133
133
  | `convert` | yes | yes | yes | yes |
134
- | `diff` | yes | yes | yes | no, convert first |
134
+ | `diff` | yes | yes | yes | yes |
135
135
  | `merge` | yes | yes | yes | yes |
136
136
 
137
- `XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
137
+ `XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`. CSV and TSV reads default to UTF-8. Pass `--encoding gbk` (or another codec name) for a different input encoding. A column that does not decode with that encoding fails the read, including a header whose name is not valid in that encoding. UTF-16 read as UTF-8 fails the same way. Writes stay UTF-8.
138
138
 
139
139
  ## Command Reference
140
140
 
@@ -258,12 +258,14 @@ Compares two datasets by key and reports:
258
258
  Notes:
259
259
 
260
260
  - `--key` is required
261
- - `diff` currently supports Parquet, CSV, and TSV inputs
262
- - the global `--delimiter` applies to both CSV inputs
263
- - XLSX files should be converted first
261
+ - inputs can be Parquet, CSV, TSV, or XLSX
262
+ - `--delimiter`, `--encoding`, and `--sheet` apply to both inputs
263
+ - XLSX diff parses the selected sheet in full, so a later value can widen a column before types are compared
264
+ - a type difference is always reported. The text from `str(value)` is not a cell edit; any other text is. Keys stay exact, so `1` and `"1"` do not match
264
265
  - duplicate keys on either side are treated as an error
266
+ - null keys are included in the sample. A list or struct column cannot be a key
265
267
  - inputs larger than 100,000 rows on a side spill into temporary files instead of holding every key in memory
266
- - `--summary-only` keeps the counts and omits sample payloads
268
+ - `--summary-only` keeps the counts and the type mismatches, and omits sample row payloads
267
269
 
268
270
  ### `merge`
269
271
 
@@ -280,7 +282,7 @@ Notes:
280
282
  - schemas must be identical or safely unifiable by Arrow
281
283
  - by default, existing output files raise an error; use `--force` / `-F` to overwrite
282
284
  - output format is inferred from the output suffix, including `.tsv`
283
- - the global `--delimiter` and `--sheet` apply to every input
285
+ - the global `--delimiter`, `--encoding`, and `--sheet` apply to every input
284
286
  - a live progress bar is shown during the merge
285
287
 
286
288
  ## Output Modes
@@ -290,10 +292,11 @@ Global options:
290
292
  - `--version`, `-v`: show version information. Place this before the subcommand: `parq --version`
291
293
  - `--output`, `-o`: select output format (`rich` | `plain` | `json`)
292
294
  - `--delimiter`, `-d`: field delimiter for CSV/TSV input (default: `,`); `.tsv` files default to `\t` automatically
295
+ - `--encoding`: character encoding for CSV/TSV input (default: `utf8`). Writes stay UTF-8
293
296
  - `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet). A value made only of digits is an index, including a sheet whose name is `1` or `2`
294
297
  - `--help`: show command help
295
298
 
296
- `--output`, `--delimiter`, and `--sheet` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
299
+ `--output`, `--delimiter`, `--sheet`, and `--encoding` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
297
300
 
298
301
  Available output modes:
299
302
 
@@ -315,11 +318,11 @@ On Windows terminals that cannot safely render emoji or extended characters, Ric
315
318
 
316
319
  ## Large File Notes
317
320
 
318
- - Parquet metadata, row counts, and previews use Arrow metadata and row-group shortcuts where available.
319
- - CSV `tail` uses a fixed-size column window instead of materializing every row as Python dicts.
321
+ - Parquet metadata and row counts come from the file footer. `head` stops after the requested rows, including when they sit inside one large row group. `tail` reads the row groups that hold the last rows.
322
+ - CSV `tail` scans the file once, keeps the trailing Arrow batches, and slices the last rows.
320
323
  - CSV `split --record-count` and CSV `convert` stream in one pass. Their progress bars stay indeterminate because the row count is not known ahead of time.
321
324
  - `meta --fast` is the best option when you need quick metadata from large CSV/XLSX inputs.
322
- - `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, and `merge` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
325
+ - `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, `merge`, and `diff` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
323
326
  - `split --record-count` keeps the progress bar indeterminate while it writes.
324
327
  - `diff` indexes at most 100,000 rows per side in memory, then hash-partitions larger inputs into temporary Parquet files.
325
328
 
@@ -94,10 +94,10 @@ parq merge chunks/*.parquet merged.parquet --force # overwrite if exists
94
94
  | `split` | yes | yes | yes | yes |
95
95
  | `stats` | yes | yes | yes | yes |
96
96
  | `convert` | yes | yes | yes | yes |
97
- | `diff` | yes | yes | yes | no, convert first |
97
+ | `diff` | yes | yes | yes | yes |
98
98
  | `merge` | yes | yes | yes | yes |
99
99
 
100
- `XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`.
100
+ `XLSX` reads use `python-calamine` and writes use `openpyxl`; both install with `parq-cli[xlsx]`. Formula cells are read from Excel's cached value, so a formula Excel has not calculated comes back empty. TSV files are auto-detected by the `.tsv` extension; a custom delimiter can be supplied with `--delimiter`. CSV and TSV reads default to UTF-8. Pass `--encoding gbk` (or another codec name) for a different input encoding. A column that does not decode with that encoding fails the read, including a header whose name is not valid in that encoding. UTF-16 read as UTF-8 fails the same way. Writes stay UTF-8.
101
101
 
102
102
  ## Command Reference
103
103
 
@@ -221,12 +221,14 @@ Compares two datasets by key and reports:
221
221
  Notes:
222
222
 
223
223
  - `--key` is required
224
- - `diff` currently supports Parquet, CSV, and TSV inputs
225
- - the global `--delimiter` applies to both CSV inputs
226
- - XLSX files should be converted first
224
+ - inputs can be Parquet, CSV, TSV, or XLSX
225
+ - `--delimiter`, `--encoding`, and `--sheet` apply to both inputs
226
+ - XLSX diff parses the selected sheet in full, so a later value can widen a column before types are compared
227
+ - a type difference is always reported. The text from `str(value)` is not a cell edit; any other text is. Keys stay exact, so `1` and `"1"` do not match
227
228
  - duplicate keys on either side are treated as an error
229
+ - null keys are included in the sample. A list or struct column cannot be a key
228
230
  - inputs larger than 100,000 rows on a side spill into temporary files instead of holding every key in memory
229
- - `--summary-only` keeps the counts and omits sample payloads
231
+ - `--summary-only` keeps the counts and the type mismatches, and omits sample row payloads
230
232
 
231
233
  ### `merge`
232
234
 
@@ -243,7 +245,7 @@ Notes:
243
245
  - schemas must be identical or safely unifiable by Arrow
244
246
  - by default, existing output files raise an error; use `--force` / `-F` to overwrite
245
247
  - output format is inferred from the output suffix, including `.tsv`
246
- - the global `--delimiter` and `--sheet` apply to every input
248
+ - the global `--delimiter`, `--encoding`, and `--sheet` apply to every input
247
249
  - a live progress bar is shown during the merge
248
250
 
249
251
  ## Output Modes
@@ -253,10 +255,11 @@ Global options:
253
255
  - `--version`, `-v`: show version information. Place this before the subcommand: `parq --version`
254
256
  - `--output`, `-o`: select output format (`rich` | `plain` | `json`)
255
257
  - `--delimiter`, `-d`: field delimiter for CSV/TSV input (default: `,`); `.tsv` files default to `\t` automatically
258
+ - `--encoding`: character encoding for CSV/TSV input (default: `utf8`). Writes stay UTF-8
256
259
  - `--sheet`: XLSX sheet name or 0-based index to read (default: active sheet). A value made only of digits is an index, including a sheet whose name is `1` or `2`
257
260
  - `--help`: show command help
258
261
 
259
- `--output`, `--delimiter`, and `--sheet` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
262
+ `--output`, `--delimiter`, `--sheet`, and `--encoding` work before the subcommand and on the subcommand. If you pass the same option in both places, the value on the subcommand wins. On `convert`, `--output` / `-o` still selects the display format; the destination file is the positional `OUTPUT` argument.
260
263
 
261
264
  Available output modes:
262
265
 
@@ -278,11 +281,11 @@ On Windows terminals that cannot safely render emoji or extended characters, Ric
278
281
 
279
282
  ## Large File Notes
280
283
 
281
- - Parquet metadata, row counts, and previews use Arrow metadata and row-group shortcuts where available.
282
- - CSV `tail` uses a fixed-size column window instead of materializing every row as Python dicts.
284
+ - Parquet metadata and row counts come from the file footer. `head` stops after the requested rows, including when they sit inside one large row group. `tail` reads the row groups that hold the last rows.
285
+ - CSV `tail` scans the file once, keeps the trailing Arrow batches, and slices the last rows.
283
286
  - CSV `split --record-count` and CSV `convert` stream in one pass. Their progress bars stay indeterminate because the row count is not known ahead of time.
284
287
  - `meta --fast` is the best option when you need quick metadata from large CSV/XLSX inputs.
285
- - `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, and `merge` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
288
+ - `parq schema` and a short `head` sample the first 1000 XLSX rows. `count`, `tail`, `stats`, `convert`, `split`, `merge`, and `diff` parse the selected sheet once, widen a column when a later value does not fit that sample, and keep those rows in memory.
286
289
  - `split --record-count` keeps the progress bar indeterminate while it writes.
287
290
  - `diff` indexes at most 100,000 rows per side in memory, then hash-partitions larger inputs into temporary Parquet files.
288
291
 
@@ -82,10 +82,10 @@ parq merge part-001.parquet part-002.parquet merged.parquet
82
82
  | `split` | 支持 | 支持 | 支持 | 支持 |
83
83
  | `stats` | 支持 | 支持 | 支持 | 支持 |
84
84
  | `convert` | 支持 | 支持 | 支持 | 支持 |
85
- | `diff` | 支持 | 支持 | 支持 | 先转换再对比 |
85
+ | `diff` | 支持 | 支持 | 支持 | 支持 |
86
86
  | `merge` | 支持 | 支持 | 支持 | 支持 |
87
87
 
88
- `XLSX` 的读取使用 `python-calamine`,写出仍使用 `openpyxl`,两者都随 `parq-cli[xlsx]` 安装。公式单元格读取的是 Excel 已经算出的缓存值,没有缓存的公式会是空值。`.tsv` 按扩展名自动使用制表符;其它分隔符用 `--delimiter`。`--sheet` 选择 XLSX 工作表,对 `merge` 同样生效。
88
+ `XLSX` 的读取使用 `python-calamine`,写出仍使用 `openpyxl`,两者都随 `parq-cli[xlsx]` 安装。公式单元格读取的是 Excel 已经算出的缓存值,没有缓存的公式会是空值。`.tsv` 按扩展名自动使用制表符;其它分隔符用 `--delimiter`。`--sheet` 选择 XLSX 工作表,对 `diff` 和 `merge` 同样生效。CSV/TSV 默认按 UTF-8 读取,其它编码用 `--encoding`(例如 `gbk`)。某一列无法按该编码解码时,读取会失败,表头对不上该编码时也一样。把 UTF-16 当成 UTF-8 读取同样会失败。写出仍然是 UTF-8。
89
89
 
90
90
  ## 命令说明
91
91
 
@@ -203,12 +203,14 @@ parq diff LEFT RIGHT --key id --summary-only
203
203
  说明:
204
204
 
205
205
  - `--key` 必填
206
- - `diff` 当前支持 Parquet、CSV 和 TSV
207
- - 全局 `--delimiter` 会同时用于左右两个 CSV 输入
208
- - XLSX 需要先转换
206
+ - 输入可以是 Parquet、CSV、TSV 或 XLSX
207
+ - `--delimiter`、`--encoding` 和 `--sheet` 会同时用于左右两个输入
208
+ - XLSX 对比会完整解析选中的工作表,第 1000 行之后的值也能放宽列类型后再比较
209
+ - 类型不同一定会报告。`str(value)` 得到的文本不算单元格修改,其它文本算。主键仍按原值匹配,`1` 和 `"1"` 对不上
209
210
  - 任一侧若出现重复 key,会直接报错
211
+ - 空主键会进入样本。列表或结构体列不能当主键
210
212
  - 单侧超过 10 万行时,会把数据按主键散列到临时文件,而不是把全部 key 放进内存
211
- - `--summary-only` 只保留计数,不输出样本内容
213
+ - `--summary-only` 保留计数和类型差异,不输出变更行样本
212
214
 
213
215
  ### `merge`
214
216
 
@@ -224,7 +226,7 @@ parq merge chunks/*.parquet merged.parquet
224
226
  - schema 必须一致,或能被 Arrow 安全统一
225
227
  - 默认遇到已存在的输出文件会报错;`--force` / `-F` 会覆盖
226
228
  - 输出格式由输出文件后缀决定,包括 `.tsv`
227
- - 全局 `--delimiter` 和 `--sheet` 会作用于每一个输入
229
+ - 全局 `--delimiter`、`--encoding` 和 `--sheet` 会作用于每一个输入
228
230
 
229
231
  ## 输出模式
230
232
 
@@ -233,10 +235,11 @@ parq merge chunks/*.parquet merged.parquet
233
235
  - `--version`, `-v`:显示版本号。放在子命令前面:`parq --version`
234
236
  - `--output`, `-o`:选择输出格式(`rich` | `plain` | `json`)
235
237
  - `--delimiter`, `-d`:CSV/TSV 输入分隔符,默认 `,`;`.tsv` 会自动使用制表符
238
+ - `--encoding`:CSV/TSV 输入编码,默认 `utf8`。写出仍然是 UTF-8
236
239
  - `--sheet`:XLSX 工作表名称或从 0 开始的序号,默认活动工作表。纯数字会按序号处理,名叫 `1` 或 `2` 的工作表也一样
237
240
  - `--help`:显示帮助
238
241
 
239
- `--output`、`--delimiter`、`--sheet` 可以写在子命令前面,也可以写在子命令上面。同一个选项两处都写时,子命令上的值生效。`convert` 的 `--output` / `-o` 仍然是显示格式,目标文件是位置参数 `OUTPUT`。
242
+ `--output`、`--delimiter`、`--sheet`、`--encoding` 可以写在子命令前面,也可以写在子命令上面。同一个选项两处都写时,子命令上的值生效。`convert` 的 `--output` / `-o` 仍然是显示格式,目标文件是位置参数 `OUTPUT`。
240
243
 
241
244
  可选输出模式:
242
245
 
@@ -256,11 +259,11 @@ parq diff left.parquet right.parquet --key id --summary-only --output json
256
259
 
257
260
  ## 大文件说明
258
261
 
259
- - Parquet 的元数据、行数和预览会优先利用 Arrow metadata 与 row group 能力。
260
- - CSV `tail` 使用固定大小列窗口,而不是把整份数据都转成 Python dict。
262
+ - Parquet 的元数据和行数来自文件 footer。`head` 拿到请求的行数就停,包括这些行落在同一个大 row group 里的情况。`tail` 只读包含末尾行的 row group。
263
+ - CSV `tail` 扫一遍文件,保留末尾的 Arrow batch,再切出最后几行。
261
264
  - CSV 的 `split --record-count` 和 `convert` 单遍流式处理,进度条保持不确定状态,因为行数要等这一遍结束才知道。
262
265
  - 对大 CSV/XLSX,`meta --fast` 是最快的元数据探查入口。
263
- - `parq schema` 和较短的 `head` 对 XLSX 只采样前 1000 行。`count`、`tail`、`stats`、`convert`、`split`、`merge` 把选中的工作表解析一次,后续值放不进样本类型时放宽该列,并把这些行留在内存里复用。
266
+ - `parq schema` 和较短的 `head` 对 XLSX 只采样前 1000 行。`count`、`tail`、`stats`、`convert`、`split`、`merge`、`diff` 把选中的工作表解析一次,后续值放不进样本类型时放宽该列,并把这些行留在内存里复用。
264
267
 
265
268
  如果你会反复处理大文件,把 CSV/XLSX 先转成 Parquet 依然是吞吐量最好的路径。
266
269
 
@@ -26,14 +26,15 @@ class _AppState:
26
26
  output_format: OutputFormat = OutputFormat.RICH
27
27
  delimiter: str = ","
28
28
  sheet: Optional[str] = None
29
+ encoding: str = "utf8"
29
30
 
30
31
 
31
32
  app = typer.Typer(
32
33
  name="parq",
33
34
  help="A powerful command-line tool for inspecting tabular files",
34
35
  epilog=(
35
- "--output, --delimiter, and --sheet work before a subcommand and on the subcommand. "
36
- "When both are set, the value on the subcommand wins."
36
+ "--output, --delimiter, --sheet, and --encoding work before a subcommand and on the "
37
+ "subcommand. When both are set, the value on the subcommand wins."
37
38
  ),
38
39
  add_completion=False,
39
40
  )
@@ -88,6 +89,14 @@ _CommandSheet = Annotated[
88
89
  show_default=False,
89
90
  ),
90
91
  ]
92
+ _CommandEncoding = Annotated[
93
+ Optional[str],
94
+ typer.Option(
95
+ "--encoding",
96
+ help="Character encoding for CSV/TSV input, such as utf8 or gbk",
97
+ show_default=False,
98
+ ),
99
+ ]
91
100
 
92
101
 
93
102
  def _apply_command_options(
@@ -95,6 +104,7 @@ def _apply_command_options(
95
104
  output_format: Optional[OutputFormat],
96
105
  delimiter: Optional[str],
97
106
  sheet: Optional[str],
107
+ encoding: Optional[str],
98
108
  ) -> _AppState:
99
109
  """Let flags after the subcommand override the same flags placed before it."""
100
110
  state = _get_state(ctx)
@@ -104,6 +114,8 @@ def _apply_command_options(
104
114
  state.delimiter = delimiter
105
115
  if sheet is not None:
106
116
  state.sheet = sheet
117
+ if encoding is not None:
118
+ state.encoding = encoding
107
119
  return state
108
120
 
109
121
 
@@ -112,7 +124,12 @@ def _get_reader(file_path: str, state: Optional[_AppState] = None) -> Any:
112
124
  from parq.reader import MultiFormatReader
113
125
 
114
126
  if state is not None:
115
- return MultiFormatReader(file_path, delimiter=state.delimiter, sheet=state.sheet)
127
+ return MultiFormatReader(
128
+ file_path,
129
+ delimiter=state.delimiter,
130
+ sheet=state.sheet,
131
+ encoding=state.encoding,
132
+ )
116
133
  return MultiFormatReader(file_path)
117
134
 
118
135
 
@@ -173,12 +190,20 @@ def main(
173
190
  ),
174
191
  ),
175
192
  ] = None,
193
+ encoding: Annotated[
194
+ str,
195
+ typer.Option(
196
+ "--encoding",
197
+ help="Character encoding for CSV/TSV input (default: utf8)",
198
+ ),
199
+ ] = "utf8",
176
200
  ) -> None:
177
201
  """A powerful command-line tool for inspecting tabular files."""
178
202
  state = ctx.ensure_object(_AppState)
179
203
  state.output_format = output
180
204
  state.delimiter = delimiter
181
205
  state.sheet = sheet
206
+ state.encoding = encoding
182
207
 
183
208
  if version:
184
209
  from parq import __version__
@@ -202,9 +227,10 @@ def meta(
202
227
  output_format: _CommandOutputFormat = None,
203
228
  delimiter: _CommandDelimiter = None,
204
229
  sheet: _CommandSheet = None,
230
+ encoding: _CommandEncoding = None,
205
231
  ) -> None:
206
232
  """Display metadata information of a tabular file."""
207
- _apply_command_options(ctx, output_format, delimiter, sheet)
233
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
208
234
 
209
235
  def operation(formatter: Any) -> None:
210
236
  with _get_reader(str(file), _get_state(ctx)) as reader:
@@ -222,9 +248,10 @@ def schema(
222
248
  output_format: _CommandOutputFormat = None,
223
249
  delimiter: _CommandDelimiter = None,
224
250
  sheet: _CommandSheet = None,
251
+ encoding: _CommandEncoding = None,
225
252
  ) -> None:
226
253
  """Display the schema of a tabular file."""
227
- _apply_command_options(ctx, output_format, delimiter, sheet)
254
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
228
255
 
229
256
  def operation(formatter: Any) -> None:
230
257
  with _get_reader(str(file), _get_state(ctx)) as reader:
@@ -247,9 +274,10 @@ def head(
247
274
  output_format: _CommandOutputFormat = None,
248
275
  delimiter: _CommandDelimiter = None,
249
276
  sheet: _CommandSheet = None,
277
+ encoding: _CommandEncoding = None,
250
278
  ) -> None:
251
279
  """Display the first N rows of a tabular file."""
252
- _apply_command_options(ctx, output_format, delimiter, sheet)
280
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
253
281
 
254
282
  def operation(formatter: Any) -> None:
255
283
  with _get_reader(str(file), _get_state(ctx)) as reader:
@@ -273,9 +301,10 @@ def tail(
273
301
  output_format: _CommandOutputFormat = None,
274
302
  delimiter: _CommandDelimiter = None,
275
303
  sheet: _CommandSheet = None,
304
+ encoding: _CommandEncoding = None,
276
305
  ) -> None:
277
306
  """Display the last N rows of a tabular file."""
278
- _apply_command_options(ctx, output_format, delimiter, sheet)
307
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
279
308
 
280
309
  def operation(formatter: Any) -> None:
281
310
  with _get_reader(str(file), _get_state(ctx)) as reader:
@@ -294,9 +323,10 @@ def count(
294
323
  output_format: _CommandOutputFormat = None,
295
324
  delimiter: _CommandDelimiter = None,
296
325
  sheet: _CommandSheet = None,
326
+ encoding: _CommandEncoding = None,
297
327
  ) -> None:
298
328
  """Display the total row count of a tabular file."""
299
- _apply_command_options(ctx, output_format, delimiter, sheet)
329
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
300
330
 
301
331
  def operation(formatter: Any) -> None:
302
332
  with _get_reader(str(file), _get_state(ctx)) as reader:
@@ -330,10 +360,11 @@ def split(
330
360
  output_format: _CommandOutputFormat = None,
331
361
  delimiter: _CommandDelimiter = None,
332
362
  sheet: _CommandSheet = None,
363
+ encoding: _CommandEncoding = None,
333
364
  ) -> None:
334
365
  """Split a source file into multiple files."""
335
366
 
336
- state = _apply_command_options(ctx, output_format, delimiter, sheet)
367
+ state = _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
337
368
  formatter = _get_formatter(ctx)
338
369
  try:
339
370
  if file_count is None and record_count is None:
@@ -431,9 +462,10 @@ def stats(
431
462
  output_format: _CommandOutputFormat = None,
432
463
  delimiter: _CommandDelimiter = None,
433
464
  sheet: _CommandSheet = None,
465
+ encoding: _CommandEncoding = None,
434
466
  ) -> None:
435
467
  """Display simple per-column statistics."""
436
- _apply_command_options(ctx, output_format, delimiter, sheet)
468
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
437
469
 
438
470
  def operation(formatter: Any) -> None:
439
471
  with _get_reader(str(file), _get_state(ctx)) as reader:
@@ -462,13 +494,14 @@ def convert(
462
494
  output_format: _CommandOutputFormat = None,
463
495
  delimiter: _CommandDelimiter = None,
464
496
  sheet: _CommandSheet = None,
497
+ encoding: _CommandEncoding = None,
465
498
  ) -> None:
466
499
  """Convert one supported input file to another supported output format.
467
500
 
468
501
  --output / -o selects rich, plain, or json. The destination file is the OUTPUT argument.
469
502
  """
470
503
 
471
- state = _apply_command_options(ctx, output_format, delimiter, sheet)
504
+ state = _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
472
505
  formatter = _get_formatter(ctx)
473
506
 
474
507
  try:
@@ -533,8 +566,12 @@ def convert(
533
566
  @app.command()
534
567
  def diff(
535
568
  ctx: typer.Context,
536
- left: Annotated[Path, typer.Argument(help="Left input file (.parquet, .csv, or .tsv)")],
537
- right: Annotated[Path, typer.Argument(help="Right input file (.parquet, .csv, or .tsv)")],
569
+ left: Annotated[
570
+ Path, typer.Argument(help="Left input file (.parquet, .csv, .tsv, or .xlsx)")
571
+ ],
572
+ right: Annotated[
573
+ Path, typer.Argument(help="Right input file (.parquet, .csv, .tsv, or .xlsx)")
574
+ ],
538
575
  key: Annotated[str, typer.Option("--key", help="Comma-separated key columns for row matching")],
539
576
  columns: Annotated[
540
577
  Optional[str],
@@ -548,9 +585,10 @@ def diff(
548
585
  output_format: _CommandOutputFormat = None,
549
586
  delimiter: _CommandDelimiter = None,
550
587
  sheet: _CommandSheet = None,
588
+ encoding: _CommandEncoding = None,
551
589
  ) -> None:
552
590
  """Compare two tabular files using one or more key columns."""
553
- _apply_command_options(ctx, output_format, delimiter, sheet)
591
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
554
592
 
555
593
  def operation(formatter: Any) -> None:
556
594
  from parq.reader import diff_files
@@ -565,6 +603,7 @@ def diff(
565
603
  summary_only=summary_only,
566
604
  delimiter=state.delimiter,
567
605
  sheet=state.sheet,
606
+ encoding=state.encoding,
568
607
  )
569
608
  formatter.print_diff_result(result)
570
609
 
@@ -585,10 +624,11 @@ def merge(
585
624
  output_format: _CommandOutputFormat = None,
586
625
  delimiter: _CommandDelimiter = None,
587
626
  sheet: _CommandSheet = None,
627
+ encoding: _CommandEncoding = None,
588
628
  ) -> None:
589
629
  """Merge multiple compatible input files into one output file."""
590
630
 
591
- _apply_command_options(ctx, output_format, delimiter, sheet)
631
+ _apply_command_options(ctx, output_format, delimiter, sheet, encoding)
592
632
  formatter = _get_formatter(ctx)
593
633
  try:
594
634
  import time
@@ -635,6 +675,7 @@ def merge(
635
675
  progress_callback=update_progress,
636
676
  delimiter=state.delimiter,
637
677
  sheet=state.sheet,
678
+ encoding=state.encoding,
638
679
  )
639
680
  else:
640
681
  total_rows = merge_files(
@@ -643,6 +684,7 @@ def merge(
643
684
  force=force,
644
685
  delimiter=state.delimiter,
645
686
  sheet=state.sheet,
687
+ encoding=state.encoding,
646
688
  )
647
689
 
648
690
  formatter.print_merge_result(input_files, output, total_rows, time.time() - start_time)
@@ -3,7 +3,7 @@ Shared utilities, data structures, and statistics functions used across
3
3
  all format-specific reader modules.
4
4
  """
5
5
 
6
- from collections import deque
6
+ import codecs
7
7
  from dataclasses import dataclass
8
8
  from typing import Any, Dict, Iterable, List, Optional
9
9
 
@@ -13,6 +13,26 @@ import pyarrow.compute as pc
13
13
  FAST_PATH_ROWS_MULTIPLIER = 10
14
14
 
15
15
 
16
+ def _validate_encoding(encoding: str) -> str:
17
+ """Return a text encoding name PyArrow can use, or raise ValueError.
18
+
19
+ A text encoding can encode a newline. Byte transforms such as ``rot_13``
20
+ cannot. ``punycode`` can encode text, but PyArrow passes a Buffer into the
21
+ decoder and that codec then raises AttributeError.
22
+ """
23
+ normalized = encoding.strip()
24
+ try:
25
+ "\n".encode(normalized)
26
+ codecs.getdecoder(normalized)(pa.py_buffer(b"\n"))
27
+ except (LookupError, UnicodeEncodeError, AttributeError, TypeError) as exc:
28
+ raise ValueError(f"Unknown encoding: {encoding}") from exc
29
+ except UnicodeError:
30
+ # The codec accepted the buffer and rejected these bytes. utf-16 does
31
+ # this for a lone newline; it is still a real text encoding.
32
+ return normalized
33
+ return normalized
34
+
35
+
16
36
  def _validate_delimiter(delimiter: str) -> None:
17
37
  """Reject delimiters PyArrow cannot use as one ASCII field separator."""
18
38
  if len(delimiter) != 1 or not delimiter.isascii() or ord(delimiter) == 0:
@@ -141,18 +161,28 @@ def _collect_tail_from_batches(
141
161
  n: int,
142
162
  schema: pa.Schema,
143
163
  ) -> pa.Table:
144
- """Collect last n rows using per-column ring buffers."""
164
+ """Collect the last n rows from trailing Arrow batches."""
145
165
  if n == 0:
146
166
  return _create_empty_table(schema)
147
- tail_columns = {field.name: deque(maxlen=n) for field in schema}
167
+ kept: list[pa.RecordBatch] = []
168
+ kept_rows = 0
169
+ first_live = 0
148
170
  for batch in batches:
149
- batch_dict = batch.to_pydict()
150
- for name in schema.names:
151
- tail_columns[name].extend(batch_dict[name])
152
- if not tail_columns or not any(tail_columns.values()):
171
+ if len(batch) == 0:
172
+ continue
173
+ kept.append(batch)
174
+ kept_rows += len(batch)
175
+ while len(kept) - first_live > 1 and kept_rows - len(kept[first_live]) >= n:
176
+ kept_rows -= len(kept[first_live])
177
+ first_live += 1
178
+ if kept_rows == 0:
153
179
  return _create_empty_table(schema)
154
- arrays = [pa.array(list(tail_columns[field.name]), type=field.type) for field in schema]
155
- return pa.Table.from_arrays(arrays, schema=schema)
180
+ if first_live:
181
+ del kept[:first_live]
182
+ table = pa.Table.from_batches(kept, schema=schema)
183
+ if kept_rows > n:
184
+ return table.slice(kept_rows - n, n)
185
+ return table
156
186
 
157
187
 
158
188
  def _collect_preview_from_batches(