pqx 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pqx-0.1.0/.github/workflows/ci.yml +57 -0
- pqx-0.1.0/.github/workflows/publish.yml +38 -0
- pqx-0.1.0/.gitignore +23 -0
- pqx-0.1.0/PKG-INFO +163 -0
- pqx-0.1.0/README.md +144 -0
- pqx-0.1.0/pqx/__init__.py +6 -0
- pqx-0.1.0/pqx/__main__.py +5 -0
- pqx-0.1.0/pqx/_terminal.py +122 -0
- pqx-0.1.0/pqx/_version.py +24 -0
- pqx-0.1.0/pqx/app.py +1634 -0
- pqx-0.1.0/pqx/app.tcss +393 -0
- pqx-0.1.0/pqx/cli.py +54 -0
- pqx-0.1.0/pqx/data.py +592 -0
- pqx-0.1.0/pqx/demo.py +114 -0
- pqx-0.1.0/pqx/fmt.py +295 -0
- pqx-0.1.0/pqx/plots.py +446 -0
- pqx-0.1.0/pqx/screens.py +329 -0
- pqx-0.1.0/pqx/widgets.py +43 -0
- pqx-0.1.0/pqx.egg-info/PKG-INFO +163 -0
- pqx-0.1.0/pqx.egg-info/SOURCES.txt +31 -0
- pqx-0.1.0/pqx.egg-info/dependency_links.txt +1 -0
- pqx-0.1.0/pqx.egg-info/entry_points.txt +2 -0
- pqx-0.1.0/pqx.egg-info/requires.txt +11 -0
- pqx-0.1.0/pqx.egg-info/scm_file_list.json +26 -0
- pqx-0.1.0/pqx.egg-info/scm_version.json +8 -0
- pqx-0.1.0/pqx.egg-info/top_level.txt +1 -0
- pqx-0.1.0/pyproject.toml +45 -0
- pqx-0.1.0/setup.cfg +4 -0
- pqx-0.1.0/tests/conftest.py +37 -0
- pqx-0.1.0/tests/test_app.py +471 -0
- pqx-0.1.0/tests/test_data.py +168 -0
- pqx-0.1.0/tests/test_fmt_plots.py +117 -0
- pqx-0.1.0/tests/test_terminal.py +154 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [master]
|
|
6
|
+
tags: ["v*"] # a release tag runs the full matrix alongside publish.yml
|
|
7
|
+
pull_request:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
concurrency:
|
|
11
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
12
|
+
cancel-in-progress: true
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
test:
|
|
16
|
+
name: tests (Python ${{ matrix.python-version }})
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
strategy:
|
|
19
|
+
fail-fast: false
|
|
20
|
+
matrix:
|
|
21
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
with:
|
|
25
|
+
fetch-depth: 0 # setuptools-scm needs the tags and history
|
|
26
|
+
- uses: actions/setup-python@v5
|
|
27
|
+
with:
|
|
28
|
+
python-version: ${{ matrix.python-version }}
|
|
29
|
+
cache: pip
|
|
30
|
+
cache-dependency-path: pyproject.toml
|
|
31
|
+
- name: Install
|
|
32
|
+
run: |
|
|
33
|
+
python -m pip install --upgrade pip
|
|
34
|
+
pip install -e ".[dev]"
|
|
35
|
+
- name: Lint
|
|
36
|
+
run: ruff check pqx tests
|
|
37
|
+
- name: Test
|
|
38
|
+
run: pytest -v
|
|
39
|
+
|
|
40
|
+
package:
|
|
41
|
+
name: build and install the package
|
|
42
|
+
runs-on: ubuntu-latest
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/checkout@v4
|
|
45
|
+
with:
|
|
46
|
+
fetch-depth: 0 # setuptools-scm needs the tags and history
|
|
47
|
+
- uses: astral-sh/setup-uv@v6
|
|
48
|
+
- name: Build sdist and wheel
|
|
49
|
+
env:
|
|
50
|
+
UV_PYTHON: "3.12"
|
|
51
|
+
run: uv build
|
|
52
|
+
- name: Install the wheel in a clean environment and run it
|
|
53
|
+
run: |
|
|
54
|
+
uv venv /tmp/pqx-wheel --python 3.12
|
|
55
|
+
uv pip install --python /tmp/pqx-wheel dist/*.whl
|
|
56
|
+
/tmp/pqx-wheel/bin/pqx --version
|
|
57
|
+
/tmp/pqx-wheel/bin/python -c "import pqx, importlib.resources as r; assert pqx.__version__ != '0.0.0.dev0'; assert (r.files('pqx') / 'app.tcss').is_file()"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Publishing a GitHub Release (which creates its vX.Y.Z tag) builds pqx and
|
|
4
|
+
# uploads it to PyPI; setuptools-scm takes the version from that tag. The same
|
|
5
|
+
# tag also runs the full test matrix (ci.yml).
|
|
6
|
+
on:
|
|
7
|
+
release:
|
|
8
|
+
types: [published]
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
id-token: write # OIDC for PyPI trusted publishing: no API token stored
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
publish:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
environment: pypi
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
with:
|
|
21
|
+
fetch-depth: 0 # setuptools-scm needs full history
|
|
22
|
+
|
|
23
|
+
- name: Install uv
|
|
24
|
+
uses: astral-sh/setup-uv@v6
|
|
25
|
+
|
|
26
|
+
- name: Build package
|
|
27
|
+
env:
|
|
28
|
+
UV_PYTHON: "3.12"
|
|
29
|
+
run: uv build
|
|
30
|
+
|
|
31
|
+
- name: Check the version matches the release tag
|
|
32
|
+
run: |
|
|
33
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
34
|
+
ls dist
|
|
35
|
+
test -f "dist/pqx-${tag}-py3-none-any.whl" || { echo "built version != tag ${GITHUB_REF_NAME}"; exit 1; }
|
|
36
|
+
|
|
37
|
+
- name: Publish to PyPI (trusted publishing)
|
|
38
|
+
run: uv publish --trusted-publishing always
|
pqx-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.egg-info/
|
|
4
|
+
*.parquet
|
|
5
|
+
!tests/data/*.parquet
|
|
6
|
+
.pytest_cache/
|
|
7
|
+
# setuptools-scm generated version file
|
|
8
|
+
pqx/_version.py
|
|
9
|
+
# build output (uv build / pip wheel)
|
|
10
|
+
build/
|
|
11
|
+
dist/
|
|
12
|
+
# Textual debug input log (TEXTUAL_DEBUG=1 writes it to the current directory)
|
|
13
|
+
keys.log
|
|
14
|
+
# coverage
|
|
15
|
+
.coverage
|
|
16
|
+
.coverage.*
|
|
17
|
+
htmlcov/
|
|
18
|
+
# OS and editor files
|
|
19
|
+
.DS_Store
|
|
20
|
+
*.swp
|
|
21
|
+
*~
|
|
22
|
+
.idea/
|
|
23
|
+
.vscode/
|
pqx-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pqx
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A fast, friendly terminal explorer for Parquet files (Textual + DuckDB)
|
|
5
|
+
Author: Mario Juric
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: textual>=1.0
|
|
10
|
+
Requires-Dist: duckdb>=1.1
|
|
11
|
+
Requires-Dist: pyarrow>=14
|
|
12
|
+
Requires-Dist: numpy>=1.23
|
|
13
|
+
Requires-Dist: pytz
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
16
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
17
|
+
Requires-Dist: pandas; extra == "dev"
|
|
18
|
+
Requires-Dist: ruff; extra == "dev"
|
|
19
|
+
|
|
20
|
+
# pqx — a terminal explorer for Parquet files
|
|
21
|
+
|
|
22
|
+
`pqx` is an interactive, keyboard-driven terminal UI for looking inside
|
|
23
|
+
Parquet files: browse rows, filter with SQL, profile columns, draw sky maps and
|
|
24
|
+
density plots, and export subsets. It is built for large LSST catalogs
|
|
25
|
+
(SSSource, SSObject, DiaSource, …) but works with any single Parquet file.
|
|
26
|
+
|
|
27
|
+
Nothing is loaded in full. The grid pulls small windows of rows, and every
|
|
28
|
+
aggregate (counts, statistics, histograms, sky maps) runs inside
|
|
29
|
+
[DuckDB](https://duckdb.org). A multi-GB file opens instantly, and jumping to
|
|
30
|
+
row 3,000,000,000 costs one row-group read.
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
pip install -e . # or: pip install -e ".[dev]" for tests
|
|
34
|
+
pqx catalog.parquet
|
|
35
|
+
pqx sssource.parquet --where "ssObjectId = 9000123"
|
|
36
|
+
python -m pqx.demo demo.parquet --rows 1000000 # a synthetic LSST-like file to play with
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## What you get
|
|
40
|
+
|
|
41
|
+
| Tab | |
|
|
42
|
+
|---|---|
|
|
43
|
+
| **Data** | A fast, scrollable grid over the entire file. Headers show type and unit, and values use astronomy-aware formatting. **d** opens a detail panel with every column of the current row at full precision, plus derived readings: MJD → UTC date, RA/Dec → sexagesimal, errors in mas, flux → AB mag. |
|
|
44
|
+
| **Schema** | Every column with its type, unit, description (from Parquet field metadata, including Felis-style `"[unit] description"`), null count, min/max from row-group statistics, compressed size, compression ratio and encodings. |
|
|
45
|
+
| **Stats** | Pick a column to see count, nulls, NaNs, distinct values, min/max, mean/std and quantiles, with a histogram for numeric and time columns or a top-values bar chart for categorical ones. |
|
|
46
|
+
| **Plot** | **Sky (Mollweide)** maps of any lon/lat pair, with RA/Dec auto-detected, and **density scatter** plots of any two numeric columns, both drawn in text with sub-character resolution and log-density colour. |
|
|
47
|
+
| **Metadata** | File overview, row-group table and key-value metadata, with JSON shown pretty-printed. |
|
|
48
|
+
|
|
49
|
+
The **filter bar** (press `/`) takes either
|
|
50
|
+
|
|
51
|
+
* a SQL `WHERE` expression: `mag < 21 and band = 'r'`, `ssObjectId is not null`,
|
|
52
|
+
`ra between 10 and 20`; or
|
|
53
|
+
* a full query over the table `t`: `select band, count(*), avg(mag) from t group by 1`.
|
|
54
|
+
|
|
55
|
+
The filter applies everywhere: grid, stats, plots and export. Column names
|
|
56
|
+
auto-complete (→ accepts), ↑/↓ recall history, and errors appear inline
|
|
57
|
+
without losing the current view.
|
|
58
|
+
|
|
59
|
+
## Keys
|
|
60
|
+
|
|
61
|
+
| key | action |
|
|
62
|
+
|---|---|
|
|
63
|
+
| `/` · `x` / `Ctrl+X` | edit filter · clear filter (`Ctrl+X` also works while typing in the filter box) |
|
|
64
|
+
| arrows, PgUp/PgDn, Ctrl+Home/End | move; the row window slides seamlessly |
|
|
65
|
+
| `g` | go to row: `1234`, `1.5M`, `50%`, `-1` |
|
|
66
|
+
| `s` | sort by the cursor column (asc → desc → off); clicking a header does the same |
|
|
67
|
+
| `=` | narrow the filter to rows equal to the cursor cell |
|
|
68
|
+
| `d` / Enter | row detail panel |
|
|
69
|
+
| `c` · `-` · `p` | choose columns · hide column · pin columns |
|
|
70
|
+
| Home / End | first / last column; `‹` `›` beside the header mark hidden columns (click to page) |
|
|
71
|
+
| `f` · `y` · `i` | raw/smart formatting · copy cell · stats for column |
|
|
72
|
+
| `1`–`5` · `Ctrl+←` `Ctrl+→` | go to a tab · previous / next tab. The strip in the panel border reads `1 Data ─ 2 Schema ─ … ^← ^→`; names underline under the mouse and switch on click |
|
|
73
|
+
| `e` | export the current view (filter + sort + visible columns) to Parquet/CSV/JSON |
|
|
74
|
+
| `m` | toggle sampling for stats and plots |
|
|
75
|
+
| `l` `L` `[` `]` | Stats: log counts, log values, fewer/more bins |
|
|
76
|
+
| click · `enter` | Plot: open a settings field's drop-down (type to narrow the column list) |
|
|
77
|
+
| `tab` · `← →` | Plot: move between settings fields · step the field's value |
|
|
78
|
+
| `r` | Plot: rotate the sky-map centre between RA 0° and 180° |
|
|
79
|
+
| Esc | cancel running queries / leave the filter bar |
|
|
80
|
+
| `?` · `q` | help · quit |
|
|
81
|
+
|
|
82
|
+
## Look
|
|
83
|
+
|
|
84
|
+
pqx draws with your terminal's own background and 16-colour palette, so it
|
|
85
|
+
matches whatever scheme you use, and it looks the same with or without 24-bit
|
|
86
|
+
colour. Panels are thin boxes, the focused one in the accent colour. Colour is
|
|
87
|
+
kept for things that mean something: the file name and other object names are
|
|
88
|
+
cyan, and status lines use ✓ (green) for done, ! (yellow) for warnings such as
|
|
89
|
+
sampling, ✗ (red) for errors, and ⠸ while running. The style follows acid's CLI
|
|
90
|
+
design language.
|
|
91
|
+
|
|
92
|
+
| option | env | |
|
|
93
|
+
|---|---|---|
|
|
94
|
+
| `--accent blue\|cyan\|magenta\|green\|yellow` | `PQX_ACCENT` | focus colour (default blue) |
|
|
95
|
+
| `--dim faint\|bright-black` | `PQX_DIM` | secondary text: the faint attribute (default), or ANSI bright black for terminals that ignore faint |
|
|
96
|
+
| `--border NAME` | `PQX_BORDER` | unfocused panel border, an ANSI colour name (default `bright_black`) |
|
|
97
|
+
| `--theme NAME` | | use a Textual theme instead of the terminal's colours |
|
|
98
|
+
|
|
99
|
+
Sky maps and density plots default to **magma**; the colormaps (magma,
|
|
100
|
+
viridis, inferno, plasma, gray) are emitted as exact xterm-256 colours, and
|
|
101
|
+
`terminal` draws density with your palette alone (faint, accent, bold).
|
|
102
|
+
|
|
103
|
+
## Large files
|
|
104
|
+
|
|
105
|
+
* **Seeking.** When no filter or sort is active, a window is fetched with DuckDB's
|
|
106
|
+
`file_row_number` pushdown. Only the row group(s) holding the window are read,
|
|
107
|
+
so paging is O(1) in file size (about 20 ms per window on a 30M-row / 850 MB file).
|
|
108
|
+
* **Filtered and sorted views** use `LIMIT/OFFSET` over the query. The total
|
|
109
|
+
row count is computed in the background, and the grid is usable before it
|
|
110
|
+
arrives.
|
|
111
|
+
* **Cancellation.** A new filter, stats request or plot interrupts the
|
|
112
|
+
now-stale DuckDB query instead of letting it run to completion. Esc cancels
|
|
113
|
+
everything that's running.
|
|
114
|
+
* **Sampling** (`m`, `--sample`) makes stats and plots read about 2M rows from
|
|
115
|
+
up to 16 evenly spaced row groups and skip the rest. It turns on
|
|
116
|
+
automatically for files over 200M rows or 8 GiB, and is worth enabling
|
|
117
|
+
whenever the storage is slow (network file systems, cold caches). Sampled
|
|
118
|
+
results are marked as such.
|
|
119
|
+
* `--threads N` caps DuckDB's parallelism on shared machines.
|
|
120
|
+
|
|
121
|
+
## Sky maps
|
|
122
|
+
|
|
123
|
+
The Mollweide renderer is a port of the one in
|
|
124
|
+
[acid](https://github.com/mjuric/acid) (`acid/io/skymap_art.py`), adapted to
|
|
125
|
+
work from an equirectangular count grid, which DuckDB bins in a single
|
|
126
|
+
`GROUP BY`, instead of a HEALPix map:
|
|
127
|
+
|
|
128
|
+
* each character cell is inverse-projected, so rendering cost doesn't depend on row count;
|
|
129
|
+
* **area** is carried by glyph shape: 2×2 quadrant sub-cells light up when at
|
|
130
|
+
least half of a sub-cell is covered, so partial coverage draws as partially
|
|
131
|
+
filled glyphs;
|
|
132
|
+
* **density** is carried by colour: log surface density in deg⁻² through a
|
|
133
|
+
selectable colormap (magma by default; viridis, inferno, plasma, gray, or
|
|
134
|
+
`terminal`), with a colorbar legend;
|
|
135
|
+
* the limb and graticule are a braille outline, and RA increases to the left.
|
|
136
|
+
|
|
137
|
+
## Versions
|
|
138
|
+
|
|
139
|
+
The version comes from git tags via [setuptools-scm](https://setuptools-scm.readthedocs.io/), as in
|
|
140
|
+
acid: a tagged commit `vX.Y.Z` is version `X.Y.Z`, and anything after it is a dev version such as
|
|
141
|
+
`0.2.dev3+g1a2b3c4`. `pqx --version` prints it.
|
|
142
|
+
|
|
143
|
+
To release, publish a GitHub Release; creating it also creates the tag:
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
gh release create v0.2.0 --generate-notes
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
That runs `.github/workflows/publish.yml`, which builds the package with `uv build` and uploads it to
|
|
150
|
+
PyPI with trusted publishing (OIDC, no stored token). The same tag runs the full test matrix in CI.
|
|
151
|
+
One-time setup on PyPI: add a trusted publisher for project `pqx`: owner `mjuric`, repository `pqx`,
|
|
152
|
+
workflow `publish.yml`, environment `pypi`.
|
|
153
|
+
|
|
154
|
+
## Development
|
|
155
|
+
|
|
156
|
+
```
|
|
157
|
+
python -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
158
|
+
.venv/bin/pytest # data layer, formatting/plots, and headless UI tests (Textual pilot)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Layout: `pqx/data.py` (DuckDB/PyArrow access layer), `pqx/fmt.py`
|
|
162
|
+
(astronomy-aware formatting), `pqx/plots.py` (text plots), `pqx/app.py` +
|
|
163
|
+
`pqx/screens.py` + `pqx/app.tcss` (the Textual UI), `pqx/demo.py` (synthetic data).
|
pqx-0.1.0/README.md
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# pqx — a terminal explorer for Parquet files
|
|
2
|
+
|
|
3
|
+
`pqx` is an interactive, keyboard-driven terminal UI for looking inside
|
|
4
|
+
Parquet files: browse rows, filter with SQL, profile columns, draw sky maps and
|
|
5
|
+
density plots, and export subsets. It is built for large LSST catalogs
|
|
6
|
+
(SSSource, SSObject, DiaSource, …) but works with any single Parquet file.
|
|
7
|
+
|
|
8
|
+
Nothing is loaded in full. The grid pulls small windows of rows, and every
|
|
9
|
+
aggregate (counts, statistics, histograms, sky maps) runs inside
|
|
10
|
+
[DuckDB](https://duckdb.org). A multi-GB file opens instantly, and jumping to
|
|
11
|
+
row 3,000,000,000 costs one row-group read.
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
pip install -e . # or: pip install -e ".[dev]" for tests
|
|
15
|
+
pqx catalog.parquet
|
|
16
|
+
pqx sssource.parquet --where "ssObjectId = 9000123"
|
|
17
|
+
python -m pqx.demo demo.parquet --rows 1000000 # a synthetic LSST-like file to play with
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## What you get
|
|
21
|
+
|
|
22
|
+
| Tab | |
|
|
23
|
+
|---|---|
|
|
24
|
+
| **Data** | A fast, scrollable grid over the entire file. Headers show type and unit, and values use astronomy-aware formatting. **d** opens a detail panel with every column of the current row at full precision, plus derived readings: MJD → UTC date, RA/Dec → sexagesimal, errors in mas, flux → AB mag. |
|
|
25
|
+
| **Schema** | Every column with its type, unit, description (from Parquet field metadata, including Felis-style `"[unit] description"`), null count, min/max from row-group statistics, compressed size, compression ratio and encodings. |
|
|
26
|
+
| **Stats** | Pick a column to see count, nulls, NaNs, distinct values, min/max, mean/std and quantiles, with a histogram for numeric and time columns or a top-values bar chart for categorical ones. |
|
|
27
|
+
| **Plot** | **Sky (Mollweide)** maps of any lon/lat pair, with RA/Dec auto-detected, and **density scatter** plots of any two numeric columns, both drawn in text with sub-character resolution and log-density colour. |
|
|
28
|
+
| **Metadata** | File overview, row-group table and key-value metadata, with JSON shown pretty-printed. |
|
|
29
|
+
|
|
30
|
+
The **filter bar** (press `/`) takes either
|
|
31
|
+
|
|
32
|
+
* a SQL `WHERE` expression: `mag < 21 and band = 'r'`, `ssObjectId is not null`,
|
|
33
|
+
`ra between 10 and 20`; or
|
|
34
|
+
* a full query over the table `t`: `select band, count(*), avg(mag) from t group by 1`.
|
|
35
|
+
|
|
36
|
+
The filter applies everywhere: grid, stats, plots and export. Column names
|
|
37
|
+
auto-complete (→ accepts), ↑/↓ recall history, and errors appear inline
|
|
38
|
+
without losing the current view.
|
|
39
|
+
|
|
40
|
+
## Keys
|
|
41
|
+
|
|
42
|
+
| key | action |
|
|
43
|
+
|---|---|
|
|
44
|
+
| `/` · `x` / `Ctrl+X` | edit filter · clear filter (`Ctrl+X` also works while typing in the filter box) |
|
|
45
|
+
| arrows, PgUp/PgDn, Ctrl+Home/End | move; the row window slides seamlessly |
|
|
46
|
+
| `g` | go to row: `1234`, `1.5M`, `50%`, `-1` |
|
|
47
|
+
| `s` | sort by the cursor column (asc → desc → off); clicking a header does the same |
|
|
48
|
+
| `=` | narrow the filter to rows equal to the cursor cell |
|
|
49
|
+
| `d` / Enter | row detail panel |
|
|
50
|
+
| `c` · `-` · `p` | choose columns · hide column · pin columns |
|
|
51
|
+
| Home / End | first / last column; `‹` `›` beside the header mark hidden columns (click to page) |
|
|
52
|
+
| `f` · `y` · `i` | raw/smart formatting · copy cell · stats for column |
|
|
53
|
+
| `1`–`5` · `Ctrl+←` `Ctrl+→` | go to a tab · previous / next tab. The strip in the panel border reads `1 Data ─ 2 Schema ─ … ^← ^→`; names underline under the mouse and switch on click |
|
|
54
|
+
| `e` | export the current view (filter + sort + visible columns) to Parquet/CSV/JSON |
|
|
55
|
+
| `m` | toggle sampling for stats and plots |
|
|
56
|
+
| `l` `L` `[` `]` | Stats: log counts, log values, fewer/more bins |
|
|
57
|
+
| click · `enter` | Plot: open a settings field's drop-down (type to narrow the column list) |
|
|
58
|
+
| `tab` · `← →` | Plot: move between settings fields · step the field's value |
|
|
59
|
+
| `r` | Plot: rotate the sky-map centre between RA 0° and 180° |
|
|
60
|
+
| Esc | cancel running queries / leave the filter bar |
|
|
61
|
+
| `?` · `q` | help · quit |
|
|
62
|
+
|
|
63
|
+
## Look
|
|
64
|
+
|
|
65
|
+
pqx draws with your terminal's own background and 16-colour palette, so it
|
|
66
|
+
matches whatever scheme you use, and it looks the same with or without 24-bit
|
|
67
|
+
colour. Panels are thin boxes, the focused one in the accent colour. Colour is
|
|
68
|
+
kept for things that mean something: the file name and other object names are
|
|
69
|
+
cyan, and status lines use ✓ (green) for done, ! (yellow) for warnings such as
|
|
70
|
+
sampling, ✗ (red) for errors, and ⠸ while running. The style follows acid's CLI
|
|
71
|
+
design language.
|
|
72
|
+
|
|
73
|
+
| option | env | |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| `--accent blue\|cyan\|magenta\|green\|yellow` | `PQX_ACCENT` | focus colour (default blue) |
|
|
76
|
+
| `--dim faint\|bright-black` | `PQX_DIM` | secondary text: the faint attribute (default), or ANSI bright black for terminals that ignore faint |
|
|
77
|
+
| `--border NAME` | `PQX_BORDER` | unfocused panel border, an ANSI colour name (default `bright_black`) |
|
|
78
|
+
| `--theme NAME` | | use a Textual theme instead of the terminal's colours |
|
|
79
|
+
|
|
80
|
+
Sky maps and density plots default to **magma**; the colormaps (magma,
|
|
81
|
+
viridis, inferno, plasma, gray) are emitted as exact xterm-256 colours, and
|
|
82
|
+
`terminal` draws density with your palette alone (faint, accent, bold).
|
|
83
|
+
|
|
84
|
+
## Large files
|
|
85
|
+
|
|
86
|
+
* **Seeking.** When no filter or sort is active, a window is fetched with DuckDB's
|
|
87
|
+
`file_row_number` pushdown. Only the row group(s) holding the window are read,
|
|
88
|
+
so paging is O(1) in file size (about 20 ms per window on a 30M-row / 850 MB file).
|
|
89
|
+
* **Filtered and sorted views** use `LIMIT/OFFSET` over the query. The total
|
|
90
|
+
row count is computed in the background, and the grid is usable before it
|
|
91
|
+
arrives.
|
|
92
|
+
* **Cancellation.** A new filter, stats request or plot interrupts the
|
|
93
|
+
now-stale DuckDB query instead of letting it run to completion. Esc cancels
|
|
94
|
+
everything that's running.
|
|
95
|
+
* **Sampling** (`m`, `--sample`) makes stats and plots read about 2M rows from
|
|
96
|
+
up to 16 evenly spaced row groups and skip the rest. It turns on
|
|
97
|
+
automatically for files over 200M rows or 8 GiB, and is worth enabling
|
|
98
|
+
whenever the storage is slow (network file systems, cold caches). Sampled
|
|
99
|
+
results are marked as such.
|
|
100
|
+
* `--threads N` caps DuckDB's parallelism on shared machines.
|
|
101
|
+
|
|
102
|
+
## Sky maps
|
|
103
|
+
|
|
104
|
+
The Mollweide renderer is a port of the one in
|
|
105
|
+
[acid](https://github.com/mjuric/acid) (`acid/io/skymap_art.py`), adapted to
|
|
106
|
+
work from an equirectangular count grid, which DuckDB bins in a single
|
|
107
|
+
`GROUP BY`, instead of a HEALPix map:
|
|
108
|
+
|
|
109
|
+
* each character cell is inverse-projected, so rendering cost doesn't depend on row count;
|
|
110
|
+
* **area** is carried by glyph shape: 2×2 quadrant sub-cells light up when at
|
|
111
|
+
least half of a sub-cell is covered, so partial coverage draws as partially
|
|
112
|
+
filled glyphs;
|
|
113
|
+
* **density** is carried by colour: log surface density in deg⁻² through a
|
|
114
|
+
selectable colormap (magma by default; viridis, inferno, plasma, gray, or
|
|
115
|
+
`terminal`), with a colorbar legend;
|
|
116
|
+
* the limb and graticule are a braille outline, and RA increases to the left.
|
|
117
|
+
|
|
118
|
+
## Versions
|
|
119
|
+
|
|
120
|
+
The version comes from git tags via [setuptools-scm](https://setuptools-scm.readthedocs.io/), as in
|
|
121
|
+
acid: a tagged commit `vX.Y.Z` is version `X.Y.Z`, and anything after it is a dev version such as
|
|
122
|
+
`0.2.dev3+g1a2b3c4`. `pqx --version` prints it.
|
|
123
|
+
|
|
124
|
+
To release, publish a GitHub Release; creating it also creates the tag:
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
gh release create v0.2.0 --generate-notes
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
That runs `.github/workflows/publish.yml`, which builds the package with `uv build` and uploads it to
|
|
131
|
+
PyPI with trusted publishing (OIDC, no stored token). The same tag runs the full test matrix in CI.
|
|
132
|
+
One-time setup on PyPI: add a trusted publisher for project `pqx`: owner `mjuric`, repository `pqx`,
|
|
133
|
+
workflow `publish.yml`, environment `pypi`.
|
|
134
|
+
|
|
135
|
+
## Development
|
|
136
|
+
|
|
137
|
+
```
|
|
138
|
+
python -m venv .venv && .venv/bin/pip install -e ".[dev]"
|
|
139
|
+
.venv/bin/pytest # data layer, formatting/plots, and headless UI tests (Textual pilot)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Layout: `pqx/data.py` (DuckDB/PyArrow access layer), `pqx/fmt.py`
|
|
143
|
+
(astronomy-aware formatting), `pqx/plots.py` (text plots), `pqx/app.py` +
|
|
144
|
+
`pqx/screens.py` + `pqx/app.tcss` (the Textual UI), `pqx/demo.py` (synthetic data).
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""pqx — a fast, friendly terminal explorer for Parquet files."""
|
|
2
|
+
|
|
3
|
+
try: # written by setuptools-scm from the git tags at build/install time
|
|
4
|
+
from pqx._version import version as __version__
|
|
5
|
+
except ImportError: # a source checkout that was never built or installed
|
|
6
|
+
__version__ = "0.0.0.dev0"
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Work-arounds for terminals and multiplexers that speak older mouse protocols.
|
|
2
|
+
|
|
3
|
+
pqx asks for SGR mouse reporting (mode 1006), like every Textual app. Some
|
|
4
|
+
multiplexers (GNU screen 4.x in particular) don't support SGR and hand the app
|
|
5
|
+
X10-encoded events instead -- ``ESC [ M`` followed by three raw bytes -- or
|
|
6
|
+
urxvt-encoded ones (``ESC [ b;x;y M``). Textual (as of 8.x) has two problems
|
|
7
|
+
with that:
|
|
8
|
+
|
|
9
|
+
* it only parses SGR mouse events, so X10/urxvt clicks are silently dropped;
|
|
10
|
+
* it decodes terminal input as strict UTF-8, and an X10 coordinate above 95
|
|
11
|
+
is a byte > 127 that is not valid UTF-8: the input thread dies with
|
|
12
|
+
``UnicodeDecodeError`` while the app keeps drawing -- it looks frozen.
|
|
13
|
+
|
|
14
|
+
:func:`install` makes input decoding lenient (an invalid byte becomes the
|
|
15
|
+
character with that code, which is exactly what an X10 coordinate means) and
|
|
16
|
+
translates X10/urxvt mouse events into SGR before Textual parses them.
|
|
17
|
+
|
|
18
|
+
It also keeps mouse coordinates in character cells. When a terminal says it
|
|
19
|
+
supports in-band resize reports (mode 2048), Textual switches the mouse to
|
|
20
|
+
pixel coordinates (mode 1016) and relies on those reports to convert back to
|
|
21
|
+
cells. iTerm2 accepts 1016 but doesn't send the reports, so every click lands
|
|
22
|
+
at a pixel position far off-screen and does nothing. Textual guards against
|
|
23
|
+
this only when ``TERM_PROGRAM``/``LC_TERMINAL`` name iTerm, which they don't
|
|
24
|
+
over ssh. pqx never needs sub-cell mouse precision, so pixel mode stays off.
|
|
25
|
+
|
|
26
|
+
Finally, quitting always leaves a clean terminal. Textual draws on the
|
|
27
|
+
alternate screen (mode 1049) and switches back on exit, which restores the
|
|
28
|
+
shell's previous contents. Where the alternate screen is unavailable (some
|
|
29
|
+
ssh/container setups, or a terminal or tmux with it turned off), pqx would
|
|
30
|
+
have drawn over the normal screen and left its last frame behind, with the
|
|
31
|
+
prompt in the middle of it. pqx clears the screen just before switching
|
|
32
|
+
back: invisible when the alternate screen works, a clean screen when not.
|
|
33
|
+
"""
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import codecs
|
|
37
|
+
import functools
|
|
38
|
+
import re
|
|
39
|
+
|
|
40
|
+
_ERRORS = "pqx-bytes"
|
|
41
|
+
_X10 = re.compile(r"\x1b\[M(.)(.)(.)\Z", re.S)
|
|
42
|
+
_URXVT = re.compile(r"\x1b\[(\d+);(\d+);(\d+)M\Z")
|
|
43
|
+
_installed = False
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _bytes_as_chars(err: UnicodeDecodeError):
|
|
47
|
+
return "".join(chr(b) for b in err.object[err.start:err.end]), err.end
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def x10_to_sgr(code: str, last_button: int = 0) -> tuple[str, int] | None:
|
|
51
|
+
"""Translate an X10 or urxvt mouse sequence to SGR. Returns (sgr, last_button)."""
|
|
52
|
+
m = _X10.match(code)
|
|
53
|
+
if m:
|
|
54
|
+
b, x, y = (ord(c) - 32 for c in m.groups())
|
|
55
|
+
else:
|
|
56
|
+
m = _URXVT.match(code)
|
|
57
|
+
if not m:
|
|
58
|
+
return None
|
|
59
|
+
b, x, y = int(m.group(1)) - 32, int(m.group(2)), int(m.group(3))
|
|
60
|
+
if x < 1 or y < 1:
|
|
61
|
+
return None
|
|
62
|
+
if b & 64 or b & 32: # wheel, or motion (with or without a button held)
|
|
63
|
+
return f"\x1b[<{b};{x};{y}M", last_button
|
|
64
|
+
if b & 3 == 3: # X10 release doesn't say which button: use the one pressed last
|
|
65
|
+
return f"\x1b[<{last_button | (b & ~3)};{x};{y}m", last_button
|
|
66
|
+
return f"\x1b[<{b};{x};{y}M", b & 3
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def install() -> None:
|
|
70
|
+
"""Patch Textual's input path (idempotent)."""
|
|
71
|
+
global _installed
|
|
72
|
+
if _installed:
|
|
73
|
+
return
|
|
74
|
+
_installed = True
|
|
75
|
+
try:
|
|
76
|
+
codecs.lookup_error(_ERRORS)
|
|
77
|
+
except LookupError:
|
|
78
|
+
codecs.register_error(_ERRORS, _bytes_as_chars)
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
from textual.drivers import linux_driver
|
|
82
|
+
|
|
83
|
+
real = codecs.getincrementaldecoder
|
|
84
|
+
|
|
85
|
+
def lenient(encoding: str):
|
|
86
|
+
return functools.partial(real(encoding), errors=_ERRORS)
|
|
87
|
+
|
|
88
|
+
linux_driver.getincrementaldecoder = lenient
|
|
89
|
+
except Exception: # pragma: no cover - other platforms / Textual versions
|
|
90
|
+
pass
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
from textual.drivers.linux_driver import LinuxDriver
|
|
94
|
+
|
|
95
|
+
LinuxDriver._enable_mouse_pixels = lambda self: None # cells, always (see above)
|
|
96
|
+
|
|
97
|
+
stop = LinuxDriver.stop_application_mode
|
|
98
|
+
|
|
99
|
+
@functools.wraps(stop)
|
|
100
|
+
def stop_application_mode(self):
|
|
101
|
+
self.write("\x1b[0m\x1b[H\x1b[2J") # reset attributes, home, clear: before leaving 1049
|
|
102
|
+
stop(self)
|
|
103
|
+
|
|
104
|
+
LinuxDriver.stop_application_mode = stop_application_mode
|
|
105
|
+
except Exception: # pragma: no cover
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
try:
|
|
109
|
+
from textual._xterm_parser import XTermParser
|
|
110
|
+
|
|
111
|
+
original = XTermParser.parse_mouse_code
|
|
112
|
+
|
|
113
|
+
@functools.wraps(original)
|
|
114
|
+
def parse_mouse_code(self, code: str):
|
|
115
|
+
translated = x10_to_sgr(code, getattr(self, "_pqx_last_button", 0))
|
|
116
|
+
if translated is not None:
|
|
117
|
+
code, self._pqx_last_button = translated
|
|
118
|
+
return original(self, code)
|
|
119
|
+
|
|
120
|
+
XTermParser.parse_mouse_code = parse_mouse_code
|
|
121
|
+
except Exception: # pragma: no cover
|
|
122
|
+
pass
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'g006519c4f'
|