scopepull 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scopepull-0.1.0/.github/workflows/ci.yml +28 -0
- scopepull-0.1.0/.github/workflows/release.yml +50 -0
- scopepull-0.1.0/.gitignore +17 -0
- scopepull-0.1.0/LICENSE +21 -0
- scopepull-0.1.0/PKG-INFO +285 -0
- scopepull-0.1.0/README.md +236 -0
- scopepull-0.1.0/docs/API.md +482 -0
- scopepull-0.1.0/docs/images/README.md +14 -0
- scopepull-0.1.0/docs/images/cli-pull.svg +14 -0
- scopepull-0.1.0/docs/images/ddd-toggle.svg +14 -0
- scopepull-0.1.0/docs/images/hero.svg +14 -0
- scopepull-0.1.0/docs/images/result.svg +14 -0
- scopepull-0.1.0/pyproject.toml +77 -0
- scopepull-0.1.0/scripts/live_recon.py +197 -0
- scopepull-0.1.0/scripts/pull_one.py +172 -0
- scopepull-0.1.0/src/scopepull/__init__.py +3 -0
- scopepull-0.1.0/src/scopepull/catalog.py +128 -0
- scopepull-0.1.0/src/scopepull/cli.py +360 -0
- scopepull-0.1.0/src/scopepull/client.py +370 -0
- scopepull-0.1.0/src/scopepull/config.py +66 -0
- scopepull-0.1.0/src/scopepull/ingest.py +286 -0
- scopepull-0.1.0/src/scopepull/manifest.py +136 -0
- scopepull-0.1.0/src/scopepull/netcheck.py +57 -0
- scopepull-0.1.0/src/scopepull/platform/__init__.py +14 -0
- scopepull-0.1.0/src/scopepull/platform/linux.py +36 -0
- scopepull-0.1.0/src/scopepull/platform/windows.py +41 -0
- scopepull-0.1.0/src/scopepull/transfer.py +137 -0
- scopepull-0.1.0/tests/__init__.py +0 -0
- scopepull-0.1.0/tests/conftest.py +24 -0
- scopepull-0.1.0/tests/fixtures/README.md +7 -0
- scopepull-0.1.0/tests/fixtures/observations_list.json +1 -0
- scopepull-0.1.0/tests/fixtures/observations_list_odyssey_fw42.json +654 -0
- scopepull-0.1.0/tests/mock_scope.py +224 -0
- scopepull-0.1.0/tests/test_catalog.py +74 -0
- scopepull-0.1.0/tests/test_cli.py +32 -0
- scopepull-0.1.0/tests/test_client.py +129 -0
- scopepull-0.1.0/tests/test_ingest.py +105 -0
- scopepull-0.1.0/tests/test_manifest.py +72 -0
- scopepull-0.1.0/tests/test_platform.py +43 -0
- scopepull-0.1.0/tests/test_transfer.py +96 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
strategy:
|
|
11
|
+
fail-fast: false
|
|
12
|
+
matrix:
|
|
13
|
+
os: [ubuntu-latest, windows-latest]
|
|
14
|
+
python: ["3.11", "3.12"]
|
|
15
|
+
runs-on: ${{ matrix.os }}
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: astral-sh/setup-uv@v4
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python }}
|
|
21
|
+
- name: Sync
|
|
22
|
+
run: uv sync --group dev
|
|
23
|
+
- name: Ruff
|
|
24
|
+
run: uv run ruff check src tests && uv run ruff format --check src tests
|
|
25
|
+
- name: Mypy
|
|
26
|
+
run: uv run mypy
|
|
27
|
+
- name: Pytest
|
|
28
|
+
run: uv run pytest -v
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
# Tag a version (e.g. v0.1.0, matching pyproject) and this builds the wheel +
|
|
4
|
+
# sdist, publishes them to PyPI via Trusted Publishing (OIDC — no stored token),
|
|
5
|
+
# and attaches them to a GitHub Release.
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
tags: ["v*"]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: astral-sh/setup-uv@v4
|
|
16
|
+
- run: uv build
|
|
17
|
+
- uses: actions/upload-artifact@v4
|
|
18
|
+
with:
|
|
19
|
+
name: dist
|
|
20
|
+
path: dist/
|
|
21
|
+
|
|
22
|
+
pypi-publish:
|
|
23
|
+
needs: build
|
|
24
|
+
runs-on: ubuntu-latest
|
|
25
|
+
environment:
|
|
26
|
+
name: pypi
|
|
27
|
+
url: https://pypi.org/p/scopepull
|
|
28
|
+
permissions:
|
|
29
|
+
id-token: write # required for PyPI Trusted Publishing (OIDC)
|
|
30
|
+
steps:
|
|
31
|
+
- uses: actions/download-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
36
|
+
|
|
37
|
+
github-release:
|
|
38
|
+
needs: build
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
permissions:
|
|
41
|
+
contents: write
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/download-artifact@v4
|
|
44
|
+
with:
|
|
45
|
+
name: dist
|
|
46
|
+
path: dist/
|
|
47
|
+
- uses: softprops/action-gh-release@v2
|
|
48
|
+
with:
|
|
49
|
+
files: dist/*
|
|
50
|
+
generate_release_notes: true
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
venv/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
*.egg-info/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.mypy_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.coverage
|
|
12
|
+
uv.lock
|
|
13
|
+
# local pull data must never land in the repo
|
|
14
|
+
unistellar_data/
|
|
15
|
+
*.fits
|
|
16
|
+
*.zip
|
|
17
|
+
!tests/fixtures/*.zip
|
scopepull-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kyle Fox Austin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
scopepull-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: scopepull
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: One-shot stack puller for Unistellar Odyssey Pro telescopes
|
|
5
|
+
Project-URL: Repository, https://github.com/kylefoxaustin/scopepull
|
|
6
|
+
Author: Kyle Fox Austin
|
|
7
|
+
License: MIT License
|
|
8
|
+
|
|
9
|
+
Copyright (c) 2026 Kyle Fox Austin
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
in the Software without restriction, including without limitation the rights
|
|
14
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
SOFTWARE.
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Classifier: Development Status :: 3 - Alpha
|
|
30
|
+
Classifier: Intended Audience :: Science/Research
|
|
31
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
32
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
33
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
36
|
+
Classifier: Topic :: Scientific/Engineering :: Astronomy
|
|
37
|
+
Requires-Python: >=3.11
|
|
38
|
+
Requires-Dist: astropy>=6.0
|
|
39
|
+
Requires-Dist: httpx>=0.27
|
|
40
|
+
Requires-Dist: imagecodecs>=2024.1
|
|
41
|
+
Requires-Dist: numpy>=1.26
|
|
42
|
+
Requires-Dist: platformdirs>=4.2
|
|
43
|
+
Requires-Dist: rich>=13.7
|
|
44
|
+
Requires-Dist: tifffile>=2024.1
|
|
45
|
+
Requires-Dist: typer>=0.12
|
|
46
|
+
Provides-Extra: tui
|
|
47
|
+
Requires-Dist: textual>=0.60; extra == 'tui'
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
<div align="center">
|
|
51
|
+
|
|
52
|
+
<img src="docs/images/hero.svg" alt="scopepull" width="100%">
|
|
53
|
+
|
|
54
|
+
# 🔭 scopepull
|
|
55
|
+
|
|
56
|
+
**One command. Raw Bayer FITS off your Unistellar Odyssey Pro — verified, organized, done.**
|
|
57
|
+
|
|
58
|
+
[](.github/workflows/ci.yml)
|
|
59
|
+
[](pyproject.toml)
|
|
60
|
+
[](#-install)
|
|
61
|
+
[](LICENSE)
|
|
62
|
+
[](tests/)
|
|
63
|
+
|
|
64
|
+
*Connect to the scope's Wi-Fi, run `scopepull`, and every observation you don't already have lands on disk as raw Bayer TIFF **and** science-ready FITS — with the master dark and the scope's own stack alongside it.*
|
|
65
|
+
|
|
66
|
+
</div>
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
> ### 🙏 Standing on prior art
|
|
71
|
+
> The hard part — reverse-engineering the scope's HTTP API — began with
|
|
72
|
+
> [**vamshikesireddy/unistellar-data-downloader**](https://github.com/vamshikesireddy/unistellar-data-downloader)
|
|
73
|
+
> (MIT), which discovered the crucial trick: **the backend won't stream data unless a client is
|
|
74
|
+
> concurrently polling `/api/event`.** scopepull builds on that knowledge (fully documented in
|
|
75
|
+
> [docs/API.md](docs/API.md)) with a rewritten async, manifest-aware pipeline. **Just want simple
|
|
76
|
+
> "pick observations, get zips"? Use their tool** — it's excellent at exactly that. scopepull is for
|
|
77
|
+
> the archival-sync use case: idempotent re-runs, verified atomic ingest, an organized archive, and
|
|
78
|
+
> the biggest observations pulled reliably where the browser drops them.
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## ✨ What it does
|
|
83
|
+
|
|
84
|
+
```mermaid
|
|
85
|
+
flowchart LR
|
|
86
|
+
A["🔭 Odyssey Pro<br/>Wi-Fi @ 192.168.100.1"] -->|"held GET + event poll"| B["📥 scopepull"]
|
|
87
|
+
B --> C["✅ verify"]
|
|
88
|
+
C --> D["📂 organized archive<br/>TIFF + FITS + dark + stack"]
|
|
89
|
+
style A fill:#1f2937,stroke:#7aa2f7,color:#e6edf3
|
|
90
|
+
style B fill:#1f2937,stroke:#9ece6a,color:#e6edf3
|
|
91
|
+
style C fill:#1f2937,stroke:#e0af68,color:#e6edf3
|
|
92
|
+
style D fill:#1f2937,stroke:#bb9af7,color:#e6edf3
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
- 🟢 **One command** — `scopepull` pulls everything you don't already have. Safe to re-run (idempotent).
|
|
96
|
+
- 🧬 **Raw science data** — deep-sky frames arrive as raw **Bayer** mosaics, converted to FITS locally with full headers (`BAYERPAT`, `EXPTIME`, `GAIN`, `RA`/`DEC`, `DATE-OBS`). `BAYERPAT` is **measured from the pixels** — the export is **RGGB as stored** even though the sensor is GBRG (see [docs/API.md](docs/API.md#bayer-pattern)).
|
|
97
|
+
- 🎁 **The whole calibration set** — every observation ships its **master dark** and the **scope's own stack**, so you can re-stack the raw frames yourself and diff against what the scope did.
|
|
98
|
+
- 🪟🐧 **Linux *and* Windows** — same code, verified pulling multi-GB observations on both.
|
|
99
|
+
- 🛟 **Survives the flaky Pi Wi-Fi** — held-connection keepalive, atomic transactional ingest (no half-observations), retry at observation granularity.
|
|
100
|
+
|
|
101
|
+
<div align="center">
|
|
102
|
+
<img src="docs/images/cli-pull.svg" alt="scopepull pull in action" width="90%">
|
|
103
|
+
</div>
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## 🚀 Install
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
uv tool install scopepull # or: pipx install scopepull
|
|
111
|
+
```
|
|
112
|
+
<sub>(pre-release: `git clone` + `uv sync`, then `uv run scopepull …`)</sub>
|
|
113
|
+
|
|
114
|
+
## ⚡ Quickstart
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
# 1. Enable Direct Data Download in the Unistellar app (once — the scope remembers)
|
|
118
|
+
# 2. Join the scope's Wi-Fi: Odyssey-xxxx (older models: UNI-xxxx / eVscope-xxxx)
|
|
119
|
+
scopepull doctor # ✅ connectivity + DDD + free-disk check
|
|
120
|
+
scopepull # ⬇️ pull everything new
|
|
121
|
+
scopepull list # 📋 what's on the scope, and what's already local
|
|
122
|
+
scopepull status # 📦 your local archive summary
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
<div align="center">
|
|
126
|
+
<img src="docs/images/ddd-toggle.svg" alt="Enable Direct Data Download" width="70%">
|
|
127
|
+
</div>
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## 🧩 How it works (the cracked protocol)
|
|
132
|
+
|
|
133
|
+
The scope builds each export **frame-by-frame on its own hardware** (~1 frame/sec) and only streams
|
|
134
|
+
the finished zip over a **single held HTTP GET** — while a concurrent event-poll keeps the "gate"
|
|
135
|
+
open. Getting this exactly right is what makes big pulls reliable:
|
|
136
|
+
|
|
137
|
+
```mermaid
|
|
138
|
+
sequenceDiagram
|
|
139
|
+
participant P as scopepull
|
|
140
|
+
participant E as /api/event
|
|
141
|
+
participant Z as /api/observations/zip
|
|
142
|
+
P->>E: start polling (the export "gate")
|
|
143
|
+
P->>Z: GET zip (one held request)
|
|
144
|
+
Note over Z: scope builds the zip<br/>frame by frame — minutes<br/>(no bytes flow yet)
|
|
145
|
+
E-->>P: status "started" · progress 1…N
|
|
146
|
+
E-->>P: status "ended"
|
|
147
|
+
Z-->>P: full archive streams down
|
|
148
|
+
Note over P: validate → unpack → TIFF→FITS → commit
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
> 💡 **Two hard-won rules** (see [docs/API.md](docs/API.md)): **never** `cancelDownload` before the
|
|
152
|
+
> GET (it suppresses the build), and keep the socket alive with TCP keepalive so a long idle build
|
|
153
|
+
> doesn't get its connection reset.
|
|
154
|
+
|
|
155
|
+
---
|
|
156
|
+
|
|
157
|
+
## 🏗️ Architecture
|
|
158
|
+
|
|
159
|
+
```mermaid
|
|
160
|
+
flowchart TD
|
|
161
|
+
subgraph net["🌐 network"]
|
|
162
|
+
SC["client.py<br/>httpx + event pump + keepalive"]
|
|
163
|
+
end
|
|
164
|
+
subgraph core["🧠 library"]
|
|
165
|
+
CAT["catalog.py<br/>parse observations"]
|
|
166
|
+
TR["transfer.py<br/>held GET, stream, verify"]
|
|
167
|
+
ING["ingest.py<br/>unpack · TIFF→FITS · atomic"]
|
|
168
|
+
MAN["manifest.py<br/>SQLite: what's local"]
|
|
169
|
+
end
|
|
170
|
+
subgraph plat["🖥️ platform/"]
|
|
171
|
+
LX["linux.py — nmcli"]
|
|
172
|
+
WN["windows.py — netsh"]
|
|
173
|
+
end
|
|
174
|
+
CLI["cli.py — typer + rich"] --> SC --> CAT --> TR --> ING --> MAN
|
|
175
|
+
SC -.SSID / Wi-Fi.-> plat
|
|
176
|
+
style net fill:#0d1117,stroke:#7aa2f7
|
|
177
|
+
style core fill:#0d1117,stroke:#9ece6a
|
|
178
|
+
style plat fill:#0d1117,stroke:#e0af68
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Everything OS-specific lives in `platform/`; the pull → verify → ingest pipeline is pure, async, and identical on Linux and Windows.
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## 📂 What lands on disk
|
|
186
|
+
|
|
187
|
+
Each observation becomes a self-contained folder — raw frames, calibration, the scope's reference stack, and full metadata:
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
~/Astro/odyssey/2026-01-31/m81-bode-s-galaxy__38102043/
|
|
191
|
+
├── frames/
|
|
192
|
+
│ ├── ..._StackInput.tiff # 🧬 raw Bayer light frames (RGGB as stored)
|
|
193
|
+
│ └── ..._StackInput.fits # ➕ same data as FITS (BAYERPAT, EXPTIME, RA/DEC…)
|
|
194
|
+
├── calibration/
|
|
195
|
+
│ ├── ..._DarkframeMean.tiff # 🌑 master dark
|
|
196
|
+
│ └── ..._DarkframeMean.fits
|
|
197
|
+
├── reference/
|
|
198
|
+
│ ├── ..._StackSum.tiff # 🔭 the scope's OWN stacked result
|
|
199
|
+
│ └── preview.jpg # 👁️ quick-look
|
|
200
|
+
├── observation.json # 📄 all metadata + pull info
|
|
201
|
+
└── SHA256SUMS # 🔐 integrity
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
```mermaid
|
|
205
|
+
flowchart LR
|
|
206
|
+
Z["observation.zip"] --> L["🧬 StackInput ×N<br/>raw Bayer lights"]
|
|
207
|
+
Z --> DK["🌑 DarkframeMean<br/>master dark"]
|
|
208
|
+
Z --> SS["🔭 StackSum<br/>scope's stack"]
|
|
209
|
+
Z --> PV["👁️ preview.jpg"]
|
|
210
|
+
L --> F["📐 per-frame FITS<br/>BAYERPAT · EXPTIME · GAIN · RA/DEC"]
|
|
211
|
+
style Z fill:#1f2937,stroke:#7aa2f7,color:#e6edf3
|
|
212
|
+
style L fill:#1f2937,stroke:#9ece6a,color:#e6edf3
|
|
213
|
+
style F fill:#1f2937,stroke:#bb9af7,color:#e6edf3
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
**Why the calibration set matters:** you get the raw lights **+** the dark **+** the scope's own stack — so you can calibrate and re-stack yourself, then compare against the scope's result. *"Did Deep Dark eat my nebula's outer shell?"* — now you can actually check.
|
|
217
|
+
|
|
218
|
+
<div align="center">
|
|
219
|
+
<img src="docs/images/result.svg" alt="A galaxy stacked from scopepull's raw frames" width="80%">
|
|
220
|
+
</div>
|
|
221
|
+
|
|
222
|
+
---
|
|
223
|
+
|
|
224
|
+
## 🛠️ CLI reference
|
|
225
|
+
|
|
226
|
+
| Command | What it does |
|
|
227
|
+
|---|---|
|
|
228
|
+
| `scopepull` | Pull everything **new** (== `pull --new`) |
|
|
229
|
+
| `scopepull pull [--new\|--all] [--since DATE] [--target TEXT]` | Pull, filtered |
|
|
230
|
+
| `scopepull list [--json]` | List observations + whether each is already local |
|
|
231
|
+
| `scopepull doctor` | Connectivity + DDD + free-disk preflight |
|
|
232
|
+
| `scopepull status` | Local archive summary |
|
|
233
|
+
| `scopepull cancel` | Clear a stuck server-side job |
|
|
234
|
+
|
|
235
|
+
---
|
|
236
|
+
|
|
237
|
+
## ⚠️ Things that matter
|
|
238
|
+
|
|
239
|
+
- **Enable Direct Data Download first** — Unistellar app → Settings → your telescope → Download → Direct Data Download. The scope remembers it.
|
|
240
|
+
- **Don't operate the scope while pulling.** Interrupted transfers have been seen to lose observations *on the scope itself* — scopepull snapshots the catalog and is transactional, but be kind to the link.
|
|
241
|
+
- **No USB data path.** The USB-A port is power-out only; Wi-Fi is the only way in.
|
|
242
|
+
- **The scope's Wi-Fi is ~10 m, 2.4 GHz, and Pi-class.** Big observations take minutes to build server-side before they stream — that's normal; the progress line shows the build.
|
|
243
|
+
|
|
244
|
+
### 🧯 Reliable pulls on a flaky link (field-tested tips)
|
|
245
|
+
|
|
246
|
+
<details>
|
|
247
|
+
<summary><b>Big pulls drop mid-transfer on Windows</b></summary>
|
|
248
|
+
|
|
249
|
+
Two common culprits, both fixable:
|
|
250
|
+
- **USB Wi-Fi adapter power-saving** — disable it (Device Manager → *USB Root Hub* → Power Management, and set the wireless adapter to Maximum Performance in the power plan).
|
|
251
|
+
- **Windows dropping the "no-internet" scope network** — set `fMinimizeConnections = 0` under `HKLM\SOFTWARE\Policies\Microsoft\Windows\WcmSvc\GroupPolicy` (or `gpedit`: Network → Windows Connection Manager → *Minimize simultaneous connections* → allow). ⚠️ Don't restart the WLAN service to apply it — reboot instead.
|
|
252
|
+
</details>
|
|
253
|
+
|
|
254
|
+
<details>
|
|
255
|
+
<summary><b>Dual-homing: internet + scope at the same time</b></summary>
|
|
256
|
+
|
|
257
|
+
Give each radio one job so they can't fight:
|
|
258
|
+
- **Adapter A → your normal Wi-Fi/Ethernet** (internet).
|
|
259
|
+
- **Adapter B → the scope only.** Remove your home-Wi-Fi profile *from adapter B* so it can't roam off the scope mid-pull.
|
|
260
|
+
|
|
261
|
+
This is exactly how the biggest observation (710 frames, 2.3 GB) was pulled reliably.
|
|
262
|
+
</details>
|
|
263
|
+
|
|
264
|
+
---
|
|
265
|
+
|
|
266
|
+
## 🧪 Development
|
|
267
|
+
|
|
268
|
+
```bash
|
|
269
|
+
uv sync --group dev
|
|
270
|
+
uv run pytest # 47 tests, incl. a FastAPI mock of the scope's protocol
|
|
271
|
+
uv run ruff check src tests
|
|
272
|
+
uv run mypy
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
CI runs the suite on **ubuntu-latest + windows-latest** × Python 3.11/3.12.
|
|
276
|
+
|
|
277
|
+
---
|
|
278
|
+
|
|
279
|
+
## 📜 License
|
|
280
|
+
|
|
281
|
+
MIT — see [LICENSE](LICENSE). Built for [Unistellar](https://www.unistellar.com/) Odyssey / Odyssey Pro (and any `evsoft` scope: eVscope, eQuinox). Not affiliated with Unistellar.
|
|
282
|
+
|
|
283
|
+
<div align="center">
|
|
284
|
+
<sub>🌌 Go stack some galaxies.</sub>
|
|
285
|
+
</div>
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<img src="docs/images/hero.svg" alt="scopepull" width="100%">
|
|
4
|
+
|
|
5
|
+
# 🔭 scopepull
|
|
6
|
+
|
|
7
|
+
**One command. Raw Bayer FITS off your Unistellar Odyssey Pro — verified, organized, done.**
|
|
8
|
+
|
|
9
|
+
[](.github/workflows/ci.yml)
|
|
10
|
+
[](pyproject.toml)
|
|
11
|
+
[](#-install)
|
|
12
|
+
[](LICENSE)
|
|
13
|
+
[](tests/)
|
|
14
|
+
|
|
15
|
+
*Connect to the scope's Wi-Fi, run `scopepull`, and every observation you don't already have lands on disk as raw Bayer TIFF **and** science-ready FITS — with the master dark and the scope's own stack alongside it.*
|
|
16
|
+
|
|
17
|
+
</div>
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
> ### 🙏 Standing on prior art
|
|
22
|
+
> The hard part — reverse-engineering the scope's HTTP API — began with
|
|
23
|
+
> [**vamshikesireddy/unistellar-data-downloader**](https://github.com/vamshikesireddy/unistellar-data-downloader)
|
|
24
|
+
> (MIT), which discovered the crucial trick: **the backend won't stream data unless a client is
|
|
25
|
+
> concurrently polling `/api/event`.** scopepull builds on that knowledge (fully documented in
|
|
26
|
+
> [docs/API.md](docs/API.md)) with a rewritten async, manifest-aware pipeline. **Just want simple
|
|
27
|
+
> "pick observations, get zips"? Use their tool** — it's excellent at exactly that. scopepull is for
|
|
28
|
+
> the archival-sync use case: idempotent re-runs, verified atomic ingest, an organized archive, and
|
|
29
|
+
> the biggest observations pulled reliably where the browser drops them.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## ✨ What it does
|
|
34
|
+
|
|
35
|
+
```mermaid
|
|
36
|
+
flowchart LR
|
|
37
|
+
A["🔭 Odyssey Pro<br/>Wi-Fi @ 192.168.100.1"] -->|"held GET + event poll"| B["📥 scopepull"]
|
|
38
|
+
B --> C["✅ verify"]
|
|
39
|
+
C --> D["📂 organized archive<br/>TIFF + FITS + dark + stack"]
|
|
40
|
+
style A fill:#1f2937,stroke:#7aa2f7,color:#e6edf3
|
|
41
|
+
style B fill:#1f2937,stroke:#9ece6a,color:#e6edf3
|
|
42
|
+
style C fill:#1f2937,stroke:#e0af68,color:#e6edf3
|
|
43
|
+
style D fill:#1f2937,stroke:#bb9af7,color:#e6edf3
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
- 🟢 **One command** — `scopepull` pulls everything you don't already have. Safe to re-run (idempotent).
|
|
47
|
+
- 🧬 **Raw science data** — deep-sky frames arrive as raw **Bayer** mosaics, converted to FITS locally with full headers (`BAYERPAT`, `EXPTIME`, `GAIN`, `RA`/`DEC`, `DATE-OBS`). `BAYERPAT` is **measured from the pixels** — the export is **RGGB as stored** even though the sensor is GBRG (see [docs/API.md](docs/API.md#bayer-pattern)).
|
|
48
|
+
- 🎁 **The whole calibration set** — every observation ships its **master dark** and the **scope's own stack**, so you can re-stack the raw frames yourself and diff against what the scope did.
|
|
49
|
+
- 🪟🐧 **Linux *and* Windows** — same code, verified pulling multi-GB observations on both.
|
|
50
|
+
- 🛟 **Survives the flaky Pi Wi-Fi** — held-connection keepalive, atomic transactional ingest (no half-observations), retry at observation granularity.
|
|
51
|
+
|
|
52
|
+
<div align="center">
|
|
53
|
+
<img src="docs/images/cli-pull.svg" alt="scopepull pull in action" width="90%">
|
|
54
|
+
</div>
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## 🚀 Install
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
uv tool install scopepull # or: pipx install scopepull
|
|
62
|
+
```
|
|
63
|
+
<sub>(pre-release: `git clone` + `uv sync`, then `uv run scopepull …`)</sub>
|
|
64
|
+
|
|
65
|
+
## ⚡ Quickstart
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
# 1. Enable Direct Data Download in the Unistellar app (once — the scope remembers)
|
|
69
|
+
# 2. Join the scope's Wi-Fi: Odyssey-xxxx (older models: UNI-xxxx / eVscope-xxxx)
|
|
70
|
+
scopepull doctor # ✅ connectivity + DDD + free-disk check
|
|
71
|
+
scopepull # ⬇️ pull everything new
|
|
72
|
+
scopepull list # 📋 what's on the scope, and what's already local
|
|
73
|
+
scopepull status # 📦 your local archive summary
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
<div align="center">
|
|
77
|
+
<img src="docs/images/ddd-toggle.svg" alt="Enable Direct Data Download" width="70%">
|
|
78
|
+
</div>
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## 🧩 How it works (the cracked protocol)
|
|
83
|
+
|
|
84
|
+
The scope builds each export **frame-by-frame on its own hardware** (~1 frame/sec) and only streams
|
|
85
|
+
the finished zip over a **single held HTTP GET** — while a concurrent event-poll keeps the "gate"
|
|
86
|
+
open. Getting this exactly right is what makes big pulls reliable:
|
|
87
|
+
|
|
88
|
+
```mermaid
|
|
89
|
+
sequenceDiagram
|
|
90
|
+
participant P as scopepull
|
|
91
|
+
participant E as /api/event
|
|
92
|
+
participant Z as /api/observations/zip
|
|
93
|
+
P->>E: start polling (the export "gate")
|
|
94
|
+
P->>Z: GET zip (one held request)
|
|
95
|
+
Note over Z: scope builds the zip<br/>frame by frame — minutes<br/>(no bytes flow yet)
|
|
96
|
+
E-->>P: status "started" · progress 1…N
|
|
97
|
+
E-->>P: status "ended"
|
|
98
|
+
Z-->>P: full archive streams down
|
|
99
|
+
Note over P: validate → unpack → TIFF→FITS → commit
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
> 💡 **Two hard-won rules** (see [docs/API.md](docs/API.md)): **never** `cancelDownload` before the
|
|
103
|
+
> GET (it suppresses the build), and keep the socket alive with TCP keepalive so a long idle build
|
|
104
|
+
> doesn't get its connection reset.
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## 🏗️ Architecture
|
|
109
|
+
|
|
110
|
+
```mermaid
|
|
111
|
+
flowchart TD
|
|
112
|
+
subgraph net["🌐 network"]
|
|
113
|
+
SC["client.py<br/>httpx + event pump + keepalive"]
|
|
114
|
+
end
|
|
115
|
+
subgraph core["🧠 library"]
|
|
116
|
+
CAT["catalog.py<br/>parse observations"]
|
|
117
|
+
TR["transfer.py<br/>held GET, stream, verify"]
|
|
118
|
+
ING["ingest.py<br/>unpack · TIFF→FITS · atomic"]
|
|
119
|
+
MAN["manifest.py<br/>SQLite: what's local"]
|
|
120
|
+
end
|
|
121
|
+
subgraph plat["🖥️ platform/"]
|
|
122
|
+
LX["linux.py — nmcli"]
|
|
123
|
+
WN["windows.py — netsh"]
|
|
124
|
+
end
|
|
125
|
+
CLI["cli.py — typer + rich"] --> SC --> CAT --> TR --> ING --> MAN
|
|
126
|
+
SC -.SSID / Wi-Fi.-> plat
|
|
127
|
+
style net fill:#0d1117,stroke:#7aa2f7
|
|
128
|
+
style core fill:#0d1117,stroke:#9ece6a
|
|
129
|
+
style plat fill:#0d1117,stroke:#e0af68
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Everything OS-specific lives in `platform/`; the pull → verify → ingest pipeline is pure, async, and identical on Linux and Windows.
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## 📂 What lands on disk
|
|
137
|
+
|
|
138
|
+
Each observation becomes a self-contained folder — raw frames, calibration, the scope's reference stack, and full metadata:
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
~/Astro/odyssey/2026-01-31/m81-bode-s-galaxy__38102043/
|
|
142
|
+
├── frames/
|
|
143
|
+
│ ├── ..._StackInput.tiff # 🧬 raw Bayer light frames (RGGB as stored)
|
|
144
|
+
│ └── ..._StackInput.fits # ➕ same data as FITS (BAYERPAT, EXPTIME, RA/DEC…)
|
|
145
|
+
├── calibration/
|
|
146
|
+
│ ├── ..._DarkframeMean.tiff # 🌑 master dark
|
|
147
|
+
│ └── ..._DarkframeMean.fits
|
|
148
|
+
├── reference/
|
|
149
|
+
│ ├── ..._StackSum.tiff # 🔭 the scope's OWN stacked result
|
|
150
|
+
│ └── preview.jpg # 👁️ quick-look
|
|
151
|
+
├── observation.json # 📄 all metadata + pull info
|
|
152
|
+
└── SHA256SUMS # 🔐 integrity
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
```mermaid
|
|
156
|
+
flowchart LR
|
|
157
|
+
Z["observation.zip"] --> L["🧬 StackInput ×N<br/>raw Bayer lights"]
|
|
158
|
+
Z --> DK["🌑 DarkframeMean<br/>master dark"]
|
|
159
|
+
Z --> SS["🔭 StackSum<br/>scope's stack"]
|
|
160
|
+
Z --> PV["👁️ preview.jpg"]
|
|
161
|
+
L --> F["📐 per-frame FITS<br/>BAYERPAT · EXPTIME · GAIN · RA/DEC"]
|
|
162
|
+
style Z fill:#1f2937,stroke:#7aa2f7,color:#e6edf3
|
|
163
|
+
style L fill:#1f2937,stroke:#9ece6a,color:#e6edf3
|
|
164
|
+
style F fill:#1f2937,stroke:#bb9af7,color:#e6edf3
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
**Why the calibration set matters:** you get the raw lights **+** the dark **+** the scope's own stack — so you can calibrate and re-stack yourself, then compare against the scope's result. *"Did Deep Dark eat my nebula's outer shell?"* — now you can actually check.
|
|
168
|
+
|
|
169
|
+
<div align="center">
|
|
170
|
+
<img src="docs/images/result.svg" alt="A galaxy stacked from scopepull's raw frames" width="80%">
|
|
171
|
+
</div>
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## 🛠️ CLI reference
|
|
176
|
+
|
|
177
|
+
| Command | What it does |
|
|
178
|
+
|---|---|
|
|
179
|
+
| `scopepull` | Pull everything **new** (== `pull --new`) |
|
|
180
|
+
| `scopepull pull [--new\|--all] [--since DATE] [--target TEXT]` | Pull, filtered |
|
|
181
|
+
| `scopepull list [--json]` | List observations + whether each is already local |
|
|
182
|
+
| `scopepull doctor` | Connectivity + DDD + free-disk preflight |
|
|
183
|
+
| `scopepull status` | Local archive summary |
|
|
184
|
+
| `scopepull cancel` | Clear a stuck server-side job |
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## ⚠️ Things that matter
|
|
189
|
+
|
|
190
|
+
- **Enable Direct Data Download first** — Unistellar app → Settings → your telescope → Download → Direct Data Download. The scope remembers it.
|
|
191
|
+
- **Don't operate the scope while pulling.** Interrupted transfers have been seen to lose observations *on the scope itself* — scopepull snapshots the catalog and is transactional, but be kind to the link.
|
|
192
|
+
- **No USB data path.** The USB-A port is power-out only; Wi-Fi is the only way in.
|
|
193
|
+
- **The scope's Wi-Fi is ~10 m, 2.4 GHz, and Pi-class.** Big observations take minutes to build server-side before they stream — that's normal; the progress line shows the build.
|
|
194
|
+
|
|
195
|
+
### 🧯 Reliable pulls on a flaky link (field-tested tips)
|
|
196
|
+
|
|
197
|
+
<details>
|
|
198
|
+
<summary><b>Big pulls drop mid-transfer on Windows</b></summary>
|
|
199
|
+
|
|
200
|
+
Two common culprits, both fixable:
|
|
201
|
+
- **USB Wi-Fi adapter power-saving** — disable it (Device Manager → *USB Root Hub* → Power Management, and set the wireless adapter to Maximum Performance in the power plan).
|
|
202
|
+
- **Windows dropping the "no-internet" scope network** — set `fMinimizeConnections = 0` under `HKLM\SOFTWARE\Policies\Microsoft\Windows\WcmSvc\GroupPolicy` (or `gpedit`: Network → Windows Connection Manager → *Minimize simultaneous connections* → allow). ⚠️ Don't restart the WLAN service to apply it — reboot instead.
|
|
203
|
+
</details>
|
|
204
|
+
|
|
205
|
+
<details>
|
|
206
|
+
<summary><b>Dual-homing: internet + scope at the same time</b></summary>
|
|
207
|
+
|
|
208
|
+
Give each radio one job so they can't fight:
|
|
209
|
+
- **Adapter A → your normal Wi-Fi/Ethernet** (internet).
|
|
210
|
+
- **Adapter B → the scope only.** Remove your home-Wi-Fi profile *from adapter B* so it can't roam off the scope mid-pull.
|
|
211
|
+
|
|
212
|
+
This is exactly how the biggest observation (710 frames, 2.3 GB) was pulled reliably.
|
|
213
|
+
</details>
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## 🧪 Development
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
uv sync --group dev
|
|
221
|
+
uv run pytest # 47 tests, incl. a FastAPI mock of the scope's protocol
|
|
222
|
+
uv run ruff check src tests
|
|
223
|
+
uv run mypy
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
CI runs the suite on **ubuntu-latest + windows-latest** × Python 3.11/3.12.
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
## 📜 License
|
|
231
|
+
|
|
232
|
+
MIT — see [LICENSE](LICENSE). Built for [Unistellar](https://www.unistellar.com/) Odyssey / Odyssey Pro (and any `evsoft` scope: eVscope, eQuinox). Not affiliated with Unistellar.
|
|
233
|
+
|
|
234
|
+
<div align="center">
|
|
235
|
+
<sub>🌌 Go stack some galaxies.</sub>
|
|
236
|
+
</div>
|