TESSify 0.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tessify-0.0.0/LICENSE +21 -0
- tessify-0.0.0/PKG-INFO +5 -0
- tessify-0.0.0/README.md +232 -0
- tessify-0.0.0/TESSify/__init__.py +730 -0
- tessify-0.0.0/TESSify.egg-info/PKG-INFO +5 -0
- tessify-0.0.0/TESSify.egg-info/SOURCES.txt +8 -0
- tessify-0.0.0/TESSify.egg-info/dependency_links.txt +1 -0
- tessify-0.0.0/TESSify.egg-info/top_level.txt +1 -0
- tessify-0.0.0/pyproject.toml +3 -0
- tessify-0.0.0/setup.cfg +4 -0
tessify-0.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 arpit290
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tessify-0.0.0/PKG-INFO
ADDED
tessify-0.0.0/README.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# TESSify
|
|
2
|
+
A Python package for helping with detecting exoplanet transit dips in TESS light curves.
|
|
3
|
+
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
## 🚀 Features
|
|
7
|
+
|
|
8
|
+
- Load and clean `.fits` light curve or target pixel files
|
|
9
|
+
- Visualize flattened light curves and shortlist transit candidates
|
|
10
|
+
- Tune the flattening window and the pixel selection (`window_length`, `aperture_mask`) to suit your targets
|
|
11
|
+
- Download and process data in bulk, with parallel downloads and optional multi-core processing
|
|
12
|
+
- Resumable at every stage: stop at any time (even with Ctrl-C) and pick up where you left off
|
|
13
|
+
- Failed downloads and plots are recorded and retried on the next run instead of crashing the batch
|
|
14
|
+
- Designed for students, researchers, and citizen scientists
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## 📦 Installation
|
|
19
|
+
|
|
20
|
+
Requires **Python 3.9 or newer**.
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install TESSify
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Or install the dependencies and run from source:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install -r requirements.txt
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Two things pip can't install for you:
|
|
33
|
+
|
|
34
|
+
- **`curl`** must be available on your PATH. It is used to run the bulk download scripts. It ships with Windows 10+, macOS, and most Linux distributions.
|
|
35
|
+
- **`tkinter`** is needed for the image viewer. It is bundled with the Windows and macOS Python installers. On Debian/Ubuntu, run `sudo apt install python3-tk`. Without it, downloading and processing still work; only `finalise()` needs it.
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## 🛠️ Usage
|
|
40
|
+
|
|
41
|
+
The typical workflow is: **create → download → process → finalise → get results**.
|
|
42
|
+
|
|
43
|
+
### Importing TESSify
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from TESSify import Project
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### Creating a Project
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
a = Project()
|
|
53
|
+
a.create("YOUR_PROJECT_NAME")
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
This makes a new folder with that name. `create` refuses to overwrite a project that already exists.
|
|
57
|
+
|
|
58
|
+
### Restoring a Project
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
a = Project()
|
|
62
|
+
a.restore("YOUR_PROJECT_NAME")
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Downloading `.fits` files in bulk
|
|
66
|
+
|
|
67
|
+
Get a bulk download script from the official [TESS archive](https://archive.stsci.edu/tess/bulk_downloads/bulk_downloads_ffi-tp-lc-dv.html). You can download either Target Pixel (`tp`) or Light Curve (`lc`) files. After downloading the script, copy its path.
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from TESSify import Project
|
|
71
|
+
|
|
72
|
+
a = Project()
|
|
73
|
+
a.create("Project1")
|
|
74
|
+
|
|
75
|
+
a.download("lc", r"C:\Users\Expert\Downloads\tesscurl_sector_89_lc.sh", 100)
|
|
76
|
+
# 1st argument: "lc" for light curves, "tp" for target pixel files
|
|
77
|
+
# 2nd argument: path to the download script
|
|
78
|
+
# 3rd argument: how many files to download
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Optional: download several files at once (default is 4).
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
a.download("lc", "tesscurl_sector_89_lc.sh", 100, workers=8)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Notes:
|
|
88
|
+
|
|
89
|
+
- Running `download` again continues where it stopped: files that are already downloaded are skipped, and files that failed are retried.
|
|
90
|
+
- To fetch more files from the same script, call `download` again with a larger amount.
|
|
91
|
+
- A project holds one file type. To work with both `lc` and `tp` files, create two projects.
|
|
92
|
+
|
|
93
|
+
### Processing the files in bulk
|
|
94
|
+
|
|
95
|
+
Processing cleans each file (removes NaNs, flattens the curve) and saves a plot of it in the project's `Graphs` folder.
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from TESSify import Project
|
|
99
|
+
|
|
100
|
+
a = Project()
|
|
101
|
+
a.restore("Project1")
|
|
102
|
+
|
|
103
|
+
a.process(100)
|
|
104
|
+
# The amount can't usefully exceed the number of files you have downloaded
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Optional: use several CPU cores (default is 1).
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
a.process(100, workers=4)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
> **Windows/macOS users:** when using `workers` greater than 1, put your script's top-level code under `if __name__ == "__main__":`. This is the standard rule for Python multiprocessing. Each worker also loads its own copy of the data, so keep `workers` modest if you are short on memory.
|
|
114
|
+
|
|
115
|
+
Files that can't be read (for example, a corrupt download) are skipped, reported in the console, and listed under `process_failed` in `config.json`. Re-running `process` retries them.
|
|
116
|
+
|
|
117
|
+
#### Customising the processing
|
|
118
|
+
|
|
119
|
+
Two optional settings control how each file is cleaned:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
a.process(100, window_length=401, aperture_mask="pipeline")
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
| Setting | Default | Applies to | What it does |
|
|
126
|
+
|---------|---------|------------|--------------|
|
|
127
|
+
| `window_length` | `101` | `lc` and `tp` | Size, in data points, of the smoothing window `flatten()` uses to remove slow brightness drift. Must be an odd integer of 3 or more. |
|
|
128
|
+
| `aperture_mask` | `"all"` | `tp` only | Which pixels are added together to build a light curve from a target pixel file. Ignored for `lc` files. |
|
|
129
|
+
|
|
130
|
+
**`window_length`:** the flattening filter smooths away anything shorter than its window, and a transit is a short dip. At the default of 101 points (about 3.4 hours for 2-minute data), a long transit can be partly flattened away. Use a window several times longer than the transits you are looking for, for example `401` or `1001` for 2-minute data. If the window is too large, slow stellar variability stays in the curve; if it is too small, real dips get erased. Longer cadences (10 or 30 minutes) need fewer points to cover the same time span.
|
|
131
|
+
|
|
132
|
+
**`aperture_mask`:** `"all"` adds up every pixel in the cutout, including faint, noisy ones. Other options are `"pipeline"` (the pixels chosen by the TESS mission pipeline, usually cleaner) and `"threshold"` (only pixels above a brightness threshold). If your target pixel plots look noisy, try `"pipeline"`.
|
|
133
|
+
|
|
134
|
+
Files that already have a plot are skipped, so changing these settings only affects files processed afterwards. To re-plot everything with new settings, delete the images in `Graphs/`, then set `"processed"` to `[]` and `"reviewed"` to `0` in `config.json`. Your shortlist (`"marked"`) is kept.
|
|
135
|
+
|
|
136
|
+
To process a single file and get the image name back (it accepts the same settings):
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
image = a.indprocess("tess2019198215352-s0014-0000000261136679-0150-s_lc.fits", window_length=401)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### Finalising the light curves and watching out for transit dips
|
|
143
|
+
|
|
144
|
+
After processing, you can flip through the graphs and shortlist any that show dips that could indicate an exoplanet.
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from TESSify import Project
|
|
148
|
+
|
|
149
|
+
a = Project()
|
|
150
|
+
a.restore("Project1")
|
|
151
|
+
|
|
152
|
+
a.finalise(100)
|
|
153
|
+
# Opens a window showing the processed graphs
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Viewer controls:
|
|
157
|
+
|
|
158
|
+
| Key | Action |
|
|
159
|
+
|-----|--------|
|
|
160
|
+
| **Right arrow** | Next graph |
|
|
161
|
+
| **Left arrow** | Previous graph |
|
|
162
|
+
| **Down arrow** | Shortlist the current graph (if you see something of interest) |
|
|
163
|
+
| **Up arrow** | Remove the current graph from the shortlist |
|
|
164
|
+
| **Escape** (or closing the window) | Save and exit |
|
|
165
|
+
|
|
166
|
+
The title bar shows your position and whether the current graph is shortlisted. Your shortlist is saved as you go, and the next call to `finalise` starts after the last graph you reached.
|
|
167
|
+
|
|
168
|
+
### Getting the list of shortlisted entities
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
from TESSify import Project
|
|
172
|
+
|
|
173
|
+
a = Project()
|
|
174
|
+
a.restore("Project1")
|
|
175
|
+
|
|
176
|
+
finalised_list = a.getresults()
|
|
177
|
+
# Returns unique TIC IDs in the order you shortlisted them
|
|
178
|
+
|
|
179
|
+
print(finalised_list)
|
|
180
|
+
# ['TIC 261136679', 'TIC 25155310']
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
### Saving the shortlisted entities as a JSON file
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
from TESSify import Project
|
|
187
|
+
|
|
188
|
+
a = Project()
|
|
189
|
+
a.restore("Project1")
|
|
190
|
+
|
|
191
|
+
a.saveresults("shortlist.json")
|
|
192
|
+
# Any path works, e.g. r"C:\Users\Expert\Documents\shortlist.json" or "/home/expert/shortlist.json"
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## 📁 Project layout
|
|
198
|
+
|
|
199
|
+
```
|
|
200
|
+
YOUR_PROJECT_NAME/
|
|
201
|
+
├── config.json # progress for every stage (safe to leave alone)
|
|
202
|
+
├── LightCurves/ # downloaded .fits light curves (type "lc")
|
|
203
|
+
├── TargetPixels/ # downloaded .fits target pixel files (type "tp")
|
|
204
|
+
└── Graphs/ # plots made by process()
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
`config.json` is written safely, so a crash can't leave it half-written. Projects created with older versions of TESSify load automatically. Re-run `process` on them, because plot names created by older versions may not match.
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## ❓ Troubleshooting
|
|
212
|
+
|
|
213
|
+
- **`FileExistsError` on `create`:** a folder with that name already exists. Use `restore` instead, or pick a new name.
|
|
214
|
+
- **`FileNotFoundError` on `restore`:** the folder doesn't contain a `config.json`. Check the name and your working directory, since projects are found relative to where you run Python.
|
|
215
|
+
- **Downloads show `FAILED`:** check your connection and that `curl` is installed, then re-run `download`. Failed files are retried automatically.
|
|
216
|
+
- **`The viewer needs tkinter and Pillow`:** install them as described under [Installation](#-installation).
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## 🧠 Contributing
|
|
221
|
+
Contributions are welcome! Open an issue or submit a pull request at [github.com/arpit290/TESSify](https://github.com/arpit290/TESSify).
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## 📜 License
|
|
226
|
+
This project is licensed under the MIT License. See `LICENSE` for details.
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
## 👤 Author
|
|
231
|
+
Arpit Bishnoi
|
|
232
|
+
[github.com/arpit290](https://github.com/arpit290) · bishnoiarpit29@gmail.com
|
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
"""TESS light-curve / target-pixel project manager (needs Python 3.10+).
|
|
2
|
+
|
|
3
|
+
Typical workflow:
|
|
4
|
+
|
|
5
|
+
project = Project()
|
|
6
|
+
project.create("my_project") # or: project.restore("my_project")
|
|
7
|
+
project.download("lc", "bulk_download.sh", 100)
|
|
8
|
+
project.process(100)
|
|
9
|
+
project.finalise(100) # Down = mark, Up = unmark, Esc = quit
|
|
10
|
+
project.saveresults("candidates.json")
|
|
11
|
+
|
|
12
|
+
Every stage is resumable: progress lives in <project>/config.json.
|
|
13
|
+
|
|
14
|
+
Note: if you call process(..., workers=N) with N > 1 on Windows/macOS, put your
|
|
15
|
+
script's top-level code under ``if __name__ == "__main__":`` (standard rule for
|
|
16
|
+
multiprocessing).
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import gc
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import shlex
|
|
25
|
+
import subprocess
|
|
26
|
+
import shutil
|
|
27
|
+
from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, wait
|
|
28
|
+
from dataclasses import asdict, dataclass, field, fields
|
|
29
|
+
from functools import lru_cache, partial, wraps
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Callable, Iterable, Iterator
|
|
32
|
+
|
|
33
|
+
try: # the viewer is optional, so headless machines can still download and process
|
|
34
|
+
import tkinter as tk
|
|
35
|
+
from PIL import Image, ImageTk
|
|
36
|
+
except ImportError:
|
|
37
|
+
tk = None
|
|
38
|
+
|
|
39
|
+
KIND_DIRS = {"lc": "LightCurves", "tp": "TargetPixels"}
|
|
40
|
+
GRAPHS_DIR = "Graphs"
|
|
41
|
+
CONFIG_NAME = "config.json"
|
|
42
|
+
IMAGE_EXT = ".jpeg"
|
|
43
|
+
SAVE_EVERY = 25 # write config.json after this many completed files (and on exit)
|
|
44
|
+
DEFAULT_WINDOW_LENGTH = 101 # lightkurve's own default for flatten()
|
|
45
|
+
DEFAULT_APERTURE_MASK = "all" # target pixel files only: which pixels form the light curve
|
|
46
|
+
TIC_PATTERN = re.compile(r"-s\d{4}-(\d{16})-")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# --------------------------------------------------------------------------- #
|
|
50
|
+
# Small helpers
|
|
51
|
+
# --------------------------------------------------------------------------- #
|
|
52
|
+
def image_name(filename: str) -> str:
|
|
53
|
+
"""'tess...-s0014-...-s_lc.fits' -> 'tess...-s0014-...-s_lc.jpeg'."""
|
|
54
|
+
return Path(filename).stem + IMAGE_EXT
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def tic_id(name: str) -> str | None:
|
|
58
|
+
"""'tess...-s0014-0000000261136679-0150-s_lc.jpeg' -> 'TIC 261136679'."""
|
|
59
|
+
match = TIC_PATTERN.search(name)
|
|
60
|
+
if match is None:
|
|
61
|
+
return None
|
|
62
|
+
return "TIC " + (match.group(1).lstrip("0") or "0")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def read_commands(script_path: str | Path, limit: int) -> dict[str, list[str]]:
|
|
66
|
+
"""Parse the first `limit` curl commands of a MAST script into {output_file: argv}."""
|
|
67
|
+
commands: dict[str, list[str]] = {}
|
|
68
|
+
with open(script_path, "r", encoding="utf-8") as script:
|
|
69
|
+
for line in script:
|
|
70
|
+
line = line.strip()
|
|
71
|
+
if not line or line.startswith("#"):
|
|
72
|
+
continue
|
|
73
|
+
if len(commands) == limit:
|
|
74
|
+
break
|
|
75
|
+
argv = shlex.split(line)
|
|
76
|
+
commands[_output_file(argv)] = argv
|
|
77
|
+
return commands
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _output_file(argv: list[str]) -> str:
|
|
81
|
+
"""Return the file name given to curl's -o / --output flag."""
|
|
82
|
+
for flag in ("-o", "--output"):
|
|
83
|
+
if flag in argv and argv.index(flag) + 1 < len(argv):
|
|
84
|
+
return Path(argv[argv.index(flag) + 1]).name
|
|
85
|
+
raise ValueError(f"No output file (-o) found in command: {' '.join(argv)}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _write_json_atomically(path: Path, data: object) -> None:
|
|
89
|
+
"""Write via a temp file so a crash never leaves a half-written config."""
|
|
90
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
91
|
+
tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
|
92
|
+
os.replace(tmp, path)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _check_window_length(window_length: int) -> None:
|
|
96
|
+
"""flatten() needs an odd integer window of at least 3 points."""
|
|
97
|
+
if not isinstance(window_length, int) or window_length < 3 or window_length % 2 == 0:
|
|
98
|
+
raise ValueError(f"window_length must be an odd integer >= 3, got {window_length!r}")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# --------------------------------------------------------------------------- #
|
|
102
|
+
# Workers (module-level so they can be sent to other processes)
|
|
103
|
+
# --------------------------------------------------------------------------- #
|
|
104
|
+
def _download(filename: str, commands: dict[str, list[str]], cwd: Path) -> None:
|
|
105
|
+
result = subprocess.run(commands[filename], cwd=cwd)
|
|
106
|
+
if result.returncode != 0 or not (cwd / filename).is_file():
|
|
107
|
+
raise RuntimeError(f"download exited with code {result.returncode}")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _render(filename: str, kind: str, source_dir: Path, graphs_dir: Path,
|
|
111
|
+
window_length: int = DEFAULT_WINDOW_LENGTH,
|
|
112
|
+
aperture_mask: str = DEFAULT_APERTURE_MASK) -> None:
|
|
113
|
+
"""Clean one FITS file and save its plot.
|
|
114
|
+
|
|
115
|
+
window_length: smoothing window (in data points) used by flatten().
|
|
116
|
+
aperture_mask: pixels summed into a light curve; only used for target pixel files.
|
|
117
|
+
"""
|
|
118
|
+
import matplotlib
|
|
119
|
+
matplotlib.use("Agg") # no GUI windows, no clash with the Tk viewer
|
|
120
|
+
import matplotlib.pyplot as plt
|
|
121
|
+
import lightkurve as lk
|
|
122
|
+
|
|
123
|
+
data = lk.read(source_dir / filename)
|
|
124
|
+
if kind == "tp":
|
|
125
|
+
data = data.to_lightcurve(aperture_mask=aperture_mask)
|
|
126
|
+
lightcurve = data.remove_nans().flatten(window_length=window_length)
|
|
127
|
+
|
|
128
|
+
fig, ax = plt.subplots()
|
|
129
|
+
try:
|
|
130
|
+
lightcurve.plot(ax=ax)
|
|
131
|
+
fig.savefig(graphs_dir / image_name(filename), bbox_inches="tight")
|
|
132
|
+
finally:
|
|
133
|
+
plt.close(fig)
|
|
134
|
+
del data, lightcurve
|
|
135
|
+
gc.collect()
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _run_jobs(job: Callable[[str], None], items: Iterable[str],
|
|
139
|
+
workers: int, executor_cls) -> Iterator[tuple[str, Exception | None]]:
|
|
140
|
+
"""Run jobs while bounding queued futures to avoid large per-file overhead."""
|
|
141
|
+
if workers <= 1: # inline: simpler tracebacks, no pool start-up cost
|
|
142
|
+
for item in items:
|
|
143
|
+
try:
|
|
144
|
+
job(item)
|
|
145
|
+
yield item, None
|
|
146
|
+
except Exception as error:
|
|
147
|
+
yield item, error
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
pool = executor_cls(max_workers=workers)
|
|
151
|
+
iterator = iter(items)
|
|
152
|
+
futures: dict[object, str] = {}
|
|
153
|
+
|
|
154
|
+
def submit_one() -> bool:
|
|
155
|
+
try:
|
|
156
|
+
item = next(iterator)
|
|
157
|
+
except StopIteration:
|
|
158
|
+
return False
|
|
159
|
+
futures[pool.submit(job, item)] = item
|
|
160
|
+
return True
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
# Keep at most 2 * workers jobs queued. This avoids allocating thousands
|
|
164
|
+
# of Future objects when a user processes a large archive.
|
|
165
|
+
for _ in range(max(1, workers * 2)):
|
|
166
|
+
if not submit_one():
|
|
167
|
+
break
|
|
168
|
+
|
|
169
|
+
while futures:
|
|
170
|
+
completed, _ = wait(tuple(futures), return_when=FIRST_COMPLETED)
|
|
171
|
+
for future in completed:
|
|
172
|
+
item = futures.pop(future)
|
|
173
|
+
try:
|
|
174
|
+
future.result()
|
|
175
|
+
error = None
|
|
176
|
+
except Exception as caught:
|
|
177
|
+
error = caught
|
|
178
|
+
yield item, error
|
|
179
|
+
submit_one()
|
|
180
|
+
finally: # on Ctrl-C: let running jobs finish, drop queued jobs
|
|
181
|
+
for future in futures:
|
|
182
|
+
future.cancel()
|
|
183
|
+
pool.shutdown(wait=True, cancel_futures=True)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# --------------------------------------------------------------------------- #
|
|
187
|
+
# Project state (saved as config.json)
|
|
188
|
+
# --------------------------------------------------------------------------- #
|
|
189
|
+
@dataclass
|
|
190
|
+
class State:
|
|
191
|
+
name: str
|
|
192
|
+
kind: str | None = None # "lc" or "tp"
|
|
193
|
+
script: str | None = None
|
|
194
|
+
downloaded: list[str] = field(default_factory=list) # FITS file names
|
|
195
|
+
download_failed: list[str] = field(default_factory=list)
|
|
196
|
+
processed: list[str] = field(default_factory=list) # image file names
|
|
197
|
+
process_failed: list[str] = field(default_factory=list)
|
|
198
|
+
reviewed: int = 0 # count of currently existing reviewed images
|
|
199
|
+
marked: list[str] = field(default_factory=list) # images flagged in the viewer
|
|
200
|
+
review_index: int = 0 # current position in the existing-image list
|
|
201
|
+
reviewed_images: list[str] = field(default_factory=list) # durable review identity
|
|
202
|
+
review_current: str | None = None # filename at review_index, if available
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def load(cls, path: Path) -> State:
|
|
206
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
207
|
+
if "Download_Progress" in data: # config written by the older versions
|
|
208
|
+
data = cls._from_legacy(data)
|
|
209
|
+
|
|
210
|
+
# Older configs only stored a high-water count. Migrate that count to
|
|
211
|
+
# filenames once, preserving the old sequential-review interpretation.
|
|
212
|
+
if "reviewed_images" not in data:
|
|
213
|
+
processed = data.get("processed", [])
|
|
214
|
+
count = data.get("reviewed", 0)
|
|
215
|
+
if not isinstance(count, int) or count < 0:
|
|
216
|
+
count = 0
|
|
217
|
+
data["reviewed_images"] = processed[:count] if isinstance(processed, list) else []
|
|
218
|
+
|
|
219
|
+
known = {f.name for f in fields(cls)}
|
|
220
|
+
state = cls(**{key: value for key, value in data.items() if key in known})
|
|
221
|
+
if not isinstance(state.reviewed_images, list):
|
|
222
|
+
state.reviewed_images = []
|
|
223
|
+
if not isinstance(state.marked, list):
|
|
224
|
+
state.marked = []
|
|
225
|
+
state.reviewed_images = list(dict.fromkeys(
|
|
226
|
+
name for name in state.reviewed_images if isinstance(name, str)
|
|
227
|
+
))
|
|
228
|
+
state.marked = list(dict.fromkeys(
|
|
229
|
+
name for name in state.marked if isinstance(name, str)
|
|
230
|
+
))
|
|
231
|
+
if (not isinstance(state.review_index, int) or isinstance(state.review_index, bool)
|
|
232
|
+
or state.review_index < 0):
|
|
233
|
+
state.review_index = 0
|
|
234
|
+
if state.review_current is not None and not isinstance(state.review_current, str):
|
|
235
|
+
state.review_current = None
|
|
236
|
+
|
|
237
|
+
# Map the saved cursor against the files that actually exist. This is
|
|
238
|
+
# important when a graph was removed between sessions: raw processed-list
|
|
239
|
+
# indices no longer match the reviewer's filtered image list.
|
|
240
|
+
graphs_dir = path.parent / GRAPHS_DIR
|
|
241
|
+
processed = state.processed if isinstance(state.processed, list) else []
|
|
242
|
+
existing = [
|
|
243
|
+
name for name in processed
|
|
244
|
+
if isinstance(name, str) and Path(name).name == name
|
|
245
|
+
and (graphs_dir / name).is_file()
|
|
246
|
+
]
|
|
247
|
+
valid = set(existing)
|
|
248
|
+
state.reviewed_images = [name for name in state.reviewed_images if name in valid]
|
|
249
|
+
state.marked = [name for name in state.marked if name in valid]
|
|
250
|
+
state.reviewed = len(state.reviewed_images)
|
|
251
|
+
if state.review_current in valid:
|
|
252
|
+
state.review_index = existing.index(state.review_current)
|
|
253
|
+
else:
|
|
254
|
+
# For configs created before review_current existed, infer the old
|
|
255
|
+
# selected filename from the unfiltered processed list before
|
|
256
|
+
# translating it into the filtered list's new index.
|
|
257
|
+
candidate = None
|
|
258
|
+
if state.review_current is None and state.review_index < len(processed):
|
|
259
|
+
candidate = processed[state.review_index]
|
|
260
|
+
if isinstance(candidate, str) and candidate in valid:
|
|
261
|
+
state.review_current = candidate
|
|
262
|
+
state.review_index = existing.index(candidate)
|
|
263
|
+
elif existing:
|
|
264
|
+
state.review_index = min(state.review_index, len(existing) - 1)
|
|
265
|
+
state.review_current = existing[state.review_index]
|
|
266
|
+
else:
|
|
267
|
+
state.review_index = 0
|
|
268
|
+
state.review_current = None
|
|
269
|
+
return state
|
|
270
|
+
|
|
271
|
+
@staticmethod
|
|
272
|
+
def _from_legacy(data: dict) -> dict:
|
|
273
|
+
download = data.get("Download_Progress", {})
|
|
274
|
+
process = data.get("Processing_Progress", {})
|
|
275
|
+
finalise = data.get("Finalising_Progress", {})
|
|
276
|
+
return {
|
|
277
|
+
"name": data.get("Name", ""),
|
|
278
|
+
"kind": download.get("Type"),
|
|
279
|
+
"script": download.get("Script"),
|
|
280
|
+
"downloaded": download.get("List", []),
|
|
281
|
+
"download_failed": download.get("Failed", []),
|
|
282
|
+
"processed": process.get("List", []),
|
|
283
|
+
"process_failed": process.get("Failed", []),
|
|
284
|
+
"reviewed": finalise.get("Finalised", 0),
|
|
285
|
+
"marked": finalise.get("List", []),
|
|
286
|
+
"review_index": 0,
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
# --------------------------------------------------------------------------- #
|
|
291
|
+
# Image viewer
|
|
292
|
+
# --------------------------------------------------------------------------- #
|
|
293
|
+
class ImageReviewer:
|
|
294
|
+
"""Tk window for stepping through plots.
|
|
295
|
+
|
|
296
|
+
Right/Left: navigate Down: mark Up: unmark Esc or close: finish
|
|
297
|
+
"""
|
|
298
|
+
|
|
299
|
+
def __init__(self, graphs_dir: Path, images: list[str],
|
|
300
|
+
marked: list[str], on_change: Callable[[], None],
|
|
301
|
+
on_review: Callable[[str], None] | None = None):
|
|
302
|
+
if tk is None:
|
|
303
|
+
raise RuntimeError("The viewer needs tkinter and Pillow (pip install pillow).")
|
|
304
|
+
self.graphs_dir = graphs_dir
|
|
305
|
+
self.images = images
|
|
306
|
+
self.marked = marked # shared with the project state, edited in place
|
|
307
|
+
self.on_change = on_change
|
|
308
|
+
self.on_review = on_review
|
|
309
|
+
self._visited: set[str] = set()
|
|
310
|
+
self.index = 0
|
|
311
|
+
self.furthest = 0
|
|
312
|
+
|
|
313
|
+
self.root = tk.Tk()
|
|
314
|
+
self.max_size = (int(self.root.winfo_screenwidth() * 0.9),
|
|
315
|
+
int(self.root.winfo_screenheight() * 0.85))
|
|
316
|
+
self.label = tk.Label(self.root)
|
|
317
|
+
self.label.pack()
|
|
318
|
+
self._load = lru_cache(maxsize=8)(self._read_photo) # cheap back-and-forth browsing
|
|
319
|
+
|
|
320
|
+
keys = {
|
|
321
|
+
"Right": lambda: self._move(+1),
|
|
322
|
+
"Left": lambda: self._move(-1),
|
|
323
|
+
"Down": self._mark,
|
|
324
|
+
"Up": self._unmark,
|
|
325
|
+
"Escape": self.root.destroy,
|
|
326
|
+
}
|
|
327
|
+
self.root.bind("<Key>", lambda event: keys.get(event.keysym, lambda: None)())
|
|
328
|
+
|
|
329
|
+
def run(self) -> int:
|
|
330
|
+
"""Block until the window closes; return how many images were reached."""
|
|
331
|
+
self._show()
|
|
332
|
+
self.root.mainloop()
|
|
333
|
+
return self.furthest + 1
|
|
334
|
+
|
|
335
|
+
def _read_photo(self, name: str):
|
|
336
|
+
with Image.open(self.graphs_dir / name) as img:
|
|
337
|
+
img.thumbnail(self.max_size) # only ever shrinks
|
|
338
|
+
return ImageTk.PhotoImage(img)
|
|
339
|
+
|
|
340
|
+
def _show(self) -> None:
|
|
341
|
+
name = self.images[self.index]
|
|
342
|
+
flag = " [MARKED]" if name in self.marked else ""
|
|
343
|
+
self.root.title(f"{self.index + 1}/{len(self.images)} {name}{flag}")
|
|
344
|
+
try:
|
|
345
|
+
photo = self._load(name)
|
|
346
|
+
except OSError as error:
|
|
347
|
+
self.label.config(image="", text=f"Cannot open {name}: {error}")
|
|
348
|
+
return
|
|
349
|
+
self.label.config(image=photo, text="")
|
|
350
|
+
self.label.image = photo
|
|
351
|
+
if name not in self._visited:
|
|
352
|
+
self._visited.add(name)
|
|
353
|
+
if self.on_review is not None:
|
|
354
|
+
self.on_review(name)
|
|
355
|
+
|
|
356
|
+
def _move(self, step: int) -> None:
|
|
357
|
+
target = self.index + step
|
|
358
|
+
if 0 <= target < len(self.images):
|
|
359
|
+
self.index = target
|
|
360
|
+
self.furthest = max(self.furthest, target)
|
|
361
|
+
self._show()
|
|
362
|
+
|
|
363
|
+
def _mark(self) -> None:
|
|
364
|
+
name = self.images[self.index]
|
|
365
|
+
if name not in self.marked:
|
|
366
|
+
self.marked.append(name)
|
|
367
|
+
self.on_change()
|
|
368
|
+
print(f"Appended: {name}")
|
|
369
|
+
self._show()
|
|
370
|
+
|
|
371
|
+
def _unmark(self) -> None:
|
|
372
|
+
name = self.images[self.index]
|
|
373
|
+
if name in self.marked:
|
|
374
|
+
self.marked.remove(name)
|
|
375
|
+
self.on_change()
|
|
376
|
+
print(f"Removed: {name}")
|
|
377
|
+
self._show()
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
# --------------------------------------------------------------------------- #
|
|
381
|
+
# Project
|
|
382
|
+
# --------------------------------------------------------------------------- #
|
|
383
|
+
def needs_project(method):
|
|
384
|
+
"""Raise a clear error if create()/restore() hasn't been called yet."""
|
|
385
|
+
@wraps(method)
|
|
386
|
+
def wrapper(self, *args, **kwargs):
|
|
387
|
+
if self.state is None:
|
|
388
|
+
raise RuntimeError("No project loaded: call create() or restore() first.")
|
|
389
|
+
return method(self, *args, **kwargs)
|
|
390
|
+
return wrapper
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
class Project:
|
|
394
|
+
def __init__(self):
|
|
395
|
+
self.root: Path | None = None
|
|
396
|
+
self.state: State | None = None
|
|
397
|
+
|
|
398
|
+
@property
|
|
399
|
+
def name(self) -> str | None:
|
|
400
|
+
return self.state.name if self.state else None
|
|
401
|
+
|
|
402
|
+
# -- lifecycle ---------------------------------------------------------- #
|
|
403
|
+
def create(self, name: str) -> None:
|
|
404
|
+
self.root = Path(name).resolve()
|
|
405
|
+
self.root.mkdir(parents=True, exist_ok=False) # never overwrite an existing project
|
|
406
|
+
for sub in (*KIND_DIRS.values(), GRAPHS_DIR):
|
|
407
|
+
(self.root / sub).mkdir()
|
|
408
|
+
self.state = State(name=self.root.name)
|
|
409
|
+
self.save()
|
|
410
|
+
print(f"Successfully created {name}!!")
|
|
411
|
+
|
|
412
|
+
def restore(self, name: str) -> None:
|
|
413
|
+
root = Path(name).resolve()
|
|
414
|
+
config_path = root / CONFIG_NAME
|
|
415
|
+
if not config_path.is_file():
|
|
416
|
+
raise FileNotFoundError(f"No {CONFIG_NAME} found in {root}")
|
|
417
|
+
self.root, self.state = root, State.load(config_path)
|
|
418
|
+
print(f"Successfully restored {name}!!")
|
|
419
|
+
|
|
420
|
+
@needs_project
|
|
421
|
+
def save(self) -> None:
|
|
422
|
+
_write_json_atomically(self.root / CONFIG_NAME, asdict(self.state))
|
|
423
|
+
|
|
424
|
+
updateconfig = save # old name
|
|
425
|
+
|
|
426
|
+
# -- stage 1: download -------------------------------------------------- #
|
|
427
|
+
@needs_project
|
|
428
|
+
def download(self, kind: str, script_path: str, amount: int, workers: int = 4,
|
|
429
|
+
progress_callback: Callable[[int, int, str, Exception | None], None] | None = None) -> None:
|
|
430
|
+
"""Download the first `amount` files of a curl script (`workers` run in parallel).
|
|
431
|
+
|
|
432
|
+
`progress_callback`, when supplied, receives (finished, total, filename, error)
|
|
433
|
+
in the calling thread. It is optional and leaves the existing API compatible.
|
|
434
|
+
"""
|
|
435
|
+
if kind not in KIND_DIRS:
|
|
436
|
+
raise ValueError(f"kind must be one of {sorted(KIND_DIRS)}, got {kind!r}")
|
|
437
|
+
if not isinstance(amount, int) or amount < 1:
|
|
438
|
+
raise ValueError("amount must be a positive integer")
|
|
439
|
+
if not isinstance(workers, int) or workers < 1:
|
|
440
|
+
raise ValueError("workers must be a positive integer")
|
|
441
|
+
script = Path(script_path).expanduser().resolve()
|
|
442
|
+
if not script.is_file():
|
|
443
|
+
raise FileNotFoundError(f"Download script not found: {script}")
|
|
444
|
+
if shutil.which("curl") is None:
|
|
445
|
+
raise FileNotFoundError("curl was not found on PATH; TESSify's bulk downloader requires curl.")
|
|
446
|
+
if self.state.kind not in (None, kind):
|
|
447
|
+
raise ValueError(f"This project holds {self.state.kind!r} files; "
|
|
448
|
+
"use a separate project for a different type.")
|
|
449
|
+
self.state.kind, self.state.script = kind, str(script)
|
|
450
|
+
|
|
451
|
+
commands = read_commands(script, amount)
|
|
452
|
+
download_dir = self.root / KIND_DIRS[kind]
|
|
453
|
+
# A stale config entry must not make a deleted FITS file look complete.
|
|
454
|
+
existing_downloads = {
|
|
455
|
+
name for name in self.state.downloaded
|
|
456
|
+
if Path(name).name == name and (download_dir / name).is_file()
|
|
457
|
+
}
|
|
458
|
+
self.state.downloaded = list(dict.fromkeys(
|
|
459
|
+
name for name in self.state.downloaded if name in existing_downloads
|
|
460
|
+
))
|
|
461
|
+
done = set(existing_downloads)
|
|
462
|
+
pending = [name for name in commands if name not in done]
|
|
463
|
+
job = partial(_download, commands=commands, cwd=download_dir)
|
|
464
|
+
|
|
465
|
+
self._run_stage(job, pending, total=len(commands), workers=workers,
|
|
466
|
+
executor_cls=ThreadPoolExecutor, # downloads wait on the network
|
|
467
|
+
done=self.state.downloaded, failed=self.state.download_failed,
|
|
468
|
+
progress_callback=progress_callback)
|
|
469
|
+
|
|
470
|
+
# -- stage 2: process --------------------------------------------------- #
|
|
471
|
+
@needs_project
|
|
472
|
+
def process(self, amount: int, workers: int = 1,
|
|
473
|
+
window_length: int = DEFAULT_WINDOW_LENGTH,
|
|
474
|
+
aperture_mask: str = DEFAULT_APERTURE_MASK,
|
|
475
|
+
progress_callback: Callable[[int, int, str, Exception | None], None] | None = None) -> None:
|
|
476
|
+
"""Plot the first `amount` downloaded files.
|
|
477
|
+
|
|
478
|
+
workers: CPU cores to use (1 = run in this process).
|
|
479
|
+
window_length: odd number of data points in flatten()'s smoothing window. Use a window
|
|
480
|
+
several times longer than the transits you are looking for.
|
|
481
|
+
aperture_mask: how target pixel ("tp") files become light curves: "all", "pipeline"
|
|
482
|
+
or "threshold". Ignored for light curve ("lc") files.
|
|
483
|
+
|
|
484
|
+
Files that already have a plot are skipped, so new settings only affect new files.
|
|
485
|
+
"""
|
|
486
|
+
_check_window_length(window_length)
|
|
487
|
+
if not isinstance(amount, int) or amount < 1:
|
|
488
|
+
raise ValueError("amount must be a positive integer")
|
|
489
|
+
if not isinstance(workers, int) or workers < 1:
|
|
490
|
+
raise ValueError("workers must be a positive integer")
|
|
491
|
+
if self.state.kind not in KIND_DIRS:
|
|
492
|
+
raise RuntimeError("No TESS data type is set for this project. Download data first.")
|
|
493
|
+
files = self.state.downloaded[:amount]
|
|
494
|
+
graphs_dir = self.root / GRAPHS_DIR
|
|
495
|
+
# If a generated plot was deleted, allow the corresponding FITS input to
|
|
496
|
+
# be processed again instead of trusting stale config state.
|
|
497
|
+
existing_processed = {
|
|
498
|
+
name for name in self.state.processed
|
|
499
|
+
if isinstance(name, str) and Path(name).name == name and (graphs_dir / name).is_file()
|
|
500
|
+
}
|
|
501
|
+
# A missing graph is going to be regenerated. Treat it as a new image:
|
|
502
|
+
# remove its old review/shortlist state so it cannot be silently skipped.
|
|
503
|
+
self.state.processed = list(dict.fromkeys(
|
|
504
|
+
name for name in self.state.processed if name in existing_processed
|
|
505
|
+
))
|
|
506
|
+
self._reconcile_review_state(self.state.processed)
|
|
507
|
+
done = set(existing_processed)
|
|
508
|
+
pending = [name for name in files if image_name(name) not in done]
|
|
509
|
+
job = partial(_render, kind=self.state.kind,
|
|
510
|
+
source_dir=self.root / KIND_DIRS.get(self.state.kind, ""),
|
|
511
|
+
graphs_dir=self.root / GRAPHS_DIR,
|
|
512
|
+
window_length=window_length, aperture_mask=aperture_mask)
|
|
513
|
+
|
|
514
|
+
self._run_stage(job, pending, total=len(files), workers=workers,
|
|
515
|
+
executor_cls=ProcessPoolExecutor, # plotting is CPU-bound
|
|
516
|
+
done=self.state.processed, failed=self.state.process_failed,
|
|
517
|
+
record=image_name, progress_callback=progress_callback)
|
|
518
|
+
|
|
519
|
+
@needs_project
|
|
520
|
+
def indprocess(self, filename: str,
|
|
521
|
+
window_length: int = DEFAULT_WINDOW_LENGTH,
|
|
522
|
+
aperture_mask: str = DEFAULT_APERTURE_MASK) -> str:
|
|
523
|
+
"""Plot a single downloaded file (same settings as process()) and return the image name."""
|
|
524
|
+
_check_window_length(window_length)
|
|
525
|
+
if self.state.kind not in KIND_DIRS:
|
|
526
|
+
raise RuntimeError("No TESS data type is set for this project. Download data first.")
|
|
527
|
+
output_name = image_name(filename)
|
|
528
|
+
was_processed = output_name in self.state.processed
|
|
529
|
+
_render(filename, self.state.kind,
|
|
530
|
+
self.root / KIND_DIRS[self.state.kind], self.root / GRAPHS_DIR,
|
|
531
|
+
window_length=window_length, aperture_mask=aperture_mask)
|
|
532
|
+
if was_processed:
|
|
533
|
+
# indprocess() overwrites the plot too, so prior review and shortlist
|
|
534
|
+
# decisions no longer describe the newly generated image.
|
|
535
|
+
self.state.reviewed_images = [
|
|
536
|
+
name for name in self.state.reviewed_images if name != output_name
|
|
537
|
+
]
|
|
538
|
+
self.state.marked = [
|
|
539
|
+
name for name in self.state.marked if name != output_name
|
|
540
|
+
]
|
|
541
|
+
if self.state.review_current == output_name:
|
|
542
|
+
self.state.review_current = None
|
|
543
|
+
self.state.review_index = 0
|
|
544
|
+
self._reconcile_review_state(self.get_review_images())
|
|
545
|
+
self.save()
|
|
546
|
+
return output_name
|
|
547
|
+
|
|
548
|
+
# -- review state API --------------------------------------------------- #
|
|
549
|
+
@needs_project
|
|
550
|
+
def get_review_images(self, amount: int | None = None) -> list[str]:
|
|
551
|
+
"""Return existing processed images, in package order, without duplicates."""
|
|
552
|
+
if amount is not None and (not isinstance(amount, int) or amount < 0):
|
|
553
|
+
raise ValueError("amount must be a non-negative integer or None")
|
|
554
|
+
names = self.state.processed if amount is None else self.state.processed[:amount]
|
|
555
|
+
graphs = self.root / GRAPHS_DIR
|
|
556
|
+
result: list[str] = []
|
|
557
|
+
seen: set[str] = set()
|
|
558
|
+
for name in names:
|
|
559
|
+
if (isinstance(name, str) and Path(name).name == name
|
|
560
|
+
and name not in seen and (graphs / name).is_file()):
|
|
561
|
+
result.append(name)
|
|
562
|
+
seen.add(name)
|
|
563
|
+
return result
|
|
564
|
+
|
|
565
|
+
def _reconcile_review_state(self, valid_images: Iterable[str]) -> None:
|
|
566
|
+
"""Drop review references to images that are no longer in the project."""
|
|
567
|
+
ordered = list(valid_images)
|
|
568
|
+
valid = set(ordered)
|
|
569
|
+
self.state.reviewed_images = list(dict.fromkeys(
|
|
570
|
+
name for name in self.state.reviewed_images if name in valid
|
|
571
|
+
))
|
|
572
|
+
self.state.marked = list(dict.fromkeys(
|
|
573
|
+
name for name in self.state.marked if name in valid
|
|
574
|
+
))
|
|
575
|
+
self.state.reviewed = len(self.state.reviewed_images)
|
|
576
|
+
if self.state.review_current in valid:
|
|
577
|
+
self.state.review_index = ordered.index(self.state.review_current)
|
|
578
|
+
else:
|
|
579
|
+
self.state.review_current = None
|
|
580
|
+
self.state.review_index = 0
|
|
581
|
+
|
|
582
|
+
def _record_reviewed(self, filename: str) -> None:
|
|
583
|
+
"""Persist one image as reviewed when the Tk viewer successfully displays it."""
|
|
584
|
+
images = self.get_review_images()
|
|
585
|
+
if filename not in images:
|
|
586
|
+
return
|
|
587
|
+
self._reconcile_review_state(images)
|
|
588
|
+
if filename not in self.state.reviewed_images:
|
|
589
|
+
self.state.reviewed_images.append(filename)
|
|
590
|
+
self.state.reviewed = len(self.state.reviewed_images)
|
|
591
|
+
self.state.review_current = filename
|
|
592
|
+
self.state.review_index = images.index(filename)
|
|
593
|
+
self.save()
|
|
594
|
+
|
|
595
|
+
@needs_project
|
|
596
|
+
def set_review_state(self, marked: Iterable[str] | None = None,
|
|
597
|
+
review_index: int | None = None) -> None:
|
|
598
|
+
"""Persist Qt review state using filenames as durable image identities."""
|
|
599
|
+
old_marked = list(self.state.marked)
|
|
600
|
+
old_reviewed_images = list(self.state.reviewed_images)
|
|
601
|
+
old_index = self.state.review_index
|
|
602
|
+
old_current = self.state.review_current
|
|
603
|
+
old_reviewed = self.state.reviewed
|
|
604
|
+
try:
|
|
605
|
+
images = self.get_review_images()
|
|
606
|
+
self._reconcile_review_state(images)
|
|
607
|
+
valid = set(images)
|
|
608
|
+
if marked is not None:
|
|
609
|
+
self.state.marked = list(dict.fromkeys(
|
|
610
|
+
name for name in marked if isinstance(name, str) and name in valid
|
|
611
|
+
))
|
|
612
|
+
else:
|
|
613
|
+
self.state.marked = [name for name in self.state.marked if name in valid]
|
|
614
|
+
|
|
615
|
+
self.state.reviewed_images = [
|
|
616
|
+
name for name in self.state.reviewed_images if name in valid
|
|
617
|
+
]
|
|
618
|
+
if review_index is not None:
|
|
619
|
+
if not isinstance(review_index, int) or isinstance(review_index, bool) or review_index < 0:
|
|
620
|
+
raise ValueError("review_index must be a non-negative integer")
|
|
621
|
+
if images:
|
|
622
|
+
index = min(review_index, len(images) - 1)
|
|
623
|
+
# Preserve the old high-water behavior: reaching image N means
|
|
624
|
+
# every image before N has necessarily been traversed as well.
|
|
625
|
+
reviewed_set = set(self.state.reviewed_images)
|
|
626
|
+
self.state.reviewed_images.extend(
|
|
627
|
+
name for name in images[:index + 1] if name not in reviewed_set
|
|
628
|
+
)
|
|
629
|
+
self.state.review_current = images[index]
|
|
630
|
+
self.state.review_index = index
|
|
631
|
+
else:
|
|
632
|
+
self.state.review_current = None
|
|
633
|
+
self.state.review_index = 0
|
|
634
|
+
|
|
635
|
+
self.state.reviewed = len(set(self.state.reviewed_images) & valid)
|
|
636
|
+
self.save()
|
|
637
|
+
except Exception:
|
|
638
|
+
self.state.marked = old_marked
|
|
639
|
+
self.state.reviewed_images = old_reviewed_images
|
|
640
|
+
self.state.review_index = old_index
|
|
641
|
+
self.state.review_current = old_current
|
|
642
|
+
self.state.reviewed = old_reviewed
|
|
643
|
+
raise
|
|
644
|
+
|
|
645
|
+
# -- stage 3: review ---------------------------------------------------- #
|
|
646
|
+
@needs_project
|
|
647
|
+
def finalise(self, amount: int) -> None:
|
|
648
|
+
"""Review unreviewed, existing graphs among the first `amount` processed images."""
|
|
649
|
+
if not isinstance(amount, int) or isinstance(amount, bool) or amount < 0:
|
|
650
|
+
raise ValueError("amount must be a non-negative integer")
|
|
651
|
+
all_images = self.get_review_images()
|
|
652
|
+
self._reconcile_review_state(all_images)
|
|
653
|
+
self.save()
|
|
654
|
+
review_scope = self.get_review_images(amount)
|
|
655
|
+
reviewed = set(self.state.reviewed_images)
|
|
656
|
+
pending = [name for name in review_scope if name not in reviewed]
|
|
657
|
+
if not pending:
|
|
658
|
+
print("Nothing left to review.")
|
|
659
|
+
return
|
|
660
|
+
reviewer = ImageReviewer(
|
|
661
|
+
self.root / GRAPHS_DIR, pending, self.state.marked, self.save,
|
|
662
|
+
on_review=self._record_reviewed,
|
|
663
|
+
)
|
|
664
|
+
reviewer.run()
|
|
665
|
+
self.save()
|
|
666
|
+
|
|
667
|
+
# -- results ------------------------------------------------------------ #
|
|
668
|
+
@needs_project
|
|
669
|
+
def getresults(self) -> list[str]:
|
|
670
|
+
"""Unique TIC IDs from marked images whose graph files still exist."""
|
|
671
|
+
valid = set(self.get_review_images())
|
|
672
|
+
ids = (tic_id(name) for name in self.state.marked if name in valid)
|
|
673
|
+
return list(dict.fromkeys(tic for tic in ids if tic))
|
|
674
|
+
|
|
675
|
+
def saveresults(self, name: str) -> None:
|
|
676
|
+
"""Export unique shortlisted TIC IDs as the package's original JSON list format."""
|
|
677
|
+
output = Path(name).expanduser().resolve()
|
|
678
|
+
tmp = output.with_name(output.name + ".tmp")
|
|
679
|
+
try:
|
|
680
|
+
tmp.write_text(json.dumps(self.getresults(), indent=2), encoding="utf-8")
|
|
681
|
+
os.replace(tmp, output)
|
|
682
|
+
finally:
|
|
683
|
+
if tmp.exists():
|
|
684
|
+
tmp.unlink()
|
|
685
|
+
|
|
686
|
+
# -- shared stage runner ------------------------------------------------ #
|
|
687
|
+
def _run_stage(self, job, pending: list[str], *, total: int, workers: int, executor_cls,
|
|
688
|
+
done: list[str], failed: list[str],
|
|
689
|
+
record: Callable[[str], str] = lambda item: item,
|
|
690
|
+
progress_callback: Callable[[int, int, str, Exception | None], None] | None = None) -> None:
|
|
691
|
+
"""Run `job` over `pending`, tracking successes/failures and saving periodically."""
|
|
692
|
+
finished = total - len(pending)
|
|
693
|
+
unsaved = 0
|
|
694
|
+
done_lookup = set(done)
|
|
695
|
+
failed_lookup = set(failed)
|
|
696
|
+
|
|
697
|
+
def notify(item: str, error: Exception | None) -> None:
|
|
698
|
+
if progress_callback is not None:
|
|
699
|
+
try:
|
|
700
|
+
progress_callback(finished, total, item, error)
|
|
701
|
+
except Exception as callback_error:
|
|
702
|
+
# UI/log callbacks must never corrupt the scientific pipeline.
|
|
703
|
+
print(f"Progress callback failed: {callback_error}")
|
|
704
|
+
|
|
705
|
+
notify("", None) # publish real initial state, including a fully resumed stage
|
|
706
|
+
try:
|
|
707
|
+
for item, error in _run_jobs(job, pending, workers, executor_cls):
|
|
708
|
+
finished += 1
|
|
709
|
+
if error is not None:
|
|
710
|
+
print(f"{finished}/{total} FAILED: {item} ({error})")
|
|
711
|
+
if item not in failed_lookup:
|
|
712
|
+
failed.append(item)
|
|
713
|
+
failed_lookup.add(item)
|
|
714
|
+
notify(item, error)
|
|
715
|
+
continue
|
|
716
|
+
recorded = record(item)
|
|
717
|
+
if recorded not in done_lookup:
|
|
718
|
+
done.append(recorded)
|
|
719
|
+
done_lookup.add(recorded)
|
|
720
|
+
if item in failed_lookup:
|
|
721
|
+
failed.remove(item)
|
|
722
|
+
failed_lookup.remove(item)
|
|
723
|
+
print(f"{finished}/{total}")
|
|
724
|
+
unsaved += 1
|
|
725
|
+
if unsaved >= SAVE_EVERY:
|
|
726
|
+
self.save()
|
|
727
|
+
unsaved = 0
|
|
728
|
+
notify(item, None)
|
|
729
|
+
finally:
|
|
730
|
+
self.save() # also runs on Ctrl-C, so nothing finished is lost
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
TESSify
|
tessify-0.0.0/setup.cfg
ADDED