TESSify 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
TESSify/__init__.py
ADDED
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
"""TESS light-curve / target-pixel project manager (needs Python 3.10+).
|
|
2
|
+
|
|
3
|
+
Typical workflow:
|
|
4
|
+
|
|
5
|
+
project = Project()
|
|
6
|
+
project.create("my_project") # or: project.restore("my_project")
|
|
7
|
+
project.download("lc", "bulk_download.sh", 100)
|
|
8
|
+
project.process(100)
|
|
9
|
+
project.finalise(100) # Down = mark, Up = unmark, Esc = quit
|
|
10
|
+
project.saveresults("candidates.json")
|
|
11
|
+
|
|
12
|
+
Every stage is resumable: progress lives in <project>/config.json.
|
|
13
|
+
|
|
14
|
+
Note: if you call process(..., workers=N) with N > 1 on Windows/macOS, put your
|
|
15
|
+
script's top-level code under ``if __name__ == "__main__":`` (standard rule for
|
|
16
|
+
multiprocessing).
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import gc
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import shlex
|
|
25
|
+
import subprocess
|
|
26
|
+
import shutil
|
|
27
|
+
from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, wait
|
|
28
|
+
from dataclasses import asdict, dataclass, field, fields
|
|
29
|
+
from functools import lru_cache, partial, wraps
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Callable, Iterable, Iterator
|
|
32
|
+
|
|
33
|
+
try: # the viewer is optional, so headless machines can still download and process
|
|
34
|
+
import tkinter as tk
|
|
35
|
+
from PIL import Image, ImageTk
|
|
36
|
+
except ImportError:
|
|
37
|
+
tk = None
|
|
38
|
+
|
|
39
|
+
KIND_DIRS = {"lc": "LightCurves", "tp": "TargetPixels"}
|
|
40
|
+
GRAPHS_DIR = "Graphs"
|
|
41
|
+
CONFIG_NAME = "config.json"
|
|
42
|
+
IMAGE_EXT = ".jpeg"
|
|
43
|
+
SAVE_EVERY = 25 # write config.json after this many completed files (and on exit)
|
|
44
|
+
DEFAULT_WINDOW_LENGTH = 101 # lightkurve's own default for flatten()
|
|
45
|
+
DEFAULT_APERTURE_MASK = "all" # target pixel files only: which pixels form the light curve
|
|
46
|
+
TIC_PATTERN = re.compile(r"-s\d{4}-(\d{16})-")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# --------------------------------------------------------------------------- #
|
|
50
|
+
# Small helpers
|
|
51
|
+
# --------------------------------------------------------------------------- #
|
|
52
|
+
def image_name(filename: str) -> str:
|
|
53
|
+
"""'tess...-s0014-...-s_lc.fits' -> 'tess...-s0014-...-s_lc.jpeg'."""
|
|
54
|
+
return Path(filename).stem + IMAGE_EXT
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def tic_id(name: str) -> str | None:
|
|
58
|
+
"""'tess...-s0014-0000000261136679-0150-s_lc.jpeg' -> 'TIC 261136679'."""
|
|
59
|
+
match = TIC_PATTERN.search(name)
|
|
60
|
+
if match is None:
|
|
61
|
+
return None
|
|
62
|
+
return "TIC " + (match.group(1).lstrip("0") or "0")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def read_commands(script_path: str | Path, limit: int) -> dict[str, list[str]]:
|
|
66
|
+
"""Parse the first `limit` curl commands of a MAST script into {output_file: argv}."""
|
|
67
|
+
commands: dict[str, list[str]] = {}
|
|
68
|
+
with open(script_path, "r", encoding="utf-8") as script:
|
|
69
|
+
for line in script:
|
|
70
|
+
line = line.strip()
|
|
71
|
+
if not line or line.startswith("#"):
|
|
72
|
+
continue
|
|
73
|
+
if len(commands) == limit:
|
|
74
|
+
break
|
|
75
|
+
argv = shlex.split(line)
|
|
76
|
+
commands[_output_file(argv)] = argv
|
|
77
|
+
return commands
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _output_file(argv: list[str]) -> str:
|
|
81
|
+
"""Return the file name given to curl's -o / --output flag."""
|
|
82
|
+
for flag in ("-o", "--output"):
|
|
83
|
+
if flag in argv and argv.index(flag) + 1 < len(argv):
|
|
84
|
+
return Path(argv[argv.index(flag) + 1]).name
|
|
85
|
+
raise ValueError(f"No output file (-o) found in command: {' '.join(argv)}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _write_json_atomically(path: Path, data: object) -> None:
|
|
89
|
+
"""Write via a temp file so a crash never leaves a half-written config."""
|
|
90
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
91
|
+
tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
|
92
|
+
os.replace(tmp, path)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _check_window_length(window_length: int) -> None:
|
|
96
|
+
"""flatten() needs an odd integer window of at least 3 points."""
|
|
97
|
+
if not isinstance(window_length, int) or window_length < 3 or window_length % 2 == 0:
|
|
98
|
+
raise ValueError(f"window_length must be an odd integer >= 3, got {window_length!r}")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# --------------------------------------------------------------------------- #
|
|
102
|
+
# Workers (module-level so they can be sent to other processes)
|
|
103
|
+
# --------------------------------------------------------------------------- #
|
|
104
|
+
def _download(filename: str, commands: dict[str, list[str]], cwd: Path) -> None:
|
|
105
|
+
result = subprocess.run(commands[filename], cwd=cwd)
|
|
106
|
+
if result.returncode != 0 or not (cwd / filename).is_file():
|
|
107
|
+
raise RuntimeError(f"download exited with code {result.returncode}")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _render(filename: str, kind: str, source_dir: Path, graphs_dir: Path,
|
|
111
|
+
window_length: int = DEFAULT_WINDOW_LENGTH,
|
|
112
|
+
aperture_mask: str = DEFAULT_APERTURE_MASK) -> None:
|
|
113
|
+
"""Clean one FITS file and save its plot.
|
|
114
|
+
|
|
115
|
+
window_length: smoothing window (in data points) used by flatten().
|
|
116
|
+
aperture_mask: pixels summed into a light curve; only used for target pixel files.
|
|
117
|
+
"""
|
|
118
|
+
import matplotlib
|
|
119
|
+
matplotlib.use("Agg") # no GUI windows, no clash with the Tk viewer
|
|
120
|
+
import matplotlib.pyplot as plt
|
|
121
|
+
import lightkurve as lk
|
|
122
|
+
|
|
123
|
+
data = lk.read(source_dir / filename)
|
|
124
|
+
if kind == "tp":
|
|
125
|
+
data = data.to_lightcurve(aperture_mask=aperture_mask)
|
|
126
|
+
lightcurve = data.remove_nans().flatten(window_length=window_length)
|
|
127
|
+
|
|
128
|
+
fig, ax = plt.subplots()
|
|
129
|
+
try:
|
|
130
|
+
lightcurve.plot(ax=ax)
|
|
131
|
+
fig.savefig(graphs_dir / image_name(filename), bbox_inches="tight")
|
|
132
|
+
finally:
|
|
133
|
+
plt.close(fig)
|
|
134
|
+
del data, lightcurve
|
|
135
|
+
gc.collect()
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _run_jobs(job: Callable[[str], None], items: Iterable[str],
|
|
139
|
+
workers: int, executor_cls) -> Iterator[tuple[str, Exception | None]]:
|
|
140
|
+
"""Run jobs while bounding queued futures to avoid large per-file overhead."""
|
|
141
|
+
if workers <= 1: # inline: simpler tracebacks, no pool start-up cost
|
|
142
|
+
for item in items:
|
|
143
|
+
try:
|
|
144
|
+
job(item)
|
|
145
|
+
yield item, None
|
|
146
|
+
except Exception as error:
|
|
147
|
+
yield item, error
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
pool = executor_cls(max_workers=workers)
|
|
151
|
+
iterator = iter(items)
|
|
152
|
+
futures: dict[object, str] = {}
|
|
153
|
+
|
|
154
|
+
def submit_one() -> bool:
|
|
155
|
+
try:
|
|
156
|
+
item = next(iterator)
|
|
157
|
+
except StopIteration:
|
|
158
|
+
return False
|
|
159
|
+
futures[pool.submit(job, item)] = item
|
|
160
|
+
return True
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
# Keep at most 2 * workers jobs queued. This avoids allocating thousands
|
|
164
|
+
# of Future objects when a user processes a large archive.
|
|
165
|
+
for _ in range(max(1, workers * 2)):
|
|
166
|
+
if not submit_one():
|
|
167
|
+
break
|
|
168
|
+
|
|
169
|
+
while futures:
|
|
170
|
+
completed, _ = wait(tuple(futures), return_when=FIRST_COMPLETED)
|
|
171
|
+
for future in completed:
|
|
172
|
+
item = futures.pop(future)
|
|
173
|
+
try:
|
|
174
|
+
future.result()
|
|
175
|
+
error = None
|
|
176
|
+
except Exception as caught:
|
|
177
|
+
error = caught
|
|
178
|
+
yield item, error
|
|
179
|
+
submit_one()
|
|
180
|
+
finally: # on Ctrl-C: let running jobs finish, drop queued jobs
|
|
181
|
+
for future in futures:
|
|
182
|
+
future.cancel()
|
|
183
|
+
pool.shutdown(wait=True, cancel_futures=True)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# --------------------------------------------------------------------------- #
|
|
187
|
+
# Project state (saved as config.json)
|
|
188
|
+
# --------------------------------------------------------------------------- #
|
|
189
|
+
@dataclass
|
|
190
|
+
class State:
|
|
191
|
+
name: str
|
|
192
|
+
kind: str | None = None # "lc" or "tp"
|
|
193
|
+
script: str | None = None
|
|
194
|
+
downloaded: list[str] = field(default_factory=list) # FITS file names
|
|
195
|
+
download_failed: list[str] = field(default_factory=list)
|
|
196
|
+
processed: list[str] = field(default_factory=list) # image file names
|
|
197
|
+
process_failed: list[str] = field(default_factory=list)
|
|
198
|
+
reviewed: int = 0 # count of currently existing reviewed images
|
|
199
|
+
marked: list[str] = field(default_factory=list) # images flagged in the viewer
|
|
200
|
+
review_index: int = 0 # current position in the existing-image list
|
|
201
|
+
reviewed_images: list[str] = field(default_factory=list) # durable review identity
|
|
202
|
+
review_current: str | None = None # filename at review_index, if available
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def load(cls, path: Path) -> State:
|
|
206
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
207
|
+
if "Download_Progress" in data: # config written by the older versions
|
|
208
|
+
data = cls._from_legacy(data)
|
|
209
|
+
|
|
210
|
+
# Older configs only stored a high-water count. Migrate that count to
|
|
211
|
+
# filenames once, preserving the old sequential-review interpretation.
|
|
212
|
+
if "reviewed_images" not in data:
|
|
213
|
+
processed = data.get("processed", [])
|
|
214
|
+
count = data.get("reviewed", 0)
|
|
215
|
+
if not isinstance(count, int) or count < 0:
|
|
216
|
+
count = 0
|
|
217
|
+
data["reviewed_images"] = processed[:count] if isinstance(processed, list) else []
|
|
218
|
+
|
|
219
|
+
known = {f.name for f in fields(cls)}
|
|
220
|
+
state = cls(**{key: value for key, value in data.items() if key in known})
|
|
221
|
+
if not isinstance(state.reviewed_images, list):
|
|
222
|
+
state.reviewed_images = []
|
|
223
|
+
if not isinstance(state.marked, list):
|
|
224
|
+
state.marked = []
|
|
225
|
+
state.reviewed_images = list(dict.fromkeys(
|
|
226
|
+
name for name in state.reviewed_images if isinstance(name, str)
|
|
227
|
+
))
|
|
228
|
+
state.marked = list(dict.fromkeys(
|
|
229
|
+
name for name in state.marked if isinstance(name, str)
|
|
230
|
+
))
|
|
231
|
+
if (not isinstance(state.review_index, int) or isinstance(state.review_index, bool)
|
|
232
|
+
or state.review_index < 0):
|
|
233
|
+
state.review_index = 0
|
|
234
|
+
if state.review_current is not None and not isinstance(state.review_current, str):
|
|
235
|
+
state.review_current = None
|
|
236
|
+
|
|
237
|
+
# Map the saved cursor against the files that actually exist. This is
|
|
238
|
+
# important when a graph was removed between sessions: raw processed-list
|
|
239
|
+
# indices no longer match the reviewer's filtered image list.
|
|
240
|
+
graphs_dir = path.parent / GRAPHS_DIR
|
|
241
|
+
processed = state.processed if isinstance(state.processed, list) else []
|
|
242
|
+
existing = [
|
|
243
|
+
name for name in processed
|
|
244
|
+
if isinstance(name, str) and Path(name).name == name
|
|
245
|
+
and (graphs_dir / name).is_file()
|
|
246
|
+
]
|
|
247
|
+
valid = set(existing)
|
|
248
|
+
state.reviewed_images = [name for name in state.reviewed_images if name in valid]
|
|
249
|
+
state.marked = [name for name in state.marked if name in valid]
|
|
250
|
+
state.reviewed = len(state.reviewed_images)
|
|
251
|
+
if state.review_current in valid:
|
|
252
|
+
state.review_index = existing.index(state.review_current)
|
|
253
|
+
else:
|
|
254
|
+
# For configs created before review_current existed, infer the old
|
|
255
|
+
# selected filename from the unfiltered processed list before
|
|
256
|
+
# translating it into the filtered list's new index.
|
|
257
|
+
candidate = None
|
|
258
|
+
if state.review_current is None and state.review_index < len(processed):
|
|
259
|
+
candidate = processed[state.review_index]
|
|
260
|
+
if isinstance(candidate, str) and candidate in valid:
|
|
261
|
+
state.review_current = candidate
|
|
262
|
+
state.review_index = existing.index(candidate)
|
|
263
|
+
elif existing:
|
|
264
|
+
state.review_index = min(state.review_index, len(existing) - 1)
|
|
265
|
+
state.review_current = existing[state.review_index]
|
|
266
|
+
else:
|
|
267
|
+
state.review_index = 0
|
|
268
|
+
state.review_current = None
|
|
269
|
+
return state
|
|
270
|
+
|
|
271
|
+
@staticmethod
|
|
272
|
+
def _from_legacy(data: dict) -> dict:
|
|
273
|
+
download = data.get("Download_Progress", {})
|
|
274
|
+
process = data.get("Processing_Progress", {})
|
|
275
|
+
finalise = data.get("Finalising_Progress", {})
|
|
276
|
+
return {
|
|
277
|
+
"name": data.get("Name", ""),
|
|
278
|
+
"kind": download.get("Type"),
|
|
279
|
+
"script": download.get("Script"),
|
|
280
|
+
"downloaded": download.get("List", []),
|
|
281
|
+
"download_failed": download.get("Failed", []),
|
|
282
|
+
"processed": process.get("List", []),
|
|
283
|
+
"process_failed": process.get("Failed", []),
|
|
284
|
+
"reviewed": finalise.get("Finalised", 0),
|
|
285
|
+
"marked": finalise.get("List", []),
|
|
286
|
+
"review_index": 0,
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
# --------------------------------------------------------------------------- #
|
|
291
|
+
# Image viewer
|
|
292
|
+
# --------------------------------------------------------------------------- #
|
|
293
|
+
class ImageReviewer:
|
|
294
|
+
"""Tk window for stepping through plots.
|
|
295
|
+
|
|
296
|
+
Right/Left: navigate Down: mark Up: unmark Esc or close: finish
|
|
297
|
+
"""
|
|
298
|
+
|
|
299
|
+
def __init__(self, graphs_dir: Path, images: list[str],
|
|
300
|
+
marked: list[str], on_change: Callable[[], None],
|
|
301
|
+
on_review: Callable[[str], None] | None = None):
|
|
302
|
+
if tk is None:
|
|
303
|
+
raise RuntimeError("The viewer needs tkinter and Pillow (pip install pillow).")
|
|
304
|
+
self.graphs_dir = graphs_dir
|
|
305
|
+
self.images = images
|
|
306
|
+
self.marked = marked # shared with the project state, edited in place
|
|
307
|
+
self.on_change = on_change
|
|
308
|
+
self.on_review = on_review
|
|
309
|
+
self._visited: set[str] = set()
|
|
310
|
+
self.index = 0
|
|
311
|
+
self.furthest = 0
|
|
312
|
+
|
|
313
|
+
self.root = tk.Tk()
|
|
314
|
+
self.max_size = (int(self.root.winfo_screenwidth() * 0.9),
|
|
315
|
+
int(self.root.winfo_screenheight() * 0.85))
|
|
316
|
+
self.label = tk.Label(self.root)
|
|
317
|
+
self.label.pack()
|
|
318
|
+
self._load = lru_cache(maxsize=8)(self._read_photo) # cheap back-and-forth browsing
|
|
319
|
+
|
|
320
|
+
keys = {
|
|
321
|
+
"Right": lambda: self._move(+1),
|
|
322
|
+
"Left": lambda: self._move(-1),
|
|
323
|
+
"Down": self._mark,
|
|
324
|
+
"Up": self._unmark,
|
|
325
|
+
"Escape": self.root.destroy,
|
|
326
|
+
}
|
|
327
|
+
self.root.bind("<Key>", lambda event: keys.get(event.keysym, lambda: None)())
|
|
328
|
+
|
|
329
|
+
def run(self) -> int:
|
|
330
|
+
"""Block until the window closes; return how many images were reached."""
|
|
331
|
+
self._show()
|
|
332
|
+
self.root.mainloop()
|
|
333
|
+
return self.furthest + 1
|
|
334
|
+
|
|
335
|
+
def _read_photo(self, name: str):
|
|
336
|
+
with Image.open(self.graphs_dir / name) as img:
|
|
337
|
+
img.thumbnail(self.max_size) # only ever shrinks
|
|
338
|
+
return ImageTk.PhotoImage(img)
|
|
339
|
+
|
|
340
|
+
def _show(self) -> None:
|
|
341
|
+
name = self.images[self.index]
|
|
342
|
+
flag = " [MARKED]" if name in self.marked else ""
|
|
343
|
+
self.root.title(f"{self.index + 1}/{len(self.images)} {name}{flag}")
|
|
344
|
+
try:
|
|
345
|
+
photo = self._load(name)
|
|
346
|
+
except OSError as error:
|
|
347
|
+
self.label.config(image="", text=f"Cannot open {name}: {error}")
|
|
348
|
+
return
|
|
349
|
+
self.label.config(image=photo, text="")
|
|
350
|
+
self.label.image = photo
|
|
351
|
+
if name not in self._visited:
|
|
352
|
+
self._visited.add(name)
|
|
353
|
+
if self.on_review is not None:
|
|
354
|
+
self.on_review(name)
|
|
355
|
+
|
|
356
|
+
def _move(self, step: int) -> None:
|
|
357
|
+
target = self.index + step
|
|
358
|
+
if 0 <= target < len(self.images):
|
|
359
|
+
self.index = target
|
|
360
|
+
self.furthest = max(self.furthest, target)
|
|
361
|
+
self._show()
|
|
362
|
+
|
|
363
|
+
def _mark(self) -> None:
|
|
364
|
+
name = self.images[self.index]
|
|
365
|
+
if name not in self.marked:
|
|
366
|
+
self.marked.append(name)
|
|
367
|
+
self.on_change()
|
|
368
|
+
print(f"Appended: {name}")
|
|
369
|
+
self._show()
|
|
370
|
+
|
|
371
|
+
def _unmark(self) -> None:
|
|
372
|
+
name = self.images[self.index]
|
|
373
|
+
if name in self.marked:
|
|
374
|
+
self.marked.remove(name)
|
|
375
|
+
self.on_change()
|
|
376
|
+
print(f"Removed: {name}")
|
|
377
|
+
self._show()
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
# --------------------------------------------------------------------------- #
|
|
381
|
+
# Project
|
|
382
|
+
# --------------------------------------------------------------------------- #
|
|
383
|
+
def needs_project(method):
|
|
384
|
+
"""Raise a clear error if create()/restore() hasn't been called yet."""
|
|
385
|
+
@wraps(method)
|
|
386
|
+
def wrapper(self, *args, **kwargs):
|
|
387
|
+
if self.state is None:
|
|
388
|
+
raise RuntimeError("No project loaded: call create() or restore() first.")
|
|
389
|
+
return method(self, *args, **kwargs)
|
|
390
|
+
return wrapper
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
class Project:
|
|
394
|
+
def __init__(self):
|
|
395
|
+
self.root: Path | None = None
|
|
396
|
+
self.state: State | None = None
|
|
397
|
+
|
|
398
|
+
@property
|
|
399
|
+
def name(self) -> str | None:
|
|
400
|
+
return self.state.name if self.state else None
|
|
401
|
+
|
|
402
|
+
# -- lifecycle ---------------------------------------------------------- #
|
|
403
|
+
def create(self, name: str) -> None:
|
|
404
|
+
self.root = Path(name).resolve()
|
|
405
|
+
self.root.mkdir(parents=True, exist_ok=False) # never overwrite an existing project
|
|
406
|
+
for sub in (*KIND_DIRS.values(), GRAPHS_DIR):
|
|
407
|
+
(self.root / sub).mkdir()
|
|
408
|
+
self.state = State(name=self.root.name)
|
|
409
|
+
self.save()
|
|
410
|
+
print(f"Successfully created {name}!!")
|
|
411
|
+
|
|
412
|
+
def restore(self, name: str) -> None:
|
|
413
|
+
root = Path(name).resolve()
|
|
414
|
+
config_path = root / CONFIG_NAME
|
|
415
|
+
if not config_path.is_file():
|
|
416
|
+
raise FileNotFoundError(f"No {CONFIG_NAME} found in {root}")
|
|
417
|
+
self.root, self.state = root, State.load(config_path)
|
|
418
|
+
print(f"Successfully restored {name}!!")
|
|
419
|
+
|
|
420
|
+
@needs_project
|
|
421
|
+
def save(self) -> None:
|
|
422
|
+
_write_json_atomically(self.root / CONFIG_NAME, asdict(self.state))
|
|
423
|
+
|
|
424
|
+
updateconfig = save # old name
|
|
425
|
+
|
|
426
|
+
# -- stage 1: download -------------------------------------------------- #
|
|
427
|
+
@needs_project
|
|
428
|
+
def download(self, kind: str, script_path: str, amount: int, workers: int = 4,
|
|
429
|
+
progress_callback: Callable[[int, int, str, Exception | None], None] | None = None) -> None:
|
|
430
|
+
"""Download the first `amount` files of a curl script (`workers` run in parallel).
|
|
431
|
+
|
|
432
|
+
`progress_callback`, when supplied, receives (finished, total, filename, error)
|
|
433
|
+
in the calling thread. It is optional and leaves the existing API compatible.
|
|
434
|
+
"""
|
|
435
|
+
if kind not in KIND_DIRS:
|
|
436
|
+
raise ValueError(f"kind must be one of {sorted(KIND_DIRS)}, got {kind!r}")
|
|
437
|
+
if not isinstance(amount, int) or amount < 1:
|
|
438
|
+
raise ValueError("amount must be a positive integer")
|
|
439
|
+
if not isinstance(workers, int) or workers < 1:
|
|
440
|
+
raise ValueError("workers must be a positive integer")
|
|
441
|
+
script = Path(script_path).expanduser().resolve()
|
|
442
|
+
if not script.is_file():
|
|
443
|
+
raise FileNotFoundError(f"Download script not found: {script}")
|
|
444
|
+
if shutil.which("curl") is None:
|
|
445
|
+
raise FileNotFoundError("curl was not found on PATH; TESSify's bulk downloader requires curl.")
|
|
446
|
+
if self.state.kind not in (None, kind):
|
|
447
|
+
raise ValueError(f"This project holds {self.state.kind!r} files; "
|
|
448
|
+
"use a separate project for a different type.")
|
|
449
|
+
self.state.kind, self.state.script = kind, str(script)
|
|
450
|
+
|
|
451
|
+
commands = read_commands(script, amount)
|
|
452
|
+
download_dir = self.root / KIND_DIRS[kind]
|
|
453
|
+
# A stale config entry must not make a deleted FITS file look complete.
|
|
454
|
+
existing_downloads = {
|
|
455
|
+
name for name in self.state.downloaded
|
|
456
|
+
if Path(name).name == name and (download_dir / name).is_file()
|
|
457
|
+
}
|
|
458
|
+
self.state.downloaded = list(dict.fromkeys(
|
|
459
|
+
name for name in self.state.downloaded if name in existing_downloads
|
|
460
|
+
))
|
|
461
|
+
done = set(existing_downloads)
|
|
462
|
+
pending = [name for name in commands if name not in done]
|
|
463
|
+
job = partial(_download, commands=commands, cwd=download_dir)
|
|
464
|
+
|
|
465
|
+
self._run_stage(job, pending, total=len(commands), workers=workers,
|
|
466
|
+
executor_cls=ThreadPoolExecutor, # downloads wait on the network
|
|
467
|
+
done=self.state.downloaded, failed=self.state.download_failed,
|
|
468
|
+
progress_callback=progress_callback)
|
|
469
|
+
|
|
470
|
+
# -- stage 2: process --------------------------------------------------- #
|
|
471
|
+
@needs_project
|
|
472
|
+
def process(self, amount: int, workers: int = 1,
|
|
473
|
+
window_length: int = DEFAULT_WINDOW_LENGTH,
|
|
474
|
+
aperture_mask: str = DEFAULT_APERTURE_MASK,
|
|
475
|
+
progress_callback: Callable[[int, int, str, Exception | None], None] | None = None) -> None:
|
|
476
|
+
"""Plot the first `amount` downloaded files.
|
|
477
|
+
|
|
478
|
+
workers: CPU cores to use (1 = run in this process).
|
|
479
|
+
window_length: odd number of data points in flatten()'s smoothing window. Use a window
|
|
480
|
+
several times longer than the transits you are looking for.
|
|
481
|
+
aperture_mask: how target pixel ("tp") files become light curves: "all", "pipeline"
|
|
482
|
+
or "threshold". Ignored for light curve ("lc") files.
|
|
483
|
+
|
|
484
|
+
Files that already have a plot are skipped, so new settings only affect new files.
|
|
485
|
+
"""
|
|
486
|
+
_check_window_length(window_length)
|
|
487
|
+
if not isinstance(amount, int) or amount < 1:
|
|
488
|
+
raise ValueError("amount must be a positive integer")
|
|
489
|
+
if not isinstance(workers, int) or workers < 1:
|
|
490
|
+
raise ValueError("workers must be a positive integer")
|
|
491
|
+
if self.state.kind not in KIND_DIRS:
|
|
492
|
+
raise RuntimeError("No TESS data type is set for this project. Download data first.")
|
|
493
|
+
files = self.state.downloaded[:amount]
|
|
494
|
+
graphs_dir = self.root / GRAPHS_DIR
|
|
495
|
+
# If a generated plot was deleted, allow the corresponding FITS input to
|
|
496
|
+
# be processed again instead of trusting stale config state.
|
|
497
|
+
existing_processed = {
|
|
498
|
+
name for name in self.state.processed
|
|
499
|
+
if isinstance(name, str) and Path(name).name == name and (graphs_dir / name).is_file()
|
|
500
|
+
}
|
|
501
|
+
# A missing graph is going to be regenerated. Treat it as a new image:
|
|
502
|
+
# remove its old review/shortlist state so it cannot be silently skipped.
|
|
503
|
+
self.state.processed = list(dict.fromkeys(
|
|
504
|
+
name for name in self.state.processed if name in existing_processed
|
|
505
|
+
))
|
|
506
|
+
self._reconcile_review_state(self.state.processed)
|
|
507
|
+
done = set(existing_processed)
|
|
508
|
+
pending = [name for name in files if image_name(name) not in done]
|
|
509
|
+
job = partial(_render, kind=self.state.kind,
|
|
510
|
+
source_dir=self.root / KIND_DIRS.get(self.state.kind, ""),
|
|
511
|
+
graphs_dir=self.root / GRAPHS_DIR,
|
|
512
|
+
window_length=window_length, aperture_mask=aperture_mask)
|
|
513
|
+
|
|
514
|
+
self._run_stage(job, pending, total=len(files), workers=workers,
|
|
515
|
+
executor_cls=ProcessPoolExecutor, # plotting is CPU-bound
|
|
516
|
+
done=self.state.processed, failed=self.state.process_failed,
|
|
517
|
+
record=image_name, progress_callback=progress_callback)
|
|
518
|
+
|
|
519
|
+
@needs_project
|
|
520
|
+
def indprocess(self, filename: str,
|
|
521
|
+
window_length: int = DEFAULT_WINDOW_LENGTH,
|
|
522
|
+
aperture_mask: str = DEFAULT_APERTURE_MASK) -> str:
|
|
523
|
+
"""Plot a single downloaded file (same settings as process()) and return the image name."""
|
|
524
|
+
_check_window_length(window_length)
|
|
525
|
+
if self.state.kind not in KIND_DIRS:
|
|
526
|
+
raise RuntimeError("No TESS data type is set for this project. Download data first.")
|
|
527
|
+
output_name = image_name(filename)
|
|
528
|
+
was_processed = output_name in self.state.processed
|
|
529
|
+
_render(filename, self.state.kind,
|
|
530
|
+
self.root / KIND_DIRS[self.state.kind], self.root / GRAPHS_DIR,
|
|
531
|
+
window_length=window_length, aperture_mask=aperture_mask)
|
|
532
|
+
if was_processed:
|
|
533
|
+
# indprocess() overwrites the plot too, so prior review and shortlist
|
|
534
|
+
# decisions no longer describe the newly generated image.
|
|
535
|
+
self.state.reviewed_images = [
|
|
536
|
+
name for name in self.state.reviewed_images if name != output_name
|
|
537
|
+
]
|
|
538
|
+
self.state.marked = [
|
|
539
|
+
name for name in self.state.marked if name != output_name
|
|
540
|
+
]
|
|
541
|
+
if self.state.review_current == output_name:
|
|
542
|
+
self.state.review_current = None
|
|
543
|
+
self.state.review_index = 0
|
|
544
|
+
self._reconcile_review_state(self.get_review_images())
|
|
545
|
+
self.save()
|
|
546
|
+
return output_name
|
|
547
|
+
|
|
548
|
+
# -- review state API --------------------------------------------------- #
|
|
549
|
+
@needs_project
|
|
550
|
+
def get_review_images(self, amount: int | None = None) -> list[str]:
|
|
551
|
+
"""Return existing processed images, in package order, without duplicates."""
|
|
552
|
+
if amount is not None and (not isinstance(amount, int) or amount < 0):
|
|
553
|
+
raise ValueError("amount must be a non-negative integer or None")
|
|
554
|
+
names = self.state.processed if amount is None else self.state.processed[:amount]
|
|
555
|
+
graphs = self.root / GRAPHS_DIR
|
|
556
|
+
result: list[str] = []
|
|
557
|
+
seen: set[str] = set()
|
|
558
|
+
for name in names:
|
|
559
|
+
if (isinstance(name, str) and Path(name).name == name
|
|
560
|
+
and name not in seen and (graphs / name).is_file()):
|
|
561
|
+
result.append(name)
|
|
562
|
+
seen.add(name)
|
|
563
|
+
return result
|
|
564
|
+
|
|
565
|
+
def _reconcile_review_state(self, valid_images: Iterable[str]) -> None:
|
|
566
|
+
"""Drop review references to images that are no longer in the project."""
|
|
567
|
+
ordered = list(valid_images)
|
|
568
|
+
valid = set(ordered)
|
|
569
|
+
self.state.reviewed_images = list(dict.fromkeys(
|
|
570
|
+
name for name in self.state.reviewed_images if name in valid
|
|
571
|
+
))
|
|
572
|
+
self.state.marked = list(dict.fromkeys(
|
|
573
|
+
name for name in self.state.marked if name in valid
|
|
574
|
+
))
|
|
575
|
+
self.state.reviewed = len(self.state.reviewed_images)
|
|
576
|
+
if self.state.review_current in valid:
|
|
577
|
+
self.state.review_index = ordered.index(self.state.review_current)
|
|
578
|
+
else:
|
|
579
|
+
self.state.review_current = None
|
|
580
|
+
self.state.review_index = 0
|
|
581
|
+
|
|
582
|
+
def _record_reviewed(self, filename: str) -> None:
|
|
583
|
+
"""Persist one image as reviewed when the Tk viewer successfully displays it."""
|
|
584
|
+
images = self.get_review_images()
|
|
585
|
+
if filename not in images:
|
|
586
|
+
return
|
|
587
|
+
self._reconcile_review_state(images)
|
|
588
|
+
if filename not in self.state.reviewed_images:
|
|
589
|
+
self.state.reviewed_images.append(filename)
|
|
590
|
+
self.state.reviewed = len(self.state.reviewed_images)
|
|
591
|
+
self.state.review_current = filename
|
|
592
|
+
self.state.review_index = images.index(filename)
|
|
593
|
+
self.save()
|
|
594
|
+
|
|
595
|
+
@needs_project
|
|
596
|
+
def set_review_state(self, marked: Iterable[str] | None = None,
|
|
597
|
+
review_index: int | None = None) -> None:
|
|
598
|
+
"""Persist Qt review state using filenames as durable image identities."""
|
|
599
|
+
old_marked = list(self.state.marked)
|
|
600
|
+
old_reviewed_images = list(self.state.reviewed_images)
|
|
601
|
+
old_index = self.state.review_index
|
|
602
|
+
old_current = self.state.review_current
|
|
603
|
+
old_reviewed = self.state.reviewed
|
|
604
|
+
try:
|
|
605
|
+
images = self.get_review_images()
|
|
606
|
+
self._reconcile_review_state(images)
|
|
607
|
+
valid = set(images)
|
|
608
|
+
if marked is not None:
|
|
609
|
+
self.state.marked = list(dict.fromkeys(
|
|
610
|
+
name for name in marked if isinstance(name, str) and name in valid
|
|
611
|
+
))
|
|
612
|
+
else:
|
|
613
|
+
self.state.marked = [name for name in self.state.marked if name in valid]
|
|
614
|
+
|
|
615
|
+
self.state.reviewed_images = [
|
|
616
|
+
name for name in self.state.reviewed_images if name in valid
|
|
617
|
+
]
|
|
618
|
+
if review_index is not None:
|
|
619
|
+
if not isinstance(review_index, int) or isinstance(review_index, bool) or review_index < 0:
|
|
620
|
+
raise ValueError("review_index must be a non-negative integer")
|
|
621
|
+
if images:
|
|
622
|
+
index = min(review_index, len(images) - 1)
|
|
623
|
+
# Preserve the old high-water behavior: reaching image N means
|
|
624
|
+
# every image before N has necessarily been traversed as well.
|
|
625
|
+
reviewed_set = set(self.state.reviewed_images)
|
|
626
|
+
self.state.reviewed_images.extend(
|
|
627
|
+
name for name in images[:index + 1] if name not in reviewed_set
|
|
628
|
+
)
|
|
629
|
+
self.state.review_current = images[index]
|
|
630
|
+
self.state.review_index = index
|
|
631
|
+
else:
|
|
632
|
+
self.state.review_current = None
|
|
633
|
+
self.state.review_index = 0
|
|
634
|
+
|
|
635
|
+
self.state.reviewed = len(set(self.state.reviewed_images) & valid)
|
|
636
|
+
self.save()
|
|
637
|
+
except Exception:
|
|
638
|
+
self.state.marked = old_marked
|
|
639
|
+
self.state.reviewed_images = old_reviewed_images
|
|
640
|
+
self.state.review_index = old_index
|
|
641
|
+
self.state.review_current = old_current
|
|
642
|
+
self.state.reviewed = old_reviewed
|
|
643
|
+
raise
|
|
644
|
+
|
|
645
|
+
# -- stage 3: review ---------------------------------------------------- #
|
|
646
|
+
@needs_project
|
|
647
|
+
def finalise(self, amount: int) -> None:
|
|
648
|
+
"""Review unreviewed, existing graphs among the first `amount` processed images."""
|
|
649
|
+
if not isinstance(amount, int) or isinstance(amount, bool) or amount < 0:
|
|
650
|
+
raise ValueError("amount must be a non-negative integer")
|
|
651
|
+
all_images = self.get_review_images()
|
|
652
|
+
self._reconcile_review_state(all_images)
|
|
653
|
+
self.save()
|
|
654
|
+
review_scope = self.get_review_images(amount)
|
|
655
|
+
reviewed = set(self.state.reviewed_images)
|
|
656
|
+
pending = [name for name in review_scope if name not in reviewed]
|
|
657
|
+
if not pending:
|
|
658
|
+
print("Nothing left to review.")
|
|
659
|
+
return
|
|
660
|
+
reviewer = ImageReviewer(
|
|
661
|
+
self.root / GRAPHS_DIR, pending, self.state.marked, self.save,
|
|
662
|
+
on_review=self._record_reviewed,
|
|
663
|
+
)
|
|
664
|
+
reviewer.run()
|
|
665
|
+
self.save()
|
|
666
|
+
|
|
667
|
+
# -- results ------------------------------------------------------------ #
|
|
668
|
+
@needs_project
|
|
669
|
+
def getresults(self) -> list[str]:
|
|
670
|
+
"""Unique TIC IDs from marked images whose graph files still exist."""
|
|
671
|
+
valid = set(self.get_review_images())
|
|
672
|
+
ids = (tic_id(name) for name in self.state.marked if name in valid)
|
|
673
|
+
return list(dict.fromkeys(tic for tic in ids if tic))
|
|
674
|
+
|
|
675
|
+
def saveresults(self, name: str) -> None:
|
|
676
|
+
"""Export unique shortlisted TIC IDs as the package's original JSON list format."""
|
|
677
|
+
output = Path(name).expanduser().resolve()
|
|
678
|
+
tmp = output.with_name(output.name + ".tmp")
|
|
679
|
+
try:
|
|
680
|
+
tmp.write_text(json.dumps(self.getresults(), indent=2), encoding="utf-8")
|
|
681
|
+
os.replace(tmp, output)
|
|
682
|
+
finally:
|
|
683
|
+
if tmp.exists():
|
|
684
|
+
tmp.unlink()
|
|
685
|
+
|
|
686
|
+
# -- shared stage runner ------------------------------------------------ #
|
|
687
|
+
def _run_stage(self, job, pending: list[str], *, total: int, workers: int, executor_cls,
|
|
688
|
+
done: list[str], failed: list[str],
|
|
689
|
+
record: Callable[[str], str] = lambda item: item,
|
|
690
|
+
progress_callback: Callable[[int, int, str, Exception | None], None] | None = None) -> None:
|
|
691
|
+
"""Run `job` over `pending`, tracking successes/failures and saving periodically."""
|
|
692
|
+
finished = total - len(pending)
|
|
693
|
+
unsaved = 0
|
|
694
|
+
done_lookup = set(done)
|
|
695
|
+
failed_lookup = set(failed)
|
|
696
|
+
|
|
697
|
+
def notify(item: str, error: Exception | None) -> None:
|
|
698
|
+
if progress_callback is not None:
|
|
699
|
+
try:
|
|
700
|
+
progress_callback(finished, total, item, error)
|
|
701
|
+
except Exception as callback_error:
|
|
702
|
+
# UI/log callbacks must never corrupt the scientific pipeline.
|
|
703
|
+
print(f"Progress callback failed: {callback_error}")
|
|
704
|
+
|
|
705
|
+
notify("", None) # publish real initial state, including a fully resumed stage
|
|
706
|
+
try:
|
|
707
|
+
for item, error in _run_jobs(job, pending, workers, executor_cls):
|
|
708
|
+
finished += 1
|
|
709
|
+
if error is not None:
|
|
710
|
+
print(f"{finished}/{total} FAILED: {item} ({error})")
|
|
711
|
+
if item not in failed_lookup:
|
|
712
|
+
failed.append(item)
|
|
713
|
+
failed_lookup.add(item)
|
|
714
|
+
notify(item, error)
|
|
715
|
+
continue
|
|
716
|
+
recorded = record(item)
|
|
717
|
+
if recorded not in done_lookup:
|
|
718
|
+
done.append(recorded)
|
|
719
|
+
done_lookup.add(recorded)
|
|
720
|
+
if item in failed_lookup:
|
|
721
|
+
failed.remove(item)
|
|
722
|
+
failed_lookup.remove(item)
|
|
723
|
+
print(f"{finished}/{total}")
|
|
724
|
+
unsaved += 1
|
|
725
|
+
if unsaved >= SAVE_EVERY:
|
|
726
|
+
self.save()
|
|
727
|
+
unsaved = 0
|
|
728
|
+
notify(item, None)
|
|
729
|
+
finally:
|
|
730
|
+
self.save() # also runs on Ctrl-C, so nothing finished is lost
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
TESSify/__init__.py,sha256=yXCW328HnbW-k3MZ1zFOt0OsVB6ssypaTBEBCafhBO8,32592
|
|
2
|
+
tessify-0.0.0.dist-info/licenses/LICENSE,sha256=-RvAW8qIL8SGIh4l2vT5fkq5QUFarxjT4aPQvNTPvIQ,1086
|
|
3
|
+
tessify-0.0.0.dist-info/METADATA,sha256=TPuxcFAHDBuGogPPtlF4-nR-cOwzKnvpX8D-_8LiAgU,100
|
|
4
|
+
tessify-0.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
5
|
+
tessify-0.0.0.dist-info/top_level.txt,sha256=dmSNY13Nndp-rUDgidjL5XYuU5SGmKeAYcvbpDxw_Fw,8
|
|
6
|
+
tessify-0.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 arpit290
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
TESSify
|