asteroidpipeline 0.2.1__py3-none-any.whl

asteroid_ui/__init__.py

@@ -0,0 +1,3 @@
+"""Terminal interface for AsteroidPipeline: image sets in, MPC reports out."""
+
+__version__ = "0.2.1"

asteroid_ui/__main__.py

@@ -0,0 +1,3 @@
+from asteroid_ui.app import main
+
+main()

asteroid_ui/app.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from textual.app import App
+from textual.reactive import reactive
+
+# textual-image finds out which graphics the terminal supports (Kitty, Sixel...)
+# by asking it with escape sequences; that only works before Textual takes the
+# terminal over, so it is imported here, at start-up.
+try:
+    import textual_image.widget  # noqa: F401
+except ImportError:
+    pass
+
+from asteroid_ui.backend import Backend
+from asteroid_ui.config import load_config
+from asteroid_ui.texts import STATUS_MESSAGES
+from asteroid_ui.theme import GRUVBOX
+
+
+class AsteroidApp(App):
+    CSS_PATH = "styles.tcss"
+    TITLE = "asteroids"
+    SUB_TITLE = "AsteroidPipeline"
+
+    engine_status: reactive[str] = reactive("Starting the analysis engine (Julia)… this can take a minute.")
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.config = load_config()
+        self.backend = Backend()
+        self.backend.on_status = lambda code: setattr(self, "engine_status", STATUS_MESSAGES.get(code, code))
+
+    def on_mount(self) -> None:
+        self.register_theme(GRUVBOX)
+        self.theme = "gruvbox-asteroides"
+        from asteroid_ui.screens.home import HomeScreen
+
+        self.push_screen(HomeScreen())
+        self.run_worker(self._start_backend())
+
+    async def _start_backend(self) -> None:
+        await self.backend.start()
+        await self.backend.ready.wait()
+        self.engine_status = f"✗ {self.backend.failed}" if self.backend.failed else "Analysis engine ready."
+
+    async def on_unmount(self) -> None:
+        await self.backend.stop()
+
+
+USAGE = """\
+usage: asteroidpipeline [--setup | --version | --help]
+
+Find moving objects in astronomical image sets (.zip files, folders or FITS
+files), check the candidates, and write MPC reports.
+
+  --setup     install the analysis engine (Julia packages) and exit;
+              the app otherwise does it on its first run
+  --version   print the version and exit
+"""
+
+
+def main() -> None:
+    import sys
+
+    args = sys.argv[1:]
+    if "--help" in args or "-h" in args:
+        print(USAGE)
+        return
+    if "--version" in args:
+        from asteroid_ui import __version__
+
+        print(f"asteroidpipeline {__version__}")
+        return
+    if "--setup" in args:
+        from asteroid_ui import engine
+
+        try:
+            julia = engine.julia_executable()
+            print(f"Setting up the analysis engine with {julia} (several minutes)…")
+            engine.setup(julia, print)
+        except engine.EngineError as e:
+            sys.exit(str(e))
+        print("Done.")
+        return
+    AsteroidApp().run()
+
+
+if __name__ == "__main__":
+    main()

asteroid_ui/backend.py

@@ -0,0 +1,126 @@
+"""Client for the analysis engine (ui/backend/server.jl).
+
+The engine is a Julia process kept open for the whole session: loading and
+compiling AsteroidPipeline takes ~20 s, so it is paid once. It speaks JSON,
+one message per line (see server.jl).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import itertools
+import json
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from asteroid_ui import engine
+from asteroid_ui.config import BACKEND_LOG
+
+Event = dict[str, Any]
+
+
+class BackendError(Exception):
+    """The engine answered with an error, or could not be started."""
+
+
+class Backend:
+    def __init__(self) -> None:
+        self._proc: asyncio.subprocess.Process | None = None
+        self._ids = itertools.count(1)
+        self._queues: dict[int, asyncio.Queue[Event]] = {}
+        self._lock = asyncio.Lock()  # the engine handles one command at a time
+        self.ready = asyncio.Event()
+        self.failed: str | None = None
+        self.on_status: Callable[[str], None] = lambda message: None
+
+    @property
+    def busy(self) -> bool:
+        return self._lock.locked()
+
+    async def start(self) -> None:
+        """Find Julia, set the engine up if this is the first run (or an upgrade), and start it."""
+        try:
+            julia = await asyncio.to_thread(engine.julia_executable)
+            if await asyncio.to_thread(engine.needs_setup):
+                self.on_status("Installing the analysis engine — the first time only, it takes several minutes…")
+                loop = asyncio.get_running_loop()
+
+                def progress(line: str) -> None:
+                    loop.call_soon_threadsafe(self.on_status, f"Installing the analysis engine: {line[:90]}")
+
+                await asyncio.to_thread(engine.setup, julia, progress)
+                self.on_status("Starting the analysis engine…")
+        except engine.EngineError as e:
+            self.failed = str(e)
+            self.ready.set()
+            return
+        BACKEND_LOG.parent.mkdir(parents=True, exist_ok=True)
+        log = open(BACKEND_LOG, "a", buffering=1)
+        self._proc = await asyncio.create_subprocess_exec(
+            *engine.server_command(julia),
+            stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=log,
+            limit=64 * 1024 * 1024,
+        )
+        asyncio.create_task(self._read_loop())
+
+    async def _read_loop(self) -> None:
+        assert self._proc and self._proc.stdout
+        while line := await self._proc.stdout.readline():
+            try:
+                event = json.loads(line)
+            except ValueError:
+                continue
+            if event.get("event") == "ready":
+                self.ready.set()
+                continue
+            if event.get("event") == "status":
+                self.on_status(event.get("message", ""))
+            queue = self._queues.get(event.get("id"))
+            if queue is not None:
+                await queue.put(event)
+        code = await self._proc.wait()
+        self.failed = f"The analysis engine stopped (exit code {code}). Details in {BACKEND_LOG}"
+        self.ready.set()
+        for queue in self._queues.values():
+            await queue.put({"event": "error", "message": self.failed})
+
+    async def request(self, cmd: str, on_event: Callable[[Event], Awaitable[None] | None] | None = None,
+                      **args: Any) -> Event:
+        """Send a command and wait for its final event (`result`, `report`, `report_check`, `inspection` or `error`).
+
+        Events along the way (progress, status) go to `on_event`.
+        """
+        await self.ready.wait()
+        if self.failed:
+            raise BackendError(self.failed)
+        async with self._lock:
+            assert self._proc and self._proc.stdin
+            request_id = next(self._ids)
+            queue: asyncio.Queue[Event] = asyncio.Queue()
+            self._queues[request_id] = queue
+            try:
+                self._proc.stdin.write((json.dumps({"cmd": cmd, "id": request_id, **args}) + "\n").encode())
+                await self._proc.stdin.drain()
+                while True:
+                    event = await queue.get()
+                    kind = event.get("event")
+                    if kind == "error":
+                        raise BackendError(event.get("message", "unknown error"))
+                    if kind in ("result", "report", "report_check", "inspection"):
+                        return event
+                    if on_event is not None:
+                        maybe = on_event(event)
+                        if asyncio.iscoroutine(maybe):
+                            await maybe
+            finally:
+                del self._queues[request_id]
+
+    async def stop(self) -> None:
+        if self._proc and self._proc.returncode is None:
+            try:
+                assert self._proc.stdin
+                self._proc.stdin.write(b'{"cmd": "quit"}\n')
+                await self._proc.stdin.drain()
+                await asyncio.wait_for(self._proc.wait(), 3)
+            except (asyncio.TimeoutError, BrokenPipeError, ConnectionResetError):
+                self._proc.kill()

asteroid_ui/clipboard.py

@@ -0,0 +1,27 @@
+"""Copy text to the clipboard.
+
+Textual's own copy (an OSC 52 escape sequence) works in most terminals, but not
+all of them allow it; the desktop's clipboard tool is used as well when there
+is one, so a copied report is really there to paste.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+
+from textual.app import App
+
+_TOOLS = (["wl-copy"], ["xclip", "-selection", "clipboard"], ["xsel", "--clipboard", "--input"])
+
+
+def copy_text(app: App, text: str) -> None:
+    app.copy_to_clipboard(text)
+    for cmd in _TOOLS:
+        if shutil.which(cmd[0]):
+            try:
+                subprocess.run(cmd, input=text.encode(), timeout=3, check=False,
+                               stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            except (OSError, subprocess.TimeoutExpired):
+                continue
+            break

asteroid_ui/config.py

@@ -0,0 +1,58 @@
+"""Paths and preferences.
+
+Everything the user can change lives in ~/.config/asteroid-ui/config.json and
+in the Settings screen; the rest follows from where this package sits in the
+repository.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+_XDG_CONFIG = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config"))
+_XDG_CACHE = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+_XDG_DATA = Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
+CONFIG_FILE = _XDG_CONFIG / "asteroid-ui" / "config.json"
+CACHE_DIR = _XDG_CACHE / "asteroid-ui"
+SETS_CACHE = CACHE_DIR / "sets"
+BACKEND_LOG = CACHE_DIR / "engine.log"
+# Every candidate a person has labelled, with its pixels: training data for a
+# future classifier. Kept apart from the cache, which can be thrown away.
+DATA_DIR = _XDG_DATA / "asteroid-ui"
+DATASET_DIR = DATA_DIR / "dataset"
+
+
+@dataclass
+class Config:
+    # Shown big on the start screen: your team's name, or the program's.
+    title: str = "NEO Hunters"
+    # Folder searched (with its subfolders) for image-set .zip files.
+    downloads: str = str(Path.home() / "Downloads")
+    # The MPC report's MEA and CON lines: who measures (you, your team) and
+    # how to reach them. The telescope's own lines (COD, OBS, TEL) come with
+    # its profile (profiles.py).
+    measurers: str = ""
+    contact: str = ""
+    # Your own temporary designations for the candidates you report: a prefix
+    # ("NHU") and the next number (NHU0002). Empty: the program's own (P2).
+    designation_prefix: str = ""
+    next_designation: int = 1
+    # A report file to check before submitting it (from this program or any other).
+    report_to_check: str = ""
+    # Animation speed, images per second (at most 3: no fast flashing).
+    fps: float = 1.5
+
+    def save(self) -> None:
+        CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)
+        CONFIG_FILE.write_text(json.dumps(asdict(self), indent=2, ensure_ascii=False))
+
+
+def load_config() -> Config:
+    try:
+        data = json.loads(CONFIG_FILE.read_text())
+    except (OSError, ValueError):
+        return Config()
+    return Config(**{k: v for k, v in data.items() if k in Config.__dataclass_fields__})

asteroid_ui/designations.py

@@ -0,0 +1,75 @@
+"""Temporary designations for the candidates you report.
+
+The MPC's 80-column format has 7 characters for an observer's own
+designation of an object it doesn't know yet. Teams number their finds with
+a prefix of their own — "NHU0001", "NHU0002"… — set in Settings. Each
+candidate keeps the designation it was first given, whenever its report is
+written again; numbers are never reused.
+
+A candidate on the MPC's NEO Confirmation Page is reported under the NEOCP's
+designation instead: those observations help confirm someone's find.
+Without a prefix, reports use the program's own: "P" and the tracklet's
+number.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from asteroid_ui.config import CONFIG_FILE
+
+DESIGNATIONS_FILE = CONFIG_FILE.parent / "designations.json"
+MAX_LENGTH = 7
+
+
+def _load() -> dict[str, Any]:
+    try:
+        return json.loads(DESIGNATIONS_FILE.read_text())
+    except (OSError, ValueError):
+        return {"assigned": {}}
+
+
+def _save(data: dict[str, Any]) -> None:
+    DESIGNATIONS_FILE.parent.mkdir(parents=True, exist_ok=True)
+    DESIGNATIONS_FILE.write_text(json.dumps(data, indent=1))
+
+
+def valid_prefix(prefix: str) -> bool:
+    return prefix == "" or (prefix.isalnum() and prefix.isascii() and len(prefix) <= MAX_LENGTH - 2)
+
+
+def neocp_designation(tracklet: dict[str, Any]) -> str | None:
+    check = tracklet.get("neocp") or {}
+    if check.get("verdict") != "on_neocp":
+        return None
+    agreeing = [o for o in check.get("objects", []) if o.get("agreement") == "motion"]
+    return agreeing[0]["designation"] if agreeing else None
+
+
+def assign(config, set_name: str, tracklets: list[dict[str, Any]]) -> dict[str, str]:
+    """Designation for each of `tracklets` (tracklet id, as a string, => designation),
+    giving new numbers to those that have none yet."""
+    data = _load()
+    assigned: dict[str, str] = data.setdefault("assigned", {})
+    out: dict[str, str] = {}
+    prefix = config.designation_prefix.strip().upper()
+    for t in tracklets:
+        neocp = neocp_designation(t)
+        if neocp:
+            out[str(t["id"])] = neocp
+            continue
+        if not prefix:
+            continue
+        key = f"{set_name}/{t['id']}"
+        if key not in assigned:
+            digits = MAX_LENGTH - len(prefix)
+            number = max(int(config.next_designation), 1)
+            if len(str(number)) > digits:
+                raise ValueError(f"{prefix} has room for {digits} digits; {number} doesn't fit")
+            assigned[key] = f"{prefix}{number:0{digits}d}"
+            config.next_designation = number + 1
+            config.save()
+        out[str(t["id"])] = assigned[key]
+    _save(data)
+    return out

asteroid_ui/engine/__init__.py

@@ -0,0 +1,129 @@
+"""The analysis engine: AsteroidPipeline.jl, run by Julia.
+
+The Julia package travels inside this Python package (julia/AsteroidPipeline,
+put there when the wheel is built; in a checkout of the repository, the
+repository itself is used). The first time the app runs — and again after an
+upgrade — it gets an environment ready for it:
+
+1. find Julia: $ASTEROIDPIPELINE_JULIA, `julia` on the PATH, or, if neither,
+   a private copy that `juliapkg` downloads;
+2. make a Julia environment in the user's data folder with AsteroidPipeline
+   and the few packages server.jl needs, and precompile it (several minutes,
+   once).
+
+After that, server.jl starts in seconds.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import subprocess
+from collections.abc import Callable
+from pathlib import Path
+
+from asteroid_ui import __version__
+from asteroid_ui.config import BACKEND_LOG, DATA_DIR
+
+ENGINE_DIR = Path(__file__).resolve().parent
+SERVER = ENGINE_DIR / "server.jl"
+_BUNDLED = ENGINE_DIR.parent / "julia" / "AsteroidPipeline"
+_CHECKOUT = ENGINE_DIR.parents[2]          # ui/asteroid_ui/engine -> repository root
+_SERVER_PACKAGES = ["FITSIO", "JSON", "WCS", "TypedTables", "Dates", "Statistics"]
+MIN_JULIA = (1, 11)
+
+
+class EngineError(Exception):
+    """The engine can't be found or set up; the message says what to do."""
+
+
+def julia_package() -> Path:
+    """Where AsteroidPipeline.jl's source is: bundled in the wheel, or this checkout."""
+    for candidate in (_BUNDLED, _CHECKOUT):
+        if (candidate / "Project.toml").is_file() and (candidate / "src" / "AsteroidPipeline.jl").is_file():
+            return candidate
+    raise EngineError("AsteroidPipeline.jl's source is missing from this installation; reinstall it.")
+
+
+def env_dir() -> Path:
+    return DATA_DIR / "engine"
+
+
+def julia_executable() -> str:
+    explicit = os.environ.get("ASTEROIDPIPELINE_JULIA")
+    if explicit:
+        return explicit
+    found = shutil.which("julia")
+    if found:
+        return found
+    try:
+        import juliapkg  # downloads a private Julia if there is none
+    except ImportError:
+        raise EngineError("Julia isn't installed. Install it from https://julialang.org/install "
+                          "(or `pip install juliapkg` to have it downloaded) and start the app again.") from None
+    return juliapkg.executable()
+
+
+def julia_version(julia: str) -> tuple[int, ...]:
+    out = subprocess.run([julia, "--startup-file=no", "-e", "print(VERSION.major, ' ', VERSION.minor)"],
+                         capture_output=True, text=True, timeout=120)
+    try:
+        return tuple(int(x) for x in out.stdout.split())
+    except ValueError:
+        raise EngineError(f"Could not run Julia ({julia}): {out.stderr.strip()[:300]}") from None
+
+
+def _fingerprint(package: Path) -> str:
+    """Changes whenever the engine must be set up again: app version, package sources' manifest, Julia."""
+    h = hashlib.sha256(__version__.encode())
+    h.update((package / "Project.toml").read_bytes())
+    h.update(str(package).encode())
+    for f in sorted((package / "src").glob("*.jl")):
+        h.update(f.name.encode())
+        h.update(str(f.stat().st_size).encode())
+    return h.hexdigest()
+
+
+def needs_setup() -> bool:
+    try:
+        stamp = (env_dir() / "ready").read_text().strip()
+    except OSError:
+        return True
+    return stamp != _fingerprint(julia_package())
+
+
+def setup(julia: str, on_line: Callable[[str], None] = lambda line: None) -> None:
+    """Make the engine's Julia environment; lines of Julia's output go to `on_line` and the log."""
+    version = julia_version(julia)
+    if version < MIN_JULIA:
+        raise EngineError(f"Julia {'.'.join(map(str, version))} is too old; AsteroidPipeline needs "
+                          f"{'.'.join(map(str, MIN_JULIA))} or newer.")
+    package = julia_package()
+    env = env_dir()
+    if env.exists():
+        shutil.rmtree(env)
+    env.mkdir(parents=True)
+    script = (
+        "using Pkg; "
+        f"Pkg.develop(path={str(package)!r}); "
+        f"Pkg.add({_SERVER_PACKAGES!r}); "
+        "Pkg.instantiate(); Pkg.precompile()"
+    ).replace("'", '"')
+    BACKEND_LOG.parent.mkdir(parents=True, exist_ok=True)
+    with open(BACKEND_LOG, "a", buffering=1) as log:
+        log.write(f"--- setting up the engine in {env} with {julia}\n")
+        proc = subprocess.Popen([julia, f"--project={env}", "--startup-file=no", "-e", script],
+                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+        assert proc.stdout is not None
+        for line in proc.stdout:
+            log.write(line)
+            if line.strip():
+                on_line(line.rstrip())
+        if proc.wait() != 0:
+            raise EngineError(f"Setting up the analysis engine failed; details in {BACKEND_LOG}")
+    (env / "ready").write_text(_fingerprint(package))
+
+
+def server_command(julia: str) -> list[str]:
+    return [julia, f"--project={env_dir()}", "--startup-file=no", str(SERVER)]