pyobs-core 2.0.0.dev66__tar.gz → 2.0.0.dev68__tar.gz

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyobs-core
-Version: 2.0.0.dev66
+Version: 2.0.0.dev68
 Summary: robotic telescope software
 Author-email: Tim-Oliver Husser <thusser@uni-goettingen.de>
 License-Expression: MIT

pyobs_core-2.0.0.dev68/pyobs/robotic/scheduler/_executor.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable, Coroutine
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, TypeVar
+
+_T = TypeVar("_T")
+
+# single worker: evaluate_constraints_and_merits calls are always awaited sequentially by the
+# caller (never fired concurrently), and DataProvider's functools.cache is not safe under real
+# concurrent access -- one worker keeps that access serialized while still freeing the main loop.
+_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="pyobs-scheduler")
+
+
+async def run_cpu_bound(coro_fn: Callable[..., Coroutine[Any, Any, _T]], *args: object) -> _T:
+    """Runs an async callable to completion on a dedicated worker thread, off the caller's loop.
+
+    Args:
+        coro_fn: An async callable whose body does not itself need to run on the caller's
+            event loop (no dependency on the caller's other tasks, timers, or comm state).
+        args: Positional args for coro_fn.
+    """
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(_executor, lambda: asyncio.run(coro_fn(*args)))
+
+
+__all__ = ["run_cpu_bound"]

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/pyobs/robotic/scheduler/constraints/moonilluminationconstraint.py

@@ -24,7 +24,7 @@ class MoonIlluminationConstraint(Constraint):
         return astroplan.MoonIlluminationConstraint(max=self.max_phase)
 
     async def __call__(self, time: Time, task: Task, data: DataProvider) -> bool:
-        moon_illumination = float(data.observer.moon_illumination(time))
+        moon_illumination = data.moon_illumination(time)
         return moon_illumination <= self.max_phase
 
 

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/pyobs/robotic/scheduler/constraints/moonseparationconstraint.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 import astroplan
-import astropy.coordinates
 import astropy.units as u
 import numpy as np
 from astropy.coordinates import SkyCoord
@@ -32,12 +31,12 @@ class MoonSeparationConstraint(Constraint):
         if task.target is None:
             return True
         coord = task.target.coordinates(time)
-        moon_separation = astropy.coordinates.get_body("moon", time).separation(coord, origin_mismatch="ignore")
+        moon_separation = data.moon(time).separation(coord, origin_mismatch="ignore")  # type: ignore[unexpected-keyword]
         return float(moon_separation.degree) >= self.min_distance
 
     async def filter_skycoord(self, time: Time, coords: SkyCoord, data: DataProvider) -> np.ndarray:
-        moon = astropy.coordinates.get_body("moon", time)
-        separations = moon.separation(coords, origin_mismatch="ignore").deg
+        moon = data.moon(time)
+        separations = moon.separation(coords, origin_mismatch="ignore").deg  # type: ignore[unexpected-keyword]
         return np.asarray(separations >= self.min_distance, dtype=np.bool_)
 
 

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/pyobs/robotic/scheduler/constraints/solarelevationconstraint.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Literal
 
 import astroplan
 import astropy.units as u
-from astropy.coordinates import get_sun
 from pydantic import Field
 
 from pyobs.utils.time import Time
@@ -34,12 +33,12 @@ class SolarElevationConstraint(Constraint):
         return astroplan.AtNightConstraint(max_solar_altitude=self.max_elevation * u.deg)
 
     async def __call__(self, time: Time, task: Task, data: DataProvider) -> bool:
-        sun = data.observer.sun_altaz(time)
+        sun = data.sun_altaz(time)
         in_range = self.min_elevation <= float(sun.alt.degree) <= self.max_elevation
         if self.direction == "both":
             return in_range
         else:
-            sun_coord = get_sun(time)
+            sun_coord = data.sun(time)
             transit = data.observer.target_meridian_transit_time(time, sun_coord, which="nearest")
             midnight = data.observer.midnight(time, which="nearest")
 

pyobs_core-2.0.0.dev68/pyobs/robotic/scheduler/dataprovider.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import datetime
+from dataclasses import dataclass
+from functools import cache
+
+import astropy.coordinates
+from astroplan import Observer
+from astropy.coordinates import SkyCoord
+
+from pyobs.robotic.scheduler.observationarchiveevolution import (
+    ObservationArchiveEvolution,
+)
+from pyobs.utils.time import Time
+
+
+@dataclass
+class TaskSuccess:
+    date: Time
+    night: Time
+
+
+class DataProvider:
+    """Data provider for Merit classes.
+
+    The ``@cache``d methods below are only safe to call from a single thread at a time -- they're
+    used exclusively from within the scheduler's dedicated single-worker executor
+    (`pyobs.robotic.scheduler._executor.run_cpu_bound`), never concurrently from the caller's main
+    event loop. A new `DataProvider` is created per `schedule()` call, so cache lifetime is bounded
+    to one schedule computation and never leaks stale values across runs.
+    """
+
+    def __init__(self, observer: Observer, archive: ObservationArchiveEvolution | None = None):
+        self.observer = observer
+        self.archive = archive if archive else ObservationArchiveEvolution(observer)
+
+    @cache
+    def last_sunset(self, time: Time) -> Time:
+        """Returns the time of the last sunset."""
+
+        # get last sunset
+        return Time(self.observer.sun_set_time(time, which="previous"))
+
+    @cache
+    def last_sunrise(self, time: Time) -> Time:
+        """Returns the time of the last sunrise."""
+
+        # get last sunset
+        return Time(self.observer.sun_rise_time(time, which="previous"))
+
+    @cache
+    def night(self, time: Time) -> datetime.date:
+        """Returns the time of the last sunset."""
+        sunset = self.last_sunset(time)
+        return sunset.to_datetime().date()
+
+    @cache
+    def sun(self, time: Time) -> SkyCoord:
+        """Returns the Sun's coordinates at the given time."""
+        return astropy.coordinates.get_sun(time)
+
+    @cache
+    def sun_altaz(self, time: Time) -> SkyCoord:
+        """Returns the Sun's AltAz coordinates at the given time, as seen by the observer."""
+        return self.observer.sun_altaz(time)
+
+    @cache
+    def moon(self, time: Time) -> SkyCoord:
+        """Returns the Moon's coordinates at the given time."""
+        return astropy.coordinates.get_body("moon", time)
+
+    @cache
+    def moon_illumination(self, time: Time) -> float:
+        """Returns the Moon's illumination fraction at the given time."""
+        return float(self.observer.moon_illumination(time))
+
+
+__all__ = ["DataProvider"]

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/pyobs/robotic/scheduler/observationarchiveevolution.py

@@ -9,6 +9,8 @@ from astroplan import Observer
 from ...utils.time import Time
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from pyobs.robotic import Observation, Task
     from pyobs.robotic.observation import ObservationList
     from pyobs.robotic.storage.observationarchive import ObservationArchive
@@ -20,6 +22,23 @@ class ObservationArchiveEvolution:
         self._obs_for_task: dict[Any, ObservationList] = {}
         self._obs_for_night: dict[datetime.date, ObservationList] = {}
         self._observer = observer
+        self._current_night: datetime.date | None = None
+        self._frozen = False
+
+    async def prefetch(self, tasks: Iterable[Task], start: Time, night: datetime.date) -> None:
+        """Populates the task cache and the one real night (anchored to `start`) up front. Call
+        once per schedule() run before any evaluation, then freeze(). Runs on the caller's
+        event loop (real I/O)."""
+        self._current_night = night
+        for task in tasks:
+            await self.observations_for_task(task)
+        await self.observations_for_night(night)
+
+    def freeze(self) -> None:
+        """Freezes observation cache. After this: a task-id miss raises RuntimeError; a night
+        miss seeds an empty list (future nights are provably empty of COMPLETED observations,
+        since `start` is pinned at least `_safety_time` ahead of real wall-clock time)."""
+        self._frozen = True
 
     async def evolve(self, scheduled_task: Observation, night: datetime.date) -> None:
         from pyobs.robotic.observation import Observation, ObservationState
@@ -50,6 +69,11 @@ class ObservationArchiveEvolution:
         from pyobs.robotic.observation import ObservationList, ObservationState
 
         if task.id not in self._obs_for_task:
+            if self._frozen:
+                raise RuntimeError(
+                    f"Task {task.id} not in observation cache after freeze(). "
+                    f"This is a bug -- all task IDs must be prefetched before freeze().",
+                )
             if self._obs_archive is not None:
                 self._obs_for_task[task.id] = await self._obs_archive.get_observations(
                     task=task, state=ObservationState.COMPLETED
@@ -70,7 +94,14 @@ class ObservationArchiveEvolution:
         from pyobs.robotic.observation import ObservationList, ObservationState
 
         if date not in self._obs_for_night:
-            if self._obs_archive is not None:
+            if self._frozen:
+                # `start` is pinned at least `_safety_time` ahead of real wall-clock time, so any
+                # night after the prefetched one cannot possibly have COMPLETED observations yet.
+                assert (
+                    self._current_night is not None and date > self._current_night
+                ), f"Night {date} not prefetched and not strictly after current night {self._current_night}."
+                self._obs_for_night[date] = ObservationList()
+            elif self._obs_archive is not None:
                 start = Time(datetime.datetime.combine(date, datetime.time(0, 0, 0)))
                 end = Time(datetime.datetime.combine(date, datetime.time(23, 59, 59)))
                 self._obs_for_night[date] = await self._obs_archive.get_observations(

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/pyobs/robotic/scheduler/ondemandscheduler.py

@@ -14,6 +14,7 @@ from pyobs.robotic.storage.observationarchive import ObservationArchive
 from pyobs.utils.time import Time
 
 from . import DataProvider
+from ._executor import run_cpu_bound
 from .constraints import Constraint
 from .observationarchiveevolution import ObservationArchiveEvolution
 from .taskscheduler import TaskScheduler
@@ -67,6 +68,12 @@ class OnDemandScheduler(TaskScheduler):
         data = DataProvider(self._observer, archive)
         projects_dict = {project.code: project for project in projects}
 
+        # prefetch historical observations on the main loop (the only place aiohttp session works),
+        # then freeze to prevent any lazy archive fetches from the worker thread
+        night = data.night(start)
+        await data.archive.prefetch(tasks, start, night)
+        data.archive.freeze()
+
         # schedule from start to end
         async for task in self.schedule_in_interval(tasks, projects_dict, start, end, data):
             # evolve archive -- night is keyed by the task's own scheduled time, not "now", since
@@ -252,7 +259,7 @@ class OnDemandScheduler(TaskScheduler):
     ) -> tuple[Task | None, float]:
 
         # evaluate all merit functions at given time
-        merits = await self.evaluate_constraints_and_merits(tasks, projects, start, end, data)
+        merits = await run_cpu_bound(self.evaluate_constraints_and_merits, tasks, projects, start, end, data)
 
         # find max one
         idx = np.argmax(merits)
@@ -276,7 +283,7 @@ class OnDemandScheduler(TaskScheduler):
         other_tasks = [t for t in tasks if t is not task]
         t = start + TimeDelta(step * u.second)
         while t < start + TimeDelta(task.estimate_duration(time=start) * u.second):
-            merits = await self.evaluate_constraints_and_merits(other_tasks, projects, t, end, data)
+            merits = await run_cpu_bound(self.evaluate_constraints_and_merits, other_tasks, projects, t, end, data)
             for i, m in enumerate(merits):
                 if m > merit:
                     return other_tasks[i], t, m
@@ -297,7 +304,9 @@ class OnDemandScheduler(TaskScheduler):
         better_start: Time = start + TimeDelta(task.estimate_duration(time=start) * u.second)
 
         # evaluate merit of better_task at new start time
-        merit = (await self.evaluate_constraints_and_merits([better_task], projects, better_start, end, data))[0]
+        merit = (
+            await run_cpu_bound(self.evaluate_constraints_and_merits, [better_task], projects, better_start, end, data)
+        )[0]
 
         # if it got better, return it, otherwise return Nones
         if merit >= better_merit:

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/pyobs/robotic/utils/archive/local_archive.py

@@ -166,7 +166,13 @@ class LocalArchive(Archive):
         return headers
 
     async def upload_frames(self, images: list[Image]) -> None:
-        pass
+        self._root_path.mkdir(parents=True, exist_ok=True)
+        for image in images:
+            filename = image.header.get("FNAME")
+            if not filename:
+                raise ValueError("Image has no FNAME header set, cannot determine output filename.")
+            image.writeto(str(self._root_path / filename), overwrite=True)
+        self._update_root()  # refresh the in-memory index so newly-written frames are queryable
 
 
 __all__ = ["LocalArchive"]

pyobs_core-2.0.0.dev68/pyobs/utils/pipeline/__init__.py

@@ -0,0 +1,4 @@
+from .pipeline import Pipeline
+from .reduction import Reduction
+
+__all__ = ["Reduction", "Pipeline"]

pyobs_core-2.0.0.dev66/pyobs/utils/pipeline/night.py → pyobs_core-2.0.0.dev68/pyobs/utils/pipeline/reduction.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import os.path
 from typing import Any
 
@@ -19,28 +20,27 @@ log = logging.getLogger(__name__)
 FILENAME = "{SITEID}{TELID}-{INSTRUME}-{DAY-OBS|date:}-{IMAGETYP}-{XBINNING}x{YBINNING}{FILTER|filter}.fits"
 
 
-class Night:
+class Reduction:
     def __init__(
         self,
         archive: dict[str, Any] | Archive,
         pipeline: dict[str, Any] | Pipeline,
-        worker_procs: int = 4,
         filenames_calib: str = FILENAME,
         min_flats: int = 10,
-        store_local: str | None = None,
+        output: str | dict[str, Any] | Archive | None = None,
         create_calibs: bool = True,
         calib_science: bool = True,
-        **kwargs: Any,
     ):
-        """Creates a Night object for reducing a given night.
+        """Creates a Reduction object for reducing a given observation period.
 
         Args:
-            archive: Archive to fetch images from and write results to.
+            archive: Archive to fetch raw and calibration frames from. Also the output
+                destination, unless output is set.
             pipeline: Science pipeline.
-            worker_procs: Number of worker processes.
             filenames_calib: Filename pattern for master calibration files.
             min_flats: Minimum number of raw frames to create flat field.
-            store_local: If True, files are stored in given local directory instead of uploaded to archive.
+            output: Where to write results. A string is a local directory path; a dict or
+                Archive is an output archive. If not set, results are written back to archive.
             create_calibs: If False, no calibration files are created for night.
             calib_science: If False, no science frames are calibrated.
         """
@@ -50,12 +50,18 @@ class Night:
         self._pipeline = get_object(pipeline, Pipeline, archive=archive)
 
         # stuff
-        self._worker_processes = worker_procs
         self._min_flats = min_flats
-        self._store_local = store_local
+        self._store_local: str | None = output if isinstance(output, str) else None
+        self._output_archive = (
+            self._archive if output is None or isinstance(output, str) else get_object(output, Archive)
+        )
         self._create_calibs = create_calibs
         self._calib_science = calib_science
 
+        # make sure the local output directory exists
+        if self._store_local:
+            os.makedirs(self._store_local, exist_ok=True)
+
         # cache for master calibration frames
         self._master_frames: dict[tuple[ImageType, str, str, str | None], Image] = {}
 
@@ -128,6 +134,7 @@ class Night:
         images = await self._archive.download_frames(infos)
         if len(images) < 3:
             log.warning("Too few (%d) frames found, skipping...", len(infos))
+            return None
 
         # create master
         calib: Image | None = None
@@ -191,7 +198,7 @@ class Night:
             calib.writeto(path, overwrite=True)
         else:
             log.info("Uploading master calibration frame as %s...", calib.header["FNAME"])
-            await self._archive.upload_frames([calib])
+            await self._output_archive.upload_frames([calib])
 
         # finished
         return calib
@@ -230,7 +237,7 @@ class Night:
                     calibrated.writeto(path, overwrite=True)
                 else:
                     log.info("(%d/%d) Uploading calibrated images as %s...", i, total, calibrated.header["FNAME"])
-                    await self._archive.upload_frames([calibrated])
+                    await self._output_archive.upload_frames([calibrated])
 
             except Exception:
                 log.exception("(%d/%d) Error processing image %s.", i, total, info.filename)
@@ -256,18 +263,32 @@ class Night:
             for binning in options["binnings"]:
                 # create bias and dark
                 if self._create_calibs:
-                    await self._create_master_calib(night, instrument, ImageType.BIAS, binning)
-                    await self._create_master_calib(night, instrument, ImageType.DARK, binning)
+                    try:
+                        await self._create_master_calib(night, instrument, ImageType.BIAS, binning)
+                    except Exception:
+                        log.exception("Error creating master bias for instrument %s, binning %s.", instrument, binning)
+                    try:
+                        await self._create_master_calib(night, instrument, ImageType.DARK, binning)
+                    except Exception:
+                        log.exception("Error creating master dark for instrument %s, binning %s.", instrument, binning)
 
                 # loop filters
                 for filter_name in options["filters"]:
                     # create flat
                     if self._create_calibs:
-                        await self._create_master_calib(night, instrument, ImageType.SKYFLAT, binning, filter_name)
+                        try:
+                            await self._create_master_calib(night, instrument, ImageType.SKYFLAT, binning, filter_name)
+                        except Exception:
+                            log.exception(
+                                "Error creating master flat for instrument %s, binning %s, filter %s.",
+                                instrument,
+                                binning,
+                                filter_name,
+                            )
 
                     # calibrate science data
                     if self._calib_science:
                         await self._calib_data(night, instrument, binning, filter_name)
 
 
-__all__ = ["Night"]
+__all__ = ["Reduction"]

{pyobs_core-2.0.0.dev66 → pyobs_core-2.0.0.dev68}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyobs-core"
-version = "2.0.0.dev66"
+version = "2.0.0.dev68"
 description = "robotic telescope software"
 authors = [{ name = "Tim-Oliver Husser", email = "thusser@uni-goettingen.de" }]
 requires-python = ">=3.11"

pyobs_core-2.0.0.dev66/pyobs/robotic/scheduler/dataprovider.py

@@ -1,49 +0,0 @@
-from __future__ import annotations
-
-import datetime
-from dataclasses import dataclass
-from functools import cache
-
-from astroplan import Observer
-
-from pyobs.robotic.scheduler.observationarchiveevolution import (
-    ObservationArchiveEvolution,
-)
-from pyobs.utils.time import Time
-
-
-@dataclass
-class TaskSuccess:
-    date: Time
-    night: Time
-
-
-class DataProvider:
-    """Data provider for Merit classes."""
-
-    def __init__(self, observer: Observer, archive: ObservationArchiveEvolution | None = None):
-        self.observer = observer
-        self.archive = archive if archive else ObservationArchiveEvolution(observer)
-
-    @cache
-    def last_sunset(self, time: Time) -> Time:
-        """Returns the time of the last sunset."""
-
-        # get last sunset
-        return Time(self.observer.sun_set_time(time, which="previous"))
-
-    @cache
-    def last_sunrise(self, time: Time) -> Time:
-        """Returns the time of the last sunrise."""
-
-        # get last sunset
-        return Time(self.observer.sun_rise_time(time, which="previous"))
-
-    @cache
-    def night(self, time: Time) -> datetime.date:
-        """Returns the time of the last sunset."""
-        sunset = self.last_sunset(time)
-        return sunset.to_datetime().date()
-
-
-__all__ = ["DataProvider"]

pyobs_core-2.0.0.dev66/pyobs/utils/pipeline/__init__.py

@@ -1,4 +0,0 @@
-from .night import Night
-from .pipeline import Pipeline
-
-__all__ = ["Night", "Pipeline"]