pyobs-core 2.2.0__tar.gz → 2.3.0__tar.gz

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyobs-core
-Version: 2.2.0
+Version: 2.3.0
 Summary: robotic telescope software
 Author-email: Tim-Oliver Husser <thusser@uni-goettingen.de>
 License-Expression: MIT

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/modules/camera/dummycamera.py

@@ -28,9 +28,11 @@ from pyobs.interfaces import (
     IImageFormat,
     ImageFormatCapabilities,
     ImageFormatState,
+    IMotion,
     IPointingRaDec,
     ITemperatures,
     IWindow,
+    MotionState,
     RaDecState,
     SensorReading,
     TemperaturesState,
@@ -39,7 +41,7 @@ from pyobs.interfaces import (
 )
 from pyobs.modules.camera.basecamera import BaseCamera
 from pyobs.utils import exceptions as exc
-from pyobs.utils.enums import ExposureStatus, ImageFormat, ImageType
+from pyobs.utils.enums import ExposureStatus, ImageFormat, ImageType, MotionStatus
 from pyobs.utils.time import Time
 
 log = logging.getLogger(__name__)
@@ -73,6 +75,7 @@ class DummyCamera(BaseCamera, IWindow, IBinning, ICooling, IGain, IImageFormat):
         max_mag: float = 20.0,
         seeing: float = 3.0,
         telescope: str | None = None,
+        roof: str | None = None,
         **kwargs: Any,
     ):
         """Creates a new dummy camera.
@@ -86,6 +89,10 @@ class DummyCamera(BaseCamera, IWindow, IBinning, ICooling, IGain, IImageFormat):
             max_mag: Maximum magnitude of simulated stars.
             seeing: Seeing in arcsec FWHM.
             telescope: Name of telescope module to read pointing from (for star simulation).
+            roof: Name of roof module to read open/closed state from. When given, the sky (flat
+                flux and stars) is only simulated while the roof is not parked -- frames come
+                back dark otherwise. Fails closed: until the roof's state has been received, the
+                roof is assumed closed.
         """
         BaseCamera.__init__(self, **kwargs)
         self.add_background_task(self._cooling_thread, True)
@@ -112,6 +119,10 @@ class DummyCamera(BaseCamera, IWindow, IBinning, ICooling, IGain, IImageFormat):
         self._telescope_module = telescope
         self._telescope_pos: SkyCoord = SkyCoord(0.0 * u.deg, 0.0 * u.deg, frame="icrs")
 
+        # roof state (updated via subscribe_state); fail closed until a state arrives
+        self._roof_module = roof
+        self._roof_open: bool = roof is None
+
         # camera state
         self._cooling = CoolingStatus()
         self._exposing = True
@@ -128,6 +139,10 @@ class DummyCamera(BaseCamera, IWindow, IBinning, ICooling, IGain, IImageFormat):
         """Update cached telescope position from IPointingRaDec state."""
         self._telescope_pos = SkyCoord(ra=state.ra * u.deg, dec=state.dec * u.deg, frame="icrs")
 
+    def _on_roof_state(self, state: MotionState) -> None:
+        """Update cached roof open/closed state from IMotion state."""
+        self._roof_open = state.status != MotionStatus.PARKED
+
     async def open(self) -> None:
         """Opens camera."""
         # publish capabilities before super().open()
@@ -154,6 +169,10 @@ class DummyCamera(BaseCamera, IWindow, IBinning, ICooling, IGain, IImageFormat):
         if self._telescope_module:
             await self.comm.subscribe_state(self._telescope_module, IPointingRaDec, self._on_telescope_state)
 
+        # subscribe to roof state if given
+        if self._roof_module:
+            await self.comm.subscribe_state(self._roof_module, IMotion, self._on_roof_state)
+
         # publish initial states
         await self.comm.set_state(
             ICooling,
@@ -206,7 +225,7 @@ class DummyCamera(BaseCamera, IWindow, IBinning, ICooling, IGain, IImageFormat):
 
         if exp_time > 0:
             data += make_noise_image(shape, distribution="gaussian", mean=exp_time / 1e4, stddev=exp_time / 1e5)
-            if open_shutter:
+            if open_shutter and self._roof_open:
                 sun_alt = self._sun_alt()
                 flat_counts = 30000 / np.exp(-1.28 * (4.209 + sun_alt)) * exp_time
                 data += make_noise_image(shape, distribution="gaussian", mean=flat_counts, stddev=flat_counts / 10.0)

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/modules/focus/focusseries.py

@@ -158,7 +158,7 @@ class AutoFocusSeries(Module, CameraSettingsMixin, IAutoFocus):
                     foc_state = focuser.get_state(IFocuser)
                     guess = foc_state.focus if foc_state is not None else 0.0
                     log.info("Using current focus of %.2fmm as initial guess.", guess)
-        except exc.PyobsError:
+        except (exc.PyobsError, ValueError):
             raise exc.FocusError("Could not fetch current focus value.")
 
         # define array of focus values to iterate
@@ -185,7 +185,7 @@ class AutoFocusSeries(Module, CameraSettingsMixin, IAutoFocus):
                     else:
                         await focuser.set_focus(float(foc))
 
-            except exc.PyobsError:
+            except (exc.PyobsError, ValueError):
                 raise exc.FocusError("Could not set new focus value.")
 
             # do exposure

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/modules/image/imagewatcher.py

@@ -5,7 +5,7 @@ import os
 import time
 import warnings
 from dataclasses import dataclass
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import Any
 
 from astropy.io import fits
@@ -47,6 +47,7 @@ class ImageWatcher(Module):
         poll_interval: int = 5,
         wait_time: int = 10,
         pattern: str = "*",
+        flatten: bool = True,
         **kwargs: Any,
     ):
         """Create a new image watcher.
@@ -59,6 +60,26 @@ class ImageWatcher(Module):
             wait_time: Time in seconds between adding a file to the list and processing it. Gives a file that is
                 still being written time to finish (relevant for poll mode and the initial scan) and spaces out
                 re-queued files after failed destination copies.
+            pattern: Only watch/process files matching this fnmatch pattern, checked against the full path (so
+                e.g. "*.fits" also matches files found in subdirectories). This is the single point of filtering,
+                applied uniformly in ``add_file`` regardless of watch mode -- inotify events, polling, and the
+                initial scan in ``open()`` all discover candidates first and let ``add_file`` decide, none of
+                them pre-filter by pattern before that.
+            flatten: For a non-templated destination (one without ``{placeholder}``s), whether to collapse the
+                file's path to just its basename under that destination (the historical behavior, default) or to
+                preserve its path relative to ``watchpath`` under the destination instead. Files found via
+                recursive watching or polling only keep their subdirectory structure at the destination when this
+                is False. Note this relies on the destination VFS root creating missing parent directories on
+                write (the default for e.g. ``LocalFile``) -- a destination configured to not do that would
+                repeatedly fail and re-queue any file whose relative path needs a parent directory that doesn't
+                exist yet.
+
+        Note:
+            If a directory holding a file that's already queued (added but not yet processed by ``_worker``) is
+            renamed before that happens, the worker's read of the file at its captured (pre-rename) path fails
+            and the file is dropped rather than picked up at its new location -- this is unrelated to, and not
+            covered by, the recursive-watching support for directories renamed after they no longer hold queued
+            files. Avoid renaming directories that may still hold queued/retrying files.
         """
         Module.__init__(self, **kwargs)
 
@@ -77,6 +98,7 @@ class ImageWatcher(Module):
         self._poll_interval = poll_interval
         self._wait_time = wait_time
         self._pattern = pattern
+        self._flatten = flatten
         self.current_file: CurrentFile | None = None
 
         # filename patterns
@@ -85,49 +107,58 @@ class ImageWatcher(Module):
         self._destinations = destinations
 
     async def _watch_inotify(self) -> None:
-        from asyncinotify import Inotify, Mask
+        from asyncinotify import Mask, RecursiveWatcher
 
         # get local directory
         local = await self.vfs.local_path(self._watchpath)
 
-        # Context manager to close the inotify handle after use
-        with Inotify() as inotify:
-            # add watch on local directory
-            inotify.add_watch(local, Mask.CLOSE_WRITE)
-
-            # iterate events forever
-            async for event in inotify:
-                # get filename by replacing local with watchpath
-                filename = str(event.path).replace(local, self._watchpath)
+        # recursively watch the local directory: watches are added for subdirectories as they're
+        # created or moved in, and removed as they're moved out (deletion needs no explicit
+        # removal -- the kernel invalidates the watch on its own, see the plan doc for details).
+        # A directory can also vanish between an event being received and asyncinotify acting on
+        # it (its own add_watch/rm_watch racing a concurrent delete), which raises OSError from
+        # inside the library rather than being handled there. Restart with a fresh watcher (a
+        # full re-walk) on that instead of letting it kill this task outright -- self-heals faster
+        # than falling back to the module's outer background-task restart-with-backoff.
+        while True:
+            try:
+                watcher = RecursiveWatcher(Path(local), Mask.CLOSE_WRITE)
+                async for event in watcher.watch_recursive():
+                    # get filename by replacing local with watchpath
+                    filename = str(event.path).replace(local, self._watchpath)
 
-                # add file
-                await self.add_file(filename)
+                    # add file
+                    await self.add_file(filename)
+            except OSError as e:
+                log.warning("Inotify watch error (directory likely vanished mid-scan), restarting watch: %s", e)
 
     async def _watch_poll(self) -> None:
-        # init list
-        files = set(await self.vfs.listdir(self._watchpath))
+        # init list (recursive, so subdirectories are picked up too; unfiltered by pattern here --
+        # add_file is the single point of pattern filtering, same as inotify mode)
+        files = set(await self.vfs.find(self._watchpath, "*"))
 
         # run forever
-        path = PurePosixPath(self._watchpath)
         while True:
+            await asyncio.sleep(self._poll_interval)
+
             # get new list
-            new_files = await self.vfs.listdir(self._watchpath)
+            new_files = set(await self.vfs.find(self._watchpath, "*"))
 
             # find all new files and add them
-            for f in new_files:
-                if f not in files:
-                    print(str(path / f))
-                    await self.add_file(str(path / f))
+            for f in new_files - files:
+                await self.add_file(os.path.join(self._watchpath, f))
 
             # store new list
-            files = set(new_files)
+            files = new_files
 
     async def open(self) -> None:
         """Open image watcher."""
         await Module.open(self)
 
-        # add all files from directory to queue
-        for filename in await self.vfs.listdir(self._watchpath):
+        # add all files from directory to queue (recursive, so subdirectories are picked up too;
+        # unfiltered by pattern here -- add_file is the single point of pattern filtering, same as
+        # inotify mode)
+        for filename in await self.vfs.find(self._watchpath, "*"):
             await self.add_file(os.path.join(self._watchpath, filename))
 
     async def close(self) -> None:
@@ -193,10 +224,15 @@ class ImageWatcher(Module):
                         if out_filename is None:
                             raise ValueError("Could not create name for file.")
 
-                    else:
+                    elif self._flatten:
                         # no formatting, so just add filename to destination
                         out_filename = os.path.join(pattern, os.path.basename(filename))
 
+                    else:
+                        # preserve the file's path relative to watchpath under the destination
+                        rel = PurePosixPath(filename).relative_to(self._watchpath)
+                        out_filename = str(PurePosixPath(pattern) / rel)
+
                     # store it
                     log.info("Storing file as %s...", out_filename)
                     self.current_file.out_filename = out_filename

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/modules/module.py

@@ -439,8 +439,8 @@ class Module(Object, IModule, IConfig):
                 caps = proxy.get_capabilities(IModule)
                 module_version = caps.version if caps is not None else ""
                 remote_location = caps.location if caps is not None else None
-        except exc.PyobsError:
-            # however this failed (transport or domain), we just skip this module
+        except (exc.PyobsError, ValueError):
+            # however this failed (transport, domain, or unresolvable peer), we just skip this module
             return True
 
         # log it

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/modules/robotic/scheduler.py

@@ -180,23 +180,38 @@ class Scheduler(Module, IRunnable, IRoboticScheduler):
         )
         log.info("Downloaded %d schedulable tasks(s).", len(tasks))
 
-        # download projects
-        self._projects = await self._task_archive.get_projects()
-
-        # compare new and old lists
+        # download projects -- keep the old list around long enough to diff against below,
+        # mirroring self._tasks (only overwritten once _need_update has been decided)
+        old_projects = self._projects
+        projects = await self._task_archive.get_projects()
+        # `updated_at` is excluded: it's a portal-side save timestamp (round-trip-only field, not
+        # scheduling-relevant), and a no-op re-save still bumps it -- comparing it in would force
+        # a reschedule on every no-op edit, not just real content changes.
+        projects_changed = {p.code: p.model_dump(exclude={"updated_at"}) for p in projects} != {
+            p.code: p.model_dump(exclude={"updated_at"}) for p in old_projects
+        }
+
+        # compare new and old lists, both by ID (removed/added) and by content (changed, same ID)
         removed, added = self._compare_task_lists(self._tasks, tasks)
+        changed = self._changed_task_ids(self._tasks, tasks)
 
         # schedule update
         self._need_update = True
 
-        # no changes?
-        if not self._trigger_on_every_update and len(removed) == 0 and len(added) == 0:
+        # no changes at all?
+        if (
+            not self._trigger_on_every_update
+            and len(removed) == 0
+            and len(added) == 0
+            and len(changed) == 0
+            and not projects_changed
+        ):
             # no need to re-schedule
             log.info("No change in list of blocks detected.")
             self._need_update = False
 
-        # has only the current block been removed?
-        log.info("Removed: %d, added: %d", len(removed), len(added))
+        # has only the current block been removed, with nothing else changed?
+        log.info("Removed: %d, added: %d, changed: %d", len(removed), len(added), len(changed))
         if len(removed) == 1:
             log.info(
                 "Found 1 removed block with ID %d. Last task ID was %s, current is %s.",
@@ -204,21 +219,20 @@ class Scheduler(Module, IRunnable, IRoboticScheduler):
                 str(self._last_task_id),
                 str(self._current_task_id),
             )
-        if len(removed) == 1 and len(added) == 0 and removed[0] == self._last_task_id:
+        if (
+            len(removed) == 1
+            and len(added) == 0
+            and len(changed) == 0
+            and not projects_changed
+            and removed[0] == self._last_task_id
+        ):
             # no need to re-schedule
             log.info("Only one removed block detected, which is the one currently running.")
             self._need_update = False
 
-        # check, if one of the removed blocks was actually in schedule
-        if len(removed) > 0 and self._need_update:
-            schedule = await self._schedule.get_schedule()
-            removed_from_schedule = [s for s in schedule if s.task.id in removed]
-            if len(removed_from_schedule) == 0:
-                log.info("Found %s tasks, but none of them was scheduled.", len(removed))
-                self._need_update = False
-
         # store blocks
         self._tasks = tasks
+        self._projects = projects
 
         # schedule update
         if self._need_update:
@@ -252,6 +266,33 @@ class Scheduler(Module, IRunnable, IRoboticScheduler):
 
         return sorted(additional1), sorted(additional2)  # type: ignore[type-var]
 
+    @staticmethod
+    def _content_dump(task: Task) -> dict[str, Any]:
+        """``task.model_dump(exclude={"updated_at"})``.
+
+        ``updated_at`` is a portal-side save timestamp, not scheduling-relevant content, and a
+        no-op re-save (e.g. an unchanged DRF PATCH) still bumps it -- leaving it in would report
+        every no-op edit as a "changed" task.
+        """
+        return task.model_dump(exclude={"updated_at"})
+
+    @classmethod
+    def _changed_task_ids(cls, tasks1: list[Task], tasks2: list[Task]) -> set[Any]:
+        """IDs present in both lists whose task content differs between them.
+
+        Comparison is via ``model_dump()`` (see :meth:`_content_dump`), not pydantic ``==``:
+        ``Task.__eq__`` also compares runtime-only ``PrivateAttr``s (e.g. ``_cant_run_reason`` set
+        by ``can_run()``, or ``_resolved_target`` set while scheduling), which would make a task
+        that has merely been scheduled/run look "changed" on the next poll. ``model_dump()``
+        excludes private attributes by construction. What's left after also dropping
+        ``updated_at`` (priority, duration, script, ...) are the fields that actually matter for
+        scheduling.
+        """
+        by_id1 = {t.id: t for t in tasks1}
+        by_id2 = {t.id: t for t in tasks2}
+        common = set(by_id1).intersection(by_id2)
+        return {i for i in common if cls._content_dump(by_id1[i]) != cls._content_dump(by_id2[i])}
+
     async def _schedule_worker(self) -> None:
         await asyncio.sleep(5)
 

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/modules/roof/dummyroof.py

@@ -122,7 +122,7 @@ class DummyRoof(BaseRoof, IRoof):
         await self._change_motion_status(MotionStatus.ABORTING)
 
         async with LockWithAbort(self._lock_motion, self._abort_motion):
-            await self._change_motion_status(MotionStatus.IDLE)
+            await self._change_motion_status(MotionStatus.PARKED if self._is_closed() else MotionStatus.IDLE)
 
 
 __all__ = ["DummyRoof"]

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/modules/telescope/basetelescope.py

@@ -78,9 +78,12 @@ class MissingObserverError(exc.MotionError):
     pass
 
 
-class AltitudeLimitError(exc.MotionError):
+class AltitudeLimitError(exc.PyobsError):
     """Destination altitude is below the configured limit -- an expected, deferrable condition
-    from a scheduler's point of view, not a failure to alert on."""
+    from a scheduler's point of view, not a failure to alert on. Deliberately not a MotionError:
+    it's not a motion fault (nothing was attempted), so it must not count towards the module's
+    repeated-motion-failure escalation (see _register_exception(exc.MotionError, ...) below) --
+    picking a target below the horizon is normal, not a sign the telescope is broken."""
 
     pass
 

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/robotic/storage/filesystem/observationarchive.py

@@ -248,6 +248,12 @@ class YamlObservationArchive(FileSystemObservationArchive):
             return ObservationList([self.pyobs_model_validate(Observation, obs) for obs in observations])
 
     async def _save_observations_to_file(self, path: str, observations: ObservationList) -> None:
+        # exclude_defaults now also applies to the nested (PolymorphicBaseModel) Task/Constraint/
+        # etc. fields, not just Observation's own -- pyobs-core#855 fixed inject_class_on_
+        # serialization() silently ignoring it. Saved YAML files drop more default-valued task
+        # fields (active, empty constraints/merits/script, priority: 1.0, ...) than before; this
+        # is intentional and round-trips fine (model_validate refills the defaults), but anyone
+        # reading these files raw will notice the shape change.
         data = [obs.model_dump(mode="json", exclude_defaults=True) for obs in observations]
         with open(path, "w") as f:
             yaml.safe_dump(data, f)

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/robotic/storage/portal/observationarchive.py

@@ -37,6 +37,9 @@ class PortalObservationArchive(ObservationArchive):
         # First failure logs immediately at ERROR (someone should know right away); repeats
         # during the same outage are throttled to at most one ERROR per minute.
         self._poll_error_throttle = LogThrottle(quiet_for=0.0, interval=60.0)
+        # Same pattern, keyed per observation id: a portal outage would otherwise re-log an
+        # ERROR every poll for as long as the cancel PUT keeps failing.
+        self._cancel_error_throttle = LogThrottle(quiet_for=0.0, interval=60.0)
 
         if auto_update:
             self.add_background_task(self._check_for_changes)
@@ -171,6 +174,39 @@ class PortalObservationArchive(ObservationArchive):
         """
         return self._observations
 
+    async def _resolve_or_cancel_pending(self, obs: Observation, task_archive: TaskArchive) -> bool:
+        """Resolve a *pending* obs.task; if the task archive has completed at least one poll and
+        the portal no longer has the task, mark obs canceled so it stops being retried on every
+        subsequent poll instead of logged-and-skipped forever.
+
+        Only for pending observations: an in-progress one should be allowed to finish even if its
+        task was deactivated mid-run, not flipped to canceled out from under the mastermind.
+
+        Returns:
+            True if obs.task resolved and obs is usable.
+        """
+        task_id = obs.task
+        await obs.fetch_task(task_archive)
+        if obs.task is not None:
+            return True
+        if await task_archive.last_changed() is None:
+            # task archive has never completed a poll (e.g. still starting up) -- a genuinely
+            # missing task looks the same as this transient race, so don't cancel on it.
+            log.error("Could not resolve task for observation %s, skipping.", obs.id)
+            return False
+        log.error("Could not resolve task for observation %s, marking canceled.", obs.id)
+        obs.task = task_id  # portal's task FK is non-nullable -- keep the id for the PUT payload
+        obs.state = ObservationState.CANCELED
+        try:
+            await self.update_observation(obs)
+            self._cancel_error_throttle.clear(str(obs.id))
+        except Exception as e:
+            if self._cancel_error_throttle.should_escalate(str(obs.id)):
+                log.error("Failed to mark observation %s canceled on portal: %s", obs.id, e)
+            else:
+                log.debug("Failed to mark observation %s canceled on portal: %s", obs.id, e)
+        return False
+
     async def get_next_observation(self, time: Time, task_archive: TaskArchive | None = None) -> Observation | None:
         """Returns the active scheduled task at the given time.
 
@@ -183,11 +219,8 @@ class PortalObservationArchive(ObservationArchive):
         """
         for obs in self._observations:
             if obs.state == "pending" and obs.start < time < obs.end:
-                if task_archive is not None:
-                    await obs.fetch_task(task_archive)
-                    if obs.task is None:
-                        log.error("Could not resolve task for observation %s, skipping.", obs.id)
-                        continue
+                if task_archive is not None and not await self._resolve_or_cancel_pending(obs, task_archive):
+                    continue
                 return obs
         else:
             return None

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/robotic/storage/portal/taskarchive.py

@@ -92,13 +92,17 @@ class PortalTaskArchive(TaskArchive):
         ``model_dump()`` rather than pydantic ``==``, which also compares runtime attributes (e.g.
         ``Task._cant_run_reason`` set by ``can_run()``) and would flag unchanged tasks as changed
         on every poll; it is keyed by ID so that a stable reordering of the same items (e.g. an
-        unordered portal queryset) is not mistaken for a change.
+        unordered portal queryset) is not mistaken for a change. ``updated_at`` is excluded from
+        both comparisons via ``exclude=`` so a no-op re-save doesn't trigger a re-download/
+        reschedule cascade (see pyobs-core#856).
         """
         projects = await self._get_projects()
         tasks = await self._get_tasks()
-        if {p.code: p.model_dump() for p in projects} != {p.code: p.model_dump() for p in self._projects} or {
-            t.id: t.model_dump() for t in tasks
-        } != {t.id: t.model_dump() for t in self._tasks}:
+        if {p.code: p.model_dump(exclude={"updated_at"}) for p in projects} != {
+            p.code: p.model_dump(exclude={"updated_at"}) for p in self._projects
+        } or {t.id: t.model_dump(exclude={"updated_at"}) for t in tasks} != {
+            t.id: t.model_dump(exclude={"updated_at"}) for t in self._tasks
+        }:
             self._projects = projects
             self._tasks = tasks
             self._last_update = Time.now()

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/robotic/task.py

@@ -155,6 +155,10 @@ class Project(BaseModel):
     priority: float | None = Field(ge=0.0, le=9999.0, default=1.0)
     users: list[str] = Field(default_factory=list)
     public: bool = False  # ingest backend `public` flag; default keeps old backends valid
+    updated_at: str | None = None
+    """Last-modification timestamp as emitted by pyobs-portal (DB-derived ``updated_at``,
+    added in pyobs-portal#134). Carried for round-trip fidelity so the strict ``Project`` model
+    (``extra="forbid"``) accepts the field the portal now sends; not used by pyobs itself."""
 
 
 __all__ = ["Task", "TaskData", "Project"]

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/utils/pipeline/reduction.py

@@ -44,6 +44,7 @@ class Reduction(ReductionBase):
         calib_science: bool = True,
         progress_callback: ProgressCallback | None = None,
         dark_exptime_tolerance: float = 0.01,
+        min_darks_per_group: int = 3,
     ):
         """Creates a Reduction object for reducing a given observation period.
 
@@ -60,6 +61,8 @@ class Reduction(ReductionBase):
             progress_callback: See ReductionBase.
             dark_exptime_tolerance: Relative tolerance for grouping a night's raw DARK frames
                 into per-exptime master series -- see pyobs.utils.exptime_grouping.
+            min_darks_per_group: Minimum number of raw frames in a per-exptime DARK group to
+                combine a master dark from; smaller groups are skipped individually.
         """
         super().__init__(archive=archive, pipeline=pipeline, min_flats=min_flats, progress_callback=progress_callback)
 
@@ -70,6 +73,7 @@ class Reduction(ReductionBase):
         self._create_calibs = create_calibs
         self._calib_science = calib_science
         self._dark_exptime_tolerance = dark_exptime_tolerance
+        self._min_darks_per_group = min_darks_per_group
 
         # make sure the local output directory exists
         if self._store_local:
@@ -195,8 +199,9 @@ class Reduction(ReductionBase):
         """Creates one master dark per exposure time present in the night's raw DARK frames
         (grouped within self._dark_exptime_tolerance) -- a night can have darks at more than
         one exptime now that DarkBiasScript can match them to the night's science exptimes
-        (#831). Frame groups with fewer than 3 members are skipped individually with a
-        warning, rather than aborting dark reduction for this instrument/binning entirely.
+        (#831). Frame groups with fewer than self._min_darks_per_group members are skipped
+        individually with a warning, rather than aborting dark reduction for this
+        instrument/binning entirely.
 
         If none of the night's raw darks carry EXPTIME at all (a fully legacy instrument, as
         opposed to a handful of untagged frames mixed in with tagged ones), grouping would
@@ -247,12 +252,12 @@ class Reduction(ReductionBase):
         for group, exptime in groups:
             label = "legacy (no exptime)" if exptime is None else f"~{exptime}s"
 
-            if len(group) < 3:
+            if len(group) < self._min_darks_per_group:
                 log.warning("Too few (%d) dark frames at %s, skipping...", len(group), label)
                 continue
 
             images = await self._archive.download_frames(group)
-            if len(images) < 3:
+            if len(images) < self._min_darks_per_group:
                 log.warning("Too few (%d) dark frames at %s, skipping...", len(images), label)
                 continue
 

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyobs/utils/serialization.py

@@ -7,7 +7,7 @@ from typing import Any, Self, TypeVar
 from astroplan import Observer
 from pydantic import BaseModel as PydanticBaseModel
 from pydantic import ConfigDict, PrivateAttr, model_serializer, model_validator
-from pydantic_core.core_schema import ValidationInfo, ValidatorFunctionWrapHandler
+from pydantic_core.core_schema import SerializationInfo, ValidationInfo, ValidatorFunctionWrapHandler
 
 from pyobs.comm import Comm
 from pyobs.object import PrivateAttrMixin
@@ -40,11 +40,57 @@ class BaseModel(PydanticBaseModel, PrivateAttrMixin):
 class PolymorphicBaseModel(BaseModel, metaclass=ABCMeta):  # type: ignore[misc]
     """Pydantic base model for pyobs sub classes that need to be serialized."""
 
+    def _flat_field_spec(self, spec: Any, info_kwarg: str) -> set[str] | None:
+        """Reduce a pydantic `exclude=`/`include=` spec to a flat set of field names.
+
+        `inject_class_on_serialization` hand-builds its output dict instead of delegating to
+        `handler`, so it also has to hand-apply `exclude`/`include` itself -- pydantic's spec
+        grammar supports nested dicts (partial excludes on sub-fields) and integer/`__all__` keys
+        (per-element excludes on sequences), forwarded into each field's own serialization. Doing
+        that generically would mean reimplementing a chunk of pydantic-core's own traversal; since
+        every current caller only ever passes a flat set of top-level field names (see
+        pyobs-core#855), only that flat form is supported. Anything else raises rather than
+        silently ignoring or partially applying the spec.
+        """
+        if spec is None:
+            return None
+        if isinstance(spec, set | frozenset):
+            return set(spec)
+        if isinstance(spec, dict) and all(v is True for v in spec.values()):
+            return set(spec.keys())
+        raise NotImplementedError(
+            f"PolymorphicBaseModel.model_dump() only supports a flat field-name {info_kwarg}= "
+            f"(no nested specs) -- got {spec!r} for {type(self).__name__}; see pyobs-core#855"
+        )
+
     @model_serializer(mode="wrap")
-    def inject_class_on_serialization(self, handler: ValidatorFunctionWrapHandler) -> dict[str, Any]:
+    def inject_class_on_serialization(
+        self, handler: ValidatorFunctionWrapHandler, info: SerializationInfo
+    ) -> dict[str, Any]:
         # Collect fields from the concrete runtime type to avoid Pydantic v2
         # resolving field schemas against the abstract base type when nested in a parent model
-        result = {field_name: getattr(self, field_name) for field_name in type(self).model_fields}
+        if info.exclude_computed_fields and type(self).__pydantic_decorators__.computed_fields:
+            raise NotImplementedError(
+                f"PolymorphicBaseModel.model_dump() doesn't support exclude_computed_fields= for "
+                f"{type(self).__name__}; see pyobs-core#855"
+            )
+        exclude = self._flat_field_spec(info.exclude, "exclude")
+        include = self._flat_field_spec(info.include, "include")
+        fields = type(self).model_fields
+        names = [n for n in fields if (include is None or n in include) and (exclude is None or n not in exclude)]
+
+        result: dict[str, Any] = {}
+        for name in names:
+            value = getattr(self, name)
+            if info.exclude_none and value is None:
+                continue
+            if info.exclude_unset and name not in self.model_fields_set:
+                continue
+            if info.exclude_defaults and value == fields[name].get_default(call_default_factory=True):
+                continue
+            alias = fields[name].alias
+            key = alias if (info.by_alias and alias) else name
+            result[key] = value
         result["class"] = f"{self.__module__}.{self.__class__.__name__}"
         return result
 

{pyobs_core-2.2.0 → pyobs_core-2.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyobs-core"
-version = "2.2.0"
+version = "2.3.0"
 description = "robotic telescope software"
 authors = [{ name = "Tim-Oliver Husser", email = "thusser@uni-goettingen.de" }]
 requires-python = ">=3.11"