pyobs-core 2.0.0.dev85__tar.gz → 2.0.0.dev87__tar.gz

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyobs-core
-Version: 2.0.0.dev85
+Version: 2.0.0.dev87
 Summary: robotic telescope software
 Author-email: Tim-Oliver Husser <thusser@uni-goettingen.de>
 License-Expression: MIT

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/comm/xmpp/xmppcomm.py

@@ -60,6 +60,21 @@ def _retry_delay(attempt: int, cap: float = 30.0, base: float = 1.0) -> float:
     return random.uniform(0, min(cap, base * (2**attempt)))
 
 
+def _log_task_exception(task: asyncio.Task[Any]) -> None:
+    """Retrieve and log a background task's exception, if it failed.
+
+    Results of asyncio.create_task() that are never awaited or callback-ed have their
+    exceptions reported by the event loop's default handler as "Task exception was never
+    retrieved" -- a noisy ERROR traceback that carries no context. Retrieving the exception
+    here turns that into a normal log line with the task's failure attached.
+    """
+    if task.cancelled():
+        return
+    exc = task.exception()
+    if exc is not None:
+        log.error("Unhandled exception in XMPP event handler task", exc_info=exc)
+
+
 # anchored at both ends -- re.match alone doesn't anchor the end, so e.g. "user@domain/res/extra"
 # would otherwise still "match" as a valid prefix
 _JID_RE = re.compile(r"([\w_\-\.]+)@([\w_\-\.]+)/([\w_\-\.]+)$")
@@ -979,7 +994,12 @@ class XmppComm(Comm):
                         for callback in callbacks:
                             callback(state_obj)
         else:
-            asyncio.create_task(self._handle_event(msg, node))
+            # Non-state notifications are handled asynchronously. Attach a done-callback so the
+            # task's exception is retrieved: without it, a failure inside _handle_event is never
+            # retrieved and asyncio reports it as "Task exception was never retrieved" (see
+            # _handle_event for the payload-less notification case that used to trigger this).
+            task = asyncio.create_task(self._handle_event(msg, node))
+            task.add_done_callback(_log_task_exception)
 
     async def _handle_event(self, msg: Any, node: str) -> None:
         """Handles an event.
@@ -994,7 +1014,16 @@ class XmppComm(Comm):
         # get body, unescape it, parse it
         # State-node messages are handled synchronously in _handle_event_sync
         # before this async task runs. By the time we get here it's a pyobs event.
-        body = json.loads(xml.sax.saxutils.unescape(msg["pubsub_event"]["items"]["item"]["payload"].text))
+        # Not every notification carries an item with a payload: retract stanzas,
+        # node purges, and nodes with deliver_payloads off all arrive payload-less.
+        # The state-node path copes via _fetch_and_dispatch_state; for events there is
+        # nothing to refetch, so just drop the notification.
+        items = msg["pubsub_event"]["items"]
+        item = items["item"] if items is not None else None
+        payload = item["payload"] if item is not None else None
+        if payload is None or payload.text is None:
+            return
+        body = json.loads(xml.sax.saxutils.unescape(payload.text))
 
         # do we have a <delay> element?
         delay = msg.xml.findall("{urn:sleekxmpp:delay}delay")

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/mixins/fitsheader.py

@@ -135,6 +135,15 @@ class FitsHeaderMixin:
             except exc.RemoteError as e:
                 log.warning("Could not fetch FITS headers from %s: %s.", client, str(e))
                 continue
+            except Exception as e:
+                # XmppComm.execute() only converts IqError/IqTimeout into RemoteError; anything
+                # else raised while making or deserializing the RPC call (e.g. a peer returning a
+                # malformed IFitsHeaderBefore/After response) escapes as a raw exception. That must
+                # not take down the whole exposure -- log and skip this peer's headers (#767).
+                log.warning(
+                    "Could not fetch FITS headers from %s: unexpected %s: %s.", client, type(e).__name__, str(e)
+                )
+                continue
 
             # add them to fits file
             if headers:

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/robotic/storage/backend/observationarchive.py

@@ -60,23 +60,43 @@ class BackendObservationArchive(ObservationArchive):
 
         while True:
             try:
-                last_update = await self.last_update_time()
-                if self._last_update is None or self._last_update < last_update:
-                    self._observations = await self._get_schedule()
-                    sorted_obs = sorted(
-                        filter(lambda o: o.state == ObservationState.PENDING, self._observations),
-                        key=lambda o: o.start,
-                    )
-                    if len(sorted_obs) == 0:
-                        log.info("Downloaded new schedule.")
-                    else:
-                        obs = sorted_obs[0]
-                        log.info("Downloaded new schedule. Next observation is task %s at %s.", obs.task, obs.start)
-                    self._last_update = last_update
+                await self._update()
             except Exception as e:
                 log.error("Failed to update observations from backend: %s", e)
             await asyncio.sleep(5)
 
+    async def _update(self) -> None:
+        """Fetch the schedule from the backend and apply it if anything changed.
+
+        Re-fetches unconditionally on every poll instead of gating on the backend's
+        ``last_observation_update`` marker (per-process Django ``LocMemCache``, unreliable across
+        gunicorn workers -- see ``BackendTaskArchive``). Changes are detected by comparing full
+        observation contents via ``model_dump(use_task_id=True)``, keyed by observation ID so a
+        stable reordering of the same items is not mistaken for a change: the backend serializes
+        ``task`` as a plain FK ID, while cached copies get their ``task`` replaced by a full
+        ``Task`` when the mastermind calls ``fetch_task()``, so the ID normalization keeps both
+        sides comparable. The comparison includes ``state`` (``Observation.__eq__`` ignores it),
+        which matters for in-set transitions such as ``pending`` -> ``in_progress``; observations
+        leaving the fetched set -- e.g. ``window_expired``, which the server-side ``state``/
+        ``end_after`` filters drop -- are picked up as list shrinkage by the unconditional
+        refetch.
+        """
+        observations = await self._get_schedule()
+        if {o.id: o.model_dump(use_task_id=True) for o in observations} != {
+            o.id: o.model_dump(use_task_id=True) for o in self._observations
+        }:
+            self._observations = observations
+            self._last_update = Time.now()
+            sorted_obs = sorted(
+                filter(lambda o: o.state == ObservationState.PENDING, self._observations),
+                key=lambda o: o.start,
+            )
+            if len(sorted_obs) == 0:
+                log.info("Downloaded new schedule.")
+            else:
+                obs = sorted_obs[0]
+                log.info("Downloaded new schedule. Next observation is task %s at %s.", obs.task, obs.start)
+
     async def last_update_time(self) -> Time:
         """Fetches last schedule update time."""
         res = await http_request_with_retries(self._session, urljoin(self._url, "/api/last_observation_update/"))
@@ -229,7 +249,7 @@ class BackendObservationArchive(ObservationArchive):
             params["end_before"] = end_before.isot
         if end_after is not None:
             params["end_after"] = end_after.isot
-        observations = await http_request_paginated(self._session, url, params=params)
+        observations = await http_request_paginated(self._session, url, params=params, strict=True)
         return ObservationList([self.pyobs_model_validate(Observation, obs) for obs in observations])
 
 

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/robotic/storage/backend/taskarchive.py

@@ -56,18 +56,36 @@ class BackendTaskArchive(TaskArchive):
         """Update tasks in background."""
         while True:
             try:
-                last_update = await self.last_update_time()
-                if self._last_update is None or self._last_update < last_update:
-                    self._projects = await self._get_projects()
-                    self._tasks = await self._get_tasks()
-                    log.info("Downloaded new tasks/projects.")
-                    self._last_update = last_update
-                    if self._on_tasks_changed is not None:
-                        await self._on_tasks_changed()
+                await self._update()
             except Exception as e:
                 log.error("Failed to update tasks from backend: %s", e)
             await asyncio.sleep(5)
 
+    async def _update(self) -> None:
+        """Fetch tasks/projects from the backend and apply them if anything changed.
+
+        Re-fetches unconditionally on every poll instead of gating on the backend's
+        ``last_task_update`` marker: that marker is computed from a per-process Django
+        ``LocMemCache`` and is unreliable across gunicorn workers, so gating on it let the
+        mastermind run stale tasks forever. Real changes are detected by comparing the downloaded
+        content against the cached copy. The comparison uses ``model_dump()`` rather than pydantic
+        ``==``, which also compares runtime attributes (e.g. ``Task._cant_run_reason`` set by
+        ``can_run()``) and would flag unchanged tasks as changed on every poll; it is keyed by ID
+        so that a stable reordering of the same items (e.g. an unordered backend queryset) is not
+        mistaken for a change.
+        """
+        projects = await self._get_projects()
+        tasks = await self._get_tasks()
+        if {p.code: p.model_dump() for p in projects} != {p.code: p.model_dump() for p in self._projects} or {
+            t.id: t.model_dump() for t in tasks
+        } != {t.id: t.model_dump() for t in self._tasks}:
+            self._projects = projects
+            self._tasks = tasks
+            self._last_update = Time.now()
+            log.info("Downloaded new tasks/projects.")
+            if self._on_tasks_changed is not None:
+                await self._on_tasks_changed()
+
     async def last_update_time(self) -> Time:
         """Fetches last schedule update time."""
         res = await http_request_with_retries(self._session, urljoin(self._url, "/api/last_task_update/"))
@@ -75,16 +93,20 @@ class BackendTaskArchive(TaskArchive):
 
     async def _get_projects(self) -> list[Project]:
         """Fetch projects from backend."""
-        projects = await http_request_paginated(self._session, urljoin(self._url, "/api/projects/"))
+        projects = await http_request_paginated(self._session, urljoin(self._url, "/api/projects/"), strict=True)
         return [self.pyobs_model_validate(Project, project) for project in projects]
 
     async def _get_tasks(self) -> list[Task]:
         """Fetch tasks from backend."""
-        tasks = await http_request_paginated(self._session, urljoin(self._url, "/api/tasks/"))
+        tasks = await http_request_paginated(self._session, urljoin(self._url, "/api/tasks/"), strict=True)
         return [self.pyobs_model_validate(Task, task) for task in tasks]
 
     async def last_changed(self) -> Time | None:
-        """Returns time when last time any tasks changed."""
+        """Returns time when last time any tasks changed (as observed by this archive).
+
+        This is the local time at which the last content change was detected by the polling loop,
+        not the backend's marker timestamp -- the marker is per-process and unreliable.
+        """
         return self._last_update
 
     async def get_projects(self) -> list[Project]:

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/robotic/task.py

@@ -150,6 +150,7 @@ class Project(BaseModel):
     name: str = ""
     priority: float | None = Field(ge=0.0, le=9999.0, default=1.0)
     users: list[str] = Field(default_factory=list)
+    public: bool = False  # ingest backend `public` flag; default keeps old backends valid
 
 
 __all__ = ["Task", "TaskData", "Project"]

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/robotic/utils/archive/archive.py

@@ -42,6 +42,7 @@ class Archive(PolymorphicBaseModel, metaclass=ABCMeta):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> dict[str, list[Any]]: ...
 
     @abstractmethod
@@ -57,6 +58,7 @@ class Archive(PolymorphicBaseModel, metaclass=ABCMeta):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> list[FrameInfo]: ...
 
     @abstractmethod

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/robotic/utils/archive/local_archive.py

@@ -48,6 +48,7 @@ class LocalArchive(Archive):
                 "site",
                 "telescope",
                 "rlevel",
+                "obsnum",
             ]
         }
         for filename in filenames:
@@ -61,6 +62,7 @@ class LocalArchive(Archive):
             columns["site"].append(hdr["SITEID"] if "SITEID" in hdr else None)
             columns["telescope"].append(hdr["TELID"] if "TELID" in hdr else None)
             columns["rlevel"].append(hdr["RLEVEL"] if "RLEVEL" in hdr else None)
+            columns["obsnum"].append(str(hdr["OBSNUM"]) if "OBSNUM" in hdr else None)
         columns["filename"] = filenames
         self._data = pd.DataFrame(columns)
 
@@ -76,6 +78,7 @@ class LocalArchive(Archive):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> pd.DataFrame:
         data = self._data
         if start is not None:
@@ -98,6 +101,8 @@ class LocalArchive(Archive):
             data = data[data["filter"] == filter_name]
         if rlevel is not None:
             data = data[data["rlevel"] == rlevel]
+        if obsnum is not None:
+            data = data[data["obsnum"] == obsnum]
         return data
 
     async def list_options(
@@ -112,9 +117,10 @@ class LocalArchive(Archive):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> dict[str, list[Any]]:
         data = self._filter_data(
-            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel
+            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel, obsnum=obsnum
         )
         return {
             "binnings": list(data["binning"].unique()),
@@ -137,9 +143,10 @@ class LocalArchive(Archive):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> list[FrameInfo]:
         data = self._filter_data(
-            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel
+            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel, obsnum=obsnum
         )
         infos: list[FrameInfo] = []
         for _, row in data.iterrows():

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/robotic/utils/archive/pyobs_archive.py

@@ -68,10 +68,11 @@ class PyobsArchive(Archive):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> dict[str, list[Any]]:
         url = urllib.parse.urljoin(self.url, "frames/aggregate/")
         params = self._build_query(
-            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel
+            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel, obsnum=obsnum
         )
         async with aiohttp.ClientSession() as session:
             async with session.get(url, params=params, headers=self._headers, timeout=self._timeout) as response:
@@ -91,10 +92,11 @@ class PyobsArchive(Archive):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> list[FrameInfo]:
         url = urllib.parse.urljoin(self.url, "frames/")
         params = self._build_query(
-            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel
+            start, end, night, site, telescope, instrument, image_type, binning, filter_name, rlevel, obsnum=obsnum
         )
         frames: list[FrameInfo] = []
         params["offset"] = 0
@@ -123,6 +125,7 @@ class PyobsArchive(Archive):
         binning: str | None = None,
         filter_name: str | None = None,
         rlevel: int | None = None,
+        obsnum: str | None = None,
     ) -> dict[str, Any]:
         params: dict[str, Any] = {}
         if start is not None:
@@ -145,6 +148,8 @@ class PyobsArchive(Archive):
             params["FILTER"] = filter_name
         if rlevel is not None:
             params["RLEVEL"] = rlevel
+        if obsnum is not None:
+            params["OBSNUM"] = obsnum
         return params
 
     async def download_frames(self, infos: list[FrameInfo]) -> list[Image]:

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyobs/utils/http.py

@@ -46,13 +46,20 @@ async def http_request_with_retries(
 
 
 async def http_request_paginated(
-    session: aiohttp.ClientSession, url: str, method: str = "get", expected_status: int = 200, **kwargs: Any
+    session: aiohttp.ClientSession,
+    url: str,
+    method: str = "get",
+    expected_status: int = 200,
+    strict: bool = False,
+    **kwargs: Any,
 ) -> list[dict[str, Any]]:
     """Fetches all pages of a DRF-style paginated list endpoint and returns the combined results.
 
     If a page beyond the first becomes invalid mid-fetch (DRF's "Invalid page." 404, which can
     happen when concurrent writes shift page boundaries on a live-growing dataset), pagination
-    stops early with whatever was already fetched instead of failing the whole request.
+    stops early with whatever was already fetched instead of failing the whole request. With
+    ``strict=True`` that truncation is raised as an error instead, so callers that treat the
+    result as authoritative (e.g. replacing a cached list) never apply a partial result silently.
     """
     results: list[dict[str, Any]] = []
     next_url: str | None = url
@@ -69,6 +76,13 @@ async def http_request_paginated(
                 and isinstance(e.body, dict)
                 and e.body.get("detail") == "Invalid page."
             ):
+                if strict:
+                    log.warning(
+                        "Pagination page became invalid mid-fetch, raising instead of returning a "
+                        "truncated result of %d item(s).",
+                        len(results),
+                    )
+                    raise
                 log.warning("Pagination page became invalid mid-fetch, stopping early with %d result(s).", len(results))
                 break
             raise

{pyobs_core-2.0.0.dev85 → pyobs_core-2.0.0.dev87}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyobs-core"
-version = "2.0.0.dev85"
+version = "2.0.0.dev87"
 description = "robotic telescope software"
 authors = [{ name = "Tim-Oliver Husser", email = "thusser@uni-goettingen.de" }]
 requires-python = ">=3.11"