pyobs-core 2.23.0__tar.gz → 2.23.2__tar.gz

{pyobs_core-2.23.0 → pyobs_core-2.23.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pyobs-core
-Version: 2.23.0
+Version: 2.23.2
 Summary: robotic telescope software
 Project-URL: Homepage, https://www.pyobs.org
 Project-URL: Documentation, https://docs.pyobs.org

{pyobs_core-2.23.0 → pyobs_core-2.23.2}/pyobs/modules/image/imagewatcher.py

@@ -143,6 +143,12 @@ class ImageWatcher(Module):
         self._watchpath = watchpath
         self._notifier: Any | None = None
         self._queue = asyncio.Queue[tuple[str, float]]()
+        # files queued by add_file and not yet picked up by the worker, mapped to the time they're
+        # ready. A repeated event (e.g. a writer closing the file twice) moves the deadline instead
+        # of queueing the file again.
+        self._pending: dict[str, float] = {}
+        # files the worker is processing right now
+        self._active: set[str] = set()
         self._poll = poll
         self._poll_interval = poll_interval
         self._wait_time = wait_time
@@ -174,16 +180,52 @@ class ImageWatcher(Module):
         # than falling back to the module's outer background-task restart-with-backoff.
         while True:
             try:
-                watcher = RecursiveWatcher(Path(local), Mask.CLOSE_WRITE)
+                # CLOSE_WRITE for files written in place, MOVED_TO for files renamed into the tree
+                # (a rename produces no CLOSE_WRITE). CREATE is only there to see new directories:
+                # asyncinotify adds the watches for them, but anything written into them before that
+                # happened is never reported, so they're scanned below.
+                watcher = RecursiveWatcher(Path(local), Mask.CLOSE_WRITE | Mask.MOVED_TO | Mask.CREATE)
                 async for event in watcher.watch_recursive():
+                    log.debug("Inotify event: mask=%s path=%s", event.mask, event.path)
+
                     # get filename by replacing local with watchpath
                     filename = str(event.path).replace(local, self._watchpath)
 
+                    # new or moved-in directory: the watches exist by now, so scanning it finds
+                    # everything written earlier, inotify reports everything written later
+                    if Mask.ISDIR in event.mask:
+                        if Mask.CREATE in event.mask or Mask.MOVED_TO in event.mask:
+                            await self._scan_directory(filename)
+                        continue
+
+                    # a file that was only created isn't finished yet, wait for its CLOSE_WRITE
+                    if not event.mask & (Mask.CLOSE_WRITE | Mask.MOVED_TO):
+                        continue
+
                     # add file
                     await self.add_file(filename)
             except OSError as e:
                 log.warning("Inotify watch error (directory likely vanished mid-scan), restarting watch: %s", e)
 
+    async def _scan_directory(self, dirname: str) -> None:
+        """Add all files already in a directory that just appeared in the watch tree.
+
+        Files that are being processed right now are skipped, so a scan can't queue a file the worker
+        is just about to remove.
+
+        Args:
+            dirname: Directory to scan (recursively), as a path below the watch path.
+        """
+        try:
+            found = await self.vfs.find(dirname, "*")
+        except Exception as e:
+            log.warning("Could not scan new directory %s: %s", dirname, e)
+            return
+        for f in found:
+            filename = os.path.join(dirname, f)
+            if filename not in self._active:
+                await self.add_file(filename)
+
     async def _watch_poll(self) -> None:
         # init list (recursive, so subdirectories are picked up too; unfiltered by pattern here --
         # add_file is the single point of pattern filtering, same as inotify mode)
@@ -237,9 +279,18 @@ class ImageWatcher(Module):
         if not fnmatch.fnmatch(filename, self._pattern):
             return
 
+        ready_at = time.time() + self._wait_time
+
+        # already queued: just restart its timer, the worker reads the file when it's processed
+        if filename in self._pending:
+            log.info("File %s changed again, restarting wait time...", filename)
+            self._pending[filename] = ready_at
+            return
+
         # log and add file
         log.info("Adding new file %s...", filename)
-        self._queue.put_nowait((filename, time.time() + self._wait_time))
+        self._pending[filename] = ready_at
+        self._queue.put_nowait((filename, ready_at))
 
     async def _worker(self) -> None:
         """Worker thread."""
@@ -248,17 +299,26 @@ class ImageWatcher(Module):
         while True:
             # get next filename and wait some time
             filename, ready_at = await self._queue.get()
-            wait = ready_at - time.time()
-            if wait > 0:
+
+            # wait until ready, the deadline may be pushed back by add_file in the meantime
+            while True:
+                ready_at = self._pending.get(filename, ready_at)
+                wait = ready_at - time.time()
+                if wait <= 0:
+                    break
                 await asyncio.sleep(wait)
+            self._pending.pop(filename, None)
             log.info("Working on file %s...", filename)
 
+            self._active.add(filename)
             try:
                 await self._process_file(filename)
             except Exception as e:
                 self._handle_failure(filename, e)
             else:
                 self._retry.pop(filename, None)
+            finally:
+                self._active.discard(filename)
 
     async def _process_file(self, filename: str) -> None:
         """Read, archive and clean up a single file. Raises on any failure short of cleanup.

{pyobs_core-2.23.0 → pyobs_core-2.23.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyobs-core"
-version = "2.23.0"
+version = "2.23.2"
 description = "robotic telescope software"
 authors = [{ name = "Tim-Oliver Husser", email = "thusser@uni-goettingen.de" }]
 requires-python = ">=3.11"