data-hub-watcher 1.2.0__tar.gz → 1.2.2__tar.gz

{data_hub_watcher-1.2.0/src/data_hub_watcher.egg-info → data_hub_watcher-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-hub-watcher
-Version: 1.2.0
+Version: 1.2.2
 Summary: File-watcher agent for lab instrument PCs that ingests data into Data Hub.
 Author-email: Arcadia Science <swe@arcadiascience.com>
 License-Expression: MIT

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data-hub-watcher"
-version = "1.2.0"
+version = "1.2.2"
 description = "File-watcher agent for lab instrument PCs that ingests data into Data Hub."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -40,7 +40,7 @@ windows-service = ["pywin32"]
 [project.urls]
 Homepage = "https://github.com/Arcadia-Science/data-hub"
 # Operator-facing install + auto-update guide — what someone landing on
-# the PyPI page actually needs. The developer-facing `developer-docs/watcher.md`
+# the PyPI page actually needs. The developer-facing `docs/developer/watcher.md`
 # describes the editable-checkout workflow and is reachable from the
 # repository link below.
 Documentation = "https://datahub.arcadiascience.com/docs/install-the-watcher"

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/api_client.py

@@ -43,6 +43,14 @@ class ApiError(Exception):
         self.status_code = status_code
         self.detail = detail
 
+    @property
+    def is_transient(self) -> bool:
+        """True when no response came back (status 0) or the server returned a 5xx.
+
+        A 4xx needs someone to fix the cause, and replaying it would fail the same way.
+        """
+        return self.status_code == 0 or self.status_code >= 500
+
 
 DEFAULT_TIMEOUT: tuple[float, float] = (5, 30)  # (connect, read) seconds
 

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/cli_catalog.py

@@ -1,10 +1,11 @@
 """Build a machine-readable catalog of the watcher Click CLI.
 
 The docs site renders its CLI reference from a committed snapshot of this
-catalog, so help text and flags come straight from ``cli.py`` instead of
-hand-maintained MDX tables. The tests here are smoke checks on the walker; they
-don't guard the docs snapshot from going stale, since that copy lives in the
-data-hub-docs repo and is refreshed manually via ``make py-watcher-cli-catalog``.
+catalog (``docs/site/src/lib/cli-catalog.snapshot.json``), so help text and
+flags come straight from ``cli.py`` instead of hand-maintained MDX tables.
+Refresh the snapshot with ``make py-watcher-cli-catalog``. The tests here are
+smoke checks on the walker; the Python test workflow regenerates the snapshot
+and fails when the committed copy is out of date.
 """
 
 from __future__ import annotations
@@ -21,8 +22,15 @@ from data_hub_watcher.constants import WATCHER_VERSION
 PROG = "data-hub-watcher"
 CATALOG_VERSION = "1"
 
-# Default on-disk location next to the watcher package root (repo checkout).
-DEFAULT_SNAPSHOT_PATH = Path(__file__).resolve().parents[2] / "cli-catalog.snapshot.json"
+# Default on-disk location of the docs site snapshot (repo checkout).
+DEFAULT_SNAPSHOT_PATH = (
+    Path(__file__).resolve().parents[3]
+    / "docs"
+    / "site"
+    / "src"
+    / "lib"
+    / "cli-catalog.snapshot.json"
+)
 
 
 def _is_unset(value: Any) -> bool:

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/constants.py

@@ -120,12 +120,20 @@ RUN_DETECTION_PRESETS: list[tuple[str, str, str, bool]] = [
 MAX_STABILITY_WAIT_SECONDS = 300
 UPLOAD_RETRY_MAX = 3
 UPLOAD_RETRY_BASE_DELAY = 1
-# How many manual-mode heartbeat polls re-attempt the same queued file
-# (missing or failing to upload) before the watcher gives up and cancels the
-# request server-side. Distinct from `UPLOAD_RETRY_MAX` (per-upload S3 PUT
-# retries). ~3 min at the 60s heartbeat: long enough to ride out a blip,
-# short enough that a stale entry from a dir change self-clears.
+# How many upload-queue polls re-attempt the same queued file (missing or
+# failing to upload) before the watcher gives up and cancels the request
+# server-side. Distinct from `UPLOAD_RETRY_MAX` (per-upload S3 PUT retries).
+# ~3 min at the 60s poll: long enough to ride out a blip, short enough that
+# a stale entry from a dir change self-clears.
 MAX_QUEUE_FILE_ATTEMPTS = 3
+# How many watcher starts retry an auto-mode upload that never finished.
+# After that the file waits for someone to request it from the run page.
+MAX_PENDING_UPLOAD_ATTEMPTS = 3
+# A run report that fails with a connection error or a 5xx is retried until it
+# succeeds, so a single-file run isn't stranded until the next restart. The
+# delay doubles from the base, then stays at the cap.
+RUN_REPORT_RETRY_BASE_DELAY_SECONDS = 5
+RUN_REPORT_RETRY_MAX_DELAY_SECONDS = 300
 # Upload records older than this are pruned from the local state DB
 # to prevent unbounded growth on long-running watcher instances.
 PRUNE_DAYS = 90
@@ -212,7 +220,7 @@ def state_db_path(config_dir: Path, environment: str) -> Path:
 
     Keyed by `environment` (not `watcher_id`) so a deregister/re-register on
     the same host reuses the existing dedup history instead of re-uploading
-    the whole backlog. See `developer-docs/watcher.md` for the rationale.
+    the whole backlog. See `docs/developer/watcher.md` for the rationale.
     """
     return config_dir / f"watcher-{environment}.db"
 

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/events.py

@@ -14,7 +14,10 @@ a database migration. Known ``kind`` values, with their per-kind
 ``details`` schemas:
 
 * ``run_report_failed`` -- POST/PATCH against
-  ``/instruments/:id/runs[/:run_id]`` raised ``ApiError``.
+  ``/instruments/:id/runs[/:run_id]`` failed. Everything except a 4xx is
+  retried until it succeeds, so only the first of a row of such failures
+  is emitted; a 4xx is emitted every time because it is not retried.
+  ``status_code`` is 0 when no HTTP status came back.
   ``details = {"kind", "run_id", "operation": "create"|"update",
   "status_code", "error", "file_count"}``.
 * ``config_sync_failed`` -- the startup ``PUT /watchers/:id/config``
@@ -36,19 +39,25 @@ a database migration. Known ``kind`` values, with their per-kind
   after one or more consecutive failures.
   ``details = {"kind", "consecutive_failures", "gap_seconds"}``.
 * ``upload_queue_poll_failed`` -- ``GET /watchers/:id/upload-queue``
-  failed in manual mode. Emitted on the 1st failure and every 10th
-  repeat so a sustained outage isn't silent.
+  failed. Emitted on the 1st failure and every 10th repeat so a
+  sustained outage isn't silent.
   ``details = {"kind", "error", "consecutive_failures"}``.
 * ``update_check_failed`` -- ``GET /watchers/:id/update-check`` failed.
   Emitted only after 3 consecutive failures so we don't alert on a
   single hourly blip. ``details = {"kind", "error", "consecutive_failures"}``.
-* ``queued_file_missing`` -- a manual-mode upload-queue file was not found
-  on disk at its resolved path. Emitted once per file id (throttled across
-  heartbeat polls). ``details = {"kind", "file_id", "expected_path"}``.
+* ``queued_file_missing`` -- an upload-queue file was not found on disk
+  at its resolved path. Emitted once per file id (throttled across
+  polls). ``details = {"kind", "file_id", "expected_path"}``.
 * ``upload_request_cancelled`` -- the watcher gave up on a queued file
   after ``MAX_QUEUE_FILE_ATTEMPTS`` failed polls (missing or upload error)
   and reverted it to ``detected`` server-side so it leaves the queue.
   ``details = {"kind", "file_id", "attempts", "reason"}``.
+* ``upload_retries_exhausted`` -- an auto-mode upload still failed after
+  ``MAX_PENDING_UPLOAD_ATTEMPTS`` watcher starts, so it is no longer
+  retried. ``details = {"kind", "relative_path", "run_id", "attempts"}``.
+* ``pending_upload_missing`` -- an auto-mode upload waiting for a retry
+  was no longer on disk at startup, so the retry was dropped.
+  ``details = {"kind", "relative_path", "run_id"}``.
 """
 
 from __future__ import annotations

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/monitor.py

@@ -114,6 +114,10 @@ class FileMonitor:
         ``MAX_STABILITY_WAIT_SECONDS``.
     on_stable_file:
         Called with the `Path` of each stable file.
+    on_tick:
+        Optional callback run at the end of every stability check, even when
+        no files are pending, on the same thread as `on_stable_file`. Lets the
+        owner do periodic work without waiting for a filesystem event.
     state_db:
         Used to skip files that have already been uploaded.
     recursive:
@@ -138,6 +142,7 @@ class FileMonitor:
         event_reporter: EventReporter | None = None,
         seed_baseline: bool = False,
         max_stability_wait_seconds: int = MAX_STABILITY_WAIT_SECONDS,
+        on_tick: Callable[[], None] | None = None,
     ) -> None:
         self._watch_dir = watch_directory
         self._patterns = file_patterns
@@ -150,6 +155,7 @@ class FileMonitor:
         self._stability_period = stability_period
         self._max_stability_wait_seconds = max_stability_wait_seconds
         self._on_stable = on_stable_file
+        self._on_tick = on_tick
         self._state_db = state_db
         self._recursive = recursive
         # Optional so unit tests that build a FileMonitor in isolation
@@ -510,6 +516,12 @@ class FileMonitor:
                         error=str(exc),
                     )
 
+        if self._on_tick is not None:
+            try:
+                self._on_tick()
+            except Exception:
+                logger.exception("Monitor tick callback failed")
+
 
 def _stat_in_dedup_index(
     index: dict[str, list[tuple[int, float]]],

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/run_detector.py

@@ -10,18 +10,26 @@ from __future__ import annotations
 import logging
 import os
 import re
+import time
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import Literal
 
 from data_hub_watcher.api_client import ApiError, DataHubClient
+from data_hub_watcher.constants import (
+    RUN_REPORT_RETRY_BASE_DELAY_SECONDS,
+    RUN_REPORT_RETRY_MAX_DELAY_SECONDS,
+)
 from data_hub_watcher.events import EventReporter, EventType, WatcherEvent
 from data_hub_watcher.heartbeat import WatcherCounters
 from data_hub_watcher.state import StateDB
 
 logger = logging.getLogger(__name__)
 
+ReportOperation = Literal["create", "update"]
+
 
 def file_created_at(st: os.stat_result) -> float:
     """Best-effort on-disk creation time for *st*.
@@ -65,13 +73,17 @@ class RunState:
     # PATCHed to the API. Used as a cursor so `_update_run` can send only
     # newly-stable files instead of re-transmitting the full manifest on
     # every call. On a failed PATCH the cursor is left untouched so the
-    # next stable file's PATCH carries the backlog plus the new entry.
+    # next PATCH, from a retry or a new file, carries the backlog.
     patched_file_count: int = 0
     # Last `acquired_at` value (seconds since epoch) we successfully sent
     # to the API for this run. Used so `_update_run` only PATCHes the
     # field when a later-stabilising file reveals an earlier creation
     # time than what the server already knows.
     acquired_at_sent: float | None = None
+    # Failed reports since the last success, and the wait before the next
+    # retry. The due time itself lives on `RunDetector._retry_at`.
+    retry_attempts: int = 0
+    retry_delay: int = 0
 
 
 def _run_acquired_at(files: list[FileInfo]) -> float | None:
@@ -132,6 +144,10 @@ class RunDetector:
         self._watch_dir = watch_directory
 
         self._runs: dict[str, RunState] = {}
+        # Due time (`time.monotonic()`) of the next retry, by run ID. A run has
+        # an entry exactly when its `retry_attempts` is non-zero. Only the
+        # monitor's stability-checker thread calls into this class, so no lock.
+        self._retry_at: dict[str, float] = {}
         # Set of parent-directory POSIX paths we've already surfaced a
         # ``kind=pattern_mismatch`` event for. Pattern misses are
         # routine for excluded files, so without throttling a
@@ -180,10 +196,89 @@ class RunDetector:
             return
         run.files.append(info)
 
-        if not run.reported:
+        # Sent right away even while a retry is waiting, so the run catches up
+        # as soon as the server is back. A failure keeps doubling the backoff.
+        self._send_report(run)
+
+    def retry_failed_reports(self) -> None:
+        """Retry the run that has waited longest, if its retry is due.
+
+        Sends at most one request per call. A request to a slow server can
+        block until `DEFAULT_TIMEOUT` runs out, and the monitor's file checks
+        share this thread.
+        """
+        now = time.monotonic()
+        due = [(at, run_id) for run_id, at in self._retry_at.items() if at <= now]
+        if not due:
+            return
+        _, run_id = min(due)
+        del self._retry_at[run_id]
+        self._send_report(self._runs[run_id])
+
+    def _send_report(self, run: RunState) -> None:
+        if run.reported:
+            self._update_run(run)
+        else:
             self._report_new_run(run)
+
+    def _clear_retry(self, run: RunState) -> None:
+        run.retry_attempts = 0
+        run.retry_delay = 0
+        self._retry_at.pop(run.run_id, None)
+
+    def _note_report_success(self, run: RunState) -> None:
+        if run.retry_attempts:
+            logger.info(
+                "Run %s recovered after %d failed attempt(s)", run.run_id, run.retry_attempts
+            )
+        self._clear_retry(run)
+
+    def _handle_report_failure(
+        self, run: RunState, operation: ReportOperation, exc: Exception
+    ) -> None:
+        """Count a failed POST or PATCH, report it, and schedule a retry if it can help."""
+        verb = "report" if operation == "create" else "update"
+        if isinstance(exc, ApiError):
+            status_code, error, retryable = exc.status_code, exc.message, exc.is_transient
+            logger.warning("Failed to %s run %s: %s", verb, run.run_id, error)
         else:
-            self._update_run(run)
+            # Anything else the client raises, such as a connection that drops
+            # mid-response or a proxy's HTML page instead of JSON, is retried
+            # like a connection error.
+            status_code, error, retryable = 0, f"{type(exc).__name__}: {exc}", True
+            logger.warning("Failed to %s run %s: %s", verb, run.run_id, error, exc_info=exc)
+        self._counters.errors += 1
+        # Report the first retryable failure in a row and every 4xx, so a long
+        # outage doesn't push other events out of the bounded queue.
+        if run.retry_attempts == 0 or not retryable:
+            self._reporter.report_error(
+                "run_report_failed",
+                f"Failed to {verb} run {run.run_id}: {error}",
+                run_id=run.run_id,
+                operation=operation,
+                status_code=status_code,
+                error=error,
+                file_count=len(run.files),
+            )
+        if retryable:
+            self._schedule_retry(run)
+        else:
+            self._clear_retry(run)
+
+    def _schedule_retry(self, run: RunState) -> None:
+        run.retry_delay = (
+            min(run.retry_delay * 2, RUN_REPORT_RETRY_MAX_DELAY_SECONDS)
+            if run.retry_delay
+            else RUN_REPORT_RETRY_BASE_DELAY_SECONDS
+        )
+        run.retry_attempts += 1
+        self._retry_at[run.run_id] = time.monotonic() + run.retry_delay
+        logger.info(
+            "Will retry run %s in %ds (%d failed attempt(s) so far)",
+            run.run_id,
+            run.retry_delay,
+            run.retry_attempts,
+        )
 
     # ------------------------------------------------------------------
     # run-ID extraction
@@ -271,35 +366,26 @@ class RunDetector:
             payload["acquired_at"] = datetime.fromtimestamp(acquired, tz=timezone.utc).isoformat()
         try:
             resp = self._client.report_run(self._instrument_id, payload)
-            run.reported = True
-            run.api_run_id = resp.id
-            run.patched_file_count = len(run.files)
-            run.acquired_at_sent = acquired
-            self._state_db.record_run_reported(run.run_id)
-            self._persist_detected_files(run)
-            self._counters.runs_reported += 1
-
-            self._reporter.queue_event(
-                WatcherEvent(
-                    event_type=EventType.RUN_REPORTED,
-                    message=f"Run {run.run_id} reported",
-                    details={"run_id": run.run_id},
-                )
-            )
-            logger.info("Reported new run %s (%d files)", run.run_id, len(run.files))
-        except ApiError as exc:
-            logger.warning("Failed to report run %s: %s (will retry)", run.run_id, exc.message)
-            self._counters.errors += 1
-            self._reporter.report_error(
-                "run_report_failed",
-                f"Failed to report run {run.run_id}: {exc.message}",
-                run_id=run.run_id,
-                operation="create",
-                status_code=exc.status_code,
-                error=exc.message,
-                file_count=len(run.files),
-            )
+        except Exception as exc:
+            self._handle_report_failure(run, "create", exc)
             return
+        run.reported = True
+        self._note_report_success(run)
+        run.api_run_id = resp.id
+        run.patched_file_count = len(run.files)
+        run.acquired_at_sent = acquired
+        self._state_db.record_run_reported(run.run_id)
+        self._persist_detected_files(run)
+        self._counters.runs_reported += 1
+
+        self._reporter.queue_event(
+            WatcherEvent(
+                event_type=EventType.RUN_REPORTED,
+                message=f"Run {run.run_id} reported",
+                details={"run_id": run.run_id},
+            )
+        )
+        logger.info("Reported new run %s (%d files)", run.run_id, len(run.files))
 
         if self._upload_cb:
             succeeded = self._upload_cb(run.run_id, list(run.files))
@@ -316,9 +402,9 @@ class RunDetector:
         API call entirely when the delta is empty avoids a needless round
         trip if `on_stable_file` re-fires for an already-reported file.
 
-        On failure the PATCH cursor is left untouched, so the next
-        stable file's PATCH naturally carries the backlog plus the new
-        entry. The upload callback continues to receive only files the
+        On failure the PATCH cursor is left untouched, so a scheduled retry
+        or the next stable file's PATCH carries the backlog (plus the new
+        entry). The upload callback continues to receive only files the
         uploader hasn't seen yet (`uploaded_file_count` is independent
         of the PATCH cursor).
         """
@@ -334,6 +420,8 @@ class RunDetector:
             else None
         )
         if not new_files and earlier_acquired is None:
+            # Nothing is waiting to be sent, so a queued retry has no work left.
+            self._clear_retry(run)
             return
 
         payload: dict[str, object] = {
@@ -345,29 +433,20 @@ class RunDetector:
             ).isoformat()
         try:
             self._client.update_run(self._instrument_id, run.run_id, payload)
-            run.patched_file_count = len(run.files)
-            if earlier_acquired is not None:
-                run.acquired_at_sent = earlier_acquired
-            self._persist_detected_files(run)
-            logger.info(
-                "Updated run %s (sent %d new file(s), %d total)",
-                run.run_id,
-                len(new_files),
-                len(run.files),
-            )
-        except ApiError as exc:
-            logger.warning("Failed to update run %s: %s", run.run_id, exc.message)
-            self._counters.errors += 1
-            self._reporter.report_error(
-                "run_report_failed",
-                f"Failed to update run {run.run_id}: {exc.message}",
-                run_id=run.run_id,
-                operation="update",
-                status_code=exc.status_code,
-                error=exc.message,
-                file_count=len(run.files),
-            )
+        except Exception as exc:
+            self._handle_report_failure(run, "update", exc)
             return
+        self._note_report_success(run)
+        run.patched_file_count = len(run.files)
+        if earlier_acquired is not None:
+            run.acquired_at_sent = earlier_acquired
+        self._persist_detected_files(run)
+        logger.info(
+            "Updated run %s (sent %d new file(s), %d total)",
+            run.run_id,
+            len(new_files),
+            len(run.files),
+        )
 
         upload_new = run.files[run.uploaded_file_count :]
         if self._upload_cb and upload_new:
@@ -416,10 +495,9 @@ class RunDetector:
                 run_id=run_id,
                 files=files,
                 reported=True,
-                # On restart every hydrated file is either already
-                # uploaded (auto mode, uploader-side dedup will skip it
-                # anyway if not) or server-driven (manual mode, where
-                # `_upload_cb` is None and this counter is unused).
+                # Auto mode retries unfinished uploads from the state DB's
+                # `pending_uploads` on the upload worker, not from here.
+                # Manual mode has no `_upload_cb`, so the counter is unused.
                 uploaded_file_count=len(files),
                 # Every hydrated file came from `detected_files`, which
                 # is only written after a successful POST/PATCH — so the

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/runtime.py

@@ -75,9 +75,8 @@ class WatcherRuntime:
     # in their stop wait so a shutdown can be triggered from any thread.
     shutdown_event: threading.Event = field(default_factory=threading.Event)
     upgrade_restart_event: threading.Event = field(default_factory=threading.Event)
-    # Manual mode only: the thread that polls the upload queue off the
-    # heartbeat. ``None`` in auto mode, where uploads run on the monitor's
-    # stability-checker thread via the run detector's callback.
+    # The thread that polls the upload queue off the heartbeat. Always set by
+    # ``build_runtime``; ``None`` only in tests that build a runtime by hand.
     upload_worker: UploadQueueWorker | None = None
 
 
@@ -256,9 +255,8 @@ def build_runtime(
 
     is_auto = inst.upload_mode == "auto"
 
-    # Shared with the manual-mode ``UploadQueueWorker`` so a shutdown can
-    # interrupt an in-flight upload's backoff and abort between queued files;
-    # unused (but harmless) in auto mode.
+    # Shared with the ``UploadQueueWorker`` so a shutdown can interrupt an
+    # in-flight upload's backoff and abort between queued files.
     upload_stop_event = threading.Event()
 
     uploader = Uploader(
@@ -275,9 +273,14 @@ def build_runtime(
         stop_event=upload_stop_event,
     )
 
-    # Manual mode polls the server queue on its own thread; auto mode uploads
-    # via the run detector's callback on the stability-checker thread instead.
-    upload_worker = None if is_auto else UploadQueueWorker(uploader, stop_event=upload_stop_event)
+    # Both modes poll the server queue so "Request upload" works everywhere.
+    # Auto mode also uploads through the run detector's callback on the
+    # stability-checker thread, and retries unfinished uploads at startup.
+    upload_worker = UploadQueueWorker(
+        uploader,
+        stop_event=upload_stop_event,
+        retry_pending_on_start=is_auto,
+    )
 
     detector = RunDetector(
         pattern=inst.run_detection.pattern,
@@ -297,15 +300,16 @@ def build_runtime(
         stability_period=inst.stability_period_seconds,
         max_stability_wait_seconds=inst.max_stability_wait_seconds,
         on_stable_file=detector.on_stable_file,
+        on_tick=detector.retry_failed_reports,
         state_db=state_db,
         recursive=inst.run_detection.recursive,
         event_reporter=reporter,
         seed_baseline=seed_baseline,
     )
 
-    # Feeds the in-process auto-updater on every tick. Manual-mode upload
-    # polling used to run here too but moved to ``UploadQueueWorker`` so a
-    # slow upload can't delay a heartbeat.
+    # Feeds the in-process auto-updater on every tick. Upload-queue polling
+    # used to run here too but moved to ``UploadQueueWorker`` so a slow
+    # upload can't delay a heartbeat.
     def _on_tick() -> None:
         try:
             updater.on_tick()
@@ -488,8 +492,8 @@ def start_runtime(rt: WatcherRuntime, *, started_message: str) -> None:
     rt.detector.hydrate_from_state_db()
 
     rt.heartbeat.start()
-    # Start manual-mode upload polling before the (potentially long) initial
-    # scan so queued uploads keep draining while the scan walks the backlog.
+    # Start upload polling before the (potentially long) initial scan so
+    # queued uploads keep draining while the scan walks the backlog.
     if rt.upload_worker is not None:
         rt.upload_worker.start()
     rt.monitor.start()

{data_hub_watcher-1.2.0 → data_hub_watcher-1.2.2}/src/data_hub_watcher/state.py

@@ -79,6 +79,15 @@ class DetectedFileRecord:
     file_created_at: float | None = None
 
 
+@dataclass
+class PendingUploadRecord:
+    """An auto-mode upload that started but has not been confirmed."""
+
+    relative_path: str
+    run_id: str
+    attempts: int
+
+
 def clean_forget_prefix(prefix: str) -> str:
     """Normalize a folder prefix for `StateDB.forget_prefix`.
 
@@ -103,6 +112,9 @@ class StateDB:
       environment was first entered and were deliberately skipped (never
       uploaded). Distinct from `uploaded_files`: a baseline row means
       "ignore", not "sent". Never pruned, unlike `uploaded_files`.
+    - `pending_uploads` — auto-mode uploads that have not succeeded yet,
+      so the next start can retry them. `detected_files` can't answer
+      this because `uploaded_files` is pruned and it is not.
     - `meta` — small key/value store (e.g. the preview deployment URL this
       DB was seeded against) used to decide when to reset on a redeploy.
     """
@@ -254,6 +266,12 @@ class StateDB:
                     mtime         REAL NOT NULL
                 );
 
+                CREATE TABLE IF NOT EXISTS pending_uploads (
+                    relative_path TEXT PRIMARY KEY,
+                    run_id        TEXT NOT NULL,
+                    attempts      INTEGER NOT NULL DEFAULT 0
+                );
+
                 CREATE TABLE IF NOT EXISTS meta (
                     key   TEXT PRIMARY KEY,
                     value TEXT NOT NULL
@@ -321,7 +339,7 @@ class StateDB:
         counts: dict[str, int] = {}
         # Table names are fixed. The prefix is bound, never interpolated.
         with self._write_lock:
-            for table in ("uploaded_files", "detected_files", "baseline_files"):
+            for table in ("uploaded_files", "detected_files", "baseline_files", "pending_uploads"):
                 cur = self._conn.execute(
                     f"DELETE FROM {table} "
                     "WHERE relative_path = ? OR substr(relative_path, 1, ?) = ?",
@@ -410,8 +428,67 @@ class StateDB:
                 "VALUES (?, ?, ?, ?, ?, ?, ?)",
                 (filename, sha256, now, s3_key, relative_path, size_bytes, mtime),
             )
+            # Covers queue-driven uploads of a file an auto-mode upload left
+            # pending, so the next start doesn't hash it again for nothing.
+            self._conn.execute(
+                "DELETE FROM pending_uploads WHERE relative_path = ?",
+                (relative_path,),
+            )
             self._conn.commit()
 
+    # ------------------------------------------------------------------
+    # pending_uploads
+    # ------------------------------------------------------------------
+
+    def record_pending_uploads(self, run_id: str, relative_paths: Iterable[str]) -> None:
+        """Mark files as handed to the auto-mode uploader.
+
+        Existing rows keep their attempt count, so a file retried within
+        one session doesn't reset its restart budget.
+        """
+        rows = [(rel_path, run_id) for rel_path in relative_paths]
+        if not rows:
+            return
+        with self._write_lock:
+            self._conn.executemany(
+                "INSERT OR IGNORE INTO pending_uploads (relative_path, run_id) VALUES (?, ?)",
+                rows,
+            )
+            self._conn.commit()
+
+    def pending_uploads(self) -> list[PendingUploadRecord]:
+        """Return every pending upload, oldest first."""
+        cur = self._conn.execute(
+            "SELECT relative_path, run_id, attempts FROM pending_uploads ORDER BY rowid"
+        )
+        return [
+            PendingUploadRecord(relative_path=row[0], run_id=row[1], attempts=row[2])
+            for row in cur.fetchall()
+        ]
+
+    def clear_pending_upload(self, relative_path: str) -> None:
+        with self._write_lock:
+            self._conn.execute(
+                "DELETE FROM pending_uploads WHERE relative_path = ?",
+                (relative_path,),
+            )
+            self._conn.commit()
+
+    def bump_pending_upload_attempts(self, relative_path: str) -> int:
+        """Count one more failed retry and return the new total."""
+        with self._write_lock:
+            self._conn.execute(
+                "UPDATE pending_uploads SET attempts = attempts + 1 WHERE relative_path = ?",
+                (relative_path,),
+            )
+            self._conn.commit()
+            cur = self._conn.execute(
+                "SELECT attempts FROM pending_uploads WHERE relative_path = ?",
+                (relative_path,),
+            )
+            row = cur.fetchone()
+        return 0 if row is None else int(row[0])
+
     # ------------------------------------------------------------------
     # detected_files
     # ------------------------------------------------------------------