data-hub-watcher 1.2.1__tar.gz → 1.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {data_hub_watcher-1.2.1/src/data_hub_watcher.egg-info → data_hub_watcher-1.2.2}/PKG-INFO +1 -1
  2. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/pyproject.toml +2 -2
  3. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/api_client.py +8 -0
  4. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/cli_catalog.py +14 -6
  5. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/constants.py +6 -1
  6. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/events.py +4 -1
  7. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/monitor.py +12 -0
  8. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/run_detector.py +135 -56
  9. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/runtime.py +1 -0
  10. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/uploader.py +1 -2
  11. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2/src/data_hub_watcher.egg-info}/PKG-INFO +1 -1
  12. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_monitor_events.py +44 -0
  13. data_hub_watcher-1.2.2/tests/test_run_detector.py +743 -0
  14. data_hub_watcher-1.2.1/tests/test_run_detector.py +0 -366
  15. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/LICENSE +0 -0
  16. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/README.md +0 -0
  17. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/setup.cfg +0 -0
  18. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/__init__.py +0 -0
  19. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/cli.py +0 -0
  20. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/config_io.py +0 -0
  21. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/heartbeat.py +0 -0
  22. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/logging_setup.py +0 -0
  23. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/models.py +0 -0
  24. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/scheduled_task.py +0 -0
  25. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/self_update.py +0 -0
  26. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/service.py +0 -0
  27. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/state.py +0 -0
  28. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/updater.py +0 -0
  29. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/upgrade_worker.py +0 -0
  30. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/util.py +0 -0
  31. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/SOURCES.txt +0 -0
  32. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/dependency_links.txt +0 -0
  33. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/entry_points.txt +0 -0
  34. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/requires.txt +0 -0
  35. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/top_level.txt +0 -0
  36. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_api_client_version_header.py +0 -0
  37. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_catalog.py +0 -0
  38. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_self_update.py +0 -0
  39. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_service_reinstall.py +0 -0
  40. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_set_environment.py +0 -0
  41. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_config_migration.py +0 -0
  42. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_digests.py +0 -0
  43. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_events.py +0 -0
  44. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_heartbeat.py +0 -0
  45. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_logging_setup.py +0 -0
  46. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_monitor_initial_scan.py +0 -0
  47. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_preview_environment.py +0 -0
  48. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_run_detection_config.py +0 -0
  49. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_run_detector_hydration.py +0 -0
  50. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_runtime.py +0 -0
  51. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_scheduled_task.py +0 -0
  52. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_self_update.py +0 -0
  53. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_service.py +0 -0
  54. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_db_baseline.py +0 -0
  55. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_db_concurrency.py +0 -0
  56. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_db_path.py +0 -0
  57. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_forget.py +0 -0
  58. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_pending_uploads.py +0 -0
  59. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_updater.py +0 -0
  60. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_upgrade_worker.py +0 -0
  61. {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_uploader.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: data-hub-watcher
3
- Version: 1.2.1
3
+ Version: 1.2.2
4
4
  Summary: File-watcher agent for lab instrument PCs that ingests data into Data Hub.
5
5
  Author-email: Arcadia Science <swe@arcadiascience.com>
6
6
  License-Expression: MIT
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "data-hub-watcher"
3
- version = "1.2.1"
3
+ version = "1.2.2"
4
4
  description = "File-watcher agent for lab instrument PCs that ingests data into Data Hub."
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12"
@@ -40,7 +40,7 @@ windows-service = ["pywin32"]
40
40
  [project.urls]
41
41
  Homepage = "https://github.com/Arcadia-Science/data-hub"
42
42
  # Operator-facing install + auto-update guide — what someone landing on
43
- # the PyPI page actually needs. The developer-facing `developer-docs/watcher.md`
43
+ # the PyPI page actually needs. The developer-facing `docs/developer/watcher.md`
44
44
  # describes the editable-checkout workflow and is reachable from the
45
45
  # repository link below.
46
46
  Documentation = "https://datahub.arcadiascience.com/docs/install-the-watcher"
@@ -43,6 +43,14 @@ class ApiError(Exception):
43
43
  self.status_code = status_code
44
44
  self.detail = detail
45
45
 
46
+ @property
47
+ def is_transient(self) -> bool:
48
+ """True when no response came back (status 0) or the server returned a 5xx.
49
+
50
+ A 4xx needs someone to fix the cause, and replaying it would fail the same way.
51
+ """
52
+ return self.status_code == 0 or self.status_code >= 500
53
+
46
54
 
47
55
  DEFAULT_TIMEOUT: tuple[float, float] = (5, 30) # (connect, read) seconds
48
56
 
@@ -1,10 +1,11 @@
1
1
  """Build a machine-readable catalog of the watcher Click CLI.
2
2
 
3
3
  The docs site renders its CLI reference from a committed snapshot of this
4
- catalog, so help text and flags come straight from ``cli.py`` instead of
5
- hand-maintained MDX tables. The tests here are smoke checks on the walker; they
6
- don't guard the docs snapshot from going stale, since that copy lives in the
7
- data-hub-docs repo and is refreshed manually via ``make py-watcher-cli-catalog``.
4
+ catalog (``docs/site/src/lib/cli-catalog.snapshot.json``), so help text and
5
+ flags come straight from ``cli.py`` instead of hand-maintained MDX tables.
6
+ Refresh the snapshot with ``make py-watcher-cli-catalog``. The tests here are
7
+ smoke checks on the walker; the Python test workflow regenerates the snapshot
8
+ and fails when the committed copy is out of date.
8
9
  """
9
10
 
10
11
  from __future__ import annotations
@@ -21,8 +22,15 @@ from data_hub_watcher.constants import WATCHER_VERSION
21
22
  PROG = "data-hub-watcher"
22
23
  CATALOG_VERSION = "1"
23
24
 
24
- # Default on-disk location next to the watcher package root (repo checkout).
25
- DEFAULT_SNAPSHOT_PATH = Path(__file__).resolve().parents[2] / "cli-catalog.snapshot.json"
25
+ # Default on-disk location of the docs site snapshot (repo checkout).
26
+ DEFAULT_SNAPSHOT_PATH = (
27
+ Path(__file__).resolve().parents[3]
28
+ / "docs"
29
+ / "site"
30
+ / "src"
31
+ / "lib"
32
+ / "cli-catalog.snapshot.json"
33
+ )
26
34
 
27
35
 
28
36
  def _is_unset(value: Any) -> bool:
@@ -129,6 +129,11 @@ MAX_QUEUE_FILE_ATTEMPTS = 3
129
129
  # How many watcher starts retry an auto-mode upload that never finished.
130
130
  # After that the file waits for someone to request it from the run page.
131
131
  MAX_PENDING_UPLOAD_ATTEMPTS = 3
132
+ # A run report that fails with a connection error or a 5xx is retried until it
133
+ # succeeds, so a single-file run isn't stranded until the next restart. The
134
+ # delay doubles from the base, then stays at the cap.
135
+ RUN_REPORT_RETRY_BASE_DELAY_SECONDS = 5
136
+ RUN_REPORT_RETRY_MAX_DELAY_SECONDS = 300
132
137
  # Upload records older than this are pruned from the local state DB
133
138
  # to prevent unbounded growth on long-running watcher instances.
134
139
  PRUNE_DAYS = 90
@@ -215,7 +220,7 @@ def state_db_path(config_dir: Path, environment: str) -> Path:
215
220
 
216
221
  Keyed by `environment` (not `watcher_id`) so a deregister/re-register on
217
222
  the same host reuses the existing dedup history instead of re-uploading
218
- the whole backlog. See `developer-docs/watcher.md` for the rationale.
223
+ the whole backlog. See `docs/developer/watcher.md` for the rationale.
219
224
  """
220
225
  return config_dir / f"watcher-{environment}.db"
221
226
 
@@ -14,7 +14,10 @@ a database migration. Known ``kind`` values, with their per-kind
14
14
  ``details`` schemas:
15
15
 
16
16
  * ``run_report_failed`` -- POST/PATCH against
17
- ``/instruments/:id/runs[/:run_id]`` raised ``ApiError``.
17
+ ``/instruments/:id/runs[/:run_id]`` failed. Everything except a 4xx is
18
+ retried until it succeeds, so only the first of a row of such failures
19
+ is emitted; a 4xx is emitted every time because it is not retried.
20
+ ``status_code`` is 0 when no HTTP status came back.
18
21
  ``details = {"kind", "run_id", "operation": "create"|"update",
19
22
  "status_code", "error", "file_count"}``.
20
23
  * ``config_sync_failed`` -- the startup ``PUT /watchers/:id/config``
@@ -114,6 +114,10 @@ class FileMonitor:
114
114
  ``MAX_STABILITY_WAIT_SECONDS``.
115
115
  on_stable_file:
116
116
  Called with the `Path` of each stable file.
117
+ on_tick:
118
+ Optional callback run at the end of every stability check, even when
119
+ no files are pending, on the same thread as `on_stable_file`. Lets the
120
+ owner do periodic work without waiting for a filesystem event.
117
121
  state_db:
118
122
  Used to skip files that have already been uploaded.
119
123
  recursive:
@@ -138,6 +142,7 @@ class FileMonitor:
138
142
  event_reporter: EventReporter | None = None,
139
143
  seed_baseline: bool = False,
140
144
  max_stability_wait_seconds: int = MAX_STABILITY_WAIT_SECONDS,
145
+ on_tick: Callable[[], None] | None = None,
141
146
  ) -> None:
142
147
  self._watch_dir = watch_directory
143
148
  self._patterns = file_patterns
@@ -150,6 +155,7 @@ class FileMonitor:
150
155
  self._stability_period = stability_period
151
156
  self._max_stability_wait_seconds = max_stability_wait_seconds
152
157
  self._on_stable = on_stable_file
158
+ self._on_tick = on_tick
153
159
  self._state_db = state_db
154
160
  self._recursive = recursive
155
161
  # Optional so unit tests that build a FileMonitor in isolation
@@ -510,6 +516,12 @@ class FileMonitor:
510
516
  error=str(exc),
511
517
  )
512
518
 
519
+ if self._on_tick is not None:
520
+ try:
521
+ self._on_tick()
522
+ except Exception:
523
+ logger.exception("Monitor tick callback failed")
524
+
513
525
 
514
526
  def _stat_in_dedup_index(
515
527
  index: dict[str, list[tuple[int, float]]],
@@ -10,18 +10,26 @@ from __future__ import annotations
10
10
  import logging
11
11
  import os
12
12
  import re
13
+ import time
13
14
  from collections.abc import Callable
14
15
  from dataclasses import dataclass, field
15
16
  from datetime import datetime, timezone
16
17
  from pathlib import Path
18
+ from typing import Literal
17
19
 
18
20
  from data_hub_watcher.api_client import ApiError, DataHubClient
21
+ from data_hub_watcher.constants import (
22
+ RUN_REPORT_RETRY_BASE_DELAY_SECONDS,
23
+ RUN_REPORT_RETRY_MAX_DELAY_SECONDS,
24
+ )
19
25
  from data_hub_watcher.events import EventReporter, EventType, WatcherEvent
20
26
  from data_hub_watcher.heartbeat import WatcherCounters
21
27
  from data_hub_watcher.state import StateDB
22
28
 
23
29
  logger = logging.getLogger(__name__)
24
30
 
31
+ ReportOperation = Literal["create", "update"]
32
+
25
33
 
26
34
  def file_created_at(st: os.stat_result) -> float:
27
35
  """Best-effort on-disk creation time for *st*.
@@ -65,13 +73,17 @@ class RunState:
65
73
  # PATCHed to the API. Used as a cursor so `_update_run` can send only
66
74
  # newly-stable files instead of re-transmitting the full manifest on
67
75
  # every call. On a failed PATCH the cursor is left untouched so the
68
- # next stable file's PATCH carries the backlog plus the new entry.
76
+ # next PATCH, from a retry or a new file, carries the backlog.
69
77
  patched_file_count: int = 0
70
78
  # Last `acquired_at` value (seconds since epoch) we successfully sent
71
79
  # to the API for this run. Used so `_update_run` only PATCHes the
72
80
  # field when a later-stabilising file reveals an earlier creation
73
81
  # time than what the server already knows.
74
82
  acquired_at_sent: float | None = None
83
+ # Failed reports since the last success, and the wait before the next
84
+ # retry. The due time itself lives on `RunDetector._retry_at`.
85
+ retry_attempts: int = 0
86
+ retry_delay: int = 0
75
87
 
76
88
 
77
89
  def _run_acquired_at(files: list[FileInfo]) -> float | None:
@@ -132,6 +144,10 @@ class RunDetector:
132
144
  self._watch_dir = watch_directory
133
145
 
134
146
  self._runs: dict[str, RunState] = {}
147
+ # Due time (`time.monotonic()`) of the next retry, by run ID. A run has
148
+ # an entry exactly when its `retry_attempts` is non-zero. Only the
149
+ # monitor's stability-checker thread calls into this class, so no lock.
150
+ self._retry_at: dict[str, float] = {}
135
151
  # Set of parent-directory POSIX paths we've already surfaced a
136
152
  # ``kind=pattern_mismatch`` event for. Pattern misses are
137
153
  # routine for excluded files, so without throttling a
@@ -180,10 +196,89 @@ class RunDetector:
180
196
  return
181
197
  run.files.append(info)
182
198
 
183
- if not run.reported:
199
+ # Sent right away even while a retry is waiting, so the run catches up
200
+ # as soon as the server is back. A failure keeps doubling the backoff.
201
+ self._send_report(run)
202
+
203
+ def retry_failed_reports(self) -> None:
204
+ """Retry the run that has waited longest, if its retry is due.
205
+
206
+ Sends at most one request per call. A request to a slow server can
207
+ block until `DEFAULT_TIMEOUT` runs out, and the monitor's file checks
208
+ share this thread.
209
+ """
210
+ now = time.monotonic()
211
+ due = [(at, run_id) for run_id, at in self._retry_at.items() if at <= now]
212
+ if not due:
213
+ return
214
+ _, run_id = min(due)
215
+ del self._retry_at[run_id]
216
+ self._send_report(self._runs[run_id])
217
+
218
+ def _send_report(self, run: RunState) -> None:
219
+ if run.reported:
220
+ self._update_run(run)
221
+ else:
184
222
  self._report_new_run(run)
223
+
224
+ def _clear_retry(self, run: RunState) -> None:
225
+ run.retry_attempts = 0
226
+ run.retry_delay = 0
227
+ self._retry_at.pop(run.run_id, None)
228
+
229
+ def _note_report_success(self, run: RunState) -> None:
230
+ if run.retry_attempts:
231
+ logger.info(
232
+ "Run %s recovered after %d failed attempt(s)", run.run_id, run.retry_attempts
233
+ )
234
+ self._clear_retry(run)
235
+
236
+ def _handle_report_failure(
237
+ self, run: RunState, operation: ReportOperation, exc: Exception
238
+ ) -> None:
239
+ """Count a failed POST or PATCH, report it, and schedule a retry if it can help."""
240
+ verb = "report" if operation == "create" else "update"
241
+ if isinstance(exc, ApiError):
242
+ status_code, error, retryable = exc.status_code, exc.message, exc.is_transient
243
+ logger.warning("Failed to %s run %s: %s", verb, run.run_id, error)
185
244
  else:
186
- self._update_run(run)
245
+ # Anything else the client raises, such as a connection that drops
246
+ # mid-response or a proxy's HTML page instead of JSON, is retried
247
+ # like a connection error.
248
+ status_code, error, retryable = 0, f"{type(exc).__name__}: {exc}", True
249
+ logger.warning("Failed to %s run %s: %s", verb, run.run_id, error, exc_info=exc)
250
+ self._counters.errors += 1
251
+ # Report the first retryable failure in a row and every 4xx, so a long
252
+ # outage doesn't push other events out of the bounded queue.
253
+ if run.retry_attempts == 0 or not retryable:
254
+ self._reporter.report_error(
255
+ "run_report_failed",
256
+ f"Failed to {verb} run {run.run_id}: {error}",
257
+ run_id=run.run_id,
258
+ operation=operation,
259
+ status_code=status_code,
260
+ error=error,
261
+ file_count=len(run.files),
262
+ )
263
+ if retryable:
264
+ self._schedule_retry(run)
265
+ else:
266
+ self._clear_retry(run)
267
+
268
+ def _schedule_retry(self, run: RunState) -> None:
269
+ run.retry_delay = (
270
+ min(run.retry_delay * 2, RUN_REPORT_RETRY_MAX_DELAY_SECONDS)
271
+ if run.retry_delay
272
+ else RUN_REPORT_RETRY_BASE_DELAY_SECONDS
273
+ )
274
+ run.retry_attempts += 1
275
+ self._retry_at[run.run_id] = time.monotonic() + run.retry_delay
276
+ logger.info(
277
+ "Will retry run %s in %ds (%d failed attempt(s) so far)",
278
+ run.run_id,
279
+ run.retry_delay,
280
+ run.retry_attempts,
281
+ )
187
282
 
188
283
  # ------------------------------------------------------------------
189
284
  # run-ID extraction
@@ -271,35 +366,26 @@ class RunDetector:
271
366
  payload["acquired_at"] = datetime.fromtimestamp(acquired, tz=timezone.utc).isoformat()
272
367
  try:
273
368
  resp = self._client.report_run(self._instrument_id, payload)
274
- run.reported = True
275
- run.api_run_id = resp.id
276
- run.patched_file_count = len(run.files)
277
- run.acquired_at_sent = acquired
278
- self._state_db.record_run_reported(run.run_id)
279
- self._persist_detected_files(run)
280
- self._counters.runs_reported += 1
281
-
282
- self._reporter.queue_event(
283
- WatcherEvent(
284
- event_type=EventType.RUN_REPORTED,
285
- message=f"Run {run.run_id} reported",
286
- details={"run_id": run.run_id},
287
- )
288
- )
289
- logger.info("Reported new run %s (%d files)", run.run_id, len(run.files))
290
- except ApiError as exc:
291
- logger.warning("Failed to report run %s: %s (will retry)", run.run_id, exc.message)
292
- self._counters.errors += 1
293
- self._reporter.report_error(
294
- "run_report_failed",
295
- f"Failed to report run {run.run_id}: {exc.message}",
296
- run_id=run.run_id,
297
- operation="create",
298
- status_code=exc.status_code,
299
- error=exc.message,
300
- file_count=len(run.files),
301
- )
369
+ except Exception as exc:
370
+ self._handle_report_failure(run, "create", exc)
302
371
  return
372
+ run.reported = True
373
+ self._note_report_success(run)
374
+ run.api_run_id = resp.id
375
+ run.patched_file_count = len(run.files)
376
+ run.acquired_at_sent = acquired
377
+ self._state_db.record_run_reported(run.run_id)
378
+ self._persist_detected_files(run)
379
+ self._counters.runs_reported += 1
380
+
381
+ self._reporter.queue_event(
382
+ WatcherEvent(
383
+ event_type=EventType.RUN_REPORTED,
384
+ message=f"Run {run.run_id} reported",
385
+ details={"run_id": run.run_id},
386
+ )
387
+ )
388
+ logger.info("Reported new run %s (%d files)", run.run_id, len(run.files))
303
389
 
304
390
  if self._upload_cb:
305
391
  succeeded = self._upload_cb(run.run_id, list(run.files))
@@ -316,9 +402,9 @@ class RunDetector:
316
402
  API call entirely when the delta is empty avoids a needless round
317
403
  trip if `on_stable_file` re-fires for an already-reported file.
318
404
 
319
- On failure the PATCH cursor is left untouched, so the next
320
- stable file's PATCH naturally carries the backlog plus the new
321
- entry. The upload callback continues to receive only files the
405
+ On failure the PATCH cursor is left untouched, so a scheduled retry
406
+ or the next stable file's PATCH carries the backlog (plus the new
407
+ entry). The upload callback continues to receive only files the
322
408
  uploader hasn't seen yet (`uploaded_file_count` is independent
323
409
  of the PATCH cursor).
324
410
  """
@@ -334,6 +420,8 @@ class RunDetector:
334
420
  else None
335
421
  )
336
422
  if not new_files and earlier_acquired is None:
423
+ # Nothing is waiting to be sent, so a queued retry has no work left.
424
+ self._clear_retry(run)
337
425
  return
338
426
 
339
427
  payload: dict[str, object] = {
@@ -345,29 +433,20 @@ class RunDetector:
345
433
  ).isoformat()
346
434
  try:
347
435
  self._client.update_run(self._instrument_id, run.run_id, payload)
348
- run.patched_file_count = len(run.files)
349
- if earlier_acquired is not None:
350
- run.acquired_at_sent = earlier_acquired
351
- self._persist_detected_files(run)
352
- logger.info(
353
- "Updated run %s (sent %d new file(s), %d total)",
354
- run.run_id,
355
- len(new_files),
356
- len(run.files),
357
- )
358
- except ApiError as exc:
359
- logger.warning("Failed to update run %s: %s", run.run_id, exc.message)
360
- self._counters.errors += 1
361
- self._reporter.report_error(
362
- "run_report_failed",
363
- f"Failed to update run {run.run_id}: {exc.message}",
364
- run_id=run.run_id,
365
- operation="update",
366
- status_code=exc.status_code,
367
- error=exc.message,
368
- file_count=len(run.files),
369
- )
436
+ except Exception as exc:
437
+ self._handle_report_failure(run, "update", exc)
370
438
  return
439
+ self._note_report_success(run)
440
+ run.patched_file_count = len(run.files)
441
+ if earlier_acquired is not None:
442
+ run.acquired_at_sent = earlier_acquired
443
+ self._persist_detected_files(run)
444
+ logger.info(
445
+ "Updated run %s (sent %d new file(s), %d total)",
446
+ run.run_id,
447
+ len(new_files),
448
+ len(run.files),
449
+ )
371
450
 
372
451
  upload_new = run.files[run.uploaded_file_count :]
373
452
  if self._upload_cb and upload_new:
@@ -300,6 +300,7 @@ def build_runtime(
300
300
  stability_period=inst.stability_period_seconds,
301
301
  max_stability_wait_seconds=inst.max_stability_wait_seconds,
302
302
  on_stable_file=detector.on_stable_file,
303
+ on_tick=detector.retry_failed_reports,
303
304
  state_db=state_db,
304
305
  recursive=inst.run_detection.recursive,
305
306
  event_reporter=reporter,
@@ -847,8 +847,7 @@ class Uploader:
847
847
  return
848
848
  except ApiError as exc:
849
849
  last_exc = exc
850
- retryable = exc.status_code == 0 or exc.status_code >= 500
851
- if not retryable:
850
+ if not exc.is_transient:
852
851
  raise
853
852
  logger.warning(
854
853
  "Complete attempt %d/%d failed: %s",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: data-hub-watcher
3
- Version: 1.2.1
3
+ Version: 1.2.2
4
4
  Summary: File-watcher agent for lab instrument PCs that ingests data into Data Hub.
5
5
  Author-email: Arcadia Science <swe@arcadiascience.com>
6
6
  License-Expression: MIT
@@ -47,6 +47,7 @@ def _make_monitor(
47
47
  on_stable_file: MagicMock | None = None,
48
48
  reporter: MagicMock | None = None,
49
49
  max_stability_wait_seconds: int = MAX_STABILITY_WAIT_SECONDS,
50
+ on_tick: MagicMock | None = None,
50
51
  ) -> FileMonitor:
51
52
  return FileMonitor(
52
53
  watch_directory=watch_dir,
@@ -54,6 +55,7 @@ def _make_monitor(
54
55
  stability_period=1,
55
56
  max_stability_wait_seconds=max_stability_wait_seconds,
56
57
  on_stable_file=on_stable_file or MagicMock(),
58
+ on_tick=on_tick,
57
59
  state_db=state_db,
58
60
  recursive=False,
59
61
  event_reporter=reporter,
@@ -177,3 +179,45 @@ class TestStableCallbackFailure:
177
179
  assert call.args[0] == "stable_callback_failed"
178
180
  assert call.kwargs["path"] == str(f)
179
181
  assert "downstream blew up" in call.kwargs["error"]
182
+
183
+
184
+ class TestTickCallback:
185
+ def test_runs_when_no_files_are_pending(self, watch_dir: Path, state_db: StateDB) -> None:
186
+ on_tick = MagicMock()
187
+ monitor = _make_monitor(watch_dir, state_db, on_tick=on_tick)
188
+
189
+ monitor._check_pending()
190
+
191
+ on_tick.assert_called_once_with()
192
+
193
+ def test_runs_after_stable_file_callbacks(self, watch_dir: Path, state_db: StateDB) -> None:
194
+ calls: list[str] = []
195
+ monitor = _make_monitor(
196
+ watch_dir,
197
+ state_db,
198
+ on_stable_file=MagicMock(side_effect=lambda _path: calls.append("stable")),
199
+ on_tick=MagicMock(side_effect=lambda: calls.append("tick")),
200
+ )
201
+ f = watch_dir / "ready.csv"
202
+ f.write_text("hello")
203
+ st = f.stat()
204
+ now = time.monotonic()
205
+ monitor._pending[f] = _PendingFile(
206
+ path=f,
207
+ size=st.st_size,
208
+ mtime=st.st_mtime,
209
+ first_seen=now - 5,
210
+ last_changed=now - 5,
211
+ )
212
+
213
+ monitor._check_pending()
214
+
215
+ assert calls == ["stable", "tick"]
216
+
217
+ def test_exception_does_not_escape(self, watch_dir: Path, state_db: StateDB) -> None:
218
+ on_tick = MagicMock(side_effect=RuntimeError("tick blew up"))
219
+ monitor = _make_monitor(watch_dir, state_db, on_tick=on_tick)
220
+
221
+ monitor._check_pending() # Must not raise, or the checker thread would die.
222
+
223
+ on_tick.assert_called_once_with()