data-hub-watcher 1.2.1__tar.gz → 1.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_hub_watcher-1.2.1/src/data_hub_watcher.egg-info → data_hub_watcher-1.2.2}/PKG-INFO +1 -1
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/pyproject.toml +2 -2
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/api_client.py +8 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/cli_catalog.py +14 -6
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/constants.py +6 -1
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/events.py +4 -1
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/monitor.py +12 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/run_detector.py +135 -56
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/runtime.py +1 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/uploader.py +1 -2
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2/src/data_hub_watcher.egg-info}/PKG-INFO +1 -1
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_monitor_events.py +44 -0
- data_hub_watcher-1.2.2/tests/test_run_detector.py +743 -0
- data_hub_watcher-1.2.1/tests/test_run_detector.py +0 -366
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/LICENSE +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/README.md +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/setup.cfg +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/__init__.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/cli.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/config_io.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/heartbeat.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/logging_setup.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/models.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/scheduled_task.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/self_update.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/service.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/state.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/updater.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/upgrade_worker.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher/util.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/SOURCES.txt +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/dependency_links.txt +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/entry_points.txt +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/requires.txt +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/src/data_hub_watcher.egg-info/top_level.txt +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_api_client_version_header.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_catalog.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_self_update.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_service_reinstall.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_cli_set_environment.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_config_migration.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_digests.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_events.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_heartbeat.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_logging_setup.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_monitor_initial_scan.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_preview_environment.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_run_detection_config.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_run_detector_hydration.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_runtime.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_scheduled_task.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_self_update.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_service.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_db_baseline.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_db_concurrency.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_db_path.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_forget.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_state_pending_uploads.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_updater.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_upgrade_worker.py +0 -0
- {data_hub_watcher-1.2.1 → data_hub_watcher-1.2.2}/tests/test_uploader.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "data-hub-watcher"
|
|
3
|
-
version = "1.2.
|
|
3
|
+
version = "1.2.2"
|
|
4
4
|
description = "File-watcher agent for lab instrument PCs that ingests data into Data Hub."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -40,7 +40,7 @@ windows-service = ["pywin32"]
|
|
|
40
40
|
[project.urls]
|
|
41
41
|
Homepage = "https://github.com/Arcadia-Science/data-hub"
|
|
42
42
|
# Operator-facing install + auto-update guide — what someone landing on
|
|
43
|
-
# the PyPI page actually needs. The developer-facing `
|
|
43
|
+
# the PyPI page actually needs. The developer-facing `docs/developer/watcher.md`
|
|
44
44
|
# describes the editable-checkout workflow and is reachable from the
|
|
45
45
|
# repository link below.
|
|
46
46
|
Documentation = "https://datahub.arcadiascience.com/docs/install-the-watcher"
|
|
@@ -43,6 +43,14 @@ class ApiError(Exception):
|
|
|
43
43
|
self.status_code = status_code
|
|
44
44
|
self.detail = detail
|
|
45
45
|
|
|
46
|
+
@property
|
|
47
|
+
def is_transient(self) -> bool:
|
|
48
|
+
"""True when no response came back (status 0) or the server returned a 5xx.
|
|
49
|
+
|
|
50
|
+
A 4xx needs someone to fix the cause, and replaying it would fail the same way.
|
|
51
|
+
"""
|
|
52
|
+
return self.status_code == 0 or self.status_code >= 500
|
|
53
|
+
|
|
46
54
|
|
|
47
55
|
DEFAULT_TIMEOUT: tuple[float, float] = (5, 30) # (connect, read) seconds
|
|
48
56
|
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
"""Build a machine-readable catalog of the watcher Click CLI.
|
|
2
2
|
|
|
3
3
|
The docs site renders its CLI reference from a committed snapshot of this
|
|
4
|
-
catalog, so help text and
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
catalog (``docs/site/src/lib/cli-catalog.snapshot.json``), so help text and
|
|
5
|
+
flags come straight from ``cli.py`` instead of hand-maintained MDX tables.
|
|
6
|
+
Refresh the snapshot with ``make py-watcher-cli-catalog``. The tests here are
|
|
7
|
+
smoke checks on the walker; the Python test workflow regenerates the snapshot
|
|
8
|
+
and fails when the committed copy is out of date.
|
|
8
9
|
"""
|
|
9
10
|
|
|
10
11
|
from __future__ import annotations
|
|
@@ -21,8 +22,15 @@ from data_hub_watcher.constants import WATCHER_VERSION
|
|
|
21
22
|
PROG = "data-hub-watcher"
|
|
22
23
|
CATALOG_VERSION = "1"
|
|
23
24
|
|
|
24
|
-
# Default on-disk location
|
|
25
|
-
DEFAULT_SNAPSHOT_PATH =
|
|
25
|
+
# Default on-disk location of the docs site snapshot (repo checkout).
|
|
26
|
+
DEFAULT_SNAPSHOT_PATH = (
|
|
27
|
+
Path(__file__).resolve().parents[3]
|
|
28
|
+
/ "docs"
|
|
29
|
+
/ "site"
|
|
30
|
+
/ "src"
|
|
31
|
+
/ "lib"
|
|
32
|
+
/ "cli-catalog.snapshot.json"
|
|
33
|
+
)
|
|
26
34
|
|
|
27
35
|
|
|
28
36
|
def _is_unset(value: Any) -> bool:
|
|
@@ -129,6 +129,11 @@ MAX_QUEUE_FILE_ATTEMPTS = 3
|
|
|
129
129
|
# How many watcher starts retry an auto-mode upload that never finished.
|
|
130
130
|
# After that the file waits for someone to request it from the run page.
|
|
131
131
|
MAX_PENDING_UPLOAD_ATTEMPTS = 3
|
|
132
|
+
# A run report that fails with a connection error or a 5xx is retried until it
|
|
133
|
+
# succeeds, so a single-file run isn't stranded until the next restart. The
|
|
134
|
+
# delay doubles from the base, then stays at the cap.
|
|
135
|
+
RUN_REPORT_RETRY_BASE_DELAY_SECONDS = 5
|
|
136
|
+
RUN_REPORT_RETRY_MAX_DELAY_SECONDS = 300
|
|
132
137
|
# Upload records older than this are pruned from the local state DB
|
|
133
138
|
# to prevent unbounded growth on long-running watcher instances.
|
|
134
139
|
PRUNE_DAYS = 90
|
|
@@ -215,7 +220,7 @@ def state_db_path(config_dir: Path, environment: str) -> Path:
|
|
|
215
220
|
|
|
216
221
|
Keyed by `environment` (not `watcher_id`) so a deregister/re-register on
|
|
217
222
|
the same host reuses the existing dedup history instead of re-uploading
|
|
218
|
-
the whole backlog. See `
|
|
223
|
+
the whole backlog. See `docs/developer/watcher.md` for the rationale.
|
|
219
224
|
"""
|
|
220
225
|
return config_dir / f"watcher-{environment}.db"
|
|
221
226
|
|
|
@@ -14,7 +14,10 @@ a database migration. Known ``kind`` values, with their per-kind
|
|
|
14
14
|
``details`` schemas:
|
|
15
15
|
|
|
16
16
|
* ``run_report_failed`` -- POST/PATCH against
|
|
17
|
-
``/instruments/:id/runs[/:run_id]``
|
|
17
|
+
``/instruments/:id/runs[/:run_id]`` failed. Everything except a 4xx is
|
|
18
|
+
retried until it succeeds, so only the first of a row of such failures
|
|
19
|
+
is emitted; a 4xx is emitted every time because it is not retried.
|
|
20
|
+
``status_code`` is 0 when no HTTP status came back.
|
|
18
21
|
``details = {"kind", "run_id", "operation": "create"|"update",
|
|
19
22
|
"status_code", "error", "file_count"}``.
|
|
20
23
|
* ``config_sync_failed`` -- the startup ``PUT /watchers/:id/config``
|
|
@@ -114,6 +114,10 @@ class FileMonitor:
|
|
|
114
114
|
``MAX_STABILITY_WAIT_SECONDS``.
|
|
115
115
|
on_stable_file:
|
|
116
116
|
Called with the `Path` of each stable file.
|
|
117
|
+
on_tick:
|
|
118
|
+
Optional callback run at the end of every stability check, even when
|
|
119
|
+
no files are pending, on the same thread as `on_stable_file`. Lets the
|
|
120
|
+
owner do periodic work without waiting for a filesystem event.
|
|
117
121
|
state_db:
|
|
118
122
|
Used to skip files that have already been uploaded.
|
|
119
123
|
recursive:
|
|
@@ -138,6 +142,7 @@ class FileMonitor:
|
|
|
138
142
|
event_reporter: EventReporter | None = None,
|
|
139
143
|
seed_baseline: bool = False,
|
|
140
144
|
max_stability_wait_seconds: int = MAX_STABILITY_WAIT_SECONDS,
|
|
145
|
+
on_tick: Callable[[], None] | None = None,
|
|
141
146
|
) -> None:
|
|
142
147
|
self._watch_dir = watch_directory
|
|
143
148
|
self._patterns = file_patterns
|
|
@@ -150,6 +155,7 @@ class FileMonitor:
|
|
|
150
155
|
self._stability_period = stability_period
|
|
151
156
|
self._max_stability_wait_seconds = max_stability_wait_seconds
|
|
152
157
|
self._on_stable = on_stable_file
|
|
158
|
+
self._on_tick = on_tick
|
|
153
159
|
self._state_db = state_db
|
|
154
160
|
self._recursive = recursive
|
|
155
161
|
# Optional so unit tests that build a FileMonitor in isolation
|
|
@@ -510,6 +516,12 @@ class FileMonitor:
|
|
|
510
516
|
error=str(exc),
|
|
511
517
|
)
|
|
512
518
|
|
|
519
|
+
if self._on_tick is not None:
|
|
520
|
+
try:
|
|
521
|
+
self._on_tick()
|
|
522
|
+
except Exception:
|
|
523
|
+
logger.exception("Monitor tick callback failed")
|
|
524
|
+
|
|
513
525
|
|
|
514
526
|
def _stat_in_dedup_index(
|
|
515
527
|
index: dict[str, list[tuple[int, float]]],
|
|
@@ -10,18 +10,26 @@ from __future__ import annotations
|
|
|
10
10
|
import logging
|
|
11
11
|
import os
|
|
12
12
|
import re
|
|
13
|
+
import time
|
|
13
14
|
from collections.abc import Callable
|
|
14
15
|
from dataclasses import dataclass, field
|
|
15
16
|
from datetime import datetime, timezone
|
|
16
17
|
from pathlib import Path
|
|
18
|
+
from typing import Literal
|
|
17
19
|
|
|
18
20
|
from data_hub_watcher.api_client import ApiError, DataHubClient
|
|
21
|
+
from data_hub_watcher.constants import (
|
|
22
|
+
RUN_REPORT_RETRY_BASE_DELAY_SECONDS,
|
|
23
|
+
RUN_REPORT_RETRY_MAX_DELAY_SECONDS,
|
|
24
|
+
)
|
|
19
25
|
from data_hub_watcher.events import EventReporter, EventType, WatcherEvent
|
|
20
26
|
from data_hub_watcher.heartbeat import WatcherCounters
|
|
21
27
|
from data_hub_watcher.state import StateDB
|
|
22
28
|
|
|
23
29
|
logger = logging.getLogger(__name__)
|
|
24
30
|
|
|
31
|
+
ReportOperation = Literal["create", "update"]
|
|
32
|
+
|
|
25
33
|
|
|
26
34
|
def file_created_at(st: os.stat_result) -> float:
|
|
27
35
|
"""Best-effort on-disk creation time for *st*.
|
|
@@ -65,13 +73,17 @@ class RunState:
|
|
|
65
73
|
# PATCHed to the API. Used as a cursor so `_update_run` can send only
|
|
66
74
|
# newly-stable files instead of re-transmitting the full manifest on
|
|
67
75
|
# every call. On a failed PATCH the cursor is left untouched so the
|
|
68
|
-
# next
|
|
76
|
+
# next PATCH, from a retry or a new file, carries the backlog.
|
|
69
77
|
patched_file_count: int = 0
|
|
70
78
|
# Last `acquired_at` value (seconds since epoch) we successfully sent
|
|
71
79
|
# to the API for this run. Used so `_update_run` only PATCHes the
|
|
72
80
|
# field when a later-stabilising file reveals an earlier creation
|
|
73
81
|
# time than what the server already knows.
|
|
74
82
|
acquired_at_sent: float | None = None
|
|
83
|
+
# Failed reports since the last success, and the wait before the next
|
|
84
|
+
# retry. The due time itself lives on `RunDetector._retry_at`.
|
|
85
|
+
retry_attempts: int = 0
|
|
86
|
+
retry_delay: int = 0
|
|
75
87
|
|
|
76
88
|
|
|
77
89
|
def _run_acquired_at(files: list[FileInfo]) -> float | None:
|
|
@@ -132,6 +144,10 @@ class RunDetector:
|
|
|
132
144
|
self._watch_dir = watch_directory
|
|
133
145
|
|
|
134
146
|
self._runs: dict[str, RunState] = {}
|
|
147
|
+
# Due time (`time.monotonic()`) of the next retry, by run ID. A run has
|
|
148
|
+
# an entry exactly when its `retry_attempts` is non-zero. Only the
|
|
149
|
+
# monitor's stability-checker thread calls into this class, so no lock.
|
|
150
|
+
self._retry_at: dict[str, float] = {}
|
|
135
151
|
# Set of parent-directory POSIX paths we've already surfaced a
|
|
136
152
|
# ``kind=pattern_mismatch`` event for. Pattern misses are
|
|
137
153
|
# routine for excluded files, so without throttling a
|
|
@@ -180,10 +196,89 @@ class RunDetector:
|
|
|
180
196
|
return
|
|
181
197
|
run.files.append(info)
|
|
182
198
|
|
|
183
|
-
|
|
199
|
+
# Sent right away even while a retry is waiting, so the run catches up
|
|
200
|
+
# as soon as the server is back. A failure keeps doubling the backoff.
|
|
201
|
+
self._send_report(run)
|
|
202
|
+
|
|
203
|
+
def retry_failed_reports(self) -> None:
|
|
204
|
+
"""Retry the run that has waited longest, if its retry is due.
|
|
205
|
+
|
|
206
|
+
Sends at most one request per call. A request to a slow server can
|
|
207
|
+
block until `DEFAULT_TIMEOUT` runs out, and the monitor's file checks
|
|
208
|
+
share this thread.
|
|
209
|
+
"""
|
|
210
|
+
now = time.monotonic()
|
|
211
|
+
due = [(at, run_id) for run_id, at in self._retry_at.items() if at <= now]
|
|
212
|
+
if not due:
|
|
213
|
+
return
|
|
214
|
+
_, run_id = min(due)
|
|
215
|
+
del self._retry_at[run_id]
|
|
216
|
+
self._send_report(self._runs[run_id])
|
|
217
|
+
|
|
218
|
+
def _send_report(self, run: RunState) -> None:
|
|
219
|
+
if run.reported:
|
|
220
|
+
self._update_run(run)
|
|
221
|
+
else:
|
|
184
222
|
self._report_new_run(run)
|
|
223
|
+
|
|
224
|
+
def _clear_retry(self, run: RunState) -> None:
|
|
225
|
+
run.retry_attempts = 0
|
|
226
|
+
run.retry_delay = 0
|
|
227
|
+
self._retry_at.pop(run.run_id, None)
|
|
228
|
+
|
|
229
|
+
def _note_report_success(self, run: RunState) -> None:
|
|
230
|
+
if run.retry_attempts:
|
|
231
|
+
logger.info(
|
|
232
|
+
"Run %s recovered after %d failed attempt(s)", run.run_id, run.retry_attempts
|
|
233
|
+
)
|
|
234
|
+
self._clear_retry(run)
|
|
235
|
+
|
|
236
|
+
def _handle_report_failure(
|
|
237
|
+
self, run: RunState, operation: ReportOperation, exc: Exception
|
|
238
|
+
) -> None:
|
|
239
|
+
"""Count a failed POST or PATCH, report it, and schedule a retry if it can help."""
|
|
240
|
+
verb = "report" if operation == "create" else "update"
|
|
241
|
+
if isinstance(exc, ApiError):
|
|
242
|
+
status_code, error, retryable = exc.status_code, exc.message, exc.is_transient
|
|
243
|
+
logger.warning("Failed to %s run %s: %s", verb, run.run_id, error)
|
|
185
244
|
else:
|
|
186
|
-
|
|
245
|
+
# Anything else the client raises, such as a connection that drops
|
|
246
|
+
# mid-response or a proxy's HTML page instead of JSON, is retried
|
|
247
|
+
# like a connection error.
|
|
248
|
+
status_code, error, retryable = 0, f"{type(exc).__name__}: {exc}", True
|
|
249
|
+
logger.warning("Failed to %s run %s: %s", verb, run.run_id, error, exc_info=exc)
|
|
250
|
+
self._counters.errors += 1
|
|
251
|
+
# Report the first retryable failure in a row and every 4xx, so a long
|
|
252
|
+
# outage doesn't push other events out of the bounded queue.
|
|
253
|
+
if run.retry_attempts == 0 or not retryable:
|
|
254
|
+
self._reporter.report_error(
|
|
255
|
+
"run_report_failed",
|
|
256
|
+
f"Failed to {verb} run {run.run_id}: {error}",
|
|
257
|
+
run_id=run.run_id,
|
|
258
|
+
operation=operation,
|
|
259
|
+
status_code=status_code,
|
|
260
|
+
error=error,
|
|
261
|
+
file_count=len(run.files),
|
|
262
|
+
)
|
|
263
|
+
if retryable:
|
|
264
|
+
self._schedule_retry(run)
|
|
265
|
+
else:
|
|
266
|
+
self._clear_retry(run)
|
|
267
|
+
|
|
268
|
+
def _schedule_retry(self, run: RunState) -> None:
|
|
269
|
+
run.retry_delay = (
|
|
270
|
+
min(run.retry_delay * 2, RUN_REPORT_RETRY_MAX_DELAY_SECONDS)
|
|
271
|
+
if run.retry_delay
|
|
272
|
+
else RUN_REPORT_RETRY_BASE_DELAY_SECONDS
|
|
273
|
+
)
|
|
274
|
+
run.retry_attempts += 1
|
|
275
|
+
self._retry_at[run.run_id] = time.monotonic() + run.retry_delay
|
|
276
|
+
logger.info(
|
|
277
|
+
"Will retry run %s in %ds (%d failed attempt(s) so far)",
|
|
278
|
+
run.run_id,
|
|
279
|
+
run.retry_delay,
|
|
280
|
+
run.retry_attempts,
|
|
281
|
+
)
|
|
187
282
|
|
|
188
283
|
# ------------------------------------------------------------------
|
|
189
284
|
# run-ID extraction
|
|
@@ -271,35 +366,26 @@ class RunDetector:
|
|
|
271
366
|
payload["acquired_at"] = datetime.fromtimestamp(acquired, tz=timezone.utc).isoformat()
|
|
272
367
|
try:
|
|
273
368
|
resp = self._client.report_run(self._instrument_id, payload)
|
|
274
|
-
|
|
275
|
-
run
|
|
276
|
-
run.patched_file_count = len(run.files)
|
|
277
|
-
run.acquired_at_sent = acquired
|
|
278
|
-
self._state_db.record_run_reported(run.run_id)
|
|
279
|
-
self._persist_detected_files(run)
|
|
280
|
-
self._counters.runs_reported += 1
|
|
281
|
-
|
|
282
|
-
self._reporter.queue_event(
|
|
283
|
-
WatcherEvent(
|
|
284
|
-
event_type=EventType.RUN_REPORTED,
|
|
285
|
-
message=f"Run {run.run_id} reported",
|
|
286
|
-
details={"run_id": run.run_id},
|
|
287
|
-
)
|
|
288
|
-
)
|
|
289
|
-
logger.info("Reported new run %s (%d files)", run.run_id, len(run.files))
|
|
290
|
-
except ApiError as exc:
|
|
291
|
-
logger.warning("Failed to report run %s: %s (will retry)", run.run_id, exc.message)
|
|
292
|
-
self._counters.errors += 1
|
|
293
|
-
self._reporter.report_error(
|
|
294
|
-
"run_report_failed",
|
|
295
|
-
f"Failed to report run {run.run_id}: {exc.message}",
|
|
296
|
-
run_id=run.run_id,
|
|
297
|
-
operation="create",
|
|
298
|
-
status_code=exc.status_code,
|
|
299
|
-
error=exc.message,
|
|
300
|
-
file_count=len(run.files),
|
|
301
|
-
)
|
|
369
|
+
except Exception as exc:
|
|
370
|
+
self._handle_report_failure(run, "create", exc)
|
|
302
371
|
return
|
|
372
|
+
run.reported = True
|
|
373
|
+
self._note_report_success(run)
|
|
374
|
+
run.api_run_id = resp.id
|
|
375
|
+
run.patched_file_count = len(run.files)
|
|
376
|
+
run.acquired_at_sent = acquired
|
|
377
|
+
self._state_db.record_run_reported(run.run_id)
|
|
378
|
+
self._persist_detected_files(run)
|
|
379
|
+
self._counters.runs_reported += 1
|
|
380
|
+
|
|
381
|
+
self._reporter.queue_event(
|
|
382
|
+
WatcherEvent(
|
|
383
|
+
event_type=EventType.RUN_REPORTED,
|
|
384
|
+
message=f"Run {run.run_id} reported",
|
|
385
|
+
details={"run_id": run.run_id},
|
|
386
|
+
)
|
|
387
|
+
)
|
|
388
|
+
logger.info("Reported new run %s (%d files)", run.run_id, len(run.files))
|
|
303
389
|
|
|
304
390
|
if self._upload_cb:
|
|
305
391
|
succeeded = self._upload_cb(run.run_id, list(run.files))
|
|
@@ -316,9 +402,9 @@ class RunDetector:
|
|
|
316
402
|
API call entirely when the delta is empty avoids a needless round
|
|
317
403
|
trip if `on_stable_file` re-fires for an already-reported file.
|
|
318
404
|
|
|
319
|
-
On failure the PATCH cursor is left untouched, so
|
|
320
|
-
stable file's PATCH
|
|
321
|
-
entry. The upload callback continues to receive only files the
|
|
405
|
+
On failure the PATCH cursor is left untouched, so a scheduled retry
|
|
406
|
+
or the next stable file's PATCH carries the backlog (plus the new
|
|
407
|
+
entry). The upload callback continues to receive only files the
|
|
322
408
|
uploader hasn't seen yet (`uploaded_file_count` is independent
|
|
323
409
|
of the PATCH cursor).
|
|
324
410
|
"""
|
|
@@ -334,6 +420,8 @@ class RunDetector:
|
|
|
334
420
|
else None
|
|
335
421
|
)
|
|
336
422
|
if not new_files and earlier_acquired is None:
|
|
423
|
+
# Nothing is waiting to be sent, so a queued retry has no work left.
|
|
424
|
+
self._clear_retry(run)
|
|
337
425
|
return
|
|
338
426
|
|
|
339
427
|
payload: dict[str, object] = {
|
|
@@ -345,29 +433,20 @@ class RunDetector:
|
|
|
345
433
|
).isoformat()
|
|
346
434
|
try:
|
|
347
435
|
self._client.update_run(self._instrument_id, run.run_id, payload)
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
run.acquired_at_sent = earlier_acquired
|
|
351
|
-
self._persist_detected_files(run)
|
|
352
|
-
logger.info(
|
|
353
|
-
"Updated run %s (sent %d new file(s), %d total)",
|
|
354
|
-
run.run_id,
|
|
355
|
-
len(new_files),
|
|
356
|
-
len(run.files),
|
|
357
|
-
)
|
|
358
|
-
except ApiError as exc:
|
|
359
|
-
logger.warning("Failed to update run %s: %s", run.run_id, exc.message)
|
|
360
|
-
self._counters.errors += 1
|
|
361
|
-
self._reporter.report_error(
|
|
362
|
-
"run_report_failed",
|
|
363
|
-
f"Failed to update run {run.run_id}: {exc.message}",
|
|
364
|
-
run_id=run.run_id,
|
|
365
|
-
operation="update",
|
|
366
|
-
status_code=exc.status_code,
|
|
367
|
-
error=exc.message,
|
|
368
|
-
file_count=len(run.files),
|
|
369
|
-
)
|
|
436
|
+
except Exception as exc:
|
|
437
|
+
self._handle_report_failure(run, "update", exc)
|
|
370
438
|
return
|
|
439
|
+
self._note_report_success(run)
|
|
440
|
+
run.patched_file_count = len(run.files)
|
|
441
|
+
if earlier_acquired is not None:
|
|
442
|
+
run.acquired_at_sent = earlier_acquired
|
|
443
|
+
self._persist_detected_files(run)
|
|
444
|
+
logger.info(
|
|
445
|
+
"Updated run %s (sent %d new file(s), %d total)",
|
|
446
|
+
run.run_id,
|
|
447
|
+
len(new_files),
|
|
448
|
+
len(run.files),
|
|
449
|
+
)
|
|
371
450
|
|
|
372
451
|
upload_new = run.files[run.uploaded_file_count :]
|
|
373
452
|
if self._upload_cb and upload_new:
|
|
@@ -300,6 +300,7 @@ def build_runtime(
|
|
|
300
300
|
stability_period=inst.stability_period_seconds,
|
|
301
301
|
max_stability_wait_seconds=inst.max_stability_wait_seconds,
|
|
302
302
|
on_stable_file=detector.on_stable_file,
|
|
303
|
+
on_tick=detector.retry_failed_reports,
|
|
303
304
|
state_db=state_db,
|
|
304
305
|
recursive=inst.run_detection.recursive,
|
|
305
306
|
event_reporter=reporter,
|
|
@@ -847,8 +847,7 @@ class Uploader:
|
|
|
847
847
|
return
|
|
848
848
|
except ApiError as exc:
|
|
849
849
|
last_exc = exc
|
|
850
|
-
|
|
851
|
-
if not retryable:
|
|
850
|
+
if not exc.is_transient:
|
|
852
851
|
raise
|
|
853
852
|
logger.warning(
|
|
854
853
|
"Complete attempt %d/%d failed: %s",
|
|
@@ -47,6 +47,7 @@ def _make_monitor(
|
|
|
47
47
|
on_stable_file: MagicMock | None = None,
|
|
48
48
|
reporter: MagicMock | None = None,
|
|
49
49
|
max_stability_wait_seconds: int = MAX_STABILITY_WAIT_SECONDS,
|
|
50
|
+
on_tick: MagicMock | None = None,
|
|
50
51
|
) -> FileMonitor:
|
|
51
52
|
return FileMonitor(
|
|
52
53
|
watch_directory=watch_dir,
|
|
@@ -54,6 +55,7 @@ def _make_monitor(
|
|
|
54
55
|
stability_period=1,
|
|
55
56
|
max_stability_wait_seconds=max_stability_wait_seconds,
|
|
56
57
|
on_stable_file=on_stable_file or MagicMock(),
|
|
58
|
+
on_tick=on_tick,
|
|
57
59
|
state_db=state_db,
|
|
58
60
|
recursive=False,
|
|
59
61
|
event_reporter=reporter,
|
|
@@ -177,3 +179,45 @@ class TestStableCallbackFailure:
|
|
|
177
179
|
assert call.args[0] == "stable_callback_failed"
|
|
178
180
|
assert call.kwargs["path"] == str(f)
|
|
179
181
|
assert "downstream blew up" in call.kwargs["error"]
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class TestTickCallback:
|
|
185
|
+
def test_runs_when_no_files_are_pending(self, watch_dir: Path, state_db: StateDB) -> None:
|
|
186
|
+
on_tick = MagicMock()
|
|
187
|
+
monitor = _make_monitor(watch_dir, state_db, on_tick=on_tick)
|
|
188
|
+
|
|
189
|
+
monitor._check_pending()
|
|
190
|
+
|
|
191
|
+
on_tick.assert_called_once_with()
|
|
192
|
+
|
|
193
|
+
def test_runs_after_stable_file_callbacks(self, watch_dir: Path, state_db: StateDB) -> None:
|
|
194
|
+
calls: list[str] = []
|
|
195
|
+
monitor = _make_monitor(
|
|
196
|
+
watch_dir,
|
|
197
|
+
state_db,
|
|
198
|
+
on_stable_file=MagicMock(side_effect=lambda _path: calls.append("stable")),
|
|
199
|
+
on_tick=MagicMock(side_effect=lambda: calls.append("tick")),
|
|
200
|
+
)
|
|
201
|
+
f = watch_dir / "ready.csv"
|
|
202
|
+
f.write_text("hello")
|
|
203
|
+
st = f.stat()
|
|
204
|
+
now = time.monotonic()
|
|
205
|
+
monitor._pending[f] = _PendingFile(
|
|
206
|
+
path=f,
|
|
207
|
+
size=st.st_size,
|
|
208
|
+
mtime=st.st_mtime,
|
|
209
|
+
first_seen=now - 5,
|
|
210
|
+
last_changed=now - 5,
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
monitor._check_pending()
|
|
214
|
+
|
|
215
|
+
assert calls == ["stable", "tick"]
|
|
216
|
+
|
|
217
|
+
def test_exception_does_not_escape(self, watch_dir: Path, state_db: StateDB) -> None:
|
|
218
|
+
on_tick = MagicMock(side_effect=RuntimeError("tick blew up"))
|
|
219
|
+
monitor = _make_monitor(watch_dir, state_db, on_tick=on_tick)
|
|
220
|
+
|
|
221
|
+
monitor._check_pending() # Must not raise, or the checker thread would die.
|
|
222
|
+
|
|
223
|
+
on_tick.assert_called_once_with()
|