data-hub-watcher 1.2.2__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_hub_watcher-1.2.2/src/data_hub_watcher.egg-info → data_hub_watcher-1.3.0}/PKG-INFO +2 -1
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/pyproject.toml +4 -1
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/cli.py +16 -2
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/models.py +8 -7
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/runtime.py +5 -2
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/uploader.py +569 -151
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0/src/data_hub_watcher.egg-info}/PKG-INFO +2 -1
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/SOURCES.txt +1 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/requires.txt +1 -0
- data_hub_watcher-1.3.0/tests/test_cli_upload.py +125 -0
- data_hub_watcher-1.3.0/tests/test_uploader.py +2922 -0
- data_hub_watcher-1.2.2/tests/test_uploader.py +0 -1476
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/LICENSE +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/README.md +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/setup.cfg +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/__init__.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/api_client.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/cli_catalog.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/config_io.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/constants.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/events.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/heartbeat.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/logging_setup.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/monitor.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/run_detector.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/scheduled_task.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/self_update.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/service.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/state.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/updater.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/upgrade_worker.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/util.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/dependency_links.txt +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/entry_points.txt +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/top_level.txt +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_api_client_version_header.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_catalog.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_self_update.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_service_reinstall.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_set_environment.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_config_migration.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_digests.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_events.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_heartbeat.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_logging_setup.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_monitor_events.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_monitor_initial_scan.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_preview_environment.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_run_detection_config.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_run_detector.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_run_detector_hydration.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_runtime.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_scheduled_task.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_self_update.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_service.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_db_baseline.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_db_concurrency.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_db_path.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_forget.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_pending_uploads.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_updater.py +0 -0
- {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_upgrade_worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: data-hub-watcher
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: File-watcher agent for lab instrument PCs that ingests data into Data Hub.
|
|
5
5
|
Author-email: Arcadia Science <swe@arcadiascience.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -24,6 +24,7 @@ Requires-Python: >=3.12
|
|
|
24
24
|
Description-Content-Type: text/markdown
|
|
25
25
|
License-File: LICENSE
|
|
26
26
|
Requires-Dist: awscrt>=0.27.0
|
|
27
|
+
Requires-Dist: certifi>=2026.10.19
|
|
27
28
|
Requires-Dist: click>=8.2.1
|
|
28
29
|
Requires-Dist: packaging>=24.0
|
|
29
30
|
Requires-Dist: pydantic>=2.11.9
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "data-hub-watcher"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.3.0"
|
|
4
4
|
description = "File-watcher agent for lab instrument PCs that ingests data into Data Hub."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -25,6 +25,9 @@ classifiers = [
|
|
|
25
25
|
]
|
|
26
26
|
dependencies = [
|
|
27
27
|
"awscrt>=0.27.0",
|
|
28
|
+
# Not imported directly. requests trusts certifi's CA bundle, and older
|
|
29
|
+
# bundles still trust the compromised eMudhra/emSign certificates.
|
|
30
|
+
"certifi>=2026.10.19",
|
|
28
31
|
"click>=8.2.1",
|
|
29
32
|
"packaging>=24.0",
|
|
30
33
|
"pydantic>=2.11.9",
|
|
@@ -7,6 +7,7 @@ import re
|
|
|
7
7
|
import signal
|
|
8
8
|
import subprocess
|
|
9
9
|
import sys
|
|
10
|
+
import threading
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
@@ -1232,6 +1233,9 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
|
|
|
1232
1233
|
state_db = StateDB(db_path)
|
|
1233
1234
|
counters = WatcherCounters()
|
|
1234
1235
|
reporter = EventReporter(client, cfg.watcher_id)
|
|
1236
|
+
# Set on Ctrl-C, so upload threads that are still running start no more
|
|
1237
|
+
# files or parts before the process exits.
|
|
1238
|
+
stop_event = threading.Event()
|
|
1235
1239
|
uploader = Uploader(
|
|
1236
1240
|
client=client,
|
|
1237
1241
|
state_db=state_db,
|
|
@@ -1240,6 +1244,8 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
|
|
|
1240
1244
|
instrument_id=inst.id,
|
|
1241
1245
|
watcher_id=cfg.watcher_id,
|
|
1242
1246
|
watch_directory=inst.watch_directory,
|
|
1247
|
+
upload_parallelism=inst.upload_parallelism,
|
|
1248
|
+
stop_event=stop_event,
|
|
1243
1249
|
)
|
|
1244
1250
|
|
|
1245
1251
|
if file_path:
|
|
@@ -1254,7 +1260,11 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
|
|
|
1254
1260
|
return
|
|
1255
1261
|
|
|
1256
1262
|
click.echo(f"Uploading {fp.name}…")
|
|
1257
|
-
|
|
1263
|
+
try:
|
|
1264
|
+
ok = uploader._upload_single(fp, run_id)
|
|
1265
|
+
except KeyboardInterrupt:
|
|
1266
|
+
stop_event.set()
|
|
1267
|
+
raise
|
|
1258
1268
|
state_db.close()
|
|
1259
1269
|
if not ok:
|
|
1260
1270
|
raise click.ClickException(f"Upload failed for {fp.name}")
|
|
@@ -1281,7 +1291,11 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
|
|
|
1281
1291
|
state_db.close()
|
|
1282
1292
|
return
|
|
1283
1293
|
|
|
1284
|
-
|
|
1294
|
+
try:
|
|
1295
|
+
uploader.poll_upload_queue()
|
|
1296
|
+
except KeyboardInterrupt:
|
|
1297
|
+
stop_event.set()
|
|
1298
|
+
raise
|
|
1285
1299
|
reporter.flush()
|
|
1286
1300
|
click.echo(click.style(f"✓ Processed {len(queue.files)} file(s).", fg="green"))
|
|
1287
1301
|
state_db.close()
|
|
@@ -48,13 +48,14 @@ class InstrumentConfig(BaseModel):
|
|
|
48
48
|
# MAX_STABILITY_WAIT_SECONDS so existing YAMLs keep today's 5-minute
|
|
49
49
|
# behaviour; raise for instruments that write large files for longer.
|
|
50
50
|
max_stability_wait_seconds: int = Field(default=MAX_STABILITY_WAIT_SECONDS, ge=1, le=86400)
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
51
|
+
# The most S3 uploads the watcher runs at once. Whole files and the parts
|
|
52
|
+
# of a large file share this one budget, across auto mode, the upload
|
|
53
|
+
# queue, pending-upload retries and the one-shot `upload` command (parts
|
|
54
|
+
# count from watcher 1.3.0; before that it only limited files). The
|
|
55
|
+
# default of 4 balances S3 throughput against lab-PC network and CPU
|
|
56
|
+
# budgets; instruments on slow shared uplinks can dial it down to 1 and
|
|
57
|
+
# PCs with fast links can push toward the cap. Defaulted, so existing
|
|
58
|
+
# config YAMLs need no migration.
|
|
58
59
|
upload_parallelism: int = Field(default=4, ge=1, le=32)
|
|
59
60
|
run_detection: RunDetectionConfig
|
|
60
61
|
|
|
@@ -267,8 +267,8 @@ def build_runtime(
|
|
|
267
267
|
instrument_id=inst.id,
|
|
268
268
|
watcher_id=watcher_id,
|
|
269
269
|
watch_directory=inst.watch_directory,
|
|
270
|
-
#
|
|
271
|
-
#
|
|
270
|
+
# The most S3 PUTs (files and parts) this watcher runs at once.
|
|
271
|
+
# Defaulted on the model, so older configs inherit it.
|
|
272
272
|
upload_parallelism=inst.upload_parallelism,
|
|
273
273
|
stop_event=upload_stop_event,
|
|
274
274
|
)
|
|
@@ -570,6 +570,9 @@ def stop_runtime(rt: WatcherRuntime, *, stopped_message: str) -> None:
|
|
|
570
570
|
# ``StateDB.close`` assumes writer threads have joined; a large upload
|
|
571
571
|
# outliving the join once wrote post-close. Stop the worker first, and if
|
|
572
572
|
# it won't stop in time, skip the close rather than race it (the OS reaps).
|
|
573
|
+
# The worker's file and part threads are daemon threads, so a PUT still in
|
|
574
|
+
# flight does not hold the process open past this join. That file is
|
|
575
|
+
# retried at the next start, from ``pending_uploads`` or the server's queue.
|
|
573
576
|
upload_worker_stopped = True
|
|
574
577
|
if rt.upload_worker is not None:
|
|
575
578
|
upload_worker_stopped = rt.upload_worker.stop(timeout=UPLOAD_WORKER_STOP_TIMEOUT_SECONDS)
|