data-hub-watcher 1.2.2__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {data_hub_watcher-1.2.2/src/data_hub_watcher.egg-info → data_hub_watcher-1.3.0}/PKG-INFO +2 -1
  2. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/pyproject.toml +4 -1
  3. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/cli.py +16 -2
  4. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/models.py +8 -7
  5. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/runtime.py +5 -2
  6. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/uploader.py +569 -151
  7. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0/src/data_hub_watcher.egg-info}/PKG-INFO +2 -1
  8. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/SOURCES.txt +1 -0
  9. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/requires.txt +1 -0
  10. data_hub_watcher-1.3.0/tests/test_cli_upload.py +125 -0
  11. data_hub_watcher-1.3.0/tests/test_uploader.py +2922 -0
  12. data_hub_watcher-1.2.2/tests/test_uploader.py +0 -1476
  13. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/LICENSE +0 -0
  14. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/README.md +0 -0
  15. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/setup.cfg +0 -0
  16. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/__init__.py +0 -0
  17. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/api_client.py +0 -0
  18. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/cli_catalog.py +0 -0
  19. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/config_io.py +0 -0
  20. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/constants.py +0 -0
  21. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/events.py +0 -0
  22. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/heartbeat.py +0 -0
  23. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/logging_setup.py +0 -0
  24. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/monitor.py +0 -0
  25. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/run_detector.py +0 -0
  26. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/scheduled_task.py +0 -0
  27. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/self_update.py +0 -0
  28. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/service.py +0 -0
  29. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/state.py +0 -0
  30. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/updater.py +0 -0
  31. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/upgrade_worker.py +0 -0
  32. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher/util.py +0 -0
  33. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/dependency_links.txt +0 -0
  34. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/entry_points.txt +0 -0
  35. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/src/data_hub_watcher.egg-info/top_level.txt +0 -0
  36. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_api_client_version_header.py +0 -0
  37. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_catalog.py +0 -0
  38. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_self_update.py +0 -0
  39. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_service_reinstall.py +0 -0
  40. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_cli_set_environment.py +0 -0
  41. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_config_migration.py +0 -0
  42. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_digests.py +0 -0
  43. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_events.py +0 -0
  44. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_heartbeat.py +0 -0
  45. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_logging_setup.py +0 -0
  46. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_monitor_events.py +0 -0
  47. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_monitor_initial_scan.py +0 -0
  48. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_preview_environment.py +0 -0
  49. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_run_detection_config.py +0 -0
  50. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_run_detector.py +0 -0
  51. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_run_detector_hydration.py +0 -0
  52. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_runtime.py +0 -0
  53. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_scheduled_task.py +0 -0
  54. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_self_update.py +0 -0
  55. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_service.py +0 -0
  56. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_db_baseline.py +0 -0
  57. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_db_concurrency.py +0 -0
  58. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_db_path.py +0 -0
  59. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_forget.py +0 -0
  60. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_state_pending_uploads.py +0 -0
  61. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_updater.py +0 -0
  62. {data_hub_watcher-1.2.2 → data_hub_watcher-1.3.0}/tests/test_upgrade_worker.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: data-hub-watcher
3
- Version: 1.2.2
3
+ Version: 1.3.0
4
4
  Summary: File-watcher agent for lab instrument PCs that ingests data into Data Hub.
5
5
  Author-email: Arcadia Science <swe@arcadiascience.com>
6
6
  License-Expression: MIT
@@ -24,6 +24,7 @@ Requires-Python: >=3.12
24
24
  Description-Content-Type: text/markdown
25
25
  License-File: LICENSE
26
26
  Requires-Dist: awscrt>=0.27.0
27
+ Requires-Dist: certifi>=2026.10.19
27
28
  Requires-Dist: click>=8.2.1
28
29
  Requires-Dist: packaging>=24.0
29
30
  Requires-Dist: pydantic>=2.11.9
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "data-hub-watcher"
3
- version = "1.2.2"
3
+ version = "1.3.0"
4
4
  description = "File-watcher agent for lab instrument PCs that ingests data into Data Hub."
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12"
@@ -25,6 +25,9 @@ classifiers = [
25
25
  ]
26
26
  dependencies = [
27
27
  "awscrt>=0.27.0",
28
+ # Not imported directly. requests trusts certifi's CA bundle, and older
29
+ # bundles still trust the compromised eMudhra/emSign certificates.
30
+ "certifi>=2026.10.19",
28
31
  "click>=8.2.1",
29
32
  "packaging>=24.0",
30
33
  "pydantic>=2.11.9",
@@ -7,6 +7,7 @@ import re
7
7
  import signal
8
8
  import subprocess
9
9
  import sys
10
+ import threading
10
11
  from pathlib import Path
11
12
  from typing import Any
12
13
 
@@ -1232,6 +1233,9 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
1232
1233
  state_db = StateDB(db_path)
1233
1234
  counters = WatcherCounters()
1234
1235
  reporter = EventReporter(client, cfg.watcher_id)
1236
+ # Set on Ctrl-C, so upload threads that are still running start no more
1237
+ # files or parts before the process exits.
1238
+ stop_event = threading.Event()
1235
1239
  uploader = Uploader(
1236
1240
  client=client,
1237
1241
  state_db=state_db,
@@ -1240,6 +1244,8 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
1240
1244
  instrument_id=inst.id,
1241
1245
  watcher_id=cfg.watcher_id,
1242
1246
  watch_directory=inst.watch_directory,
1247
+ upload_parallelism=inst.upload_parallelism,
1248
+ stop_event=stop_event,
1243
1249
  )
1244
1250
 
1245
1251
  if file_path:
@@ -1254,7 +1260,11 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
1254
1260
  return
1255
1261
 
1256
1262
  click.echo(f"Uploading {fp.name}…")
1257
- ok = uploader._upload_single(fp, run_id)
1263
+ try:
1264
+ ok = uploader._upload_single(fp, run_id)
1265
+ except KeyboardInterrupt:
1266
+ stop_event.set()
1267
+ raise
1258
1268
  state_db.close()
1259
1269
  if not ok:
1260
1270
  raise click.ClickException(f"Upload failed for {fp.name}")
@@ -1281,7 +1291,11 @@ def upload(ctx: click.Context, file_path: str | None, run_id: str | None, dry_ru
1281
1291
  state_db.close()
1282
1292
  return
1283
1293
 
1284
- uploader.poll_upload_queue()
1294
+ try:
1295
+ uploader.poll_upload_queue()
1296
+ except KeyboardInterrupt:
1297
+ stop_event.set()
1298
+ raise
1285
1299
  reporter.flush()
1286
1300
  click.echo(click.style(f"✓ Processed {len(queue.files)} file(s).", fg="green"))
1287
1301
  state_db.close()
@@ -48,13 +48,14 @@ class InstrumentConfig(BaseModel):
48
48
  # MAX_STABILITY_WAIT_SECONDS so existing YAMLs keep today's 5-minute
49
49
  # behaviour; raise for instruments that write large files for longer.
50
50
  max_stability_wait_seconds: int = Field(default=MAX_STABILITY_WAIT_SECONDS, ge=1, le=86400)
51
- # Maximum number of files to upload concurrently for a single
52
- # detected run (auto mode) or queue poll. The default of 4 balances
53
- # S3 throughput against lab-PC network and CPU budgets; instruments
54
- # on slow shared uplinks can dial it down to 1 and high-bandwidth
55
- # microscopy hosts can push toward the cap. New field, defaulted
56
- # for backward compatibility -- existing config YAMLs need no
57
- # migration.
51
+ # The most S3 uploads the watcher runs at once. Whole files and the parts
52
+ # of a large file share this one budget, across auto mode, the upload
53
+ # queue, pending-upload retries and the one-shot `upload` command (parts
54
+ # count from watcher 1.3.0; before that it only limited files). The
55
+ # default of 4 balances S3 throughput against lab-PC network and CPU
56
+ # budgets; instruments on slow shared uplinks can dial it down to 1 and
57
+ # PCs with fast links can push toward the cap. Defaulted, so existing
58
+ # config YAMLs need no migration.
58
59
  upload_parallelism: int = Field(default=4, ge=1, le=32)
59
60
  run_detection: RunDetectionConfig
60
61
 
@@ -267,8 +267,8 @@ def build_runtime(
267
267
  instrument_id=inst.id,
268
268
  watcher_id=watcher_id,
269
269
  watch_directory=inst.watch_directory,
270
- # Per-instrument knob, defaulted on the model so older configs
271
- # transparently inherit the new parallel-upload behaviour.
270
+ # The most S3 PUTs (files and parts) this watcher runs at once.
271
+ # Defaulted on the model, so older configs inherit it.
272
272
  upload_parallelism=inst.upload_parallelism,
273
273
  stop_event=upload_stop_event,
274
274
  )
@@ -570,6 +570,9 @@ def stop_runtime(rt: WatcherRuntime, *, stopped_message: str) -> None:
570
570
  # ``StateDB.close`` assumes writer threads have joined; a large upload
571
571
  # outliving the join once wrote post-close. Stop the worker first, and if
572
572
  # it won't stop in time, skip the close rather than race it (the OS reaps).
573
+ # The worker's file and part threads are daemon threads, so a PUT still in
574
+ # flight does not hold the process open past this join. That file is
575
+ # retried at the next start, from ``pending_uploads`` or the server's queue.
573
576
  upload_worker_stopped = True
574
577
  if rt.upload_worker is not None:
575
578
  upload_worker_stopped = rt.upload_worker.stop(timeout=UPLOAD_WORKER_STOP_TIMEOUT_SECONDS)