fleetpull 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (252) hide show
  1. fleetpull/__init__.py +36 -0
  2. fleetpull/api/__init__.py +32 -0
  3. fleetpull/api/auth_ingress.py +216 -0
  4. fleetpull/api/catalog.py +139 -0
  5. fleetpull/api/fetch.py +229 -0
  6. fleetpull/api/identity.py +90 -0
  7. fleetpull/api/sync.py +735 -0
  8. fleetpull/cli.py +146 -0
  9. fleetpull/config/__init__.py +47 -0
  10. fleetpull/config/base.py +18 -0
  11. fleetpull/config/example.py +90 -0
  12. fleetpull/config/geotab.py +78 -0
  13. fleetpull/config/http.py +32 -0
  14. fleetpull/config/loading.py +195 -0
  15. fleetpull/config/logger.py +139 -0
  16. fleetpull/config/providers.py +520 -0
  17. fleetpull/config/rate_limit.py +50 -0
  18. fleetpull/config/resolution.py +156 -0
  19. fleetpull/config/retry.py +69 -0
  20. fleetpull/config/root.py +157 -0
  21. fleetpull/config/sections.py +146 -0
  22. fleetpull/endpoints/__init__.py +24 -0
  23. fleetpull/endpoints/geotab/__init__.py +12 -0
  24. fleetpull/endpoints/geotab/_requests.py +427 -0
  25. fleetpull/endpoints/geotab/annotation_logs.py +73 -0
  26. fleetpull/endpoints/geotab/audits.py +70 -0
  27. fleetpull/endpoints/geotab/devices.py +79 -0
  28. fleetpull/endpoints/geotab/driver_changes.py +73 -0
  29. fleetpull/endpoints/geotab/duty_status_logs.py +75 -0
  30. fleetpull/endpoints/geotab/dvir_logs.py +76 -0
  31. fleetpull/endpoints/geotab/exception_events.py +119 -0
  32. fleetpull/endpoints/geotab/fault_data.py +72 -0
  33. fleetpull/endpoints/geotab/fill_ups.py +79 -0
  34. fleetpull/endpoints/geotab/fuel_and_energy_used.py +74 -0
  35. fleetpull/endpoints/geotab/fuel_tax_details.py +75 -0
  36. fleetpull/endpoints/geotab/log_records.py +74 -0
  37. fleetpull/endpoints/geotab/media_files.py +74 -0
  38. fleetpull/endpoints/geotab/shipment_logs.py +71 -0
  39. fleetpull/endpoints/geotab/status_data.py +71 -0
  40. fleetpull/endpoints/geotab/text_messages.py +77 -0
  41. fleetpull/endpoints/geotab/trips.py +102 -0
  42. fleetpull/endpoints/geotab/users.py +81 -0
  43. fleetpull/endpoints/motive/__init__.py +12 -0
  44. fleetpull/endpoints/motive/_spec_builders.py +94 -0
  45. fleetpull/endpoints/motive/driver_idle_rollups.py +113 -0
  46. fleetpull/endpoints/motive/driving_periods.py +85 -0
  47. fleetpull/endpoints/motive/groups.py +61 -0
  48. fleetpull/endpoints/motive/idle_events.py +94 -0
  49. fleetpull/endpoints/motive/users.py +64 -0
  50. fleetpull/endpoints/motive/vehicle_locations.py +170 -0
  51. fleetpull/endpoints/motive/vehicle_utilizations.py +122 -0
  52. fleetpull/endpoints/motive/vehicles.py +106 -0
  53. fleetpull/endpoints/registry.py +280 -0
  54. fleetpull/endpoints/samsara/__init__.py +12 -0
  55. fleetpull/endpoints/samsara/_spec_builders.py +202 -0
  56. fleetpull/endpoints/samsara/addresses.py +77 -0
  57. fleetpull/endpoints/samsara/asset_locations.py +217 -0
  58. fleetpull/endpoints/samsara/driver_fuel_energy_reports.py +124 -0
  59. fleetpull/endpoints/samsara/driver_vehicle_assignments.py +211 -0
  60. fleetpull/endpoints/samsara/drivers.py +169 -0
  61. fleetpull/endpoints/samsara/engine_states.py +114 -0
  62. fleetpull/endpoints/samsara/gps_readings.py +113 -0
  63. fleetpull/endpoints/samsara/idling_events.py +195 -0
  64. fleetpull/endpoints/samsara/odometer_readings.py +116 -0
  65. fleetpull/endpoints/samsara/trips.py +217 -0
  66. fleetpull/endpoints/samsara/vehicle_fuel_energy_reports.py +139 -0
  67. fleetpull/endpoints/samsara/vehicles.py +121 -0
  68. fleetpull/endpoints/shared/__init__.py +57 -0
  69. fleetpull/endpoints/shared/base.py +529 -0
  70. fleetpull/endpoints/shared/request_shape.py +227 -0
  71. fleetpull/endpoints/shared/resume.py +74 -0
  72. fleetpull/endpoints/shared/spec_builders.py +59 -0
  73. fleetpull/endpoints/shared/sync_mode.py +159 -0
  74. fleetpull/endpoints/shared/url_paths.py +149 -0
  75. fleetpull/exceptions.py +320 -0
  76. fleetpull/incremental/__init__.py +27 -0
  77. fleetpull/incremental/cursor.py +66 -0
  78. fleetpull/incremental/resolution.py +152 -0
  79. fleetpull/incremental/seed.py +53 -0
  80. fleetpull/incremental/window.py +94 -0
  81. fleetpull/logger/__init__.py +5 -0
  82. fleetpull/logger/setup.py +145 -0
  83. fleetpull/model_contract/__init__.py +6 -0
  84. fleetpull/model_contract/coercions.py +33 -0
  85. fleetpull/model_contract/response.py +45 -0
  86. fleetpull/models/__init__.py +7 -0
  87. fleetpull/models/geotab/__init__.py +166 -0
  88. fleetpull/models/geotab/annotation_log.py +109 -0
  89. fleetpull/models/geotab/audit.py +56 -0
  90. fleetpull/models/geotab/device.py +217 -0
  91. fleetpull/models/geotab/driver_change.py +102 -0
  92. fleetpull/models/geotab/duty_status_log.py +200 -0
  93. fleetpull/models/geotab/dvir_log.py +184 -0
  94. fleetpull/models/geotab/exception_event.py +135 -0
  95. fleetpull/models/geotab/fault_data.py +168 -0
  96. fleetpull/models/geotab/fill_up.py +189 -0
  97. fleetpull/models/geotab/fuel_and_energy_used.py +81 -0
  98. fleetpull/models/geotab/fuel_tax_detail.py +150 -0
  99. fleetpull/models/geotab/log_record.py +68 -0
  100. fleetpull/models/geotab/media_file.py +133 -0
  101. fleetpull/models/geotab/shared.py +221 -0
  102. fleetpull/models/geotab/shipment_log.py +105 -0
  103. fleetpull/models/geotab/status_data.py +108 -0
  104. fleetpull/models/geotab/text_message.py +125 -0
  105. fleetpull/models/geotab/trip.py +162 -0
  106. fleetpull/models/geotab/user.py +187 -0
  107. fleetpull/models/motive/__init__.py +43 -0
  108. fleetpull/models/motive/driver_idle_rollup.py +109 -0
  109. fleetpull/models/motive/driving_period.py +82 -0
  110. fleetpull/models/motive/group.py +49 -0
  111. fleetpull/models/motive/idle_event.py +77 -0
  112. fleetpull/models/motive/shared.py +192 -0
  113. fleetpull/models/motive/user.py +217 -0
  114. fleetpull/models/motive/vehicle.py +162 -0
  115. fleetpull/models/motive/vehicle_location.py +127 -0
  116. fleetpull/models/motive/vehicle_utilization.py +128 -0
  117. fleetpull/models/samsara/__init__.py +108 -0
  118. fleetpull/models/samsara/address.py +149 -0
  119. fleetpull/models/samsara/asset_location.py +131 -0
  120. fleetpull/models/samsara/driver.py +232 -0
  121. fleetpull/models/samsara/driver_fuel_energy_report.py +153 -0
  122. fleetpull/models/samsara/driver_vehicle_assignment.py +168 -0
  123. fleetpull/models/samsara/engine_state.py +92 -0
  124. fleetpull/models/samsara/gps_reading.py +146 -0
  125. fleetpull/models/samsara/idling_event.py +174 -0
  126. fleetpull/models/samsara/odometer_reading.py +90 -0
  127. fleetpull/models/samsara/trip.py +199 -0
  128. fleetpull/models/samsara/vehicle.py +167 -0
  129. fleetpull/models/samsara/vehicle_fuel_energy_report.py +191 -0
  130. fleetpull/network/__init__.py +9 -0
  131. fleetpull/network/auth/__init__.py +15 -0
  132. fleetpull/network/auth/authenticate.py +271 -0
  133. fleetpull/network/auth/manager.py +223 -0
  134. fleetpull/network/auth/models.py +57 -0
  135. fleetpull/network/auth/strategies.py +169 -0
  136. fleetpull/network/classifiers/__init__.py +11 -0
  137. fleetpull/network/classifiers/geotab.py +145 -0
  138. fleetpull/network/classifiers/motive.py +79 -0
  139. fleetpull/network/classifiers/samsara.py +80 -0
  140. fleetpull/network/client/__init__.py +25 -0
  141. fleetpull/network/client/page.py +32 -0
  142. fleetpull/network/client/profile.py +27 -0
  143. fleetpull/network/client/registry.py +103 -0
  144. fleetpull/network/client/registry_base.py +150 -0
  145. fleetpull/network/client/runtime.py +44 -0
  146. fleetpull/network/client/transport.py +381 -0
  147. fleetpull/network/contract/__init__.py +50 -0
  148. fleetpull/network/contract/auth.py +48 -0
  149. fleetpull/network/contract/classifier.py +189 -0
  150. fleetpull/network/contract/envelope_fetcher.py +33 -0
  151. fleetpull/network/contract/envelopes.py +189 -0
  152. fleetpull/network/contract/outcome.py +43 -0
  153. fleetpull/network/contract/page_decoder.py +101 -0
  154. fleetpull/network/contract/request.py +97 -0
  155. fleetpull/network/decoders/__init__.py +40 -0
  156. fleetpull/network/decoders/_window_stamp.py +78 -0
  157. fleetpull/network/decoders/geotab.py +309 -0
  158. fleetpull/network/decoders/motive.py +204 -0
  159. fleetpull/network/decoders/motive_reports.py +114 -0
  160. fleetpull/network/decoders/samsara.py +343 -0
  161. fleetpull/network/decoders/samsara_reports.py +119 -0
  162. fleetpull/network/decoders/single_page.py +55 -0
  163. fleetpull/network/limits/__init__.py +13 -0
  164. fleetpull/network/limits/bucket_math.py +58 -0
  165. fleetpull/network/limits/limiter.py +153 -0
  166. fleetpull/network/limits/registry.py +96 -0
  167. fleetpull/network/posture/__init__.py +6 -0
  168. fleetpull/network/posture/client_options.py +96 -0
  169. fleetpull/network/retry/__init__.py +13 -0
  170. fleetpull/network/retry/decision.py +155 -0
  171. fleetpull/network/tls/__init__.py +5 -0
  172. fleetpull/network/tls/truststore_context.py +35 -0
  173. fleetpull/orchestrator/__init__.py +30 -0
  174. fleetpull/orchestrator/backfill.py +127 -0
  175. fleetpull/orchestrator/batch.py +155 -0
  176. fleetpull/orchestrator/bisection.py +271 -0
  177. fleetpull/orchestrator/drivers.py +350 -0
  178. fleetpull/orchestrator/entry.py +347 -0
  179. fleetpull/orchestrator/executors.py +128 -0
  180. fleetpull/orchestrator/fanout.py +163 -0
  181. fleetpull/orchestrator/feed_drive.py +266 -0
  182. fleetpull/orchestrator/metadata_projection.py +195 -0
  183. fleetpull/orchestrator/outcome.py +53 -0
  184. fleetpull/orchestrator/recording.py +85 -0
  185. fleetpull/orchestrator/resume.py +143 -0
  186. fleetpull/orchestrator/roster_harvest.py +108 -0
  187. fleetpull/orchestrator/roster_refresh.py +363 -0
  188. fleetpull/orchestrator/runner.py +262 -0
  189. fleetpull/orchestrator/shape_resolution.py +221 -0
  190. fleetpull/orchestrator/spine.py +166 -0
  191. fleetpull/orchestrator/streaming.py +153 -0
  192. fleetpull/orchestrator/unit_loop.py +273 -0
  193. fleetpull/orchestrator/watermark_drive.py +455 -0
  194. fleetpull/paths/__init__.py +13 -0
  195. fleetpull/paths/datasets.py +38 -0
  196. fleetpull/paths/partitions.py +41 -0
  197. fleetpull/paths/resolution.py +98 -0
  198. fleetpull/polars_typing/__init__.py +20 -0
  199. fleetpull/py.typed +0 -0
  200. fleetpull/records/__init__.py +19 -0
  201. fleetpull/records/convert.py +53 -0
  202. fleetpull/records/dataframe.py +63 -0
  203. fleetpull/records/event_time.py +70 -0
  204. fleetpull/records/fields.py +190 -0
  205. fleetpull/records/flatten.py +54 -0
  206. fleetpull/records/roster_members.py +74 -0
  207. fleetpull/records/schema.py +73 -0
  208. fleetpull/records/validation.py +62 -0
  209. fleetpull/resources/__init__.py +10 -0
  210. fleetpull/resources/config.example.yaml +222 -0
  211. fleetpull/roster/__init__.py +16 -0
  212. fleetpull/roster/definition.py +46 -0
  213. fleetpull/roster/key.py +40 -0
  214. fleetpull/roster/registry.py +93 -0
  215. fleetpull/state/__init__.py +36 -0
  216. fleetpull/state/cursors.py +417 -0
  217. fleetpull/state/database.py +463 -0
  218. fleetpull/state/migrations.py +430 -0
  219. fleetpull/state/reconcile.py +127 -0
  220. fleetpull/state/rosters.py +159 -0
  221. fleetpull/state/run_ledger.py +572 -0
  222. fleetpull/state/work_units.py +648 -0
  223. fleetpull/storage/__init__.py +30 -0
  224. fleetpull/storage/append.py +187 -0
  225. fleetpull/storage/atomic.py +161 -0
  226. fleetpull/storage/files.py +176 -0
  227. fleetpull/storage/frames.py +98 -0
  228. fleetpull/storage/metadata.py +161 -0
  229. fleetpull/storage/partitioned.py +205 -0
  230. fleetpull/storage/pruning.py +169 -0
  231. fleetpull/storage/read.py +35 -0
  232. fleetpull/storage/result.py +34 -0
  233. fleetpull/storage/single_file.py +130 -0
  234. fleetpull/storage/splitting.py +88 -0
  235. fleetpull/storage/staging.py +178 -0
  236. fleetpull/storage/writers.py +145 -0
  237. fleetpull/timing/__init__.py +21 -0
  238. fleetpull/timing/canon.py +92 -0
  239. fleetpull/timing/clock.py +192 -0
  240. fleetpull/timing/codec.py +122 -0
  241. fleetpull/timing/sleeper.py +65 -0
  242. fleetpull/vocabulary/__init__.py +16 -0
  243. fleetpull/vocabulary/json_types.py +20 -0
  244. fleetpull/vocabulary/provider.py +29 -0
  245. fleetpull/vocabulary/quota_scope.py +47 -0
  246. fleetpull/vocabulary/response_category.py +32 -0
  247. fleetpull-0.1.0.dist-info/METADATA +248 -0
  248. fleetpull-0.1.0.dist-info/RECORD +252 -0
  249. fleetpull-0.1.0.dist-info/WHEEL +5 -0
  250. fleetpull-0.1.0.dist-info/entry_points.txt +2 -0
  251. fleetpull-0.1.0.dist-info/licenses/LICENSE +202 -0
  252. fleetpull-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,143 @@
1
+ # src/fleetpull/orchestrator/resume.py
2
+ """The incremental arms' pure resume decisions: interpret the stored cursor.
3
+
4
+ Pure functions, no I/O and no ``self``: ``resolve_watermark_start`` turns the
5
+ stored cursor into resume arm 1 (the watermark less the lookback margin), carrying
6
+ the future-watermark guard (Guard A) and the cross-mode rejection that
7
+ ``incremental/resolution.py`` deliberately omits (that module is cursor-free, so
8
+ cursor interpretation and its guards are the orchestrator's policy);
9
+ ``resolve_feed_resume`` is the feed arm's mirror -- the stored token used
10
+ directly, the cold-start ``FeedSeed`` when none is stored (the seed-once
11
+ invariant's structural half, DESIGN section 14 I4), and the same cross-mode
12
+ rejection in the other direction. The
13
+ strictly-forward advance discipline that once lived beside them moved into the
14
+ cursor store's atomic ``advance_watermark_forward`` with the prefix-advance rule
15
+ (DESIGN section 5, 2026-07-20) -- concurrent unit completions cannot enforce
16
+ monotonicity race-free from outside the statement. The runner reads and writes
17
+ the cursor; this only computes and validates, following the ``batch.py``
18
+ precedent that the runner's pure logic lives beside it, not on the class.
19
+ """
20
+
21
+ from datetime import datetime, timedelta
22
+
23
+ from fleetpull.exceptions import ConfigurationError
24
+ from fleetpull.incremental import (
25
+ DateWatermark,
26
+ FeedResume,
27
+ FeedSeed,
28
+ FeedToken,
29
+ IncrementalCursor,
30
+ )
31
+ from fleetpull.vocabulary import Provider
32
+
33
+ __all__: list[str] = ['resolve_feed_resume', 'resolve_watermark_start']
34
+
35
+
36
+ def resolve_watermark_start(
37
+ stored: IncrementalCursor | None,
38
+ lookback: timedelta,
39
+ now: datetime,
40
+ provider: Provider,
41
+ endpoint: str,
42
+ ) -> datetime | None:
43
+ """Resolve resume arm 1 from the stored cursor, guarding a future date.
44
+
45
+ Arm 1 of the resume precedence (DESIGN section 4): the committed watermark less
46
+ the lookback re-fetch margin, or ``None`` when no watermark is committed (the
47
+ caller then falls through to the coverage frontier and the cold-start
48
+ anchor). The future-watermark guard (Guard A) and the cross-mode rejection
49
+ live here -- the resolution math in ``incremental/resolution.py`` is
50
+ deliberately cursor-free, so cursor interpretation and its guards are the
51
+ orchestrator's policy.
52
+
53
+ Args:
54
+ stored: The persisted cursor, or ``None`` when none is committed.
55
+ lookback: The watermark mode's late-arrival re-fetch margin.
56
+ now: The run's clock instant.
57
+ provider: The endpoint's provider (error context).
58
+ endpoint: The endpoint name (error context).
59
+
60
+ Returns:
61
+ The watermark instant less ``lookback`` when a ``DateWatermark`` is
62
+ stored, else ``None``. This is the datetime-granular arm-1 candidate;
63
+ ``resolve_resume_start`` floors whichever arm it picks to the UTC
64
+ midnight of its date (the floored-window invariant).
65
+
66
+ Raises:
67
+ ConfigurationError: The stored watermark is dated after ``now`` (Guard A
68
+ -- it would otherwise stall the endpoint as a permanent caught-up),
69
+ or a ``FeedToken`` is stored for this watermark endpoint (cross-mode
70
+ state corruption).
71
+
72
+ Side Effects:
73
+ None -- pure.
74
+ """
75
+ match stored:
76
+ case None:
77
+ return None
78
+ case DateWatermark(watermark=watermark):
79
+ if watermark > now:
80
+ raise ConfigurationError(
81
+ 'stored watermark is in the future',
82
+ provider=provider.value,
83
+ endpoint=endpoint,
84
+ detail=(
85
+ f'watermark {watermark.isoformat()} is after the '
86
+ f'run clock {now.isoformat()}'
87
+ ),
88
+ )
89
+ return watermark - lookback
90
+ case _:
91
+ raise ConfigurationError(
92
+ 'feed cursor stored for a watermark endpoint',
93
+ provider=provider.value,
94
+ endpoint=endpoint,
95
+ )
96
+
97
+
98
+ def resolve_feed_resume(
99
+ stored: IncrementalCursor | None,
100
+ default_start: datetime,
101
+ provider: Provider,
102
+ endpoint: str,
103
+ ) -> FeedResume:
104
+ """Resolve the feed arm's resume value from the stored cursor.
105
+
106
+ The feed mirror of ``resolve_watermark_start``: a stored ``FeedToken``
107
+ resumes directly (no transformation — DESIGN section 4), and no stored
108
+ cursor seeds the feed at the sync-wide cold-start anchor. Constructing
109
+ the ``FeedSeed`` here, and ONLY here, is the structural half of the
110
+ seed-once invariant (section 14's I4): a resumed run can never carry a
111
+ seed, because the seed exists only on the no-cursor branch. The
112
+ cross-mode rejection guards the other direction of the watermark
113
+ resolver's — a watermark stored for a feed endpoint is state corruption.
114
+
115
+ Args:
116
+ stored: The persisted cursor, or ``None`` when none is committed.
117
+ default_start: The sync-wide cold-start anchor
118
+ (``sync.default_start_datetime``) the seed carries.
119
+ provider: The endpoint's provider (error context).
120
+ endpoint: The endpoint name (error context).
121
+
122
+ Returns:
123
+ The stored ``FeedToken``, or a ``FeedSeed`` at ``default_start``
124
+ when no cursor is committed.
125
+
126
+ Raises:
127
+ ConfigurationError: A ``DateWatermark`` is stored for this feed
128
+ endpoint (cross-mode state corruption).
129
+
130
+ Side Effects:
131
+ None -- pure.
132
+ """
133
+ match stored:
134
+ case None:
135
+ return FeedSeed(start=default_start)
136
+ case FeedToken():
137
+ return stored
138
+ case _:
139
+ raise ConfigurationError(
140
+ 'watermark cursor stored for a feed endpoint',
141
+ provider=provider.value,
142
+ endpoint=endpoint,
143
+ )
@@ -0,0 +1,108 @@
1
+ # src/fleetpull/orchestrator/roster_harvest.py
2
+ """Roster harvest: a feeder's complete current membership as a set of strings.
3
+
4
+ ``harvest_roster_members`` drives a feeder endpoint to a full listing and returns
5
+ the distinct values of its roster column -- the complete current membership the
6
+ refresh hands to ``reconcile``. It drives ``stream_processed_batches`` over the
7
+ feeder (validate-and-frame only, no window filter, no write) and unions
8
+ ``extract_roster_members`` across the streamed batches.
9
+
10
+ The harvest is always a full listing: ``reconcile`` diffs the complete membership
11
+ against the stored roster (anything absent from the listing increments toward
12
+ eviction), so a partial listing would not under-update the roster -- it would evict
13
+ live members. ``resume`` and ``context`` are therefore fixed at ``None`` (the
14
+ validate-and-frame-only path that preserves every row); they are not parameters,
15
+ because a windowed harvest is not a harvest. The feeder must be a full-listing
16
+ (snapshot) endpoint for the same reason; the coordinator that resolves the feeder
17
+ binding guards that, so this stays ``sync_mode``-blind, driving whatever binding it
18
+ is handed.
19
+
20
+ This owns no state and resolves no client or binding: the coordinator resolves the
21
+ roster's ``source_endpoint`` to its ``EndpointDefinition``, builds the
22
+ ``SingleRequestDriver``, opens the provider's client, and calls this.
23
+ """
24
+
25
+ from fleetpull.endpoints.shared import EndpointDefinition, SnapshotMode
26
+ from fleetpull.exceptions import ConfigurationError
27
+ from fleetpull.model_contract import ResponseModel
28
+ from fleetpull.network.client import TransportClient
29
+ from fleetpull.orchestrator.drivers import RequestDriver
30
+ from fleetpull.orchestrator.streaming import stream_processed_batches
31
+ from fleetpull.records import extract_roster_members
32
+
33
+ __all__: list[str] = ['harvest_roster_members', 'require_snapshot_feeder']
34
+
35
+
36
+ def require_snapshot_feeder(
37
+ definition: EndpointDefinition[ResponseModel], source_endpoint: str
38
+ ) -> None:
39
+ """Reject a roster feeder that is not a snapshot endpoint.
40
+
41
+ The full-listing requirement, stated once for BOTH reconcile routes --
42
+ the orchestration entry's feeder tap and the refresh coordinator's
43
+ harvest: ``reconcile`` is only correct over a complete listing, which
44
+ only a snapshot-mode feeder produces (a windowed run's partial listing
45
+ would mass-count absences against every member outside the window). A
46
+ non-snapshot feeder is a wiring bug, raised loudly before anything runs.
47
+
48
+ Args:
49
+ definition: The feeder endpoint's resolved binding.
50
+ source_endpoint: The feeder's endpoint name, for the error context.
51
+
52
+ Raises:
53
+ ConfigurationError: The feeder is not snapshot-mode.
54
+ """
55
+ if not isinstance(definition.sync_mode, SnapshotMode):
56
+ raise ConfigurationError(
57
+ 'roster feeder is not a snapshot endpoint',
58
+ provider=definition.provider.value,
59
+ endpoint=source_endpoint,
60
+ detail=(
61
+ 'a roster source must be a full-listing (snapshot) endpoint: '
62
+ 'reconcile is only correct over a complete listing'
63
+ ),
64
+ )
65
+
66
+
67
+ def harvest_roster_members(
68
+ definition: EndpointDefinition[ResponseModel],
69
+ driver: RequestDriver,
70
+ client: TransportClient,
71
+ column: str,
72
+ ) -> set[str]:
73
+ """Harvest a feeder's complete current membership as a set of strings.
74
+
75
+ Drives the feeder to a full listing and returns the distinct stringified values
76
+ of ``column`` across every batch -- the complete membership ``reconcile``
77
+ consumes. The stream runs validate-and-frame only (``resume`` and ``context``
78
+ ``None``), so no row is filtered out and the listing stays complete.
79
+
80
+ Args:
81
+ definition: The resolved feeder binding to list (its response model, spec
82
+ builder, and page decoder). Must be a full-listing endpoint; the
83
+ coordinator guards that before calling.
84
+ driver: The request driver the feeder is listed with -- a
85
+ ``SingleRequestDriver`` for the full single-chain listing.
86
+ client: The feeder provider's open transport client.
87
+ column: The feeder-frame column whose distinct values are the members (the
88
+ roster's ``source_column``).
89
+
90
+ Returns:
91
+ The complete current membership -- the distinct values of ``column`` as
92
+ strings, unioned across batches; empty when the feeder lists nothing.
93
+
94
+ Raises:
95
+ ValueError: ``column`` is absent from a feeder frame (from
96
+ ``extract_roster_members``) -- surfaced to the coordinator, whose
97
+ best-effort refresh keeps the existing roster on failure. Null and
98
+ empty-string values do not raise; the extractor filters them loudly.
99
+ FleetpullError: A fetch, validation, or framing failure from the stream.
100
+
101
+ Side Effects:
102
+ Issues the feeder's request chain through ``client`` (network I/O); writes
103
+ nothing.
104
+ """
105
+ members: set[str] = set()
106
+ for batch in stream_processed_batches(definition, driver, client, None, None):
107
+ members |= extract_roster_members(batch.frame, column)
108
+ return members
@@ -0,0 +1,363 @@
1
+ # src/fleetpull/orchestrator/roster_refresh.py
2
+ """The roster refresh coordinator: make a stale roster's membership current.
3
+
4
+ The coupling rule this module upholds (with the orchestration entry's feeder
5
+ tap): **every execution of a feeder endpoint updates its rosters and records a
6
+ run in the ledger**; parquet is written only when the user asked for that
7
+ endpoint. A coordinator harvest is such an execution -- it fetches, reconciles
8
+ the roster, and records a ``runs`` row for ``(provider, source_endpoint)`` --
9
+ so ``RunLedger.last_success_at`` is a sound staleness key in both directions:
10
+ a harvest is visible to the next staleness check, and a runner-driven feeder
11
+ run (which the entry taps into ``apply_listing``) never leaves the roster
12
+ behind the ledger. A run row certifies execution of the endpoint, not parquet
13
+ freshness.
14
+
15
+ ``refresh_if_stale`` is the demand-driven refresh a fan-out consumer calls
16
+ before it reads a roster. Given the roster's ``RosterDefinition`` -- the
17
+ caller resolves the ``RosterKey`` against the roster registry, the same way
18
+ the run executor is handed an already-resolved ``EndpointDefinition`` -- it
19
+ asks the ledger when the feeder last succeeded, and only if a refresh is due
20
+ (``is_roster_stale``, or an empty stored roster, which is stale regardless of
21
+ the ledger verdict -- ledger history may predate the harvest/ledger coupling)
22
+ re-lists the feeder, reconciles the listing against the stored roster, applies
23
+ the delta, and completes the run. ``apply_listing`` is the feeder-tap handoff
24
+ and the reconcile choke point: the entry collects a successful feeder run's
25
+ listed members and hands them here, and the harvest routes its own listing
26
+ through the same method -- so the reconcile guard (a roster is never
27
+ reconciled to empty; an empty listing is a failed refresh, not a membership
28
+ fact) covers both routes once. Staleness gates only whether the coordinator
29
+ *initiates* a harvest, never whether an executed run's non-empty listing is
30
+ applied; the run row on the tap path was already recorded by the run
31
+ executor.
32
+
33
+ The full-listing requirement lives here: a roster needs its feeder's complete
34
+ current membership each refresh, so the feeder must be a snapshot endpoint. The
35
+ coordinator resolves the definition's ``source_endpoint`` to the feeder binding and
36
+ guards that (``ConfigurationError`` on a non-snapshot feeder) before harvesting; the
37
+ harvester itself stays ``sync_mode``-blind.
38
+
39
+ ``refresh_if_stale`` is single-flight per roster key (added with the
40
+ intra-provider grain, DESIGN section 7, 2026-07-20): concurrent consumers of
41
+ one stale roster -- both cold-starting with their feeder unselected -- would
42
+ each harvest, wasting quota and recording duplicate ledger snapshot runs.
43
+ The coordinator therefore holds a per-``RosterKey`` lock across the method's
44
+ whole body (an outer lock guards get-or-create of the per-key locks), so the
45
+ second entrant re-runs the freshness check under the lock and returns early
46
+ on the now-fresh roster. Holding the lock across the harvest network call is
47
+ intended -- the waiting consumer needs the membership before it can fan out
48
+ -- and no lock nests inside (the store and ledger are
49
+ connection-per-operation), so there is no deadlock surface. Locks are
50
+ per-key: distinct rosters refresh concurrently.
51
+
52
+ Failure is best-effort with one loud exception. A harvest ``FleetpullError`` (the
53
+ feeder is unreachable, or a page fails to validate) or ``ValueError`` (a missing
54
+ source column) marks the run failed and degrades to the existing
55
+ roster -- a stale verdict is a refresh *attempt*, not a barrier to the fan-out. The
56
+ exception is cold start: when the store holds no members for the key, there is
57
+ nothing to fall back to and a silent empty roster would fan out over nothing, so the
58
+ failure re-raises (still marking the run failed). Wiring errors -- no feeder
59
+ registered for the ``source_endpoint``, a non-snapshot feeder -- raise before any
60
+ run row is opened; they are bugs, not executions. The crash order inside a
61
+ successful refresh mirrors the run executor's output-first discipline:
62
+ ``RosterStore.apply`` (one transaction) lands before ``complete_run``, so a crash
63
+ between the two leaves the roster current and the run merely ``running`` -- the
64
+ next check re-harvests idempotently, never the reverse mode where a fresh ledger
65
+ masks a stale roster.
66
+
67
+ Collaborators are injected, not assembled: the pure ``reconcile`` /
68
+ ``is_roster_stale`` are called directly; the ``EndpointRegistry`` is the immutable
69
+ catalog passed concrete; the stateful surfaces are narrow Protocols (``RosterAccess``,
70
+ ``FeederRunLedger``, and the spine's shared ``ClientSource``) the composition
71
+ root satisfies with the real store, ledger, and client registry.
72
+ """
73
+
74
+ import logging
75
+ import threading
76
+ from datetime import datetime
77
+ from functools import partial
78
+ from typing import Protocol
79
+
80
+ from fleetpull.endpoints import EndpointRegistry
81
+ from fleetpull.exceptions import FleetpullError, ProviderResponseError
82
+ from fleetpull.orchestrator.drivers import SingleRequestDriver
83
+ from fleetpull.orchestrator.recording import record_failure_safely
84
+ from fleetpull.orchestrator.roster_harvest import (
85
+ harvest_roster_members,
86
+ require_snapshot_feeder,
87
+ )
88
+ from fleetpull.orchestrator.spine import ClientSource
89
+ from fleetpull.roster import RosterDefinition, RosterKey
90
+ from fleetpull.state import RosterDelta, is_roster_stale, reconcile
91
+ from fleetpull.timing import Clock
92
+ from fleetpull.vocabulary import Provider
93
+
94
+ __all__: list[str] = [
95
+ 'FeederRunLedger',
96
+ 'RosterAccess',
97
+ 'RosterRefreshCoordinator',
98
+ ]
99
+
100
+ logger = logging.getLogger(__name__)
101
+
102
+
103
+ class RosterAccess(Protocol):
104
+ """The roster store surface the coordinator needs (a subset of RosterStore)."""
105
+
106
+ def read_counts(self, key: RosterKey) -> dict[str, int]:
107
+ """Return the roster as ``{member: absence_count}``."""
108
+ ...
109
+
110
+ def apply(self, key: RosterKey, delta: RosterDelta) -> None:
111
+ """Apply a reconciliation delta in one transaction."""
112
+ ...
113
+
114
+
115
+ class FeederRunLedger(Protocol):
116
+ """The ledger surface the coordinator needs (a subset of ``RunLedger``).
117
+
118
+ ``last_success_at`` keys the staleness decision; the run-recording trio
119
+ makes a coordinator harvest visible to that same key -- every execution of
120
+ a feeder endpoint records a run, so the freshness signal and the freshness
121
+ events cannot diverge.
122
+ """
123
+
124
+ def last_success_at(self, provider: Provider, endpoint: str) -> datetime | None:
125
+ """Return the feeder's latest successful run end, or ``None``."""
126
+ ...
127
+
128
+ def start_snapshot_run(self, provider: Provider, endpoint: str) -> int:
129
+ """Open a snapshot run for a harvest and return its id."""
130
+ ...
131
+
132
+ def complete_run(self, run_id: int, *, row_count: int) -> None:
133
+ """Close a run as succeeded with its row count."""
134
+ ...
135
+
136
+ def fail_run(self, run_id: int, *, error_detail: str) -> None:
137
+ """Close a run as failed with an error detail."""
138
+ ...
139
+
140
+
141
+ class RosterRefreshCoordinator:
142
+ """Refresh a roster's stored membership when its feeder listing has gone stale.
143
+
144
+ Built once with the endpoint catalog and the store / ledger / client surfaces, it
145
+ answers ``refresh_if_stale(definition)``. Beyond the per-key single-flight
146
+ locks (the coordinator is the stateful orchestrator here, so the lock map
147
+ lives on it) it holds no per-refresh state and opens no clients -- the
148
+ injected client source hands it the feeder provider's open client.
149
+
150
+ Args:
151
+ endpoint_registry: Resolves the feeder's ``(provider, name)`` to its binding.
152
+ store: The roster store (read counts, apply a delta).
153
+ ledger: The feeder's ledger surface -- the last-success read for the
154
+ staleness decision, and the run recording a harvest performs.
155
+ client_source: The open transport client per provider.
156
+ clock: The clock supplying ``now`` for the staleness decision.
157
+ """
158
+
159
+ def __init__(
160
+ self,
161
+ endpoint_registry: EndpointRegistry,
162
+ store: RosterAccess,
163
+ ledger: FeederRunLedger,
164
+ client_source: ClientSource,
165
+ clock: Clock,
166
+ ) -> None:
167
+ self._endpoint_registry = endpoint_registry
168
+ self._store = store
169
+ self._ledger = ledger
170
+ self._client_source = client_source
171
+ self._clock = clock
172
+ self._key_locks_guard = threading.Lock()
173
+ self._key_locks: dict[RosterKey, threading.Lock] = {}
174
+
175
+ def refresh_if_stale(self, definition: RosterDefinition) -> None:
176
+ """Re-list the feeder, reconcile the roster, and record the run, if stale.
177
+
178
+ Single-flight per roster key: the whole body runs under the key's
179
+ lock, so of two concurrent consumers hitting the same stale roster,
180
+ exactly one harvests -- the second re-runs the freshness check under
181
+ the lock and returns early on the now-fresh roster. The lock is held
182
+ across the harvest network call deliberately (the waiting consumer
183
+ needs the membership before it can fan out), and distinct keys never
184
+ contend.
185
+
186
+ Returns early when the roster is fresh -- non-empty and inside the
187
+ staleness bound. An empty stored roster is stale regardless of the
188
+ ledger verdict (ledger history may predate the harvest/ledger
189
+ coupling, and a fresh ledger must not mask a roster with nothing to
190
+ fan out over). When a refresh is due, the harvest is an execution of
191
+ the feeder endpoint: it opens a snapshot run, harvests the complete
192
+ membership, applies the reconciliation, and completes the run -- store
193
+ before ledger, so a crash between the two re-harvests rather than
194
+ masking. A harvest failure marks the run failed and degrades to the
195
+ existing roster, unless the store is empty (cold start), where it
196
+ re-raises.
197
+
198
+ Args:
199
+ definition: The roster to refresh -- its key, feeder ``source_endpoint``
200
+ and ``source_column``, and staleness / eviction policy. The caller
201
+ resolved it from a ``RosterKey`` against the roster registry.
202
+
203
+ Raises:
204
+ ConfigurationError: No feeder is registered for ``source_endpoint``, or the
205
+ resolved feeder is not a snapshot endpoint -- wiring bugs, raised
206
+ loudly before any run row is opened.
207
+ FleetpullError: The harvest failed on a cold start (no existing roster to
208
+ fall back to); the underlying failure propagates.
209
+ ValueError: The harvest failed on a cold start with a missing source
210
+ column; the underlying failure propagates.
211
+
212
+ Side Effects:
213
+ Holds the roster's single-flight lock for the duration. On a due
214
+ refresh: issues the feeder's request chain and records a run in
215
+ the ledger; on success, also writes the reconciled delta to the
216
+ store. Otherwise touches nothing.
217
+ """
218
+ with self._refresh_lock(definition.key):
219
+ self._refresh_if_due(definition)
220
+
221
+ def _refresh_lock(self, key: RosterKey) -> threading.Lock:
222
+ """Get or create the roster's single-flight lock.
223
+
224
+ Args:
225
+ key: The roster whose lock to return (hashable frozen identity).
226
+
227
+ Returns:
228
+ The per-key lock, created on first use under the outer guard.
229
+ """
230
+ with self._key_locks_guard:
231
+ return self._key_locks.setdefault(key, threading.Lock())
232
+
233
+ def _refresh_if_due(self, definition: RosterDefinition) -> None:
234
+ """The refresh body ``refresh_if_stale`` runs under the key's lock."""
235
+ provider = definition.key.provider
236
+ current = self._store.read_counts(definition.key)
237
+ last_success = self._ledger.last_success_at(
238
+ provider, definition.source_endpoint
239
+ )
240
+ now = self._clock.now_utc()
241
+ if current and not is_roster_stale(last_success, now, definition.max_age):
242
+ return
243
+ feeder = self._endpoint_registry.get(provider, definition.source_endpoint)
244
+ require_snapshot_feeder(feeder, definition.source_endpoint)
245
+ client = self._client_source.client_for(provider)
246
+ logger.info(
247
+ 'roster refresh started: provider=%s roster=%s feeder=%s members_held=%d',
248
+ provider.value,
249
+ definition.key.name,
250
+ definition.source_endpoint,
251
+ len(current),
252
+ )
253
+ run_id = self._ledger.start_snapshot_run(provider, definition.source_endpoint)
254
+ try:
255
+ listed = harvest_roster_members(
256
+ feeder, SingleRequestDriver(), client, definition.source_column
257
+ )
258
+ # Routed through the shared reconcile body so its guard (a
259
+ # roster is never reconciled to empty) covers the harvest inside
260
+ # this same failed-refresh path: an empty listing marks the run
261
+ # failed and degrades exactly like a failed HTTP refresh. The
262
+ # key's single-flight lock is already held here, so the locked
263
+ # ``apply_listing`` wrapper would deadlock.
264
+ self._reconcile_listing(definition, listed)
265
+ except (FleetpullError, ValueError) as failure:
266
+ record_failure_safely(
267
+ partial(self._ledger.fail_run, run_id, error_detail=str(failure)),
268
+ f'harvest run {run_id}',
269
+ )
270
+ if not current:
271
+ raise
272
+ logger.warning(
273
+ 'roster %s/%s refresh failed; keeping %d existing members: %s',
274
+ provider.value,
275
+ definition.key.name,
276
+ len(current),
277
+ failure,
278
+ )
279
+ return
280
+ # A harvest run's row count is the distinct-member count of the
281
+ # listing -- the rows this execution produced for its consumer.
282
+ self._ledger.complete_run(run_id, row_count=len(listed))
283
+ logger.info(
284
+ 'roster refreshed: provider=%s roster=%s members=%d',
285
+ provider.value,
286
+ definition.key.name,
287
+ len(listed),
288
+ )
289
+
290
+ def apply_listing(self, definition: RosterDefinition, listed: set[str]) -> None:
291
+ """Reconcile an executed feeder run's listed membership into the store.
292
+
293
+ The feeder tap's route into the reconcile choke point
294
+ (``_reconcile_listing``): the orchestration entry hands a successful
295
+ runner-driven feeder run's distinct ``source_column`` values here.
296
+ Runs under the roster key's single-flight lock -- the same lock
297
+ ``refresh_if_stale`` holds around its harvest -- so every reconcile
298
+ for one key serializes, whichever route wrote it: two
299
+ read-reconcile-write sequences on one roster can never interleave
300
+ and corrupt absence counts. Records no ledger row: the tap route's
301
+ run was recorded by the run executor.
302
+
303
+ Args:
304
+ definition: The sourced roster -- its key and eviction policy.
305
+ listed: The feeder run's complete listed membership.
306
+
307
+ Raises:
308
+ ProviderResponseError: ``listed`` is empty (the reconcile guard);
309
+ the store is untouched.
310
+
311
+ Side Effects:
312
+ Holds the roster's single-flight lock for the duration; writes
313
+ the reconciled delta to the store (one transaction).
314
+ """
315
+ with self._refresh_lock(definition.key):
316
+ self._reconcile_listing(definition, listed)
317
+
318
+ def _reconcile_listing(
319
+ self, definition: RosterDefinition, listed: set[str]
320
+ ) -> None:
321
+ """The reconcile body both write routes run under the key's lock.
322
+
323
+ The reconcile guard lives here so both routes -- the feeder tap
324
+ (``apply_listing``) and the coordinator's own harvest
325
+ (``_refresh_if_due``, which already holds the lock) -- are covered
326
+ once: **a roster is never reconciled to empty**. An empty listing --
327
+ the provider returned nothing, or every record's member value
328
+ filtered out -- is a failed refresh, not a membership fact:
329
+ reconciling it would mass-increment absence counts and, with an
330
+ eviction threshold, evict the entire roster through systematic
331
+ provider garbage. The prior roster stays intact and the caller's
332
+ failed-refresh semantics apply (the harvest degrades exactly like a
333
+ failed HTTP refresh; the tap propagates, failing the endpoint
334
+ loudly). A non-empty listing is applied whole -- staleness gates
335
+ only whether the coordinator *initiates* a harvest, never whether an
336
+ executed run's complete listing is reconciled.
337
+
338
+ Args:
339
+ definition: The sourced roster -- its key and eviction policy.
340
+ listed: The feeder run's complete listed membership.
341
+
342
+ Raises:
343
+ ProviderResponseError: ``listed`` is empty (the reconcile guard);
344
+ the store is untouched.
345
+
346
+ Side Effects:
347
+ Writes the reconciled delta to the store (one transaction).
348
+ """
349
+ if not listed:
350
+ raise ProviderResponseError(
351
+ provider=definition.key.provider.value,
352
+ endpoint=definition.source_endpoint,
353
+ detail=(
354
+ f'feeder listed no members for roster '
355
+ f'{definition.key.name!r}; a roster is never reconciled '
356
+ f'to empty -- the prior membership stands'
357
+ ),
358
+ )
359
+ current = self._store.read_counts(definition.key)
360
+ self._store.apply(
361
+ definition.key,
362
+ reconcile(current, listed, definition.eviction_threshold),
363
+ )