fleetpull 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fleetpull/__init__.py +36 -0
- fleetpull/api/__init__.py +32 -0
- fleetpull/api/auth_ingress.py +216 -0
- fleetpull/api/catalog.py +139 -0
- fleetpull/api/fetch.py +229 -0
- fleetpull/api/identity.py +90 -0
- fleetpull/api/sync.py +735 -0
- fleetpull/cli.py +146 -0
- fleetpull/config/__init__.py +47 -0
- fleetpull/config/base.py +18 -0
- fleetpull/config/example.py +90 -0
- fleetpull/config/geotab.py +78 -0
- fleetpull/config/http.py +32 -0
- fleetpull/config/loading.py +195 -0
- fleetpull/config/logger.py +139 -0
- fleetpull/config/providers.py +520 -0
- fleetpull/config/rate_limit.py +50 -0
- fleetpull/config/resolution.py +156 -0
- fleetpull/config/retry.py +69 -0
- fleetpull/config/root.py +157 -0
- fleetpull/config/sections.py +146 -0
- fleetpull/endpoints/__init__.py +24 -0
- fleetpull/endpoints/geotab/__init__.py +12 -0
- fleetpull/endpoints/geotab/_requests.py +427 -0
- fleetpull/endpoints/geotab/annotation_logs.py +73 -0
- fleetpull/endpoints/geotab/audits.py +70 -0
- fleetpull/endpoints/geotab/devices.py +79 -0
- fleetpull/endpoints/geotab/driver_changes.py +73 -0
- fleetpull/endpoints/geotab/duty_status_logs.py +75 -0
- fleetpull/endpoints/geotab/dvir_logs.py +76 -0
- fleetpull/endpoints/geotab/exception_events.py +119 -0
- fleetpull/endpoints/geotab/fault_data.py +72 -0
- fleetpull/endpoints/geotab/fill_ups.py +79 -0
- fleetpull/endpoints/geotab/fuel_and_energy_used.py +74 -0
- fleetpull/endpoints/geotab/fuel_tax_details.py +75 -0
- fleetpull/endpoints/geotab/log_records.py +74 -0
- fleetpull/endpoints/geotab/media_files.py +74 -0
- fleetpull/endpoints/geotab/shipment_logs.py +71 -0
- fleetpull/endpoints/geotab/status_data.py +71 -0
- fleetpull/endpoints/geotab/text_messages.py +77 -0
- fleetpull/endpoints/geotab/trips.py +102 -0
- fleetpull/endpoints/geotab/users.py +81 -0
- fleetpull/endpoints/motive/__init__.py +12 -0
- fleetpull/endpoints/motive/_spec_builders.py +94 -0
- fleetpull/endpoints/motive/driver_idle_rollups.py +113 -0
- fleetpull/endpoints/motive/driving_periods.py +85 -0
- fleetpull/endpoints/motive/groups.py +61 -0
- fleetpull/endpoints/motive/idle_events.py +94 -0
- fleetpull/endpoints/motive/users.py +64 -0
- fleetpull/endpoints/motive/vehicle_locations.py +170 -0
- fleetpull/endpoints/motive/vehicle_utilizations.py +122 -0
- fleetpull/endpoints/motive/vehicles.py +106 -0
- fleetpull/endpoints/registry.py +280 -0
- fleetpull/endpoints/samsara/__init__.py +12 -0
- fleetpull/endpoints/samsara/_spec_builders.py +202 -0
- fleetpull/endpoints/samsara/addresses.py +77 -0
- fleetpull/endpoints/samsara/asset_locations.py +217 -0
- fleetpull/endpoints/samsara/driver_fuel_energy_reports.py +124 -0
- fleetpull/endpoints/samsara/driver_vehicle_assignments.py +211 -0
- fleetpull/endpoints/samsara/drivers.py +169 -0
- fleetpull/endpoints/samsara/engine_states.py +114 -0
- fleetpull/endpoints/samsara/gps_readings.py +113 -0
- fleetpull/endpoints/samsara/idling_events.py +195 -0
- fleetpull/endpoints/samsara/odometer_readings.py +116 -0
- fleetpull/endpoints/samsara/trips.py +217 -0
- fleetpull/endpoints/samsara/vehicle_fuel_energy_reports.py +139 -0
- fleetpull/endpoints/samsara/vehicles.py +121 -0
- fleetpull/endpoints/shared/__init__.py +57 -0
- fleetpull/endpoints/shared/base.py +529 -0
- fleetpull/endpoints/shared/request_shape.py +227 -0
- fleetpull/endpoints/shared/resume.py +74 -0
- fleetpull/endpoints/shared/spec_builders.py +59 -0
- fleetpull/endpoints/shared/sync_mode.py +159 -0
- fleetpull/endpoints/shared/url_paths.py +149 -0
- fleetpull/exceptions.py +320 -0
- fleetpull/incremental/__init__.py +27 -0
- fleetpull/incremental/cursor.py +66 -0
- fleetpull/incremental/resolution.py +152 -0
- fleetpull/incremental/seed.py +53 -0
- fleetpull/incremental/window.py +94 -0
- fleetpull/logger/__init__.py +5 -0
- fleetpull/logger/setup.py +145 -0
- fleetpull/model_contract/__init__.py +6 -0
- fleetpull/model_contract/coercions.py +33 -0
- fleetpull/model_contract/response.py +45 -0
- fleetpull/models/__init__.py +7 -0
- fleetpull/models/geotab/__init__.py +166 -0
- fleetpull/models/geotab/annotation_log.py +109 -0
- fleetpull/models/geotab/audit.py +56 -0
- fleetpull/models/geotab/device.py +217 -0
- fleetpull/models/geotab/driver_change.py +102 -0
- fleetpull/models/geotab/duty_status_log.py +200 -0
- fleetpull/models/geotab/dvir_log.py +184 -0
- fleetpull/models/geotab/exception_event.py +135 -0
- fleetpull/models/geotab/fault_data.py +168 -0
- fleetpull/models/geotab/fill_up.py +189 -0
- fleetpull/models/geotab/fuel_and_energy_used.py +81 -0
- fleetpull/models/geotab/fuel_tax_detail.py +150 -0
- fleetpull/models/geotab/log_record.py +68 -0
- fleetpull/models/geotab/media_file.py +133 -0
- fleetpull/models/geotab/shared.py +221 -0
- fleetpull/models/geotab/shipment_log.py +105 -0
- fleetpull/models/geotab/status_data.py +108 -0
- fleetpull/models/geotab/text_message.py +125 -0
- fleetpull/models/geotab/trip.py +162 -0
- fleetpull/models/geotab/user.py +187 -0
- fleetpull/models/motive/__init__.py +43 -0
- fleetpull/models/motive/driver_idle_rollup.py +109 -0
- fleetpull/models/motive/driving_period.py +82 -0
- fleetpull/models/motive/group.py +49 -0
- fleetpull/models/motive/idle_event.py +77 -0
- fleetpull/models/motive/shared.py +192 -0
- fleetpull/models/motive/user.py +217 -0
- fleetpull/models/motive/vehicle.py +162 -0
- fleetpull/models/motive/vehicle_location.py +127 -0
- fleetpull/models/motive/vehicle_utilization.py +128 -0
- fleetpull/models/samsara/__init__.py +108 -0
- fleetpull/models/samsara/address.py +149 -0
- fleetpull/models/samsara/asset_location.py +131 -0
- fleetpull/models/samsara/driver.py +232 -0
- fleetpull/models/samsara/driver_fuel_energy_report.py +153 -0
- fleetpull/models/samsara/driver_vehicle_assignment.py +168 -0
- fleetpull/models/samsara/engine_state.py +92 -0
- fleetpull/models/samsara/gps_reading.py +146 -0
- fleetpull/models/samsara/idling_event.py +174 -0
- fleetpull/models/samsara/odometer_reading.py +90 -0
- fleetpull/models/samsara/trip.py +199 -0
- fleetpull/models/samsara/vehicle.py +167 -0
- fleetpull/models/samsara/vehicle_fuel_energy_report.py +191 -0
- fleetpull/network/__init__.py +9 -0
- fleetpull/network/auth/__init__.py +15 -0
- fleetpull/network/auth/authenticate.py +271 -0
- fleetpull/network/auth/manager.py +223 -0
- fleetpull/network/auth/models.py +57 -0
- fleetpull/network/auth/strategies.py +169 -0
- fleetpull/network/classifiers/__init__.py +11 -0
- fleetpull/network/classifiers/geotab.py +145 -0
- fleetpull/network/classifiers/motive.py +79 -0
- fleetpull/network/classifiers/samsara.py +80 -0
- fleetpull/network/client/__init__.py +25 -0
- fleetpull/network/client/page.py +32 -0
- fleetpull/network/client/profile.py +27 -0
- fleetpull/network/client/registry.py +103 -0
- fleetpull/network/client/registry_base.py +150 -0
- fleetpull/network/client/runtime.py +44 -0
- fleetpull/network/client/transport.py +381 -0
- fleetpull/network/contract/__init__.py +50 -0
- fleetpull/network/contract/auth.py +48 -0
- fleetpull/network/contract/classifier.py +189 -0
- fleetpull/network/contract/envelope_fetcher.py +33 -0
- fleetpull/network/contract/envelopes.py +189 -0
- fleetpull/network/contract/outcome.py +43 -0
- fleetpull/network/contract/page_decoder.py +101 -0
- fleetpull/network/contract/request.py +97 -0
- fleetpull/network/decoders/__init__.py +40 -0
- fleetpull/network/decoders/_window_stamp.py +78 -0
- fleetpull/network/decoders/geotab.py +309 -0
- fleetpull/network/decoders/motive.py +204 -0
- fleetpull/network/decoders/motive_reports.py +114 -0
- fleetpull/network/decoders/samsara.py +343 -0
- fleetpull/network/decoders/samsara_reports.py +119 -0
- fleetpull/network/decoders/single_page.py +55 -0
- fleetpull/network/limits/__init__.py +13 -0
- fleetpull/network/limits/bucket_math.py +58 -0
- fleetpull/network/limits/limiter.py +153 -0
- fleetpull/network/limits/registry.py +96 -0
- fleetpull/network/posture/__init__.py +6 -0
- fleetpull/network/posture/client_options.py +96 -0
- fleetpull/network/retry/__init__.py +13 -0
- fleetpull/network/retry/decision.py +155 -0
- fleetpull/network/tls/__init__.py +5 -0
- fleetpull/network/tls/truststore_context.py +35 -0
- fleetpull/orchestrator/__init__.py +30 -0
- fleetpull/orchestrator/backfill.py +127 -0
- fleetpull/orchestrator/batch.py +155 -0
- fleetpull/orchestrator/bisection.py +271 -0
- fleetpull/orchestrator/drivers.py +350 -0
- fleetpull/orchestrator/entry.py +347 -0
- fleetpull/orchestrator/executors.py +128 -0
- fleetpull/orchestrator/fanout.py +163 -0
- fleetpull/orchestrator/feed_drive.py +266 -0
- fleetpull/orchestrator/metadata_projection.py +195 -0
- fleetpull/orchestrator/outcome.py +53 -0
- fleetpull/orchestrator/recording.py +85 -0
- fleetpull/orchestrator/resume.py +143 -0
- fleetpull/orchestrator/roster_harvest.py +108 -0
- fleetpull/orchestrator/roster_refresh.py +363 -0
- fleetpull/orchestrator/runner.py +262 -0
- fleetpull/orchestrator/shape_resolution.py +221 -0
- fleetpull/orchestrator/spine.py +166 -0
- fleetpull/orchestrator/streaming.py +153 -0
- fleetpull/orchestrator/unit_loop.py +273 -0
- fleetpull/orchestrator/watermark_drive.py +455 -0
- fleetpull/paths/__init__.py +13 -0
- fleetpull/paths/datasets.py +38 -0
- fleetpull/paths/partitions.py +41 -0
- fleetpull/paths/resolution.py +98 -0
- fleetpull/polars_typing/__init__.py +20 -0
- fleetpull/py.typed +0 -0
- fleetpull/records/__init__.py +19 -0
- fleetpull/records/convert.py +53 -0
- fleetpull/records/dataframe.py +63 -0
- fleetpull/records/event_time.py +70 -0
- fleetpull/records/fields.py +190 -0
- fleetpull/records/flatten.py +54 -0
- fleetpull/records/roster_members.py +74 -0
- fleetpull/records/schema.py +73 -0
- fleetpull/records/validation.py +62 -0
- fleetpull/resources/__init__.py +10 -0
- fleetpull/resources/config.example.yaml +222 -0
- fleetpull/roster/__init__.py +16 -0
- fleetpull/roster/definition.py +46 -0
- fleetpull/roster/key.py +40 -0
- fleetpull/roster/registry.py +93 -0
- fleetpull/state/__init__.py +36 -0
- fleetpull/state/cursors.py +417 -0
- fleetpull/state/database.py +463 -0
- fleetpull/state/migrations.py +430 -0
- fleetpull/state/reconcile.py +127 -0
- fleetpull/state/rosters.py +159 -0
- fleetpull/state/run_ledger.py +572 -0
- fleetpull/state/work_units.py +648 -0
- fleetpull/storage/__init__.py +30 -0
- fleetpull/storage/append.py +187 -0
- fleetpull/storage/atomic.py +161 -0
- fleetpull/storage/files.py +176 -0
- fleetpull/storage/frames.py +98 -0
- fleetpull/storage/metadata.py +161 -0
- fleetpull/storage/partitioned.py +205 -0
- fleetpull/storage/pruning.py +169 -0
- fleetpull/storage/read.py +35 -0
- fleetpull/storage/result.py +34 -0
- fleetpull/storage/single_file.py +130 -0
- fleetpull/storage/splitting.py +88 -0
- fleetpull/storage/staging.py +178 -0
- fleetpull/storage/writers.py +145 -0
- fleetpull/timing/__init__.py +21 -0
- fleetpull/timing/canon.py +92 -0
- fleetpull/timing/clock.py +192 -0
- fleetpull/timing/codec.py +122 -0
- fleetpull/timing/sleeper.py +65 -0
- fleetpull/vocabulary/__init__.py +16 -0
- fleetpull/vocabulary/json_types.py +20 -0
- fleetpull/vocabulary/provider.py +29 -0
- fleetpull/vocabulary/quota_scope.py +47 -0
- fleetpull/vocabulary/response_category.py +32 -0
- fleetpull-0.1.0.dist-info/METADATA +248 -0
- fleetpull-0.1.0.dist-info/RECORD +252 -0
- fleetpull-0.1.0.dist-info/WHEEL +5 -0
- fleetpull-0.1.0.dist-info/entry_points.txt +2 -0
- fleetpull-0.1.0.dist-info/licenses/LICENSE +202 -0
- fleetpull-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# src/fleetpull/orchestrator/resume.py
|
|
2
|
+
"""The incremental arms' pure resume decisions: interpret the stored cursor.
|
|
3
|
+
|
|
4
|
+
Pure functions, no I/O and no ``self``: ``resolve_watermark_start`` turns the
|
|
5
|
+
stored cursor into resume arm 1 (the watermark less the lookback margin), carrying
|
|
6
|
+
the future-watermark guard (Guard A) and the cross-mode rejection that
|
|
7
|
+
``incremental/resolution.py`` deliberately omits (that module is cursor-free, so
|
|
8
|
+
cursor interpretation and its guards are the orchestrator's policy);
|
|
9
|
+
``resolve_feed_resume`` is the feed arm's mirror -- the stored token used
|
|
10
|
+
directly, the cold-start ``FeedSeed`` when none is stored (the seed-once
|
|
11
|
+
invariant's structural half, DESIGN section 14 I4), and the same cross-mode
|
|
12
|
+
rejection in the other direction. The
|
|
13
|
+
strictly-forward advance discipline that once lived beside them moved into the
|
|
14
|
+
cursor store's atomic ``advance_watermark_forward`` with the prefix-advance rule
|
|
15
|
+
(DESIGN section 5, 2026-07-20) -- concurrent unit completions cannot enforce
|
|
16
|
+
monotonicity race-free from outside the statement. The runner reads and writes
|
|
17
|
+
the cursor; this only computes and validates, following the ``batch.py``
|
|
18
|
+
precedent that the runner's pure logic lives beside it, not on the class.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from datetime import datetime, timedelta
|
|
22
|
+
|
|
23
|
+
from fleetpull.exceptions import ConfigurationError
|
|
24
|
+
from fleetpull.incremental import (
|
|
25
|
+
DateWatermark,
|
|
26
|
+
FeedResume,
|
|
27
|
+
FeedSeed,
|
|
28
|
+
FeedToken,
|
|
29
|
+
IncrementalCursor,
|
|
30
|
+
)
|
|
31
|
+
from fleetpull.vocabulary import Provider
|
|
32
|
+
|
|
33
|
+
__all__: list[str] = ['resolve_feed_resume', 'resolve_watermark_start']
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def resolve_watermark_start(
|
|
37
|
+
stored: IncrementalCursor | None,
|
|
38
|
+
lookback: timedelta,
|
|
39
|
+
now: datetime,
|
|
40
|
+
provider: Provider,
|
|
41
|
+
endpoint: str,
|
|
42
|
+
) -> datetime | None:
|
|
43
|
+
"""Resolve resume arm 1 from the stored cursor, guarding a future date.
|
|
44
|
+
|
|
45
|
+
Arm 1 of the resume precedence (DESIGN section 4): the committed watermark less
|
|
46
|
+
the lookback re-fetch margin, or ``None`` when no watermark is committed (the
|
|
47
|
+
caller then falls through to the coverage frontier and the cold-start
|
|
48
|
+
anchor). The future-watermark guard (Guard A) and the cross-mode rejection
|
|
49
|
+
live here -- the resolution math in ``incremental/resolution.py`` is
|
|
50
|
+
deliberately cursor-free, so cursor interpretation and its guards are the
|
|
51
|
+
orchestrator's policy.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
stored: The persisted cursor, or ``None`` when none is committed.
|
|
55
|
+
lookback: The watermark mode's late-arrival re-fetch margin.
|
|
56
|
+
now: The run's clock instant.
|
|
57
|
+
provider: The endpoint's provider (error context).
|
|
58
|
+
endpoint: The endpoint name (error context).
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
The watermark instant less ``lookback`` when a ``DateWatermark`` is
|
|
62
|
+
stored, else ``None``. This is the datetime-granular arm-1 candidate;
|
|
63
|
+
``resolve_resume_start`` floors whichever arm it picks to the UTC
|
|
64
|
+
midnight of its date (the floored-window invariant).
|
|
65
|
+
|
|
66
|
+
Raises:
|
|
67
|
+
ConfigurationError: The stored watermark is dated after ``now`` (Guard A
|
|
68
|
+
-- it would otherwise stall the endpoint as a permanent caught-up),
|
|
69
|
+
or a ``FeedToken`` is stored for this watermark endpoint (cross-mode
|
|
70
|
+
state corruption).
|
|
71
|
+
|
|
72
|
+
Side Effects:
|
|
73
|
+
None -- pure.
|
|
74
|
+
"""
|
|
75
|
+
match stored:
|
|
76
|
+
case None:
|
|
77
|
+
return None
|
|
78
|
+
case DateWatermark(watermark=watermark):
|
|
79
|
+
if watermark > now:
|
|
80
|
+
raise ConfigurationError(
|
|
81
|
+
'stored watermark is in the future',
|
|
82
|
+
provider=provider.value,
|
|
83
|
+
endpoint=endpoint,
|
|
84
|
+
detail=(
|
|
85
|
+
f'watermark {watermark.isoformat()} is after the '
|
|
86
|
+
f'run clock {now.isoformat()}'
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
return watermark - lookback
|
|
90
|
+
case _:
|
|
91
|
+
raise ConfigurationError(
|
|
92
|
+
'feed cursor stored for a watermark endpoint',
|
|
93
|
+
provider=provider.value,
|
|
94
|
+
endpoint=endpoint,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def resolve_feed_resume(
|
|
99
|
+
stored: IncrementalCursor | None,
|
|
100
|
+
default_start: datetime,
|
|
101
|
+
provider: Provider,
|
|
102
|
+
endpoint: str,
|
|
103
|
+
) -> FeedResume:
|
|
104
|
+
"""Resolve the feed arm's resume value from the stored cursor.
|
|
105
|
+
|
|
106
|
+
The feed mirror of ``resolve_watermark_start``: a stored ``FeedToken``
|
|
107
|
+
resumes directly (no transformation — DESIGN section 4), and no stored
|
|
108
|
+
cursor seeds the feed at the sync-wide cold-start anchor. Constructing
|
|
109
|
+
the ``FeedSeed`` here, and ONLY here, is the structural half of the
|
|
110
|
+
seed-once invariant (section 14's I4): a resumed run can never carry a
|
|
111
|
+
seed, because the seed exists only on the no-cursor branch. The
|
|
112
|
+
cross-mode rejection guards the other direction of the watermark
|
|
113
|
+
resolver's — a watermark stored for a feed endpoint is state corruption.
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
stored: The persisted cursor, or ``None`` when none is committed.
|
|
117
|
+
default_start: The sync-wide cold-start anchor
|
|
118
|
+
(``sync.default_start_datetime``) the seed carries.
|
|
119
|
+
provider: The endpoint's provider (error context).
|
|
120
|
+
endpoint: The endpoint name (error context).
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
The stored ``FeedToken``, or a ``FeedSeed`` at ``default_start``
|
|
124
|
+
when no cursor is committed.
|
|
125
|
+
|
|
126
|
+
Raises:
|
|
127
|
+
ConfigurationError: A ``DateWatermark`` is stored for this feed
|
|
128
|
+
endpoint (cross-mode state corruption).
|
|
129
|
+
|
|
130
|
+
Side Effects:
|
|
131
|
+
None -- pure.
|
|
132
|
+
"""
|
|
133
|
+
match stored:
|
|
134
|
+
case None:
|
|
135
|
+
return FeedSeed(start=default_start)
|
|
136
|
+
case FeedToken():
|
|
137
|
+
return stored
|
|
138
|
+
case _:
|
|
139
|
+
raise ConfigurationError(
|
|
140
|
+
'watermark cursor stored for a feed endpoint',
|
|
141
|
+
provider=provider.value,
|
|
142
|
+
endpoint=endpoint,
|
|
143
|
+
)
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# src/fleetpull/orchestrator/roster_harvest.py
|
|
2
|
+
"""Roster harvest: a feeder's complete current membership as a set of strings.
|
|
3
|
+
|
|
4
|
+
``harvest_roster_members`` drives a feeder endpoint to a full listing and returns
|
|
5
|
+
the distinct values of its roster column -- the complete current membership the
|
|
6
|
+
refresh hands to ``reconcile``. It drives ``stream_processed_batches`` over the
|
|
7
|
+
feeder (validate-and-frame only, no window filter, no write) and unions
|
|
8
|
+
``extract_roster_members`` across the streamed batches.
|
|
9
|
+
|
|
10
|
+
The harvest is always a full listing: ``reconcile`` diffs the complete membership
|
|
11
|
+
against the stored roster (anything absent from the listing increments toward
|
|
12
|
+
eviction), so a partial listing would not under-update the roster -- it would evict
|
|
13
|
+
live members. ``resume`` and ``context`` are therefore fixed at ``None`` (the
|
|
14
|
+
validate-and-frame-only path that preserves every row); they are not parameters,
|
|
15
|
+
because a windowed harvest is not a harvest. The feeder must be a full-listing
|
|
16
|
+
(snapshot) endpoint for the same reason; the coordinator that resolves the feeder
|
|
17
|
+
binding guards that, so this stays ``sync_mode``-blind, driving whatever binding it
|
|
18
|
+
is handed.
|
|
19
|
+
|
|
20
|
+
This owns no state and resolves no client or binding: the coordinator resolves the
|
|
21
|
+
roster's ``source_endpoint`` to its ``EndpointDefinition``, builds the
|
|
22
|
+
``SingleRequestDriver``, opens the provider's client, and calls this.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from fleetpull.endpoints.shared import EndpointDefinition, SnapshotMode
|
|
26
|
+
from fleetpull.exceptions import ConfigurationError
|
|
27
|
+
from fleetpull.model_contract import ResponseModel
|
|
28
|
+
from fleetpull.network.client import TransportClient
|
|
29
|
+
from fleetpull.orchestrator.drivers import RequestDriver
|
|
30
|
+
from fleetpull.orchestrator.streaming import stream_processed_batches
|
|
31
|
+
from fleetpull.records import extract_roster_members
|
|
32
|
+
|
|
33
|
+
__all__: list[str] = ['harvest_roster_members', 'require_snapshot_feeder']
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def require_snapshot_feeder(
|
|
37
|
+
definition: EndpointDefinition[ResponseModel], source_endpoint: str
|
|
38
|
+
) -> None:
|
|
39
|
+
"""Reject a roster feeder that is not a snapshot endpoint.
|
|
40
|
+
|
|
41
|
+
The full-listing requirement, stated once for BOTH reconcile routes --
|
|
42
|
+
the orchestration entry's feeder tap and the refresh coordinator's
|
|
43
|
+
harvest: ``reconcile`` is only correct over a complete listing, which
|
|
44
|
+
only a snapshot-mode feeder produces (a windowed run's partial listing
|
|
45
|
+
would mass-count absences against every member outside the window). A
|
|
46
|
+
non-snapshot feeder is a wiring bug, raised loudly before anything runs.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
definition: The feeder endpoint's resolved binding.
|
|
50
|
+
source_endpoint: The feeder's endpoint name, for the error context.
|
|
51
|
+
|
|
52
|
+
Raises:
|
|
53
|
+
ConfigurationError: The feeder is not snapshot-mode.
|
|
54
|
+
"""
|
|
55
|
+
if not isinstance(definition.sync_mode, SnapshotMode):
|
|
56
|
+
raise ConfigurationError(
|
|
57
|
+
'roster feeder is not a snapshot endpoint',
|
|
58
|
+
provider=definition.provider.value,
|
|
59
|
+
endpoint=source_endpoint,
|
|
60
|
+
detail=(
|
|
61
|
+
'a roster source must be a full-listing (snapshot) endpoint: '
|
|
62
|
+
'reconcile is only correct over a complete listing'
|
|
63
|
+
),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def harvest_roster_members(
|
|
68
|
+
definition: EndpointDefinition[ResponseModel],
|
|
69
|
+
driver: RequestDriver,
|
|
70
|
+
client: TransportClient,
|
|
71
|
+
column: str,
|
|
72
|
+
) -> set[str]:
|
|
73
|
+
"""Harvest a feeder's complete current membership as a set of strings.
|
|
74
|
+
|
|
75
|
+
Drives the feeder to a full listing and returns the distinct stringified values
|
|
76
|
+
of ``column`` across every batch -- the complete membership ``reconcile``
|
|
77
|
+
consumes. The stream runs validate-and-frame only (``resume`` and ``context``
|
|
78
|
+
``None``), so no row is filtered out and the listing stays complete.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
definition: The resolved feeder binding to list (its response model, spec
|
|
82
|
+
builder, and page decoder). Must be a full-listing endpoint; the
|
|
83
|
+
coordinator guards that before calling.
|
|
84
|
+
driver: The request driver the feeder is listed with -- a
|
|
85
|
+
``SingleRequestDriver`` for the full single-chain listing.
|
|
86
|
+
client: The feeder provider's open transport client.
|
|
87
|
+
column: The feeder-frame column whose distinct values are the members (the
|
|
88
|
+
roster's ``source_column``).
|
|
89
|
+
|
|
90
|
+
Returns:
|
|
91
|
+
The complete current membership -- the distinct values of ``column`` as
|
|
92
|
+
strings, unioned across batches; empty when the feeder lists nothing.
|
|
93
|
+
|
|
94
|
+
Raises:
|
|
95
|
+
ValueError: ``column`` is absent from a feeder frame (from
|
|
96
|
+
``extract_roster_members``) -- surfaced to the coordinator, whose
|
|
97
|
+
best-effort refresh keeps the existing roster on failure. Null and
|
|
98
|
+
empty-string values do not raise; the extractor filters them loudly.
|
|
99
|
+
FleetpullError: A fetch, validation, or framing failure from the stream.
|
|
100
|
+
|
|
101
|
+
Side Effects:
|
|
102
|
+
Issues the feeder's request chain through ``client`` (network I/O); writes
|
|
103
|
+
nothing.
|
|
104
|
+
"""
|
|
105
|
+
members: set[str] = set()
|
|
106
|
+
for batch in stream_processed_batches(definition, driver, client, None, None):
|
|
107
|
+
members |= extract_roster_members(batch.frame, column)
|
|
108
|
+
return members
|
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
# src/fleetpull/orchestrator/roster_refresh.py
|
|
2
|
+
"""The roster refresh coordinator: make a stale roster's membership current.
|
|
3
|
+
|
|
4
|
+
The coupling rule this module upholds (with the orchestration entry's feeder
|
|
5
|
+
tap): **every execution of a feeder endpoint updates its rosters and records a
|
|
6
|
+
run in the ledger**; parquet is written only when the user asked for that
|
|
7
|
+
endpoint. A coordinator harvest is such an execution -- it fetches, reconciles
|
|
8
|
+
the roster, and records a ``runs`` row for ``(provider, source_endpoint)`` --
|
|
9
|
+
so ``RunLedger.last_success_at`` is a sound staleness key in both directions:
|
|
10
|
+
a harvest is visible to the next staleness check, and a runner-driven feeder
|
|
11
|
+
run (which the entry taps into ``apply_listing``) never leaves the roster
|
|
12
|
+
behind the ledger. A run row certifies execution of the endpoint, not parquet
|
|
13
|
+
freshness.
|
|
14
|
+
|
|
15
|
+
``refresh_if_stale`` is the demand-driven refresh a fan-out consumer calls
|
|
16
|
+
before it reads a roster. Given the roster's ``RosterDefinition`` -- the
|
|
17
|
+
caller resolves the ``RosterKey`` against the roster registry, the same way
|
|
18
|
+
the run executor is handed an already-resolved ``EndpointDefinition`` -- it
|
|
19
|
+
asks the ledger when the feeder last succeeded, and only if a refresh is due
|
|
20
|
+
(``is_roster_stale``, or an empty stored roster, which is stale regardless of
|
|
21
|
+
the ledger verdict -- ledger history may predate the harvest/ledger coupling)
|
|
22
|
+
re-lists the feeder, reconciles the listing against the stored roster, applies
|
|
23
|
+
the delta, and completes the run. ``apply_listing`` is the feeder-tap handoff
|
|
24
|
+
and the reconcile choke point: the entry collects a successful feeder run's
|
|
25
|
+
listed members and hands them here, and the harvest routes its own listing
|
|
26
|
+
through the same method -- so the reconcile guard (a roster is never
|
|
27
|
+
reconciled to empty; an empty listing is a failed refresh, not a membership
|
|
28
|
+
fact) covers both routes once. Staleness gates only whether the coordinator
|
|
29
|
+
*initiates* a harvest, never whether an executed run's non-empty listing is
|
|
30
|
+
applied; the run row on the tap path was already recorded by the run
|
|
31
|
+
executor.
|
|
32
|
+
|
|
33
|
+
The full-listing requirement lives here: a roster needs its feeder's complete
|
|
34
|
+
current membership each refresh, so the feeder must be a snapshot endpoint. The
|
|
35
|
+
coordinator resolves the definition's ``source_endpoint`` to the feeder binding and
|
|
36
|
+
guards that (``ConfigurationError`` on a non-snapshot feeder) before harvesting; the
|
|
37
|
+
harvester itself stays ``sync_mode``-blind.
|
|
38
|
+
|
|
39
|
+
``refresh_if_stale`` is single-flight per roster key (added with the
|
|
40
|
+
intra-provider grain, DESIGN section 7, 2026-07-20): concurrent consumers of
|
|
41
|
+
one stale roster -- both cold-starting with their feeder unselected -- would
|
|
42
|
+
each harvest, wasting quota and recording duplicate ledger snapshot runs.
|
|
43
|
+
The coordinator therefore holds a per-``RosterKey`` lock across the method's
|
|
44
|
+
whole body (an outer lock guards get-or-create of the per-key locks), so the
|
|
45
|
+
second entrant re-runs the freshness check under the lock and returns early
|
|
46
|
+
on the now-fresh roster. Holding the lock across the harvest network call is
|
|
47
|
+
intended -- the waiting consumer needs the membership before it can fan out
|
|
48
|
+
-- and no lock nests inside (the store and ledger are
|
|
49
|
+
connection-per-operation), so there is no deadlock surface. Locks are
|
|
50
|
+
per-key: distinct rosters refresh concurrently.
|
|
51
|
+
|
|
52
|
+
Failure is best-effort with one loud exception. A harvest ``FleetpullError`` (the
|
|
53
|
+
feeder is unreachable, or a page fails to validate) or ``ValueError`` (a missing
|
|
54
|
+
source column) marks the run failed and degrades to the existing
|
|
55
|
+
roster -- a stale verdict is a refresh *attempt*, not a barrier to the fan-out. The
|
|
56
|
+
exception is cold start: when the store holds no members for the key, there is
|
|
57
|
+
nothing to fall back to and a silent empty roster would fan out over nothing, so the
|
|
58
|
+
failure re-raises (still marking the run failed). Wiring errors -- no feeder
|
|
59
|
+
registered for the ``source_endpoint``, a non-snapshot feeder -- raise before any
|
|
60
|
+
run row is opened; they are bugs, not executions. The crash order inside a
|
|
61
|
+
successful refresh mirrors the run executor's output-first discipline:
|
|
62
|
+
``RosterStore.apply`` (one transaction) lands before ``complete_run``, so a crash
|
|
63
|
+
between the two leaves the roster current and the run merely ``running`` -- the
|
|
64
|
+
next check re-harvests idempotently, never the reverse mode where a fresh ledger
|
|
65
|
+
masks a stale roster.
|
|
66
|
+
|
|
67
|
+
Collaborators are injected, not assembled: the pure ``reconcile`` /
|
|
68
|
+
``is_roster_stale`` are called directly; the ``EndpointRegistry`` is the immutable
|
|
69
|
+
catalog passed concrete; the stateful surfaces are narrow Protocols (``RosterAccess``,
|
|
70
|
+
``FeederRunLedger``, and the spine's shared ``ClientSource``) the composition
|
|
71
|
+
root satisfies with the real store, ledger, and client registry.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
import logging
|
|
75
|
+
import threading
|
|
76
|
+
from datetime import datetime
|
|
77
|
+
from functools import partial
|
|
78
|
+
from typing import Protocol
|
|
79
|
+
|
|
80
|
+
from fleetpull.endpoints import EndpointRegistry
|
|
81
|
+
from fleetpull.exceptions import FleetpullError, ProviderResponseError
|
|
82
|
+
from fleetpull.orchestrator.drivers import SingleRequestDriver
|
|
83
|
+
from fleetpull.orchestrator.recording import record_failure_safely
|
|
84
|
+
from fleetpull.orchestrator.roster_harvest import (
|
|
85
|
+
harvest_roster_members,
|
|
86
|
+
require_snapshot_feeder,
|
|
87
|
+
)
|
|
88
|
+
from fleetpull.orchestrator.spine import ClientSource
|
|
89
|
+
from fleetpull.roster import RosterDefinition, RosterKey
|
|
90
|
+
from fleetpull.state import RosterDelta, is_roster_stale, reconcile
|
|
91
|
+
from fleetpull.timing import Clock
|
|
92
|
+
from fleetpull.vocabulary import Provider
|
|
93
|
+
|
|
94
|
+
__all__: list[str] = [
|
|
95
|
+
'FeederRunLedger',
|
|
96
|
+
'RosterAccess',
|
|
97
|
+
'RosterRefreshCoordinator',
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
logger = logging.getLogger(__name__)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class RosterAccess(Protocol):
|
|
104
|
+
"""The roster store surface the coordinator needs (a subset of RosterStore)."""
|
|
105
|
+
|
|
106
|
+
def read_counts(self, key: RosterKey) -> dict[str, int]:
|
|
107
|
+
"""Return the roster as ``{member: absence_count}``."""
|
|
108
|
+
...
|
|
109
|
+
|
|
110
|
+
def apply(self, key: RosterKey, delta: RosterDelta) -> None:
|
|
111
|
+
"""Apply a reconciliation delta in one transaction."""
|
|
112
|
+
...
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class FeederRunLedger(Protocol):
|
|
116
|
+
"""The ledger surface the coordinator needs (a subset of ``RunLedger``).
|
|
117
|
+
|
|
118
|
+
``last_success_at`` keys the staleness decision; the run-recording trio
|
|
119
|
+
makes a coordinator harvest visible to that same key -- every execution of
|
|
120
|
+
a feeder endpoint records a run, so the freshness signal and the freshness
|
|
121
|
+
events cannot diverge.
|
|
122
|
+
"""
|
|
123
|
+
|
|
124
|
+
def last_success_at(self, provider: Provider, endpoint: str) -> datetime | None:
|
|
125
|
+
"""Return the feeder's latest successful run end, or ``None``."""
|
|
126
|
+
...
|
|
127
|
+
|
|
128
|
+
def start_snapshot_run(self, provider: Provider, endpoint: str) -> int:
|
|
129
|
+
"""Open a snapshot run for a harvest and return its id."""
|
|
130
|
+
...
|
|
131
|
+
|
|
132
|
+
def complete_run(self, run_id: int, *, row_count: int) -> None:
|
|
133
|
+
"""Close a run as succeeded with its row count."""
|
|
134
|
+
...
|
|
135
|
+
|
|
136
|
+
def fail_run(self, run_id: int, *, error_detail: str) -> None:
|
|
137
|
+
"""Close a run as failed with an error detail."""
|
|
138
|
+
...
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class RosterRefreshCoordinator:
|
|
142
|
+
"""Refresh a roster's stored membership when its feeder listing has gone stale.
|
|
143
|
+
|
|
144
|
+
Built once with the endpoint catalog and the store / ledger / client surfaces, it
|
|
145
|
+
answers ``refresh_if_stale(definition)``. Beyond the per-key single-flight
|
|
146
|
+
locks (the coordinator is the stateful orchestrator here, so the lock map
|
|
147
|
+
lives on it) it holds no per-refresh state and opens no clients -- the
|
|
148
|
+
injected client source hands it the feeder provider's open client.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
endpoint_registry: Resolves the feeder's ``(provider, name)`` to its binding.
|
|
152
|
+
store: The roster store (read counts, apply a delta).
|
|
153
|
+
ledger: The feeder's ledger surface -- the last-success read for the
|
|
154
|
+
staleness decision, and the run recording a harvest performs.
|
|
155
|
+
client_source: The open transport client per provider.
|
|
156
|
+
clock: The clock supplying ``now`` for the staleness decision.
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
def __init__(
|
|
160
|
+
self,
|
|
161
|
+
endpoint_registry: EndpointRegistry,
|
|
162
|
+
store: RosterAccess,
|
|
163
|
+
ledger: FeederRunLedger,
|
|
164
|
+
client_source: ClientSource,
|
|
165
|
+
clock: Clock,
|
|
166
|
+
) -> None:
|
|
167
|
+
self._endpoint_registry = endpoint_registry
|
|
168
|
+
self._store = store
|
|
169
|
+
self._ledger = ledger
|
|
170
|
+
self._client_source = client_source
|
|
171
|
+
self._clock = clock
|
|
172
|
+
self._key_locks_guard = threading.Lock()
|
|
173
|
+
self._key_locks: dict[RosterKey, threading.Lock] = {}
|
|
174
|
+
|
|
175
|
+
def refresh_if_stale(self, definition: RosterDefinition) -> None:
|
|
176
|
+
"""Re-list the feeder, reconcile the roster, and record the run, if stale.
|
|
177
|
+
|
|
178
|
+
Single-flight per roster key: the whole body runs under the key's
|
|
179
|
+
lock, so of two concurrent consumers hitting the same stale roster,
|
|
180
|
+
exactly one harvests -- the second re-runs the freshness check under
|
|
181
|
+
the lock and returns early on the now-fresh roster. The lock is held
|
|
182
|
+
across the harvest network call deliberately (the waiting consumer
|
|
183
|
+
needs the membership before it can fan out), and distinct keys never
|
|
184
|
+
contend.
|
|
185
|
+
|
|
186
|
+
Returns early when the roster is fresh -- non-empty and inside the
|
|
187
|
+
staleness bound. An empty stored roster is stale regardless of the
|
|
188
|
+
ledger verdict (ledger history may predate the harvest/ledger
|
|
189
|
+
coupling, and a fresh ledger must not mask a roster with nothing to
|
|
190
|
+
fan out over). When a refresh is due, the harvest is an execution of
|
|
191
|
+
the feeder endpoint: it opens a snapshot run, harvests the complete
|
|
192
|
+
membership, applies the reconciliation, and completes the run -- store
|
|
193
|
+
before ledger, so a crash between the two re-harvests rather than
|
|
194
|
+
masking. A harvest failure marks the run failed and degrades to the
|
|
195
|
+
existing roster, unless the store is empty (cold start), where it
|
|
196
|
+
re-raises.
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
definition: The roster to refresh -- its key, feeder ``source_endpoint``
|
|
200
|
+
and ``source_column``, and staleness / eviction policy. The caller
|
|
201
|
+
resolved it from a ``RosterKey`` against the roster registry.
|
|
202
|
+
|
|
203
|
+
Raises:
|
|
204
|
+
ConfigurationError: No feeder is registered for ``source_endpoint``, or the
|
|
205
|
+
resolved feeder is not a snapshot endpoint -- wiring bugs, raised
|
|
206
|
+
loudly before any run row is opened.
|
|
207
|
+
FleetpullError: The harvest failed on a cold start (no existing roster to
|
|
208
|
+
fall back to); the underlying failure propagates.
|
|
209
|
+
ValueError: The harvest failed on a cold start with a missing source
|
|
210
|
+
column; the underlying failure propagates.
|
|
211
|
+
|
|
212
|
+
Side Effects:
|
|
213
|
+
Holds the roster's single-flight lock for the duration. On a due
|
|
214
|
+
refresh: issues the feeder's request chain and records a run in
|
|
215
|
+
the ledger; on success, also writes the reconciled delta to the
|
|
216
|
+
store. Otherwise touches nothing.
|
|
217
|
+
"""
|
|
218
|
+
with self._refresh_lock(definition.key):
|
|
219
|
+
self._refresh_if_due(definition)
|
|
220
|
+
|
|
221
|
+
def _refresh_lock(self, key: RosterKey) -> threading.Lock:
|
|
222
|
+
"""Get or create the roster's single-flight lock.
|
|
223
|
+
|
|
224
|
+
Args:
|
|
225
|
+
key: The roster whose lock to return (hashable frozen identity).
|
|
226
|
+
|
|
227
|
+
Returns:
|
|
228
|
+
The per-key lock, created on first use under the outer guard.
|
|
229
|
+
"""
|
|
230
|
+
with self._key_locks_guard:
|
|
231
|
+
return self._key_locks.setdefault(key, threading.Lock())
|
|
232
|
+
|
|
233
|
+
def _refresh_if_due(self, definition: RosterDefinition) -> None:
|
|
234
|
+
"""The refresh body ``refresh_if_stale`` runs under the key's lock."""
|
|
235
|
+
provider = definition.key.provider
|
|
236
|
+
current = self._store.read_counts(definition.key)
|
|
237
|
+
last_success = self._ledger.last_success_at(
|
|
238
|
+
provider, definition.source_endpoint
|
|
239
|
+
)
|
|
240
|
+
now = self._clock.now_utc()
|
|
241
|
+
if current and not is_roster_stale(last_success, now, definition.max_age):
|
|
242
|
+
return
|
|
243
|
+
feeder = self._endpoint_registry.get(provider, definition.source_endpoint)
|
|
244
|
+
require_snapshot_feeder(feeder, definition.source_endpoint)
|
|
245
|
+
client = self._client_source.client_for(provider)
|
|
246
|
+
logger.info(
|
|
247
|
+
'roster refresh started: provider=%s roster=%s feeder=%s members_held=%d',
|
|
248
|
+
provider.value,
|
|
249
|
+
definition.key.name,
|
|
250
|
+
definition.source_endpoint,
|
|
251
|
+
len(current),
|
|
252
|
+
)
|
|
253
|
+
run_id = self._ledger.start_snapshot_run(provider, definition.source_endpoint)
|
|
254
|
+
try:
|
|
255
|
+
listed = harvest_roster_members(
|
|
256
|
+
feeder, SingleRequestDriver(), client, definition.source_column
|
|
257
|
+
)
|
|
258
|
+
# Routed through the shared reconcile body so its guard (a
|
|
259
|
+
# roster is never reconciled to empty) covers the harvest inside
|
|
260
|
+
# this same failed-refresh path: an empty listing marks the run
|
|
261
|
+
# failed and degrades exactly like a failed HTTP refresh. The
|
|
262
|
+
# key's single-flight lock is already held here, so the locked
|
|
263
|
+
# ``apply_listing`` wrapper would deadlock.
|
|
264
|
+
self._reconcile_listing(definition, listed)
|
|
265
|
+
except (FleetpullError, ValueError) as failure:
|
|
266
|
+
record_failure_safely(
|
|
267
|
+
partial(self._ledger.fail_run, run_id, error_detail=str(failure)),
|
|
268
|
+
f'harvest run {run_id}',
|
|
269
|
+
)
|
|
270
|
+
if not current:
|
|
271
|
+
raise
|
|
272
|
+
logger.warning(
|
|
273
|
+
'roster %s/%s refresh failed; keeping %d existing members: %s',
|
|
274
|
+
provider.value,
|
|
275
|
+
definition.key.name,
|
|
276
|
+
len(current),
|
|
277
|
+
failure,
|
|
278
|
+
)
|
|
279
|
+
return
|
|
280
|
+
# A harvest run's row count is the distinct-member count of the
|
|
281
|
+
# listing -- the rows this execution produced for its consumer.
|
|
282
|
+
self._ledger.complete_run(run_id, row_count=len(listed))
|
|
283
|
+
logger.info(
|
|
284
|
+
'roster refreshed: provider=%s roster=%s members=%d',
|
|
285
|
+
provider.value,
|
|
286
|
+
definition.key.name,
|
|
287
|
+
len(listed),
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
def apply_listing(self, definition: RosterDefinition, listed: set[str]) -> None:
|
|
291
|
+
"""Reconcile an executed feeder run's listed membership into the store.
|
|
292
|
+
|
|
293
|
+
The feeder tap's route into the reconcile choke point
|
|
294
|
+
(``_reconcile_listing``): the orchestration entry hands a successful
|
|
295
|
+
runner-driven feeder run's distinct ``source_column`` values here.
|
|
296
|
+
Runs under the roster key's single-flight lock -- the same lock
|
|
297
|
+
``refresh_if_stale`` holds around its harvest -- so every reconcile
|
|
298
|
+
for one key serializes, whichever route wrote it: two
|
|
299
|
+
read-reconcile-write sequences on one roster can never interleave
|
|
300
|
+
and corrupt absence counts. Records no ledger row: the tap route's
|
|
301
|
+
run was recorded by the run executor.
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
definition: The sourced roster -- its key and eviction policy.
|
|
305
|
+
listed: The feeder run's complete listed membership.
|
|
306
|
+
|
|
307
|
+
Raises:
|
|
308
|
+
ProviderResponseError: ``listed`` is empty (the reconcile guard);
|
|
309
|
+
the store is untouched.
|
|
310
|
+
|
|
311
|
+
Side Effects:
|
|
312
|
+
Holds the roster's single-flight lock for the duration; writes
|
|
313
|
+
the reconciled delta to the store (one transaction).
|
|
314
|
+
"""
|
|
315
|
+
with self._refresh_lock(definition.key):
|
|
316
|
+
self._reconcile_listing(definition, listed)
|
|
317
|
+
|
|
318
|
+
def _reconcile_listing(
|
|
319
|
+
self, definition: RosterDefinition, listed: set[str]
|
|
320
|
+
) -> None:
|
|
321
|
+
"""The reconcile body both write routes run under the key's lock.
|
|
322
|
+
|
|
323
|
+
The reconcile guard lives here so both routes -- the feeder tap
|
|
324
|
+
(``apply_listing``) and the coordinator's own harvest
|
|
325
|
+
(``_refresh_if_due``, which already holds the lock) -- are covered
|
|
326
|
+
once: **a roster is never reconciled to empty**. An empty listing --
|
|
327
|
+
the provider returned nothing, or every record's member value
|
|
328
|
+
filtered out -- is a failed refresh, not a membership fact:
|
|
329
|
+
reconciling it would mass-increment absence counts and, with an
|
|
330
|
+
eviction threshold, evict the entire roster through systematic
|
|
331
|
+
provider garbage. The prior roster stays intact and the caller's
|
|
332
|
+
failed-refresh semantics apply (the harvest degrades exactly like a
|
|
333
|
+
failed HTTP refresh; the tap propagates, failing the endpoint
|
|
334
|
+
loudly). A non-empty listing is applied whole -- staleness gates
|
|
335
|
+
only whether the coordinator *initiates* a harvest, never whether an
|
|
336
|
+
executed run's complete listing is reconciled.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
definition: The sourced roster -- its key and eviction policy.
|
|
340
|
+
listed: The feeder run's complete listed membership.
|
|
341
|
+
|
|
342
|
+
Raises:
|
|
343
|
+
ProviderResponseError: ``listed`` is empty (the reconcile guard);
|
|
344
|
+
the store is untouched.
|
|
345
|
+
|
|
346
|
+
Side Effects:
|
|
347
|
+
Writes the reconciled delta to the store (one transaction).
|
|
348
|
+
"""
|
|
349
|
+
if not listed:
|
|
350
|
+
raise ProviderResponseError(
|
|
351
|
+
provider=definition.key.provider.value,
|
|
352
|
+
endpoint=definition.source_endpoint,
|
|
353
|
+
detail=(
|
|
354
|
+
f'feeder listed no members for roster '
|
|
355
|
+
f'{definition.key.name!r}; a roster is never reconciled '
|
|
356
|
+
f'to empty -- the prior membership stands'
|
|
357
|
+
),
|
|
358
|
+
)
|
|
359
|
+
current = self._store.read_counts(definition.key)
|
|
360
|
+
self._store.apply(
|
|
361
|
+
definition.key,
|
|
362
|
+
reconcile(current, listed, definition.eviction_threshold),
|
|
363
|
+
)
|