fleetpull 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fleetpull/__init__.py +36 -0
- fleetpull/api/__init__.py +32 -0
- fleetpull/api/auth_ingress.py +216 -0
- fleetpull/api/catalog.py +139 -0
- fleetpull/api/fetch.py +229 -0
- fleetpull/api/identity.py +90 -0
- fleetpull/api/sync.py +735 -0
- fleetpull/cli.py +146 -0
- fleetpull/config/__init__.py +47 -0
- fleetpull/config/base.py +18 -0
- fleetpull/config/example.py +90 -0
- fleetpull/config/geotab.py +78 -0
- fleetpull/config/http.py +32 -0
- fleetpull/config/loading.py +195 -0
- fleetpull/config/logger.py +139 -0
- fleetpull/config/providers.py +520 -0
- fleetpull/config/rate_limit.py +50 -0
- fleetpull/config/resolution.py +156 -0
- fleetpull/config/retry.py +69 -0
- fleetpull/config/root.py +157 -0
- fleetpull/config/sections.py +146 -0
- fleetpull/endpoints/__init__.py +24 -0
- fleetpull/endpoints/geotab/__init__.py +12 -0
- fleetpull/endpoints/geotab/_requests.py +427 -0
- fleetpull/endpoints/geotab/annotation_logs.py +73 -0
- fleetpull/endpoints/geotab/audits.py +70 -0
- fleetpull/endpoints/geotab/devices.py +79 -0
- fleetpull/endpoints/geotab/driver_changes.py +73 -0
- fleetpull/endpoints/geotab/duty_status_logs.py +75 -0
- fleetpull/endpoints/geotab/dvir_logs.py +76 -0
- fleetpull/endpoints/geotab/exception_events.py +119 -0
- fleetpull/endpoints/geotab/fault_data.py +72 -0
- fleetpull/endpoints/geotab/fill_ups.py +79 -0
- fleetpull/endpoints/geotab/fuel_and_energy_used.py +74 -0
- fleetpull/endpoints/geotab/fuel_tax_details.py +75 -0
- fleetpull/endpoints/geotab/log_records.py +74 -0
- fleetpull/endpoints/geotab/media_files.py +74 -0
- fleetpull/endpoints/geotab/shipment_logs.py +71 -0
- fleetpull/endpoints/geotab/status_data.py +71 -0
- fleetpull/endpoints/geotab/text_messages.py +77 -0
- fleetpull/endpoints/geotab/trips.py +102 -0
- fleetpull/endpoints/geotab/users.py +81 -0
- fleetpull/endpoints/motive/__init__.py +12 -0
- fleetpull/endpoints/motive/_spec_builders.py +94 -0
- fleetpull/endpoints/motive/driver_idle_rollups.py +113 -0
- fleetpull/endpoints/motive/driving_periods.py +85 -0
- fleetpull/endpoints/motive/groups.py +61 -0
- fleetpull/endpoints/motive/idle_events.py +94 -0
- fleetpull/endpoints/motive/users.py +64 -0
- fleetpull/endpoints/motive/vehicle_locations.py +170 -0
- fleetpull/endpoints/motive/vehicle_utilizations.py +122 -0
- fleetpull/endpoints/motive/vehicles.py +106 -0
- fleetpull/endpoints/registry.py +280 -0
- fleetpull/endpoints/samsara/__init__.py +12 -0
- fleetpull/endpoints/samsara/_spec_builders.py +202 -0
- fleetpull/endpoints/samsara/addresses.py +77 -0
- fleetpull/endpoints/samsara/asset_locations.py +217 -0
- fleetpull/endpoints/samsara/driver_fuel_energy_reports.py +124 -0
- fleetpull/endpoints/samsara/driver_vehicle_assignments.py +211 -0
- fleetpull/endpoints/samsara/drivers.py +169 -0
- fleetpull/endpoints/samsara/engine_states.py +114 -0
- fleetpull/endpoints/samsara/gps_readings.py +113 -0
- fleetpull/endpoints/samsara/idling_events.py +195 -0
- fleetpull/endpoints/samsara/odometer_readings.py +116 -0
- fleetpull/endpoints/samsara/trips.py +217 -0
- fleetpull/endpoints/samsara/vehicle_fuel_energy_reports.py +139 -0
- fleetpull/endpoints/samsara/vehicles.py +121 -0
- fleetpull/endpoints/shared/__init__.py +57 -0
- fleetpull/endpoints/shared/base.py +529 -0
- fleetpull/endpoints/shared/request_shape.py +227 -0
- fleetpull/endpoints/shared/resume.py +74 -0
- fleetpull/endpoints/shared/spec_builders.py +59 -0
- fleetpull/endpoints/shared/sync_mode.py +159 -0
- fleetpull/endpoints/shared/url_paths.py +149 -0
- fleetpull/exceptions.py +320 -0
- fleetpull/incremental/__init__.py +27 -0
- fleetpull/incremental/cursor.py +66 -0
- fleetpull/incremental/resolution.py +152 -0
- fleetpull/incremental/seed.py +53 -0
- fleetpull/incremental/window.py +94 -0
- fleetpull/logger/__init__.py +5 -0
- fleetpull/logger/setup.py +145 -0
- fleetpull/model_contract/__init__.py +6 -0
- fleetpull/model_contract/coercions.py +33 -0
- fleetpull/model_contract/response.py +45 -0
- fleetpull/models/__init__.py +7 -0
- fleetpull/models/geotab/__init__.py +166 -0
- fleetpull/models/geotab/annotation_log.py +109 -0
- fleetpull/models/geotab/audit.py +56 -0
- fleetpull/models/geotab/device.py +217 -0
- fleetpull/models/geotab/driver_change.py +102 -0
- fleetpull/models/geotab/duty_status_log.py +200 -0
- fleetpull/models/geotab/dvir_log.py +184 -0
- fleetpull/models/geotab/exception_event.py +135 -0
- fleetpull/models/geotab/fault_data.py +168 -0
- fleetpull/models/geotab/fill_up.py +189 -0
- fleetpull/models/geotab/fuel_and_energy_used.py +81 -0
- fleetpull/models/geotab/fuel_tax_detail.py +150 -0
- fleetpull/models/geotab/log_record.py +68 -0
- fleetpull/models/geotab/media_file.py +133 -0
- fleetpull/models/geotab/shared.py +221 -0
- fleetpull/models/geotab/shipment_log.py +105 -0
- fleetpull/models/geotab/status_data.py +108 -0
- fleetpull/models/geotab/text_message.py +125 -0
- fleetpull/models/geotab/trip.py +162 -0
- fleetpull/models/geotab/user.py +187 -0
- fleetpull/models/motive/__init__.py +43 -0
- fleetpull/models/motive/driver_idle_rollup.py +109 -0
- fleetpull/models/motive/driving_period.py +82 -0
- fleetpull/models/motive/group.py +49 -0
- fleetpull/models/motive/idle_event.py +77 -0
- fleetpull/models/motive/shared.py +192 -0
- fleetpull/models/motive/user.py +217 -0
- fleetpull/models/motive/vehicle.py +162 -0
- fleetpull/models/motive/vehicle_location.py +127 -0
- fleetpull/models/motive/vehicle_utilization.py +128 -0
- fleetpull/models/samsara/__init__.py +108 -0
- fleetpull/models/samsara/address.py +149 -0
- fleetpull/models/samsara/asset_location.py +131 -0
- fleetpull/models/samsara/driver.py +232 -0
- fleetpull/models/samsara/driver_fuel_energy_report.py +153 -0
- fleetpull/models/samsara/driver_vehicle_assignment.py +168 -0
- fleetpull/models/samsara/engine_state.py +92 -0
- fleetpull/models/samsara/gps_reading.py +146 -0
- fleetpull/models/samsara/idling_event.py +174 -0
- fleetpull/models/samsara/odometer_reading.py +90 -0
- fleetpull/models/samsara/trip.py +199 -0
- fleetpull/models/samsara/vehicle.py +167 -0
- fleetpull/models/samsara/vehicle_fuel_energy_report.py +191 -0
- fleetpull/network/__init__.py +9 -0
- fleetpull/network/auth/__init__.py +15 -0
- fleetpull/network/auth/authenticate.py +271 -0
- fleetpull/network/auth/manager.py +223 -0
- fleetpull/network/auth/models.py +57 -0
- fleetpull/network/auth/strategies.py +169 -0
- fleetpull/network/classifiers/__init__.py +11 -0
- fleetpull/network/classifiers/geotab.py +145 -0
- fleetpull/network/classifiers/motive.py +79 -0
- fleetpull/network/classifiers/samsara.py +80 -0
- fleetpull/network/client/__init__.py +25 -0
- fleetpull/network/client/page.py +32 -0
- fleetpull/network/client/profile.py +27 -0
- fleetpull/network/client/registry.py +103 -0
- fleetpull/network/client/registry_base.py +150 -0
- fleetpull/network/client/runtime.py +44 -0
- fleetpull/network/client/transport.py +381 -0
- fleetpull/network/contract/__init__.py +50 -0
- fleetpull/network/contract/auth.py +48 -0
- fleetpull/network/contract/classifier.py +189 -0
- fleetpull/network/contract/envelope_fetcher.py +33 -0
- fleetpull/network/contract/envelopes.py +189 -0
- fleetpull/network/contract/outcome.py +43 -0
- fleetpull/network/contract/page_decoder.py +101 -0
- fleetpull/network/contract/request.py +97 -0
- fleetpull/network/decoders/__init__.py +40 -0
- fleetpull/network/decoders/_window_stamp.py +78 -0
- fleetpull/network/decoders/geotab.py +309 -0
- fleetpull/network/decoders/motive.py +204 -0
- fleetpull/network/decoders/motive_reports.py +114 -0
- fleetpull/network/decoders/samsara.py +343 -0
- fleetpull/network/decoders/samsara_reports.py +119 -0
- fleetpull/network/decoders/single_page.py +55 -0
- fleetpull/network/limits/__init__.py +13 -0
- fleetpull/network/limits/bucket_math.py +58 -0
- fleetpull/network/limits/limiter.py +153 -0
- fleetpull/network/limits/registry.py +96 -0
- fleetpull/network/posture/__init__.py +6 -0
- fleetpull/network/posture/client_options.py +96 -0
- fleetpull/network/retry/__init__.py +13 -0
- fleetpull/network/retry/decision.py +155 -0
- fleetpull/network/tls/__init__.py +5 -0
- fleetpull/network/tls/truststore_context.py +35 -0
- fleetpull/orchestrator/__init__.py +30 -0
- fleetpull/orchestrator/backfill.py +127 -0
- fleetpull/orchestrator/batch.py +155 -0
- fleetpull/orchestrator/bisection.py +271 -0
- fleetpull/orchestrator/drivers.py +350 -0
- fleetpull/orchestrator/entry.py +347 -0
- fleetpull/orchestrator/executors.py +128 -0
- fleetpull/orchestrator/fanout.py +163 -0
- fleetpull/orchestrator/feed_drive.py +266 -0
- fleetpull/orchestrator/metadata_projection.py +195 -0
- fleetpull/orchestrator/outcome.py +53 -0
- fleetpull/orchestrator/recording.py +85 -0
- fleetpull/orchestrator/resume.py +143 -0
- fleetpull/orchestrator/roster_harvest.py +108 -0
- fleetpull/orchestrator/roster_refresh.py +363 -0
- fleetpull/orchestrator/runner.py +262 -0
- fleetpull/orchestrator/shape_resolution.py +221 -0
- fleetpull/orchestrator/spine.py +166 -0
- fleetpull/orchestrator/streaming.py +153 -0
- fleetpull/orchestrator/unit_loop.py +273 -0
- fleetpull/orchestrator/watermark_drive.py +455 -0
- fleetpull/paths/__init__.py +13 -0
- fleetpull/paths/datasets.py +38 -0
- fleetpull/paths/partitions.py +41 -0
- fleetpull/paths/resolution.py +98 -0
- fleetpull/polars_typing/__init__.py +20 -0
- fleetpull/py.typed +0 -0
- fleetpull/records/__init__.py +19 -0
- fleetpull/records/convert.py +53 -0
- fleetpull/records/dataframe.py +63 -0
- fleetpull/records/event_time.py +70 -0
- fleetpull/records/fields.py +190 -0
- fleetpull/records/flatten.py +54 -0
- fleetpull/records/roster_members.py +74 -0
- fleetpull/records/schema.py +73 -0
- fleetpull/records/validation.py +62 -0
- fleetpull/resources/__init__.py +10 -0
- fleetpull/resources/config.example.yaml +222 -0
- fleetpull/roster/__init__.py +16 -0
- fleetpull/roster/definition.py +46 -0
- fleetpull/roster/key.py +40 -0
- fleetpull/roster/registry.py +93 -0
- fleetpull/state/__init__.py +36 -0
- fleetpull/state/cursors.py +417 -0
- fleetpull/state/database.py +463 -0
- fleetpull/state/migrations.py +430 -0
- fleetpull/state/reconcile.py +127 -0
- fleetpull/state/rosters.py +159 -0
- fleetpull/state/run_ledger.py +572 -0
- fleetpull/state/work_units.py +648 -0
- fleetpull/storage/__init__.py +30 -0
- fleetpull/storage/append.py +187 -0
- fleetpull/storage/atomic.py +161 -0
- fleetpull/storage/files.py +176 -0
- fleetpull/storage/frames.py +98 -0
- fleetpull/storage/metadata.py +161 -0
- fleetpull/storage/partitioned.py +205 -0
- fleetpull/storage/pruning.py +169 -0
- fleetpull/storage/read.py +35 -0
- fleetpull/storage/result.py +34 -0
- fleetpull/storage/single_file.py +130 -0
- fleetpull/storage/splitting.py +88 -0
- fleetpull/storage/staging.py +178 -0
- fleetpull/storage/writers.py +145 -0
- fleetpull/timing/__init__.py +21 -0
- fleetpull/timing/canon.py +92 -0
- fleetpull/timing/clock.py +192 -0
- fleetpull/timing/codec.py +122 -0
- fleetpull/timing/sleeper.py +65 -0
- fleetpull/vocabulary/__init__.py +16 -0
- fleetpull/vocabulary/json_types.py +20 -0
- fleetpull/vocabulary/provider.py +29 -0
- fleetpull/vocabulary/quota_scope.py +47 -0
- fleetpull/vocabulary/response_category.py +32 -0
- fleetpull-0.1.0.dist-info/METADATA +248 -0
- fleetpull-0.1.0.dist-info/RECORD +252 -0
- fleetpull-0.1.0.dist-info/WHEEL +5 -0
- fleetpull-0.1.0.dist-info/entry_points.txt +2 -0
- fleetpull-0.1.0.dist-info/licenses/LICENSE +202 -0
- fleetpull-0.1.0.dist-info/top_level.txt +1 -0
fleetpull/exceptions.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
# src/fleetpull/exceptions.py
|
|
2
|
+
"""The fleetpull exception hierarchy: operational errors consumers catch.
|
|
3
|
+
|
|
4
|
+
The hierarchy mirrors the response-classification vocabulary
|
|
5
|
+
(``ResponseCategory``) and inherits its closure invariant: a new
|
|
6
|
+
exception type is admissible only if it demands a distinct consumer
|
|
7
|
+
action. Programming errors stay stdlib ``ValueError`` /
|
|
8
|
+
``RuntimeError`` — a hierarchy that absorbs caller bugs invites broad
|
|
9
|
+
``except`` clauses that silence them.
|
|
10
|
+
|
|
11
|
+
Every member is a plain data carrier: typed fields for programmatic
|
|
12
|
+
handling, a composed human message for ``str()``. Instances never
|
|
13
|
+
carry raw request or response material (headers, bodies, request
|
|
14
|
+
specs, credentials-adjacent values) — every instance is safe to log.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
|
|
19
|
+
from fleetpull.vocabulary import ResponseCategory
|
|
20
|
+
|
|
21
|
+
__all__: list[str] = [
|
|
22
|
+
'AuthenticationError',
|
|
23
|
+
'ConfigurationError',
|
|
24
|
+
'EndpointFailure',
|
|
25
|
+
'FleetpullError',
|
|
26
|
+
'ProviderResponseError',
|
|
27
|
+
'RetriesExhaustedError',
|
|
28
|
+
'SyncFailuresError',
|
|
29
|
+
'UnknownQuotaScopeError',
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _compose_message(
|
|
34
|
+
head: str,
|
|
35
|
+
provider: str | None,
|
|
36
|
+
endpoint: str | None,
|
|
37
|
+
detail: str | None,
|
|
38
|
+
) -> str:
|
|
39
|
+
"""
|
|
40
|
+
Compose the human-readable message every hierarchy member carries.
|
|
41
|
+
|
|
42
|
+
Single composition point so all members read uniformly:
|
|
43
|
+
``<head> [provider=..., endpoint=...]: <detail>``, with absent
|
|
44
|
+
fields omitted.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
head: Summary phrase supplied by the concrete class.
|
|
48
|
+
provider: Provider name, when known.
|
|
49
|
+
endpoint: Endpoint name, when known.
|
|
50
|
+
detail: Human-readable context; programmatic handling never
|
|
51
|
+
reads it.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
The composed message.
|
|
55
|
+
"""
|
|
56
|
+
message: str = head
|
|
57
|
+
context_parts: list[str] = []
|
|
58
|
+
if provider is not None:
|
|
59
|
+
context_parts.append(f'provider={provider}')
|
|
60
|
+
if endpoint is not None:
|
|
61
|
+
context_parts.append(f'endpoint={endpoint}')
|
|
62
|
+
if context_parts:
|
|
63
|
+
message = f'{message} [{", ".join(context_parts)}]'
|
|
64
|
+
if detail is not None:
|
|
65
|
+
message = f'{message}: {detail}'
|
|
66
|
+
return message
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class FleetpullError(Exception):
|
|
70
|
+
"""
|
|
71
|
+
Root of the hierarchy. Never raised directly — raise the concrete
|
|
72
|
+
member whose consumer action matches. Catching this means "any
|
|
73
|
+
operational fleetpull failure."
|
|
74
|
+
|
|
75
|
+
Attributes:
|
|
76
|
+
provider: Provider name, when known.
|
|
77
|
+
endpoint: Endpoint name, when known.
|
|
78
|
+
detail: Human-readable context for the failure. Programmatic
|
|
79
|
+
handling never reads it.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
provider: str | None
|
|
83
|
+
endpoint: str | None
|
|
84
|
+
detail: str | None
|
|
85
|
+
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
message: str,
|
|
89
|
+
*,
|
|
90
|
+
provider: str | None = None,
|
|
91
|
+
endpoint: str | None = None,
|
|
92
|
+
detail: str | None = None,
|
|
93
|
+
) -> None:
|
|
94
|
+
"""
|
|
95
|
+
Store the shared fields and compose the ``str()`` message.
|
|
96
|
+
|
|
97
|
+
Keyword-only fields keep raise sites self-describing. The
|
|
98
|
+
signature breaks ``BaseException``'s default pickle round-trip
|
|
99
|
+
(which replays positional ``args``); pickling is deliberately
|
|
100
|
+
unsupported — fleetpull concurrency is threads, and exceptions
|
|
101
|
+
never cross a process boundary in this package.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
message: Summary phrase; the concrete class supplies it.
|
|
105
|
+
provider: Provider name, when known.
|
|
106
|
+
endpoint: Endpoint name, when known.
|
|
107
|
+
detail: Human-readable context; never read programmatically.
|
|
108
|
+
"""
|
|
109
|
+
super().__init__(_compose_message(message, provider, endpoint, detail))
|
|
110
|
+
self.provider = provider
|
|
111
|
+
self.endpoint = endpoint
|
|
112
|
+
self.detail = detail
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class ConfigurationError(FleetpullError):
|
|
116
|
+
"""
|
|
117
|
+
Local configuration or wiring is wrong; fix it before rerunning.
|
|
118
|
+
|
|
119
|
+
The message is caller-supplied because configuration failures are
|
|
120
|
+
bespoke by nature. Inherits the base ``__init__`` unchanged.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class UnknownQuotaScopeError(ConfigurationError):
|
|
125
|
+
"""
|
|
126
|
+
A request named a quota scope the registry was never configured
|
|
127
|
+
with — a configuration bug. No defaults, no fallbacks.
|
|
128
|
+
|
|
129
|
+
Attributes:
|
|
130
|
+
scope: The unconfigured quota scope.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
scope: str
|
|
134
|
+
|
|
135
|
+
def __init__(self, scope: str, *, detail: str | None = None) -> None:
|
|
136
|
+
"""
|
|
137
|
+
Compose the message from the offending scope.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
scope: The unconfigured quota scope.
|
|
141
|
+
detail: Human-readable context (e.g. the configured
|
|
142
|
+
scopes); never read programmatically.
|
|
143
|
+
"""
|
|
144
|
+
super().__init__(f'unknown quota scope: {scope!r}', detail=detail)
|
|
145
|
+
self.scope = scope
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class AuthenticationError(FleetpullError):
|
|
149
|
+
"""
|
|
150
|
+
Authentication failed fatally: bad credentials or revoked access.
|
|
151
|
+
Rerunning without fixing credentials cannot succeed.
|
|
152
|
+
|
|
153
|
+
Raised when the one-retry-per-attempt-sequence auth path fails a
|
|
154
|
+
second time, or when the authentication call itself is rejected.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
def __init__(
|
|
158
|
+
self,
|
|
159
|
+
*,
|
|
160
|
+
provider: str | None = None,
|
|
161
|
+
endpoint: str | None = None,
|
|
162
|
+
detail: str | None = None,
|
|
163
|
+
) -> None:
|
|
164
|
+
"""
|
|
165
|
+
Compose the fixed head with the shared context fields.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
provider: Provider name, when known.
|
|
169
|
+
endpoint: Endpoint name, when known.
|
|
170
|
+
detail: Human-readable context; never read programmatically.
|
|
171
|
+
"""
|
|
172
|
+
super().__init__(
|
|
173
|
+
'authentication failed',
|
|
174
|
+
provider=provider,
|
|
175
|
+
endpoint=endpoint,
|
|
176
|
+
detail=detail,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class ProviderResponseError(FleetpullError):
|
|
181
|
+
"""
|
|
182
|
+
The provider returned a response that fleetpull classified as
|
|
183
|
+
non-retryable or contract-violating.
|
|
184
|
+
|
|
185
|
+
Covers both definitive refusals and structurally unusable
|
|
186
|
+
responses: malformed envelopes, bodies that violate the provider
|
|
187
|
+
contract, unrecognized failure types, unexpected status codes.
|
|
188
|
+
|
|
189
|
+
Attributes:
|
|
190
|
+
status_code: HTTP status of the offending response, when one
|
|
191
|
+
exists. A ``category`` field is deliberately absent: every
|
|
192
|
+
raise of this class corresponds to the FATAL
|
|
193
|
+
classification, so the field would carry no information.
|
|
194
|
+
"""
|
|
195
|
+
|
|
196
|
+
status_code: int | None
|
|
197
|
+
|
|
198
|
+
def __init__(
|
|
199
|
+
self,
|
|
200
|
+
*,
|
|
201
|
+
provider: str | None = None,
|
|
202
|
+
endpoint: str | None = None,
|
|
203
|
+
detail: str | None = None,
|
|
204
|
+
status_code: int | None = None,
|
|
205
|
+
) -> None:
|
|
206
|
+
"""
|
|
207
|
+
Compose the head, folding in the status code when present.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
provider: Provider name, when known.
|
|
211
|
+
endpoint: Endpoint name, when known.
|
|
212
|
+
detail: Human-readable context; never read programmatically.
|
|
213
|
+
status_code: HTTP status of the offending response, when
|
|
214
|
+
one exists.
|
|
215
|
+
"""
|
|
216
|
+
head: str = 'non-retryable provider response'
|
|
217
|
+
if status_code is not None:
|
|
218
|
+
head = f'{head} (HTTP {status_code})'
|
|
219
|
+
super().__init__(head, provider=provider, endpoint=endpoint, detail=detail)
|
|
220
|
+
self.status_code = status_code
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class RetriesExhaustedError(FleetpullError):
|
|
224
|
+
"""
|
|
225
|
+
A retryable classification kept recurring until its attempt budget
|
|
226
|
+
ran out. Rerunning later is reasonable.
|
|
227
|
+
|
|
228
|
+
A ``status_code`` field is deliberately absent, mirroring the
|
|
229
|
+
``ProviderResponseError`` precedent: ``category`` subsumes the raw
|
|
230
|
+
status's programmatic value, and the final attempt's specifics are
|
|
231
|
+
diagnostics — ``detail``'s job, folded in by the raise site.
|
|
232
|
+
|
|
233
|
+
Attributes:
|
|
234
|
+
category: The classification that exhausted its budget
|
|
235
|
+
(TRANSIENT or RATE_LIMITED).
|
|
236
|
+
attempt_count: Attempts made before giving up.
|
|
237
|
+
"""
|
|
238
|
+
|
|
239
|
+
category: ResponseCategory | None
|
|
240
|
+
attempt_count: int | None
|
|
241
|
+
|
|
242
|
+
def __init__(
|
|
243
|
+
self,
|
|
244
|
+
*,
|
|
245
|
+
provider: str | None = None,
|
|
246
|
+
endpoint: str | None = None,
|
|
247
|
+
detail: str | None = None,
|
|
248
|
+
category: ResponseCategory | None = None,
|
|
249
|
+
attempt_count: int | None = None,
|
|
250
|
+
) -> None:
|
|
251
|
+
"""
|
|
252
|
+
Compose the head from the attempt count and category.
|
|
253
|
+
|
|
254
|
+
Args:
|
|
255
|
+
provider: Provider name, when known.
|
|
256
|
+
endpoint: Endpoint name, when known.
|
|
257
|
+
detail: Human-readable context; never read programmatically.
|
|
258
|
+
category: The classification that exhausted its budget.
|
|
259
|
+
attempt_count: Attempts made before giving up.
|
|
260
|
+
"""
|
|
261
|
+
head: str = 'retry budget exhausted'
|
|
262
|
+
if attempt_count is not None:
|
|
263
|
+
head = f'{head} after {attempt_count} attempts'
|
|
264
|
+
if category is not None:
|
|
265
|
+
head = f'{head} ({category} responses)'
|
|
266
|
+
super().__init__(head, provider=provider, endpoint=endpoint, detail=detail)
|
|
267
|
+
self.category = category
|
|
268
|
+
self.attempt_count = attempt_count
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@dataclass(frozen=True, slots=True)
|
|
272
|
+
class EndpointFailure:
|
|
273
|
+
"""One endpoint's failure inside a sync run, as the aggregate carries it.
|
|
274
|
+
|
|
275
|
+
Attributes:
|
|
276
|
+
provider: The failed endpoint's provider name (the lowercase value).
|
|
277
|
+
endpoint: The failed endpoint's name.
|
|
278
|
+
error: The caught operational failure, unmodified.
|
|
279
|
+
"""
|
|
280
|
+
|
|
281
|
+
provider: str
|
|
282
|
+
endpoint: str
|
|
283
|
+
error: FleetpullError
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class SyncFailuresError(FleetpullError):
|
|
287
|
+
"""
|
|
288
|
+
One or more endpoints failed inside a sync run that let its
|
|
289
|
+
siblings continue. Raised after every selected endpoint has run;
|
|
290
|
+
each carried failure is itself a public-family exception.
|
|
291
|
+
|
|
292
|
+
Deliberately not an ``ExceptionGroup``: the documented catch
|
|
293
|
+
contract stays ``except FleetpullError:``, which a group would
|
|
294
|
+
silently bypass.
|
|
295
|
+
|
|
296
|
+
Attributes:
|
|
297
|
+
failures: The per-endpoint failures, in queue order within each
|
|
298
|
+
provider -- feeders then consumers, config order within each
|
|
299
|
+
-- providers in config order (a sync runs one staged queue
|
|
300
|
+
per provider, the queues concurrent).
|
|
301
|
+
"""
|
|
302
|
+
|
|
303
|
+
failures: tuple[EndpointFailure, ...]
|
|
304
|
+
|
|
305
|
+
def __init__(self, failures: tuple[EndpointFailure, ...]) -> None:
|
|
306
|
+
"""
|
|
307
|
+
Compose the message from the failed endpoints.
|
|
308
|
+
|
|
309
|
+
Args:
|
|
310
|
+
failures: The per-endpoint failures, in queue order within
|
|
311
|
+
each provider, providers in config order.
|
|
312
|
+
"""
|
|
313
|
+
failed_names = ', '.join(
|
|
314
|
+
f'{failure.provider}.{failure.endpoint}' for failure in failures
|
|
315
|
+
)
|
|
316
|
+
super().__init__(
|
|
317
|
+
f'sync failed for {len(failures)} endpoint(s)',
|
|
318
|
+
detail=failed_names,
|
|
319
|
+
)
|
|
320
|
+
self.failures = failures
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# src/fleetpull/incremental/__init__.py
|
|
2
|
+
"""Per-endpoint incremental resume state: the cursors, the resume values (the watermark window and the feed seed), and the pure functions that resolve them."""
|
|
3
|
+
|
|
4
|
+
from fleetpull.incremental.cursor import (
|
|
5
|
+
DateWatermark,
|
|
6
|
+
FeedToken,
|
|
7
|
+
IncrementalCursor,
|
|
8
|
+
)
|
|
9
|
+
from fleetpull.incremental.resolution import (
|
|
10
|
+
resolve_resume_start,
|
|
11
|
+
resolve_trailing_edge,
|
|
12
|
+
window_or_none,
|
|
13
|
+
)
|
|
14
|
+
from fleetpull.incremental.seed import FeedResume, FeedSeed
|
|
15
|
+
from fleetpull.incremental.window import DateWindow
|
|
16
|
+
|
|
17
|
+
__all__: list[str] = [
|
|
18
|
+
'DateWatermark',
|
|
19
|
+
'DateWindow',
|
|
20
|
+
'FeedResume',
|
|
21
|
+
'FeedSeed',
|
|
22
|
+
'FeedToken',
|
|
23
|
+
'IncrementalCursor',
|
|
24
|
+
'resolve_resume_start',
|
|
25
|
+
'resolve_trailing_edge',
|
|
26
|
+
'window_or_none',
|
|
27
|
+
]
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# src/fleetpull/incremental/cursor.py
|
|
2
|
+
"""Per-endpoint incremental resume state: the cursor a fetch resumes from.
|
|
3
|
+
|
|
4
|
+
A pure, dependency-free leaf: ``DateWatermark`` imports only stdlib
|
|
5
|
+
``datetime``; ``FeedToken`` imports nothing. The two members are the closed
|
|
6
|
+
set of incremental strategies (DESIGN §4):
|
|
7
|
+
|
|
8
|
+
- ``DateWatermark`` (Motive/Samsara) — the maximum event timestamp seen.
|
|
9
|
+
The next fetch resumes from ``watermark - lookback`` (the orchestrator's
|
|
10
|
+
arithmetic, not this module's).
|
|
11
|
+
- ``FeedToken`` (GeoTab GetFeed) — the opaque ``toVersion`` cursor, sent
|
|
12
|
+
back as ``fromVersion``. fleetpull never interprets it.
|
|
13
|
+
|
|
14
|
+
Pure data, no behavior. Resume arithmetic is the orchestrator's,
|
|
15
|
+
resume-param construction is the endpoint's, and serialization is the state
|
|
16
|
+
layer's (it owns the SQLite representation and calls ``timing`` to turn a
|
|
17
|
+
watermark into ISO text) — keeping all three out of here is what lets the
|
|
18
|
+
cursor import nothing internal.
|
|
19
|
+
|
|
20
|
+
An endpoint with no incremental state has no cursor — absence, handled at the
|
|
21
|
+
endpoint/orchestrator layer, not a third union member. The set is exactly
|
|
22
|
+
two.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from datetime import datetime
|
|
27
|
+
|
|
28
|
+
__all__: list[str] = [
|
|
29
|
+
'DateWatermark',
|
|
30
|
+
'FeedToken',
|
|
31
|
+
'IncrementalCursor',
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True, slots=True)
|
|
36
|
+
class DateWatermark:
|
|
37
|
+
"""
|
|
38
|
+
Date-windowed resume state (Motive/Samsara).
|
|
39
|
+
|
|
40
|
+
Attributes:
|
|
41
|
+
watermark: The maximum event timestamp seen in the data fetched so
|
|
42
|
+
far, timezone-aware UTC. The next fetch resumes from
|
|
43
|
+
``watermark - lookback``; that arithmetic belongs to the
|
|
44
|
+
orchestrator, and UTC validity is enforced at the codec
|
|
45
|
+
serialization boundary, so this carrier neither computes nor
|
|
46
|
+
re-checks.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
watermark: datetime
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True, slots=True)
|
|
53
|
+
class FeedToken:
|
|
54
|
+
"""
|
|
55
|
+
Feed-cursor resume state (GeoTab GetFeed).
|
|
56
|
+
|
|
57
|
+
Attributes:
|
|
58
|
+
from_version: The opaque version cursor — GeoTab's ``toVersion`` from
|
|
59
|
+
the last page, sent back as ``fromVersion`` to resume. fleetpull
|
|
60
|
+
never parses or orders it; it is a token, not a timestamp.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
from_version: str
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
type IncrementalCursor = DateWatermark | FeedToken
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# src/fleetpull/incremental/resolution.py
|
|
2
|
+
"""Pure window-resolution helpers: the stateless pieces that compute a run's window.
|
|
3
|
+
|
|
4
|
+
The three pure functions a watermark run's window is built from, composed by the
|
|
5
|
+
(stateful) orchestrator -- which does the SQLite reads and the clock read and feeds
|
|
6
|
+
the values in. Each is single-concern and individually testable; none touches a
|
|
7
|
+
clock, a cursor store, a run ledger, or any I/O.
|
|
8
|
+
|
|
9
|
+
- ``resolve_trailing_edge`` -- the window's end: the current UTC day floored to
|
|
10
|
+
midnight, held back by the configured cutoff.
|
|
11
|
+
- ``resolve_resume_start`` -- the window's start, by the resume precedence
|
|
12
|
+
(DESIGN section 4): the watermark-derived start if one exists, else the
|
|
13
|
+
coverage frontier, else the cold-start default -- the chosen arm floored
|
|
14
|
+
to the UTC midnight of its date.
|
|
15
|
+
- ``window_or_none`` -- the assembly: a ``DateWindow`` when ``start < end``, else
|
|
16
|
+
``None`` (caught up -- no work this run, not an error).
|
|
17
|
+
|
|
18
|
+
Together they are the watermark resume mechanism (DESIGN section 4): the resolver
|
|
19
|
+
day-aligns both bounds -- the start floored to its UTC midnight, the trailing edge
|
|
20
|
+
floored and held back -- and returns ``None`` for a caught-up window
|
|
21
|
+
(``start >= end``) rather than raising, so a watermark sitting inside the
|
|
22
|
+
still-arriving day is a "no work this run" verdict, not an error. Day alignment
|
|
23
|
+
is the floored-window invariant the date-partitioned writers document: requests
|
|
24
|
+
and partitions are day-granular, so every covered date must be refetched in
|
|
25
|
+
full for wholesale partition replacement and the prune to be safe.
|
|
26
|
+
|
|
27
|
+
The start candidates are pre-resolved datetimes, not cursors: the orchestrator
|
|
28
|
+
extracts the watermark moment and subtracts the lookback (arm 1's one line) before
|
|
29
|
+
calling, so this module stays pure datetime math with no cursor dependency. The
|
|
30
|
+
cutoff is expected to be a whole-day ``timedelta`` (its only source is
|
|
31
|
+
``timedelta(days=cutoff_days)``), which keeps the trailing edge date-aligned -- the
|
|
32
|
+
partition-wholeness invariant storage relies on. UTC validity is not checked here;
|
|
33
|
+
the bounds cross the codec boundary when a spec-builder serializes them, and that
|
|
34
|
+
boundary raises on naive/non-UTC, exactly as for ``DateWindow``.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from datetime import UTC, datetime, time, timedelta
|
|
38
|
+
|
|
39
|
+
from fleetpull.incremental.window import DateWindow
|
|
40
|
+
|
|
41
|
+
__all__: list[str] = [
|
|
42
|
+
'resolve_resume_start',
|
|
43
|
+
'resolve_trailing_edge',
|
|
44
|
+
'window_or_none',
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _utc_midnight(moment: datetime) -> datetime:
|
|
49
|
+
"""The UTC midnight beginning ``moment``'s UTC date (the day-alignment floor)."""
|
|
50
|
+
return datetime.combine(moment.date(), time.min, tzinfo=UTC)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_trailing_edge(now: datetime, cutoff: timedelta) -> datetime:
|
|
54
|
+
"""
|
|
55
|
+
Compute the window's exclusive end: today's UTC midnight, less the cutoff.
|
|
56
|
+
|
|
57
|
+
Floors ``now`` to the start of its UTC day (the most recent midnight at or before
|
|
58
|
+
``now``) so the still-arriving current day is never inside the window, then holds
|
|
59
|
+
the edge back by ``cutoff`` for providers whose data settles later. With
|
|
60
|
+
``cutoff`` zero the end is today's midnight, covering through yesterday -- the
|
|
61
|
+
last complete day.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
now: The current instant, timezone-aware UTC (the caller reads it from its
|
|
65
|
+
``Clock``); its UTC date is the day floored to.
|
|
66
|
+
cutoff: Trailing-edge holdback, a whole-day ``timedelta`` (zero or more);
|
|
67
|
+
whole days keep the returned edge midnight-aligned.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
The exclusive end, a UTC-midnight datetime with ``tzinfo`` ``datetime.UTC``.
|
|
71
|
+
|
|
72
|
+
Side Effects:
|
|
73
|
+
None -- pure function.
|
|
74
|
+
"""
|
|
75
|
+
return _utc_midnight(now) - cutoff
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def resolve_resume_start(
|
|
79
|
+
watermark_start: datetime | None,
|
|
80
|
+
frontier: datetime | None,
|
|
81
|
+
default_start: datetime,
|
|
82
|
+
) -> datetime:
|
|
83
|
+
"""
|
|
84
|
+
Pick the window's inclusive start by the resume precedence, floored to
|
|
85
|
+
the UTC midnight of its date.
|
|
86
|
+
|
|
87
|
+
First present wins (DESIGN section 4): the watermark-derived start when a
|
|
88
|
+
committed watermark exists, else the coverage frontier (the furthest forward a
|
|
89
|
+
succeeded run has covered, even empty), else the cold-start default anchor.
|
|
90
|
+
``default_start`` is never ``None``, so a start is always produced.
|
|
91
|
+
|
|
92
|
+
The chosen start is floored to its UTC midnight because day-granular
|
|
93
|
+
endpoints request and replace whole days: the effective window must be
|
|
94
|
+
day-aligned on both bounds so every covered date is refetched in full --
|
|
95
|
+
the floored-window invariant the date-partitioned writers' wholesale
|
|
96
|
+
replacement and prune are safe under. Unfloored, a watermark's
|
|
97
|
+
``23:59:59`` less the lookback covers its boundary date by a seconds-wide
|
|
98
|
+
sliver: the day-granular request fetches the whole day, the window filter
|
|
99
|
+
keeps only the sliver, and replacement/prune then destroy the boundary
|
|
100
|
+
partition (the live 6-row sliver defect). Floored, lookback reads as
|
|
101
|
+
"re-cover N whole days before the watermark's day". Arms 2 and 3 are
|
|
102
|
+
midnight-aligned by construction, so the floor is idempotent there and
|
|
103
|
+
load-bearing on arm 1. This rule is for snapshot/point-event endpoints
|
|
104
|
+
(each record a single point in time); duration/span endpoints (e.g. HOS
|
|
105
|
+
periods crossing midnight) need their own boundary policy when they
|
|
106
|
+
arrive and must not blindly reuse it.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
watermark_start: The watermark moment less the lookback (arm 1), or ``None``
|
|
110
|
+
when no watermark is committed. Pre-computed by the caller.
|
|
111
|
+
frontier: The coverage frontier (arm 2), or ``None`` when no succeeded run
|
|
112
|
+
has covered anything yet.
|
|
113
|
+
default_start: The cold-start anchor (arm 3); always present.
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
The chosen start floored to the UTC midnight beginning its date,
|
|
117
|
+
timezone-aware UTC.
|
|
118
|
+
|
|
119
|
+
Side Effects:
|
|
120
|
+
None -- pure function.
|
|
121
|
+
"""
|
|
122
|
+
if watermark_start is not None:
|
|
123
|
+
return _utc_midnight(watermark_start)
|
|
124
|
+
if frontier is not None:
|
|
125
|
+
return _utc_midnight(frontier)
|
|
126
|
+
return _utc_midnight(default_start)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def window_or_none(start: datetime, end: datetime) -> DateWindow | None:
|
|
130
|
+
"""
|
|
131
|
+
Assemble the window, or ``None`` when there is no work this run.
|
|
132
|
+
|
|
133
|
+
Returns ``DateWindow(start, end)`` when ``start < end``. When ``start >= end`` the
|
|
134
|
+
resume point has reached or passed the trailing edge -- caught up -- and the
|
|
135
|
+
result is ``None``: a verdict ("nothing to fetch"), not an error. An inverted
|
|
136
|
+
window is therefore never constructed through this path, so ``DateWindow``'s own
|
|
137
|
+
``start < end`` invariant is a backstop for a direct construction bug, never
|
|
138
|
+
tripped here.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
start: The window's inclusive start, timezone-aware UTC.
|
|
142
|
+
end: The window's exclusive end, timezone-aware UTC.
|
|
143
|
+
|
|
144
|
+
Returns:
|
|
145
|
+
The ``DateWindow`` when ``start < end``, else ``None``.
|
|
146
|
+
|
|
147
|
+
Side Effects:
|
|
148
|
+
None -- pure function.
|
|
149
|
+
"""
|
|
150
|
+
if start < end:
|
|
151
|
+
return DateWindow(start=start, end=end)
|
|
152
|
+
return None
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# src/fleetpull/incremental/seed.py
|
|
2
|
+
"""The feed arm's cold-start resume value — the historical seed.
|
|
3
|
+
|
|
4
|
+
A pure, stdlib-only leaf beside the cursors and the window (DESIGN §4).
|
|
5
|
+
``FeedSeed`` is the resume value a tokenless feed endpoint's FIRST request is
|
|
6
|
+
built from: the spec-builder renders it as ``search.fromDate``, which starts
|
|
7
|
+
the feed at a version covering all entities with date >= that instant
|
|
8
|
+
(wire-proven 2026-07-21 — the probe record in §4). It is never persisted —
|
|
9
|
+
the moment the first page returns, the page's ``toVersion`` commits as the
|
|
10
|
+
``FeedToken`` cursor and the seed is gone forever, which is what makes the
|
|
11
|
+
seed-once invariant (§14's I4) structural: only a run that read no stored
|
|
12
|
+
token can ever construct one.
|
|
13
|
+
|
|
14
|
+
``FeedResume`` is the union a feed spec-builder consumes — the seed on the
|
|
15
|
+
cold first run, the stored token on every run after. There is no ``None``
|
|
16
|
+
arm: a feed endpoint always resumes from something, so an absent resume
|
|
17
|
+
reaching a feed builder is a wiring bug the shared guard
|
|
18
|
+
(``require_feed_resume``) rejects loudly.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from datetime import datetime
|
|
23
|
+
|
|
24
|
+
from fleetpull.incremental.cursor import FeedToken
|
|
25
|
+
|
|
26
|
+
__all__: list[str] = ['FeedResume', 'FeedSeed']
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True, slots=True)
|
|
30
|
+
class FeedSeed:
|
|
31
|
+
"""
|
|
32
|
+
The feed arm's cold-start resume value: seed the feed at a date.
|
|
33
|
+
|
|
34
|
+
Rendered by the spec-builder as the first request's ``search.fromDate``
|
|
35
|
+
(and never a ``fromVersion``); superseded by the ``FeedToken`` cursor the
|
|
36
|
+
moment the first page commits. UTC validity defers to the codec boundary
|
|
37
|
+
when the builder serializes ``start``, exactly as ``DateWatermark`` and
|
|
38
|
+
``DateWindow`` defer it.
|
|
39
|
+
|
|
40
|
+
Attributes:
|
|
41
|
+
start: The instant the feed's history begins from — the run's
|
|
42
|
+
configured cold-start anchor (``sync.default_start_date``),
|
|
43
|
+
timezone-aware UTC.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
start: datetime
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# The resume value a feed spec-builder consumes: the cold-start seed, or the
|
|
50
|
+
# stored token. Deliberately no None arm — a feed endpoint always resumes
|
|
51
|
+
# from one of the two (the runner constructs the seed exactly when no token
|
|
52
|
+
# is stored).
|
|
53
|
+
type FeedResume = FeedSeed | FeedToken
|