fleetpull 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fleetpull/__init__.py +36 -0
- fleetpull/api/__init__.py +32 -0
- fleetpull/api/auth_ingress.py +216 -0
- fleetpull/api/catalog.py +139 -0
- fleetpull/api/fetch.py +229 -0
- fleetpull/api/identity.py +90 -0
- fleetpull/api/sync.py +735 -0
- fleetpull/cli.py +146 -0
- fleetpull/config/__init__.py +47 -0
- fleetpull/config/base.py +18 -0
- fleetpull/config/example.py +90 -0
- fleetpull/config/geotab.py +78 -0
- fleetpull/config/http.py +32 -0
- fleetpull/config/loading.py +195 -0
- fleetpull/config/logger.py +139 -0
- fleetpull/config/providers.py +520 -0
- fleetpull/config/rate_limit.py +50 -0
- fleetpull/config/resolution.py +156 -0
- fleetpull/config/retry.py +69 -0
- fleetpull/config/root.py +157 -0
- fleetpull/config/sections.py +146 -0
- fleetpull/endpoints/__init__.py +24 -0
- fleetpull/endpoints/geotab/__init__.py +12 -0
- fleetpull/endpoints/geotab/_requests.py +427 -0
- fleetpull/endpoints/geotab/annotation_logs.py +73 -0
- fleetpull/endpoints/geotab/audits.py +70 -0
- fleetpull/endpoints/geotab/devices.py +79 -0
- fleetpull/endpoints/geotab/driver_changes.py +73 -0
- fleetpull/endpoints/geotab/duty_status_logs.py +75 -0
- fleetpull/endpoints/geotab/dvir_logs.py +76 -0
- fleetpull/endpoints/geotab/exception_events.py +119 -0
- fleetpull/endpoints/geotab/fault_data.py +72 -0
- fleetpull/endpoints/geotab/fill_ups.py +79 -0
- fleetpull/endpoints/geotab/fuel_and_energy_used.py +74 -0
- fleetpull/endpoints/geotab/fuel_tax_details.py +75 -0
- fleetpull/endpoints/geotab/log_records.py +74 -0
- fleetpull/endpoints/geotab/media_files.py +74 -0
- fleetpull/endpoints/geotab/shipment_logs.py +71 -0
- fleetpull/endpoints/geotab/status_data.py +71 -0
- fleetpull/endpoints/geotab/text_messages.py +77 -0
- fleetpull/endpoints/geotab/trips.py +102 -0
- fleetpull/endpoints/geotab/users.py +81 -0
- fleetpull/endpoints/motive/__init__.py +12 -0
- fleetpull/endpoints/motive/_spec_builders.py +94 -0
- fleetpull/endpoints/motive/driver_idle_rollups.py +113 -0
- fleetpull/endpoints/motive/driving_periods.py +85 -0
- fleetpull/endpoints/motive/groups.py +61 -0
- fleetpull/endpoints/motive/idle_events.py +94 -0
- fleetpull/endpoints/motive/users.py +64 -0
- fleetpull/endpoints/motive/vehicle_locations.py +170 -0
- fleetpull/endpoints/motive/vehicle_utilizations.py +122 -0
- fleetpull/endpoints/motive/vehicles.py +106 -0
- fleetpull/endpoints/registry.py +280 -0
- fleetpull/endpoints/samsara/__init__.py +12 -0
- fleetpull/endpoints/samsara/_spec_builders.py +202 -0
- fleetpull/endpoints/samsara/addresses.py +77 -0
- fleetpull/endpoints/samsara/asset_locations.py +217 -0
- fleetpull/endpoints/samsara/driver_fuel_energy_reports.py +124 -0
- fleetpull/endpoints/samsara/driver_vehicle_assignments.py +211 -0
- fleetpull/endpoints/samsara/drivers.py +169 -0
- fleetpull/endpoints/samsara/engine_states.py +114 -0
- fleetpull/endpoints/samsara/gps_readings.py +113 -0
- fleetpull/endpoints/samsara/idling_events.py +195 -0
- fleetpull/endpoints/samsara/odometer_readings.py +116 -0
- fleetpull/endpoints/samsara/trips.py +217 -0
- fleetpull/endpoints/samsara/vehicle_fuel_energy_reports.py +139 -0
- fleetpull/endpoints/samsara/vehicles.py +121 -0
- fleetpull/endpoints/shared/__init__.py +57 -0
- fleetpull/endpoints/shared/base.py +529 -0
- fleetpull/endpoints/shared/request_shape.py +227 -0
- fleetpull/endpoints/shared/resume.py +74 -0
- fleetpull/endpoints/shared/spec_builders.py +59 -0
- fleetpull/endpoints/shared/sync_mode.py +159 -0
- fleetpull/endpoints/shared/url_paths.py +149 -0
- fleetpull/exceptions.py +320 -0
- fleetpull/incremental/__init__.py +27 -0
- fleetpull/incremental/cursor.py +66 -0
- fleetpull/incremental/resolution.py +152 -0
- fleetpull/incremental/seed.py +53 -0
- fleetpull/incremental/window.py +94 -0
- fleetpull/logger/__init__.py +5 -0
- fleetpull/logger/setup.py +145 -0
- fleetpull/model_contract/__init__.py +6 -0
- fleetpull/model_contract/coercions.py +33 -0
- fleetpull/model_contract/response.py +45 -0
- fleetpull/models/__init__.py +7 -0
- fleetpull/models/geotab/__init__.py +166 -0
- fleetpull/models/geotab/annotation_log.py +109 -0
- fleetpull/models/geotab/audit.py +56 -0
- fleetpull/models/geotab/device.py +217 -0
- fleetpull/models/geotab/driver_change.py +102 -0
- fleetpull/models/geotab/duty_status_log.py +200 -0
- fleetpull/models/geotab/dvir_log.py +184 -0
- fleetpull/models/geotab/exception_event.py +135 -0
- fleetpull/models/geotab/fault_data.py +168 -0
- fleetpull/models/geotab/fill_up.py +189 -0
- fleetpull/models/geotab/fuel_and_energy_used.py +81 -0
- fleetpull/models/geotab/fuel_tax_detail.py +150 -0
- fleetpull/models/geotab/log_record.py +68 -0
- fleetpull/models/geotab/media_file.py +133 -0
- fleetpull/models/geotab/shared.py +221 -0
- fleetpull/models/geotab/shipment_log.py +105 -0
- fleetpull/models/geotab/status_data.py +108 -0
- fleetpull/models/geotab/text_message.py +125 -0
- fleetpull/models/geotab/trip.py +162 -0
- fleetpull/models/geotab/user.py +187 -0
- fleetpull/models/motive/__init__.py +43 -0
- fleetpull/models/motive/driver_idle_rollup.py +109 -0
- fleetpull/models/motive/driving_period.py +82 -0
- fleetpull/models/motive/group.py +49 -0
- fleetpull/models/motive/idle_event.py +77 -0
- fleetpull/models/motive/shared.py +192 -0
- fleetpull/models/motive/user.py +217 -0
- fleetpull/models/motive/vehicle.py +162 -0
- fleetpull/models/motive/vehicle_location.py +127 -0
- fleetpull/models/motive/vehicle_utilization.py +128 -0
- fleetpull/models/samsara/__init__.py +108 -0
- fleetpull/models/samsara/address.py +149 -0
- fleetpull/models/samsara/asset_location.py +131 -0
- fleetpull/models/samsara/driver.py +232 -0
- fleetpull/models/samsara/driver_fuel_energy_report.py +153 -0
- fleetpull/models/samsara/driver_vehicle_assignment.py +168 -0
- fleetpull/models/samsara/engine_state.py +92 -0
- fleetpull/models/samsara/gps_reading.py +146 -0
- fleetpull/models/samsara/idling_event.py +174 -0
- fleetpull/models/samsara/odometer_reading.py +90 -0
- fleetpull/models/samsara/trip.py +199 -0
- fleetpull/models/samsara/vehicle.py +167 -0
- fleetpull/models/samsara/vehicle_fuel_energy_report.py +191 -0
- fleetpull/network/__init__.py +9 -0
- fleetpull/network/auth/__init__.py +15 -0
- fleetpull/network/auth/authenticate.py +271 -0
- fleetpull/network/auth/manager.py +223 -0
- fleetpull/network/auth/models.py +57 -0
- fleetpull/network/auth/strategies.py +169 -0
- fleetpull/network/classifiers/__init__.py +11 -0
- fleetpull/network/classifiers/geotab.py +145 -0
- fleetpull/network/classifiers/motive.py +79 -0
- fleetpull/network/classifiers/samsara.py +80 -0
- fleetpull/network/client/__init__.py +25 -0
- fleetpull/network/client/page.py +32 -0
- fleetpull/network/client/profile.py +27 -0
- fleetpull/network/client/registry.py +103 -0
- fleetpull/network/client/registry_base.py +150 -0
- fleetpull/network/client/runtime.py +44 -0
- fleetpull/network/client/transport.py +381 -0
- fleetpull/network/contract/__init__.py +50 -0
- fleetpull/network/contract/auth.py +48 -0
- fleetpull/network/contract/classifier.py +189 -0
- fleetpull/network/contract/envelope_fetcher.py +33 -0
- fleetpull/network/contract/envelopes.py +189 -0
- fleetpull/network/contract/outcome.py +43 -0
- fleetpull/network/contract/page_decoder.py +101 -0
- fleetpull/network/contract/request.py +97 -0
- fleetpull/network/decoders/__init__.py +40 -0
- fleetpull/network/decoders/_window_stamp.py +78 -0
- fleetpull/network/decoders/geotab.py +309 -0
- fleetpull/network/decoders/motive.py +204 -0
- fleetpull/network/decoders/motive_reports.py +114 -0
- fleetpull/network/decoders/samsara.py +343 -0
- fleetpull/network/decoders/samsara_reports.py +119 -0
- fleetpull/network/decoders/single_page.py +55 -0
- fleetpull/network/limits/__init__.py +13 -0
- fleetpull/network/limits/bucket_math.py +58 -0
- fleetpull/network/limits/limiter.py +153 -0
- fleetpull/network/limits/registry.py +96 -0
- fleetpull/network/posture/__init__.py +6 -0
- fleetpull/network/posture/client_options.py +96 -0
- fleetpull/network/retry/__init__.py +13 -0
- fleetpull/network/retry/decision.py +155 -0
- fleetpull/network/tls/__init__.py +5 -0
- fleetpull/network/tls/truststore_context.py +35 -0
- fleetpull/orchestrator/__init__.py +30 -0
- fleetpull/orchestrator/backfill.py +127 -0
- fleetpull/orchestrator/batch.py +155 -0
- fleetpull/orchestrator/bisection.py +271 -0
- fleetpull/orchestrator/drivers.py +350 -0
- fleetpull/orchestrator/entry.py +347 -0
- fleetpull/orchestrator/executors.py +128 -0
- fleetpull/orchestrator/fanout.py +163 -0
- fleetpull/orchestrator/feed_drive.py +266 -0
- fleetpull/orchestrator/metadata_projection.py +195 -0
- fleetpull/orchestrator/outcome.py +53 -0
- fleetpull/orchestrator/recording.py +85 -0
- fleetpull/orchestrator/resume.py +143 -0
- fleetpull/orchestrator/roster_harvest.py +108 -0
- fleetpull/orchestrator/roster_refresh.py +363 -0
- fleetpull/orchestrator/runner.py +262 -0
- fleetpull/orchestrator/shape_resolution.py +221 -0
- fleetpull/orchestrator/spine.py +166 -0
- fleetpull/orchestrator/streaming.py +153 -0
- fleetpull/orchestrator/unit_loop.py +273 -0
- fleetpull/orchestrator/watermark_drive.py +455 -0
- fleetpull/paths/__init__.py +13 -0
- fleetpull/paths/datasets.py +38 -0
- fleetpull/paths/partitions.py +41 -0
- fleetpull/paths/resolution.py +98 -0
- fleetpull/polars_typing/__init__.py +20 -0
- fleetpull/py.typed +0 -0
- fleetpull/records/__init__.py +19 -0
- fleetpull/records/convert.py +53 -0
- fleetpull/records/dataframe.py +63 -0
- fleetpull/records/event_time.py +70 -0
- fleetpull/records/fields.py +190 -0
- fleetpull/records/flatten.py +54 -0
- fleetpull/records/roster_members.py +74 -0
- fleetpull/records/schema.py +73 -0
- fleetpull/records/validation.py +62 -0
- fleetpull/resources/__init__.py +10 -0
- fleetpull/resources/config.example.yaml +222 -0
- fleetpull/roster/__init__.py +16 -0
- fleetpull/roster/definition.py +46 -0
- fleetpull/roster/key.py +40 -0
- fleetpull/roster/registry.py +93 -0
- fleetpull/state/__init__.py +36 -0
- fleetpull/state/cursors.py +417 -0
- fleetpull/state/database.py +463 -0
- fleetpull/state/migrations.py +430 -0
- fleetpull/state/reconcile.py +127 -0
- fleetpull/state/rosters.py +159 -0
- fleetpull/state/run_ledger.py +572 -0
- fleetpull/state/work_units.py +648 -0
- fleetpull/storage/__init__.py +30 -0
- fleetpull/storage/append.py +187 -0
- fleetpull/storage/atomic.py +161 -0
- fleetpull/storage/files.py +176 -0
- fleetpull/storage/frames.py +98 -0
- fleetpull/storage/metadata.py +161 -0
- fleetpull/storage/partitioned.py +205 -0
- fleetpull/storage/pruning.py +169 -0
- fleetpull/storage/read.py +35 -0
- fleetpull/storage/result.py +34 -0
- fleetpull/storage/single_file.py +130 -0
- fleetpull/storage/splitting.py +88 -0
- fleetpull/storage/staging.py +178 -0
- fleetpull/storage/writers.py +145 -0
- fleetpull/timing/__init__.py +21 -0
- fleetpull/timing/canon.py +92 -0
- fleetpull/timing/clock.py +192 -0
- fleetpull/timing/codec.py +122 -0
- fleetpull/timing/sleeper.py +65 -0
- fleetpull/vocabulary/__init__.py +16 -0
- fleetpull/vocabulary/json_types.py +20 -0
- fleetpull/vocabulary/provider.py +29 -0
- fleetpull/vocabulary/quota_scope.py +47 -0
- fleetpull/vocabulary/response_category.py +32 -0
- fleetpull-0.1.0.dist-info/METADATA +248 -0
- fleetpull-0.1.0.dist-info/RECORD +252 -0
- fleetpull-0.1.0.dist-info/WHEEL +5 -0
- fleetpull-0.1.0.dist-info/entry_points.txt +2 -0
- fleetpull-0.1.0.dist-info/licenses/LICENSE +202 -0
- fleetpull-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# src/fleetpull/network/classifiers/samsara.py
|
|
2
|
+
"""Samsara response classifier (sources: scrubbed provider-behavior
|
|
3
|
+
verification, June 2026; rate-limit contract from official Samsara
|
|
4
|
+
documentation).
|
|
5
|
+
|
|
6
|
+
Classification reads status codes and structured fields, never
|
|
7
|
+
human-readable message text; ``detail`` carries messages for humans,
|
|
8
|
+
decisions never read them. Branch logic deliberately resembles sibling
|
|
9
|
+
classifiers without sharing code: provider classifiers evolve
|
|
10
|
+
independently (blast-radius over DRY).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from http import HTTPStatus
|
|
16
|
+
|
|
17
|
+
from fleetpull.network.contract import (
|
|
18
|
+
SERVER_ERROR_FLOOR,
|
|
19
|
+
SUCCESS_STATUS_RANGE,
|
|
20
|
+
ClassifiedResponse,
|
|
21
|
+
ResponseClassifier,
|
|
22
|
+
body_snippet,
|
|
23
|
+
retry_after_seconds_from_headers,
|
|
24
|
+
)
|
|
25
|
+
from fleetpull.vocabulary import JsonValue, ResponseCategory
|
|
26
|
+
|
|
27
|
+
__all__: list[str] = ['SamsaraResponseClassifier']
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _auth_failure_detail(body_text: str) -> str:
|
|
31
|
+
"""Extract the body's ``message`` when JSON, else a snippet."""
|
|
32
|
+
try:
|
|
33
|
+
parsed_body: JsonValue = json.loads(body_text)
|
|
34
|
+
except json.JSONDecodeError:
|
|
35
|
+
return body_snippet(body_text)
|
|
36
|
+
if isinstance(parsed_body, dict):
|
|
37
|
+
message: JsonValue = parsed_body.get('message')
|
|
38
|
+
if isinstance(message, str):
|
|
39
|
+
return message
|
|
40
|
+
return body_snippet(body_text)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class SamsaraResponseClassifier(ResponseClassifier):
|
|
44
|
+
"""Classifies Samsara REST responses (observed shape:
|
|
45
|
+
``{"message": "invalid token", "requestId": ...}`` on 401)."""
|
|
46
|
+
|
|
47
|
+
def classify_response(
|
|
48
|
+
self,
|
|
49
|
+
status_code: int,
|
|
50
|
+
headers: Mapping[str, str],
|
|
51
|
+
body_text: str,
|
|
52
|
+
) -> ClassifiedResponse:
|
|
53
|
+
"""Classify one Samsara response by status code."""
|
|
54
|
+
match status_code:
|
|
55
|
+
case code if code in SUCCESS_STATUS_RANGE:
|
|
56
|
+
return ClassifiedResponse(category=ResponseCategory.SUCCESS)
|
|
57
|
+
case HTTPStatus.TOO_MANY_REQUESTS:
|
|
58
|
+
# Retry-After is documented as fractional seconds
|
|
59
|
+
# (e.g. 0.40235).
|
|
60
|
+
return ClassifiedResponse(
|
|
61
|
+
category=ResponseCategory.RATE_LIMITED,
|
|
62
|
+
retry_after_seconds=retry_after_seconds_from_headers(headers),
|
|
63
|
+
)
|
|
64
|
+
case HTTPStatus.UNAUTHORIZED | HTTPStatus.FORBIDDEN:
|
|
65
|
+
return ClassifiedResponse(
|
|
66
|
+
category=ResponseCategory.AUTH_FAILURE,
|
|
67
|
+
detail=_auth_failure_detail(body_text),
|
|
68
|
+
)
|
|
69
|
+
case code if code >= SERVER_ERROR_FLOOR:
|
|
70
|
+
# Documented Samsara behavior: 5xx bodies are plain
|
|
71
|
+
# strings, not JSON — never attempt JSON parsing here.
|
|
72
|
+
return ClassifiedResponse(
|
|
73
|
+
category=ResponseCategory.TRANSIENT,
|
|
74
|
+
detail=f'HTTP {status_code}: {body_snippet(body_text)}',
|
|
75
|
+
)
|
|
76
|
+
case _:
|
|
77
|
+
return ClassifiedResponse(
|
|
78
|
+
category=ResponseCategory.FATAL,
|
|
79
|
+
detail=f'HTTP {status_code}: {body_snippet(body_text)}',
|
|
80
|
+
)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# src/fleetpull/network/client/__init__.py
|
|
2
|
+
"""The transport client face: the assembled HTTP fetch loop and its inputs.
|
|
3
|
+
|
|
4
|
+
``TransportClient`` runs the per-attempt pipeline and page loop against a
|
|
5
|
+
per-provider ``ProviderProfile`` (auth + classifier) and a process-global
|
|
6
|
+
``ClientRuntime`` (configs, limiter registry, jitter, sleeper), emitting
|
|
7
|
+
``FetchedPage`` objects. ``ProviderClientRegistry`` owns one open client per
|
|
8
|
+
provider, keyed by ``Provider``, and hands the right one to the run executor.
|
|
9
|
+
External callers import these names here."""
|
|
10
|
+
|
|
11
|
+
from fleetpull.network.client.page import FetchedPage
|
|
12
|
+
from fleetpull.network.client.profile import ProviderProfile
|
|
13
|
+
from fleetpull.network.client.registry import ProviderClientRegistry
|
|
14
|
+
from fleetpull.network.client.registry_base import ProviderResourceRegistry
|
|
15
|
+
from fleetpull.network.client.runtime import ClientRuntime
|
|
16
|
+
from fleetpull.network.client.transport import TransportClient
|
|
17
|
+
|
|
18
|
+
__all__: list[str] = [
|
|
19
|
+
'ClientRuntime',
|
|
20
|
+
'FetchedPage',
|
|
21
|
+
'ProviderClientRegistry',
|
|
22
|
+
'ProviderProfile',
|
|
23
|
+
'ProviderResourceRegistry',
|
|
24
|
+
'TransportClient',
|
|
25
|
+
]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# src/fleetpull/network/client/page.py
|
|
2
|
+
"""The client's emit type: one page of records plus resume progress."""
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from fleetpull.vocabulary import JsonObject
|
|
7
|
+
|
|
8
|
+
__all__: list[str] = ['FetchedPage']
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True, slots=True)
|
|
12
|
+
class FetchedPage:
|
|
13
|
+
"""
|
|
14
|
+
One page emitted by the transport client: its records and the opaque
|
|
15
|
+
resume cursor.
|
|
16
|
+
|
|
17
|
+
The records are the page decoder's wire-shape extraction — a list of JSON
|
|
18
|
+
objects, validated as record-bearing but not yet validated into the
|
|
19
|
+
per-record response model (that is the records layer's job). Attempt counts
|
|
20
|
+
and timings are not here until a consumer demands them — the client is
|
|
21
|
+
state-blind.
|
|
22
|
+
|
|
23
|
+
Attributes:
|
|
24
|
+
records: The page's records, each a JSON object, as extracted by the
|
|
25
|
+
endpoint's page decoder. Per-record model validation is downstream.
|
|
26
|
+
durable_progress: Opaque resume cursor that must outlive the fetch
|
|
27
|
+
(GeoTab ``toVersion``), or None for fetch-private cursors
|
|
28
|
+
(Motive, Samsara). The orchestrator owns its interpretation.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
records: list[JsonObject]
|
|
32
|
+
durable_progress: str | None
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# src/fleetpull/network/client/profile.py
|
|
2
|
+
"""The per-provider strategy bundle the client receives at construction."""
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from fleetpull.network.contract import AuthStrategy, ResponseClassifier
|
|
7
|
+
|
|
8
|
+
__all__: list[str] = ['ProviderProfile']
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True, slots=True)
|
|
12
|
+
class ProviderProfile:
|
|
13
|
+
"""
|
|
14
|
+
The per-provider, per-construction dependencies of a transport client.
|
|
15
|
+
|
|
16
|
+
Auth strategy and classifier are shared across all of a provider's
|
|
17
|
+
endpoints (one session auth, one classifier). The page decoder and
|
|
18
|
+
quota scope are deliberately NOT here — they are per-endpoint and arrive
|
|
19
|
+
on each ``fetch_pages`` call.
|
|
20
|
+
|
|
21
|
+
Attributes:
|
|
22
|
+
auth: Credential injection for this provider.
|
|
23
|
+
classifier: Response and transport classification for this provider.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
auth: AuthStrategy
|
|
27
|
+
classifier: ResponseClassifier
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# src/fleetpull/network/client/registry.py
|
|
2
|
+
"""Provider-keyed registry of transport clients: one open pool per provider.
|
|
3
|
+
|
|
4
|
+
The seam between endpoint execution and provider transport identity. A
|
|
5
|
+
``TransportClient`` is provider-scoped (it carries a provider's ``ProviderProfile``
|
|
6
|
+
and owns that provider's pooled ``httpx.Client``); an endpoint run is
|
|
7
|
+
endpoint-scoped. This registry lets the run executor ask for the client of
|
|
8
|
+
``definition.provider`` without owning a single client or pretending one client can
|
|
9
|
+
authenticate every provider. It owns the clients' lifecycle -- open every configured
|
|
10
|
+
provider's client on enter, close every pool on exit -- and nothing else; it builds
|
|
11
|
+
no profiles and reads no credentials, which is the composition root's job. The
|
|
12
|
+
lifecycle machinery itself (publish-on-success enter, closed-before-release
|
|
13
|
+
exit, the RuntimeError-vs-ConfigurationError lookup split) is the generic
|
|
14
|
+
``ProviderResourceRegistry``'s (``registry_base.py``); this subclass supplies
|
|
15
|
+
client construction and the error nouns.
|
|
16
|
+
|
|
17
|
+
The one shared ``ClientRuntime`` passed to every client is what keeps cross-provider
|
|
18
|
+
quota enforced: every page attempt routes through that runtime's single
|
|
19
|
+
``RateLimiterRegistry`` (DESIGN §7, §14).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from collections.abc import Mapping
|
|
23
|
+
from contextlib import ExitStack
|
|
24
|
+
from typing import ClassVar
|
|
25
|
+
|
|
26
|
+
from fleetpull.network.client.profile import ProviderProfile
|
|
27
|
+
from fleetpull.network.client.registry_base import ProviderResourceRegistry
|
|
28
|
+
from fleetpull.network.client.runtime import ClientRuntime
|
|
29
|
+
from fleetpull.network.client.transport import TransportClient
|
|
30
|
+
from fleetpull.vocabulary import Provider
|
|
31
|
+
|
|
32
|
+
__all__: list[str] = ['ProviderClientRegistry']
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ProviderClientRegistry(ProviderResourceRegistry[TransportClient]):
|
|
36
|
+
"""Owns one open ``TransportClient`` per provider, keyed by ``Provider``.
|
|
37
|
+
|
|
38
|
+
A resource-owning context manager (the generic base's semantics).
|
|
39
|
+
``client_for`` returns a provider's client; use it only inside the
|
|
40
|
+
``with`` block::
|
|
41
|
+
|
|
42
|
+
with ProviderClientRegistry(profiles, runtime) as clients:
|
|
43
|
+
client = clients.client_for(definition.provider)
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
_resource_noun: ClassVar[str] = 'transport client'
|
|
47
|
+
_lookup_description: ClassVar[str] = 'client_for'
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
profiles: Mapping[Provider, ProviderProfile],
|
|
52
|
+
runtime: ClientRuntime,
|
|
53
|
+
) -> None:
|
|
54
|
+
"""
|
|
55
|
+
Args:
|
|
56
|
+
profiles: The per-provider auth/classifier bundle for each configured
|
|
57
|
+
provider. A provider absent here has no client and is rejected by
|
|
58
|
+
``client_for`` while the registry is open.
|
|
59
|
+
runtime: The one process-global transport runtime shared by every
|
|
60
|
+
client (its limiter registry enforces cross-provider quota).
|
|
61
|
+
|
|
62
|
+
Side Effects:
|
|
63
|
+
None -- clients are opened on ``__enter__``, not here.
|
|
64
|
+
"""
|
|
65
|
+
super().__init__(profiles)
|
|
66
|
+
self._profiles: dict[Provider, ProviderProfile] = dict(profiles)
|
|
67
|
+
self._runtime: ClientRuntime = runtime
|
|
68
|
+
|
|
69
|
+
def _open_resource(self, stack: ExitStack, provider: Provider) -> TransportClient:
|
|
70
|
+
"""Open one provider's transport client (its pooled ``httpx.Client``).
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
stack: The enter's unwind stack; the client's pool close registers
|
|
74
|
+
here.
|
|
75
|
+
provider: The provider whose client to open.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
The provider's open ``TransportClient``.
|
|
79
|
+
|
|
80
|
+
Side Effects:
|
|
81
|
+
Constructs one pooled ``httpx.Client``.
|
|
82
|
+
"""
|
|
83
|
+
return stack.enter_context(
|
|
84
|
+
TransportClient(self._profiles[provider], self._runtime)
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
def client_for(self, provider: Provider) -> TransportClient:
|
|
88
|
+
"""Return the transport client for a provider.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
provider: The provider whose client to return (e.g.
|
|
92
|
+
``definition.provider``).
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
The provider's open ``TransportClient``.
|
|
96
|
+
|
|
97
|
+
Raises:
|
|
98
|
+
RuntimeError: The registry is not open -- ``client_for`` was called
|
|
99
|
+
outside an active ``with`` block (a caller bug).
|
|
100
|
+
ConfigurationError: The registry is open but the provider has no
|
|
101
|
+
configured client.
|
|
102
|
+
"""
|
|
103
|
+
return self._resource_for(provider)
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# src/fleetpull/network/client/registry_base.py
|
|
2
|
+
"""The generic provider-keyed resource registry the concrete registries subclass.
|
|
3
|
+
|
|
4
|
+
One resource per provider, owned as a context manager: the publish-on-success
|
|
5
|
+
enter (resources built off to the side on an ``ExitStack`` and assigned only
|
|
6
|
+
after every provider's opened, so a mid-open failure unwinds what opened and
|
|
7
|
+
leaves the registry unentered), the closed-before-release exit (the instance
|
|
8
|
+
is marked closed before the stack unwinds, so a close error still leaves the
|
|
9
|
+
registry unusable rather than apparently-open), and the two-way lookup split
|
|
10
|
+
-- ``RuntimeError`` for a lookup outside the ``with`` block (a caller bug),
|
|
11
|
+
``ConfigurationError`` for an unconfigured provider while open -- are each
|
|
12
|
+
stated once here. ``ProviderClientRegistry`` (transport clients) and the
|
|
13
|
+
orchestrator's ``FetchPoolRegistry`` (fetch worker pools) are thin
|
|
14
|
+
subclasses supplying construction and the error nouns.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from abc import ABC, abstractmethod
|
|
18
|
+
from collections.abc import Iterable
|
|
19
|
+
from contextlib import ExitStack
|
|
20
|
+
from types import TracebackType
|
|
21
|
+
from typing import ClassVar, Self
|
|
22
|
+
|
|
23
|
+
from fleetpull.exceptions import ConfigurationError
|
|
24
|
+
from fleetpull.vocabulary import Provider
|
|
25
|
+
|
|
26
|
+
__all__: list[str] = ['ProviderResourceRegistry']
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ProviderResourceRegistry[ResourceT](ABC):
|
|
30
|
+
"""Owns one open resource per provider, keyed by ``Provider``.
|
|
31
|
+
|
|
32
|
+
A resource-owning context manager: ``__enter__`` opens every configured
|
|
33
|
+
provider's resource and returns self; ``__exit__`` releases them all.
|
|
34
|
+
Subclasses bind the vocabulary (``_resource_noun`` for the unconfigured-
|
|
35
|
+
provider error, ``_lookup_description`` for the not-open error), open one
|
|
36
|
+
provider's resource in ``_open_resource``, and expose their named lookup
|
|
37
|
+
over ``_resource_for``.
|
|
38
|
+
|
|
39
|
+
Attributes:
|
|
40
|
+
_resource_noun: The resource's error noun (e.g. ``'transport
|
|
41
|
+
client'``), completing ``'no <noun> configured for provider'``.
|
|
42
|
+
_lookup_description: The public lookup method's name (e.g.
|
|
43
|
+
``'client_for'``), naming the misuse in the not-open error.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
_resource_noun: ClassVar[str]
|
|
47
|
+
_lookup_description: ClassVar[str]
|
|
48
|
+
|
|
49
|
+
def __init__(self, providers: Iterable[Provider]) -> None:
|
|
50
|
+
"""
|
|
51
|
+
Args:
|
|
52
|
+
providers: The providers to open a resource for on ``__enter__``.
|
|
53
|
+
A provider absent here has no resource and is rejected by the
|
|
54
|
+
lookup while the registry is open.
|
|
55
|
+
|
|
56
|
+
Side Effects:
|
|
57
|
+
None -- resources are opened on ``__enter__``, not here.
|
|
58
|
+
"""
|
|
59
|
+
self._providers: tuple[Provider, ...] = tuple(providers)
|
|
60
|
+
self._resources: dict[Provider, ResourceT] = {}
|
|
61
|
+
self._stack: ExitStack = ExitStack()
|
|
62
|
+
self._open: bool = False
|
|
63
|
+
|
|
64
|
+
@abstractmethod
|
|
65
|
+
def _open_resource(self, stack: ExitStack, provider: Provider) -> ResourceT:
|
|
66
|
+
"""Open one provider's resource, registering its release on ``stack``.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
stack: The enter's unwind stack; register whatever must release
|
|
70
|
+
on exit (or on a later provider's open failure) here.
|
|
71
|
+
provider: The provider whose resource to open.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
The opened resource.
|
|
75
|
+
"""
|
|
76
|
+
...
|
|
77
|
+
|
|
78
|
+
def __enter__(self) -> Self:
|
|
79
|
+
"""Open one resource per configured provider; publish only on full success.
|
|
80
|
+
|
|
81
|
+
Builds the provider-to-resource map in a local dict and assigns it to
|
|
82
|
+
the instance only after every resource has opened, so a mid-open
|
|
83
|
+
failure unwinds the resources already opened and leaves the registry
|
|
84
|
+
unentered (``_resources`` untouched, ``_open`` false).
|
|
85
|
+
|
|
86
|
+
Side Effects:
|
|
87
|
+
Opens one resource per provider via ``_open_resource``. On a later
|
|
88
|
+
open failure, the earlier resources are released before
|
|
89
|
+
propagating.
|
|
90
|
+
"""
|
|
91
|
+
resources: dict[Provider, ResourceT] = {}
|
|
92
|
+
with ExitStack() as stack:
|
|
93
|
+
for provider in self._providers:
|
|
94
|
+
resources[provider] = self._open_resource(stack, provider)
|
|
95
|
+
self._stack = stack.pop_all()
|
|
96
|
+
self._resources = resources
|
|
97
|
+
self._open = True
|
|
98
|
+
return self
|
|
99
|
+
|
|
100
|
+
def __exit__(
|
|
101
|
+
self,
|
|
102
|
+
exc_type: type[BaseException] | None,
|
|
103
|
+
exc_value: BaseException | None,
|
|
104
|
+
traceback: TracebackType | None,
|
|
105
|
+
) -> bool:
|
|
106
|
+
"""Release every provider's resource, forwarding the exit context.
|
|
107
|
+
|
|
108
|
+
Forwards ``exc_*`` to the owned ``ExitStack`` so each resource's
|
|
109
|
+
``__exit__`` receives the exit context and the suppression decision
|
|
110
|
+
passes through. The instance is marked closed before the resources
|
|
111
|
+
are released, so a release error still leaves the registry unusable
|
|
112
|
+
rather than apparently-open.
|
|
113
|
+
|
|
114
|
+
Side Effects:
|
|
115
|
+
Releases every resource opened in ``__enter__``.
|
|
116
|
+
"""
|
|
117
|
+
self._open = False
|
|
118
|
+
self._resources = {}
|
|
119
|
+
return bool(self._stack.__exit__(exc_type, exc_value, traceback))
|
|
120
|
+
|
|
121
|
+
def _resource_for(self, provider: Provider) -> ResourceT:
|
|
122
|
+
"""Return the provider's open resource -- the shared lookup body.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
provider: The provider whose resource to return.
|
|
126
|
+
|
|
127
|
+
Returns:
|
|
128
|
+
The provider's open resource.
|
|
129
|
+
|
|
130
|
+
Raises:
|
|
131
|
+
RuntimeError: The registry is not open -- the lookup was called
|
|
132
|
+
outside an active ``with`` block (a caller bug), kept distinct
|
|
133
|
+
from a genuinely unconfigured provider.
|
|
134
|
+
ConfigurationError: The registry is open but the provider has no
|
|
135
|
+
configured resource.
|
|
136
|
+
"""
|
|
137
|
+
if not self._open:
|
|
138
|
+
raise RuntimeError(
|
|
139
|
+
f'{type(self).__name__} is not open; call '
|
|
140
|
+
f'{self._lookup_description} inside its `with` block'
|
|
141
|
+
)
|
|
142
|
+
resource = self._resources.get(provider)
|
|
143
|
+
if resource is None:
|
|
144
|
+
configured = ', '.join(sorted(p.value for p in self._resources)) or 'none'
|
|
145
|
+
raise ConfigurationError(
|
|
146
|
+
f'no {self._resource_noun} configured for provider',
|
|
147
|
+
provider=provider.value,
|
|
148
|
+
detail=f'configured providers: {configured}',
|
|
149
|
+
)
|
|
150
|
+
return resource
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# src/fleetpull/network/client/runtime.py
|
|
2
|
+
"""Process-global transport infrastructure, shared by every provider's client."""
|
|
3
|
+
|
|
4
|
+
import random
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from fleetpull.config import HttpConfig, RetryConfig
|
|
8
|
+
from fleetpull.network.limits import RateLimiterRegistry
|
|
9
|
+
from fleetpull.network.retry import RandomFractionGenerator
|
|
10
|
+
from fleetpull.timing import Sleeper, SystemSleeper
|
|
11
|
+
|
|
12
|
+
__all__: list[str] = ['ClientRuntime']
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True, slots=True)
|
|
16
|
+
class ClientRuntime:
|
|
17
|
+
"""
|
|
18
|
+
The process-global dependencies a transport client runs against.
|
|
19
|
+
|
|
20
|
+
Built once per composition root and shared across every per-provider
|
|
21
|
+
client that root constructs — only the ``ProviderProfile`` varies per
|
|
22
|
+
provider. Bundled because these always travel together at the sites
|
|
23
|
+
that construct clients, and no module-level singletons are permitted.
|
|
24
|
+
|
|
25
|
+
Attributes:
|
|
26
|
+
http_config: Connect/read timeouts and TLS posture (consumed once to
|
|
27
|
+
build the client's connection pool).
|
|
28
|
+
retry_config: Per-category failure budgets, backoff shape, fallback
|
|
29
|
+
penalty.
|
|
30
|
+
limiter_registry: Shared rate limiters, keyed by quota scope.
|
|
31
|
+
random_source: Jitter seam for retry backoff. Defaults to the
|
|
32
|
+
production ``random.Random``; injectable so tests make every
|
|
33
|
+
jittered delay exact arithmetic. No composition root needs to
|
|
34
|
+
know the seam exists.
|
|
35
|
+
sleeper: Sleep seam for TRANSIENT backoff. Defaults to the
|
|
36
|
+
production ``SystemSleeper``; injectable so tests record delays
|
|
37
|
+
instead of waiting. Same stance as ``random_source``.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
http_config: HttpConfig
|
|
41
|
+
retry_config: RetryConfig
|
|
42
|
+
limiter_registry: RateLimiterRegistry
|
|
43
|
+
random_source: RandomFractionGenerator = field(default_factory=random.Random)
|
|
44
|
+
sleeper: Sleeper = field(default_factory=SystemSleeper)
|