fleetpull 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fleetpull/__init__.py +36 -0
- fleetpull/api/__init__.py +32 -0
- fleetpull/api/auth_ingress.py +216 -0
- fleetpull/api/catalog.py +139 -0
- fleetpull/api/fetch.py +229 -0
- fleetpull/api/identity.py +90 -0
- fleetpull/api/sync.py +735 -0
- fleetpull/cli.py +146 -0
- fleetpull/config/__init__.py +47 -0
- fleetpull/config/base.py +18 -0
- fleetpull/config/example.py +90 -0
- fleetpull/config/geotab.py +78 -0
- fleetpull/config/http.py +32 -0
- fleetpull/config/loading.py +195 -0
- fleetpull/config/logger.py +139 -0
- fleetpull/config/providers.py +520 -0
- fleetpull/config/rate_limit.py +50 -0
- fleetpull/config/resolution.py +156 -0
- fleetpull/config/retry.py +69 -0
- fleetpull/config/root.py +157 -0
- fleetpull/config/sections.py +146 -0
- fleetpull/endpoints/__init__.py +24 -0
- fleetpull/endpoints/geotab/__init__.py +12 -0
- fleetpull/endpoints/geotab/_requests.py +427 -0
- fleetpull/endpoints/geotab/annotation_logs.py +73 -0
- fleetpull/endpoints/geotab/audits.py +70 -0
- fleetpull/endpoints/geotab/devices.py +79 -0
- fleetpull/endpoints/geotab/driver_changes.py +73 -0
- fleetpull/endpoints/geotab/duty_status_logs.py +75 -0
- fleetpull/endpoints/geotab/dvir_logs.py +76 -0
- fleetpull/endpoints/geotab/exception_events.py +119 -0
- fleetpull/endpoints/geotab/fault_data.py +72 -0
- fleetpull/endpoints/geotab/fill_ups.py +79 -0
- fleetpull/endpoints/geotab/fuel_and_energy_used.py +74 -0
- fleetpull/endpoints/geotab/fuel_tax_details.py +75 -0
- fleetpull/endpoints/geotab/log_records.py +74 -0
- fleetpull/endpoints/geotab/media_files.py +74 -0
- fleetpull/endpoints/geotab/shipment_logs.py +71 -0
- fleetpull/endpoints/geotab/status_data.py +71 -0
- fleetpull/endpoints/geotab/text_messages.py +77 -0
- fleetpull/endpoints/geotab/trips.py +102 -0
- fleetpull/endpoints/geotab/users.py +81 -0
- fleetpull/endpoints/motive/__init__.py +12 -0
- fleetpull/endpoints/motive/_spec_builders.py +94 -0
- fleetpull/endpoints/motive/driver_idle_rollups.py +113 -0
- fleetpull/endpoints/motive/driving_periods.py +85 -0
- fleetpull/endpoints/motive/groups.py +61 -0
- fleetpull/endpoints/motive/idle_events.py +94 -0
- fleetpull/endpoints/motive/users.py +64 -0
- fleetpull/endpoints/motive/vehicle_locations.py +170 -0
- fleetpull/endpoints/motive/vehicle_utilizations.py +122 -0
- fleetpull/endpoints/motive/vehicles.py +106 -0
- fleetpull/endpoints/registry.py +280 -0
- fleetpull/endpoints/samsara/__init__.py +12 -0
- fleetpull/endpoints/samsara/_spec_builders.py +202 -0
- fleetpull/endpoints/samsara/addresses.py +77 -0
- fleetpull/endpoints/samsara/asset_locations.py +217 -0
- fleetpull/endpoints/samsara/driver_fuel_energy_reports.py +124 -0
- fleetpull/endpoints/samsara/driver_vehicle_assignments.py +211 -0
- fleetpull/endpoints/samsara/drivers.py +169 -0
- fleetpull/endpoints/samsara/engine_states.py +114 -0
- fleetpull/endpoints/samsara/gps_readings.py +113 -0
- fleetpull/endpoints/samsara/idling_events.py +195 -0
- fleetpull/endpoints/samsara/odometer_readings.py +116 -0
- fleetpull/endpoints/samsara/trips.py +217 -0
- fleetpull/endpoints/samsara/vehicle_fuel_energy_reports.py +139 -0
- fleetpull/endpoints/samsara/vehicles.py +121 -0
- fleetpull/endpoints/shared/__init__.py +57 -0
- fleetpull/endpoints/shared/base.py +529 -0
- fleetpull/endpoints/shared/request_shape.py +227 -0
- fleetpull/endpoints/shared/resume.py +74 -0
- fleetpull/endpoints/shared/spec_builders.py +59 -0
- fleetpull/endpoints/shared/sync_mode.py +159 -0
- fleetpull/endpoints/shared/url_paths.py +149 -0
- fleetpull/exceptions.py +320 -0
- fleetpull/incremental/__init__.py +27 -0
- fleetpull/incremental/cursor.py +66 -0
- fleetpull/incremental/resolution.py +152 -0
- fleetpull/incremental/seed.py +53 -0
- fleetpull/incremental/window.py +94 -0
- fleetpull/logger/__init__.py +5 -0
- fleetpull/logger/setup.py +145 -0
- fleetpull/model_contract/__init__.py +6 -0
- fleetpull/model_contract/coercions.py +33 -0
- fleetpull/model_contract/response.py +45 -0
- fleetpull/models/__init__.py +7 -0
- fleetpull/models/geotab/__init__.py +166 -0
- fleetpull/models/geotab/annotation_log.py +109 -0
- fleetpull/models/geotab/audit.py +56 -0
- fleetpull/models/geotab/device.py +217 -0
- fleetpull/models/geotab/driver_change.py +102 -0
- fleetpull/models/geotab/duty_status_log.py +200 -0
- fleetpull/models/geotab/dvir_log.py +184 -0
- fleetpull/models/geotab/exception_event.py +135 -0
- fleetpull/models/geotab/fault_data.py +168 -0
- fleetpull/models/geotab/fill_up.py +189 -0
- fleetpull/models/geotab/fuel_and_energy_used.py +81 -0
- fleetpull/models/geotab/fuel_tax_detail.py +150 -0
- fleetpull/models/geotab/log_record.py +68 -0
- fleetpull/models/geotab/media_file.py +133 -0
- fleetpull/models/geotab/shared.py +221 -0
- fleetpull/models/geotab/shipment_log.py +105 -0
- fleetpull/models/geotab/status_data.py +108 -0
- fleetpull/models/geotab/text_message.py +125 -0
- fleetpull/models/geotab/trip.py +162 -0
- fleetpull/models/geotab/user.py +187 -0
- fleetpull/models/motive/__init__.py +43 -0
- fleetpull/models/motive/driver_idle_rollup.py +109 -0
- fleetpull/models/motive/driving_period.py +82 -0
- fleetpull/models/motive/group.py +49 -0
- fleetpull/models/motive/idle_event.py +77 -0
- fleetpull/models/motive/shared.py +192 -0
- fleetpull/models/motive/user.py +217 -0
- fleetpull/models/motive/vehicle.py +162 -0
- fleetpull/models/motive/vehicle_location.py +127 -0
- fleetpull/models/motive/vehicle_utilization.py +128 -0
- fleetpull/models/samsara/__init__.py +108 -0
- fleetpull/models/samsara/address.py +149 -0
- fleetpull/models/samsara/asset_location.py +131 -0
- fleetpull/models/samsara/driver.py +232 -0
- fleetpull/models/samsara/driver_fuel_energy_report.py +153 -0
- fleetpull/models/samsara/driver_vehicle_assignment.py +168 -0
- fleetpull/models/samsara/engine_state.py +92 -0
- fleetpull/models/samsara/gps_reading.py +146 -0
- fleetpull/models/samsara/idling_event.py +174 -0
- fleetpull/models/samsara/odometer_reading.py +90 -0
- fleetpull/models/samsara/trip.py +199 -0
- fleetpull/models/samsara/vehicle.py +167 -0
- fleetpull/models/samsara/vehicle_fuel_energy_report.py +191 -0
- fleetpull/network/__init__.py +9 -0
- fleetpull/network/auth/__init__.py +15 -0
- fleetpull/network/auth/authenticate.py +271 -0
- fleetpull/network/auth/manager.py +223 -0
- fleetpull/network/auth/models.py +57 -0
- fleetpull/network/auth/strategies.py +169 -0
- fleetpull/network/classifiers/__init__.py +11 -0
- fleetpull/network/classifiers/geotab.py +145 -0
- fleetpull/network/classifiers/motive.py +79 -0
- fleetpull/network/classifiers/samsara.py +80 -0
- fleetpull/network/client/__init__.py +25 -0
- fleetpull/network/client/page.py +32 -0
- fleetpull/network/client/profile.py +27 -0
- fleetpull/network/client/registry.py +103 -0
- fleetpull/network/client/registry_base.py +150 -0
- fleetpull/network/client/runtime.py +44 -0
- fleetpull/network/client/transport.py +381 -0
- fleetpull/network/contract/__init__.py +50 -0
- fleetpull/network/contract/auth.py +48 -0
- fleetpull/network/contract/classifier.py +189 -0
- fleetpull/network/contract/envelope_fetcher.py +33 -0
- fleetpull/network/contract/envelopes.py +189 -0
- fleetpull/network/contract/outcome.py +43 -0
- fleetpull/network/contract/page_decoder.py +101 -0
- fleetpull/network/contract/request.py +97 -0
- fleetpull/network/decoders/__init__.py +40 -0
- fleetpull/network/decoders/_window_stamp.py +78 -0
- fleetpull/network/decoders/geotab.py +309 -0
- fleetpull/network/decoders/motive.py +204 -0
- fleetpull/network/decoders/motive_reports.py +114 -0
- fleetpull/network/decoders/samsara.py +343 -0
- fleetpull/network/decoders/samsara_reports.py +119 -0
- fleetpull/network/decoders/single_page.py +55 -0
- fleetpull/network/limits/__init__.py +13 -0
- fleetpull/network/limits/bucket_math.py +58 -0
- fleetpull/network/limits/limiter.py +153 -0
- fleetpull/network/limits/registry.py +96 -0
- fleetpull/network/posture/__init__.py +6 -0
- fleetpull/network/posture/client_options.py +96 -0
- fleetpull/network/retry/__init__.py +13 -0
- fleetpull/network/retry/decision.py +155 -0
- fleetpull/network/tls/__init__.py +5 -0
- fleetpull/network/tls/truststore_context.py +35 -0
- fleetpull/orchestrator/__init__.py +30 -0
- fleetpull/orchestrator/backfill.py +127 -0
- fleetpull/orchestrator/batch.py +155 -0
- fleetpull/orchestrator/bisection.py +271 -0
- fleetpull/orchestrator/drivers.py +350 -0
- fleetpull/orchestrator/entry.py +347 -0
- fleetpull/orchestrator/executors.py +128 -0
- fleetpull/orchestrator/fanout.py +163 -0
- fleetpull/orchestrator/feed_drive.py +266 -0
- fleetpull/orchestrator/metadata_projection.py +195 -0
- fleetpull/orchestrator/outcome.py +53 -0
- fleetpull/orchestrator/recording.py +85 -0
- fleetpull/orchestrator/resume.py +143 -0
- fleetpull/orchestrator/roster_harvest.py +108 -0
- fleetpull/orchestrator/roster_refresh.py +363 -0
- fleetpull/orchestrator/runner.py +262 -0
- fleetpull/orchestrator/shape_resolution.py +221 -0
- fleetpull/orchestrator/spine.py +166 -0
- fleetpull/orchestrator/streaming.py +153 -0
- fleetpull/orchestrator/unit_loop.py +273 -0
- fleetpull/orchestrator/watermark_drive.py +455 -0
- fleetpull/paths/__init__.py +13 -0
- fleetpull/paths/datasets.py +38 -0
- fleetpull/paths/partitions.py +41 -0
- fleetpull/paths/resolution.py +98 -0
- fleetpull/polars_typing/__init__.py +20 -0
- fleetpull/py.typed +0 -0
- fleetpull/records/__init__.py +19 -0
- fleetpull/records/convert.py +53 -0
- fleetpull/records/dataframe.py +63 -0
- fleetpull/records/event_time.py +70 -0
- fleetpull/records/fields.py +190 -0
- fleetpull/records/flatten.py +54 -0
- fleetpull/records/roster_members.py +74 -0
- fleetpull/records/schema.py +73 -0
- fleetpull/records/validation.py +62 -0
- fleetpull/resources/__init__.py +10 -0
- fleetpull/resources/config.example.yaml +222 -0
- fleetpull/roster/__init__.py +16 -0
- fleetpull/roster/definition.py +46 -0
- fleetpull/roster/key.py +40 -0
- fleetpull/roster/registry.py +93 -0
- fleetpull/state/__init__.py +36 -0
- fleetpull/state/cursors.py +417 -0
- fleetpull/state/database.py +463 -0
- fleetpull/state/migrations.py +430 -0
- fleetpull/state/reconcile.py +127 -0
- fleetpull/state/rosters.py +159 -0
- fleetpull/state/run_ledger.py +572 -0
- fleetpull/state/work_units.py +648 -0
- fleetpull/storage/__init__.py +30 -0
- fleetpull/storage/append.py +187 -0
- fleetpull/storage/atomic.py +161 -0
- fleetpull/storage/files.py +176 -0
- fleetpull/storage/frames.py +98 -0
- fleetpull/storage/metadata.py +161 -0
- fleetpull/storage/partitioned.py +205 -0
- fleetpull/storage/pruning.py +169 -0
- fleetpull/storage/read.py +35 -0
- fleetpull/storage/result.py +34 -0
- fleetpull/storage/single_file.py +130 -0
- fleetpull/storage/splitting.py +88 -0
- fleetpull/storage/staging.py +178 -0
- fleetpull/storage/writers.py +145 -0
- fleetpull/timing/__init__.py +21 -0
- fleetpull/timing/canon.py +92 -0
- fleetpull/timing/clock.py +192 -0
- fleetpull/timing/codec.py +122 -0
- fleetpull/timing/sleeper.py +65 -0
- fleetpull/vocabulary/__init__.py +16 -0
- fleetpull/vocabulary/json_types.py +20 -0
- fleetpull/vocabulary/provider.py +29 -0
- fleetpull/vocabulary/quota_scope.py +47 -0
- fleetpull/vocabulary/response_category.py +32 -0
- fleetpull-0.1.0.dist-info/METADATA +248 -0
- fleetpull-0.1.0.dist-info/RECORD +252 -0
- fleetpull-0.1.0.dist-info/WHEEL +5 -0
- fleetpull-0.1.0.dist-info/entry_points.txt +2 -0
- fleetpull-0.1.0.dist-info/licenses/LICENSE +202 -0
- fleetpull-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# src/fleetpull/storage/single_file.py
|
|
2
|
+
"""The single-file writer family: one ``data.parquet`` rewritten each run.
|
|
3
|
+
|
|
4
|
+
``SingleFileWriter`` is the family's shared substance -- accumulate the run's
|
|
5
|
+
frames, then dedup the subclass's finalized frame and atomically rewrite the
|
|
6
|
+
single file -- and ``SnapshotWriter`` is the shipped ``(SINGLE, SnapshotMode)``
|
|
7
|
+
cell (full replacement, the prior file never read). The unbuilt single-file
|
|
8
|
+
date-window cell will join this family as the combine subclass (DESIGN §3).
|
|
9
|
+
``select_writer`` (``storage/writers.py``) is the routing face that constructs
|
|
10
|
+
these.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from abc import ABC, abstractmethod
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import polars as pl
|
|
17
|
+
|
|
18
|
+
from fleetpull.storage.atomic import atomic_write_parquet
|
|
19
|
+
from fleetpull.storage.files import data_file
|
|
20
|
+
from fleetpull.storage.frames import dedup_counting
|
|
21
|
+
from fleetpull.storage.result import WriteResult
|
|
22
|
+
|
|
23
|
+
__all__: list[str] = ['SingleFileWriter', 'SnapshotWriter']
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SingleFileWriter(ABC):
|
|
27
|
+
"""Writers whose dataset is one ``data.parquet`` rewritten each run.
|
|
28
|
+
|
|
29
|
+
Shared substance: accumulate the run's frames, and on ``finalize`` dedup the
|
|
30
|
+
subclass's finalized frame and atomically rewrite the single file. Subclasses
|
|
31
|
+
supply ``_finalize_frame`` -- the per-cell write semantic, which decides
|
|
32
|
+
whether the prior file is read at all. A snapshot does not read it (it
|
|
33
|
+
replaces); the unbuilt watermark single-file cell will (it combines with
|
|
34
|
+
prior rows), reading through ``read_parquet_if_exists`` itself. The
|
|
35
|
+
base never reads -- the read is opt-in by the subclass.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, target_dir: Path, *, drop_duplicates: bool = True) -> None:
|
|
39
|
+
"""Bind the writer to an endpoint directory.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
target_dir: The endpoint directory holding ``data.parquet``.
|
|
43
|
+
drop_duplicates: Whether ``finalize`` drops exact-duplicate rows
|
|
44
|
+
(``storage.drop_exact_duplicates``, default on).
|
|
45
|
+
|
|
46
|
+
Side Effects:
|
|
47
|
+
None.
|
|
48
|
+
"""
|
|
49
|
+
self._target_dir = target_dir
|
|
50
|
+
self._drop_duplicates = drop_duplicates
|
|
51
|
+
self._frames: list[pl.DataFrame] = []
|
|
52
|
+
|
|
53
|
+
def write(self, frame: pl.DataFrame) -> None:
|
|
54
|
+
"""Accumulate one frame for this run.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
frame: A validated, flattened frame for this endpoint.
|
|
58
|
+
|
|
59
|
+
Side Effects:
|
|
60
|
+
Holds a reference to ``frame`` until ``finalize``.
|
|
61
|
+
"""
|
|
62
|
+
self._frames.append(frame)
|
|
63
|
+
|
|
64
|
+
def finalize(self) -> WriteResult:
|
|
65
|
+
"""Dedup (unless off) the subclass's finalized frame and write it.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
The write report (``files_written`` is always ``1``).
|
|
69
|
+
|
|
70
|
+
Side Effects:
|
|
71
|
+
Atomically rewrites ``data.parquet`` under the endpoint directory.
|
|
72
|
+
"""
|
|
73
|
+
written, duplicates_dropped = dedup_counting(
|
|
74
|
+
self._finalize_frame(), enabled=self._drop_duplicates
|
|
75
|
+
)
|
|
76
|
+
atomic_write_parquet(written, data_file(self._target_dir))
|
|
77
|
+
return WriteResult(
|
|
78
|
+
rows_written=written.height,
|
|
79
|
+
duplicates_dropped=duplicates_dropped,
|
|
80
|
+
files_written=1,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
@abstractmethod
|
|
84
|
+
def _finalize_frame(self) -> pl.DataFrame:
|
|
85
|
+
"""Produce the frame to dedup and write for this run.
|
|
86
|
+
|
|
87
|
+
The per-cell write semantic. A replace cell returns its accumulated frame;
|
|
88
|
+
a combine cell reads the prior file (via ``read_parquet_if_exists``) and
|
|
89
|
+
merges it with the accumulated frame. Reading the prior file is the
|
|
90
|
+
subclass's choice -- the base never reads.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
The frame to dedup and write.
|
|
94
|
+
|
|
95
|
+
Side Effects:
|
|
96
|
+
A combine subclass reads ``data.parquet``; a replace subclass does not.
|
|
97
|
+
"""
|
|
98
|
+
...
|
|
99
|
+
|
|
100
|
+
def _accumulated(self) -> pl.DataFrame:
|
|
101
|
+
"""This run's accumulated ``write`` frames as one frame.
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
The concatenation of every frame passed to ``write``. ``write`` must
|
|
105
|
+
have been called at least once.
|
|
106
|
+
|
|
107
|
+
Side Effects:
|
|
108
|
+
None.
|
|
109
|
+
"""
|
|
110
|
+
return pl.concat(self._frames)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class SnapshotWriter(SingleFileWriter):
|
|
114
|
+
"""``snapshot`` + ``single``: full replacement of the current-state dataset.
|
|
115
|
+
|
|
116
|
+
A snapshot re-fetches its whole current-state dataset every run, so the result
|
|
117
|
+
is just this run's accumulated frame and the prior file is never read -- it is
|
|
118
|
+
overwritten by the atomic rename (DESIGN §3).
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def _finalize_frame(self) -> pl.DataFrame:
|
|
122
|
+
"""Return this run's accumulated frame; the prior file is not read.
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
The accumulated frame, replacing the prior dataset wholesale.
|
|
126
|
+
|
|
127
|
+
Side Effects:
|
|
128
|
+
None -- the prior ``data.parquet`` is overwritten without being read.
|
|
129
|
+
"""
|
|
130
|
+
return self._accumulated()
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# src/fleetpull/storage/splitting.py
|
|
2
|
+
"""Date-partition splitting: group a records frame into per-UTC-date sub-frames.
|
|
3
|
+
|
|
4
|
+
The write-unit decomposition the date-partitioned layout (the ``StorageKind``
|
|
5
|
+
date_partitioned arm) iterates over. A fetch arrives as one frame whose rows may
|
|
6
|
+
span several calendar dates; the date-partitioned layout writes one parquet file
|
|
7
|
+
per date (``date=YYYY-MM-DD/part.parquet``), so something must say which rows
|
|
8
|
+
belong to which date's file. That is this module's only job: read the event-time
|
|
9
|
+
column, take the UTC calendar date off each row, and group the rows by it.
|
|
10
|
+
|
|
11
|
+
A pure leaf -- stdlib ``date`` and Polars only, nothing internal. It does not
|
|
12
|
+
touch the filesystem, does not know what a partition path looks like, does not
|
|
13
|
+
read existing data, and does not merge, dedup, or filter by window; those are the
|
|
14
|
+
layout's and the merge's concerns. Pure frame -> list of (date, frame).
|
|
15
|
+
|
|
16
|
+
The date is the UTC calendar date, because every timestamp in fleetpull is UTC
|
|
17
|
+
end to end; the partition boundary is therefore UTC midnight, which lines the
|
|
18
|
+
partitions up exactly with the half-open ``[start, end)`` fetch window so no
|
|
19
|
+
event double-counts at an edge. Row order within a partition is the input's order
|
|
20
|
+
unchanged -- this function imposes no sort, that being out of scope (parquet is
|
|
21
|
+
order-indifferent and the end use is not fleetpull's to anticipate).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from datetime import date
|
|
25
|
+
|
|
26
|
+
import polars as pl
|
|
27
|
+
|
|
28
|
+
__all__: list[str] = ['split_by_date']
|
|
29
|
+
|
|
30
|
+
_PARTITION_DATE_COLUMN: str = '_partition_date'
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def split_by_date(
|
|
34
|
+
frame: pl.DataFrame, event_time_column: str
|
|
35
|
+
) -> list[tuple[date, pl.DataFrame]]:
|
|
36
|
+
"""Group a frame's rows into per-UTC-date sub-frames.
|
|
37
|
+
|
|
38
|
+
Derives the UTC calendar date of each row from ``event_time_column`` and
|
|
39
|
+
partitions the frame on it, returning one ``(date, sub_frame)`` pair per
|
|
40
|
+
distinct date present. The derived date is dropped from each sub-frame before
|
|
41
|
+
return -- the date lives in the partition path (hive ``date=YYYY-MM-DD``), not
|
|
42
|
+
inside the file, so storing it would be redundant and could collide with the
|
|
43
|
+
path-derived column on read. Every other column, and the precise
|
|
44
|
+
``event_time_column`` value itself, is carried through unchanged.
|
|
45
|
+
|
|
46
|
+
An empty frame yields an empty list: an empty frame has no dates, hence zero
|
|
47
|
+
partitions, so the caller writes nothing and no empty ``date=.../`` directory
|
|
48
|
+
is created -- a missing date in the on-disk tree therefore always reflects a
|
|
49
|
+
real absence of data, never a silent gap.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
frame: The records frame to split; ``event_time_column`` must be a UTC
|
|
53
|
+
datetime column. Expected to hold one fetch's worth of records (e.g.
|
|
54
|
+
one vehicle's locations over a window), kept small upstream so the
|
|
55
|
+
materialized per-date sub-frames stay small.
|
|
56
|
+
event_time_column: Name of the UTC datetime column whose calendar date
|
|
57
|
+
keys the partitions (e.g. ``'located_at'``).
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
One ``(date, sub_frame)`` pair per distinct UTC date, each key a
|
|
61
|
+
``datetime.date`` and each sub-frame that date's rows with the derived
|
|
62
|
+
date column removed. Empty when ``frame`` is empty. Rows within a
|
|
63
|
+
sub-frame keep their input order; the order of the pairs is unspecified
|
|
64
|
+
beyond being deterministic for a given input.
|
|
65
|
+
|
|
66
|
+
Raises:
|
|
67
|
+
polars.exceptions.ColumnNotFoundError: ``event_time_column`` is absent
|
|
68
|
+
from ``frame`` -- a caller bug, surfaced unguarded by Polars.
|
|
69
|
+
polars.exceptions.PolarsError: ``event_time_column`` is not a temporal
|
|
70
|
+
dtype, so the date accessor cannot apply -- a caller bug, left to
|
|
71
|
+
propagate as Polars' own error.
|
|
72
|
+
|
|
73
|
+
Side Effects:
|
|
74
|
+
None -- pure function; ``frame`` is not mutated.
|
|
75
|
+
"""
|
|
76
|
+
if frame.height == 0:
|
|
77
|
+
return []
|
|
78
|
+
with_partition_date = frame.with_columns(
|
|
79
|
+
pl.col(event_time_column).dt.date().alias(_PARTITION_DATE_COLUMN)
|
|
80
|
+
)
|
|
81
|
+
partitions = with_partition_date.partition_by(
|
|
82
|
+
_PARTITION_DATE_COLUMN, maintain_order=True
|
|
83
|
+
)
|
|
84
|
+
result: list[tuple[date, pl.DataFrame]] = []
|
|
85
|
+
for partition_frame in partitions:
|
|
86
|
+
partition_date: date = partition_frame.get_column(_PARTITION_DATE_COLUMN)[0]
|
|
87
|
+
result.append((partition_date, partition_frame.drop(_PARTITION_DATE_COLUMN)))
|
|
88
|
+
return result
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# src/fleetpull/storage/staging.py
|
|
2
|
+
"""Date-partition staging and compaction: the write half of the date-partitioned
|
|
3
|
+
path (the prune in ``pruning.py`` is the delete half).
|
|
4
|
+
|
|
5
|
+
A date-partitioned watermark endpoint fans out -- the writer receives this run's
|
|
6
|
+
records one piece at a time. ``stage_shard`` lands each piece immediately as date-split shards under
|
|
7
|
+
``date=YYYY-MM-DD/staging/``, so no piece is held in memory waiting for the rest;
|
|
8
|
+
``compact_partition`` folds each date's shards into that date's single
|
|
9
|
+
``part.parquet``; ``clear_partition_staging`` removes staging the writer is done with
|
|
10
|
+
-- at construction a crashed run's stale shards, at finalize the shards just folded.
|
|
11
|
+
Three stateless functions, each one concern; the partitioned writer family
|
|
12
|
+
(``partitioned.py``) orchestrates
|
|
13
|
+
them and decides per cell whether compaction folds in the existing partition and
|
|
14
|
+
whether the run prunes (DESIGN §3).
|
|
15
|
+
|
|
16
|
+
Shards carry a ``.shard`` extension, not ``.parquet``, so a hive read of the live
|
|
17
|
+
dataset (``scan_parquet`` globbing ``**/*.parquet``, a BigQuery external table)
|
|
18
|
+
never picks up a half-staged partition's shards mid-run -- the queryable surface is
|
|
19
|
+
the ``part.parquet`` files alone. Compaction reads the shards back by explicit
|
|
20
|
+
path, which is format-driven, not extension-driven.
|
|
21
|
+
|
|
22
|
+
Memory is bounded by the chunk, not the endpoint (DESIGN §3): a partition holds one
|
|
23
|
+
chunk's rows for one date, materialized only at compaction and released after.
|
|
24
|
+
High-volume endpoints stay in bounds by a smaller date-chunk, not by streaming.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
import shutil
|
|
28
|
+
from collections.abc import Collection
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from datetime import date
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
import polars as pl
|
|
34
|
+
|
|
35
|
+
from fleetpull.storage.atomic import atomic_write_parquet
|
|
36
|
+
from fleetpull.storage.files import (
|
|
37
|
+
partition_dir,
|
|
38
|
+
partition_part_file,
|
|
39
|
+
partition_staging_dir,
|
|
40
|
+
partition_staging_shard,
|
|
41
|
+
)
|
|
42
|
+
from fleetpull.storage.frames import dedup_counting
|
|
43
|
+
from fleetpull.storage.splitting import split_by_date
|
|
44
|
+
|
|
45
|
+
__all__: list[str] = [
|
|
46
|
+
'CompactionResult',
|
|
47
|
+
'clear_partition_staging',
|
|
48
|
+
'compact_partition',
|
|
49
|
+
'stage_shard',
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True, slots=True)
|
|
54
|
+
class CompactionResult:
|
|
55
|
+
"""The row counts from compacting one partition.
|
|
56
|
+
|
|
57
|
+
Attributes:
|
|
58
|
+
rows_written: Rows in the written ``part.parquet`` (after dedup).
|
|
59
|
+
duplicates_dropped: Exact-duplicate rows removed during compaction.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
rows_written: int
|
|
63
|
+
duplicates_dropped: int
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def stage_shard(
|
|
67
|
+
endpoint_dir: Path, frame: pl.DataFrame, event_time_column: str
|
|
68
|
+
) -> set[date]:
|
|
69
|
+
"""Land one fetched piece as date-split shards under the partitions' staging.
|
|
70
|
+
|
|
71
|
+
Splits ``frame`` by the UTC date of ``event_time_column`` and writes each
|
|
72
|
+
date's rows as a uniquely-named ``.shard`` file under that date's ``staging/``
|
|
73
|
+
directory, atomically. An empty frame writes nothing. Returns the dates
|
|
74
|
+
touched so the writer knows which partitions to compact.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
endpoint_dir: The endpoint directory holding the ``date=`` partitions.
|
|
78
|
+
frame: One fetched piece (e.g. one vehicle's rows over the window).
|
|
79
|
+
event_time_column: The UTC datetime column whose date keys the partitions.
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
The set of UTC dates that received a shard (empty for an empty frame).
|
|
83
|
+
|
|
84
|
+
Side Effects:
|
|
85
|
+
Writes one ``.shard`` file per date present, creating staging directories
|
|
86
|
+
as needed.
|
|
87
|
+
"""
|
|
88
|
+
touched: set[date] = set()
|
|
89
|
+
for partition_date, sub_frame in split_by_date(frame, event_time_column):
|
|
90
|
+
atomic_write_parquet(
|
|
91
|
+
sub_frame, partition_staging_shard(endpoint_dir, partition_date)
|
|
92
|
+
)
|
|
93
|
+
touched.add(partition_date)
|
|
94
|
+
return touched
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def compact_partition(
|
|
98
|
+
endpoint_dir: Path,
|
|
99
|
+
partition_date: date,
|
|
100
|
+
existing: pl.DataFrame | None,
|
|
101
|
+
*,
|
|
102
|
+
drop_duplicates: bool = True,
|
|
103
|
+
) -> CompactionResult:
|
|
104
|
+
"""Fold one date's staged shards into its ``part.parquet``.
|
|
105
|
+
|
|
106
|
+
Reads every ``.shard`` under the date's ``staging/`` directory, concatenates
|
|
107
|
+
them (and ``existing`` if the cell folds in the prior partition), drops exact
|
|
108
|
+
duplicates unless the flag turns that off, and writes the result atomically
|
|
109
|
+
to ``part.parquet``, replacing any prior file. The whole partition is
|
|
110
|
+
materialized here; it is bounded by the chunk (DESIGN §3), not the endpoint.
|
|
111
|
+
Folds and nothing else -- the writer clears the staging afterward
|
|
112
|
+
(``clear_partition_staging``), keeping this single-concern.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
endpoint_dir: The endpoint directory holding the ``date=`` partitions.
|
|
116
|
+
partition_date: The date whose staged shards to compact; its ``staging/``
|
|
117
|
+
directory must hold at least one shard.
|
|
118
|
+
existing: The prior ``part.parquet`` contents to fold in (append cells), or
|
|
119
|
+
``None`` to replace the partition wholesale (watermark cells).
|
|
120
|
+
drop_duplicates: Whether to drop exact-duplicate rows (DESIGN §6 --
|
|
121
|
+
``storage.drop_exact_duplicates``, default on). ``False`` writes the
|
|
122
|
+
combined rows byte-for-byte.
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
The written partition's row counts.
|
|
126
|
+
|
|
127
|
+
Side Effects:
|
|
128
|
+
Writes ``part.parquet``. Leaves the staging shards in place for the caller
|
|
129
|
+
to clear.
|
|
130
|
+
"""
|
|
131
|
+
staging_dir = partition_staging_dir(endpoint_dir, partition_date)
|
|
132
|
+
shard_frames = [
|
|
133
|
+
pl.read_parquet(shard) for shard in sorted(staging_dir.glob('*.shard'))
|
|
134
|
+
]
|
|
135
|
+
if existing is not None:
|
|
136
|
+
shard_frames.append(existing)
|
|
137
|
+
combined = pl.concat(shard_frames)
|
|
138
|
+
written, duplicates_dropped = dedup_counting(combined, enabled=drop_duplicates)
|
|
139
|
+
atomic_write_parquet(written, partition_part_file(endpoint_dir, partition_date))
|
|
140
|
+
return CompactionResult(
|
|
141
|
+
rows_written=written.height, duplicates_dropped=duplicates_dropped
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def clear_partition_staging(
|
|
146
|
+
endpoint_dir: Path, partition_dates: Collection[date]
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Remove these dates' staging, dropping any ``date=`` directory it empties.
|
|
149
|
+
|
|
150
|
+
For each date: remove its ``staging/`` directory if present, then remove the
|
|
151
|
+
enclosing ``date=`` directory if that left it empty. The writer calls this in two
|
|
152
|
+
places. At construction, to sweep a crashed prior run's stale shards before
|
|
153
|
+
staging anything, so a later ``compact_partition`` folds only the live run's
|
|
154
|
+
shards; a date that died after ``stage_shard`` but before any ``part.parquet``
|
|
155
|
+
holds only ``staging/``, and an empty ``date=`` directory must not survive (it
|
|
156
|
+
would read as a partition with no data, the invariant ``delete_partition``
|
|
157
|
+
upholds), so it goes too. At finalize, to clear the shards ``compact_partition``
|
|
158
|
+
just folded; that date now holds a ``part.parquet``, so its directory survives.
|
|
159
|
+
Only a genuinely empty directory is removed -- a ``part.parquet`` or a crashed
|
|
160
|
+
atomic-write temp keeps it. Lenient on a missing ``staging/``: a date with none
|
|
161
|
+
is a no-op, not an error (DESIGN §3/§14).
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
endpoint_dir: The endpoint directory holding the ``date=`` partitions.
|
|
165
|
+
partition_dates: The dates whose staging to clear.
|
|
166
|
+
|
|
167
|
+
Side Effects:
|
|
168
|
+
Removes any ``staging/`` directory under those dates, and any ``date=``
|
|
169
|
+
directory left empty by that removal.
|
|
170
|
+
"""
|
|
171
|
+
for partition_date in partition_dates:
|
|
172
|
+
staging_dir = partition_staging_dir(endpoint_dir, partition_date)
|
|
173
|
+
if not staging_dir.exists():
|
|
174
|
+
continue
|
|
175
|
+
shutil.rmtree(staging_dir)
|
|
176
|
+
date_dir = partition_dir(endpoint_dir, partition_date)
|
|
177
|
+
if not any(date_dir.iterdir()):
|
|
178
|
+
date_dir.rmdir()
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# src/fleetpull/storage/writers.py
|
|
2
|
+
"""The dataset-writer contract and its routing face.
|
|
3
|
+
|
|
4
|
+
A ``DatasetWriter`` accepts this run's records in one or more pieces and finalizes
|
|
5
|
+
them onto disk as the endpoint's dataset. Each ``(StorageKind, SyncMode)`` cell is
|
|
6
|
+
its own writer, because the write semantics are not freely composable across the
|
|
7
|
+
two axes -- a floored watermark write *replaces* under date partitioning but
|
|
8
|
+
*clears and appends* under a single file, so the semantic depends on both axes at
|
|
9
|
+
once. The writer families live with their layouts -- the single-file family in
|
|
10
|
+
``storage/single_file.py``, the date-partitioned family in
|
|
11
|
+
``storage/partitioned.py``, the append-log feed cell in ``storage/append.py`` --
|
|
12
|
+
and this module owns the contract (``DatasetWriter``) and the one routing point
|
|
13
|
+
(``select_writer``).
|
|
14
|
+
|
|
15
|
+
The orchestrator drives every endpoint identically: ``select_writer`` returns the
|
|
16
|
+
cell's writer, the orchestrator calls ``write`` for each frame it has (once for a
|
|
17
|
+
snapshot, once per fanned-out unit for a partitioned watermark endpoint), then
|
|
18
|
+
``finalize`` once. The single-file date-window cell remains unbuilt.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from typing import Protocol
|
|
22
|
+
|
|
23
|
+
import polars as pl
|
|
24
|
+
|
|
25
|
+
from fleetpull.endpoints.shared import (
|
|
26
|
+
EndpointDefinition,
|
|
27
|
+
FeedMode,
|
|
28
|
+
SnapshotMode,
|
|
29
|
+
StorageKind,
|
|
30
|
+
WatermarkMode,
|
|
31
|
+
)
|
|
32
|
+
from fleetpull.incremental import DateWindow
|
|
33
|
+
from fleetpull.model_contract import ResponseModel
|
|
34
|
+
from fleetpull.paths import PathInput, endpoint_directory
|
|
35
|
+
from fleetpull.storage.append import FeedAppendWriter
|
|
36
|
+
from fleetpull.storage.partitioned import WatermarkPartitionedWriter
|
|
37
|
+
from fleetpull.storage.result import WriteResult
|
|
38
|
+
from fleetpull.storage.single_file import SnapshotWriter
|
|
39
|
+
|
|
40
|
+
__all__: list[str] = ['DatasetWriter', 'select_writer']
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class DatasetWriter(Protocol):
|
|
44
|
+
"""The storage write surface: accept records in pieces, finalize to disk.
|
|
45
|
+
|
|
46
|
+
An implementation is constructed for one endpoint's one run, with any runtime
|
|
47
|
+
context that run needs (e.g. an incremental ``window``). The orchestrator calls
|
|
48
|
+
``write`` for each frame it has, then ``finalize`` once. A plain Protocol --
|
|
49
|
+
composed and called through ``select_writer``, never dynamically verified.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def write(self, frame: pl.DataFrame) -> None:
|
|
53
|
+
"""Accept one piece of this run's records.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
frame: A validated, flattened frame for this endpoint. Called at least
|
|
57
|
+
once per run (a zero-row but typed frame is valid).
|
|
58
|
+
"""
|
|
59
|
+
...
|
|
60
|
+
|
|
61
|
+
def finalize(self) -> WriteResult:
|
|
62
|
+
"""Finalize the accepted records onto disk and report the write.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
The write report.
|
|
66
|
+
"""
|
|
67
|
+
...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def select_writer(
|
|
71
|
+
definition: EndpointDefinition[ResponseModel],
|
|
72
|
+
dataset_root: PathInput,
|
|
73
|
+
*,
|
|
74
|
+
window: DateWindow | None = None,
|
|
75
|
+
drop_duplicates: bool = True,
|
|
76
|
+
) -> DatasetWriter:
|
|
77
|
+
"""Construct the writer for an endpoint's ``(StorageKind, SyncMode)`` cell.
|
|
78
|
+
|
|
79
|
+
The single routing point. Resolves the endpoint directory under
|
|
80
|
+
``dataset_root`` and returns the cell's writer, constructed with the runtime
|
|
81
|
+
context that cell needs. ``window`` is the run's resume window -- consumed by
|
|
82
|
+
the incremental writers (the prune, the window-clear) and forbidden for the
|
|
83
|
+
snapshot cell, which has no resume.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
definition: The endpoint binding; supplies the provider / endpoint
|
|
87
|
+
directory names and the storage-kind / sync-mode cell.
|
|
88
|
+
dataset_root: The dataset root directory.
|
|
89
|
+
window: The run's half-open resume window, for incremental cells.
|
|
90
|
+
drop_duplicates: Whether the writer's compaction drops exact-duplicate
|
|
91
|
+
rows (``storage.drop_exact_duplicates``, default on).
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
The cell's ``DatasetWriter``.
|
|
95
|
+
|
|
96
|
+
Raises:
|
|
97
|
+
ValueError: A ``window`` was supplied for the snapshot or feed cell.
|
|
98
|
+
RuntimeError: An event-time-requiring cell carries no
|
|
99
|
+
``event_time_column`` (via ``required_event_time_column`` --
|
|
100
|
+
impossible past construction validation).
|
|
101
|
+
NotImplementedError: The endpoint's cell is not yet built (the
|
|
102
|
+
single-file date-window cell).
|
|
103
|
+
|
|
104
|
+
Side Effects:
|
|
105
|
+
None.
|
|
106
|
+
"""
|
|
107
|
+
target_dir = endpoint_directory(
|
|
108
|
+
dataset_root, definition.provider.value, definition.name
|
|
109
|
+
)
|
|
110
|
+
match (definition.storage_kind, definition.sync_mode):
|
|
111
|
+
case (StorageKind.SINGLE, SnapshotMode()):
|
|
112
|
+
if window is not None:
|
|
113
|
+
raise ValueError(
|
|
114
|
+
f'{definition.provider.value}.{definition.name}: snapshot '
|
|
115
|
+
f'endpoints have no resume window.'
|
|
116
|
+
)
|
|
117
|
+
return SnapshotWriter(target_dir, drop_duplicates=drop_duplicates)
|
|
118
|
+
case (StorageKind.APPEND_LOG, FeedMode()):
|
|
119
|
+
if window is not None:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
f'{definition.provider.value}.{definition.name}: feed '
|
|
122
|
+
f'endpoints have no resume window.'
|
|
123
|
+
)
|
|
124
|
+
# drop_duplicates is deliberately not threaded: the append-log
|
|
125
|
+
# cell performs no write-time dedup by design (stored-as-emitted,
|
|
126
|
+
# DESIGN section 4; storage/append.py carries the rationale).
|
|
127
|
+
return FeedAppendWriter(target_dir, definition.required_event_time_column)
|
|
128
|
+
case (StorageKind.DATE_PARTITIONED, WatermarkMode()):
|
|
129
|
+
if window is None:
|
|
130
|
+
raise ValueError(
|
|
131
|
+
f'{definition.provider.value}.{definition.name}: a watermark '
|
|
132
|
+
f'date-partitioned endpoint requires a resume window.'
|
|
133
|
+
)
|
|
134
|
+
return WatermarkPartitionedWriter(
|
|
135
|
+
target_dir,
|
|
136
|
+
definition.required_event_time_column,
|
|
137
|
+
window,
|
|
138
|
+
drop_duplicates=drop_duplicates,
|
|
139
|
+
)
|
|
140
|
+
case _:
|
|
141
|
+
raise NotImplementedError(
|
|
142
|
+
f'no writer for storage_kind={definition.storage_kind} '
|
|
143
|
+
f'sync_mode={type(definition.sync_mode).__name__} yet '
|
|
144
|
+
f'(the single-file watermark cell is not built)'
|
|
145
|
+
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# src/fleetpull/timing/__init__.py
|
|
2
|
+
"""Injectable time abstraction (Clock, Sleeper), pure UTC datetime conversions,
|
|
3
|
+
and the canonical-UTC surface (ensure_utc / require_utc)."""
|
|
4
|
+
|
|
5
|
+
from fleetpull.timing.canon import ensure_utc, require_utc
|
|
6
|
+
from fleetpull.timing.clock import Clock, FrozenClock, SystemClock
|
|
7
|
+
from fleetpull.timing.codec import from_iso8601, to_iso8601, to_utc_date_string
|
|
8
|
+
from fleetpull.timing.sleeper import Sleeper, SystemSleeper
|
|
9
|
+
|
|
10
|
+
__all__: list[str] = [
|
|
11
|
+
'Clock',
|
|
12
|
+
'FrozenClock',
|
|
13
|
+
'Sleeper',
|
|
14
|
+
'SystemClock',
|
|
15
|
+
'SystemSleeper',
|
|
16
|
+
'ensure_utc',
|
|
17
|
+
'from_iso8601',
|
|
18
|
+
'require_utc',
|
|
19
|
+
'to_iso8601',
|
|
20
|
+
'to_utc_date_string',
|
|
21
|
+
]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# src/fleetpull/timing/canon.py
|
|
2
|
+
"""The canonical-UTC surface: normalize at ingress, require in the interior.
|
|
3
|
+
|
|
4
|
+
The canonical interior temporal form is exactly one: a timezone-aware
|
|
5
|
+
``datetime`` whose ``tzinfo is datetime.UTC`` -- identity, not offset-equality.
|
|
6
|
+
``datetime.date`` serves calendar concepts (timezone-free by nature); strings
|
|
7
|
+
exist only at wire/storage edges via the codec. Two verbs enforce the form:
|
|
8
|
+
|
|
9
|
+
- **Ingress normalizes** (``ensure_utc``): any function bringing a temporal
|
|
10
|
+
value into the domain -- from a string, from a Polars frame, from config
|
|
11
|
+
-- converts it to canonical form, rejecting only the genuinely ambiguous
|
|
12
|
+
(a naive value is never assumed UTC). ``from_iso8601`` is the string
|
|
13
|
+
twin of this datetime-object verb.
|
|
14
|
+
- **Interior and egress require** (``require_utc``): strict identity
|
|
15
|
+
checks, never loosened. A strict failure in the interior means an
|
|
16
|
+
ingress was missed -- the fix is adding the missing ``ensure_utc``
|
|
17
|
+
boundary, never weakening the guard.
|
|
18
|
+
|
|
19
|
+
Identity rather than offset-equality is deliberate: a zero-offset foreign
|
|
20
|
+
tzinfo (Polars materializes ``zoneinfo.ZoneInfo('UTC')``; pydantic-core tags
|
|
21
|
+
its own ``TzInfo``) is the fingerprint of a value that entered the domain
|
|
22
|
+
without normalizing. An offset-equality check would accept it and mask the
|
|
23
|
+
missed ingress; the identity check finds it -- it is what caught the live
|
|
24
|
+
watermark-serialization crash.
|
|
25
|
+
|
|
26
|
+
Bad input raises stdlib ``ValueError``, never a ``FleetpullError`` -- a bad
|
|
27
|
+
temporal value is a caller bug or malformed input, and keeping the raise
|
|
28
|
+
stdlib is what lets ``timing`` stay a leaf below ``exceptions``, importing
|
|
29
|
+
nothing internal (the codec stance).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from datetime import UTC, datetime
|
|
33
|
+
|
|
34
|
+
__all__: list[str] = [
|
|
35
|
+
'ensure_utc',
|
|
36
|
+
'require_utc',
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def ensure_utc(moment: datetime) -> datetime:
|
|
41
|
+
"""
|
|
42
|
+
Normalize an aware datetime to canonical UTC; reject a naive one.
|
|
43
|
+
|
|
44
|
+
The ingress verb: converts via ``astimezone(UTC)`` so the result's tzinfo
|
|
45
|
+
*is* ``datetime.UTC`` regardless of the source tag (a foreign
|
|
46
|
+
``ZoneInfo('UTC')``, a fixed offset). A naive datetime is ambiguous and is
|
|
47
|
+
rejected, never assumed UTC -- the ``from_iso8601`` stance.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
moment: The aware datetime to normalize.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
The same instant with ``tzinfo`` ``datetime.UTC``.
|
|
54
|
+
|
|
55
|
+
Raises:
|
|
56
|
+
ValueError: If ``moment`` is naive.
|
|
57
|
+
"""
|
|
58
|
+
if moment.tzinfo is None:
|
|
59
|
+
raise ValueError(
|
|
60
|
+
'datetime must be timezone-aware to normalize; got a naive value '
|
|
61
|
+
'(never assumed UTC)'
|
|
62
|
+
)
|
|
63
|
+
return moment.astimezone(UTC)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def require_utc(moment: datetime) -> datetime:
|
|
67
|
+
"""
|
|
68
|
+
Validate that ``moment`` is canonical UTC; return it unchanged.
|
|
69
|
+
|
|
70
|
+
The guard verb: the check is identity against ``datetime.UTC``. The whole
|
|
71
|
+
codebase produces UTC datetimes via ``datetime.UTC`` / ``tz=UTC``, so a
|
|
72
|
+
different tzinfo -- even a zero-offset one -- signals a value that did not
|
|
73
|
+
pass an ingress (``ensure_utc`` / ``from_iso8601``) and is rejected rather
|
|
74
|
+
than silently coerced.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
moment: The datetime to validate.
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
``moment`` unchanged, once validated.
|
|
81
|
+
|
|
82
|
+
Raises:
|
|
83
|
+
ValueError: If ``moment`` is naive, or its tzinfo is not
|
|
84
|
+
``datetime.UTC``.
|
|
85
|
+
"""
|
|
86
|
+
if moment.tzinfo is None:
|
|
87
|
+
raise ValueError('datetime must be timezone-aware (UTC); got a naive value')
|
|
88
|
+
if moment.tzinfo is not UTC:
|
|
89
|
+
raise ValueError(
|
|
90
|
+
f'datetime must use datetime.UTC; got tzinfo={moment.tzinfo!r}'
|
|
91
|
+
)
|
|
92
|
+
return moment
|