fleetpull 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (252) hide show
  1. fleetpull/__init__.py +36 -0
  2. fleetpull/api/__init__.py +32 -0
  3. fleetpull/api/auth_ingress.py +216 -0
  4. fleetpull/api/catalog.py +139 -0
  5. fleetpull/api/fetch.py +229 -0
  6. fleetpull/api/identity.py +90 -0
  7. fleetpull/api/sync.py +735 -0
  8. fleetpull/cli.py +146 -0
  9. fleetpull/config/__init__.py +47 -0
  10. fleetpull/config/base.py +18 -0
  11. fleetpull/config/example.py +90 -0
  12. fleetpull/config/geotab.py +78 -0
  13. fleetpull/config/http.py +32 -0
  14. fleetpull/config/loading.py +195 -0
  15. fleetpull/config/logger.py +139 -0
  16. fleetpull/config/providers.py +520 -0
  17. fleetpull/config/rate_limit.py +50 -0
  18. fleetpull/config/resolution.py +156 -0
  19. fleetpull/config/retry.py +69 -0
  20. fleetpull/config/root.py +157 -0
  21. fleetpull/config/sections.py +146 -0
  22. fleetpull/endpoints/__init__.py +24 -0
  23. fleetpull/endpoints/geotab/__init__.py +12 -0
  24. fleetpull/endpoints/geotab/_requests.py +427 -0
  25. fleetpull/endpoints/geotab/annotation_logs.py +73 -0
  26. fleetpull/endpoints/geotab/audits.py +70 -0
  27. fleetpull/endpoints/geotab/devices.py +79 -0
  28. fleetpull/endpoints/geotab/driver_changes.py +73 -0
  29. fleetpull/endpoints/geotab/duty_status_logs.py +75 -0
  30. fleetpull/endpoints/geotab/dvir_logs.py +76 -0
  31. fleetpull/endpoints/geotab/exception_events.py +119 -0
  32. fleetpull/endpoints/geotab/fault_data.py +72 -0
  33. fleetpull/endpoints/geotab/fill_ups.py +79 -0
  34. fleetpull/endpoints/geotab/fuel_and_energy_used.py +74 -0
  35. fleetpull/endpoints/geotab/fuel_tax_details.py +75 -0
  36. fleetpull/endpoints/geotab/log_records.py +74 -0
  37. fleetpull/endpoints/geotab/media_files.py +74 -0
  38. fleetpull/endpoints/geotab/shipment_logs.py +71 -0
  39. fleetpull/endpoints/geotab/status_data.py +71 -0
  40. fleetpull/endpoints/geotab/text_messages.py +77 -0
  41. fleetpull/endpoints/geotab/trips.py +102 -0
  42. fleetpull/endpoints/geotab/users.py +81 -0
  43. fleetpull/endpoints/motive/__init__.py +12 -0
  44. fleetpull/endpoints/motive/_spec_builders.py +94 -0
  45. fleetpull/endpoints/motive/driver_idle_rollups.py +113 -0
  46. fleetpull/endpoints/motive/driving_periods.py +85 -0
  47. fleetpull/endpoints/motive/groups.py +61 -0
  48. fleetpull/endpoints/motive/idle_events.py +94 -0
  49. fleetpull/endpoints/motive/users.py +64 -0
  50. fleetpull/endpoints/motive/vehicle_locations.py +170 -0
  51. fleetpull/endpoints/motive/vehicle_utilizations.py +122 -0
  52. fleetpull/endpoints/motive/vehicles.py +106 -0
  53. fleetpull/endpoints/registry.py +280 -0
  54. fleetpull/endpoints/samsara/__init__.py +12 -0
  55. fleetpull/endpoints/samsara/_spec_builders.py +202 -0
  56. fleetpull/endpoints/samsara/addresses.py +77 -0
  57. fleetpull/endpoints/samsara/asset_locations.py +217 -0
  58. fleetpull/endpoints/samsara/driver_fuel_energy_reports.py +124 -0
  59. fleetpull/endpoints/samsara/driver_vehicle_assignments.py +211 -0
  60. fleetpull/endpoints/samsara/drivers.py +169 -0
  61. fleetpull/endpoints/samsara/engine_states.py +114 -0
  62. fleetpull/endpoints/samsara/gps_readings.py +113 -0
  63. fleetpull/endpoints/samsara/idling_events.py +195 -0
  64. fleetpull/endpoints/samsara/odometer_readings.py +116 -0
  65. fleetpull/endpoints/samsara/trips.py +217 -0
  66. fleetpull/endpoints/samsara/vehicle_fuel_energy_reports.py +139 -0
  67. fleetpull/endpoints/samsara/vehicles.py +121 -0
  68. fleetpull/endpoints/shared/__init__.py +57 -0
  69. fleetpull/endpoints/shared/base.py +529 -0
  70. fleetpull/endpoints/shared/request_shape.py +227 -0
  71. fleetpull/endpoints/shared/resume.py +74 -0
  72. fleetpull/endpoints/shared/spec_builders.py +59 -0
  73. fleetpull/endpoints/shared/sync_mode.py +159 -0
  74. fleetpull/endpoints/shared/url_paths.py +149 -0
  75. fleetpull/exceptions.py +320 -0
  76. fleetpull/incremental/__init__.py +27 -0
  77. fleetpull/incremental/cursor.py +66 -0
  78. fleetpull/incremental/resolution.py +152 -0
  79. fleetpull/incremental/seed.py +53 -0
  80. fleetpull/incremental/window.py +94 -0
  81. fleetpull/logger/__init__.py +5 -0
  82. fleetpull/logger/setup.py +145 -0
  83. fleetpull/model_contract/__init__.py +6 -0
  84. fleetpull/model_contract/coercions.py +33 -0
  85. fleetpull/model_contract/response.py +45 -0
  86. fleetpull/models/__init__.py +7 -0
  87. fleetpull/models/geotab/__init__.py +166 -0
  88. fleetpull/models/geotab/annotation_log.py +109 -0
  89. fleetpull/models/geotab/audit.py +56 -0
  90. fleetpull/models/geotab/device.py +217 -0
  91. fleetpull/models/geotab/driver_change.py +102 -0
  92. fleetpull/models/geotab/duty_status_log.py +200 -0
  93. fleetpull/models/geotab/dvir_log.py +184 -0
  94. fleetpull/models/geotab/exception_event.py +135 -0
  95. fleetpull/models/geotab/fault_data.py +168 -0
  96. fleetpull/models/geotab/fill_up.py +189 -0
  97. fleetpull/models/geotab/fuel_and_energy_used.py +81 -0
  98. fleetpull/models/geotab/fuel_tax_detail.py +150 -0
  99. fleetpull/models/geotab/log_record.py +68 -0
  100. fleetpull/models/geotab/media_file.py +133 -0
  101. fleetpull/models/geotab/shared.py +221 -0
  102. fleetpull/models/geotab/shipment_log.py +105 -0
  103. fleetpull/models/geotab/status_data.py +108 -0
  104. fleetpull/models/geotab/text_message.py +125 -0
  105. fleetpull/models/geotab/trip.py +162 -0
  106. fleetpull/models/geotab/user.py +187 -0
  107. fleetpull/models/motive/__init__.py +43 -0
  108. fleetpull/models/motive/driver_idle_rollup.py +109 -0
  109. fleetpull/models/motive/driving_period.py +82 -0
  110. fleetpull/models/motive/group.py +49 -0
  111. fleetpull/models/motive/idle_event.py +77 -0
  112. fleetpull/models/motive/shared.py +192 -0
  113. fleetpull/models/motive/user.py +217 -0
  114. fleetpull/models/motive/vehicle.py +162 -0
  115. fleetpull/models/motive/vehicle_location.py +127 -0
  116. fleetpull/models/motive/vehicle_utilization.py +128 -0
  117. fleetpull/models/samsara/__init__.py +108 -0
  118. fleetpull/models/samsara/address.py +149 -0
  119. fleetpull/models/samsara/asset_location.py +131 -0
  120. fleetpull/models/samsara/driver.py +232 -0
  121. fleetpull/models/samsara/driver_fuel_energy_report.py +153 -0
  122. fleetpull/models/samsara/driver_vehicle_assignment.py +168 -0
  123. fleetpull/models/samsara/engine_state.py +92 -0
  124. fleetpull/models/samsara/gps_reading.py +146 -0
  125. fleetpull/models/samsara/idling_event.py +174 -0
  126. fleetpull/models/samsara/odometer_reading.py +90 -0
  127. fleetpull/models/samsara/trip.py +199 -0
  128. fleetpull/models/samsara/vehicle.py +167 -0
  129. fleetpull/models/samsara/vehicle_fuel_energy_report.py +191 -0
  130. fleetpull/network/__init__.py +9 -0
  131. fleetpull/network/auth/__init__.py +15 -0
  132. fleetpull/network/auth/authenticate.py +271 -0
  133. fleetpull/network/auth/manager.py +223 -0
  134. fleetpull/network/auth/models.py +57 -0
  135. fleetpull/network/auth/strategies.py +169 -0
  136. fleetpull/network/classifiers/__init__.py +11 -0
  137. fleetpull/network/classifiers/geotab.py +145 -0
  138. fleetpull/network/classifiers/motive.py +79 -0
  139. fleetpull/network/classifiers/samsara.py +80 -0
  140. fleetpull/network/client/__init__.py +25 -0
  141. fleetpull/network/client/page.py +32 -0
  142. fleetpull/network/client/profile.py +27 -0
  143. fleetpull/network/client/registry.py +103 -0
  144. fleetpull/network/client/registry_base.py +150 -0
  145. fleetpull/network/client/runtime.py +44 -0
  146. fleetpull/network/client/transport.py +381 -0
  147. fleetpull/network/contract/__init__.py +50 -0
  148. fleetpull/network/contract/auth.py +48 -0
  149. fleetpull/network/contract/classifier.py +189 -0
  150. fleetpull/network/contract/envelope_fetcher.py +33 -0
  151. fleetpull/network/contract/envelopes.py +189 -0
  152. fleetpull/network/contract/outcome.py +43 -0
  153. fleetpull/network/contract/page_decoder.py +101 -0
  154. fleetpull/network/contract/request.py +97 -0
  155. fleetpull/network/decoders/__init__.py +40 -0
  156. fleetpull/network/decoders/_window_stamp.py +78 -0
  157. fleetpull/network/decoders/geotab.py +309 -0
  158. fleetpull/network/decoders/motive.py +204 -0
  159. fleetpull/network/decoders/motive_reports.py +114 -0
  160. fleetpull/network/decoders/samsara.py +343 -0
  161. fleetpull/network/decoders/samsara_reports.py +119 -0
  162. fleetpull/network/decoders/single_page.py +55 -0
  163. fleetpull/network/limits/__init__.py +13 -0
  164. fleetpull/network/limits/bucket_math.py +58 -0
  165. fleetpull/network/limits/limiter.py +153 -0
  166. fleetpull/network/limits/registry.py +96 -0
  167. fleetpull/network/posture/__init__.py +6 -0
  168. fleetpull/network/posture/client_options.py +96 -0
  169. fleetpull/network/retry/__init__.py +13 -0
  170. fleetpull/network/retry/decision.py +155 -0
  171. fleetpull/network/tls/__init__.py +5 -0
  172. fleetpull/network/tls/truststore_context.py +35 -0
  173. fleetpull/orchestrator/__init__.py +30 -0
  174. fleetpull/orchestrator/backfill.py +127 -0
  175. fleetpull/orchestrator/batch.py +155 -0
  176. fleetpull/orchestrator/bisection.py +271 -0
  177. fleetpull/orchestrator/drivers.py +350 -0
  178. fleetpull/orchestrator/entry.py +347 -0
  179. fleetpull/orchestrator/executors.py +128 -0
  180. fleetpull/orchestrator/fanout.py +163 -0
  181. fleetpull/orchestrator/feed_drive.py +266 -0
  182. fleetpull/orchestrator/metadata_projection.py +195 -0
  183. fleetpull/orchestrator/outcome.py +53 -0
  184. fleetpull/orchestrator/recording.py +85 -0
  185. fleetpull/orchestrator/resume.py +143 -0
  186. fleetpull/orchestrator/roster_harvest.py +108 -0
  187. fleetpull/orchestrator/roster_refresh.py +363 -0
  188. fleetpull/orchestrator/runner.py +262 -0
  189. fleetpull/orchestrator/shape_resolution.py +221 -0
  190. fleetpull/orchestrator/spine.py +166 -0
  191. fleetpull/orchestrator/streaming.py +153 -0
  192. fleetpull/orchestrator/unit_loop.py +273 -0
  193. fleetpull/orchestrator/watermark_drive.py +455 -0
  194. fleetpull/paths/__init__.py +13 -0
  195. fleetpull/paths/datasets.py +38 -0
  196. fleetpull/paths/partitions.py +41 -0
  197. fleetpull/paths/resolution.py +98 -0
  198. fleetpull/polars_typing/__init__.py +20 -0
  199. fleetpull/py.typed +0 -0
  200. fleetpull/records/__init__.py +19 -0
  201. fleetpull/records/convert.py +53 -0
  202. fleetpull/records/dataframe.py +63 -0
  203. fleetpull/records/event_time.py +70 -0
  204. fleetpull/records/fields.py +190 -0
  205. fleetpull/records/flatten.py +54 -0
  206. fleetpull/records/roster_members.py +74 -0
  207. fleetpull/records/schema.py +73 -0
  208. fleetpull/records/validation.py +62 -0
  209. fleetpull/resources/__init__.py +10 -0
  210. fleetpull/resources/config.example.yaml +222 -0
  211. fleetpull/roster/__init__.py +16 -0
  212. fleetpull/roster/definition.py +46 -0
  213. fleetpull/roster/key.py +40 -0
  214. fleetpull/roster/registry.py +93 -0
  215. fleetpull/state/__init__.py +36 -0
  216. fleetpull/state/cursors.py +417 -0
  217. fleetpull/state/database.py +463 -0
  218. fleetpull/state/migrations.py +430 -0
  219. fleetpull/state/reconcile.py +127 -0
  220. fleetpull/state/rosters.py +159 -0
  221. fleetpull/state/run_ledger.py +572 -0
  222. fleetpull/state/work_units.py +648 -0
  223. fleetpull/storage/__init__.py +30 -0
  224. fleetpull/storage/append.py +187 -0
  225. fleetpull/storage/atomic.py +161 -0
  226. fleetpull/storage/files.py +176 -0
  227. fleetpull/storage/frames.py +98 -0
  228. fleetpull/storage/metadata.py +161 -0
  229. fleetpull/storage/partitioned.py +205 -0
  230. fleetpull/storage/pruning.py +169 -0
  231. fleetpull/storage/read.py +35 -0
  232. fleetpull/storage/result.py +34 -0
  233. fleetpull/storage/single_file.py +130 -0
  234. fleetpull/storage/splitting.py +88 -0
  235. fleetpull/storage/staging.py +178 -0
  236. fleetpull/storage/writers.py +145 -0
  237. fleetpull/timing/__init__.py +21 -0
  238. fleetpull/timing/canon.py +92 -0
  239. fleetpull/timing/clock.py +192 -0
  240. fleetpull/timing/codec.py +122 -0
  241. fleetpull/timing/sleeper.py +65 -0
  242. fleetpull/vocabulary/__init__.py +16 -0
  243. fleetpull/vocabulary/json_types.py +20 -0
  244. fleetpull/vocabulary/provider.py +29 -0
  245. fleetpull/vocabulary/quota_scope.py +47 -0
  246. fleetpull/vocabulary/response_category.py +32 -0
  247. fleetpull-0.1.0.dist-info/METADATA +248 -0
  248. fleetpull-0.1.0.dist-info/RECORD +252 -0
  249. fleetpull-0.1.0.dist-info/WHEEL +5 -0
  250. fleetpull-0.1.0.dist-info/entry_points.txt +2 -0
  251. fleetpull-0.1.0.dist-info/licenses/LICENSE +202 -0
  252. fleetpull-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,130 @@
1
+ # src/fleetpull/storage/single_file.py
2
+ """The single-file writer family: one ``data.parquet`` rewritten each run.
3
+
4
+ ``SingleFileWriter`` is the family's shared substance -- accumulate the run's
5
+ frames, then dedup the subclass's finalized frame and atomically rewrite the
6
+ single file -- and ``SnapshotWriter`` is the shipped ``(SINGLE, SnapshotMode)``
7
+ cell (full replacement, the prior file never read). The unbuilt single-file
8
+ date-window cell will join this family as the combine subclass (DESIGN §3).
9
+ ``select_writer`` (``storage/writers.py``) is the routing face that constructs
10
+ these.
11
+ """
12
+
13
+ from abc import ABC, abstractmethod
14
+ from pathlib import Path
15
+
16
+ import polars as pl
17
+
18
+ from fleetpull.storage.atomic import atomic_write_parquet
19
+ from fleetpull.storage.files import data_file
20
+ from fleetpull.storage.frames import dedup_counting
21
+ from fleetpull.storage.result import WriteResult
22
+
23
+ __all__: list[str] = ['SingleFileWriter', 'SnapshotWriter']
24
+
25
+
26
+ class SingleFileWriter(ABC):
27
+ """Writers whose dataset is one ``data.parquet`` rewritten each run.
28
+
29
+ Shared substance: accumulate the run's frames, and on ``finalize`` dedup the
30
+ subclass's finalized frame and atomically rewrite the single file. Subclasses
31
+ supply ``_finalize_frame`` -- the per-cell write semantic, which decides
32
+ whether the prior file is read at all. A snapshot does not read it (it
33
+ replaces); the unbuilt watermark single-file cell will (it combines with
34
+ prior rows), reading through ``read_parquet_if_exists`` itself. The
35
+ base never reads -- the read is opt-in by the subclass.
36
+ """
37
+
38
+ def __init__(self, target_dir: Path, *, drop_duplicates: bool = True) -> None:
39
+ """Bind the writer to an endpoint directory.
40
+
41
+ Args:
42
+ target_dir: The endpoint directory holding ``data.parquet``.
43
+ drop_duplicates: Whether ``finalize`` drops exact-duplicate rows
44
+ (``storage.drop_exact_duplicates``, default on).
45
+
46
+ Side Effects:
47
+ None.
48
+ """
49
+ self._target_dir = target_dir
50
+ self._drop_duplicates = drop_duplicates
51
+ self._frames: list[pl.DataFrame] = []
52
+
53
+ def write(self, frame: pl.DataFrame) -> None:
54
+ """Accumulate one frame for this run.
55
+
56
+ Args:
57
+ frame: A validated, flattened frame for this endpoint.
58
+
59
+ Side Effects:
60
+ Holds a reference to ``frame`` until ``finalize``.
61
+ """
62
+ self._frames.append(frame)
63
+
64
+ def finalize(self) -> WriteResult:
65
+ """Dedup (unless off) the subclass's finalized frame and write it.
66
+
67
+ Returns:
68
+ The write report (``files_written`` is always ``1``).
69
+
70
+ Side Effects:
71
+ Atomically rewrites ``data.parquet`` under the endpoint directory.
72
+ """
73
+ written, duplicates_dropped = dedup_counting(
74
+ self._finalize_frame(), enabled=self._drop_duplicates
75
+ )
76
+ atomic_write_parquet(written, data_file(self._target_dir))
77
+ return WriteResult(
78
+ rows_written=written.height,
79
+ duplicates_dropped=duplicates_dropped,
80
+ files_written=1,
81
+ )
82
+
83
+ @abstractmethod
84
+ def _finalize_frame(self) -> pl.DataFrame:
85
+ """Produce the frame to dedup and write for this run.
86
+
87
+ The per-cell write semantic. A replace cell returns its accumulated frame;
88
+ a combine cell reads the prior file (via ``read_parquet_if_exists``) and
89
+ merges it with the accumulated frame. Reading the prior file is the
90
+ subclass's choice -- the base never reads.
91
+
92
+ Returns:
93
+ The frame to dedup and write.
94
+
95
+ Side Effects:
96
+ A combine subclass reads ``data.parquet``; a replace subclass does not.
97
+ """
98
+ ...
99
+
100
+ def _accumulated(self) -> pl.DataFrame:
101
+ """This run's accumulated ``write`` frames as one frame.
102
+
103
+ Returns:
104
+ The concatenation of every frame passed to ``write``. ``write`` must
105
+ have been called at least once.
106
+
107
+ Side Effects:
108
+ None.
109
+ """
110
+ return pl.concat(self._frames)
111
+
112
+
113
+ class SnapshotWriter(SingleFileWriter):
114
+ """``snapshot`` + ``single``: full replacement of the current-state dataset.
115
+
116
+ A snapshot re-fetches its whole current-state dataset every run, so the result
117
+ is just this run's accumulated frame and the prior file is never read -- it is
118
+ overwritten by the atomic rename (DESIGN §3).
119
+ """
120
+
121
+ def _finalize_frame(self) -> pl.DataFrame:
122
+ """Return this run's accumulated frame; the prior file is not read.
123
+
124
+ Returns:
125
+ The accumulated frame, replacing the prior dataset wholesale.
126
+
127
+ Side Effects:
128
+ None -- the prior ``data.parquet`` is overwritten without being read.
129
+ """
130
+ return self._accumulated()
@@ -0,0 +1,88 @@
1
+ # src/fleetpull/storage/splitting.py
2
+ """Date-partition splitting: group a records frame into per-UTC-date sub-frames.
3
+
4
+ The write-unit decomposition the date-partitioned layout (the ``StorageKind``
5
+ date_partitioned arm) iterates over. A fetch arrives as one frame whose rows may
6
+ span several calendar dates; the date-partitioned layout writes one parquet file
7
+ per date (``date=YYYY-MM-DD/part.parquet``), so something must say which rows
8
+ belong to which date's file. That is this module's only job: read the event-time
9
+ column, take the UTC calendar date off each row, and group the rows by it.
10
+
11
+ A pure leaf -- stdlib ``date`` and Polars only, nothing internal. It does not
12
+ touch the filesystem, does not know what a partition path looks like, does not
13
+ read existing data, and does not merge, dedup, or filter by window; those are the
14
+ layout's and the merge's concerns. Pure frame -> list of (date, frame).
15
+
16
+ The date is the UTC calendar date, because every timestamp in fleetpull is UTC
17
+ end to end; the partition boundary is therefore UTC midnight, which lines the
18
+ partitions up exactly with the half-open ``[start, end)`` fetch window so no
19
+ event double-counts at an edge. Row order within a partition is the input's order
20
+ unchanged -- this function imposes no sort, that being out of scope (parquet is
21
+ order-indifferent and the end use is not fleetpull's to anticipate).
22
+ """
23
+
24
+ from datetime import date
25
+
26
+ import polars as pl
27
+
28
+ __all__: list[str] = ['split_by_date']
29
+
30
+ _PARTITION_DATE_COLUMN: str = '_partition_date'
31
+
32
+
33
+ def split_by_date(
34
+ frame: pl.DataFrame, event_time_column: str
35
+ ) -> list[tuple[date, pl.DataFrame]]:
36
+ """Group a frame's rows into per-UTC-date sub-frames.
37
+
38
+ Derives the UTC calendar date of each row from ``event_time_column`` and
39
+ partitions the frame on it, returning one ``(date, sub_frame)`` pair per
40
+ distinct date present. The derived date is dropped from each sub-frame before
41
+ return -- the date lives in the partition path (hive ``date=YYYY-MM-DD``), not
42
+ inside the file, so storing it would be redundant and could collide with the
43
+ path-derived column on read. Every other column, and the precise
44
+ ``event_time_column`` value itself, is carried through unchanged.
45
+
46
+ An empty frame yields an empty list: an empty frame has no dates, hence zero
47
+ partitions, so the caller writes nothing and no empty ``date=.../`` directory
48
+ is created -- a missing date in the on-disk tree therefore always reflects a
49
+ real absence of data, never a silent gap.
50
+
51
+ Args:
52
+ frame: The records frame to split; ``event_time_column`` must be a UTC
53
+ datetime column. Expected to hold one fetch's worth of records (e.g.
54
+ one vehicle's locations over a window), kept small upstream so the
55
+ materialized per-date sub-frames stay small.
56
+ event_time_column: Name of the UTC datetime column whose calendar date
57
+ keys the partitions (e.g. ``'located_at'``).
58
+
59
+ Returns:
60
+ One ``(date, sub_frame)`` pair per distinct UTC date, each key a
61
+ ``datetime.date`` and each sub-frame that date's rows with the derived
62
+ date column removed. Empty when ``frame`` is empty. Rows within a
63
+ sub-frame keep their input order; the order of the pairs is unspecified
64
+ beyond being deterministic for a given input.
65
+
66
+ Raises:
67
+ polars.exceptions.ColumnNotFoundError: ``event_time_column`` is absent
68
+ from ``frame`` -- a caller bug, surfaced unguarded by Polars.
69
+ polars.exceptions.PolarsError: ``event_time_column`` is not a temporal
70
+ dtype, so the date accessor cannot apply -- a caller bug, left to
71
+ propagate as Polars' own error.
72
+
73
+ Side Effects:
74
+ None -- pure function; ``frame`` is not mutated.
75
+ """
76
+ if frame.height == 0:
77
+ return []
78
+ with_partition_date = frame.with_columns(
79
+ pl.col(event_time_column).dt.date().alias(_PARTITION_DATE_COLUMN)
80
+ )
81
+ partitions = with_partition_date.partition_by(
82
+ _PARTITION_DATE_COLUMN, maintain_order=True
83
+ )
84
+ result: list[tuple[date, pl.DataFrame]] = []
85
+ for partition_frame in partitions:
86
+ partition_date: date = partition_frame.get_column(_PARTITION_DATE_COLUMN)[0]
87
+ result.append((partition_date, partition_frame.drop(_PARTITION_DATE_COLUMN)))
88
+ return result
@@ -0,0 +1,178 @@
1
+ # src/fleetpull/storage/staging.py
2
+ """Date-partition staging and compaction: the write half of the date-partitioned
3
+ path (the prune in ``pruning.py`` is the delete half).
4
+
5
+ A date-partitioned watermark endpoint fans out -- the writer receives this run's
6
+ records one piece at a time. ``stage_shard`` lands each piece immediately as date-split shards under
7
+ ``date=YYYY-MM-DD/staging/``, so no piece is held in memory waiting for the rest;
8
+ ``compact_partition`` folds each date's shards into that date's single
9
+ ``part.parquet``; ``clear_partition_staging`` removes staging the writer is done with
10
+ -- at construction a crashed run's stale shards, at finalize the shards just folded.
11
+ Three stateless functions, each one concern; the partitioned writer family
12
+ (``partitioned.py``) orchestrates
13
+ them and decides per cell whether compaction folds in the existing partition and
14
+ whether the run prunes (DESIGN §3).
15
+
16
+ Shards carry a ``.shard`` extension, not ``.parquet``, so a hive read of the live
17
+ dataset (``scan_parquet`` globbing ``**/*.parquet``, a BigQuery external table)
18
+ never picks up a half-staged partition's shards mid-run -- the queryable surface is
19
+ the ``part.parquet`` files alone. Compaction reads the shards back by explicit
20
+ path, which is format-driven, not extension-driven.
21
+
22
+ Memory is bounded by the chunk, not the endpoint (DESIGN §3): a partition holds one
23
+ chunk's rows for one date, materialized only at compaction and released after.
24
+ High-volume endpoints stay in bounds by a smaller date-chunk, not by streaming.
25
+ """
26
+
27
+ import shutil
28
+ from collections.abc import Collection
29
+ from dataclasses import dataclass
30
+ from datetime import date
31
+ from pathlib import Path
32
+
33
+ import polars as pl
34
+
35
+ from fleetpull.storage.atomic import atomic_write_parquet
36
+ from fleetpull.storage.files import (
37
+ partition_dir,
38
+ partition_part_file,
39
+ partition_staging_dir,
40
+ partition_staging_shard,
41
+ )
42
+ from fleetpull.storage.frames import dedup_counting
43
+ from fleetpull.storage.splitting import split_by_date
44
+
45
+ __all__: list[str] = [
46
+ 'CompactionResult',
47
+ 'clear_partition_staging',
48
+ 'compact_partition',
49
+ 'stage_shard',
50
+ ]
51
+
52
+
53
+ @dataclass(frozen=True, slots=True)
54
+ class CompactionResult:
55
+ """The row counts from compacting one partition.
56
+
57
+ Attributes:
58
+ rows_written: Rows in the written ``part.parquet`` (after dedup).
59
+ duplicates_dropped: Exact-duplicate rows removed during compaction.
60
+ """
61
+
62
+ rows_written: int
63
+ duplicates_dropped: int
64
+
65
+
66
+ def stage_shard(
67
+ endpoint_dir: Path, frame: pl.DataFrame, event_time_column: str
68
+ ) -> set[date]:
69
+ """Land one fetched piece as date-split shards under the partitions' staging.
70
+
71
+ Splits ``frame`` by the UTC date of ``event_time_column`` and writes each
72
+ date's rows as a uniquely-named ``.shard`` file under that date's ``staging/``
73
+ directory, atomically. An empty frame writes nothing. Returns the dates
74
+ touched so the writer knows which partitions to compact.
75
+
76
+ Args:
77
+ endpoint_dir: The endpoint directory holding the ``date=`` partitions.
78
+ frame: One fetched piece (e.g. one vehicle's rows over the window).
79
+ event_time_column: The UTC datetime column whose date keys the partitions.
80
+
81
+ Returns:
82
+ The set of UTC dates that received a shard (empty for an empty frame).
83
+
84
+ Side Effects:
85
+ Writes one ``.shard`` file per date present, creating staging directories
86
+ as needed.
87
+ """
88
+ touched: set[date] = set()
89
+ for partition_date, sub_frame in split_by_date(frame, event_time_column):
90
+ atomic_write_parquet(
91
+ sub_frame, partition_staging_shard(endpoint_dir, partition_date)
92
+ )
93
+ touched.add(partition_date)
94
+ return touched
95
+
96
+
97
+ def compact_partition(
98
+ endpoint_dir: Path,
99
+ partition_date: date,
100
+ existing: pl.DataFrame | None,
101
+ *,
102
+ drop_duplicates: bool = True,
103
+ ) -> CompactionResult:
104
+ """Fold one date's staged shards into its ``part.parquet``.
105
+
106
+ Reads every ``.shard`` under the date's ``staging/`` directory, concatenates
107
+ them (and ``existing`` if the cell folds in the prior partition), drops exact
108
+ duplicates unless the flag turns that off, and writes the result atomically
109
+ to ``part.parquet``, replacing any prior file. The whole partition is
110
+ materialized here; it is bounded by the chunk (DESIGN §3), not the endpoint.
111
+ Folds and nothing else -- the writer clears the staging afterward
112
+ (``clear_partition_staging``), keeping this single-concern.
113
+
114
+ Args:
115
+ endpoint_dir: The endpoint directory holding the ``date=`` partitions.
116
+ partition_date: The date whose staged shards to compact; its ``staging/``
117
+ directory must hold at least one shard.
118
+ existing: The prior ``part.parquet`` contents to fold in (append cells), or
119
+ ``None`` to replace the partition wholesale (watermark cells).
120
+ drop_duplicates: Whether to drop exact-duplicate rows (DESIGN §6 --
121
+ ``storage.drop_exact_duplicates``, default on). ``False`` writes the
122
+ combined rows byte-for-byte.
123
+
124
+ Returns:
125
+ The written partition's row counts.
126
+
127
+ Side Effects:
128
+ Writes ``part.parquet``. Leaves the staging shards in place for the caller
129
+ to clear.
130
+ """
131
+ staging_dir = partition_staging_dir(endpoint_dir, partition_date)
132
+ shard_frames = [
133
+ pl.read_parquet(shard) for shard in sorted(staging_dir.glob('*.shard'))
134
+ ]
135
+ if existing is not None:
136
+ shard_frames.append(existing)
137
+ combined = pl.concat(shard_frames)
138
+ written, duplicates_dropped = dedup_counting(combined, enabled=drop_duplicates)
139
+ atomic_write_parquet(written, partition_part_file(endpoint_dir, partition_date))
140
+ return CompactionResult(
141
+ rows_written=written.height, duplicates_dropped=duplicates_dropped
142
+ )
143
+
144
+
145
+ def clear_partition_staging(
146
+ endpoint_dir: Path, partition_dates: Collection[date]
147
+ ) -> None:
148
+ """Remove these dates' staging, dropping any ``date=`` directory it empties.
149
+
150
+ For each date: remove its ``staging/`` directory if present, then remove the
151
+ enclosing ``date=`` directory if that left it empty. The writer calls this in two
152
+ places. At construction, to sweep a crashed prior run's stale shards before
153
+ staging anything, so a later ``compact_partition`` folds only the live run's
154
+ shards; a date that died after ``stage_shard`` but before any ``part.parquet``
155
+ holds only ``staging/``, and an empty ``date=`` directory must not survive (it
156
+ would read as a partition with no data, the invariant ``delete_partition``
157
+ upholds), so it goes too. At finalize, to clear the shards ``compact_partition``
158
+ just folded; that date now holds a ``part.parquet``, so its directory survives.
159
+ Only a genuinely empty directory is removed -- a ``part.parquet`` or a crashed
160
+ atomic-write temp keeps it. Lenient on a missing ``staging/``: a date with none
161
+ is a no-op, not an error (DESIGN §3/§14).
162
+
163
+ Args:
164
+ endpoint_dir: The endpoint directory holding the ``date=`` partitions.
165
+ partition_dates: The dates whose staging to clear.
166
+
167
+ Side Effects:
168
+ Removes any ``staging/`` directory under those dates, and any ``date=``
169
+ directory left empty by that removal.
170
+ """
171
+ for partition_date in partition_dates:
172
+ staging_dir = partition_staging_dir(endpoint_dir, partition_date)
173
+ if not staging_dir.exists():
174
+ continue
175
+ shutil.rmtree(staging_dir)
176
+ date_dir = partition_dir(endpoint_dir, partition_date)
177
+ if not any(date_dir.iterdir()):
178
+ date_dir.rmdir()
@@ -0,0 +1,145 @@
1
+ # src/fleetpull/storage/writers.py
2
+ """The dataset-writer contract and its routing face.
3
+
4
+ A ``DatasetWriter`` accepts this run's records in one or more pieces and finalizes
5
+ them onto disk as the endpoint's dataset. Each ``(StorageKind, SyncMode)`` cell is
6
+ its own writer, because the write semantics are not freely composable across the
7
+ two axes -- a floored watermark write *replaces* under date partitioning but
8
+ *clears and appends* under a single file, so the semantic depends on both axes at
9
+ once. The writer families live with their layouts -- the single-file family in
10
+ ``storage/single_file.py``, the date-partitioned family in
11
+ ``storage/partitioned.py``, the append-log feed cell in ``storage/append.py`` --
12
+ and this module owns the contract (``DatasetWriter``) and the one routing point
13
+ (``select_writer``).
14
+
15
+ The orchestrator drives every endpoint identically: ``select_writer`` returns the
16
+ cell's writer, the orchestrator calls ``write`` for each frame it has (once for a
17
+ snapshot, once per fanned-out unit for a partitioned watermark endpoint), then
18
+ ``finalize`` once. The single-file date-window cell remains unbuilt.
19
+ """
20
+
21
+ from typing import Protocol
22
+
23
+ import polars as pl
24
+
25
+ from fleetpull.endpoints.shared import (
26
+ EndpointDefinition,
27
+ FeedMode,
28
+ SnapshotMode,
29
+ StorageKind,
30
+ WatermarkMode,
31
+ )
32
+ from fleetpull.incremental import DateWindow
33
+ from fleetpull.model_contract import ResponseModel
34
+ from fleetpull.paths import PathInput, endpoint_directory
35
+ from fleetpull.storage.append import FeedAppendWriter
36
+ from fleetpull.storage.partitioned import WatermarkPartitionedWriter
37
+ from fleetpull.storage.result import WriteResult
38
+ from fleetpull.storage.single_file import SnapshotWriter
39
+
40
+ __all__: list[str] = ['DatasetWriter', 'select_writer']
41
+
42
+
43
+ class DatasetWriter(Protocol):
44
+ """The storage write surface: accept records in pieces, finalize to disk.
45
+
46
+ An implementation is constructed for one endpoint's one run, with any runtime
47
+ context that run needs (e.g. an incremental ``window``). The orchestrator calls
48
+ ``write`` for each frame it has, then ``finalize`` once. A plain Protocol --
49
+ composed and called through ``select_writer``, never dynamically verified.
50
+ """
51
+
52
+ def write(self, frame: pl.DataFrame) -> None:
53
+ """Accept one piece of this run's records.
54
+
55
+ Args:
56
+ frame: A validated, flattened frame for this endpoint. Called at least
57
+ once per run (a zero-row but typed frame is valid).
58
+ """
59
+ ...
60
+
61
+ def finalize(self) -> WriteResult:
62
+ """Finalize the accepted records onto disk and report the write.
63
+
64
+ Returns:
65
+ The write report.
66
+ """
67
+ ...
68
+
69
+
70
+ def select_writer(
71
+ definition: EndpointDefinition[ResponseModel],
72
+ dataset_root: PathInput,
73
+ *,
74
+ window: DateWindow | None = None,
75
+ drop_duplicates: bool = True,
76
+ ) -> DatasetWriter:
77
+ """Construct the writer for an endpoint's ``(StorageKind, SyncMode)`` cell.
78
+
79
+ The single routing point. Resolves the endpoint directory under
80
+ ``dataset_root`` and returns the cell's writer, constructed with the runtime
81
+ context that cell needs. ``window`` is the run's resume window -- consumed by
82
+ the incremental writers (the prune, the window-clear) and forbidden for the
83
+ snapshot cell, which has no resume.
84
+
85
+ Args:
86
+ definition: The endpoint binding; supplies the provider / endpoint
87
+ directory names and the storage-kind / sync-mode cell.
88
+ dataset_root: The dataset root directory.
89
+ window: The run's half-open resume window, for incremental cells.
90
+ drop_duplicates: Whether the writer's compaction drops exact-duplicate
91
+ rows (``storage.drop_exact_duplicates``, default on).
92
+
93
+ Returns:
94
+ The cell's ``DatasetWriter``.
95
+
96
+ Raises:
97
+ ValueError: A ``window`` was supplied for the snapshot or feed cell.
98
+ RuntimeError: An event-time-requiring cell carries no
99
+ ``event_time_column`` (via ``required_event_time_column`` --
100
+ impossible past construction validation).
101
+ NotImplementedError: The endpoint's cell is not yet built (the
102
+ single-file date-window cell).
103
+
104
+ Side Effects:
105
+ None.
106
+ """
107
+ target_dir = endpoint_directory(
108
+ dataset_root, definition.provider.value, definition.name
109
+ )
110
+ match (definition.storage_kind, definition.sync_mode):
111
+ case (StorageKind.SINGLE, SnapshotMode()):
112
+ if window is not None:
113
+ raise ValueError(
114
+ f'{definition.provider.value}.{definition.name}: snapshot '
115
+ f'endpoints have no resume window.'
116
+ )
117
+ return SnapshotWriter(target_dir, drop_duplicates=drop_duplicates)
118
+ case (StorageKind.APPEND_LOG, FeedMode()):
119
+ if window is not None:
120
+ raise ValueError(
121
+ f'{definition.provider.value}.{definition.name}: feed '
122
+ f'endpoints have no resume window.'
123
+ )
124
+ # drop_duplicates is deliberately not threaded: the append-log
125
+ # cell performs no write-time dedup by design (stored-as-emitted,
126
+ # DESIGN section 4; storage/append.py carries the rationale).
127
+ return FeedAppendWriter(target_dir, definition.required_event_time_column)
128
+ case (StorageKind.DATE_PARTITIONED, WatermarkMode()):
129
+ if window is None:
130
+ raise ValueError(
131
+ f'{definition.provider.value}.{definition.name}: a watermark '
132
+ f'date-partitioned endpoint requires a resume window.'
133
+ )
134
+ return WatermarkPartitionedWriter(
135
+ target_dir,
136
+ definition.required_event_time_column,
137
+ window,
138
+ drop_duplicates=drop_duplicates,
139
+ )
140
+ case _:
141
+ raise NotImplementedError(
142
+ f'no writer for storage_kind={definition.storage_kind} '
143
+ f'sync_mode={type(definition.sync_mode).__name__} yet '
144
+ f'(the single-file watermark cell is not built)'
145
+ )
@@ -0,0 +1,21 @@
1
+ # src/fleetpull/timing/__init__.py
2
+ """Injectable time abstraction (Clock, Sleeper), pure UTC datetime conversions,
3
+ and the canonical-UTC surface (ensure_utc / require_utc)."""
4
+
5
+ from fleetpull.timing.canon import ensure_utc, require_utc
6
+ from fleetpull.timing.clock import Clock, FrozenClock, SystemClock
7
+ from fleetpull.timing.codec import from_iso8601, to_iso8601, to_utc_date_string
8
+ from fleetpull.timing.sleeper import Sleeper, SystemSleeper
9
+
10
+ __all__: list[str] = [
11
+ 'Clock',
12
+ 'FrozenClock',
13
+ 'Sleeper',
14
+ 'SystemClock',
15
+ 'SystemSleeper',
16
+ 'ensure_utc',
17
+ 'from_iso8601',
18
+ 'require_utc',
19
+ 'to_iso8601',
20
+ 'to_utc_date_string',
21
+ ]
@@ -0,0 +1,92 @@
1
+ # src/fleetpull/timing/canon.py
2
+ """The canonical-UTC surface: normalize at ingress, require in the interior.
3
+
4
+ The canonical interior temporal form is exactly one: a timezone-aware
5
+ ``datetime`` whose ``tzinfo is datetime.UTC`` -- identity, not offset-equality.
6
+ ``datetime.date`` serves calendar concepts (timezone-free by nature); strings
7
+ exist only at wire/storage edges via the codec. Two verbs enforce the form:
8
+
9
+ - **Ingress normalizes** (``ensure_utc``): any function bringing a temporal
10
+ value into the domain -- from a string, from a Polars frame, from config
11
+ -- converts it to canonical form, rejecting only the genuinely ambiguous
12
+ (a naive value is never assumed UTC). ``from_iso8601`` is the string
13
+ twin of this datetime-object verb.
14
+ - **Interior and egress require** (``require_utc``): strict identity
15
+ checks, never loosened. A strict failure in the interior means an
16
+ ingress was missed -- the fix is adding the missing ``ensure_utc``
17
+ boundary, never weakening the guard.
18
+
19
+ Identity rather than offset-equality is deliberate: a zero-offset foreign
20
+ tzinfo (Polars materializes ``zoneinfo.ZoneInfo('UTC')``; pydantic-core tags
21
+ its own ``TzInfo``) is the fingerprint of a value that entered the domain
22
+ without normalizing. An offset-equality check would accept it and mask the
23
+ missed ingress; the identity check finds it -- it is what caught the live
24
+ watermark-serialization crash.
25
+
26
+ Bad input raises stdlib ``ValueError``, never a ``FleetpullError`` -- a bad
27
+ temporal value is a caller bug or malformed input, and keeping the raise
28
+ stdlib is what lets ``timing`` stay a leaf below ``exceptions``, importing
29
+ nothing internal (the codec stance).
30
+ """
31
+
32
+ from datetime import UTC, datetime
33
+
34
+ __all__: list[str] = [
35
+ 'ensure_utc',
36
+ 'require_utc',
37
+ ]
38
+
39
+
40
+ def ensure_utc(moment: datetime) -> datetime:
41
+ """
42
+ Normalize an aware datetime to canonical UTC; reject a naive one.
43
+
44
+ The ingress verb: converts via ``astimezone(UTC)`` so the result's tzinfo
45
+ *is* ``datetime.UTC`` regardless of the source tag (a foreign
46
+ ``ZoneInfo('UTC')``, a fixed offset). A naive datetime is ambiguous and is
47
+ rejected, never assumed UTC -- the ``from_iso8601`` stance.
48
+
49
+ Args:
50
+ moment: The aware datetime to normalize.
51
+
52
+ Returns:
53
+ The same instant with ``tzinfo`` ``datetime.UTC``.
54
+
55
+ Raises:
56
+ ValueError: If ``moment`` is naive.
57
+ """
58
+ if moment.tzinfo is None:
59
+ raise ValueError(
60
+ 'datetime must be timezone-aware to normalize; got a naive value '
61
+ '(never assumed UTC)'
62
+ )
63
+ return moment.astimezone(UTC)
64
+
65
+
66
+ def require_utc(moment: datetime) -> datetime:
67
+ """
68
+ Validate that ``moment`` is canonical UTC; return it unchanged.
69
+
70
+ The guard verb: the check is identity against ``datetime.UTC``. The whole
71
+ codebase produces UTC datetimes via ``datetime.UTC`` / ``tz=UTC``, so a
72
+ different tzinfo -- even a zero-offset one -- signals a value that did not
73
+ pass an ingress (``ensure_utc`` / ``from_iso8601``) and is rejected rather
74
+ than silently coerced.
75
+
76
+ Args:
77
+ moment: The datetime to validate.
78
+
79
+ Returns:
80
+ ``moment`` unchanged, once validated.
81
+
82
+ Raises:
83
+ ValueError: If ``moment`` is naive, or its tzinfo is not
84
+ ``datetime.UTC``.
85
+ """
86
+ if moment.tzinfo is None:
87
+ raise ValueError('datetime must be timezone-aware (UTC); got a naive value')
88
+ if moment.tzinfo is not UTC:
89
+ raise ValueError(
90
+ f'datetime must use datetime.UTC; got tzinfo={moment.tzinfo!r}'
91
+ )
92
+ return moment