fleetpull 0.1.0__py3-none-any.whl

fleetpull/resources/config.example.yaml

@@ -0,0 +1,222 @@
+# fleetpull configuration -- copy this file (e.g. to config.yaml), edit the
+# required values, and hand its path to the sync entry point
+# (FleetpullConfig.from_yaml). Commented-out keys are optional and show the
+# default that applies when the key is absent. Unknown keys are rejected at
+# any level. Every path key is normalized (~ expansion, relative-to-absolute)
+# at load time.
+
+sync:
+  # REQUIRED. The cold-start anchor: the UTC calendar date history begins
+  # from on a watermark endpoint's very first run. Goes inert once observed
+  # data or completed coverage exists.
+  default_start_date: 2026-06-01
+
+  # Package-wide late-arrival window knobs, in whole days. A value declared
+  # here fans into every provider that does not set its own key; a
+  # per-provider key always wins (provider key > this > provider default).
+  # lookback_days: 7
+  # cutoff_days: 0
+
+  # The width, in whole days, of the work units a windowed run is planned
+  # into. Every windowed pull -- the daily sync and a long backfill alike --
+  # is tiled into units that fetch and commit independently (partitions,
+  # ledger row, watermark), so a crash resumes from the last completed unit
+  # instead of refetching the whole window. Smaller chunks mean finer
+  # crash-resume granularity at the cost of more commits; a window smaller
+  # than one chunk is simply one unit. An endpoint whose rows are
+  # per-request-window rollups declares its own fixed unit width, which
+  # wins over this value (e.g. the Samsara fuel-energy report pair is
+  # pinned to 1-day units). Optional; default 7.
+  # backfill_chunk_days: 7
+
+  # How many of a windowed endpoint's work units drive concurrently. The
+  # watermark stays truthful under any completion order (the prefix-advance
+  # rule), and the provider's rate budget still governs at the wire, so
+  # workers beyond the budget simply pace on rate-limit tokens. 1 is the
+  # serial path. Optional; default 4.
+  # backfill_unit_workers: 4
+
+storage:
+  # REQUIRED. The root directory the parquet dataset is written under
+  # ({root}/{provider}/{endpoint}/). Use a real local path -- never a
+  # cloud-synced folder (OneDrive, Dropbox, Google Drive): sync clients
+  # fight the writer's atomic renames.
+  dataset_root: /data/fleetpull
+
+  # Whether write-time compaction drops exact-duplicate rows (semantic
+  # dedup is out of scope by design). Optional; default true.
+  # drop_exact_duplicates: true
+
+# state:
+#   # Where operational state (SQLite) lives. Keep it on a real local disk
+#   # even when dataset_root sits on a network filesystem. Optional; default:
+#   database_path: /data/fleetpull/.fleetpull/state.sqlite3
+
+# logging:
+#   # Minimum level for console output (name or standard integer). Optional.
+#   console_level: INFO
+#   # Either file key enables file logging; the missing partner is defaulted
+#   # (file_level -> DEBUG, file_path -> <dataset_root>/.fleetpull/fleetpull.log).
+#   # file_level: DEBUG
+#   # file_path: /var/log/fleetpull.log
+
+# http:
+#   # Build TLS contexts from the operating system's trust store -- required
+#   # behind TLS-intercepting corporate proxies (Zscaler and kin). Optional.
+#   use_truststore: false
+#   connect_timeout_seconds: 10.0
+#   read_timeout_seconds: 30.0
+
+# retry:
+#   # Attempt budgets and backoff shape; the defaults suit almost everyone.
+#   transient_max_failures: 3
+#   transient_backoff_base_seconds: 1.0
+#   transient_backoff_cap_seconds: 30.0
+#   rate_limited_max_failures: 10
+#   fallback_penalty_seconds: 60.0
+
+providers:
+  motive:
+    # Set your API key as an environment variable rather than writing it here:
+    #   Linux/macOS (current shell):   export MOTIVE_API_KEY='...'
+    #     persistent: add that line to ~/.bashrc or ~/.zshrc
+    #   Windows PowerShell (current shell):   $env:MOTIVE_API_KEY = '...'
+    #     persistent: setx MOTIVE_API_KEY "..."
+    #     (setx takes effect in NEW shells only -- reopen your terminal)
+    # A key written here wins over the environment variable.
+    # api_key: ''
+
+    # The endpoints to sync, by catalog name. A provider with endpoints
+    # listed requires a resolvable credential; a credential with no
+    # endpoints leaves the provider disabled (with a load-time warning).
+    endpoints:
+      - vehicles
+      - vehicle_locations
+      - driving_periods
+      - idle_events
+      - groups
+      - users
+      - vehicle_utilizations
+      - driver_idle_rollups
+
+    # base_url: https://api.gomotive.com   # optional; Motive's production host
+    # records_per_page: 100                # optional; 1..100
+
+    # Per-provider window knobs, in whole days. Optional overrides: a key
+    # set here beats sync's package-wide value, which beats this default.
+    # lookback_days: 7
+    # cutoff_days: 0
+
+    # rate_limit:                          # optional; conservative defaults
+    #   requests_per_period: 60
+    #   period_seconds: 60.0
+    #   burst: 10
+    #   max_concurrency: 2
+
+  samsara:
+    # Set your API token as an environment variable rather than writing it
+    # here (same shell mechanics as MOTIVE_API_KEY above):
+    #   export SAMSARA_API_KEY='...'
+    # A token written here wins over the environment variable.
+    # api_key: ''
+
+    # The endpoints to sync, by catalog name.
+    endpoints:
+      - vehicles
+      - drivers
+      - trips
+      - idling_events
+      - addresses
+      - engine_states
+      - gps_readings
+      - odometer_readings
+      - asset_locations
+      - driver_vehicle_assignments
+      - vehicle_fuel_energy_reports
+      - driver_fuel_energy_reports
+
+    # base_url: https://api.samsara.com    # optional; Samsara's production host
+
+    # Per-provider window knobs for watermark endpoints (Samsara trips,
+    # idling_events, the vehicle-stats triple, asset_locations,
+    # driver_vehicle_assignments, and the fuel-energy report pair
+    # vehicle_fuel_energy_reports / driver_fuel_energy_reports), in
+    # whole days. Optional overrides: a key set here beats sync's
+    # package-wide value, which beats this default.
+    # lookback_days: 7
+    # cutoff_days: 0
+
+    # rate_limit:                          # optional; defaults to the
+    #   requests_per_period: 100           # tightest documented endpoint
+    #   period_seconds: 60.0               # tier (see SamsaraConfig)
+    #   burst: 10
+    #   max_concurrency: 2
+
+  geotab:
+    # The four-field GeoTab credential, nested. Set the password as an
+    # environment variable rather than writing it here:
+    #   Linux/macOS:   export GEOTAB_PASSWORD='...'
+    #   Windows PowerShell:   $env:GEOTAB_PASSWORD = '...'
+    # A password written here wins over the environment variable; the
+    # other three fields are not secrets and always come from this file.
+    auth:
+      username: user@example.com
+      database: exampledb
+      # server: my.geotab.com              # optional; the authentication host
+
+    # The endpoints to sync, by catalog name. The feed endpoints
+    # (log_records through media_files) drive GeoTab's GetFeed
+    # version-token stream into append-only storage.
+    endpoints:
+      - devices
+      - users
+      - trips
+      - exception_events
+      - log_records
+      - status_data
+      - fill_ups
+      - fuel_and_energy_used
+      - fuel_tax_details
+      - fault_data
+      - duty_status_logs
+      - driver_changes
+      - dvir_logs
+      - annotation_logs
+      - shipment_logs
+      - audits
+      - text_messages
+      - media_files
+
+    # Per-provider window knobs for watermark endpoints (GeoTab trips and
+    # exception_events; the feed endpoints resume from a version token and
+    # take no window), in whole days. Optional overrides: a key set here
+    # beats sync's package-wide value, which beats this default. The
+    # lookback margin is also what absorbs GeoTab's Trip recalculation.
+    # lookback_days: 7
+    # cutoff_days: 0
+
+    # The Get method-class budget (GeoTab meters per method class). The
+    # default cites the captured 2026-07-09 rate headers (~650/min; a
+    # single datum, so burst errs conservative).
+    # rate_limit:
+    #   requests_per_period: 650
+    #   period_seconds: 60.0
+    #   burst: 100
+    #   max_concurrency: 2
+
+    # The GetFeed method-class budget the feed endpoints spend from
+    # (~60/min, the 2026-07-21 header-decrement probe).
+    # feed_rate_limit:
+    #   requests_per_period: 60
+    #   period_seconds: 60.0
+    #   burst: 10
+    #   max_concurrency: 2
+
+    # The Authenticate method-class budget: 10/min (June 2026 capture;
+    # docs Status: Active). Authenticate fires rarely behind the session
+    # manager's single-flight lock.
+    # authenticate_rate_limit:
+    #   requests_per_period: 10
+    #   period_seconds: 60.0
+    #   burst: 2
+    #   max_concurrency: 1

fleetpull/roster/__init__.py

@@ -0,0 +1,16 @@
+# src/fleetpull/roster/__init__.py
+"""The roster layer: the fan-out roster's identity, declaration, and catalog.
+
+A leaf both the endpoints layer (the roster-backed shapes on a consuming endpoint) and
+the state layer (the ``RosterStore``) key by -- ``state`` cannot import ``endpoints``,
+so the shared roster identity sits below both. ``RosterKey`` is the opaque handle a
+consumer carries; ``RosterDefinition`` is the registry's record of where a key's
+members come from and the refresh policy; ``RosterRegistry`` resolves a key to its
+definition. Imports only ``vocabulary`` and ``exceptions``, nothing higher.
+"""
+
+from fleetpull.roster.definition import RosterDefinition
+from fleetpull.roster.key import RosterKey
+from fleetpull.roster.registry import RosterRegistry
+
+__all__: list[str] = ['RosterDefinition', 'RosterKey', 'RosterRegistry']

fleetpull/roster/definition.py

@@ -0,0 +1,46 @@
+# src/fleetpull/roster/definition.py
+"""The roster declaration: ``RosterDefinition`` -- a key, its source, and its policy.
+
+The registry's record for one ``RosterKey``: the feeder endpoint and column its
+members are listed from, and the staleness and eviction policy a refresh applies. The
+consuming endpoint never sees this -- it carries only the ``RosterKey``; the registry
+resolves the key to this when a refresh runs. Homed beside the key it references.
+"""
+
+from dataclasses import dataclass
+from datetime import timedelta
+
+from fleetpull.roster.key import RosterKey
+
+__all__: list[str] = ['RosterDefinition']
+
+
+@dataclass(frozen=True, slots=True)
+class RosterDefinition:
+    """One roster's source and refresh policy, keyed by its ``RosterKey``.
+
+    Maps a ``RosterKey`` to the feeder endpoint and column its members are listed
+    from, plus the staleness bound and eviction threshold a refresh applies. The
+    source is named by endpoint and column (strings), not an ``EndpointDefinition`` --
+    resolving the name to a runnable binding is the coordinator's job, against the
+    endpoint catalog. ``max_age`` and ``eviction_threshold`` are the policy the pure
+    ``is_roster_stale`` and ``reconcile`` already take.
+
+    Attributes:
+        key: The roster this defines.
+        source_endpoint: The feeder endpoint whose listing supplies the members
+            (e.g. ``'vehicles'``).
+        source_column: The feeder frame column whose distinct values are the members
+            (e.g. ``'vehicle_id'``) -- the model field name after the records-layer
+            flatten, not the wire key.
+        max_age: The staleness bound -- a refresh is due once the feeder's last
+            success is older than this.
+        eviction_threshold: Consecutive-miss count past which a member is evicted, or
+            ``None`` for append-only (never evict).
+    """
+
+    key: RosterKey
+    source_endpoint: str
+    source_column: str
+    max_age: timedelta
+    eviction_threshold: int | None

fleetpull/roster/key.py

@@ -0,0 +1,40 @@
+# src/fleetpull/roster/key.py
+"""The roster identity: ``RosterKey``, the opaque handle a consumer references.
+
+A ``RosterKey`` names one roster -- ``(provider, name)`` -- and is the only roster
+fact a fan-out consumer carries (its shape's ``roster`` field): the consumer knows
+*that* a roster of its keys exists, never where those keys come from. The mapping
+from a key to its source endpoint and column lives in the ``RosterRegistry`` (the
+``RosterDefinition``), and the persisted members live in the ``RosterStore``, both
+keyed by this. Homed in this leaf so both the endpoints layer (the binding) and the
+state layer (the store) can key by it -- ``state`` cannot import ``endpoints``, so the
+shared identity sits below both.
+
+``name`` is the logical roster, deliberately not the source column: ``'vehicle_ids'``
+the roster, not ``'vehicle_id'`` the feeder field -- the gap is the decoupling.
+"""
+
+from dataclasses import dataclass
+
+from fleetpull.vocabulary import Provider
+
+__all__: list[str] = ['RosterKey']
+
+
+@dataclass(frozen=True, slots=True)
+class RosterKey:
+    """The identity of one roster: a provider and a logical roster name.
+
+    The opaque handle a fan-out consumer references and the key the registry and
+    store map from. Two rosters are the same iff their provider and name match;
+    nothing else (the source endpoint, the column, the policy) is part of the
+    identity -- those live in the ``RosterDefinition`` the registry holds.
+
+    Attributes:
+        provider: The provider whose roster this is.
+        name: The logical roster name (e.g. ``'vehicle_ids'``), distinct from the
+            feeder column its members are listed from.
+    """
+
+    provider: Provider
+    name: str

fleetpull/roster/registry.py

@@ -0,0 +1,93 @@
+# src/fleetpull/roster/registry.py
+"""The roster catalog: ``RosterRegistry``, ``RosterKey`` -> ``RosterDefinition``.
+
+Holds the roster declarations and resolves them both ways. Forward
+(``get``): a ``RosterKey`` to its ``RosterDefinition`` -- the source endpoint,
+column, and policy a refresh needs; the consuming endpoint carries only the
+key, and the coordinator asks the registry for the key's definition when it
+refreshes. Reverse (``sourced_by``): which roster definitions a feeder
+``(provider, endpoint)`` sources -- the lookup the feeder tap reads so every
+execution of a feeder endpoint reconciles its rosters. Construction rejects
+two definitions claiming the same key.
+"""
+
+from collections.abc import Iterable
+
+from fleetpull.exceptions import ConfigurationError
+from fleetpull.roster.definition import RosterDefinition
+from fleetpull.roster.key import RosterKey
+from fleetpull.vocabulary import Provider
+
+__all__: list[str] = ['RosterRegistry']
+
+
+class RosterRegistry:
+    """An immutable catalog mapping each ``RosterKey`` to its ``RosterDefinition``.
+
+    Built once from the roster definitions, it answers ``get(key)``. The map is
+    private and frozen at construction; a duplicate key is a wiring bug and raises.
+
+    Args:
+        definitions: The roster definitions to catalog; their keys must be distinct.
+
+    Raises:
+        ConfigurationError: Two definitions share a ``RosterKey`` -- a wiring bug.
+    """
+
+    def __init__(self, definitions: Iterable[RosterDefinition]) -> None:
+        by_key: dict[RosterKey, RosterDefinition] = {}
+        for definition in definitions:
+            if definition.key in by_key:
+                raise ConfigurationError(
+                    'duplicate roster definition',
+                    provider=definition.key.provider.value,
+                    detail=f'roster {definition.key.name!r} is defined twice',
+                )
+            by_key[definition.key] = definition
+        self._by_key: dict[RosterKey, RosterDefinition] = by_key
+
+    def get(self, key: RosterKey) -> RosterDefinition:
+        """Return the definition for a roster key.
+
+        Args:
+            key: The roster to resolve.
+
+        Returns:
+            The roster's definition (source endpoint, column, and policy).
+
+        Raises:
+            ConfigurationError: No definition is registered for ``key`` -- a consumer
+                references a roster the catalog does not declare.
+        """
+        try:
+            return self._by_key[key]
+        except KeyError:
+            raise ConfigurationError(
+                'unknown roster',
+                provider=key.provider.value,
+                detail=f'no roster definition registered for {key.name!r}',
+            ) from None
+
+    def sourced_by(
+        self, provider: Provider, endpoint: str
+    ) -> tuple[RosterDefinition, ...]:
+        """Return the roster definitions sourced by one feeder endpoint.
+
+        The reverse lookup the feeder tap reads: which rosters must be
+        reconciled when this ``(provider, endpoint)`` runs. A linear scan --
+        the catalog holds a handful of definitions -- in registration order.
+
+        Args:
+            provider: The feeder's provider.
+            endpoint: The feeder's endpoint name (e.g. ``'vehicles'``).
+
+        Returns:
+            The definitions whose ``source_endpoint`` is this endpoint; empty
+            when the endpoint sources no roster.
+        """
+        return tuple(
+            definition
+            for definition in self._by_key.values()
+            if definition.key.provider is provider
+            and definition.source_endpoint == endpoint
+        )

fleetpull/state/__init__.py

@@ -0,0 +1,36 @@
+# src/fleetpull/state/__init__.py
+"""SQLite operational state store: the database lifecycle (connections, schema
+migrations, integrity verification) and the stores over it -- incremental
+cursors, the run ledger, feeder rosters, and the backfill work-unit queue."""
+
+from fleetpull.state.cursors import CursorKind, CursorStore
+from fleetpull.state.database import StateDatabase
+from fleetpull.state.migrations import migrate_to_head
+from fleetpull.state.reconcile import RosterDelta, is_roster_stale, reconcile
+from fleetpull.state.rosters import RosterStore
+from fleetpull.state.run_ledger import RunLedger, RunStatus
+from fleetpull.state.work_units import (
+    ClaimedWorkUnit,
+    WorkUnitProgress,
+    WorkUnitSpec,
+    WorkUnitStatus,
+    WorkUnitStore,
+)
+
+__all__: list[str] = [
+    'ClaimedWorkUnit',
+    'CursorKind',
+    'CursorStore',
+    'RosterDelta',
+    'RosterStore',
+    'RunLedger',
+    'RunStatus',
+    'StateDatabase',
+    'WorkUnitProgress',
+    'WorkUnitSpec',
+    'WorkUnitStatus',
+    'WorkUnitStore',
+    'is_roster_stale',
+    'migrate_to_head',
+    'reconcile',
+]