fleetpull 0.1.0__py3-none-any.whl

fleetpull/config/rate_limit.py

@@ -0,0 +1,50 @@
+# src/fleetpull/config/rate_limit.py
+"""Rate-limit configuration for a single quota scope.
+
+One ``RateLimitConfig`` describes the request budget for one quota scope
+(default: one provider). The limiter registry (``network/limits/``) maps
+quota-scope strings to these configs; the limiter reads its refill rate and
+capacity from here. Homed in ``config/`` with the other YAML-facing models;
+each provider config carries its scope's values with a documented default.
+"""
+
+from pydantic import Field
+
+from fleetpull.config.base import ConfigModel
+
+__all__: list[str] = ['RateLimitConfig']
+
+
+class RateLimitConfig(ConfigModel):
+    """Token-bucket and concurrency settings for one quota scope.
+
+    ``burst`` is the bucket CAPACITY: the maximum number of tokens the
+    bucket holds, and therefore the number of requests a cold-started
+    limiter can fire immediately before settling to the steady rate of
+    ``requests_per_period / period_seconds``. This semantics is a settled
+    design decision (DESIGN.md §7).
+
+    Attributes:
+        requests_per_period: Requests allowed per period (>= 1).
+        period_seconds: Length of the rate period in seconds (> 0).
+        burst: Bucket capacity in tokens (>= 1).
+        max_concurrency: Maximum requests in flight at once (>= 1) --
+            enforced by the limiter's semaphore, and also the size of the
+            per-provider fetch executor a fan-out run's workers come from
+            (``max_workers``, DESIGN section 7), so the pool can never
+            outrun the semaphore.
+    """
+
+    requests_per_period: int = Field(ge=1)
+    period_seconds: float = Field(gt=0)
+    burst: int = Field(ge=1)
+    max_concurrency: int = Field(ge=1)
+
+    @property
+    def refill_rate_per_second(self) -> float:
+        """Token refill rate: ``requests_per_period / period_seconds``.
+
+        Returns:
+            Tokens added to the bucket per second.
+        """
+        return self.requests_per_period / self.period_seconds

fleetpull/config/resolution.py

@@ -0,0 +1,156 @@
+# src/fleetpull/config/resolution.py
+"""Pure cross-section resolution over raw YAML documents.
+
+The single-concern functions behind ``FleetpullConfig``'s
+``mode='before'`` validators (house style: validator bodies delegate to
+functions). Each takes a raw document mapping and returns one -- no
+environment access, no I/O, no logging -- and each acts only where the
+relevant sections are raw mappings with the relevant keys present:
+already-constructed section models and malformed sections pass through
+unchanged, so shape errors surface from validation with their real key
+paths.
+
+The three settled rules (DESIGN section 10):
+    - Window-knob precedence per provider: a provider's own
+      ``lookback_days`` / ``cutoff_days`` key stands; else a declared
+      ``sync`` value fans in; else the provider model's default
+      validates in naturally.
+    - ``state.database_path`` defaults to
+      ``<dataset_root>/.fleetpull/state.sqlite3`` when absent.
+    - Either logging file key enables file logging: ``file_level``
+      without ``file_path`` injects the default path
+      (``<dataset_root>/.fleetpull/fleetpull.log``); ``file_path`` alone
+      already works via the model's DEBUG default.
+"""
+
+from collections.abc import Mapping
+from pathlib import Path
+
+__all__: list[str] = [
+    'with_log_path_defaulted',
+    'with_provider_knobs_applied',
+    'with_state_path_defaulted',
+]
+
+# The internal directory convention under dataset_root (DESIGN section 5).
+_INTERNAL_SUBDIRECTORY: str = '.fleetpull'
+_STATE_FILENAME: str = 'state.sqlite3'
+_LOG_FILENAME: str = 'fleetpull.log'
+
+# The knobs the sync section can declare package-wide.
+_WINDOW_KNOB_KEYS: tuple[str, ...] = ('lookback_days', 'cutoff_days')
+
+
+# typing-justified: raw YAML documents are arbitrary until validated
+def with_provider_knobs_applied(document: Mapping[str, object]) -> dict[str, object]:
+    """Fan declared ``sync`` window knobs into providers lacking their own key.
+
+    Args:
+        document: The raw document mapping.
+
+    Returns:
+        The document with each provider mapping gaining any ``sync``-declared
+        knob it does not set itself; everything else unchanged. A provider key
+        always wins over the ``sync`` value (proven by the precedence tests).
+    """
+    sync_section = document.get('sync')
+    providers_section = document.get('providers')
+    if not isinstance(sync_section, Mapping) or not isinstance(
+        providers_section, Mapping
+    ):
+        return dict(document)
+    declared = {
+        key: sync_section[key] for key in _WINDOW_KNOB_KEYS if key in sync_section
+    }
+    if not declared:
+        return dict(document)
+    # typing-justified: raw provider entries, arbitrary until validated
+    fanned_providers: dict[object, object] = {}
+    for provider_name, provider_section in providers_section.items():
+        # typing-justified: a raw provider section, arbitrary until validated
+        fanned_section: object = provider_section
+        if isinstance(provider_section, Mapping):
+            missing = {
+                key: value
+                for key, value in declared.items()
+                if key not in provider_section
+            }
+            if missing:
+                fanned_section = {**provider_section, **missing}
+        fanned_providers[provider_name] = fanned_section
+    return {**document, 'providers': fanned_providers}
+
+
+# typing-justified: raw YAML documents are arbitrary until validated
+def with_state_path_defaulted(document: Mapping[str, object]) -> dict[str, object]:
+    """Default ``state.database_path`` under ``dataset_root`` when absent.
+
+    Args:
+        document: The raw document mapping.
+
+    Returns:
+        The document with ``state.database_path`` set to the DESIGN
+        section 5 convention when the key is absent and ``dataset_root``
+        is readable; otherwise unchanged.
+    """
+    dataset_root = _raw_dataset_root(document)
+    if dataset_root is None:
+        return dict(document)
+    state_section = document.get('state', {})
+    if not isinstance(state_section, Mapping) or 'database_path' in state_section:
+        return dict(document)
+    defaulted = {
+        **state_section,
+        'database_path': dataset_root / _INTERNAL_SUBDIRECTORY / _STATE_FILENAME,
+    }
+    return {**document, 'state': defaulted}
+
+
+# typing-justified: raw YAML documents are arbitrary until validated
+def with_log_path_defaulted(document: Mapping[str, object]) -> dict[str, object]:
+    """Inject the default log path when ``file_level`` is set without a path.
+
+    Args:
+        document: The raw document mapping.
+
+    Returns:
+        The document with ``logging.file_path`` defaulted when
+        ``file_level`` is present without it and ``dataset_root`` is
+        readable; otherwise unchanged. Neither file key present means
+        file logging stays off; ``file_path`` alone needs no help (the
+        model's DEBUG default covers the level).
+    """
+    logging_section = document.get('logging')
+    if not isinstance(logging_section, Mapping):
+        return dict(document)
+    if 'file_level' not in logging_section or 'file_path' in logging_section:
+        return dict(document)
+    dataset_root = _raw_dataset_root(document)
+    if dataset_root is None:
+        return dict(document)
+    defaulted = {
+        **logging_section,
+        'file_path': dataset_root / _INTERNAL_SUBDIRECTORY / _LOG_FILENAME,
+    }
+    return {**document, 'logging': defaulted}
+
+
+# typing-justified: reads a raw YAML value, arbitrary until validated
+def _raw_dataset_root(document: Mapping[str, object]) -> Path | None:
+    """The raw ``storage.dataset_root`` as a Path, or None when unreadable.
+
+    Unreadable (missing section, missing key, non-path type) returns
+    ``None`` so the caller passes the document through and validation
+    names the real problem at its real key path. The value may still be
+    unnormalized here; the derived defaults normalize with everything
+    else at field validation (``resolve_path``).
+    """
+    storage_section = document.get('storage')
+    if not isinstance(storage_section, Mapping):
+        return None
+    dataset_root = storage_section.get('dataset_root')
+    if isinstance(dataset_root, str) and dataset_root.strip():
+        return Path(dataset_root)
+    if isinstance(dataset_root, Path):
+        return dataset_root
+    return None

fleetpull/config/retry.py

@@ -0,0 +1,69 @@
+# src/fleetpull/config/retry.py
+"""Retry policy configuration: attempt budgets and backoff shape.
+
+Vocabulary (shared with ``network/retry/decision.py``): a *failure
+count* is one-based within the current retryable category — the first
+TRANSIENT failure of an attempt sequence is 1. ``*_max_failures = N``
+means failures 1..N are each answered with a retry and the (N+1)th
+exhausts the budget: at most N + 1 requests. Counters are independent
+per category within an attempt sequence; a RATE_LIMITED failure
+neither resets nor advances the TRANSIENT count.
+"""
+
+from typing import Self
+
+from pydantic import Field, model_validator
+
+from fleetpull.config.base import ConfigModel
+
+__all__: list[str] = ['RetryConfig']
+
+
+class RetryConfig(ConfigModel):
+    """
+    User-facing retry policy, one instance per run.
+
+    Attributes:
+        transient_max_failures: Highest TRANSIENT failure count still
+            retried. 0 means TRANSIENT failures are never retried.
+        transient_backoff_base_seconds: Full-jitter base; the delay
+            envelope for failure ``n`` is
+            ``min(cap, base * 2 ** (n - 1))``.
+        transient_backoff_cap_seconds: Ceiling on the delay envelope.
+            Only reachable when an operator raises the failure budget.
+        rate_limited_max_failures: Highest RATE_LIMITED failure count
+            still retried. A circuit breaker against 429 storms, not a
+            pacer — pacing is the limiter's job, and RATE_LIMITED
+            retries never sleep locally.
+        fallback_penalty_seconds: Quota-scope penalty the client
+            applies when a rate-limited response carries no usable
+            Retry-After. Fires only when a provider is already
+            misbehaving, so it errs long; the penalty log line carries
+            the raw header value to keep the case diagnosable.
+    """
+
+    transient_max_failures: int = Field(default=3, ge=0)
+    transient_backoff_base_seconds: float = Field(default=1.0, gt=0)
+    transient_backoff_cap_seconds: float = Field(default=30.0, gt=0)
+    rate_limited_max_failures: int = Field(default=10, ge=0)
+    fallback_penalty_seconds: float = Field(default=60.0, gt=0)
+
+    @model_validator(mode='after')
+    def _cap_not_below_base(self) -> Self:
+        """
+        Reject a cap below the base: the envelope would be constant at
+        the cap from the first failure, which is never what a config
+        author meant.
+
+        Returns:
+            The validated model.
+
+        Raises:
+            ValueError: When the cap is below the base.
+        """
+        if self.transient_backoff_cap_seconds < self.transient_backoff_base_seconds:
+            raise ValueError(
+                'transient_backoff_cap_seconds must be >= '
+                'transient_backoff_base_seconds'
+            )
+        return self

fleetpull/config/root.py

@@ -0,0 +1,157 @@
+# src/fleetpull/config/root.py
+"""The whole-document configuration root: ``FleetpullConfig`` and ``from_yaml``.
+
+The one model family that IS the schema (DESIGN section 10 -- the YAML
+schema is sync's API): the root composes the section models exactly as
+the YAML reads, and cross-section resolution runs as ``mode='before'``
+validation on the root, where key presence is still visible. Each
+validator body is a thin wrapper over a single-concern pure function in
+``config/resolution.py``; the loading steps behind ``from_yaml`` live in
+``config/loading.py``. No masks, no injections, no post-validation
+rewriting -- the sections and the schema agree, so nothing needs
+compensating for.
+
+The invariant, precisely: any ``FleetpullConfig`` validated from a raw
+document is fully resolved -- every path field normalized through
+``paths.resolve_path``, ``state.database_path`` and the log-file default
+composed against ``storage.dataset_root``, and every provider's window
+knobs settled by precedence (provider key > ``sync`` key > provider
+default). Direct construction from already-built section models skips
+raw-document resolution but never the enablement invariant: endpoints
+listed with no credential raise ``ConfigurationError`` at validation
+either way.
+"""
+
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Self
+
+from pydantic import Field, ValidationError, model_validator
+
+from fleetpull.config.base import ConfigModel
+from fleetpull.config.http import HttpConfig
+from fleetpull.config.loading import (
+    read_yaml_document,
+    validation_detail,
+    warn_disabled_providers,
+    with_environment_credentials,
+)
+from fleetpull.config.logger import LoggerConfig
+from fleetpull.config.providers import ProvidersConfig, require_provider_credentials
+from fleetpull.config.resolution import (
+    with_log_path_defaulted,
+    with_provider_knobs_applied,
+    with_state_path_defaulted,
+)
+from fleetpull.config.retry import RetryConfig
+from fleetpull.config.sections import StateConfig, StorageConfig, SyncConfig
+from fleetpull.exceptions import ConfigurationError
+
+__all__: list[str] = ['FleetpullConfig']
+
+
+class FleetpullConfig(ConfigModel):
+    """
+    The validated whole-document configuration, one instance per run.
+
+    Attributes:
+        sync: Sync-wide settings: the cold-start anchor and the optional
+            package-wide window knobs.
+        storage: Where the parquet dataset lives (``dataset_root``'s one
+            and only home).
+        state: Where operational SQLite state lives; defaulted under
+            ``dataset_root`` by raw-document resolution.
+        logging: Console and file logging; the file-path default composes
+            under ``dataset_root`` when only ``file_level`` is given.
+        http: Transport timeouts and TLS posture.
+        retry: Attempt budgets and backoff shape.
+        providers: The per-provider sections; a provider listing
+            endpoints without a credential fails validation.
+    """
+
+    sync: SyncConfig
+    storage: StorageConfig
+    state: StateConfig = Field(default_factory=StateConfig)
+    logging: LoggerConfig = Field(default_factory=LoggerConfig)
+    http: HttpConfig = Field(default_factory=HttpConfig)
+    retry: RetryConfig = Field(default_factory=RetryConfig)
+    providers: ProvidersConfig
+
+    @model_validator(mode='before')
+    @classmethod
+    # typing-justified: mode='before' input is the arbitrary raw document
+    def _apply_provider_knob_precedence(cls, document: object) -> object:
+        """Fan declared ``sync`` knobs into providers lacking their own key."""
+        if not isinstance(document, Mapping):
+            return document
+        return with_provider_knobs_applied(document)
+
+    @model_validator(mode='before')
+    @classmethod
+    # typing-justified: mode='before' input is the arbitrary raw document
+    def _apply_state_path_default(cls, document: object) -> object:
+        """Default ``state.database_path`` under ``dataset_root``."""
+        if not isinstance(document, Mapping):
+            return document
+        return with_state_path_defaulted(document)
+
+    @model_validator(mode='before')
+    @classmethod
+    # typing-justified: mode='before' input is the arbitrary raw document
+    def _apply_log_path_default(cls, document: object) -> object:
+        """Inject the default log path when ``file_level`` is set alone."""
+        if not isinstance(document, Mapping):
+            return document
+        return with_log_path_defaulted(document)
+
+    @model_validator(mode='after')
+    def _require_credentialed_providers(self) -> Self:
+        """Enforce enablement's credential half; see the providers family.
+
+        Raises:
+            ConfigurationError: A provider lists endpoints with no
+                credential. Raised (not a ``ValueError``) so it reaches
+                the caller as itself -- Pydantic wraps only
+                ``ValueError``/``AssertionError`` into validation errors.
+        """
+        require_provider_credentials(self.providers)
+        return self
+
+    @classmethod
+    def from_yaml(cls, path: Path | str) -> Self:
+        """Load, validate, and resolve one fleetpull YAML configuration file.
+
+        The loading API: read the file, merge conventional credential
+        environment variables (a YAML literal wins; empty counts as
+        unset), validate -- which resolves every cross-section default --
+        and warn about credentialed providers with no endpoints.
+
+        Args:
+            path: The configuration file to read; a string coerces.
+
+        Returns:
+            The fully resolved configuration (the class docstring's
+            invariant).
+
+        Raises:
+            ConfigurationError: The file is missing (naming the path), is
+                not valid YAML (naming the line), violates the schema
+                (naming each offending key path, never a raw value), or
+                lists endpoints for a provider with no resolvable
+                credential (naming the YAML field and the environment
+                variable).
+
+        Side Effects:
+            Reads the file and the process environment; logs one WARNING
+            per credentialed provider whose endpoint list is empty.
+        """
+        document = read_yaml_document(Path(path))
+        merged = with_environment_credentials(document)
+        try:
+            config = cls.model_validate(merged)
+        except ValidationError as validation_error:
+            raise ConfigurationError(
+                'invalid configuration', detail=validation_detail(validation_error)
+            ) from None
+        warn_disabled_providers(config.providers)
+        return config

fleetpull/config/sections.py

@@ -0,0 +1,146 @@
+# src/fleetpull/config/sections.py
+"""The run-scoped standalone sections: ``sync``, ``storage``, ``state``.
+
+One model family per file (house rule): these three sections describe
+run-wide facts -- when history starts, where the dataset lands, where
+operational state lives -- and change together as the run vocabulary
+grows. Path fields normalize through ``paths.resolve_path`` at
+validation, so a validated section is fully resolved by construction and
+downstream code never normalizes.
+"""
+
+from datetime import UTC, date, datetime, time
+from pathlib import Path
+
+from pydantic import Field, field_validator
+
+from fleetpull.config.base import ConfigModel
+from fleetpull.paths import resolve_path
+
+__all__: list[str] = ['StateConfig', 'StorageConfig', 'SyncConfig']
+
+
+class SyncConfig(ConfigModel):
+    """
+    User-facing sync-wide settings, one instance per run.
+
+    ``dataset_root`` deliberately does not live here. It did once; the
+    config rebuild moved it to ``StorageConfig`` for real, so the YAML
+    section and this model are the same shape and no loader machinery
+    bridges them. Consumers that need both the anchor and the dataset
+    root (the runner) take the root ``FleetpullConfig`` and read each
+    from its own section.
+
+    Attributes:
+        default_start_date: The cold-start backfill anchor -- the UTC calendar date
+            a watermark endpoint's history begins from on its very first run, before
+            any committed watermark or completed coverage exists (DESIGN section 4/5
+            resume precedence arm 3). Required: there is no safe default for "where
+            our history begins", so it must be declared. It goes inert the moment
+            observed data or completed coverage exists; thereafter the start is
+            derived from those, never from this.
+        lookback_days: Package-wide late-arrival re-fetch margin in whole days.
+            Optional; ``None`` means no package-wide value is declared. Root-level
+            resolution fans a declared value into every provider section that does
+            not set its own key (provider key > this > provider default); the
+            field itself is never read as a runtime knob.
+        cutoff_days: Package-wide trailing-edge holdback in whole days, the
+            complement of ``lookback_days``; same precedence and ``None``
+            semantics.
+        backfill_chunk_days: The width, in whole days, of the work units a
+            windowed run's plan tiles its window into (DESIGN sections 4/5).
+            Every windowed pull is planned as units and each unit commits
+            independently, so smaller chunks mean finer crash-resume
+            granularity at the cost of more per-unit commits; a window
+            smaller than one chunk degenerates to a single unit (the daily
+            run). Sync-level only, deliberately without a per-provider
+            override: chunk size is fleetpull's transactional knob, not a
+            provider-latency fact. Applies to newly planned units only --
+            already-persisted unit boundaries are honored on resume. An
+            endpoint whose ``WatermarkMode`` declares ``fixed_unit_days``
+            (a window-grain rollup surface, where the unit width is part
+            of the row's meaning) tiles at exactly that width instead;
+            this knob remains the default for every other endpoint.
+        backfill_unit_workers: How many of a windowed endpoint's work units
+            drive concurrently (DESIGN section 5's prefix-advance rule keeps
+            the watermark truthful under any completion order). ``1`` is the
+            serial path. Sync-level like ``backfill_chunk_days`` -- unit
+            concurrency is fleetpull's execution knob; the per-provider rate
+            budget stays the limiter's job at the transport boundary, so
+            workers beyond a provider's budget simply pace on tokens.
+    """
+
+    default_start_date: date
+    lookback_days: int | None = Field(default=None, ge=0)
+    cutoff_days: int | None = Field(default=None, ge=0)
+    backfill_chunk_days: int = Field(default=7, ge=1)
+    backfill_unit_workers: int = Field(default=4, ge=1)
+
+    @property
+    def default_start_datetime(self) -> datetime:
+        """The cold-start anchor as a timezone-aware UTC midnight instant.
+
+        ``default_start_date`` is the human-authored calendar date history
+        begins from; the resume resolver (``resolve_resume_start``) composes it
+        with watermark and frontier datetimes, so it is lifted to the start of
+        that UTC day here. Deriving it keeps the stored field a ``date`` (a
+        time-of-day on a backfill anchor is meaningless) while handing the
+        orchestrator the ``datetime`` it needs, the conversion defined in one
+        place.
+
+        Returns:
+            ``default_start_date`` at 00:00:00 with ``tzinfo`` ``datetime.UTC``.
+        """
+        return datetime.combine(self.default_start_date, time.min, tzinfo=UTC)
+
+
+class StorageConfig(ConfigModel):
+    """
+    User-facing storage settings, one instance per run.
+
+    Attributes:
+        dataset_root: The root directory the dataset is written under
+            (``{root}/{provider}/{endpoint}/``, DESIGN section 3).
+            Required: there is no safe default for where output lands.
+            Use a real local path -- never a cloud-synced folder
+            (OneDrive and kin), whose sync clients fight the writer's
+            atomic renames. Normalized through ``resolve_path`` at
+            validation.
+        drop_exact_duplicates: Whether write-time compaction drops
+            exact-duplicate rows (DESIGN section 6 -- the default-on
+            dedup with a config flag off; semantic dedup stays out of
+            scope). ``False`` preserves duplicates byte-for-byte.
+    """
+
+    dataset_root: Path
+    drop_exact_duplicates: bool = True
+
+    @field_validator('dataset_root')
+    @classmethod
+    def _resolve(cls, value: Path) -> Path:
+        """Normalize the path lexically; see ``paths.resolve_path``."""
+        return resolve_path(value)
+
+
+class StateConfig(ConfigModel):
+    """
+    User-facing operational-state settings, one instance per run.
+
+    Attributes:
+        database_path: Where the SQLite state database lives -- separable
+            from ``dataset_root`` so SQLite stays on a real local disk
+            even when the parquet dataset sits on a network filesystem
+            (DESIGN section 5). Optional in YAML: root-level resolution
+            defaults it to ``<dataset_root>/.fleetpull/state.sqlite3``,
+            so any ``FleetpullConfig`` validated from a raw document
+            carries a real, normalized path. ``None`` survives only
+            direct section construction.
+    """
+
+    database_path: Path | None = None
+
+    @field_validator('database_path')
+    @classmethod
+    def _resolve(cls, value: Path | None) -> Path | None:
+        """Normalize the path lexically when present; ``None`` passes through."""
+        return None if value is None else resolve_path(value)

fleetpull/endpoints/__init__.py

@@ -0,0 +1,24 @@
+# src/fleetpull/endpoints/__init__.py
+"""The endpoints layer: per-endpoint bindings and the discovery catalogs.
+
+Provider bindings live in subpackage faces consumers import directly --
+``fleetpull.endpoints.shared`` for the ``EndpointDefinition`` binding and the
+shared spec-builders, ``fleetpull.endpoints.motive`` (and future provider
+packages) for the binding factories. The catalogs over those bindings --
+``EndpointRegistry`` / ``build_endpoint_registry`` and the roster sibling
+``build_roster_registry`` -- are re-exported here as the layer's public
+lookup surface, so a consumer routes through this face rather than reaching
+the ``registry`` submodule directly.
+"""
+
+from fleetpull.endpoints.registry import (
+    EndpointRegistry,
+    build_endpoint_registry,
+    build_roster_registry,
+)
+
+__all__: list[str] = [
+    'EndpointRegistry',
+    'build_endpoint_registry',
+    'build_roster_registry',
+]

fleetpull/endpoints/geotab/__init__.py

@@ -0,0 +1,12 @@
+# src/fleetpull/endpoints/geotab/__init__.py
+"""The GeoTab endpoints package.
+
+A provider package under the endpoints layer. It exposes no factory gather:
+endpoint leaves are discovered by ``build_endpoint_registry`` walking this
+package for modules exposing ``build_endpoint``, so a new GeoTab endpoint is
+a new leaf module here with nothing to register. Import a specific factory
+from its leaf module (``fleetpull.endpoints.geotab.devices``) when one is
+needed directly.
+"""
+
+__all__: list[str] = []