fleetpull 0.1.0__py3-none-any.whl

fleetpull/cli.py

@@ -0,0 +1,146 @@
+# src/fleetpull/cli.py
+"""The fleetpull command line: config-driven sync and config scaffolding.
+
+Two subcommands. ``fleetpull sync <config>`` is the shell form of
+``Sync(config).run()`` (DESIGN §10): parse the arguments, run the sync, and
+translate the operational-failure family (``FleetpullError``) into a stderr
+line and exit code 1. No logging setup happens here — ``Sync.run`` applies
+the config's logging section itself — and an unexpected exception propagates
+with its traceback: a bug report, not an operational outcome. ``fleetpull
+init-config [path]`` writes the packaged annotated example configuration to
+disk, the onboarding path for a pip-installed user who has no repository to
+copy it from.
+"""
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from fleetpull.api import Sync
+from fleetpull.config import EXAMPLE_CONFIG_FILENAME, write_example_config
+from fleetpull.exceptions import FleetpullError
+
+__all__: list[str] = ['main']
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    """Build the ``fleetpull`` argument parser.
+
+    Two required subcommands: ``sync`` (run a config-driven sync) and
+    ``init-config`` (write the example configuration).
+
+    Returns:
+        The configured parser.
+    """
+    parser = argparse.ArgumentParser(
+        prog='fleetpull',
+        description=(
+            'Pull fleet telematics data from provider APIs into typed parquet.'
+        ),
+    )
+    subcommands = parser.add_subparsers(dest='command', required=True)
+
+    sync_parser = subcommands.add_parser(
+        'sync', help='run a config-driven sync: fleetpull sync <config.yaml>'
+    )
+    sync_parser.add_argument(
+        'config', type=Path, help='path to the fleetpull YAML configuration file'
+    )
+
+    init_parser = subcommands.add_parser(
+        'init-config',
+        help=(
+            'write the annotated example configuration to a path '
+            f'(default ./{EXAMPLE_CONFIG_FILENAME})'
+        ),
+    )
+    init_parser.add_argument(
+        'path',
+        type=Path,
+        nargs='?',
+        default=Path(EXAMPLE_CONFIG_FILENAME),
+        help=(
+            'destination file, or an existing directory to write '
+            f'{EXAMPLE_CONFIG_FILENAME} into '
+            f'(default ./{EXAMPLE_CONFIG_FILENAME})'
+        ),
+    )
+    init_parser.add_argument(
+        '--force',
+        action='store_true',
+        help='overwrite the destination if it already exists',
+    )
+    return parser
+
+
+def _run_sync(config: Path) -> int:
+    """Run a config-driven sync, mapping operational failures to exit code 1.
+
+    Args:
+        config: The YAML configuration path.
+
+    Returns:
+        ``0`` on success; ``1`` on a ``FleetpullError``-family failure
+        (whose message lands on stderr).
+
+    Side Effects:
+        Runs the sync (network, parquet, state, logging per the config);
+        writes the failure message to ``sys.stderr`` on an operational
+        error.
+    """
+    try:
+        Sync(config).run()
+    except FleetpullError as error:
+        sys.stderr.write(f'{error}\n')
+        return 1
+    return 0
+
+
+def _run_init_config(path: Path, *, force: bool) -> int:
+    """Write the example configuration, mapping a refusal to exit code 1.
+
+    Args:
+        path: The destination file, or an existing directory.
+        force: Overwrite an existing destination when ``True``.
+
+    Returns:
+        ``0`` after writing; ``1`` when the destination exists (and
+        ``force`` is unset) or the write fails, the reason on stderr.
+
+    Side Effects:
+        Writes the example config to disk; prints the written path to
+        stdout on success, the failure to ``sys.stderr`` otherwise.
+    """
+    try:
+        written = write_example_config(path, force=force)
+    except OSError as error:
+        sys.stderr.write(f'{error}\n')
+        return 1
+    sys.stdout.write(f'wrote example configuration to {written}\n')
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the fleetpull CLI and return its exit code.
+
+    Args:
+        argv: The argument vector to parse, or ``None`` for ``sys.argv[1:]``
+            (the console-script entry point).
+
+    Returns:
+        ``0`` on success; ``1`` on an operational failure (a
+        ``FleetpullError``-family sync error, or a refused/failed config
+        write), whose message lands on stderr.
+
+    Raises:
+        SystemExit: Argparse's exit on a missing or unknown argument
+            (exit code 2).
+
+    Side Effects:
+        Per subcommand: runs the sync, or writes the example config.
+    """
+    arguments = _build_parser().parse_args(argv)
+    if arguments.command == 'init-config':
+        return _run_init_config(arguments.path, force=arguments.force)
+    return _run_sync(arguments.config)

fleetpull/config/__init__.py

@@ -0,0 +1,47 @@
+"""Pydantic models for user-provided YAML configuration, one model family per
+file, with ``FleetpullConfig.from_yaml`` as the loading API."""
+
+from fleetpull.config.base import ConfigModel
+from fleetpull.config.example import (
+    EXAMPLE_CONFIG_FILENAME,
+    read_example_config,
+    write_example_config,
+)
+from fleetpull.config.geotab import DEFAULT_GEOTAB_SERVER, GeotabAuthConfig
+from fleetpull.config.http import HttpConfig
+from fleetpull.config.logger import LoggerConfig
+from fleetpull.config.providers import (
+    GeotabConfig,
+    MotiveConfig,
+    ProviderConfig,
+    ProvidersConfig,
+    SamsaraConfig,
+    default_provider_sections,
+)
+from fleetpull.config.rate_limit import RateLimitConfig
+from fleetpull.config.retry import RetryConfig
+from fleetpull.config.root import FleetpullConfig
+from fleetpull.config.sections import StateConfig, StorageConfig, SyncConfig
+
+__all__: list[str] = [
+    'DEFAULT_GEOTAB_SERVER',
+    'EXAMPLE_CONFIG_FILENAME',
+    'ConfigModel',
+    'FleetpullConfig',
+    'GeotabAuthConfig',
+    'GeotabConfig',
+    'HttpConfig',
+    'LoggerConfig',
+    'MotiveConfig',
+    'ProviderConfig',
+    'ProvidersConfig',
+    'RateLimitConfig',
+    'RetryConfig',
+    'SamsaraConfig',
+    'StateConfig',
+    'StorageConfig',
+    'SyncConfig',
+    'default_provider_sections',
+    'read_example_config',
+    'write_example_config',
+]

fleetpull/config/base.py

@@ -0,0 +1,18 @@
+# src/fleetpull/config/base.py
+"""The shared configuration-model base: the policy, stated exactly once.
+
+Every config model inherits ``ConfigModel``: frozen so a loaded config
+cannot mutate mid-run, ``extra='forbid'`` so a misspelled YAML key is
+rejected rather than silently dropped, and ``validate_default=True`` so
+defaulted values pass the same validators as supplied ones.
+"""
+
+from pydantic import BaseModel, ConfigDict
+
+__all__: list[str] = ['ConfigModel']
+
+
+class ConfigModel(BaseModel):
+    """Base for every fleetpull configuration model; carries the policy only."""
+
+    model_config = ConfigDict(frozen=True, extra='forbid', validate_default=True)

fleetpull/config/example.py

@@ -0,0 +1,90 @@
+# src/fleetpull/config/example.py
+"""The packaged example configuration: read it, or materialize it to disk.
+
+``config.example.yaml`` ships inside the wheel (``fleetpull.resources``),
+so a pip-installed user has no repository to copy it from. ``fleetpull
+init-config`` writes it to a path of their choosing through
+``write_example_config``; ``read_example_config`` returns its text for any
+programmatic caller. Read through ``importlib.resources`` so the file
+resolves identically in a built wheel and an editable checkout.
+"""
+
+from importlib import resources
+from pathlib import Path
+
+from fleetpull.paths import PathInput, resolve_path
+
+__all__: list[str] = [
+    'EXAMPLE_CONFIG_FILENAME',
+    'read_example_config',
+    'write_example_config',
+]
+
+# The resource's package and name; the file lives in ``fleetpull/resources``.
+_RESOURCE_PACKAGE = 'fleetpull.resources'
+_RESOURCE_NAME = 'config.example.yaml'
+
+# The default filename ``init-config`` writes when given only a directory
+# (or nothing) -- not the resource's own name, so the materialized file
+# reads as the user's config rather than "the example".
+EXAMPLE_CONFIG_FILENAME = 'fleetpull_config.yaml'
+
+
+def read_example_config() -> str:
+    """Return the packaged example configuration's text.
+
+    Returns:
+        The full ``config.example.yaml`` document, verbatim UTF-8.
+
+    Side Effects:
+        Reads the packaged resource.
+    """
+    return (
+        resources.files(_RESOURCE_PACKAGE)
+        .joinpath(_RESOURCE_NAME)
+        .read_text(encoding='utf-8')
+    )
+
+
+def _resolve_destination(destination: PathInput) -> Path:
+    """Resolve the write target, defaulting a directory to the config filename.
+
+    Args:
+        destination: The user-supplied path -- a file path, or a directory
+            (existing) into which the default filename is written.
+
+    Returns:
+        The concrete file path to write.
+    """
+    resolved = resolve_path(destination)
+    if resolved.is_dir():
+        return resolved / EXAMPLE_CONFIG_FILENAME
+    return resolved
+
+
+def write_example_config(destination: PathInput, *, force: bool = False) -> Path:
+    """Write the packaged example configuration to ``destination``.
+
+    Args:
+        destination: Where to write -- a file path, or an existing
+            directory (the default filename ``fleetpull_config.yaml`` is
+            appended). The parent directory must already exist.
+        force: Overwrite an existing target when ``True``; otherwise a
+            pre-existing target is a loud refusal (never clobber a config a
+            user may have edited).
+
+    Returns:
+        The path actually written.
+
+    Raises:
+        FileExistsError: The target exists and ``force`` is ``False``.
+        OSError: The parent directory is missing or the write fails.
+
+    Side Effects:
+        Writes a file to disk.
+    """
+    target = _resolve_destination(destination)
+    if target.exists() and not force:
+        raise FileExistsError(f'{target} already exists; pass force to overwrite it')
+    target.write_text(read_example_config(), encoding='utf-8')
+    return target

fleetpull/config/geotab.py

@@ -0,0 +1,78 @@
+# src/fleetpull/config/geotab.py
+"""GeoTab authentication configuration model.
+
+GeotabAuthConfig is the GeoTab authentication section of fleetpull's
+user-provided YAML configuration (nested under ``providers.geotab.auth``;
+the provider section itself lives with its family in
+``config/providers.py``). The password is a ``SecretStr``: its value is
+extracted with ``.get_secret_value()`` only at the moment of use (inside
+the real authenticate function), and is never logged or included in
+``repr()``/``str()`` output.
+"""
+
+from pydantic import Field, SecretStr, field_validator
+
+from fleetpull.config.base import ConfigModel
+
+__all__: list[str] = ['DEFAULT_GEOTAB_SERVER', 'GeotabAuthConfig']
+
+# The documented default authentication host. Hoisted as a named constant
+# (and exported through the config face) because it is a shared provider
+# fact: the auth section defaults to it, and the GeoTab spec builders use
+# it as the pre-auth placeholder host for a credential-less config.
+DEFAULT_GEOTAB_SERVER: str = 'my.geotab.com'
+
+
+class GeotabAuthConfig(ConfigModel):
+    """
+    GeoTab authentication credentials and target database.
+
+    Attributes:
+        username: GeoTab account username (non-empty).
+        password: GeoTab account password; masked in all output.
+        database: GeoTab database name (non-empty).
+        server: The authentication host — a bare hostname like
+            ``my.geotab.com`` (no scheme, path, or whitespace). The
+            authenticator builds ``https://{server}/apiv1`` from it.
+            ``Authenticate`` may redirect subsequent calls to a
+            different resolved host — that is session state, not
+            configuration, so it never lives here.
+    """
+
+    username: str = Field(min_length=1)
+    password: SecretStr
+    database: str = Field(min_length=1)
+    server: str = Field(default=DEFAULT_GEOTAB_SERVER, min_length=1)
+
+    @field_validator('server')
+    @classmethod
+    def _server_is_bare_hostname(cls, server: str) -> str:
+        """
+        Reject anything but a bare hostname.
+
+        The authenticator builds ``https://{server}/apiv1``; a scheme,
+        path, or stray whitespace would corrupt that URL. Caught here at
+        the config boundary with a message that says exactly what to
+        write.
+
+        Args:
+            server: The configured server value.
+
+        Returns:
+            The value unchanged when it is a bare hostname.
+
+        Raises:
+            ValueError: When the value carries a scheme, a path, a
+                slash, or whitespace.
+        """
+        if '/' in server:
+            raise ValueError(
+                f'server must be a bare hostname like "my.geotab.com" — no '
+                f'scheme or path; got {server!r} (write "https://my.geotab.com" '
+                f'as "my.geotab.com")'
+            )
+        if any(character.isspace() for character in server):
+            raise ValueError(
+                f'server must be a bare hostname with no whitespace; got {server!r}'
+            )
+        return server

fleetpull/config/http.py

@@ -0,0 +1,32 @@
+# src/fleetpull/config/http.py
+"""HTTP transport configuration: timeouts and TLS posture.
+
+Every network call in the package requires explicit timeouts (house
+rule); this model is their single YAML-facing source.
+"""
+
+from pydantic import Field
+
+from fleetpull.config.base import ConfigModel
+
+__all__: list[str] = ['HttpConfig']
+
+
+class HttpConfig(ConfigModel):
+    """
+    User-facing HTTP transport settings, one instance per run.
+
+    Attributes:
+        connect_timeout_seconds: Timeout for establishing a connection.
+        read_timeout_seconds: Timeout for reading a response.
+        use_truststore: Build SSL contexts from the operating system's
+            trust store (``network/tls/``) — required behind
+            TLS-intercepting corporate proxies. Default False:
+            unproxied environments (production deployment targets) use
+            httpx's bundled CA store; the trust-store path is opt-in
+            where the proxy is the exception, not the rule.
+    """
+
+    connect_timeout_seconds: float = Field(default=10.0, gt=0)
+    read_timeout_seconds: float = Field(default=30.0, gt=0)
+    use_truststore: bool = False

fleetpull/config/loading.py

@@ -0,0 +1,195 @@
+# src/fleetpull/config/loading.py
+"""The single-concern loading steps behind ``FleetpullConfig.from_yaml``.
+
+Reading and parsing the file, merging conventional credential
+environment variables into the raw document, shaping validation failures
+into actionable messages, and the post-validation disabled-provider
+warning. Environment access lives here and nowhere else in the config
+layer; none of it runs inside validators.
+
+Every failure surfaces as a ``ConfigurationError`` a user can act on: a
+missing file names the path; a parse failure names the line the parser
+reports, composed from the parser's ``problem`` text and never the
+marked source snippet (a snippet could echo a malformed credential
+line); a validation failure carries one ``key.path: message`` entry per
+error and no raw input values.
+"""
+
+import logging
+import os
+from pathlib import Path
+
+import yaml
+from pydantic import SecretStr, ValidationError
+
+from fleetpull.config.providers import PROVIDER_CREDENTIAL_ENV_VARS, ProvidersConfig
+from fleetpull.exceptions import ConfigurationError
+
+__all__: list[str] = [
+    'read_yaml_document',
+    'validation_detail',
+    'warn_disabled_providers',
+    'with_environment_credentials',
+]
+
+logger = logging.getLogger(__name__)
+
+
+# typing-justified: YAML values are arbitrary user input until validated
+def read_yaml_document(config_path: Path) -> dict[str, object]:
+    """Read and parse the file into the raw top-level mapping.
+
+    Args:
+        config_path: The file to read.
+
+    Returns:
+        The parsed top-level mapping; an empty file is an empty mapping
+        (validation then names the missing required sections).
+
+    Raises:
+        ConfigurationError: The file is missing, unparseable (naming the
+            line the parser reports), or not a mapping at the top level.
+    """
+    if not config_path.is_file():
+        raise ConfigurationError('config file not found', detail=str(config_path))
+    try:
+        loaded = yaml.safe_load(config_path.read_text(encoding='utf-8'))
+    except yaml.YAMLError as parse_error:
+        raise ConfigurationError(
+            'config file is not valid YAML',
+            detail=_parse_error_detail(config_path, parse_error),
+        ) from None
+    if loaded is None:
+        return {}
+    if not isinstance(loaded, dict):
+        raise ConfigurationError(
+            'config file must be a YAML mapping at the top level',
+            detail=f'{config_path}: got {type(loaded).__name__}',
+        )
+    return loaded
+
+
+def _parse_error_detail(config_path: Path, parse_error: yaml.YAMLError) -> str:
+    """Locate a parse failure at the line the parser reports.
+
+    Composes from the parser's ``problem`` description, never the marked
+    source snippet -- a snippet could echo a malformed credential line.
+    """
+    problem: str = getattr(parse_error, 'problem', None) or type(parse_error).__name__
+    mark = getattr(parse_error, 'problem_mark', None)
+    if isinstance(mark, yaml.Mark):
+        return f'{config_path}, line {mark.line + 1}: {problem}'
+    return f'{config_path}: {problem}'
+
+
+def _with_api_key_from_env(
+    # typing-justified: rewrites the raw provider sections before validation
+    providers: dict[str, object],
+    provider_name: str,
+    # typing-justified: rewrites the raw provider sections before validation
+) -> dict[str, object]:
+    """Merge one static-key provider's environment credential, if resolvable.
+
+    The whole-credential half of the documented asymmetry: the provider's
+    conventional variable supplies ``api_key`` outright, merged only when
+    the key is absent from the YAML section (a YAML literal wins) and the
+    variable carries a non-empty value (empty counts as unset). Shared
+    verbatim by the Motive and Samsara arms.
+
+    Args:
+        providers: The raw ``providers`` mapping.
+        provider_name: The static-key provider to merge (``'motive'`` /
+            ``'samsara'``).
+
+    Returns:
+        The providers mapping with the credential merged, or unchanged.
+
+    Side Effects:
+        Reads the process environment.
+    """
+    section = providers.get(provider_name)
+    if not isinstance(section, dict) or 'api_key' in section:
+        return providers
+    value = os.environ.get(PROVIDER_CREDENTIAL_ENV_VARS[provider_name])
+    if not value:
+        return providers
+    return {**providers, provider_name: {**section, 'api_key': SecretStr(value)}}
+
+
+# typing-justified: rewrites the raw document before validation
+def with_environment_credentials(document: dict[str, object]) -> dict[str, object]:
+    """Merge conventional credential environment variables into the document.
+
+    Applied per provider from ``PROVIDER_CREDENTIAL_ENV_VARS``, only when
+    the credential is absent from the YAML (a YAML literal wins) and the
+    variable carries a non-empty value (empty counts as unset). The shape
+    is per-provider (the mapping's documented asymmetry): Motive's and
+    Samsara's variables each supply the whole credential (``api_key``);
+    GeoTab's fills only the ``password`` field of a YAML-present ``auth``
+    section --
+    username, database, and server are not secrets and always come from
+    the YAML, so an absent ``auth`` section is left for the enablement
+    guard to reject. The value is wrapped in ``SecretStr`` here, so the
+    raw string never travels in the document.
+
+    Args:
+        document: The raw document mapping.
+
+    Returns:
+        The document with resolvable credentials merged; otherwise
+        unchanged.
+
+    Side Effects:
+        Reads the process environment -- the only place in the config
+        layer that does.
+    """
+    providers_section = document.get('providers')
+    if not isinstance(providers_section, dict):
+        return document
+    merged_providers = _with_api_key_from_env(dict(providers_section), 'motive')
+    merged_providers = _with_api_key_from_env(merged_providers, 'samsara')
+    geotab_section = merged_providers.get('geotab')
+    if isinstance(geotab_section, dict):
+        auth_section = geotab_section.get('auth')
+        if isinstance(auth_section, dict) and 'password' not in auth_section:
+            geotab_value = os.environ.get(PROVIDER_CREDENTIAL_ENV_VARS['geotab'])
+            if geotab_value:
+                merged_providers['geotab'] = {
+                    **geotab_section,
+                    'auth': {**auth_section, 'password': SecretStr(geotab_value)},
+                }
+    return {**document, 'providers': merged_providers}
+
+
+def validation_detail(validation_error: ValidationError) -> str:
+    """Summarize validation errors as ``key.path: message`` entries.
+
+    Uses each error's location and message only -- never the offending
+    input value, which could be a credential.
+    """
+    return '; '.join(
+        f'{".".join(str(item) for item in entry["loc"])}: {entry["msg"]}'
+        for entry in validation_error.errors()
+    )
+
+
+def warn_disabled_providers(providers: ProvidersConfig) -> None:
+    """Log one WARNING per provider with a credential but no endpoints.
+
+    The provider is merely disabled, not misconfigured, so this is a
+    post-validation side effect of loading -- never a validator's job.
+
+    Args:
+        providers: The validated providers container.
+
+    Side Effects:
+        Logs through this module's logger.
+    """
+    for name, section in providers.named_sections():
+        if section is None or section.credential is None or section.endpoints:
+            continue
+        logger.warning(
+            "providers.%s: a credential resolves but 'endpoints' is empty; "
+            'the provider is disabled for this run.',
+            name,
+        )