aaos-logging-drive 0.1.0__py3-none-any.whl

aaos_logging_drive/__init__.py

@@ -0,0 +1,20 @@
+"""Read-only AAOS Logging Drive client and recorded vehicle data processing."""
+
+from .api import DriveAuthError, DriveError, DriveNotFoundError, ReadOnlyDrive, folder_id
+from .battery_capacity import capacity_telemetry
+from .client import AAOSDriveClient
+from .const import READ_SCOPE, readonly_grant
+from .energy import EnergyReview, effective_telemetry, review_energy
+from .history import HistoryStore, sync_history
+from .model import Artifact, ExportNotReady, InvalidExport, Manifest, Snapshot, archive_trips, trip_summary
+from .repository import DriveRepository
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AAOSDriveClient", "Artifact", "DriveAuthError", "DriveError", "DriveNotFoundError",
+    "DriveRepository", "EnergyReview", "ExportNotReady", "HistoryStore", "InvalidExport",
+    "Manifest", "READ_SCOPE", "ReadOnlyDrive", "Snapshot", "archive_trips",
+    "capacity_telemetry", "effective_telemetry", "folder_id", "readonly_grant",
+    "review_energy", "sync_history", "trip_summary",
+]

aaos_logging_drive/api.py

@@ -0,0 +1,134 @@
+"""Google Drive API reads only. OAuth token refresh belongs to the caller."""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import re
+from typing import Awaitable, Callable
+
+import aiohttp
+
+from .const import API_ROOT, FOLDER_MIME, MAX_FILE_BYTES
+from .model import InvalidExport
+
+
+class DriveError(Exception):
+    """A transient Drive read failed."""
+
+
+class DriveAuthError(DriveError):
+    """The Drive grant needs renewed consent."""
+
+
+class DriveNotFoundError(DriveError):
+    """A file vanished or is not yet visible during a generation replacement."""
+
+
+class RetryableDriveError(DriveError):
+    """A rate limit or temporary Google service error."""
+
+    def __init__(self, message, retry_after=0):
+        super().__init__(message)
+        self.retry_after = retry_after
+
+
+def folder_id(value: str) -> str:
+    value = value.strip()
+    match = re.fullmatch(r"https://drive\.google\.com/drive/(?:u/\d+/)?folders/([\w-]+)(?:\?.*)?", value)
+    result = match[1] if match else value
+    if not re.fullmatch(r"[A-Za-z0-9_-]+", result):
+        raise InvalidExport("Enter a Google Drive folder ID or folder URL")
+    return result
+
+
+class ReadOnlyDrive:
+    def __init__(self, session: aiohttp.ClientSession, token: Callable[[], Awaitable[str]]):
+        self.session = session
+        self.token = token
+
+    async def _read(self, path: str, params: dict, limit: int, media: bool = False):
+        for attempt in range(3):
+            try:
+                return await self._read_once(path, params, limit, media)
+            except (RetryableDriveError, aiohttp.ClientError, TimeoutError) as err:
+                if attempt == 2:
+                    raise DriveError("Google Drive is temporarily unavailable after three read attempts") from err
+                delay = max(2 ** attempt + random.random(), getattr(err, "retry_after", 0))
+                await asyncio.sleep(min(delay, 60))
+
+    async def _read_once(self, path: str, params: dict, limit: int, media: bool = False):
+        access_token = await self.token()
+        async with self.session.get(
+            f"{API_ROOT}/{path}", params=params,
+            headers={"Authorization": f"Bearer {access_token}"},
+            timeout=aiohttp.ClientTimeout(total=180), allow_redirects=False,
+        ) as response:
+            if response.status == 401:
+                raise DriveAuthError("Google Drive authorization expired")
+            if response.status == 403:
+                # Quota exhaustion is transient; insufficient grants require reauth.
+                try:
+                    error = await response.json()
+                    reasons = {e.get("reason") for e in error.get("error", {}).get("errors", [])}
+                except (ValueError, aiohttp.ClientError):
+                    reasons = set()
+                if reasons & {"insufficientPermissions", "authError"}:
+                    raise DriveAuthError("Google Drive read-only permission is required")
+                if reasons & {"rateLimitExceeded", "userRateLimitExceeded"}:
+                    raise RetryableDriveError("Google Drive read rate limit reached")
+                raise DriveError("Google Drive denied the read (permission, API configuration or quota)")
+            if response.status in {429, 500, 502, 503, 504}:
+                try:
+                    retry_after = max(0, min(float(getattr(response, "headers", {}).get("Retry-After", "0")), 60))
+                except ValueError:
+                    retry_after = 0
+                raise RetryableDriveError(f"Temporary Google Drive HTTP {response.status}", retry_after)
+            if response.status == 404:
+                raise DriveNotFoundError("A Google Drive file is not visible or accessible")
+            if response.status >= 300:
+                raise DriveError(f"Google Drive read failed with HTTP {response.status}")
+            data = bytearray()
+            async for chunk in response.content.iter_chunked(64 * 1024):
+                data.extend(chunk)
+                if len(data) > limit:
+                    raise InvalidExport("Drive response exceeds the import limit")
+            if media:
+                return bytes(data)
+            from .model import json_object
+            return json_object(bytes(data))
+
+    async def metadata(self, file_id: str) -> dict:
+        return await self._read(f"files/{folder_id(file_id)}", {
+            "fields": "id,name,mimeType,size,sha256Checksum,trashed", "supportsAllDrives": "true",
+        }, 1024 * 1024)
+
+    async def children(self, parent: str) -> list[dict]:
+        parent = folder_id(parent)
+        result, page_token = [], None
+        while True:
+            params = {
+                "q": f"'{parent}' in parents and trashed = false", "spaces": "drive",
+                "fields": "nextPageToken,incompleteSearch,files(id,name,mimeType,size,sha256Checksum)",
+                "pageSize": "1000", "supportsAllDrives": "true", "includeItemsFromAllDrives": "true",
+            }
+            if page_token:
+                params["pageToken"] = page_token
+            page = await self._read("files", params, 8 * 1024 * 1024)
+            if page.get("incompleteSearch"):
+                raise DriveError("Drive returned an incomplete folder listing; retry later")
+            result.extend(page.get("files", []))
+            page_token = page.get("nextPageToken")
+            if not page_token:
+                return result
+
+    async def child_folder(self, parent: str, name: str) -> str:
+        matches = [f for f in await self.children(parent) if f.get("name") == name and f.get("mimeType") == FOLDER_MIME]
+        if len(matches) != 1:
+            raise InvalidExport(f"Expected one {name} folder; sync AAOS Logging V3 first")
+        return matches[0]["id"]
+
+    async def download(self, file_id: str, limit: int = MAX_FILE_BYTES) -> bytes:
+        return await self._read(f"files/{folder_id(file_id)}", {
+            "alt": "media", "supportsAllDrives": "true",
+        }, limit, media=True)

aaos_logging_drive/battery_capacity.py

@@ -0,0 +1,27 @@
+"""Paired, timestamped usable-capacity estimate; never a certified SoH sensor."""
+
+from math import isfinite
+
+FIELDS = frozenset({"batteryCurrentCapacityWh", "batteryNominalCapacityWh", "batteryCapacityRetentionPercent"})
+NOTICE = ("Estimated battery capacity retention, not verified battery state of health. "
+          "Current usable capacity / nominal usable capacity when new × 100. "
+          "Temperature and battery-management estimates affect this value; it does not measure permanent degradation. "
+          "Values above 100% are not capped. Missing capacities are not inferred from charge level, range or remaining energy.")
+
+
+def positive(value):
+    try:
+        return value if type(value) in (int, float) and isfinite(value) and value > 0 else None
+    except OverflowError:
+        return None
+
+
+def capacity_telemetry(point):
+    """Return a derived view, without modifying the original exported trip."""
+    result = dict(point)
+    current = positive(point.get("batteryCurrentCapacityWh"))
+    nominal = positive(point.get("batteryNominalCapacityWh"))
+    ratio = current / nominal * 100 if current is not None and nominal is not None else None
+    result.update(batteryCurrentCapacityWh=current, batteryNominalCapacityWh=nominal,
+                  batteryCapacityRetentionPercent=positive(ratio))
+    return result

aaos_logging_drive/client.py

@@ -0,0 +1,116 @@
+"""Framework-independent synchronization and local-history API."""
+
+from __future__ import annotations
+
+import asyncio
+from copy import deepcopy
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+from .api import DriveNotFoundError, ReadOnlyDrive, folder_id
+from .history import HistoryStore, sync_history
+from .model import ExportNotReady, Snapshot
+from .repository import DriveRepository
+
+
+class AAOSDriveClient:
+    """Coordinate complete imports, without owning credentials or HTTP sessions.
+
+    The low-level constructor accepts a repository and an initialized optional
+    HistoryStore. Use from_google_drive for automatic history initialization.
+    Persist accepted_head in the host application to retain rollback protection
+    across restarts; restore it through the factory's accepted_head argument.
+    """
+
+    def __init__(self, repository: DriveRepository, history: HistoryStore | None = None,
+                 *, executor=asyncio.to_thread):
+        self.repository = repository
+        self.history = history
+        self.executor = executor
+        self._lock = asyncio.Lock()
+        self._initialize_history = False
+
+    @classmethod
+    def from_google_drive(cls, session: aiohttp.ClientSession, token_provider, vehicle_folder: str,
+                          *, history_path: str | Path | None = None,
+                          expected_identity: dict | None = None, accepted_head: dict | None = None,
+                          executor=asyncio.to_thread) -> AAOSDriveClient:
+        """Use a caller-owned aiohttp session and async access-token provider.
+
+        Google OAuth consent and token refresh remain the caller's responsibility.
+        No credential is written to the local history database.
+        """
+        repository = DriveRepository(ReadOnlyDrive(session, token_provider), folder_id(vehicle_folder),
+                                     executor, expected_identity)
+        repository.accepted_head = dict(accepted_head) if accepted_head else None
+        history = HistoryStore(history_path) if history_path is not None else None
+        client = cls(repository, history, executor=executor)
+        client._initialize_history = history is not None
+        return client
+
+    @property
+    def snapshot(self) -> Snapshot | None:
+        return self.repository.snapshot
+
+    @property
+    def accepted_head(self) -> dict | None:
+        head = self.repository.accepted_head
+        return deepcopy(head) if head else None
+
+    async def refresh(self) -> Snapshot:
+        """Read the latest complete generation and retain old data on failure.
+
+        Publication races are retried three times. A generation is accepted only
+        after every required archive is validated and local history is complete.
+        """
+        async with self._lock:
+            if self._initialize_history:
+                await self.executor(self.history.initialize)
+                self._initialize_history = False
+            for attempt in range(3):
+                try:
+                    snapshot = await self.repository.refresh(accept=False)
+                    if self.history is not None:
+                        await sync_history(self.history, self.repository, snapshot, self.executor)
+                    break
+                except (ExportNotReady, DriveNotFoundError):
+                    if attempt == 2:
+                        raise
+                    await asyncio.sleep(2 ** attempt)
+            self.repository.accept_snapshot(snapshot)
+            snapshot.checked_at = datetime.now(timezone.utc)
+            if snapshot.previous_identity is not None:
+                self.repository.expected_identity = {
+                    key: snapshot.manifest.raw[key] for key in ("datasetId", "vehicleId")
+                }
+            return snapshot
+
+    async def _query(self, method: str, *args) -> Any:
+        async with self._lock:
+            if self.history is None:
+                raise ValueError("Configure history_path to query complete recorded history")
+            if self._initialize_history:
+                await self.executor(self.history.initialize)
+                self._initialize_history = False
+            if not (await self.executor(self.history.state))["generation_id"]:
+                raise ValueError("Import a complete generation before querying history")
+            return await self.executor(getattr(self.history, method), *args)
+
+    async def history_fields(self) -> dict:
+        return await self._query("overview")
+
+    async def list_trips(self, period: str = "") -> list[dict]:
+        return await self._query("summaries", period)
+
+    async def get_trip(self, trip_id: str = "") -> dict:
+        """Return the exact original trip, including its raw sample arrays."""
+        return await self._query("trip", trip_id)
+
+    async def get_history(self, field: str, *, start: int | None = None, end: int | None = None,
+                          limit: int = 2000, cursor: dict | None = None,
+                          trip_id: str | None = None) -> dict:
+        """Return timestamped readings and a generation-bound next_cursor."""
+        return await self._query("points", field, start, end, limit, cursor, trip_id)

aaos_logging_drive/const.py

@@ -0,0 +1,64 @@
+"""AAOS Logging import constants."""
+
+READ_SCOPE = "https://www.googleapis.com/auth/drive.readonly"
+API_ROOT = "https://www.googleapis.com/drive/v3"
+FOLDER_MIME = "application/vnd.google-apps.folder"
+MAX_FILE_BYTES = 128 * 1024 * 1024
+MAX_ZIP_BYTES = 512 * 1024 * 1024
+HISTORY_PROCESSING_VERSION = 4
+
+
+def readonly_grant(token: dict) -> bool:
+    scopes = set(token.get("scope", "").split())
+    return READ_SCOPE in scopes and not any(
+        s.startswith("https://www.googleapis.com/auth/drive") and s != READ_SCOPE for s in scopes
+    )
+
+# Exact keys from AAOS Logging's TelemetryJsonContract (trip schema 17).
+TELEMETRY_FIELDS = (
+    "recordedAt", "vehicleType", "vehicleTypeSource", "fuelLevelLiters",
+    "fuelCapacityLiters", "fuelLevelLow", "fuelDoorOpen", "fuelConsumptionLitersPer100Km",
+    "speedKph", "acceleratorPercent", "brakePercent", "lateralG",
+    "longitudinalG", "powerKw", "powerSource", "powerSignalDirection", "batteryEnergyWh", "turnSignalState",
+    "steeringAngleDeg", "batteryPercent", "batteryTemperatureC",
+    "batteryCurrentCapacityWh", "batteryNominalCapacityWh", "batteryCapacityRetentionPercent",
+    "vehicleOutsideTemperatureC", "chargePortConnected", "chargePortOpen", "chargeState",
+    "tirePressureKpaLeftFront", "tirePressureKpaRightFront", "tirePressureKpaLeftRear",
+    "tirePressureKpaRightRear", "brakeFluidLevelLow", "brakePadWearPercentLeftFront",
+    "brakePadWearPercentRightFront", "brakePadWearPercentLeftRear",
+    "brakePadWearPercentRightRear", "headlightsState", "highBeamLightsState",
+    "fogLightsState", "frontFogLightsState", "rearFogLightsState", "hazardLightsState",
+    "windshieldWipersState", "absActive", "tractionControlActive",
+    "electronicStabilityControlState", "currentGear", "parkingBrakeOn",
+    "parkingBrakeAutoApply", "regenerativeBrakingLevel", "evStoppingMode", "nightMode",
+    "wheelTickResetCount", "wheelTickLeftFront", "wheelTickRightFront", "wheelTickRightRear",
+    "wheelTickLeftRear", "seatOccupancyFrontLeft", "seatOccupancyFrontRight",
+    "seatOccupancyRearLeft", "seatOccupancyRearRight", "engineRpm",
+    "engineCoolantTemperatureC", "engineOilTemperatureC", "engineOilLevel",
+    "engineOilPressureKpa", "forwardCollisionWarningState", "automaticEmergencyBrakingState",
+    "laneDepartureWarningState", "laneKeepAssistState", "laneCenteringAssistState",
+    "blindSpotWarningLeftState", "blindSpotWarningRightState", "cruiseControlState",
+    "cruiseControlTargetSpeedKph", "adaptiveCruiseLeadDistanceM", "rangeKm", "gearSelection",
+)
+BINARY_FIELDS = frozenset({
+    "chargePortConnected", "chargePortOpen", "brakeFluidLevelLow", "absActive",
+    "tractionControlActive", "parkingBrakeOn", "parkingBrakeAutoApply", "nightMode",
+    "fuelLevelLow", "fuelDoorOpen",
+})
+FUEL_USAGE_MEASUREMENTS = (
+    "movingFuelLiters", "movingConsumptionLitersPer100Km", "coveredDistanceKm",
+    "observedDistanceKm", "tripDistanceKm", "engineRunningSeconds", "engineObservedSeconds",
+    "engineOffMovingSeconds", "coveragePercent",
+)
+FUEL_USAGE_FIELDS = (*FUEL_USAGE_MEASUREMENTS, "source", "includesStationaryFuel", "limitations")
+TRIP_FIELDS = (
+    "tripId", "localId", "startedAt", "endedAt", "storageUnitSystem", "distanceKm",
+    "energyKwh", "energySource", "powerSignalDirection", "averageSpeedKph", "maxSpeedKph", "maxAccelerationG",
+    "maxBrakingG", "maxLateralG", "startLat", "startLon", "endLat", "endLon", "routeKey",
+    "durationSeconds", "efficiencyKwhPer100Km", "telemetryCount", "routePointCount",
+    "weatherSampleCount",
+    "energyQuality", "energyReviewVersion", "energyDescription", "reportedEnergyKwh", "reportedEnergySource",
+    "vehicleType", "vehicleTypeSource", "powertrainSchemaVersion",
+    *(f"fuelUsage.{key}" for key in FUEL_USAGE_FIELDS),
+)
+WEATHER_FIELDS = ("temperatureC", "windKph", "precipitationMm", "capturedAt", "latitude", "longitude")

aaos_logging_drive/energy.py

@@ -0,0 +1,143 @@
+"""Conservative battery-energy recovery, shared in behaviour with both Android apps.
+
+Never mutates an export. Trip energy uses qualified endpoints, not integration
+of the smoothed power estimate. Fuel use is never inferred from battery depletion.
+"""
+
+from collections import deque
+from dataclasses import dataclass
+import math
+
+DERIVED = {"derived_battery_energy", "mixed_vehicle_sensor_and_derived"}
+KNOWN = DERIVED | {"vehicle_sensor"}
+REVIEW_VERSION = 1
+QUALITY_FIELDS = {"reportedPowerKw", "reportedPowerSource", "powerQuality"}
+
+
+def finite(value):
+    return type(value) in (int, float) and math.isfinite(value)
+
+
+def charging(point):
+    return point.get("chargePortConnected") is True or point.get("chargeState") == 1
+
+
+class BatteryEvidence:
+    def __init__(self):
+        self.first_at = self.last_at = None
+        self.first_wh = self.last_wh = None
+        self.previous_at = self.previous_wh = self.observation_at = None
+        self.invalid = False
+        self.changes = self.moving_ms = self.power_count = self.nonzero_count = 0
+
+    def add(self, point):
+        at, power, wh, speed = (point.get(k) for k in ("recordedAt", "powerKw", "batteryEnergyWh", "speedKph"))
+        if finite(power):
+            self.power_count += 1
+            self.nonzero_count += abs(power) > .01
+        if self.observation_at is not None and finite(speed) and abs(speed) >= 3 and 1 <= at - self.observation_at <= 15000:
+            self.moving_ms += at - self.observation_at
+        self.observation_at = at
+        if charging(point) or not finite(wh) or wh < 0:
+            self.invalid = True
+            return
+        if self.previous_at is None:
+            self.first_at, self.first_wh = at, wh
+        else:
+            elapsed = at - self.previous_at
+            delta = abs(wh - self.previous_wh)
+            if not 1 <= elapsed <= 15000 or delta > max(50, max(0, elapsed) * 500 / 3600):
+                self.invalid = True
+            self.changes += delta >= 1
+        self.previous_at, self.previous_wh = at, wh
+        self.last_at, self.last_wh = at, wh
+
+    def battery_kwh(self, start, end):
+        if (self.invalid or self.first_wh is None or not finite(end) or end <= start or
+                self.last_at - self.first_at < 10000 or abs(self.first_at - start) > 10000 or abs(end - self.last_at) > 10000):
+            return None
+        energy = (self.first_wh - self.last_wh) / 1000
+        return energy if abs(energy) * 3600000 / (end - start) <= 250 else None
+
+    def suspect_zero(self, start, end):
+        if not finite(end) or end - start < 30000 or self.nonzero_count or self.power_count < 4 or self.moving_ms < 10000:
+            return False
+        contradiction = (self.changes >= 3 and self.first_wh is not None and self.last_wh is not None and
+                         abs(self.first_wh - self.last_wh) >= 20)
+        return contradiction or self.battery_kwh(start, end) is None
+
+
+class BatteryPowerWindow:
+    def __init__(self):
+        self.samples = deque()
+
+    def add(self, point):
+        at, wh = point.get("recordedAt"), point.get("batteryEnergyWh")
+        if charging(point) or not finite(wh) or wh < 0:
+            self.samples.clear()
+            return None
+        if self.samples:
+            last_at, last_wh = self.samples[-1]
+            elapsed = at - last_at
+            if not 1 <= elapsed <= 15000 or abs(wh - last_wh) > max(50, max(0, elapsed) * 500 / 3600):
+                self.samples.clear()
+        self.samples.append((at, wh))
+        while len(self.samples) > 1 and at - self.samples[0][0] > 60000:
+            self.samples.popleft()
+        first_at, first_wh = self.samples[0]
+        if at - first_at < 10000:
+            return None
+        power = (first_wh - wh) * 3600 / (at - first_at)
+        return power if finite(power) and abs(power) <= 250 else None
+
+
+@dataclass(frozen=True)
+class EnergyReview:
+    energy: float | None
+    source: str | None
+    quality: str
+    repair_power: bool = False
+
+
+def review_energy(trip):
+    energy, source = trip.get("energyKwh"), trip.get("energySource")
+    if trip.get("vehicleType") == "FUEL":
+        usable = finite(energy) and source in KNOWN
+        return EnergyReview(energy if usable else None, source, "estimated" if usable and source in DERIVED else "measured" if usable else "unavailable")
+    # Normal measured history needs no battery scan and has no artificial power cap.
+    if finite(energy) and energy != 0 and source != "unavailable_energy":
+        return EnergyReview(energy, source, "estimated" if source in DERIVED else "measured")
+    evidence = BatteryEvidence()
+    for point in trip.get("telemetry", []):
+        evidence.add(point)
+    start, end = trip["startedAt"], trip.get("endedAt")
+    suspect = energy == 0 and evidence.suspect_zero(start, end)
+    missing = (source == "unavailable_energy" or not finite(energy) or
+               energy == 0 and evidence.power_count == 0 and source not in KNOWN)
+    if suspect or missing:
+        recovered = evidence.battery_kwh(start, end)
+        if recovered is not None:
+            return EnergyReview(recovered, "derived_battery_energy", "recovered", True)
+        return EnergyReview(None, "unavailable_energy", "unavailable", True)
+    return EnergyReview(energy, source, "estimated" if source in DERIVED else "measured")
+
+
+def effective_telemetry(trip, review):
+    """One bounded rolling window per affected trip; all other readings stay exact."""
+    window = BatteryPowerWindow()
+    for point in trip.get("telemetry", []):
+        if not review.repair_power:
+            yield point
+            continue
+        power = window.add(point) if review.quality == "recovered" else None
+        yield {**point, "reportedPowerKw": point.get("powerKw"), "reportedPowerSource": point.get("powerSource"),
+               "powerKw": power, "powerSource": review.source,
+               "powerQuality": "estimated" if power is not None else "unavailable"}
+
+
+def energy_description(quality):
+    if quality in {"recovered", "estimated"}:
+        return "Estimated net battery energy, including regeneration and auxiliaries. Smoothed power cannot show precise peaks. Hybrid fuel use is separate."
+    if quality == "unavailable":
+        return "Battery consumption unavailable: insufficient reliable energy readings; excluded from consumption averages."
+    return "Vehicle-measured battery energy."