waze-logs 1.0.0__py3-none-any.whl

database.py

@@ -0,0 +1,176 @@
+# database.py
+import sqlite3
+from pathlib import Path
+from typing import Optional, List, Any
+
+class Database:
+    def __init__(self, db_path: str, check_same_thread: bool = True):
+        self.db_path = db_path
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        # timeout=30 waits up to 30 seconds for locks instead of failing immediately
+        self.conn = sqlite3.connect(db_path, check_same_thread=check_same_thread, timeout=30)
+        self.conn.row_factory = sqlite3.Row
+        # Enable WAL mode for better concurrent write performance
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self.conn.execute("PRAGMA busy_timeout=30000")  # 30 second busy timeout
+        self._create_tables()
+
+    def _create_tables(self):
+        self.conn.executescript("""
+            CREATE TABLE IF NOT EXISTS events (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                event_hash TEXT UNIQUE NOT NULL,
+                username TEXT NOT NULL,
+                latitude REAL NOT NULL,
+                longitude REAL NOT NULL,
+                timestamp_utc TEXT NOT NULL,
+                timestamp_ms INTEGER NOT NULL,
+                report_type TEXT NOT NULL,
+                subtype TEXT,
+                raw_json TEXT,
+                collected_at TEXT NOT NULL,
+                grid_cell TEXT NOT NULL
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_events_username ON events(username);
+            CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp_ms);
+            CREATE INDEX IF NOT EXISTS idx_events_location ON events(latitude, longitude);
+            CREATE INDEX IF NOT EXISTS idx_events_type ON events(report_type);
+
+            CREATE TABLE IF NOT EXISTS collection_runs (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                started_at TEXT NOT NULL,
+                completed_at TEXT,
+                grid_cell TEXT NOT NULL,
+                events_found INTEGER DEFAULT 0,
+                events_new INTEGER DEFAULT 0,
+                status TEXT DEFAULT 'running'
+            );
+
+            CREATE TABLE IF NOT EXISTS tracked_users (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                username TEXT UNIQUE NOT NULL,
+                first_seen TEXT NOT NULL,
+                last_seen TEXT NOT NULL,
+                event_count INTEGER DEFAULT 1,
+                notes TEXT
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_tracked_users_username ON tracked_users(username);
+
+            CREATE TABLE IF NOT EXISTS daily_stats (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                date TEXT UNIQUE NOT NULL,
+                events_collected INTEGER DEFAULT 0,
+                unique_users INTEGER DEFAULT 0,
+                api_requests INTEGER DEFAULT 0,
+                api_errors INTEGER DEFAULT 0,
+                grid_cells_scanned INTEGER DEFAULT 0,
+                by_type_json TEXT
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_daily_stats_date ON daily_stats(date);
+        """)
+        self.conn.commit()
+
+    def upsert_tracked_user(self, username: str, timestamp: str) -> bool:
+        """Track a user, updating last_seen and event_count if exists."""
+        try:
+            self.conn.execute("""
+                INSERT INTO tracked_users (username, first_seen, last_seen, event_count)
+                VALUES (?, ?, ?, 1)
+                ON CONFLICT(username) DO UPDATE SET
+                    last_seen = excluded.last_seen,
+                    event_count = event_count + 1
+            """, (username, timestamp, timestamp))
+            self.conn.commit()
+            return True
+        except Exception:
+            return False
+
+    def get_tracked_users(self, limit: int = 100):
+        """Get tracked users ordered by event count."""
+        return self.conn.execute("""
+            SELECT * FROM tracked_users
+            ORDER BY event_count DESC
+            LIMIT ?
+        """, (limit,)).fetchall()
+
+    def update_daily_stats(self, date: str, events: int = 0, users: int = 0,
+                           requests: int = 0, errors: int = 0, cells: int = 0,
+                           by_type: dict = None):
+        """Update daily collection statistics."""
+        import json
+        by_type_json = json.dumps(by_type) if by_type else None
+
+        self.conn.execute("""
+            INSERT INTO daily_stats (date, events_collected, unique_users,
+                                     api_requests, api_errors, grid_cells_scanned, by_type_json)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(date) DO UPDATE SET
+                events_collected = events_collected + excluded.events_collected,
+                unique_users = excluded.unique_users,
+                api_requests = api_requests + excluded.api_requests,
+                api_errors = api_errors + excluded.api_errors,
+                grid_cells_scanned = grid_cells_scanned + excluded.grid_cells_scanned,
+                by_type_json = excluded.by_type_json
+        """, (date, events, users, requests, errors, cells, by_type_json))
+        self.conn.commit()
+
+    def get_daily_stats(self, days: int = 30):
+        """Get daily stats for the last N days."""
+        return self.conn.execute("""
+            SELECT * FROM daily_stats
+            ORDER BY date DESC
+            LIMIT ?
+        """, (days,)).fetchall()
+
+    def get_collection_summary(self):
+        """Get overall collection summary."""
+        result = self.conn.execute("""
+            SELECT
+                COUNT(*) as total_events,
+                COUNT(DISTINCT username) as unique_users,
+                COUNT(DISTINCT DATE(timestamp_utc)) as days_collected,
+                MIN(timestamp_utc) as first_event,
+                MAX(timestamp_utc) as last_event,
+                COUNT(DISTINCT grid_cell) as grid_cells_used
+            FROM events
+        """).fetchone()
+        return dict(result) if result else {}
+
+    def execute(self, query: str, params: tuple = ()) -> sqlite3.Cursor:
+        return self.conn.execute(query, params)
+
+    def commit(self):
+        self.conn.commit()
+
+    def close(self):
+        self.conn.close()
+
+    def insert_event(self, event: dict) -> bool:
+        """Insert event, return True if inserted, False if duplicate."""
+        try:
+            self.conn.execute("""
+                INSERT INTO events (
+                    event_hash, username, latitude, longitude,
+                    timestamp_utc, timestamp_ms, report_type, subtype,
+                    raw_json, collected_at, grid_cell
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (
+                event["event_hash"],
+                event["username"],
+                event["latitude"],
+                event["longitude"],
+                event["timestamp_utc"],
+                event["timestamp_ms"],
+                event["report_type"],
+                event.get("subtype"),
+                event.get("raw_json"),
+                event["collected_at"],
+                event["grid_cell"]
+            ))
+            self.conn.commit()
+            return True
+        except sqlite3.IntegrityError:
+            return False

waze_client.py

@@ -0,0 +1,234 @@
+# waze_client.py
+import requests
+import time
+import random
+from typing import Tuple, List, Dict, Any
+
+
+class RateLimiter:
+    """Simple rate limiter with exponential backoff."""
+
+    def __init__(self, min_delay: float = 1.0, max_delay: float = 10.0, backoff_factor: float = 2.0):
+        self.min_delay = min_delay
+        self.max_delay = max_delay
+        self.backoff_factor = backoff_factor
+        self.current_delay = min_delay
+        self.last_request_time = 0
+        self.consecutive_errors = 0
+
+    def wait(self):
+        """Wait before making next request."""
+        elapsed = time.time() - self.last_request_time
+        # Add jitter to avoid synchronized requests
+        jitter = random.uniform(0, 0.5)
+        wait_time = max(0, self.current_delay + jitter - elapsed)
+        if wait_time > 0:
+            time.sleep(wait_time)
+        self.last_request_time = time.time()
+
+    def success(self):
+        """Call after successful request to reset backoff."""
+        self.consecutive_errors = 0
+        self.current_delay = self.min_delay
+
+    def error(self):
+        """Call after failed request to increase backoff."""
+        self.consecutive_errors += 1
+        self.current_delay = min(
+            self.max_delay,
+            self.min_delay * (self.backoff_factor ** self.consecutive_errors)
+        )
+
+
+class WazeClient:
+    """Client for querying Waze live traffic data directly."""
+
+    WAZE_API_URL = "https://www.waze.com/live-map/api/georss"
+
+    def __init__(self, server_url: str = None, timeout: int = 30):
+        """
+        Initialize WazeClient.
+
+        Args:
+            server_url: Ignored - kept for backwards compatibility.
+                       We now query Waze API directly.
+            timeout: Request timeout in seconds.
+        """
+        self.timeout = timeout
+        self.rate_limiter = RateLimiter(min_delay=1.5, max_delay=30.0)
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            "Referer": "https://www.waze.com/live-map",
+            "Accept": "application/json",
+            "Accept-Language": "en-US,en;q=0.9,es;q=0.8",
+        })
+
+    def get_traffic_notifications(
+        self,
+        lat_top: float,
+        lat_bottom: float,
+        lon_left: float,
+        lon_right: float,
+        max_retries: int = 3
+    ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+        """
+        Query Waze for traffic notifications in a bounding box.
+        Returns (alerts, jams) tuple.
+        Implements retry logic with exponential backoff.
+        """
+        last_error = None
+
+        for attempt in range(max_retries):
+            try:
+                # Wait according to rate limiter
+                self.rate_limiter.wait()
+
+                response = self.session.get(
+                    self.WAZE_API_URL,
+                    params={
+                        "top": str(lat_top),
+                        "bottom": str(lat_bottom),
+                        "left": str(lon_left),
+                        "right": str(lon_right),
+                        "env": "row",
+                        "types": "alerts,traffic,users"
+                    },
+                    timeout=self.timeout
+                )
+
+                # Check for rate limiting responses
+                if response.status_code == 429:
+                    self.rate_limiter.error()
+                    retry_after = int(response.headers.get("Retry-After", 60))
+                    time.sleep(retry_after)
+                    continue
+
+                if response.status_code == 403:
+                    self.rate_limiter.error()
+                    # Possible IP block, wait longer
+                    time.sleep(30 + random.uniform(0, 30))
+                    continue
+
+                response.raise_for_status()
+                self.rate_limiter.success()
+
+                data = response.json()
+
+                # Transform alerts to normalize the location format
+                alerts = []
+                for alert in data.get("alerts", []):
+                    # Extract location from nested structure
+                    loc = alert.get("location", {})
+                    transformed = {
+                        **alert,
+                        "latitude": loc.get("y", alert.get("latitude")),
+                        "longitude": loc.get("x", alert.get("longitude")),
+                        # Extract username from wazeData if available
+                        "reportBy": self._extract_username(alert)
+                    }
+                    alerts.append(transformed)
+
+                return alerts, data.get("jams", [])
+
+            except requests.exceptions.RequestException as e:
+                self.rate_limiter.error()
+                last_error = e
+                if attempt < max_retries - 1:
+                    # Wait with exponential backoff before retry
+                    wait_time = (2 ** attempt) + random.uniform(0, 1)
+                    time.sleep(wait_time)
+
+        # All retries failed
+        raise last_error or Exception("Failed to fetch data after retries")
+
+    def _extract_username(self, alert: Dict[str, Any]) -> str:
+        """Extract username from alert data."""
+        # wazeData format: "world,lon,lat,uuid" or sometimes contains username
+        waze_data = alert.get("wazeData", "")
+        if waze_data:
+            parts = waze_data.split(",")
+            if len(parts) >= 1:
+                # First part is often the username prefix (e.g., "world")
+                # or could be an actual username
+                return parts[0] if parts[0] != "world" else f"world_{parts[-1][:8]}"
+
+        # Fallback: use uuid as identifier
+        uuid = alert.get("uuid", "")
+        if uuid:
+            return f"user_{uuid[:8]}"
+
+        return "anonymous"
+
+    def get_users(
+        self,
+        lat_top: float,
+        lat_bottom: float,
+        lon_left: float,
+        lon_right: float
+    ) -> List[Dict[str, Any]]:
+        """Get active Waze users in a bounding box."""
+        self.rate_limiter.wait()
+
+        try:
+            response = self.session.get(
+                self.WAZE_API_URL,
+                params={
+                    "top": str(lat_top),
+                    "bottom": str(lat_bottom),
+                    "left": str(lon_left),
+                    "right": str(lon_right),
+                    "env": "row",
+                    "types": "users"
+                },
+                timeout=self.timeout
+            )
+            response.raise_for_status()
+            self.rate_limiter.success()
+            data = response.json()
+
+            users = []
+            for user in data.get("users", []):
+                loc = user.get("location", {})
+                users.append({
+                    **user,
+                    "latitude": loc.get("y"),
+                    "longitude": loc.get("x"),
+                })
+            return users
+        except requests.RequestException:
+            self.rate_limiter.error()
+            raise
+
+    def health_check(self) -> bool:
+        """Check if the Waze API is responding."""
+        try:
+            self.rate_limiter.wait()
+            response = self.session.get(
+                self.WAZE_API_URL,
+                params={
+                    "top": "40.43",
+                    "bottom": "40.42",
+                    "left": "-3.71",
+                    "right": "-3.70",
+                    "env": "row",
+                    "types": "alerts"
+                },
+                timeout=5
+            )
+            if response.status_code == 200:
+                self.rate_limiter.success()
+                return True
+            self.rate_limiter.error()
+            return False
+        except requests.RequestException:
+            self.rate_limiter.error()
+            return False
+
+    def get_rate_limit_status(self) -> dict:
+        """Get current rate limiter status."""
+        return {
+            "current_delay": self.rate_limiter.current_delay,
+            "consecutive_errors": self.rate_limiter.consecutive_errors,
+            "last_request": self.rate_limiter.last_request_time
+        }