quickbase-extract 0.1.0__py3-none-any.whl

quickbase_extract/__init__.py

@@ -0,0 +1,98 @@
+"""Quickbase Extract - Extract and cache Quickbase report data.
+
+A Python package for efficiently retrieving, transforming, and caching data
+from Quickbase reports with built-in error handling, retry logic, and S3 support
+for Lambda environments.
+
+Quick Start:
+    >>> from quickbase_extract import get_qb_client, refresh_all, load_report_metadata_batch
+    >>> from quickbase_extract import get_data_parallel
+    >>>
+    >>> # Initialize client
+    >>> client = get_qb_client(realm="example.quickbase.com", user_token="...")
+    >>>
+    >>> # Refresh metadata cache
+    >>> refresh_all(client, report_configs)
+    >>>
+    >>> # Load metadata and fetch data
+    >>> metadata = load_report_metadata_batch(report_configs)
+    >>> data = get_data_parallel(client, metadata, ["report1", "report2"], cache=True)
+"""
+
+import logging
+
+# API operations with error handling
+from quickbase_extract.api_handlers import QuickbaseOperationError, handle_delete, handle_query, handle_upsert
+
+# Cache monitoring
+from quickbase_extract.cache_freshness import (
+    CacheFileInfo,
+    CacheSummary,
+    check_cache_freshness,
+    get_cache_files,
+    get_cache_summary,
+)
+
+# Cache management
+from quickbase_extract.cache_manager import CacheManager, get_cache_manager
+from quickbase_extract.cache_sync import is_cache_synced, sync_from_s3_once
+
+# Client
+from quickbase_extract.client import get_qb_client
+
+# Report data retrieval
+from quickbase_extract.report_data import get_data, get_data_parallel, load_data, load_data_batch
+
+# Report metadata
+from quickbase_extract.report_metadata import (
+    get_report_metadata,
+    get_report_metadata_parallel,
+    load_report_metadata,
+    load_report_metadata_batch,
+    refresh_all,
+)
+
+# Utilities
+from quickbase_extract.utils import find_report, normalize_name
+
+__version__ = "0.1.0"
+
+# Configure logging
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+__all__ = [
+    # Version
+    "__version__",
+    # Client
+    "get_qb_client",
+    # Cache management
+    "CacheManager",
+    "get_cache_manager",
+    "sync_from_s3_once",
+    "is_cache_synced",
+    # Cache monitoring
+    "CacheFileInfo",
+    "CacheSummary",
+    "check_cache_freshness",
+    "get_cache_files",
+    "get_cache_summary",
+    # API operations
+    "QuickbaseOperationError",
+    "handle_delete",
+    "handle_query",
+    "handle_upsert",
+    # Report metadata
+    "get_report_metadata",
+    "get_report_metadata_parallel",
+    "load_report_metadata",
+    "load_report_metadata_batch",
+    "refresh_all",
+    # Report data
+    "get_data",
+    "get_data_parallel",
+    "load_data",
+    "load_data_batch",
+    # Utilities
+    "find_report",
+    "normalize_name",
+]

quickbase_extract/api_handlers.py

@@ -0,0 +1,210 @@
+"""Error handling utilities for Quickbase operations.
+
+Provides retry logic for rate-limited requests, standardized error handling,
+and logging for Quickbase API operations.
+"""
+
+import logging
+import random
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class QuickbaseOperationError(Exception):
+    """Raised when a Quickbase API operation fails."""
+
+    def __init__(self, operation: str, details: str = ""):
+        self.operation = operation
+        self.details = details
+        super().__init__(f"Quickbase {operation} failed: {details}")
+
+
+def handle_upsert(
+    client,
+    table_id: str,
+    data: list[dict],
+    description: str = "",
+    max_retries: int = 3,
+) -> dict:
+    """Execute a Quickbase upsert with error handling, retry logic, and logging.
+
+    Retries on rate limiting (429 errors) with exponential backoff and jitter.
+    Wait time is capped at 60 seconds per retry.
+
+    Args:
+        client: Quickbase API client.
+        table_id: Target table ID.
+        data: List of record dicts to upsert.
+        description: Human-readable description for logging. Defaults to empty string.
+        max_retries: Maximum number of retry attempts. Defaults to 3.
+
+    Returns:
+        API response dict containing metadata about created/updated/unchanged records.
+
+    Raises:
+        QuickbaseOperationError: If the upsert fails after all retries.
+
+    Example:
+        >>> records = [{"6": {"value": "John"}, "7": {"value": "Doe"}}]
+        >>> result = handle_upsert(client, "bq8xyx9z", records, "customer records")
+    """
+    for attempt in range(max_retries):
+        try:
+            result = client.upsert_records(table_id, data=data)
+
+            created = result.get("metadata", {}).get("createdRecordIds", [])
+            updated = result.get("metadata", {}).get("updatedRecordIds", [])
+            unchanged = result.get("metadata", {}).get("unchangedRecordIds", [])
+
+            logger.info(
+                f"Upsert {description}: {len(created)} created, {len(updated)} updated, " f"{len(unchanged)} unchanged"
+            )
+
+            return result
+
+        except Exception as e:  # noqa: BLE001  # Need to catch all exceptions for retry logic
+            error_str = str(e)
+
+            # Retry on 429 (rate limit)
+            if "429" in error_str and attempt < max_retries - 1:
+                wait_time = min(2**attempt, 60) + random.uniform(0, 1)
+                logger.warning(
+                    f"Rate limited on upsert {description} (attempt {attempt + 1}/{max_retries}), "
+                    f"retrying in {wait_time:.1f}s"
+                )
+                time.sleep(wait_time)
+            else:
+                logger.error(f"Upsert {description} failed: {error_str}")
+                raise QuickbaseOperationError("upsert", error_str) from e
+
+
+def handle_delete(
+    client,
+    table_id: str,
+    where: str,
+    description: str = "",
+    max_retries: int = 3,
+) -> int:
+    """Execute a Quickbase delete with error handling, logging, and rate limit retry.
+
+    Only retries on rate limiting (429 errors) with exponential backoff and jitter.
+    Other errors fail immediately for safety. Wait time is capped at 60 seconds per retry.
+
+    Args:
+        client: Quickbase API client.
+        table_id: Target table ID.
+        where: Quickbase filter string specifying records to delete.
+        description: Human-readable description for logging. Defaults to empty string.
+        max_retries: Maximum number of retry attempts for rate limits. Defaults to 3.
+
+    Returns:
+        Number of records deleted.
+
+    Raises:
+        QuickbaseOperationError: If the delete fails.
+
+    Example:
+        >>> deleted = handle_delete(client, "bq8xyx9z", "{3.EX.'test'}", "test records")
+
+    Note:
+        For safety, only 429 (rate limit) errors are retried. All other errors
+        fail immediately to prevent unintended deletions.
+    """
+    for attempt in range(max_retries):
+        try:
+            deleted = client.delete_records(table_id, where=where)
+            logger.info(f"Delete {description}: {deleted} records deleted")
+            return deleted
+
+        except Exception as e:  # noqa: BLE001  # Need to catch all exceptions for retry logic
+            error_str = str(e)
+
+            # Only retry on 429 (rate limit) - other errors are too risky to retry
+            if "429" in error_str and attempt < max_retries - 1:
+                wait_time = min(2**attempt, 60) + random.uniform(0, 1)
+                logger.warning(
+                    f"Rate limited on delete {description} (attempt {attempt + 1}/{max_retries}), "
+                    f"retrying in {wait_time:.1f}s"
+                )
+                time.sleep(wait_time)
+            else:
+                logger.error(f"Delete {description} failed: {error_str}")
+                raise QuickbaseOperationError("delete", error_str) from e
+
+
+def handle_query(
+    client,
+    table_id: str,
+    *,
+    select: list[int] = None,
+    where: str = None,
+    sort_by: list[dict] = None,
+    group_by: list[dict] = None,
+    options: dict = None,
+    description: str = "",
+    max_retries: int = 3,
+) -> dict:
+    """Execute a Quickbase query with error handling, retry logic, and logging.
+
+    Retries on rate limiting (429 errors) with exponential backoff and jitter.
+    Wait time is capped at 60 seconds per retry.
+
+    Args:
+        client: Quickbase API client.
+        table_id: Target table ID.
+        select: List of field IDs to return. If omitted, returns fields from
+            the default report.
+        where: A Quickbase query string (e.g., "{12.EX.'VPF'}").
+        sort_by: Sort order, e.g., [{"fieldId": 6, "order": "ASC"}].
+        group_by: Grouping, e.g., [{"fieldId": 6, "grouping": "equal-values"}].
+        options: Additional options, e.g.,
+            {"skip": 0, "top": 100, "compareWithAppLocalTime": False}.
+        description: Human-readable description for logging. Defaults to empty string.
+        max_retries: Maximum number of retry attempts. Defaults to 3.
+
+    Returns:
+        API response dict containing query results.
+
+    Raises:
+        QuickbaseOperationError: If the query fails after all retries.
+
+    Example:
+        >>> result = handle_query(
+        ...     client,
+        ...     "bq8xyx9z",
+        ...     select=[6, 7, 8],
+        ...     where="{12.EX.'Active'}",
+        ...     description="active customers"
+        ... )
+    """
+    for attempt in range(max_retries):
+        try:
+            result = client.query_for_data(
+                table_id,
+                select=select,
+                where=where,
+                sort_by=sort_by,
+                group_by=group_by,
+                options=options,
+            )
+            record_count = len(result.get("data", []))
+            desc_str = f" {description}" if description else ""
+            logger.info(f"Query{desc_str} returned {record_count} records")
+            return result
+
+        except Exception as e:  # noqa: BLE001  # Need to catch all exceptions for retry logic
+            error_str = str(e)
+
+            if "429" in error_str and attempt < max_retries - 1:
+                wait_time = min(2**attempt, 60) + random.uniform(0, 1)
+                desc_str = f" {description}" if description else f" table {table_id}"
+                logger.warning(
+                    f"Rate limited on query{desc_str} (attempt {attempt + 1}/{max_retries}), "
+                    f"retrying in {wait_time:.1f}s"
+                )
+                time.sleep(wait_time)
+            else:
+                desc_str = f" {description}" if description else f" on table {table_id}"
+                logger.error(f"Query{desc_str} failed: {error_str}")
+                raise QuickbaseOperationError("query", error_str) from e

quickbase_extract/cache_freshness.py

@@ -0,0 +1,199 @@
+"""Cache monitoring and freshness detection.
+
+Inspects cached JSON files, checks their age, and identifies stale cache entries.
+Works with both local and Lambda environments via CacheManager.
+"""
+
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import TypedDict
+
+from quickbase_extract.cache_manager import get_cache_manager
+
+logger = logging.getLogger(__name__)
+
+# Cache freshness thresholds (in hours)
+# Metadata rarely changes, so longer threshold is acceptable
+DEFAULT_METADATA_STALE_HOURS = 168  # 7 days
+# Data should be refreshed more frequently
+DEFAULT_DATA_STALE_HOURS = 24  # 1 day
+# General default when cache type isn't specified
+DEFAULT_STALE_THRESHOLD_HOURS = 36
+
+
+class CacheFileInfo(TypedDict):
+    """Information about a cached file."""
+
+    file: str
+    path: Path
+    size_bytes: int
+    size_mb: float
+    modified: datetime
+    age_hours: float
+
+
+class CacheSummary(TypedDict):
+    """Summary statistics for cache directory."""
+
+    cache_dir: str
+    total_files: int
+    total_size_mb: float
+    oldest_file: str | None
+    oldest_age_hours: float
+    newest_file: str | None
+    newest_age_hours: float
+
+
+def get_cache_files(cache_root: Path | None = None) -> list[CacheFileInfo]:
+    """Get all cached JSON files with their metadata.
+
+    Scans report_data and report_metadata directories for JSON files.
+
+    Args:
+        cache_root: Optional cache root path. If not provided, uses CacheManager default.
+
+    Returns:
+        List of dicts with file path, size, and modification time.
+        Sorted by age in descending order (oldest files first).
+
+    Raises:
+        FileNotFoundError: If cache directory doesn't exist.
+        PermissionError: If cache directory is not readable.
+
+    Example:
+        >>> files = get_cache_files()
+        >>> print(f"Oldest file: {files[0]['file']}, age: {files[0]['age_hours']}h")
+    """
+    if cache_root is None:
+        cache_mgr = get_cache_manager()
+        cache_root = cache_mgr.cache_root
+    else:
+        cache_root = Path(cache_root)
+
+    if not cache_root.exists():
+        raise FileNotFoundError(f"Cache directory does not exist: {cache_root}")
+
+    if not cache_root.is_dir():
+        raise NotADirectoryError(f"Cache path is not a directory: {cache_root}")
+
+    try:
+        files = []
+
+        # Scan all JSON files in cache directory
+        for json_file in cache_root.rglob("*.json"):
+            try:
+                stat = json_file.stat()
+                age_hours = (time.time() - stat.st_mtime) / 3600
+
+                files.append(
+                    {
+                        "file": str(json_file.relative_to(cache_root)),
+                        "path": json_file,
+                        "size_bytes": stat.st_size,
+                        "size_mb": round(stat.st_size / (1024 * 1024), 2),
+                        "modified": datetime.fromtimestamp(stat.st_mtime),
+                        "age_hours": round(age_hours, 1),
+                    }
+                )
+            except (OSError, ValueError) as e:
+                # File might have been deleted or become inaccessible
+                logger.warning(f"Failed to stat file {json_file}: {e}")
+                continue
+
+        # Sort by age descending (oldest first)
+        return sorted(files, key=lambda x: x["age_hours"], reverse=True)
+
+    except PermissionError as e:
+        raise PermissionError(f"Cannot read cache directory {cache_root}: {e}") from e
+
+
+def check_cache_freshness(
+    threshold_hours: float = DEFAULT_STALE_THRESHOLD_HOURS,
+    cache_root: Path | None = None,
+) -> list[CacheFileInfo]:
+    """Check for stale cache files.
+
+    Args:
+        threshold_hours: Files older than this are considered stale. Defaults to
+            DEFAULT_STALE_THRESHOLD_HOURS (36 hours).
+        cache_root: Optional cache root path. If not provided, uses CacheManager default.
+
+    Returns:
+        List of stale file info dicts, sorted by age descending (oldest first).
+        Empty list if all files are fresh.
+
+    Raises:
+        FileNotFoundError: If cache directory doesn't exist.
+        PermissionError: If cache directory is not readable.
+
+    Example:
+        >>> stale = check_cache_freshness(threshold_hours=24)
+        >>> if stale:
+        ...     print(f"Found {len(stale)} files older than 24 hours")
+    """
+    files = get_cache_files(cache_root=cache_root)
+    stale = [f for f in files if f["age_hours"] > threshold_hours]
+
+    if stale:
+        logger.warning(
+            f"Found {len(stale)} stale cache files (older than {threshold_hours}h). "
+            f"Oldest: {stale[0]['file']} ({stale[0]['age_hours']}h)"
+        )
+    else:
+        logger.info(f"All {len(files)} cache files are fresh (within {threshold_hours}h)")
+
+    return stale
+
+
+def get_cache_summary(cache_root: Path | None = None) -> CacheSummary:
+    """Get a summary of the cache directory.
+
+    Args:
+        cache_root: Optional cache root path. If not provided, uses CacheManager default.
+
+    Returns:
+        Dict with total files, size, oldest/newest file info.
+
+    Raises:
+        FileNotFoundError: If cache directory doesn't exist.
+        PermissionError: If cache directory is not readable.
+
+    Example:
+        >>> summary = get_cache_summary()
+        >>> print(f"Cache: {summary['total_files']} files, {summary['total_size_mb']} MB")
+        >>> print(f"Oldest: {summary['oldest_file']} ({summary['oldest_age_hours']}h)")
+    """
+    if cache_root is None:
+        cache_mgr = get_cache_manager()
+        cache_root = cache_mgr.cache_root
+    else:
+        cache_root = Path(cache_root)
+
+    files = get_cache_files(cache_root=cache_root)
+
+    if not files:
+        return {
+            "cache_dir": str(cache_root),
+            "total_files": 0,
+            "total_size_mb": 0.0,
+            "oldest_file": None,
+            "oldest_age_hours": 0.0,
+            "newest_file": None,
+            "newest_age_hours": 0.0,
+        }
+
+    total_size = sum(f["size_bytes"] for f in files)
+    oldest = files[0]  # First item in descending age order
+    newest = files[-1]  # Last item in descending age order
+
+    return {
+        "cache_dir": str(cache_root),
+        "total_files": len(files),
+        "total_size_mb": round(total_size / (1024 * 1024), 1),
+        "oldest_file": oldest["file"],
+        "oldest_age_hours": oldest["age_hours"],
+        "newest_file": newest["file"],
+        "newest_age_hours": newest["age_hours"],
+    }