quickbase-extract 0.2.1__tar.gz → 0.3.1__tar.gz

{quickbase_extract-0.2.1 → quickbase_extract-0.3.1}/.pre-commit-config.yaml

@@ -8,14 +8,13 @@ repos:
       - id: check-yaml
       - id: debug-statements    # Prevents accidental pdb/breakpoint commits
 
-  - repo: https://github.com/psf/black
-    rev: 23.12.1
-    hooks:
-      - id: black
-        language_version: python3.12
-
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.1.11
     hooks:
       - id: ruff
         args: [--fix]
+        stages: [pre-commit]
+      - id: ruff-format
+        stages: [pre-commit]
+      - id: ruff  # Run again to verify no more changes
+        stages: [pre-commit]

{quickbase_extract-0.2.1 → quickbase_extract-0.3.1}/CHANGELOG.md

@@ -5,6 +5,35 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.1] - 2026-04-27
+
+### Fixed
+
+- `complete_cache_refresh()` function now properly exported from `quickbase_extract` package for use in Lambda handlers
+
+## [0.3.0] - 2026-04-27
+
+### Added
+
+- `complete_cache_refresh()` function in `cache_sync` module for selective cache refresh in development
+- Support for granular cache refresh control:
+  - `force_all=True`: Refresh both metadata and data caches
+  - `force_metadata=True`: Refresh only metadata cache
+  - `force_data=True`: Refresh only data cache
+- Cache refresh workflow: clear /tmp → fetch fresh from Quickbase → update S3 → re-sync to /tmp
+- Manual development toggles in Lambda handlers for testing cache refresh without API changes
+- Comprehensive test suite for complete cache refresh functionality
+
+### Changed
+
+- Lambda workers can now force complete cache refresh for development/debugging by toggling code variables
+- Cache refresh strategy clarified: selective refresh based on what changed (metadata vs data)
+
+### Fixed
+
+- Dev workflow now supports forcing cache refresh without modifying Lambda event API
+- Ensures /tmp and S3 are synchronized after cache refresh operations
+
 ## [0.2.1] - 2026-04-25
 
 ### Fixed

{quickbase_extract-0.2.1 → quickbase_extract-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quickbase-extract
-Version: 0.2.1
+Version: 0.3.1
 Summary: Extract and cache Quickbase report data with built-in error handling and S3 support
 Project-URL: Homepage, https://github.com/tbrezler/quickbase-extract
 Project-URL: Repository, https://github.com/tbrezler/quickbase-extract.git
@@ -19,7 +19,6 @@ Requires-Python: >=3.12
 Requires-Dist: boto3>=1.26.0
 Requires-Dist: quickbase-api>=0.3.1
 Provides-Extra: dev
-Requires-Dist: black>=23.0; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
 Requires-Dist: ruff>=0.1.0; extra == 'dev'
@@ -955,10 +954,30 @@ def lambda_handler(event, context):
     - Cold start: Syncs cache from S3
     - Cache freshness: Auto-refreshes if stale
     - Data fetching: Queries Quickbase and caches results
+    - Development: Supports forcing complete cache refresh for testing
     """
     client = get_client()
     cache_mgr = get_cache_manager()
 
+    # OPTIONAL: Set to True to force cache refresh (for development/debugging only)
+    # Only one should be True at a time. force_all overrides the others.
+    FORCE_COMPLETE_CACHE_REFRESH_ALL = False
+    FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
+    FORCE_COMPLETE_CACHE_REFRESH_DATA = False
+
+    # Force complete cache refresh if needed (dev/debugging only)
+    if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
+            or FORCE_COMPLETE_CACHE_REFRESH_DATA):
+        from quickbase_extract import complete_cache_refresh
+        complete_cache_refresh(
+            cache_manager=cache_mgr,
+            client=client,
+            report_configs=get_all_reports(),
+            force_all=FORCE_COMPLETE_CACHE_REFRESH_ALL,
+            force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
+            force_data=FORCE_COMPLETE_CACHE_REFRESH_DATA,
+        )
+
     # Step 1: Sync cache from S3 on cold start (only on first invocation)
     sync_from_s3_once(cache_mgr)
 
@@ -1563,6 +1582,48 @@ def lambda_handler(event, context):
     metadata = load_report_metadata_batch(cache_mgr, config)
 ```
 
+#### `complete_cache_refresh(cache_manager, client, report_configs, force_all=False, force_metadata=False, force_data=False)`
+
+Completely refresh cache for development/debugging: clear /tmp, fetch fresh from Quickbase, update S3, re-sync to /tmp.
+
+This is a development utility for forcing a complete cache refresh when report metadata or configurations change. Clears local /tmp cache, fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
+
+**Parameters:**
+- `cache_manager` (CacheManager): Cache manager instance
+- `client`: Quickbase API client for fetching fresh data
+- `report_configs` (list[ReportConfig]): List of all ReportConfig instances to refresh
+- `force_all` (bool): If True, refresh both metadata and data. Defaults to False.
+- `force_metadata` (bool): If True (and `force_all` is False), refresh only metadata. Defaults to False.
+- `force_data` (bool): If True (and `force_all` is False), refresh only data. Defaults to False.
+
+**Returns:** None
+
+**Raises:**
+- `Exception`: If cache clearing or refresh operations fail
+
+**Example:**
+```python
+from quickbase_extract import complete_cache_refresh
+
+# Refresh only metadata (after changing report configurations)
+complete_cache_refresh(
+    cache_manager=cache_mgr,
+    client=qb_client,
+    report_configs=get_all_reports(),
+    force_metadata=True
+)
+
+# Refresh all (metadata + data)
+complete_cache_refresh(
+    cache_manager=cache_mgr,
+    client=qb_client,
+    report_configs=get_all_reports(),
+    force_all=True
+)
+```
+
+**Note:** This function is designed for development/debugging. To use in Lambda, add toggles to your handler (see "Development/Debug Mode" section below).
+
 ### Query Execution with Retry Logic
 
 #### `handle_query(client, table_id, *, select=None, where=None, sort_by=None, group_by=None, options=None, description="", max_retries=3)`
@@ -2243,6 +2304,47 @@ ask_values = {"ask1": "value"}    # No underscores
 
 ---
 
+### Issue: Lambda has old cached data after I changed report metadata
+
+**Symptom:** You updated a Quickbase report's fields, filters, or configuration, but your Lambda returns stale data.
+
+**Cause:** Cache was loaded before your changes, and it hasn't become "stale" enough to auto-refresh yet.
+
+**Solutions:**
+
+1. **Quick fix: Use force refresh toggle (development only)**
+
+   In your Lambda handler, temporarily set:
+   ```python
+   FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
+   ```
+
+   Upload new build, invoke Lambda, check logs for refresh. Then revert flag to `False`.
+
+2. **For immediate production fix:**
+
+   Manually delete files from S3:
+   ```bash
+   aws s3 rm s3://my-quickbase-cache-bucket/prod/cache/report_metadata/ --recursive
+   ```
+
+   Next Lambda cold start will re-fetch fresh metadata.
+
+3. **To prevent in future:**
+
+   Reduce `metadata_stale_hours` threshold:
+   ```python
+   ensure_cache_freshness(
+       client=client,
+       cache_manager=cache_mgr,
+       report_configs_all=get_all_reports(),
+       report_configs_to_cache=get_reports_to_cache(),
+       metadata_stale_hours=24  # Check daily instead of 30 days
+   )
+   ```
+
+---
+
 ### Issue: Performance degradation over time
 
 **Symptom:** First request fast, subsequent requests slow.
@@ -2622,6 +2724,95 @@ def scheduled_cache_refresh(event, context):
     return {"statusCode": 200, "message": "Cache refreshed"}
 ```
 
+### Development/Debug Mode: Forcing Complete Cache Refresh
+
+When you modify report metadata or configurations in Quickbase, your Lambda may still use stale cached data. Use the force refresh toggles to clear everything and fetch fresh data.
+
+#### When to Use
+
+- You changed a report's filters or fields in Quickbase
+- You added/removed fields from a report
+- You renamed a report or table
+- You need to verify fresh data is being fetched
+
+#### How to Use
+
+1. Open your Lambda handler code
+2. Set one of the toggle flags to `True`:
+
+```python
+# In lambda_handler, find these lines:
+FORCE_COMPLETE_CACHE_REFRESH_ALL = False
+FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
+FORCE_COMPLETE_CACHE_REFRESH_DATA = False
+
+# Change to (example: refresh only metadata):
+FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
+```
+
+3. Upload new Lambda build
+4. Invoke your Lambda (via API or CloudWatch event)
+5. Check CloudWatch logs for cache refresh messages
+6. **Revert the flag back to `False`** for normal operation
+
+#### Flag Options
+
+| Flag | What Gets Refreshed | Use When |
+|------|---------------------|----------|
+| `force_all=True` | Both metadata + data | Complete cache overhaul needed |
+| `force_metadata=True` | Only metadata | Report configuration changed |
+| `force_data=True` | Only data | Data needs fresh pull |
+
+#### What Happens
+
+When you trigger a force refresh:
+
+1. ✓ `/tmp` cache directories are deleted
+2. ✓ Fresh data fetched from Quickbase API
+3. ✓ Data written to S3
+4. ✓ `/tmp` re-synced from updated S3
+
+Your Lambda now has fresh data from Quickbase.
+
+#### Example
+
+```python
+def lambda_handler(event, context):
+    client = get_client()
+    cache_mgr = get_cache_manager()
+
+    # Metadata changed in Quickbase? Force refresh it:
+    FORCE_COMPLETE_CACHE_REFRESH_METADATA = True  # ← Toggle this
+
+    if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
+            or FORCE_COMPLETE_CACHE_REFRESH_DATA):
+        from quickbase_extract import complete_cache_refresh
+        complete_cache_refresh(
+            cache_manager=cache_mgr,
+            client=client,
+            report_configs=get_all_reports(),
+            force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
+        )
+
+    # Rest of handler...
+```
+
+**CloudWatch logs will show:**
+```
+WARNING: Starting complete cache refresh for: metadata (clearing /tmp, refreshing from Quickbase, updating S3...)
+DEBUG: Reset cache sync flag
+INFO: Fetching fresh data from Quickbase...
+INFO: Re-syncing /tmp from S3...
+WARNING: Complete cache refresh finished for metadata: /tmp and S3 now have fresh data from Quickbase
+```
+
+#### Important Notes
+
+- **Don't leave toggles set to `True`** — revert to `False` after testing
+- **Only for development** — not a production workflow
+- Logs will show exactly what was refreshed
+- Safe to use — doesn't affect running processes, only next Lambda invocation
+
 ## Advanced Usage
 
 ### Custom Report Configurations

{quickbase_extract-0.2.1 → quickbase_extract-0.3.1}/README.md

@@ -928,10 +928,30 @@ def lambda_handler(event, context):
     - Cold start: Syncs cache from S3
     - Cache freshness: Auto-refreshes if stale
     - Data fetching: Queries Quickbase and caches results
+    - Development: Supports forcing complete cache refresh for testing
     """
     client = get_client()
     cache_mgr = get_cache_manager()
 
+    # OPTIONAL: Set to True to force cache refresh (for development/debugging only)
+    # Only one should be True at a time. force_all overrides the others.
+    FORCE_COMPLETE_CACHE_REFRESH_ALL = False
+    FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
+    FORCE_COMPLETE_CACHE_REFRESH_DATA = False
+
+    # Force complete cache refresh if needed (dev/debugging only)
+    if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
+            or FORCE_COMPLETE_CACHE_REFRESH_DATA):
+        from quickbase_extract import complete_cache_refresh
+        complete_cache_refresh(
+            cache_manager=cache_mgr,
+            client=client,
+            report_configs=get_all_reports(),
+            force_all=FORCE_COMPLETE_CACHE_REFRESH_ALL,
+            force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
+            force_data=FORCE_COMPLETE_CACHE_REFRESH_DATA,
+        )
+
     # Step 1: Sync cache from S3 on cold start (only on first invocation)
     sync_from_s3_once(cache_mgr)
 
@@ -1536,6 +1556,48 @@ def lambda_handler(event, context):
     metadata = load_report_metadata_batch(cache_mgr, config)
 ```
 
+#### `complete_cache_refresh(cache_manager, client, report_configs, force_all=False, force_metadata=False, force_data=False)`
+
+Completely refresh cache for development/debugging: clear /tmp, fetch fresh from Quickbase, update S3, re-sync to /tmp.
+
+This is a development utility for forcing a complete cache refresh when report metadata or configurations change. Clears local /tmp cache, fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
+
+**Parameters:**
+- `cache_manager` (CacheManager): Cache manager instance
+- `client`: Quickbase API client for fetching fresh data
+- `report_configs` (list[ReportConfig]): List of all ReportConfig instances to refresh
+- `force_all` (bool): If True, refresh both metadata and data. Defaults to False.
+- `force_metadata` (bool): If True (and `force_all` is False), refresh only metadata. Defaults to False.
+- `force_data` (bool): If True (and `force_all` is False), refresh only data. Defaults to False.
+
+**Returns:** None
+
+**Raises:**
+- `Exception`: If cache clearing or refresh operations fail
+
+**Example:**
+```python
+from quickbase_extract import complete_cache_refresh
+
+# Refresh only metadata (after changing report configurations)
+complete_cache_refresh(
+    cache_manager=cache_mgr,
+    client=qb_client,
+    report_configs=get_all_reports(),
+    force_metadata=True
+)
+
+# Refresh all (metadata + data)
+complete_cache_refresh(
+    cache_manager=cache_mgr,
+    client=qb_client,
+    report_configs=get_all_reports(),
+    force_all=True
+)
+```
+
+**Note:** This function is designed for development/debugging. To use in Lambda, add toggles to your handler (see "Development/Debug Mode" section below).
+
 ### Query Execution with Retry Logic
 
 #### `handle_query(client, table_id, *, select=None, where=None, sort_by=None, group_by=None, options=None, description="", max_retries=3)`
@@ -2216,6 +2278,47 @@ ask_values = {"ask1": "value"}    # No underscores
 
 ---
 
+### Issue: Lambda has old cached data after I changed report metadata
+
+**Symptom:** You updated a Quickbase report's fields, filters, or configuration, but your Lambda returns stale data.
+
+**Cause:** Cache was loaded before your changes, and it hasn't become "stale" enough to auto-refresh yet.
+
+**Solutions:**
+
+1. **Quick fix: Use force refresh toggle (development only)**
+
+   In your Lambda handler, temporarily set:
+   ```python
+   FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
+   ```
+
+   Upload new build, invoke Lambda, check logs for refresh. Then revert flag to `False`.
+
+2. **For immediate production fix:**
+
+   Manually delete files from S3:
+   ```bash
+   aws s3 rm s3://my-quickbase-cache-bucket/prod/cache/report_metadata/ --recursive
+   ```
+
+   Next Lambda cold start will re-fetch fresh metadata.
+
+3. **To prevent in future:**
+
+   Reduce `metadata_stale_hours` threshold:
+   ```python
+   ensure_cache_freshness(
+       client=client,
+       cache_manager=cache_mgr,
+       report_configs_all=get_all_reports(),
+       report_configs_to_cache=get_reports_to_cache(),
+       metadata_stale_hours=24  # Check daily instead of 30 days
+   )
+   ```
+
+---
+
 ### Issue: Performance degradation over time
 
 **Symptom:** First request fast, subsequent requests slow.
@@ -2595,6 +2698,95 @@ def scheduled_cache_refresh(event, context):
     return {"statusCode": 200, "message": "Cache refreshed"}
 ```
 
+### Development/Debug Mode: Forcing Complete Cache Refresh
+
+When you modify report metadata or configurations in Quickbase, your Lambda may still use stale cached data. Use the force refresh toggles to clear everything and fetch fresh data.
+
+#### When to Use
+
+- You changed a report's filters or fields in Quickbase
+- You added/removed fields from a report
+- You renamed a report or table
+- You need to verify fresh data is being fetched
+
+#### How to Use
+
+1. Open your Lambda handler code
+2. Set one of the toggle flags to `True`:
+
+```python
+# In lambda_handler, find these lines:
+FORCE_COMPLETE_CACHE_REFRESH_ALL = False
+FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
+FORCE_COMPLETE_CACHE_REFRESH_DATA = False
+
+# Change to (example: refresh only metadata):
+FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
+```
+
+3. Upload new Lambda build
+4. Invoke your Lambda (via API or CloudWatch event)
+5. Check CloudWatch logs for cache refresh messages
+6. **Revert the flag back to `False`** for normal operation
+
+#### Flag Options
+
+| Flag | What Gets Refreshed | Use When |
+|------|---------------------|----------|
+| `force_all=True` | Both metadata + data | Complete cache overhaul needed |
+| `force_metadata=True` | Only metadata | Report configuration changed |
+| `force_data=True` | Only data | Data needs fresh pull |
+
+#### What Happens
+
+When you trigger a force refresh:
+
+1. ✓ `/tmp` cache directories are deleted
+2. ✓ Fresh data fetched from Quickbase API
+3. ✓ Data written to S3
+4. ✓ `/tmp` re-synced from updated S3
+
+Your Lambda now has fresh data from Quickbase.
+
+#### Example
+
+```python
+def lambda_handler(event, context):
+    client = get_client()
+    cache_mgr = get_cache_manager()
+
+    # Metadata changed in Quickbase? Force refresh it:
+    FORCE_COMPLETE_CACHE_REFRESH_METADATA = True  # ← Toggle this
+
+    if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
+            or FORCE_COMPLETE_CACHE_REFRESH_DATA):
+        from quickbase_extract import complete_cache_refresh
+        complete_cache_refresh(
+            cache_manager=cache_mgr,
+            client=client,
+            report_configs=get_all_reports(),
+            force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
+        )
+
+    # Rest of handler...
+```
+
+**CloudWatch logs will show:**
+```
+WARNING: Starting complete cache refresh for: metadata (clearing /tmp, refreshing from Quickbase, updating S3...)
+DEBUG: Reset cache sync flag
+INFO: Fetching fresh data from Quickbase...
+INFO: Re-syncing /tmp from S3...
+WARNING: Complete cache refresh finished for metadata: /tmp and S3 now have fresh data from Quickbase
+```
+
+#### Important Notes
+
+- **Don't leave toggles set to `True`** — revert to `False` after testing
+- **Only for development** — not a production workflow
+- Logs will show exactly what was refreshed
+- Safe to use — doesn't affect running processes, only next Lambda invocation
+
 ## Advanced Usage
 
 ### Custom Report Configurations

{quickbase_extract-0.2.1 → quickbase_extract-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "quickbase-extract"
-version = "0.2.1"
+version = "0.3.1"
 description = "Extract and cache Quickbase report data with built-in error handling and S3 support"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -31,7 +31,6 @@ dependencies = [
 dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",
-    "black>=23.0",
     "ruff>=0.1.0",
 ]
 

{quickbase_extract-0.2.1 → quickbase_extract-0.3.1}/src/quickbase_extract/__init__.py

@@ -28,12 +28,7 @@ import logging
 from importlib.metadata import version
 
 # API operations with error handling
-from quickbase_extract.api_handlers import (
-    QuickbaseOperationError,
-    handle_delete,
-    handle_query,
-    handle_upsert,
-)
+from quickbase_extract.api_handlers import QuickbaseOperationError, handle_delete, handle_query, handle_upsert
 
 # Cache management
 from quickbase_extract.cache_manager import CacheManager
@@ -42,18 +37,13 @@ from quickbase_extract.cache_manager import CacheManager
 from quickbase_extract.cache_orchestration import ensure_cache_freshness
 
 # Cache sync
-from quickbase_extract.cache_sync import is_cache_synced, sync_from_s3_once
+from quickbase_extract.cache_sync import complete_cache_refresh, is_cache_synced, sync_from_s3_once
 
 # Config
 from quickbase_extract.config import ReportConfig
 
 # Report data retrieval
-from quickbase_extract.report_data import (
-    get_data,
-    get_data_parallel,
-    load_data,
-    load_data_batch,
-)
+from quickbase_extract.report_data import get_data, get_data_parallel, load_data, load_data_batch
 
 # Report metadata
 from quickbase_extract.report_metadata import (
@@ -77,6 +67,7 @@ __all__ = [
     # Cache management
     "CacheManager",
     "ensure_cache_freshness",
+    "complete_cache_refresh",
     "sync_from_s3_once",
     "is_cache_synced",
     # Config