quickbase-extract 0.2.1__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/CHANGELOG.md +23 -0
  2. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/PKG-INFO +193 -1
  3. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/README.md +192 -0
  4. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/pyproject.toml +1 -1
  5. quickbase_extract-0.3.0/src/quickbase_extract/cache_sync.py +191 -0
  6. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/conftest.py +10 -0
  7. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_api_handlers.py +20 -3
  8. quickbase_extract-0.3.0/tests/test_cache_sync.py +360 -0
  9. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_report_data.py +11 -2
  10. quickbase_extract-0.2.1/src/quickbase_extract/cache_sync.py +0 -82
  11. quickbase_extract-0.2.1/tests/test_cache_sync.py +0 -119
  12. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.editorconfig +0 -0
  13. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.gitignore +0 -0
  14. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.pre-commit-config.yaml +0 -0
  15. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.python-version +0 -0
  16. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/LICENSE.txt +0 -0
  17. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/TODO.md +0 -0
  18. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/__init__.py +0 -0
  19. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/api_handlers.py +0 -0
  20. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/cache_manager.py +0 -0
  21. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/cache_orchestration.py +0 -0
  22. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/config.py +0 -0
  23. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/py.typed +0 -0
  24. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/report_data.py +0 -0
  25. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/report_metadata.py +0 -0
  26. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/utils.py +0 -0
  27. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_cache_manager.py +0 -0
  28. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_cache_orchestration.py +0 -0
  29. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_report_metadata.py +0 -0
  30. {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_utils.py +0 -0
@@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [0.3.0] - 2026-04-27
9
+
10
+ ### Added
11
+
12
+ - `complete_cache_refresh()` function in `cache_sync` module for selective cache refresh in development
13
+ - Support for granular cache refresh control:
14
+ - `force_all=True`: Refresh both metadata and data caches
15
+ - `force_metadata=True`: Refresh only metadata cache
16
+ - `force_data=True`: Refresh only data cache
17
+ - Cache refresh workflow: clear /tmp → fetch fresh from Quickbase → update S3 → re-sync to /tmp
18
+ - Manual development toggles in Lambda handlers for testing cache refresh without API changes
19
+ - Comprehensive test suite for complete cache refresh functionality
20
+
21
+ ### Changed
22
+
23
+ - Lambda workers can now force complete cache refresh for development/debugging by toggling code variables
24
+ - Cache refresh strategy clarified: selective refresh based on what changed (metadata vs data)
25
+
26
+ ### Fixed
27
+
28
+ - Dev workflow now supports forcing cache refresh without modifying Lambda event API
29
+ - Ensures /tmp and S3 are synchronized after cache refresh operations
30
+
8
31
  ## [0.2.1] - 2026-04-25
9
32
 
10
33
  ### Fixed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quickbase-extract
3
- Version: 0.2.1
3
+ Version: 0.3.0
4
4
  Summary: Extract and cache Quickbase report data with built-in error handling and S3 support
5
5
  Project-URL: Homepage, https://github.com/tbrezler/quickbase-extract
6
6
  Project-URL: Repository, https://github.com/tbrezler/quickbase-extract.git
@@ -955,10 +955,30 @@ def lambda_handler(event, context):
955
955
  - Cold start: Syncs cache from S3
956
956
  - Cache freshness: Auto-refreshes if stale
957
957
  - Data fetching: Queries Quickbase and caches results
958
+ - Development: Supports forcing complete cache refresh for testing
958
959
  """
959
960
  client = get_client()
960
961
  cache_mgr = get_cache_manager()
961
962
 
963
+ # OPTIONAL: Set to True to force cache refresh (for development/debugging only)
964
+ # Only one should be True at a time. force_all overrides the others.
965
+ FORCE_COMPLETE_CACHE_REFRESH_ALL = False
966
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
967
+ FORCE_COMPLETE_CACHE_REFRESH_DATA = False
968
+
969
+ # Force complete cache refresh if needed (dev/debugging only)
970
+ if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
971
+ or FORCE_COMPLETE_CACHE_REFRESH_DATA):
972
+ from quickbase_extract import complete_cache_refresh
973
+ complete_cache_refresh(
974
+ cache_manager=cache_mgr,
975
+ client=client,
976
+ report_configs=get_all_reports(),
977
+ force_all=FORCE_COMPLETE_CACHE_REFRESH_ALL,
978
+ force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
979
+ force_data=FORCE_COMPLETE_CACHE_REFRESH_DATA,
980
+ )
981
+
962
982
  # Step 1: Sync cache from S3 on cold start (only on first invocation)
963
983
  sync_from_s3_once(cache_mgr)
964
984
 
@@ -1563,6 +1583,48 @@ def lambda_handler(event, context):
1563
1583
  metadata = load_report_metadata_batch(cache_mgr, config)
1564
1584
  ```
1565
1585
 
1586
+ #### `complete_cache_refresh(cache_manager, client, report_configs, force_all=False, force_metadata=False, force_data=False)`
1587
+
1588
+ Completely refresh cache for development/debugging: clear /tmp, fetch fresh from Quickbase, update S3, re-sync to /tmp.
1589
+
1590
+ This is a development utility for forcing a complete cache refresh when report metadata or configurations change. Clears local /tmp cache, fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
1591
+
1592
+ **Parameters:**
1593
+ - `cache_manager` (CacheManager): Cache manager instance
1594
+ - `client`: Quickbase API client for fetching fresh data
1595
+ - `report_configs` (list[ReportConfig]): List of all ReportConfig instances to refresh
1596
+ - `force_all` (bool): If True, refresh both metadata and data. Defaults to False.
1597
+ - `force_metadata` (bool): If True (and `force_all` is False), refresh only metadata. Defaults to False.
1598
+ - `force_data` (bool): If True (and `force_all` is False), refresh only data. Defaults to False.
1599
+
1600
+ **Returns:** None
1601
+
1602
+ **Raises:**
1603
+ - `Exception`: If cache clearing or refresh operations fail
1604
+
1605
+ **Example:**
1606
+ ```python
1607
+ from quickbase_extract import complete_cache_refresh
1608
+
1609
+ # Refresh only metadata (after changing report configurations)
1610
+ complete_cache_refresh(
1611
+ cache_manager=cache_mgr,
1612
+ client=qb_client,
1613
+ report_configs=get_all_reports(),
1614
+ force_metadata=True
1615
+ )
1616
+
1617
+ # Refresh all (metadata + data)
1618
+ complete_cache_refresh(
1619
+ cache_manager=cache_mgr,
1620
+ client=qb_client,
1621
+ report_configs=get_all_reports(),
1622
+ force_all=True
1623
+ )
1624
+ ```
1625
+
1626
+ **Note:** This function is designed for development/debugging. To use in Lambda, add toggles to your handler (see "Development/Debug Mode" section below).
1627
+
1566
1628
  ### Query Execution with Retry Logic
1567
1629
 
1568
1630
  #### `handle_query(client, table_id, *, select=None, where=None, sort_by=None, group_by=None, options=None, description="", max_retries=3)`
@@ -2243,6 +2305,47 @@ ask_values = {"ask1": "value"} # No underscores
2243
2305
 
2244
2306
  ---
2245
2307
 
2308
+ ### Issue: Lambda has old cached data after I changed report metadata
2309
+
2310
+ **Symptom:** You updated a Quickbase report's fields, filters, or configuration, but your Lambda returns stale data.
2311
+
2312
+ **Cause:** Cache was loaded before your changes, and it hasn't become "stale" enough to auto-refresh yet.
2313
+
2314
+ **Solutions:**
2315
+
2316
+ 1. **Quick fix: Use force refresh toggle (development only)**
2317
+
2318
+ In your Lambda handler, temporarily set:
2319
+ ```python
2320
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
2321
+ ```
2322
+
2323
+ Upload new build, invoke Lambda, check logs for refresh. Then revert flag to `False`.
2324
+
2325
+ 2. **For immediate production fix:**
2326
+
2327
+ Manually delete files from S3:
2328
+ ```bash
2329
+ aws s3 rm s3://my-quickbase-cache-bucket/prod/cache/report_metadata/ --recursive
2330
+ ```
2331
+
2332
+ Next Lambda cold start will re-fetch fresh metadata.
2333
+
2334
+ 3. **To prevent in future:**
2335
+
2336
+ Reduce `metadata_stale_hours` threshold:
2337
+ ```python
2338
+ ensure_cache_freshness(
2339
+ client=client,
2340
+ cache_manager=cache_mgr,
2341
+ report_configs_all=get_all_reports(),
2342
+ report_configs_to_cache=get_reports_to_cache(),
2343
+ metadata_stale_hours=24 # Check daily instead of 30 days
2344
+ )
2345
+ ```
2346
+
2347
+ ---
2348
+
2246
2349
  ### Issue: Performance degradation over time
2247
2350
 
2248
2351
  **Symptom:** First request fast, subsequent requests slow.
@@ -2622,6 +2725,95 @@ def scheduled_cache_refresh(event, context):
2622
2725
  return {"statusCode": 200, "message": "Cache refreshed"}
2623
2726
  ```
2624
2727
 
2728
+ ### Development/Debug Mode: Forcing Complete Cache Refresh
2729
+
2730
+ When you modify report metadata or configurations in Quickbase, your Lambda may still use stale cached data. Use the force refresh toggles to clear everything and fetch fresh data.
2731
+
2732
+ #### When to Use
2733
+
2734
+ - You changed a report's filters or fields in Quickbase
2735
+ - You added/removed fields from a report
2736
+ - You renamed a report or table
2737
+ - You need to verify fresh data is being fetched
2738
+
2739
+ #### How to Use
2740
+
2741
+ 1. Open your Lambda handler code
2742
+ 2. Set one of the toggle flags to `True`:
2743
+
2744
+ ```python
2745
+ # In lambda_handler, find these lines:
2746
+ FORCE_COMPLETE_CACHE_REFRESH_ALL = False
2747
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
2748
+ FORCE_COMPLETE_CACHE_REFRESH_DATA = False
2749
+
2750
+ # Change to (example: refresh only metadata):
2751
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
2752
+ ```
2753
+
2754
+ 3. Upload new Lambda build
2755
+ 4. Invoke your Lambda (via API or CloudWatch event)
2756
+ 5. Check CloudWatch logs for cache refresh messages
2757
+ 6. **Revert the flag back to `False`** for normal operation
2758
+
2759
+ #### Flag Options
2760
+
2761
+ | Flag | What Gets Refreshed | Use When |
2762
+ |------|---------------------|----------|
2763
+ | `force_all=True` | Both metadata + data | Complete cache overhaul needed |
2764
+ | `force_metadata=True` | Only metadata | Report configuration changed |
2765
+ | `force_data=True` | Only data | Data needs fresh pull |
2766
+
2767
+ #### What Happens
2768
+
2769
+ When you trigger a force refresh:
2770
+
2771
+ 1. ✓ `/tmp` cache directories are deleted
2772
+ 2. ✓ Fresh data fetched from Quickbase API
2773
+ 3. ✓ Data written to S3
2774
+ 4. ✓ `/tmp` re-synced from updated S3
2775
+
2776
+ Your Lambda now has fresh data from Quickbase.
2777
+
2778
+ #### Example
2779
+
2780
+ ```python
2781
+ def lambda_handler(event, context):
2782
+ client = get_client()
2783
+ cache_mgr = get_cache_manager()
2784
+
2785
+ # Metadata changed in Quickbase? Force refresh it:
2786
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = True # ← Toggle this
2787
+
2788
+ if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
2789
+ or FORCE_COMPLETE_CACHE_REFRESH_DATA):
2790
+ from quickbase_extract import complete_cache_refresh
2791
+ complete_cache_refresh(
2792
+ cache_manager=cache_mgr,
2793
+ client=client,
2794
+ report_configs=get_all_reports(),
2795
+ force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
2796
+ )
2797
+
2798
+ # Rest of handler...
2799
+ ```
2800
+
2801
+ **CloudWatch logs will show:**
2802
+ ```
2803
+ WARNING: Starting complete cache refresh for: metadata (clearing /tmp, refreshing from Quickbase, updating S3...)
2804
+ DEBUG: Reset cache sync flag
2805
+ INFO: Fetching fresh data from Quickbase...
2806
+ INFO: Re-syncing /tmp from S3...
2807
+ WARNING: Complete cache refresh finished for metadata: /tmp and S3 now have fresh data from Quickbase
2808
+ ```
2809
+
2810
+ #### Important Notes
2811
+
2812
+ - **Don't leave toggles set to `True`** — revert to `False` after testing
2813
+ - **Only for development** — not a production workflow
2814
+ - Logs will show exactly what was refreshed
2815
+ - Safe to use — doesn't affect running processes, only next Lambda invocation
2816
+
2625
2817
  ## Advanced Usage
2626
2818
 
2627
2819
  ### Custom Report Configurations
@@ -928,10 +928,30 @@ def lambda_handler(event, context):
928
928
  - Cold start: Syncs cache from S3
929
929
  - Cache freshness: Auto-refreshes if stale
930
930
  - Data fetching: Queries Quickbase and caches results
931
+ - Development: Supports forcing complete cache refresh for testing
931
932
  """
932
933
  client = get_client()
933
934
  cache_mgr = get_cache_manager()
934
935
 
936
+ # OPTIONAL: Set to True to force cache refresh (for development/debugging only)
937
+ # Only one should be True at a time. force_all overrides the others.
938
+ FORCE_COMPLETE_CACHE_REFRESH_ALL = False
939
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
940
+ FORCE_COMPLETE_CACHE_REFRESH_DATA = False
941
+
942
+ # Force complete cache refresh if needed (dev/debugging only)
943
+ if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
944
+ or FORCE_COMPLETE_CACHE_REFRESH_DATA):
945
+ from quickbase_extract import complete_cache_refresh
946
+ complete_cache_refresh(
947
+ cache_manager=cache_mgr,
948
+ client=client,
949
+ report_configs=get_all_reports(),
950
+ force_all=FORCE_COMPLETE_CACHE_REFRESH_ALL,
951
+ force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
952
+ force_data=FORCE_COMPLETE_CACHE_REFRESH_DATA,
953
+ )
954
+
935
955
  # Step 1: Sync cache from S3 on cold start (only on first invocation)
936
956
  sync_from_s3_once(cache_mgr)
937
957
 
@@ -1536,6 +1556,48 @@ def lambda_handler(event, context):
1536
1556
  metadata = load_report_metadata_batch(cache_mgr, config)
1537
1557
  ```
1538
1558
 
1559
+ #### `complete_cache_refresh(cache_manager, client, report_configs, force_all=False, force_metadata=False, force_data=False)`
1560
+
1561
+ Completely refresh cache for development/debugging: clear /tmp, fetch fresh from Quickbase, update S3, re-sync to /tmp.
1562
+
1563
+ This is a development utility for forcing a complete cache refresh when report metadata or configurations change. Clears local /tmp cache, fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
1564
+
1565
+ **Parameters:**
1566
+ - `cache_manager` (CacheManager): Cache manager instance
1567
+ - `client`: Quickbase API client for fetching fresh data
1568
+ - `report_configs` (list[ReportConfig]): List of all ReportConfig instances to refresh
1569
+ - `force_all` (bool): If True, refresh both metadata and data. Defaults to False.
1570
+ - `force_metadata` (bool): If True (and `force_all` is False), refresh only metadata. Defaults to False.
1571
+ - `force_data` (bool): If True (and `force_all` is False), refresh only data. Defaults to False.
1572
+
1573
+ **Returns:** None
1574
+
1575
+ **Raises:**
1576
+ - `Exception`: If cache clearing or refresh operations fail
1577
+
1578
+ **Example:**
1579
+ ```python
1580
+ from quickbase_extract import complete_cache_refresh
1581
+
1582
+ # Refresh only metadata (after changing report configurations)
1583
+ complete_cache_refresh(
1584
+ cache_manager=cache_mgr,
1585
+ client=qb_client,
1586
+ report_configs=get_all_reports(),
1587
+ force_metadata=True
1588
+ )
1589
+
1590
+ # Refresh all (metadata + data)
1591
+ complete_cache_refresh(
1592
+ cache_manager=cache_mgr,
1593
+ client=qb_client,
1594
+ report_configs=get_all_reports(),
1595
+ force_all=True
1596
+ )
1597
+ ```
1598
+
1599
+ **Note:** This function is designed for development/debugging. To use in Lambda, add toggles to your handler (see "Development/Debug Mode" section below).
1600
+
1539
1601
  ### Query Execution with Retry Logic
1540
1602
 
1541
1603
  #### `handle_query(client, table_id, *, select=None, where=None, sort_by=None, group_by=None, options=None, description="", max_retries=3)`
@@ -2216,6 +2278,47 @@ ask_values = {"ask1": "value"} # No underscores
2216
2278
 
2217
2279
  ---
2218
2280
 
2281
+ ### Issue: Lambda has old cached data after I changed report metadata
2282
+
2283
+ **Symptom:** You updated a Quickbase report's fields, filters, or configuration, but your Lambda returns stale data.
2284
+
2285
+ **Cause:** Cache was loaded before your changes, and it hasn't become "stale" enough to auto-refresh yet.
2286
+
2287
+ **Solutions:**
2288
+
2289
+ 1. **Quick fix: Use force refresh toggle (development only)**
2290
+
2291
+ In your Lambda handler, temporarily set:
2292
+ ```python
2293
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
2294
+ ```
2295
+
2296
+ Upload new build, invoke Lambda, check logs for refresh. Then revert flag to `False`.
2297
+
2298
+ 2. **For immediate production fix:**
2299
+
2300
+ Manually delete files from S3:
2301
+ ```bash
2302
+ aws s3 rm s3://my-quickbase-cache-bucket/prod/cache/report_metadata/ --recursive
2303
+ ```
2304
+
2305
+ Next Lambda cold start will re-fetch fresh metadata.
2306
+
2307
+ 3. **To prevent in future:**
2308
+
2309
+ Reduce `metadata_stale_hours` threshold:
2310
+ ```python
2311
+ ensure_cache_freshness(
2312
+ client=client,
2313
+ cache_manager=cache_mgr,
2314
+ report_configs_all=get_all_reports(),
2315
+ report_configs_to_cache=get_reports_to_cache(),
2316
+ metadata_stale_hours=24 # Check daily instead of 30 days
2317
+ )
2318
+ ```
2319
+
2320
+ ---
2321
+
2219
2322
  ### Issue: Performance degradation over time
2220
2323
 
2221
2324
  **Symptom:** First request fast, subsequent requests slow.
@@ -2595,6 +2698,95 @@ def scheduled_cache_refresh(event, context):
2595
2698
  return {"statusCode": 200, "message": "Cache refreshed"}
2596
2699
  ```
2597
2700
 
2701
+ ### Development/Debug Mode: Forcing Complete Cache Refresh
2702
+
2703
+ When you modify report metadata or configurations in Quickbase, your Lambda may still use stale cached data. Use the force refresh toggles to clear everything and fetch fresh data.
2704
+
2705
+ #### When to Use
2706
+
2707
+ - You changed a report's filters or fields in Quickbase
2708
+ - You added/removed fields from a report
2709
+ - You renamed a report or table
2710
+ - You need to verify fresh data is being fetched
2711
+
2712
+ #### How to Use
2713
+
2714
+ 1. Open your Lambda handler code
2715
+ 2. Set one of the toggle flags to `True`:
2716
+
2717
+ ```python
2718
+ # In lambda_handler, find these lines:
2719
+ FORCE_COMPLETE_CACHE_REFRESH_ALL = False
2720
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
2721
+ FORCE_COMPLETE_CACHE_REFRESH_DATA = False
2722
+
2723
+ # Change to (example: refresh only metadata):
2724
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
2725
+ ```
2726
+
2727
+ 3. Upload new Lambda build
2728
+ 4. Invoke your Lambda (via API or CloudWatch event)
2729
+ 5. Check CloudWatch logs for cache refresh messages
2730
+ 6. **Revert the flag back to `False`** for normal operation
2731
+
2732
+ #### Flag Options
2733
+
2734
+ | Flag | What Gets Refreshed | Use When |
2735
+ |------|---------------------|----------|
2736
+ | `force_all=True` | Both metadata + data | Complete cache overhaul needed |
2737
+ | `force_metadata=True` | Only metadata | Report configuration changed |
2738
+ | `force_data=True` | Only data | Data needs fresh pull |
2739
+
2740
+ #### What Happens
2741
+
2742
+ When you trigger a force refresh:
2743
+
2744
+ 1. ✓ `/tmp` cache directories are deleted
2745
+ 2. ✓ Fresh data fetched from Quickbase API
2746
+ 3. ✓ Data written to S3
2747
+ 4. ✓ `/tmp` re-synced from updated S3
2748
+
2749
+ Your Lambda now has fresh data from Quickbase.
2750
+
2751
+ #### Example
2752
+
2753
+ ```python
2754
+ def lambda_handler(event, context):
2755
+ client = get_client()
2756
+ cache_mgr = get_cache_manager()
2757
+
2758
+ # Metadata changed in Quickbase? Force refresh it:
2759
+ FORCE_COMPLETE_CACHE_REFRESH_METADATA = True # ← Toggle this
2760
+
2761
+ if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
2762
+ or FORCE_COMPLETE_CACHE_REFRESH_DATA):
2763
+ from quickbase_extract import complete_cache_refresh
2764
+ complete_cache_refresh(
2765
+ cache_manager=cache_mgr,
2766
+ client=client,
2767
+ report_configs=get_all_reports(),
2768
+ force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
2769
+ )
2770
+
2771
+ # Rest of handler...
2772
+ ```
2773
+
2774
+ **CloudWatch logs will show:**
2775
+ ```
2776
+ WARNING: Starting complete cache refresh for: metadata (clearing /tmp, refreshing from Quickbase, updating S3...)
2777
+ DEBUG: Reset cache sync flag
2778
+ INFO: Fetching fresh data from Quickbase...
2779
+ INFO: Re-syncing /tmp from S3...
2780
+ WARNING: Complete cache refresh finished for metadata: /tmp and S3 now have fresh data from Quickbase
2781
+ ```
2782
+
2783
+ #### Important Notes
2784
+
2785
+ - **Don't leave toggles set to `True`** — revert to `False` after testing
2786
+ - **Only for development** — not a production workflow
2787
+ - Logs will show exactly what was refreshed
2788
+ - Safe to use — doesn't affect running processes, only next Lambda invocation
2789
+
2598
2790
  ## Advanced Usage
2599
2791
 
2600
2792
  ### Custom Report Configurations
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "quickbase-extract"
7
- version = "0.2.1"
7
+ version = "0.3.0"
8
8
  description = "Extract and cache Quickbase report data with built-in error handling and S3 support"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.12"
@@ -0,0 +1,191 @@
1
+ """S3-backed cache sync for Lambda environments."""
2
+
3
+ import logging
4
+ import shutil
5
+
6
+ from quickbase_extract.cache_manager import CacheManager
7
+ from quickbase_extract.cache_orchestration import ensure_cache_freshness
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ # Module-level flag to track if we've synced this Lambda invocation
12
+ _CACHE_SYNCED = False
13
+
14
+
15
+ def complete_cache_refresh(
16
+ client,
17
+ cache_manager: CacheManager,
18
+ report_configs: list,
19
+ force_all: bool = False,
20
+ force_metadata: bool = False,
21
+ force_data: bool = False,
22
+ ) -> None:
23
+ """Completely refresh cache: clear /tmp, fetch from Quickbase, update S3, re-sync to /tmp.
24
+
25
+ This is a dev/debugging utility for forcing a cache refresh when report
26
+ metadata or configurations change. Clears specified local /tmp cache,
27
+ fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
28
+
29
+ Only the cache types specified by force flags are cleared and refreshed.
30
+ For example, if only force_metadata=True, only metadata cache is touched.
31
+
32
+ Args:
33
+ cache_manager: CacheManager instance for cache operations.
34
+ client: Quickbase API client for fetching fresh data.
35
+ report_configs: List of all ReportConfig instances to refresh.
36
+ force_all: If True, refresh both metadata and data. Overrides individual flags.
37
+ Defaults to False.
38
+ force_metadata: If True (and force_all is False), refresh only metadata.
39
+ Defaults to False.
40
+ force_data: If True (and force_all is False), refresh only data.
41
+ Defaults to False.
42
+
43
+ Raises:
44
+ Exception: If cache clearing or refresh operations fail.
45
+
46
+ Example:
47
+ >>> # Refresh only metadata
48
+ >>> complete_cache_refresh(
49
+ ... cache_manager=cache_mgr,
50
+ ... client=qb_client,
51
+ ... report_configs=report_config,
52
+ ... force_metadata=True,
53
+ ... )
54
+ >>>
55
+ >>> # Refresh all
56
+ >>> complete_cache_refresh(
57
+ ... cache_manager=cache_mgr,
58
+ ... client=qb_client,
59
+ ... report_configs=report_config,
60
+ ... force_all=True,
61
+ ... )
62
+ """
63
+ global _CACHE_SYNCED
64
+
65
+ # Determine which caches to refresh (force_all overrides individual flags)
66
+ should_refresh_metadata = force_all or force_metadata
67
+ should_refresh_data = force_all or force_data
68
+
69
+ if not (should_refresh_metadata or should_refresh_data):
70
+ logger.debug("No cache refresh flags set, skipping complete cache refresh")
71
+ return
72
+
73
+ refresh_types = []
74
+ if should_refresh_metadata:
75
+ refresh_types.append("metadata")
76
+ if should_refresh_data:
77
+ refresh_types.append("data")
78
+
79
+ logger.warning(
80
+ f"Starting complete cache refresh for: {', '.join(refresh_types)} "
81
+ "(clearing /tmp, refreshing from Quickbase, updating S3...)"
82
+ )
83
+
84
+ # Step 1: Clear specified /tmp cache directories
85
+ if should_refresh_metadata:
86
+ metadata_dir = cache_manager.cache_root / "report_metadata"
87
+ if metadata_dir.exists():
88
+ shutil.rmtree(metadata_dir)
89
+ logger.info(f"Cleared metadata cache directory: {metadata_dir}")
90
+
91
+ if should_refresh_data:
92
+ data_dir = cache_manager.cache_root / "report_data"
93
+ if data_dir.exists():
94
+ shutil.rmtree(data_dir)
95
+ logger.info(f"Cleared data cache directory: {data_dir}")
96
+
97
+ # Step 2: Reset sync flag so fresh data will be fetched
98
+ _CACHE_SYNCED = False
99
+ logger.debug("Reset cache sync flag")
100
+
101
+ # Step 3: Fetch fresh data from Quickbase and write to S3
102
+ logger.info("Fetching fresh data from Quickbase...")
103
+ ensure_cache_freshness(
104
+ client=client,
105
+ cache_manager=cache_manager,
106
+ report_configs_all=report_configs,
107
+ force_all=force_all,
108
+ force_metadata=force_metadata,
109
+ force_data=force_data,
110
+ )
111
+
112
+ # Step 4: Re-sync /tmp from S3 for cleared caches
113
+ logger.info("Re-syncing /tmp from S3...")
114
+ sync_from_s3_once(cache_manager, force=True)
115
+
116
+ logger.warning(
117
+ f"Complete cache refresh finished for {', '.join(refresh_types)}: "
118
+ "/tmp and S3 now have fresh data from Quickbase"
119
+ )
120
+
121
+
122
+ def sync_from_s3_once(cache_manager: CacheManager, force: bool = False) -> None:
123
+ """Download cache from S3 to /tmp on Lambda cold start.
124
+
125
+ Only syncs if cache hasn't been synced in this invocation.
126
+ Subsequent calls are no-ops unless force=True.
127
+
128
+ On Lambda, the sync flag persists across warm invocations within the same
129
+ container, so warm starts skip the sync (Lambda /tmp persists). Only cold
130
+ starts trigger a sync.
131
+
132
+ On local environments, automatically detects if CACHE_BUCKET is configured.
133
+ If not configured, does nothing (local caching only).
134
+
135
+ Args:
136
+ cache_manager: CacheManager instance for cache operations.
137
+ force: If True, sync even if already synced in this invocation.
138
+ Defaults to False.
139
+
140
+ Raises:
141
+ Exception: If S3 operations fail.
142
+
143
+ Example:
144
+ >>> # In Lambda handler initialization
145
+ >>> cache_manager = CacheManager(
146
+ ... cache_root=Path("/tmp/my_project/dev/cache"),
147
+ ... s3_bucket="mit-bio-quickbase",
148
+ ... s3_prefix="my_project/dev/cache",
149
+ ... )
150
+ >>> sync_from_s3_once(cache_manager) # Syncs on cold start
151
+ >>> sync_from_s3_once(cache_manager) # No-op on same invocation
152
+ >>>
153
+ >>> # Force re-sync if needed (programmatically)
154
+ >>> sync_from_s3_once(cache_manager, force=True)
155
+ """
156
+ global _CACHE_SYNCED
157
+
158
+ # Check for force refresh
159
+ already_synced = _CACHE_SYNCED and not force
160
+
161
+ if already_synced:
162
+ logger.debug("Cache already synced in this invocation, skipping")
163
+ return
164
+
165
+ cache_manager.sync_from_s3() # Handles Lambda detection internally
166
+ _CACHE_SYNCED = True
167
+ logger.info("Cache synced from S3")
168
+
169
+
170
+ def is_cache_synced() -> bool:
171
+ """Check if cache has been synced in this invocation.
172
+
173
+ Returns:
174
+ True if cache has been synced, False otherwise.
175
+
176
+ Example:
177
+ >>> if not is_cache_synced():
178
+ ... print("Cache needs syncing")
179
+ """
180
+ return _CACHE_SYNCED
181
+
182
+
183
+ def _reset_cache_sync() -> None:
184
+ """Reset the cache sync flag. For testing only.
185
+
186
+ Example:
187
+ >>> # In test teardown
188
+ >>> _reset_cache_sync()
189
+ """
190
+ global _CACHE_SYNCED
191
+ _CACHE_SYNCED = False