quickbase-extract 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/CHANGELOG.md +23 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/PKG-INFO +193 -1
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/README.md +192 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/pyproject.toml +1 -1
- quickbase_extract-0.3.0/src/quickbase_extract/cache_sync.py +191 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/conftest.py +10 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_api_handlers.py +20 -3
- quickbase_extract-0.3.0/tests/test_cache_sync.py +360 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_report_data.py +11 -2
- quickbase_extract-0.2.1/src/quickbase_extract/cache_sync.py +0 -82
- quickbase_extract-0.2.1/tests/test_cache_sync.py +0 -119
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.editorconfig +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.gitignore +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.pre-commit-config.yaml +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/.python-version +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/LICENSE.txt +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/TODO.md +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/__init__.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/api_handlers.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/cache_manager.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/cache_orchestration.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/config.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/py.typed +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/report_data.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/report_metadata.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/src/quickbase_extract/utils.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_cache_manager.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_cache_orchestration.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_report_metadata.py +0 -0
- {quickbase_extract-0.2.1 → quickbase_extract-0.3.0}/tests/test_utils.py +0 -0
|
@@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.3.0] - 2026-04-27
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `complete_cache_refresh()` function in `cache_sync` module for selective cache refresh in development
|
|
13
|
+
- Support for granular cache refresh control:
|
|
14
|
+
- `force_all=True`: Refresh both metadata and data caches
|
|
15
|
+
- `force_metadata=True`: Refresh only metadata cache
|
|
16
|
+
- `force_data=True`: Refresh only data cache
|
|
17
|
+
- Cache refresh workflow: clear /tmp → fetch fresh from Quickbase → update S3 → re-sync to /tmp
|
|
18
|
+
- Manual development toggles in Lambda handlers for testing cache refresh without API changes
|
|
19
|
+
- Comprehensive test suite for complete cache refresh functionality
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
|
|
23
|
+
- Lambda workers can now force complete cache refresh for development/debugging by toggling code variables
|
|
24
|
+
- Cache refresh strategy clarified: selective refresh based on what changed (metadata vs data)
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- Dev workflow now supports forcing cache refresh without modifying Lambda event API
|
|
29
|
+
- Ensures /tmp and S3 are synchronized after cache refresh operations
|
|
30
|
+
|
|
8
31
|
## [0.2.1] - 2026-04-25
|
|
9
32
|
|
|
10
33
|
### Fixed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quickbase-extract
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Extract and cache Quickbase report data with built-in error handling and S3 support
|
|
5
5
|
Project-URL: Homepage, https://github.com/tbrezler/quickbase-extract
|
|
6
6
|
Project-URL: Repository, https://github.com/tbrezler/quickbase-extract.git
|
|
@@ -955,10 +955,30 @@ def lambda_handler(event, context):
|
|
|
955
955
|
- Cold start: Syncs cache from S3
|
|
956
956
|
- Cache freshness: Auto-refreshes if stale
|
|
957
957
|
- Data fetching: Queries Quickbase and caches results
|
|
958
|
+
- Development: Supports forcing complete cache refresh for testing
|
|
958
959
|
"""
|
|
959
960
|
client = get_client()
|
|
960
961
|
cache_mgr = get_cache_manager()
|
|
961
962
|
|
|
963
|
+
# OPTIONAL: Set to True to force cache refresh (for development/debugging only)
|
|
964
|
+
# Only one should be True at a time. force_all overrides the others.
|
|
965
|
+
FORCE_COMPLETE_CACHE_REFRESH_ALL = False
|
|
966
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
|
|
967
|
+
FORCE_COMPLETE_CACHE_REFRESH_DATA = False
|
|
968
|
+
|
|
969
|
+
# Force complete cache refresh if needed (dev/debugging only)
|
|
970
|
+
if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
|
|
971
|
+
or FORCE_COMPLETE_CACHE_REFRESH_DATA):
|
|
972
|
+
from quickbase_extract import complete_cache_refresh
|
|
973
|
+
complete_cache_refresh(
|
|
974
|
+
cache_manager=cache_mgr,
|
|
975
|
+
client=client,
|
|
976
|
+
report_configs=get_all_reports(),
|
|
977
|
+
force_all=FORCE_COMPLETE_CACHE_REFRESH_ALL,
|
|
978
|
+
force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
|
|
979
|
+
force_data=FORCE_COMPLETE_CACHE_REFRESH_DATA,
|
|
980
|
+
)
|
|
981
|
+
|
|
962
982
|
# Step 1: Sync cache from S3 on cold start (only on first invocation)
|
|
963
983
|
sync_from_s3_once(cache_mgr)
|
|
964
984
|
|
|
@@ -1563,6 +1583,48 @@ def lambda_handler(event, context):
|
|
|
1563
1583
|
metadata = load_report_metadata_batch(cache_mgr, config)
|
|
1564
1584
|
```
|
|
1565
1585
|
|
|
1586
|
+
#### `complete_cache_refresh(cache_manager, client, report_configs, force_all=False, force_metadata=False, force_data=False)`
|
|
1587
|
+
|
|
1588
|
+
Completely refresh cache for development/debugging: clear /tmp, fetch fresh from Quickbase, update S3, re-sync to /tmp.
|
|
1589
|
+
|
|
1590
|
+
This is a development utility for forcing a complete cache refresh when report metadata or configurations change. Clears local /tmp cache, fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
|
|
1591
|
+
|
|
1592
|
+
**Parameters:**
|
|
1593
|
+
- `cache_manager` (CacheManager): Cache manager instance
|
|
1594
|
+
- `client`: Quickbase API client for fetching fresh data
|
|
1595
|
+
- `report_configs` (list[ReportConfig]): List of all ReportConfig instances to refresh
|
|
1596
|
+
- `force_all` (bool): If True, refresh both metadata and data. Defaults to False.
|
|
1597
|
+
- `force_metadata` (bool): If True (and `force_all` is False), refresh only metadata. Defaults to False.
|
|
1598
|
+
- `force_data` (bool): If True (and `force_all` is False), refresh only data. Defaults to False.
|
|
1599
|
+
|
|
1600
|
+
**Returns:** None
|
|
1601
|
+
|
|
1602
|
+
**Raises:**
|
|
1603
|
+
- `Exception`: If cache clearing or refresh operations fail
|
|
1604
|
+
|
|
1605
|
+
**Example:**
|
|
1606
|
+
```python
|
|
1607
|
+
from quickbase_extract import complete_cache_refresh
|
|
1608
|
+
|
|
1609
|
+
# Refresh only metadata (after changing report configurations)
|
|
1610
|
+
complete_cache_refresh(
|
|
1611
|
+
cache_manager=cache_mgr,
|
|
1612
|
+
client=qb_client,
|
|
1613
|
+
report_configs=get_all_reports(),
|
|
1614
|
+
force_metadata=True
|
|
1615
|
+
)
|
|
1616
|
+
|
|
1617
|
+
# Refresh all (metadata + data)
|
|
1618
|
+
complete_cache_refresh(
|
|
1619
|
+
cache_manager=cache_mgr,
|
|
1620
|
+
client=qb_client,
|
|
1621
|
+
report_configs=get_all_reports(),
|
|
1622
|
+
force_all=True
|
|
1623
|
+
)
|
|
1624
|
+
```
|
|
1625
|
+
|
|
1626
|
+
**Note:** This function is designed for development/debugging. To use in Lambda, add toggles to your handler (see "Development/Debug Mode" section below).
|
|
1627
|
+
|
|
1566
1628
|
### Query Execution with Retry Logic
|
|
1567
1629
|
|
|
1568
1630
|
#### `handle_query(client, table_id, *, select=None, where=None, sort_by=None, group_by=None, options=None, description="", max_retries=3)`
|
|
@@ -2243,6 +2305,47 @@ ask_values = {"ask1": "value"} # No underscores
|
|
|
2243
2305
|
|
|
2244
2306
|
---
|
|
2245
2307
|
|
|
2308
|
+
### Issue: Lambda has old cached data after I changed report metadata
|
|
2309
|
+
|
|
2310
|
+
**Symptom:** You updated a Quickbase report's fields, filters, or configuration, but your Lambda returns stale data.
|
|
2311
|
+
|
|
2312
|
+
**Cause:** Cache was loaded before your changes, and it hasn't become "stale" enough to auto-refresh yet.
|
|
2313
|
+
|
|
2314
|
+
**Solutions:**
|
|
2315
|
+
|
|
2316
|
+
1. **Quick fix: Use force refresh toggle (development only)**
|
|
2317
|
+
|
|
2318
|
+
In your Lambda handler, temporarily set:
|
|
2319
|
+
```python
|
|
2320
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
|
|
2321
|
+
```
|
|
2322
|
+
|
|
2323
|
+
Upload new build, invoke Lambda, check logs for refresh. Then revert flag to `False`.
|
|
2324
|
+
|
|
2325
|
+
2. **For immediate production fix:**
|
|
2326
|
+
|
|
2327
|
+
Manually delete files from S3:
|
|
2328
|
+
```bash
|
|
2329
|
+
aws s3 rm s3://my-quickbase-cache-bucket/prod/cache/report_metadata/ --recursive
|
|
2330
|
+
```
|
|
2331
|
+
|
|
2332
|
+
Next Lambda cold start will re-fetch fresh metadata.
|
|
2333
|
+
|
|
2334
|
+
3. **To prevent in future:**
|
|
2335
|
+
|
|
2336
|
+
Reduce `metadata_stale_hours` threshold:
|
|
2337
|
+
```python
|
|
2338
|
+
ensure_cache_freshness(
|
|
2339
|
+
client=client,
|
|
2340
|
+
cache_manager=cache_mgr,
|
|
2341
|
+
report_configs_all=get_all_reports(),
|
|
2342
|
+
report_configs_to_cache=get_reports_to_cache(),
|
|
2343
|
+
metadata_stale_hours=24 # Check daily instead of 30 days
|
|
2344
|
+
)
|
|
2345
|
+
```
|
|
2346
|
+
|
|
2347
|
+
---
|
|
2348
|
+
|
|
2246
2349
|
### Issue: Performance degradation over time
|
|
2247
2350
|
|
|
2248
2351
|
**Symptom:** First request fast, subsequent requests slow.
|
|
@@ -2622,6 +2725,95 @@ def scheduled_cache_refresh(event, context):
|
|
|
2622
2725
|
return {"statusCode": 200, "message": "Cache refreshed"}
|
|
2623
2726
|
```
|
|
2624
2727
|
|
|
2728
|
+
### Development/Debug Mode: Forcing Complete Cache Refresh
|
|
2729
|
+
|
|
2730
|
+
When you modify report metadata or configurations in Quickbase, your Lambda may still use stale cached data. Use the force refresh toggles to clear everything and fetch fresh data.
|
|
2731
|
+
|
|
2732
|
+
#### When to Use
|
|
2733
|
+
|
|
2734
|
+
- You changed a report's filters or fields in Quickbase
|
|
2735
|
+
- You added/removed fields from a report
|
|
2736
|
+
- You renamed a report or table
|
|
2737
|
+
- You need to verify fresh data is being fetched
|
|
2738
|
+
|
|
2739
|
+
#### How to Use
|
|
2740
|
+
|
|
2741
|
+
1. Open your Lambda handler code
|
|
2742
|
+
2. Set one of the toggle flags to `True`:
|
|
2743
|
+
|
|
2744
|
+
```python
|
|
2745
|
+
# In lambda_handler, find these lines:
|
|
2746
|
+
FORCE_COMPLETE_CACHE_REFRESH_ALL = False
|
|
2747
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
|
|
2748
|
+
FORCE_COMPLETE_CACHE_REFRESH_DATA = False
|
|
2749
|
+
|
|
2750
|
+
# Change to (example: refresh only metadata):
|
|
2751
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
|
|
2752
|
+
```
|
|
2753
|
+
|
|
2754
|
+
3. Upload new Lambda build
|
|
2755
|
+
4. Invoke your Lambda (via API or CloudWatch event)
|
|
2756
|
+
5. Check CloudWatch logs for cache refresh messages
|
|
2757
|
+
6. **Revert the flag back to `False`** for normal operation
|
|
2758
|
+
|
|
2759
|
+
#### Flag Options
|
|
2760
|
+
|
|
2761
|
+
| Flag | What Gets Refreshed | Use When |
|
|
2762
|
+
|------|---------------------|----------|
|
|
2763
|
+
| `force_all=True` | Both metadata + data | Complete cache overhaul needed |
|
|
2764
|
+
| `force_metadata=True` | Only metadata | Report configuration changed |
|
|
2765
|
+
| `force_data=True` | Only data | Data needs fresh pull |
|
|
2766
|
+
|
|
2767
|
+
#### What Happens
|
|
2768
|
+
|
|
2769
|
+
When you trigger a force refresh:
|
|
2770
|
+
|
|
2771
|
+
1. ✓ `/tmp` cache directories are deleted
|
|
2772
|
+
2. ✓ Fresh data fetched from Quickbase API
|
|
2773
|
+
3. ✓ Data written to S3
|
|
2774
|
+
4. ✓ `/tmp` re-synced from updated S3
|
|
2775
|
+
|
|
2776
|
+
Your Lambda now has fresh data from Quickbase.
|
|
2777
|
+
|
|
2778
|
+
#### Example
|
|
2779
|
+
|
|
2780
|
+
```python
|
|
2781
|
+
def lambda_handler(event, context):
|
|
2782
|
+
client = get_client()
|
|
2783
|
+
cache_mgr = get_cache_manager()
|
|
2784
|
+
|
|
2785
|
+
# Metadata changed in Quickbase? Force refresh it:
|
|
2786
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = True # ← Toggle this
|
|
2787
|
+
|
|
2788
|
+
if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
|
|
2789
|
+
or FORCE_COMPLETE_CACHE_REFRESH_DATA):
|
|
2790
|
+
from quickbase_extract import complete_cache_refresh
|
|
2791
|
+
complete_cache_refresh(
|
|
2792
|
+
cache_manager=cache_mgr,
|
|
2793
|
+
client=client,
|
|
2794
|
+
report_configs=get_all_reports(),
|
|
2795
|
+
force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
|
|
2796
|
+
)
|
|
2797
|
+
|
|
2798
|
+
# Rest of handler...
|
|
2799
|
+
```
|
|
2800
|
+
|
|
2801
|
+
**CloudWatch logs will show:**
|
|
2802
|
+
```
|
|
2803
|
+
WARNING: Starting complete cache refresh for: metadata (clearing /tmp, refreshing from Quickbase, updating S3...)
|
|
2804
|
+
DEBUG: Reset cache sync flag
|
|
2805
|
+
INFO: Fetching fresh data from Quickbase...
|
|
2806
|
+
INFO: Re-syncing /tmp from S3...
|
|
2807
|
+
WARNING: Complete cache refresh finished for metadata: /tmp and S3 now have fresh data from Quickbase
|
|
2808
|
+
```
|
|
2809
|
+
|
|
2810
|
+
#### Important Notes
|
|
2811
|
+
|
|
2812
|
+
- **Don't leave toggles set to `True`** — revert to `False` after testing
|
|
2813
|
+
- **Only for development** — not a production workflow
|
|
2814
|
+
- Logs will show exactly what was refreshed
|
|
2815
|
+
- Safe to use — doesn't affect running processes, only next Lambda invocation
|
|
2816
|
+
|
|
2625
2817
|
## Advanced Usage
|
|
2626
2818
|
|
|
2627
2819
|
### Custom Report Configurations
|
|
@@ -928,10 +928,30 @@ def lambda_handler(event, context):
|
|
|
928
928
|
- Cold start: Syncs cache from S3
|
|
929
929
|
- Cache freshness: Auto-refreshes if stale
|
|
930
930
|
- Data fetching: Queries Quickbase and caches results
|
|
931
|
+
- Development: Supports forcing complete cache refresh for testing
|
|
931
932
|
"""
|
|
932
933
|
client = get_client()
|
|
933
934
|
cache_mgr = get_cache_manager()
|
|
934
935
|
|
|
936
|
+
# OPTIONAL: Set to True to force cache refresh (for development/debugging only)
|
|
937
|
+
# Only one should be True at a time. force_all overrides the others.
|
|
938
|
+
FORCE_COMPLETE_CACHE_REFRESH_ALL = False
|
|
939
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
|
|
940
|
+
FORCE_COMPLETE_CACHE_REFRESH_DATA = False
|
|
941
|
+
|
|
942
|
+
# Force complete cache refresh if needed (dev/debugging only)
|
|
943
|
+
if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
|
|
944
|
+
or FORCE_COMPLETE_CACHE_REFRESH_DATA):
|
|
945
|
+
from quickbase_extract import complete_cache_refresh
|
|
946
|
+
complete_cache_refresh(
|
|
947
|
+
cache_manager=cache_mgr,
|
|
948
|
+
client=client,
|
|
949
|
+
report_configs=get_all_reports(),
|
|
950
|
+
force_all=FORCE_COMPLETE_CACHE_REFRESH_ALL,
|
|
951
|
+
force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
|
|
952
|
+
force_data=FORCE_COMPLETE_CACHE_REFRESH_DATA,
|
|
953
|
+
)
|
|
954
|
+
|
|
935
955
|
# Step 1: Sync cache from S3 on cold start (only on first invocation)
|
|
936
956
|
sync_from_s3_once(cache_mgr)
|
|
937
957
|
|
|
@@ -1536,6 +1556,48 @@ def lambda_handler(event, context):
|
|
|
1536
1556
|
metadata = load_report_metadata_batch(cache_mgr, config)
|
|
1537
1557
|
```
|
|
1538
1558
|
|
|
1559
|
+
#### `complete_cache_refresh(cache_manager, client, report_configs, force_all=False, force_metadata=False, force_data=False)`
|
|
1560
|
+
|
|
1561
|
+
Completely refresh cache for development/debugging: clear /tmp, fetch fresh from Quickbase, update S3, re-sync to /tmp.
|
|
1562
|
+
|
|
1563
|
+
This is a development utility for forcing a complete cache refresh when report metadata or configurations change. Clears local /tmp cache, fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
|
|
1564
|
+
|
|
1565
|
+
**Parameters:**
|
|
1566
|
+
- `cache_manager` (CacheManager): Cache manager instance
|
|
1567
|
+
- `client`: Quickbase API client for fetching fresh data
|
|
1568
|
+
- `report_configs` (list[ReportConfig]): List of all ReportConfig instances to refresh
|
|
1569
|
+
- `force_all` (bool): If True, refresh both metadata and data. Defaults to False.
|
|
1570
|
+
- `force_metadata` (bool): If True (and `force_all` is False), refresh only metadata. Defaults to False.
|
|
1571
|
+
- `force_data` (bool): If True (and `force_all` is False), refresh only data. Defaults to False.
|
|
1572
|
+
|
|
1573
|
+
**Returns:** None
|
|
1574
|
+
|
|
1575
|
+
**Raises:**
|
|
1576
|
+
- `Exception`: If cache clearing or refresh operations fail
|
|
1577
|
+
|
|
1578
|
+
**Example:**
|
|
1579
|
+
```python
|
|
1580
|
+
from quickbase_extract import complete_cache_refresh
|
|
1581
|
+
|
|
1582
|
+
# Refresh only metadata (after changing report configurations)
|
|
1583
|
+
complete_cache_refresh(
|
|
1584
|
+
cache_manager=cache_mgr,
|
|
1585
|
+
client=qb_client,
|
|
1586
|
+
report_configs=get_all_reports(),
|
|
1587
|
+
force_metadata=True
|
|
1588
|
+
)
|
|
1589
|
+
|
|
1590
|
+
# Refresh all (metadata + data)
|
|
1591
|
+
complete_cache_refresh(
|
|
1592
|
+
cache_manager=cache_mgr,
|
|
1593
|
+
client=qb_client,
|
|
1594
|
+
report_configs=get_all_reports(),
|
|
1595
|
+
force_all=True
|
|
1596
|
+
)
|
|
1597
|
+
```
|
|
1598
|
+
|
|
1599
|
+
**Note:** This function is designed for development/debugging. To use in Lambda, add toggles to your handler (see "Development/Debug Mode" section below).
|
|
1600
|
+
|
|
1539
1601
|
### Query Execution with Retry Logic
|
|
1540
1602
|
|
|
1541
1603
|
#### `handle_query(client, table_id, *, select=None, where=None, sort_by=None, group_by=None, options=None, description="", max_retries=3)`
|
|
@@ -2216,6 +2278,47 @@ ask_values = {"ask1": "value"} # No underscores
|
|
|
2216
2278
|
|
|
2217
2279
|
---
|
|
2218
2280
|
|
|
2281
|
+
### Issue: Lambda has old cached data after I changed report metadata
|
|
2282
|
+
|
|
2283
|
+
**Symptom:** You updated a Quickbase report's fields, filters, or configuration, but your Lambda returns stale data.
|
|
2284
|
+
|
|
2285
|
+
**Cause:** Cache was loaded before your changes, and it hasn't become "stale" enough to auto-refresh yet.
|
|
2286
|
+
|
|
2287
|
+
**Solutions:**
|
|
2288
|
+
|
|
2289
|
+
1. **Quick fix: Use force refresh toggle (development only)**
|
|
2290
|
+
|
|
2291
|
+
In your Lambda handler, temporarily set:
|
|
2292
|
+
```python
|
|
2293
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
|
|
2294
|
+
```
|
|
2295
|
+
|
|
2296
|
+
Upload new build, invoke Lambda, check logs for refresh. Then revert flag to `False`.
|
|
2297
|
+
|
|
2298
|
+
2. **For immediate production fix:**
|
|
2299
|
+
|
|
2300
|
+
Manually delete files from S3:
|
|
2301
|
+
```bash
|
|
2302
|
+
aws s3 rm s3://my-quickbase-cache-bucket/prod/cache/report_metadata/ --recursive
|
|
2303
|
+
```
|
|
2304
|
+
|
|
2305
|
+
Next Lambda cold start will re-fetch fresh metadata.
|
|
2306
|
+
|
|
2307
|
+
3. **To prevent in future:**
|
|
2308
|
+
|
|
2309
|
+
Reduce `metadata_stale_hours` threshold:
|
|
2310
|
+
```python
|
|
2311
|
+
ensure_cache_freshness(
|
|
2312
|
+
client=client,
|
|
2313
|
+
cache_manager=cache_mgr,
|
|
2314
|
+
report_configs_all=get_all_reports(),
|
|
2315
|
+
report_configs_to_cache=get_reports_to_cache(),
|
|
2316
|
+
metadata_stale_hours=24 # Check daily instead of 30 days
|
|
2317
|
+
)
|
|
2318
|
+
```
|
|
2319
|
+
|
|
2320
|
+
---
|
|
2321
|
+
|
|
2219
2322
|
### Issue: Performance degradation over time
|
|
2220
2323
|
|
|
2221
2324
|
**Symptom:** First request fast, subsequent requests slow.
|
|
@@ -2595,6 +2698,95 @@ def scheduled_cache_refresh(event, context):
|
|
|
2595
2698
|
return {"statusCode": 200, "message": "Cache refreshed"}
|
|
2596
2699
|
```
|
|
2597
2700
|
|
|
2701
|
+
### Development/Debug Mode: Forcing Complete Cache Refresh
|
|
2702
|
+
|
|
2703
|
+
When you modify report metadata or configurations in Quickbase, your Lambda may still use stale cached data. Use the force refresh toggles to clear everything and fetch fresh data.
|
|
2704
|
+
|
|
2705
|
+
#### When to Use
|
|
2706
|
+
|
|
2707
|
+
- You changed a report's filters or fields in Quickbase
|
|
2708
|
+
- You added/removed fields from a report
|
|
2709
|
+
- You renamed a report or table
|
|
2710
|
+
- You need to verify fresh data is being fetched
|
|
2711
|
+
|
|
2712
|
+
#### How to Use
|
|
2713
|
+
|
|
2714
|
+
1. Open your Lambda handler code
|
|
2715
|
+
2. Set one of the toggle flags to `True`:
|
|
2716
|
+
|
|
2717
|
+
```python
|
|
2718
|
+
# In lambda_handler, find these lines:
|
|
2719
|
+
FORCE_COMPLETE_CACHE_REFRESH_ALL = False
|
|
2720
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = False
|
|
2721
|
+
FORCE_COMPLETE_CACHE_REFRESH_DATA = False
|
|
2722
|
+
|
|
2723
|
+
# Change to (example: refresh only metadata):
|
|
2724
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = True
|
|
2725
|
+
```
|
|
2726
|
+
|
|
2727
|
+
3. Upload new Lambda build
|
|
2728
|
+
4. Invoke your Lambda (via API or CloudWatch event)
|
|
2729
|
+
5. Check CloudWatch logs for cache refresh messages
|
|
2730
|
+
6. **Revert the flag back to `False`** for normal operation
|
|
2731
|
+
|
|
2732
|
+
#### Flag Options
|
|
2733
|
+
|
|
2734
|
+
| Flag | What Gets Refreshed | Use When |
|
|
2735
|
+
|------|---------------------|----------|
|
|
2736
|
+
| `force_all=True` | Both metadata + data | Complete cache overhaul needed |
|
|
2737
|
+
| `force_metadata=True` | Only metadata | Report configuration changed |
|
|
2738
|
+
| `force_data=True` | Only data | Data needs fresh pull |
|
|
2739
|
+
|
|
2740
|
+
#### What Happens
|
|
2741
|
+
|
|
2742
|
+
When you trigger a force refresh:
|
|
2743
|
+
|
|
2744
|
+
1. ✓ `/tmp` cache directories are deleted
|
|
2745
|
+
2. ✓ Fresh data fetched from Quickbase API
|
|
2746
|
+
3. ✓ Data written to S3
|
|
2747
|
+
4. ✓ `/tmp` re-synced from updated S3
|
|
2748
|
+
|
|
2749
|
+
Your Lambda now has fresh data from Quickbase.
|
|
2750
|
+
|
|
2751
|
+
#### Example
|
|
2752
|
+
|
|
2753
|
+
```python
|
|
2754
|
+
def lambda_handler(event, context):
|
|
2755
|
+
client = get_client()
|
|
2756
|
+
cache_mgr = get_cache_manager()
|
|
2757
|
+
|
|
2758
|
+
# Metadata changed in Quickbase? Force refresh it:
|
|
2759
|
+
FORCE_COMPLETE_CACHE_REFRESH_METADATA = True # ← Toggle this
|
|
2760
|
+
|
|
2761
|
+
if (FORCE_COMPLETE_CACHE_REFRESH_ALL or FORCE_COMPLETE_CACHE_REFRESH_METADATA
|
|
2762
|
+
or FORCE_COMPLETE_CACHE_REFRESH_DATA):
|
|
2763
|
+
from quickbase_extract import complete_cache_refresh
|
|
2764
|
+
complete_cache_refresh(
|
|
2765
|
+
cache_manager=cache_mgr,
|
|
2766
|
+
client=client,
|
|
2767
|
+
report_configs=get_all_reports(),
|
|
2768
|
+
force_metadata=FORCE_COMPLETE_CACHE_REFRESH_METADATA,
|
|
2769
|
+
)
|
|
2770
|
+
|
|
2771
|
+
# Rest of handler...
|
|
2772
|
+
```
|
|
2773
|
+
|
|
2774
|
+
**CloudWatch logs will show:**
|
|
2775
|
+
```
|
|
2776
|
+
WARNING: Starting complete cache refresh for: metadata (clearing /tmp, refreshing from Quickbase, updating S3...)
|
|
2777
|
+
DEBUG: Reset cache sync flag
|
|
2778
|
+
INFO: Fetching fresh data from Quickbase...
|
|
2779
|
+
INFO: Re-syncing /tmp from S3...
|
|
2780
|
+
WARNING: Complete cache refresh finished for metadata: /tmp and S3 now have fresh data from Quickbase
|
|
2781
|
+
```
|
|
2782
|
+
|
|
2783
|
+
#### Important Notes
|
|
2784
|
+
|
|
2785
|
+
- **Don't leave toggles set to `True`** — revert to `False` after testing
|
|
2786
|
+
- **Only for development** — not a production workflow
|
|
2787
|
+
- Logs will show exactly what was refreshed
|
|
2788
|
+
- Safe to use — doesn't affect running processes, only next Lambda invocation
|
|
2789
|
+
|
|
2598
2790
|
## Advanced Usage
|
|
2599
2791
|
|
|
2600
2792
|
### Custom Report Configurations
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "quickbase-extract"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Extract and cache Quickbase report data with built-in error handling and S3 support"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""S3-backed cache sync for Lambda environments."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import shutil
|
|
5
|
+
|
|
6
|
+
from quickbase_extract.cache_manager import CacheManager
|
|
7
|
+
from quickbase_extract.cache_orchestration import ensure_cache_freshness
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
# Module-level flag to track if we've synced this Lambda invocation
|
|
12
|
+
_CACHE_SYNCED = False
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def complete_cache_refresh(
|
|
16
|
+
client,
|
|
17
|
+
cache_manager: CacheManager,
|
|
18
|
+
report_configs: list,
|
|
19
|
+
force_all: bool = False,
|
|
20
|
+
force_metadata: bool = False,
|
|
21
|
+
force_data: bool = False,
|
|
22
|
+
) -> None:
|
|
23
|
+
"""Completely refresh cache: clear /tmp, fetch from Quickbase, update S3, re-sync to /tmp.
|
|
24
|
+
|
|
25
|
+
This is a dev/debugging utility for forcing a cache refresh when report
|
|
26
|
+
metadata or configurations change. Clears specified local /tmp cache,
|
|
27
|
+
fetches fresh data from Quickbase, writes to S3, and re-syncs to /tmp.
|
|
28
|
+
|
|
29
|
+
Only the cache types specified by force flags are cleared and refreshed.
|
|
30
|
+
For example, if only force_metadata=True, only metadata cache is touched.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
cache_manager: CacheManager instance for cache operations.
|
|
34
|
+
client: Quickbase API client for fetching fresh data.
|
|
35
|
+
report_configs: List of all ReportConfig instances to refresh.
|
|
36
|
+
force_all: If True, refresh both metadata and data. Overrides individual flags.
|
|
37
|
+
Defaults to False.
|
|
38
|
+
force_metadata: If True (and force_all is False), refresh only metadata.
|
|
39
|
+
Defaults to False.
|
|
40
|
+
force_data: If True (and force_all is False), refresh only data.
|
|
41
|
+
Defaults to False.
|
|
42
|
+
|
|
43
|
+
Raises:
|
|
44
|
+
Exception: If cache clearing or refresh operations fail.
|
|
45
|
+
|
|
46
|
+
Example:
|
|
47
|
+
>>> # Refresh only metadata
|
|
48
|
+
>>> complete_cache_refresh(
|
|
49
|
+
... cache_manager=cache_mgr,
|
|
50
|
+
... client=qb_client,
|
|
51
|
+
... report_configs=report_config,
|
|
52
|
+
... force_metadata=True,
|
|
53
|
+
... )
|
|
54
|
+
>>>
|
|
55
|
+
>>> # Refresh all
|
|
56
|
+
>>> complete_cache_refresh(
|
|
57
|
+
... cache_manager=cache_mgr,
|
|
58
|
+
... client=qb_client,
|
|
59
|
+
... report_configs=report_config,
|
|
60
|
+
... force_all=True,
|
|
61
|
+
... )
|
|
62
|
+
"""
|
|
63
|
+
global _CACHE_SYNCED
|
|
64
|
+
|
|
65
|
+
# Determine which caches to refresh (force_all overrides individual flags)
|
|
66
|
+
should_refresh_metadata = force_all or force_metadata
|
|
67
|
+
should_refresh_data = force_all or force_data
|
|
68
|
+
|
|
69
|
+
if not (should_refresh_metadata or should_refresh_data):
|
|
70
|
+
logger.debug("No cache refresh flags set, skipping complete cache refresh")
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
refresh_types = []
|
|
74
|
+
if should_refresh_metadata:
|
|
75
|
+
refresh_types.append("metadata")
|
|
76
|
+
if should_refresh_data:
|
|
77
|
+
refresh_types.append("data")
|
|
78
|
+
|
|
79
|
+
logger.warning(
|
|
80
|
+
f"Starting complete cache refresh for: {', '.join(refresh_types)} "
|
|
81
|
+
"(clearing /tmp, refreshing from Quickbase, updating S3...)"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
# Step 1: Clear specified /tmp cache directories
|
|
85
|
+
if should_refresh_metadata:
|
|
86
|
+
metadata_dir = cache_manager.cache_root / "report_metadata"
|
|
87
|
+
if metadata_dir.exists():
|
|
88
|
+
shutil.rmtree(metadata_dir)
|
|
89
|
+
logger.info(f"Cleared metadata cache directory: {metadata_dir}")
|
|
90
|
+
|
|
91
|
+
if should_refresh_data:
|
|
92
|
+
data_dir = cache_manager.cache_root / "report_data"
|
|
93
|
+
if data_dir.exists():
|
|
94
|
+
shutil.rmtree(data_dir)
|
|
95
|
+
logger.info(f"Cleared data cache directory: {data_dir}")
|
|
96
|
+
|
|
97
|
+
# Step 2: Reset sync flag so fresh data will be fetched
|
|
98
|
+
_CACHE_SYNCED = False
|
|
99
|
+
logger.debug("Reset cache sync flag")
|
|
100
|
+
|
|
101
|
+
# Step 3: Fetch fresh data from Quickbase and write to S3
|
|
102
|
+
logger.info("Fetching fresh data from Quickbase...")
|
|
103
|
+
ensure_cache_freshness(
|
|
104
|
+
client=client,
|
|
105
|
+
cache_manager=cache_manager,
|
|
106
|
+
report_configs_all=report_configs,
|
|
107
|
+
force_all=force_all,
|
|
108
|
+
force_metadata=force_metadata,
|
|
109
|
+
force_data=force_data,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
# Step 4: Re-sync /tmp from S3 for cleared caches
|
|
113
|
+
logger.info("Re-syncing /tmp from S3...")
|
|
114
|
+
sync_from_s3_once(cache_manager, force=True)
|
|
115
|
+
|
|
116
|
+
logger.warning(
|
|
117
|
+
f"Complete cache refresh finished for {', '.join(refresh_types)}: "
|
|
118
|
+
"/tmp and S3 now have fresh data from Quickbase"
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def sync_from_s3_once(cache_manager: CacheManager, force: bool = False) -> None:
|
|
123
|
+
"""Download cache from S3 to /tmp on Lambda cold start.
|
|
124
|
+
|
|
125
|
+
Only syncs if cache hasn't been synced in this invocation.
|
|
126
|
+
Subsequent calls are no-ops unless force=True.
|
|
127
|
+
|
|
128
|
+
On Lambda, the sync flag persists across warm invocations within the same
|
|
129
|
+
container, so warm starts skip the sync (Lambda /tmp persists). Only cold
|
|
130
|
+
starts trigger a sync.
|
|
131
|
+
|
|
132
|
+
On local environments, automatically detects if CACHE_BUCKET is configured.
|
|
133
|
+
If not configured, does nothing (local caching only).
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
cache_manager: CacheManager instance for cache operations.
|
|
137
|
+
force: If True, sync even if already synced in this invocation.
|
|
138
|
+
Defaults to False.
|
|
139
|
+
|
|
140
|
+
Raises:
|
|
141
|
+
Exception: If S3 operations fail.
|
|
142
|
+
|
|
143
|
+
Example:
|
|
144
|
+
>>> # In Lambda handler initialization
|
|
145
|
+
>>> cache_manager = CacheManager(
|
|
146
|
+
... cache_root=Path("/tmp/my_project/dev/cache"),
|
|
147
|
+
... s3_bucket="mit-bio-quickbase",
|
|
148
|
+
... s3_prefix="my_project/dev/cache",
|
|
149
|
+
... )
|
|
150
|
+
>>> sync_from_s3_once(cache_manager) # Syncs on cold start
|
|
151
|
+
>>> sync_from_s3_once(cache_manager) # No-op on same invocation
|
|
152
|
+
>>>
|
|
153
|
+
>>> # Force re-sync if needed (programmatically)
|
|
154
|
+
>>> sync_from_s3_once(cache_manager, force=True)
|
|
155
|
+
"""
|
|
156
|
+
global _CACHE_SYNCED
|
|
157
|
+
|
|
158
|
+
# Check for force refresh
|
|
159
|
+
already_synced = _CACHE_SYNCED and not force
|
|
160
|
+
|
|
161
|
+
if already_synced:
|
|
162
|
+
logger.debug("Cache already synced in this invocation, skipping")
|
|
163
|
+
return
|
|
164
|
+
|
|
165
|
+
cache_manager.sync_from_s3() # Handles Lambda detection internally
|
|
166
|
+
_CACHE_SYNCED = True
|
|
167
|
+
logger.info("Cache synced from S3")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def is_cache_synced() -> bool:
|
|
171
|
+
"""Check if cache has been synced in this invocation.
|
|
172
|
+
|
|
173
|
+
Returns:
|
|
174
|
+
True if cache has been synced, False otherwise.
|
|
175
|
+
|
|
176
|
+
Example:
|
|
177
|
+
>>> if not is_cache_synced():
|
|
178
|
+
... print("Cache needs syncing")
|
|
179
|
+
"""
|
|
180
|
+
return _CACHE_SYNCED
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _reset_cache_sync() -> None:
|
|
184
|
+
"""Reset the cache sync flag. For testing only.
|
|
185
|
+
|
|
186
|
+
Example:
|
|
187
|
+
>>> # In test teardown
|
|
188
|
+
>>> _reset_cache_sync()
|
|
189
|
+
"""
|
|
190
|
+
global _CACHE_SYNCED
|
|
191
|
+
_CACHE_SYNCED = False
|