quickbase-extract 0.3.1__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/.pre-commit-config.yaml +1 -1
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/CHANGELOG.md +34 -3
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/PKG-INFO +4 -4
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/README.md +1 -1
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/pyproject.toml +4 -4
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/__init__.py +15 -20
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/api_handlers.py +5 -7
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/cache_manager.py +1 -1
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/cache_orchestration.py +15 -2
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/report_data.py +117 -34
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/report_metadata.py +57 -4
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/conftest.py +1 -2
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/test_api_handlers.py +1 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/test_cache_manager.py +1 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/test_cache_orchestration.py +2 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/test_cache_sync.py +1 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/test_report_data.py +197 -5
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/test_report_metadata.py +57 -11
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/.editorconfig +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/.gitignore +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/.python-version +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/LICENSE.txt +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/TODO.md +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/cache_sync.py +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/config.py +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/py.typed +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/utils.py +0 -0
- {quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/tests/test_utils.py +0 -0
|
@@ -5,6 +5,37 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.4.1] - 2026-05-01
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- `load_report_metadata()` return type changed from `dict` to `dict[ReportConfig, dict]` to match `load_report_metadata_batch()` and ensure consistent API with `get_data()`
|
|
13
|
+
- Fixed incompatibility where `load_report_metadata()` result could not be directly passed to `get_data()` due to format mismatch
|
|
14
|
+
|
|
15
|
+
## [0.4.0] - 2026-04-29
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- Support for multiple values in `ask_values` parameters - values can now be strings or lists of strings
|
|
20
|
+
- Automatic OR condition expansion: list values like `{"ask1": ["val1", "val2", "val3"]}` expand to `({'25'.EX.'val1'}OR{'25'.EX.'val2'}OR{'25'.EX.'val3'})`
|
|
21
|
+
- Proper grouping with parentheses when multiple values are used to preserve filter logic precedence
|
|
22
|
+
- Validation for empty lists in `ask_values` - raises `ValueError` if empty list provided
|
|
23
|
+
- `filter_metadata_by_table()` helper function for convenient report metadata lookup by table name, with optional app name filtering to resolve ambiguous table names across apps
|
|
24
|
+
- Helper functions `_validate_ask_values()` and `_normalize_ask_values()` for cleaner separation of concerns in placeholder replacement
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- `ask_values` type hint updated from `dict[str, str]` to `dict[str, str | list[str]]` in `get_data()` and `get_data_parallel()`
|
|
29
|
+
- `_replace_ask_placeholders()` refactored to use helper functions for validation and normalization, improving testability and maintainability
|
|
30
|
+
- Filter replacement now extracts and replaces full condition blocks (e.g., `{'25'.EX.'_ask1_'}`) instead of just placeholders
|
|
31
|
+
- Placeholder replacement now processes all occurrences using `re.finditer()` instead of only the first match
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- Complex filters with multiple conditions now maintain proper precedence when list values are expanded
|
|
36
|
+
- **BREAKING FIX**: Multiple condition blocks with the same placeholder are now all replaced correctly (previously only first occurrence was replaced)
|
|
37
|
+
- String position preservation during replacement to handle complex multi-placeholder filters correctly
|
|
38
|
+
|
|
8
39
|
## [0.3.1] - 2026-04-27
|
|
9
40
|
|
|
10
41
|
### Fixed
|
|
@@ -83,9 +114,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
83
114
|
|
|
84
115
|
- `cache_freshness.py` module — functionality consolidated into `cache_manager.py` (use `ensure_cache_freshness()` instead)
|
|
85
116
|
- `check_cache_freshness()`, `get_cache_files()`, `get_cache_summary()` functions — use `CacheManager` methods directly or `ensure_cache_freshness()` for orchestration
|
|
86
|
-
- `refresh_all()` function
|
|
87
|
-
- `client.py` module
|
|
88
|
-
- `get_cache_manager()` singleton
|
|
117
|
+
- `refresh_all()` function\*\*: Use `ensure_cache_freshness()` for cache management
|
|
118
|
+
- `client.py` module\*\*: Users must create Quickbase clients directly using `quickbase-api` package
|
|
119
|
+
- `get_cache_manager()` singleton\*\*: Users must create `CacheManager` instances explicitly
|
|
89
120
|
- `find_report()` function from utils - no longer needed with `ReportConfig`
|
|
90
121
|
- Dict-based report config format - all configs must use `ReportConfig` NamedTuple
|
|
91
122
|
- Nested `report` object from metadata - simplified structure
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quickbase-extract
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Extract and cache Quickbase report data with built-in error handling and S3 support
|
|
5
5
|
Project-URL: Homepage, https://github.com/tbrezler/quickbase-extract
|
|
6
6
|
Project-URL: Repository, https://github.com/tbrezler/quickbase-extract.git
|
|
@@ -20,8 +20,8 @@ Requires-Dist: boto3>=1.26.0
|
|
|
20
20
|
Requires-Dist: quickbase-api>=0.3.1
|
|
21
21
|
Provides-Extra: dev
|
|
22
22
|
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
23
|
-
Requires-Dist: pytest>=
|
|
24
|
-
Requires-Dist: ruff>=0.
|
|
23
|
+
Requires-Dist: pytest>=9.0.3; extra == 'dev'
|
|
24
|
+
Requires-Dist: ruff>=0.15.11; extra == 'dev'
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
|
|
27
27
|
# Quickbase Extract
|
|
@@ -47,7 +47,7 @@ pip install quickbase-extract
|
|
|
47
47
|
|
|
48
48
|
### Requirements
|
|
49
49
|
|
|
50
|
-
- Python 3.
|
|
50
|
+
- Python 3.12+
|
|
51
51
|
- `quickbase-api` - Quickbase API client
|
|
52
52
|
- `boto3` - AWS SDK (for Lambda/S3 support)
|
|
53
53
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "quickbase-extract"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.1"
|
|
8
8
|
description = "Extract and cache Quickbase report data with built-in error handling and S3 support"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -29,9 +29,9 @@ dependencies = [
|
|
|
29
29
|
|
|
30
30
|
[project.optional-dependencies]
|
|
31
31
|
dev = [
|
|
32
|
-
"pytest>=
|
|
32
|
+
"pytest>=9.0.3",
|
|
33
33
|
"pytest-cov>=4.0",
|
|
34
|
-
"ruff>=0.
|
|
34
|
+
"ruff>=0.15.11",
|
|
35
35
|
]
|
|
36
36
|
|
|
37
37
|
[project.urls]
|
|
@@ -48,5 +48,5 @@ line-length = 120
|
|
|
48
48
|
target-version = "py312"
|
|
49
49
|
|
|
50
50
|
[tool.ruff.lint]
|
|
51
|
-
select = ["
|
|
51
|
+
select = ["F", "E", "W", "B", "BLE", "I", "UP", "A", "RUF", "T10"]
|
|
52
52
|
ignore = ["E501"]
|
|
@@ -42,11 +42,12 @@ from quickbase_extract.cache_sync import complete_cache_refresh, is_cache_synced
|
|
|
42
42
|
# Config
|
|
43
43
|
from quickbase_extract.config import ReportConfig
|
|
44
44
|
|
|
45
|
-
# Report data
|
|
45
|
+
# Report data
|
|
46
46
|
from quickbase_extract.report_data import get_data, get_data_parallel, load_data, load_data_batch
|
|
47
47
|
|
|
48
48
|
# Report metadata
|
|
49
49
|
from quickbase_extract.report_metadata import (
|
|
50
|
+
filter_metadata_by_table,
|
|
50
51
|
get_report_metadata,
|
|
51
52
|
get_report_metadata_parallel,
|
|
52
53
|
load_report_metadata,
|
|
@@ -62,31 +63,25 @@ __version__ = version("quickbase-extract")
|
|
|
62
63
|
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
63
64
|
|
|
64
65
|
__all__ = [
|
|
65
|
-
# Version
|
|
66
|
-
"__version__",
|
|
67
|
-
# Cache management
|
|
68
66
|
"CacheManager",
|
|
69
|
-
"ensure_cache_freshness",
|
|
70
|
-
"complete_cache_refresh",
|
|
71
|
-
"sync_from_s3_once",
|
|
72
|
-
"is_cache_synced",
|
|
73
|
-
# Config
|
|
74
|
-
"ReportConfig",
|
|
75
|
-
# API operations
|
|
76
67
|
"QuickbaseOperationError",
|
|
68
|
+
"ReportConfig",
|
|
69
|
+
"__version__",
|
|
70
|
+
"complete_cache_refresh",
|
|
71
|
+
"ensure_cache_freshness",
|
|
72
|
+
"filter_metadata_by_table",
|
|
73
|
+
"get_data",
|
|
74
|
+
"get_data_parallel",
|
|
75
|
+
"get_report_metadata",
|
|
76
|
+
"get_report_metadata_parallel",
|
|
77
77
|
"handle_delete",
|
|
78
78
|
"handle_query",
|
|
79
79
|
"handle_upsert",
|
|
80
|
-
|
|
81
|
-
"get_report_metadata",
|
|
82
|
-
"get_report_metadata_parallel",
|
|
83
|
-
"load_report_metadata",
|
|
84
|
-
"load_report_metadata_batch",
|
|
85
|
-
# Report data
|
|
86
|
-
"get_data",
|
|
87
|
-
"get_data_parallel",
|
|
80
|
+
"is_cache_synced",
|
|
88
81
|
"load_data",
|
|
89
82
|
"load_data_batch",
|
|
90
|
-
|
|
83
|
+
"load_report_metadata",
|
|
84
|
+
"load_report_metadata_batch",
|
|
91
85
|
"normalize_name",
|
|
86
|
+
"sync_from_s3_once",
|
|
92
87
|
]
|
|
@@ -4,8 +4,6 @@ Provides retry logic for rate-limited requests, standardized error handling,
|
|
|
4
4
|
and logging for Quickbase API operations.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
# ruff: noqa: BLE001
|
|
8
|
-
|
|
9
7
|
import logging
|
|
10
8
|
import random
|
|
11
9
|
import time
|
|
@@ -139,11 +137,11 @@ def handle_query(
|
|
|
139
137
|
client,
|
|
140
138
|
table_id: str,
|
|
141
139
|
*,
|
|
142
|
-
select: list[int] = None,
|
|
143
|
-
where: str = None,
|
|
144
|
-
sort_by: list[dict] = None,
|
|
145
|
-
group_by: list[dict] = None,
|
|
146
|
-
options: dict = None,
|
|
140
|
+
select: list[int] | None = None,
|
|
141
|
+
where: str | None = None,
|
|
142
|
+
sort_by: list[dict] | None = None,
|
|
143
|
+
group_by: list[dict] | None = None,
|
|
144
|
+
options: dict | None = None,
|
|
147
145
|
description: str = "",
|
|
148
146
|
max_retries: int = 3,
|
|
149
147
|
) -> dict:
|
{quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/cache_orchestration.py
RENAMED
|
@@ -123,6 +123,7 @@ def _refresh_data_cache(
|
|
|
123
123
|
cache_manager: CacheManager,
|
|
124
124
|
reports_to_refresh: list[ReportConfig],
|
|
125
125
|
reasons: list[str],
|
|
126
|
+
ask_values: dict[ReportConfig, dict[str, str | list[str]] | None] | None = None,
|
|
126
127
|
) -> None:
|
|
127
128
|
"""Refresh data cache for specified reports.
|
|
128
129
|
|
|
@@ -131,6 +132,8 @@ def _refresh_data_cache(
|
|
|
131
132
|
cache_manager: CacheManager instance.
|
|
132
133
|
reports_to_refresh: Reports to refresh data for.
|
|
133
134
|
reasons: List of reasons for refresh (for logging).
|
|
135
|
+
ask_values: Optional dict mapping ReportConfig -> ask_values dict.
|
|
136
|
+
Per-report "ask the user" filter values.
|
|
134
137
|
|
|
135
138
|
Raises:
|
|
136
139
|
CacheRefreshError: If data refresh fails.
|
|
@@ -148,6 +151,7 @@ def _refresh_data_cache(
|
|
|
148
151
|
report_configs=reports_to_refresh,
|
|
149
152
|
report_metadata=metadata,
|
|
150
153
|
cache=True,
|
|
154
|
+
ask_values=ask_values,
|
|
151
155
|
)
|
|
152
156
|
logger.info("Data cache refresh completed successfully")
|
|
153
157
|
except Exception as e:
|
|
@@ -160,6 +164,7 @@ def ensure_cache_freshness(
|
|
|
160
164
|
cache_manager: CacheManager,
|
|
161
165
|
report_configs_all: list[ReportConfig],
|
|
162
166
|
report_configs_to_cache: list[ReportConfig] | None = None,
|
|
167
|
+
ask_values: dict[ReportConfig, dict[str, str | list[str]] | None] | None = None,
|
|
163
168
|
metadata_stale_hours: float | None = None,
|
|
164
169
|
data_stale_hours: float | None = None,
|
|
165
170
|
cache_all_data: bool = False,
|
|
@@ -184,6 +189,12 @@ def ensure_cache_freshness(
|
|
|
184
189
|
report_configs_to_cache: Optional subset of ReportConfig instances to
|
|
185
190
|
cache data for. If cache_all_data is True, this parameter is ignored
|
|
186
191
|
and all reports' data is cached instead.
|
|
192
|
+
ask_values: Optional dict mapping ReportConfig -> ask_values dict.
|
|
193
|
+
Per-report "ask the user" filter values. Only used when refreshing
|
|
194
|
+
data cache. Example: {
|
|
195
|
+
ReportConfig("bq8x", "Accounts", "Python"): {"ask1": "abc"},
|
|
196
|
+
ReportConfig("bq9y", "Contacts", "Active"): {"ask1": "def"}
|
|
197
|
+
}
|
|
187
198
|
metadata_stale_hours: Threshold (hours) for metadata staleness.
|
|
188
199
|
If not provided, reads from METADATA_STALE_HOURS env var,
|
|
189
200
|
falls back to DEFAULT_METADATA_STALE_HOURS (168 hours / 7 days).
|
|
@@ -221,12 +232,14 @@ def ensure_cache_freshness(
|
|
|
221
232
|
... report_configs_to_cache=get_reports_to_cache(),
|
|
222
233
|
... )
|
|
223
234
|
>>>
|
|
224
|
-
>>> # Cache all reports' data
|
|
235
|
+
>>> # Cache all reports' data with ask_values
|
|
236
|
+
>>> ask_vals = {get_all_reports()[0]: {"ask1": "value1"}}
|
|
225
237
|
>>> ensure_cache_freshness(
|
|
226
238
|
... client=client,
|
|
227
239
|
... cache_manager=cache_manager,
|
|
228
240
|
... report_configs_all=get_all_reports(),
|
|
229
241
|
... cache_all_data=True,
|
|
242
|
+
... ask_values=ask_vals,
|
|
230
243
|
... )
|
|
231
244
|
"""
|
|
232
245
|
# Resolve thresholds from arguments, environment, or defaults
|
|
@@ -295,4 +308,4 @@ def ensure_cache_freshness(
|
|
|
295
308
|
|
|
296
309
|
# Refresh data if needed
|
|
297
310
|
if data_needs_refresh:
|
|
298
|
-
_refresh_data_cache(client, cache_manager, reports_to_refresh_data, data_reasons)
|
|
311
|
+
_refresh_data_cache(client, cache_manager, reports_to_refresh_data, reasons=data_reasons, ask_values=ask_values)
|
|
@@ -12,70 +12,153 @@ from quickbase_extract.config import ReportConfig
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def _validate_ask_values(
|
|
16
|
+
ask_values: dict[str, str | list[str]],
|
|
17
|
+
placeholders_in_filter: set[str],
|
|
18
|
+
report_config: ReportConfig,
|
|
19
|
+
) -> None:
|
|
20
|
+
"""Validate that ask_values matches placeholders in filter.
|
|
21
|
+
|
|
22
|
+
Ensures that:
|
|
23
|
+
- All placeholders in the filter have corresponding values in ask_values
|
|
24
|
+
- All values in ask_values are actually used in the filter
|
|
25
|
+
- No empty lists are provided
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
ask_values: Dict mapping placeholder keys (e.g., "ask1") to values or lists of values.
|
|
29
|
+
placeholders_in_filter: Set of placeholder strings found in filter (e.g., {"_ask1_", "_ask2_"}).
|
|
30
|
+
report_config: ReportConfig for error messages.
|
|
31
|
+
|
|
32
|
+
Raises:
|
|
33
|
+
ValueError: If values are missing, unused, or if any list is empty.
|
|
34
|
+
"""
|
|
35
|
+
# Check for missing values
|
|
36
|
+
missing_values = [p for p in placeholders_in_filter if p.strip("_") not in ask_values]
|
|
37
|
+
if missing_values:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
f"Report {report_config} filter requires values for {missing_values}, "
|
|
40
|
+
f"but they were not provided in ask_values."
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# Check for unused values
|
|
44
|
+
unused_values = [k for k in ask_values.keys() if f"_{k}_" not in placeholders_in_filter]
|
|
45
|
+
if unused_values:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"Report {report_config} received ask_values {unused_values} "
|
|
48
|
+
f"that are not used in the filter. Available placeholders: {list(placeholders_in_filter)}"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# Check for empty lists
|
|
52
|
+
empty_list_keys = [k for k, v in ask_values.items() if isinstance(v, list) and len(v) == 0]
|
|
53
|
+
if empty_list_keys:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"Report {report_config} received empty lists for ask_values keys: {empty_list_keys}. "
|
|
56
|
+
f"Empty lists are not allowed."
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _normalize_ask_values(ask_values: dict[str, str | list[str]]) -> dict[str, list[str]]:
|
|
61
|
+
"""Normalize all ask_values to lists for consistent processing.
|
|
62
|
+
|
|
63
|
+
Converts single string values to single-element lists, leaving list values
|
|
64
|
+
unchanged. This allows downstream logic to handle all values uniformly.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
ask_values: Dict with values that are either strings or lists of strings.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
Dict with all values as lists of strings.
|
|
71
|
+
|
|
72
|
+
Example:
|
|
73
|
+
>>> _normalize_ask_values({"ask1": "value", "ask2": ["a", "b"]})
|
|
74
|
+
{"ask1": ["value"], "ask2": ["a", "b"]}
|
|
75
|
+
"""
|
|
76
|
+
normalized = {}
|
|
77
|
+
for key, value in ask_values.items():
|
|
78
|
+
normalized[key] = [value] if isinstance(value, str) else value
|
|
79
|
+
return normalized
|
|
80
|
+
|
|
81
|
+
|
|
15
82
|
def _replace_ask_placeholders(
|
|
16
83
|
report_filter: str,
|
|
17
|
-
ask_values: dict[str, str],
|
|
84
|
+
ask_values: dict[str, str | list[str]],
|
|
18
85
|
report_config: ReportConfig,
|
|
19
86
|
) -> str:
|
|
20
87
|
"""Replace ask-the-user placeholders in a Quickbase filter with actual values.
|
|
21
88
|
|
|
89
|
+
Finds all placeholders (e.g., _ask1_, _ask2_) in the filter and replaces them
|
|
90
|
+
with provided values. Single values are replaced directly, while multiple values
|
|
91
|
+
in a list are expanded into OR conditions with proper grouping to preserve
|
|
92
|
+
filter logic precedence.
|
|
93
|
+
|
|
22
94
|
Args:
|
|
23
95
|
report_filter: The filter string from report metadata (e.g., "{'25'.EX.'_ask1_'}").
|
|
24
|
-
ask_values: Dict mapping placeholder keys to values
|
|
96
|
+
ask_values: Dict mapping placeholder keys to values or lists of values.
|
|
97
|
+
Keys like "ask1", "ask2". Values can be strings or lists of strings.
|
|
98
|
+
Example: {"ask1": "abc123"} or {"ask1": ["abc123", "abc456"]}
|
|
25
99
|
report_config: ReportConfig for error messages.
|
|
26
100
|
|
|
27
101
|
Returns:
|
|
28
102
|
Modified filter string with placeholders replaced.
|
|
29
103
|
|
|
30
104
|
Raises:
|
|
31
|
-
ValueError: If required placeholders are missing values
|
|
105
|
+
ValueError: If required placeholders are missing values, unused values provided,
|
|
106
|
+
empty lists provided, or condition blocks cannot be found.
|
|
32
107
|
|
|
33
108
|
Example:
|
|
34
109
|
>>> filter_str = "{'25'.EX.'_ask1_'}AND{'10'.AF.'today'}"
|
|
35
110
|
>>> config = ReportConfig("bq8xyx9z", "Accounts", "Python")
|
|
111
|
+
>>> # Single value
|
|
36
112
|
>>> _replace_ask_placeholders(filter_str, {"ask1": "abc123"}, config)
|
|
37
113
|
"{'25'.EX.'abc123'}AND{'10'.AF.'today'}"
|
|
114
|
+
>>> # Multiple values
|
|
115
|
+
>>> _replace_ask_placeholders(filter_str, {"ask1": ["val1", "val2"]}, config)
|
|
116
|
+
"(({'25'.EX.'val1'}OR{'25'.EX.'val2'}))AND{'10'.AF.'today'}"
|
|
117
|
+
>>> # Multiple condition blocks with same placeholder
|
|
118
|
+
>>> _replace_ask_placeholders(
|
|
119
|
+
... "({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'})",
|
|
120
|
+
... {"ask1": ["a", "b"]},
|
|
121
|
+
... config
|
|
122
|
+
... )
|
|
123
|
+
"(({'41'.EX.'a'}OR{'41'.EX.'b'})OR({'40'.EX.'a'}OR{'40'.EX.'b'}))"
|
|
38
124
|
"""
|
|
39
|
-
# Find all placeholders in the filter (e.g., _ask1_, _ask2_)
|
|
40
125
|
placeholders_in_filter = set(re.findall(r"_ask\d+_", report_filter))
|
|
41
126
|
|
|
42
127
|
if not placeholders_in_filter:
|
|
43
|
-
# No placeholders found - nothing to replace
|
|
44
128
|
return report_filter
|
|
45
129
|
|
|
46
|
-
# Validate
|
|
47
|
-
|
|
130
|
+
# Validate and normalize
|
|
131
|
+
_validate_ask_values(ask_values, placeholders_in_filter, report_config)
|
|
132
|
+
normalized_values = _normalize_ask_values(ask_values)
|
|
133
|
+
|
|
134
|
+
# Replace placeholders
|
|
135
|
+
modified_filter = report_filter
|
|
48
136
|
for placeholder in placeholders_in_filter:
|
|
49
|
-
# Convert _ask1_ to ask1 for lookup
|
|
50
137
|
key = placeholder.strip("_")
|
|
51
|
-
|
|
52
|
-
missing_values.append(placeholder)
|
|
138
|
+
values = normalized_values[key]
|
|
53
139
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
)
|
|
140
|
+
# Find ALL condition blocks containing this placeholder
|
|
141
|
+
# Pattern: {...'_ask1_'...} where we capture the entire {...}
|
|
142
|
+
condition_pattern = rf"(\{{[^}}]*{re.escape(placeholder)}[^{{]*\}})"
|
|
143
|
+
matches = list(re.finditer(condition_pattern, modified_filter))
|
|
59
144
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
for key in ask_values.keys():
|
|
63
|
-
placeholder = f"_{key}_"
|
|
64
|
-
if placeholder not in placeholders_in_filter:
|
|
65
|
-
unused_values.append(key)
|
|
145
|
+
if not matches:
|
|
146
|
+
raise ValueError(f"Report {report_config} could not find condition block for placeholder {placeholder}")
|
|
66
147
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
f"that are not used in the filter. Available placeholders: {list(placeholders_in_filter)}"
|
|
71
|
-
)
|
|
148
|
+
# Process matches in reverse order to maintain string positions during replacement
|
|
149
|
+
for match in reversed(matches):
|
|
150
|
+
original_condition = match.group(1)
|
|
72
151
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
152
|
+
if len(values) == 1:
|
|
153
|
+
# Single value: simple replacement
|
|
154
|
+
new_condition = original_condition.replace(placeholder, values[0])
|
|
155
|
+
else:
|
|
156
|
+
# Multiple values: replicate condition and join with OR
|
|
157
|
+
replicated_conditions = [original_condition.replace(placeholder, v) for v in values]
|
|
158
|
+
new_condition = f"({('OR'.join(replicated_conditions))})"
|
|
159
|
+
|
|
160
|
+
# Replace using string slicing to preserve positions for other replacements
|
|
161
|
+
modified_filter = modified_filter[: match.start()] + new_condition + modified_filter[match.end() :]
|
|
79
162
|
|
|
80
163
|
return modified_filter
|
|
81
164
|
|
|
@@ -132,7 +215,7 @@ def get_data(
|
|
|
132
215
|
report_config: ReportConfig,
|
|
133
216
|
report_metadata: dict[ReportConfig, dict],
|
|
134
217
|
cache: bool = False,
|
|
135
|
-
ask_values: dict[str, str] | None = None,
|
|
218
|
+
ask_values: dict[str, str | list[str]] | None = None,
|
|
136
219
|
) -> list[dict]:
|
|
137
220
|
"""Query a Quickbase table for data using cached report metadata.
|
|
138
221
|
|
|
@@ -217,7 +300,7 @@ def get_data_parallel(
|
|
|
217
300
|
report_metadata: dict,
|
|
218
301
|
cache: bool = False,
|
|
219
302
|
max_workers: int = 8,
|
|
220
|
-
ask_values: dict[ReportConfig, dict[str, str] | None] | None = None,
|
|
303
|
+
ask_values: dict[ReportConfig, dict[str, str | list[str]] | None] | None = None,
|
|
221
304
|
) -> dict[ReportConfig, list[dict]]:
|
|
222
305
|
"""Fetch multiple reports in parallel using cached report metadata.
|
|
223
306
|
|
{quickbase_extract-0.3.1 → quickbase_extract-0.4.1}/src/quickbase_extract/report_metadata.py
RENAMED
|
@@ -238,7 +238,7 @@ def get_report_metadata_parallel(
|
|
|
238
238
|
def load_report_metadata(
|
|
239
239
|
cache_manager: CacheManager,
|
|
240
240
|
report_config: ReportConfig,
|
|
241
|
-
) -> dict:
|
|
241
|
+
) -> dict[ReportConfig, dict]:
|
|
242
242
|
"""Load cached report metadata from disk.
|
|
243
243
|
|
|
244
244
|
Args:
|
|
@@ -246,7 +246,8 @@ def load_report_metadata(
|
|
|
246
246
|
report_config: ReportConfig identifying the report to load.
|
|
247
247
|
|
|
248
248
|
Returns:
|
|
249
|
-
Dict
|
|
249
|
+
Dict mapping ReportConfig -> metadata dict (table ID, field mappings,
|
|
250
|
+
query config, and filters).
|
|
250
251
|
|
|
251
252
|
Raises:
|
|
252
253
|
FileNotFoundError: If cached metadata does not exist.
|
|
@@ -255,6 +256,7 @@ def load_report_metadata(
|
|
|
255
256
|
>>> cache_manager = CacheManager(cache_root=Path("my_project/dev/cache"))
|
|
256
257
|
>>> config = ReportConfig("bq8xyx9z", "Accounts", "Python")
|
|
257
258
|
>>> metadata = load_report_metadata(cache_manager, config)
|
|
259
|
+
>>> config_metadata = metadata[config]
|
|
258
260
|
"""
|
|
259
261
|
# Normalize names to match how they were saved
|
|
260
262
|
app_name = normalize_name(report_config.app_name)
|
|
@@ -268,7 +270,8 @@ def load_report_metadata(
|
|
|
268
270
|
f"Report metadata not found for {report_config}. Run get_report_metadata() first. Expected: {md_path}"
|
|
269
271
|
)
|
|
270
272
|
|
|
271
|
-
|
|
273
|
+
metadata_dict = json.loads(cache_manager.read_file(md_path))
|
|
274
|
+
return {report_config: metadata_dict}
|
|
272
275
|
|
|
273
276
|
|
|
274
277
|
def load_report_metadata_batch(
|
|
@@ -304,6 +307,56 @@ def load_report_metadata_batch(
|
|
|
304
307
|
|
|
305
308
|
metadata: dict[ReportConfig, dict] = {}
|
|
306
309
|
for config in report_configs:
|
|
307
|
-
metadata
|
|
310
|
+
metadata.update(load_report_metadata(cache_manager, config))
|
|
308
311
|
|
|
309
312
|
return metadata
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def filter_metadata_by_table(
|
|
316
|
+
report_metadata: dict[ReportConfig, dict],
|
|
317
|
+
table_name: str,
|
|
318
|
+
app_name: str | None = None,
|
|
319
|
+
) -> dict:
|
|
320
|
+
"""Retrieve metadata for a specific table, optionally filtered by app.
|
|
321
|
+
|
|
322
|
+
Args:
|
|
323
|
+
report_metadata: dict from load_report_metadata_batch()
|
|
324
|
+
table_name: Table name (e.g., "Accounts", "Billing Runs")
|
|
325
|
+
app_name: Optional app name. If not provided, table_name must be unique
|
|
326
|
+
across all apps.
|
|
327
|
+
|
|
328
|
+
Returns:
|
|
329
|
+
Metadata dict for the specified table (and app if provided).
|
|
330
|
+
|
|
331
|
+
Raises:
|
|
332
|
+
ValueError: If not found, or if table_name is ambiguous (multiple apps)
|
|
333
|
+
without app_name specified.
|
|
334
|
+
|
|
335
|
+
Example:
|
|
336
|
+
>>> metadata = load_report_metadata_batch(cache_manager, configs)
|
|
337
|
+
>>> runs_md = filter_metadata_by_app_table(metadata, "Billing Runs")
|
|
338
|
+
>>> accounts = filter_metadata_by_app_table(metadata, "Accounts", app_name="date_lake")
|
|
339
|
+
"""
|
|
340
|
+
if app_name:
|
|
341
|
+
# Filter by both app and table
|
|
342
|
+
results = [
|
|
343
|
+
data
|
|
344
|
+
for config, data in report_metadata.items()
|
|
345
|
+
if config.app_name == app_name and config.table_name == table_name
|
|
346
|
+
]
|
|
347
|
+
error_msg = f"app={app_name}, table={table_name}"
|
|
348
|
+
else:
|
|
349
|
+
# Filter by table only
|
|
350
|
+
results = [data for config, data in report_metadata.items() if config.table_name == table_name]
|
|
351
|
+
error_msg = f"table={table_name}"
|
|
352
|
+
|
|
353
|
+
if not results:
|
|
354
|
+
raise ValueError(f"No metadata found for {error_msg}")
|
|
355
|
+
|
|
356
|
+
if len(results) > 1:
|
|
357
|
+
available_apps = [config.app_name for config, _ in report_metadata.items() if config.table_name == table_name]
|
|
358
|
+
raise ValueError(
|
|
359
|
+
f"Multiple apps have table '{table_name}': {available_apps}. Please specify app_name: {available_apps[0]}"
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
return results[0]
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from unittest.mock import MagicMock
|
|
4
4
|
|
|
5
5
|
import pytest
|
|
6
|
+
|
|
6
7
|
from quickbase_extract.config import ReportConfig
|
|
7
8
|
|
|
8
9
|
|
|
@@ -217,5 +218,3 @@ def mock_s3_client():
|
|
|
217
218
|
from unittest.mock import MagicMock
|
|
218
219
|
|
|
219
220
|
return MagicMock()
|
|
220
|
-
return MagicMock()
|
|
221
|
-
return MagicMock()
|
|
@@ -6,6 +6,7 @@ import time
|
|
|
6
6
|
from unittest.mock import MagicMock, patch
|
|
7
7
|
|
|
8
8
|
import pytest
|
|
9
|
+
|
|
9
10
|
from quickbase_extract.cache_manager import CacheManager
|
|
10
11
|
from quickbase_extract.cache_orchestration import (
|
|
11
12
|
CacheRefreshError,
|
|
@@ -264,6 +265,7 @@ class TestRefreshDataCache:
|
|
|
264
265
|
report_configs=sample_report_configs,
|
|
265
266
|
report_metadata={config: {} for config in sample_report_configs},
|
|
266
267
|
cache=True,
|
|
268
|
+
ask_values=None,
|
|
267
269
|
)
|
|
268
270
|
assert "Data cache refresh needed: data stale" in caplog.text
|
|
269
271
|
assert "Data cache refresh completed successfully" in caplog.text
|
|
@@ -3,11 +3,14 @@
|
|
|
3
3
|
import json
|
|
4
4
|
|
|
5
5
|
import pytest
|
|
6
|
+
|
|
6
7
|
from quickbase_extract.cache_manager import CacheManager
|
|
7
8
|
from quickbase_extract.config import ReportConfig
|
|
8
9
|
from quickbase_extract.report_data import (
|
|
9
10
|
_extract_report_names,
|
|
11
|
+
_normalize_ask_values,
|
|
10
12
|
_replace_ask_placeholders,
|
|
13
|
+
_validate_ask_values,
|
|
11
14
|
get_data,
|
|
12
15
|
get_data_parallel,
|
|
13
16
|
load_data,
|
|
@@ -19,6 +22,103 @@ from quickbase_extract.report_metadata import (
|
|
|
19
22
|
)
|
|
20
23
|
|
|
21
24
|
|
|
25
|
+
class TestValidateAskValues:
|
|
26
|
+
"""Tests for _validate_ask_values helper function."""
|
|
27
|
+
|
|
28
|
+
def test_valid_single_placeholder_value(self, sample_report_configs):
|
|
29
|
+
"""Test validation passes with valid single placeholder and value."""
|
|
30
|
+
config = sample_report_configs[0]
|
|
31
|
+
placeholders = {"_ask1_"}
|
|
32
|
+
ask_values = {"ask1": "value"}
|
|
33
|
+
|
|
34
|
+
# Should not raise
|
|
35
|
+
_validate_ask_values(ask_values, placeholders, config)
|
|
36
|
+
|
|
37
|
+
def test_valid_multiple_placeholders(self, sample_report_configs):
|
|
38
|
+
"""Test validation passes with multiple placeholders and values."""
|
|
39
|
+
config = sample_report_configs[0]
|
|
40
|
+
placeholders = {"_ask1_", "_ask2_"}
|
|
41
|
+
ask_values = {"ask1": "value1", "ask2": ["value2", "value3"]}
|
|
42
|
+
|
|
43
|
+
# Should not raise
|
|
44
|
+
_validate_ask_values(ask_values, placeholders, config)
|
|
45
|
+
|
|
46
|
+
def test_missing_placeholder_value_raises_error(self, sample_report_configs):
|
|
47
|
+
"""Test error when placeholder has no corresponding value."""
|
|
48
|
+
config = sample_report_configs[0]
|
|
49
|
+
placeholders = {"_ask1_", "_ask2_"}
|
|
50
|
+
ask_values = {"ask1": "value1"} # Missing ask2
|
|
51
|
+
|
|
52
|
+
with pytest.raises(ValueError, match=r"requires values for.*_ask2_"):
|
|
53
|
+
_validate_ask_values(ask_values, placeholders, config)
|
|
54
|
+
|
|
55
|
+
def test_unused_ask_value_raises_error(self, sample_report_configs):
|
|
56
|
+
"""Test error when provided value is not used in filter."""
|
|
57
|
+
config = sample_report_configs[0]
|
|
58
|
+
placeholders = {"_ask1_"}
|
|
59
|
+
ask_values = {"ask1": "value1", "ask2": "value2"} # ask2 not in filter
|
|
60
|
+
|
|
61
|
+
with pytest.raises(ValueError, match=r"received ask_values.*ask2.*not used"):
|
|
62
|
+
_validate_ask_values(ask_values, placeholders, config)
|
|
63
|
+
|
|
64
|
+
def test_empty_list_raises_error(self, sample_report_configs):
|
|
65
|
+
"""Test error when empty list provided."""
|
|
66
|
+
config = sample_report_configs[0]
|
|
67
|
+
placeholders = {"_ask1_"}
|
|
68
|
+
ask_values = {"ask1": []} # Empty list
|
|
69
|
+
|
|
70
|
+
with pytest.raises(ValueError, match="empty lists"):
|
|
71
|
+
_validate_ask_values(ask_values, placeholders, config)
|
|
72
|
+
|
|
73
|
+
def test_empty_placeholders_set_validates(self, sample_report_configs):
|
|
74
|
+
"""Test that empty placeholders set validates successfully with empty ask_values."""
|
|
75
|
+
config = sample_report_configs[0]
|
|
76
|
+
placeholders = set()
|
|
77
|
+
ask_values = {}
|
|
78
|
+
|
|
79
|
+
# Should not raise
|
|
80
|
+
_validate_ask_values(ask_values, placeholders, config)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class TestNormalizeAskValues:
|
|
84
|
+
"""Tests for _normalize_ask_values helper function."""
|
|
85
|
+
|
|
86
|
+
def test_normalize_string_to_list(self):
|
|
87
|
+
"""Test that string values are converted to single-element lists."""
|
|
88
|
+
ask_values = {"ask1": "value"}
|
|
89
|
+
result = _normalize_ask_values(ask_values)
|
|
90
|
+
|
|
91
|
+
assert result == {"ask1": ["value"]}
|
|
92
|
+
|
|
93
|
+
def test_normalize_list_unchanged(self):
|
|
94
|
+
"""Test that list values remain unchanged."""
|
|
95
|
+
ask_values = {"ask1": ["val1", "val2"]}
|
|
96
|
+
result = _normalize_ask_values(ask_values)
|
|
97
|
+
|
|
98
|
+
assert result == {"ask1": ["val1", "val2"]}
|
|
99
|
+
|
|
100
|
+
def test_normalize_mixed_values(self):
|
|
101
|
+
"""Test normalization with mixed string and list values."""
|
|
102
|
+
ask_values = {"ask1": "single", "ask2": ["multi1", "multi2"]}
|
|
103
|
+
result = _normalize_ask_values(ask_values)
|
|
104
|
+
|
|
105
|
+
assert result == {"ask1": ["single"], "ask2": ["multi1", "multi2"]}
|
|
106
|
+
|
|
107
|
+
def test_normalize_empty_dict(self):
|
|
108
|
+
"""Test normalization with empty dict."""
|
|
109
|
+
ask_values = {}
|
|
110
|
+
result = _normalize_ask_values(ask_values)
|
|
111
|
+
|
|
112
|
+
assert result == {}
|
|
113
|
+
|
|
114
|
+
def test_normalize_preserves_order(self):
|
|
115
|
+
"""Test that normalization preserves dict key order."""
|
|
116
|
+
ask_values = {"ask3": "val3", "ask1": "val1", "ask2": "val2"}
|
|
117
|
+
result = _normalize_ask_values(ask_values)
|
|
118
|
+
|
|
119
|
+
assert list(result.keys()) == ["ask3", "ask1", "ask2"]
|
|
120
|
+
|
|
121
|
+
|
|
22
122
|
class TestReplaceAskPlaceholders:
|
|
23
123
|
"""Tests for _replace_ask_placeholders function."""
|
|
24
124
|
|
|
@@ -33,7 +133,7 @@ class TestReplaceAskPlaceholders:
|
|
|
33
133
|
assert result == "{'25'.EX.'abc123'}"
|
|
34
134
|
|
|
35
135
|
def test_replace_multiple_placeholders(self, sample_report_configs):
|
|
36
|
-
"""Test replacing multiple
|
|
136
|
+
"""Test replacing multiple different placeholders."""
|
|
37
137
|
config = sample_report_configs[0]
|
|
38
138
|
filter_str = "({'25'.EX.'_ask1_'}AND{'40'.EX.'_ask2_'})"
|
|
39
139
|
ask_values = {"ask1": "value1", "ask2": "value2"}
|
|
@@ -42,6 +142,84 @@ class TestReplaceAskPlaceholders:
|
|
|
42
142
|
|
|
43
143
|
assert result == "({'25'.EX.'value1'}AND{'40'.EX.'value2'})"
|
|
44
144
|
|
|
145
|
+
def test_replace_placeholder_with_list_values(self, sample_report_configs):
|
|
146
|
+
"""Test replacing placeholder with multiple values creates OR condition."""
|
|
147
|
+
config = sample_report_configs[0]
|
|
148
|
+
filter_str = "{'25'.EX.'_ask1_'}"
|
|
149
|
+
ask_values = {"ask1": ["val1", "val2", "val3"]}
|
|
150
|
+
|
|
151
|
+
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
152
|
+
|
|
153
|
+
expected = "({'25'.EX.'val1'}OR{'25'.EX.'val2'}OR{'25'.EX.'val3'})"
|
|
154
|
+
assert result == expected
|
|
155
|
+
|
|
156
|
+
def test_replace_list_with_single_element(self, sample_report_configs):
|
|
157
|
+
"""Test that list with single element is treated as single value."""
|
|
158
|
+
config = sample_report_configs[0]
|
|
159
|
+
filter_str = "{'25'.EX.'_ask1_'}"
|
|
160
|
+
ask_values = {"ask1": ["only_value"]}
|
|
161
|
+
|
|
162
|
+
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
163
|
+
|
|
164
|
+
# Single element list should not create OR
|
|
165
|
+
assert result == "{'25'.EX.'only_value'}"
|
|
166
|
+
|
|
167
|
+
def test_empty_list_raises_error(self, sample_report_configs):
|
|
168
|
+
"""Test that empty list in ask_values raises ValueError."""
|
|
169
|
+
config = sample_report_configs[0]
|
|
170
|
+
filter_str = "{'25'.EX.'_ask1_'}"
|
|
171
|
+
ask_values = {"ask1": []}
|
|
172
|
+
|
|
173
|
+
with pytest.raises(ValueError, match="empty lists"):
|
|
174
|
+
_replace_ask_placeholders(filter_str, ask_values, config)
|
|
175
|
+
|
|
176
|
+
def test_mixed_string_and_list_values(self, sample_report_configs):
|
|
177
|
+
"""Test replacing multiple placeholders with mixed string and list values."""
|
|
178
|
+
config = sample_report_configs[0]
|
|
179
|
+
filter_str = "({'25'.EX.'_ask1_'}AND{'40'.EX.'_ask2_'})"
|
|
180
|
+
ask_values = {"ask1": "single_value", "ask2": ["val1", "val2"]}
|
|
181
|
+
|
|
182
|
+
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
183
|
+
|
|
184
|
+
# ask1 should be simple replacement, ask2 should have OR
|
|
185
|
+
assert "{'25'.EX.'single_value'}" in result
|
|
186
|
+
assert "({'40'.EX.'val1'}OR{'40'.EX.'val2'})" in result
|
|
187
|
+
|
|
188
|
+
def test_list_values_in_complex_filter(self, sample_report_configs):
|
|
189
|
+
"""Test list value expansion maintains filter logic in complex expressions."""
|
|
190
|
+
config = sample_report_configs[0]
|
|
191
|
+
filter_str = "({'15'.EX.'Pending'}AND{'41'.EX.'_ask1_'})"
|
|
192
|
+
ask_values = {"ask1": ["urgent", "high"]}
|
|
193
|
+
|
|
194
|
+
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
195
|
+
|
|
196
|
+
# Should preserve outer AND structure with inner OR
|
|
197
|
+
assert "{'15'.EX.'Pending'}" in result
|
|
198
|
+
assert "({'41'.EX.'urgent'}OR{'41'.EX.'high'})" in result
|
|
199
|
+
|
|
200
|
+
def test_multiple_condition_blocks_same_placeholder(self, sample_report_configs):
|
|
201
|
+
"""Test replacing multiple condition blocks with same placeholder."""
|
|
202
|
+
config = sample_report_configs[0]
|
|
203
|
+
filter_str = "({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'})"
|
|
204
|
+
ask_values = {"ask1": "urgent"}
|
|
205
|
+
|
|
206
|
+
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
207
|
+
|
|
208
|
+
expected = "({'41'.EX.'urgent'}OR{'40'.EX.'urgent'})"
|
|
209
|
+
assert result == expected
|
|
210
|
+
|
|
211
|
+
def test_multiple_blocks_same_placeholder_with_list_values(self, sample_report_configs):
|
|
212
|
+
"""Test replacing multiple condition blocks with same placeholder using list values."""
|
|
213
|
+
config = sample_report_configs[0]
|
|
214
|
+
filter_str = "({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'})"
|
|
215
|
+
ask_values = {"ask1": ["a", "b"]}
|
|
216
|
+
|
|
217
|
+
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
218
|
+
|
|
219
|
+
# Each condition block should be expanded
|
|
220
|
+
assert "({'41'.EX.'a'}OR{'41'.EX.'b'})" in result
|
|
221
|
+
assert "({'40'.EX.'a'}OR{'40'.EX.'b'})" in result
|
|
222
|
+
|
|
45
223
|
def test_no_placeholders_returns_unchanged(self, sample_report_configs):
|
|
46
224
|
"""Test that filter without placeholders is returned unchanged."""
|
|
47
225
|
config = sample_report_configs[0]
|
|
@@ -58,7 +236,7 @@ class TestReplaceAskPlaceholders:
|
|
|
58
236
|
filter_str = "{'25'.EX.'_ask1_'}AND{'40'.EX.'_ask2_'}"
|
|
59
237
|
ask_values = {"ask1": "value1"} # Missing ask2
|
|
60
238
|
|
|
61
|
-
with pytest.raises(ValueError, match="requires values for.*_ask2_"):
|
|
239
|
+
with pytest.raises(ValueError, match=r"requires values for.*_ask2_"):
|
|
62
240
|
_replace_ask_placeholders(filter_str, ask_values, config)
|
|
63
241
|
|
|
64
242
|
def test_unused_placeholder_value_raises_error(self, sample_report_configs):
|
|
@@ -67,7 +245,7 @@ class TestReplaceAskPlaceholders:
|
|
|
67
245
|
filter_str = "{'25'.EX.'_ask1_'}"
|
|
68
246
|
ask_values = {"ask1": "value1", "ask2": "value2"} # ask2 not in filter
|
|
69
247
|
|
|
70
|
-
with pytest.raises(ValueError, match="received ask_values.*ask2.*not used"):
|
|
248
|
+
with pytest.raises(ValueError, match=r"received ask_values.*ask2.*not used"):
|
|
71
249
|
_replace_ask_placeholders(filter_str, ask_values, config)
|
|
72
250
|
|
|
73
251
|
def test_placeholder_with_special_characters(self, sample_report_configs):
|
|
@@ -81,14 +259,28 @@ class TestReplaceAskPlaceholders:
|
|
|
81
259
|
assert result == "{'25'.EX.'value with spaces & symbols!'}"
|
|
82
260
|
|
|
83
261
|
def test_complex_filter_with_mixed_content(self, sample_report_configs):
|
|
84
|
-
"""Test replacing placeholders in complex filter."""
|
|
262
|
+
"""Test replacing placeholders in complex filter with both fixed and dynamic values."""
|
|
85
263
|
config = sample_report_configs[0]
|
|
86
264
|
filter_str = "({'15'.EX.'Pending'}AND({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'}))"
|
|
87
265
|
ask_values = {"ask1": "urgent"}
|
|
88
266
|
|
|
89
267
|
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
90
268
|
|
|
91
|
-
|
|
269
|
+
expected = "({'15'.EX.'Pending'}AND({'41'.EX.'urgent'}OR{'40'.EX.'urgent'}))"
|
|
270
|
+
assert result == expected
|
|
271
|
+
|
|
272
|
+
def test_complex_filter_with_mixed_content_and_list(self, sample_report_configs):
|
|
273
|
+
"""Test complex filter with multiple condition blocks and list values."""
|
|
274
|
+
config = sample_report_configs[0]
|
|
275
|
+
filter_str = "({'15'.EX.'Pending'}AND({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'}))"
|
|
276
|
+
ask_values = {"ask1": ["urgent", "high"]}
|
|
277
|
+
|
|
278
|
+
result = _replace_ask_placeholders(filter_str, ask_values, config)
|
|
279
|
+
|
|
280
|
+
# Should have outer structure preserved and inner ORs expanded
|
|
281
|
+
assert "{'15'.EX.'Pending'}" in result
|
|
282
|
+
assert "({'41'.EX.'urgent'}OR{'41'.EX.'high'})" in result
|
|
283
|
+
assert "({'40'.EX.'urgent'}OR{'40'.EX.'high'})" in result
|
|
92
284
|
|
|
93
285
|
|
|
94
286
|
class TestExtractReportNames:
|
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
import json
|
|
4
4
|
|
|
5
5
|
import pytest
|
|
6
|
+
|
|
6
7
|
from quickbase_extract.cache_manager import CacheManager
|
|
7
8
|
from quickbase_extract.config import ReportConfig
|
|
8
9
|
from quickbase_extract.report_metadata import (
|
|
9
10
|
fetch_report_metadata_api,
|
|
11
|
+
filter_metadata_by_table,
|
|
10
12
|
get_report_metadata,
|
|
11
13
|
get_report_metadata_parallel,
|
|
12
14
|
load_report_metadata,
|
|
@@ -70,7 +72,7 @@ class TestFetchReportMetadataApi:
|
|
|
70
72
|
{"id": "rptABC", "name": "Default"},
|
|
71
73
|
]
|
|
72
74
|
|
|
73
|
-
with pytest.raises(ValueError, match="Report .* not found"):
|
|
75
|
+
with pytest.raises(ValueError, match=r"Report .* not found"):
|
|
74
76
|
fetch_report_metadata_api(
|
|
75
77
|
mock_qb_api,
|
|
76
78
|
"appXYZ123",
|
|
@@ -166,7 +168,7 @@ class TestGetReportMetadata:
|
|
|
166
168
|
report_name="Nonexistent",
|
|
167
169
|
)
|
|
168
170
|
|
|
169
|
-
with pytest.raises(ValueError, match="Report .* not found"):
|
|
171
|
+
with pytest.raises(ValueError, match=r"Report .* not found"):
|
|
170
172
|
get_report_metadata(
|
|
171
173
|
mock_qb_api,
|
|
172
174
|
cache_mgr,
|
|
@@ -298,10 +300,12 @@ class TestLoadReportMetadata:
|
|
|
298
300
|
# Now load it
|
|
299
301
|
metadata = load_report_metadata(cache_mgr, config)
|
|
300
302
|
|
|
301
|
-
|
|
302
|
-
assert
|
|
303
|
-
assert "
|
|
304
|
-
assert "
|
|
303
|
+
# Metadata is now dict[ReportConfig, dict]
|
|
304
|
+
assert config in metadata
|
|
305
|
+
assert metadata[config]["table_id"] == "tblXYZ123"
|
|
306
|
+
assert metadata[config]["table_name"] == "test_table"
|
|
307
|
+
assert "fields" in metadata[config]
|
|
308
|
+
assert "filter" in metadata[config]
|
|
305
309
|
|
|
306
310
|
def test_load_nonexistent_metadata(self, temp_cache_dir, sample_report_configs):
|
|
307
311
|
"""Test error when loading non-cached metadata."""
|
|
@@ -333,7 +337,8 @@ class TestLoadReportMetadata:
|
|
|
333
337
|
|
|
334
338
|
# Should be able to load with original config
|
|
335
339
|
metadata = load_report_metadata(cache_mgr, config)
|
|
336
|
-
assert
|
|
340
|
+
assert config in metadata
|
|
341
|
+
assert metadata[config] is not None
|
|
337
342
|
|
|
338
343
|
|
|
339
344
|
class TestLoadReportMetadataBatch:
|
|
@@ -392,7 +397,48 @@ class TestLoadReportMetadataBatch:
|
|
|
392
397
|
# Should be able to look up by config
|
|
393
398
|
config1 = sample_report_configs[0]
|
|
394
399
|
assert all_metadata[config1]["table_id"] == "tblXYZ123"
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
class TestFilterMetadataByTable:
|
|
403
|
+
"""Tests for filter_metadata_by_table function."""
|
|
404
|
+
|
|
405
|
+
def test_filter_metadata_by_table_unique(self, sample_report_metadata, sample_report_configs):
|
|
406
|
+
"""Test retrieving metadata for a unique table."""
|
|
407
|
+
|
|
408
|
+
table_name = sample_report_configs[0].table_name
|
|
409
|
+
result = filter_metadata_by_table(sample_report_metadata, table_name)
|
|
410
|
+
|
|
411
|
+
assert result["table_name"] == "test_table"
|
|
412
|
+
assert result["table_id"] == "tblXYZ123"
|
|
413
|
+
|
|
414
|
+
def test_get_metadata_by_table_with_app_name(self, sample_report_metadata, sample_report_configs):
|
|
415
|
+
"""Test retrieving metadata filtering by both app and table."""
|
|
416
|
+
|
|
417
|
+
config = sample_report_configs[0]
|
|
418
|
+
result = filter_metadata_by_table(sample_report_metadata, config.table_name, app_name=config.app_name)
|
|
419
|
+
|
|
420
|
+
assert result["table_name"] == "test_table"
|
|
421
|
+
assert result["app_name"] == "test_app"
|
|
422
|
+
|
|
423
|
+
def test_get_metadata_by_table_not_found(self, sample_report_metadata):
|
|
424
|
+
"""Test error when table not found."""
|
|
425
|
+
|
|
426
|
+
with pytest.raises(ValueError, match="No metadata found"):
|
|
427
|
+
filter_metadata_by_table(sample_report_metadata, "Nonexistent Table")
|
|
428
|
+
|
|
429
|
+
def test_get_metadata_by_table_ambiguous_without_app(self, sample_report_metadata):
|
|
430
|
+
"""Test error when table exists in multiple apps without app_name specified."""
|
|
431
|
+
|
|
432
|
+
# Create a second metadata entry with same table name but different app
|
|
433
|
+
# This would require a fixture with duplicate table names across apps
|
|
434
|
+
# For now, this documents expected behavior
|
|
435
|
+
pass
|
|
436
|
+
|
|
437
|
+
def test_get_metadata_by_table_ambiguous_with_app(self, sample_report_metadata, sample_report_configs):
|
|
438
|
+
"""Test successful lookup when table is ambiguous but app_name is provided."""
|
|
439
|
+
|
|
440
|
+
config = sample_report_configs[0]
|
|
441
|
+
# Should not raise error because app_name disambiguates
|
|
442
|
+
result = filter_metadata_by_table(sample_report_metadata, config.table_name, app_name=config.app_name)
|
|
443
|
+
|
|
444
|
+
assert result is not None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|