quickbase-extract 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/.pre-commit-config.yaml +5 -6
  2. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/CHANGELOG.md +30 -0
  3. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/PKG-INFO +1 -2
  4. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/pyproject.toml +2 -3
  5. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/__init__.py +18 -32
  6. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/api_handlers.py +5 -7
  7. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/cache_manager.py +1 -1
  8. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/cache_orchestration.py +15 -2
  9. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/report_data.py +117 -34
  10. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/report_metadata.py +50 -0
  11. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/test_cache_orchestration.py +1 -0
  12. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/test_cache_sync.py +6 -1
  13. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/test_report_data.py +196 -5
  14. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/test_report_metadata.py +48 -6
  15. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/.editorconfig +0 -0
  16. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/.gitignore +0 -0
  17. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/.python-version +0 -0
  18. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/LICENSE.txt +0 -0
  19. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/README.md +0 -0
  20. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/TODO.md +0 -0
  21. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/cache_sync.py +0 -0
  22. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/config.py +0 -0
  23. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/py.typed +0 -0
  24. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/src/quickbase_extract/utils.py +0 -0
  25. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/conftest.py +0 -0
  26. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/test_api_handlers.py +0 -0
  27. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/test_cache_manager.py +0 -0
  28. {quickbase_extract-0.3.0 → quickbase_extract-0.4.0}/tests/test_utils.py +0 -0
@@ -8,14 +8,13 @@ repos:
8
8
  - id: check-yaml
9
9
  - id: debug-statements # Prevents accidental pdb/breakpoint commits
10
10
 
11
- - repo: https://github.com/psf/black
12
- rev: 23.12.1
13
- hooks:
14
- - id: black
15
- language_version: python3.12
16
-
17
11
  - repo: https://github.com/astral-sh/ruff-pre-commit
18
12
  rev: v0.1.11
19
13
  hooks:
20
14
  - id: ruff
21
15
  args: [--fix]
16
+ stages: [pre-commit]
17
+ - id: ruff-format
18
+ stages: [pre-commit]
19
+ - id: ruff # Run again to verify no more changes
20
+ stages: [pre-commit]
@@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [0.4.0] - 2026-04-29
9
+
10
+ ### Added
11
+
12
+ - Support for multiple values in `ask_values` parameters - values can now be strings or lists of strings
13
+ - Automatic OR condition expansion: list values like `{"ask1": ["val1", "val2", "val3"]}` expand to `({'25'.EX.'val1'}OR{'25'.EX.'val2'}OR{'25'.EX.'val3'})`
14
+ - Proper grouping with parentheses when multiple values are used to preserve filter logic precedence
15
+ - Validation for empty lists in `ask_values` - raises `ValueError` if empty list provided
16
+ - `filter_metadata_by_table()` helper function for convenient report metadata lookup by table name, with optional app name filtering to resolve ambiguous table names across apps
17
+ - Helper functions `_validate_ask_values()` and `_normalize_ask_values()` for cleaner separation of concerns in placeholder replacement
18
+
19
+ ### Changed
20
+
21
+ - `ask_values` type hint updated from `dict[str, str]` to `dict[str, str | list[str]]` in `get_data()` and `get_data_parallel()`
22
+ - `_replace_ask_placeholders()` refactored to use helper functions for validation and normalization, improving testability and maintainability
23
+ - Filter replacement now extracts and replaces full condition blocks (e.g., `{'25'.EX.'_ask1_'}`) instead of just placeholders
24
+ - Placeholder replacement now processes all occurrences using `re.finditer()` instead of only the first match
25
+
26
+ ### Fixed
27
+
28
+ - Complex filters with multiple conditions now maintain proper precedence when list values are expanded
29
+ - **BREAKING FIX**: Multiple condition blocks with the same placeholder are now all replaced correctly (previously only first occurrence was replaced)
30
+ - String position preservation during replacement to handle complex multi-placeholder filters correctly
31
+
32
+ ## [0.3.1] - 2026-04-27
33
+
34
+ ### Fixed
35
+
36
+ - `complete_cache_refresh()` function now properly exported from `quickbase_extract` package for use in Lambda handlers
37
+
8
38
  ## [0.3.0] - 2026-04-27
9
39
 
10
40
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quickbase-extract
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Extract and cache Quickbase report data with built-in error handling and S3 support
5
5
  Project-URL: Homepage, https://github.com/tbrezler/quickbase-extract
6
6
  Project-URL: Repository, https://github.com/tbrezler/quickbase-extract.git
@@ -19,7 +19,6 @@ Requires-Python: >=3.12
19
19
  Requires-Dist: boto3>=1.26.0
20
20
  Requires-Dist: quickbase-api>=0.3.1
21
21
  Provides-Extra: dev
22
- Requires-Dist: black>=23.0; extra == 'dev'
23
22
  Requires-Dist: pytest-cov>=4.0; extra == 'dev'
24
23
  Requires-Dist: pytest>=7.0; extra == 'dev'
25
24
  Requires-Dist: ruff>=0.1.0; extra == 'dev'
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "quickbase-extract"
7
- version = "0.3.0"
7
+ version = "0.4.0"
8
8
  description = "Extract and cache Quickbase report data with built-in error handling and S3 support"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.12"
@@ -31,7 +31,6 @@ dependencies = [
31
31
  dev = [
32
32
  "pytest>=7.0",
33
33
  "pytest-cov>=4.0",
34
- "black>=23.0",
35
34
  "ruff>=0.1.0",
36
35
  ]
37
36
 
@@ -49,5 +48,5 @@ line-length = 120
49
48
  target-version = "py312"
50
49
 
51
50
  [tool.ruff.lint]
52
- select = ["E", "F", "W", "I", "UP", "N", "C4", "BLE", "A"]
51
+ select = ["F", "E", "W", "B", "BLE", "I", "UP", "A", "RUF", "T10"]
53
52
  ignore = ["E501"]
@@ -28,12 +28,7 @@ import logging
28
28
  from importlib.metadata import version
29
29
 
30
30
  # API operations with error handling
31
- from quickbase_extract.api_handlers import (
32
- QuickbaseOperationError,
33
- handle_delete,
34
- handle_query,
35
- handle_upsert,
36
- )
31
+ from quickbase_extract.api_handlers import QuickbaseOperationError, handle_delete, handle_query, handle_upsert
37
32
 
38
33
  # Cache management
39
34
  from quickbase_extract.cache_manager import CacheManager
@@ -42,21 +37,17 @@ from quickbase_extract.cache_manager import CacheManager
42
37
  from quickbase_extract.cache_orchestration import ensure_cache_freshness
43
38
 
44
39
  # Cache sync
45
- from quickbase_extract.cache_sync import is_cache_synced, sync_from_s3_once
40
+ from quickbase_extract.cache_sync import complete_cache_refresh, is_cache_synced, sync_from_s3_once
46
41
 
47
42
  # Config
48
43
  from quickbase_extract.config import ReportConfig
49
44
 
50
- # Report data retrieval
51
- from quickbase_extract.report_data import (
52
- get_data,
53
- get_data_parallel,
54
- load_data,
55
- load_data_batch,
56
- )
45
+ # Report data
46
+ from quickbase_extract.report_data import get_data, get_data_parallel, load_data, load_data_batch
57
47
 
58
48
  # Report metadata
59
49
  from quickbase_extract.report_metadata import (
50
+ filter_metadata_by_table,
60
51
  get_report_metadata,
61
52
  get_report_metadata_parallel,
62
53
  load_report_metadata,
@@ -72,30 +63,25 @@ __version__ = version("quickbase-extract")
72
63
  logging.getLogger(__name__).addHandler(logging.NullHandler())
73
64
 
74
65
  __all__ = [
75
- # Version
76
- "__version__",
77
- # Cache management
78
66
  "CacheManager",
79
- "ensure_cache_freshness",
80
- "sync_from_s3_once",
81
- "is_cache_synced",
82
- # Config
83
- "ReportConfig",
84
- # API operations
85
67
  "QuickbaseOperationError",
68
+ "ReportConfig",
69
+ "__version__",
70
+ "complete_cache_refresh",
71
+ "ensure_cache_freshness",
72
+ "filter_metadata_by_table",
73
+ "get_data",
74
+ "get_data_parallel",
75
+ "get_report_metadata",
76
+ "get_report_metadata_parallel",
86
77
  "handle_delete",
87
78
  "handle_query",
88
79
  "handle_upsert",
89
- # Report metadata
90
- "get_report_metadata",
91
- "get_report_metadata_parallel",
92
- "load_report_metadata",
93
- "load_report_metadata_batch",
94
- # Report data
95
- "get_data",
96
- "get_data_parallel",
80
+ "is_cache_synced",
97
81
  "load_data",
98
82
  "load_data_batch",
99
- # Utilities
83
+ "load_report_metadata",
84
+ "load_report_metadata_batch",
100
85
  "normalize_name",
86
+ "sync_from_s3_once",
101
87
  ]
@@ -4,8 +4,6 @@ Provides retry logic for rate-limited requests, standardized error handling,
4
4
  and logging for Quickbase API operations.
5
5
  """
6
6
 
7
- # ruff: noqa: BLE001
8
-
9
7
  import logging
10
8
  import random
11
9
  import time
@@ -139,11 +137,11 @@ def handle_query(
139
137
  client,
140
138
  table_id: str,
141
139
  *,
142
- select: list[int] = None,
143
- where: str = None,
144
- sort_by: list[dict] = None,
145
- group_by: list[dict] = None,
146
- options: dict = None,
140
+ select: list[int] | None = None,
141
+ where: str | None = None,
142
+ sort_by: list[dict] | None = None,
143
+ group_by: list[dict] | None = None,
144
+ options: dict | None = None,
147
145
  description: str = "",
148
146
  max_retries: int = 3,
149
147
  ) -> dict:
@@ -308,7 +308,7 @@ class CacheManager:
308
308
  return 0
309
309
 
310
310
  oldest_mtime = min(f.stat().st_mtime for f in json_files)
311
- # 60 sec × 60 min = 3600
311
+ # 60 sec * 60 min = 3600
312
312
  age_hours = (time.time() - oldest_mtime) / 3600
313
313
 
314
314
  return round(age_hours, 1)
@@ -123,6 +123,7 @@ def _refresh_data_cache(
123
123
  cache_manager: CacheManager,
124
124
  reports_to_refresh: list[ReportConfig],
125
125
  reasons: list[str],
126
+ ask_values: dict[ReportConfig, dict[str, str | list[str]] | None] | None = None,
126
127
  ) -> None:
127
128
  """Refresh data cache for specified reports.
128
129
 
@@ -131,6 +132,8 @@ def _refresh_data_cache(
131
132
  cache_manager: CacheManager instance.
132
133
  reports_to_refresh: Reports to refresh data for.
133
134
  reasons: List of reasons for refresh (for logging).
135
+ ask_values: Optional dict mapping ReportConfig -> ask_values dict.
136
+ Per-report "ask the user" filter values.
134
137
 
135
138
  Raises:
136
139
  CacheRefreshError: If data refresh fails.
@@ -148,6 +151,7 @@ def _refresh_data_cache(
148
151
  report_configs=reports_to_refresh,
149
152
  report_metadata=metadata,
150
153
  cache=True,
154
+ ask_values=ask_values,
151
155
  )
152
156
  logger.info("Data cache refresh completed successfully")
153
157
  except Exception as e:
@@ -160,6 +164,7 @@ def ensure_cache_freshness(
160
164
  cache_manager: CacheManager,
161
165
  report_configs_all: list[ReportConfig],
162
166
  report_configs_to_cache: list[ReportConfig] | None = None,
167
+ ask_values: dict[ReportConfig, dict[str, str | list[str]] | None] | None = None,
163
168
  metadata_stale_hours: float | None = None,
164
169
  data_stale_hours: float | None = None,
165
170
  cache_all_data: bool = False,
@@ -184,6 +189,12 @@ def ensure_cache_freshness(
184
189
  report_configs_to_cache: Optional subset of ReportConfig instances to
185
190
  cache data for. If cache_all_data is True, this parameter is ignored
186
191
  and all reports' data is cached instead.
192
+ ask_values: Optional dict mapping ReportConfig -> ask_values dict.
193
+ Per-report "ask the user" filter values. Only used when refreshing
194
+ data cache. Example: {
195
+ ReportConfig("bq8x", "Accounts", "Python"): {"ask1": "abc"},
196
+ ReportConfig("bq9y", "Contacts", "Active"): {"ask1": "def"}
197
+ }
187
198
  metadata_stale_hours: Threshold (hours) for metadata staleness.
188
199
  If not provided, reads from METADATA_STALE_HOURS env var,
189
200
  falls back to DEFAULT_METADATA_STALE_HOURS (168 hours / 7 days).
@@ -221,12 +232,14 @@ def ensure_cache_freshness(
221
232
  ... report_configs_to_cache=get_reports_to_cache(),
222
233
  ... )
223
234
  >>>
224
- >>> # Cache all reports' data
235
+ >>> # Cache all reports' data with ask_values
236
+ >>> ask_vals = {get_all_reports()[0]: {"ask1": "value1"}}
225
237
  >>> ensure_cache_freshness(
226
238
  ... client=client,
227
239
  ... cache_manager=cache_manager,
228
240
  ... report_configs_all=get_all_reports(),
229
241
  ... cache_all_data=True,
242
+ ... ask_values=ask_vals,
230
243
  ... )
231
244
  """
232
245
  # Resolve thresholds from arguments, environment, or defaults
@@ -295,4 +308,4 @@ def ensure_cache_freshness(
295
308
 
296
309
  # Refresh data if needed
297
310
  if data_needs_refresh:
298
- _refresh_data_cache(client, cache_manager, reports_to_refresh_data, data_reasons)
311
+ _refresh_data_cache(client, cache_manager, reports_to_refresh_data, reasons=data_reasons, ask_values=ask_values)
@@ -12,70 +12,153 @@ from quickbase_extract.config import ReportConfig
12
12
  logger = logging.getLogger(__name__)
13
13
 
14
14
 
15
+ def _validate_ask_values(
16
+ ask_values: dict[str, str | list[str]],
17
+ placeholders_in_filter: set[str],
18
+ report_config: ReportConfig,
19
+ ) -> None:
20
+ """Validate that ask_values matches placeholders in filter.
21
+
22
+ Ensures that:
23
+ - All placeholders in the filter have corresponding values in ask_values
24
+ - All values in ask_values are actually used in the filter
25
+ - No empty lists are provided
26
+
27
+ Args:
28
+ ask_values: Dict mapping placeholder keys (e.g., "ask1") to values or lists of values.
29
+ placeholders_in_filter: Set of placeholder strings found in filter (e.g., {"_ask1_", "_ask2_"}).
30
+ report_config: ReportConfig for error messages.
31
+
32
+ Raises:
33
+ ValueError: If values are missing, unused, or if any list is empty.
34
+ """
35
+ # Check for missing values
36
+ missing_values = [p for p in placeholders_in_filter if p.strip("_") not in ask_values]
37
+ if missing_values:
38
+ raise ValueError(
39
+ f"Report {report_config} filter requires values for {missing_values}, "
40
+ f"but they were not provided in ask_values."
41
+ )
42
+
43
+ # Check for unused values
44
+ unused_values = [k for k in ask_values.keys() if f"_{k}_" not in placeholders_in_filter]
45
+ if unused_values:
46
+ raise ValueError(
47
+ f"Report {report_config} received ask_values {unused_values} "
48
+ f"that are not used in the filter. Available placeholders: {list(placeholders_in_filter)}"
49
+ )
50
+
51
+ # Check for empty lists
52
+ empty_list_keys = [k for k, v in ask_values.items() if isinstance(v, list) and len(v) == 0]
53
+ if empty_list_keys:
54
+ raise ValueError(
55
+ f"Report {report_config} received empty lists for ask_values keys: {empty_list_keys}. "
56
+ f"Empty lists are not allowed."
57
+ )
58
+
59
+
60
+ def _normalize_ask_values(ask_values: dict[str, str | list[str]]) -> dict[str, list[str]]:
61
+ """Normalize all ask_values to lists for consistent processing.
62
+
63
+ Converts single string values to single-element lists, leaving list values
64
+ unchanged. This allows downstream logic to handle all values uniformly.
65
+
66
+ Args:
67
+ ask_values: Dict with values that are either strings or lists of strings.
68
+
69
+ Returns:
70
+ Dict with all values as lists of strings.
71
+
72
+ Example:
73
+ >>> _normalize_ask_values({"ask1": "value", "ask2": ["a", "b"]})
74
+ {"ask1": ["value"], "ask2": ["a", "b"]}
75
+ """
76
+ normalized = {}
77
+ for key, value in ask_values.items():
78
+ normalized[key] = [value] if isinstance(value, str) else value
79
+ return normalized
80
+
81
+
15
82
  def _replace_ask_placeholders(
16
83
  report_filter: str,
17
- ask_values: dict[str, str],
84
+ ask_values: dict[str, str | list[str]],
18
85
  report_config: ReportConfig,
19
86
  ) -> str:
20
87
  """Replace ask-the-user placeholders in a Quickbase filter with actual values.
21
88
 
89
+ Finds all placeholders (e.g., _ask1_, _ask2_) in the filter and replaces them
90
+ with provided values. Single values are replaced directly, while multiple values
91
+ in a list are expanded into OR conditions with proper grouping to preserve
92
+ filter logic precedence.
93
+
22
94
  Args:
23
95
  report_filter: The filter string from report metadata (e.g., "{'25'.EX.'_ask1_'}").
24
- ask_values: Dict mapping placeholder keys to values (e.g., {"ask1": "abc123"}).
96
+ ask_values: Dict mapping placeholder keys to values or lists of values.
97
+ Keys like "ask1", "ask2". Values can be strings or lists of strings.
98
+ Example: {"ask1": "abc123"} or {"ask1": ["abc123", "abc456"]}
25
99
  report_config: ReportConfig for error messages.
26
100
 
27
101
  Returns:
28
102
  Modified filter string with placeholders replaced.
29
103
 
30
104
  Raises:
31
- ValueError: If required placeholders are missing values or unused values provided.
105
+ ValueError: If required placeholders are missing values, unused values provided,
106
+ empty lists provided, or condition blocks cannot be found.
32
107
 
33
108
  Example:
34
109
  >>> filter_str = "{'25'.EX.'_ask1_'}AND{'10'.AF.'today'}"
35
110
  >>> config = ReportConfig("bq8xyx9z", "Accounts", "Python")
111
+ >>> # Single value
36
112
  >>> _replace_ask_placeholders(filter_str, {"ask1": "abc123"}, config)
37
113
  "{'25'.EX.'abc123'}AND{'10'.AF.'today'}"
114
+ >>> # Multiple values
115
+ >>> _replace_ask_placeholders(filter_str, {"ask1": ["val1", "val2"]}, config)
116
+ "(({'25'.EX.'val1'}OR{'25'.EX.'val2'}))AND{'10'.AF.'today'}"
117
+ >>> # Multiple condition blocks with same placeholder
118
+ >>> _replace_ask_placeholders(
119
+ ... "({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'})",
120
+ ... {"ask1": ["a", "b"]},
121
+ ... config
122
+ ... )
123
+ "(({'41'.EX.'a'}OR{'41'.EX.'b'})OR({'40'.EX.'a'}OR{'40'.EX.'b'}))"
38
124
  """
39
- # Find all placeholders in the filter (e.g., _ask1_, _ask2_)
40
125
  placeholders_in_filter = set(re.findall(r"_ask\d+_", report_filter))
41
126
 
42
127
  if not placeholders_in_filter:
43
- # No placeholders found - nothing to replace
44
128
  return report_filter
45
129
 
46
- # Validate: all placeholders in filter must have corresponding values
47
- missing_values = []
130
+ # Validate and normalize
131
+ _validate_ask_values(ask_values, placeholders_in_filter, report_config)
132
+ normalized_values = _normalize_ask_values(ask_values)
133
+
134
+ # Replace placeholders
135
+ modified_filter = report_filter
48
136
  for placeholder in placeholders_in_filter:
49
- # Convert _ask1_ to ask1 for lookup
50
137
  key = placeholder.strip("_")
51
- if key not in ask_values:
52
- missing_values.append(placeholder)
138
+ values = normalized_values[key]
53
139
 
54
- if missing_values:
55
- raise ValueError(
56
- f"Report {report_config} filter requires values for {missing_values}, "
57
- f"but they were not provided in ask_values."
58
- )
140
+ # Find ALL condition blocks containing this placeholder
141
+ # Pattern: {...'_ask1_'...} where we capture the entire {...}
142
+ condition_pattern = rf"(\{{[^}}]*{re.escape(placeholder)}[^{{]*\}})"
143
+ matches = list(re.finditer(condition_pattern, modified_filter))
59
144
 
60
- # Validate: all provided values must be used in filter
61
- unused_values = []
62
- for key in ask_values.keys():
63
- placeholder = f"_{key}_"
64
- if placeholder not in placeholders_in_filter:
65
- unused_values.append(key)
145
+ if not matches:
146
+ raise ValueError(f"Report {report_config} could not find condition block for placeholder {placeholder}")
66
147
 
67
- if unused_values:
68
- raise ValueError(
69
- f"Report {report_config} received ask_values {unused_values} "
70
- f"that are not used in the filter. Available placeholders: {list(placeholders_in_filter)}"
71
- )
148
+ # Process matches in reverse order to maintain string positions during replacement
149
+ for match in reversed(matches):
150
+ original_condition = match.group(1)
72
151
 
73
- # Replace placeholders with actual values
74
- modified_filter = report_filter
75
- for placeholder in placeholders_in_filter:
76
- key = placeholder.strip("_")
77
- value = ask_values[key]
78
- modified_filter = modified_filter.replace(placeholder, value)
152
+ if len(values) == 1:
153
+ # Single value: simple replacement
154
+ new_condition = original_condition.replace(placeholder, values[0])
155
+ else:
156
+ # Multiple values: replicate condition and join with OR
157
+ replicated_conditions = [original_condition.replace(placeholder, v) for v in values]
158
+ new_condition = f"({('OR'.join(replicated_conditions))})"
159
+
160
+ # Replace using string slicing to preserve positions for other replacements
161
+ modified_filter = modified_filter[: match.start()] + new_condition + modified_filter[match.end() :]
79
162
 
80
163
  return modified_filter
81
164
 
@@ -132,7 +215,7 @@ def get_data(
132
215
  report_config: ReportConfig,
133
216
  report_metadata: dict[ReportConfig, dict],
134
217
  cache: bool = False,
135
- ask_values: dict[str, str] | None = None,
218
+ ask_values: dict[str, str | list[str]] | None = None,
136
219
  ) -> list[dict]:
137
220
  """Query a Quickbase table for data using cached report metadata.
138
221
 
@@ -217,7 +300,7 @@ def get_data_parallel(
217
300
  report_metadata: dict,
218
301
  cache: bool = False,
219
302
  max_workers: int = 8,
220
- ask_values: dict[ReportConfig, dict[str, str] | None] | None = None,
303
+ ask_values: dict[ReportConfig, dict[str, str | list[str]] | None] | None = None,
221
304
  ) -> dict[ReportConfig, list[dict]]:
222
305
  """Fetch multiple reports in parallel using cached report metadata.
223
306
 
@@ -307,3 +307,53 @@ def load_report_metadata_batch(
307
307
  metadata[config] = load_report_metadata(cache_manager, config)
308
308
 
309
309
  return metadata
310
+
311
+
312
+ def filter_metadata_by_table(
313
+ report_metadata: dict[ReportConfig, dict],
314
+ table_name: str,
315
+ app_name: str | None = None,
316
+ ) -> dict:
317
+ """Retrieve metadata for a specific table, optionally filtered by app.
318
+
319
+ Args:
320
+ report_metadata: dict from load_report_metadata_batch()
321
+ table_name: Table name (e.g., "Accounts", "Billing Runs")
322
+ app_name: Optional app name. If not provided, table_name must be unique
323
+ across all apps.
324
+
325
+ Returns:
326
+ Metadata dict for the specified table (and app if provided).
327
+
328
+ Raises:
329
+ ValueError: If not found, or if table_name is ambiguous (multiple apps)
330
+ without app_name specified.
331
+
332
+ Example:
333
+ >>> metadata = load_report_metadata_batch(cache_manager, configs)
334
+ >>> runs_md = filter_metadata_by_app_table(metadata, "Billing Runs")
335
+ >>> accounts = filter_metadata_by_app_table(metadata, "Accounts", app_name="date_lake")
336
+ """
337
+ if app_name:
338
+ # Filter by both app and table
339
+ results = [
340
+ data
341
+ for config, data in report_metadata.items()
342
+ if config.app_name == app_name and config.table_name == table_name
343
+ ]
344
+ error_msg = f"app={app_name}, table={table_name}"
345
+ else:
346
+ # Filter by table only
347
+ results = [data for config, data in report_metadata.items() if config.table_name == table_name]
348
+ error_msg = f"table={table_name}"
349
+
350
+ if not results:
351
+ raise ValueError(f"No metadata found for {error_msg}")
352
+
353
+ if len(results) > 1:
354
+ available_apps = [config.app_name for config, _ in report_metadata.items() if config.table_name == table_name]
355
+ raise ValueError(
356
+ f"Multiple apps have table '{table_name}': {available_apps}. Please specify app_name: {available_apps[0]}"
357
+ )
358
+
359
+ return results[0]
@@ -265,6 +265,7 @@ class TestRefreshDataCache:
265
265
  report_configs=sample_report_configs,
266
266
  report_metadata={config: {} for config in sample_report_configs},
267
267
  cache=True,
268
+ ask_values=None,
268
269
  )
269
270
  assert "Data cache refresh needed: data stale" in caplog.text
270
271
  assert "Data cache refresh completed successfully" in caplog.text
@@ -5,7 +5,12 @@ from unittest.mock import MagicMock, patch
5
5
  import pytest
6
6
 
7
7
  from quickbase_extract.cache_manager import CacheManager
8
- from quickbase_extract.cache_sync import _reset_cache_sync, complete_cache_refresh, is_cache_synced, sync_from_s3_once
8
+ from quickbase_extract.cache_sync import (
9
+ _reset_cache_sync,
10
+ complete_cache_refresh,
11
+ is_cache_synced,
12
+ sync_from_s3_once,
13
+ )
9
14
 
10
15
 
11
16
  class TestSyncFromS3Once:
@@ -8,7 +8,9 @@ from quickbase_extract.cache_manager import CacheManager
8
8
  from quickbase_extract.config import ReportConfig
9
9
  from quickbase_extract.report_data import (
10
10
  _extract_report_names,
11
+ _normalize_ask_values,
11
12
  _replace_ask_placeholders,
13
+ _validate_ask_values,
12
14
  get_data,
13
15
  get_data_parallel,
14
16
  load_data,
@@ -20,6 +22,103 @@ from quickbase_extract.report_metadata import (
20
22
  )
21
23
 
22
24
 
25
+ class TestValidateAskValues:
26
+ """Tests for _validate_ask_values helper function."""
27
+
28
+ def test_valid_single_placeholder_value(self, sample_report_configs):
29
+ """Test validation passes with valid single placeholder and value."""
30
+ config = sample_report_configs[0]
31
+ placeholders = {"_ask1_"}
32
+ ask_values = {"ask1": "value"}
33
+
34
+ # Should not raise
35
+ _validate_ask_values(ask_values, placeholders, config)
36
+
37
+ def test_valid_multiple_placeholders(self, sample_report_configs):
38
+ """Test validation passes with multiple placeholders and values."""
39
+ config = sample_report_configs[0]
40
+ placeholders = {"_ask1_", "_ask2_"}
41
+ ask_values = {"ask1": "value1", "ask2": ["value2", "value3"]}
42
+
43
+ # Should not raise
44
+ _validate_ask_values(ask_values, placeholders, config)
45
+
46
+ def test_missing_placeholder_value_raises_error(self, sample_report_configs):
47
+ """Test error when placeholder has no corresponding value."""
48
+ config = sample_report_configs[0]
49
+ placeholders = {"_ask1_", "_ask2_"}
50
+ ask_values = {"ask1": "value1"} # Missing ask2
51
+
52
+ with pytest.raises(ValueError, match=r"requires values for.*_ask2_"):
53
+ _validate_ask_values(ask_values, placeholders, config)
54
+
55
+ def test_unused_ask_value_raises_error(self, sample_report_configs):
56
+ """Test error when provided value is not used in filter."""
57
+ config = sample_report_configs[0]
58
+ placeholders = {"_ask1_"}
59
+ ask_values = {"ask1": "value1", "ask2": "value2"} # ask2 not in filter
60
+
61
+ with pytest.raises(ValueError, match=r"received ask_values.*ask2.*not used"):
62
+ _validate_ask_values(ask_values, placeholders, config)
63
+
64
+ def test_empty_list_raises_error(self, sample_report_configs):
65
+ """Test error when empty list provided."""
66
+ config = sample_report_configs[0]
67
+ placeholders = {"_ask1_"}
68
+ ask_values = {"ask1": []} # Empty list
69
+
70
+ with pytest.raises(ValueError, match="empty lists"):
71
+ _validate_ask_values(ask_values, placeholders, config)
72
+
73
+ def test_empty_placeholders_set_validates(self, sample_report_configs):
74
+ """Test that empty placeholders set validates successfully with empty ask_values."""
75
+ config = sample_report_configs[0]
76
+ placeholders = set()
77
+ ask_values = {}
78
+
79
+ # Should not raise
80
+ _validate_ask_values(ask_values, placeholders, config)
81
+
82
+
83
+ class TestNormalizeAskValues:
84
+ """Tests for _normalize_ask_values helper function."""
85
+
86
+ def test_normalize_string_to_list(self):
87
+ """Test that string values are converted to single-element lists."""
88
+ ask_values = {"ask1": "value"}
89
+ result = _normalize_ask_values(ask_values)
90
+
91
+ assert result == {"ask1": ["value"]}
92
+
93
+ def test_normalize_list_unchanged(self):
94
+ """Test that list values remain unchanged."""
95
+ ask_values = {"ask1": ["val1", "val2"]}
96
+ result = _normalize_ask_values(ask_values)
97
+
98
+ assert result == {"ask1": ["val1", "val2"]}
99
+
100
+ def test_normalize_mixed_values(self):
101
+ """Test normalization with mixed string and list values."""
102
+ ask_values = {"ask1": "single", "ask2": ["multi1", "multi2"]}
103
+ result = _normalize_ask_values(ask_values)
104
+
105
+ assert result == {"ask1": ["single"], "ask2": ["multi1", "multi2"]}
106
+
107
+ def test_normalize_empty_dict(self):
108
+ """Test normalization with empty dict."""
109
+ ask_values = {}
110
+ result = _normalize_ask_values(ask_values)
111
+
112
+ assert result == {}
113
+
114
+ def test_normalize_preserves_order(self):
115
+ """Test that normalization preserves dict key order."""
116
+ ask_values = {"ask3": "val3", "ask1": "val1", "ask2": "val2"}
117
+ result = _normalize_ask_values(ask_values)
118
+
119
+ assert list(result.keys()) == ["ask3", "ask1", "ask2"]
120
+
121
+
23
122
  class TestReplaceAskPlaceholders:
24
123
  """Tests for _replace_ask_placeholders function."""
25
124
 
@@ -34,7 +133,7 @@ class TestReplaceAskPlaceholders:
34
133
  assert result == "{'25'.EX.'abc123'}"
35
134
 
36
135
  def test_replace_multiple_placeholders(self, sample_report_configs):
37
- """Test replacing multiple ask placeholders."""
136
+ """Test replacing multiple different placeholders."""
38
137
  config = sample_report_configs[0]
39
138
  filter_str = "({'25'.EX.'_ask1_'}AND{'40'.EX.'_ask2_'})"
40
139
  ask_values = {"ask1": "value1", "ask2": "value2"}
@@ -43,6 +142,84 @@ class TestReplaceAskPlaceholders:
43
142
 
44
143
  assert result == "({'25'.EX.'value1'}AND{'40'.EX.'value2'})"
45
144
 
145
+ def test_replace_placeholder_with_list_values(self, sample_report_configs):
146
+ """Test replacing placeholder with multiple values creates OR condition."""
147
+ config = sample_report_configs[0]
148
+ filter_str = "{'25'.EX.'_ask1_'}"
149
+ ask_values = {"ask1": ["val1", "val2", "val3"]}
150
+
151
+ result = _replace_ask_placeholders(filter_str, ask_values, config)
152
+
153
+ expected = "({'25'.EX.'val1'}OR{'25'.EX.'val2'}OR{'25'.EX.'val3'})"
154
+ assert result == expected
155
+
156
+ def test_replace_list_with_single_element(self, sample_report_configs):
157
+ """Test that list with single element is treated as single value."""
158
+ config = sample_report_configs[0]
159
+ filter_str = "{'25'.EX.'_ask1_'}"
160
+ ask_values = {"ask1": ["only_value"]}
161
+
162
+ result = _replace_ask_placeholders(filter_str, ask_values, config)
163
+
164
+ # Single element list should not create OR
165
+ assert result == "{'25'.EX.'only_value'}"
166
+
167
+ def test_empty_list_raises_error(self, sample_report_configs):
168
+ """Test that empty list in ask_values raises ValueError."""
169
+ config = sample_report_configs[0]
170
+ filter_str = "{'25'.EX.'_ask1_'}"
171
+ ask_values = {"ask1": []}
172
+
173
+ with pytest.raises(ValueError, match="empty lists"):
174
+ _replace_ask_placeholders(filter_str, ask_values, config)
175
+
176
+ def test_mixed_string_and_list_values(self, sample_report_configs):
177
+ """Test replacing multiple placeholders with mixed string and list values."""
178
+ config = sample_report_configs[0]
179
+ filter_str = "({'25'.EX.'_ask1_'}AND{'40'.EX.'_ask2_'})"
180
+ ask_values = {"ask1": "single_value", "ask2": ["val1", "val2"]}
181
+
182
+ result = _replace_ask_placeholders(filter_str, ask_values, config)
183
+
184
+ # ask1 should be simple replacement, ask2 should have OR
185
+ assert "{'25'.EX.'single_value'}" in result
186
+ assert "({'40'.EX.'val1'}OR{'40'.EX.'val2'})" in result
187
+
188
+ def test_list_values_in_complex_filter(self, sample_report_configs):
189
+ """Test list value expansion maintains filter logic in complex expressions."""
190
+ config = sample_report_configs[0]
191
+ filter_str = "({'15'.EX.'Pending'}AND{'41'.EX.'_ask1_'})"
192
+ ask_values = {"ask1": ["urgent", "high"]}
193
+
194
+ result = _replace_ask_placeholders(filter_str, ask_values, config)
195
+
196
+ # Should preserve outer AND structure with inner OR
197
+ assert "{'15'.EX.'Pending'}" in result
198
+ assert "({'41'.EX.'urgent'}OR{'41'.EX.'high'})" in result
199
+
200
+ def test_multiple_condition_blocks_same_placeholder(self, sample_report_configs):
201
+ """Test replacing multiple condition blocks with same placeholder."""
202
+ config = sample_report_configs[0]
203
+ filter_str = "({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'})"
204
+ ask_values = {"ask1": "urgent"}
205
+
206
+ result = _replace_ask_placeholders(filter_str, ask_values, config)
207
+
208
+ expected = "({'41'.EX.'urgent'}OR{'40'.EX.'urgent'})"
209
+ assert result == expected
210
+
211
+ def test_multiple_blocks_same_placeholder_with_list_values(self, sample_report_configs):
212
+ """Test replacing multiple condition blocks with same placeholder using list values."""
213
+ config = sample_report_configs[0]
214
+ filter_str = "({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'})"
215
+ ask_values = {"ask1": ["a", "b"]}
216
+
217
+ result = _replace_ask_placeholders(filter_str, ask_values, config)
218
+
219
+ # Each condition block should be expanded
220
+ assert "({'41'.EX.'a'}OR{'41'.EX.'b'})" in result
221
+ assert "({'40'.EX.'a'}OR{'40'.EX.'b'})" in result
222
+
46
223
  def test_no_placeholders_returns_unchanged(self, sample_report_configs):
47
224
  """Test that filter without placeholders is returned unchanged."""
48
225
  config = sample_report_configs[0]
@@ -59,7 +236,7 @@ class TestReplaceAskPlaceholders:
59
236
  filter_str = "{'25'.EX.'_ask1_'}AND{'40'.EX.'_ask2_'}"
60
237
  ask_values = {"ask1": "value1"} # Missing ask2
61
238
 
62
- with pytest.raises(ValueError, match="requires values for.*_ask2_"):
239
+ with pytest.raises(ValueError, match=r"requires values for.*_ask2_"):
63
240
  _replace_ask_placeholders(filter_str, ask_values, config)
64
241
 
65
242
  def test_unused_placeholder_value_raises_error(self, sample_report_configs):
@@ -68,7 +245,7 @@ class TestReplaceAskPlaceholders:
68
245
  filter_str = "{'25'.EX.'_ask1_'}"
69
246
  ask_values = {"ask1": "value1", "ask2": "value2"} # ask2 not in filter
70
247
 
71
- with pytest.raises(ValueError, match="received ask_values.*ask2.*not used"):
248
+ with pytest.raises(ValueError, match=r"received ask_values.*ask2.*not used"):
72
249
  _replace_ask_placeholders(filter_str, ask_values, config)
73
250
 
74
251
  def test_placeholder_with_special_characters(self, sample_report_configs):
@@ -82,14 +259,28 @@ class TestReplaceAskPlaceholders:
82
259
  assert result == "{'25'.EX.'value with spaces & symbols!'}"
83
260
 
84
261
  def test_complex_filter_with_mixed_content(self, sample_report_configs):
85
- """Test replacing placeholders in complex filter."""
262
+ """Test replacing placeholders in complex filter with both fixed and dynamic values."""
86
263
  config = sample_report_configs[0]
87
264
  filter_str = "({'15'.EX.'Pending'}AND({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'}))"
88
265
  ask_values = {"ask1": "urgent"}
89
266
 
90
267
  result = _replace_ask_placeholders(filter_str, ask_values, config)
91
268
 
92
- assert result == ("({'15'.EX.'Pending'}AND({'41'.EX.'urgent'}OR{'40'.EX.'urgent'}))")
269
+ expected = "({'15'.EX.'Pending'}AND({'41'.EX.'urgent'}OR{'40'.EX.'urgent'}))"
270
+ assert result == expected
271
+
272
+ def test_complex_filter_with_mixed_content_and_list(self, sample_report_configs):
273
+ """Test complex filter with multiple condition blocks and list values."""
274
+ config = sample_report_configs[0]
275
+ filter_str = "({'15'.EX.'Pending'}AND({'41'.EX.'_ask1_'}OR{'40'.EX.'_ask1_'}))"
276
+ ask_values = {"ask1": ["urgent", "high"]}
277
+
278
+ result = _replace_ask_placeholders(filter_str, ask_values, config)
279
+
280
+ # Should have outer structure preserved and inner ORs expanded
281
+ assert "{'15'.EX.'Pending'}" in result
282
+ assert "({'41'.EX.'urgent'}OR{'41'.EX.'high'})" in result
283
+ assert "({'40'.EX.'urgent'}OR{'40'.EX.'high'})" in result
93
284
 
94
285
 
95
286
  class TestExtractReportNames:
@@ -8,6 +8,7 @@ from quickbase_extract.cache_manager import CacheManager
8
8
  from quickbase_extract.config import ReportConfig
9
9
  from quickbase_extract.report_metadata import (
10
10
  fetch_report_metadata_api,
11
+ filter_metadata_by_table,
11
12
  get_report_metadata,
12
13
  get_report_metadata_parallel,
13
14
  load_report_metadata,
@@ -71,7 +72,7 @@ class TestFetchReportMetadataApi:
71
72
  {"id": "rptABC", "name": "Default"},
72
73
  ]
73
74
 
74
- with pytest.raises(ValueError, match="Report .* not found"):
75
+ with pytest.raises(ValueError, match=r"Report .* not found"):
75
76
  fetch_report_metadata_api(
76
77
  mock_qb_api,
77
78
  "appXYZ123",
@@ -167,7 +168,7 @@ class TestGetReportMetadata:
167
168
  report_name="Nonexistent",
168
169
  )
169
170
 
170
- with pytest.raises(ValueError, match="Report .* not found"):
171
+ with pytest.raises(ValueError, match=r"Report .* not found"):
171
172
  get_report_metadata(
172
173
  mock_qb_api,
173
174
  cache_mgr,
@@ -393,7 +394,48 @@ class TestLoadReportMetadataBatch:
393
394
  # Should be able to look up by config
394
395
  config1 = sample_report_configs[0]
395
396
  assert all_metadata[config1]["table_id"] == "tblXYZ123"
396
- assert all_metadata[config1]["table_id"] == "tblXYZ123"
397
- assert all_metadata[config1]["table_id"] == "tblXYZ123"
398
- assert all_metadata[config1]["table_id"] == "tblXYZ123"
399
- assert all_metadata[config1]["table_id"] == "tblXYZ123"
397
+
398
+
399
+ class TestFilterMetadataByTable:
400
+ """Tests for filter_metadata_by_table function."""
401
+
402
+ def test_filter_metadata_by_table_unique(self, sample_report_metadata, sample_report_configs):
403
+ """Test retrieving metadata for a unique table."""
404
+
405
+ table_name = sample_report_configs[0].table_name
406
+ result = filter_metadata_by_table(sample_report_metadata, table_name)
407
+
408
+ assert result["table_name"] == "test_table"
409
+ assert result["table_id"] == "tblXYZ123"
410
+
411
+ def test_get_metadata_by_table_with_app_name(self, sample_report_metadata, sample_report_configs):
412
+ """Test retrieving metadata filtering by both app and table."""
413
+
414
+ config = sample_report_configs[0]
415
+ result = filter_metadata_by_table(sample_report_metadata, config.table_name, app_name=config.app_name)
416
+
417
+ assert result["table_name"] == "test_table"
418
+ assert result["app_name"] == "test_app"
419
+
420
+ def test_get_metadata_by_table_not_found(self, sample_report_metadata):
421
+ """Test error when table not found."""
422
+
423
+ with pytest.raises(ValueError, match="No metadata found"):
424
+ filter_metadata_by_table(sample_report_metadata, "Nonexistent Table")
425
+
426
+ def test_get_metadata_by_table_ambiguous_without_app(self, sample_report_metadata):
427
+ """Test error when table exists in multiple apps without app_name specified."""
428
+
429
+ # Create a second metadata entry with same table name but different app
430
+ # This would require a fixture with duplicate table names across apps
431
+ # For now, this documents expected behavior
432
+ pass
433
+
434
+ def test_get_metadata_by_table_ambiguous_with_app(self, sample_report_metadata, sample_report_configs):
435
+ """Test successful lookup when table is ambiguous but app_name is provided."""
436
+
437
+ config = sample_report_configs[0]
438
+ # Should not raise error because app_name disambiguates
439
+ result = filter_metadata_by_table(sample_report_metadata, config.table_name, app_name=config.app_name)
440
+
441
+ assert result is not None