dataface 0.1.6.dev360__py3-none-any.whl → 0.1.6.dev426__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataface/DATAFACE_SYNTAX.md +26 -0
- dataface/agent_api/docs/yaml-reference.md +4 -0
- dataface/agent_api/project_session.py +6 -6
- dataface/agent_api/render_face.py +12 -4
- dataface/ai/skills/dashboard-build/SKILL.md +2 -0
- dataface/core/compile/channel.py +10 -3
- dataface/core/compile/models/chart/authored.py +11 -0
- dataface/core/compile/models/chart/normalized.py +7 -0
- dataface/core/compile/models/style/resolved.py +2 -0
- dataface/core/compile/models/style/theme.py +33 -0
- dataface/core/compile/style_cascade.py +4 -0
- dataface/core/dashboard.py +2 -2
- dataface/core/defaults/themes/_base.yaml +5 -0
- dataface/core/execute/_duckdb_cache_base.py +272 -0
- dataface/core/execute/adapters/schema_adapter.py +83 -0
- dataface/core/execute/batch.py +1 -2
- dataface/core/execute/cache_backend.py +1 -35
- dataface/core/execute/duckdb_cache.py +25 -333
- dataface/core/execute/executor.py +0 -158
- dataface/core/execute/parallel.py +7 -114
- dataface/core/execute/trivial_local_cache.py +183 -0
- dataface/core/inspect/templates/categorical_column.yml +1 -1
- dataface/core/inspect/templates/date_column.yml +1 -1
- dataface/core/inspect/templates/model.yml +2 -2
- dataface/core/inspect/templates/numeric_column.yml +1 -1
- dataface/core/inspect/templates/quality.yml +1 -1
- dataface/core/inspect/templates/string_column.yml +1 -1
- dataface/core/registered_views/link_keys.py +17 -14
- dataface/core/registered_views/render_pipeline.py +11 -2
- dataface/core/registered_views/variable_planner.py +19 -19
- dataface/core/render/chart/auto_link.py +349 -72
- dataface/core/render/chart/decisions.py +18 -10
- dataface/core/render/chart/render_single.py +292 -23
- dataface/core/render/chart/table.py +123 -17
- dataface/core/render/chart/table_support.py +49 -4
- dataface/core/render/chart/vega_lite_types.py +17 -0
- dataface/core/render/chart/vl_field_maps.py +2 -0
- dataface/core/render/chrome_css.py +56 -0
- dataface/core/render/face_api.py +2 -2
- dataface/core/render/renderer.py +37 -33
- dataface/core/render/templates/controls/_styles.css +60 -62
- dataface/core/render/templates/nav/nav-fragment.html +1 -0
- dataface/core/render/templates/nav/nav.css +18 -19
- dataface/core/render/templates/scripts/variables.js +4 -4
- dataface/core/render/templates/variable_controls/container.html +2 -8
- dataface/core/render/variable_controls.py +2 -9
- dataface/core/render/warnings/base.py +17 -0
- dataface/core/render/warnings/pie_dominant_segment.py +68 -0
- dataface/core/render/warnings/pie_too_many_segments.py +47 -0
- dataface/core/render/warnings/registry.py +8 -0
- dataface/core/render/warnings/too_many_color_categories.py +68 -0
- dataface/core/render/warnings/too_many_x_categories.py +63 -0
- dataface/core/serve/alias_index.py +52 -4
- dataface/core/serve/server.py +19 -12
- {dataface-0.1.6.dev360.dist-info → dataface-0.1.6.dev426.dist-info}/METADATA +1 -1
- {dataface-0.1.6.dev360.dist-info → dataface-0.1.6.dev426.dist-info}/RECORD +59 -52
- {dataface-0.1.6.dev360.dist-info → dataface-0.1.6.dev426.dist-info}/WHEEL +0 -0
- {dataface-0.1.6.dev360.dist-info → dataface-0.1.6.dev426.dist-info}/entry_points.txt +0 -0
- {dataface-0.1.6.dev360.dist-info → dataface-0.1.6.dev426.dist-info}/licenses/LICENSE +0 -0
dataface/DATAFACE_SYNTAX.md
CHANGED
|
@@ -253,6 +253,32 @@ Rules:
|
|
|
253
253
|
- Each alias must be unique across the project — two faces claiming the same alias is a startup error.
|
|
254
254
|
- Aliasing a generated system route (data or inspector view) is allowed: the redirect takes precedence, letting you override what that URL serves.
|
|
255
255
|
|
|
256
|
+
#### Parameterized aliases (capture a path segment into a variable)
|
|
257
|
+
|
|
258
|
+
An alias may contain `<name>` capture segments. A request matching the pattern
|
|
259
|
+
302-redirects to the face's canonical URL with each captured segment appended as a
|
|
260
|
+
query param of the same name — which the face then reads as its variable. This
|
|
261
|
+
gives a single detail face a clean per-entity URL:
|
|
262
|
+
|
|
263
|
+
```yaml
|
|
264
|
+
# faces/milestone.yml (canonical URL: /milestone)
|
|
265
|
+
aliases:
|
|
266
|
+
- /milestones/<name>
|
|
267
|
+
variables:
|
|
268
|
+
name: { input: select, options: { query: milestone_options, column: slug } }
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Now `/milestones/m3-public-launch` redirects to `/milestone/?name=m3-public-launch`,
|
|
272
|
+
and `milestone.yml` renders with `name` bound to `m3-public-launch`.
|
|
273
|
+
|
|
274
|
+
- Capture names use the same grammar as built-in routes: `<name>` matches exactly
|
|
275
|
+
one non-empty path segment (no `/`). Use a capture name that matches the
|
|
276
|
+
variable you want it to fill.
|
|
277
|
+
- A real face file always wins over a pattern alias, so `/milestones/` (the list
|
|
278
|
+
face) and `/milestones/<name>` (the detail redirect) coexist without conflict.
|
|
279
|
+
- Plain aliases (no `<...>`) redirect as before; only aliases with a capture are
|
|
280
|
+
treated as patterns.
|
|
281
|
+
|
|
256
282
|
There are two ways to override a data/inspector route for a given path:
|
|
257
283
|
- **Face file at that path** — create `faces/data/warehouse/schema/table.yml` and the server renders it directly (no redirect).
|
|
258
284
|
- **`aliases:` entry** — any face can declare `/data/…/` as an alias; the server issues a 302 to the declaring face's canonical URL.
|
|
@@ -287,6 +287,7 @@ Authored patch for KPI (key performance indicator) charts.
|
|
|
287
287
|
| `label` | str | ✓ | KPI label rendered above the headline value. |
|
|
288
288
|
| `support` | [KpiSupportConfig](#kpisupportconfig) | ✓ | Optional support line beneath the KPI value. |
|
|
289
289
|
| `style` | [KpiChartStyle](#kpichartstyle) | ✓ | Chart-local style overrides. |
|
|
290
|
+
| `background` | str \| dict[str, Any] | ✓ | Gradient background channel — {column, scale} shape. Paints the card background by the value's position in the scale. See channel.py parse_style_channel for the full grammar. |
|
|
290
291
|
|
|
291
292
|
<a id="tablechart"></a>
|
|
292
293
|
## TableChart
|
|
@@ -848,6 +849,7 @@ Authored overlay for TableChartStyle. Table chart style overrides layered on top
|
|
|
848
849
|
| `more_rows` | [TableEdgeStyle](#tableedgestyle) | ✓ | Style for the 'more rows' edge-case indicator. |
|
|
849
850
|
| `empty_state` | [TableEdgeStyle](#tableedgestyle) | ✓ | Style for the empty-state (no data) indicator. |
|
|
850
851
|
| `spark` | [SparkStyle](#sparkstyle) | ✓ | Inline sparkline defaults for table cells. |
|
|
852
|
+
| `transpose` | bool | ✓ | When True, render a single wide data row as N (label, value) rows — one per column. Raises ChartDataError when data has more than one row. Used for Looker single-value summary tiles with multiple measures. |
|
|
851
853
|
| `text_baseline_offset` | float | ✓ | Vertical offset to align SVG text baseline with cell grid in pixels. |
|
|
852
854
|
| `title_baseline_offset` | float | ✓ | Vertical offset to align SVG title baseline in pixels. |
|
|
853
855
|
| `title_subtitle_gap` | float | ✓ | Gap between table title and subtitle lines in pixels. |
|
|
@@ -1348,6 +1350,7 @@ Authored overlay for LegendStyle.
|
|
|
1348
1350
|
| `title` | [LegendElementStyle](#legendelementstyle) | ✓ | Legend title style. |
|
|
1349
1351
|
| `visible` | bool | ✓ | Show the legend. None = legend visible; False = explicitly suppressed. |
|
|
1350
1352
|
| `interactive_legend` | bool | ✓ | Emit a dft_legend point-selection param for single-click series toggle. |
|
|
1353
|
+
| `symbol_limit` | int | ✓ | Maximum number of legend entries to display; maps to VL symbolLimit. None uses Vega-Lite's default (no cap). Set to a positive integer to prevent legend overflow on high-cardinality series. |
|
|
1351
1354
|
|
|
1352
1355
|
<a id="tooltipstyle"></a>
|
|
1353
1356
|
## TooltipStyle
|
|
@@ -1569,6 +1572,7 @@ Authored overlay for TableColumnsStyle.
|
|
|
1569
1572
|
| `default_width` | float | ✓ | Default column width in pixels. |
|
|
1570
1573
|
| `cell_padding` | float | ✓ | Horizontal padding inside table cells in pixels. |
|
|
1571
1574
|
| `width_similarity_threshold` | float | ✓ | Auto-width columns whose min/max ratio >= this threshold are snapped to a shared width before budget allocation. 1.0 disables clustering; 0.0 forces all auto-columns to equal width. |
|
|
1575
|
+
| `content_headroom` | float | ✓ | Fractional breathing room added above the raw p95 column demand when pinning compact columns in mixed (compact + text) tables. 0.10 means each compact column is pinned at 10 % above its measured demand; the extra width is funded by the text-column budget. Has no effect on all-compact tables (proportional scaling already fills the budget). 0.0 disables headroom. |
|
|
1572
1576
|
|
|
1573
1577
|
<a id="tableheaderstyle"></a>
|
|
1574
1578
|
## TableHeaderStyle
|
|
@@ -35,7 +35,7 @@ from dataface.core.execute.adapters.adapter_registry import (
|
|
|
35
35
|
AdapterRegistry,
|
|
36
36
|
build_adapter_registry,
|
|
37
37
|
)
|
|
38
|
-
from dataface.core.execute.
|
|
38
|
+
from dataface.core.execute.cache_backend import QueryResultCache
|
|
39
39
|
from dataface.core.execute.observability import WarehouseObserver
|
|
40
40
|
from dataface.core.inspect.query_validator import (
|
|
41
41
|
QueryDiagnostic,
|
|
@@ -68,7 +68,7 @@ class ProjectSession:
|
|
|
68
68
|
"""
|
|
69
69
|
|
|
70
70
|
project: Project
|
|
71
|
-
cache:
|
|
71
|
+
cache: QueryResultCache | None
|
|
72
72
|
_owns_registry: bool
|
|
73
73
|
_read_only: bool
|
|
74
74
|
_connection_string: str | None
|
|
@@ -82,7 +82,7 @@ class ProjectSession:
|
|
|
82
82
|
def __init__(
|
|
83
83
|
self,
|
|
84
84
|
project: Project,
|
|
85
|
-
cache:
|
|
85
|
+
cache: QueryResultCache | None = None,
|
|
86
86
|
adapter_registry: AdapterRegistry | None = None,
|
|
87
87
|
read_only: bool = True,
|
|
88
88
|
observers: list[WarehouseObserver] | None = None,
|
|
@@ -110,7 +110,7 @@ class ProjectSession:
|
|
|
110
110
|
cls,
|
|
111
111
|
project_dir: Path | str,
|
|
112
112
|
*,
|
|
113
|
-
cache:
|
|
113
|
+
cache: QueryResultCache | None = None,
|
|
114
114
|
read_only: bool = True,
|
|
115
115
|
connection_string: str | None = None,
|
|
116
116
|
dialect: str = "duckdb",
|
|
@@ -150,7 +150,7 @@ class ProjectSession:
|
|
|
150
150
|
cls,
|
|
151
151
|
project: Project,
|
|
152
152
|
*,
|
|
153
|
-
cache:
|
|
153
|
+
cache: QueryResultCache | None = None,
|
|
154
154
|
read_only: bool = True,
|
|
155
155
|
connection_string: str | None = None,
|
|
156
156
|
dialect: str = "duckdb",
|
|
@@ -183,7 +183,7 @@ class ProjectSession:
|
|
|
183
183
|
face_file: Path | str,
|
|
184
184
|
*,
|
|
185
185
|
read_only: bool = True,
|
|
186
|
-
cache:
|
|
186
|
+
cache: QueryResultCache | None = None,
|
|
187
187
|
) -> Self:
|
|
188
188
|
"""Open a ProjectSession rooted at the project directory discovered upward from face_file."""
|
|
189
189
|
resolved = Path(face_file).resolve()
|
|
@@ -13,8 +13,8 @@ from dataface.core.compile.markdown import (
|
|
|
13
13
|
is_markdown_face,
|
|
14
14
|
markdown_to_yaml,
|
|
15
15
|
)
|
|
16
|
-
from dataface.core.execute.
|
|
17
|
-
from dataface.core.project import Project
|
|
16
|
+
from dataface.core.execute.cache_backend import QueryResultCache
|
|
17
|
+
from dataface.core.project import Project, ProjectFile
|
|
18
18
|
from dataface.core.render.face_api import compile_and_render
|
|
19
19
|
|
|
20
20
|
|
|
@@ -26,7 +26,7 @@ def render_face(
|
|
|
26
26
|
project: Project | None = None,
|
|
27
27
|
variables: dict[str, str] | None = None,
|
|
28
28
|
use_cache: bool = True,
|
|
29
|
-
cache:
|
|
29
|
+
cache: QueryResultCache | None = None,
|
|
30
30
|
**options: Any,
|
|
31
31
|
) -> str | bytes:
|
|
32
32
|
"""Render a Dataface face file (.yml or .md) to embeddable output.
|
|
@@ -70,7 +70,15 @@ def render_face(
|
|
|
70
70
|
project_session = ProjectSession.from_face(face_file, cache=cache)
|
|
71
71
|
try:
|
|
72
72
|
p = project_session.project
|
|
73
|
-
|
|
73
|
+
# The face may live outside an explicitly-supplied project root — e.g.
|
|
74
|
+
# the looker_migrate eval renders faces under evals/runs/ against a
|
|
75
|
+
# project rooted at runtime/ for its sources/schema. Such a face has no
|
|
76
|
+
# project-relative handle; nested sub-file refs are unsupported in that
|
|
77
|
+
# mode (relative reads still resolve via face_dir below).
|
|
78
|
+
try:
|
|
79
|
+
base_file: ProjectFile | None = p.file_for_path(face_file)
|
|
80
|
+
except ValueError:
|
|
81
|
+
base_file = None
|
|
74
82
|
return compile_and_render(
|
|
75
83
|
yaml_content,
|
|
76
84
|
p,
|
|
@@ -129,6 +129,8 @@ Skip this step when you are previewing ephemerally — see "Previewing vs. savin
|
|
|
129
129
|
|
|
130
130
|
Fix any errors `{{ s_validate_dashboard }}` reports, then re-run until it passes.
|
|
131
131
|
|
|
132
|
+
**YAML style:** write block style — one key per line. Avoid JSON-like inline flow maps (`{ type: bar, x: month }`); they read and diff worse than indented blocks. Inline arrays (`y: [revenue, cost]`) are fine.
|
|
133
|
+
|
|
132
134
|
## Parameterized Queries — Use From the Start
|
|
133
135
|
|
|
134
136
|
Use `{{ variable_name }}` syntax for any configurable values, even during exploration:
|
dataface/core/compile/channel.py
CHANGED
|
@@ -130,7 +130,7 @@ def validate_channel_fields(
|
|
|
130
130
|
)
|
|
131
131
|
|
|
132
132
|
|
|
133
|
-
_COLOR_CHANNELS = frozenset({"color"})
|
|
133
|
+
_COLOR_CHANNELS = frozenset({"color", "background"})
|
|
134
134
|
_NUMERIC_CHANNELS: frozenset[str] = frozenset()
|
|
135
135
|
|
|
136
136
|
|
|
@@ -165,8 +165,10 @@ def normalize_chart_channels(
|
|
|
165
165
|
) -> dict[str, ResolvedStyleChannel]:
|
|
166
166
|
"""Parse and validate chart-level style channels.
|
|
167
167
|
|
|
168
|
-
|
|
169
|
-
|
|
168
|
+
``color`` is a data channel at chart root for all chart types.
|
|
169
|
+
``background`` is a data channel at chart root for KPI charts only
|
|
170
|
+
(gradient background painted by value position in scale).
|
|
171
|
+
Other paint fields (opacity, stroke) are not data channels.
|
|
170
172
|
|
|
171
173
|
``style_color``: when a ``StyleColorConfig`` with a scale is provided, the
|
|
172
174
|
series color channel is upgraded to gradient mode. A plain string value
|
|
@@ -182,6 +184,11 @@ def normalize_chart_channels(
|
|
|
182
184
|
if color_raw is not None:
|
|
183
185
|
channels["color"] = parse_style_channel(color_raw, "color")
|
|
184
186
|
|
|
187
|
+
if chart_type == "kpi":
|
|
188
|
+
background_raw = chart.background
|
|
189
|
+
if background_raw is not None:
|
|
190
|
+
channels["background"] = parse_style_channel(background_raw, "background")
|
|
191
|
+
|
|
185
192
|
# Upgrade series channel to gradient when style.color carries a scale.
|
|
186
193
|
if isinstance(style_color, StyleColorConfig):
|
|
187
194
|
if "color" not in channels:
|
|
@@ -1538,6 +1538,17 @@ class KpiChart(_BaseChartFields):
|
|
|
1538
1538
|
KpiChartStylePatch | None,
|
|
1539
1539
|
Field(default=None, description="Chart-local style overrides."),
|
|
1540
1540
|
]
|
|
1541
|
+
background: Annotated[
|
|
1542
|
+
str | dict[str, Any] | None,
|
|
1543
|
+
Field(
|
|
1544
|
+
default=None,
|
|
1545
|
+
description=(
|
|
1546
|
+
"Gradient background channel — {column, scale} shape. "
|
|
1547
|
+
"Paints the card background by the value's position in the scale. "
|
|
1548
|
+
"See channel.py parse_style_channel for the full grammar."
|
|
1549
|
+
),
|
|
1550
|
+
),
|
|
1551
|
+
]
|
|
1541
1552
|
|
|
1542
1553
|
@field_validator("value", mode="before")
|
|
1543
1554
|
@classmethod
|
|
@@ -201,6 +201,13 @@ class Chart(BaseModel):
|
|
|
201
201
|
support: KpiSupportConfig | None = Field(
|
|
202
202
|
default=None, description="Optional support line beneath the KPI value."
|
|
203
203
|
)
|
|
204
|
+
background: str | dict[str, Any] | None = Field(
|
|
205
|
+
default=None,
|
|
206
|
+
description=(
|
|
207
|
+
"KPI gradient background channel — {column, scale} shape. "
|
|
208
|
+
"Parsed at render time via parse_style_channel. None for all other chart types."
|
|
209
|
+
),
|
|
210
|
+
)
|
|
204
211
|
projection: str | Projection | None = Field(
|
|
205
212
|
default=None, description="Map projection name or Vega-Lite projection config."
|
|
206
213
|
)
|
|
@@ -195,6 +195,7 @@ class ResolvedLegendStyle:
|
|
|
195
195
|
title: ResolvedLegendElementStyle
|
|
196
196
|
interactive_legend: bool
|
|
197
197
|
visible: bool | None = None
|
|
198
|
+
symbol_limit: int | None = None
|
|
198
199
|
|
|
199
200
|
|
|
200
201
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -885,6 +886,7 @@ def _build_resolved_legend(legend: LegendStyle) -> ResolvedLegendStyle:
|
|
|
885
886
|
direction=legend.direction,
|
|
886
887
|
visible=legend.visible,
|
|
887
888
|
interactive_legend=legend.interactive_legend,
|
|
889
|
+
symbol_limit=legend.symbol_limit,
|
|
888
890
|
label=ResolvedLegendElementStyle(
|
|
889
891
|
font=resolve_cascaded_font(legend.label.font, "charts.legend.label.font"),
|
|
890
892
|
padding=legend.label.padding,
|
|
@@ -1089,6 +1089,15 @@ class LegendStyle(BaseModel):
|
|
|
1089
1089
|
interactive_legend: bool = Field(
|
|
1090
1090
|
description="Emit a dft_legend point-selection param for single-click series toggle.",
|
|
1091
1091
|
)
|
|
1092
|
+
symbol_limit: int | None = Field(
|
|
1093
|
+
default=None,
|
|
1094
|
+
ge=1,
|
|
1095
|
+
description=(
|
|
1096
|
+
"Maximum number of legend entries to display; maps to VL symbolLimit. "
|
|
1097
|
+
"None uses Vega-Lite's default (no cap). "
|
|
1098
|
+
"Set to a positive integer to prevent legend overflow on high-cardinality series."
|
|
1099
|
+
),
|
|
1100
|
+
)
|
|
1092
1101
|
|
|
1093
1102
|
|
|
1094
1103
|
# Inference
|
|
@@ -2586,6 +2595,18 @@ class TableColumnsStyle(BaseModel):
|
|
|
2586
2595
|
"0.0 forces all auto-columns to equal width."
|
|
2587
2596
|
),
|
|
2588
2597
|
)
|
|
2598
|
+
content_headroom: float = Field(
|
|
2599
|
+
ge=0.0,
|
|
2600
|
+
le=1.0,
|
|
2601
|
+
description=(
|
|
2602
|
+
"Fractional breathing room added above the raw p95 column demand when "
|
|
2603
|
+
"pinning compact columns in mixed (compact + text) tables. 0.10 means "
|
|
2604
|
+
"each compact column is pinned at 10 % above its measured demand; the "
|
|
2605
|
+
"extra width is funded by the text-column budget. Has no effect on "
|
|
2606
|
+
"all-compact tables (proportional scaling already fills the budget). "
|
|
2607
|
+
"0.0 disables headroom."
|
|
2608
|
+
),
|
|
2609
|
+
)
|
|
2589
2610
|
|
|
2590
2611
|
|
|
2591
2612
|
class TableHeaderStyle(BaseModel):
|
|
@@ -2988,6 +3009,18 @@ class TableChartStyle(_ChartStyleBaseAllOptional):
|
|
|
2988
3009
|
)
|
|
2989
3010
|
spark: SparkStyle = Field(description="Inline sparkline defaults for table cells.")
|
|
2990
3011
|
|
|
3012
|
+
# Authored convenience — not a theme constant; defaults False so themes need
|
|
3013
|
+
# not supply it. When True the renderer pivots a single wide data row into
|
|
3014
|
+
# N (label, value) rows, one per column.
|
|
3015
|
+
transpose: bool = Field(
|
|
3016
|
+
default=False,
|
|
3017
|
+
description=(
|
|
3018
|
+
"When True, render a single wide data row as N (label, value) rows — "
|
|
3019
|
+
"one per column. Raises ChartDataError when data has more than one row. "
|
|
3020
|
+
"Used for Looker single-value summary tiles with multiple measures."
|
|
3021
|
+
),
|
|
3022
|
+
)
|
|
3023
|
+
|
|
2991
3024
|
# SVG layout constants (all-config per ADR-002)
|
|
2992
3025
|
text_baseline_offset: float = Field(
|
|
2993
3026
|
description="Vertical offset to align SVG text baseline with cell grid in pixels."
|
|
@@ -121,6 +121,10 @@ def _merge_legend(
|
|
|
121
121
|
updates["direction"] = patch.direction
|
|
122
122
|
if patch.visible is not None:
|
|
123
123
|
updates["visible"] = patch.visible
|
|
124
|
+
# symbol_limit=None on the patch means "unset / inherit base"; only an
|
|
125
|
+
# explicitly authored integer overrides the resolved value.
|
|
126
|
+
if patch.symbol_limit is not None: # pyright: ignore[reportUnnecessaryComparison]
|
|
127
|
+
updates["symbol_limit"] = patch.symbol_limit
|
|
124
128
|
for sub_name in ("label", "title"):
|
|
125
129
|
sub_patch = getattr(patch, sub_name)
|
|
126
130
|
if sub_patch is None:
|
dataface/core/dashboard.py
CHANGED
|
@@ -43,7 +43,7 @@ from dataface.core.errors import (
|
|
|
43
43
|
from dataface.core.errors.structured import NextCommand
|
|
44
44
|
from dataface.core.execute import ExecutionError, Executor
|
|
45
45
|
from dataface.core.execute.adapters import AdapterRegistry
|
|
46
|
-
from dataface.core.execute.
|
|
46
|
+
from dataface.core.execute.cache_backend import QueryResultCache
|
|
47
47
|
from dataface.core.project import Project
|
|
48
48
|
from dataface.core.render.warnings.base import RenderWarning
|
|
49
49
|
from dataface.core.render.warnings.suppression import partition as _partition_warnings
|
|
@@ -164,7 +164,7 @@ def render_dashboard(
|
|
|
164
164
|
scale: float | None = None,
|
|
165
165
|
as_link: bool = False,
|
|
166
166
|
link_context: LinkContext | None = None,
|
|
167
|
-
duckdb_cache:
|
|
167
|
+
duckdb_cache: QueryResultCache | None,
|
|
168
168
|
ignore_codes: set[str] | None = None,
|
|
169
169
|
url_mount_dir: str = "",
|
|
170
170
|
max_workers: int | None = None,
|
|
@@ -231,6 +231,7 @@ style:
|
|
|
231
231
|
legend:
|
|
232
232
|
position: right
|
|
233
233
|
direction: vertical
|
|
234
|
+
symbol_limit: 20
|
|
234
235
|
label:
|
|
235
236
|
font:
|
|
236
237
|
size: *size_label
|
|
@@ -276,6 +277,10 @@ style:
|
|
|
276
277
|
default_width: 100.0
|
|
277
278
|
cell_padding: 8.0
|
|
278
279
|
width_similarity_threshold: 0.8
|
|
280
|
+
# Opt-in: 0.0 globally (no width change to existing dashboards). The
|
|
281
|
+
# Looker migrator sets this per-table so migrated tables get breathing
|
|
282
|
+
# room without perturbing every authored table's column widths.
|
|
283
|
+
content_headroom: 0.0
|
|
279
284
|
title_row:
|
|
280
285
|
height: 40.0
|
|
281
286
|
more_rows:
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""Shared base and helpers for DuckDB-backed QueryResultCache implementations.
|
|
2
|
+
|
|
3
|
+
Stage: EXECUTE (Cache)
|
|
4
|
+
Purpose: Hold the connection lifecycle, SQL helpers, and failure store
|
|
5
|
+
that are identical across DuckDBCache (full history) and
|
|
6
|
+
TrivialDuckDBCache (replace-on-write).
|
|
7
|
+
|
|
8
|
+
The two subclasses override only the parts that genuinely diverge:
|
|
9
|
+
- get/put — result-table schema and sequencing logic differ
|
|
10
|
+
- _ensure_result_table — seq columns (DuckDBCache) vs none (Trivial)
|
|
11
|
+
- _drop_*_tables — legacy-table cleanup differs
|
|
12
|
+
|
|
13
|
+
Public helpers (_q, _result_table_name, _cache_safe_value, _infer_type)
|
|
14
|
+
live here so both subclasses share one definition and duckdb_cache.py
|
|
15
|
+
can re-export them without a circular import.
|
|
16
|
+
|
|
17
|
+
Dependencies:
|
|
18
|
+
- duckdb (optional at module load; required at construction)
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import hashlib
|
|
24
|
+
import json
|
|
25
|
+
import logging
|
|
26
|
+
import threading
|
|
27
|
+
import traceback as tb_mod
|
|
28
|
+
from datetime import datetime
|
|
29
|
+
from decimal import Decimal
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import TYPE_CHECKING, Any
|
|
32
|
+
|
|
33
|
+
if TYPE_CHECKING:
|
|
34
|
+
import duckdb
|
|
35
|
+
|
|
36
|
+
HAS_DUCKDB = True
|
|
37
|
+
else:
|
|
38
|
+
try:
|
|
39
|
+
import duckdb
|
|
40
|
+
|
|
41
|
+
HAS_DUCKDB = True
|
|
42
|
+
except ImportError:
|
|
43
|
+
duckdb = None
|
|
44
|
+
HAS_DUCKDB = False
|
|
45
|
+
|
|
46
|
+
from dataface._install_hint import install_hint
|
|
47
|
+
from dataface.core.execute.cache_backend import CachedQueryFailure
|
|
48
|
+
|
|
49
|
+
logger = logging.getLogger(__name__)
|
|
50
|
+
|
|
51
|
+
# How long to keep a combined hash prefix for table names
|
|
52
|
+
_HASH_PREFIX = 12
|
|
53
|
+
|
|
54
|
+
# Exported: these helpers are defined here to avoid duplication but used by
|
|
55
|
+
# duckdb_cache.py and trivial_local_cache.py (subclass modules).
|
|
56
|
+
__all__ = [
|
|
57
|
+
"_DuckDBResultCacheBase",
|
|
58
|
+
"_cache_safe_value",
|
|
59
|
+
"_infer_type",
|
|
60
|
+
"_q",
|
|
61
|
+
"_result_table_name",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
66
|
+
# SQL helper functions (re-exported by duckdb_cache for backward compat)
|
|
67
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _q(identifier: str) -> str:
|
|
71
|
+
"""Quote a SQL identifier (already assumed safe)."""
|
|
72
|
+
escaped = identifier.replace('"', '""')
|
|
73
|
+
return f'"{escaped}"'
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _result_table_name(source_hash: str, query_hash: str, variables_hash: str) -> str:
|
|
77
|
+
"""Generate the table name for a cached result.
|
|
78
|
+
|
|
79
|
+
Format: _r_{combined_prefix} where combined is a short hash of all three key parts.
|
|
80
|
+
Short enough to stay under DuckDB's identifier limits.
|
|
81
|
+
"""
|
|
82
|
+
combined = hashlib.sha256(
|
|
83
|
+
f"{source_hash}:{query_hash}:{variables_hash}".encode()
|
|
84
|
+
).hexdigest()[:_HASH_PREFIX]
|
|
85
|
+
return f"_r_{combined}"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _cache_safe_value(value: Any) -> Any:
|
|
89
|
+
"""Coerce a Python value to something DuckDB's parameterized INSERT accepts.
|
|
90
|
+
|
|
91
|
+
Decimals become float: DuckDB's DECIMAL is HUGEINT-backed and capped at
|
|
92
|
+
precision 38, but BigQuery NUMERIC can return precision > 38 (e.g. NUMERIC(47,38)
|
|
93
|
+
on fan-out-deduplicated SUM aggregates). float's 15 significant digits are
|
|
94
|
+
plenty for dashboard rendering. list/dict become JSON text for the JSON column.
|
|
95
|
+
"""
|
|
96
|
+
if isinstance(value, Decimal):
|
|
97
|
+
return float(value)
|
|
98
|
+
if isinstance(value, (list, dict)):
|
|
99
|
+
return json.dumps(value)
|
|
100
|
+
return value
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _infer_type(value: Any) -> str:
|
|
104
|
+
if value is None:
|
|
105
|
+
return "VARCHAR"
|
|
106
|
+
if isinstance(value, bool):
|
|
107
|
+
return "BOOLEAN"
|
|
108
|
+
if isinstance(value, int):
|
|
109
|
+
return "BIGINT"
|
|
110
|
+
if isinstance(value, (float, Decimal)):
|
|
111
|
+
return "DOUBLE"
|
|
112
|
+
if isinstance(value, datetime):
|
|
113
|
+
return "TIMESTAMP"
|
|
114
|
+
if isinstance(value, (list, dict)):
|
|
115
|
+
return "JSON"
|
|
116
|
+
return "VARCHAR"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
120
|
+
# Abstract base class
|
|
121
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _DuckDBResultCacheBase:
|
|
125
|
+
"""Abstract base for DuckDB-backed result caches.
|
|
126
|
+
|
|
127
|
+
Manages: connection lifecycle, failure store (_query_failures).
|
|
128
|
+
|
|
129
|
+
Subclasses must implement: get, put, and any storage-specific setup
|
|
130
|
+
(_drop_*_tables, _ensure_result_table, _insert_rows).
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
def __init__(
|
|
134
|
+
self,
|
|
135
|
+
db_path: Path | None = None,
|
|
136
|
+
failure_ttl_seconds: int = 900,
|
|
137
|
+
result_ttl_seconds: int | None = None,
|
|
138
|
+
*,
|
|
139
|
+
_class_name: str,
|
|
140
|
+
) -> None:
|
|
141
|
+
if not HAS_DUCKDB:
|
|
142
|
+
raise ImportError(
|
|
143
|
+
f"DuckDB is required to use {_class_name}. "
|
|
144
|
+
f"Install it with: {install_hint('duckdb')}"
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
self.db_path = db_path
|
|
148
|
+
self.failure_ttl_seconds = failure_ttl_seconds
|
|
149
|
+
self.result_ttl_seconds = result_ttl_seconds
|
|
150
|
+
self._lock = threading.RLock()
|
|
151
|
+
|
|
152
|
+
if db_path:
|
|
153
|
+
self.conn = duckdb.connect(str(db_path))
|
|
154
|
+
else:
|
|
155
|
+
self.conn = duckdb.connect(":memory:")
|
|
156
|
+
|
|
157
|
+
self.conn.execute("SET enable_object_cache = true")
|
|
158
|
+
|
|
159
|
+
def get_failure(
|
|
160
|
+
self, source_hash: str, query_hash: str, variables_hash: str
|
|
161
|
+
) -> CachedQueryFailure | None:
|
|
162
|
+
"""Return a cached failure if within TTL, else None."""
|
|
163
|
+
if self.failure_ttl_seconds == 0:
|
|
164
|
+
return None
|
|
165
|
+
with self._lock:
|
|
166
|
+
row = self.conn.execute(
|
|
167
|
+
"""
|
|
168
|
+
SELECT error_class, error_message, traceback, failed_at
|
|
169
|
+
FROM _query_failures
|
|
170
|
+
WHERE source_hash = ? AND query_hash = ? AND variables_hash = ?
|
|
171
|
+
""",
|
|
172
|
+
[source_hash, query_hash, variables_hash],
|
|
173
|
+
).fetchone()
|
|
174
|
+
if row is None:
|
|
175
|
+
return None
|
|
176
|
+
error_class, error_message, traceback, failed_at = row
|
|
177
|
+
elapsed = (datetime.now() - failed_at).total_seconds()
|
|
178
|
+
if elapsed > self.failure_ttl_seconds:
|
|
179
|
+
return None
|
|
180
|
+
return CachedQueryFailure(
|
|
181
|
+
error_class=error_class,
|
|
182
|
+
error_message=error_message,
|
|
183
|
+
traceback=traceback,
|
|
184
|
+
failed_at=failed_at,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
def put_failure(
|
|
188
|
+
self,
|
|
189
|
+
source_hash: str,
|
|
190
|
+
query_hash: str,
|
|
191
|
+
variables_hash: str,
|
|
192
|
+
exception: Exception,
|
|
193
|
+
*,
|
|
194
|
+
face_slug: str,
|
|
195
|
+
query_name: str,
|
|
196
|
+
) -> None:
|
|
197
|
+
"""Store a query failure keyed by (source_hash, query_hash, variables_hash)."""
|
|
198
|
+
with self._lock:
|
|
199
|
+
self.conn.execute(
|
|
200
|
+
"""
|
|
201
|
+
INSERT OR REPLACE INTO _query_failures
|
|
202
|
+
(source_hash, query_hash, variables_hash,
|
|
203
|
+
face_slug, query_name,
|
|
204
|
+
error_class, error_message, traceback, failed_at)
|
|
205
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
206
|
+
""",
|
|
207
|
+
[
|
|
208
|
+
source_hash,
|
|
209
|
+
query_hash,
|
|
210
|
+
variables_hash,
|
|
211
|
+
face_slug,
|
|
212
|
+
query_name,
|
|
213
|
+
type(exception).__name__,
|
|
214
|
+
str(exception),
|
|
215
|
+
"".join(tb_mod.format_exception(exception)),
|
|
216
|
+
datetime.now(),
|
|
217
|
+
],
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
def clear(self, source_hash: str, query_hash: str, variables_hash: str) -> None:
|
|
221
|
+
"""Remove success and failure entries for this key."""
|
|
222
|
+
with self._lock:
|
|
223
|
+
self.conn.execute(
|
|
224
|
+
"DELETE FROM _query_failures WHERE source_hash = ? AND query_hash = ? AND variables_hash = ?",
|
|
225
|
+
[source_hash, query_hash, variables_hash],
|
|
226
|
+
)
|
|
227
|
+
tbl = _result_table_name(source_hash, query_hash, variables_hash)
|
|
228
|
+
if self._table_exists(tbl):
|
|
229
|
+
self.conn.execute(f"DROP TABLE {_q(tbl)}")
|
|
230
|
+
|
|
231
|
+
def close(self) -> None:
|
|
232
|
+
with self._lock:
|
|
233
|
+
self.conn.close()
|
|
234
|
+
|
|
235
|
+
def _ensure_schema(self) -> None:
|
|
236
|
+
"""Create the system tables if they don't exist."""
|
|
237
|
+
self.conn.execute(
|
|
238
|
+
"""
|
|
239
|
+
CREATE TABLE IF NOT EXISTS _query_failures (
|
|
240
|
+
source_hash TEXT NOT NULL,
|
|
241
|
+
query_hash TEXT NOT NULL,
|
|
242
|
+
variables_hash TEXT NOT NULL,
|
|
243
|
+
face_slug TEXT NOT NULL,
|
|
244
|
+
query_name TEXT NOT NULL,
|
|
245
|
+
error_class TEXT NOT NULL,
|
|
246
|
+
error_message TEXT NOT NULL,
|
|
247
|
+
traceback TEXT,
|
|
248
|
+
failed_at TIMESTAMP NOT NULL,
|
|
249
|
+
PRIMARY KEY (source_hash, query_hash, variables_hash)
|
|
250
|
+
)
|
|
251
|
+
"""
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
def _table_exists(self, name: str) -> bool:
|
|
255
|
+
try:
|
|
256
|
+
result = self.conn.execute(
|
|
257
|
+
"SELECT COUNT(*) FROM information_schema.tables WHERE table_name = ?",
|
|
258
|
+
[name],
|
|
259
|
+
).fetchone()
|
|
260
|
+
return bool(result and result[0] > 0)
|
|
261
|
+
except duckdb.CatalogException:
|
|
262
|
+
return False
|
|
263
|
+
|
|
264
|
+
def _column_names(self, table_name: str) -> set[str]:
|
|
265
|
+
try:
|
|
266
|
+
rows = self.conn.execute(
|
|
267
|
+
"SELECT column_name FROM information_schema.columns WHERE table_name = ?",
|
|
268
|
+
[table_name],
|
|
269
|
+
).fetchall()
|
|
270
|
+
return {r[0] for r in rows}
|
|
271
|
+
except duckdb.CatalogException:
|
|
272
|
+
return set()
|