codepbi 0.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codepbi/__init__.py +17 -0
- codepbi/ids.py +11 -0
- codepbi/model.py +408 -0
- codepbi/semantic/__init__.py +48 -0
- codepbi/semantic/check_drift.py +48 -0
- codepbi/semantic/check_schema_vs_sql.py +61 -0
- codepbi/semantic/config.py +63 -0
- codepbi/semantic/dbt_sql.py +69 -0
- codepbi/semantic/schema_loader.py +138 -0
- codepbi/semantic/sync_descriptions.py +147 -0
- codepbi/semantic/sync_measures.py +127 -0
- codepbi/semantic/sync_relationships.py +232 -0
- codepbi/semantic/tom_utils.py +120 -0
- codepbi/semantic/validate_tmdl.py +231 -0
- codepbi/writer.py +238 -0
- codepbi-0.1.7.dist-info/METADATA +73 -0
- codepbi-0.1.7.dist-info/RECORD +20 -0
- codepbi-0.1.7.dist-info/WHEEL +5 -0
- codepbi-0.1.7.dist-info/licenses/LICENSE +21 -0
- codepbi-0.1.7.dist-info/top_level.txt +1 -0
codepbi/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from .model import (
|
|
2
|
+
Field,
|
|
3
|
+
Filter,
|
|
4
|
+
Page,
|
|
5
|
+
Position,
|
|
6
|
+
Report,
|
|
7
|
+
Visual,
|
|
8
|
+
column_width,
|
|
9
|
+
drillthrough_binding,
|
|
10
|
+
field_ref,
|
|
11
|
+
visual_interaction,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"Field", "Filter", "Page", "Position", "Report", "Visual",
|
|
16
|
+
"column_width", "drillthrough_binding", "field_ref", "visual_interaction",
|
|
17
|
+
]
|
codepbi/ids.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import secrets
|
|
2
|
+
import uuid
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def hex_id(length: int = 20) -> str:
|
|
6
|
+
"""Matches the folder-name id style PBI Desktop generates for pages/visuals (e.g. '953e48bb4f7e4f9a82ce')."""
|
|
7
|
+
return secrets.token_hex(length // 2)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def guid() -> str:
|
|
11
|
+
return str(uuid.uuid4())
|
codepbi/model.py
ADDED
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
from .ids import hex_id
|
|
6
|
+
|
|
7
|
+
# Single-role visuals: visualType -> the one query role every projected field goes under.
|
|
8
|
+
# slicer was wrongly "Field" in earlier versions -- confirmed "Values" (like everything else
|
|
9
|
+
# here) via a real Desktop-saved slicer. See docs/codepbi-lessons-learned.md in the project
|
|
10
|
+
# that found this.
|
|
11
|
+
SINGLE_ROLE_KEY = {
|
|
12
|
+
"slicer": "Values",
|
|
13
|
+
"tableEx": "Values",
|
|
14
|
+
"card": "Values",
|
|
15
|
+
"multiRowCard": "Values",
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def field_ref(entity: str, property_: str, is_measure: bool = False) -> dict:
|
|
20
|
+
"""The {"Column": {...}} / {"Measure": {...}} expression wrapper every field reference
|
|
21
|
+
uses -- shared by projections, filters, and drillthrough page bindings."""
|
|
22
|
+
wrapper = "Measure" if is_measure else "Column"
|
|
23
|
+
return {
|
|
24
|
+
wrapper: {
|
|
25
|
+
"Expression": {"SourceRef": {"Entity": entity}},
|
|
26
|
+
"Property": property_,
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def drillthrough_binding(fields: list[Field], name: str = "drillthrough") -> dict:
|
|
32
|
+
"""pageBinding for a page with page_type="Drillthrough". One BindingParameter per field
|
|
33
|
+
that should trigger/filter the drillthrough page. Structure confirmed against the
|
|
34
|
+
published page/2.1.0 JSON Schema (pageBinding/BindingParameter definitions), not inferred
|
|
35
|
+
from an example.
|
|
36
|
+
|
|
37
|
+
Usage: page.page_type = "Drillthrough"; page.page_binding = drillthrough_binding([field])
|
|
38
|
+
"""
|
|
39
|
+
return {
|
|
40
|
+
"name": name,
|
|
41
|
+
"type": "Drillthrough",
|
|
42
|
+
"parameters": [
|
|
43
|
+
{"name": f.query_ref, "fieldExpr": field_ref(f.entity, f.property, f.is_measure)}
|
|
44
|
+
for f in fields
|
|
45
|
+
],
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
INTERACTION_TYPES = {"Default", "DataFilter", "HighlightFilter", "NoFilter"}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def visual_interaction(source: Visual, target: Visual, interaction_type: str) -> dict:
|
|
53
|
+
"""A page.visual_interactions entry overriding how selecting a data point on `source`
|
|
54
|
+
affects `target`. interaction_type is one of INTERACTION_TYPES. Structure confirmed
|
|
55
|
+
against the published page/2.1.0 JSON Schema (VisualInteraction definition).
|
|
56
|
+
|
|
57
|
+
Usage: page.visual_interactions.append(visual_interaction(chart_a, chart_b, "NoFilter"))
|
|
58
|
+
"""
|
|
59
|
+
if interaction_type not in INTERACTION_TYPES:
|
|
60
|
+
raise ValueError(f"interaction_type must be one of {sorted(INTERACTION_TYPES)}")
|
|
61
|
+
return {"source": source.name, "target": target.name, "type": interaction_type}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _literal(value: str) -> dict:
|
|
65
|
+
return {"expr": {"Literal": {"Value": value}}}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def column_width(query_ref: str, pixels: float) -> dict:
|
|
69
|
+
"""An `objects.columnWidth` entry for a table/matrix column, keyed by the column's
|
|
70
|
+
queryRef (e.g. "Organizations.Organization Name" -- the same string Field.query_ref
|
|
71
|
+
produces). Confirmed from a real Desktop-saved manual column resize: table/matrix
|
|
72
|
+
columns are NOT sized from container width alone -- Desktop persists an explicit
|
|
73
|
+
per-column pixel width once you drag one, and that's the only mechanical lever
|
|
74
|
+
available (there's no formula from container width + data, since actual column
|
|
75
|
+
width depends on live data this generator never sees).
|
|
76
|
+
|
|
77
|
+
Usage: matrix.objects["columnWidth"] = [column_width("Orders.Region", 200)]
|
|
78
|
+
"""
|
|
79
|
+
return {
|
|
80
|
+
"properties": {"value": {"expr": {"Literal": {"Value": f"{pixels}D"}}}},
|
|
81
|
+
"selector": {"metadata": query_ref},
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class Field:
|
|
87
|
+
"""A column or measure reference projected onto a visual, e.g.
|
|
88
|
+
Field("Projects", "Project Title") for a column, or
|
|
89
|
+
Field("Orders", "Total Sales", is_measure=True) for a measure."""
|
|
90
|
+
|
|
91
|
+
entity: str
|
|
92
|
+
property: str
|
|
93
|
+
native_name: str | None = None
|
|
94
|
+
is_measure: bool = False
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def query_ref(self) -> str:
|
|
98
|
+
return f"{self.entity}.{self.property}"
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def native_query_ref(self) -> str:
|
|
102
|
+
return self.native_name or self.property
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass
|
|
106
|
+
class Position:
|
|
107
|
+
x: float
|
|
108
|
+
y: float
|
|
109
|
+
width: float
|
|
110
|
+
height: float
|
|
111
|
+
z: float = 0
|
|
112
|
+
tab_order: float | None = None
|
|
113
|
+
|
|
114
|
+
def __post_init__(self) -> None:
|
|
115
|
+
if self.tab_order is None:
|
|
116
|
+
self.tab_order = self.z
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass
|
|
120
|
+
class Filter:
|
|
121
|
+
"""A filter pane entry on a visual. Use `categorical` to just expose a field as filterable,
|
|
122
|
+
or `equals` to pre-apply a fixed condition (matches what PBI Desktop itself emits)."""
|
|
123
|
+
|
|
124
|
+
entity: str
|
|
125
|
+
property: str
|
|
126
|
+
kind: str # "Categorical" or "Advanced"
|
|
127
|
+
value: str | None = None
|
|
128
|
+
name: str = field(default_factory=hex_id)
|
|
129
|
+
|
|
130
|
+
@staticmethod
|
|
131
|
+
def categorical(entity: str, property: str) -> Filter:
|
|
132
|
+
return Filter(entity, property, kind="Categorical")
|
|
133
|
+
|
|
134
|
+
@staticmethod
|
|
135
|
+
def equals(entity: str, property: str, value: str) -> Filter:
|
|
136
|
+
return Filter(entity, property, kind="Advanced", value=value)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass
|
|
140
|
+
class Visual:
|
|
141
|
+
"""roles maps a visual-type-specific query role name (e.g. "Category", "Y", "Series" for a
|
|
142
|
+
chart; "Values" for a table/card; "Field" for a slicer) to the fields projected onto it.
|
|
143
|
+
Role names are NOT uniform across visual types.
|
|
144
|
+
|
|
145
|
+
objects / visual_container_objects are passed straight through to the visual's own JSON
|
|
146
|
+
(per-visual-type formatting and title/background/border respectively)."""
|
|
147
|
+
|
|
148
|
+
visual_type: str
|
|
149
|
+
roles: dict[str, list[Field]]
|
|
150
|
+
position: Position
|
|
151
|
+
filters: list[Filter] = field(default_factory=list)
|
|
152
|
+
name: str = field(default_factory=hex_id)
|
|
153
|
+
drill_filter_other_visuals: bool = True
|
|
154
|
+
objects: dict = field(default_factory=dict)
|
|
155
|
+
visual_container_objects: dict = field(default_factory=dict)
|
|
156
|
+
# If set, this container IS a group (visual_type/roles are ignored) -- confirmed via the
|
|
157
|
+
# visualContainer schema: {"displayName": str, "groupMode": "ScaleMode" | "ScrollMode"}.
|
|
158
|
+
visual_group: dict | None = None
|
|
159
|
+
# Name of the group Visual this one belongs to, if any.
|
|
160
|
+
parent_group_name: str | None = None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@dataclass
|
|
164
|
+
class Page:
|
|
165
|
+
display_name: str
|
|
166
|
+
name: str = field(default_factory=hex_id)
|
|
167
|
+
height: float = 720
|
|
168
|
+
width: float = 1280
|
|
169
|
+
display_option: str = "FitToPage"
|
|
170
|
+
visuals: list[Visual] = field(default_factory=list)
|
|
171
|
+
# "Drillthrough" or "Tooltip"; pairs with page_binding. None = a normal page.
|
|
172
|
+
page_type: str | None = None
|
|
173
|
+
page_binding: dict | None = None
|
|
174
|
+
# Overrides for how selecting a data point on one visual affects another -- see
|
|
175
|
+
# visual_interaction(). Default (no entry) leaves Desktop's own inferred behavior.
|
|
176
|
+
visual_interactions: list[dict] = field(default_factory=list)
|
|
177
|
+
# Page-level formatting passthrough, e.g. {"background": [{"properties": {...}}]} --
|
|
178
|
+
# confirmed property names (pageInformation, pageSize, background, displayArea, outspace,
|
|
179
|
+
# outspacePane, filterCard, pageRefresh, personalizeVisual) via the published page schema.
|
|
180
|
+
objects: dict = field(default_factory=dict)
|
|
181
|
+
# Page-level filters, applied on top of visual-level filters and under report-level ones.
|
|
182
|
+
# Same FilterContainer shape as Visual.filters (confirmed via filterConfiguration schema).
|
|
183
|
+
filters: list[Filter] = field(default_factory=list)
|
|
184
|
+
|
|
185
|
+
def add_visual(
|
|
186
|
+
self,
|
|
187
|
+
visual_type: str,
|
|
188
|
+
roles: dict[str, list[Field]],
|
|
189
|
+
position: Position,
|
|
190
|
+
filters: list[Filter] | None = None,
|
|
191
|
+
objects: dict | None = None,
|
|
192
|
+
visual_container_objects: dict | None = None,
|
|
193
|
+
) -> Visual:
|
|
194
|
+
visual = Visual(
|
|
195
|
+
visual_type, roles, position, filters or [],
|
|
196
|
+
objects=objects or {}, visual_container_objects=visual_container_objects or {},
|
|
197
|
+
)
|
|
198
|
+
self.visuals.append(visual)
|
|
199
|
+
return visual
|
|
200
|
+
|
|
201
|
+
def add_group(self, display_name: str, position: Position, group_mode: str = "ScaleMode") -> Visual:
|
|
202
|
+
"""A grouping container -- add visuals normally, then set their `.parent_group_name`
|
|
203
|
+
to this group's `.name` to make them members."""
|
|
204
|
+
group = Visual(
|
|
205
|
+
visual_type="", roles={}, position=position,
|
|
206
|
+
visual_group={"displayName": display_name, "groupMode": group_mode},
|
|
207
|
+
)
|
|
208
|
+
self.visuals.append(group)
|
|
209
|
+
return group
|
|
210
|
+
|
|
211
|
+
def add_text_box(
|
|
212
|
+
self, text: str, position: Position, font_size: str | None = None, bold: bool = False,
|
|
213
|
+
) -> Visual:
|
|
214
|
+
"""A static text box -- confirmed shape from a real Desktop-saved textbox. No query
|
|
215
|
+
roles; content lives entirely in objects.general.paragraphs. Note the text `value` is
|
|
216
|
+
a plain string here, NOT the usual Literal/expr wrapper every other property uses.
|
|
217
|
+
There is no vertical-alignment property for this visual (checked, and confirmed absent
|
|
218
|
+
in a real example) -- center it relative to other visuals by sizing its box tightly
|
|
219
|
+
around the text and aligning box centers, not by looking for an alignment property."""
|
|
220
|
+
text_style = {}
|
|
221
|
+
if font_size:
|
|
222
|
+
text_style["fontSize"] = font_size
|
|
223
|
+
if bold:
|
|
224
|
+
text_style["fontWeight"] = "bold"
|
|
225
|
+
text_run = {"value": text}
|
|
226
|
+
if text_style:
|
|
227
|
+
text_run["textStyle"] = text_style
|
|
228
|
+
objects = {"general": [{"properties": {"paragraphs": [{"textRuns": [text_run]}]}}]}
|
|
229
|
+
return self.add_visual("textbox", {}, position, objects=objects)
|
|
230
|
+
|
|
231
|
+
def add_image(
|
|
232
|
+
self, resource_name: str, position: Position,
|
|
233
|
+
display_name: str | None = None, scaling: str = "Normal",
|
|
234
|
+
) -> Visual:
|
|
235
|
+
"""A static image, referencing a resource already registered via
|
|
236
|
+
Report.add_image_resource(resource_name, local_file_path). Confirmed shape from a real
|
|
237
|
+
Desktop-saved image visual (logo). No query roles."""
|
|
238
|
+
objects = {
|
|
239
|
+
"image": [{
|
|
240
|
+
"properties": {
|
|
241
|
+
"sourceType": _literal("'image'"),
|
|
242
|
+
"sourceFile": {
|
|
243
|
+
"image": {
|
|
244
|
+
"name": _literal(f"'{display_name or resource_name}'"),
|
|
245
|
+
"url": {
|
|
246
|
+
"expr": {
|
|
247
|
+
"ResourcePackageItem": {
|
|
248
|
+
"PackageName": "RegisteredResources",
|
|
249
|
+
"PackageType": 1,
|
|
250
|
+
"ItemName": resource_name,
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
},
|
|
254
|
+
"scaling": _literal(f"'{scaling}'"),
|
|
255
|
+
}
|
|
256
|
+
},
|
|
257
|
+
}
|
|
258
|
+
}]
|
|
259
|
+
}
|
|
260
|
+
return self.add_visual("image", {}, position, objects=objects)
|
|
261
|
+
|
|
262
|
+
def _add_single_role(self, visual_type: str, fields: list[Field], position: Position,
|
|
263
|
+
filters: list[Filter] | None = None) -> Visual:
|
|
264
|
+
return self.add_visual(visual_type, {SINGLE_ROLE_KEY[visual_type]: fields}, position, filters)
|
|
265
|
+
|
|
266
|
+
def add_table(self, fields: list[Field], position: Position, filters: list[Filter] | None = None) -> Visual:
|
|
267
|
+
return self._add_single_role("tableEx", fields, position, filters)
|
|
268
|
+
|
|
269
|
+
def add_slicer(self, field_: Field, position: Position) -> Visual:
|
|
270
|
+
return self._add_single_role("slicer", [field_], position)
|
|
271
|
+
|
|
272
|
+
def add_card(self, field_: Field, position: Position) -> Visual:
|
|
273
|
+
return self._add_single_role("card", [field_], position)
|
|
274
|
+
|
|
275
|
+
def add_multi_row_card(self, fields: list[Field], position: Position) -> Visual:
|
|
276
|
+
return self._add_single_role("multiRowCard", fields, position)
|
|
277
|
+
|
|
278
|
+
def add_matrix(
|
|
279
|
+
self, rows: list[Field], values: list[Field], position: Position,
|
|
280
|
+
columns: list[Field] | None = None,
|
|
281
|
+
) -> Visual:
|
|
282
|
+
roles = {"Rows": rows, "Values": values}
|
|
283
|
+
if columns:
|
|
284
|
+
roles["Columns"] = columns
|
|
285
|
+
return self.add_visual("pivotTable", roles, position)
|
|
286
|
+
|
|
287
|
+
def add_bar_chart(
|
|
288
|
+
self, category: Field, values: list[Field], position: Position,
|
|
289
|
+
series: Field | None = None, clustered: bool = True,
|
|
290
|
+
) -> Visual:
|
|
291
|
+
roles = {"Category": [category], "Y": values}
|
|
292
|
+
if series:
|
|
293
|
+
roles["Series"] = [series]
|
|
294
|
+
visual_type = "clusteredBarChart" if clustered else "barChart"
|
|
295
|
+
return self.add_visual(visual_type, roles, position)
|
|
296
|
+
|
|
297
|
+
def add_column_chart(
|
|
298
|
+
self, category: Field, values: list[Field], position: Position,
|
|
299
|
+
series: Field | None = None, clustered: bool = True,
|
|
300
|
+
) -> Visual:
|
|
301
|
+
roles = {"Category": [category], "Y": values}
|
|
302
|
+
if series:
|
|
303
|
+
roles["Series"] = [series]
|
|
304
|
+
visual_type = "clusteredColumnChart" if clustered else "columnChart"
|
|
305
|
+
return self.add_visual(visual_type, roles, position)
|
|
306
|
+
|
|
307
|
+
def add_line_chart(
|
|
308
|
+
self, category: Field, values: list[Field], position: Position,
|
|
309
|
+
series: Field | None = None,
|
|
310
|
+
) -> Visual:
|
|
311
|
+
roles = {"Category": [category], "Y": values}
|
|
312
|
+
if series:
|
|
313
|
+
roles["Series"] = [series]
|
|
314
|
+
return self.add_visual("lineChart", roles, position)
|
|
315
|
+
|
|
316
|
+
def add_combo_chart(
|
|
317
|
+
self, category: Field, column_values: list[Field], line_values: list[Field],
|
|
318
|
+
position: Position,
|
|
319
|
+
) -> Visual:
|
|
320
|
+
roles = {"Category": [category], "Y": column_values, "Y2": line_values}
|
|
321
|
+
return self.add_visual("lineClusteredColumnComboChart", roles, position)
|
|
322
|
+
|
|
323
|
+
def add_pie_chart(
|
|
324
|
+
self, category: Field, values: list[Field], position: Position, donut: bool = False,
|
|
325
|
+
) -> Visual:
|
|
326
|
+
roles = {"Category": [category], "Y": values}
|
|
327
|
+
return self.add_visual("donutChart" if donut else "pieChart", roles, position)
|
|
328
|
+
|
|
329
|
+
def add_scatter_chart(
|
|
330
|
+
self, x: Field, y: Field, position: Position,
|
|
331
|
+
category: Field | None = None, size: Field | None = None,
|
|
332
|
+
) -> Visual:
|
|
333
|
+
# Desktop's UI labels this well "Details", but (like every other chart here) the
|
|
334
|
+
# underlying role key is "Category" -- see docs/codepbi-lessons-learned.md.
|
|
335
|
+
roles = {"X": [x], "Y": [y]}
|
|
336
|
+
if category:
|
|
337
|
+
roles["Category"] = [category]
|
|
338
|
+
if size:
|
|
339
|
+
roles["Size"] = [size]
|
|
340
|
+
return self.add_visual("scatterChart", roles, position)
|
|
341
|
+
|
|
342
|
+
def add_gauge(
|
|
343
|
+
self, value: Field, position: Position,
|
|
344
|
+
min_value: Field | None = None, max_value: Field | None = None, target_value: Field | None = None,
|
|
345
|
+
) -> Visual:
|
|
346
|
+
roles = {"Y": [value]}
|
|
347
|
+
if min_value:
|
|
348
|
+
roles["MinValue"] = [min_value]
|
|
349
|
+
if max_value:
|
|
350
|
+
roles["MaxValue"] = [max_value]
|
|
351
|
+
if target_value:
|
|
352
|
+
roles["TargetValue"] = [target_value]
|
|
353
|
+
return self.add_visual("gauge", roles, position)
|
|
354
|
+
|
|
355
|
+
def add_treemap(
|
|
356
|
+
self, group: Field, values: list[Field], position: Position, details: Field | None = None,
|
|
357
|
+
) -> Visual:
|
|
358
|
+
roles = {"Group": [group], "Values": values}
|
|
359
|
+
if details:
|
|
360
|
+
roles["Details"] = [details]
|
|
361
|
+
return self.add_visual("treemap", roles, position)
|
|
362
|
+
|
|
363
|
+
def add_filled_map(
|
|
364
|
+
self, location: Field, position: Position,
|
|
365
|
+
values: list[Field] | None = None, legend: Field | None = None,
|
|
366
|
+
) -> Visual:
|
|
367
|
+
roles = {"Category": [location]}
|
|
368
|
+
if values:
|
|
369
|
+
roles["Values"] = values
|
|
370
|
+
if legend:
|
|
371
|
+
roles["Legend"] = [legend]
|
|
372
|
+
return self.add_visual("filledMap", roles, position)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
@dataclass
|
|
376
|
+
class Report:
|
|
377
|
+
name: str
|
|
378
|
+
semantic_model_path: str
|
|
379
|
+
pages: list[Page] = field(default_factory=list)
|
|
380
|
+
active_page_name: str | None = None
|
|
381
|
+
# A custom color theme -- Microsoft's public Report Theme JSON format
|
|
382
|
+
# (https://learn.microsoft.com/power-bi/create-reports/desktop-report-themes), e.g.
|
|
383
|
+
# {"name": "Brand", "dataColors": ["#005493", "#008fb4", ...]}. None = Desktop's default
|
|
384
|
+
# theme (still valid -- themeCollection is required but can be empty, see
|
|
385
|
+
# docs/codepbi-lessons-learned.md in the consuming project).
|
|
386
|
+
theme: dict | None = None
|
|
387
|
+
# Report-level filters, applied on top of page- and visual-level filters. Same
|
|
388
|
+
# FilterContainer shape as Visual/Page filters -- confirmed via the filterConfiguration
|
|
389
|
+
# schema (report.json's top-level "filterConfig" property).
|
|
390
|
+
filters: list[Filter] = field(default_factory=list)
|
|
391
|
+
# (resource_name, local_file_path) pairs bundled into the report's RegisteredResources
|
|
392
|
+
# package (type "Image") -- reference one from a page via Page.add_image(resource_name, ...).
|
|
393
|
+
images: list[tuple[str, str]] = field(default_factory=list)
|
|
394
|
+
|
|
395
|
+
def add_image_resource(self, name: str, file_path: str) -> None:
|
|
396
|
+
self.images.append((name, file_path))
|
|
397
|
+
|
|
398
|
+
def add_page(self, display_name: str, **kwargs) -> Page:
|
|
399
|
+
page = Page(display_name=display_name, **kwargs)
|
|
400
|
+
self.pages.append(page)
|
|
401
|
+
if self.active_page_name is None:
|
|
402
|
+
self.active_page_name = page.name
|
|
403
|
+
return page
|
|
404
|
+
|
|
405
|
+
def save(self, parent_dir: str) -> str:
|
|
406
|
+
from .writer import write_report
|
|
407
|
+
|
|
408
|
+
return write_report(self, parent_dir)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Sync a Power BI TMDL semantic model from a dbt project.
|
|
2
|
+
|
|
3
|
+
Treats dbt's schema YAML (`meta.entity` / `meta.grain` / `meta.display_name` / ...)
|
|
4
|
+
as the source of truth for columns, and a dbt-semantic-layer-style
|
|
5
|
+
`semantic_models/*.yml` (entities + measures) as the source of truth for joins
|
|
6
|
+
and DAX measures. Applies both to TMDL via the Tabular Object Model (TOM), so
|
|
7
|
+
Power BI Desktop keeps working normally against the result.
|
|
8
|
+
|
|
9
|
+
Typical flow:
|
|
10
|
+
|
|
11
|
+
from codepbi.semantic import TmdlProjectConfig, check_drift, sync_descriptions, \\
|
|
12
|
+
sync_measures, sync_relationships
|
|
13
|
+
|
|
14
|
+
config = TmdlProjectConfig(
|
|
15
|
+
dbt_project_dir="path/to/dbt_project",
|
|
16
|
+
semantic_model_definition_dir="path/to/Model.SemanticModel/definition",
|
|
17
|
+
tom_lib_dir="path/to/analysis-services-dlls",
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
sync_measures.sync(config)
|
|
21
|
+
sync_descriptions.sync(config) # also runs validate_tmdl at the end
|
|
22
|
+
sync_relationships.sync(config)
|
|
23
|
+
check_drift.check(config)
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from . import (
|
|
27
|
+
check_drift,
|
|
28
|
+
check_schema_vs_sql,
|
|
29
|
+
schema_loader,
|
|
30
|
+
sync_descriptions,
|
|
31
|
+
sync_measures,
|
|
32
|
+
sync_relationships,
|
|
33
|
+
tom_utils,
|
|
34
|
+
validate_tmdl,
|
|
35
|
+
)
|
|
36
|
+
from .config import TmdlProjectConfig
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"TmdlProjectConfig",
|
|
40
|
+
"check_drift",
|
|
41
|
+
"check_schema_vs_sql",
|
|
42
|
+
"schema_loader",
|
|
43
|
+
"sync_descriptions",
|
|
44
|
+
"sync_measures",
|
|
45
|
+
"sync_relationships",
|
|
46
|
+
"tom_utils",
|
|
47
|
+
"validate_tmdl",
|
|
48
|
+
]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# CI drift detection: compares dbt columns (schema YAML) against TMDL sourceColumns.
|
|
2
|
+
# Returns 1 if TMDL references columns that don't exist in dbt.
|
|
3
|
+
|
|
4
|
+
from . import tom_utils
|
|
5
|
+
from .config import TmdlProjectConfig
|
|
6
|
+
from .schema_loader import load_schema
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def check(config: TmdlProjectConfig) -> int:
|
|
10
|
+
dbt_models = {
|
|
11
|
+
pbi_table: {col["source"] for col in spec["columns"]}
|
|
12
|
+
for pbi_table, spec in load_schema(config).items()
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
errors = []
|
|
16
|
+
warnings = []
|
|
17
|
+
|
|
18
|
+
for table_name, table in sorted(tom_utils.read_tables(config).items()):
|
|
19
|
+
if table_name not in dbt_models:
|
|
20
|
+
continue
|
|
21
|
+
|
|
22
|
+
dbt_cols = dbt_models[table_name]
|
|
23
|
+
tmdl_sources = {col["source"] for col in table["columns"].values() if col["source"]}
|
|
24
|
+
|
|
25
|
+
# TMDL references columns not in dbt (breaking -- column was removed/renamed)
|
|
26
|
+
orphaned = tmdl_sources - dbt_cols
|
|
27
|
+
for col in sorted(orphaned):
|
|
28
|
+
errors.append(f' {table_name}: TMDL sourceColumn "{col}" not found in schema YAML')
|
|
29
|
+
|
|
30
|
+
# dbt has columns not in TMDL (informational -- new column not yet exposed in PBI)
|
|
31
|
+
missing = dbt_cols - tmdl_sources
|
|
32
|
+
for col in sorted(missing):
|
|
33
|
+
warnings.append(f' {table_name}: dbt column "{col}" not in TMDL')
|
|
34
|
+
|
|
35
|
+
if warnings:
|
|
36
|
+
print(f"INFO: {len(warnings)} dbt columns not yet in TMDL:")
|
|
37
|
+
for w in warnings:
|
|
38
|
+
print(w)
|
|
39
|
+
print()
|
|
40
|
+
|
|
41
|
+
if errors:
|
|
42
|
+
print(f"ERROR: {len(errors)} TMDL columns reference missing dbt columns:")
|
|
43
|
+
for e in errors:
|
|
44
|
+
print(e)
|
|
45
|
+
return 1
|
|
46
|
+
|
|
47
|
+
print(f"OK: all TMDL sourceColumns exist in schema YAML ({len(dbt_models)} tables checked)")
|
|
48
|
+
return 0
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Drift detection check_drift can't do: it only compares schema YAML against
|
|
2
|
+
# TMDL, so if schema YAML itself is wrong (a column moved/renamed in a model's
|
|
3
|
+
# SQL without schema YAML being updated to match), TMDL happily syncs to the
|
|
4
|
+
# same wrong name and both "agree" on a column that doesn't exist -- the bug
|
|
5
|
+
# only surfaces later when Power BI queries it live. This checks schema YAML's
|
|
6
|
+
# declared columns against what each model's own compiled SQL actually SELECTs.
|
|
7
|
+
# Requires `dbt compile` (or `dbt run`) to have already produced target/compiled/....
|
|
8
|
+
|
|
9
|
+
from .config import TmdlProjectConfig
|
|
10
|
+
from .dbt_sql import load_model_data, sql_exprs
|
|
11
|
+
from .schema_loader import load_schema
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def check(config: TmdlProjectConfig) -> int:
|
|
15
|
+
if config.dbt_manifest_path is None or config.dbt_compiled_dir is None:
|
|
16
|
+
raise SystemExit(
|
|
17
|
+
"check_schema_vs_sql requires config.dbt_manifest_path and "
|
|
18
|
+
"config.dbt_compiled_dir to be set."
|
|
19
|
+
)
|
|
20
|
+
schema = load_schema(config)
|
|
21
|
+
model_data = load_model_data(config.dbt_manifest_path, config.dbt_compiled_dir)
|
|
22
|
+
errors = []
|
|
23
|
+
checked = 0
|
|
24
|
+
|
|
25
|
+
for table_name, spec in sorted(schema.items()):
|
|
26
|
+
dbt_model = spec["model"]
|
|
27
|
+
if not dbt_model:
|
|
28
|
+
continue
|
|
29
|
+
|
|
30
|
+
if not model_data.get(dbt_model, {}).get("compiled_exists"):
|
|
31
|
+
errors.append(f'{table_name}: no compiled SQL found for model "{dbt_model}" '
|
|
32
|
+
f'(did `dbt compile` run?)')
|
|
33
|
+
continue
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
exprs = sql_exprs(model_data[dbt_model]["sql"])
|
|
37
|
+
except Exception as e:
|
|
38
|
+
errors.append(f'{table_name}: failed to parse compiled SQL for model "{dbt_model}": {e}')
|
|
39
|
+
continue
|
|
40
|
+
|
|
41
|
+
if set(exprs) == {"*"}:
|
|
42
|
+
errors.append(f'{table_name}: model "{dbt_model}" appears to end in a bare '
|
|
43
|
+
f'SELECT * -- cannot verify its columns')
|
|
44
|
+
continue
|
|
45
|
+
|
|
46
|
+
exprs_lower = {k.lower() for k in exprs}
|
|
47
|
+
for col in spec["columns"]:
|
|
48
|
+
checked += 1
|
|
49
|
+
source_col = col["source"]
|
|
50
|
+
if source_col.lower() not in exprs_lower:
|
|
51
|
+
errors.append(f'{table_name}.{col["display"]}: schema YAML source column '
|
|
52
|
+
f'"{source_col}" not found in "{dbt_model}"\'s compiled SELECT list')
|
|
53
|
+
|
|
54
|
+
if errors:
|
|
55
|
+
print(f"ERROR: {len(errors)} schema YAML column(s) not found in their model's compiled SQL:")
|
|
56
|
+
for e in errors:
|
|
57
|
+
print(f" {e}")
|
|
58
|
+
return 1
|
|
59
|
+
|
|
60
|
+
print(f"OK: all schema YAML columns exist in their model's compiled SQL ({checked} columns checked)")
|
|
61
|
+
return 0
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class TmdlProjectConfig:
|
|
9
|
+
"""Points the semantic-sync tooling at a specific dbt project + TMDL semantic model.
|
|
10
|
+
|
|
11
|
+
dbt_project_dir: the dbt project root (contains dbt_project.yml, models/, seeds/).
|
|
12
|
+
semantic_model_definition_dir: the `<Model>.SemanticModel/definition` folder PBI Desktop
|
|
13
|
+
reads/writes (contains model.tmdl, tables/, relationships.tmdl).
|
|
14
|
+
tom_lib_dir: folder containing the Microsoft.AnalysisServices .NET DLLs (from the
|
|
15
|
+
`Microsoft.AnalysisServices.NetCore.retail.amd64` NuGet package) that pythonnet
|
|
16
|
+
loads to talk to TOM. Not redistributed here — obtain separately.
|
|
17
|
+
models_glob: glob (relative to dbt_project_dir) for schema YAML files to read.
|
|
18
|
+
Excludes anything under semantic_models_subdir and any `sources.yml` automatically.
|
|
19
|
+
semantic_models_subdir: relative path to the folder of per-table semantic model YAML
|
|
20
|
+
files (entities + measures, dbt-semantic-layer style) that are the join and
|
|
21
|
+
DAX-measure authority.
|
|
22
|
+
skip_tables: TMDL table names to ignore entirely (e.g. a reference/glossary table with
|
|
23
|
+
no backing dbt model).
|
|
24
|
+
extra_column_meta: additional `meta.*` keys to pass through verbatim on each column dict
|
|
25
|
+
(key -> default value used when a column doesn't declare it), for callers that track
|
|
26
|
+
extra per-column metadata (e.g. data-governance fields) beyond what TMDL sync needs.
|
|
27
|
+
Unused by anything in this package; purely a passthrough so callers don't have to
|
|
28
|
+
re-parse schema YAML themselves just to read one more field.
|
|
29
|
+
extra_table_meta: same idea as extra_column_meta, but for per-table `meta.*` keys.
|
|
30
|
+
dbt_manifest_path: path to target/manifest.json, only needed for check_schema_vs_sql.
|
|
31
|
+
dbt_compiled_dir: folder compiled SQL is actually written under -- dbt's real layout is
|
|
32
|
+
target/compiled/<dbt_project_name>/<model-paths>/..., which isn't generically
|
|
33
|
+
derivable, so this must be the fully resolved path. Only needed for
|
|
34
|
+
check_schema_vs_sql.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
dbt_project_dir: Path
|
|
38
|
+
semantic_model_definition_dir: Path
|
|
39
|
+
tom_lib_dir: Path
|
|
40
|
+
models_glob: str = "models/**/*.yml"
|
|
41
|
+
semantic_models_subdir: str = "models/semantic_models"
|
|
42
|
+
skip_tables: set[str] = field(default_factory=set)
|
|
43
|
+
extra_column_meta: dict[str, str] = field(default_factory=dict)
|
|
44
|
+
extra_table_meta: dict[str, str] = field(default_factory=dict)
|
|
45
|
+
dbt_manifest_path: Path | None = None
|
|
46
|
+
dbt_compiled_dir: Path | None = None
|
|
47
|
+
|
|
48
|
+
def __post_init__(self) -> None:
|
|
49
|
+
self.dbt_project_dir = Path(self.dbt_project_dir)
|
|
50
|
+
self.semantic_model_definition_dir = Path(self.semantic_model_definition_dir)
|
|
51
|
+
self.tom_lib_dir = Path(self.tom_lib_dir)
|
|
52
|
+
if self.dbt_manifest_path is not None:
|
|
53
|
+
self.dbt_manifest_path = Path(self.dbt_manifest_path)
|
|
54
|
+
if self.dbt_compiled_dir is not None:
|
|
55
|
+
self.dbt_compiled_dir = Path(self.dbt_compiled_dir)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def tables_dir(self) -> Path:
|
|
59
|
+
return self.semantic_model_definition_dir / "tables"
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def semantic_models_dir(self) -> Path:
|
|
63
|
+
return self.dbt_project_dir / self.semantic_models_subdir
|