codepbi 0.1.7__py3-none-any.whl

codepbi/semantic/dbt_sql.py

@@ -0,0 +1,69 @@
+# Generic helpers for reading a dbt model's own compiled SELECT list, used by
+# check_schema_vs_sql. Requires `dbt compile` (or `dbt run`) to have already
+# produced target/manifest.json and target/compiled/... .
+
+import json
+from pathlib import Path
+
+from sqlglot import exp, parse_one
+
+
+def sql_exprs(sql: str, dialect: str = "tsql") -> dict:
+    """A model's own SELECT list: {output column: raw SQL expression text}."""
+    stmt = parse_one(sql, dialect=dialect)
+    cte_map = {
+        cte.alias.lower(): {
+            s.alias_or_name.lower(): str(s.this if isinstance(s, exp.Alias) else s)
+            for s in cte.this.selects if s.alias_or_name
+        }
+        for cte in stmt.find_all(exp.CTE)
+    }
+    result = {}
+    for s in stmt.selects:
+        alias = s.alias_or_name
+        expr = s.this if isinstance(s, exp.Alias) else s
+        if isinstance(expr, exp.Column) and not expr.table:
+            result[alias] = next(
+                (c[expr.name.lower()] for c in cte_map.values() if expr.name.lower() in c),
+                str(expr),
+            )
+        else:
+            result[alias] = str(expr)
+    return result
+
+
+def load_model_data(manifest_path: Path, compiled_dir: Path) -> dict:
+    """dbt model name -> {dbt_db, dbt_schema, sql, compiled_path, compiled_exists}.
+
+    manifest_path: path to target/manifest.json.
+    compiled_dir: the folder compiled SQL is actually written under -- dbt's real
+        layout is target/compiled/<dbt_project_name>/<model-paths>/..., and the
+        project name and model-paths config aren't generically derivable, so the
+        caller must resolve this themselves (it's just target/compiled/<your
+        project name>/models for the common case).
+
+    The compiled path comes from the manifest node's own `path` (relative to
+    model-paths), not from the bare model name -- dbt writes to
+    <compiled_dir>/<subfolder>/<name>.sql, so a name-only lookup misses the
+    moment models live in folders.
+
+    The raw_code fallback is kept for models that genuinely have no compiled
+    output, but it is Jinja, not SQL: sqlglot parses it into garbage rather than
+    failing, which yields silently wrong results. compiled_exists lets callers
+    refuse instead.
+    """
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    result = {}
+    for node in manifest["nodes"].values():
+        if node.get("resource_type") != "model":
+            continue
+        p = compiled_dir / node.get("path", f"{node['name']}.sql")
+        exists = p.exists()
+        result[node["name"]] = {
+            "dbt_db": node.get("database", ""),
+            "dbt_schema": node.get("schema", ""),
+            "sql": p.read_text(encoding="utf-8") if exists else node.get("raw_code", ""),
+            "compiled_path": p,
+            "compiled_exists": exists,
+        }
+    return result

codepbi/semantic/schema_loader.py

@@ -0,0 +1,138 @@
+# Shared dbt schema YAML parser. Every generator/sync function that needs column or
+# table metadata calls `load_schema` from here instead of parsing schema YAML
+# itself -- schema YAML is the single source of truth for column metadata, so
+# there is exactly one reader for it.
+
+import yaml
+
+from . import tom_utils
+from .config import TmdlProjectConfig
+
+# schema YAML data_type -> TMDL sourceProviderType + formatString.
+# 'format' is only the default for a bare data_type; meta.format_string overrides it.
+# double has no default format on purpose: a money column must declare one, or it
+# renders as a raw number with no totals.
+DATATYPE_MAP = {
+    "int64": {"provider": "int", "format": "0"},
+    "string": {"provider": "nvarchar(255)", "format": None},
+    "dateTime": {"provider": "datetime2", "format": "General Date"},
+    "boolean": {"provider": "bit", "format": '"TRUE";"TRUE";"FALSE"'},
+    "double": {"provider": "float", "format": None},
+}
+
+# meta.summarize_by -> TOM AggregateFunction member name. 'none' makes a numeric column
+# behave as an attribute: no implicit aggregation and no totals row. That is correct for
+# semi-additive snapshots (a point-in-time balance must not be summed across periods) and
+# wrong for an ordinary money column.
+SUMMARIZE_MAP = {
+    "none": "None", "default": "Default", "sum": "Sum", "average": "Average",
+    "count": "Count", "distinctCount": "DistinctCount", "min": "Min", "max": "Max",
+}
+
+# TOM DataType name -> schema YAML data_type, for the drift check in sync_descriptions.
+TOM_TO_YAML = {"String": "string", "Int64": "int64", "DateTime": "dateTime",
+               "Double": "double", "Boolean": "boolean"}
+
+
+def aggregate_function(value, source):
+    from Microsoft.AnalysisServices.Tabular import AggregateFunction
+
+    if value not in SUMMARIZE_MAP:
+        raise SystemExit(
+            f"Unknown summarize_by {value!r} for column {source!r}. "
+            f"Must be one of: {sorted(SUMMARIZE_MAP)}"
+        )
+    return getattr(AggregateFunction, SUMMARIZE_MAP[value])
+
+
+def _model_blocks(config: TmdlProjectConfig):
+    """Every `models:` entry across the configured schema YAML glob, sorted by model name.
+
+    Lets schema YAML be split across multiple files (e.g. per layer) without every
+    caller needing to know the split. semantic_models_subdir and any `sources.yml`
+    are excluded: they are different documents that also carry a top-level `models`-
+    shaped key we would otherwise misread.
+    """
+    sm_dir = config.semantic_models_dir
+    blocks, seen = [], {}
+    for path in sorted(config.dbt_project_dir.glob(config.models_glob)):
+        if sm_dir in path.parents or path.name == "sources.yml":
+            continue
+        with open(path, encoding="utf-8") as f:
+            doc = yaml.safe_load(f) or {}
+        for model in doc.get("models", []) or []:
+            name = model.get("name")
+            if name in seen:
+                raise SystemExit(
+                    f"duplicate model {name!r} in schema YAML: "
+                    f"{seen[name]} and {path}. "
+                    "A silent last-write-wins here loses a model's column metadata."
+                )
+            seen[name] = path
+            blocks.append(model)
+    blocks.sort(key=lambda m: m.get("name") or "")
+    return blocks
+
+
+def load_schema(config: TmdlProjectConfig) -> dict:
+    """Schema YAML -> {pbi_table: {'entity','grain','columns': [colinfo]}}.
+
+    colinfo carries everything TMDL needs; the schema YAML is the only source of truth.
+    """
+    out = {}
+    for model in _model_blocks(config):
+        meta = model.get("meta", {}) or {}
+        pbi_table = tom_utils.pbi_table_of(meta)
+        if not pbi_table:
+            continue
+        cols = []
+        for col in model.get("columns", []):
+            cm = col.get("meta", {}) or {}
+            colinfo = {
+                "source": col["name"],
+                "display": cm.get("display_name", col["name"]),
+                "desc": col.get("description", ""),
+                "folder": cm.get("displayFolder", ""),
+                "hidden": bool(cm.get("hidden", False)),
+                "data_type": cm.get("data_type"),
+                # None means "not declared" -> leave whatever TMDL has. Declaring these
+                # is what makes a money column survive a table rebuild.
+                "format_string": cm.get("format_string"),
+                "summarize_by": cm.get("summarize_by"),
+            }
+            for key, default in config.extra_column_meta.items():
+                colinfo[key] = cm.get(key, default)
+            cols.append(colinfo)
+        table_info = {
+            "entity": meta.get("entity", ""),
+            "grain": meta.get("grain", ""),
+            # The dbt model name, so generators can emit ref() without reading the
+            # TMDL partition to find out which model backs the table.
+            "model": model["name"],
+            "columns": cols,
+        }
+        for key, default in config.extra_table_meta.items():
+            table_info[key] = meta.get(key, default)
+        out[pbi_table] = table_info
+    return out
+
+
+def model_to_table(config: TmdlProjectConfig) -> dict:
+    return {spec["model"]: name for name, spec in load_schema(config).items()}
+
+
+def iter_semantic_models(config: TmdlProjectConfig):
+    """Yield (path, doc, sm, table_name) for every semantic-model YAML file with a
+    table resolvable via schema YAML. Shared by every function that reads
+    entities/measures out of those files, so the glob-and-parse loop exists once."""
+    m2t = model_to_table(config)
+    for path in sorted(config.semantic_models_dir.glob("*.yml")):
+        if path.name.startswith("_"):
+            continue
+        with open(path, encoding="utf-8") as f:
+            doc = yaml.safe_load(f) or {}
+        for sm in doc.get("semantic_models") or []:
+            dbt_model = sm["model"][len("ref('"):-len("')")]
+            table_name = m2t.get(dbt_model)
+            if table_name:
+                yield path, doc, sm, table_name

codepbi/semantic/sync_descriptions.py

@@ -0,0 +1,147 @@
+# Sync schema YAML -> TMDL columns, descriptions, folders, hidden flags, format,
+# summarization. Inserts columns that don't exist yet. Descriptions are tagged
+# [synced] so hand-written ones (untagged) are never overwritten.
+#
+# TMDL is read and written through TOM (see tom_utils) -- never regex.
+
+import uuid
+
+from . import tom_utils
+from .config import TmdlProjectConfig
+from .schema_loader import DATATYPE_MAP, TOM_TO_YAML, aggregate_function, load_schema
+from .validate_tmdl import validate_tmdl
+
+SYNCED_TAG = "[synced]"
+
+
+def _new_column(info):
+    """Build a TOM DataColumn from a schema YAML column entry."""
+    from Microsoft.AnalysisServices.Tabular import AggregateFunction, Annotation, DataColumn, DataType
+
+    data_type = info["data_type"]
+    if data_type not in DATATYPE_MAP:
+        raise SystemExit(
+            f"Unknown data_type {data_type!r} for column {info['source']!r}. "
+            f"Must be one of: {sorted(DATATYPE_MAP)}"
+        )
+    spec = DATATYPE_MAP[data_type]
+
+    col = DataColumn()
+    col.Name = info["display"]
+    col.DataType = getattr(DataType, data_type[0].upper() + data_type[1:])
+    col.SourceColumn = info["source"]
+    col.SourceProviderType = spec["provider"]
+    fmt = info["format_string"] or spec["format"]
+    if fmt:
+        col.FormatString = fmt
+    if info["folder"]:
+        col.DisplayFolder = info["folder"]
+    if info["hidden"]:
+        col.IsHidden = True
+    if info["desc"]:
+        col.Description = info["desc"]
+    col.LineageTag = str(uuid.uuid4())
+    col.SummarizeBy = (
+        aggregate_function(info["summarize_by"], info["source"])
+        if info["summarize_by"]
+        else getattr(AggregateFunction, "None")
+    )
+
+    ann = Annotation()
+    ann.Name = "SummarizationSetBy"
+    ann.Value = "Automatic"
+    col.Annotations.Add(ann)
+    return col
+
+
+def sync(config: TmdlProjectConfig):
+    """Sync schema YAML -> TMDL: display names, descriptions, folders, hidden flags.
+
+    Inserts columns that don't exist yet. Applies schema YAML first, then validates
+    the result -- validating first would deadlock, since the fix for a TMDL error
+    often lives in schema YAML and can only land by syncing.
+    """
+    wanted = load_schema(config)
+    db, model = tom_utils.load_model(config)
+
+    inserts, renames, updates = [], [], 0
+    type_drift = []
+
+    for pbi_table, spec in wanted.items():
+        if not model.Tables.ContainsName(pbi_table):
+            continue
+        table = model.Tables[pbi_table]
+        by_source = {c.SourceColumn: c for c in table.Columns if c.SourceColumn}
+
+        for info in spec["columns"]:
+            col = by_source.get(info["source"])
+
+            # A column's type is only ever set at creation, so a declaration that
+            # disagrees with TMDL is silently inert. This also catches Power BI
+            # Desktop retyping a column behind your back (it does this sometimes
+            # when a view's inferred type looks different on refresh).
+            if col is not None and info["data_type"]:
+                actual = TOM_TO_YAML.get(str(col.DataType))
+                if actual and actual != info["data_type"]:
+                    type_drift.append(
+                        f"{pbi_table}.{info['display']} ({info['source']}): "
+                        f"schema YAML says {info['data_type']}, TMDL has {actual}"
+                    )
+
+            if col is None:
+                if not info["data_type"]:
+                    raise SystemExit(
+                        f"{pbi_table}: new column {info['source']!r} missing "
+                        f"meta.data_type in schema YAML (required for auto-insert)"
+                    )
+                table.Columns.Add(_new_column(info))
+                inserts.append(f"{pbi_table}.{info['source']}")
+                continue
+
+            if col.Name != info["display"]:
+                renames.append(f"{pbi_table}: {col.Name!r} -> {info['display']!r}")
+                col.Name = info["display"]
+
+            # Only manage descriptions this sync owns -- leave untagged ones alone.
+            current = col.Description or ""
+            if info["desc"] and (not current or SYNCED_TAG in current):
+                updates += tom_utils.set_if_changed(col, "Description", f"{SYNCED_TAG} {info['desc']}")
+
+            updates += tom_utils.set_if_changed(col, "IsHidden", info["hidden"], bool)
+
+            if info["folder"]:
+                updates += tom_utils.set_if_changed(col, "DisplayFolder", info["folder"])
+
+            # Only touch these where schema YAML declares them -- applying a default to
+            # every column would strip the format off columns that already carry one.
+            if info["format_string"]:
+                updates += tom_utils.set_if_changed(col, "FormatString", info["format_string"])
+
+            if info["summarize_by"]:
+                want = aggregate_function(info["summarize_by"], info["source"])
+                updates += tom_utils.set_if_changed(col, "SummarizeBy", want, lambda v: v)
+
+    tom_utils.save_model(config, db)
+
+    print(f"TMDL: {len(inserts)} new column(s) inserted.")
+    for i in inserts:
+        print(f"  inserted {i}")
+    print(f"TMDL: {updates} property update(s).")
+    if renames:
+        print(f"TMDL: {len(renames)} column display-name rename(s):")
+        for r in renames:
+            print(f"  {r}")
+        print("Review DAX/relationship/role refs -- sync only renames the column header.")
+
+    if type_drift:
+        print(f"\nERROR: {len(type_drift)} column data_type mismatch(es):")
+        for d in type_drift:
+            print(f"  {d}")
+        print("\nA column type is only applied when the column is first created, so one "
+              "of the two is wrong and neither will change on its own.")
+        print("Check what the underlying SQL actually returns, then fix whichever side "
+              "disagrees with it. Only dataType is checked -- sourceProviderType is left "
+              "to Power BI Desktop, which legitimately refines it (e.g. datetime2 -> date).")
+        raise SystemExit(1)
+
+    validate_tmdl(config)

codepbi/semantic/sync_measures.py

@@ -0,0 +1,127 @@
+# Apply semantic-model YAML `measures:` -> TMDL, and export back the other way.
+#
+# semantic-model YAML is the measure authority -- DAX lives in each measure's
+# `description` (after the trailing "DAX: " marker), format/folder in
+# `config.meta`. Only touches measures -- columns, partitions, relationships,
+# entities and dimensions untouched. Idempotent: rerun with same config = same
+# output.
+
+import re
+
+import yaml
+
+from .config import TmdlProjectConfig
+from .schema_loader import iter_semantic_models, load_schema
+from .tom_utils import load_model, save_model, set_if_changed
+
+_DAX_RE = re.compile(r"DAX:\s*(.*)$", re.DOTALL)
+
+
+def _dax_from_description(description):
+    m = _DAX_RE.search(description or "")
+    return m.group(1).strip() if m else ""
+
+
+def load_by_table(config: TmdlProjectConfig) -> dict:
+    """semantic-model YAML -> {tmdl_table_name: [{name, dax, formatString, displayFolder}]}."""
+    by_table = {}
+    for _path, _doc, sm, table_name in iter_semantic_models(config):
+        for m in sm.get("measures") or []:
+            meta = ((m.get("config") or {}).get("meta")) or {}
+            by_table.setdefault(table_name, []).append({
+                "name": m.get("label") or m["name"],
+                "dax": _dax_from_description(m.get("description", "")),
+                "formatString": meta.get("format_string", ""),
+                "displayFolder": meta.get("displayFolder", "Calculations"),
+            })
+    return by_table
+
+
+def export(config: TmdlProjectConfig):
+    """Export TMDL measures back into each table's semantic-model YAML, replacing
+    only its `measures:` block -- entities/dimensions/description untouched."""
+    table_to_model = {name: spec["model"] for name, spec in load_schema(config).items()}
+
+    db, model = load_model(config)
+    by_table = {}
+    for table in model.Tables:
+        entries = []
+        for m in table.Measures:
+            entries.append({
+                "name": m.Name,
+                "agg": "sum",
+                "label": m.Name,
+                "description": f"DAX: {m.Expression.strip()}",
+                "config": {"meta": {
+                    "format_string": m.FormatString or "",
+                    "displayFolder": m.DisplayFolder or "Calculations",
+                }},
+            })
+        if entries:
+            entries.sort(key=lambda e: e["name"])
+            by_table[table.Name] = entries
+
+    written = 0
+    for table_name, measures in by_table.items():
+        dbt_model = table_to_model.get(table_name)
+        if not dbt_model:
+            print(f'  WARNING: "{table_name}" has TMDL measures but no schema YAML entry, skipping')
+            continue
+        out_file = config.semantic_models_dir / f"{re.sub(r'[^A-Za-z0-9]+', '_', table_name).strip('_').lower()}.yml"
+        if not out_file.exists():
+            print(f'  WARNING: no semantic model file for "{table_name}" at {out_file.name}, skipping')
+            continue
+        with open(out_file, encoding="utf-8") as f:
+            doc = yaml.safe_load(f)
+        doc["semantic_models"][0]["measures"] = measures
+        text = yaml.safe_dump(doc, default_flow_style=False, sort_keys=False, allow_unicode=True, width=200)
+        out_file.write_text(text, encoding="utf-8")
+        written += 1
+
+    total = sum(len(v) for v in by_table.values())
+    print(f"Exported {total} measure(s) across {written} file(s)")
+
+
+def sync(config: TmdlProjectConfig):
+    """Apply semantic-model YAML measures -> TMDL via TOM."""
+    by_table = load_by_table(config)
+    total = sum(len(v) for v in by_table.values())
+    if not total:
+        print("No measures found in semantic-model YAML")
+        return
+
+    db, model = load_model(config)
+    from Microsoft.AnalysisServices.Tabular import Measure
+
+    updated = created = 0
+
+    for table_name, entries in by_table.items():
+        table = model.Tables.Find(table_name)
+        if table is None:
+            print(f'  WARNING: table "{table_name}" not found, skipping its measures')
+            continue
+
+        for entry in entries:
+            measure_name = entry["name"]
+            dax = entry["dax"]
+            fmt = entry.get("formatString", "")
+            folder = entry.get("displayFolder", "Calculations")
+
+            existing = table.Measures.Find(measure_name)
+            if existing:
+                changed = set_if_changed(existing, "Expression", dax.strip(), lambda v: (v or "").strip())
+                changed |= set_if_changed(existing, "FormatString", fmt)
+                changed |= set_if_changed(existing, "DisplayFolder", folder)
+                if changed:
+                    updated += 1
+            else:
+                m = Measure()
+                m.Name = measure_name
+                m.Expression = dax
+                m.FormatString = fmt
+                m.DisplayFolder = folder
+                table.Measures.Add(m)
+                created += 1
+
+    save_model(config, db)
+    print(f"Measures synced: {created} created, {updated} updated ({total} total in config)")