codepbi 0.1.7__py3-none-any.whl

codepbi/semantic/sync_relationships.py

@@ -0,0 +1,232 @@
+# Apply semantic-model YAML `entities:` -> TMDL relationships, and export back
+# the other way.
+#
+# semantic-model YAML `entities:` blocks are the source of truth for joins --
+# the same role schema YAML plays for columns and each measure's own block plays
+# for measures. A join is declared the way dbt/MetricFlow natively does it: a
+# `primary` entity in the "one" side's file and a `foreign` entity of the same
+# name in the "many" side's file. TMDL is the build artifact: Power BI still
+# reads relationships.tmdl, this module is what keeps it in sync.
+#
+# Cardinality isn't declared here: every relationship this module creates is
+# Many -> One. If your model needs other cardinalities, extend `sync()` with an
+# explicit cardinality field rather than assuming.
+#
+# inactive (config.meta.inactive on the foreign entity): a relationship whose
+# two tables already connect via a different active path. Analysis Services
+# refuses to save a second *active* path between the same two tables (a real
+# engine-level rejection, not something this module checks) -- inactive lets
+# the join exist for DAX measures to opt into via USERELATIONSHIP, without
+# fighting the existing default path.
+
+import re
+import uuid
+
+import yaml
+
+from . import tom_utils
+from .config import TmdlProjectConfig
+from .schema_loader import iter_semantic_models, load_schema
+
+
+def _entity_name(column_friendly):
+    name = re.sub(r"[^A-Za-z0-9]+", "_", column_friendly).strip("_").lower()
+    if name.endswith("_id"):
+        name = name[:-3]
+    return name or "id"
+
+
+def validate(config: TmdlProjectConfig) -> list[str]:
+    """Every entity's expr must be a real column source on its table, every
+    'foreign' entity must have a matching same-named 'primary' entity somewhere.
+
+    Returns a list of error strings; empty means clean. Called both standalone
+    (sync() refuses to touch TOM if this fails) and from validate_tmdl.
+    """
+    schema = load_schema(config)
+    source_by_table = {t: {c["source"] for c in spec["columns"]} for t, spec in schema.items()}
+
+    errors = []
+    primaries = {}  # entity name -> table
+    foreigns = []  # (entity name, table)
+
+    for path, _doc, sm, table_name in iter_semantic_models(config):
+        valid_sources = source_by_table.get(table_name, set())
+        for e in sm.get("entities") or []:
+            if e.get("expr") not in valid_sources:
+                errors.append(
+                    f"{path.name} entity {e['name']!r}: expr {e.get('expr')!r} "
+                    f"is not a column on {table_name!r}"
+                )
+            if e["type"] == "primary":
+                if e["name"] in primaries:
+                    errors.append(
+                        f"duplicate primary entity {e['name']!r}: "
+                        f"{primaries[e['name']]!r} and {table_name!r}"
+                    )
+                primaries[e["name"]] = table_name
+            elif e["type"] == "foreign":
+                foreigns.append((e["name"], table_name))
+
+    for name, table_name in foreigns:
+        if name not in primaries:
+            errors.append(
+                f"foreign entity {name!r} on {table_name!r} has no matching "
+                f"primary entity anywhere in the model"
+            )
+    return errors
+
+
+def load(config: TmdlProjectConfig) -> list[dict]:
+    """Reconstruct joins by pairing same-named primary/foreign entities across
+    semantic-model YAML -> list of {from, from_column, to, to_column, both_directions?}."""
+    schema = load_schema(config)
+    display_by_source = {t: {c["source"]: c["display"] for c in spec["columns"]} for t, spec in schema.items()}
+
+    primaries = {}  # entity name -> (table, column_friendly)
+    foreigns = []  # (entity name, table, column_friendly, both_directions)
+
+    for _path, _doc, sm, table_name in iter_semantic_models(config):
+        col_lookup = display_by_source.get(table_name, {})
+        for e in sm.get("entities") or []:
+            col_friendly = col_lookup.get(e["expr"], e["expr"])
+            if e["type"] == "primary":
+                primaries[e["name"]] = (table_name, col_friendly)
+            elif e["type"] == "foreign":
+                meta = ((e.get("config") or {}).get("meta")) or {}
+                foreigns.append((e["name"], table_name, col_friendly,
+                                  bool(meta.get("both_directions")), bool(meta.get("inactive"))))
+
+    rels = []
+    for name, from_table, from_col, both_dir, inactive in foreigns:
+        if name not in primaries:
+            continue  # validate() already flags this
+        to_table, to_col = primaries[name]
+        row = {"from": from_table, "from_column": from_col, "to": to_table, "to_column": to_col}
+        if both_dir:
+            row["both_directions"] = True
+        if inactive:
+            row["inactive"] = True
+        rels.append(row)
+    return rels
+
+
+def _key(r):
+    return (r["from"], r["from_column"], r["to"], r["to_column"])
+
+
+def export(config: TmdlProjectConfig):
+    """Dump the current TMDL relationships into each table's semantic-model YAML,
+    replacing only its `entities:` block -- dimensions/measures/description untouched.
+    Use this to seed the files the first time, or to recover if TMDL and the YAML
+    ever disagree about what 'current' means."""
+    schema = load_schema(config)
+    table_to_model = {name: spec["model"] for name, spec in schema.items()}
+
+    _, model = tom_utils.load_model(config)
+
+    entities_by_table = {}  # table -> {entity_name: entity dict}
+    for r in model.Relationships:
+        to_table, to_col = r.ToTable.Name, r.ToColumn.Name
+        from_table, from_col = r.FromTable.Name, r.FromColumn.Name
+        name = _entity_name(to_col)
+
+        entities_by_table.setdefault(to_table, {})[name] = {
+            "name": name, "type": "primary", "expr": to_col,
+        }
+        foreign_entity = {"name": name, "type": "foreign", "expr": from_col}
+        fmeta = {}
+        if str(r.CrossFilteringBehavior) != "OneDirection":
+            fmeta["both_directions"] = True
+        if not r.IsActive:
+            fmeta["inactive"] = True
+        if fmeta:
+            foreign_entity["config"] = {"meta": fmeta}
+        entities_by_table.setdefault(from_table, {})[name] = foreign_entity
+
+    written = 0
+    for table_name, entities in entities_by_table.items():
+        dbt_model = table_to_model.get(table_name)
+        if not dbt_model:
+            print(f'  WARNING: "{table_name}" has TMDL relationships but no schema YAML entry, skipping')
+            continue
+        out_file = config.semantic_models_dir / f"{re.sub(r'[^A-Za-z0-9]+', '_', table_name).strip('_').lower()}.yml"
+        if not out_file.exists():
+            print(f'  WARNING: no semantic model file for "{table_name}" at {out_file.name}, skipping')
+            continue
+        with open(out_file, encoding="utf-8") as f:
+            doc = yaml.safe_load(f)
+        ordered = sorted(entities.values(), key=lambda e: e["name"])
+        doc["semantic_models"][0]["entities"] = ordered
+        text = yaml.safe_dump(doc, default_flow_style=False, sort_keys=False, allow_unicode=True, width=200)
+        out_file.write_text(text, encoding="utf-8")
+        written += 1
+
+    print(f"Exported {len(model.Relationships)} relationship(s) across {written} file(s)")
+
+
+def sync(config: TmdlProjectConfig):
+    """Apply semantic-model YAML entities to TMDL: add/remove/update so TMDL matches."""
+    wanted_rows = load(config)
+    errors = validate(config)
+    if errors:
+        for e in errors:
+            print(f"RELATIONSHIPS: {e}")
+        raise SystemExit(f"semantic-model YAML relationship validation failed: {len(errors)} issue(s)")
+
+    wanted = {_key(r): r for r in wanted_rows}
+
+    # load_model() triggers tom_utils._ensure_loaded(), which adds the CLR
+    # references -- this import must come after, or it fails with
+    # ModuleNotFoundError: No module named 'Microsoft'.
+    db, model = tom_utils.load_model(config)
+    from Microsoft.AnalysisServices.Tabular import (
+        CrossFilteringBehavior, RelationshipEndCardinality, SingleColumnRelationship)
+
+    existing = {}
+    for r in model.Relationships:
+        k = (r.FromTable.Name, r.FromColumn.Name, r.ToTable.Name, r.ToColumn.Name)
+        existing[k] = r
+
+    removed = added = updated = 0
+
+    # Relationships out first -- same discipline as removing a column that one
+    # still references, just applied to the relationship side.
+    for k, r in list(existing.items()):
+        if k not in wanted:
+            model.Relationships.Remove(r)
+            removed += 1
+
+    for k, spec in wanted.items():
+        want_cf = (CrossFilteringBehavior.BothDirections if spec.get("both_directions")
+                   else CrossFilteringBehavior.OneDirection)
+        want_active = not spec.get("inactive")
+        if k in existing:
+            r = existing[k]
+            changed = False
+            if r.CrossFilteringBehavior != want_cf:
+                r.CrossFilteringBehavior = want_cf
+                changed = True
+            if r.IsActive != want_active:
+                r.IsActive = want_active
+                changed = True
+            if changed:
+                updated += 1
+            continue
+
+        from_table, from_col, to_table, to_col = k
+        rel = SingleColumnRelationship()
+        rel.Name = str(uuid.uuid4())
+        rel.FromTable = model.Tables[from_table]
+        rel.FromColumn = model.Tables[from_table].Columns[from_col]
+        rel.ToTable = model.Tables[to_table]
+        rel.ToColumn = model.Tables[to_table].Columns[to_col]
+        rel.FromCardinality = RelationshipEndCardinality.Many
+        rel.ToCardinality = RelationshipEndCardinality.One
+        rel.CrossFilteringBehavior = want_cf
+        rel.IsActive = want_active
+        model.Relationships.Add(rel)
+        added += 1
+
+    tom_utils.save_model(config, db)
+    print(f"RELATIONSHIPS: {removed} removed, {added} added, {updated} updated.")

codepbi/semantic/tom_utils.py

@@ -0,0 +1,120 @@
+# Shared TOM (Tabular Object Model) utilities via pythonnet.
+#
+# Requires the Microsoft.AnalysisServices .NET DLLs (NuGet package
+# Microsoft.AnalysisServices.NetCore.retail.amd64) at config.tom_lib_dir. Not
+# redistributed by this package -- obtain them yourself and point tom_lib_dir
+# at the folder you placed them in.
+
+import re
+
+from .config import TmdlProjectConfig
+
+_DLLS = [
+    "Microsoft.AnalysisServices.Core.dll",
+    "Microsoft.AnalysisServices.Tabular.dll",
+    "Microsoft.AnalysisServices.Tabular.Json.dll",
+]
+
+_loaded = False
+
+
+def _ensure_loaded(config: TmdlProjectConfig) -> None:
+    global _loaded
+    if _loaded:
+        return
+
+    import clr
+
+    for name in _DLLS:
+        dll = config.tom_lib_dir / name
+        if dll.exists():
+            clr.AddReference(str(dll))
+
+    try:
+        from Microsoft.AnalysisServices.Tabular import TmdlSerializer  # noqa: F401
+    except ImportError:
+        raise SystemExit(
+            f"TOM DLLs not found in {config.tom_lib_dir}. Install the "
+            f"Microsoft.AnalysisServices.NetCore.retail.amd64 NuGet package and point "
+            f"tom_lib_dir at its lib folder."
+        )
+
+    _loaded = True
+
+
+def load_model(config: TmdlProjectConfig):
+    _ensure_loaded(config)
+    from Microsoft.AnalysisServices.Tabular import TmdlSerializer
+
+    db = TmdlSerializer.DeserializeDatabaseFromFolder(str(config.semantic_model_definition_dir))
+    return db, db.Model
+
+
+def save_model(config: TmdlProjectConfig, db) -> None:
+    _ensure_loaded(config)
+    from Microsoft.AnalysisServices.Tabular import TmdlSerializer
+
+    TmdlSerializer.SerializeDatabaseToFolder(db, str(config.semantic_model_definition_dir))
+
+
+def set_if_changed(obj, attr, want, normalize=lambda v: (v or "")):
+    """Set obj.attr to want if it differs (after normalize); return whether it changed.
+
+    Shared shape behind every sync's per-property TOM update: compare, set, flag.
+    normalize handles the None-vs-'' and .strip() variations between properties
+    (an Expression is stripped, FormatString/DisplayFolder are not).
+    """
+    if normalize(getattr(obj, attr)) != want:
+        setattr(obj, attr, want)
+        return True
+    return False
+
+
+def pbi_table_of(meta: dict) -> str | None:
+    """Derive the PBI table name from schema YAML meta {entity, grain}."""
+    if not meta:
+        return None
+    entity = meta.get("entity")
+    if not entity:
+        return None
+    grain = meta.get("grain")
+    return f"{entity} - {grain}" if grain else entity
+
+
+_ITEM_RE = re.compile(r'Item="([^"]+)"')
+
+
+def _source_model(table) -> str | None:
+    """dbt model name behind a table, read from its M partition expression."""
+    for p in table.Partitions:
+        expr = getattr(p.Source, "Expression", None)
+        if expr:
+            m = _ITEM_RE.search(str(expr))
+            if m:
+                return m.group(1)
+    return None
+
+
+def read_tables(config: TmdlProjectConfig, skip: bool = True) -> dict:
+    """Read the model into plain dicts: {table_name: {model, columns}}.
+
+    columns maps display name -> {source, hidden, folder, desc, data_type}.
+    Loads the whole model once via TOM rather than parsing TMDL text.
+    """
+    _, model = load_model(config)
+    out = {}
+    for table in model.Tables:
+        if skip and table.Name in config.skip_tables:
+            continue
+        columns = {}
+        for col in table.Columns:
+            dt = str(col.DataType)
+            columns[col.Name] = {
+                "source": col.SourceColumn or None,
+                "hidden": bool(col.IsHidden),
+                "folder": col.DisplayFolder or "",
+                "desc": col.Description or "",
+                "data_type": dt[0].lower() + dt[1:],
+            }
+        out[table.Name] = {"model": _source_model(table), "columns": columns}
+    return out

codepbi/semantic/validate_tmdl.py

@@ -0,0 +1,231 @@
+# Validates the TMDL semantic model: file hygiene, duplicate lineageTags,
+# ambiguous display names, duplicate/ambiguous relationships, DAX references
+# that don't resolve, and measures missing from semantic-model YAML.
+# Raises SystemExit on any failure. Called both standalone and from
+# sync_descriptions.sync() at the end of a column sync.
+
+import re
+from collections import Counter, defaultdict
+
+from . import tom_utils
+from .config import TmdlProjectConfig
+
+_BAD_LINE_LITERALS = {r"\r", r"\n", r"\t", r"\\r", r"\\n", r"\\t"}
+
+# A DAX column reference: 'Table Name'[Column Name]
+_DAX_REF_RE = re.compile(r"'([^']+)'\[([^\]]+)\]")
+
+
+def _check_file_hygiene(config: TmdlProjectConfig) -> list[str]:
+    """Byte-level tripwires for hand edits: BOMs, trailing whitespace, stray escapes."""
+    errors = []
+    for path in sorted(config.tables_dir.glob("*.tmdl")):
+        raw = path.read_bytes()
+        if raw.startswith(b"\xef\xbb\xbf"):
+            errors.append(f"{path.name}: BOM at file start")
+        sep = b"\r\n" if b"\r\n" in raw else b"\n"
+        for i, line in enumerate(raw.split(sep), 1):
+            try:
+                text = line.decode("utf-8")
+            except UnicodeDecodeError:
+                errors.append(f"{path.name}:{i}: non-UTF-8 bytes")
+                continue
+            if text.strip() in _BAD_LINE_LITERALS:
+                errors.append(
+                    f"{path.name}:{i}: literal escape sequence as line content ({text.strip()!r})"
+                )
+            if text != text.rstrip() and text.strip():
+                errors.append(f"{path.name}:{i}: trailing whitespace")
+    return errors
+
+
+def _ambiguous_paths(model):
+    """Table pairs reachable by more than one filter path.
+
+    Filters propagate from the 'one' side to the 'many' side -- in TMDL that is
+    toColumn -> fromColumn -- and both ways on a bidirectional relationship. Power
+    BI permits exactly one path between any two tables; more than one and it
+    refuses to load the model.
+
+    Returns [(from_table, to_table, [path, ...]), ...] with each pair reported once.
+    """
+    edges = defaultdict(set)
+    for rel in model.Relationships:
+        if not rel.IsActive:
+            continue
+        one, many = rel.ToTable.Name, rel.FromTable.Name
+        edges[one].add(many)
+        if str(rel.CrossFilteringBehavior) != "OneDirection":
+            edges[many].add(one)
+
+    def walk(src, dst, seen):
+        if src == dst:
+            return [[src]]
+        found = []
+        for nxt in edges.get(src, ()):
+            if nxt not in seen:
+                found += [[src] + p for p in walk(nxt, dst, seen | {nxt})]
+        return found
+
+    names = [t.Name for t in model.Tables]
+    out = []
+    for a in names:
+        for b in names:
+            if a == b:
+                continue
+            routes = walk(a, b, {a})
+            if len(routes) > 1:
+                out.append((a, b, routes))
+    return out
+
+
+def validate_tmdl(config: TmdlProjectConfig) -> None:
+    """Validate the semantic model. Raises SystemExit on any failure.
+
+    A dangling relationship endpoint makes TOM deserialization fail outright,
+    so that class of error is caught by simply loading the model.
+    """
+    from .sync_measures import load_by_table as _load_measures_by_table
+    from .sync_relationships import validate as validate_relationships
+
+    errors = _check_file_hygiene(config)
+
+    try:
+        _, model = tom_utils.load_model(config)
+    except Exception as exc:
+        for e in errors:
+            print(f"TMDL VALIDATE: {e}")
+        raise SystemExit(
+            f"TMDL failed to load -- a reference does not resolve (often a relationship "
+            f"pointing at a deleted table or renamed column):\n  {exc}"
+        )
+
+    tables = tom_utils.read_tables(config)
+
+    # Duplicate lineageTags across the model
+    tags = defaultdict(list)
+    for table in model.Tables:
+        for col in table.Columns:
+            if col.LineageTag:
+                tags[col.LineageTag].append(f"{table.Name}[{col.Name}]")
+    for tag, locs in tags.items():
+        if len(locs) > 1:
+            errors.append(f"duplicate lineageTag {tag!r} at {', '.join(locs)}")
+
+    # No display name may be visible on two tables at once. Two variants:
+    #   A: same name, different sourceColumn  -> genuinely different things
+    #   B: same name, same sourceColumn       -> the same thing in two places
+    # Both confuse the field list, so both are errors. Measures share the same
+    # namespace as columns (a table object model requires every column AND
+    # measure name to be unique within that table), so they go in the same map --
+    # a column/measure pair with the same name on the same table is a Desktop
+    # "Issues were found" failure on open, not just a field-list ambiguity.
+    name_map = defaultdict(list)
+    for tname, t in tables.items():
+        for cname, c in t["columns"].items():
+            name_map[cname].append((tname, c["source"] or "", c["hidden"], "column"))
+    for table in model.Tables:
+        for measure in table.Measures:
+            name_map[measure.Name].append((table.Name, "", False, "measure"))
+
+    for name, entries in name_map.items():
+        by_table = defaultdict(list)
+        for t, s, h, kind in entries:
+            by_table[t].append((s, h, kind))
+        for tname, items in by_table.items():
+            if len(items) > 1:
+                kinds = ", ".join(f'{kind} ({"hidden" if h else "visible"})' for _, h, kind in items)
+                errors.append(
+                    f"{name!r} defined more than once on table {tname!r}: {kinds}. "
+                    f"A column and a measure (or two measures) cannot share a name "
+                    f"on the same table -- rename or remove one."
+                )
+
+        visible = [(t, s) for t, s, h, kind in entries if not h]
+        if len(visible) < 2:
+            continue
+        if len({s for _, s in visible}) > 1:
+            details = ", ".join(f'{t}[{s or "-"}]' for t, s in visible)
+            errors.append(
+                f"ambiguous display name {name!r} on visible tables with "
+                f"different sourceColumns: {details}. Rename to disambiguate grain."
+            )
+        else:
+            errors.append(
+                f"column {name!r} visible on multiple tables (same source): "
+                f"{sorted(t for t, _ in visible)}. Either hide it where a "
+                f"relationship already supplies it, or rename to make the grain "
+                f"explicit. Do not hide it if nothing else supplies it."
+            )
+
+    # semantic-model YAML owns measures. A measure only in TMDL -- hand-written, or
+    # built in Power BI Desktop -- is outside source control, so flag it rather than
+    # delete it: destroying someone's Desktop work on a sync would be worse.
+    in_yaml = {
+        (table, m["name"])
+        for table, entries in _load_measures_by_table(config).items()
+        for m in entries
+    }
+    for table in model.Tables:
+        for measure in table.Measures:
+            if (table.Name, measure.Name) not in in_yaml:
+                errors.append(
+                    f"measure {table.Name}[{measure.Name}] is not in semantic-model YAML. "
+                    f"Add it there (semantic-model YAML is the source of truth for "
+                    f"measures) or delete it from TMDL."
+                )
+
+    # Every 'Table'[Column] reference in measure DAX must resolve. Renaming a
+    # column only rewrites the column header, so measures silently rot otherwise.
+    known = {
+        t.Name: {c.Name for c in t.Columns} | {x.Name for x in t.Measures}
+        for t in model.Tables
+    }
+    for table in model.Tables:
+        for measure in table.Measures:
+            for ref_tbl, ref_col in _DAX_REF_RE.findall(str(measure.Expression)):
+                if ref_tbl not in known:
+                    errors.append(
+                        f"measure {table.Name}[{measure.Name}] references table "
+                        f"{ref_tbl!r}, which does not exist."
+                    )
+                elif ref_col not in known[ref_tbl]:
+                    errors.append(
+                        f"measure {table.Name}[{measure.Name}] references "
+                        f"{ref_tbl!r}[{ref_col!r}], which does not exist. "
+                        f"A rename only rewrites the column header -- update the DAX too."
+                    )
+
+    # 2+ active relationships between the same table pair (PBI ambiguity)
+    pair_counts = Counter(
+        frozenset((r.FromTable.Name, r.ToTable.Name))
+        for r in model.Relationships if r.IsActive
+    )
+    for pair, n in pair_counts.items():
+        if n >= 2:
+            errors.append(
+                f"{n} active relationships between {sorted(pair)}. "
+                f"PBI will throw 'ambiguous paths'. Mark at most one active per pair."
+            )
+
+    # More than one filter path between any two tables, however many hops -- the
+    # pair-count check above only sees duplicate relationships on a single pair,
+    # so it misses longer cycles.
+    for a, b, routes in _ambiguous_paths(model):
+        shown = "; ".join(" -> ".join(p) for p in routes[:3])
+        errors.append(
+            f"{len(routes)} filter paths from {a!r} to {b!r}: {shown}. "
+            f"PBI allows only one. Drop a relationship or mark one inactive."
+        )
+
+    # semantic-model YAML entities are the join authority; every entity's expr
+    # must name a real column, and every foreign entity needs a matching primary
+    # elsewhere, or a typo silently drops a join instead of failing loudly.
+    for e in validate_relationships(config):
+        errors.append(f"semantic model: {e}")
+
+    if errors:
+        for e in errors:
+            print(f"TMDL VALIDATE: {e}")
+        raise SystemExit(f"TMDL validation failed: {len(errors)} issue(s)")
+    print("TMDL: validation passed.")