metabase-sync 0.1.0__py3-none-any.whl

metabase_sync/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

metabase_sync/__main__.py

@@ -0,0 +1,4 @@
+from metabase_sync.cli import main
+
+if __name__ == "__main__":
+    main()

metabase_sync/apply/__init__.py

@@ -0,0 +1,30 @@
+"""`metabase-sync apply` orchestration.
+
+Public entry point is `run()`. The rest of this package is internal: each
+`_<resource>.py` module contains a single `apply_<resource>(ctx)` function
+that mutates the plan and (in apply mode) calls the Metabase API.
+"""
+
+from metabase_sync.diff import build_remote_index
+
+from ._pulses import _build_pulse_payload  # re-exported for unit tests
+from ._runner import (
+    _concurrency_check,
+    _database_preflight,
+    _reference_preflight,
+    run,
+)
+from ._shared import ApplyContext
+from ._shared import resolve_card_path as _resolve_card_path
+
+__all__ = [
+    "ApplyContext",
+    "run",
+    "build_remote_index",
+    # Internals re-exported for test access only — not part of the supported API.
+    "_build_pulse_payload",
+    "_concurrency_check",
+    "_database_preflight",
+    "_reference_preflight",
+    "_resolve_card_path",
+]

metabase_sync/apply/_cards.py

@@ -0,0 +1,261 @@
+"""Apply cards — native + GUI.
+
+Two interesting bits:
+
+* `dataset_query` is rebuilt from disk: SQL body for native cards, the stored
+  full dataset_query dict for GUI cards (we accept either classic or MBQL5).
+* `result_metadata` is reset to `[]` only when the dataset_query itself
+  changed. For cosmetic edits (display, viz settings) we pass the server's
+  existing metadata through so Metabase doesn't re-run the underlying query.
+"""
+
+from __future__ import annotations
+
+import difflib
+import json
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from metabase_sync.plan import Change
+from metabase_sync.serialize.cards import read_card_file
+from metabase_sync.serialize.yamlio import write_frontmatter_sql, write_yaml
+
+from ._shared import (
+    ApplyContext,
+    normalize,
+    normalize_dataset_query,
+    summarize_diffs,
+)
+
+# Diff field labels that indicate the dataset_query itself changed, so we
+# must reset result_metadata. Anything else is metadata-only.
+_DATASET_QUERY_DIFF_FIELDS = frozenset(
+    {"SQL", "query_type", "MBQL query structure", "template_tags"}
+)
+
+
+def apply_cards(ctx: ApplyContext) -> None:
+    for path, fm, body, gui_query in _iter_card_files(ctx.state_dir):
+        relpath = str(path.relative_to(ctx.state_dir))
+        eid = fm.get("entity_id")
+        remote_card = ctx.remote.cards_by_entity.get(eid) if eid else None
+        collection_id = _resolve_collection_for_card(
+            path, ctx.collection_id_by_disk_path
+        )
+        db_name = fm.get("database")
+        db_id = ctx.db_id_by_name.get(db_name) if db_name else None
+        dataset_query = _build_dataset_query(fm, body, gui_query, db_id)
+
+        desired: dict[str, Any] = {
+            "name": fm["name"],
+            "description": fm.get("description"),
+            "type": fm.get("type", "question"),
+            "display": fm.get("display", "table"),
+            "visualization_settings": fm.get("visualization_settings", {}),
+            "parameters": fm.get("parameters", []),
+            "collection_id": collection_id,
+            "database_id": db_id,
+            "dataset_query": dataset_query,
+            "enable_embedding": fm.get("enable_embedding", False),
+            "embedding_params": fm.get("embedding_params"),
+            "cache_ttl": fm.get("cache_ttl"),
+            "archived": fm.get("archived", False),
+        }
+
+        is_model = fm.get("type") == "model"
+
+        if remote_card is None:
+            # New card: ship stored result_metadata for models so curated
+            # column descriptions land on first apply; let Metabase recompute
+            # for ordinary questions.
+            desired["result_metadata"] = (
+                list(fm.get("result_metadata") or []) if is_model else []
+            )
+            ctx.plan.add(
+                Change(
+                    resource="cards",
+                    action="create",
+                    relpath=relpath,
+                    name=fm["name"],
+                    summary=_create_summary(fm, body),
+                    details={"display": desired["display"], "database": db_name},
+                )
+            )
+            if ctx.mode == "apply":
+                created = ctx.client.post("/api/card", desired)
+                ctx.card_id_by_disk_path[path.resolve()] = int(created["id"])
+                fm["entity_id"] = created.get("entity_id")
+                _rewrite_card_file(path, fm, body, gui_query)
+            continue
+
+        ctx.card_id_by_disk_path[path.resolve()] = int(remote_card["id"])
+        diffs = _diff_card(desired, remote_card)
+        if diffs:
+            dq_changed = any(d[0] in _DATASET_QUERY_DIFF_FIELDS for d in diffs)
+            if is_model:
+                # Never wipe a model's curated result_metadata. If the user
+                # has stored an updated copy on disk, prefer it; otherwise
+                # pass the remote's existing one through.
+                desired["result_metadata"] = list(
+                    fm.get("result_metadata")
+                    or remote_card.get("result_metadata")
+                    or []
+                )
+            else:
+                desired["result_metadata"] = (
+                    [] if dq_changed else remote_card.get("result_metadata") or []
+                )
+            ctx.plan.add(
+                Change(
+                    resource="cards",
+                    action="update",
+                    relpath=relpath,
+                    name=fm["name"],
+                    summary=summarize_diffs(diffs),
+                    details={"changes": diffs},
+                )
+            )
+            if ctx.mode == "apply":
+                ctx.client.put(f"/api/card/{remote_card['id']}", desired)
+        else:
+            ctx.plan.add(
+                Change(
+                    resource="cards", action="skip", relpath=relpath, name=fm["name"]
+                )
+            )
+
+
+# --- iteration + path resolution -----------------------------------------------
+
+
+def _iter_card_files(
+    state_dir: Path,
+) -> Iterator[tuple[Path, dict, str | None, dict | None]]:
+    root = state_dir / "collections"
+    if not root.exists():
+        return
+    for path in sorted(root.rglob("cards/*.sql")):
+        fm, body, gui = read_card_file(path)
+        yield path, fm, body, gui
+    for path in sorted(root.rglob("cards/*.yaml")):
+        fm, body, gui = read_card_file(path)
+        yield path, fm, body, gui
+
+
+def _resolve_collection_for_card(
+    card_path: Path, collection_id_by_disk_path: dict[Path, int]
+) -> int | None:
+    cards_dir = card_path.parent
+    parent = cards_dir.parent
+    # Dashboard-internal card path: <collection>/dashboards/<dash>/cards/<card>.sql
+    if parent.parent.name == "dashboards":
+        return collection_id_by_disk_path.get(parent.parent.parent.resolve())
+    return collection_id_by_disk_path.get(parent.resolve())
+
+
+# --- dataset_query build + write-back ------------------------------------------
+
+
+def _build_dataset_query(
+    fm: dict[str, Any],
+    body: str | None,
+    gui_query: dict[str, Any] | None,
+    db_id: int | None,
+) -> dict[str, Any]:
+    if body is not None:
+        return {
+            "database": db_id,
+            "type": "native",
+            "native": {
+                "query": body,
+                "template-tags": fm.get("template_tags") or {},
+            },
+        }
+    dq = dict(gui_query or {})
+    dq["database"] = db_id
+    return dq
+
+
+def _rewrite_card_file(
+    path: Path, fm: dict[str, Any], body: str | None, gui_query: dict[str, Any] | None
+) -> None:
+    if body is not None:
+        write_frontmatter_sql(path, fm, body)
+    else:
+        write_yaml(path, fm | {"query": gui_query or {}})
+
+
+# --- diff ---------------------------------------------------------------------
+
+
+def _diff_card(
+    desired: dict[str, Any], remote: dict[str, Any]
+) -> list[tuple[str, Any, Any]]:
+    diffs: list[tuple[str, Any, Any]] = []
+    for k in ("name", "description", "display", "collection_id", "archived"):
+        if desired.get(k) != remote.get(k):
+            diffs.append((k, remote.get(k), desired.get(k)))
+    remote_legacy = remote.get("legacy_query")
+    if remote_legacy:
+        try:
+            remote_dq = json.loads(remote_legacy)
+        except json.JSONDecodeError:
+            diffs.append(("dataset_query", "<unparseable>", "<rebuilt>"))
+            return diffs
+        if normalize_dataset_query(desired["dataset_query"]) != normalize_dataset_query(
+            remote_dq
+        ):
+            diffs.extend(_dataset_query_diffs(remote_dq, desired["dataset_query"]))
+    return diffs
+
+
+def _dataset_query_diffs(
+    remote_dq: dict[str, Any], desired_dq: dict[str, Any]
+) -> list[tuple[str, Any, Any]]:
+    """Break a dataset_query change into specific sub-diffs so the plan report
+    is useful (e.g. 'SQL: 76 → 78 lines, +2 −0' rather than '<old> → <new>')."""
+    diffs: list[tuple[str, Any, Any]] = []
+    if remote_dq.get("type") != desired_dq.get("type"):
+        diffs.append(("query_type", remote_dq.get("type"), desired_dq.get("type")))
+        return diffs
+
+    if desired_dq.get("type") == "native":
+        remote_native = remote_dq.get("native") or {}
+        desired_native = desired_dq.get("native") or {}
+        remote_sql = remote_native.get("query") or ""
+        desired_sql = desired_native.get("query") or ""
+        if remote_sql != desired_sql:
+            diffs.append(("SQL", _sql_change_summary(remote_sql, desired_sql), None))
+        if (remote_native.get("template-tags") or {}) != (
+            desired_native.get("template-tags") or {}
+        ):
+            r = list((remote_native.get("template-tags") or {}).keys())
+            d = list((desired_native.get("template-tags") or {}).keys())
+            diffs.append(("template_tags", r, d))
+    else:
+        remote_q = remote_dq.get("query") or {}
+        desired_q = desired_dq.get("query") or {}
+        if normalize(remote_q) != normalize(desired_q):
+            diffs.append(("MBQL query structure", "(changed)", None))
+    return diffs
+
+
+def _sql_change_summary(before: str, after: str) -> str:
+    before_lines = before.splitlines()
+    after_lines = after.splitlines()
+    added = removed = 0
+    for line in difflib.ndiff(before_lines, after_lines):
+        if line.startswith("+ "):
+            added += 1
+        elif line.startswith("- "):
+            removed += 1
+    return f"{len(before_lines)} → {len(after_lines)} lines, +{added} −{removed}"
+
+
+def _create_summary(fm: dict[str, Any], body: str | None) -> str:
+    base = fm.get("name", "")
+    if body is not None:
+        preview = body.strip().split("\n", 1)[0][:60]
+        return f"{base} — {preview}"
+    return f"{base} (GUI)"

metabase_sync/apply/_collections.py

@@ -0,0 +1,99 @@
+"""Apply collections — POST new, PUT changed, persist server-allocated entity_ids."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from metabase_sync.diff import RemoteIndex
+from metabase_sync.plan import Change
+from metabase_sync.serialize.collections import read_collections
+from metabase_sync.serialize.yamlio import write_yaml
+
+from ._shared import ApplyContext, diff_fields, summarize_diffs
+
+
+def apply_collections(ctx: ApplyContext) -> None:
+    for directory, manifest in read_collections(ctx.state_dir):
+        if directory.name == "root" and directory.parent.name == "collections":
+            continue
+        parent_dir = directory.parent
+        parent_id: int | None = (
+            ctx.collection_id_by_disk_path.get(parent_dir.resolve())
+            if parent_dir.name != "collections"
+            else None
+        )
+        relpath = str(directory.relative_to(ctx.state_dir))
+
+        eid = manifest.get("entity_id")
+        remote_collection = (
+            ctx.remote.collections_by_entity.get(eid)
+            if eid
+            else _find_by_name_and_parent(ctx.remote, parent_id, manifest["name"])
+        )
+        desired = {
+            "name": manifest["name"],
+            "description": manifest.get("description"),
+            "parent_id": parent_id,
+            "authority_level": manifest.get("authority_level"),
+        }
+
+        if remote_collection is None:
+            ctx.plan.add(
+                Change(
+                    resource="collections",
+                    action="create",
+                    relpath=relpath,
+                    name=manifest["name"],
+                    summary=manifest["name"],
+                    details={"desired": desired},
+                )
+            )
+            if ctx.mode == "apply":
+                created = ctx.client.post("/api/collection", desired)
+                ctx.collection_id_by_disk_path[directory.resolve()] = int(created["id"])
+                manifest["entity_id"] = created.get("entity_id")
+                write_yaml(directory / "_collection.yaml", manifest)
+            continue
+
+        ctx.collection_id_by_disk_path[directory.resolve()] = int(
+            remote_collection["id"]
+        )
+        diffs = diff_fields(
+            desired,
+            remote_collection,
+            ("name", "description", "parent_id", "authority_level"),
+        )
+        if diffs:
+            ctx.plan.add(
+                Change(
+                    resource="collections",
+                    action="update",
+                    relpath=relpath,
+                    name=manifest["name"],
+                    summary=summarize_diffs(diffs),
+                    details={"changes": diffs},
+                )
+            )
+            if ctx.mode == "apply":
+                ctx.client.put(f"/api/collection/{remote_collection['id']}", desired)
+        else:
+            ctx.plan.add(
+                Change(
+                    resource="collections",
+                    action="skip",
+                    relpath=relpath,
+                    name=manifest["name"],
+                )
+            )
+        if eid != remote_collection.get("entity_id") and ctx.mode == "apply":
+            manifest["entity_id"] = remote_collection.get("entity_id")
+            write_yaml(directory / "_collection.yaml", manifest)
+
+
+def _find_by_name_and_parent(
+    remote: RemoteIndex, parent_id: int | None, name: str
+) -> dict[str, Any] | None:
+    for c in remote.collections_by_id.values():
+        if c.get("parent_id") == parent_id and c.get("name") == name:
+            return c
+    return None

metabase_sync/apply/_dashboards.py

@@ -0,0 +1,257 @@
+"""Apply dashboards — single PUT carries metadata + tabs + dashcards together
+so a dashboard can never be left half-applied."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from metabase_sync.client import MetabaseClient
+from metabase_sync.diff import RemoteIndex
+from metabase_sync.errors import ReferenceResolutionError
+from metabase_sync.plan import Change
+from metabase_sync.serialize.dashboards import read_dashboard_files
+from metabase_sync.serialize.yamlio import write_yaml
+
+from ._shared import ApplyContext, diff_fields, resolve_card_path, summarize_diffs
+
+
+def apply_dashboards(ctx: ApplyContext) -> None:
+    for directory, doc in read_dashboard_files(ctx.state_dir):
+        dashboard_file = directory / "dashboard.yaml"
+        relpath = str(dashboard_file.relative_to(ctx.state_dir))
+        eid = doc.get("entity_id")
+        collection_id = _resolve_collection_for_dashboard(
+            directory, ctx.collection_id_by_disk_path
+        )
+        remote_dashboard = (
+            ctx.remote.dashboards_by_entity.get(eid)
+            if eid
+            else _find_by_collection_and_name(ctx.remote, collection_id, doc["name"])
+        )
+
+        desired: dict[str, Any] = {
+            "name": doc["name"],
+            "description": doc.get("description"),
+            "collection_id": collection_id,
+            "parameters": doc.get("parameters", []),
+            "auto_apply_filters": doc.get("auto_apply_filters", True),
+            "cache_ttl": doc.get("cache_ttl"),
+            "enable_embedding": doc.get("enable_embedding", False),
+            "embedding_params": doc.get("embedding_params"),
+            "width": doc.get("width", "fixed"),
+            "archived": doc.get("archived", False),
+        }
+
+        if remote_dashboard is None:
+            ctx.plan.add(
+                Change(
+                    resource="dashboards",
+                    action="create",
+                    relpath=relpath,
+                    name=doc["name"],
+                    summary=f"{doc['name']} ({len(doc.get('dashcards') or [])} dashcards)",
+                )
+            )
+            if ctx.mode == "apply":
+                # Two-call create: POST allocates an id; PUT installs the full
+                # metadata + tabs + dashcards in one go (the create POST
+                # doesn't accept nested dashcards on most Metabase versions).
+                created = ctx.client.post(
+                    "/api/dashboard",
+                    {"name": doc["name"], "collection_id": collection_id},
+                )
+                dashboard_id = int(created["id"])
+                ctx.dashboard_id_by_disk_path[directory.resolve()] = dashboard_id
+                doc["entity_id"] = created.get("entity_id")
+                write_yaml(dashboard_file, doc)
+                _put_full(
+                    ctx.client,
+                    dashboard_id,
+                    desired,
+                    doc,
+                    directory,
+                    ctx.card_id_by_disk_path,
+                )
+            continue
+
+        ctx.dashboard_id_by_disk_path[directory.resolve()] = int(remote_dashboard["id"])
+        diffs = diff_fields(
+            desired,
+            remote_dashboard,
+            ("name", "description", "collection_id", "auto_apply_filters", "archived"),
+        )
+        contents_diff = _contents_diff(
+            doc, remote_dashboard, directory, ctx.card_id_by_disk_path
+        )
+        if diffs or contents_diff:
+            summary_parts = []
+            if diffs:
+                summary_parts.append(summarize_diffs(diffs))
+            if contents_diff:
+                summary_parts.append(contents_diff)
+            ctx.plan.add(
+                Change(
+                    resource="dashboards",
+                    action="update",
+                    relpath=relpath,
+                    name=doc["name"],
+                    summary="; ".join(summary_parts),
+                    details={"changes": diffs, "contents": contents_diff},
+                )
+            )
+            if ctx.mode == "apply":
+                _put_full(
+                    ctx.client,
+                    int(remote_dashboard["id"]),
+                    desired,
+                    doc,
+                    directory,
+                    ctx.card_id_by_disk_path,
+                )
+        else:
+            ctx.plan.add(
+                Change(
+                    resource="dashboards",
+                    action="skip",
+                    relpath=relpath,
+                    name=doc["name"],
+                )
+            )
+
+
+def _resolve_collection_for_dashboard(
+    dashboard_dir: Path, collection_id_by_disk_path: dict[Path, int]
+) -> int | None:
+    return collection_id_by_disk_path.get(dashboard_dir.parent.parent.resolve())
+
+
+def _find_by_collection_and_name(
+    remote: RemoteIndex, collection_id: int | None, name: str
+) -> dict[str, Any] | None:
+    for d in remote.dashboards_by_id.values():
+        if d.get("collection_id") == collection_id and d.get("name") == name:
+            return d
+    return None
+
+
+def _contents_diff(
+    doc: dict[str, Any],
+    remote_dashboard: dict[str, Any],
+    dashboard_dir: Path,
+    card_id_by_disk_path: dict[Path, int],
+) -> str:
+    """One-line summary if disk dashcards/tabs differ from the remote, else ''.
+
+    Surfaces both tabs and dashcards drift in the same summary so a dashboard
+    that changed both doesn't show only the tabs delta.
+    """
+    parts: list[str] = []
+
+    desired_tabs = [(t.get("name"), t.get("position")) for t in (doc.get("tabs") or [])]
+    remote_tabs = [
+        (t.get("name"), t.get("position")) for t in (remote_dashboard.get("tabs") or [])
+    ]
+    if desired_tabs != remote_tabs:
+        parts.append(f"tabs: {len(remote_tabs)} → {len(desired_tabs)}")
+
+    desired_dc = [
+        (
+            resolve_card_path(dc.get("card_path"), dashboard_dir, card_id_by_disk_path),
+            dc.get("row"),
+            dc.get("col"),
+            dc.get("size_x"),
+            dc.get("size_y"),
+        )
+        for dc in (doc.get("dashcards") or [])
+    ]
+    remote_dc = [
+        (
+            dc.get("card_id"),
+            dc.get("row"),
+            dc.get("col"),
+            dc.get("size_x"),
+            dc.get("size_y"),
+        )
+        for dc in (remote_dashboard.get("dashcards") or [])
+    ]
+    if desired_dc != remote_dc:
+        parts.append(f"dashcards: {len(remote_dc)} → {len(desired_dc)}")
+
+    return "; ".join(parts)
+
+
+# --- write side --------------------------------------------------------------
+
+
+def _put_full(
+    client: MetabaseClient,
+    dashboard_id: int,
+    metadata: dict[str, Any],
+    doc: dict[str, Any],
+    dashboard_dir: Path,
+    card_id_by_disk_path: dict[Path, int],
+) -> None:
+    """Single PUT carrying metadata + tabs + dashcards.
+
+    Metabase's dashboard PUT accepts all these fields in one request, so we
+    fire one call rather than two — eliminates the half-applied window where
+    metadata had been updated but dashcards hadn't.
+    """
+    tabs_payload, pos_to_temp_id = _build_tabs_payload(doc)
+    dashcards_payload = _build_dashcards_payload(
+        doc, dashboard_dir, card_id_by_disk_path, pos_to_temp_id
+    )
+    body = {**metadata, "tabs": tabs_payload, "dashcards": dashcards_payload}
+    client.put(f"/api/dashboard/{dashboard_id}", body)
+
+
+def _build_tabs_payload(
+    doc: dict[str, Any],
+) -> tuple[list[dict[str, Any]], dict[int, int]]:
+    tabs_payload: list[dict[str, Any]] = []
+    pos_to_temp_id: dict[int, int] = {}
+    for i, tab in enumerate(doc.get("tabs", []) or []):
+        temp_id = -(i + 1)
+        pos_to_temp_id[int(tab["position"])] = temp_id
+        tabs_payload.append(
+            {"id": temp_id, "name": tab["name"], "position": tab["position"]}
+        )
+    return tabs_payload, pos_to_temp_id
+
+
+def _build_dashcards_payload(
+    doc: dict[str, Any],
+    dashboard_dir: Path,
+    card_id_by_disk_path: dict[Path, int],
+    pos_to_temp_id: dict[int, int],
+) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for i, dc in enumerate(doc.get("dashcards", []) or []):
+        card_id = resolve_card_path(
+            dc.get("card_path"), dashboard_dir, card_id_by_disk_path
+        )
+        if card_id is None and dc.get("card_path") is not None:
+            raise ReferenceResolutionError(
+                f"dashboard {dashboard_dir}: dashcard references "
+                f"card_path={dc['card_path']} but no such card file exists"
+            )
+        tab_pos = dc.get("tab_position")
+        out.append(
+            {
+                "id": -(i + 1),
+                "card_id": card_id,
+                "dashboard_tab_id": pos_to_temp_id.get(tab_pos)
+                if tab_pos is not None
+                else None,
+                "row": dc["row"],
+                "col": dc["col"],
+                "size_x": dc["size_x"],
+                "size_y": dc["size_y"],
+                "parameter_mappings": dc.get("parameter_mappings", []),
+                "visualization_settings": dc.get("visualization_settings", {}),
+                "series": dc.get("series", []),
+                "action_id": dc.get("action_id"),
+            }
+        )
+    return out