fabcontext 0.1.0__py3-none-any.whl

fabcontext/__init__.py

@@ -0,0 +1,187 @@
+"""One call: name a workspace, get back a lakehouse holding the ranked context of it.
+
+    %pip install fabcontext
+    from fabcontext import harvest
+    url = harvest("My Workspace")
+
+The first call creates the lakehouse; every later call updates it. Nothing else to configure
+and nothing kept on the machine that ran it - the lakehouse is the only state, and the URL it
+returns is the whole contract with whatever answers questions from it later.
+
+It is built for the Fabric Python runtime and asks for nothing that runtime does not already
+have, so the install replaces no native library and needs no kernel restart.
+"""
+from __future__ import annotations
+
+import os
+import tempfile
+import time
+from typing import Dict, List, Optional, Sequence, Union
+
+__version__ = "0.1.0"
+
+__all__ = ["harvest", "build_and_publish", "open_context", "__version__"]
+
+DEFAULT_LAKEHOUSE = "context_layer"
+
+
+def _steps(log):
+    """A per-step timing table. A step that dies leaves the ones before it published, and the
+    table at the end says which failed and why - a half-built context that reports honestly
+    beats one that vanishes with a traceback."""
+    rows = []
+
+    def step(name, fn, *args, **kwargs):
+        start = time.time()
+        try:
+            out = fn(*args, **kwargs)
+        except Exception as exc:                    # noqa: BLE001 - recorded, then re-raised
+            rows.append((name, round(time.time() - start, 1), "FAILED"))
+            log("[FAILED] " + name + "  " + exc.__class__.__name__ + ": " + str(exc)[:400])
+            raise
+        rows.append((name, round(time.time() - start, 1), "ok"))
+        log("[ok] " + name + "  " + str(round(time.time() - start, 1)) + "s  "
+            + str(out)[:200])
+        return out
+
+    return step, rows
+
+
+def _work_dirs(work: str):
+    raw = os.path.join(work, "raw")
+    build = os.path.join(work, "build")
+    wiki_dir = os.path.join(work, "wiki")
+    for folder in (raw, build, wiki_dir):
+        os.makedirs(folder, exist_ok=True)
+    return raw, build, wiki_dir, os.path.join(work, "graph.html")
+
+
+def _target(to: Optional[str], first_workspace: str):
+    """(workspace, lakehouse) for `to`, or the default: a lakehouse called `context_layer` in
+    the first workspace named, so the context lives beside what it describes."""
+    if not to:
+        return first_workspace, DEFAULT_LAKEHOUSE
+    workspace, _sep, lakehouse = to.partition("/")
+    if not workspace or not lakehouse:
+        raise ValueError("`to` is <workspace>/<lakehouse>; got " + repr(to))
+    return workspace, lakehouse
+
+
+def build_and_publish(work: str, store, *, profile: bool = True, values: bool = True,
+                      wiki: bool = True, aliases: Optional[Dict[str, List[str]]] = None,
+                      push: bool = True, log=print) -> Dict:
+    """Everything downstream of the fetch: parse `work/raw`, rank it, publish, render, push.
+
+    Split out from `harvest` because it needs no tenant - a folder of harvested JSON and a
+    local store are enough to run the whole second half, which is how the offline suite
+    exercises the real publish rather than a mock.
+    """
+    from . import files, graph, parse, profiling, publish
+
+    step, rows = _steps(log)
+    raw, build, wiki_dir, graph_html = _work_dirs(work)
+
+    def _build():
+        parse.build(raw, build)
+        return graph.build(build, aliases)
+
+    con, counts = step("build", _build)
+    try:
+        if profile:
+            step("profile", profiling.run, con, raw, values=values)
+            con.close()
+            con, counts = step("build (with profiles)", _build)
+        published = step("publish", publish.publish, con, store)
+        if wiki:
+            from . import viz
+            from . import wiki as wiki_mod
+            step("wiki", wiki_mod.render, con, wiki_dir)
+            step("graph.html", viz.render, con, graph_html)
+    finally:
+        con.close()
+    if push:
+        step("push files", files.push, store, work, files.ITEMS, False, lambda _m: None)
+    return {"counts": counts, "tables": published, "steps": rows}
+
+
+def harvest(workspaces: Union[str, Sequence[str]], to: Optional[str] = None, *,
+            days: int = 28, query_log: bool = False, refresh: bool = False,
+            profile: bool = True, values: bool = True, wiki: bool = True,
+            folder: Optional[str] = "context", stale_after_days: float = 1.0,
+            aliases: Optional[Dict[str, List[str]]] = None, work: Optional[str] = None,
+            log=print) -> str:
+    """Harvest `workspaces` into a lakehouse and return its URL.
+
+    `workspaces` is a name, a GUID, or a list of either. `to` names the lakehouse to publish
+    into as `<workspace>/<lakehouse>`; by default it is `context_layer` in the first workspace
+    given. `aliases` merges spellings the word lists cannot, as {term_id: [spelling, ...]}.
+
+    **Create or update.** The lakehouse is made if it is not there and reused if it is, and an
+    existing one has its previous `Files/raw` pulled down first - which is what makes a second
+    run incremental rather than a full re-read of the tenant.
+
+    `query_log` additionally reads each workspace's monitoring Eventhouse for the DAX that
+    actually ran, the one signal that says a measure was evaluated rather than merely written
+    into a report. Off by default: monitoring bills against the capacity, and a workspace
+    without it is skipped.
+
+    Two levels of permission, and it says which it got rather than failing: **contributor** on
+    the target workspace to create the lakehouse, and **Fabric admin** for the Scanner API and
+    the audit log. Without admin the graph still builds, but loses endorsement, cross-workspace
+    lineage and usage.
+    """
+    from . import common, fetch, files, publish
+
+    names = [workspaces] if isinstance(workspaces, str) else list(workspaces)
+    if not names:
+        raise ValueError("name at least one workspace to harvest")
+    workspace, lakehouse = _target(to, names[0])
+    common.set_aliases(aliases)
+
+    step, _rows = _steps(log)
+    store, created = step("open lakehouse", publish.open_lakehouse,
+                          workspace, lakehouse, folder)
+
+    own_work = work is None
+    work = work or tempfile.mkdtemp(prefix="fabcontext_")
+    raw, _build, _wiki, _html = _work_dirs(work)
+    try:
+        if not created:
+            # The previous harvest. Its absence is not an error - a lakehouse someone made by
+            # hand, or a run that died before its first push, simply starts from scratch.
+            step("stage previous raw/", files.pull, store, work, ["raw"], lambda _m: None)
+        step("harvest", fetch.harvest, raw, names, days, refresh, True, True,
+             query_log, stale_after_days)
+        out = build_and_publish(work, store, profile=profile, values=values, wiki=wiki,
+                                aliases=aliases, log=log)
+    finally:
+        if own_work:
+            import shutil
+            shutil.rmtree(work, ignore_errors=True)
+
+    log("")
+    log("published " + str(sum(out["tables"].values())) + " rows across "
+        + str(len(out["tables"])) + " tables")
+    log(store.tables_root)
+    return store.tables_root
+
+
+def open_context(url: str, storage_options: Optional[Dict[str, str]] = None):
+    """A read-only DuckDB connection over a published context.
+
+    `url` is what `harvest` returned. This is the harvest side reading back what it wrote -
+    for looking at a context without rebuilding it.
+    """
+    from ._fabric import auth, onelake
+    from . import graph
+
+    class _Store:
+        tables_root = url
+        storage_options = None
+
+    store = _Store()
+    if storage_options is not None:
+        store.storage_options = storage_options
+    elif url.startswith("abfss://"):
+        store.storage_options = onelake.storage_options(auth.onelake_token())
+    return graph.read_published(store)

fabcontext/__main__.py

@@ -0,0 +1,64 @@
+"""`python -m fabcontext "<workspace>"` - the one call, from a command line.
+
+The package does one thing, so this does one thing. It exists for a terminal session and for
+CI; inside a notebook, call `harvest()` directly.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __doc__ as _doc
+from . import __version__, harvest
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m fabcontext", description=_doc,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("workspace", nargs="+",
+                        help="workspace name or GUID; repeat for more")
+    parser.add_argument("--to", default=None,
+                        help="the lakehouse to publish into, <workspace>/<lakehouse>. "
+                             "Default: context_layer in the first workspace named")
+    parser.add_argument("--days", type=int, default=28,
+                        help="days of activity events to read (the audit log keeps 30)")
+    parser.add_argument("--query-log", action="store_true",
+                        help="also read each workspace's monitoring Eventhouse for the DAX "
+                             "that actually ran. Needs monitoring enabled there, which bills "
+                             "against the capacity; a workspace without it is skipped")
+    parser.add_argument("--refresh", action="store_true",
+                        help="refetch everything instead of only what changed")
+    parser.add_argument("--stale-after-days", type=float, default=1.0,
+                        help="refetch the scanner and the store table lists once older than "
+                             "this; they carry no freshness signal of their own")
+    parser.add_argument("--no-profile", action="store_true",
+                        help="skip reading lakehouse columns, stats and values")
+    parser.add_argument("--no-values", action="store_true",
+                        help="profile from the Delta log only, without the distinct-value scan")
+    parser.add_argument("--no-wiki", action="store_true",
+                        help="skip the markdown wiki and graph.html")
+    parser.add_argument("--folder", default="context",
+                        help="workspace folder to put the lakehouse in; empty for the root")
+    parser.add_argument("--work", default=None,
+                        help="keep the working files here instead of a temporary folder")
+    parser.add_argument("--version", action="version", version="fabcontext " + __version__)
+    args = parser.parse_args(argv)
+
+    # DuckDB prints tables with box-drawing characters and Fabric names are not all ASCII;
+    # a stock Windows console is cp1252 and would raise on both.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(errors="replace")
+        except (AttributeError, ValueError):
+            pass
+
+    harvest(args.workspace, to=args.to, days=args.days, query_log=args.query_log,
+            refresh=args.refresh, profile=not args.no_profile, values=not args.no_values,
+            wiki=not args.no_wiki, folder=args.folder or None,
+            stale_after_days=args.stale_after_days, work=args.work)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fabcontext/_fabric/__init__.py

@@ -0,0 +1,20 @@
+"""Everything that talks to Fabric, and nothing that understands the tenant.
+
+The split is deliberate: above this package the code is about semantic models, terms and
+ranking; inside it, about tokens, retries, Delta files and REST pagination. The harvest used
+to get all of this from a third-party library that pins duckdb and deltalake to versions the
+Fabric Python runtime does not ship - which forced a pip upgrade of two native libraries and
+a kernel restart before anything could import them.
+
+So it is vendored here instead, sized to exactly what the harvest does, against the versions
+Fabric already has. Nothing in this package imports that library, and nothing should.
+"""
+from __future__ import annotations
+
+from . import auth, delta, onelake, patterns, rest
+from .onelake import LocalStore, OneLakeStore
+from .rest import FabricError
+from .workspace import Workspace
+
+__all__ = ["auth", "delta", "onelake", "patterns", "rest",
+           "LocalStore", "OneLakeStore", "FabricError", "Workspace"]

fabcontext/_fabric/auth.py

@@ -0,0 +1,167 @@
+"""Bearer tokens, one function per audience.
+
+Five audiences are in play and they are genuinely different: OneLake storage, the Fabric
+control plane, the Power BI REST API, an Eventhouse cluster, and the SQL endpoint. A token
+for one 401s on another, so nothing here is interchangeable.
+
+Acquisition order, cheapest first:
+
+1. **inside a Fabric notebook** - `notebookutils.credentials.getToken`, which is the only
+   path that matters in production and needs no sign-in at all;
+2. an already-minted token in the environment;
+3. **azure-identity** - Azure CLI, then an interactive browser but only on a TTY, so a
+   headless run can never hang waiting for a redirect that will not come.
+
+Tokens are cached per (tenant, scope) and re-acquired near expiry, read out of the JWT
+itself. A failed re-acquire keeps the token it has rather than raising over one that is
+merely inside the refresh margin.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import os
+import sys
+import threading
+import time
+from typing import Callable, Dict, Optional
+
+STORAGE_SCOPE = "https://storage.azure.com/.default"
+FABRIC_SCOPE = "https://api.fabric.microsoft.com/.default"
+POWERBI_SCOPE = "https://analysis.windows.net/powerbi/api/.default"
+SQL_SCOPE = "https://database.windows.net/.default"
+
+# Env vars honoured per scope, so a CI run can inject a token instead of signing in.
+_ENV = {STORAGE_SCOPE: "AZURE_STORAGE_TOKEN", FABRIC_SCOPE: "FABRIC_TOKEN",
+        POWERBI_SCOPE: "POWERBI_TOKEN", SQL_SCOPE: "SQL_TOKEN"}
+
+_CACHE: Dict[tuple, str] = {}
+_LOCK = threading.RLock()
+_EXPIRY: Dict[str, Optional[float]] = {}
+
+
+def _notebook_token(audience: str) -> Optional[str]:
+    """A token from the Fabric notebook runtime, or None when not running in one."""
+    try:
+        import notebookutils                        # noqa: F401 - Fabric runtime only
+    except ImportError:
+        return None
+    try:
+        return notebookutils.credentials.getToken(audience) or None
+    except Exception:                               # noqa: BLE001 - audience may be unknown
+        return None
+
+
+def azure_identity_token(scope: str, interactive: bool = True) -> Optional[str]:
+    """A token for `scope` through azure-identity, or None when it cannot be had.
+
+    The browser credential is appended only on a TTY: on a headless runner it would try to
+    open a browser and block on a local redirect listener, which reads as a hang rather than
+    a failure.
+    """
+    try:
+        from azure.identity import AzureCliCredential
+    except ImportError:
+        return None
+    chain = [AzureCliCredential]
+    if interactive and sys.stdin is not None and sys.stdin.isatty():
+        try:
+            from azure.identity import InteractiveBrowserCredential
+            chain.append(InteractiveBrowserCredential)
+        except ImportError:
+            pass
+    debug = bool(os.environ.get("FABCONTEXT_AUTH_DEBUG"))
+    for credential in chain:
+        try:
+            return credential().get_token(scope).token
+        except Exception as exc:                    # noqa: BLE001 - try the next credential
+            if debug:
+                print("[auth] " + credential.__name__ + " failed for " + scope + ": "
+                      + repr(exc), flush=True)
+    return None
+
+
+def _expiry(token: str) -> Optional[float]:
+    """The `exp` of a JWT, or None when it is not a decodable one. No signature check - the
+    only question is when to refresh."""
+    if token in _EXPIRY:
+        return _EXPIRY[token]
+    try:
+        seg = token.split(".")[1]
+        seg += "=" * (-len(seg) % 4)                # restore base64url padding
+        exp = float(json.loads(base64.urlsafe_b64decode(seg.encode())).get("exp"))
+    except Exception:                               # noqa: BLE001 - not a JWT
+        exp = None
+    with _LOCK:
+        if len(_EXPIRY) >= 16:                      # a session holds a handful of tokens
+            _EXPIRY.clear()
+        _EXPIRY[token] = exp
+    return exp
+
+
+def is_expiring(token: Optional[str], margin_seconds: int = 600) -> bool:
+    """Whether `token` is a JWT within `margin_seconds` of expiry. A token whose expiry
+    cannot be read is left alone rather than churned."""
+    if not token:
+        return False
+    exp = _expiry(token)
+    return exp is not None and time.time() >= exp - margin_seconds
+
+
+def _cached(scope: str, acquire: Callable[[], Optional[str]]) -> Optional[str]:
+    key = (os.environ.get("AZURE_TENANT_ID") or "", scope)
+    with _LOCK:
+        held = _CACHE.get(key)
+        if held and not is_expiring(held):
+            return held
+        token = acquire()
+        if token:
+            _CACHE[key] = token
+        # A blip re-acquiring must not discard a token that is merely inside the margin: it
+        # is still valid, and raising while holding a working token helps nobody.
+        return token or held
+
+
+def _token(scope: str, audience: str, hint: str) -> str:
+    token = _cached(scope, lambda: (_notebook_token(audience)
+                                    or os.environ.get(_ENV.get(scope, ""))
+                                    or azure_identity_token(scope)))
+    if token:
+        return token
+    raise RuntimeError(
+        "could not acquire a token for " + hint + ". Inside a Fabric notebook this is "
+        "automatic; elsewhere run `az login --scope " + scope + "`"
+        + (", or set " + _ENV[scope] if scope in _ENV else ""))
+
+
+def onelake_token() -> str:
+    """OneLake storage. Goes into delta-rs `storage_options` and the DFS credential."""
+    return _token(STORAGE_SCOPE, "storage", "OneLake storage")
+
+
+def fabric_token() -> str:
+    """The Fabric control plane. In a notebook the `pbi` audience covers it."""
+    return _token(FABRIC_SCOPE, "pbi", "the Fabric API")
+
+
+def powerbi_token() -> str:
+    """The Power BI REST API - the scanner and the audit log."""
+    return _token(POWERBI_SCOPE, "pbi", "the Power BI API")
+
+
+def sql_token() -> str:
+    """A SQL analytics endpoint, over TDS."""
+    return _token(SQL_SCOPE, "pbi", "the SQL endpoint")
+
+
+def kusto_token(cluster_uri: str) -> str:
+    """An Eventhouse query endpoint. The audience is the cluster itself, so this one cannot
+    be cached against a fixed scope like the others."""
+    token = _notebook_token(cluster_uri)
+    if token:
+        return token
+    scope = cluster_uri.rstrip("/") + "/.default"
+    token = _cached(scope, lambda: azure_identity_token(scope))
+    if token:
+        return token
+    raise RuntimeError("no token for " + cluster_uri + "; run `az login --scope " + scope + "`")

fabcontext/_fabric/delta.py

@@ -0,0 +1,136 @@
+"""Delta Lake in and out, through delta-rs alone.
+
+Three things worth knowing about why it is shaped this way:
+
+- **Writing streams.** `write_deltalake` accepts anything exporting an Arrow C stream, and a
+  DuckDB relation is one, so a table goes from the in-memory graph to Delta without ever
+  being materialised as an Arrow table in between.
+- **Reading uses DataFusion, not `delta_scan`.** DuckDB's `delta` and `azure` extensions are
+  not bundled with the Fabric runtime, so `delta_scan` would fetch them over the network in
+  every notebook session. `QueryBuilder` ships inside the deltalake wheel and hands back a
+  `RecordBatchReader`, which DuckDB consumes through the same Arrow capsule. No extension,
+  no download, no pyarrow.
+- **Opening logs is latency, not work.** Replaying a Delta log over OneLake is a handful of
+  round trips, so a batch of tables is opened concurrently and the results stay positional.
+"""
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+# Opening one Delta log is pure network wait; more threads than this buys nothing and starts
+# to look like a burst to OneLake.
+MAX_OPEN_WORKERS = 16
+
+
+def table_url(root: str, schema: str, table: str) -> str:
+    """Where one table lives under a Tables root. A local root stays a local path so the
+    offline test exercises the real write, not a mock."""
+    if "://" in root:
+        return root.rstrip("/") + "/" + schema + "/" + table
+    return os.path.join(root, schema, table)
+
+
+def write_table(url: str, source: Any, storage_options: Optional[Dict[str, str]] = None) -> None:
+    """Replace whatever is at `url` with `source`, schema and all.
+
+    `source` is normally a DuckDB relation. `schema_mode="overwrite"` matters: the published
+    tables gain columns as the graph gains attributes, and without it a new column would fail
+    against the previous version's schema.
+    """
+    from deltalake import write_deltalake
+
+    write_deltalake(url, source, mode="overwrite", schema_mode="overwrite",
+                    storage_options=storage_options or None)
+
+
+def open_table(url: str, storage_options: Optional[Dict[str, str]] = None):
+    """The `DeltaTable` at `url`, or None when there is no Delta log there. A missing table is
+    an ordinary outcome when probing for one, not an error."""
+    from deltalake import DeltaTable
+
+    try:
+        return DeltaTable(url, storage_options=storage_options or None)
+    except Exception:                               # noqa: BLE001 - absent or unreadable
+        return None
+
+
+def open_tables(targets: Sequence[Tuple[str, Optional[Dict[str, str]]]]) -> List[Optional[Any]]:
+    """One `DeltaTable` (or None) per target, opened concurrently.
+
+    The result is **positional**: callers index it against the input to retry a different URL
+    shape for the ones that came back None.
+    """
+    if not targets:
+        return []
+    workers = min(MAX_OPEN_WORKERS, len(targets))
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        return list(pool.map(lambda t: open_table(t[0], t[1]), targets))
+
+
+def query(dt, sql: str, name: str = "t"):
+    """Run `sql` against one Delta table and return a `RecordBatchReader`.
+
+    The dialect is DataFusion's, not DuckDB's - `approx_distinct`, not
+    `approx_count_distinct`.
+    """
+    from deltalake import QueryBuilder
+
+    return QueryBuilder().register(name, dt).execute(sql)
+
+
+def query_rows(dt, sql: str, name: str = "t") -> List[tuple]:
+    """`query`, materialised as a list of tuples. For the small results - a distinct-value
+    listing, a row of cardinality estimates - where streaming buys nothing."""
+    import duckdb
+
+    con = duckdb.connect()
+    try:
+        con.register("_q", query(dt, sql, name))
+        return con.execute("SELECT * FROM _q").fetchall()
+    finally:
+        con.close()
+
+
+def read_into(con, name: str, url: str,
+              storage_options: Optional[Dict[str, str]] = None) -> bool:
+    """Copy the Delta table at `url` into `con` as `name`. False when it is not there, so an
+    optional table simply does not arrive rather than failing the read."""
+    dt = open_table(url, storage_options)
+    if dt is None:
+        return False
+    # Registered by name rather than left to DuckDB's replacement scan, which would resolve
+    # the reader out of this frame's locals - true, but not something to rely on.
+    handle = "_reader_" + name
+    con.register(handle, query(dt, "SELECT * FROM t"))
+    try:
+        con.execute("CREATE OR REPLACE TABLE " + name + " AS SELECT * FROM " + handle)
+    finally:
+        con.unregister(handle)
+    return True
+
+
+def arrow_fields(dt) -> List[Any]:
+    """A table's columns as Arrow fields, whichever spelling this deltalake version has. The
+    probe is deliberate: this is the one place the versions disagree."""
+    schema = dt.schema()
+    for attr in ("to_arrow", "to_pyarrow"):
+        if hasattr(schema, attr):
+            return list(getattr(schema, attr)())
+    return [f for f in schema.fields]               # delta-rs fields carry .name and .type
+
+
+def add_actions(dt):
+    """The Delta log's add actions as something DuckDB can register.
+
+    `get_add_actions` hands back an arro3 `RecordBatch`, which exports an Arrow C *array* but
+    not a *stream*, and DuckDB only accepts the latter. Wrapping it in a one-batch table is
+    the whole fix.
+    """
+    adds = dt.get_add_actions(flatten=True)
+    if hasattr(adds, "__arrow_c_stream__"):
+        return adds
+    from arro3.core import Table
+
+    return Table.from_batches([adds])