semantic-link-labs 0.9.10__py3-none-any.whl → 0.10.0__py3-none-any.whl

sempy_labs/_variable_libraries.py

@@ -0,0 +1,89 @@
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    resolve_workspace_id,
+    _base_api,
+    _create_dataframe,
+    _update_dataframe_datatypes,
+    delete_item,
+)
+import pandas as pd
+from typing import Optional
+from uuid import UUID
+
+
+def list_variable_libraries(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
+    """
+    Shows the variable libraries within a workspace.
+
+    This is a wrapper function for the following API: `Items - List Variable Libraries <https://learn.microsoft.com/rest/api/fabric/variablelibrary/items/list-variable-libraries>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the variable libraries within a workspace.
+    """
+
+    columns = {
+        "Variable Library Name": "string",
+        "Variable Library Id": "string",
+        "Description": "string",
+        "Active Value Set Name": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/VariableLibraries",
+        uses_pagination=True,
+        client="fabric_sp",
+    )
+
+    dfs = []
+    for r in responses:
+        for v in r.get("value", []):
+            prop = v.get("properties", {})
+
+            new_data = {
+                "Variable Library Name": v.get("displayName"),
+                "Variable Library Id": v.get("id"),
+                "Description": v.get("description"),
+                "Active Value Set Name": prop.get("activeValueSetName"),
+            }
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df
+
+
+def delete_variable_library(
+    variable_library: str | UUID, workspace: Optional[str | UUID] = None
+):
+    """
+    Deletes a variable library.
+
+    This is a wrapper function for the following API: `Items - Delete Variable Library <https://learn.microsoft.com/rest/api/fabric/warehouse/items/delete-variable-library>`_.
+
+    Parameters
+    ----------
+    navariable_libraryme: str | uuid.UUID
+        Name or ID of the variable library.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    delete_item(item=variable_library, type="VariableLibrary", workspace=workspace)

sempy_labs/_vpax.py

@@ -0,0 +1,388 @@
+import sempy
+import re
+from urllib.parse import urlparse
+import sempy.fabric as fabric
+import sys
+from pathlib import Path
+from typing import Optional
+from uuid import UUID
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    resolve_dataset_name_and_id,
+    resolve_lakehouse_name_and_id,
+    _mount,
+    _get_column_aggregate,
+    resolve_item_type,
+    file_exists,
+    create_abfss_path_from_path,
+)
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+import zipfile
+import requests
+
+
+VPA_VERSION = "1.10.0"
+NUGET_BASE_URL = "https://www.nuget.org/api/v2/package"
+ASSEMBLIES = [
+    "Dax.Metadata",
+    "Dax.Model.Extractor",
+    "Dax.ViewVpaExport",
+    "Dax.Vpax",
+]
+
+_vpa_initialized = False
+current_dir = Path(__file__).parent
+nuget_dir = current_dir / "nuget_dlls"
+
+
+def find_lib_folder(pkg_folder: Path) -> Path:
+    lib_base = pkg_folder / "lib"
+    if not lib_base.exists():
+        raise FileNotFoundError(f"No 'lib' directory in package {pkg_folder}")
+
+    # Prefer netstandard2.0 if available
+    candidates = sorted(lib_base.iterdir())
+    for preferred in ["netstandard2.0", "net6.0", "net5.0", "netcoreapp3.1", "net472"]:
+        if (lib_base / preferred).exists():
+            return lib_base / preferred
+
+    # Fallback: first available folder
+    for candidate in candidates:
+        if candidate.is_dir():
+            return candidate
+
+    raise FileNotFoundError(f"No usable framework folder found in {lib_base}")
+
+
+def download_and_extract_package(
+    package_name: str, version: str, target_dir: Path
+) -> Path:
+    nupkg_url = f"{NUGET_BASE_URL}/{package_name}/{version}"
+    nupkg_path = target_dir / f"{package_name}.{version}.nupkg"
+
+    if not nupkg_path.exists():
+        r = requests.get(nupkg_url)
+        r.raise_for_status()
+        target_dir.mkdir(parents=True, exist_ok=True)
+        with open(nupkg_path, "wb") as f:
+            f.write(r.content)
+
+    extract_path = target_dir / f"{package_name}_{version}"
+    if not extract_path.exists():
+        with zipfile.ZipFile(nupkg_path, "r") as zip_ref:
+            zip_ref.extractall(extract_path)
+    return extract_path
+
+
+def download_and_load_nuget_package(
+    package_name, version, target_dir: Path = None, load_assembly=True
+):
+
+    from System.Reflection import Assembly
+
+    if target_dir is None:
+        target_dir = nuget_dir
+
+    # Download and extract
+    pkg_folder = download_and_extract_package(package_name, version, target_dir)
+    lib_folder = find_lib_folder(pkg_folder)
+
+    dll_path = lib_folder / f"{package_name}.dll"
+    if not dll_path.exists():
+        raise FileNotFoundError(f"{dll_path} not found")
+
+    sys.path.append(str(lib_folder))
+    if load_assembly:
+        Assembly.LoadFile(str(dll_path))
+
+
+def init_vertipaq_analyzer():
+    global _vpa_initialized
+    if _vpa_initialized:
+        return
+
+    from clr_loader import get_coreclr
+    from pythonnet import set_runtime
+
+    # Load the runtime and set it BEFORE importing clr
+    runtime_config_path = current_dir / "dotnet_lib" / "dotnet.runtime.config.json"
+    rt = get_coreclr(runtime_config=str(runtime_config_path))
+    set_runtime(rt)
+
+    sempy.fabric._client._utils._init_analysis_services()
+
+    from System.Reflection import Assembly
+
+    for name in ASSEMBLIES:
+        download_and_load_nuget_package(
+            name, VPA_VERSION, nuget_dir, load_assembly=False
+        )
+
+    download_and_load_nuget_package("Newtonsoft.Json", "13.0.1")
+    download_and_load_nuget_package("System.IO.Packaging", "7.0.0")
+
+    # For some reason I have to load these after and not inside the download_and_load_nuget_package function
+    dll_paths = [
+        f"{nuget_dir}/Dax.Model.Extractor_1.10.0/lib/net6.0/Dax.Model.Extractor.dll",
+        f"{nuget_dir}/Dax.Metadata_1.10.0/lib/netstandard2.0/Dax.Metadata.dll",
+        f"{nuget_dir}/Dax.ViewVpaExport_1.10.0/lib/netstandard2.0/Dax.ViewVpaExport.dll",
+        f"{nuget_dir}/Dax.Vpax_1.10.0/lib/net6.0/Dax.Vpax.dll",
+    ]
+    for dll_path in dll_paths:
+        Assembly.LoadFile(dll_path)
+
+    _vpa_initialized = True
+
+
+@log
+def create_vpax(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    lakehouse: Optional[str | UUID] = None,
+    lakehouse_workspace: Optional[str | UUID] = None,
+    file_path: Optional[str] = None,
+    read_stats_from_data: bool = False,
+    read_direct_query_stats: bool = False,
+    direct_lake_stats_mode: str = "ResidentOnly",
+    overwrite: bool = False,
+):
+    """
+    Creates a .vpax file for a semantic model and saves it to a lakehouse. This is based on `SQL BI's VertiPaq Analyzer <https://www.sqlbi.com/tools/vertipaq-analyzer/>`_.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    lakehouse : str | uuid.UUID, default=None
+        The lakehouse name or ID.
+        Defaults to None which resolves to the attached lakehouse.
+    lakehouse_workspace : str | uuid.UUID, default=None
+        The workspace name or ID of the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse.
+    file_path : str, default=None
+        The path where the .vpax file will be saved in the lakehouse.
+        Defaults to None which resolves to the dataset name.
+    read_stats_from_data : bool, default=False
+        Whether to read statistics from the data.
+    read_direct_query_stats : bool, default=False
+        Whether to analyze DirectQuery tables.
+    direct_lake_stats_mode : str, default='ResidentOnly'
+        The Direct Lake extraction mode. Options are 'ResidentOnly' or 'Full'. This parameter is ignored if read_stats_from_data is False. This parameter is only relevant for tables which use Direct Lake mode.
+        If set to 'ResidentOnly', column statistics are obtained only for the columns which are in memory.
+        If set to 'Full', column statistics are obtained for all columns - pending the proper identification of the Direct Lake source.
+    overwrite : bool, default=False
+        Whether to overwrite the .vpax file if it already exists in the lakehouse.
+    """
+
+    init_vertipaq_analyzer()
+
+    import notebookutils
+    from Dax.Metadata import DirectLakeExtractionMode
+    from Dax.Model.Extractor import TomExtractor
+    from Dax.Vpax.Tools import VpaxTools
+    from Dax.ViewVpaExport import Model
+    from System.IO import MemoryStream, FileMode, FileStream, FileAccess, FileShare
+
+    direct_lake_stats_mode = direct_lake_stats_mode.capitalize()
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+    (lakehouse_workspace_name, lakehouse_workspace_id) = resolve_workspace_name_and_id(
+        lakehouse_workspace
+    )
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=lakehouse_workspace_id
+    )
+
+    local_path = _mount(lakehouse=lakehouse_id, workspace=lakehouse_workspace_id)
+    if file_path is None:
+        file_path = dataset_name
+
+    if file_path.endswith(".vpax"):
+        file_path = file_path[:-5]
+    save_location = f"Files/{file_path}.vpax"
+    path = f"{local_path}/{save_location}"
+
+    # Check if the .vpax file already exists in the lakehouse
+    if not overwrite:
+        new_path = create_abfss_path_from_path(
+            lakehouse_id, lakehouse_workspace_id, save_location
+        )
+        if file_exists(new_path):
+            print(
+                f"{icons.warning} The {save_location} file already exists in the '{lakehouse_name}' lakehouse. Set overwrite=True to overwrite the file."
+            )
+            return
+
+    vpax_stream = MemoryStream()
+    extractor_app_name = "VPAX Notebook"
+    extractor_app_version = "1.0"
+    column_batch_size = 50
+    token = notebookutils.credentials.getToken("pbi")
+    connection_string = f"data source=powerbi://api.powerbi.com/v1.0/myorg/{workspace_name};initial catalog={dataset_name};User ID=;Password={token};Persist Security Info=True;Impersonation Level=Impersonate"
+
+    print(f"{icons.in_progress} Extracting .vpax metadata...")
+
+    # Get stats for the model; for direct lake only get is_resident
+    dax_model = TomExtractor.GetDaxModel(
+        connection_string,
+        extractor_app_name,
+        extractor_app_version,
+        read_stats_from_data,
+        0,
+        read_direct_query_stats,
+        DirectLakeExtractionMode.ResidentOnly,
+        column_batch_size,
+    )
+    vpa_model = Model(dax_model)
+    tom_database = TomExtractor.GetDatabase(connection_string)
+
+    # Calculate Direct Lake stats for columns which are IsResident=False
+    from sempy_labs.tom import connect_semantic_model
+
+    with connect_semantic_model(dataset=dataset, workspace=workspace) as tom:
+        is_direct_lake = tom.is_direct_lake()
+        if read_stats_from_data and is_direct_lake and direct_lake_stats_mode == "Full":
+
+            df_not_resident = fabric.evaluate_dax(
+                dataset=dataset,
+                workspace=workspace,
+                dax_string=""" SELECT [DIMENSION_NAME] AS [TableName], [ATTRIBUTE_NAME] AS [ColumnName] FROM $SYSTEM.DISCOVER_STORAGE_TABLE_COLUMNS WHERE NOT [ISROWNUMBER] AND NOT [DICTIONARY_ISRESIDENT]""",
+            )
+
+            import Microsoft.AnalysisServices.Tabular as TOM
+
+            print(f"{icons.in_progress} Calculating Direct Lake statistics...")
+
+            # For SQL endpoints (do once)
+            dfI = fabric.list_items(workspace=workspace)
+            # Get list of tables in Direct Lake mode which have columns that are not resident
+            tbls = [
+                t
+                for t in tom.model.Tables
+                if t.Name in df_not_resident["TableName"].values
+                and any(p.Mode == TOM.ModeType.DirectLake for p in t.Partitions)
+            ]
+            for t in tbls:
+                column_cardinalities = {}
+                table_name = t.Name
+                partition = next(p for p in t.Partitions)
+                entity_name = partition.Source.EntityName
+                schema_name = partition.Source.SchemaName
+                if len(schema_name) == 0 or schema_name == "dbo":
+                    schema_name = None
+                expr_name = partition.Source.ExpressionSource.Name
+                expr = tom.model.Expressions[expr_name].Expression
+                item_id = None
+                if "Sql.Database(" in expr:
+                    matches = re.findall(r'"([^"]+)"', expr)
+                    sql_endpoint_id = matches[1]
+                    dfI_filt = dfI[dfI["Id"] == sql_endpoint_id]
+                    item_name = (
+                        dfI_filt["Display Name"].iloc[0] if not dfI_filt.empty else None
+                    )
+                    dfI_filt2 = dfI[
+                        (dfI["Display Name"] == item_name)
+                        & (dfI["Type"].isin(["Lakehouse", "Warehouse"]))
+                    ]
+                    item_id = dfI_filt2["Id"].iloc[0]
+                    item_type = dfI_filt2["Type"].iloc[0]
+                    item_workspace_id = workspace_id
+                elif "AzureStorage.DataLake(" in expr:
+                    match = re.search(r'AzureStorage\.DataLake\("([^"]+)"', expr)
+                    if match:
+                        url = match.group(1)
+                        path_parts = urlparse(url).path.strip("/").split("/")
+                        if len(path_parts) >= 2:
+                            item_workspace_id, item_id = (
+                                path_parts[0],
+                                path_parts[1],
+                            )
+                            item_type = resolve_item_type(
+                                item_id=item_id, workspace=workspace_id
+                            )
+                else:
+                    raise NotImplementedError(
+                        f"Direct Lake source '{expr}' is not supported. Please report this issue on GitHub (https://github.com/microsoft/semantic-link-labs/issues)."
+                    )
+
+                if not item_id:
+                    print(
+                        f"{icons.info} Cannot determine the Direct Lake source of the '{table_name}' table."
+                    )
+                elif item_type == "Warehouse":
+                    print(
+                        f"{icons.info} The '{table_name}' table references a warehouse. Warehouses are not yet supported for this method."
+                    )
+                else:
+                    df_not_resident_cols = df_not_resident[
+                        df_not_resident["TableName"] == table_name
+                    ]
+                    col_dict = {
+                        c.Name: c.SourceColumn
+                        for c in t.Columns
+                        if c.Type != TOM.ColumnType.RowNumber
+                        and c.Name in df_not_resident_cols["ColumnName"].values
+                    }
+                    col_agg = _get_column_aggregate(
+                        lakehouse=item_id,
+                        workspace=item_workspace_id,
+                        table_name=entity_name,
+                        schema_name=schema_name,
+                        column_name=list(col_dict.values()),
+                        function="distinct",
+                    )
+                    column_cardinalities = {
+                        column_name: col_agg[source_column]
+                        for column_name, source_column in col_dict.items()
+                        if source_column in col_agg
+                    }
+
+                    # Update the dax_model file with column cardinalities
+                    tbl = next(
+                        table
+                        for table in dax_model.Tables
+                        if str(table.TableName) == table_name
+                    )
+                    # print(
+                    #    f"{icons.in_progress} Calculating column cardinalities for the '{table_name}' table..."
+                    # )
+                    cols = [
+                        col
+                        for col in tbl.Columns
+                        if str(col.ColumnType) != "RowNumber"
+                        and str(col.ColumnName) in column_cardinalities
+                    ]
+                    for col in cols:
+                        # print(str(col.ColumnName), col.ColumnCardinality)
+                        col.ColumnCardinality = column_cardinalities.get(
+                            str(col.ColumnName)
+                        )
+
+    VpaxTools.ExportVpax(vpax_stream, dax_model, vpa_model, tom_database)
+
+    print(f"{icons.in_progress} Exporting .vpax file...")
+
+    mode = FileMode.Create
+    file_stream = FileStream(path, mode, FileAccess.Write, FileShare.Read)
+    vpax_stream.CopyTo(file_stream)
+    file_stream.Close()
+
+    print(
+        f"{icons.green_dot} The {file_path}.vpax file has been saved in the '{lakehouse_name}' lakehouse within the '{lakehouse_workspace_name}' workspace."
+    )
+
+
+def _dax_distinctcount(table_name, columns):
+
+    dax = "EVALUATE\nROW("
+    for c in columns:
+        full_name = f"'{table_name}'[{c}]"
+        dax += f"""\n"{c}", DISTINCTCOUNT({full_name}),"""
+
+    return f"{dax.rstrip(',')}\n)"

sempy_labs/admin/__init__.py

@@ -84,6 +84,11 @@ from sempy_labs.admin._git import (
 from sempy_labs.admin._dataflows import (
     export_dataflow,
 )
+from sempy_labs.admin._tags import (
+    list_tags,
+    create_tags,
+    delete_tag,
+)
 
 __all__ = [
     "list_items",
@@ -139,4 +144,7 @@ __all__ = [
     "list_report_subscriptions",
     "get_refreshables",
     "export_dataflow",
+    "list_tags",
+    "create_tags",
+    "delete_tag",
 ]

sempy_labs/admin/_tags.py

@@ -0,0 +1,126 @@
+from sempy_labs._helper_functions import (
+    _base_api,
+    _is_valid_uuid,
+)
+from uuid import UUID
+from sempy_labs._tags import list_tags
+import sempy_labs._icons as icons
+from typing import List
+
+
+def resolve_tag_id(tag: str | UUID):
+
+    if _is_valid_uuid(tag):
+        tag_id = tag
+    else:
+        df = list_tags()
+        df[df["Tag Name"] == tag]
+        if df.empty:
+            raise ValueError(f"{icons.red_dot} The '{tag}' tag does not exist.")
+        tag_id = df.iloc[0]["Tag Id"]
+
+    return tag_id
+
+
+def create_tags(tags: str | List[str]):
+    """
+    Creates a new tag or tags.
+
+    This is a wrapper function for the following API: `Tags - Bulk Create Tags <https://learn.microsoft.com/rest/api/fabric/admin/tags/bulk-create-tags>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    tags : str | List[str]
+        The name of the tag or tags to create.
+    """
+
+    if isinstance(tags, str):
+        tags = [tags]
+
+    # Check the length of the tags
+    for tag in tags:
+        if len(tag) > 40:
+            raise ValueError(
+                f"{icons.red_dot} The '{tag}' tag name is too long. It must be 40 characters or less."
+            )
+
+    # Check if the tags already exist
+    df = list_tags()
+    existing_names = df["Tag Name"].tolist()
+    existing_ids = df["Tag Id"].tolist()
+
+    available_tags = [
+        tag for tag in tags if tag not in existing_names and tag not in existing_ids
+    ]
+    unavailable_tags = [
+        tag for tag in tags if tag in existing_names or tag in existing_ids
+    ]
+
+    print(f"{icons.warning} The following tags already exist: {unavailable_tags}")
+    if not available_tags:
+        print(f"{icons.info} No new tags to create.")
+        return
+
+    payload = [{"displayName": name} for name in available_tags]
+
+    for tag in tags:
+        _base_api(
+            request="/v1/admin/bulkCreateTags",
+            client="fabric_sp",
+            method="post",
+            payload=payload,
+            status_codes=201,
+        )
+
+    print(f"{icons.green_dot} The '{available_tags}' tag(s) have been created.")
+
+
+def delete_tag(tag: str | UUID):
+    """
+    Deletes a tag.
+
+    This is a wrapper function for the following API: `Tags - Delete Tag <https://learn.microsoft.com/rest/api/fabric/admin/tags/delete-tag>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    tag : str | uuid.UUID
+        The name or ID of the tag to delete.
+    """
+
+    tag_id = resolve_tag_id(tag)
+
+    _base_api(request=f"/v1/admin/tags/{tag_id}", client="fabric_sp", method="delete")
+
+    print(f"{icons.green_dot} The '{tag}' tag has been deleted.")
+
+
+def update_tag(name: str, tag: str | UUID):
+    """
+    Updates the name of a tag.
+
+    This is a wrapper function for the following API: `Tags - Update Tag <https://learn.microsoft.com/rest/api/fabric/admin/tags/update-tag>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    name : str
+        The new name of the tag.
+    tag : str | uuid.UUID
+        The name or ID of the tag to update.
+    """
+
+    tag_id = resolve_tag_id(tag)
+
+    _base_api(
+        request=f"/v1/admin/tags/{tag_id}",
+        client="fabric_sp",
+        method="patch",
+        payload={"displayName": name},
+    )
+
+    print(f"{icons.green_dot} The '{tag}' tag has been renamed to '{name}'.")

sempy_labs/directlake/_generate_shared_expression.py

@@ -3,6 +3,7 @@ from sempy_labs._helper_functions import (
     _base_api,
     resolve_lakehouse_name_and_id,
     resolve_item_name_and_id,
+    _get_fabric_context_setting,
 )
 from typing import Optional
 import sempy_labs._icons as icons
@@ -85,4 +86,7 @@ def generate_shared_expression(
         return f"{start_expr}{mid_expr}{end_expr}"
     else:
         # Build DL/OL expression
-        return f"""let\n\tSource = AzureStorage.DataLake("onelake.dfs.fabric.microsoft.com/{workspace_id}/{item_id}")\nin\n\tSource"""
+        env = _get_fabric_context_setting("spark.trident.pbienv").lower()
+        env = "" if env == "prod" else f"{env}-"
+
+        return f"""let\n\tSource = AzureStorage.DataLake("https://{env}onelake.dfs.fabric.microsoft.com/{workspace_id}/{item_id}")\nin\n\tSource"""