fdsempy 1.3.0__tar.gz

fdsempy-1.3.0/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: fdsempy
+Version: 1.3.0
+Summary: Fairday Sempy - semantic model utilities
+Author: Peter Johnson - Fairday Research Limited
+Requires-Dist: semantic-link-labs
+Requires-Dist: pandas
+Requires-Dist: json
+Requires-Dist: requests
+Requires-Dist: sempy
+Requires-Dist: notebookutils
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: summary

fdsempy-1.3.0/fdsempy/fdsempy.py

@@ -0,0 +1,376 @@
+#######################################
+#   fdsempy.py
+#   (c)2025 Fairday Research Limited
+#   Dependencies: semantic-link-labs
+#######################################
+
+### imports ###
+import pandas as pd
+import json
+import requests
+from sempy.fabric import list_tables, list_columns, list_workspaces
+from sempy_labs import deploy_semantic_model, refresh_semantic_model
+from sempy_labs.directlake import update_direct_lake_model_connection
+from sempy_labs.environment import list_environments 
+from sempy_labs.report import clone_report
+from sempy_labs.tom import connect_semantic_model
+from sempy_labs.warehouse import get_warehouse_columns 
+from notebookutils.notebook import list, getDefinition, updateDefinition, update, create
+from notebookutils.lakehouse import get as lhGet
+from notebookutils.credentials import getToken
+
+########################################
+# Helper functions
+########################################
+def _wh_convert_dtype(input_data_type: str) -> str:
+
+    if not input_data_type:
+        return None
+
+    input_data_type = input_data_type.lower()
+
+    data_type_mapping = {
+        "varchar": "String",
+        "int": "Int64",
+        "tinyint": "Int64",
+        "smallint": "Int64",
+        "bigint": "Int64",
+        "boolean": "Boolean",
+        "timestamp": "DateTime",
+        "date": "DateTime",
+        "decimal": "Double",
+        "numeric": "Double",
+        "double": "Double",
+        "float": "Double",
+        "binary": "Boolean",
+        "bit": "Boolean",
+        "long": "Int64",
+    }
+
+    if input_data_type.startswith("decimal"):
+        return "Double"
+    elif input_data_type.startswith("datetime"):
+        return "DateTime"
+
+    return data_type_mapping.get(input_data_type)
+
+########################################
+# Version
+########################################
+def libver():
+    print("fdsempy.py (c)2025 Fairday Research Ltd\nVersion 1.3.0 - July 2026")
+
+########################################
+# Add warehouse table to semantic model
+########################################
+def sempy_add_warehouse_table(workspace: str, dataset: str, table_name: str, warehouse: str, schema: str, source: str):
+    # get meta data of warehouse columns
+    meta_df = get_warehouse_columns(warehouse=warehouse, workspace=workspace)
+    # filter on target schema & table
+    tbl_meta_df = meta_df[(meta_df['Schema'] == schema) & (meta_df['Table Name'] == source)]
+
+    with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:
+        # create table
+        tom.add_table(name=table_name)
+        tom.add_entity_partition(table_name=table_name, entity_name=source, schema_name=schema)
+
+        # iterate through columns and convert SQ data types to SM data types
+        # then add the columns to the new table
+        count = 0
+        for _, row in tbl_meta_df.iterrows():
+            cname = row["Column Name"]
+            src_dtype = row["Data Type"]
+            tom_dtype = _wh_convert_dtype(src_dtype)
+            tom.add_data_column(
+                table_name=table_name,
+                column_name=cname,
+                source_column=cname,
+                data_type=tom_dtype,
+            )
+            count+=1
+    print(f"\U00002705 Columns added: {count}")
+
+########################################
+# Format semantic model tables
+########################################
+def sempy_format_tables(workspace: str, dataset: str):
+
+    # read tables & columns from model
+    tables = list_tables(dataset=dataset, workspace=workspace)
+
+    # read tables from mapping file and join to sempy list
+    fmtmap = pd.read_csv("/lakehouse/default/Files/fdsempymap.csv").set_index("Name")
+    fmttabs = tables.join(fmtmap, on="Name", how="left", lsuffix="_sm", rsuffix="_map")
+
+    with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:
+        count = 0
+        # rename tables
+        for _, row in fmttabs.iterrows():
+            tablename = row["Name"]
+            # rename table
+            if not pd.isna(row["Model Name"]):
+                print(f"Changing {tablename} to {row['Model Name']}")
+                tablename = row["Name"]
+                tab = tom.model.Tables[tablename]
+                tab.Name = row["Model Name"]
+                tom.add_changed_property(tab,"Name")
+                count+=1
+                if not pd.isna(row["Description_map"]):
+                    tab.Description = row["Description_map"]
+                    tom.add_changed_property(tab,"Description")
+    print(f"\U00002705 Formatted tables: {count}")
+
+########################################
+# Format semantic model columns
+########################################
+def sempy_format_columns(workspace: str, dataset: str, table: str):
+
+    cols = list_columns(dataset=dataset, workspace=workspace)
+    hide_n = 0
+    date_n = 0
+    dp6_n = 0
+    dp4_n = 0
+    int_n = 0
+    with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:
+        print("Updating columns")
+        # go through columns making changes
+        for _, row in cols.iterrows():
+            if (table == None) | (row["Table Name"] == table):
+                # hide columns
+                if row["Column Name"].startswith("scd_") | row["Column Name"].startswith("nk_"):     # hide scd/nk columns
+                    #print(f"Set {row['Table Name']}.{row['Column Name']} to hidden")
+                    tom.update_column(table_name=row["Table Name"], column_name=row["Column Name"], hidden=True)
+                    hide_n+=1
+                if row["Column Name"].endswith("id") & (row["Data Type"] == "Int64"):                # hide numeric id columns
+                    #print(f"Set {row['Table Name']}.{row['Column Name']} to hidden")
+                    tom.update_column(table_name=row["Table Name"], column_name=row["Column Name"], hidden=True)
+                    hide_n+=1
+                # set date format
+                if row["Data Type"].startswith("Date") & (not row["Column Name"].startswith("scd")):    # change non scd date formats
+                    #print(f"Set {row['Table Name']}.{row['Column Name']} to short date")
+                    tom.update_column(table_name=row["Table Name"], column_name=row["Column Name"], format_string='Short Date')
+                    date_n+=1
+                if (row["Data Type"] == "Double") & ("(6 dp)" in row['Column Name']):                    # decimal type add comma, set dps
+                    # set thousands separator (comma) and six dps
+                    #print(f"Set {row['Table Name']}.{row['Column Name']} to #,0.000000")
+                    tom.update_column(table_name=row["Table Name"], column_name=row["Column Name"], format_string='#,0.000000')
+                    dp6_n+=1
+                elif row["Data Type"] == "Double":                                                   # decimal type add comma, set dps
+                    # set thousands separator (comma) and four dps
+                    #print(f"Set {row['Table Name']}.{row['Column Name']} to #,0.0000")
+                    tom.update_column(table_name=row["Table Name"], column_name=row["Column Name"], format_string='#,0.0000')
+                    dp4_n+=1
+                if row["Data Type"].startswith("Int") & (not row["Column Name"].endswith("id")) & (not row["Table Name"].startswith("Date"))  & (not row["Table Name"].startswith("dim_date")):        # int type addcomma
+                    # set thousands separator (comma) and four dps
+                    #print(f"Set {row['Table Name']}.{row['Column Name']} to #,0")
+                    tom.update_column(table_name=row["Table Name"], column_name=row["Column Name"], format_string='#,0')
+                    int_n+=1
+    print(f"\U00002705 Hidden: {hide_n}, Date formatted: {date_n}, 6dp formatted {dp6_n}, 4dp formatted: {dp4_n}, int formatted: {int_n}")
+
+########################################
+# Deploy Notebooks to Workspace
+########################################
+def sempy_notebooks_deploy(src_workspace_id: str, tgt_workspace_id: str, core_notebooks: list, util_notebooks: list=[]):
+    ## get the id of the DWH environment in the target workspace (assume that it exists)
+    dwh_env = 'DWH'
+    env_df = list_environments(tgt_workspace_id) # get all the environments in the target workspace
+    tgt_dwh_env_id = env_df.loc[env_df['Environment Name']==dwh_env]['Environment Id'].values[0] # get the id of the first one that matches DWH
+    
+    # create a dict in the expected json format
+    tgt_env_ws_dict = dict(environmentId=tgt_dwh_env_id, workspaceId=tgt_workspace_id)
+    
+    # feedback to user
+    aw_df = list_workspaces()
+    
+    src_name = aw_df[aw_df['Id'] == src_workspace_id]['Name'].values[0]
+    tgt_name = aw_df[aw_df['Id'] == tgt_workspace_id]['Name'].values[0]
+    
+    print(f"\U00002139 Cloning notebooks from \033[1m{src_name}\033[0m to \033[1m{tgt_name}\033[0m")
+    for nb in core_notebooks:
+        print(f"\U000025AA {nb}")
+    if "_dev_" in tgt_name:
+        for nb in util_notebooks:
+            print(f"\U000025AA {nb}")   
+
+    count = 0
+    # get a list of artifacts in target
+    print(f"Building lists from target workspace...")
+    namelist = []
+    artifacts = list(tgt_workspace_id)
+    for artifact in artifacts:
+        if (artifact.type == 'Notebook') & ((artifact.displayName in core_notebooks) | (("_dev_" in tgt_name) & (artifact.displayName in util_notebooks))):
+            namelist.append(artifact.displayName)
+    
+    # Loop through notebook artifacts in source:
+    artifacts = list(src_workspace_id)
+    for artifact in artifacts:
+        json_hacked = False
+        if (artifact.type == 'Notebook') & ((artifact.displayName in core_notebooks) | (("_dev_" in tgt_name) & (artifact.displayName in util_notebooks))):  
+            print(f"\U000025B6 Processing {artifact.displayName}")
+            nb_name = artifact.displayName  # grab notebook name
+            print(f"\U0000231B getting definition of {artifact.displayName} from {src_name}...")
+            content = getDefinition(name=nb_name,workspaceId=src_workspace_id) # get notebook ipynb definition
+    
+            ### hack of target nb json ###
+            # turn content into json
+            json_def = json.loads(content)
+    
+            # if content contains a default lakehouse name
+            if json_def.get("metadata", {}).get("dependencies", {}).get("lakehouse", {}).get("default_lakehouse_name", {}):
+                # grab the default lakehouse name
+                src_nb_lh_name = json_def.get("metadata", {}).get("dependencies", {}).get("lakehouse", {}).get("default_lakehouse_name", {})
+    
+                # then get id of tgt lakehouse id with same name as default lakehouse in src notebook
+                tgt_lh = lhGet(name=src_nb_lh_name, workspaceId=tgt_workspace_id)
+                tgt_def_lh_id = tgt_lh.id
+    
+                # then create a dict containing tgt lakehouse id and workspace id
+                tgt_def_lh_dict_list = [dict(id=tgt_def_lh_id)]
+    
+                # if the known_lakehouses key is populated in the json
+                if json_def.get("metadata", {}).get("dependencies", {}).get("lakehouse", {}).get("known_lakehouses", {}):
+                    print(f"\U0000231B modifying [lakehouses] json key...")
+                    json_def["metadata"]["dependencies"]["lakehouse"]["known_lakehouses"] = tgt_def_lh_dict_list  
+                    json_def["metadata"]["dependencies"]["lakehouse"]["default_lakehouse"] = tgt_def_lh_id
+                    json_def["metadata"]["dependencies"]["lakehouse"]["default_lakehouse_workspace_id"] = workspaceId=tgt_workspace_id
+                    #print(f"\U0000267B new: %s" % (json_def["metadata"]["dependencies"]["lakehouse"]))
+                    json_hacked = True
+    
+            # if the environment is populated in the json
+            if json_def.get("metadata", {}).get("dependencies", {}).get("environment", {}):
+                print("\U0000231B modifying [environment] json key...")
+                json_def["metadata"]["dependencies"]["environment"] = tgt_env_ws_dict
+                #print(f"\U0000267B new: %s" % (json_def["metadata"]["dependencies"]["environment"]))
+                json_hacked = True
+    
+            if json_hacked:
+                # save this definition to a string. Otherwise we'll use the original content
+                content = json.dumps(json_def)
+    
+            # check if notebook exists in target (from artifact list)
+            if nb_name in namelist: # if your name's not on the list you ain't getting in mate
+                print(f"\U0000231B updating {nb_name} in {tgt_name}")
+                # if it does exist update the content:
+                updateDefinition(name=nb_name, content=content, workspaceId=tgt_workspace_id) 
+                # then update the properties
+                update(name=nb_name, newName=nb_name, description=artifact.description, workspaceId=tgt_workspace_id)
+                count+=1
+            else:
+            # if notebook doesn't exist create it 
+                print(f"\U0000231B creating {nb_name} in {tgt_name}")
+                create(name=nb_name, description=artifact.description, content=content, workspaceId=tgt_workspace_id)
+                count+=1
+            print(f"\U0001F197 {artifact.displayName} complete\n")
+    
+    # and we're done
+    print(f"\U00002705 Done. Notebooks processed: {count}")
+
+
+
+########################################
+# Deploy Models to Workspace
+########################################
+def sempy_models_deploy(src_workspace: str, tgt_workspace: str, models: list, source: str, source_type: str, refresh: bool=False):
+    count = 0
+    for dataset in models:
+        deploy_semantic_model(
+            source_dataset=dataset, 
+            source_workspace=src_workspace,
+            target_dataset=dataset,
+            target_workspace=tgt_workspace,
+            refresh_target_dataset=False,
+            overwrite=True
+        ) 
+    
+        # update models to point to local workspace
+        update_direct_lake_model_connection(
+            dataset = dataset,
+            workspace = tgt_workspace,
+            source = source,
+            source_type = source_type,
+            source_workspace = tgt_workspace
+        )
+        count+=1
+        # refresh the model if selected
+        if refresh:
+            refresh_semantic_model(dataset=dataset, workspace=tgt_workspace)
+    print(f"\U00002705 Models deployed: {count}")
+    
+########################################
+# Deploy Reports to Workspace
+########################################
+def sempy_reports_deploy(src_workspace: str, tgt_workspace: str, dataset: str, reports: list):
+    count = 0
+    for report in reports:
+        clone_report(
+            report = report,
+            cloned_report = report,
+            workspace = src_workspace,
+            target_workspace = tgt_workspace,
+            target_dataset = dataset,
+            target_dataset_workspace = tgt_workspace
+        )
+        count+=1
+    print(f"\U00002705 Reports deployed: {count}")
+
+########################################
+# Advanvced Deploy Reports to Workspace
+########################################
+def sempy_reports_deploy_adv(src_workspace: str, tgt_workspace: str, dataset: str, ds_workspace: str, reports: list):
+    count = 0
+    for (src_report, tgt_report) in reports:
+        if tgt_report == None:
+            tgt_report = src_report
+        clone_report(
+            report = src_report,
+            cloned_report = tgt_report,
+            workspace = src_workspace,
+            target_workspace = tgt_workspace,
+            target_dataset = dataset,
+            target_dataset_workspace = ds_workspace
+        )
+        count+=1
+    print(f"\U00002705 Reports deployed: {count}")
+
+########################################
+# Bind SM Connection (REST API call)
+########################################
+def set_conn(workspace_id: str, semantic_model_id: str, connection_id: str, connection_path: str) -> str:
+    # Authentication/token stuff
+    access_url = "https://analysis.windows.net/powerbi/api"
+    token = getToken(access_url)
+
+    bind_url = f"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/semanticModels/{semantic_model_id}/bindConnection"
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json"
+    }
+
+    payload = {
+        "connectionBinding": {
+            "id": connection_id,
+            "connectivityType": "ShareableCloud",
+            "connectionDetails": {
+            "type": "SQL",
+            "path": connection_path
+            }
+        }
+    }
+
+    # do it
+    response = requests.post(bind_url, json=payload, headers=headers)
+    if response.status_code == 200:
+        ret = 'Success'
+    else:
+        ret = f'Error: {response.text}'
+    return ret
+
+########################################
+#  Connection selector
+########################################
+def sel_conn(connections: list, workspace: str) -> str:
+    for ws, conn in connections:
+        if ws == workspace:
+            ret = conn
+    return(ret)

fdsempy-1.3.0/fdsempy.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: fdsempy
+Version: 1.3.0
+Summary: Fairday Sempy - semantic model utilities
+Author: Peter Johnson - Fairday Research Limited
+Requires-Dist: semantic-link-labs
+Requires-Dist: pandas
+Requires-Dist: json
+Requires-Dist: requests
+Requires-Dist: sempy
+Requires-Dist: notebookutils
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: summary

fdsempy-1.3.0/fdsempy.egg-info/SOURCES.txt

@@ -0,0 +1,8 @@
+setup.py
+fdsempy/__init__.py
+fdsempy/fdsempy.py
+fdsempy.egg-info/PKG-INFO
+fdsempy.egg-info/SOURCES.txt
+fdsempy.egg-info/dependency_links.txt
+fdsempy.egg-info/requires.txt
+fdsempy.egg-info/top_level.txt

fdsempy-1.3.0/fdsempy.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

fdsempy-1.3.0/fdsempy.egg-info/requires.txt

@@ -0,0 +1,6 @@
+semantic-link-labs
+pandas
+json
+requests
+sempy
+notebookutils

fdsempy-1.3.0/fdsempy.egg-info/top_level.txt

@@ -0,0 +1 @@
+fdsempy

fdsempy-1.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fdsempy-1.3.0/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup
+setup(name='fdsempy',
+      description='Fairday Sempy - semantic model utilities',
+      author='Peter Johnson - Fairday Research Limited',
+      version='1.3.0',
+      packages=['fdsempy'],
+      install_requires=[
+          'semantic-link-labs',
+          'pandas',
+          'json',
+          'requests',
+          'sempy',
+          'notebookutils'
+      ]
+      )