semantic-link-labs 0.9.10__py3-none-any.whl → 0.10.0__py3-none-any.whl

sempy_labs/_environments.py

@@ -3,6 +3,7 @@ import sempy_labs._icons as icons
 from typing import Optional
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
+    resolve_workspace_id,
     _base_api,
     _create_dataframe,
     resolve_item_id,
@@ -67,10 +68,16 @@ def list_environments(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         "Environment Name": "string",
         "Environment Id": "string",
         "Description": "string",
+        "Publish State": "string",
+        "Publish Target Version": "string",
+        "Publish Start Time": "string",
+        "Publish End Time": "string",
+        "Spark Libraries State": "string",
+        "Spark Settings State": "string",
     }
     df = _create_dataframe(columns=columns)
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     responses = _base_api(
         request=f"/v1/workspaces/{workspace_id}/environments",
@@ -80,10 +87,21 @@ def list_environments(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
 
     for r in responses:
         for v in r.get("value", []):
+            pub = v.get("properties", {}).get("publishDetails", {})
             new_data = {
                 "Environment Name": v.get("displayName"),
                 "Environment Id": v.get("id"),
                 "Description": v.get("description"),
+                "Publish State": pub.get("state"),
+                "Publish Target Version": pub.get("targetVersion"),
+                "Publish Start Time": pub.get("startTime"),
+                "Publish End Time": pub.get("endTime"),
+                "Spark Libraries State": pub.get("componentPublishInfo", {})
+                .get("sparkLibraries", {})
+                .get("state"),
+                "Spark Settings State": pub.get("componentPublishInfo", {})
+                .get("sparkSettings", {})
+                .get("state"),
             }
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 

sempy_labs/_generate_semantic_model.py

@@ -493,7 +493,7 @@ def get_semantic_model_size(
     Returns
     -------
     int
-        The size of the semantic model in
+        The size of the semantic model in bytes
     """
 
     dict = fabric.evaluate_dax(

sempy_labs/_helper_functions.py

@@ -17,6 +17,8 @@ import numpy as np
 from IPython.display import display, HTML
 import requests
 import sempy_labs._authentication as auth
+from jsonpath_ng.ext import parse
+from jsonpath_ng.jsonpath import Fields, Index
 
 
 def _build_url(url: str, params: dict) -> str:
@@ -74,6 +76,15 @@ def create_abfss_path(
     return path
 
 
+def create_abfss_path_from_path(
+    lakehouse_id: UUID, workspace_id: UUID, file_path: str
+) -> str:
+
+    fp = _get_default_file_path()
+
+    return f"abfss://{workspace_id}@{fp}/{lakehouse_id}/{file_path}"
+
+
 def _get_default_file_path() -> str:
 
     default_file_storage = _get_fabric_context_setting(name="fs.defaultFS")
@@ -266,7 +277,7 @@ def create_item(
         lro_return_status_code=True,
     )
     print(
-        f"{icons.green_dot} The '{name}' {item_type} has been successfully created within the in the '{workspace_name}' workspace."
+        f"{icons.green_dot} The '{name}' {item_type} has been successfully created within the '{workspace_name}' workspace."
     )
 
 
@@ -278,10 +289,9 @@ def get_item_definition(
     return_dataframe: bool = True,
     decode: bool = True,
 ):
-
     from sempy_labs._utils import item_types
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
     item_id = resolve_item_id(item, type, workspace_id)
     item_type_url = item_types.get(type)[1]
     path = item_types.get(type)[2]
@@ -304,92 +314,11 @@ def get_item_definition(
         p.get("payload") for p in result["definition"]["parts"] if p.get("path") == path
     )
     if decode:
-        json.loads(_decode_b64(value))
+        return json.loads(_decode_b64(value))
     else:
         return value
 
 
-def resolve_item_id(
-    item: str | UUID, type: Optional[str] = None, workspace: Optional[str | UUID] = None
-) -> UUID:
-
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    item_id = None
-
-    if _is_valid_uuid(item):
-        # Check (optional)
-        item_id = item
-        try:
-            _base_api(
-                request=f"/v1/workspaces/{workspace_id}/items/{item_id}",
-                client="fabric_sp",
-            )
-        except FabricHTTPException:
-            raise ValueError(
-                f"{icons.red_dot} The '{item_id}' item was not found in the '{workspace_name}' workspace."
-            )
-    else:
-        if type is None:
-            raise ValueError(
-                f"{icons.red_dot} The 'type' parameter is required if specifying an item name."
-            )
-        responses = _base_api(
-            request=f"/v1/workspaces/{workspace_id}/items?type={type}",
-            client="fabric_sp",
-            uses_pagination=True,
-        )
-        for r in responses:
-            for v in r.get("value", []):
-                display_name = v.get("displayName")
-                if display_name == item:
-                    item_id = v.get("id")
-                    break
-
-    if item_id is None:
-        raise ValueError(
-            f"{icons.red_dot} There's no item '{item}' of type '{type}' in the '{workspace_name}' workspace."
-        )
-
-    return item_id
-
-
-def resolve_item_name_and_id(
-    item: str | UUID, type: Optional[str] = None, workspace: Optional[str | UUID] = None
-) -> Tuple[str, UUID]:
-
-    workspace_id = resolve_workspace_id(workspace)
-    item_id = resolve_item_id(item=item, type=type, workspace=workspace_id)
-    item_name = (
-        _base_api(
-            request=f"/v1/workspaces/{workspace_id}/items/{item_id}", client="fabric_sp"
-        )
-        .json()
-        .get("displayName")
-    )
-
-    return item_name, item_id
-
-
-def resolve_item_name(item_id: UUID, workspace: Optional[str | UUID] = None) -> str:
-
-    workspace_id = resolve_workspace_id(workspace)
-    try:
-        item_name = (
-            _base_api(
-                request=f"/v1/workspaces/{workspace_id}/items/{item_id}",
-                client="fabric_sp",
-            )
-            .json()
-            .get("displayName")
-        )
-    except FabricHTTPException:
-        raise ValueError(
-            f"{icons.red_dot} The '{item_id}' item was not found in the '{workspace_id}' workspace."
-        )
-
-    return item_name
-
-
 def resolve_lakehouse_name_and_id(
     lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
 ) -> Tuple[str, UUID]:
@@ -732,7 +661,7 @@ def save_as_delta_table(
             "bool": ("pa", pa.bool_(), BooleanType()),
             "boolean": ("pa", pa.bool_(), BooleanType()),
             "date": ("pa", pa.date32(), DateType()),
-            "timestamp": ("pa", pa.timestamp("ms"), TimestampType()),
+            "timestamp": ("pa", pa.timestamp("us"), TimestampType()),
         }
         return {k: v[1] if pure_python else v[2] for k, v in common_mapping.items()}
 
@@ -936,6 +865,87 @@ def resolve_workspace_name_and_id(
     return workspace_name, workspace_id
 
 
+def resolve_item_id(
+    item: str | UUID, type: Optional[str] = None, workspace: Optional[str | UUID] = None
+) -> UUID:
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    item_id = None
+
+    if _is_valid_uuid(item):
+        # Check (optional)
+        item_id = item
+        try:
+            _base_api(
+                request=f"/v1/workspaces/{workspace_id}/items/{item_id}",
+                client="fabric_sp",
+            )
+        except FabricHTTPException:
+            raise ValueError(
+                f"{icons.red_dot} The '{item_id}' item was not found in the '{workspace_name}' workspace."
+            )
+    else:
+        if type is None:
+            raise ValueError(
+                f"{icons.red_dot} The 'type' parameter is required if specifying an item name."
+            )
+        responses = _base_api(
+            request=f"/v1/workspaces/{workspace_id}/items?type={type}",
+            client="fabric_sp",
+            uses_pagination=True,
+        )
+        for r in responses:
+            for v in r.get("value", []):
+                display_name = v.get("displayName")
+                if display_name == item:
+                    item_id = v.get("id")
+                    break
+
+    if item_id is None:
+        raise ValueError(
+            f"{icons.red_dot} There's no item '{item}' of type '{type}' in the '{workspace_name}' workspace."
+        )
+
+    return item_id
+
+
+def resolve_item_name_and_id(
+    item: str | UUID, type: Optional[str] = None, workspace: Optional[str | UUID] = None
+) -> Tuple[str, UUID]:
+
+    workspace_id = resolve_workspace_id(workspace)
+    item_id = resolve_item_id(item=item, type=type, workspace=workspace_id)
+    item_name = (
+        _base_api(
+            request=f"/v1/workspaces/{workspace_id}/items/{item_id}", client="fabric_sp"
+        )
+        .json()
+        .get("displayName")
+    )
+
+    return item_name, item_id
+
+
+def resolve_item_name(item_id: UUID, workspace: Optional[str | UUID] = None) -> str:
+
+    workspace_id = resolve_workspace_id(workspace)
+    try:
+        item_name = (
+            _base_api(
+                request=f"/v1/workspaces/{workspace_id}/items/{item_id}",
+                client="fabric_sp",
+            )
+            .json()
+            .get("displayName")
+        )
+    except FabricHTTPException:
+        raise ValueError(
+            f"{icons.red_dot} The '{item_id}' item was not found in the '{workspace_id}' workspace."
+        )
+
+    return item_name
+
+
 def _extract_json(dataframe: pd.DataFrame) -> dict:
 
     payload = dataframe["payload"].iloc[0]
@@ -1540,38 +1550,18 @@ def _get_column_aggregate(
     workspace: Optional[str | UUID] = None,
     function: str = "max",
     default_value: int = 0,
+    schema_name: Optional[str] = None,
 ) -> int | Dict[str, int]:
 
     workspace_id = resolve_workspace_id(workspace)
     lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
-    path = create_abfss_path(lakehouse_id, workspace_id, table_name)
+    path = create_abfss_path(lakehouse_id, workspace_id, table_name, schema_name)
     df = _read_delta_table(path)
 
-    if isinstance(column_name, str):
-        result = _get_aggregate(
-            df=df,
-            column_name=column_name,
-            function=function,
-            default_value=default_value,
-        )
-    elif isinstance(column_name, list):
-        result = {}
-        for col in column_name:
-            result[col] = _get_aggregate(
-                df=df,
-                column_name=col,
-                function=function,
-                default_value=default_value,
-            )
-    else:
-        raise TypeError("column_name must be a string or a list of strings.")
-
-    return result
-
-
-def _get_aggregate(df, column_name, function, default_value: int = 0) -> int:
+    function = function.lower()
 
-    function = function.upper()
+    if isinstance(column_name, str):
+        column_name = [column_name]
 
     if _pure_python_notebook():
         import polars as pl
@@ -1581,36 +1571,82 @@ def _get_aggregate(df, column_name, function, default_value: int = 0) -> int:
 
         df = pl.from_pandas(df)
 
-        # Perform aggregation
-        if "DISTINCT" in function:
-            if isinstance(df[column_name].dtype, pl.Decimal):
-                result = df[column_name].cast(pl.Float64).n_unique()
+        def get_expr(col):
+            col_dtype = df.schema[col]
+
+            if "approx" in function:
+                return pl.col(col).unique().count().alias(col)
+            elif "distinct" in function:
+                if col_dtype == pl.Decimal:
+                    return pl.col(col).cast(pl.Float64).n_unique().alias(col)
+                else:
+                    return pl.col(col).n_unique().alias(col)
+            elif function == "sum":
+                return pl.col(col).sum().alias(col)
+            elif function == "min":
+                return pl.col(col).min().alias(col)
+            elif function == "max":
+                return pl.col(col).max().alias(col)
+            elif function == "count":
+                return pl.col(col).count().alias(col)
+            elif function in {"avg", "mean"}:
+                return pl.col(col).mean().alias(col)
             else:
-                result = df[column_name].n_unique()
-        elif "APPROX" in function:
-            result = df[column_name].unique().shape[0]
-        else:
-            try:
-                result = getattr(df[column_name], function.lower())()
-            except AttributeError:
                 raise ValueError(f"Unsupported function: {function}")
 
-        return result if result is not None else default_value
+        exprs = [get_expr(col) for col in column_name]
+        aggs = df.select(exprs).to_dict(as_series=False)
+
+        if len(column_name) == 1:
+            result = aggs[column_name[0]][0] or default_value
+        else:
+            result = {col: aggs[col][0] for col in column_name}
     else:
-        from pyspark.sql.functions import approx_count_distinct
-        from pyspark.sql import functions as F
+        from pyspark.sql.functions import (
+            count,
+            sum,
+            min,
+            max,
+            avg,
+            approx_count_distinct,
+            countDistinct,
+        )
+
+        result = None
+        if "approx" in function:
+            spark_func = approx_count_distinct
+        elif "distinct" in function:
+            spark_func = countDistinct
+        elif function == "count":
+            spark_func = count
+        elif function == "sum":
+            spark_func = sum
+        elif function == "min":
+            spark_func = min
+        elif function == "max":
+            spark_func = max
+        elif function == "avg":
+            spark_func = avg
+        else:
+            raise ValueError(f"Unsupported function: {function}")
 
-        if isinstance(df, pd.DataFrame):
-            df = _create_spark_dataframe(df)
+        agg_exprs = []
+        for col in column_name:
+            agg_exprs.append(spark_func(col).alias(col))
 
-        if "DISTINCT" in function:
-            result = df.select(F.count_distinct(F.col(column_name)))
-        elif "APPROX" in function:
-            result = df.select(approx_count_distinct(column_name))
+        aggs = df.agg(*agg_exprs).collect()[0]
+        if len(column_name) == 1:
+            result = aggs[0] or default_value
         else:
-            result = df.selectExpr(f"{function}({column_name})")
+            result = {col: aggs[col] for col in column_name}
+
+    return result
 
-        return result.collect()[0][0] or default_value
+
+def _create_spark_dataframe(df: pd.DataFrame):
+
+    spark = _create_spark_session()
+    return spark.createDataFrame(df)
 
 
 def _make_list_unique(my_list):
@@ -1705,6 +1741,9 @@ def _process_and_display_chart(df, title, widget):
     df["Start"] = df["Start"] - Offset
     df["End"] = df["End"] - Offset
 
+    unique_objects = df["Object Name"].nunique()
+    height = min(max(400, unique_objects * 30), 1000)
+
     # Vega-Lite spec for Gantt chart
     spec = (
         """{
@@ -1714,7 +1753,9 @@ def _process_and_display_chart(df, title, widget):
         + df.to_json(orient="records")
         + """ },
         "width": 700,
-        "height": 400,
+        "height": """
+        + str(height)
+        + """,
         "mark": "bar",
         "encoding": {
             "y": {
@@ -2211,3 +2252,186 @@ def _xml_to_dict(element):
             element.text.strip() if element.text and element.text.strip() else None
         )
     return data
+
+
+def file_exists(file_path: str) -> bool:
+    """
+    Check if a file exists in the given path.
+
+    Parameters
+    ----------
+    file_path : str
+        The path to the file.
+
+    Returns
+    -------
+    bool
+        True if the file exists, False otherwise.
+    """
+
+    import notebookutils
+
+    return len(notebookutils.fs.ls(file_path)) > 0
+
+
+def generate_number_guid():
+
+    guid = uuid.uuid4()
+    return str(guid.int & ((1 << 64) - 1))
+
+
+def get_url_content(url: str):
+
+    if "github.com" in url and "/blob/" in url:
+        url = url.replace("github.com", "raw.githubusercontent.com")
+        url = url.replace("/blob/", "/")
+
+    response = requests.get(url)
+    if response.ok:
+        try:
+            data = response.json()  # Only works if the response is valid JSON
+        except ValueError:
+            data = response.text  # Fallback: get raw text content
+        return data
+    else:
+        print(f"Failed to fetch raw content: {response.status_code}")
+
+
+def generate_hex(length: int = 10) -> str:
+    """
+    Generate a random hex string of the specified length. Used for generating IDs for report objects (page, visual, bookmark etc.).
+    """
+    import secrets
+
+    return secrets.token_hex(length)
+
+
+def decode_payload(payload):
+
+    if is_base64(payload):
+        try:
+            decoded_payload = json.loads(base64.b64decode(payload).decode("utf-8"))
+        except Exception:
+            decoded_payload = base64.b64decode(payload)
+    elif isinstance(payload, dict):
+        decoded_payload = payload
+    else:
+        raise ValueError("Payload must be a dictionary or a base64 encoded value.")
+
+    return decoded_payload
+
+
+def is_base64(s):
+    try:
+        # Add padding if needed
+        s_padded = s + "=" * (-len(s) % 4)
+        decoded = base64.b64decode(s_padded, validate=True)
+        # Optional: check if re-encoding gives the original (excluding padding)
+        return base64.b64encode(decoded).decode().rstrip("=") == s.rstrip("=")
+    except Exception:
+        return False
+
+
+def get_jsonpath_value(
+    data, path, default=None, remove_quotes=False, fix_true: bool = False
+):
+    matches = parse(path).find(data)
+    result = matches[0].value if matches else default
+    if result and remove_quotes and isinstance(result, str):
+        if result.startswith("'") and result.endswith("'"):
+            result = result[1:-1]
+    if fix_true and isinstance(result, str):
+        if result.lower() == "true":
+            result = True
+        elif result.lower() == "false":
+            result = False
+    return result
+
+
+def set_json_value(payload: dict, json_path: str, json_value: str | dict | List):
+
+    jsonpath_expr = parse(json_path)
+    matches = jsonpath_expr.find(payload)
+
+    if matches:
+        # Update all matches
+        for match in matches:
+            parent = match.context.value
+            path = match.path
+            if isinstance(path, Fields):
+                parent[path.fields[0]] = json_value
+            elif isinstance(path, Index):
+                parent[path.index] = json_value
+    else:
+        # Handle creation
+        parts = json_path.lstrip("$").strip(".").split(".")
+        current = payload
+
+        for i, part in enumerate(parts):
+            is_last = i == len(parts) - 1
+
+            # Detect list syntax like "lockAspect[*]"
+            list_match = re.match(r"(\w+)\[\*\]", part)
+            if list_match:
+                list_key = list_match.group(1)
+                if list_key not in current or not isinstance(current[list_key], list):
+                    # Initialize with one dict element
+                    current[list_key] = [{}]
+
+                for item in current[list_key]:
+                    if is_last:
+                        # Last part, assign value
+                        item = json_value
+                    else:
+                        # Proceed to next level
+                        if not isinstance(item, dict):
+                            raise ValueError(
+                                f"Expected dict in list for key '{list_key}', got {type(item)}"
+                            )
+                        next_part = ".".join(parts[i + 1 :])
+                        set_json_value(item, "$." + next_part, json_value)
+                return payload
+            else:
+                if part not in current or not isinstance(current[part], dict):
+                    current[part] = {} if not is_last else json_value
+                elif is_last:
+                    current[part] = json_value
+                current = current[part]
+
+    return payload
+
+
+def remove_json_value(path: str, payload: dict, json_path: str, verbose: bool = True):
+
+    if not isinstance(payload, dict):
+        raise ValueError(
+            f"{icons.red_dot} Cannot apply json_path to non-dictionary payload in '{path}'."
+        )
+
+    jsonpath_expr = parse(json_path)
+    matches = jsonpath_expr.find(payload)
+
+    if not matches and verbose:
+        print(
+            f"{icons.red_dot} No match found for '{json_path}' in '{path}'. Skipping."
+        )
+        return payload
+
+    for match in matches:
+        parent = match.context.value
+        path_expr = match.path
+
+        if isinstance(path_expr, Fields):
+            key = path_expr.fields[0]
+            if key in parent:
+                del parent[key]
+                if verbose:
+                    print(f"{icons.green_dot} Removed key '{key}' from '{path}'.")
+        elif isinstance(path_expr, Index):
+            index = path_expr.index
+            if isinstance(parent, list) and 0 <= index < len(parent):
+                parent.pop(index)
+                if verbose:
+                    print(f"{icons.green_dot} Removed index [{index}] from '{path}'.")
+
+    return payload