clickzetta-semantic-model-generator 1.0.1__tar.gz → 1.0.3__tar.gz

{clickzetta_semantic_model_generator-1.0.1 → clickzetta_semantic_model_generator-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: clickzetta-semantic-model-generator
-Version: 1.0.1
+Version: 1.0.3
 Summary: Curate a Semantic Model for ClickZetta Lakehouse
 License: Apache Software License; BSD License
 Author: qililiang
@@ -13,12 +13,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: looker
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: clickzetta-connector-python (==0.8.92)
-Requires-Dist: clickzetta-zettapark-python (==0.1.3)
+Requires-Dist: clickzetta-connector-python (>=0.8.92)
+Requires-Dist: clickzetta-zettapark-python (>=0.1.3)
 Requires-Dist: dashscope (>=1.22.2,<2.0.0)
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: looker-sdk (>=24.14.0,<25.0.0) ; extra == "looker"
-Requires-Dist: numpy (>=1.26.4,<2.0.0)
+Requires-Dist: numpy (>=1.26.4,<3.0.0)
 Requires-Dist: pandas (>=2.0.1,<3.0.0)
 Requires-Dist: protobuf (==5.26.1)
 Requires-Dist: pyarrow (==14.0.2)
@@ -31,7 +31,7 @@ Requires-Dist: streamlit (==1.36.0)
 Requires-Dist: streamlit-extras (==0.4.0)
 Requires-Dist: strictyaml (>=1.7.3,<2.0.0)
 Requires-Dist: tqdm (>=4.66.5,<5.0.0)
-Requires-Dist: urllib3 (>=1.26.19,<2.0.0)
+Requires-Dist: urllib3 (>=1.26.19,<3.0.0)
 Description-Content-Type: text/markdown
 
 # semantic-model-generator

{clickzetta_semantic_model_generator-1.0.1 → clickzetta_semantic_model_generator-1.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "clickzetta-semantic-model-generator"
-version = "1.0.1"
+version = "1.0.3"
 description = "Curate a Semantic Model for ClickZetta Lakehouse"
 authors = ["qililiang <qililiang@clickzetta.com>"]
 license = "Apache Software License; BSD License"
@@ -11,7 +11,7 @@ packages = [{include = "semantic_model_generator"}]
 python = ">=3.9,<3.9.7 || >3.9.7,<3.12"
 pandas = "^2.0.1"
 loguru = "^0.7.2"
-clickzetta-connector-python = "0.8.92"
+clickzetta-connector-python = ">=0.8.92"
 protobuf = "5.26.1"
 pydantic = ">=2.8.2,<3.0.0"
 PyYAML = "^6.0.1"
@@ -22,11 +22,11 @@ sqlglot = "25.10.0"
 strictyaml = "^1.7.3"
 streamlit = "1.36.0"
 streamlit-extras = "0.4.0"
-numpy = "^1.26.4"
+numpy = ">=1.26.4,<3.0.0"
 python-dotenv = "^1.0.1"
-urllib3 = "^1.26.19"
+urllib3 = ">=1.26.19,<3.0.0"
 requests = "^2.32.3"
-clickzetta-zettapark-python = "0.1.3"
+clickzetta-zettapark-python = ">=0.1.3"
 dashscope = "^1.22.2"
 
 # Optional dependencies for functionality such as partner semantic model support.

{clickzetta_semantic_model_generator-1.0.1 → clickzetta_semantic_model_generator-1.0.3}/semantic_model_generator/clickzetta_utils/clickzetta_connector.py

@@ -4,7 +4,6 @@ import concurrent.futures
 import re
 from collections import defaultdict
 from contextlib import contextmanager
-from types import SimpleNamespace
 from typing import Any, Dict, Generator, List, Optional, TypeVar, Union
 
 import pandas as pd
@@ -12,10 +11,7 @@ from clickzetta.zettapark.session import Session
 from loguru import logger
 
 from semantic_model_generator.clickzetta_utils import env_vars
-from semantic_model_generator.clickzetta_utils.utils import (
-    clickzetta_connection,
-    create_session,
-)
+from semantic_model_generator.clickzetta_utils.utils import create_session
 from semantic_model_generator.data_processing.data_types import Column, Table
 
 ConnectionType = TypeVar("ConnectionType", bound=Session)
@@ -115,7 +111,9 @@ class ClickzettaCursor:
     def execute(self, query: str) -> "ClickzettaCursor":
         self._df = _execute_query_to_pandas(self._session, query)
         columns = [] if self._df is None else list(self._df.columns)
-        self.description = [(col, None, None, None, None, None, None) for col in columns]
+        self.description = [
+            (col, None, None, None, None, None, None) for col in columns
+        ]
         return self
 
     def fetchone(self) -> Optional[tuple[Any, ...]]:
@@ -159,7 +157,11 @@ def _quote_identifier(name: str) -> str:
 
 def _qualify_table(workspace: str, schema_name: str, table_name: str) -> str:
     return ".".join(
-        [_quote_identifier(workspace), _quote_identifier(schema_name), _quote_identifier(table_name)]
+        [
+            _quote_identifier(workspace),
+            _quote_identifier(schema_name),
+            _quote_identifier(table_name),
+        ]
     )
 
 
@@ -214,12 +216,18 @@ def _fetch_distinct_values(
     column_name: str,
     ndv: int,
 ) -> Optional[List[str]]:
-    workspace_part = _sanitize_identifier(workspace, workspace).upper() if workspace else ""
-    schema_part = _sanitize_identifier(schema_name, schema_name).upper() if schema_name else ""
+    workspace_part = (
+        _sanitize_identifier(workspace, workspace).upper() if workspace else ""
+    )
+    schema_part = (
+        _sanitize_identifier(schema_name, schema_name).upper() if schema_name else ""
+    )
     table_part = _sanitize_identifier(table_name, table_name).upper()
     column_part = _sanitize_identifier(column_name, column_name).upper()
 
-    qualified_parts = [part for part in (workspace_part, schema_part, table_part) if part]
+    qualified_parts = [
+        part for part in (workspace_part, schema_part, table_part) if part
+    ]
     qualified_table = ".".join(qualified_parts)
 
     query = f"SELECT DISTINCT {column_part} FROM {qualified_table} LIMIT {ndv}"
@@ -257,7 +265,6 @@ def _get_column_representation(
     else:
         column_datatype = str(column_datatype_raw)
     column_datatype = _normalize_column_type(column_datatype)
-    normalized_type = column_datatype.split("(")[0].strip()
     column_values = (
         _fetch_distinct_values(
             session=session,
@@ -351,7 +358,14 @@ def _catalog_category(session: Session, workspace: str) -> str:
         return "UNKNOWN"
 
     df.columns = [str(col).upper() for col in df.columns]
-    name_col = next((col for col in ("WORKSPACE_NAME", "NAME", "CATALOG_NAME") if col in df.columns), None)
+    name_col = next(
+        (
+            col
+            for col in ("WORKSPACE_NAME", "NAME", "CATALOG_NAME")
+            if col in df.columns
+        ),
+        None,
+    )
     category_col = next((col for col in ("CATEGORY",) if col in df.columns), None)
     if not name_col or not category_col:
         _CATALOG_CATEGORY_CACHE[workspace_upper] = "UNKNOWN"
@@ -408,7 +422,9 @@ ORDER BY kc.ordinal_position
         if result is not None:
             return result
     except Exception:
-        logger.debug("Primary key lookup via sys.information_schema failed; falling back.")
+        logger.debug(
+            "Primary key lookup via sys.information_schema failed; falling back."
+        )
 
     fallback_query = f"""
 SELECT kc.column_name
@@ -423,7 +439,13 @@ ORDER BY kc.ordinal_position
         if result is not None:
             return result
     except Exception as exc:
-        logger.warning("Primary key lookup failed for {}.{}.{}: {}", workspace, schema_name, table_name, exc)
+        logger.warning(
+            "Primary key lookup failed for {}.{}.{}: {}",
+            workspace,
+            schema_name,
+            table_name,
+            exc,
+        )
     return None
 
 
@@ -432,9 +454,7 @@ def _build_information_schema_query(
     table_schema: Optional[str],
     table_names: Optional[List[str]],
 ) -> str:
-    where_conditions: List[str] = [
-        "1=1"
-    ]
+    where_conditions: List[str] = ["1=1"]
     if table_schema:
         where_conditions.append(f"upper(t.table_schema) = '{table_schema.upper()}'")
     if table_names:
@@ -442,7 +462,6 @@ def _build_information_schema_query(
         where_conditions.append(f"upper(t.table_name) IN ({formatted_names})")
 
     where_clause = " AND ".join(where_conditions)
-    base = "information_schema"
     return f"""
 SELECT
     t.table_schema AS {_TABLE_SCHEMA_COL},
@@ -474,27 +493,48 @@ def _fetch_columns_via_show(
     schema = table_schema.upper() if table_schema else ""
 
     for table_name in table_names:
-        qualified_parts = [part for part in (catalog, schema, table_name.upper()) if part]
+        qualified_parts = [
+            part for part in (catalog, schema, table_name.upper()) if part
+        ]
         qualified_table = ".".join(qualified_parts)
         query = f"SHOW COLUMNS IN {qualified_table}"
         try:
             df = session.sql(query).to_pandas()
         except Exception as exc:
-            logger.debug("SHOW COLUMNS fallback failed for {}: {}", qualified_table, exc)
+            logger.debug(
+                "SHOW COLUMNS fallback failed for {}: {}", qualified_table, exc
+            )
             continue
         if df.empty:
             continue
         df.columns = [str(col).upper() for col in df.columns]
-        schema_col = next((col for col in ("TABLE_SCHEMA", "SCHEMA_NAME") if col in df.columns), None)
-        table_col = next((col for col in ("TABLE_NAME", "NAME") if col in df.columns), None)
-        column_col = next((col for col in ("COLUMN_NAME", "NAME") if col in df.columns and col != table_col), None)
-        datatype_col = next((col for col in ("DATA_TYPE", "TYPE") if col in df.columns), None)
-        comment_col = next((col for col in ("COMMENT", "COLUMN_COMMENT") if col in df.columns), None)
+        schema_col = next(
+            (col for col in ("TABLE_SCHEMA", "SCHEMA_NAME") if col in df.columns), None
+        )
+        table_col = next(
+            (col for col in ("TABLE_NAME", "NAME") if col in df.columns), None
+        )
+        column_col = next(
+            (
+                col
+                for col in ("COLUMN_NAME", "NAME")
+                if col in df.columns and col != table_col
+            ),
+            None,
+        )
+        datatype_col = next(
+            (col for col in ("DATA_TYPE", "TYPE") if col in df.columns), None
+        )
+        comment_col = next(
+            (col for col in ("COMMENT", "COLUMN_COMMENT") if col in df.columns), None
+        )
 
         normalized = pd.DataFrame()
         normalized[_TABLE_SCHEMA_COL] = df[schema_col] if schema_col else table_schema
         normalized[_TABLE_NAME_COL] = df[table_col] if table_col else table_name
-        normalized[_COLUMN_NAME_COL] = df[column_col] if column_col else df.index.astype(str)
+        normalized[_COLUMN_NAME_COL] = (
+            df[column_col] if column_col else df.index.astype(str)
+        )
         normalized[_DATATYPE_COL] = df[datatype_col] if datatype_col else ""
         normalized[_COLUMN_COMMENT_ALIAS] = df[comment_col] if comment_col else ""
         normalized[_TABLE_COMMENT_COL] = ""
@@ -552,6 +592,7 @@ def get_valid_schemas_tables_columns_df(
     if _TABLE_SCHEMA_COL in result.columns:
         result[_TABLE_SCHEMA_COL] = result[_TABLE_SCHEMA_COL].astype(str).str.upper()
     if _IS_PRIMARY_KEY_COL in result.columns:
+
         def _normalize_pk(value: Any) -> bool:
             if isinstance(value, bool):
                 return value
@@ -617,10 +658,11 @@ def fetch_tables_views_in_schema(
     workspace_upper = workspace.upper()
     schema_upper = schema.upper()
 
-    target = ""
     try:
         if workspace_upper and schema_upper:
-            df = session.sql(f"SHOW TABLES IN {workspace_upper}.{schema_upper}").to_pandas()
+            df = session.sql(
+                f"SHOW TABLES IN {workspace_upper}.{schema_upper}"
+            ).to_pandas()
         else:
             df = session.sql("SHOW TABLES").to_pandas()
     except Exception as exc:  # pragma: no cover
@@ -634,17 +676,27 @@ def fetch_tables_views_in_schema(
     df.columns = [str(col).upper() for col in df.columns]
     name_column = "TABLE_NAME" if "TABLE_NAME" in df.columns else df.columns[0]
     schema_column = next(
-        (col for col in ("SCHEMA_NAME", "TABLE_SCHEMA", "NAMESPACE") if col in df.columns),
+        (
+            col
+            for col in ("SCHEMA_NAME", "TABLE_SCHEMA", "NAMESPACE")
+            if col in df.columns
+        ),
         None,
     )
     catalog_column = next(
-        (col for col in ("CATALOG_NAME", "WORKSPACE_NAME", "TABLE_CATALOG") if col in df.columns),
+        (
+            col
+            for col in ("CATALOG_NAME", "WORKSPACE_NAME", "TABLE_CATALOG")
+            if col in df.columns
+        ),
         None,
     )
 
     results: List[str] = []
     for _, row in df.iterrows():
-        if _value_is_true(row.get("IS_VIEW")) and not _value_is_true(row.get("IS_MATERIALIZED_VIEW")):
+        if _value_is_true(row.get("IS_VIEW")) and not _value_is_true(
+            row.get("IS_MATERIALIZED_VIEW")
+        ):
             continue
         if _value_is_true(row.get("IS_EXTERNAL")):
             continue
@@ -756,7 +808,11 @@ def fetch_yaml_names_in_stage(
     if stage.lower().startswith("volume:user://"):
         volume_body = stage[len("volume:") :]
         # Normalize relative directory
-        relative = volume_body[len("user://") :] if volume_body.startswith("user://") else volume_body
+        relative = (
+            volume_body[len("user://") :]
+            if volume_body.startswith("user://")
+            else volume_body
+        )
         relative = relative.lstrip("~/")
         relative = relative.strip("/")
 
@@ -842,7 +898,9 @@ def create_table_in_schema(
     table_fqn: str,
     columns_schema: Dict[str, str],
 ) -> bool:
-    fields = ", ".join(f"{_quote_identifier(name)} {dtype}" for name, dtype in columns_schema.items())
+    fields = ", ".join(
+        f"{_quote_identifier(name)} {dtype}" for name, dtype in columns_schema.items()
+    )
     query = f"CREATE TABLE IF NOT EXISTS {table_fqn} ({fields})"
     try:
         session.sql(query).collect()

{clickzetta_semantic_model_generator-1.0.1 → clickzetta_semantic_model_generator-1.0.3}/semantic_model_generator/clickzetta_utils/env_vars.py

@@ -20,7 +20,9 @@ _CONFIG_PATHS = [
 _ACTIVE_CONFIG_PATH: Optional[str] = None
 
 
-def _load_config_from_file() -> Tuple[Optional[Dict[str, str]], Dict[str, Dict[str, str]]]:
+def _load_config_from_file() -> (
+    Tuple[Optional[Dict[str, str]], Dict[str, Dict[str, str]]]
+):
     global _ACTIVE_CONFIG_PATH
     _ACTIVE_CONFIG_PATH = None
     for path in _CONFIG_PATHS:
@@ -91,7 +93,10 @@ def _deep_lookup(mapping: Any, key: str) -> Optional[Any]:
         if isinstance(current, dict):
             for candidate_key, candidate_value in current.items():
                 candidate_key_str = str(candidate_key).lower()
-                if candidate_key_str == normalized_key and candidate_value not in (None, ""):
+                if candidate_key_str == normalized_key and candidate_value not in (
+                    None,
+                    "",
+                ):
                     return candidate_value
                 if isinstance(candidate_value, (dict, list)):
                     queue.append(candidate_value)