openmetadata-ingestion 1.6.3.0.dev0__py3-none-any.whl → 1.6.3.2__py3-none-any.whl

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-01-31T10:46:58+00:00
+#   timestamp: 2025-02-04T15:21:36+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -496,9 +496,13 @@ class LookerSource(DashboardServiceSource):
             try:
                 project_parser = self.parser.get(explore.project_name)
                 if project_parser:
-                    return project_parser.parsed_files.get(
+                    explore_sql = project_parser.parsed_files.get(
                         Includes(get_path_from_link(explore.lookml_link))
                     )
+                    logger.debug(
+                        f"Explore SQL for project {explore.project_name}: \n{explore_sql}"
+                    )
+                    return explore_sql
             except Exception as err:
                 logger.warning(f"Exception getting the model sql: {err}")
 
@@ -597,6 +601,7 @@ class LookerSource(DashboardServiceSource):
                 sql_query = view.derived_table.sql
                 if not sql_query:
                     return
+                logger.debug(f"Processing view [{view.name}] with SQL: \n[{sql_query}]")
                 for db_service_name in db_service_names or []:
                     lineage_parser = LineageParser(
                         sql_query,

metadata/ingestion/source/dashboard/looker/parser.py

@@ -181,6 +181,9 @@ class LkmlParser:
         Otherwise, return None
         """
         if view_name in self._views_cache:
+            logger.debug(
+                f"Found view [{view_name}] in cache: \n{self._views_cache[view_name]}"
+            )
             return self._views_cache[view_name]
 
         return None

metadata/ingestion/source/database/databricks/metadata.py

@@ -113,11 +113,6 @@ _type_map.update(
 )
 
 
-def format_schema_name(schema):
-    # Adds back quotes(``) if hyphen(-) in schema name
-    return f"`{schema}`" if "-" in schema else schema
-
-
 # This method is from hive dialect originally but
 # is overridden to optimize DESCRIBE query execution
 def _get_table_columns(self, connection, table_name, schema, db_name):
@@ -158,7 +153,6 @@ def _get_table_columns(self, connection, table_name, schema, db_name):
 
 def _get_column_rows(self, connection, table_name, schema, db_name):
     # get columns and strip whitespace
-    schema = format_schema_name(schema=schema)
     table_columns = _get_table_columns(  # pylint: disable=protected-access
         self, connection, table_name, schema, db_name
     )
@@ -212,11 +206,10 @@ def get_columns(self, connection, table_name, schema=None, **kw):
             "system_data_type": raw_col_type,
         }
         if col_type in {"array", "struct", "map"}:
-            col_name = f"`{col_name}`" if "." in col_name else col_name
             try:
                 rows = dict(
                     connection.execute(
-                        f"DESCRIBE TABLE {kw.get('db_name')}.{schema}.{table_name} {col_name}"
+                        f"DESCRIBE TABLE `{kw.get('db_name')}`.`{schema}`.`{table_name}` `{col_name}`"
                     ).fetchall()
                 )
                 col_info["system_data_type"] = rows["data_type"]
@@ -393,8 +386,7 @@ def get_table_type(self, connection, database, schema, table):
                 database_name=database, schema_name=schema, table_name=table
             )
         else:
-            schema = format_schema_name(schema=schema)
-            query = f"DESCRIBE TABLE EXTENDED {schema}.{table}"
+            query = f"DESCRIBE TABLE EXTENDED `{schema}`.`{table}`"
         rows = get_table_comment_result(
             self,
             connection=connection,
@@ -761,7 +753,6 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
     ) -> str:
         description = None
         try:
-            schema_name = format_schema_name(schema=schema_name)
             query = DATABRICKS_GET_TABLE_COMMENTS.format(
                 database_name=self.context.get().database,
                 schema_name=schema_name,
@@ -816,9 +807,7 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
         try:
             query = DATABRICKS_GET_TABLE_COMMENTS.format(
                 database_name=self.context.get().database,
-                schema_name=format_schema_name(
-                    schema=self.context.get().database_schema
-                ),
+                schema_name=self.context.get().database_schema,
                 table_name=table_name,
             )
             result = self.inspector.dialect.get_table_comment_result(

metadata/ingestion/source/database/databricks/queries.py

@@ -25,27 +25,27 @@ DATABRICKS_VIEW_DEFINITIONS = textwrap.dedent(
 )
 
 DATABRICKS_GET_TABLE_COMMENTS = (
-    "DESCRIBE TABLE EXTENDED {database_name}.{schema_name}.{table_name}"
+    "DESCRIBE TABLE EXTENDED `{database_name}`.`{schema_name}`.`{table_name}`"
 )
 
 DATABRICKS_GET_CATALOGS = "SHOW CATALOGS"
 
 DATABRICKS_GET_CATALOGS_TAGS = textwrap.dedent(
-    """SELECT * FROM {database_name}.information_schema.catalog_tags;"""
+    """SELECT * FROM `{database_name}`.information_schema.catalog_tags;"""
 )
 
 DATABRICKS_GET_SCHEMA_TAGS = textwrap.dedent(
     """
     SELECT 
         * 
-    FROM {database_name}.information_schema.schema_tags"""
+    FROM `{database_name}`.information_schema.schema_tags"""
 )
 
 DATABRICKS_GET_TABLE_TAGS = textwrap.dedent(
     """
     SELECT 
         * 
-    FROM {database_name}.information_schema.table_tags 
+    FROM `{database_name}`.information_schema.table_tags 
     """
 )
 
@@ -53,8 +53,8 @@ DATABRICKS_GET_COLUMN_TAGS = textwrap.dedent(
     """
     SELECT 
         * 
-    FROM {database_name}.information_schema.column_tags 
+    FROM `{database_name}`.information_schema.column_tags 
     """
 )
 
-DATABRICKS_DDL = "SHOW CREATE TABLE {table_name}"
+DATABRICKS_DDL = "SHOW CREATE TABLE `{table_name}`"

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -15,7 +15,7 @@ DBT service Topology.
 from abc import ABC, abstractmethod
 from typing import Iterable, List
 
-from dbt_artifacts_parser.parser import (
+from collate_dbt_artifacts_parser.parser import (
     parse_catalog,
     parse_manifest,
     parse_run_results,

metadata/ingestion/source/database/snowflake/queries.py

@@ -299,6 +299,7 @@ SELECT
 FROM SNOWFLAKE.ACCOUNT_USAGE.PROCEDURES
 WHERE PROCEDURE_CATALOG = '{database_name}'
   AND PROCEDURE_SCHEMA = '{schema_name}'
+  AND DELETED IS NULL
     """
 )
 
@@ -315,6 +316,7 @@ SELECT
 FROM SNOWFLAKE.ACCOUNT_USAGE.FUNCTIONS
 WHERE FUNCTION_CATALOG = '{database_name}'
   AND FUNCTION_SCHEMA = '{schema_name}'
+  AND DELETED IS NULL
     """
 )
 

metadata/profiler/metrics/window/first_quartile.py

@@ -56,14 +56,14 @@ class FirstQuartile(StaticMetric, PercentilMixin):
             # col fullname is only needed for MySQL and SQLite
             return self._compute_sqa_fn(
                 column(self.col.name, self.col.type),
-                self.col.table.fullname if self.col.table is not None else None,
+                self.col.table.name if self.col.table is not None else None,
                 0.25,
             )
 
         if is_concatenable(self.col.type):
             return self._compute_sqa_fn(
                 LenFn(column(self.col.name, self.col.type)),
-                self.col.table.fullname if self.col.table is not None else None,
+                self.col.table.name if self.col.table is not None else None,
                 0.25,
             )
 

metadata/profiler/metrics/window/median.py

@@ -56,14 +56,14 @@ class Median(StaticMetric, PercentilMixin):
             # col fullname is only needed for MySQL and SQLite
             return self._compute_sqa_fn(
                 column(self.col.name, self.col.type),
-                self.col.table.fullname if self.col.table is not None else None,
+                self.col.table.name if self.col.table is not None else None,
                 0.5,
             )
 
         if is_concatenable(self.col.type):
             return self._compute_sqa_fn(
                 LenFn(column(self.col.name, self.col.type)),
-                self.col.table.fullname if self.col.table is not None else None,
+                self.col.table.name if self.col.table is not None else None,
                 0.5,
             )
 

metadata/profiler/metrics/window/third_quartile.py

@@ -56,14 +56,14 @@ class ThirdQuartile(StaticMetric, PercentilMixin):
             # col fullname is only needed for MySQL and SQLite
             return self._compute_sqa_fn(
                 column(self.col.name, self.col.type),
-                self.col.table.fullname if self.col.table is not None else None,
+                self.col.table.name if self.col.table is not None else None,
                 0.75,
             )
 
         if is_concatenable(self.col.type):
             return self._compute_sqa_fn(
                 LenFn(column(self.col.name, self.col.type)),
-                self.col.table.fullname if self.col.table is not None else None,
+                self.col.table.name if self.col.table is not None else None,
                 0.75,
             )
 

metadata/profiler/processor/core.py

@@ -550,12 +550,12 @@ class Profiler(Generic[TMetric]):
                 createDateTime=raw_create_date,
                 sizeInByte=self._table_results.get("sizeInBytes"),
                 profileSample=(
-                    self.profiler_interface.sampler.sample_config.profile_sample
+                    self.profiler_interface.sampler.sample_config.profileSample
                     if self.profiler_interface.sampler.sample_config
                     else None
                 ),
                 profileSampleType=(
-                    self.profiler_interface.sampler.sample_config.profile_sample_type
+                    self.profiler_interface.sampler.sample_config.profileSampleType
                     if self.profiler_interface.sampler.sample_config
                     else None
                 ),

metadata/profiler/source/database/base/profiler_source.py

@@ -150,9 +150,9 @@ class ProfilerSource(ProfilerSourceInterface):
             database_entity=database_entity,
             table_config=config,
             default_sample_config=SampleConfig(
-                profile_sample=self.source_config.profileSample,
-                profile_sample_type=self.source_config.profileSampleType,
-                sampling_method_type=self.source_config.samplingMethodType,
+                profileSample=self.source_config.profileSample,
+                profileSampleType=self.source_config.profileSampleType,
+                samplingMethodType=self.source_config.samplingMethodType,
             ),
         )
 

metadata/sampler/config.py

@@ -119,9 +119,9 @@ def get_profile_sample_config(
         try:
             if config and config.profileSample:
                 return SampleConfig(
-                    profile_sample=config.profileSample,
-                    profile_sample_type=config.profileSampleType,
-                    sampling_method_type=config.samplingMethodType,
+                    profileSample=config.profileSample,
+                    profileSampleType=config.profileSampleType,
+                    samplingMethodType=config.samplingMethodType,
                 )
         except AttributeError:
             pass

metadata/sampler/models.py

@@ -101,6 +101,6 @@ class SamplerResponse(ConfigModel):
 class SampleConfig(ConfigModel):
     """Profile Sample Config"""
 
-    profile_sample: Optional[Union[float, int]] = None
-    profile_sample_type: Optional[ProfileSampleType] = ProfileSampleType.PERCENTAGE
-    sampling_method_type: Optional[SamplingMethodType] = None
+    profileSample: Optional[Union[float, int]] = None
+    profileSampleType: Optional[ProfileSampleType] = ProfileSampleType.PERCENTAGE
+    samplingMethodType: Optional[SamplingMethodType] = None

metadata/sampler/nosql/sampler.py

@@ -82,10 +82,10 @@ class NoSQLSampler(SamplerInterface):
 
     def _get_limit(self) -> Optional[int]:
         num_rows = self.client.item_count(self.raw_dataset)
-        if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
-            limit = num_rows * (self.sample_config.profile_sample / 100)
-        elif self.sample_config.profile_sample_type == ProfileSampleType.ROWS:
-            limit = self.sample_config.profile_sample
+        if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
+            limit = num_rows * (self.sample_config.profileSample or 100 / 100)
+        elif self.sample_config.profileSampleType == ProfileSampleType.ROWS:
+            limit = self.sample_config.profileSample
         else:
             limit = SAMPLE_DATA_DEFAULT_COUNT
         return limit

metadata/sampler/pandas/sampler.py

@@ -123,13 +123,13 @@ class DatalakeSampler(SamplerInterface, PandasInterfaceMixin):
         """
         random.shuffle(self.raw_dataset)  # we'll shuffle the list of dataframes
         # sampling data based on profiler config (if any)
-        if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
+        if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
             try:
-                profile_sample = (self.sample_config.profile_sample or 100) / 100
+                profile_sample = (self.sample_config.profileSample or 100) / 100
             except TypeError:
                 # if the profile sample is not a number or is None
                 # we'll set it to 100
-                profile_sample = self.sample_config.profile_sample = 100
+                profile_sample = self.sample_config.profileSample = 100
             return [
                 df.sample(
                     frac=profile_sample,
@@ -141,7 +141,7 @@ class DatalakeSampler(SamplerInterface, PandasInterfaceMixin):
 
         # we'll distribute the sample size equally among the dataframes
         sample_rows_per_chunk: int = math.floor(
-            (self.sample_config.profile_sample or 100) / len(self.raw_dataset)
+            (self.sample_config.profileSample or 100) / len(self.raw_dataset)
         )
         num_rows = sum(len(df) for df in self.raw_dataset)
 
@@ -187,9 +187,9 @@ class DatalakeSampler(SamplerInterface, PandasInterfaceMixin):
         if self.partition_details:
             self._table = self._partitioned_table()
 
-        if not self.sample_config.profile_sample or (
-            self.sample_config.profile_sample == 100
-            and self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE
+        if not self.sample_config.profileSample or (
+            self.sample_config.profileSample == 100
+            and self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE
         ):
             return self.raw_dataset
         return self._get_sampled_dataframe()

metadata/sampler/sampler_interface.py

@@ -76,8 +76,8 @@ class SamplerInterface(ABC):
         self._columns: Optional[List[SQALikeColumn]] = None
         self.sample_config = sample_config
 
-        if not self.sample_config.profile_sample:
-            self.sample_config.profile_sample = 100
+        if not self.sample_config.profileSample:
+            self.sample_config.profileSample = 100
 
         self.entity = entity
         self.include_columns = include_columns

metadata/sampler/sqlalchemy/bigquery/sampler.py

@@ -76,11 +76,11 @@ class BigQuerySampler(SQASampler):
             selectable (Table): Table object
         """
         if (
-            self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE
+            self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE
             and self.raw_dataset_type != TableType.View
         ):
             return selectable.tablesample(
-                text(f"{self.sample_config.profile_sample or 100} PERCENT")
+                text(f"{self.sample_config.profileSample or 100} PERCENT")
             )
 
         return selectable
@@ -116,7 +116,7 @@ class BigQuerySampler(SQASampler):
         """get query for sample data"""
         # TABLESAMPLE SYSTEM is not supported for views
         if (
-            self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE
+            self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE
             and self.raw_dataset_type != TableType.View
         ):
             return self._base_sample_query(column).cte(

metadata/sampler/sqlalchemy/mssql/sampler.py

@@ -17,7 +17,7 @@ for the profiler
 from sqlalchemy import Table, text
 from sqlalchemy.sql.selectable import CTE
 
-from metadata.generated.schema.entity.data.table import ProfileSampleType
+from metadata.generated.schema.entity.data.table import ProfileSampleType, TableType
 from metadata.sampler.sqlalchemy.sampler import SQASampler
 
 
@@ -32,14 +32,16 @@ class MssqlSampler(SQASampler):
         Args:
             selectable (Table): _description_
         """
-        if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
+        if self.entity.tableType != TableType.View:
+            if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
+                return selectable.tablesample(
+                    text(f"{self.sample_config.profileSample or 100} PERCENT")
+                )
+
             return selectable.tablesample(
-                text(f"{self.sample_config.profile_sample or 100} PERCENT")
+                text(f"{int(self.sample_config.profileSample or 100)} ROWS")
             )
-
-        return selectable.tablesample(
-            text(f"{int(self.sample_config.profile_sample or 100)} ROWS")
-        )
+        return selectable
 
     def get_sample_query(self, *, column=None) -> CTE:
         """get query for sample data"""

metadata/sampler/sqlalchemy/postgres/sampler.py

@@ -66,7 +66,7 @@ class PostgresSampler(SQASampler):
         self.sampling_method_type = SamplingMethodType.BERNOULLI
         if (
             sample_config
-            and sample_config.sampling_method_type == SamplingMethodType.SYSTEM
+            and sample_config.samplingMethodType == SamplingMethodType.SYSTEM
         ):
             self.sampling_fn = func.system
 
@@ -75,15 +75,15 @@ class PostgresSampler(SQASampler):
         Args:
             selectable (Table): _description_
         """
-        if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
+        if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
             return selectable.tablesample(
-                self.sampling_fn(self.sample_config.profile_sample or 100)
+                self.sampling_fn(self.sample_config.profileSample or 100)
             )
 
         return selectable
 
     def get_sample_query(self, *, column=None) -> Query:
-        if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
+        if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
             return self._base_sample_query(column).cte(
                 f"{self.raw_dataset.__tablename__}_rnd"
             )

metadata/sampler/sqlalchemy/sampler.py

@@ -111,14 +111,14 @@ class SQASampler(SamplerInterface, SQAInterfaceMixin):
 
     def get_sample_query(self, *, column=None) -> Query:
         """get query for sample data"""
-        if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
+        if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
             rnd = self._base_sample_query(
                 column,
                 (ModuloFn(RandomNumFn(), 100)).label(RANDOM_LABEL),
             ).cte(f"{self.raw_dataset.__tablename__}_rnd")
             session_query = self.client.query(rnd)
             return session_query.where(
-                rnd.c.random <= self.sample_config.profile_sample
+                rnd.c.random <= self.sample_config.profileSample
             ).cte(f"{self.raw_dataset.__tablename__}_sample")
 
         table_query = self.client.query(self.raw_dataset)
@@ -128,7 +128,7 @@ class SQASampler(SamplerInterface, SQAInterfaceMixin):
         )
         return (
             session_query.order_by(RANDOM_LABEL)
-            .limit(self.sample_config.profile_sample)
+            .limit(self.sample_config.profileSample)
             .cte(f"{self.raw_dataset.__tablename__}_rnd")
         )
 
@@ -140,10 +140,7 @@ class SQASampler(SamplerInterface, SQAInterfaceMixin):
         if self.sample_query:
             return self._rdn_sample_from_user_query()
 
-        if not self.sample_config.profile_sample or (
-            int(self.sample_config.profile_sample) == 100
-            and self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE
-        ):
+        if not self.sample_config.profileSample:
             if self.partition_details:
                 partitioned = self._partitioned_table()
                 return partitioned.cte(f"{self.raw_dataset.__tablename__}_partitioned")

metadata/sampler/sqlalchemy/snowflake/sampler.py

@@ -68,7 +68,7 @@ class SnowflakeSampler(SQASampler):
         self.sampling_method_type = func.bernoulli
         if (
             sample_config
-            and sample_config.sampling_method_type == SamplingMethodType.SYSTEM
+            and sample_config.samplingMethodType == SamplingMethodType.SYSTEM
         ):
             self.sampling_method_type = func.system
 
@@ -77,13 +77,13 @@ class SnowflakeSampler(SQASampler):
         Args:
             selectable (Table): _description_
         """
-        if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
+        if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE:
             return selectable.tablesample(
-                self.sampling_method_type(self.sample_config.profile_sample or 100)
+                self.sampling_method_type(self.sample_config.profileSample or 100)
             )
 
         return selectable.tablesample(
-            func.ROW(text(f"{self.sample_config.profile_sample or 100} ROWS"))
+            func.ROW(text(f"{self.sample_config.profileSample or 100} ROWS"))
         )
 
     def get_sample_query(self, *, column=None) -> CTE:

metadata/sampler/sqlalchemy/trino/sampler.py

@@ -41,7 +41,7 @@ class TrinoSampler(SQASampler):
         return self.client.query(entity, label).where(
             or_(
                 *[
-                    text(f"is_nan({cols.name}) = False")
+                    text(f'is_nan("{cols.name}") = False')
                     for cols in sqa_columns
                     if type(cols.type) in FLOAT_SET
                 ]