openmetadata-ingestion 1.5.0.0rc1__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

metadata/ingestion/source/pipeline/dbtcloud/metadata.py

@@ -111,6 +111,7 @@ class DbtcloudSource(PipelineServiceSource):
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Failed to get tasks list due to : {exc}")
+        return None
 
     def yield_pipeline(
         self, pipeline_details: DBTJob
@@ -119,16 +120,22 @@ class DbtcloudSource(PipelineServiceSource):
         Method to Get Pipeline Entity
         """
         try:
-            connection_url = f"{clean_uri(self.service_connection.host)}/deploy/{self.service_connection.accountId}/projects/{pipeline_details.project_id}/jobs/{pipeline_details.id}"
+            connection_url = (
+                f"{clean_uri(self.service_connection.host)}/deploy/"
+                f"{self.service_connection.accountId}/projects/"
+                f"{pipeline_details.project_id}/jobs/{pipeline_details.id}"
+            )
 
             pipeline_request = CreatePipelineRequest(
                 name=EntityName(pipeline_details.name),
                 description=Markdown(pipeline_details.description),
                 sourceUrl=SourceUrl(connection_url),
                 tasks=self._get_task_list(job_id=int(pipeline_details.id)),
-                scheduleInterval=str(pipeline_details.schedule.cron)
-                if pipeline_details.schedule
-                else None,
+                scheduleInterval=(
+                    str(pipeline_details.schedule.cron)
+                    if pipeline_details.schedule
+                    else None
+                ),
                 service=FullyQualifiedEntityName(self.context.get().pipeline_service),
             )
             yield Either(right=pipeline_request)
@@ -148,77 +155,88 @@ class DbtcloudSource(PipelineServiceSource):
         """
         Get lineage between pipeline and data sources
         """
-        try:
-            pipeline_fqn = fqn.build(
-                metadata=self.metadata,
-                entity_type=Pipeline,
-                service_name=self.context.get().pipeline_service,
-                pipeline_name=self.context.get().pipeline,
-            )
-
-            pipeline_entity = self.metadata.get_by_name(
-                entity=Pipeline, fqn=pipeline_fqn
-            )
+        try:  # pylint: disable=too-many-nested-blocks
+            if self.source_config.lineageInformation:
+                pipeline_fqn = fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Pipeline,
+                    service_name=self.context.get().pipeline_service,
+                    pipeline_name=self.context.get().pipeline,
+                )
 
-            lineage_details = LineageDetails(
-                pipeline=EntityReference(id=pipeline_entity.id.root, type="pipeline"),
-                source=LineageSource.PipelineLineage,
-            )
+                pipeline_entity = self.metadata.get_by_name(
+                    entity=Pipeline, fqn=pipeline_fqn
+                )
 
-            dbt_models = self.client.get_model_details(
-                job_id=pipeline_details.id, run_id=self.context.get().latest_run_id
-            )
+                lineage_details = LineageDetails(
+                    pipeline=EntityReference(
+                        id=pipeline_entity.id.root, type="pipeline"
+                    ),
+                    source=LineageSource.PipelineLineage,
+                )
 
-            for model in dbt_models or []:
-                for dbservicename in (
-                    self.source_config.lineageInformation.dbServiceNames or []
-                ):
-                    to_entity = self.metadata.get_by_name(
-                        entity=Table,
-                        fqn=fqn.build(
-                            metadata=self.metadata,
-                            entity_type=Table,
-                            table_name=model.alias,
-                            database_name=model.database,
-                            schema_name=model.dbtschema,
-                            service_name=dbservicename,
-                        ),
-                    )
+                dbt_models = self.client.get_model_details(
+                    job_id=pipeline_details.id, run_id=self.context.get().latest_run_id
+                )
 
-                    if to_entity is None:
-                        continue
+                dbt_parents = self.client.get_models_and_seeds_details(
+                    job_id=pipeline_details.id, run_id=self.context.get().latest_run_id
+                )
 
-                    for dest in model.parentsSources or []:
-                        from_entity = self.metadata.get_by_name(
+                for model in dbt_models or []:
+                    for dbservicename in (
+                        self.source_config.lineageInformation.dbServiceNames or []
+                    ):
+                        to_entity = self.metadata.get_by_name(
                             entity=Table,
                             fqn=fqn.build(
                                 metadata=self.metadata,
                                 entity_type=Table,
-                                table_name=dest.name,
-                                database_name=dest.database,
-                                schema_name=dest.dbtschema,
+                                table_name=model.name,
+                                database_name=model.database,
+                                schema_name=model.dbtschema,
                                 service_name=dbservicename,
                             ),
                         )
 
-                        if from_entity is None:
+                        if to_entity is None:
                             continue
 
-                        yield Either(
-                            right=AddLineageRequest(
-                                edge=EntitiesEdge(
-                                    fromEntity=EntityReference(
-                                        id=from_entity.id,
-                                        type="table",
-                                    ),
-                                    toEntity=EntityReference(
-                                        id=to_entity.id,
-                                        type="table",
+                        for unique_id in model.dependsOn or []:
+                            parents = [
+                                d for d in dbt_parents if d.uniqueId == unique_id
+                            ]
+                            if parents:
+                                from_entity = self.metadata.get_by_name(
+                                    entity=Table,
+                                    fqn=fqn.build(
+                                        metadata=self.metadata,
+                                        entity_type=Table,
+                                        table_name=parents[0].name,
+                                        database_name=parents[0].database,
+                                        schema_name=parents[0].dbtschema,
+                                        service_name=dbservicename,
                                     ),
-                                    lineageDetails=lineage_details,
                                 )
-                            )
-                        )
+
+                                if from_entity is None:
+                                    continue
+
+                                yield Either(
+                                    right=AddLineageRequest(
+                                        edge=EntitiesEdge(
+                                            fromEntity=EntityReference(
+                                                id=from_entity.id,
+                                                type="table",
+                                            ),
+                                            toEntity=EntityReference(
+                                                id=to_entity.id,
+                                                type="table",
+                                            ),
+                                            lineageDetails=lineage_details,
+                                        )
+                                    )
+                                )
 
         except Exception as exc:
             yield Either(
@@ -238,7 +256,6 @@ class DbtcloudSource(PipelineServiceSource):
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Failed to get pipeline list due to : {exc}")
-        return None
 
     def get_pipeline_name(self, pipeline_details: DBTJob) -> str:
         """
@@ -263,21 +280,27 @@ class DbtcloudSource(PipelineServiceSource):
                 TaskStatus(
                     name=str(task.id),
                     executionStatus=STATUS_MAP.get(task.state, StatusType.Pending),
-                    startTime=Timestamp(
-                        datetime_to_ts(
-                            datetime.strptime(task.started_at, "%Y-%m-%d %H:%M:%S.%f%z")
-                            if task.started_at
-                            else datetime.now()
+                    startTime=(
+                        Timestamp(
+                            datetime_to_ts(
+                                datetime.strptime(
+                                    task.started_at, "%Y-%m-%d %H:%M:%S.%f%z"
+                                )
+                            )
                         )
+                        if task.started_at
+                        else None
                     ),
-                    endTime=Timestamp(
-                        datetime_to_ts(
-                            datetime.strptime(
-                                task.finished_at, "%Y-%m-%d %H:%M:%S.%f%z"
+                    endTime=(
+                        Timestamp(
+                            datetime_to_ts(
+                                datetime.strptime(
+                                    task.finished_at, "%Y-%m-%d %H:%M:%S.%f%z"
+                                )
                             )
-                            if task.finished_at
-                            else datetime.now()
                         )
+                        if task.finished_at
+                        else None
                     ),
                 )
                 for task in self.client.get_runs(job_id=int(pipeline_details.id)) or []
@@ -294,7 +317,7 @@ class DbtcloudSource(PipelineServiceSource):
                             pipeline_details.created_at, "%Y-%m-%dT%H:%M:%S.%f%z"
                         )
                         if pipeline_details.created_at
-                        else datetime.now()
+                        else None
                     )
                 ),
             )

metadata/ingestion/source/pipeline/dbtcloud/models.py

@@ -35,7 +35,7 @@ class DBTJob(BaseModel):
 
 
 class DBTJobList(BaseModel):
-    Jobs: Optional[List[DBTJob]] = Field(None, alias="data")
+    Jobs: Optional[List[DBTJob]] = Field([], alias="data")
 
 
 class DBTRun(BaseModel):
@@ -50,7 +50,7 @@ class DBTRun(BaseModel):
 
 
 class DBTRunList(BaseModel):
-    Runs: Optional[List[DBTRun]] = Field(None, alias="data")
+    Runs: Optional[List[DBTRun]] = Field([], alias="data")
 
 
 class DBTSources(BaseModel):
@@ -60,13 +60,13 @@ class DBTSources(BaseModel):
 
 
 class DBTModel(BaseModel):
+    uniqueId: Optional[str] = None
     name: Optional[str] = None
-    alias: Optional[str] = None
     dbtschema: Optional[str] = Field(None, alias="schema")
     database: Optional[str] = None
-    rawSql: Optional[str] = None
-    parentsSources: Optional[List[DBTSources]] = None
+    dependsOn: Optional[List[str]] = None
 
 
 class DBTModelList(BaseModel):
-    models: Optional[List[DBTModel]] = None
+    models: Optional[List[DBTModel]] = []
+    seeds: Optional[List[DBTModel]] = []

metadata/ingestion/source/pipeline/dbtcloud/queries.py

@@ -12,22 +12,33 @@
 GraphQL Queries used during ingestion
 """
 
-DBT_QUERY = """
+DBT_GET_MODEL_DEPENDS_ON = """
 query Query($jobId: BigInt!, $runId: BigInt) {
   job(id: $jobId, runId: $runId) {
     models {
-      name #destinationTable
-      alias
+      name
       database
       schema
-      rawSql
-      materializedType
-      parentsSources {
-        database
-        name
-        schema
-        sourceName #sourceTable
-      }
+      dependsOn
+    }
+  }
+}
+"""
+
+DBT_GET_MODELS_SEEDS = """
+query Query($jobId: BigInt!, $runId: BigInt) {
+  job(id: $jobId, runId: $runId) {
+    models {
+      uniqueId
+      name
+      database
+      schema
+    }
+    seeds {
+      uniqueId
+      name
+      schema
+      database
     }
   }
 }

metadata/ingestion/source/pipeline/domopipeline/metadata.py

@@ -89,7 +89,7 @@ class DomopipelineSource(PipelineServiceSource):
         self, pipeline_details
     ) -> Iterable[Either[CreatePipelineRequest]]:
         try:
-            pipeline_name = pipeline_details["id"]
+            pipeline_name = str(pipeline_details["id"])
             source_url = self.get_source_url(pipeline_id=pipeline_name)
             task = Task(
                 name=pipeline_name,

metadata/ingestion/source/pipeline/fivetran/client.py

@@ -20,6 +20,7 @@ from metadata.generated.schema.entity.services.connections.pipeline.fivetranConn
     FivetranConnection,
 )
 from metadata.ingestion.ometa.client import REST, ClientConfig
+from metadata.utils.helpers import clean_uri
 
 
 class FivetranClient:
@@ -36,7 +37,7 @@ class FivetranClient:
         )
 
         client_config: ClientConfig = ClientConfig(
-            base_url=str(self.config.hostPort),
+            base_url=clean_uri(str(self.config.hostPort)),
             api_version="v1",
             auth_header="Authorization",
             auth_token=lambda: (api_token[2:-1], 0),

metadata/ingestion/source/storage/gcs/client.py

@@ -16,8 +16,6 @@ from google import auth
 from google.cloud.monitoring_v3 import MetricServiceClient
 from google.cloud.storage import Client
 
-NoProject = object()
-
 
 class MultiProjectClient:
     """Google Cloud Client does not support ad-hoc project switching. This class wraps the client and allows
@@ -37,19 +35,16 @@ class MultiProjectClient:
         project_ids: Optional[List[str]] = None,
         **client_kwargs,
     ):
+        self.default_project = None
         if project_ids:
             self.clients = {
                 project_id: client_class(project=project_id, **client_kwargs)
                 for project_id in project_ids
             }
         else:
-            self.clients = {NoProject: client_class(**client_kwargs)}
-
-    def project_ids(self):
-        if NoProject in self.clients:
             _, project_id = auth.default()
-            return [project_id]
-        return list(self.clients.keys())
+            self.default_project = project_id
+            self.clients = {project_id: client_class(**client_kwargs)}
 
     def __getattr__(self, client_method):
         """Return the underlying client method as a partial function so we can inject the project_id."""
@@ -57,7 +52,7 @@ class MultiProjectClient:
 
     def _call(self, method, project_id, *args, **kwargs):
         """Call the method on the client for the given project_id. The args and kwargs are passed through."""
-        client = self.clients.get(project_id, self.clients.get(NoProject))
+        client = self.clients.get(project_id, self.clients.get(self.default_project))
         if not client:
             raise ValueError(f"Project {project_id} not found")
         return getattr(client, method)(*args, **kwargs)

metadata/ingestion/source/storage/gcs/metadata.py

@@ -259,6 +259,10 @@ class GcsSource(StorageServiceSource):
             try:
                 bucket = client.get_bucket(bucket_name)
             except NotFound:
+                logger.warning(
+                    f"Bucket {bucket_name} not found in project {project_id}"
+                )
+                self.status.warning(f"{project_id}.{bucket_name}", "Bucket Not Found")
                 continue
             return GCSBucketResponse(
                 name=bucket.name,

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -16,6 +16,7 @@ supporting sqlalchemy abstraction layer
 """
 
 import concurrent.futures
+import math
 import threading
 import traceback
 from collections import defaultdict
@@ -450,6 +451,14 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     column=metric_func.column,
                     sample=sample,
                 )
+                if row:
+                    for k, v in row.items():
+                        # Replace NaN values with None
+                        if isinstance(v, float) and math.isnan(v):
+                            logger.warning(
+                                "NaN data detected and will be cast to null in OpenMetadata to maintain database parity"
+                            )
+                            row[k] = None
             except Exception as exc:
                 error = (
                     f"{metric_func.column if metric_func.column is not None else metric_func.table.__tablename__} "

metadata/profiler/metrics/composed/null_ratio.py

@@ -49,12 +49,9 @@ class NullRatio(ComposedMetric):
         Safely compute null ratio based on the profiler
         results of other Metrics
         """
-        import pandas as pd
 
-        res_count = res.get(Count.name())
-        res_null = res.get(NullCount.name())
-
-        if not pd.isnull(res_count) and not pd.isnull(res_null):
-            result = res_null / (res_null + res_count)
-            return None if pd.isnull(result) else result
-        return None
+        count = res.get(Count.name())
+        null_count = res.get(NullCount.name())
+        if count + null_count == 0:
+            return None
+        return null_count / (null_count + count)

metadata/profiler/metrics/hybrid/histogram.py

@@ -25,7 +25,11 @@ from metadata.profiler.metrics.static.count import Count
 from metadata.profiler.metrics.static.max import Max
 from metadata.profiler.metrics.static.min import Min
 from metadata.profiler.orm.functions.length import LenFn
-from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
+from metadata.profiler.orm.registry import (
+    is_concatenable,
+    is_quantifiable,
+    is_value_non_numeric,
+)
 from metadata.utils.helpers import format_large_string_numbers
 from metadata.utils.logger import profiler_logger
 
@@ -120,7 +124,6 @@ class Histogram(HybridMetric):
             # freedman-diaconis rule
             bin_width = self._get_bin_width(float(res_iqr), res_row_count)  # type: ignore
             num_bins = math.ceil((res_max - res_min) / bin_width)  # type: ignore
-
         # sturge's rule
         if res_iqr is None or num_bins > max_bin_count:
             num_bins = int(math.ceil(math.log2(res_row_count) + 1))
@@ -148,7 +151,10 @@ class Histogram(HybridMetric):
                 "We are missing the session attribute to compute the Histogram."
             )
 
-        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)):
+        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)) or (
+            is_value_non_numeric(res.get(Min.name()))
+            or is_value_non_numeric(res.get(Max.name()))
+        ):
             return None
 
         # get the metric need for the freedman-diaconis rule

metadata/profiler/metrics/static/null_count.py

@@ -16,6 +16,7 @@ Null Count Metric definition
 
 
 from sqlalchemy import case, column
+from sqlalchemy.sql.functions import coalesce
 
 from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
@@ -48,8 +49,9 @@ class NullCount(StaticMetric):
     @_label
     def fn(self):
         """sqlalchemy function"""
-        return SumFn(
-            case([(column(self.col.name, self.col.type).is_(None), 1)], else_=0)
+        return coalesce(
+            SumFn(case([(column(self.col.name, self.col.type).is_(None), 1)], else_=0)),
+            0,
         )
 
     def df_fn(self, dfs=None):

metadata/profiler/orm/functions/sum.py

@@ -45,7 +45,7 @@ def _(element, compiler, **kw):
 def _(element, compiler, **kw):
     """Cast to DECIMAL to address cannot cast nan to bigint"""
     proc = compiler.process(element.clauses, **kw)
-    return f"SUM(TRY_CAST({proc} AS DECIMAL))"
+    return f"COALESCE(SUM(CAST({proc} AS DECIMAL)),0)"
 
 
 @compiles(SumFn, Dialects.BigQuery)

metadata/profiler/orm/functions/table_metric_computer.py

@@ -194,7 +194,7 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
                     Column("object_name").label("table_name"),
                     Column("created"),
                 ],
-                self._build_table("all_objects", None),
+                self._build_table("DBA_OBJECTS", None),
                 [
                     func.lower(Column("owner")) == self.schema_name.lower(),
                     func.lower(Column("object_name")) == self.table_name.lower(),
@@ -209,7 +209,7 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
                     Column("table_name"),
                     Column("NUM_ROWS"),
                 ],
-                self._build_table("all_tables", None),
+                self._build_table("DBA_TABLES", None),
                 [
                     func.lower(Column("owner")) == self.schema_name.lower(),
                     func.lower(Column("table_name")) == self.table_name.lower(),

metadata/profiler/orm/registry.py

@@ -13,6 +13,8 @@
 Custom types' registry for easy access
 without having an import mess
 """
+import math
+
 import sqlalchemy
 from sqlalchemy import Date, DateTime, Integer, Numeric, Time
 from sqlalchemy.sql.sqltypes import Concatenable, Enum
@@ -171,3 +173,12 @@ def is_concatenable(_type) -> bool:
     if isinstance(_type, DataType):
         return _type.value in CONCATENABLE_SET
     return issubclass(_type.__class__, Concatenable)
+
+
+def is_value_non_numeric(value) -> bool:
+    try:
+        if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
+            return True
+        return False
+    except Exception:
+        return False

metadata/profiler/processor/default.py

@@ -77,9 +77,12 @@ class DefaultProfiler(Profiler):
         include_columns: Optional[List[ColumnProfilerConfig]] = None,
         exclude_columns: Optional[List[str]] = None,
         global_profiler_configuration: Optional[ProfilerConfiguration] = None,
+        db_service=None,
     ):
         _metrics = get_default_metrics(
-            table=profiler_interface.table, ometa_client=profiler_interface.ometa_client
+            table=profiler_interface.table,
+            ometa_client=profiler_interface.ometa_client,
+            db_service=db_service,
         )
 
         super().__init__(

metadata/profiler/processor/sampler/sampler_factory.py

@@ -15,6 +15,9 @@ Factory class for creating sampler objects
 
 from typing import Union
 
+from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import (
+    AzureSQLConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
@@ -36,6 +39,9 @@ from metadata.generated.schema.entity.services.connections.database.trinoConnect
 from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
 from metadata.profiler.processor.sampler.nosql.sampler import NoSQLSampler
 from metadata.profiler.processor.sampler.pandas.sampler import DatalakeSampler
+from metadata.profiler.processor.sampler.sqlalchemy.azuresql.sampler import (
+    AzureSQLSampler,
+)
 from metadata.profiler.processor.sampler.sqlalchemy.bigquery.sampler import (
     BigQuerySampler,
 )
@@ -89,3 +95,6 @@ sampler_factory_.register(
 sampler_factory_.register(
     source_type=DynamoDBConnection.__name__, sampler_class=NoSQLSampler
 )
+sampler_factory_.register(
+    source_type=AzureSQLConnection.__name__, sampler_class=AzureSQLSampler
+)

metadata/profiler/processor/sampler/sqlalchemy/azuresql/sampler.py

@@ -0,0 +1,40 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Helper module to handle data sampling
+for the profiler
+"""
+from typing import List, Optional
+
+from sqlalchemy import Column
+
+from metadata.generated.schema.entity.data.table import TableData
+from metadata.profiler.processor.sampler.sqlalchemy.sampler import SQASampler
+
+
+class AzureSQLSampler(SQASampler):
+    """
+    Generates a sample of the data to not
+    run the query in the whole table.
+    """
+
+    # These types are not supported by pyodbc - it throws
+    # an error when trying to fetch data from these columns
+    # pyodbc.ProgrammingError: ('ODBC SQL type -151 is not yet supported.  column-index=x  type=-151', 'HY106')
+    NOT_COMPUTE_PYODBC = {"SQASGeography", "UndeterminedType"}
+
+    def fetch_sample_data(self, columns: Optional[List[Column]] = None) -> TableData:
+        sqa_columns = []
+        if columns:
+            for col in columns:
+                if col.type.__class__.__name__ not in self.NOT_COMPUTE_PYODBC:
+                    sqa_columns.append(col)
+        return super().fetch_sample_data(sqa_columns or columns)

metadata/profiler/source/base/profiler_source.py

@@ -277,6 +277,7 @@ class ProfilerSource(ProfilerSourceInterface):
                 include_columns=self._get_include_columns(entity, table_config),
                 exclude_columns=self._get_exclude_columns(entity, table_config),
                 global_profiler_configuration=self.global_profiler_configuration,
+                db_service=db_service,
             )
 
         metrics = (

metadata/readers/file/api_reader.py

@@ -21,12 +21,15 @@ from metadata.generated.schema.security.credentials.bitbucketCredentials import
 from metadata.generated.schema.security.credentials.githubCredentials import (
     GitHubCredentials,
 )
+from metadata.generated.schema.security.credentials.gitlabCredentials import (
+    GitlabCredentials,
+)
 from metadata.readers.file.base import Reader
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
-ReadersCredentials = Union[GitHubCredentials, BitBucketCredentials]
+ReadersCredentials = Union[GitHubCredentials, BitBucketCredentials, GitlabCredentials]
 
 
 class ApiReader(Reader, ABC):