openmetadata-ingestion 1.6.7.3__py3-none-any.whl → 1.6.8.0__py3-none-any.whl

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-04-04T11:34:24+00:00
+#   timestamp: 2025-04-08T13:18:29+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/api/rest/connection.py

@@ -30,6 +30,8 @@ from metadata.ingestion.connections.test_connections import test_connection_step
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.constants import THREE_MIN
 
+ACCEPTED_CONTENT_TYPES = ["application/json", "application/vnd.oai.openapi+json"]
+
 
 class SchemaURLError(Exception):
     """
@@ -66,9 +68,9 @@ def test_connection(
     """
 
     def custom_url_exec():
-        if (
-            "application/json" in client.headers.get("content-type")
-            and client.status_code == 200
+        if client.status_code == 200 and any(
+            content_type in client.headers.get("content-type")
+            for content_type in ACCEPTED_CONTENT_TYPES
         ):
             return []
         raise SchemaURLError(

metadata/ingestion/source/api/rest/metadata.py

@@ -41,6 +41,8 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.api.api_service import ApiServiceSource
 from metadata.ingestion.source.api.rest.models import RESTCollection, RESTEndpoint
 from metadata.utils import fqn
+from metadata.utils.filters import filter_by_collection
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -76,21 +78,33 @@ class RestSource(ApiServiceSource):
         """
         try:
             self.json_response = self.connection.json()
+            collections_list = []
+            tags_collection_set = set()
             if self.json_response.get("tags", []):
                 # Works only if list of tags are present in schema so we can fetch collection names
                 for collection in self.json_response.get("tags", []):
                     if not collection.get("name"):
                         continue
-                    yield RESTCollection(**collection)
-            else:
-                # in other case collect tags from paths because we have to yield collection/tags first
-                collections_set = set()
-                for path, methods in self.json_response.get("paths", {}).items():
-                    for method_type, info in methods.items():
-                        collections_set.update({tag for tag in info.get("tags", [])})
-                for collection_name in collections_set:
-                    data = {"name": collection_name}
-                    yield RESTCollection(**data)
+                    collections_list.append(collection)
+                    tags_collection_set.update({collection.get("name")})
+            # iterate through paths if there's any missing collection not present in tags
+            collections_set = set()
+            for path, methods in self.json_response.get("paths", {}).items():
+                for method_type, info in methods.items():
+                    collections_set.update({tag for tag in info.get("tags", [])})
+            for collection_name in collections_set:
+                if collection_name not in tags_collection_set:
+                    collections_list.append({"name": collection_name})
+            for collection in collections_list:
+                if filter_by_collection(
+                    self.source_config.apiCollectionFilterPattern,
+                    collection.get("name"),
+                ):
+                    self.status.filter(
+                        collection.get("name"), "Collection filtered out"
+                    )
+                    continue
+                yield RESTCollection(**collection)
         except Exception as err:
             logger.error(f"Error while fetching collections from schema URL :{err}")
 
@@ -126,7 +140,9 @@ class RestSource(ApiServiceSource):
         for path, methods in filtered_endpoints.items():
             for method_type, info in methods.items():
                 try:
-                    endpoint = self._prepare_endpoint_data(path, method_type, info)
+                    endpoint = self._prepare_endpoint_data(
+                        path, method_type, info, collection
+                    )
                     if not endpoint:
                         continue
                     yield Either(
@@ -176,12 +192,14 @@ class RestSource(ApiServiceSource):
             )
             return None
 
-    def _prepare_endpoint_data(self, path, method_type, info) -> Optional[RESTEndpoint]:
+    def _prepare_endpoint_data(
+        self, path, method_type, info, collection
+    ) -> Optional[RESTEndpoint]:
         try:
             endpoint = RESTEndpoint(**info)
-            endpoint.url = self._generate_endpoint_url(endpoint.name)
-            if not endpoint.name:
-                endpoint.name = f"{path} - {method_type}"
+            endpoint.name = f"{path}/{method_type}"
+            endpoint.display_name = f"{path}"
+            endpoint.url = self._generate_endpoint_url(collection, endpoint)
             return endpoint
         except Exception as err:
             logger.warning(f"Error while parsing endpoint data: {err}")
@@ -190,20 +208,39 @@ class RestSource(ApiServiceSource):
     def _generate_collection_url(self, collection_name: str) -> Optional[AnyUrl]:
         """generate collection url"""
         try:
-            if collection_name:
-                return AnyUrl(
-                    f"{self.config.serviceConnection.root.config.openAPISchemaURL}#tag/{collection_name}"
+            base_url = self.config.serviceConnection.root.config.docURL
+            if not base_url:
+                logger.debug(
+                    f"Could not generate collection url for {collection_name}"
+                    " because docURL is not present"
                 )
+                return self.config.serviceConnection.root.config.openAPISchemaURL
+            base_url = str(base_url)
+            if base_url.endswith("#/") or base_url.endswith("#"):
+                base_url = base_url.split("#")[0]
+            return AnyUrl(f"{clean_uri(base_url)}/#/{collection_name}")
         except Exception as err:
-            logger.warning(f"Error while generating collection url: {err}")
-        return None
+            logger.warning(
+                f"Error while generating collection url for {collection_name}: {err}"
+            )
+        return self.config.serviceConnection.root.config.openAPISchemaURL
 
-    def _generate_endpoint_url(self, endpoint_name: str) -> AnyUrl:
+    def _generate_endpoint_url(
+        self, collection: RESTCollection, endpoint: RESTEndpoint
+    ) -> AnyUrl:
         """generate endpoint url"""
-        base_url = self.config.serviceConnection.root.config.openAPISchemaURL
-        if endpoint_name:
-            return AnyUrl(f"{base_url}#operation/{endpoint_name}")
-        return AnyUrl(base_url)
+        try:
+            if not collection.url or not endpoint.operationId:
+                logger.debug(
+                    f"Could not generate endpoint url for {str(endpoint.name)},"
+                    f" collection url: {str(collection.url)},"
+                    f" endpoint operation id: {str(endpoint.operationId)}"
+                )
+                return self.config.serviceConnection.root.config.openAPISchemaURL
+            return AnyUrl(f"{str(collection.url)}/{endpoint.operationId}")
+        except Exception as err:
+            logger.warning(f"Error while generating collection url: {err}")
+        return self.config.serviceConnection.root.config.openAPISchemaURL
 
     def _get_api_request_method(self, method_type: str) -> Optional[str]:
         """fetch endpoint request method"""

metadata/ingestion/source/api/rest/models.py

@@ -13,7 +13,7 @@ OpenAPI REST API Models
 """
 from typing import Optional
 
-from pydantic import AnyUrl, BaseModel, Field
+from pydantic import AnyUrl, BaseModel
 
 from metadata.generated.schema.entity.data.apiEndpoint import ApiRequestMethod
 from metadata.generated.schema.type import basic
@@ -32,10 +32,11 @@ class RESTCollection(BaseModel):
 class RESTEndpoint(BaseModel):
     """REST endpoint model"""
 
-    name: Optional[str] = Field(None, alias="operationId")
+    name: Optional[str] = None
     display_name: Optional[str] = None
     description: Optional[basic.Markdown] = None
     url: Optional[AnyUrl] = None
+    operationId: Optional[str] = None
     request_method: Optional[ApiRequestMethod] = None
     request_schema: Optional[APISchema] = None
     response_schema: Optional[APISchema] = None

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -859,6 +859,7 @@ class TableauSource(DashboardServiceSource):
                         [
                             TableAndQuery(table=table, query=custom_sql_table.query)
                             for table in from_entities
+                            if table is not None
                         ]
                     )
 

metadata/ingestion/source/dashboard/tableau/models.py

@@ -144,6 +144,14 @@ class UpstreamTable(BaseModel):
     database: Optional[TableauDatabase] = None
     referencedByQueries: Optional[List[CustomSQLTable]] = None
 
+    @validator("referencedByQueries", pre=True)
+    @classmethod
+    def filter_none_queries(cls, v):
+        """Filter out CustomSQLTable items where query==None."""
+        if v is None:
+            return None
+        return [item for item in v if item.get("query") is not None]
+
 
 class DataSource(BaseModel):
     id: str

metadata/ingestion/source/pipeline/dbtcloud/client.py

@@ -75,17 +75,37 @@ class DBTCloudClient:
         fetch jobs for an account in dbt cloud
         """
         job_list = []
+        # we will get 100 jobs at a time
+        query_params = {"offset": 0, "limit": 100}
         try:
             job_path = f"{job_id}/" if job_id else ""
             project_path = f"?project_id={project_id}" if project_id else ""
             result = self.client.get(
-                f"/accounts/{self.config.accountId}/jobs/{job_path}{project_path}"
-            )
-            job_list = (
-                [DBTJob.model_validate(result["data"])]
-                if job_id
-                else DBTJobList.model_validate(result).Jobs
+                f"/accounts/{self.config.accountId}/jobs/{job_path}{project_path}",
+                data=query_params,
             )
+
+            if job_id:
+                job_list = [DBTJob.model_validate(result["data"])]
+            else:
+                job_list_response = DBTJobList.model_validate(result)
+                job_list = job_list_response.Jobs
+
+                while job_list_response.extra and job_list_response.extra.pagination:
+                    total_count = job_list_response.extra.pagination.total_count
+                    current_count = job_list_response.extra.pagination.count
+
+                    if current_count >= total_count:
+                        break
+
+                    query_params["offset"] += query_params["limit"]
+                    result = self.client.get(
+                        f"/accounts/{self.config.accountId}/jobs/{job_path}{project_path}",
+                        data=query_params,
+                    )
+                    job_list_response = DBTJobList.model_validate(result)
+                    job_list.extend(job_list_response.Jobs)
+
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(
@@ -147,13 +167,42 @@ class DBTCloudClient:
         list runs for a job in dbt cloud
         """
         try:
+            number_of_runs = self.config.numberOfRuns
+            runs = []
+            # we will get 100 runs at a time and order by created_at in descending order
+            query_params = {
+                "job_definition_id": job_id,
+                "offset": 0,
+                "limit": min(100, number_of_runs) if number_of_runs else 100,
+                "order_by": "-created_at",
+            }
+
             result = self.client.get(
-                f"/accounts/{self.config.accountId}/runs/",
-                data={"job_definition_id": job_id},
+                f"/accounts/{self.config.accountId}/runs/", data=query_params
             )
-            if result:
-                run_list = DBTRunList.model_validate(result).Runs
-                return run_list
+            run_list_response = DBTRunList.model_validate(result)
+            runs.extend(run_list_response.Runs)
+
+            while (
+                (number_of_runs is None or len(runs) < number_of_runs)
+                and run_list_response.extra
+                and run_list_response.extra.pagination
+            ):
+                total_count = run_list_response.extra.pagination.total_count
+                current_count = run_list_response.extra.pagination.count
+
+                if current_count >= total_count:
+                    break
+
+                query_params["offset"] += query_params["limit"]
+                result = self.client.get(
+                    f"/accounts/{self.config.accountId}/runs/",
+                    data=query_params,
+                )
+                run_list_response = DBTRunList.model_validate(result)
+                runs.extend(run_list_response.Runs)
+
+            return runs[:number_of_runs] if number_of_runs is not None else runs
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Unable to get run info :{exc}")

metadata/ingestion/source/pipeline/dbtcloud/metadata.py

@@ -105,7 +105,7 @@ class DbtcloudSource(PipelineServiceSource):
                     )
                     task_list.append(task)
                 self.context.get().latest_run_id = (
-                    task_list[-1].name if task_list else None
+                    task_list[0].name if task_list else None
                 )
             return task_list or None
         except Exception as exc:

metadata/ingestion/source/pipeline/dbtcloud/models.py

@@ -34,8 +34,18 @@ class DBTJob(BaseModel):
     project_id: int
 
 
+class Pagination(BaseModel):
+    count: int
+    total_count: int
+
+
+class Extra(BaseModel):
+    pagination: Optional[Pagination] = None
+
+
 class DBTJobList(BaseModel):
     Jobs: List[DBTJob] = Field(alias="data")
+    extra: Optional[Extra] = None
 
 
 class DBTRun(BaseModel):
@@ -51,12 +61,14 @@ class DBTRun(BaseModel):
 
 class DBTRunList(BaseModel):
     Runs: Optional[List[DBTRun]] = Field([], alias="data")
+    extra: Optional[Extra] = None
 
 
 class DBTSources(BaseModel):
     name: Optional[str] = None
     dbtschema: Optional[str] = Field(None, alias="schema")
     database: Optional[str] = None
+    extra: Optional[Extra] = None
 
 
 class DBTModel(BaseModel):
@@ -70,3 +82,4 @@ class DBTModel(BaseModel):
 class DBTModelList(BaseModel):
     models: Optional[List[DBTModel]] = []
     seeds: Optional[List[DBTModel]] = []
+    extra: Optional[Extra] = None

metadata/profiler/orm/functions/median.py

@@ -50,12 +50,6 @@ def _(elements, compiler, **kwargs):
     ]
     return "percentile_cont(%s , %s) OVER()" % (col, percentile)
 
-@compiles(MedianFn, Dialects.Databricks)
-def _(elements, compiler, **kwargs):
-    col, _, percentile = [
-        compiler.process(element, **kwargs) for element in elements.clauses
-    ]
-    return "percentile_approx(%s , %s)" % (col, percentile)
 
 @compiles(MedianFn, Dialects.ClickHouse)
 def _(elements, compiler, **kwargs):

metadata/utils/filters.py

@@ -295,3 +295,18 @@ def filter_by_classification(
     :return: True for filtering, False otherwise
     """
     return _filter(classification_pattern, classification_name)
+
+
+def filter_by_collection(
+    collection_pattern: Optional[FilterPattern], collection_name: str
+) -> bool:
+    """
+    Return True if the models needs to be filtered, False otherwise
+
+    Include takes precedence over exclude
+
+    :param collection_pattern: Model defining collection filtering logic
+    :param collection_name: collection name
+    :return: True for filtering, False otherwise
+    """
+    return _filter(collection_pattern, collection_name)