openmetadata-ingestion 1.5.2.0__py3-none-any.whl → 1.5.3.0__py3-none-any.whl

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHierarchy.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-09-02T10:08:12+00:00
+#   timestamp: 2024-09-09T14:10:10+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/es_mixin.py

@@ -19,11 +19,11 @@ import traceback
 from typing import Generic, Iterable, List, Optional, Set, Type, TypeVar
 
 from pydantic import BaseModel
-from requests.utils import quote
 
 from metadata.generated.schema.entity.data.container import Container
 from metadata.generated.schema.entity.data.query import Query
 from metadata.ingestion.ometa.client import REST, APIError
+from metadata.ingestion.ometa.utils import quote
 from metadata.utils.elasticsearch import ES_INDEX_MAP
 from metadata.utils.logger import ometa_logger
 

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -28,7 +28,7 @@ from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.parser import LINEAGE_PARSING_TIMEOUT
 from metadata.ingestion.models.patch_request import build_patch
 from metadata.ingestion.ometa.client import REST, APIError
-from metadata.ingestion.ometa.utils import get_entity_type
+from metadata.ingestion.ometa.utils import get_entity_type, quote
 from metadata.utils.logger import ometa_logger
 from metadata.utils.lru_cache import LRU_CACHE_SIZE, LRUCache
 
@@ -279,7 +279,7 @@ class OMetaLineageMixin(Generic[T]):
         """
         return self._get_lineage(
             entity=entity,
-            path=f"name/{fqn}",
+            path=f"name/{quote(fqn)}",
             up_depth=up_depth,
             down_depth=down_depth,
         )

metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -157,7 +157,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
 
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.error(f"Error trying to PATCH {get_log_name(source)}: {exc}")
+            logger.warning(f"Error trying to PATCH {get_log_name(source)}: {exc}")
 
         return None
 

metadata/ingestion/ometa/mixins/table_mixin.py

@@ -16,8 +16,7 @@ To be used by OpenMetadata class
 import traceback
 from typing import List, Optional, Type, TypeVar
 
-from pydantic import BaseModel
-from requests.utils import quote
+from pydantic import BaseModel, validate_call
 
 from metadata.generated.schema.api.data.createTableProfile import (
     CreateTableProfileRequest,
@@ -39,7 +38,7 @@ from metadata.generated.schema.type.basic import FullyQualifiedEntityName, Uuid
 from metadata.generated.schema.type.usageRequest import UsageRequest
 from metadata.ingestion.ometa.client import REST
 from metadata.ingestion.ometa.models import EntityList
-from metadata.ingestion.ometa.utils import model_str
+from metadata.ingestion.ometa.utils import model_str, quote
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
@@ -227,6 +226,7 @@ class OMetaTableMixin:
 
         return None
 
+    @validate_call
     def get_profile_data(
         self,
         fqn: str,
@@ -253,7 +253,6 @@ class OMetaTableMixin:
         Returns:
             EntityList: EntityList list object
         """
-
         url_after = f"&after={after}" if after else ""
         profile_type_url = profile_type.__name__[0].lower() + profile_type.__name__[1:]
 
@@ -290,7 +289,7 @@ class OMetaTableMixin:
         Returns:
             Optional[Table]: OM table object
         """
-        return self._get(Table, f"{quote(model_str(fqn), safe='')}/tableProfile/latest")
+        return self._get(Table, f"{quote(fqn)}/tableProfile/latest")
 
     def create_or_update_custom_metric(
         self, custom_metric: CreateCustomMetricRequest, table_id: str

metadata/ingestion/ometa/mixins/tests_mixin.py

@@ -17,7 +17,6 @@ To be used by OpenMetadata class
 import traceback
 from datetime import datetime
 from typing import List, Optional, Type, Union
-from urllib.parse import quote
 from uuid import UUID
 
 from metadata.generated.schema.api.tests.createLogicalTestCases import (
@@ -46,7 +45,7 @@ from metadata.generated.schema.tests.testDefinition import (
 from metadata.generated.schema.tests.testSuite import TestSuite
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.ometa.client import REST
-from metadata.ingestion.ometa.utils import model_str
+from metadata.ingestion.ometa.utils import model_str, quote
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
@@ -76,7 +75,7 @@ class OMetaTestsMixin:
             _type_: _description_
         """
         resp = self.client.put(
-            f"{self.get_suffix(TestCase)}/{quote(test_case_fqn,safe='')}/testCaseResult",
+            f"{self.get_suffix(TestCase)}/{quote(test_case_fqn)}/testCaseResult",
             test_results.model_dump_json(),
         )
 

metadata/ingestion/ometa/ometa_api.py

@@ -18,7 +18,6 @@ import traceback
 from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
 
 from pydantic import BaseModel
-from requests.utils import quote
 
 from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import (
     CreateIngestionPipelineRequest,
@@ -57,7 +56,7 @@ from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
 from metadata.ingestion.ometa.mixins.version_mixin import OMetaVersionMixin
 from metadata.ingestion.ometa.models import EntityList
 from metadata.ingestion.ometa.routes import ROUTES
-from metadata.ingestion.ometa.utils import get_entity_type, model_str
+from metadata.ingestion.ometa.utils import get_entity_type, model_str, quote
 from metadata.utils.logger import ometa_logger
 from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory
 from metadata.utils.ssl_registry import get_verify_ssl_fn
@@ -293,7 +292,7 @@ class OpenMetadata(
 
         return self._get(
             entity=entity,
-            path=f"name/{quote(model_str(fqn), safe='')}",
+            path=f"name/{quote(fqn)}",
             fields=fields,
             nullable=nullable,
         )

metadata/ingestion/ometa/utils.py

@@ -17,6 +17,9 @@ import string
 from typing import Any, Type, TypeVar, Union
 
 from pydantic import BaseModel
+from requests.utils import quote as url_quote
+
+from metadata.generated.schema.type.basic import FullyQualifiedEntityName
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -74,3 +77,11 @@ def model_str(arg: Any) -> str:
         return str(arg.root)
 
     return str(arg)
+
+
+def quote(fqn: Union[FullyQualifiedEntityName, str]) -> str:
+    """
+    Quote the FQN so that it's safe to pass to the API.
+    E.g., `"foo.bar/baz"` -> `%22foo.bar%2Fbaz%22`
+    """
+    return url_quote(model_str(fqn), safe="")

metadata/ingestion/sink/metadata_rest.py

@@ -586,11 +586,9 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
                 table=record.table, column_tags=record.column_tags
             )
             if not patched:
-                self.status.failed(
-                    StackTraceError(
-                        name=table.fullyQualifiedName.root,
-                        error="Error patching tags for table",
-                    )
+                self.status.warning(
+                    key=table.fullyQualifiedName.root,
+                    reason="Error patching tags for table",
                 )
             else:
                 logger.debug(

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -148,6 +148,24 @@ class TableauSource(DashboardServiceSource):
             logger.warning(f"Could not fetch owner data due to {err}")
         return None
 
+    @staticmethod
+    def _get_data_models_tags(dataModels: [DataSource]) -> Set[str]:
+        """
+        Get the tags from the data model in the upstreamDatasources
+        """
+        tags = set()
+        try:
+            for data_model in dataModels:
+                # tags seems to be available for upstreamDatasources only, not for dataModels
+                for upstream_source in data_model.upstreamDatasources or []:
+                    for tag in upstream_source.tags:
+                        tags.add(tag.name)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error fetching tags from data models: {exc}")
+
+        return tags
+
     def yield_tags(
         self, dashboard_details: TableauDashboard
     ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -160,8 +178,14 @@ class TableauSource(DashboardServiceSource):
                 for elem in container:
                     tags.update(elem.tags)
 
+            _tags = {tag.label for tag in tags}
+            # retrieve tags from data models
+            _data_models_tags = self._get_data_models_tags(dashboard_details.dataModels)
+
+            _all_tags = _tags.union(_data_models_tags)
+
             yield from get_ometa_tag_and_classification(
-                tags=[tag.label for tag in tags],
+                tags=[tag for tag in _all_tags],
                 classification_name=TABLEAU_TAG_CATEGORY,
                 tag_description="Tableau Tag",
                 classification_description="Tags associated with tableau entities",
@@ -201,13 +225,23 @@ class TableauSource(DashboardServiceSource):
             self.status.filter(data_model_name, "Data model filtered out.")
             return
         try:
+            data_model_tags = data_model.tags or []
             data_model_request = CreateDashboardDataModelRequest(
                 name=EntityName(data_model.id),
                 displayName=data_model_name,
+                description=Markdown(data_model.description)
+                if data_model.description
+                else None,
                 service=FullyQualifiedEntityName(self.context.get().dashboard_service),
                 dataModelType=data_model_type.value,
                 serviceType=DashboardServiceType.Tableau.value,
                 columns=self.get_column_info(data_model),
+                tags=get_tag_labels(
+                    metadata=self.metadata,
+                    tags=[tag.name for tag in data_model_tags],
+                    classification_name=TABLEAU_TAG_CATEGORY,
+                    include_tags=self.source_config.includeTags,
+                ),
                 sql=self._get_datamodel_sql_query(data_model=data_model),
                 owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )

metadata/ingestion/source/dashboard/tableau/models.py

@@ -58,6 +58,14 @@ class TableauTag(BaseModel):
     label: str
 
 
+class TableauDataModelTag(BaseModel):
+    """
+    Aux class for Tag object for Tableau Data Model
+    """
+
+    name: str
+
+
 class TableauOwner(TableauBaseModel):
     """
     Aux class for Owner object of the tableau_api_lib response
@@ -121,6 +129,8 @@ class UpstreamTable(BaseModel):
 class DataSource(BaseModel):
     id: str
     name: Optional[str] = None
+    description: Optional[str] = None
+    tags: Optional[List[TableauDataModelTag]] = []
     fields: Optional[List[DatasourceField]] = None
     upstreamTables: Optional[List[UpstreamTable]] = None
     upstreamDatasources: Optional[List["DataSource"]] = None

metadata/ingestion/source/dashboard/tableau/queries.py

@@ -26,6 +26,10 @@ workbooks(filter:{{luid: "{workbook_id}"}}){{
       upstreamDatasources{{
         id
         name
+        description
+        tags {{
+          name
+        }}
         fields {{
           id
           name

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -202,7 +202,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
             try:
                 logger.debug("Requesting [dbt_run_results]")
                 dbt_run_results = client.get(
-                    f"/accounts/{account_id}/runs/{run_id}/artifacts/{DBT_RUN_RESULTS_FILE_NAME}"
+                    f"/accounts/{account_id}/runs/{run_id}/artifacts/{DBT_RUN_RESULTS_FILE_NAME}.json"
                 )
             except Exception as exc:
                 logger.debug(

metadata/pii/processor.py

@@ -66,6 +66,7 @@ class PIIProcessor(Processor):
         )  # Used to satisfy type checked
 
         self._ner_scanner = None
+        self.name_scanner = ColumnNameScanner()
         self.confidence_threshold = self.source_config.confidence
 
     @property
@@ -128,7 +129,7 @@ class PIIProcessor(Processor):
             return None
 
         # Scan by column name. If no results there, check the sample data, if any
-        tag_and_confidence = ColumnNameScanner.scan(column.name.root) or (
+        tag_and_confidence = self.name_scanner.scan(column.name.root) or (
             self.ner_scanner.scan([row[idx] for row in table_data.rows])
             if table_data
             else None

metadata/pii/scanners/base.py

@@ -0,0 +1,23 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Basic Scanner ABC
+"""
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseScanner(ABC):
+    """Basic scanner abstract class"""
+
+    @abstractmethod
+    def scan(self, data: Any):
+        """Scan the given data from a column"""

metadata/pii/scanners/column_name_scanner.py

@@ -17,17 +17,18 @@ from typing import Optional
 from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.pii.constants import PII
 from metadata.pii.models import TagAndConfidence, TagType
+from metadata.pii.scanners.base import BaseScanner
 from metadata.utils import fqn
 
 
-class ColumnNameScanner:
+class ColumnNameScanner(BaseScanner):
     """
     Column Name Scanner to scan column name
     """
 
     sensitive_regex = {
         "PASSWORD": re.compile("^.*password.*$", re.IGNORECASE),
-        "SSN": re.compile("^.*(ssn|social).*$", re.IGNORECASE),
+        "US_SSN": re.compile("^.*(ssn|social).*$", re.IGNORECASE),
         "CREDIT_CARD": re.compile("^.*(credit).*(card).*$", re.IGNORECASE),
         "BANK_ACCOUNT": re.compile("^.*bank.*(acc|num).*$", re.IGNORECASE),
         "EMAIL_ADDRESS": re.compile("^.*(email|e-mail|mail).*$", re.IGNORECASE),
@@ -53,14 +54,13 @@ class ColumnNameScanner:
         "PHONE_NUMBER": re.compile("^.*(phone).*$", re.IGNORECASE),
     }
 
-    @classmethod
-    def scan(cls, column_name: str) -> Optional[TagAndConfidence]:
+    def scan(self, data: str) -> Optional[TagAndConfidence]:
         """
         Check the column name against the regex patterns and prepare the
         sensitive or non-sensitive tag
         """
-        for pii_type_pattern in cls.sensitive_regex.values():
-            if pii_type_pattern.match(column_name) is not None:
+        for pii_type_pattern in self.sensitive_regex.values():
+            if pii_type_pattern.match(data) is not None:
                 return TagAndConfidence(
                     tag_fqn=fqn.build(
                         metadata=None,
@@ -71,8 +71,8 @@ class ColumnNameScanner:
                     confidence=1,
                 )
 
-        for pii_type_pattern in cls.non_sensitive_regex.values():
-            if pii_type_pattern.match(column_name) is not None:
+        for pii_type_pattern in self.non_sensitive_regex.values():
+            if pii_type_pattern.match(data) is not None:
                 return TagAndConfidence(
                     tag_fqn=fqn.build(
                         metadata=None,