openmetadata-ingestion 1.4.6.0__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/readers/dataframe/reader_factory.py

@@ -42,6 +42,9 @@ class SupportedTypes(Enum):
     JSON = "json"
     JSONGZ = "json.gz"
     JSONZIP = "json.zip"
+    JSONL = "jsonl"
+    JSONLGZ = "jsonl.gz"
+    JSONLZIP = "jsonl.zip"
 
 
 DF_READER_MAP = {
@@ -52,6 +55,9 @@ DF_READER_MAP = {
     SupportedTypes.JSON.value: JSONDataFrameReader,
     SupportedTypes.JSONGZ.value: JSONDataFrameReader,
     SupportedTypes.JSONZIP.value: JSONDataFrameReader,
+    SupportedTypes.JSONL.value: JSONDataFrameReader,
+    SupportedTypes.JSONLGZ.value: JSONDataFrameReader,
+    SupportedTypes.JSONLZIP.value: JSONDataFrameReader,
 }
 
 

metadata/readers/file/api_reader.py

@@ -46,7 +46,7 @@ class ApiReader(Reader, ABC):
         """
         if self._auth_headers is None and self.credentials.token:
             self._auth_headers = {
-                "Authorization": f"Bearer {self.credentials.token.__root__.get_secret_value()}"
+                "Authorization": f"Bearer {self.credentials.token.root.get_secret_value()}"
             }
 
         return self._auth_headers

metadata/readers/file/bitbucket.py

@@ -56,8 +56,8 @@ class BitBucketReader(ApiReader):
                 self._build_url(
                     HOST,
                     UrlParts.REPOS.value,
-                    self.credentials.repositoryOwner.__root__,
-                    self.credentials.repositoryName.__root__,
+                    self.credentials.repositoryOwner.root,
+                    self.credentials.repositoryName.root,
                     UrlParts.SRC.value,
                     self.credentials.branch,
                     path,
@@ -114,8 +114,8 @@ class BitBucketReader(ApiReader):
         url = self._build_url(
             HOST,
             UrlParts.REPOS.value,
-            self.credentials.repositoryOwner.__root__,
-            self.credentials.repositoryName.__root__,
+            self.credentials.repositoryOwner.root,
+            self.credentials.repositoryName.root,
             UrlParts.SRC.value,
             self.credentials.branch,
         )

metadata/readers/file/credentials.py

@@ -26,8 +26,8 @@ def update_repository_name(
     Given an original set of credentials and a new repository name,
     return the updated credentials
     """
-    updated = original.copy(deep=True)
-    updated.repositoryName = RepositoryName(__root__=name)
+    updated = original.model_copy(deep=True)
+    updated.repositoryName = RepositoryName(name)
 
     return updated
 
@@ -47,16 +47,16 @@ def get_credentials_from_url(
     up information, which would still not happen since we work with a single
     token which cannot have permissions on different owners.
     """
-    if original.repositoryOwner.__root__ not in url:
+    if original.repositoryOwner.root not in url:
         logger.warning(
-            f"Default repository owner [{original.repositoryOwner.__root__}] not found in [{url}]."
+            f"Default repository owner [{original.repositoryOwner.root}] not found in [{url}]."
             " We'll use the default reader credentials."
         )
         return original
 
     # Your typical URL is git@bitbucket.org:owner/repo.git
     # or git@github.com:owner/repo.git
-    url_repository = url.split(original.repositoryOwner.__root__ + "/")[-1]
+    url_repository = url.split(original.repositoryOwner.root + "/")[-1]
     repo_name = url_repository.replace(".git", "")
 
     return update_repository_name(original=original, name=repo_name)

metadata/readers/file/github.py

@@ -66,8 +66,8 @@ class GitHubReader(ApiReader):
                 self._build_url(
                     HOST,
                     UrlParts.REPOS.value,
-                    self.credentials.repositoryOwner.__root__,
-                    self.credentials.repositoryName.__root__,
+                    self.credentials.repositoryOwner.root,
+                    self.credentials.repositoryName.root,
                     UrlParts.CONTENTS.value,
                     path,
                 ),
@@ -94,8 +94,8 @@ class GitHubReader(ApiReader):
             self._build_url(
                 HOST,
                 UrlParts.REPOS.value,
-                self.credentials.repositoryOwner.__root__,
-                self.credentials.repositoryName.__root__,
+                self.credentials.repositoryOwner.root,
+                self.credentials.repositoryName.root,
             ),
             headers=self.auth_headers,
             timeout=30,
@@ -118,8 +118,8 @@ class GitHubReader(ApiReader):
                 self._build_url(
                     HOST,
                     UrlParts.REPOS.value,
-                    self.credentials.repositoryOwner.__root__,
-                    self.credentials.repositoryName.__root__,
+                    self.credentials.repositoryOwner.root,
+                    self.credentials.repositoryName.root,
                     "git",
                     "trees",
                     f"{branch}?recursive=1",

metadata/utils/class_helper.py

@@ -103,16 +103,14 @@ def _clean(source_type: str):
     return source_type
 
 
-def get_pipeline_type_from_source_config(
-    source_config_type: SourceConfig.__fields__["config"].type_,
-) -> PipelineType:
+def get_pipeline_type_from_source_config(source_config: SourceConfig) -> PipelineType:
     """From the YAML serviceType, get the Ingestion Pipeline Type"""
     pipeline_type = SOURCE_CONFIG_TYPE_INGESTION.get(
-        source_config_type.__class__.__name__
+        source_config.config.__class__.__name__
     )
     if not pipeline_type:
         raise ValueError(
-            f"Cannot find Pipeline Type for SourceConfig {source_config_type}"
+            f"Cannot find Pipeline Type for SourceConfig {source_config.config}"
         )
     return pipeline_type
 

metadata/utils/credentials.py

@@ -105,7 +105,7 @@ def build_google_credentials_dict(
 
         return {
             "type": gcp_values.type,
-            "project_id": gcp_values.projectId.__root__,
+            "project_id": gcp_values.projectId.root,
             "private_key_id": gcp_values.privateKeyId,
             "private_key": private_key_str,
             "client_email": gcp_values.clientEmail,
@@ -134,7 +134,7 @@ def set_google_credentials(gcp_credentials: GCPCredentials) -> None:
     :param gcp_credentials: GCPCredentials
     """
     if isinstance(gcp_credentials.gcpConfig, GcpCredentialsPath):
-        os.environ[GOOGLE_CREDENTIALS] = str(gcp_credentials.gcpConfig.__root__)
+        os.environ[GOOGLE_CREDENTIALS] = str(gcp_credentials.gcpConfig.path)
         return
 
     if (

metadata/utils/datalake/datalake_utils.py

@@ -452,11 +452,13 @@ class ParquetDataFrameColumnParser:
 
             if parsed_column["dataType"] == DataType.BINARY:
                 try:
-                    data_length = type(column.type).byte_width
-                except AttributeError:
+                    # Either we an int number or -1
+                    data_length = int(type(column.type).byte_width)
+                except Exception as exc:
                     # if the byte width is not specified, we will set it to -1
                     # following pyarrow convention
                     data_length = -1
+                    logger.debug("Could not extract binary field length due to %s", exc)
                 parsed_column["dataLength"] = data_length
 
             if parsed_column["dataType"] == DataType.STRUCT:

metadata/utils/entity_link.py

@@ -57,6 +57,14 @@ def split(str_: str) -> List[str]:
 def get_decoded_column(entity_link: str) -> str:
     """From an URL encoded entity link get the decoded column name
 
+    Examples:
+        >>> get_decoded_column("<#E::table::rds.dev.dbt_jaffle.column_w_space::columns::first_name>")
+        'first name'
+        >>> get_decoded_column("<#E::table::rds.dev.dbt_jaffle.column_w_space::columns::随机的>")
+        '随机的'
+        >>> get_decoded_column("<#E::table::rds.dev.dbt_jaffle.table_w_space>")
+        ''
+
     Args:
         entity_link: entity link
     """

metadata/utils/execution_time_tracker.py

@@ -17,7 +17,7 @@ import threading
 from copy import deepcopy
 from functools import wraps
 from time import perf_counter
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 from pydantic import BaseModel
 
@@ -81,7 +81,7 @@ class ExecutionTimeTrackerState(metaclass=Singleton):
 
     def __init__(self):
         """Initializes the state and the lock."""
-        self.state = {}
+        self.state: Dict[str, float] = {}
         self.lock = threading.Lock()
 
     def add(self, context: ExecutionTimeTrackerContext, elapsed: float):

metadata/utils/filters.py

@@ -27,12 +27,12 @@ class InvalidPatternException(Exception):
     """
 
 
-def validate_regex(regex_list: List[str]) -> None:
+def validate_regex(regex_list: Optional[List[str]]) -> None:
     """
     Check that the given include/exclude regexes
     are well formatted
     """
-    for regex in regex_list:
+    for regex in regex_list or []:
         try:
             re.compile(regex)
         except re.error as err:
@@ -40,7 +40,7 @@ def validate_regex(regex_list: List[str]) -> None:
             raise InvalidPatternException(msg) from err
 
 
-def _filter(filter_pattern: Optional[FilterPattern], name: str) -> bool:
+def _filter(filter_pattern: Optional[FilterPattern], name: Optional[str]) -> bool:
     """
     Return True if the name needs to be filtered, False otherwise
 
@@ -58,24 +58,32 @@ def _filter(filter_pattern: Optional[FilterPattern], name: str) -> bool:
         # Filter pattern is present but not the name so we'll filter it out
         return True
 
+    validate_regex(filter_pattern.includes)
+    validate_regex(filter_pattern.excludes)
+
+    if filter_pattern.includes and filter_pattern.excludes:
+        return not any(
+            name
+            for regex in filter_pattern.includes
+            if (re.match(regex, name, re.IGNORECASE))
+        ) or any(
+            name
+            for regex in filter_pattern.excludes
+            if (re.match(regex, name, re.IGNORECASE))
+        )
+
     if filter_pattern.includes:
-        validate_regex(filter_pattern.includes)
-        return not any(  # pylint: disable=use-a-generator
-            [
-                name
-                for regex in filter_pattern.includes
-                if (re.match(regex, name, re.IGNORECASE))
-            ]
+        return not any(
+            name
+            for regex in filter_pattern.includes
+            if (re.match(regex, name, re.IGNORECASE))
         )
 
     if filter_pattern.excludes:
-        validate_regex(filter_pattern.excludes)
-        return any(  # pylint: disable=use-a-generator
-            [
-                name
-                for regex in filter_pattern.excludes
-                if (re.match(regex, name, re.IGNORECASE))
-            ]
+        return any(
+            name
+            for regex in filter_pattern.excludes
+            if (re.match(regex, name, re.IGNORECASE))
         )
 
     return False

metadata/utils/fqn.py

@@ -65,8 +65,8 @@ class SplitTestCaseFqn(BaseModel):
     database: str
     schema_: str = Field(alias="schema")
     table: str
-    column: Optional[str]
-    test_case: Optional[str]
+    column: Optional[str] = None
+    test_case: Optional[str] = None
 
 
 def split(str_: str) -> List[str]:
@@ -184,9 +184,9 @@ def _(
         fqn = _build(service_name, database_name, schema_name, table_name)
         return [fqn] if fetch_multiple_entities else fqn
     if entity and fetch_multiple_entities:
-        return [str(table.fullyQualifiedName.__root__) for table in entity]
+        return [str(table.fullyQualifiedName.root) for table in entity]
     if entity:
-        return str(entity.fullyQualifiedName.__root__)
+        return str(entity.fullyQualifiedName.root)
     return None
 
 
@@ -215,9 +215,9 @@ def _(
         fqn = _build(service_name, database_name, schema_name)
         return [fqn] if fetch_multiple_entities else fqn
     if entity and fetch_multiple_entities:
-        return [str(table.fullyQualifiedName.__root__) for table in entity]
+        return [str(table.fullyQualifiedName.root) for table in entity]
     if entity:
-        return str(entity.fullyQualifiedName.__root__)
+        return str(entity.fullyQualifiedName.root)
 
     return None
 
@@ -290,16 +290,33 @@ def _(_: Optional[OpenMetadata], *, table_fqn: str) -> str:
 
 @fqn_build_registry.add(Topic)
 def _(
-    _: Optional[OpenMetadata],  # ES Index not necessary for Topic FQN building
+    metadata: Optional[OpenMetadata],
     *,
     service_name: str,
     topic_name: str,
-) -> str:
-    if not service_name or not topic_name:
+    skip_es_search: bool = True,
+) -> Optional[str]:
+    entity: Optional[Topic] = None
+
+    if not skip_es_search:
+        entity = search_topic_from_es(
+            metadata=metadata, service_name=service_name, topic_name=topic_name
+        )
+
+    # if entity not found in ES proceed to build FQN with database_name and schema_name
+    if not entity and service_name and topic_name:
+        fqn = _build(service_name, topic_name)
+        return fqn
+
+    if entity:
+        return str(entity.fullyQualifiedName.root)
+
+    if not all([service_name, topic_name]):
         raise FQNBuildingException(
             f"Args should be informed, but got service=`{service_name}`, topic=`{topic_name}``"
         )
-    return _build(service_name, topic_name)
+
+    return None
 
 
 @fqn_build_registry.add(Container)
@@ -422,8 +439,8 @@ def _(
     if not entity:
         return None
     if fetch_multiple_entities:
-        return [str(user.fullyQualifiedName.__root__) for user in entity]
-    return str(entity.fullyQualifiedName.__root__)
+        return [str(user.fullyQualifiedName.root) for user in entity]
+    return str(entity.fullyQualifiedName.root)
 
 
 @fqn_build_registry.add(Team)
@@ -452,8 +469,8 @@ def _(
     if not entity:
         return None
     if fetch_multiple_entities:
-        return [str(user.fullyQualifiedName.__root__) for user in entity]
-    return str(entity.fullyQualifiedName.__root__)
+        return [str(user.fullyQualifiedName.root) for user in entity]
+    return str(entity.fullyQualifiedName.root)
 
 
 @fqn_build_registry.add(TestCase)
@@ -681,6 +698,34 @@ def search_database_from_es(
     )
 
 
+def search_topic_from_es(
+    metadata: OpenMetadata,
+    topic_name: str,
+    service_name: Optional[str],
+    fields: Optional[str] = None,
+):
+    """
+    Search Topic entity from ES
+    """
+
+    if not topic_name:
+        raise FQNBuildingException(
+            f"Topic Name should be informed, but got topic=`{topic_name}`"
+        )
+
+    fqn_search_string = _build(service_name or "*", topic_name)
+
+    es_result = metadata.es_search_from_fqn(
+        entity_type=Topic,
+        fqn_search_string=fqn_search_string,
+        fields=fields,
+    )
+
+    return get_entity_from_es_result(
+        entity_list=es_result, fetch_multiple_entities=False
+    )
+
+
 def get_query_checksum(query: str) -> str:
     """
     Prepare the query checksum from its string representation.

metadata/utils/helpers.py

@@ -19,12 +19,13 @@ import itertools
 import re
 import shutil
 import sys
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from math import floor, log
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 
 import sqlparse
+from pydantic_core import Url
 from sqlparse.sql import Statement
 
 from metadata.generated.schema.entity.data.chart import ChartType
@@ -117,12 +118,12 @@ def pretty_print_time_duration(duration: Union[int, float]) -> str:
     return f"{seconds}s"
 
 
-def get_start_and_end(duration: int = 0):
+def get_start_and_end(duration: int = 0) -> Tuple[datetime, datetime]:
     """
     Method to return start and end time based on duration
     """
 
-    today = datetime.utcnow()
+    today = datetime.now(timezone.utc)
     start = (today + timedelta(0 - duration)).replace(
         hour=0, minute=0, second=0, microsecond=0
     )
@@ -206,7 +207,7 @@ def find_column_in_table(
         return first.lower() == second.lower()
 
     return next(
-        (col for col in table.columns if equals(col.name.__root__, column_name)), None
+        (col for col in table.columns if equals(col.name.root, column_name)), None
     )
 
 
@@ -222,7 +223,7 @@ def find_suggestion(
         (
             sugg
             for sugg in suggestions
-            if sugg.type == suggestion_type and sugg.entityLink == entity_link
+            if sugg.root.type == suggestion_type and sugg.root.entityLink == entity_link
         ),
         None,
     )
@@ -244,7 +245,7 @@ def find_column_in_table_with_index(
         (
             (col_index, col)
             for col_index, col in enumerate(table.columns)
-            if str(col.name.__root__).lower() == column_name.lower()
+            if str(col.name.root).lower() == column_name.lower()
         ),
         (None, None),
     )
@@ -325,11 +326,7 @@ def get_entity_tier_from_tags(tags: list[TagLabel]) -> Optional[str]:
     if not tags:
         return None
     return next(
-        (
-            tag.tagFQN.__root__
-            for tag in tags
-            if tag.tagFQN.__root__.lower().startswith("tier")
-        ),
+        (tag.tagFQN.root for tag in tags if tag.tagFQN.root.lower().startswith("tier")),
         None,
     )
 
@@ -351,12 +348,15 @@ def format_large_string_numbers(number: Union[float, int]) -> str:
     return f"{number / constant_k**magnitude:.3f}{units[magnitude]}"
 
 
-def clean_uri(uri: str) -> str:
+def clean_uri(uri: Union[str, Url]) -> str:
     """
     if uri is like http://localhost:9000/
     then remove the end / and
     make it http://localhost:9000
     """
+    # force a string of the given Uri if needed
+    if isinstance(uri, Url):
+        uri = str(uri)
     return uri[:-1] if uri.endswith("/") else uri
 
 

metadata/utils/importer.py

@@ -18,6 +18,7 @@ from typing import Any, Callable, Optional, Type, TypeVar
 
 from pydantic import BaseModel
 
+from metadata.data_quality.validations.base_test_handler import BaseTestValidator
 from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
     OpenMetadataConnection,
 )
@@ -170,7 +171,7 @@ def get_sink(
     from the given configs
     """
     sink_class = import_sink_class(sink_type=sink_type, from_=from_)
-    sink_config = sink_config.dict().get("config", {})
+    sink_config = sink_config.model_dump().get("config", {})
     sink: Sink = sink_class.create(sink_config, metadata_config)
     logger.debug(f"Sink type:{sink_type}, {sink_class} configured")
 
@@ -220,7 +221,7 @@ def import_test_case_class(
     test_type: str,
     runner_type: str,
     test_definition: str,
-) -> Callable:
+) -> Type[BaseTestValidator]:
     """_summary_
 
     Args:

metadata/utils/life_cycle_utils.py

@@ -58,11 +58,11 @@ def _get_query_type_from_regex(create_query) -> Optional[Any]:
     """
     Method to get the query type from regex
     """
-    if re.match(create_pattern, create_query.query.__root__):
+    if re.match(create_pattern, create_query.query.root):
         return "created"
-    if re.match(update_pattern, create_query.query.__root__):
+    if re.match(update_pattern, create_query.query.root):
         return "updated"
-    if re.match(select_pattern, create_query.query.__root__):
+    if re.match(select_pattern, create_query.query.root):
         return "accessed"
     return None
 

metadata/utils/logger.py

@@ -52,7 +52,6 @@ class Loggers(Enum):
     GREAT_EXPECTATIONS = "GreatExpectations"
     PROFILER_INTERFACE = "ProfilerInterface"
     TEST_SUITE = "TestSuite"
-    DATA_INSIGHT = "DataInsight"
     QUERY_RUNNER = "QueryRunner"
     APP = "App"
 
@@ -130,14 +129,6 @@ def ingestion_logger():
     return logging.getLogger(Loggers.INGESTION.value)
 
 
-def data_insight_logger():
-    """
-    Function to get the DATA INSIGHT logger
-    """
-
-    return logging.getLogger(Loggers.DATA_INSIGHT.value)
-
-
 def utils_logger():
     """
     Method to get the UTILS logger
@@ -190,8 +181,8 @@ def log_ansi_encoded_string(
 def get_log_name(record: Entity) -> Optional[str]:
     try:
         if hasattr(record, "name"):
-            return f"{type(record).__name__} [{getattr(record, 'name').__root__}]"
-        return f"{type(record).__name__} [{record.entity.name.__root__}]"
+            return f"{type(record).__name__} [{getattr(record, 'name').root}]"
+        return f"{type(record).__name__} [{record.entity.name.root}]"
     except Exception:
         return str(record)
 
@@ -202,11 +193,7 @@ def _(record: OMetaTagAndClassification) -> str:
     Given a LineageRequest, parse its contents to return
     a string that we can log
     """
-    name = (
-        record.fqn.__root__
-        if record.fqn
-        else record.classification_request.name.__root__
-    )
+    name = record.fqn.root if record.fqn else record.classification_request.name.root
     return f"{type(record).__name__} [{name}]"
 
 
@@ -218,7 +205,7 @@ def _(record: AddLineageRequest) -> str:
     """
 
     # id and type will always be informed
-    id_ = record.edge.fromEntity.id.__root__
+    id_ = record.edge.fromEntity.id.root
     type_ = record.edge.fromEntity.type
 
     # name can be informed or not
@@ -234,7 +221,7 @@ def _(record: DeleteEntity) -> str:
     """
     Capture information about the deleted Entity
     """
-    return f"{type(record.entity).__name__} [{record.entity.name.__root__}]"
+    return f"{type(record.entity).__name__} [{record.entity.name.root}]"
 
 
 @get_log_name.register
@@ -248,22 +235,20 @@ def _(record: OMetaLifeCycleData) -> str:
 @get_log_name.register
 def _(record: TableAndTests) -> str:
     if record.table:
-        return f"Tests for [{record.table.fullyQualifiedName.__root__}]"
+        return f"Tests for [{record.table.fullyQualifiedName.root}]"
 
-    return f"Test Suite [{record.executable_test_suite.name.__root__}]"
+    return f"Test Suite [{record.executable_test_suite.name.root}]"
 
 
 @get_log_name.register
 def _(record: TestCaseResults) -> str:
     """We don't want to log this in the status"""
-    return ",".join(
-        set(result.testCase.name.__root__ for result in record.test_results)
-    )
+    return ",".join(set(result.testCase.name.root for result in record.test_results))
 
 
 @get_log_name.register
 def _(record: TestCaseResultResponse) -> str:
-    return record.testCase.fullyQualifiedName.__root__
+    return record.testCase.fullyQualifiedName.root
 
 
 @get_log_name.register

metadata/utils/lru_cache.py

@@ -25,6 +25,9 @@ class LRUCache:
         self._cache = OrderedDict()
         self.capacity = capacity
 
+    def clear(self):
+        self._cache = OrderedDict()
+
     def get(self, key):
         """
         Returns the value associated to `key` if it exists,

metadata/utils/metadata_service_helper.py

@@ -11,7 +11,9 @@ SERVICE_TYPE_MAPPER = {
         "service_name": "DeltaLake",
         "connection": {
             "config": {
-                "metastoreConnection": {"metastoreHostPort": "http://localhost:9083"}
+                "configSource": {
+                    "connection": {"metastoreHostPort": "http://localhost:9083"}
+                }
             }
         },
     },