openmetadata-ingestion 1.5.11.0__py3-none-any.whl → 1.5.12.0__py3-none-any.whl

metadata/ingestion/source/dashboard/mstr/metadata.py

@@ -91,6 +91,18 @@ class MstrSource(DashboardServiceSource):
         """
         return dashboard.name
 
+    def get_project_name(self, dashboard_details: MstrDashboard) -> Optional[str]:
+        """
+        Get dashboard project name
+        """
+        try:
+            return dashboard_details.projectName
+        except Exception as exc:
+            logger.debug(
+                f"Cannot get project name from dashboard [{dashboard_details.name}] - [{exc}]"
+            )
+        return None
+
     def get_dashboard_details(self, dashboard: MstrDashboard) -> MstrDashboardDetails:
         """
         Get Dashboard Details
@@ -106,40 +118,41 @@ class MstrSource(DashboardServiceSource):
         """
         Method to Get Dashboard Entity
         """
-        try:
-            dashboard_url = (
-                f"{clean_uri(self.service_connection.hostPort)}/MicroStrategyLibrary/app/"
-                f"{dashboard_details.projectId}/{dashboard_details.id}"
-            )
-            dashboard_request = CreateDashboardRequest(
-                name=EntityName(dashboard_details.id),
-                displayName=dashboard_details.name,
-                sourceUrl=SourceUrl(dashboard_url),
-                project=dashboard_details.projectName,
-                charts=[
-                    FullyQualifiedEntityName(
-                        fqn.build(
-                            self.metadata,
-                            entity_type=Chart,
-                            service_name=self.context.get().dashboard_service,
-                            chart_name=chart,
+        if dashboard_details:
+            try:
+                dashboard_url = (
+                    f"{clean_uri(self.service_connection.hostPort)}/MicroStrategyLibrary/app/"
+                    f"{dashboard_details.projectId}/{dashboard_details.id}"
+                )
+                dashboard_request = CreateDashboardRequest(
+                    name=EntityName(dashboard_details.id),
+                    displayName=dashboard_details.name,
+                    sourceUrl=SourceUrl(dashboard_url),
+                    project=dashboard_details.projectName,
+                    charts=[
+                        FullyQualifiedEntityName(
+                            fqn.build(
+                                self.metadata,
+                                entity_type=Chart,
+                                service_name=self.context.get().dashboard_service,
+                                chart_name=chart,
+                            )
                         )
+                        for chart in self.context.get().charts or []
+                    ],
+                    service=self.context.get().dashboard_service,
+                    owners=self.get_owner_ref(dashboard_details=dashboard_details),
+                )
+                yield Either(right=dashboard_request)
+                self.register_record(dashboard_request=dashboard_request)
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name=dashboard_details.id,
+                        error=f"Error yielding dashboard for {dashboard_details}: {exc}",
+                        stackTrace=traceback.format_exc(),
                     )
-                    for chart in self.context.get().charts or []
-                ],
-                service=self.context.get().dashboard_service,
-                owners=self.get_owner_ref(dashboard_details=dashboard_details),
-            )
-            yield Either(right=dashboard_request)
-            self.register_record(dashboard_request=dashboard_request)
-        except Exception as exc:
-            yield Either(
-                left=StackTraceError(
-                    name=dashboard_details.id,
-                    error=f"Error yielding dashboard for {dashboard_details}: {exc}",
-                    stackTrace=traceback.format_exc(),
                 )
-            )
 
     def yield_dashboard_lineage_details(
         self, dashboard_details: MstrDashboardDetails, db_service_name: str
@@ -156,14 +169,15 @@ class MstrSource(DashboardServiceSource):
         Returns:
             Iterable[CreateChartRequest]
         """
-        try:
-            for chapter in dashboard_details.chapters:
-                for page in chapter.pages:
-                    yield from self._yield_chart_from_visualization(page)
+        if dashboard_details:
+            try:
+                for chapter in dashboard_details.chapters:
+                    for page in chapter.pages:
+                        yield from self._yield_chart_from_visualization(page)
 
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Error creating dashboard: {exc}")
+            except Exception as exc:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error creating dashboard: {exc}")
 
     def _yield_chart_from_visualization(
         self, page: MstrPage
@@ -192,3 +206,8 @@ class MstrSource(DashboardServiceSource):
                         stackTrace=traceback.format_exc(),
                     )
                 )
+
+    def close(self):
+        # close the api session
+        self.client.close_api_session()
+        self.metadata.close()

metadata/ingestion/source/dashboard/mstr/models.py

@@ -12,7 +12,7 @@
 MSTR Models
 """
 from datetime import datetime
-from typing import List, Optional
+from typing import Any, List, Optional
 
 from pydantic import BaseModel
 
@@ -69,7 +69,8 @@ class MstrSearchResult(BaseModel):
 
 
 class MstrSearchResultList(BaseModel):
-    results: Optional[List[MstrSearchResult]]
+    totalItems: Optional[int] = 0
+    result: Optional[List[MstrSearchResult]] = None
 
 
 class MstrDashboard(BaseModel):
@@ -142,3 +143,8 @@ class MstrDashboardDetails(BaseModel):
     projectName: str
     currentChapter: str
     chapters: List[MstrChapter]
+
+
+class AuthHeaderCookie(BaseModel):
+    auth_header: dict
+    auth_cookies: Any

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -201,7 +201,7 @@ class SupersetDBSource(SupersetSourceMixin):
             except Exception as exc:
                 yield Either(
                     left=StackTraceError(
-                        name=chart_json.id,
+                        name=str(chart_json.id),
                         error=f"Error yielding Chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
                         stackTrace=traceback.format_exc(),
                     )

metadata/ingestion/source/database/athena/metadata.py

@@ -157,6 +157,14 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
             return True, partition_details
         return False, None
 
+    def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]:
+        """
+        Method to fetch the location path of the table
+        """
+        return self.external_location_map.get(
+            (self.context.get().database, schema_name, table_name)
+        )
+
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:

metadata/ingestion/source/database/common_nosql_source.py

@@ -54,6 +54,7 @@ from metadata.utils.constants import DEFAULT_DATABASE
 from metadata.utils.datalake.datalake_utils import DataFrameColumnParser
 from metadata.utils.filters import filter_by_schema, filter_by_table
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import check_ssl_and_init
 
 logger = ingestion_logger()
 
@@ -75,7 +76,13 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
         )
         self.metadata = metadata
         self.service_connection = self.config.serviceConnection.root.config
+        self.ssl_manager = check_ssl_and_init(self.service_connection)
+        if self.ssl_manager:
+            self.service_connection = self.ssl_manager.setup_ssl(
+                self.service_connection
+            )
         self.connection_obj = get_connection(self.service_connection)
+
         self.test_connection()
 
     def prepare(self):

metadata/ingestion/source/database/mysql/connection.py

@@ -38,6 +38,7 @@ from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.mysql.queries import MYSQL_TEST_GET_QUERIES
 from metadata.utils.constants import THREE_MIN
 
 
@@ -73,10 +74,14 @@ def test_connection(
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
+    queries = {
+        "GetQueries": MYSQL_TEST_GET_QUERIES,
+    }
     return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         timeout_seconds=timeout_seconds,
+        queries=queries,
     )

metadata/ingestion/source/database/mysql/lineage.py

@@ -0,0 +1,30 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+MYSQL lineage module
+"""
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.ingestion.source.database.mysql.queries import MYSQL_SQL_STATEMENT
+from metadata.ingestion.source.database.mysql.query_parser import MysqlQueryParserSource
+
+
+class MysqlLineageSource(MysqlQueryParserSource, LineageSource):
+
+    sql_stmt = MYSQL_SQL_STATEMENT
+
+    filters = """
+        AND (
+            lower(argument) LIKE '%%create%%table%%select%%'
+            OR lower(argument) LIKE '%%insert%%into%%select%%'
+            OR lower(argument) LIKE '%%update%%'
+            OR lower(argument) LIKE '%%merge%%'
+        )
+    """

metadata/ingestion/source/database/mysql/queries.py

@@ -0,0 +1,45 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SQL Queries used during ingestion
+"""
+
+import textwrap
+
+MYSQL_SQL_STATEMENT = textwrap.dedent(
+    """
+SELECT 
+	NULL `database_name`,
+	argument `query_text`,
+	event_time `start_time`,
+    NULL `end_time`,
+	NULL `duration`,
+	NULL `schema_name`,
+    NULL `query_type`,
+    NULL `user_name`,
+    NULL `aborted`
+FROM mysql.general_log
+WHERE command_type = 'Query' 
+    AND event_time between '{start_time}' and '{end_time}'
+    AND argument NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
+    AND argument NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+    {filters}
+ORDER BY event_time desc
+LIMIT {result_limit};
+"""
+)
+
+
+MYSQL_TEST_GET_QUERIES = textwrap.dedent(
+    """
+SELECT `argument` from mysql.general_log limit 1;
+"""
+)

metadata/ingestion/source/database/mysql/query_parser.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Mysql query parser module
+"""
+from abc import ABC
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
+    MysqlConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.query_parser_source import QueryParserSource
+
+
+class MysqlQueryParserSource(QueryParserSource, ABC):
+    """
+    Mysql base for Usage and Lineage
+    """
+
+    filters: str
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: MysqlConnection = config.serviceConnection.root.config
+        if not isinstance(connection, MysqlConnection):
+            raise InvalidSourceException(
+                f"Expected MysqlConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/mysql/usage.py

@@ -0,0 +1,26 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+MYSQL usage module
+"""
+from metadata.ingestion.source.database.mysql.queries import MYSQL_SQL_STATEMENT
+from metadata.ingestion.source.database.mysql.query_parser import MysqlQueryParserSource
+from metadata.ingestion.source.database.usage_source import UsageSource
+
+
+class MysqlUsageSource(MysqlQueryParserSource, UsageSource):
+
+    sql_stmt = MYSQL_SQL_STATEMENT
+
+    filters = ""
+
+    def format_query(self, query: bytes) -> str:
+        return query.decode(errors="ignore").replace("\\n", "\n")

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -217,9 +217,24 @@ class StoredProcedureMixin(ABC):
                     for query_by_procedure in (
                         queries_dict.get(procedure.name.root.lower()) or []
                     ):
-                        yield from self.yield_procedure_lineage(
-                            query_by_procedure=query_by_procedure, procedure=procedure
-                        )
-                        yield from self.yield_procedure_query(
-                            query_by_procedure=query_by_procedure, procedure=procedure
-                        )
+                        try:
+                            yield from self.yield_procedure_lineage(
+                                query_by_procedure=query_by_procedure,
+                                procedure=procedure,
+                            )
+                        except Exception as exc:
+                            logger.debug(traceback.format_exc())
+                            logger.warning(
+                                f"Could not get lineage for store procedure '{procedure_fqn}' due to [{exc}]."
+                            )
+
+                        try:
+                            yield from self.yield_procedure_query(
+                                query_by_procedure=query_by_procedure,
+                                procedure=procedure,
+                            )
+                        except Exception as exc:
+                            logger.debug(traceback.format_exc())
+                            logger.warning(
+                                f"Could not get query for store procedure '{procedure_fqn}' due to [{exc}]."
+                            )

metadata/utils/logger.py

@@ -284,4 +284,4 @@ def _(record: QueryParserData) -> str:
     queries = "\n------\n".join(
         mask_query(query.sql, query.dialect) for query in record.parsedData
     )
-    return f"Usage ParsedData [{queries}]"
+    return f"Usage ParsedData [{queries}]"

metadata/utils/ssl_manager.py

@@ -21,6 +21,9 @@ from typing import Optional, Union, cast
 
 from pydantic import SecretStr
 
+from metadata.generated.schema.entity.services.connections.connectionBasicType import (
+    ConnectionOptions,
+)
 from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import (
     QlikSenseConnection,
 )
@@ -30,6 +33,9 @@ from metadata.generated.schema.entity.services.connections.database.dorisConnect
 from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
     GreenplumConnection,
 )
+from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
+    MongoDBConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
@@ -158,6 +164,20 @@ class SSLManager:
             "check_hostname": connection.validateHostName,
         }
 
+    @setup_ssl.register(MongoDBConnection)
+    def _(self, connection: MongoDBConnection):
+        connection.connectionOptions = (
+            connection.connectionOptions or ConnectionOptions(root={})
+        )
+        connection.connectionOptions.root.update(
+            {
+                "tls": "true",
+                "tlsCertificateKeyFile": self.key_file_path,
+                "tlsCAFile": self.ca_file_path,
+            }
+        )
+        return connection
+
     @setup_ssl.register(KafkaConnection)
     def _(self, connection):
         connection = cast(KafkaConnection, connection)
@@ -170,7 +190,7 @@ class SSLManager:
 
 
 @singledispatch
-def check_ssl_and_init(_) -> None:
+def check_ssl_and_init(_) -> Optional[SSLManager]:
     return None
 
 
@@ -203,6 +223,24 @@ def _(connection):
     return None
 
 
+@check_ssl_and_init.register(MongoDBConnection)
+def _(connection):
+    service_connection = cast(Union[MysqlConnection, DorisConnection], connection)
+    ssl: Optional[verifySSLConfig.SslConfig] = service_connection.sslConfig
+    if ssl and ssl.root.sslCertificate:
+        raise ValueError(
+            "MongoDB connection does not support SSL certificate. Only CA certificate is supported.\n"
+            "More information about configuring MongoDB connection can be found at:\n"
+            "https://www.mongodb.com/docs/manual/tutorial/configure-ssl-clients/#mongodb-shell"
+        )
+    if ssl and (ssl.root.caCertificate or ssl.root.sslKey):
+        return SSLManager(
+            ca=ssl.root.caCertificate,
+            key=ssl.root.sslKey,
+        )
+    return None
+
+
 @check_ssl_and_init.register(PostgresConnection)
 @check_ssl_and_init.register(RedshiftConnection)
 @check_ssl_and_init.register(GreenplumConnection)