openmetadata-ingestion 1.5.11.0__py3-none-any.whl → 1.6.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/sample_data.py

@@ -136,12 +136,13 @@ from metadata.parsers.schema_parsers import (
     InvalidSchemaTypeException,
     schema_parser_config_registry,
 )
+from metadata.profiler.api.models import ProfilerResponse, SampleData
 from metadata.utils import entity_link, fqn
 from metadata.utils.constants import UTF_8
 from metadata.utils.fqn import FQN_SEPARATOR
 from metadata.utils.helpers import get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
-from metadata.utils.time_utils import convert_timestamp_to_milliseconds
+from metadata.utils.time_utils import datetime_to_timestamp
 
 logger = ingestion_logger()
 
@@ -242,6 +243,41 @@ class SampleDataSource(
             entity=DatabaseService,
             config=WorkflowSource(**self.glue_database_service_json),
         )
+
+        # MYSQL service for er diagrams
+        self.mysql_database_service_json = json.load(
+            open(  # pylint: disable=consider-using-with
+                sample_data_folder + "/mysql/database_service.json",
+                "r",
+                encoding=UTF_8,
+            )
+        )
+        self.mysql_database = json.load(
+            open(  # pylint: disable=consider-using-with
+                sample_data_folder + "/mysql/database.json",
+                "r",
+                encoding=UTF_8,
+            )
+        )
+        self.mysql_database_schema = json.load(
+            open(  # pylint: disable=consider-using-with
+                sample_data_folder + "/mysql/database_schema.json",
+                "r",
+                encoding=UTF_8,
+            )
+        )
+        self.mysql_tables = json.load(
+            open(  # pylint: disable=consider-using-with
+                sample_data_folder + "/mysql/tables.json",
+                "r",
+                encoding=UTF_8,
+            )
+        )
+        self.mysql_database_service = self.metadata.get_service_or_create(
+            entity=DatabaseService,
+            config=WorkflowSource(**self.mysql_database_service_json),
+        )
+
         self.database_service_json = json.load(
             open(  # pylint: disable=consider-using-with
                 sample_data_folder + "/datasets/service.json",
@@ -615,6 +651,7 @@ class SampleDataSource(
         yield from self.ingest_users()
         yield from self.ingest_tables()
         yield from self.ingest_glue()
+        yield from self.ingest_mysql()
         yield from self.ingest_stored_procedures()
         yield from self.ingest_topics()
         yield from self.ingest_charts()
@@ -666,6 +703,55 @@ class SampleDataSource(
 
             yield Either(right=team_to_ingest)
 
+    def ingest_mysql(self) -> Iterable[Either[Entity]]:
+        """Ingest Sample Data for mysql database source including ER diagrams metadata"""
+
+        db = CreateDatabaseRequest(
+            name=self.mysql_database["name"],
+            service=self.mysql_database_service.fullyQualifiedName,
+        )
+
+        yield Either(right=db)
+
+        database_entity = fqn.build(
+            self.metadata,
+            entity_type=Database,
+            service_name=self.mysql_database_service.fullyQualifiedName.root,
+            database_name=db.name.root,
+        )
+
+        database_object = self.metadata.get_by_name(
+            entity=Database, fqn=database_entity
+        )
+        schema = CreateDatabaseSchemaRequest(
+            name=self.mysql_database_schema["name"],
+            database=database_object.fullyQualifiedName,
+        )
+        yield Either(right=schema)
+
+        database_schema_entity = fqn.build(
+            self.metadata,
+            entity_type=DatabaseSchema,
+            service_name=self.mysql_database_service.fullyQualifiedName.root,
+            database_name=db.name.root,
+            schema_name=schema.name.root,
+        )
+
+        database_schema_object = self.metadata.get_by_name(
+            entity=DatabaseSchema, fqn=database_schema_entity
+        )
+
+        for table in self.mysql_tables["tables"]:
+            table_request = CreateTableRequest(
+                name=table["name"],
+                description=table["description"],
+                columns=table["columns"],
+                databaseSchema=database_schema_object.fullyQualifiedName,
+                tableConstraints=table.get("tableConstraints"),
+                tableType=table["tableType"],
+            )
+            yield Either(right=table_request)
+
     def ingest_glue(self) -> Iterable[Either[Entity]]:
         """Ingest Sample Data for glue database source"""
 
@@ -813,10 +899,24 @@ class SampleDataSource(
 
                 self.metadata.ingest_table_sample_data(
                     table_entity,
-                    TableData(
-                        rows=table["sampleData"]["rows"],
-                        columns=table["sampleData"]["columns"],
-                    ),
+                    ProfilerResponse(
+                        table=table_entity,
+                        profile=CreateTableProfileRequest(
+                            tableProfile=TableProfile(
+                                timestamp=Timestamp(
+                                    int(datetime.now().timestamp() * 1000)
+                                ),
+                                columnCount=1.0,
+                                rowCount=3.0,
+                            )
+                        ),
+                        sample_data=SampleData(
+                            data=TableData(
+                                rows=table["sampleData"]["rows"],
+                                columns=table["sampleData"]["columns"],
+                            )
+                        ),
+                    ).sample_data.data,
                 )
 
             if table.get("customMetrics"):
@@ -1246,12 +1346,14 @@ class SampleDataSource(
                     description=model["description"],
                     algorithm=model["algorithm"],
                     dashboard=dashboard.fullyQualifiedName.root,
-                    mlStore=MlStore(
-                        storage=model["mlStore"]["storage"],
-                        imageRepository=model["mlStore"]["imageRepository"],
-                    )
-                    if model.get("mlStore")
-                    else None,
+                    mlStore=(
+                        MlStore(
+                            storage=model["mlStore"]["storage"],
+                            imageRepository=model["mlStore"]["imageRepository"],
+                        )
+                        if model.get("mlStore")
+                        else None
+                    ),
                     server=model.get("server"),
                     target=model.get("target"),
                     mlFeatures=self.get_ml_features(model),
@@ -1290,9 +1392,11 @@ class SampleDataSource(
                     name=container["name"],
                     displayName=container["displayName"],
                     description=container["description"],
-                    parent=EntityReference(id=parent_container.id, type="container")
-                    if parent_container_fqn
-                    else None,
+                    parent=(
+                        EntityReference(id=parent_container.id, type="container")
+                        if parent_container_fqn
+                        else None
+                    ),
                     prefix=container["prefix"],
                     dataModel=container.get("dataModel"),
                     numberOfObjects=container.get("numberOfObjects"),
@@ -1330,11 +1434,13 @@ class SampleDataSource(
                     yield Either(
                         right=CreateContainerRequest(
                             name=name,
-                            parent=EntityReference(
-                                id=parent_container.id, type="container"
-                            )
-                            if parent_container
-                            else None,
+                            parent=(
+                                EntityReference(
+                                    id=parent_container.id, type="container"
+                                )
+                                if parent_container
+                                else None
+                            ),
                             service=self.storage_service.fullyQualifiedName,
                         )
                     )
@@ -1646,11 +1752,10 @@ class SampleDataSource(
             life_cycle_data.created = AccessDetails(
                 timestamp=Timestamp(
                     int(
-                        convert_timestamp_to_milliseconds(
-                            (
-                                datetime.now()
-                                - timedelta(days=life_cycle["created"]["days"])
-                            ).timestamp()
+                        datetime_to_timestamp(
+                            datetime_value=datetime.now()
+                            - timedelta(days=life_cycle["created"]["days"]),
+                            milliseconds=True,
                         )
                     )
                 ),
@@ -1660,11 +1765,10 @@ class SampleDataSource(
             life_cycle_data.updated = AccessDetails(
                 timestamp=Timestamp(
                     int(
-                        convert_timestamp_to_milliseconds(
-                            (
-                                datetime.now()
-                                - timedelta(days=life_cycle["updated"]["days"])
-                            ).timestamp()
+                        datetime_to_timestamp(
+                            datetime_value=datetime.now()
+                            - timedelta(days=life_cycle["updated"]["days"]),
+                            milliseconds=True,
                         )
                     ),
                 ),
@@ -1674,11 +1778,10 @@ class SampleDataSource(
             life_cycle_data.accessed = AccessDetails(
                 timestamp=Timestamp(
                     int(
-                        convert_timestamp_to_milliseconds(
-                            (
-                                datetime.now()
-                                - timedelta(days=life_cycle["accessed"]["days"])
-                            ).timestamp()
+                        datetime_to_timestamp(
+                            datetime_value=datetime.now()
+                            - timedelta(days=life_cycle["accessed"]["days"]),
+                            milliseconds=True,
                         )
                     ),
                 ),

metadata/ingestion/source/database/saperp/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.saperp.metadata import SaperpSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=SaperpSource)

metadata/ingestion/source/database/saphana/service_spec.py

@@ -0,0 +1,7 @@
+from metadata.ingestion.source.database.saphana.lineage import SaphanaLineageSource
+from metadata.ingestion.source.database.saphana.metadata import SaphanaSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=SaphanaSource, lineage_source_class=SaphanaLineageSource
+)

metadata/ingestion/source/database/sas/metadata.py

@@ -19,7 +19,7 @@ import json
 import re
 import traceback
 from datetime import datetime, timezone
-from typing import Any, Iterable, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple
 
 from requests.exceptions import HTTPError
 
@@ -28,7 +28,6 @@ from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequ
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
 )
-from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
@@ -881,9 +880,6 @@ class SasSource(
     ) -> Iterable[Either[OMetaTagAndClassification]]:
         """No tags to send"""
 
-    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
-        yield from []
-
     def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, list]]]:
         """Not implemented"""
 
@@ -900,11 +896,6 @@ class SasSource(
     ) -> Iterable[Either[CreateStoredProcedureRequest]]:
         """Not implemented"""
 
-    def yield_procedure_lineage_and_queries(
-        self,
-    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
-        yield from []
-
     def close(self) -> None:
         pass
 

metadata/ingestion/source/database/sas/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.database.sas.metadata import SasSource
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=SasSource)

metadata/ingestion/source/database/singlestore/lineage.py

@@ -0,0 +1,46 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Singlestore lineage module
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.services.connections.database.singleStoreConnection import (
+    SingleStoreConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.lineage_source import LineageSource
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class SinglestoreLineageSource(LineageSource):
+    """
+    Singlestore lineage source implements view lineage
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        """Create class instance"""
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: SingleStoreConnection = config.serviceConnection.root.config
+        if not isinstance(connection, SingleStoreConnection):
+            raise InvalidSourceException(
+                f"Expected SingleStoreConnection, but got {connection}"
+            )
+        return cls(config, metadata)

metadata/ingestion/source/database/singlestore/service_spec.py

@@ -0,0 +1,9 @@
+from metadata.ingestion.source.database.singlestore.metadata import SinglestoreSource
+from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface import (
+    SingleStoreProfilerInterface,
+)
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(
+    metadata_source_class=SinglestoreSource, profiler_class=SingleStoreProfilerInterface
+)

metadata/ingestion/source/database/snowflake/connection.py

@@ -48,6 +48,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_TEST_FETCH_TAG,
     SNOWFLAKE_TEST_GET_QUERIES,
     SNOWFLAKE_TEST_GET_TABLES,
+    SNOWFLAKE_TEST_GET_VIEWS,
 )
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
@@ -181,7 +182,11 @@ def test_connection(
             statement=SNOWFLAKE_TEST_GET_TABLES,
             engine_wrapper=engine_wrapper,
         ),
-        "GetViews": partial(execute_inspector_func, engine_wrapper, "get_view_names"),
+        "GetViews": partial(
+            test_table_query,
+            statement=SNOWFLAKE_TEST_GET_VIEWS,
+            engine_wrapper=engine_wrapper,
+        ),
         "GetQueries": partial(
             test_query, statement=SNOWFLAKE_TEST_GET_QUERIES, engine=engine
         ),

metadata/ingestion/source/database/snowflake/lineage.py

@@ -12,14 +12,26 @@
 Snowflake lineage module
 """
 
+from typing import Dict, List
+
 from metadata.ingestion.source.database.lineage_source import LineageSource
-from metadata.ingestion.source.database.snowflake.queries import SNOWFLAKE_SQL_STATEMENT
+from metadata.ingestion.source.database.snowflake.queries import (
+    SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES,
+    SNOWFLAKE_SQL_STATEMENT,
+)
 from metadata.ingestion.source.database.snowflake.query_parser import (
     SnowflakeQueryParserSource,
 )
+from metadata.ingestion.source.database.stored_procedures_mixin import (
+    QueryByProcedure,
+    StoredProcedureLineageMixin,
+)
+from metadata.utils.helpers import get_start_and_end
 
 
-class SnowflakeLineageSource(SnowflakeQueryParserSource, LineageSource):
+class SnowflakeLineageSource(
+    SnowflakeQueryParserSource, StoredProcedureLineageMixin, LineageSource
+):
     """
     Snowflake class for Lineage
     """
@@ -32,3 +44,20 @@ class SnowflakeLineageSource(SnowflakeQueryParserSource, LineageSource):
             OR (QUERY_TYPE = 'INSERT' and query_text ILIKE '%%insert%%into%%select%%')
         )
     """
+
+    stored_procedure_query = SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES
+
+    def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]]:
+        """
+        Return the dictionary associating stored procedures to the
+        queries they triggered
+        """
+        start, _ = get_start_and_end(self.source_config.queryLogDuration)
+        query = self.stored_procedure_query.format(
+            start_date=start,
+        )
+        queries_dict = self.procedure_queries_dict(
+            query=query,
+        )
+
+        return queries_dict

metadata/ingestion/source/database/snowflake/metadata.py

@@ -14,7 +14,7 @@ Snowflake source module
 import json
 import traceback
 from datetime import datetime
-from typing import Dict, Iterable, List, Optional, Tuple
+from typing import Iterable, List, Optional, Tuple
 
 import sqlparse
 from snowflake.sqlalchemy.custom_types import VARIANT
@@ -80,7 +80,6 @@ from metadata.ingestion.source.database.snowflake.queries import (
     SNOWFLAKE_GET_EXTERNAL_LOCATIONS,
     SNOWFLAKE_GET_ORGANIZATION_NAME,
     SNOWFLAKE_GET_SCHEMA_COMMENTS,
-    SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES,
     SNOWFLAKE_GET_STORED_PROCEDURES,
     SNOWFLAKE_LIFE_CYCLE_QUERY,
     SNOWFLAKE_SESSION_TAG_QUERY,
@@ -102,13 +101,8 @@ from metadata.ingestion.source.database.snowflake.utils import (
     get_view_names_reflection,
     normalize_names,
 )
-from metadata.ingestion.source.database.stored_procedures_mixin import (
-    QueryByProcedure,
-    StoredProcedureMixin,
-)
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_database
-from metadata.utils.helpers import get_start_and_end
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import get_all_table_comments, get_all_table_ddls
 from metadata.utils.tag_utils import get_ometa_tag_and_classification
@@ -145,7 +139,6 @@ SnowflakeDialect._get_schema_foreign_keys = get_schema_foreign_keys
 
 
 class SnowflakeSource(
-    StoredProcedureMixin,
     ExternalTableLineageMixin,
     CommonDbSourceService,
     MultiDBSource,
@@ -712,22 +705,6 @@ class SnowflakeSource(
                 )
             )
 
-    def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]]:
-        """
-        Return the dictionary associating stored procedures to the
-        queries they triggered
-        """
-        start, _ = get_start_and_end(self.source_config.queryLogDuration)
-        query = SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES.format(
-            start_date=start,
-        )
-
-        queries_dict = self.procedure_queries_dict(
-            query=query,
-        )
-
-        return queries_dict
-
     def mark_tables_as_deleted(self):
         """
         Use the current inspector to mark tables as deleted

metadata/ingestion/source/database/snowflake/models.py

@@ -15,11 +15,19 @@ import urllib
 from datetime import datetime
 from typing import List, Optional
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, TypeAdapter, field_validator
 from requests.utils import quote
+from sqlalchemy import text
+from sqlalchemy.orm import Session
 
 from metadata.generated.schema.entity.data.storedProcedure import Language
+from metadata.ingestion.source.database.snowflake.queries import (
+    SNOWFLAKE_QUERY_LOG_QUERY,
+)
+from metadata.profiler.metrics.system.dml_operation import DatabaseDMLOperations
+from metadata.utils.dict import ExtendedDict
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.profiler_utils import QueryResult
 
 logger = ingestion_logger()
 
@@ -95,3 +103,44 @@ class SnowflakeTableList(BaseModel):
 
     def get_not_deleted(self) -> List[SnowflakeTable]:
         return [table for table in self.tables if not table.deleted]
+
+
+class SnowflakeQueryLogEntry(BaseModel):
+    """Entry for a Snowflake query log at SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
+    More info at: https://docs.snowflake.com/en/sql-reference/account-usage/query_history
+    """
+
+    query_id: str
+    database_name: Optional[str] = None
+    schema_name: Optional[str] = None
+    query_type: str
+    start_time: datetime
+    query_text: Optional[str] = None
+    rows_inserted: Optional[int] = None
+    rows_updated: Optional[int] = None
+    rows_deleted: Optional[int] = None
+
+    @staticmethod
+    def get_for_table(session: Session, tablename: str):
+        rows = session.execute(
+            text(
+                SNOWFLAKE_QUERY_LOG_QUERY.format(
+                    tablename=tablename,  # type: ignore
+                    insert=DatabaseDMLOperations.INSERT.value,
+                    update=DatabaseDMLOperations.UPDATE.value,
+                    delete=DatabaseDMLOperations.DELETE.value,
+                    merge=DatabaseDMLOperations.MERGE.value,
+                )
+            )
+        )
+        return TypeAdapter(List[SnowflakeQueryLogEntry]).validate_python(
+            [ExtendedDict(r).lower_case_keys() for r in rows]
+        )
+
+
+class SnowflakeQueryResult(QueryResult):
+    """Snowflake system metric query result"""
+
+    rows_inserted: Optional[int] = None
+    rows_updated: Optional[int] = None
+    rows_deleted: Optional[int] = None

metadata/ingestion/source/database/snowflake/profiler/profiler.py

@@ -0,0 +1,26 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Profiler for Snowflake
+"""
+from metadata.ingestion.source.database.snowflake.profiler.system import (
+    SnowflakeSystemMetricsComputer,
+)
+from metadata.profiler.interface.sqlalchemy.snowflake.profiler_interface import (
+    SnowflakeProfilerInterface,
+)
+from metadata.profiler.metrics.system.system import SystemMetricsComputer
+
+
+class SnowflakeProfiler(SnowflakeProfilerInterface):
+    def initialize_system_metrics_computer(self) -> SystemMetricsComputer:
+        return SnowflakeSystemMetricsComputer(session=self.session)