openmetadata-ingestion 1.3.3.0rc2__py3-none-any.whl → 1.4.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/life_cycle_query_mixin.py

@@ -15,7 +15,7 @@ import traceback
 from collections import defaultdict
 from datetime import datetime
 from functools import lru_cache
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional, Type
 
 from pydantic import BaseModel, Field
 from sqlalchemy.engine import Engine
@@ -31,7 +31,7 @@ from metadata.generated.schema.type.lifeCycle import AccessDetails, LifeCycle
 from metadata.ingestion.api.models import Either, Entity
 from metadata.ingestion.api.status import Status
 from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
-from metadata.ingestion.models.topology import TopologyContext
+from metadata.ingestion.models.topology import TopologyContextManager
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
@@ -57,7 +57,7 @@ class LifeCycleQueryMixin:
     Module stores the methods to query the sources and get life cycle information
     """
 
-    context: TopologyContext
+    context: TopologyContextManager
     status: Status
     source_config: DatabaseServiceMetadataPipeline
     engine: Engine
@@ -92,34 +92,35 @@ class LifeCycleQueryMixin:
 
         return queries_dict
 
-    def get_life_cycle_data(self, entity: Entity, query: str):
+    def get_life_cycle_data(
+        self, entity: Type[Entity], entity_name: str, entity_fqn: str, query: str
+    ):
         """
         Get the life cycle data
         """
-        if entity:
-            try:
-                life_cycle_data = self.life_cycle_query_dict(query=query).get(
-                    entity.name.__root__
-                )
-                if life_cycle_data:
-                    life_cycle = LifeCycle(
-                        created=AccessDetails(
-                            timestamp=convert_timestamp_to_milliseconds(
-                                life_cycle_data.created_at.timestamp()
-                            )
+        try:
+            life_cycle_data = self.life_cycle_query_dict(query=query).get(entity_name)
+            if life_cycle_data:
+                life_cycle = LifeCycle(
+                    created=AccessDetails(
+                        timestamp=convert_timestamp_to_milliseconds(
+                            life_cycle_data.created_at.timestamp()
                         )
                     )
-                    yield Either(
-                        right=OMetaLifeCycleData(entity=entity, life_cycle=life_cycle)
-                    )
-            except Exception as exc:
+                )
                 yield Either(
-                    left=StackTraceError(
-                        name=entity.name.__root__,
-                        error=f"Unable to get the table life cycle data for table {entity.name.__root__}: {exc}",
-                        stackTrace=traceback.format_exc(),
+                    right=OMetaLifeCycleData(
+                        entity=entity, entity_fqn=entity_fqn, life_cycle=life_cycle
                     )
                 )
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=entity_name,
+                    error=f"Unable to get the table life cycle data for table {entity_name}: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
 
     def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]:
         """
@@ -129,21 +130,23 @@ class LifeCycleQueryMixin:
             table_fqn = fqn.build(
                 self.metadata,
                 entity_type=Table,
-                service_name=self.context.database_service,
-                database_name=self.context.database,
-                schema_name=self.context.database_schema,
-                table_name=self.context.table,
+                service_name=self.context.get().database_service,
+                database_name=self.context.get().database,
+                schema_name=self.context.get().database_schema,
+                table_name=self.context.get().table,
                 skip_es_search=True,
             )
-            table = self.metadata.get_by_name(entity=Table, fqn=table_fqn)
-            if table:
-                yield from self.get_life_cycle_data(
-                    entity=table,
-                    query=self.life_cycle_query.format(
-                        database_name=table.database.name,
-                        schema_name=table.databaseSchema.name,
-                    ),
-                )
+            # table = self.metadata.get_by_name(entity=Table, fqn=table_fqn)
+            # if table:
+            yield from self.get_life_cycle_data(
+                entity=Table,
+                entity_name=self.context.get().table,
+                entity_fqn=table_fqn,
+                query=self.life_cycle_query.format(
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
+                ),
+            )
         except Exception as exc:
             yield Either(
                 left=StackTraceError(

metadata/ingestion/source/database/mariadb/metadata.py

@@ -11,6 +11,8 @@
 """
 MariaDB source module
 """
+from typing import Optional
+
 from sqlalchemy.dialects.mysql.base import ischema_names
 from sqlalchemy.dialects.mysql.reflection import MySQLTableDefinitionParser
 
@@ -40,7 +42,9 @@ class MariadbSource(CommonDbSourceService):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: MariaDBConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, MariaDBConnection):

metadata/ingestion/source/database/mongodb/metadata.py

@@ -13,7 +13,7 @@ MongoDB source methods.
 """
 
 import traceback
-from typing import Dict, List, Union
+from typing import Dict, List, Optional, Union
 
 from pymongo.errors import OperationFailure
 
@@ -45,7 +45,9 @@ class MongodbSource(CommonNoSQLSource):
         self.mongodb = self.connection_obj
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: MongoDBConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, MongoDBConnection):

metadata/ingestion/source/database/mssql/metadata.py

@@ -95,7 +95,9 @@ class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         """Create class instance"""
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: MssqlConnection = config.serviceConnection.__root__.config
@@ -123,7 +125,7 @@ class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
                 database_fqn = fqn.build(
                     self.metadata,
                     entity_type=Database,
-                    service_name=self.context.database_service,
+                    service_name=self.context.get().database_service,
                     database_name=new_database,
                 )
 
@@ -150,8 +152,8 @@ class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
         if self.source_config.includeStoredProcedures:
             results = self.engine.execute(
                 MSSQL_GET_STORED_PROCEDURES.format(
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                 )
             ).all()
             for row in results:
@@ -184,9 +186,9 @@ class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                 ),
             )
             yield Either(right=stored_procedure_request)

metadata/ingestion/source/database/mssql/queries.py

@@ -46,27 +46,27 @@ MSSQL_GET_TABLE_COMMENTS = textwrap.dedent(
 SELECT obj.name AS table_name,
         ep.value AS table_comment,
         s.name AS "schema"
-FROM sys.tables AS obj
+FROM sys.objects AS obj
 LEFT JOIN sys.extended_properties AS ep
-    ON obj.object_id = ep.major_id AND ep.minor_id = 0
+    ON obj.object_id = ep.major_id AND ep.minor_id = 0 AND ep.name = 'MS_Description'
 JOIN sys.schemas AS s
     ON obj.schema_id = s.schema_id
-WHERE ep.name = 'MS_Description'
+WHERE
+    obj.type IN ('U', 'V') /* User tables and views */
 """
 )
 
 MSSQL_ALL_VIEW_DEFINITIONS = textwrap.dedent(
     """
-select
-	definition view_def,
-	views.name view_name,
-	sch.name "schema"
-from sys.sql_modules as mod,
-sys.views as views,
-sys.schemas as sch
- where
-mod.object_id=views.object_id and
-views.schema_id=sch.schema_id
+SELECT
+    definition view_def,
+    views.name view_name,
+    sch.name "schema"
+FROM sys.sql_modules as mod
+INNER JOIN sys.views as views
+    ON mod.object_id = views.object_id
+INNER JOIN sys.schemas as sch
+    ON views.schema_id = sch.schema_id
 """
 )
 
@@ -191,10 +191,12 @@ MSSQL_GET_STORED_PROCEDURES = textwrap.dedent(
     """
 SELECT
   ROUTINE_NAME AS name,
-  NULL AS owner,
+  NULL AS owner,            
   ROUTINE_BODY AS language,
-  ROUTINE_DEFINITION AS definition
-FROM INFORMATION_SCHEMA.ROUTINES
+  l.definition AS definition
+FROM INFORMATION_SCHEMA.ROUTINES r
+JOIN sys.procedures p ON p.name = r.ROUTINE_NAME 
+JOIN sys.sql_modules l on l.object_id = p.object_id
  WHERE ROUTINE_TYPE = 'PROCEDURE'
    AND ROUTINE_CATALOG = '{database_name}'
    AND ROUTINE_SCHEMA = '{schema_name}' 

metadata/ingestion/source/database/mssql/query_parser.py

@@ -12,6 +12,7 @@
 MSSQL usage module
 """
 from abc import ABC
+from typing import Optional
 
 from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
     MssqlConnection,
@@ -32,7 +33,9 @@ class MssqlQueryParserSource(QueryParserSource, ABC):
     filters: str
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         """Create class instance"""
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: MssqlConnection = config.serviceConnection.__root__.config

metadata/ingestion/source/database/mssql/utils.py

@@ -124,6 +124,7 @@ def get_columns(
         Column("minor_id", Integer, primary_key=True),
         Column("name", String, primary_key=True),
         Column("value", String),
+        Column("class_desc", String),
         schema="sys",
     )
     sys_columns = alias(
@@ -181,6 +182,8 @@ def get_columns(
             onclause=sql.and_(
                 extended_properties.c.major_id == sys_columns.c.object_id,
                 extended_properties.c.minor_id == sys_columns.c.column_id,
+                extended_properties.c.class_desc == "OBJECT_OR_COLUMN",
+                extended_properties.c.name == "MS_Description",
             ),
             isouter=True,
         )

metadata/ingestion/source/database/mysql/connection.py

@@ -30,7 +30,6 @@ from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
     get_connection_url_common,
-    init_empty_connection_options,
 )
 from metadata.ingestion.connections.test_connections import (
     test_connection_db_schema_sources,
@@ -52,16 +51,6 @@ def get_connection(connection: MysqlConnection) -> Engine:
             *connection.authType.azureConfig.scopes.split(",")
         )
         connection.authType = BasicAuth(password=access_token_obj.token)
-    if connection.sslCA or connection.sslCert or connection.sslKey:
-        if not connection.connectionOptions:
-            connection.connectionOptions = init_empty_connection_options()
-        if connection.sslCA:
-            connection.connectionOptions.__root__["ssl_ca"] = connection.sslCA
-        if connection.sslCert:
-            connection.connectionOptions.__root__["ssl_cert"] = connection.sslCert
-        if connection.sslKey:
-            connection.connectionOptions.__root__["ssl_key"] = connection.sslKey
-
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url_common,

metadata/ingestion/source/database/mysql/metadata.py

@@ -9,7 +9,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """Mysql source module"""
-from typing import cast
+from typing import Optional, cast
 
 from sqlalchemy.dialects.mysql.base import ischema_names
 from sqlalchemy.dialects.mysql.reflection import MySQLTableDefinitionParser
@@ -40,7 +40,9 @@ class MysqlSource(CommonDbSourceService):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection = cast(MysqlConnection, config.serviceConnection.__root__.config)
         if not isinstance(connection, MysqlConnection):

metadata/ingestion/source/database/oracle/connection.py

@@ -38,7 +38,7 @@ from metadata.ingestion.connections.builders import (
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.database.oracle.queries import CHECK_ACCESS_TO_DBA
+from metadata.ingestion.source.database.oracle.queries import CHECK_ACCESS_TO_ALL
 from metadata.utils.logger import ingestion_logger
 
 CX_ORACLE_LIB_VERSION = "8.3.0"
@@ -138,7 +138,7 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    test_conn_queries = {"CheckAccess": CHECK_ACCESS_TO_DBA}
+    test_conn_queries = {"CheckAccess": CHECK_ACCESS_TO_ALL}
 
     test_connection_db_common(
         metadata=metadata,

metadata/ingestion/source/database/oracle/metadata.py

@@ -104,7 +104,9 @@ class OracleSource(StoredProcedureMixin, CommonDbSourceService):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config = WorkflowSource.parse_obj(config_dict)
         connection: OracleConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, OracleConnection):
@@ -179,7 +181,7 @@ class OracleSource(StoredProcedureMixin, CommonDbSourceService):
         if self.source_config.includeStoredProcedures:
             results: FetchProcedureList = self.engine.execute(
                 ORACLE_GET_STORED_PROCEDURES.format(
-                    schema=self.context.database_schema.upper()
+                    schema=self.context.get().database_schema.upper()
                 )
             ).all()
             results = self.process_result(data=results)
@@ -207,9 +209,9 @@ class OracleSource(StoredProcedureMixin, CommonDbSourceService):
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                 ),
             )
             yield Either(right=stored_procedure_request)

metadata/ingestion/source/database/oracle/queries.py

@@ -20,7 +20,7 @@ SELECT
 	comments table_comment,
 	LOWER(table_name) "table_name",
 	LOWER(owner) "schema" 	
-FROM dba_tab_comments
+FROM ALL_TAB_COMMENTS
 where comments is not null and owner not in ('SYSTEM', 'SYS')
 """
 )
@@ -32,33 +32,33 @@ SELECT
 LOWER(view_name) AS "view_name",
 LOWER(owner) AS "schema",
 DBMS_METADATA.GET_DDL('VIEW', view_name, owner) AS view_def
-FROM DBA_VIEWS
+FROM ALL_VIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 UNION ALL
 SELECT
 LOWER(mview_name) AS "view_name",
 LOWER(owner) AS "schema",
 DBMS_METADATA.GET_DDL('MATERIALIZED_VIEW', mview_name, owner) AS view_def
-FROM DBA_MVIEWS
+FROM ALL_MVIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 """
 )
 
 GET_MATERIALIZED_VIEW_NAMES = textwrap.dedent(
     """
-SELECT mview_name FROM DBA_MVIEWS WHERE owner = :owner
+SELECT mview_name FROM ALL_MVIEWS WHERE owner = :owner
 """
 )
 
 ORACLE_GET_TABLE_NAMES = textwrap.dedent(
     """
-SELECT table_name FROM DBA_TABLES WHERE 
+SELECT table_name FROM ALL_TABLES WHERE 
 {tablespace}
 OWNER = :owner  
 AND IOT_NAME IS NULL 
 AND DURATION IS NULL
 AND TABLE_NAME NOT IN 
-(SELECT mview_name FROM DBA_MVIEWS WHERE owner = :owner)
+(SELECT mview_name FROM ALL_MVIEWS WHERE owner = :owner)
 """
 )
 
@@ -67,7 +67,7 @@ ORACLE_IDENTITY_TYPE = textwrap.dedent(
 col.default_on_null,
 (
 	SELECT id.generation_type || ',' || id.IDENTITY_OPTIONS
-	FROM DBA_TAB_IDENTITY_COLS{dblink} id
+	FROM ALL_TAB_IDENTITY_COLS{dblink} id
 	WHERE col.table_name = id.table_name
 	AND col.column_name = id.column_name
 	AND col.owner = id.owner
@@ -83,12 +83,12 @@ SELECT
     LINE,
     TEXT
 FROM
-    DBA_SOURCE
+    ALL_SOURCE
 WHERE
     type = 'PROCEDURE' and owner = '{schema}'
 """
 )
-CHECK_ACCESS_TO_DBA = "SELECT table_name FROM DBA_TABLES where ROWNUM < 2"
+CHECK_ACCESS_TO_ALL = "SELECT table_name FROM ALL_TABLES where ROWNUM < 2"
 ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
     """
 WITH SP_HISTORY AS (SELECT
@@ -97,7 +97,7 @@ WITH SP_HISTORY AS (SELECT
     TO_TIMESTAMP(LAST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') + NUMTODSINTERVAL(ELAPSED_TIME / 1000, 'SECOND') AS end_time,
     PARSING_SCHEMA_NAME as user_name
   FROM gv$sql
-  WHERE sql_text LIKE 'CALL%%'
+  WHERE UPPER(sql_text) LIKE 'CALL%%'
   AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') >= TO_TIMESTAMP('{start_date}', 'YYYY-MM-DD HH24:MI:SS')
  ),
  Q_HISTORY AS (SELECT
@@ -114,7 +114,7 @@ WITH SP_HISTORY AS (SELECT
       PARSING_SCHEMA_NAME AS SCHEMA_NAME,
       NULL AS DATABASE_NAME
     FROM gv$sql
-    WHERE sql_text NOT LIKE '%CALL%'
+    WHERE UPPER(sql_text) NOT LIKE '%CALL%'
       AND SQL_FULLTEXT NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
       AND SQL_FULLTEXT NOT LIKE '/* {{"app": "dbt", %%}} */%%'
       AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') 
@@ -152,8 +152,8 @@ ORACLE_GET_COLUMNS = textwrap.dedent(
             com.comments,
             col.virtual_column,
             {identity_cols}
-        FROM DBA_TAB_COLS{dblink} col
-        LEFT JOIN all_col_comments{dblink} com
+        FROM ALL_TAB_COLS{dblink} col
+        LEFT JOIN ALL_COL_COMMENTS{dblink} com
         ON col.table_name = com.table_name
         AND col.column_name = com.column_name
         AND col.owner = com.owner

metadata/ingestion/source/database/oracle/query_parser.py

@@ -12,6 +12,7 @@
 Oracle query parsing module
 """
 from abc import ABC
+from typing import Optional
 
 from metadata.generated.schema.entity.services.connections.database.oracleConnection import (
     OracleConnection,
@@ -35,7 +36,9 @@ class OracleQueryParserSource(QueryParserSource, ABC):
     filters: str
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: OracleConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, OracleConnection):

metadata/ingestion/source/database/pinotdb/metadata.py

@@ -9,6 +9,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """PinotDb source module"""
+from typing import Optional
 
 from metadata.generated.schema.entity.services.connections.database.pinotDBConnection import (
     PinotDBConnection,
@@ -28,7 +29,9 @@ class PinotdbSource(CommonDbSourceService):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: PinotDBConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, PinotDBConnection):

metadata/ingestion/source/database/postgres/connection.py

@@ -26,13 +26,11 @@ from metadata.generated.schema.entity.services.connections.database.common.basic
 )
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
-    SslMode,
 )
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
     get_connection_url_common,
-    init_empty_connection_arguments,
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -61,14 +59,6 @@ def get_connection(connection: PostgresConnection) -> Engine:
             *connection.authType.azureConfig.scopes.split(",")
         )
         connection.authType = BasicAuth(password=access_token_obj.token)
-    if connection.sslMode:
-        if not connection.connectionArguments:
-            connection.connectionArguments = init_empty_connection_arguments()
-        connection.connectionArguments.__root__["sslmode"] = connection.sslMode.value
-        if connection.sslMode in (SslMode.verify_ca, SslMode.verify_full):
-            connection.connectionArguments.__root__[
-                "sslrootcert"
-            ] = connection.sslConfig.__root__.certificatePath
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url_common,