openmetadata-ingestion 1.3.3.0rc2__py3-none-any.whl → 1.4.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/postgres/metadata.py

@@ -15,13 +15,15 @@ import traceback
 from collections import namedtuple
 from typing import Iterable, Optional, Tuple
 
+from sqlalchemy import String as SqlAlchemyString
 from sqlalchemy import sql
 from sqlalchemy.dialects.postgresql.base import PGDialect, ischema_names
 from sqlalchemy.engine import Inspector
 
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.table import (
-    IntervalType,
+    PartitionColumnDetails,
+    PartitionIntervalTypes,
     TablePartition,
     TableType,
 )
@@ -54,6 +56,7 @@ from metadata.ingestion.source.database.postgres.utils import (
     get_column_info,
     get_columns,
     get_etable_owner,
+    get_foreign_keys,
     get_table_comment,
     get_table_owner,
     get_view_definition,
@@ -74,9 +77,9 @@ logger = ingestion_logger()
 
 
 INTERVAL_TYPE_MAP = {
-    "list": IntervalType.COLUMN_VALUE.value,
-    "hash": IntervalType.COLUMN_VALUE.value,
-    "range": IntervalType.TIME_UNIT.value,
+    "list": PartitionIntervalTypes.COLUMN_VALUE,
+    "hash": PartitionIntervalTypes.COLUMN_VALUE,
+    "range": PartitionIntervalTypes.TIME_UNIT,
 }
 
 RELKIND_MAP = {
@@ -95,6 +98,7 @@ ischema_names.update(
         "point": POINT,
         "polygon": POLYGON,
         "box": create_sqlalchemy_type("BOX"),
+        "bpchar": SqlAlchemyString,
         "circle": create_sqlalchemy_type("CIRCLE"),
         "line": create_sqlalchemy_type("LINE"),
         "lseg": create_sqlalchemy_type("LSEG"),
@@ -103,6 +107,7 @@ ischema_names.update(
         "pg_snapshot": create_sqlalchemy_type("PG_SNAPSHOT"),
         "tsquery": create_sqlalchemy_type("TSQUERY"),
         "txid_snapshot": create_sqlalchemy_type("TXID_SNAPSHOT"),
+        "xid": SqlAlchemyString,
         "xml": create_sqlalchemy_type("XML"),
     }
 )
@@ -121,6 +126,8 @@ PGDialect.ischema_names = ischema_names
 
 Inspector.get_table_owner = get_etable_owner
 
+PGDialect.get_foreign_keys = get_foreign_keys
+
 
 class PostgresSource(CommonDbSourceService, MultiDBSource):
     """
@@ -129,7 +136,9 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: PostgresConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, PostgresConnection):
@@ -175,15 +184,17 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
                 database_fqn = fqn.build(
                     self.metadata,
                     entity_type=Database,
-                    service_name=self.context.database_service,
+                    service_name=self.context.get().database_service,
                     database_name=new_database,
                 )
 
                 if filter_by_database(
                     self.source_config.databaseFilterPattern,
-                    database_fqn
-                    if self.source_config.useFqnForFiltering
-                    else new_database,
+                    (
+                        database_fqn
+                        if self.source_config.useFqnForFiltering
+                        else new_database
+                    ),
                 ):
                     self.status.filter(database_fqn, "Database Filtered Out")
                     continue
@@ -203,12 +214,20 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
         result = self.engine.execute(
             POSTGRES_PARTITION_DETAILS, table_name=table_name, schema_name=schema_name
         ).all()
+
         if result:
             partition_details = TablePartition(
-                intervalType=INTERVAL_TYPE_MAP.get(
-                    result[0].partition_strategy, IntervalType.COLUMN_VALUE.value
-                ),
-                columns=[row.column_name for row in result if row.column_name],
+                columns=[
+                    PartitionColumnDetails(
+                        columnName=row.column_name,
+                        intervalType=INTERVAL_TYPE_MAP.get(
+                            row.partition_strategy, PartitionIntervalTypes.COLUMN_VALUE
+                        ),
+                        interval=None,
+                    )
+                    for row in result
+                    if row.column_name
+                ]
             )
             return True, partition_details
         return False, None
@@ -222,7 +241,7 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
         try:
             result = self.engine.execute(
                 POSTGRES_GET_ALL_TABLE_PG_POLICY.format(
-                    database_name=self.context.database,
+                    database_name=self.context.get().database,
                     schema_name=schema_name,
                 )
             ).all()
@@ -231,7 +250,7 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
                 fqn_elements = [name for name in row[2:] if name]
                 yield from get_ometa_tag_and_classification(
                     tag_fqn=fqn._build(  # pylint: disable=protected-access
-                        self.context.database_service, *fqn_elements
+                        self.context.get().database_service, *fqn_elements
                     ),
                     tags=[row[1]],
                     classification_name=self.service_connection.classificationName,

metadata/ingestion/source/database/postgres/queries.py

@@ -76,7 +76,7 @@ POSTGRES_PARTITION_DETAILS = textwrap.dedent(
 )
 
 POSTGRES_GET_ALL_TABLE_PG_POLICY = """
-SELECT object_id, polname, table_catalog , table_schema ,table_name  
+SELECT object_id, polname, table_catalog, table_schema, table_name  
 FROM information_schema.tables AS it
 JOIN (SELECT pc.oid as object_id, pc.relname, pp.*
       FROM pg_policy AS pp
@@ -188,3 +188,19 @@ POSTGRES_SQL_COLUMNS = """
 POSTGRES_GET_SERVER_VERSION = """
 show server_version
 """
+
+POSTGRES_FETCH_FK = """
+    SELECT r.conname,
+        pg_catalog.pg_get_constraintdef(r.oid, true) as condef,
+        n.nspname as conschema,
+        d.datname AS con_db_name
+    FROM  pg_catalog.pg_constraint r,
+        pg_namespace n,
+        pg_class c
+    JOIN pg_database d ON d.datname = current_database()
+    WHERE r.conrelid = :table AND
+        r.contype = 'f' AND
+        c.oid = confrelid AND
+        n.oid = c.relnamespace
+    ORDER BY 1
+"""

metadata/ingestion/source/database/postgres/query_parser.py

@@ -13,7 +13,7 @@ Postgres Query parser module
 """
 import traceback
 from abc import ABC
-from typing import Iterable
+from typing import Iterable, Optional
 
 from sqlalchemy.engine.base import Engine
 
@@ -53,7 +53,9 @@ class PostgresQueryParserSource(QueryParserSource, ABC):
         self.start, self.end = get_start_and_end(duration)
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: PostgresConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, PostgresConnection):

metadata/ingestion/source/database/postgres/utils.py

@@ -25,6 +25,7 @@ from sqlalchemy.sql import sqltypes
 
 from metadata.ingestion.source.database.postgres.queries import (
     POSTGRES_COL_IDENTITY,
+    POSTGRES_FETCH_FK,
     POSTGRES_GET_SERVER_VERSION,
     POSTGRES_SQL_COLUMNS,
     POSTGRES_TABLE_COMMENTS,
@@ -62,6 +63,103 @@ def get_etable_owner(
         )
 
 
+@reflection.cache
+def get_foreign_keys(
+    self, connection, table_name, schema=None, postgresql_ignore_search_path=False, **kw
+):
+    preparer = self.identifier_preparer
+    table_oid = self.get_table_oid(
+        connection, table_name, schema, info_cache=kw.get("info_cache")
+    )
+
+    # https://www.postgresql.org/docs/9.0/static/sql-createtable.html
+    FK_REGEX = re.compile(
+        r"FOREIGN KEY \((.*?)\) REFERENCES (?:(.*?)\.)?(.*?)\((.*?)\)"
+        r"[\s]?(MATCH (FULL|PARTIAL|SIMPLE)+)?"
+        r"[\s]?(ON UPDATE "
+        r"(CASCADE|RESTRICT|NO ACTION|SET NULL|SET DEFAULT)+)?"
+        r"[\s]?(ON DELETE "
+        r"(CASCADE|RESTRICT|NO ACTION|SET NULL|SET DEFAULT)+)?"
+        r"[\s]?(DEFERRABLE|NOT DEFERRABLE)?"
+        r"[\s]?(INITIALLY (DEFERRED|IMMEDIATE)+)?"
+    )
+
+    t = sql.text(POSTGRES_FETCH_FK).columns(
+        conname=sqltypes.Unicode, condef=sqltypes.Unicode, con_db_name=sqltypes.Unicode
+    )
+    c = connection.execute(t, dict(table=table_oid))
+    fkeys = []
+    for conname, condef, conschema, con_db_name in c.fetchall():
+        m = re.search(FK_REGEX, condef).groups()
+
+        (
+            constrained_columns,
+            referred_schema,
+            referred_table,
+            referred_columns,
+            _,
+            match,
+            _,
+            onupdate,
+            _,
+            ondelete,
+            deferrable,
+            _,
+            initially,
+        ) = m
+
+        if deferrable is not None:
+            deferrable = True if deferrable == "DEFERRABLE" else False
+        constrained_columns = tuple(re.split(r"\s*,\s*", constrained_columns))
+        constrained_columns = [
+            preparer._unquote_identifier(x) for x in constrained_columns
+        ]
+
+        if postgresql_ignore_search_path:
+            # when ignoring search path, we use the actual schema
+            # provided it isn't the "default" schema
+            if conschema != self.default_schema_name:
+                referred_schema = conschema
+            else:
+                referred_schema = schema
+        elif referred_schema:
+            # referred_schema is the schema that we regexp'ed from
+            # pg_get_constraintdef().  If the schema is in the search
+            # path, pg_get_constraintdef() will give us None.
+            referred_schema = preparer._unquote_identifier(referred_schema)
+        elif schema is not None and schema == conschema:
+            # If the actual schema matches the schema of the table
+            # we're reflecting, then we will use that.
+            referred_schema = schema
+
+        referred_table = preparer._unquote_identifier(referred_table)
+        referred_columns = tuple(re.split(r"\s*,\s", referred_columns))
+        referred_columns = [preparer._unquote_identifier(x) for x in referred_columns]
+        options = {
+            k: v
+            for k, v in [
+                ("onupdate", onupdate),
+                ("ondelete", ondelete),
+                ("initially", initially),
+                ("deferrable", deferrable),
+                ("match", match),
+            ]
+            if v is not None and v != "NO ACTION"
+        }
+        referred_database = con_db_name if con_db_name else ""
+        fkey_d = {
+            "name": conname,
+            "constrained_columns": constrained_columns,
+            "referred_schema": referred_schema,
+            "referred_table": referred_table,
+            "referred_columns": referred_columns,
+            "options": options,
+            "referred_database": referred_database,
+        }
+        fkeys.append(fkey_d)
+    return fkeys
+
+
 @reflection.cache
 def get_table_owner(
     self, connection, table_name, schema=None, **kw

metadata/ingestion/source/database/presto/connection.py

@@ -28,6 +28,7 @@ from metadata.generated.schema.entity.services.connections.database.prestoConnec
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
+    init_empty_connection_arguments,
 )
 from metadata.ingestion.connections.test_connections import (
     execute_inspector_func,
@@ -58,6 +59,19 @@ def get_connection(connection: PrestoConnection) -> Engine:
     """
     Create connection
     """
+    connection.connectionArguments = (
+        connection.connectionArguments or init_empty_connection_arguments()
+    )
+    if connection.protocol:
+        connection.connectionArguments.__root__["protocol"] = connection.protocol
+    if connection.verify:
+        connection.connectionArguments = (
+            connection.connectionArguments or init_empty_connection_arguments()
+        )
+        connection.connectionArguments.__root__["requests_kwargs"] = {
+            "verify": connection.verify
+        }
+
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url,

metadata/ingestion/source/database/presto/metadata.py

@@ -15,10 +15,10 @@ Presto source module
 import re
 import traceback
 from copy import deepcopy
-from typing import Iterable
+from typing import Iterable, Optional
 
 from pyhive.sqlalchemy_presto import PrestoDialect, _type_map
-from sqlalchemy import inspect, types, util
+from sqlalchemy import types, util
 from sqlalchemy.engine import reflection
 
 from metadata.generated.schema.entity.data.database import Database
@@ -118,7 +118,9 @@ class PrestoSource(CommonDbSourceService):
     """
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config = WorkflowSource.parse_obj(config_dict)
         connection: PrestoConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, PrestoConnection):
@@ -138,7 +140,8 @@ class PrestoSource(CommonDbSourceService):
         new_service_connection = deepcopy(self.service_connection)
         new_service_connection.catalog = database_name
         self.engine = get_connection(new_service_connection)
-        self.inspector = inspect(self.engine)
+        self._connection_map = {}  # Lazy init as well
+        self._inspector_map = {}
 
     def get_database_names(self) -> Iterable[str]:
         configured_catalog = self.service_connection.catalog
@@ -153,7 +156,7 @@ class PrestoSource(CommonDbSourceService):
                     database_fqn = fqn.build(
                         self.metadata,
                         entity_type=Database,
-                        service_name=self.context.database_service,
+                        service_name=self.context.get().database_service,
                         database_name=new_catalog,
                     )
                     if filter_by_database(

metadata/ingestion/source/database/query/lineage.py

@@ -11,6 +11,8 @@
 """
 Common Query Log Connector
 """
+from typing import Optional
+
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
@@ -20,7 +22,9 @@ from metadata.ingestion.source.database.lineage_source import LineageSource
 
 class QueryLogLineageSource(LineageSource):
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         return cls(config, metadata)
 

metadata/ingestion/source/database/query/usage.py

@@ -12,6 +12,7 @@
 Common Query Log Connector
 """
 from datetime import datetime
+from typing import Optional
 
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
@@ -26,7 +27,9 @@ class QueryLogUsageSource(UsageSource):
         self.analysis_date = datetime.today().strftime("%Y-%m-%d %H:%M:%S")
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         return cls(config, metadata)
 

metadata/ingestion/source/database/redshift/connection.py

@@ -21,13 +21,11 @@ from metadata.generated.schema.entity.automations.workflow import (
 )
 from metadata.generated.schema.entity.services.connections.database.redshiftConnection import (
     RedshiftConnection,
-    SslMode,
 )
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
     get_connection_url_common,
-    init_empty_connection_arguments,
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -42,14 +40,6 @@ def get_connection(connection: RedshiftConnection) -> Engine:
     """
     Create connection
     """
-    if connection.sslMode:
-        if not connection.connectionArguments:
-            connection.connectionArguments = init_empty_connection_arguments()
-        connection.connectionArguments.__root__["sslmode"] = connection.sslMode.value
-        if connection.sslMode in (SslMode.verify_ca, SslMode.verify_full):
-            connection.connectionArguments.__root__[
-                "sslrootcert"
-            ] = connection.sslConfig.__root__.certificatePath
     return create_generic_db_connection(
         connection=connection,
         get_connection_url_fn=get_connection_url_common,

metadata/ingestion/source/database/redshift/incremental_table_processor.py

@@ -0,0 +1,192 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Incremental Processor for Redshift
+"""
+import re
+from datetime import datetime
+from typing import Iterable, List, Optional, Tuple
+
+from sqlalchemy.engine import Connection
+
+from metadata.ingestion.source.database.redshift.models import (
+    RedshiftTable,
+    RedshiftTableChangeQueryRegex,
+    RedshiftTableMap,
+    SchemaName,
+    TableName,
+)
+from metadata.ingestion.source.database.redshift.queries import (
+    REDSHIFT_TABLE_CHANGES_QUERY,
+)
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+# ---- Regex Definitions
+TABLE_NAME_RE = r"(\w+\.){0,2}\w+"
+
+CREATE_TABLE = rf"^.*CREATE\s+(LOCAL\s+|EXTERNAL\s+)?(TEMPORARY\s+|TEMP\s+)?TABLE\s+(IF\s+NOT\s+EXISTS\s+)?(?P<table>{TABLE_NAME_RE}).*$"
+
+ALTER_TABLE = rf"^.*(ALTER\s+TABLE)\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+DROP_TABLE = rf"^.*DROP\s+TABLE\s+(IF\s+EXISTS\s+)?(?P<table>{TABLE_NAME_RE}).*$"
+
+CREATE_VIEW = rf"^.*CREATE\s+(OR\s+REPLACE\s+)?(EXTERNAL\s+|MATERIALIZED\s+)?VIEW\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+ALTER_VIEW = rf"^.*ALTER\s+(EXTERNAL\s+)?VIEW\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+DROP_VIEW = rf"^.*DROP\s+(EXTERNAL\s+|MATERIALIZED\s+)?VIEW\s+(IF\s+EXISTS\s+)?(?P<table>{TABLE_NAME_RE}).*$"
+
+# Not supporting Comment changes on Constraint
+COMMENT = rf"^.*COMMENT\s+ON\s+(TABLE|COLUMN|VIEW)\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+
+REGEX_LIST = [
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(ALTER_TABLE, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(CREATE_TABLE, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(DROP_TABLE, re.IGNORECASE), deleted=True
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(ALTER_VIEW, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(CREATE_VIEW, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(DROP_VIEW, re.IGNORECASE), deleted=True
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(COMMENT, re.IGNORECASE), deleted=False
+    ),
+]
+
+
+class RedshiftIncrementalTableProcessor:
+    """Incremental Table Processor class.
+
+    Responsible for the Incremental Extraction flow."""
+
+    def __init__(
+        self,
+        table_map: RedshiftTableMap,
+        table_changes_query: str,
+        regex_list: List[RedshiftTableChangeQueryRegex],
+        connection: Connection,
+        default_schema: SchemaName,
+    ):
+        self.table_map = table_map
+        self.table_changes_query = table_changes_query
+        self.regex_list = regex_list
+        self.connection = connection
+        self.default_schema = default_schema
+
+    @classmethod
+    def create(
+        cls, connection: Connection, default_schema: SchemaName
+    ) -> "RedshiftIncrementalTableProcessor":
+        """Creates a new instance based on a connection and the default schema."""
+        return cls(
+            table_map=RedshiftTableMap.default(),
+            table_changes_query=REDSHIFT_TABLE_CHANGES_QUERY,
+            regex_list=REGEX_LIST,
+            connection=connection,
+            default_schema=default_schema,
+        )
+
+    def _query_for_changes(self, database: str, start_date: datetime) -> Iterable[str]:
+        """Queries the Redshift database for the Table Changes."""
+        for row in (
+            self.connection.execute(
+                self.table_changes_query.format(
+                    database=database, start_date=start_date
+                )
+            )
+            or []
+        ):
+            yield row[0]
+
+    def _clean_statement(self, statement: str) -> str:
+        """Gets rid of unwanted characters"""
+        return (
+            statement.replace("\n", " ")
+            .replace("\t", " ")
+            .replace("\v", " ")
+            .replace('"', "")
+        )  # Removes '"' to make the Regex Match easier.
+
+    def _get_schema_and_table(
+        self, full_table_name: str, statement: str
+    ) -> Tuple[SchemaName, TableName]:
+        """From the full table name, retrieves the Schema and Table Name.
+        If no Schema is present, falls back to the default schema."""
+        full_table_name_as_list = full_table_name.split(".")
+
+        # Check if Schema is present
+        if len(full_table_name_as_list) >= 2:
+            schema = full_table_name_as_list[-2]
+        else:
+            schema = self.default_schema
+            logger.debug(
+                "Schema not present for statement %s. TableName: %s",
+                statement,
+                full_table_name,
+            )
+            logger.debug("Falling back to the Default Schema: %s", self.default_schema)
+
+        table_name = full_table_name_as_list[-1]
+
+        return schema, table_name
+
+    def set_table_map(self, database: str, start_date: datetime):
+        """Sets the RedshiftTableMap for the given database, filtering by the given start_date."""
+        for statement in self._query_for_changes(database, start_date):
+            statement = self._clean_statement(statement)
+            match_found = False
+
+            for possible_match in self.regex_list:
+                match = possible_match.regex.match(statement)
+
+                if not match:
+                    continue
+
+                else:
+                    match_found = True
+                    schema, table_name = self._get_schema_and_table(
+                        match.group("table"), statement
+                    )
+                    deleted = possible_match.deleted
+
+                    self.table_map.update(
+                        schema, RedshiftTable(name=table_name, deleted=deleted)
+                    )
+
+                    break
+
+            if not match_found:
+                logger.debug("Match not found for %s", statement)
+
+    def get_deleted(
+        self, schema_name: Optional[SchemaName] = None
+    ) -> List[Tuple[SchemaName, TableName]]:
+        """Returns the deleted table names present in the table_map for a given schema."""
+        return self.table_map.get_deleted(schema_name)
+
+    def get_not_deleted(self, schema_name: SchemaName) -> List[TableName]:
+        """Returns the not deleted table names present in the table_map for a given schema."""
+        return self.table_map.get_not_deleted(schema_name)