openmetadata-ingestion 1.2.5.2__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/database/athena/metadata.py

@@ -11,32 +11,52 @@
 
 """Athena source module"""
 
-from typing import Iterable
+import traceback
+from typing import Iterable, Tuple
 
 from pyathena.sqlalchemy.base import AthenaDialect
 from sqlalchemy import types
 from sqlalchemy.engine import reflection
+from sqlalchemy.engine.reflection import Inspector
 
-from metadata.generated.schema.entity.data.table import TableType
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    IntervalType,
+    Table,
+    TablePartition,
+    TableType,
+)
 from metadata.generated.schema.entity.services.connections.database.athenaConnection import (
     AthenaConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source import sqa_types
+from metadata.ingestion.source.database.athena.client import AthenaLakeFormationClient
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.common_db_source import (
     CommonDbSourceService,
     TableNameAndType,
 )
+from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import is_complex_type
+from metadata.utils.tag_utils import get_ometa_tag_and_classification
 
 logger = ingestion_logger()
 
+ATHENA_TAG = "ATHENA TAG"
+ATHENA_TAG_CLASSIFICATION = "ATHENA TAG CLASSIFICATION"
+
 
 def _get_column_type(self, type_):
     """
@@ -73,20 +93,14 @@ def _get_column_type(self, type_):
         "struct": sqa_types.SQAStruct,
         "row": sqa_types.SQAStruct,
         "map": sqa_types.SQAMap,
+        "decimal": types.DECIMAL,
+        "varchar": types.VARCHAR,
+        "char": types.CHAR,
     }
-    if name in ["decimal"]:
-        col_type = types.DECIMAL
-        if length:
-            precision, scale = length.split(",")
-            args = [int(precision), int(scale)]
-    elif name in ["char"]:
-        col_type = types.CHAR
+    if name in ["decimal", "char", "varchar"]:
+        col_type = col_map[name]
         if length:
-            args = [int(length)]
-    elif name in ["varchar"]:
-        col_type = types.VARCHAR
-        if length:
-            args = [int(length)]
+            args = [int(l) for l in length.split(",")]
     elif type_.startswith("array"):
         parsed_type = (
             ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
@@ -94,7 +108,13 @@ def _get_column_type(self, type_):
             )
         )
         col_type = col_map["array"]
-        args = [col_map.get(parsed_type.get("arrayDataType").lower(), types.String)]
+        if parsed_type["arrayDataType"].lower().startswith("array"):
+            # as OpenMetadata doesn't store any details on children of array, we put
+            # in type as string as default to avoid Array item_type required issue
+            # from sqlalchemy types
+            args = [types.String]
+        else:
+            args = [col_map.get(parsed_type.get("arrayDataType").lower(), types.String)]
     elif col_map.get(name):
         col_type = col_map.get(name)
     else:
@@ -121,10 +141,14 @@ def get_columns(self, connection, table_name, schema=None, **kw):
             "comment": c.comment,
             "system_data_type": c.type,
             "is_complex": is_complex_type(c.type),
-            "dialect_options": {"awsathena_partition": None},
+            "dialect_options": {"awsathena_partition": True},
         }
-        for c in metadata.columns
+        for c in metadata.partition_keys
     ]
+
+    if kw.get("only_partition_columns"):
+        return columns
+
     columns += [
         {
             "name": c.name,
@@ -135,10 +159,11 @@ def get_columns(self, connection, table_name, schema=None, **kw):
             "comment": c.comment,
             "system_data_type": c.type,
             "is_complex": is_complex_type(c.type),
-            "dialect_options": {"awsathena_partition": True},
+            "dialect_options": {"awsathena_partition": None},
         }
-        for c in metadata.partition_keys
+        for c in metadata.columns
     ]
+
     return columns
 
 
@@ -176,6 +201,16 @@ class AthenaSource(CommonDbSourceService):
             )
         return cls(config, metadata)
 
+    def __init__(
+        self,
+        config: WorkflowSource,
+        metadata: OpenMetadata,
+    ):
+        super().__init__(config, metadata)
+        self.athena_lake_formation_client = AthenaLakeFormationClient(
+            connection=self.service_connection
+        )
+
     def query_table_names_and_types(
         self, schema_name: str
     ) -> Iterable[TableNameAndType]:
@@ -185,3 +220,110 @@ class AthenaSource(CommonDbSourceService):
             TableNameAndType(name=name, type_=TableType.External)
             for name in self.inspector.get_table_names(schema_name)
         ]
+
+    def get_table_partition_details(
+        self, table_name: str, schema_name: str, inspector: Inspector
+    ) -> Tuple[bool, TablePartition]:
+        columns = inspector.get_columns(
+            table_name=table_name, schema=schema_name, only_partition_columns=True
+        )
+        if columns:
+            partition_details = TablePartition(
+                intervalType=IntervalType.COLUMN_VALUE.value,
+                columns=[column["name"] for column in columns],
+            )
+            return True, partition_details
+        return False, None
+
+    def yield_tag(
+        self, schema_name: str
+    ) -> Iterable[Either[OMetaTagAndClassification]]:
+        """
+        Method to yield schema tags
+        """
+        if self.source_config.includeTags:
+            try:
+                tags = self.athena_lake_formation_client.get_database_tags(
+                    name=schema_name
+                )
+                for tag in tags or []:
+                    yield from get_ometa_tag_and_classification(
+                        tag_fqn=fqn.build(
+                            self.metadata,
+                            DatabaseSchema,
+                            service_name=self.context.database_service,
+                            database_name=self.context.database,
+                            schema_name=schema_name,
+                        ),
+                        tags=tag.TagValues,
+                        classification_name=tag.TagKey,
+                        tag_description=ATHENA_TAG,
+                        classification_description=ATHENA_TAG_CLASSIFICATION,
+                    )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name="Tags and Classifications",
+                        error=f"Failed to fetch database tags due to [{exc}]",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def yield_table_tags(
+        self, table_name_and_type: Tuple[str, TableType]
+    ) -> Iterable[Either[OMetaTagAndClassification]]:
+        """
+        Method to yield table and column tags
+        """
+        if self.source_config.includeTags:
+            try:
+                table_name, _ = table_name_and_type
+                table_tags = (
+                    self.athena_lake_formation_client.get_table_and_column_tags(
+                        schema_name=self.context.database_schema, table_name=table_name
+                    )
+                )
+
+                # yield the table tags
+                for tag in table_tags.LFTagsOnTable or []:
+                    yield from get_ometa_tag_and_classification(
+                        tag_fqn=fqn.build(
+                            self.metadata,
+                            Table,
+                            service_name=self.context.database_service,
+                            database_name=self.context.database,
+                            schema_name=self.context.database_schema,
+                            table_name=table_name,
+                        ),
+                        tags=tag.TagValues,
+                        classification_name=tag.TagKey,
+                        tag_description=ATHENA_TAG,
+                        classification_description=ATHENA_TAG_CLASSIFICATION,
+                    )
+
+                # yield the column tags
+                for column in table_tags.LFTagsOnColumns or []:
+                    for tag in column.LFTags or []:
+                        yield from get_ometa_tag_and_classification(
+                            tag_fqn=fqn.build(
+                                self.metadata,
+                                Column,
+                                service_name=self.context.database_service,
+                                database_name=self.context.database,
+                                schema_name=self.context.database_schema,
+                                table_name=table_name,
+                                column_name=column.Name,
+                            ),
+                            tags=tag.TagValues,
+                            classification_name=tag.TagKey,
+                            tag_description=ATHENA_TAG,
+                            classification_description=ATHENA_TAG_CLASSIFICATION,
+                        )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name="Tags and Classifications",
+                        error=f"Failed to fetch table/column tags due to [{exc}]",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )

metadata/ingestion/source/database/azuresql/metadata.py

@@ -77,7 +77,6 @@ class AzuresqlSource(CommonDbSourceService, MultiDBSource):
         yield from self._execute_database_query(AZURE_SQL_GET_DATABASES)
 
     def get_database_names(self) -> Iterable[str]:
-
         if not self.config.serviceConnection.__root__.config.ingestAllDatabases:
             configured_db = self.config.serviceConnection.__root__.config.database
             self.set_inspector(database_name=configured_db)

metadata/ingestion/source/database/bigquery/connection.py

@@ -44,6 +44,9 @@ from metadata.ingestion.connections.test_connections import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.bigquery.queries import BIGQUERY_TEST_STATEMENT
 from metadata.utils.credentials import set_google_credentials
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
 
 
 def get_connection_url(connection: BigQueryConnection) -> str:
@@ -107,9 +110,27 @@ def test_connection(
             return policy_tags
 
     def test_tags():
-        taxonomies = PolicyTagManagerClient().list_taxonomies(
-            parent=f"projects/{engine.url.host}/locations/{service_connection.taxonomyLocation}"
-        )
+        taxonomy_project_ids = []
+        if engine.url.host:
+            taxonomy_project_ids.append(engine.url.host)
+        if service_connection.taxonomyProjectID:
+            taxonomy_project_ids.extend(service_connection.taxonomyProjectID)
+        if not taxonomy_project_ids:
+            logger.info("'taxonomyProjectID' is not set, so skipping this test.")
+            return None
+
+        taxonomy_location = service_connection.taxonomyLocation
+        if not taxonomy_location:
+            logger.info("'taxonomyLocation' is not set, so skipping this test.")
+            return None
+
+        taxonomies = []
+        for project_id in taxonomy_project_ids:
+            taxonomies.extend(
+                PolicyTagManagerClient().list_taxonomies(
+                    parent=f"projects/{project_id}/locations/{taxonomy_location}"
+                )
+            )
         return get_tags(taxonomies)
 
     def test_connection_inner(engine):

metadata/ingestion/source/database/bigquery/metadata.py

@@ -29,7 +29,7 @@ from metadata.generated.schema.api.data.createDatabaseSchema import (
 from metadata.generated.schema.api.data.createStoredProcedure import (
     CreateStoredProcedureRequest,
 )
-from metadata.generated.schema.entity.data.database import Database, EntityName
+from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode
 from metadata.generated.schema.entity.data.table import (
@@ -40,15 +40,18 @@ from metadata.generated.schema.entity.data.table import (
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.security.credentials.gcpValues import (
     GcpCredentialsValues,
 )
-from metadata.generated.schema.type.basic import SourceUrl
+from metadata.generated.schema.type.basic import EntityName, SourceUrl
 from metadata.generated.schema.type.tagLabel import TagLabel
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -61,6 +64,7 @@ from metadata.ingestion.source.database.bigquery.models import (
 from metadata.ingestion.source.database.bigquery.queries import (
     BIGQUERY_GET_STORED_PROCEDURE_QUERIES,
     BIGQUERY_GET_STORED_PROCEDURES,
+    BIGQUERY_LIFE_CYCLE_QUERY,
     BIGQUERY_SCHEMA_DESCRIPTION,
     BIGQUERY_TABLE_AND_TYPE,
 )
@@ -69,6 +73,9 @@ from metadata.ingestion.source.database.common_db_source import (
     CommonDbSourceService,
     TableNameAndType,
 )
+from metadata.ingestion.source.database.life_cycle_query_mixin import (
+    LifeCycleQueryMixin,
+)
 from metadata.ingestion.source.database.multi_db_source import MultiDBSource
 from metadata.ingestion.source.database.stored_procedures_mixin import (
     QueryByProcedure,
@@ -80,11 +87,8 @@ from metadata.utils.filters import filter_by_database
 from metadata.utils.helpers import get_start_and_end
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import is_complex_type
-from metadata.utils.tag_utils import (
-    get_ometa_tag_and_classification,
-    get_tag_label,
-    get_tag_labels,
-)
+from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_label
+from metadata.utils.tag_utils import get_tag_labels as fetch_tag_labels_om
 
 _bigquery_table_types = {
     "BASE TABLE": TableType.Regular,
@@ -192,7 +196,9 @@ BigQueryDialect._build_formatted_table_id = (  # pylint: disable=protected-acces
 )
 
 
-class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
+class BigquerySource(
+    LifeCycleQueryMixin, StoredProcedureMixin, CommonDbSourceService, MultiDBSource
+):
     """
     Implements the necessary methods to extract
     Database metadata from Bigquery Source
@@ -204,12 +210,14 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
         # as per service connection config, which would result in an error.
         self.test_connection = lambda: None
         super().__init__(config, metadata)
-        self.temp_credentials = None
         self.client = None
+        # Used to delete temp json file created while initializing bigquery client
+        self.temp_credentials_file_path = []
         # Upon invoking the set_project_id method, we retrieve a comprehensive
         # list of all project IDs. Subsequently, after the invokation,
         # we proceed to test the connections for each of these project IDs
         self.project_ids = self.set_project_id()
+        self.life_cycle_query = BIGQUERY_LIFE_CYCLE_QUERY
         self.test_connection = self._test_connection
         self.test_connection()
 
@@ -237,6 +245,8 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
             test_connection_fn(
                 self.metadata, inspector_details.engine, self.service_connection
             )
+            if os.environ[GOOGLE_CREDENTIALS]:
+                self.temp_credentials_file_path.append(os.environ[GOOGLE_CREDENTIALS])
 
     def query_table_names_and_types(
         self, schema_name: str
@@ -281,6 +291,7 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
                         classification_name=key,
                         tag_description="Bigquery Dataset Label",
                         classification_description="",
+                        include_tags=self.source_config.includeTags,
                     )
             # Fetching policy tags on the column level
             list_project_ids = [self.context.database]
@@ -300,13 +311,14 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
                         classification_name=taxonomy.display_name,
                         tag_description="Bigquery Policy Tag",
                         classification_description="",
+                        include_tags=self.source_config.includeTags,
                     )
         except Exception as exc:
             yield Either(
                 left=StackTraceError(
                     name="Tags and Classifications",
                     error=f"Skipping Policy Tag ingestion due to: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -353,18 +365,18 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
                 schema_name=schema_name,
             ),
         )
-
-        dataset_obj = self.client.get_dataset(schema_name)
-        if dataset_obj.labels:
-            database_schema_request_obj.tags = []
-            for label_classification, label_tag_name in dataset_obj.labels.items():
-                database_schema_request_obj.tags.append(
-                    get_tag_label(
+        if self.source_config.includeTags:
+            dataset_obj = self.client.get_dataset(schema_name)
+            if dataset_obj.labels:
+                database_schema_request_obj.tags = []
+                for label_classification, label_tag_name in dataset_obj.labels.items():
+                    tag_label = get_tag_label(
                         metadata=self.metadata,
                         tag_name=label_tag_name,
                         classification_name=label_classification,
                     )
-                )
+                    if tag_label:
+                        database_schema_request_obj.tags.append(tag_label)
         yield Either(right=database_schema_request_obj)
 
     def get_table_obj(self, table_name: str):
@@ -373,7 +385,7 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
         bq_table_fqn = fqn._build(database, schema_name, table_name)
         return self.client.get_table(bq_table_fqn)
 
-    def yield_table_tag_details(self, table_name_and_type: Tuple[str, str]):
+    def yield_table_tags(self, table_name_and_type: Tuple[str, str]):
         table_name, _ = table_name_and_type
         table_obj = self.get_table_obj(table_name=table_name)
         if table_obj.labels:
@@ -383,6 +395,7 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
                     classification_name=key,
                     tag_description="Bigquery Table Label",
                     classification_description="",
+                    include_tags=self.source_config.includeTags,
                 )
 
     def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]:
@@ -411,7 +424,7 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
         is properly informed
         """
         if column.get("policy_tags"):
-            return get_tag_labels(
+            return fetch_tag_labels_om(
                 metadata=self.metadata,
                 tags=[column["policy_tags"]],
                 classification_name=column["taxonomy"],
@@ -423,7 +436,7 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
         inspector_details = get_inspector_details(
             database_name=database_name, service_connection=self.service_connection
         )
-
+        self.temp_credentials_file_path.append(os.environ[GOOGLE_CREDENTIALS])
         self.client = inspector_details.client
         self.engine = inspector_details.engine
         self.inspector = inspector_details.inspector
@@ -516,15 +529,14 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
 
     def close(self):
         super().close()
-        if self.temp_credentials:
-            os.unlink(self.temp_credentials)
         os.environ.pop("GOOGLE_CLOUD_PROJECT", "")
         if isinstance(
             self.service_connection.credentials.gcpConfig, GcpCredentialsValues
         ) and (GOOGLE_CREDENTIALS in os.environ):
-            tmp_credentials_file = os.environ[GOOGLE_CREDENTIALS]
-            os.remove(tmp_credentials_file)
             del os.environ[GOOGLE_CREDENTIALS]
+            for temp_file_path in self.temp_credentials_file_path:
+                if os.path.exists(temp_file_path):
+                    os.remove(temp_file_path)
 
     def _get_source_url(
         self,
@@ -635,7 +647,7 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
                 left=StackTraceError(
                     name=stored_procedure.name,
                     error=f"Error yielding Stored Procedure [{stored_procedure.name}] due to [{exc}]",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 

metadata/ingestion/source/database/bigquery/queries.py

@@ -75,7 +75,6 @@ BIGQUERY_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
     """
 WITH SP_HISTORY AS (
   SELECT
-    job_id,
     query AS query_text,
     start_time,
     end_time,
@@ -90,7 +89,6 @@ WITH SP_HISTORY AS (
 ),
 Q_HISTORY AS (
   SELECT
-    job_id,
     project_id as database_name,
     user_email as user_name,
     statement_type as query_type,
@@ -109,8 +107,6 @@ Q_HISTORY AS (
     AND error_result is NULL
 )
 SELECT
-  SP.job_id as procedure_id,
-  Q.job_id as query_id,
   Q.query_type as query_type,
   SP.query_text as procedure_text,
   Q.query_text as query_text,
@@ -130,3 +126,14 @@ JOIN Q_HISTORY Q
 ORDER BY procedure_start_time DESC
 """
 )
+
+BIGQUERY_LIFE_CYCLE_QUERY = textwrap.dedent(
+    """
+select 
+table_name as table_name,
+creation_time as created_at
+from `{schema_name}`.INFORMATION_SCHEMA.TABLES
+where table_schema = '{schema_name}'
+and table_catalog = '{database_name}'
+"""
+)

metadata/ingestion/source/database/bigquery/query_parser.py

@@ -14,6 +14,7 @@ Handle big query usage extraction
 from abc import ABC
 from copy import deepcopy
 from datetime import datetime
+from typing import Optional
 
 from google import auth
 
@@ -86,3 +87,15 @@ class BigqueryQueryParserSource(QueryParserSource, ABC):
                 yield inspector_details.engine
         else:
             yield self.engine
+
+    def check_life_cycle_query(
+        self, query_type: Optional[str], query_text: Optional[str]
+    ) -> bool:
+        """
+        returns true if query is to be used for life cycle processing.
+
+        Override if we have specific parameters
+        """
+        if query_type != "SELECT":
+            return True
+        return False

metadata/ingestion/source/database/bigquery/usage.py

@@ -26,6 +26,4 @@ class BigqueryUsageSource(BigqueryQueryParserSource, UsageSource):
 
     sql_stmt = BIGQUERY_STATEMENT
 
-    filters = """
-        AND statement_type = "SELECT"
-    """
+    filters = ""

metadata/ingestion/source/database/column_helpers.py

@@ -14,16 +14,6 @@ their raw information from the source
 """
 
 
-def remove_table_from_column_name(table_name: str, raw_column_name: str) -> str:
-    """
-    Given a column `table.column`, return only `column`.
-
-    Note that we might have columns which have real dots
-    "." in the name, so we cannot just split.
-    """
-    return raw_column_name.replace(table_name + ".", "")
-
-
 def truncate_column_name(col_name: str):
     """
     OpenMetadata table column specification limits column name to 128 characters.

metadata/ingestion/source/database/column_type_parser.py

@@ -369,7 +369,9 @@ class ColumnTypeParser:
 
     @staticmethod
     def _parse_struct_fields_string(stuct_type: str) -> Dict[str, object]:
-        parts = ColumnTypeParser._ignore_brackets_split(stuct_type, ",")
+        parts = ColumnTypeParser._ignore_brackets_split(
+            stuct_type, ",", skip_no_child_validation=True
+        )
         columns = []
         for part in parts:
             name_and_type = ColumnTypeParser._ignore_brackets_split(part, ":")
@@ -387,9 +389,10 @@ class ColumnTypeParser:
             field_type = ColumnTypeParser._parse_datatype_string(name_and_type[1])
             field_type["name"] = field_name
             columns.append(field_type)
+
         return {
             "children": columns,
-            "dataTypeDisplay": f"struct<{stuct_type}>",
+            "dataTypeDisplay": f"struct<{stuct_type or 'unknown'}>",
             "dataType": "STRUCT",
         }
 
@@ -431,7 +434,9 @@ class ColumnTypeParser:
         }
 
     @staticmethod
-    def _ignore_brackets_split(string: str, separator: str) -> List[str]:
+    def _ignore_brackets_split(
+        string: str, separator: str, skip_no_child_validation: bool = False
+    ) -> List[str]:
         parts = []
         buf = ""
         level = 0
@@ -452,9 +457,10 @@ class ColumnTypeParser:
             else:
                 buf += char
 
-        if len(buf) == 0:
+        if len(buf) == 0 and not skip_no_child_validation:
             raise ValueError(f"The {separator} cannot be the last char: {string}")
-        parts.append(buf)
+        if buf:
+            parts.append(buf)
         return parts
 
     @staticmethod