acryl-datahub 1.3.0.1rc9__py3-none-any.whl → 1.3.1__py3-none-any.whl

datahub/ingestion/source/elastic_search.py

@@ -407,12 +407,78 @@ class ElasticsearchSource(StatefulIngestionSourceBase):
         for mcp in self._get_data_stream_index_count_mcps():
             yield mcp.as_workunit()
         if self.source_config.ingest_index_templates:
-            templates = self.client.indices.get_template()
-            for template in templates:
+            # Fetch legacy index templates
+            legacy_templates = self.client.indices.get_template()
+            for template in legacy_templates:
                 if self.source_config.index_template_pattern.allowed(template):
                     for mcp in self._extract_mcps(template, is_index=False):
                         yield mcp.as_workunit()
 
+            # Fetch composable index templates (ES 7.8+ / OpenSearch)
+            try:
+                composable_templates = self.client.indices.get_index_template()
+                for template_info in composable_templates.get("index_templates", []):
+                    template = template_info.get("name")
+                    if template and self.source_config.index_template_pattern.allowed(
+                        template
+                    ):
+                        for mcp in self._extract_mcps(
+                            template, is_index=False, is_composable_template=True
+                        ):
+                            yield mcp.as_workunit()
+            except Exception as e:
+                logger.warning(f"Unable to fetch composable index templates: {e}")
+
+    def _get_template_metadata(
+        self, template_name: str, is_composable: bool
+    ) -> Dict[str, Any]:
+        """Fetch template metadata from Elasticsearch/OpenSearch."""
+        if is_composable:
+            # For composable templates (ES 7.8+ / OpenSearch)
+            raw_response = self.client.indices.get_index_template(name=template_name)
+            template_data = raw_response.get("index_templates", [{}])[0]
+            return template_data.get("index_template", {})
+        else:
+            # For legacy templates
+            raw_response = self.client.indices.get_template(name=template_name)
+            return raw_response[template_name]
+
+    def _extract_template_custom_properties(
+        self, raw_metadata: Dict[str, Any], is_composable: bool
+    ) -> Dict[str, str]:
+        """Extract custom properties from template metadata."""
+        custom_properties: Dict[str, str] = {}
+
+        # Extract aliases
+        if is_composable:
+            aliases_dict = raw_metadata.get("template", {}).get("aliases", {})
+        else:
+            aliases_dict = raw_metadata.get("aliases", {})
+        index_aliases: List[str] = list(aliases_dict.keys()) if aliases_dict else []
+        if index_aliases:
+            custom_properties["aliases"] = ",".join(index_aliases)
+
+        # Extract index_patterns
+        index_patterns: List[str] = raw_metadata.get("index_patterns", [])
+        if index_patterns:
+            custom_properties["index_patterns"] = ",".join(index_patterns)
+
+        # Extract settings
+        if is_composable:
+            index_settings: Dict[str, Any] = (
+                raw_metadata.get("template", {}).get("settings", {}).get("index", {})
+            )
+        else:
+            index_settings = raw_metadata.get("settings", {}).get("index", {})
+        num_shards: str = index_settings.get("number_of_shards", "")
+        if num_shards:
+            custom_properties["num_shards"] = num_shards
+        num_replicas: str = index_settings.get("number_of_replicas", "")
+        if num_replicas:
+            custom_properties["num_replicas"] = num_replicas
+
+        return custom_properties
+
     def _get_data_stream_index_count_mcps(
         self,
     ) -> Iterable[MetadataChangeProposalWrapper]:
@@ -434,9 +500,11 @@ class ElasticsearchSource(StatefulIngestionSourceBase):
             )
 
     def _extract_mcps(
-        self, index: str, is_index: bool = True
+        self, index: str, is_index: bool = True, is_composable_template: bool = False
     ) -> Iterable[MetadataChangeProposalWrapper]:
-        logger.debug(f"index='{index}', is_index={is_index}")
+        logger.debug(
+            f"index='{index}', is_index={is_index}, is_composable_template={is_composable_template}"
+        )
 
         if is_index:
             raw_index = self.client.indices.get(index=index)
@@ -451,15 +519,20 @@ class ElasticsearchSource(StatefulIngestionSourceBase):
                     # This is a duplicate, skip processing it further.
                     return
         else:
-            raw_index = self.client.indices.get_template(name=index)
-            raw_index_metadata = raw_index[index]
+            raw_index_metadata = self._get_template_metadata(
+                index, is_composable_template
+            )
         collapsed_index_name = collapse_name(
             name=index, collapse_urns=self.source_config.collapse_urns
         )
 
         # 1. Construct and emit the schemaMetadata aspect
         # 1.1 Generate the schema fields from ES mappings.
-        index_mappings = raw_index_metadata["mappings"]
+        # For composable templates, mappings are under 'template.mappings'
+        if is_composable_template:
+            index_mappings = raw_index_metadata.get("template", {}).get("mappings", {})
+        else:
+            index_mappings = raw_index_metadata.get("mappings", {})
         index_mappings_json_str: str = json.dumps(index_mappings)
         md5_hash = md5(index_mappings_json_str.encode()).hexdigest()
         schema_fields = list(
@@ -517,28 +590,32 @@ class ElasticsearchSource(StatefulIngestionSourceBase):
             ),
         )
 
-        # 4. Construct and emit properties if needed. Will attempt to get the following properties
-        custom_properties: Dict[str, str] = {}
-        # 4.1 aliases
-        index_aliases: List[str] = raw_index_metadata.get("aliases", {}).keys()
-        if index_aliases:
-            custom_properties["aliases"] = ",".join(index_aliases)
-        # 4.2 index_patterns
-        index_patterns: List[str] = raw_index_metadata.get("index_patterns", [])
-        if index_patterns:
-            custom_properties["index_patterns"] = ",".join(index_patterns)
-
-        # 4.3 number_of_shards
-        index_settings: Dict[str, Any] = raw_index_metadata.get("settings", {}).get(
-            "index", {}
-        )
-        num_shards: str = index_settings.get("number_of_shards", "")
-        if num_shards:
-            custom_properties["num_shards"] = num_shards
-        # 4.4 number_of_replicas
-        num_replicas: str = index_settings.get("number_of_replicas", "")
-        if num_replicas:
-            custom_properties["num_replicas"] = num_replicas
+        # 4. Construct and emit properties
+        if is_index:
+            custom_properties: Dict[str, str] = {}
+            # Extract properties for indices
+            index_aliases: List[str] = list(
+                raw_index_metadata.get("aliases", {}).keys()
+            )
+            if index_aliases:
+                custom_properties["aliases"] = ",".join(index_aliases)
+            index_patterns: List[str] = raw_index_metadata.get("index_patterns", [])
+            if index_patterns:
+                custom_properties["index_patterns"] = ",".join(index_patterns)
+            index_settings: Dict[str, Any] = raw_index_metadata.get("settings", {}).get(
+                "index", {}
+            )
+            num_shards: str = index_settings.get("number_of_shards", "")
+            if num_shards:
+                custom_properties["num_shards"] = num_shards
+            num_replicas: str = index_settings.get("number_of_replicas", "")
+            if num_replicas:
+                custom_properties["num_replicas"] = num_replicas
+        else:
+            # Extract properties for templates
+            custom_properties = self._extract_template_custom_properties(
+                raw_index_metadata, is_composable_template
+            )
 
         yield MetadataChangeProposalWrapper(
             entityUrn=dataset_urn,

datahub/ingestion/source/snowflake/snowflake_queries.py

@@ -78,6 +78,7 @@ from datahub.utilities.file_backed_collections import (
     ConnectionWrapper,
     FileBackedList,
 )
+from datahub.utilities.lossy_collections import LossyList
 from datahub.utilities.perf_timer import PerfTimer
 
 logger = logging.getLogger(__name__)
@@ -169,6 +170,10 @@ class SnowflakeQueriesExtractorReport(Report):
     num_stream_queries_observed: int = 0
     num_create_temp_view_queries_observed: int = 0
     num_users: int = 0
+    num_queries_with_empty_column_name: int = 0
+    queries_with_empty_column_name: LossyList[str] = dataclasses.field(
+        default_factory=LossyList
+    )
 
 
 @dataclass
@@ -626,9 +631,28 @@ class SnowflakeQueriesExtractor(SnowflakeStructuredReportMixin, Closeable):
 
             columns = set()
             for modified_column in obj["columns"]:
-                columns.add(
-                    self.identifiers.snowflake_identifier(modified_column["columnName"])
-                )
+                column_name = modified_column["columnName"]
+                # An empty column name in the audit log would cause an error when creating column URNs.
+                # To avoid this and still extract lineage, the raw query text is parsed as a fallback.
+                if not column_name or not column_name.strip():
+                    query_id = res["query_id"]
+                    self.report.num_queries_with_empty_column_name += 1
+                    self.report.queries_with_empty_column_name.append(query_id)
+                    logger.info(f"Query {query_id} has empty column name in audit log.")
+
+                    return ObservedQuery(
+                        query=query_text,
+                        session_id=res["session_id"],
+                        timestamp=timestamp,
+                        user=user,
+                        default_db=res["default_db"],
+                        default_schema=res["default_schema"],
+                        query_hash=get_query_fingerprint(
+                            query_text, self.identifiers.platform, fast=True
+                        ),
+                        extra_info=extra_info,
+                    )
+                columns.add(self.identifiers.snowflake_identifier(column_name))
 
             upstreams.append(dataset)
             column_usage[dataset] = columns

datahub/ingestion/source/sql_queries.py

@@ -1,11 +1,13 @@
 import json
 import logging
 import os
-from dataclasses import dataclass
+import re
+from dataclasses import dataclass, field
 from datetime import datetime
 from functools import partial
-from typing import ClassVar, Iterable, List, Optional, Union
+from typing import ClassVar, Iterable, List, Optional, Union, cast
 
+import smart_open
 from pydantic import BaseModel, Field, validator
 
 from datahub.configuration.common import HiddenFromDocs
@@ -36,12 +38,13 @@ from datahub.ingestion.api.source import (
     SourceCapability,
     SourceReport,
 )
-from datahub.ingestion.api.source_helpers import auto_workunit_reporter
+from datahub.ingestion.api.source_helpers import auto_workunit, auto_workunit_reporter
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
 from datahub.ingestion.source.usage.usage_common import BaseUsageConfig
 from datahub.metadata.urns import CorpUserUrn, DatasetUrn
-from datahub.sql_parsing.schema_resolver import SchemaResolver
+from datahub.sql_parsing.schema_resolver import SchemaResolver, SchemaResolverReport
 from datahub.sql_parsing.sql_parsing_aggregator import (
     KnownQueryLineageInfo,
     ObservedQuery,
@@ -82,6 +85,24 @@ class SqlQueriesSourceConfig(
         None,
         description="The SQL dialect to use when parsing queries. Overrides automatic dialect detection.",
     )
+    temp_table_patterns: List[str] = Field(
+        description="Regex patterns for temporary tables to filter in lineage ingestion. "
+        "Specify regex to match the entire table name. This is useful for platforms like Athena "
+        "that don't have native temp tables but use naming patterns for fake temp tables.",
+        default=[],
+    )
+
+    enable_lazy_schema_loading: bool = Field(
+        default=True,
+        description="Enable lazy schema loading for better performance. When enabled, schemas are fetched on-demand "
+        "instead of bulk loading all schemas upfront, reducing startup time and memory usage.",
+    )
+
+    # AWS/S3 configuration
+    aws_config: Optional[AwsConnectionConfig] = Field(
+        default=None,
+        description="AWS configuration for S3 access. Required when query_file is an S3 URI (s3://).",
+    )
 
 
 @dataclass
@@ -89,8 +110,13 @@ class SqlQueriesSourceReport(SourceReport):
     num_entries_processed: int = 0
     num_entries_failed: int = 0
     num_queries_aggregator_failures: int = 0
+    num_queries_processed_sequential: int = 0
+    num_temp_tables_detected: int = 0
+    temp_table_patterns_used: List[str] = field(default_factory=list)
+    peak_memory_usage_mb: float = 0.0
 
     sql_aggregator: Optional[SqlAggregatorReport] = None
+    schema_resolver_report: Optional[SchemaResolverReport] = None
 
 
 @platform_name("SQL Queries", id="sql-queries")
@@ -115,6 +141,18 @@ class SqlQueriesSource(Source):
     - upstream_tables (optional): string[] - Fallback list of tables the query reads from,
      used if the query can't be parsed.
 
+    **Lazy Schema Loading**:
+    - Fetches schemas on-demand during query parsing instead of bulk loading all schemas upfront
+    - Caches fetched schemas for future lookups to avoid repeated network requests
+    - Reduces initial startup time and memory usage significantly
+    - Automatically handles large platforms efficiently without memory issues
+
+    **Query Processing**:
+    - Loads the entire query file into memory at once
+    - Processes all queries sequentially before generating metadata work units
+    - Preserves temp table mappings and lineage relationships to ensure consistent lineage tracking
+    - Query deduplication is handled automatically by the SQL parsing aggregator
+
     ### Incremental Lineage
     When `incremental_lineage` is enabled, this source will emit lineage as patches rather than full overwrites.
     This allows you to add lineage edges without removing existing ones, which is useful for:
@@ -124,6 +162,12 @@ class SqlQueriesSource(Source):
 
     Note: Incremental lineage only applies to UpstreamLineage aspects. Other aspects like queries and usage
     statistics will still be emitted normally.
+
+    ### Temporary Table Support
+    For platforms like Athena that don't have native temporary tables, you can use the `temp_table_patterns`
+    configuration to specify regex patterns that identify fake temporary tables. This allows the source to
+    process these tables like other sources that support native temp tables, enabling proper lineage tracking
+    across temporary table operations.
     """
 
     schema_resolver: Optional[SchemaResolver]
@@ -141,13 +185,19 @@ class SqlQueriesSource(Source):
         self.report = SqlQueriesSourceReport()
 
         if self.config.use_schema_resolver:
-            # TODO: `initialize_schema_resolver_from_datahub` does a  bulk initialization by fetching all schemas
-            # for the given platform, platform instance, and env. Instead this should be configurable:
-            # bulk initialization vs lazy on-demand schema fetching.
-            self.schema_resolver = self.graph.initialize_schema_resolver_from_datahub(
+            # Create schema resolver report for tracking
+            self.report.schema_resolver_report = SchemaResolverReport()
+
+            # Use lazy loading - schemas will be fetched on-demand and cached
+            logger.info(
+                "Using lazy schema loading - schemas will be fetched on-demand and cached"
+            )
+            self.schema_resolver = SchemaResolver(
                 platform=self.config.platform,
                 platform_instance=self.config.platform_instance,
                 env=self.config.env,
+                graph=self.graph,
+                report=self.report.schema_resolver_report,
             )
         else:
             self.schema_resolver = None
@@ -156,7 +206,9 @@ class SqlQueriesSource(Source):
             platform=self.config.platform,
             platform_instance=self.config.platform_instance,
             env=self.config.env,
-            schema_resolver=self.schema_resolver,
+            schema_resolver=cast(SchemaResolver, self.schema_resolver)
+            if self.schema_resolver
+            else None,
             eager_graph_load=False,
             generate_lineage=True,  # TODO: make this configurable
             generate_queries=True,  # TODO: make this configurable
@@ -165,7 +217,9 @@ class SqlQueriesSource(Source):
             generate_usage_statistics=True,
             generate_operations=True,  # TODO: make this configurable
             usage_config=self.config.usage,
-            is_temp_table=None,
+            is_temp_table=self.is_temp_table
+            if self.config.temp_table_patterns
+            else None,
             is_allowed_table=None,
             format_queries=False,
         )
@@ -193,20 +247,73 @@ class SqlQueriesSource(Source):
     ) -> Iterable[Union[MetadataWorkUnit, MetadataChangeProposalWrapper]]:
         logger.info(f"Parsing queries from {os.path.basename(self.config.query_file)}")
 
+        logger.info("Processing all queries in batch mode")
+        yield from self._process_queries_batch()
+
+    def _process_queries_batch(
+        self,
+    ) -> Iterable[Union[MetadataWorkUnit, MetadataChangeProposalWrapper]]:
+        """Process all queries in memory (original behavior)."""
         with self.report.new_stage("Collecting queries from file"):
             queries = list(self._parse_query_file())
             logger.info(f"Collected {len(queries)} queries for processing")
 
         with self.report.new_stage("Processing queries through SQL parsing aggregator"):
-            for query_entry in queries:
-                self._add_query_to_aggregator(query_entry)
+            logger.info("Using sequential processing")
+            self._process_queries_sequential(queries)
 
         with self.report.new_stage("Generating metadata work units"):
             logger.info("Generating workunits from SQL parsing aggregator")
-            yield from self.aggregator.gen_metadata()
+            yield from auto_workunit(self.aggregator.gen_metadata())
 
-    def _parse_query_file(self) -> Iterable["QueryEntry"]:
-        """Parse the query file and yield QueryEntry objects."""
+    def _is_s3_uri(self, path: str) -> bool:
+        """Check if the path is an S3 URI."""
+        return path.startswith("s3://")
+
+    def _parse_s3_query_file(self) -> Iterable["QueryEntry"]:
+        """Parse query file from S3 using smart_open."""
+        if not self.config.aws_config:
+            raise ValueError("AWS configuration required for S3 file access")
+
+        logger.info(f"Reading query file from S3: {self.config.query_file}")
+
+        try:
+            # Use smart_open for efficient S3 streaming, similar to S3FileSystem
+            s3_client = self.config.aws_config.get_s3_client()
+
+            with smart_open.open(
+                self.config.query_file, mode="r", transport_params={"client": s3_client}
+            ) as file_stream:
+                for line in file_stream:
+                    if line.strip():
+                        try:
+                            query_dict = json.loads(line, strict=False)
+                            entry = QueryEntry.create(query_dict, config=self.config)
+                            self.report.num_entries_processed += 1
+                            if self.report.num_entries_processed % 1000 == 0:
+                                logger.info(
+                                    f"Processed {self.report.num_entries_processed} query entries from S3"
+                                )
+                            yield entry
+                        except Exception as e:
+                            self.report.num_entries_failed += 1
+                            self.report.warning(
+                                title="Error processing query from S3",
+                                message="Query skipped due to parsing error",
+                                context=line.strip(),
+                                exc=e,
+                            )
+        except Exception as e:
+            self.report.warning(
+                title="Error reading S3 file",
+                message="Failed to read S3 file",
+                context=self.config.query_file,
+                exc=e,
+            )
+            raise
+
+    def _parse_local_query_file(self) -> Iterable["QueryEntry"]:
+        """Parse local query file (existing logic)."""
         with open(self.config.query_file) as f:
             for line in f:
                 try:
@@ -227,6 +334,30 @@ class SqlQueriesSource(Source):
                         exc=e,
                     )
 
+    def _parse_query_file(self) -> Iterable["QueryEntry"]:
+        """Parse the query file and yield QueryEntry objects."""
+        if self._is_s3_uri(self.config.query_file):
+            yield from self._parse_s3_query_file()
+        else:
+            yield from self._parse_local_query_file()
+
+    def _process_queries_sequential(self, queries: List["QueryEntry"]) -> None:
+        """Process queries sequentially."""
+        total_queries = len(queries)
+        logger.info(f"Processing {total_queries} queries sequentially")
+
+        # Process each query sequentially
+        for i, query_entry in enumerate(queries):
+            self._add_query_to_aggregator(query_entry)
+            self.report.num_queries_processed_sequential += 1
+
+            # Simple progress reporting every 1000 queries
+            if (i + 1) % 1000 == 0:
+                progress_pct = ((i + 1) / total_queries) * 100
+                logger.info(
+                    f"Processed {i + 1}/{total_queries} queries ({progress_pct:.1f}%)"
+                )
+
     def _add_query_to_aggregator(self, query_entry: "QueryEntry") -> None:
         """Add a query to the SQL parsing aggregator."""
         try:
@@ -285,6 +416,24 @@ class SqlQueriesSource(Source):
                 exc=e,
             )
 
+    def is_temp_table(self, name: str) -> bool:
+        """Check if a table name matches any of the configured temp table patterns."""
+        if not self.config.temp_table_patterns:
+            return False
+
+        try:
+            for pattern in self.config.temp_table_patterns:
+                if re.match(pattern, name, flags=re.IGNORECASE):
+                    logger.debug(
+                        f"Table '{name}' matched temp table pattern: {pattern}"
+                    )
+                    self.report.num_temp_tables_detected += 1
+                    return True
+        except re.error as e:
+            logger.warning(f"Invalid regex pattern '{pattern}': {e}")
+
+        return False
+
 
 class QueryEntry(BaseModel):
     query: str

datahub/metadata/_internal_schema_classes.py

@@ -511,6 +511,7 @@ class AssertionInfoClass(_Aspect):
         source: Union[None, "AssertionSourceClass"]=None,
         lastUpdated: Union[None, "AuditStampClass"]=None,
         description: Union[None, str]=None,
+        note: Union[None, "AssertionNoteClass"]=None,
     ):
         super().__init__()
         
@@ -531,6 +532,7 @@ class AssertionInfoClass(_Aspect):
         self.source = source
         self.lastUpdated = lastUpdated
         self.description = description
+        self.note = note
     
     def _restore_defaults(self) -> None:
         self.customProperties = dict()
@@ -546,6 +548,7 @@ class AssertionInfoClass(_Aspect):
         self.source = self.RECORD_SCHEMA.fields_dict["source"].default
         self.lastUpdated = self.RECORD_SCHEMA.fields_dict["lastUpdated"].default
         self.description = self.RECORD_SCHEMA.fields_dict["description"].default
+        self.note = self.RECORD_SCHEMA.fields_dict["note"].default
     
     
     @property
@@ -570,7 +573,7 @@ class AssertionInfoClass(_Aspect):
     
     @property
     def type(self) -> Union[str, "AssertionTypeClass"]:
-        """Type of assertion. Assertion types can evolve to span Datasets, Flows (Pipelines), Models, Features etc."""
+        """Type of assertion."""
         return self._inner_dict.get('type')  # type: ignore
     
     @type.setter
@@ -682,6 +685,55 @@ class AssertionInfoClass(_Aspect):
         self._inner_dict['description'] = value
     
     
+    @property
+    def note(self) -> Union[None, "AssertionNoteClass"]:
+        """An optional note to give technical owners more context about the assertion, and how to troubleshoot it.
+    The UI will render this in markdown format."""
+        return self._inner_dict.get('note')  # type: ignore
+    
+    @note.setter
+    def note(self, value: Union[None, "AssertionNoteClass"]) -> None:
+        self._inner_dict['note'] = value
+    
+    
+class AssertionNoteClass(DictWrapper):
+    # No docs available.
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.assertion.AssertionNote")
+    def __init__(self,
+        content: str,
+        lastModified: "AuditStampClass",
+    ):
+        super().__init__()
+        
+        self.content = content
+        self.lastModified = lastModified
+    
+    def _restore_defaults(self) -> None:
+        self.content = str()
+        self.lastModified = AuditStampClass._construct_with_defaults()
+    
+    
+    @property
+    def content(self) -> str:
+        """The note to give technical owners more context about the assertion, and how to troubleshoot it."""
+        return self._inner_dict.get('content')  # type: ignore
+    
+    @content.setter
+    def content(self, value: str) -> None:
+        self._inner_dict['content'] = value
+    
+    
+    @property
+    def lastModified(self) -> "AuditStampClass":
+        """The time at which the note was last modified."""
+        return self._inner_dict.get('lastModified')  # type: ignore
+    
+    @lastModified.setter
+    def lastModified(self, value: "AuditStampClass") -> None:
+        self._inner_dict['lastModified'] = value
+    
+    
 class AssertionResultClass(DictWrapper):
     """The result of running an assertion"""
     
@@ -1337,7 +1389,7 @@ class AssertionStdParametersClass(DictWrapper):
     
     
 class AssertionTypeClass(object):
-    # No docs available.
+    """Type of assertion. Assertion types can evolve to span Datasets, Flows (Pipelines), Models, Features etc."""
     
     DATASET = "DATASET"
     """A single-dataset assertion.
@@ -12623,6 +12675,9 @@ class NotificationSinkTypeClass(object):
     EMAIL = "EMAIL"
     """Email target type."""
     
+    TEAMS = "TEAMS"
+    """Microsoft Teams target type."""
+    
     
     
 class EmailNotificationSettingsClass(DictWrapper):
@@ -20439,6 +20494,9 @@ class DataHubPageModuleTypeClass(object):
     PLATFORMS = "PLATFORMS"
     """Module displaying the platforms in an instance"""
     
+    UNKNOWN = "UNKNOWN"
+    """Unknown module type - this can occur with corrupted data or rolling back to versions without new modules"""
+    
     
     
 class DataHubPageModuleVisibilityClass(DictWrapper):
@@ -27742,6 +27800,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.assertion.AssertionActionType': AssertionActionTypeClass,
     'com.linkedin.pegasus2avro.assertion.AssertionActions': AssertionActionsClass,
     'com.linkedin.pegasus2avro.assertion.AssertionInfo': AssertionInfoClass,
+    'com.linkedin.pegasus2avro.assertion.AssertionNote': AssertionNoteClass,
     'com.linkedin.pegasus2avro.assertion.AssertionResult': AssertionResultClass,
     'com.linkedin.pegasus2avro.assertion.AssertionResultError': AssertionResultErrorClass,
     'com.linkedin.pegasus2avro.assertion.AssertionResultErrorType': AssertionResultErrorTypeClass,
@@ -28268,6 +28327,7 @@ __SCHEMA_TYPES = {
     'AssertionActionType': AssertionActionTypeClass,
     'AssertionActions': AssertionActionsClass,
     'AssertionInfo': AssertionInfoClass,
+    'AssertionNote': AssertionNoteClass,
     'AssertionResult': AssertionResultClass,
     'AssertionResultError': AssertionResultErrorClass,
     'AssertionResultErrorType': AssertionResultErrorTypeClass,

datahub/metadata/com/linkedin/pegasus2avro/assertion/__init__.py

@@ -11,6 +11,7 @@ from .....schema_classes import AssertionActionClass
 from .....schema_classes import AssertionActionTypeClass
 from .....schema_classes import AssertionActionsClass
 from .....schema_classes import AssertionInfoClass
+from .....schema_classes import AssertionNoteClass
 from .....schema_classes import AssertionResultClass
 from .....schema_classes import AssertionResultErrorClass
 from .....schema_classes import AssertionResultErrorTypeClass
@@ -64,6 +65,7 @@ AssertionAction = AssertionActionClass
 AssertionActionType = AssertionActionTypeClass
 AssertionActions = AssertionActionsClass
 AssertionInfo = AssertionInfoClass
+AssertionNote = AssertionNoteClass
 AssertionResult = AssertionResultClass
 AssertionResultError = AssertionResultErrorClass
 AssertionResultErrorType = AssertionResultErrorTypeClass