openmetadata-ingestion 1.10.2.0__py3-none-any.whl → 1.10.3.0__py3-none-any.whl

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship/__init__.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship/nodeInformation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship/nodeInformation.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/layerPaging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/layerPaging.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/patternRecognizer.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/patternRecognizer.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/personaPreferences.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/personaPreferences.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/piiEntity.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/piiEntity.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/predefinedRecognizer.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/predefinedRecognizer.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/recognizer.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/recognizer.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/recognizerFeedback.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/recognizerFeedback.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/recognizers/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00

metadata/generated/schema/type/recognizers/patterns.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/recognizers/patterns.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/recognizers/regexFlags.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/recognizers/regexFlags.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/status.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/status.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-10-17T13:06:31+00:00
+#   timestamp: 2025-10-22T04:03:21+00:00
 
 from __future__ import annotations
 

metadata/ingestion/ometa/mixins/tests_mixin.py

@@ -253,7 +253,7 @@ class OMetaTestsMixin:
         }
 
         resp = self.client.get(
-            f"{self.get_suffix(TestCaseResult)}/{test_case_fqn}",
+            f"{self.get_suffix(TestCaseResult)}/{quote(test_case_fqn)}",
             params,
         )
 

metadata/ingestion/source/database/salesforce/metadata.py

@@ -184,9 +184,11 @@ class SalesforceSource(DatabaseServiceSource):
                     )
                     if filter_by_table(
                         self.config.sourceConfig.config.tableFilterPattern,
-                        table_fqn
-                        if self.config.sourceConfig.config.useFqnForFiltering
-                        else table_name,
+                        (
+                            table_fqn
+                            if self.config.sourceConfig.config.useFqnForFiltering
+                            else table_name
+                        ),
                     ):
                         self.status.filter(
                             table_fqn,
@@ -238,7 +240,7 @@ class SalesforceSource(DatabaseServiceSource):
         all_column_description = None
         try:
             result = self.client.toolingexecute(
-                f"query/?q=SELECT+Description+FROM+FieldDefinition+WHERE+"
+                f"query/?q=SELECT+Description, QualifiedApiName+FROM+FieldDefinition+WHERE+"
                 f"EntityDefinition.QualifiedApiName='{table_name}'"
             )
             all_column_description = result["records"]
@@ -313,7 +315,7 @@ class SalesforceSource(DatabaseServiceSource):
             for item in all_column_description:
                 try:
                     if item.get("Description") is not None:
-                        column_name = item["attributes"]["url"].split(".")[-1]
+                        column_name = item["QualifiedApiName"]
                         column_description_mapping.update(
                             {column_name: item["Description"]}
                         )

metadata/ingestion/source/pipeline/databrickspipeline/kafka_parser.py

@@ -33,6 +33,12 @@ VARIABLE_ASSIGNMENT_PATTERN = re.compile(
     re.MULTILINE,
 )
 
+# Pattern to extract boolean variable assignments like: snapshot_required = True
+BOOL_ASSIGNMENT_PATTERN = re.compile(
+    r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(True|False)\s*$",
+    re.MULTILINE,
+)
+
 # Pattern to extract DLT table decorators: @dlt.table(name="table_name", ...) or @dlt.table(name=func())
 DLT_TABLE_PATTERN = re.compile(
     r"@dlt\.table\s*\(",
@@ -98,16 +104,25 @@ def _extract_variables(source_code: str) -> dict:
     Examples:
         TOPIC = "events"
         KAFKA_BROKER = "localhost:9092"
+        snapshot_required = True
 
-    Returns dict like: {"TOPIC": "events", "KAFKA_BROKER": "localhost:9092"}
+    Returns dict like: {"TOPIC": "events", "KAFKA_BROKER": "localhost:9092", "snapshot_required": "True"}
     """
     variables = {}
     try:
+        # Extract string variables
         for match in VARIABLE_ASSIGNMENT_PATTERN.finditer(source_code):
             var_name = match.group(1)
             var_value = match.group(2)
             variables[var_name] = var_value
             logger.debug(f"Found variable: {var_name} = {var_value}")
+
+        # Extract boolean variables
+        for match in BOOL_ASSIGNMENT_PATTERN.finditer(source_code):
+            var_name = match.group(1)
+            var_value = match.group(2)
+            variables[var_name] = var_value
+            logger.debug(f"Found boolean variable: {var_name} = {var_value}")
     except Exception as exc:
         logger.debug(f"Error extracting variables: {exc}")
     return variables
@@ -313,24 +328,43 @@ def _infer_table_name_from_function(
     Strategies:
     1. Look for entity_name variable and use it to build table name
     2. Extract keywords from function name (e.g., "event_log" from "generate_event_log_table_name")
+    3. Handle Materializer pattern: entity_name + suffix from function name
     """
     try:
         # Extract variables to find entity_name or similar
         variables = _extract_variables(source_code)
 
-        # Strategy 1: Use entity_name variable if present
+        # Strategy 1: Materializer pattern - entity_name + suffix from function
+        # Handles: @dlt.table(name=materializer.generate_event_log_table_name())
+        # where entity_name = "customerEvent" should produce "customerevent_event_log"
         entity_name = (
             variables.get("entity_name")
             or variables.get("entity")
             or variables.get("table_name")
         )
+
+        if entity_name and "generate_event_log_table_name" in function_call.lower():
+            table_name = f"{entity_name.lower()}_event_log"
+            logger.debug(
+                f"Inferred event_log table from Materializer pattern: {table_name}"
+            )
+            return table_name
+
+        if entity_name and "generate_snapshot_table_name" in function_call.lower():
+            table_name = f"{entity_name.lower()}_snapshot"
+            logger.debug(
+                f"Inferred snapshot table from Materializer pattern: {table_name}"
+            )
+            return table_name
+
+        # Strategy 2: Use entity_name variable if present (fallback)
         if entity_name:
             logger.debug(
                 f"Inferred table name from entity_name variable: {entity_name}"
             )
             return entity_name
 
-        # Strategy 2: Extract from function name (e.g., "event_log" from "generate_event_log_table_name")
+        # Strategy 3: Extract from function name (e.g., "event_log" from "generate_event_log_table_name")
         # Common patterns: generate_X_table_name, create_X_table, build_X_dataframe
         match = re.search(
             r"(?:generate|create|build)_([a-z_]+?)(?:_table|_dataframe)",
@@ -421,8 +455,22 @@ def extract_dlt_table_dependencies(source_code: str) -> List[DLTTableDependency]
                     continue
 
                 # Check if it reads from Kafka
+                # Direct pattern: spark.readStream.format("kafka")
                 reads_from_kafka = bool(KAFKA_STREAM_PATTERN.search(function_block))
 
+                # Materializer pattern: materializer.build_event_log_dataframe()
+                # This method internally reads from Kafka, so if we find this pattern
+                # and the table name matches event_log pattern, mark as Kafka reader
+                if (
+                    not reads_from_kafka
+                    and "materializer.build_event_log_dataframe" in function_block
+                ):
+                    if "event_log" in table_name:
+                        reads_from_kafka = True
+                        logger.debug(
+                            f"Table {table_name} reads from Kafka via Materializer"
+                        )
+
                 # Check if it reads from S3
                 s3_locations = []
                 for s3_match in S3_PATH_PATTERN.finditer(function_block):
@@ -464,6 +512,60 @@ def extract_dlt_table_dependencies(source_code: str) -> List[DLTTableDependency]
                 logger.debug(f"Error parsing function block: {exc}")
                 continue
 
+        # Handle Materializer snapshot pattern:
+        # if snapshot_required:
+        #     materializer.build_snapshot_dataframe()
+        # This creates a snapshot table without @dlt.table decorator
+        try:
+            variables = _extract_variables(source_code)
+            snapshot_required = variables.get("snapshot_required")
+            entity_name = (
+                variables.get("entity_name")
+                or variables.get("entity")
+                or variables.get("table_name")
+            )
+
+            # Check if snapshot table is built
+            # snapshot_required can be "True" (string) or True (boolean)
+            is_snapshot_enabled = (
+                snapshot_required and str(snapshot_required).lower() == "true"
+            )
+
+            if (
+                is_snapshot_enabled
+                and entity_name
+                and "build_snapshot_dataframe" in source_code
+            ):
+                snapshot_table_name = f"{entity_name.lower()}_snapshot"
+                event_log_table_name = f"{entity_name.lower()}_event_log"
+
+                # Find the event_log table in existing dependencies
+                event_log_dep = next(
+                    (d for d in dependencies if d.table_name == event_log_table_name),
+                    None,
+                )
+
+                if event_log_dep:
+                    # Create snapshot table that depends on event_log
+                    snapshot_dependency = DLTTableDependency(
+                        table_name=snapshot_table_name,
+                        depends_on=[event_log_table_name],
+                        reads_from_kafka=False,
+                        reads_from_s3=False,
+                        s3_locations=[],
+                    )
+                    dependencies.append(snapshot_dependency)
+                    logger.debug(
+                        f"Extracted Materializer snapshot table: {snapshot_table_name} depends on {event_log_table_name}"
+                    )
+                else:
+                    logger.debug(
+                        f"Found snapshot pattern but event_log table {event_log_table_name} not found in dependencies"
+                    )
+
+        except Exception as exc:
+            logger.debug(f"Error extracting Materializer snapshot pattern: {exc}")
+
     except Exception as exc:
         logger.warning(f"Error extracting DLT table dependencies: {exc}")
 

metadata/ingestion/source/pipeline/databrickspipeline/metadata.py

@@ -216,7 +216,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
     def get_tasks(self, pipeline_details: DataBrickPipelineDetails) -> List[Task]:
         try:
             if not pipeline_details.settings or not pipeline_details.settings.tasks:
-                return None
+                return []
 
             job_url = f"https://{self.service_connection.hostPort}/#job/{pipeline_details.job_id}"
 
@@ -237,7 +237,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Failed to get tasks list due to : {exc}")
-        return None
+        return []
 
     def yield_pipeline_status(
         self, pipeline_details: DataBrickPipelineDetails
@@ -523,7 +523,7 @@ class DatabrickspipelineSource(PipelineServiceSource):
         """
         Find Kafka topic in OpenMetadata using Elasticsearch search
 
-        Handles topic names with dots (e.g., "dev.ern.cashout.moneyRequest_v1")
+        Handles topic names with dots (e.g., "dev.example.transactions.customerEvent_v1")
         by searching with wildcard pattern: *.topic_name
 
         Topic FQN format: MessagingServiceName.TopicName
@@ -896,14 +896,11 @@ class DatabrickspipelineSource(PipelineServiceSource):
                                     f"   ✓ Topic found: {kafka_topic.fullyQualifiedName.root if hasattr(kafka_topic.fullyQualifiedName, 'root') else kafka_topic.fullyQualifiedName}"
                                 )
 
-                                # Find tables that read directly from Kafka OR from an external source view
+                                # Find tables that read DIRECTLY from Kafka
+                                # Only create Kafka -> table lineage for direct consumers
+                                # Downstream tables get table -> table lineage in Step 2
                                 for dep in dlt_dependencies:
-                                    is_kafka_consumer = dep.reads_from_kafka or any(
-                                        src in external_sources_map
-                                        for src in dep.depends_on
-                                    )
-
-                                    if is_kafka_consumer:
+                                    if dep.reads_from_kafka:
                                         logger.info(
                                             f"   🔍 Processing table: {dep.table_name}"
                                         )