acryl-datahub 1.0.0rc18__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/source/dbt/dbt_core.py

@@ -1,3 +1,4 @@
+import dataclasses
 import json
 import logging
 import re
@@ -12,7 +13,9 @@ from pydantic import BaseModel, Field, validator
 
 from datahub.configuration.git import GitReference
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
+from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
+    SourceCapability,
     SupportStatus,
     capability,
     config_class,
@@ -21,7 +24,6 @@ from datahub.ingestion.api.decorators import (
 )
 from datahub.ingestion.api.source import (
     CapabilityReport,
-    SourceCapability,
     TestableSource,
     TestConnectionReport,
 )
@@ -40,19 +42,28 @@ from datahub.ingestion.source.dbt.dbt_tests import DBTTest, DBTTestResult
 logger = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass
+class DBTCoreReport(DBTSourceReport):
+    catalog_info: Optional[dict] = None
+    manifest_info: Optional[dict] = None
+
+
 class DBTCoreConfig(DBTCommonConfig):
     manifest_path: str = Field(
-        description="Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json Note "
-        "this can be a local file or a URI."
+        description="Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json. "
+        "This can be a local file or a URI."
     )
-    catalog_path: str = Field(
-        description="Path to dbt catalog JSON. See https://docs.getdbt.com/reference/artifacts/catalog-json Note this "
-        "can be a local file or a URI."
+    catalog_path: Optional[str] = Field(
+        None,
+        description="Path to dbt catalog JSON. See https://docs.getdbt.com/reference/artifacts/catalog-json. "
+        "This file is optional, but highly recommended. Without it, some metadata like column info will be incomplete or missing. "
+        "This can be a local file or a URI.",
     )
     sources_path: Optional[str] = Field(
         default=None,
-        description="Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. If not "
-        "specified, last-modified fields will not be populated. Note this can be a local file or a URI.",
+        description="Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. "
+        "If not specified, last-modified fields will not be populated. "
+        "This can be a local file or a URI.",
     )
     run_results_paths: List[str] = Field(
         default=[],
@@ -161,7 +172,7 @@ def get_columns(
 
 def extract_dbt_entities(
     all_manifest_entities: Dict[str, Dict[str, Any]],
-    all_catalog_entities: Dict[str, Dict[str, Any]],
+    all_catalog_entities: Optional[Dict[str, Dict[str, Any]]],
     sources_results: List[Dict[str, Any]],
     manifest_adapter: str,
     use_identifiers: bool,
@@ -186,15 +197,6 @@ def extract_dbt_entities(
         ):
             name = manifest_node["alias"]
 
-        # initialize comment to "" for consistency with descriptions
-        # (since dbt null/undefined descriptions as "")
-        comment = ""
-
-        if key in all_catalog_entities and all_catalog_entities[key]["metadata"].get(
-            "comment"
-        ):
-            comment = all_catalog_entities[key]["metadata"]["comment"]
-
         materialization = None
         if "materialized" in manifest_node.get("config", {}):
             # It's a model
@@ -204,8 +206,9 @@ def extract_dbt_entities(
         if "depends_on" in manifest_node and "nodes" in manifest_node["depends_on"]:
             upstream_nodes = manifest_node["depends_on"]["nodes"]
 
-        # It's a source
-        catalog_node = all_catalog_entities.get(key)
+        catalog_node = (
+            all_catalog_entities.get(key) if all_catalog_entities is not None else None
+        )
         missing_from_catalog = catalog_node is None
         catalog_type = None
 
@@ -214,16 +217,23 @@ def extract_dbt_entities(
                 # Test and ephemeral nodes will never show up in the catalog.
                 missing_from_catalog = False
             else:
-                if not only_include_if_in_catalog:
+                if all_catalog_entities is not None and not only_include_if_in_catalog:
+                    # If the catalog file is missing, we have already generated a general message.
                     report.warning(
                         title="Node missing from catalog",
                         message="Found a node in the manifest file but not in the catalog. "
                         "This usually means the catalog file was not generated by `dbt docs generate` and so is incomplete. "
-                        "Some metadata, such as column types and descriptions, will be impacted.",
+                        "Some metadata, particularly schema information, will be impacted.",
                         context=key,
                     )
         else:
-            catalog_type = all_catalog_entities[key]["metadata"]["type"]
+            catalog_type = catalog_node["metadata"]["type"]
+
+        # initialize comment to "" for consistency with descriptions
+        # (since dbt null/undefined descriptions as "")
+        comment = ""
+        if catalog_node is not None and catalog_node.get("metadata", {}).get("comment"):
+            comment = catalog_node["metadata"]["comment"]
 
         query_tag_props = manifest_node.get("query_tag", {})
 
@@ -231,12 +241,15 @@ def extract_dbt_entities(
 
         owner = meta.get("owner")
         if owner is None:
-            owner = manifest_node.get("config", {}).get("meta", {}).get("owner")
+            owner = (manifest_node.get("config", {}).get("meta") or {}).get("owner")
+
+        if not meta:
+            # On older versions of dbt, the meta field was nested under config
+            # for some node types.
+            meta = manifest_node.get("config", {}).get("meta") or {}
 
         tags = manifest_node.get("tags", [])
         tags = [tag_prefix + tag for tag in tags]
-        if not meta:
-            meta = manifest_node.get("config", {}).get("meta", {})
 
         max_loaded_at_str = sources_by_id.get(key, {}).get("max_loaded_at")
         max_loaded_at = None
@@ -343,6 +356,9 @@ class DBTRunResult(BaseModel):
     def timing_map(self) -> Dict[str, DBTRunTiming]:
         return {x.name: x for x in self.timing if x.name}
 
+    def has_success_status(self) -> bool:
+        return self.status in ("pass", "success")
+
 
 class DBTRunMetadata(BaseModel):
     dbt_schema_version: str
@@ -355,12 +371,7 @@ def _parse_test_result(
     dbt_metadata: DBTRunMetadata,
     run_result: DBTRunResult,
 ) -> Optional[DBTTestResult]:
-    if run_result.status == "success":
-        # This was probably a docs generate run result, so this isn't actually
-        # a test result.
-        return None
-
-    if run_result.status != "pass":
+    if not run_result.has_success_status():
         native_results = {"message": run_result.message or ""}
         if run_result.failures:
             native_results.update({"failures": str(run_result.failures)})
@@ -455,15 +466,19 @@ def load_run_results(
 @platform_name("dbt")
 @config_class(DBTCoreConfig)
 @support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
-@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class DBTCoreSource(DBTSourceBase, TestableSource):
     config: DBTCoreConfig
+    report: DBTCoreReport
+
+    def __init__(self, config: DBTCommonConfig, ctx: PipelineContext):
+        super().__init__(config, ctx)
+        self.report = DBTCoreReport()
 
     @classmethod
     def create(cls, config_dict, ctx):
         config = DBTCoreConfig.parse_obj(config_dict)
-        return cls(config, ctx, "dbt")
+        return cls(config, ctx)
 
     @staticmethod
     def test_connection(config_dict: dict) -> TestConnectionReport:
@@ -473,9 +488,10 @@ class DBTCoreSource(DBTSourceBase, TestableSource):
             DBTCoreSource.load_file_as_json(
                 source_config.manifest_path, source_config.aws_connection
             )
-            DBTCoreSource.load_file_as_json(
-                source_config.catalog_path, source_config.aws_connection
-            )
+            if source_config.catalog_path is not None:
+                DBTCoreSource.load_file_as_json(
+                    source_config.catalog_path, source_config.aws_connection
+                )
             test_report.basic_connectivity = CapabilityReport(capable=True)
         except Exception as e:
             test_report.basic_connectivity = CapabilityReport(
@@ -513,11 +529,31 @@ class DBTCoreSource(DBTSourceBase, TestableSource):
         dbt_manifest_json = self.load_file_as_json(
             self.config.manifest_path, self.config.aws_connection
         )
-
-        dbt_catalog_json = self.load_file_as_json(
-            self.config.catalog_path, self.config.aws_connection
+        dbt_manifest_metadata = dbt_manifest_json["metadata"]
+        self.report.manifest_info = dict(
+            generated_at=dbt_manifest_metadata.get("generated_at", "unknown"),
+            dbt_version=dbt_manifest_metadata.get("dbt_version", "unknown"),
+            project_name=dbt_manifest_metadata.get("project_name", "unknown"),
         )
 
+        dbt_catalog_json = None
+        dbt_catalog_metadata = None
+        if self.config.catalog_path is not None:
+            dbt_catalog_json = self.load_file_as_json(
+                self.config.catalog_path, self.config.aws_connection
+            )
+            dbt_catalog_metadata = dbt_catalog_json.get("metadata", {})
+            self.report.catalog_info = dict(
+                generated_at=dbt_catalog_metadata.get("generated_at", "unknown"),
+                dbt_version=dbt_catalog_metadata.get("dbt_version", "unknown"),
+                project_name=dbt_catalog_metadata.get("project_name", "unknown"),
+            )
+        else:
+            self.report.warning(
+                title="No catalog file configured",
+                message="Some metadata, particularly schema information, will be missing.",
+            )
+
         if self.config.sources_path is not None:
             dbt_sources_json = self.load_file_as_json(
                 self.config.sources_path, self.config.aws_connection
@@ -530,18 +566,23 @@ class DBTCoreSource(DBTSourceBase, TestableSource):
         manifest_version = dbt_manifest_json["metadata"].get("dbt_version")
         manifest_adapter = dbt_manifest_json["metadata"].get("adapter_type")
 
-        catalog_schema = dbt_catalog_json.get("metadata", {}).get("dbt_schema_version")
-        catalog_version = dbt_catalog_json.get("metadata", {}).get("dbt_version")
+        catalog_schema = None
+        catalog_version = None
+        if dbt_catalog_metadata is not None:
+            catalog_schema = dbt_catalog_metadata.get("dbt_schema_version")
+            catalog_version = dbt_catalog_metadata.get("dbt_version")
 
         manifest_nodes = dbt_manifest_json["nodes"]
         manifest_sources = dbt_manifest_json["sources"]
 
         all_manifest_entities = {**manifest_nodes, **manifest_sources}
 
-        catalog_nodes = dbt_catalog_json["nodes"]
-        catalog_sources = dbt_catalog_json["sources"]
+        all_catalog_entities = None
+        if dbt_catalog_json is not None:
+            catalog_nodes = dbt_catalog_json["nodes"]
+            catalog_sources = dbt_catalog_json["sources"]
 
-        all_catalog_entities = {**catalog_nodes, **catalog_sources}
+            all_catalog_entities = {**catalog_nodes, **catalog_sources}
 
         nodes = extract_dbt_entities(
             all_manifest_entities=all_manifest_entities,
@@ -592,7 +633,7 @@ class DBTCoreSource(DBTSourceBase, TestableSource):
                 )
         except Exception as e:
             self.report.info(
-                title="Dbt Catalog Version",
+                title="dbt Catalog Version",
                 message="Failed to determine the catalog version",
                 exc=e,
             )

datahub/ingestion/source/dbt/dbt_tests.py

@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union
 
 from datahub.emitter import mce_builder
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.schema_classes import (
     AssertionInfoClass,
     AssertionResultClass,
@@ -43,6 +42,9 @@ class DBTTestResult:
 
     native_results: Dict[str, str]
 
+    def has_success_status(self) -> bool:
+        return self.status in ("pass", "success")
+
 
 def _get_name_for_relationship_test(kw_args: Dict[str, str]) -> Optional[str]:
     """
@@ -157,7 +159,7 @@ def make_assertion_from_test(
     node: "DBTNode",
     assertion_urn: str,
     upstream_urn: str,
-) -> MetadataWorkUnit:
+) -> MetadataChangeProposalWrapper:
     assert node.test_info
     qualified_test_name = node.test_info.qualified_test_name
     column_name = node.test_info.column_name
@@ -231,7 +233,7 @@ def make_assertion_from_test(
     return MetadataChangeProposalWrapper(
         entityUrn=assertion_urn,
         aspect=assertion_info,
-    ).as_workunit()
+    )
 
 
 def make_assertion_result_from_test(
@@ -240,7 +242,7 @@ def make_assertion_result_from_test(
     assertion_urn: str,
     upstream_urn: str,
     test_warnings_are_errors: bool,
-) -> MetadataWorkUnit:
+) -> MetadataChangeProposalWrapper:
     assertionResult = AssertionRunEventClass(
         timestampMillis=int(test_result.execution_time.timestamp() * 1000.0),
         assertionUrn=assertion_urn,
@@ -249,7 +251,7 @@ def make_assertion_result_from_test(
         result=AssertionResultClass(
             type=(
                 AssertionResultTypeClass.SUCCESS
-                if test_result.status == "pass"
+                if test_result.has_success_status()
                 or (not test_warnings_are_errors and test_result.status == "warn")
                 else AssertionResultTypeClass.FAILURE
             ),
@@ -261,4 +263,4 @@ def make_assertion_result_from_test(
     return MetadataChangeProposalWrapper(
         entityUrn=assertion_urn,
         aspect=assertionResult,
-    ).as_workunit()
+    )

datahub/ingestion/source/debug/datahub_debug.py

@@ -0,0 +1,300 @@
+import logging
+import socket
+import time
+from typing import Iterable, Optional
+from urllib.parse import urlparse
+
+import dns.exception
+import dns.resolver
+import requests
+
+from datahub.configuration.common import ConfigModel
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+
+logger = logging.getLogger(__name__)
+
+
+class DataHubDebugSourceConfig(ConfigModel):
+    dns_probe_url: Optional[str] = None
+
+
+@platform_name("DataHubDebug")
+@config_class(DataHubDebugSourceConfig)
+@support_status(SupportStatus.TESTING)
+class DataHubDebugSource(Source):
+    """
+    DataHubDebugSource is helper to debug things in executor where ingestion is running.
+
+    This source can perform the following tasks:
+    1. Network probe of a URL. Different from test connection in sources as that is after source starts.
+
+    """
+
+    def __init__(self, ctx: PipelineContext, config: DataHubDebugSourceConfig):
+        self.ctx = ctx
+        self.config = config
+        self.report = SourceReport()
+        self.report.event_not_produced_warn = False
+
+    @classmethod
+    def create(cls, config_dict, ctx):
+        config = DataHubDebugSourceConfig.parse_obj(config_dict)
+        return cls(ctx, config)
+
+    def perform_dns_probe(self, url: str) -> None:
+        """
+        Perform comprehensive DNS probe and network connectivity tests.
+        Logs detailed information to help diagnose network issues.
+        """
+        logger.info(f"Starting DNS probe for URL: {url}")
+        logger.info("=" * 60)
+        logger.info(f"DNS PROBE REPORT FOR: {url}")
+        logger.info("=" * 60)
+
+        try:
+            # Parse the URL to extract hostname
+            parsed_url = urlparse(
+                url if url.startswith(("http://", "https://")) else f"http://{url}"
+            )
+            hostname = parsed_url.hostname or parsed_url.netloc
+            port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80)
+
+            logger.info(f"Parsed hostname: {hostname}")
+            logger.info(f"Target port: {port}")
+            logger.info(f"URL scheme: {parsed_url.scheme}")
+            logger.info("-" * 60)
+
+            # Test 1: Enhanced DNS resolution with dnspython if available
+            logger.info("1. DNS RESOLUTION TEST")
+            self._dns_probe_with_dnspython(hostname)
+
+            logger.info("-" * 60)
+
+            # Test 2: HTTP/HTTPS connectivity test with requests if available
+            logger.info("2. HTTP CONNECTIVITY TEST")
+            self._http_probe_with_requests(url)
+
+            logger.info("-" * 60)
+
+            # Test 3: System network information
+            logger.info("3. SYSTEM NETWORK INFORMATION")
+            self._log_system_network_info()
+
+        except Exception as e:
+            logger.error(f"DNS probe failed with unexpected error: {e}", exc_info=True)
+
+        logger.info("=" * 60)
+        logger.info("DNS PROBE COMPLETED")
+        logger.info("=" * 60)
+
+    def _dns_probe_with_dnspython(self, hostname: str) -> None:
+        """Enhanced DNS probing using dnspython library"""
+        try:
+            # Test different record types
+            record_types = ["A", "AAAA", "CNAME", "MX"]
+
+            for record_type in record_types:
+                try:
+                    start_time = time.time()
+                    answers = dns.resolver.resolve(hostname, record_type)
+                    dns_time = time.time() - start_time
+
+                    logger.info(
+                        f"✓ {record_type} record resolution successful ({dns_time:.3f}s)"
+                    )
+                    for answer in answers:
+                        logger.info(f"  - {record_type}: {answer}")
+
+                except dns.resolver.NXDOMAIN:
+                    logger.info(f"✗ {record_type} record: Domain does not exist")
+                except dns.resolver.NoAnswer:
+                    logger.info(
+                        f"- {record_type} record: No answer (record type not available)"
+                    )
+                except dns.exception.Timeout:
+                    logger.error(f"✗ {record_type} record: DNS query timed out")
+                except Exception as e:
+                    logger.error(f"✗ {record_type} record query failed: {e}")
+
+            # Test different DNS servers
+            logger.info("Testing with different DNS servers:")
+            dns_servers = ["8.8.8.8", "1.1.1.1", "208.67.222.222"]
+
+            for dns_server in dns_servers:
+                try:
+                    resolver = dns.resolver.Resolver()
+                    resolver.nameservers = [dns_server]
+                    resolver.timeout = 5
+
+                    start_time = time.time()
+                    answers = resolver.resolve(hostname, "A")
+                    dns_time = time.time() - start_time
+
+                    logger.info(
+                        f"✓ DNS server {dns_server} responded ({dns_time:.3f}s)"
+                    )
+                    for answer in answers:
+                        logger.info(f"  - A: {answer}")
+
+                except Exception as e:
+                    logger.error(f"✗ DNS server {dns_server} failed: {e}")
+
+        except Exception as e:
+            logger.error(f"Enhanced DNS probe failed: {e}", exc_info=True)
+
+    def _http_probe_with_requests(self, url: str) -> None:
+        """HTTP connectivity test using requests library"""
+        try:
+            # Test with different timeouts and methods
+            timeout = 10
+            allow_redirects_head = True
+            allow_redirects_get = False
+
+            # Test HEAD request
+            try:
+                logger.info(f"Testing HEAD request with timeout {timeout}s")
+                start_time = time.time()
+
+                response = requests.head(
+                    url, timeout=timeout, allow_redirects=allow_redirects_head
+                )
+
+                request_time = time.time() - start_time
+
+                logger.info(f"✓ HEAD request successful ({request_time:.3f}s)")
+                logger.info(f"  Status code: {response.status_code}")
+                logger.info(
+                    f"  Response headers: {dict(list(response.headers.items())[:5])}"
+                )
+
+                if hasattr(response, "url") and response.url != url:
+                    logger.info(f"  Final URL after redirects: {response.url}")
+
+            except requests.exceptions.Timeout:
+                logger.error(f"✗ HEAD request timed out after {timeout}s")
+            except requests.exceptions.ConnectionError as e:
+                logger.error(f"✗ HEAD connection error: {e}")
+            except requests.exceptions.RequestException as e:
+                logger.error(f"✗ HEAD request failed: {e}")
+            except Exception as e:
+                logger.error(f"✗ HEAD unexpected error: {e}")
+
+            # Test GET request
+            try:
+                logger.info(f"Testing GET request with timeout {timeout}s")
+                start_time = time.time()
+
+                response = requests.get(
+                    url, timeout=timeout, allow_redirects=allow_redirects_get
+                )
+
+                request_time = time.time() - start_time
+
+                logger.info(f"✓ GET request successful ({request_time:.3f}s)")
+                logger.info(f"  Status code: {response.status_code}")
+                logger.info(
+                    f"  Response headers: {dict(list(response.headers.items())[:5])}"
+                )
+
+                if hasattr(response, "url") and response.url != url:
+                    logger.info(f"  Final URL after redirects: {response.url}")
+
+            except requests.exceptions.Timeout:
+                logger.error(f"✗ GET request timed out after {timeout}s")
+            except requests.exceptions.ConnectionError as e:
+                logger.error(f"✗ GET connection error: {e}")
+            except requests.exceptions.RequestException as e:
+                logger.error(f"✗ GET request failed: {e}")
+            except Exception as e:
+                logger.error(f"✗ GET unexpected error: {e}")
+
+        except Exception as e:
+            logger.error(f"HTTP probe failed: {e}", exc_info=True)
+
+    def _log_dns_troubleshooting(self) -> None:
+        """Log DNS troubleshooting information"""
+        logger.info("DNS TROUBLESHOOTING SUGGESTIONS:")
+        logger.info("- Check if the hostname is correct")
+        logger.info("- Verify DNS server configuration")
+        logger.info("- Check network connectivity")
+        logger.info("- Try using a different DNS server (8.8.8.8, 1.1.1.1)")
+        logger.info("- Check if there are firewall restrictions")
+
+    def _log_system_network_info(self) -> None:
+        """Log system network configuration information"""
+        try:
+            local_hostname = socket.gethostname()
+            logger.info(f"Local hostname: {local_hostname}")
+
+            try:
+                local_ips = socket.getaddrinfo(local_hostname, None)
+                logger.info("Local IP addresses:")
+                for addr_info in local_ips:
+                    if addr_info[0] in [socket.AF_INET, socket.AF_INET6]:
+                        family = "IPv4" if addr_info[0] == socket.AF_INET else "IPv6"
+                        logger.info(f"  - {addr_info[4][0]} ({family})")
+            except Exception as e:
+                logger.warning(f"Could not retrieve local IP addresses: {e}")
+
+            logger.info("DNS Server Connectivity:")
+            dns_servers = ["8.8.8.8", "1.1.1.1", "208.67.222.222"]
+            for dns_server in dns_servers:
+                try:
+                    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                    sock.settimeout(5)
+                    result = sock.connect_ex((dns_server, 53))
+                    if result == 0:
+                        logger.info(f"  ✓ Can reach {dns_server}:53")
+                    else:
+                        logger.error(f"  ✗ Cannot reach {dns_server}:53")
+                    sock.close()
+                except Exception as e:
+                    logger.error(f"  ✗ Error testing {dns_server}:53 - {e}")
+
+        except Exception as e:
+            logger.warning(f"Could not gather system network info: {e}")
+
+    def _test_alternative_dns(self, hostname: str) -> None:
+        """Test hostname resolution using alternative methods"""
+        try:
+            families = [(socket.AF_INET, "IPv4"), (socket.AF_INET6, "IPv6")]
+
+            for family, family_name in families:
+                try:
+                    result = socket.getaddrinfo(hostname, None, family)
+                    if result:
+                        logger.info(f"✓ {family_name} resolution successful:")
+                        for addr_info in result[:3]:
+                            logger.info(f"  - {addr_info[4][0]}")
+                    else:
+                        logger.warning(
+                            f"✗ {family_name} resolution returned no results"
+                        )
+                except socket.gaierror:
+                    logger.error(f"✗ {family_name} resolution failed")
+                except Exception as e:
+                    logger.error(f"✗ {family_name} resolution error: {e}")
+
+        except Exception as e:
+            logger.error(f"Alternative DNS test failed: {e}")
+
+    def get_workunits_internal(
+        self,
+    ) -> Iterable[MetadataWorkUnit]:
+        if self.config.dns_probe_url is not None:
+            # Perform DNS probe
+            logger.info(f"Performing DNS probe for: {self.config.dns_probe_url}")
+            self.perform_dns_probe(self.config.dns_probe_url)
+
+        yield from []
+
+    def get_report(self) -> SourceReport:
+        return self.report

datahub/ingestion/source/delta_lake/config.py

@@ -13,8 +13,9 @@ from datahub.configuration.source_common import (
 )
 from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
 from datahub.ingestion.source.aws.s3_util import is_s3_uri
-from datahub.ingestion.source.state.stateful_ingestion_base import (
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StatefulIngestionConfigBase,
+    StatefulStaleMetadataRemovalConfig,
 )
 
 # hide annoying debug errors from py4j
@@ -39,9 +40,7 @@ class S3(ConfigModel):
 
 
 class DeltaLakeSourceConfig(
-    PlatformInstanceConfigMixin,
-    EnvConfigMixin,
-    StatefulIngestionConfigBase,
+    PlatformInstanceConfigMixin, EnvConfigMixin, StatefulIngestionConfigBase
 ):
     base_path: str = Field(
         description="Path to table (s3 or local file system). If path is not a delta table path "
@@ -78,7 +77,12 @@ class DeltaLakeSourceConfig(
         "When set to `False`, number_of_files in delta table can not be reported.",
     )
 
-    s3: Optional[S3] = Field()
+    s3: Optional[S3] = Field(None)
+
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
+        default=None,
+        description="Stateful Ingestion Config with stale metadata removal",
+    )
 
     @cached_property
     def is_s3(self):