acryl-datahub 1.1.0.5rc3__py3-none-any.whl → 1.1.0.5rc5__py3-none-any.whl

datahub/ingestion/source/fivetran/fivetran.py

@@ -1,8 +1,8 @@
 import logging
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional, Union
 
 import datahub.emitter.mce_builder as builder
-from datahub.api.entities.datajob import DataFlow, DataJob
+from datahub.api.entities.datajob import DataJob as DataJobV1
 from datahub.api.entities.dataprocess.dataprocess_instance import (
     DataProcessInstance,
     InstanceRunResult,
@@ -42,8 +42,10 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     FineGrainedLineageDownstreamType,
     FineGrainedLineageUpstreamType,
 )
-from datahub.utilities.urns.data_flow_urn import DataFlowUrn
-from datahub.utilities.urns.dataset_urn import DatasetUrn
+from datahub.metadata.urns import CorpUserUrn, DataFlowUrn, DatasetUrn
+from datahub.sdk.dataflow import DataFlow
+from datahub.sdk.datajob import DataJob
+from datahub.sdk.entity import Entity
 
 # Logger instance
 logger = logging.getLogger(__name__)
@@ -75,8 +77,8 @@ class FivetranSource(StatefulIngestionSourceBase):
         self.audit_log = FivetranLogAPI(self.config.fivetran_log_config)
 
     def _extend_lineage(self, connector: Connector, datajob: DataJob) -> Dict[str, str]:
-        input_dataset_urn_list: List[DatasetUrn] = []
-        output_dataset_urn_list: List[DatasetUrn] = []
+        input_dataset_urn_list: List[Union[str, DatasetUrn]] = []
+        output_dataset_urn_list: List[Union[str, DatasetUrn]] = []
         fine_grained_lineage: List[FineGrainedLineage] = []
 
         # TODO: Once Fivetran exposes the database via the API, we shouldn't ask for it via config.
@@ -178,9 +180,9 @@ class FivetranSource(StatefulIngestionSourceBase):
                         )
                     )
 
-        datajob.inlets.extend(input_dataset_urn_list)
-        datajob.outlets.extend(output_dataset_urn_list)
-        datajob.fine_grained_lineages.extend(fine_grained_lineage)
+        datajob.set_inlets(input_dataset_urn_list)
+        datajob.set_outlets(output_dataset_urn_list)
+        datajob.set_fine_grained_lineages(fine_grained_lineage)
 
         return dict(
             **{
@@ -197,10 +199,10 @@ class FivetranSource(StatefulIngestionSourceBase):
 
     def _generate_dataflow_from_connector(self, connector: Connector) -> DataFlow:
         return DataFlow(
-            orchestrator=Constant.ORCHESTRATOR,
-            id=connector.connector_id,
+            platform=Constant.ORCHESTRATOR,
+            name=connector.connector_id,
             env=self.config.env,
-            name=connector.connector_name,
+            display_name=connector.connector_name,
             platform_instance=self.config.platform_instance,
         )
 
@@ -213,11 +215,11 @@ class FivetranSource(StatefulIngestionSourceBase):
         )
         owner_email = self.audit_log.get_user_email(connector.user_id)
         datajob = DataJob(
-            id=connector.connector_id,
+            name=connector.connector_id,
             flow_urn=dataflow_urn,
             platform_instance=self.config.platform_instance,
-            name=connector.connector_name,
-            owners={owner_email} if owner_email else set(),
+            display_name=connector.connector_name,
+            owners=[CorpUserUrn(owner_email)] if owner_email else None,
         )
 
         # Map connector source and destination table with dataset entity
@@ -232,16 +234,24 @@ class FivetranSource(StatefulIngestionSourceBase):
             "sync_frequency": str(connector.sync_frequency),
             "destination_id": connector.destination_id,
         }
-        datajob.properties = {
-            **connector_properties,
-            **lineage_properties,
-        }
+
+        datajob.set_custom_properties({**connector_properties, **lineage_properties})
 
         return datajob
 
     def _generate_dpi_from_job(self, job: Job, datajob: DataJob) -> DataProcessInstance:
+        # hack: convert to old instance for DataProcessInstance.from_datajob compatibility
+        datajob_v1 = DataJobV1(
+            id=datajob.name,
+            flow_urn=datajob.flow_urn,
+            platform_instance=self.config.platform_instance,
+            name=datajob.name,
+            inlets=datajob.inlets,
+            outlets=datajob.outlets,
+            fine_grained_lineages=datajob.fine_grained_lineages,
+        )
         return DataProcessInstance.from_datajob(
-            datajob=datajob,
+            datajob=datajob_v1,
             id=job.job_id,
             clone_inlets=True,
             clone_outlets=True,
@@ -278,17 +288,15 @@ class FivetranSource(StatefulIngestionSourceBase):
 
     def _get_connector_workunits(
         self, connector: Connector
-    ) -> Iterable[MetadataWorkUnit]:
+    ) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         self.report.report_connectors_scanned()
         # Create dataflow entity with same name as connector name
         dataflow = self._generate_dataflow_from_connector(connector)
-        for mcp in dataflow.generate_mcp():
-            yield mcp.as_workunit()
+        yield dataflow
 
         # Map Fivetran's connector entity with Datahub's datajob entity
         datajob = self._generate_datajob_from_connector(connector)
-        for mcp in datajob.generate_mcp(materialize_iolets=False):
-            yield mcp.as_workunit()
+        yield datajob
 
         # Map Fivetran's job/sync history entity with Datahub's data process entity
         if len(connector.jobs) >= MAX_JOBS_PER_CONNECTOR:
@@ -310,7 +318,7 @@ class FivetranSource(StatefulIngestionSourceBase):
             ).workunit_processor,
         ]
 
-    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+    def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         """
         Datahub Ingestion framework invoke this method
         """

datahub/ingestion/source/hex/api.py

@@ -5,7 +5,9 @@ from typing import Any, Dict, Generator, List, Optional, Union
 
 import requests
 from pydantic import BaseModel, Field, ValidationError, validator
+from requests.adapters import HTTPAdapter
 from typing_extensions import assert_never
+from urllib3.util.retry import Retry
 
 from datahub.ingestion.api.source import SourceReport
 from datahub.ingestion.source.hex.constants import (
@@ -220,6 +222,7 @@ class HexApi:
         self.base_url = base_url
         self.report = report
         self.page_size = page_size
+        self.session = self._create_retry_session()
 
     def _list_projects_url(self):
         return f"{self.base_url}/projects"
@@ -227,6 +230,28 @@ class HexApi:
     def _auth_header(self):
         return {"Authorization": f"Bearer {self.token}"}
 
+    def _create_retry_session(self) -> requests.Session:
+        """Create a requests session with retry logic for rate limiting.
+
+        Hex API rate limit: 60 requests per minute
+        https://learn.hex.tech/docs/api/api-overview#kernel-and-rate-limits
+        """
+        session = requests.Session()
+
+        # Configure retry strategy for 429 (Too Many Requests) with exponential backoff
+        retry_strategy = Retry(
+            total=5,  # Maximum number of retries
+            status_forcelist=[429],  # Only retry on 429 status code
+            backoff_factor=2,  # Exponential backoff: 2, 4, 8, 16, 32 seconds
+            raise_on_status=True,  # Raise exception after max retries
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
+
+        return session
+
     def fetch_projects(
         self,
         include_components: bool = True,
@@ -259,7 +284,7 @@ class HexApi:
         logger.debug(f"Fetching projects page with params: {params}")
         self.report.fetch_projects_page_calls += 1
         try:
-            response = requests.get(
+            response = self.session.get(
                 url=self._list_projects_url(),
                 headers=self._auth_header(),
                 params=params,

datahub/ingestion/source/kafka_connect/sink_connectors.py

@@ -1,3 +1,4 @@
+import logging
 import re
 from dataclasses import dataclass
 from typing import Dict, Iterable, List, Optional, Tuple
@@ -9,6 +10,81 @@ from datahub.ingestion.source.kafka_connect.common import (
     KafkaConnectLineage,
 )
 
+logger = logging.getLogger(__name__)
+
+
+class RegexRouterTransform:
+    """Helper class to handle RegexRouter transformations for topic/table names."""
+
+    def __init__(self, config: Dict[str, str]) -> None:
+        self.transforms = self._parse_transforms(config)
+
+    def _parse_transforms(self, config: Dict[str, str]) -> List[Dict[str, str]]:
+        """Parse transforms configuration from connector config."""
+        transforms_list: List[Dict[str, str]] = []
+
+        # Get the transforms parameter
+        transforms_param: str = config.get("transforms", "")
+        if not transforms_param:
+            return transforms_list
+
+        # Parse individual transforms
+        transform_names: List[str] = [
+            name.strip() for name in transforms_param.split(",")
+        ]
+
+        for transform_name in transform_names:
+            if not transform_name:
+                continue
+            transform_config: Dict[str, str] = {}
+            transform_prefix: str = f"transforms.{transform_name}."
+
+            # Extract transform configuration
+            for key, value in config.items():
+                if key.startswith(transform_prefix):
+                    config_key: str = key[len(transform_prefix) :]
+                    transform_config[config_key] = value
+
+            # Only process RegexRouter transforms
+            if (
+                transform_config.get("type")
+                == "org.apache.kafka.connect.transforms.RegexRouter"
+            ):
+                transform_config["name"] = transform_name
+                transforms_list.append(transform_config)
+
+        return transforms_list
+
+    def apply_transforms(self, topic_name: str) -> str:
+        """Apply RegexRouter transforms to the topic name using Java regex."""
+        result: str = topic_name
+
+        for transform in self.transforms:
+            regex_pattern: Optional[str] = transform.get("regex")
+            replacement: str = transform.get("replacement", "")
+
+            if regex_pattern:
+                try:
+                    # Use Java Pattern and Matcher for exact Kafka Connect compatibility
+                    from java.util.regex import Pattern
+
+                    pattern = Pattern.compile(regex_pattern)
+                    matcher = pattern.matcher(result)
+
+                    if matcher.find():
+                        # Reset matcher to beginning for replaceFirst
+                        matcher.reset()
+                        result = matcher.replaceFirst(replacement)
+                        logger.debug(
+                            f"Applied transform {transform['name']}: {topic_name} -> {result}"
+                        )
+                except Exception as e:
+                    logger.warning(
+                        f"Invalid regex pattern in transform {transform['name']}: {e}"
+                    )
+
+        return str(result)
+
 
 @dataclass
 class ConfluentS3SinkConnector(BaseConnector):
@@ -18,28 +94,35 @@ class ConfluentS3SinkConnector(BaseConnector):
         bucket: str
         topics_dir: str
         topics: Iterable[str]
+        regex_router: RegexRouterTransform
 
     def _get_parser(self, connector_manifest: ConnectorManifest) -> S3SinkParser:
         # https://docs.confluent.io/kafka-connectors/s3-sink/current/configuration_options.html#s3
-        bucket = connector_manifest.config.get("s3.bucket.name")
+        bucket: Optional[str] = connector_manifest.config.get("s3.bucket.name")
         if not bucket:
             raise ValueError(
                 "Could not find 's3.bucket.name' in connector configuration"
             )
 
         # https://docs.confluent.io/kafka-connectors/s3-sink/current/configuration_options.html#storage
-        topics_dir = connector_manifest.config.get("topics.dir", "topics")
+        topics_dir: str = connector_manifest.config.get("topics.dir", "topics")
+
+        # Create RegexRouterTransform instance
+        regex_router: RegexRouterTransform = RegexRouterTransform(
+            connector_manifest.config
+        )
 
         return self.S3SinkParser(
             target_platform="s3",
             bucket=bucket,
             topics_dir=topics_dir,
             topics=connector_manifest.topic_names,
+            regex_router=regex_router,
         )
 
     def extract_flow_property_bag(self) -> Dict[str, str]:
         # Mask/Remove properties that may reveal credentials
-        flow_property_bag = {
+        flow_property_bag: Dict[str, str] = {
             k: v
             for k, v in self.connector_manifest.config.items()
             if k
@@ -54,11 +137,17 @@ class ConfluentS3SinkConnector(BaseConnector):
 
     def extract_lineages(self) -> List[KafkaConnectLineage]:
         try:
-            parser = self._get_parser(self.connector_manifest)
+            parser: ConfluentS3SinkConnector.S3SinkParser = self._get_parser(
+                self.connector_manifest
+            )
 
             lineages: List[KafkaConnectLineage] = list()
             for topic in parser.topics:
-                target_dataset = f"{parser.bucket}/{parser.topics_dir}/{topic}"
+                # Apply RegexRouter transformations using the RegexRouterTransform class
+                transformed_topic: str = parser.regex_router.apply_transforms(topic)
+                target_dataset: str = (
+                    f"{parser.bucket}/{parser.topics_dir}/{transformed_topic}"
+                )
 
                 lineages.append(
                     KafkaConnectLineage(
@@ -86,6 +175,7 @@ class SnowflakeSinkConnector(BaseConnector):
         database_name: str
         schema_name: str
         topics_to_tables: Dict[str, str]
+        regex_router: RegexRouterTransform
 
     def get_table_name_from_topic_name(self, topic_name: str) -> str:
         """
@@ -93,7 +183,7 @@ class SnowflakeSinkConnector(BaseConnector):
         Refer below link for more info
         https://docs.snowflake.com/en/user-guide/kafka-connector-overview#target-tables-for-kafka-topics
         """
-        table_name = re.sub("[^a-zA-Z0-9_]", "_", topic_name)
+        table_name: str = re.sub("[^a-zA-Z0-9_]", "_", topic_name)
         if re.match("^[^a-zA-Z_].*", table_name):
             table_name = "_" + table_name
         # Connector  may append original topic's hash code as suffix for conflict resolution
@@ -106,8 +196,13 @@ class SnowflakeSinkConnector(BaseConnector):
         self,
         connector_manifest: ConnectorManifest,
     ) -> SnowflakeParser:
-        database_name = connector_manifest.config["snowflake.database.name"]
-        schema_name = connector_manifest.config["snowflake.schema.name"]
+        database_name: str = connector_manifest.config["snowflake.database.name"]
+        schema_name: str = connector_manifest.config["snowflake.schema.name"]
+
+        # Create RegexRouterTransform instance
+        regex_router: RegexRouterTransform = RegexRouterTransform(
+            connector_manifest.config
+        )
 
         # Fetch user provided topic to table map
         provided_topics_to_tables: Dict[str, str] = {}
@@ -121,24 +216,30 @@ class SnowflakeSinkConnector(BaseConnector):
         topics_to_tables: Dict[str, str] = {}
         # Extract lineage for only those topics whose data ingestion started
         for topic in connector_manifest.topic_names:
+            # Apply transforms first to get the transformed topic name
+            transformed_topic: str = regex_router.apply_transforms(topic)
+
             if topic in provided_topics_to_tables:
                 # If user provided which table to get mapped with this topic
                 topics_to_tables[topic] = provided_topics_to_tables[topic]
             else:
-                # Else connector converts topic name to a valid Snowflake table name.
-                topics_to_tables[topic] = self.get_table_name_from_topic_name(topic)
+                # Use the transformed topic name to generate table name
+                topics_to_tables[topic] = self.get_table_name_from_topic_name(
+                    transformed_topic
+                )
 
         return self.SnowflakeParser(
             database_name=database_name,
             schema_name=schema_name,
             topics_to_tables=topics_to_tables,
+            regex_router=regex_router,
         )
 
     def extract_flow_property_bag(self) -> Dict[str, str]:
         # For all snowflake sink connector properties, refer below link
         # https://docs.snowflake.com/en/user-guide/kafka-connector-install#configuring-the-kafka-connector
         # remove private keys, secrets from properties
-        flow_property_bag = {
+        flow_property_bag: Dict[str, str] = {
             k: v
             for k, v in self.connector_manifest.config.items()
             if k
@@ -153,10 +254,12 @@ class SnowflakeSinkConnector(BaseConnector):
 
     def extract_lineages(self) -> List[KafkaConnectLineage]:
         lineages: List[KafkaConnectLineage] = list()
-        parser = self.get_parser(self.connector_manifest)
+        parser: SnowflakeSinkConnector.SnowflakeParser = self.get_parser(
+            self.connector_manifest
+        )
 
         for topic, table in parser.topics_to_tables.items():
-            target_dataset = f"{parser.database_name}.{parser.schema_name}.{table}"
+            target_dataset: str = f"{parser.database_name}.{parser.schema_name}.{table}"
             lineages.append(
                 KafkaConnectLineage(
                     source_dataset=topic,
@@ -176,7 +279,8 @@ class BigQuerySinkConnector(BaseConnector):
         project: str
         target_platform: str
         sanitizeTopics: bool
-        transforms: list
+        transforms: List[Dict[str, str]]
+        regex_router: RegexRouterTransform
         topicsToTables: Optional[str] = None
         datasets: Optional[str] = None
         defaultDataset: Optional[str] = None
@@ -186,16 +290,18 @@ class BigQuerySinkConnector(BaseConnector):
         self,
         connector_manifest: ConnectorManifest,
     ) -> BQParser:
-        project = connector_manifest.config["project"]
-        sanitizeTopics = connector_manifest.config.get("sanitizeTopics") or "false"
-        transform_names = (
+        project: str = connector_manifest.config["project"]
+        sanitizeTopics: str = connector_manifest.config.get("sanitizeTopics") or "false"
+
+        # Parse ALL transforms (original BigQuery logic)
+        transform_names: List[str] = (
             self.connector_manifest.config.get("transforms", "").split(",")
             if self.connector_manifest.config.get("transforms")
             else []
         )
-        transforms = []
+        transforms: List[Dict[str, str]] = []
         for name in transform_names:
-            transform = {"name": name}
+            transform: Dict[str, str] = {"name": name}
             transforms.append(transform)
             for key in self.connector_manifest.config:
                 if key.startswith(f"transforms.{name}."):
@@ -203,8 +309,13 @@ class BigQuerySinkConnector(BaseConnector):
                         self.connector_manifest.config[key]
                     )
 
+        # Create RegexRouterTransform instance for RegexRouter-specific handling
+        regex_router: RegexRouterTransform = RegexRouterTransform(
+            connector_manifest.config
+        )
+
         if "defaultDataset" in connector_manifest.config:
-            defaultDataset = connector_manifest.config["defaultDataset"]
+            defaultDataset: str = connector_manifest.config["defaultDataset"]
             return self.BQParser(
                 project=project,
                 defaultDataset=defaultDataset,
@@ -212,11 +323,14 @@ class BigQuerySinkConnector(BaseConnector):
                 sanitizeTopics=sanitizeTopics.lower() == "true",
                 version="v2",
                 transforms=transforms,
+                regex_router=regex_router,
             )
         else:
             # version 1.6.x and similar configs supported
-            datasets = connector_manifest.config["datasets"]
-            topicsToTables = connector_manifest.config.get("topicsToTables")
+            datasets: str = connector_manifest.config["datasets"]
+            topicsToTables: Optional[str] = connector_manifest.config.get(
+                "topicsToTables"
+            )
 
             return self.BQParser(
                 project=project,
@@ -225,10 +339,11 @@ class BigQuerySinkConnector(BaseConnector):
                 target_platform="bigquery",
                 sanitizeTopics=sanitizeTopics.lower() == "true",
                 transforms=transforms,
+                regex_router=regex_router,
             )
 
     def get_list(self, property: str) -> Iterable[Tuple[str, str]]:
-        entries = property.split(",")
+        entries: List[str] = property.split(",")
         for entry in entries:
             key, val = entry.rsplit("=")
             yield (key.strip(), val.strip())
@@ -243,7 +358,7 @@ class BigQuerySinkConnector(BaseConnector):
                 return dataset
         return None
 
-    def sanitize_table_name(self, table_name):
+    def sanitize_table_name(self, table_name: str) -> str:
         table_name = re.sub("[^a-zA-Z0-9_]", "_", table_name)
         if re.match("^[^a-zA-Z_].*", table_name):
             table_name = "_" + table_name
@@ -254,8 +369,8 @@ class BigQuerySinkConnector(BaseConnector):
         self, topic: str, parser: BQParser
     ) -> Optional[str]:
         if parser.version == "v2":
-            dataset = parser.defaultDataset
-            parts = topic.split(":")
+            dataset: Optional[str] = parser.defaultDataset
+            parts: List[str] = topic.split(":")
             if len(parts) == 2:
                 dataset = parts[0]
                 table = parts[1]
@@ -283,21 +398,9 @@ class BigQuerySinkConnector(BaseConnector):
             table = self.sanitize_table_name(table)
         return f"{dataset}.{table}"
 
-    def apply_transformations(
-        self, topic: str, transforms: List[Dict[str, str]]
-    ) -> str:
-        for transform in transforms:
-            if transform["type"] == "org.apache.kafka.connect.transforms.RegexRouter":
-                regex = transform["regex"]
-                replacement = transform["replacement"]
-                pattern = re.compile(regex)
-                if pattern.match(topic):
-                    topic = pattern.sub(replacement, topic, count=1)
-        return topic
-
     def extract_flow_property_bag(self) -> Dict[str, str]:
         # Mask/Remove properties that may reveal credentials
-        flow_property_bag = {
+        flow_property_bag: Dict[str, str] = {
             k: v
             for k, v in self.connector_manifest.config.items()
             if k not in ["keyfile"]
@@ -307,27 +410,33 @@ class BigQuerySinkConnector(BaseConnector):
 
     def extract_lineages(self) -> List[KafkaConnectLineage]:
         lineages: List[KafkaConnectLineage] = list()
-        parser = self.get_parser(self.connector_manifest)
+        parser: BigQuerySinkConnector.BQParser = self.get_parser(
+            self.connector_manifest
+        )
         if not parser:
             return lineages
-        target_platform = parser.target_platform
-        project = parser.project
-        transforms = parser.transforms
+        target_platform: str = parser.target_platform
+        project: str = parser.project
 
         for topic in self.connector_manifest.topic_names:
-            transformed_topic = self.apply_transformations(topic, transforms)
-            dataset_table = self.get_dataset_table_for_topic(transformed_topic, parser)
+            # Apply RegexRouter transformations using the RegexRouterTransform class
+            transformed_topic: str = parser.regex_router.apply_transforms(topic)
+
+            # Use the transformed topic to determine dataset/table
+            dataset_table: Optional[str] = self.get_dataset_table_for_topic(
+                transformed_topic, parser
+            )
             if dataset_table is None:
                 self.report.warning(
                     "Could not find target dataset for topic, please check your connector configuration"
                     f"{self.connector_manifest.name} : {transformed_topic} ",
                 )
                 continue
-            target_dataset = f"{project}.{dataset_table}"
+            target_dataset: str = f"{project}.{dataset_table}"
 
             lineages.append(
                 KafkaConnectLineage(
-                    source_dataset=transformed_topic,
+                    source_dataset=topic,  # Keep original topic as source
                     source_platform=KAFKA,
                     target_dataset=target_dataset,
                     target_platform=target_platform,

datahub/ingestion/source/mock_data/datahub_mock_data.py

@@ -15,6 +15,7 @@ from datahub.ingestion.api.decorators import (
 )
 from datahub.ingestion.api.source import Source, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import DatasetSubTypes
 from datahub.ingestion.source.mock_data.datahub_mock_data_report import (
     DataHubMockDataReport,
 )
@@ -211,15 +212,19 @@ class DataHubMockDataSource(Source):
         pattern = self.config.gen_1.subtype_pattern
 
         if pattern == SubTypePattern.ALTERNATING:
-            return "Table" if table_index % 2 == 0 else "View"
+            return (
+                DatasetSubTypes.TABLE if table_index % 2 == 0 else DatasetSubTypes.VIEW
+            )
         elif pattern == SubTypePattern.LEVEL_BASED:
-            return self.config.gen_1.level_subtypes.get(table_level, "Table")
+            return self.config.gen_1.level_subtypes.get(
+                table_level, DatasetSubTypes.TABLE
+            )
         elif pattern == SubTypePattern.ALL_TABLE:
-            return "Table"
+            return DatasetSubTypes.TABLE
         elif pattern == SubTypePattern.ALL_VIEW:
-            return "View"
+            return DatasetSubTypes.VIEW
         else:
-            return "Table"  # default
+            return DatasetSubTypes.TABLE  # default
 
     def _get_subtypes_aspect(
         self, table_name: str, table_level: int, table_index: int
@@ -261,11 +266,8 @@ class DataHubMockDataSource(Source):
             fan_out, hops, fan_out_after_first
         )
 
-        logger.info(
-            f"About to create {tables_to_be_created} tables for lineage testing"
-        )
+        logger.info(f"About to create {tables_to_be_created} datasets mock data")
 
-        current_progress = 0
         for i in range(hops + 1):
             tables_at_level = tables_at_levels[i]
 
@@ -286,12 +288,6 @@ class DataHubMockDataSource(Source):
                     tables_at_levels=tables_at_levels,
                 )
 
-                current_progress += 1
-                if current_progress % 1000 == 0:
-                    logger.info(
-                        f"Progress: {current_progress}/{tables_to_be_created} tables processed"
-                    )
-
     def _generate_lineage_for_table(
         self,
         table_name: str,

datahub/ingestion/source/salesforce.py

@@ -33,7 +33,10 @@ from datahub.ingestion.api.decorators import (
 )
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.common.subtypes import DatasetSubTypes
+from datahub.ingestion.source.common.subtypes import (
+    DatasetSubTypes,
+    SourceCapabilityModifier,
+)
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
     StaleEntityRemovalSourceReport,
@@ -532,11 +535,11 @@ class SalesforceApi:
 @capability(
     capability_name=SourceCapability.DATA_PROFILING,
     description="Only table level profiling is supported via `profiling.enabled` config field",
+    subtype_modifier=[SourceCapabilityModifier.TABLE],
 )
 @capability(
     capability_name=SourceCapability.DELETION_DETECTION,
-    description="Not supported yet",
-    supported=False,
+    description="Enabled by default via stateful ingestion",
 )
 @capability(
     capability_name=SourceCapability.SCHEMA_METADATA,

datahub/ingestion/source/slack/slack.py

@@ -23,6 +23,7 @@ from datahub.ingestion.api.source import (
     SourceReport,
 )
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import DatasetSubTypes
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalHandler,
     StaleEntityRemovalSourceReport,
@@ -493,7 +494,7 @@ class SlackSource(StatefulIngestionSourceBase):
                     mcp=MetadataChangeProposalWrapper(
                         entityUrn=urn_channel,
                         aspect=SubTypesClass(
-                            typeNames=["Slack Channel"],
+                            typeNames=[DatasetSubTypes.SLACK_CHANNEL],
                         ),
                     ),
                 )