acryl-datahub 1.1.1rc3__py3-none-any.whl → 1.2.0__py3-none-any.whl

datahub/ingestion/source/mock_data/datahub_mock_data.py

@@ -0,0 +1,472 @@
+import logging
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from pydantic import Field
+
+from datahub.configuration.common import ConfigModel
+from datahub.emitter.mce_builder import make_dataset_urn
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import DatasetSubTypes
+from datahub.ingestion.source.mock_data.datahub_mock_data_report import (
+    DataHubMockDataReport,
+)
+from datahub.ingestion.source.mock_data.table_naming_helper import TableNamingHelper
+from datahub.metadata.schema_classes import (
+    CalendarIntervalClass,
+    DatasetLineageTypeClass,
+    DatasetProfileClass,
+    DatasetUsageStatisticsClass,
+    StatusClass,
+    SubTypesClass,
+    TimeWindowSizeClass,
+    UpstreamClass,
+    UpstreamLineageClass,
+)
+from datahub.utilities.str_enum import StrEnum
+
+logger = logging.getLogger(__name__)
+
+
+class SubTypePattern(StrEnum):
+    ALTERNATING = "alternating"
+    ALL_TABLE = "all_table"
+    ALL_VIEW = "all_view"
+    LEVEL_BASED = "level_based"
+
+
+class LineageConfigGen1(ConfigModel):
+    """
+    Configuration for generating mock lineage data for testing purposes.
+
+    This configuration controls how the mock data source generates a hierarchical
+    lineage graph with multiple levels of upstream/downstream relationships.
+
+    The lineage graph is structured as follows:
+    - Level 0: 1 table (root)
+    - Level 1: lineage_fan_out tables (each connected to the root)
+    - Level 2+: If lineage_fan_out_after_first_hop is set, uses that value;
+                otherwise uses lineage_fan_out^level tables (each connected to a level 1 table)
+    - ... and so on for lineage_hops levels
+
+    Examples:
+        - With lineage_fan_out=2, lineage_hops=1: Creates 3 tables total
+          (1 root + 2 downstream) with 2 lineage relationships
+        - With lineage_fan_out=3, lineage_hops=2: Creates 13 tables total
+          (1 + 3 + 9) with 12 lineage relationships
+        - With lineage_fan_out=4, lineage_hops=1: Creates 5 tables total
+          (1 + 4) with 4 lineage relationships
+        - With lineage_fan_out=3, lineage_hops=3, lineage_fan_out_after_first_hop=2:
+          Creates 1 + 3 + 6 + 12 = 22 tables total (prevents exponential growth)
+
+    Table naming convention: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+    """
+
+    enabled: bool = Field(
+        default=False,
+        description="Whether this source is enabled",
+    )
+
+    emit_lineage: bool = Field(
+        default=True,
+        description="Whether to emit lineage data for testing purposes. When False, no lineage data is generated regardless of other settings.",
+    )
+    emit_usage: bool = Field(
+        default=True,
+        description="Whether to emit usage data for testing purposes. When False, no usage data is generated regardless of other settings.",
+    )
+
+    lineage_fan_out: int = Field(
+        default=3,
+        description="Number of downstream tables that each upstream table connects to. This controls the 'width' of the lineage graph. Higher values create more parallel downstream tables per level.",
+    )
+
+    lineage_hops: int = Field(
+        default=2,
+        description="Number of hops (levels) in the lineage graph. This controls the 'depth' of the lineage graph. Level 0 is the root table, and each subsequent level contains downstream tables. Higher values create deeper lineage chains.",
+    )
+
+    lineage_fan_out_after_first_hop: Optional[int] = Field(
+        default=None,
+        description="Optional limit on fanout for hops after the first hop. When set, prevents exponential growth by limiting the number of downstream tables per upstream table at levels 2 and beyond. When None, uses the standard exponential growth (lineage_fan_out^level).",
+    )
+
+    subtype_pattern: SubTypePattern = Field(
+        default=SubTypePattern.ALTERNATING,
+        description="Pattern for determining SubTypes. Options: 'alternating', 'all_table', 'all_view', 'level_based'",
+    )
+
+    level_subtypes: Dict[int, str] = Field(
+        default={0: "Table", 1: "View", 2: "Table"},
+        description="Mapping of level to subtype for level_based pattern",
+    )
+
+
+class DataHubMockDataConfig(ConfigModel):
+    enabled: bool = Field(
+        default=True,
+        description="Whether this source is enabled",
+    )
+    throw_uncaught_exceptions: bool = Field(
+        default=False,
+        description="Whether to throw an uncaught exception for testing",
+    )
+    num_errors: int = Field(
+        default=0,
+        description="Number of errors to add in report for testing",
+    )
+    num_warnings: int = Field(
+        default=0,
+        description="Number of warnings to add in report for testing",
+    )
+
+    gen_1: LineageConfigGen1 = Field(
+        default_factory=LineageConfigGen1,
+        description="Configuration for lineage data generation",
+    )
+
+
+@platform_name("DataHubMockData")
+@config_class(DataHubMockDataConfig)
+@support_status(SupportStatus.TESTING)
+class DataHubMockDataSource(Source):
+    """
+    This source is for generating mock data for testing purposes.
+    Expect breaking changes as we iterate on the mock data source.
+    """
+
+    def __init__(self, ctx: PipelineContext, config: DataHubMockDataConfig):
+        self.ctx = ctx
+        self.config = config
+        self.report = DataHubMockDataReport()
+
+    def get_workunits(self) -> Iterable[MetadataWorkUnit]:
+        if self.config.throw_uncaught_exceptions:
+            raise Exception("This is a test exception")
+
+        if self.config.num_errors > 0:
+            for i in range(self.config.num_errors):
+                self.report.failure(
+                    message="This is test error message",
+                    title="Test Error",
+                    context=f"This is test error {i}",
+                )
+
+        if self.config.num_warnings > 0:
+            for i in range(self.config.num_warnings):
+                self.report.warning(
+                    message="This is test warning",
+                    title="Test Warning",
+                    context=f"This is test warning {i}",
+                )
+
+        # We don't want any implicit aspects to be produced
+        # so we are not using get_workunits_internal
+        if self.config.gen_1.enabled:
+            for wu in self._data_gen_1():
+                if self.report.first_urn_seen is None:
+                    self.report.first_urn_seen = wu.get_urn()
+                self.report.report_workunit(wu)
+                yield wu
+
+        yield from []
+
+    def _calculate_lineage_tables(
+        self, fan_out: int, hops: int, fan_out_after_first: Optional[int] = None
+    ) -> Tuple[int, List[int]]:
+        """
+        Calculate the total number of tables and tables at each level for lineage generation.
+
+        Args:
+            fan_out: Number of downstream tables per upstream table at level 1
+            hops: Number of hops (levels) in the lineage graph
+            fan_out_after_first: Optional limit on fanout for hops after the first hop
+
+        Returns:
+            Tuple of (total_tables, tables_at_levels) where tables_at_levels is a list
+            containing the number of tables at each level (index 0 = level 0, etc.)
+        """
+        tables_to_be_created = 0
+        tables_at_levels: List[int] = []
+
+        for i in range(hops + 1):
+            if i == 0:
+                # Level 0: always 1 table
+                tables_at_level = 1
+            elif i == 1:
+                # Level 1: uses lineage_fan_out
+                tables_at_level = fan_out
+            else:
+                # Level 2+: use fan_out_after_first_hop if set, otherwise exponential growth
+                if fan_out_after_first is not None:
+                    # Each table at previous level creates fan_out_after_first tables
+                    tables_at_level = tables_at_levels[i - 1] * fan_out_after_first
+                else:
+                    # Original exponential behavior
+                    tables_at_level = fan_out**i
+
+            tables_at_levels.append(tables_at_level)
+            tables_to_be_created += tables_at_level
+
+        return tables_to_be_created, tables_at_levels
+
+    def _calculate_fanout_for_level(
+        self, level: int, fan_out: int, fan_out_after_first: Optional[int] = None
+    ) -> int:
+        """
+        Calculate the fanout (number of downstream tables) for a specific level.
+
+        Args:
+            level: The current level (0-based)
+            fan_out: Number of downstream tables per upstream table at level 1
+            fan_out_after_first: Optional limit on fanout for hops after the first hop
+
+        Returns:
+            The number of downstream tables that each table at this level should connect to
+        """
+        if level == 0:
+            # Level 0: uses the standard fan_out
+            return fan_out
+        else:
+            # Level 1+: use fan_out_after_first if set, otherwise use fan_out
+            return fan_out_after_first if fan_out_after_first is not None else fan_out
+
+    def _determine_subtype(
+        self, table_name: str, table_level: int, table_index: int
+    ) -> str:
+        """
+        Determine subtype based on configured pattern.
+
+        Args:
+            table_name: Name of the table
+            table_level: Level of the table in the lineage graph
+            table_index: Index of the table within its level
+
+        Returns:
+            The determined subtype ("Table" or "View")
+        """
+        pattern = self.config.gen_1.subtype_pattern
+
+        if pattern == SubTypePattern.ALTERNATING:
+            return (
+                DatasetSubTypes.TABLE if table_index % 2 == 0 else DatasetSubTypes.VIEW
+            )
+        elif pattern == SubTypePattern.LEVEL_BASED:
+            return self.config.gen_1.level_subtypes.get(
+                table_level, DatasetSubTypes.TABLE
+            )
+        elif pattern == SubTypePattern.ALL_TABLE:
+            return DatasetSubTypes.TABLE
+        elif pattern == SubTypePattern.ALL_VIEW:
+            return DatasetSubTypes.VIEW
+        else:
+            return DatasetSubTypes.TABLE  # default
+
+    def _get_subtypes_aspect(
+        self, table_name: str, table_level: int, table_index: int
+    ) -> MetadataWorkUnit:
+        """
+        Create a SubTypes aspect for a table based on deterministic pattern.
+
+        Args:
+            table_name: Name of the table
+            table_level: Level of the table in the lineage graph
+            table_index: Index of the table within its level
+
+        Returns:
+            MetadataWorkUnit containing the SubTypes aspect
+        """
+        # Determine subtype based on pattern
+        subtype = self._determine_subtype(table_name, table_level, table_index)
+
+        urn = make_dataset_urn(platform="fake", name=table_name)
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=urn,
+            entityType="dataset",
+            aspect=SubTypesClass(typeNames=[subtype]),
+        )
+        return mcp.as_workunit()
+
+    def _data_gen_1(self) -> Iterable[MetadataWorkUnit]:
+        """Generate mock lineage data for testing purposes."""
+        gen_1 = self.config.gen_1
+        fan_out = gen_1.lineage_fan_out
+        hops = gen_1.lineage_hops
+        fan_out_after_first = gen_1.lineage_fan_out_after_first_hop
+
+        logger.info(
+            f"Generating lineage data with fan_out={fan_out}, hops={hops}, fan_out_after_first={fan_out_after_first}"
+        )
+
+        tables_to_be_created, tables_at_levels = self._calculate_lineage_tables(
+            fan_out, hops, fan_out_after_first
+        )
+
+        logger.info(f"About to create {tables_to_be_created} datasets mock data")
+
+        for i in range(hops + 1):
+            tables_at_level = tables_at_levels[i]
+
+            for j in range(tables_at_level):
+                table_name = TableNamingHelper.generate_table_name(hops, fan_out, i, j)
+
+                yield self._get_status_aspect(table_name)
+
+                yield self._get_subtypes_aspect(table_name, i, j)
+
+                yield self._get_profile_aspect(table_name)
+
+                if self.config.gen_1.emit_usage:
+                    yield self._get_usage_aspect(table_name)
+
+                if self.config.gen_1.emit_lineage:
+                    yield from self._generate_lineage_for_table(
+                        table_name=table_name,
+                        table_level=i,
+                        table_index=j,
+                        hops=hops,
+                        fan_out=fan_out,
+                        fan_out_after_first=fan_out_after_first,
+                        tables_at_levels=tables_at_levels,
+                    )
+
+    def _generate_lineage_for_table(
+        self,
+        table_name: str,
+        table_level: int,
+        table_index: int,
+        hops: int,
+        fan_out: int,
+        fan_out_after_first: Optional[int],
+        tables_at_levels: List[int],
+    ) -> Iterable[MetadataWorkUnit]:
+        """Generate lineage relationships for a specific table."""
+        # Only generate lineage if there are downstream levels
+        if table_level + 1 > hops:
+            return
+
+        current_fan_out = self._calculate_fanout_for_level(
+            table_level, fan_out, fan_out_after_first
+        )
+
+        yield from self._generate_downstream_lineage(
+            upstream_table_name=table_name,
+            upstream_table_index=table_index,
+            upstream_table_level=table_level,
+            current_fan_out=current_fan_out,
+            hops=hops,
+            fan_out=fan_out,
+            tables_at_levels=tables_at_levels,
+        )
+
+    def _generate_downstream_lineage(
+        self,
+        upstream_table_name: str,
+        upstream_table_index: int,
+        upstream_table_level: int,
+        current_fan_out: int,
+        hops: int,
+        fan_out: int,
+        tables_at_levels: List[int],
+    ) -> Iterable[MetadataWorkUnit]:
+        """Generate lineage relationships to downstream tables."""
+        downstream_level = upstream_table_level + 1
+        downstream_tables_count = tables_at_levels[downstream_level]
+
+        # Calculate range of downstream tables this upstream table connects to
+        start_downstream = upstream_table_index * current_fan_out
+        end_downstream = min(
+            (upstream_table_index + 1) * current_fan_out, downstream_tables_count
+        )
+
+        for downstream_index in range(start_downstream, end_downstream):
+            downstream_table_name = TableNamingHelper.generate_table_name(
+                hops, fan_out, downstream_level, downstream_index
+            )
+            yield self._get_upstream_aspect(
+                upstream_table=upstream_table_name,
+                downstream_table=downstream_table_name,
+            )
+
+    def _get_status_aspect(self, table: str) -> MetadataWorkUnit:
+        urn = make_dataset_urn(
+            platform="fake",
+            name=table,
+        )
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=urn,
+            entityType="dataset",
+            aspect=StatusClass(removed=False),
+        )
+        return mcp.as_workunit()
+
+    def _get_upstream_aspect(
+        self, upstream_table: str, downstream_table: str
+    ) -> MetadataWorkUnit:
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=make_dataset_urn(
+                platform="fake",
+                name=downstream_table,
+            ),
+            entityType="dataset",
+            aspect=UpstreamLineageClass(
+                upstreams=[
+                    UpstreamClass(
+                        dataset=make_dataset_urn(
+                            platform="fake",
+                            name=upstream_table,
+                        ),
+                        type=DatasetLineageTypeClass.TRANSFORMED,
+                    )
+                ],
+            ),
+        )
+        return mcp.as_workunit()
+
+    def _get_profile_aspect(self, table: str) -> MetadataWorkUnit:
+        urn = make_dataset_urn(
+            platform="fake",
+            name=table,
+        )
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=urn,
+            entityType="dataset",
+            aspect=DatasetProfileClass(
+                timestampMillis=0,
+                rowCount=100,
+                columnCount=10,
+                sizeInBytes=1000,
+            ),
+        )
+        return mcp.as_workunit()
+
+    def _get_usage_aspect(self, table: str) -> MetadataWorkUnit:
+        urn = make_dataset_urn(
+            platform="fake",
+            name=table,
+        )
+        mcp = MetadataChangeProposalWrapper(
+            entityUrn=urn,
+            entityType="dataset",
+            aspect=DatasetUsageStatisticsClass(
+                timestampMillis=0,
+                eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY),
+                uniqueUserCount=0,
+                totalSqlQueries=0,
+                topSqlQueries=[],
+                userCounts=[],
+                fieldCounts=[],
+            ),
+        )
+        return mcp.as_workunit()
+
+    def get_report(self) -> SourceReport:
+        return self.report

datahub/ingestion/source/mock_data/datahub_mock_data_report.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass, field
+from typing import Optional
+
+from datahub.ingestion.api.source import SourceReport
+
+
+@dataclass
+class DataHubMockDataReport(SourceReport):
+    first_urn_seen: Optional[str] = field(
+        default=None,
+        metadata={"description": "The first URN encountered during ingestion"},
+    )

datahub/ingestion/source/mock_data/table_naming_helper.py

@@ -0,0 +1,91 @@
+from typing import Dict
+
+
+class TableNamingHelper:
+    """
+    Helper class for managing table naming conventions in mock data generation.
+
+    Table naming pattern: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+    """
+
+    @staticmethod
+    def generate_table_name(
+        lineage_hops: int, lineage_fan_out: int, level: int, table_index: int
+    ) -> str:
+        """
+        Generate a table name following the standard naming convention.
+
+        Args:
+            lineage_hops: Total number of hops in the lineage graph
+            lineage_fan_out: Number of downstream tables per upstream table
+            level: Level of the table in the lineage graph (0-based)
+            table_index: Index of the table within its level (0-based)
+
+        Returns:
+            Table name following the pattern: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+        """
+        return f"hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+
+    @staticmethod
+    def parse_table_name(table_name: str) -> Dict[str, int]:
+        """
+        Parse a table name to extract its components.
+
+        Args:
+            table_name: Table name following the standard naming convention
+
+        Returns:
+            Dictionary containing parsed components:
+            - lineage_hops: Total number of hops in the lineage graph
+            - lineage_fan_out: Number of downstream tables per upstream table
+            - level: Level of the table in the lineage graph (0-based)
+            - table_index: Index of the table within its level (0-based)
+
+        Raises:
+            ValueError: If the table name doesn't follow the expected pattern
+        """
+        try:
+            # Expected pattern: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+            parts = table_name.split("_")
+
+            if (
+                len(parts) != 6
+                or parts[0] != "hops"
+                or parts[2] != "f"
+                or not parts[4].startswith("h")
+                or not parts[5].startswith("t")
+            ):
+                raise ValueError(f"Invalid table name format: {table_name}")
+
+            lineage_hops = int(parts[1])
+            lineage_fan_out = int(parts[3])  # lineage_fan_out is at index 3
+            level = int(parts[4][1:])  # Remove 'h' prefix from parts[4]
+            table_index = int(parts[5][1:])  # Remove 't' prefix from parts[5]
+
+            return {
+                "lineage_hops": lineage_hops,
+                "lineage_fan_out": lineage_fan_out,
+                "level": level,
+                "table_index": table_index,
+            }
+        except (ValueError, IndexError) as e:
+            raise ValueError(
+                f"Failed to parse table name '{table_name}': {str(e)}"
+            ) from e
+
+    @staticmethod
+    def is_valid_table_name(table_name: str) -> bool:
+        """
+        Check if a table name follows the expected naming convention.
+
+        Args:
+            table_name: Table name to validate
+
+        Returns:
+            True if the table name follows the expected pattern, False otherwise
+        """
+        try:
+            TableNamingHelper.parse_table_name(table_name)
+            return True
+        except ValueError:
+            return False

datahub/ingestion/source/nifi.py

@@ -72,7 +72,7 @@ NIFI = "nifi"
 # and here - https://github.com/psf/requests/issues/1573
 class SSLAdapter(HTTPAdapter):
     def __init__(self, certfile, keyfile, password=None):
-        self.context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        self.context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
         self.context.load_cert_chain(
             certfile=certfile, keyfile=keyfile, password=password
         )

datahub/ingestion/source/openapi.py

@@ -82,6 +82,9 @@ class OpenApiConfig(ConfigModel):
     get_token: dict = Field(
         default={}, description="Retrieving a token from the endpoint."
     )
+    verify_ssl: bool = Field(
+        default=True, description="Enable SSL certificate verification"
+    )
 
     @validator("bearer_token", always=True)
     def ensure_only_one_token(
@@ -129,12 +132,14 @@ class OpenApiConfig(ConfigModel):
                     tok_url=url4req,
                     method=self.get_token["request_type"],
                     proxies=self.proxies,
+                    verify_ssl=self.verify_ssl,
                 )
             sw_dict = get_swag_json(
                 self.url,
                 token=self.token,
                 swagger_file=self.swagger_file,
                 proxies=self.proxies,
+                verify_ssl=self.verify_ssl,
             )  # load the swagger file
 
         else:  # using basic auth for accessing endpoints
@@ -144,6 +149,7 @@ class OpenApiConfig(ConfigModel):
                 password=self.password,
                 swagger_file=self.swagger_file,
                 proxies=self.proxies,
+                verify_ssl=self.verify_ssl,
             )
         return sw_dict
 
@@ -343,6 +349,7 @@ class APISource(Source, ABC):
                         tot_url,
                         token=config.token,
                         proxies=config.proxies,
+                        verify_ssl=config.verify_ssl,
                     )
                 else:
                     response = request_call(
@@ -350,6 +357,7 @@ class APISource(Source, ABC):
                         username=config.username,
                         password=config.password,
                         proxies=config.proxies,
+                        verify_ssl=config.verify_ssl,
                     )
                 if response.status_code == 200:
                     fields2add, root_dataset_samples[dataset_name] = extract_fields(
@@ -380,6 +388,7 @@ class APISource(Source, ABC):
                             tot_url,
                             token=config.token,
                             proxies=config.proxies,
+                            verify_ssl=config.verify_ssl,
                         )
                     else:
                         response = request_call(
@@ -387,6 +396,7 @@ class APISource(Source, ABC):
                             username=config.username,
                             password=config.password,
                             proxies=config.proxies,
+                            verify_ssl=config.verify_ssl,
                         )
                     if response.status_code == 200:
                         fields2add, _ = extract_fields(response, dataset_name)
@@ -415,6 +425,7 @@ class APISource(Source, ABC):
                             tot_url,
                             token=config.token,
                             proxies=config.proxies,
+                            verify_ssl=config.verify_ssl,
                         )
                     else:
                         response = request_call(
@@ -422,6 +433,7 @@ class APISource(Source, ABC):
                             username=config.username,
                             password=config.password,
                             proxies=config.proxies,
+                            verify_ssl=config.verify_ssl,
                         )
                     if response.status_code == 200:
                         fields2add, _ = extract_fields(response, dataset_name)