acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.2.0.1rc1__py3-none-any.whl

datahub/ingestion/source/data_lake_common/object_store.py

@@ -1,3 +1,4 @@
+import re
 from abc import ABC, abstractmethod
 
 # Add imports for source customization
@@ -236,42 +237,76 @@ class ABSObjectStore(ObjectStoreInterface):
     """Implementation of ObjectStoreInterface for Azure Blob Storage."""
 
     PREFIX = "abfss://"
+    HTTPS_REGEX = re.compile(r"(https?://[a-z0-9]{3,24}\.blob\.core\.windows\.net/)")
 
     @classmethod
     def is_uri(cls, uri: str) -> bool:
-        return uri.startswith(cls.PREFIX)
+        return uri.startswith(cls.PREFIX) or bool(cls.HTTPS_REGEX.match(uri))
 
     @classmethod
     def get_prefix(cls, uri: str) -> Optional[str]:
         if uri.startswith(cls.PREFIX):
             return cls.PREFIX
+
+        # Check for HTTPS format
+        match = cls.HTTPS_REGEX.match(uri)
+        if match:
+            return match.group(1)
+
         return None
 
     @classmethod
     def strip_prefix(cls, uri: str) -> str:
-        prefix = cls.get_prefix(uri)
-        if not prefix:
-            raise ValueError(f"Not an ABS URI. Must start with prefix: {cls.PREFIX}")
-        return uri[len(prefix) :]
+        if uri.startswith(cls.PREFIX):
+            return uri[len(cls.PREFIX) :]
+
+        # Handle HTTPS format
+        match = cls.HTTPS_REGEX.match(uri)
+        if match:
+            return uri[len(match.group(1)) :]
+
+        raise ValueError(
+            f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
+        )
 
     @classmethod
     def get_bucket_name(cls, uri: str) -> str:
         if not cls.is_uri(uri):
-            raise ValueError(f"Not an ABS URI. Must start with prefix: {cls.PREFIX}")
-        return cls.strip_prefix(uri).split("@")[0]
+            raise ValueError(
+                f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
+            )
+
+        if uri.startswith(cls.PREFIX):
+            # abfss://container@account.dfs.core.windows.net/path
+            return cls.strip_prefix(uri).split("@")[0]
+        else:
+            # https://account.blob.core.windows.net/container/path
+            return cls.strip_prefix(uri).split("/")[0]
 
     @classmethod
     def get_object_key(cls, uri: str) -> str:
         if not cls.is_uri(uri):
-            raise ValueError(f"Not an ABS URI. Must start with prefix: {cls.PREFIX}")
-        parts = cls.strip_prefix(uri).split("@", 1)
-        if len(parts) < 2:
-            return ""
-        account_path = parts[1]
-        path_parts = account_path.split("/", 1)
-        if len(path_parts) < 2:
-            return ""
-        return path_parts[1]
+            raise ValueError(
+                f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
+            )
+
+        if uri.startswith(cls.PREFIX):
+            # abfss://container@account.dfs.core.windows.net/path
+            parts = cls.strip_prefix(uri).split("@", 1)
+            if len(parts) < 2:
+                return ""
+            account_path = parts[1]
+            path_parts = account_path.split("/", 1)
+            if len(path_parts) < 2:
+                return ""
+            return path_parts[1]
+        else:
+            # https://account.blob.core.windows.net/container/path
+            stripped = cls.strip_prefix(uri)
+            parts = stripped.split("/", 1)
+            if len(parts) < 2:
+                return ""
+            return parts[1]
 
 
 # Registry of all object store implementations
@@ -331,6 +366,12 @@ def get_object_store_bucket_name(uri: str) -> str:
         return uri[prefix_length:].split("/")[0]
     elif uri.startswith(ABSObjectStore.PREFIX):
         return uri[len(ABSObjectStore.PREFIX) :].split("@")[0]
+    elif ABSObjectStore.HTTPS_REGEX.match(uri):
+        # Handle HTTPS Azure Blob Storage URLs
+        match = ABSObjectStore.HTTPS_REGEX.match(uri)
+        if match:
+            stripped = uri[len(match.group(1)) :]
+            return stripped.split("/")[0]
 
     raise ValueError(f"Unsupported URI format: {uri}")
 
@@ -470,18 +511,25 @@ class ObjectStoreSourceAdapter:
         if not ABSObjectStore.is_uri(table_data.table_path):
             return None
 
-        # Parse the ABS URI
         try:
-            # URI format: abfss://container@account.dfs.core.windows.net/path
-            path_without_prefix = ABSObjectStore.strip_prefix(table_data.table_path)
-            parts = path_without_prefix.split("@", 1)
-            if len(parts) < 2:
-                return None
-
-            container_name = parts[0]
-            account_parts = parts[1].split("/", 1)
-            account_domain = account_parts[0]
-            account_name = account_domain.split(".")[0]
+            if table_data.table_path.startswith("abfss://"):
+                # URI format: abfss://container@account.dfs.core.windows.net/path
+                path_without_prefix = ABSObjectStore.strip_prefix(table_data.table_path)
+                parts = path_without_prefix.split("@", 1)
+                if len(parts) < 2:
+                    return None
+
+                container_name = parts[0]
+                account_parts = parts[1].split("/", 1)
+                account_domain = account_parts[0]
+                account_name = account_domain.split(".")[0]
+            else:
+                # Handle HTTPS format: https://account.blob.core.windows.net/container/path
+                container_name = ABSObjectStore.get_bucket_name(table_data.table_path)
+                if "blob.core.windows.net" in table_data.table_path:
+                    account_name = table_data.table_path.split("//")[1].split(".")[0]
+                else:
+                    return None
 
             # Construct Azure portal URL
             return f"https://portal.azure.com/#blade/Microsoft_Azure_Storage/ContainerMenuBlade/overview/storageAccountId/{account_name}/containerName/{container_name}"
@@ -519,6 +567,13 @@ class ObjectStoreSourceAdapter:
                 "get_external_url",
                 lambda table_data: self.get_gcs_external_url(table_data),
             )
+            # Fix URI mismatch issue in pattern matching
+            self.register_customization(
+                "_normalize_uri_for_pattern_matching",
+                self._normalize_gcs_uri_for_pattern_matching,
+            )
+            # Fix URI handling in schema extraction - override strip_s3_prefix for GCS
+            self.register_customization("strip_s3_prefix", self._strip_gcs_prefix)
         elif platform == "s3":
             self.register_customization("is_s3_platform", lambda: True)
             self.register_customization("create_s3_path", self.create_s3_path)
@@ -612,6 +667,39 @@ class ObjectStoreSourceAdapter:
             return self.get_abs_external_url(table_data)
         return None
 
+    def _normalize_gcs_uri_for_pattern_matching(self, uri: str) -> str:
+        """
+        Normalize GCS URI for pattern matching.
+
+        This method converts gs:// URIs to s3:// URIs for pattern matching purposes,
+        fixing the URI mismatch issue in GCS ingestion.
+
+        Args:
+            uri: The URI to normalize
+
+        Returns:
+            The normalized URI for pattern matching
+        """
+        if uri.startswith("gs://"):
+            return uri.replace("gs://", "s3://", 1)
+        return uri
+
+    def _strip_gcs_prefix(self, uri: str) -> str:
+        """
+        Strip GCS prefix from URI.
+
+        This method removes the gs:// prefix from GCS URIs for path processing.
+
+        Args:
+            uri: The URI to strip the prefix from
+
+        Returns:
+            The URI without the gs:// prefix
+        """
+        if uri.startswith("gs://"):
+            return uri[5:]  # Remove "gs://" prefix
+        return uri
+
 
 # Factory function to create an adapter for a specific platform
 def create_object_store_adapter(

datahub/ingestion/source/data_lake_common/path_spec.py

@@ -166,7 +166,6 @@ class PathSpec(ConfigModel):
         return False
 
     def allowed(self, path: str, ignore_ext: bool = False) -> bool:
-        logger.debug(f"Checking file to inclusion: {path}")
         if self.is_path_hidden(path) and not self.include_hidden_folders:
             return False
 
@@ -174,19 +173,17 @@ class PathSpec(ConfigModel):
             self.glob_include, flags=pathlib.GLOBSTAR
         ):
             return False
-        logger.debug(f"{path} matched include ")
+
         if self.exclude:
             for exclude_path in self.exclude:
                 if pathlib.PurePath(path).globmatch(
                     exclude_path, flags=pathlib.GLOBSTAR
                 ):
                     return False
-        logger.debug(f"{path} is not excluded")
 
         table_name, _ = self.extract_table_name_and_path(path)
         if not self.tables_filter_pattern.allowed(table_name):
             return False
-        logger.debug(f"{path} is passed table name check")
 
         ext = os.path.splitext(path)[1].strip(".")
 
@@ -196,8 +193,6 @@ class PathSpec(ConfigModel):
             ):
                 return False
 
-            logger.debug(f"{path} had selected extension {ext}")
-            logger.debug(f"{path} allowed for dataset creation")
         return True
 
     def dir_allowed(self, path: str) -> bool:
@@ -219,10 +214,8 @@ class PathSpec(ConfigModel):
         for _ in range(slash_to_remove_from_glob):
             glob_include = glob_include.rsplit("/", 1)[0]
 
-        logger.debug(f"Checking dir to inclusion: {path}")
         if not pathlib.PurePath(path).globmatch(glob_include, flags=pathlib.GLOBSTAR):
             return False
-        logger.debug(f"{path} matched include ")
         if self.exclude:
             for exclude_path in self.exclude:
                 if pathlib.PurePath(path.rstrip("/")).globmatch(
@@ -236,7 +229,7 @@ class PathSpec(ConfigModel):
         )
         if not self.tables_filter_pattern.allowed(table_name):
             return False
-        logger.debug(f"{path} is passed table name check")
+        # logger.debug(f"{path} is passed table name check")
 
         return True
 
@@ -246,10 +239,10 @@ class PathSpec(ConfigModel):
         if parsable_include.endswith("/{table}/**"):
             # Remove the last two characters to make it parsable if it ends with {table}/** which marks autodetect partition
             parsable_include = parsable_include[:-2]
-        else:
-            # Replace all * with {folder[i]} to make it parsable
-            for i in range(parsable_include.count("*")):
-                parsable_include = parsable_include.replace("*", f"{{folder[{i}]}}", 1)
+
+        # Replace all * with {folder[i]} to make it parsable
+        for i in range(parsable_include.count("*")):
+            parsable_include = parsable_include.replace("*", f"{{folder[{i}]}}", 1)
         return parsable_include
 
     def get_named_vars(self, path: str) -> Union[None, parse.Result, parse.Match]:
@@ -330,8 +323,6 @@ class PathSpec(ConfigModel):
             if "{table}" in values["include"]:
                 v = "{table}"
         else:
-            logger.debug(f"include fields: {compiled_include.named_fields}")
-            logger.debug(f"table_name fields: {parse.compile(v).named_fields}")
             if not all(
                 x in compiled_include.named_fields
                 for x in parse.compile(v).named_fields
@@ -356,9 +347,7 @@ class PathSpec(ConfigModel):
     @cached_property
     def compiled_include(self):
         parsable_include = PathSpec.get_parsable_include(self.include)
-        logger.debug(f"parsable_include: {parsable_include}")
         compiled_include = parse.compile(parsable_include)
-        logger.debug(f"Setting compiled_include: {compiled_include}")
         return compiled_include
 
     @cached_property
@@ -366,9 +355,8 @@ class PathSpec(ConfigModel):
         parsable_folder_include = PathSpec.get_parsable_include(self.include).rsplit(
             "/", 1
         )[0]
-        logger.debug(f"parsable_folder_include: {parsable_folder_include}")
         compiled_folder_include = parse.compile(parsable_folder_include)
-        logger.debug(f"Setting compiled_folder_include: {compiled_folder_include}")
+
         return compiled_folder_include
 
     @cached_property
@@ -376,7 +364,8 @@ class PathSpec(ConfigModel):
         # Regular expression to find all substrings enclosed in {}
         pattern = r"\{(.*?)\}"
         # Find all matches
-        matches = re.findall(pattern, self.include.split("{table}/")[1])
+        split_parts = self.include.split("{table}/")
+        matches = re.findall(pattern, split_parts[1]) if len(split_parts) > 1 else []
         return matches
 
     def get_partition_from_path(self, path: str) -> Optional[List[Tuple[str, str]]]:
@@ -563,7 +552,7 @@ class PathSpec(ConfigModel):
                         f"{{{template_key}}}", var[key]
                     )
             else:
-                partition_format.replace(f"{{{var_key}}}", var)
+                partition_format = partition_format.replace(f"{{{var_key}}}", var)
         return datetime.datetime.strptime(partition_format, datetime_format).replace(
             tzinfo=datetime.timezone.utc
         )

datahub/ingestion/source/datahub/datahub_database_reader.py

@@ -12,7 +12,7 @@ from datahub.emitter.serialization_helper import post_json_transform
 from datahub.ingestion.source.datahub.config import DataHubSourceConfig
 from datahub.ingestion.source.datahub.report import DataHubSourceReport
 from datahub.ingestion.source.sql.sql_config import SQLAlchemyConnectionConfig
-from datahub.metadata.schema_classes import ChangeTypeClass, SystemMetadataClass
+from datahub.metadata.schema_classes import SystemMetadataClass
 from datahub.utilities.lossy_collections import LossyDict, LossyList
 
 logger = logging.getLogger(__name__)
@@ -374,7 +374,6 @@ class DataHubDatabaseReader:
                 entityUrn=row["urn"],
                 aspect=ASPECT_MAP[row["aspect"]].from_obj(json_aspect),
                 systemMetadata=system_metadata,
-                changeType=ChangeTypeClass.UPSERT,
             )
         except Exception as e:
             logger.warning(

datahub/ingestion/source/dbt/dbt_cloud.py

@@ -9,7 +9,9 @@ import requests
 from pydantic import Field, root_validator
 
 from datahub.ingestion.api.decorators import (
+    SourceCapability,
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
@@ -24,6 +26,7 @@ from datahub.ingestion.source.dbt.dbt_common import (
     DBTCommonConfig,
     DBTNode,
     DBTSourceBase,
+    DBTSourceReport,
 )
 from datahub.ingestion.source.dbt.dbt_tests import DBTTest, DBTTestResult
 
@@ -261,8 +264,10 @@ query DatahubMetadataQuery_{type}($jobId: BigInt!, $runId: BigInt) {{
 @platform_name("dbt")
 @config_class(DBTCloudConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class DBTCloudSource(DBTSourceBase, TestableSource):
     config: DBTCloudConfig
+    report: DBTSourceReport  # nothing cloud-specific in the report
 
     @classmethod
     def create(cls, config_dict, ctx):
@@ -401,8 +406,11 @@ class DBTCloudSource(DBTSourceBase, TestableSource):
         if node["resourceType"] in {"model", "seed", "snapshot"}:
             status = node["status"]
             if status is None and materialization != "ephemeral":
-                self.report.report_warning(
-                    key, "node is missing a status, schema metadata will be incomplete"
+                self.report.warning(
+                    title="Schema information may be incomplete",
+                    message="Some nodes are missing the `status` field, which dbt uses to track the status of the node in the target database.",
+                    context=key,
+                    log=False,
                 )
 
             # The code fields are new in dbt 1.3, and replace the sql ones.

datahub/ingestion/source/dbt/dbt_common.py

@@ -355,7 +355,7 @@ class DBTCommonConfig(
     # override default value to True.
     incremental_lineage: bool = Field(
         default=True,
-        description="When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run.",
+        description="When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run. This would also require enabling 'incremental_lineage' in the counterpart warehouse ingestion (_e.g._ BigQuery, Redshift, etc).",
     )
 
     _remove_use_compiled_code = pydantic_removed_field("use_compiled_code")
@@ -823,7 +823,9 @@ def get_column_type(
 @platform_name("dbt")
 @config_class(DBTCommonConfig)
 @support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
+@capability(
+    SourceCapability.DELETION_DETECTION, "Enabled by default via stateful ingestion"
+)
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 @capability(
     SourceCapability.LINEAGE_FINE,
@@ -1334,6 +1336,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
             self.config.tag_prefix,
             "SOURCE_CONTROL",
             self.config.strip_user_ids_from_email,
+            match_nested_props=True,
         )
 
         action_processor_tag = OperationProcessor(
@@ -1705,6 +1708,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
             self.config.tag_prefix,
             "SOURCE_CONTROL",
             self.config.strip_user_ids_from_email,
+            match_nested_props=True,
         )
 
         canonical_schema: List[SchemaField] = []

datahub/ingestion/source/dbt/dbt_core.py

@@ -15,7 +15,9 @@ from datahub.configuration.git import GitReference
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
+    SourceCapability,
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
@@ -464,6 +466,7 @@ def load_run_results(
 @platform_name("dbt")
 @config_class(DBTCoreConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
 class DBTCoreSource(DBTSourceBase, TestableSource):
     config: DBTCoreConfig
     report: DBTCoreReport