acryl-datahub 1.2.0.3rc2__py3-none-any.whl → 1.2.0.4__py3-none-any.whl

datahub/cli/quickstart_versioning.py

@@ -44,7 +44,7 @@ def get_minimum_supported_version_message(version: str) -> str:
 class QuickstartExecutionPlan(BaseModel):
     composefile_git_ref: str
     docker_tag: str
-    mysql_tag: Optional[str]
+    mysql_tag: Optional[str] = None
 
 
 def _is_it_a_version(version: str) -> bool:

datahub/cli/specific/assertions_cli.py

@@ -1,3 +1,8 @@
+"""
+DEPRECATED: This assertions CLI is no longer supported and will be removed in a future version.
+Please use alternative methods for managing assertions in DataHub.
+"""
+
 import logging
 import os
 from pathlib import Path
@@ -26,7 +31,18 @@ REPORT_FILE_NAME = "compile_report.json"
 
 @click.group(cls=DefaultGroup, default="upsert")
 def assertions() -> None:
-    """A group of commands to interact with the Assertion entity in DataHub."""
+    """A group of commands to interact with the Assertion entity in DataHub.
+
+    ⚠️  DEPRECATED: This assertions CLI is no longer supported and will be removed
+    in a future version. Please use alternative methods for managing assertions in DataHub.
+    """
+    click.secho(
+        "⚠️  WARNING: The assertions CLI is deprecated and no longer supported. "
+        "It may be removed in a future version. Please use alternative methods for managing assertions in DataHub.",
+        fg="yellow",
+        bold=True,
+        err=True,
+    )
     pass
 
 
@@ -34,7 +50,16 @@ def assertions() -> None:
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
 @upgrade.check_upgrade
 def upsert(file: str) -> None:
-    """Upsert (create or update) a set of assertions in DataHub."""
+    """Upsert (create or update) a set of assertions in DataHub.
+
+    ⚠️  DEPRECATED: This command is deprecated and no longer supported.
+    """
+    click.secho(
+        "⚠️  WARNING: The 'upsert' command is deprecated and no longer supported.",
+        fg="yellow",
+        bold=True,
+        err=True,
+    )
 
     assertions_spec: AssertionsConfigSpec = AssertionsConfigSpec.from_yaml(file)
 
@@ -78,7 +103,15 @@ def compile(
     In future, we may introduce separate command to automatically apply these compiled changes
     in assertion platform. Currently, generated result artifacts are stored in target folder
     unless another folder is specified using option `--output-to <folder>`.
+
+    ⚠️  DEPRECATED: This command is deprecated and no longer supported.
     """
+    click.secho(
+        "⚠️  WARNING: The 'compile' command is deprecated and no longer supported.",
+        fg="yellow",
+        bold=True,
+        err=True,
+    )
 
     if platform not in ASSERTION_PLATFORMS:
         click.secho(
@@ -146,3 +179,5 @@ def extras_list_to_dict(extras: List[str]) -> Dict[str, str]:
 # Later:
 # 3. execute compiled assertions on assertion platform (Later, requires connection details to platform),
 # 4. cleanup assertions from assertion platform (generate artifacts. optionally execute)
+#
+# NOTE: This entire assertions CLI is deprecated and these TODOs will not be implemented.

datahub/cli/specific/datacontract_cli.py

@@ -1,4 +1,5 @@
 import logging
+import warnings
 from typing import Optional
 
 import click
@@ -14,15 +15,52 @@ logger = logging.getLogger(__name__)
 
 @click.group(cls=DefaultGroup, default="upsert")
 def datacontract() -> None:
-    """A group of commands to interact with the DataContract entity in DataHub."""
-    pass
+    """
+    A group of commands to interact with the DataContract entity in DataHub.
+
+    WARNING: This CLI is DEPRECATED and no longer supported.
+    Please migrate to alternative data contract solutions.
+    """
+    # Issue deprecation warning
+    warnings.warn(
+        "The datacontract CLI is deprecated and no longer supported. "
+        "Please migrate to alternative data contract solutions.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    # Log deprecation message for runtime visibility
+    logger.warning(
+        "DEPRECATED: The datacontract CLI is no longer supported and will be removed in a future version. "
+        "Please migrate to alternative data contract solutions."
+    )
+
+    # Display deprecation message to user
+    click.secho(
+        "⚠️  WARNING: This datacontract CLI is DEPRECATED and no longer supported.",
+        fg="yellow",
+        bold=True,
+    )
+    click.secho("Please migrate to alternative data contract solutions.", fg="yellow")
 
 
 @datacontract.command()
 @click.option("-f", "--file", required=True, type=click.Path(exists=True))
 @upgrade.check_upgrade
 def upsert(file: str) -> None:
-    """Upsert (create or update) a Data Contract in DataHub."""
+    """
+    Upsert (create or update) a Data Contract in DataHub.
+
+    WARNING: This command is DEPRECATED and no longer supported.
+    """
+
+    click.secho(
+        "⚠️  WARNING: The 'upsert' command is deprecated and no longer supported.",
+        fg="yellow",
+        bold=True,
+    )
+
+    logger.warning("DEPRECATED: datacontract upsert command is no longer supported")
 
     data_contract: DataContract = DataContract.from_yaml(file)
     urn = data_contract.urn
@@ -59,7 +97,19 @@ def upsert(file: str) -> None:
 @click.option("--hard/--soft", required=False, is_flag=True, default=False)
 @upgrade.check_upgrade
 def delete(urn: Optional[str], file: Optional[str], hard: bool) -> None:
-    """Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
+    """
+    Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata.
+
+    WARNING: This command is DEPRECATED and no longer supported.
+    """
+
+    click.secho(
+        "⚠️  WARNING: The 'delete' command is deprecated and no longer supported.",
+        fg="yellow",
+        bold=True,
+    )
+
+    logger.warning("DEPRECATED: datacontract delete command is no longer supported")
 
     if not urn:
         if not file:

datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py

@@ -90,7 +90,7 @@ class EnsureAspectSizeProcessor:
         on GMS side and failure of the entire ingestion. This processor will attempt to trim suspected aspects.
         """
         for wu in stream:
-            logger.debug(f"Ensuring size of workunit: {wu.id}")
+            # logger.debug(f"Ensuring size of workunit: {wu.id}")
 
             if schema := wu.get_aspect_of_type(SchemaMetadataClass):
                 self.ensure_schema_metadata_size(wu.get_urn(), schema)

datahub/ingestion/api/report.py

@@ -186,11 +186,19 @@ class ExamplesReport(Report, Closeable):
     aspects: Dict[str, Dict[str, int]] = field(
         default_factory=lambda: defaultdict(lambda: defaultdict(int))
     )
+    # This counts existence of aspects for each entity/subtype
+    # This is used for the UI to calculate %age of entities with the aspect
     aspects_by_subtypes: Dict[str, Dict[str, Dict[str, int]]] = field(
         default_factory=lambda: defaultdict(
             lambda: defaultdict(lambda: defaultdict(int))
         )
     )
+    # This counts all aspects for each entity/subtype
+    aspects_by_subtypes_full_count: Dict[str, Dict[str, Dict[str, int]]] = field(
+        default_factory=lambda: defaultdict(
+            lambda: defaultdict(lambda: defaultdict(int))
+        )
+    )
     samples: Dict[str, Dict[str, List[str]]] = field(
         default_factory=lambda: defaultdict(lambda: defaultdict(list))
     )
@@ -399,6 +407,9 @@ class ExamplesReport(Report, Closeable):
         entity_subtype_aspect_counts: Dict[str, Dict[str, Dict[str, int]]] = (
             defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
         )
+        entity_subtype_aspect_counts_exist: Dict[str, Dict[str, Dict[str, int]]] = (
+            defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
+        )
         for row in self._file_based_dict.sql_query(query):
             entity_type = row["entityType"]
             sub_type = row["subTypes"]
@@ -410,15 +421,23 @@ class ExamplesReport(Report, Closeable):
                 entity_subtype_aspect_counts[entity_type][sub_type][aspect] += (
                     aspect_count * count
                 )
+                entity_subtype_aspect_counts_exist[entity_type][sub_type][aspect] += (
+                    count
+                )
 
         self.aspects.clear()
         self.aspects_by_subtypes.clear()
-        _aspects_seen: Set[str] = set()
+        self.aspects_by_subtypes_full_count.clear()
         for entity_type, subtype_counts in entity_subtype_aspect_counts.items():
             for sub_type, aspect_counts in subtype_counts.items():
                 for aspect, count in aspect_counts.items():
                     self.aspects[entity_type][aspect] += count
-                    _aspects_seen.add(aspect)
+                self.aspects_by_subtypes_full_count[entity_type][sub_type] = dict(
+                    aspect_counts
+                )
+
+        for entity_type, subtype_counts in entity_subtype_aspect_counts_exist.items():
+            for sub_type, aspect_counts in subtype_counts.items():
                 self.aspects_by_subtypes[entity_type][sub_type] = dict(aspect_counts)
 
         self.samples.clear()

datahub/ingestion/api/source.py

@@ -81,11 +81,24 @@ class StructuredLogLevel(Enum):
     ERROR = logging.ERROR
 
 
+class StructuredLogCategory(Enum):
+    """
+    This is used to categorise the errors mainly based on the biggest impact area
+    This is to be used to help in self-serve understand the impact of any log entry
+    More enums to be added as logs are updated to be self-serve
+    """
+
+    LINEAGE = "LINEAGE"
+    USAGE = "USAGE"
+    PROFILING = "PROFILING"
+
+
 @dataclass
 class StructuredLogEntry(Report):
     title: Optional[str]
     message: str
     context: LossyList[str]
+    log_category: Optional[StructuredLogCategory] = None
 
 
 @dataclass
@@ -108,9 +121,10 @@ class StructuredLogs(Report):
         exc: Optional[BaseException] = None,
         log: bool = False,
         stacklevel: int = 1,
+        log_category: Optional[StructuredLogCategory] = None,
     ) -> None:
         """
-        Report a user-facing warning for the ingestion run.
+        Report a user-facing log for the ingestion run.
 
         Args:
             level: The level of the log entry.
@@ -118,6 +132,9 @@ class StructuredLogs(Report):
             title: The category / heading to present on for this message in the UI.
             context: Additional context (e.g. where, how) for the log entry.
             exc: The exception associated with the event. We'll show the stack trace when in debug mode.
+            log_category: The type of the log entry. This is used to categorise the log entry.
+            log: Whether to log the entry to the console.
+            stacklevel: The stack level to use for the log entry.
         """
 
         # One for this method, and one for the containing report_* call.
@@ -160,6 +177,7 @@ class StructuredLogs(Report):
                 title=title,
                 message=message,
                 context=context_list,
+                log_category=log_category,
             )
         else:
             if context is not None:
@@ -219,9 +237,19 @@ class SourceReport(ExamplesReport):
         context: Optional[str] = None,
         title: Optional[LiteralString] = None,
         exc: Optional[BaseException] = None,
+        log_category: Optional[StructuredLogCategory] = None,
     ) -> None:
+        """
+        See docs of StructuredLogs.report_log for details of args
+        """
         self._structured_logs.report_log(
-            StructuredLogLevel.WARN, message, title, context, exc, log=False
+            StructuredLogLevel.WARN,
+            message,
+            title,
+            context,
+            exc,
+            log=False,
+            log_category=log_category,
         )
 
     def warning(
@@ -231,9 +259,19 @@ class SourceReport(ExamplesReport):
         title: Optional[LiteralString] = None,
         exc: Optional[BaseException] = None,
         log: bool = True,
+        log_category: Optional[StructuredLogCategory] = None,
     ) -> None:
+        """
+        See docs of StructuredLogs.report_log for details of args
+        """
         self._structured_logs.report_log(
-            StructuredLogLevel.WARN, message, title, context, exc, log=log
+            StructuredLogLevel.WARN,
+            message,
+            title,
+            context,
+            exc,
+            log=log,
+            log_category=log_category,
         )
 
     def report_failure(
@@ -243,9 +281,19 @@ class SourceReport(ExamplesReport):
         title: Optional[LiteralString] = None,
         exc: Optional[BaseException] = None,
         log: bool = True,
+        log_category: Optional[StructuredLogCategory] = None,
     ) -> None:
+        """
+        See docs of StructuredLogs.report_log for details of args
+        """
         self._structured_logs.report_log(
-            StructuredLogLevel.ERROR, message, title, context, exc, log=log
+            StructuredLogLevel.ERROR,
+            message,
+            title,
+            context,
+            exc,
+            log=log,
+            log_category=log_category,
         )
 
     def failure(
@@ -255,9 +303,19 @@ class SourceReport(ExamplesReport):
         title: Optional[LiteralString] = None,
         exc: Optional[BaseException] = None,
         log: bool = True,
+        log_category: Optional[StructuredLogCategory] = None,
     ) -> None:
+        """
+        See docs of StructuredLogs.report_log for details of args
+        """
         self._structured_logs.report_log(
-            StructuredLogLevel.ERROR, message, title, context, exc, log=log
+            StructuredLogLevel.ERROR,
+            message,
+            title,
+            context,
+            exc,
+            log=log,
+            log_category=log_category,
         )
 
     def info(
@@ -267,9 +325,19 @@ class SourceReport(ExamplesReport):
         title: Optional[LiteralString] = None,
         exc: Optional[BaseException] = None,
         log: bool = True,
+        log_category: Optional[StructuredLogCategory] = None,
     ) -> None:
+        """
+        See docs of StructuredLogs.report_log for details of args
+        """
         self._structured_logs.report_log(
-            StructuredLogLevel.INFO, message, title, context, exc, log=log
+            StructuredLogLevel.INFO,
+            message,
+            title,
+            context,
+            exc,
+            log=log,
+            log_category=log_category,
         )
 
     @contextlib.contextmanager
@@ -279,6 +347,7 @@ class SourceReport(ExamplesReport):
         title: Optional[LiteralString] = None,
         context: Optional[str] = None,
         level: StructuredLogLevel = StructuredLogLevel.ERROR,
+        log_category: Optional[StructuredLogCategory] = None,
     ) -> Iterator[None]:
         # Convenience method that helps avoid boilerplate try/except blocks.
         # TODO: I'm not super happy with the naming here - it's not obvious that this
@@ -287,7 +356,12 @@ class SourceReport(ExamplesReport):
             yield
         except Exception as exc:
             self._structured_logs.report_log(
-                level, message=message, title=title, context=context, exc=exc
+                level,
+                message=message,
+                title=title,
+                context=context,
+                exc=exc,
+                log_category=log_category,
             )
 
     def __post_init__(self) -> None:

datahub/ingestion/autogenerated/capability_summary.json

@@ -1,9 +1,18 @@
 {
-  "generated_at": "2025-07-24T13:24:05.751563+00:00",
+  "generated_at": "2025-07-31T12:54:30.557618+00:00",
   "generated_by": "metadata-ingestion/scripts/capability_summary.py",
   "plugin_details": {
     "abs": {
       "capabilities": [
+        {
+          "capability": "CONTAINERS",
+          "description": "Extract ABS containers and folders",
+          "subtype_modifier": [
+            "Folder",
+            "ABS container"
+          ],
+          "supported": true
+        },
         {
           "capability": "DATA_PROFILING",
           "description": "Optionally enabled via configuration",
@@ -468,7 +477,9 @@
         {
           "capability": "CONTAINERS",
           "description": "Enabled by default",
-          "subtype_modifier": null,
+          "subtype_modifier": [
+            "Database"
+          ],
           "supported": true
         },
         {
@@ -531,13 +542,6 @@
       "platform_name": "File Based Lineage",
       "support_status": "CERTIFIED"
     },
-    "datahub-mock-data": {
-      "capabilities": [],
-      "classname": "datahub.ingestion.source.mock_data.datahub_mock_data.DataHubMockDataSource",
-      "platform_id": "datahubmockdata",
-      "platform_name": "DataHubMockData",
-      "support_status": "TESTING"
-    },
     "dbt": {
       "capabilities": [
         {
@@ -607,7 +611,9 @@
         {
           "capability": "CONTAINERS",
           "description": "Enabled by default",
-          "subtype_modifier": null,
+          "subtype_modifier": [
+            "Folder"
+          ],
           "supported": true
         },
         {
@@ -643,6 +649,14 @@
           "subtype_modifier": null,
           "supported": true
         },
+        {
+          "capability": "LINEAGE_FINE",
+          "description": "Extract column-level lineage",
+          "subtype_modifier": [
+            "Table"
+          ],
+          "supported": true
+        },
         {
           "capability": "DATA_PROFILING",
           "description": "Optionally enabled via configuration",
@@ -688,7 +702,9 @@
         {
           "capability": "LINEAGE_COARSE",
           "description": "Enabled by default",
-          "subtype_modifier": null,
+          "subtype_modifier": [
+            "Table"
+          ],
           "supported": true
         }
       ],
@@ -1229,8 +1245,7 @@
           "capability": "CONTAINERS",
           "description": "Enabled by default",
           "subtype_modifier": [
-            "Database",
-            "Schema"
+            "Catalog"
           ],
           "supported": true
         },
@@ -2387,8 +2402,9 @@
         },
         {
           "capability": "LINEAGE_COARSE",
-          "description": "Enabled by default to get lineage for views via `include_view_lineage`",
+          "description": "Extract table-level lineage",
           "subtype_modifier": [
+            "Table",
             "View"
           ],
           "supported": true
@@ -2411,8 +2427,7 @@
           "capability": "CONTAINERS",
           "description": "Enabled by default",
           "subtype_modifier": [
-            "Database",
-            "Schema"
+            "Catalog"
           ],
           "supported": true
         },
@@ -2598,7 +2613,8 @@
           "capability": "CONTAINERS",
           "description": "Enabled by default",
           "subtype_modifier": [
-            "Database"
+            "Database",
+            "Schema"
           ],
           "supported": true
         },
@@ -2812,6 +2828,15 @@
           "description": "Enabled by default",
           "subtype_modifier": null,
           "supported": true
+        },
+        {
+          "capability": "LINEAGE_COARSE",
+          "description": "Extract table-level lineage for Salesforce objects",
+          "subtype_modifier": [
+            "Custom Object",
+            "Object"
+          ],
+          "supported": true
         }
       ],
       "classname": "datahub.ingestion.source.salesforce.SalesforceSource",
@@ -3207,7 +3232,9 @@
         {
           "capability": "CONTAINERS",
           "description": "Enabled by default",
-          "subtype_modifier": null,
+          "subtype_modifier": [
+            "Database"
+          ],
           "supported": true
         },
         {
@@ -3339,8 +3366,9 @@
         },
         {
           "capability": "LINEAGE_COARSE",
-          "description": "Enabled by default to get lineage for views via `include_view_lineage`",
+          "description": "Extract table-level lineage",
           "subtype_modifier": [
+            "Table",
             "View"
           ],
           "supported": true

datahub/ingestion/source/abs/config.py

@@ -151,7 +151,7 @@ class DataLakeSourceConfig(
             raise ValueError("platform must not be empty")
         return platform
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def ensure_profiling_pattern_is_passed_to_profiling(
         cls, values: Dict[str, Any]
     ) -> Dict[str, Any]:

datahub/ingestion/source/abs/datalake_profiler_config.py

@@ -72,7 +72,7 @@ class DataLakeProfilerConfig(ConfigModel):
         description="Whether to profile for the sample values for all columns.",
     )
 
-    @pydantic.root_validator()
+    @pydantic.root_validator(skip_on_failure=True)
     def ensure_field_level_settings_are_normalized(
         cls: "DataLakeProfilerConfig", values: Dict[str, Any]
     ) -> Dict[str, Any]:

datahub/ingestion/source/abs/source.py

@@ -44,6 +44,7 @@ from datahub.ingestion.source.azure.abs_utils import (
     get_key_prefix,
     strip_abs_prefix,
 )
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
 from datahub.ingestion.source.data_lake_common.data_lake_utils import (
     ContainerWUCreator,
     add_partition_columns_to_schema,
@@ -128,6 +129,14 @@ class TableData:
 @support_status(SupportStatus.INCUBATING)
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
 @capability(SourceCapability.TAGS, "Can extract ABS object/container tags if enabled")
+@capability(
+    SourceCapability.CONTAINERS,
+    "Extract ABS containers and folders",
+    subtype_modifier=[
+        SourceCapabilityModifier.FOLDER,
+        SourceCapabilityModifier.ABS_CONTAINER,
+    ],
+)
 class ABSSource(StatefulIngestionSourceBase):
     source_config: DataLakeSourceConfig
     report: DataLakeSourceReport

datahub/ingestion/source/aws/glue.py

@@ -395,7 +395,7 @@ class GlueSource(StatefulIngestionSourceBase):
                     t = LakeFormationTag(
                         key=tag_key,
                         value=tag_value,
-                        catalog_id=catalog_id,
+                        catalog=catalog_id,
                     )
                     tags.append(t)
             return tags
@@ -438,7 +438,7 @@ class GlueSource(StatefulIngestionSourceBase):
                     t = LakeFormationTag(
                         key=tag_key,
                         value=tag_value,
-                        catalog_id=catalog_id,
+                        catalog=catalog_id,
                     )
                     tags.append(t)
             return tags
@@ -522,6 +522,14 @@ class GlueSource(StatefulIngestionSourceBase):
         bucket = url.netloc
         key = url.path[1:]
 
+        # validate that we have a non-empty key
+        if not key:
+            self.report.num_job_script_location_invalid += 1
+            logger.warning(
+                f"Error parsing DAG for Glue job. The script {script_path} is not a valid S3 path for flow urn: {flow_urn}."
+            )
+            return None
+
         # download the script contents
         # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_object
         try:
@@ -533,6 +541,14 @@ class GlueSource(StatefulIngestionSourceBase):
             )
             self.report.num_job_script_failed_download += 1
             return None
+        except botocore.exceptions.ParamValidationError as e:
+            self.report_warning(
+                flow_urn,
+                f"Invalid S3 path for Glue job script {script_path}: {e}",
+            )
+            self.report.num_job_script_location_invalid += 1
+            return None
+
         script = obj["Body"].read().decode("utf-8")
 
         try:

datahub/ingestion/source/aws/tag_entities.py

@@ -37,7 +37,7 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
 
     tag_key: str
     tag_value: Optional[str] = None
-    platform_instance: Optional[str]
+    platform_instance: Optional[str] = None
     catalog: Optional[str] = None
     exists_in_lake_formation: bool = False
     persisted: bool = False
@@ -88,8 +88,8 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
             return existing_platform_resource
 
         return LakeFormationTagPlatformResourceId(
-            tag_key=tag.key,
-            tag_value=tag.value if tag.value is not None else None,
+            tag_key=str(tag.key),
+            tag_value=str(tag.value) if tag.value is not None else None,
             platform_instance=platform_instance,
             exists_in_lake_formation=exists_in_lake_formation,
             catalog=catalog,
@@ -227,7 +227,7 @@ class LakeFormationTagPlatformResource(BaseModel, ExternalEntity):
     datahub_urns: LinkedResourceSet
     managed_by_datahub: bool
     id: LakeFormationTagPlatformResourceId
-    allowed_values: Optional[List[str]]
+    allowed_values: Optional[List[str]] = None
 
     def get_id(self) -> ExternalEntityId:
         return self.id