acryl-datahub 1.1.0.5rc5__py3-none-any.whl → 1.1.0.5rc7__py3-none-any.whl

datahub/ingestion/autogenerated/lineage.json

@@ -0,0 +1,401 @@
+{
+  "entities": {
+    "dataJob": {
+      "dataJobInputOutput": {
+        "aspect": "dataJobInputOutput",
+        "fields": [
+          {
+            "name": "inputDatasets",
+            "path": "inputDatasets",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputDatasetEdges",
+            "path": "inputDatasetEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "outputDatasets",
+            "path": "outputDatasets",
+            "isLineage": true,
+            "relationship": {
+              "name": "Produces",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "outputDatasetEdges",
+            "path": "outputDatasetEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Produces",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputDatajobs",
+            "path": "inputDatajobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "DownstreamOf",
+              "entityTypes": [
+                "dataJob"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputDatajobEdges",
+            "path": "inputDatajobEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "DownstreamOf",
+              "entityTypes": [
+                "dataJob"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dataProcessInstance": {
+      "dataProcessInstanceOutput": {
+        "aspect": "dataProcessInstanceOutput",
+        "fields": [
+          {
+            "name": "outputEdges",
+            "path": "outputEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "DataProcessInstanceProduces",
+              "entityTypes": [
+                "dataset",
+                "mlModel",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      },
+      "dataProcessInstanceInput": {
+        "aspect": "dataProcessInstanceInput",
+        "fields": [
+          {
+            "name": "inputEdges",
+            "path": "inputEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "DataProcessInstanceConsumes",
+              "entityTypes": [
+                "dataset",
+                "mlModel",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dataProcess": {
+      "dataProcessInfo": {
+        "aspect": "dataProcessInfo",
+        "fields": [
+          {
+            "name": "inputs",
+            "path": "inputs",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "outputs",
+            "path": "outputs",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dataset": {
+      "upstreamLineage": {
+        "aspect": "upstreamLineage",
+        "fields": [
+          {
+            "name": "dataset",
+            "path": "upstreams.dataset",
+            "isLineage": true,
+            "relationship": {
+              "name": "DownstreamOf",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "chart": {
+      "chartInfo": {
+        "aspect": "chartInfo",
+        "fields": [
+          {
+            "name": "inputs",
+            "path": "inputs",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputEdges",
+            "path": "inputEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dashboard": {
+      "dashboardInfo": {
+        "aspect": "dashboardInfo",
+        "fields": [
+          {
+            "name": "charts",
+            "path": "charts",
+            "isLineage": true,
+            "relationship": {
+              "name": "Contains",
+              "entityTypes": [
+                "chart"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "chartEdges",
+            "path": "chartEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Contains",
+              "entityTypes": [
+                "chart"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "datasets",
+            "path": "datasets",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "datasetEdges",
+            "path": "datasetEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "dashboards",
+            "path": "dashboards",
+            "isLineage": true,
+            "relationship": {
+              "name": "DashboardContainsDashboard",
+              "entityTypes": [
+                "dashboard"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlModelGroup": {
+      "mlModelGroupProperties": {
+        "aspect": "mlModelGroupProperties",
+        "fields": [
+          {
+            "name": "trainingJobs",
+            "path": "trainingJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "TrainedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "downstreamJobs",
+            "path": "downstreamJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "UsedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlFeature": {
+      "mlFeatureProperties": {
+        "aspect": "mlFeatureProperties",
+        "fields": [
+          {
+            "name": "sources",
+            "path": "sources",
+            "isLineage": true,
+            "relationship": {
+              "name": "DerivedFrom",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlPrimaryKey": {
+      "mlPrimaryKeyProperties": {
+        "aspect": "mlPrimaryKeyProperties",
+        "fields": [
+          {
+            "name": "sources",
+            "path": "sources",
+            "isLineage": true,
+            "relationship": {
+              "name": "DerivedFrom",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlModel": {
+      "mlModelProperties": {
+        "aspect": "mlModelProperties",
+        "fields": [
+          {
+            "name": "trainingJobs",
+            "path": "trainingJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "TrainedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "downstreamJobs",
+            "path": "downstreamJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "UsedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "mlFeatures",
+            "path": "mlFeatures",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "mlFeature"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "groups",
+            "path": "groups",
+            "isLineage": true,
+            "relationship": {
+              "name": "MemberOf",
+              "entityTypes": [
+                "mlModelGroup"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    }
+  },
+  "generated_by": "metadata-ingestion/scripts/modeldocgen.py",
+  "generated_at": "2025-07-01T10:49:03.713749+00:00"
+}

datahub/ingestion/autogenerated/lineage_helper.py

@@ -1,9 +1,8 @@
 import json
 import logging
+from functools import lru_cache
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Set
-
-from datahub.utilities.urns.urn import guess_entity_type
+from typing import Dict, List, Optional
 
 logger = logging.getLogger(__name__)
 
@@ -18,10 +17,9 @@ def _load_lineage_data() -> Dict:
     Load lineage data from the autogenerated lineage.json file.
 
     Returns:
-        Dict containing the lineage information
+        Dict containing the lineage information, or empty dict if file doesn't exist
 
     Raises:
-        FileNotFoundError: If lineage.json doesn't exist
         json.JSONDecodeError: If lineage.json is malformed
     """
     global _lineage_data
@@ -34,151 +32,55 @@ def _load_lineage_data() -> Dict:
     lineage_file = current_file.parent / "lineage.json"
 
     if not lineage_file.exists():
-        raise FileNotFoundError(f"Lineage file not found: {lineage_file}")
+        logger.warning(
+            f"Lineage file not found: {lineage_file}. "
+            "This may indicate a packaging issue. Lineage detection will be disabled."
+        )
+        _lineage_data = {}
+        return _lineage_data
 
     try:
         with open(lineage_file, "r") as f:
             _lineage_data = json.load(f)
         return _lineage_data
     except json.JSONDecodeError as e:
-        raise json.JSONDecodeError(
-            f"Failed to parse lineage.json: {e}", e.doc, e.pos
-        ) from e
-
-
-def get_lineage_fields(entity_type: str, aspect_name: str) -> List[Dict]:
-    """
-    This is experimental internal API subject to breaking changes without prior notice.
-
-    Get lineage fields for a specific entity type and aspect.
-
-    Args:
-        entity_type: The entity type (e.g., 'dataset', 'dataJob')
-        aspect_name: The aspect name (e.g., 'upstreamLineage', 'dataJobInputOutput')
-
-    Returns:
-        List of lineage field dictionaries, each containing:
-        - name: field name
-        - path: dot-notation path to the field
-        - isLineage: boolean indicating if it's lineage
-        - relationship: relationship information
-
-    Raises:
-        FileNotFoundError: If lineage.json doesn't exist
-        json.JSONDecodeError: If lineage.json is malformed
-    """
-    lineage_data = _load_lineage_data()
-
-    entity_data = lineage_data.get("entities", {}).get(entity_type, {})
-    aspect_data = entity_data.get(aspect_name, {})
-
-    return aspect_data.get("fields", [])
-
-
-def is_lineage_field(urn: str, aspect_name: str, field_path: str) -> bool:
-    """
-    This is experimental internal API subject to breaking changes without prior notice.
-
-    Check if a specific field path is lineage-related.
-
-    Args:
-        urn: The entity URN (e.g., 'urn:li:dataset:(urn:li:dataPlatform:mysql,test_db.test_table,PROD)')
-        aspect_name: The aspect name (e.g., 'upstreamLineage', 'dataJobInputOutput')
-        field_path: The dot-notation path to the field (e.g., 'upstreams.dataset')
-
-    Returns:
-        True if the field is lineage-related, False otherwise
-
-    Raises:
-        FileNotFoundError: If lineage.json doesn't exist
-        json.JSONDecodeError: If lineage.json is malformed
-        AssertionError: If URN doesn't start with 'urn:li:'
-    """
-    entity_type = guess_entity_type(urn)
-    lineage_fields = get_lineage_fields(entity_type, aspect_name)
-
-    for field in lineage_fields:
-        if field.get("path") == field_path:
-            return field.get("isLineage", False)
-
-    return False
+        logger.error(
+            f"Failed to parse lineage.json: {e}. Lineage detection will be disabled."
+        )
+        _lineage_data = {}
+        return _lineage_data
 
 
-def has_lineage(urn: str, aspect: Any) -> bool:
+def _get_fields(entity_type: str, aspect_name: str) -> List[Dict]:
     """
     This is experimental internal API subject to breaking changes without prior notice.
-
-    Check if an aspect has any lineage fields.
-
-    Args:
-        urn: The entity URN (e.g., 'urn:li:dataset:(urn:li:dataPlatform:mysql,test_db.test_table,PROD)')
-        aspect: The aspect object
-
-    Returns:
-        True if the aspect has lineage fields, False otherwise
-
-    Raises:
-        FileNotFoundError: If lineage.json doesn't exist
-        json.JSONDecodeError: If lineage.json is malformed
-        AssertionError: If URN doesn't start with 'urn:li:'
     """
-    entity_type = guess_entity_type(urn)
-    aspect_class = getattr(aspect, "__class__", None)
-    aspect_name = (
-        aspect_class.__name__ if aspect_class is not None else str(type(aspect))
+    return (
+        _load_lineage_data()
+        .get("entities", {})
+        .get(entity_type, {})
+        .get(aspect_name, {})
+        .get("fields", [])
     )
 
-    lineage_fields = get_lineage_fields(entity_type, aspect_name)
-    return len(lineage_fields) > 0
-
 
-def has_lineage_aspect(entity_type: str, aspect_name: str) -> bool:
+def _get_lineage_fields(entity_type: str, aspect_name: str) -> List[Dict]:
     """
     This is experimental internal API subject to breaking changes without prior notice.
-
-    Check if an aspect has any lineage fields.
-
-    Args:
-        entity_type: The entity type (e.g., 'dataset', 'dataJob')
-        aspect_name: The aspect name (e.g., 'upstreamLineage', 'dataJobInputOutput')
-
-    Returns:
-        True if the aspect has lineage fields, False otherwise
-
-    Raises:
-        FileNotFoundError: If lineage.json doesn't exist
-        json.JSONDecodeError: If lineage.json is malformed
     """
-    lineage_fields = get_lineage_fields(entity_type, aspect_name)
-    return len(lineage_fields) > 0
+    return [
+        field
+        for field in _get_fields(entity_type, aspect_name)
+        if field.get("isLineage", False)
+    ]
 
 
-def get_all_lineage_aspects(entity_type: str) -> Set[str]:
+@lru_cache(maxsize=128)
+def is_lineage_aspect(entity_type: str, aspect_name: str) -> bool:
     """
     This is experimental internal API subject to breaking changes without prior notice.
-
-    Get all aspects that have lineage fields for a given entity type.
-
-    Args:
-        entity_type: The entity type (e.g., 'dataset', 'dataJob')
-
-    Returns:
-        Set of aspect names that have lineage fields
-
-    Raises:
-        FileNotFoundError: If lineage.json doesn't exist
-        json.JSONDecodeError: If lineage.json is malformed
     """
-    lineage_data = _load_lineage_data()
-
-    entity_data = lineage_data.get("entities", {}).get(entity_type, {})
-    lineage_aspects = set()
-
-    for aspect_name, aspect_data in entity_data.items():
-        if aspect_data.get("fields"):
-            lineage_aspects.add(aspect_name)
-
-    return lineage_aspects
+    return len(_get_lineage_fields(entity_type, aspect_name)) > 0
 
 
 def clear_cache() -> None:

datahub/ingestion/run/pipeline.py

@@ -171,7 +171,10 @@ class Pipeline:
         self.last_time_printed = int(time.time())
         self.cli_report = CliReport()
 
-        with contextlib.ExitStack() as exit_stack, contextlib.ExitStack() as inner_exit_stack:
+        with (
+            contextlib.ExitStack() as exit_stack,
+            contextlib.ExitStack() as inner_exit_stack,
+        ):
             self.graph: Optional[DataHubGraph] = None
             with _add_init_error_context("connect to DataHub"):
                 if self.config.datahub_api:

datahub/ingestion/source/bigquery_v2/bigquery.py

@@ -288,28 +288,29 @@ class BigqueryV2Source(StatefulIngestionSourceBase, TestableSource):
             ):
                 return
 
-            with self.report.new_stage(
-                f"*: {QUERIES_EXTRACTION}"
-            ), BigQueryQueriesExtractor(
-                connection=self.config.get_bigquery_client(),
-                schema_api=self.bq_schema_extractor.schema_api,
-                config=BigQueryQueriesExtractorConfig(
-                    window=self.config,
-                    user_email_pattern=self.config.usage.user_email_pattern,
-                    include_lineage=self.config.include_table_lineage,
-                    include_usage_statistics=self.config.include_usage_statistics,
-                    include_operations=self.config.usage.include_operational_stats,
-                    include_queries=self.config.include_queries,
-                    include_query_usage_statistics=self.config.include_query_usage_statistics,
-                    top_n_queries=self.config.usage.top_n_queries,
-                    region_qualifiers=self.config.region_qualifiers,
-                ),
-                structured_report=self.report,
-                filters=self.filters,
-                identifiers=self.identifiers,
-                schema_resolver=self.sql_parser_schema_resolver,
-                discovered_tables=self.bq_schema_extractor.table_refs,
-            ) as queries_extractor:
+            with (
+                self.report.new_stage(f"*: {QUERIES_EXTRACTION}"),
+                BigQueryQueriesExtractor(
+                    connection=self.config.get_bigquery_client(),
+                    schema_api=self.bq_schema_extractor.schema_api,
+                    config=BigQueryQueriesExtractorConfig(
+                        window=self.config,
+                        user_email_pattern=self.config.usage.user_email_pattern,
+                        include_lineage=self.config.include_table_lineage,
+                        include_usage_statistics=self.config.include_usage_statistics,
+                        include_operations=self.config.usage.include_operational_stats,
+                        include_queries=self.config.include_queries,
+                        include_query_usage_statistics=self.config.include_query_usage_statistics,
+                        top_n_queries=self.config.usage.top_n_queries,
+                        region_qualifiers=self.config.region_qualifiers,
+                    ),
+                    structured_report=self.report,
+                    filters=self.filters,
+                    identifiers=self.identifiers,
+                    schema_resolver=self.sql_parser_schema_resolver,
+                    discovered_tables=self.bq_schema_extractor.table_refs,
+                ) as queries_extractor,
+            ):
                 self.report.queries_extractor = queries_extractor.report
                 yield from queries_extractor.get_workunits_internal()
         else:

datahub/ingestion/source/bigquery_v2/queries.py

@@ -46,11 +46,11 @@ SELECT
   t.is_insertable_into,
   t.ddl,
   ts.row_count as row_count,
-  ts.size_bytes as size_bytes,
+  ts.size_bytes as bytes,
   p.num_partitions,
   p.max_partition_id,
   p.active_billable_bytes as active_billable_bytes,
-  -- IFNULL(p.long_term_billable_bytes, 0) as long_term_billable_bytes,
+  IFNULL(p.long_term_billable_bytes, 0) as long_term_billable_bytes,
   REGEXP_EXTRACT(t.table_name, r"(?:(?:.+\\D)[_$]?)(\\d\\d\\d\\d(?:0[1-9]|1[012])(?:0[1-9]|[12][0-9]|3[01]))$") as table_suffix,
   REGEXP_REPLACE(t.table_name, r"(?:[_$]?)(\\d\\d\\d\\d(?:0[1-9]|1[012])(?:0[1-9]|[12][0-9]|3[01]))$", "") as table_base