acryl-datahub 1.0.0rc18__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/autogenerated/lineage.json

@@ -0,0 +1,402 @@
+{
+  "entities": {
+    "dataJob": {
+      "dataJobInputOutput": {
+        "aspect": "dataJobInputOutput",
+        "fields": [
+          {
+            "name": "inputDatasets",
+            "path": "inputDatasets",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputDatasetEdges",
+            "path": "inputDatasetEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "outputDatasets",
+            "path": "outputDatasets",
+            "isLineage": true,
+            "relationship": {
+              "name": "Produces",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "outputDatasetEdges",
+            "path": "outputDatasetEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Produces",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputDatajobs",
+            "path": "inputDatajobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "DownstreamOf",
+              "entityTypes": [
+                "dataJob"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputDatajobEdges",
+            "path": "inputDatajobEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "DownstreamOf",
+              "entityTypes": [
+                "dataJob"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dataProcessInstance": {
+      "dataProcessInstanceOutput": {
+        "aspect": "dataProcessInstanceOutput",
+        "fields": [
+          {
+            "name": "outputEdges",
+            "path": "outputEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "DataProcessInstanceProduces",
+              "entityTypes": [
+                "dataset",
+                "mlModel",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      },
+      "dataProcessInstanceInput": {
+        "aspect": "dataProcessInstanceInput",
+        "fields": [
+          {
+            "name": "inputEdges",
+            "path": "inputEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "DataProcessInstanceConsumes",
+              "entityTypes": [
+                "dataset",
+                "mlModel",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dataProcess": {
+      "dataProcessInfo": {
+        "aspect": "dataProcessInfo",
+        "fields": [
+          {
+            "name": "inputs",
+            "path": "inputs",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "outputs",
+            "path": "outputs",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dataset": {
+      "upstreamLineage": {
+        "aspect": "upstreamLineage",
+        "fields": [
+          {
+            "name": "dataset",
+            "path": "upstreams.dataset",
+            "isLineage": true,
+            "relationship": {
+              "name": "DownstreamOf",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "chart": {
+      "chartInfo": {
+        "aspect": "chartInfo",
+        "fields": [
+          {
+            "name": "inputs",
+            "path": "inputs",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "inputEdges",
+            "path": "inputEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset",
+                "chart"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "dashboard": {
+      "dashboardInfo": {
+        "aspect": "dashboardInfo",
+        "fields": [
+          {
+            "name": "charts",
+            "path": "charts",
+            "isLineage": true,
+            "relationship": {
+              "name": "Contains",
+              "entityTypes": [
+                "chart"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "chartEdges",
+            "path": "chartEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Contains",
+              "entityTypes": [
+                "chart"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "datasets",
+            "path": "datasets",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "datasetEdges",
+            "path": "datasetEdges",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "dashboards",
+            "path": "dashboards",
+            "isLineage": true,
+            "relationship": {
+              "name": "DashboardContainsDashboard",
+              "entityTypes": [
+                "dashboard"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlModelGroup": {
+      "mlModelGroupProperties": {
+        "aspect": "mlModelGroupProperties",
+        "fields": [
+          {
+            "name": "trainingJobs",
+            "path": "trainingJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "TrainedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "downstreamJobs",
+            "path": "downstreamJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "UsedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlFeature": {
+      "mlFeatureProperties": {
+        "aspect": "mlFeatureProperties",
+        "fields": [
+          {
+            "name": "sources",
+            "path": "sources",
+            "isLineage": true,
+            "relationship": {
+              "name": "DerivedFrom",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlPrimaryKey": {
+      "mlPrimaryKeyProperties": {
+        "aspect": "mlPrimaryKeyProperties",
+        "fields": [
+          {
+            "name": "sources",
+            "path": "sources",
+            "isLineage": true,
+            "relationship": {
+              "name": "DerivedFrom",
+              "entityTypes": [
+                "dataset"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    },
+    "mlModel": {
+      "mlModelProperties": {
+        "aspect": "mlModelProperties",
+        "fields": [
+          {
+            "name": "trainingJobs",
+            "path": "trainingJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "TrainedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "downstreamJobs",
+            "path": "downstreamJobs",
+            "isLineage": true,
+            "relationship": {
+              "name": "UsedBy",
+              "entityTypes": [
+                "dataJob",
+                "dataProcessInstance"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "mlFeatures",
+            "path": "mlFeatures",
+            "isLineage": true,
+            "relationship": {
+              "name": "Consumes",
+              "entityTypes": [
+                "mlFeature"
+              ],
+              "isLineage": true
+            }
+          },
+          {
+            "name": "groups",
+            "path": "groups",
+            "isLineage": true,
+            "relationship": {
+              "name": "MemberOf",
+              "entityTypes": [
+                "mlModelGroup"
+              ],
+              "isLineage": true
+            }
+          }
+        ]
+      }
+    }
+  },
+  "generated_by": "metadata-ingestion/scripts/modeldocgen.py",
+  "generated_at": "2025-08-05T19:29:49.306404+00:00"
+}

datahub/ingestion/autogenerated/lineage_helper.py

@@ -0,0 +1,177 @@
+import json
+import logging
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Global cache for lineage data to avoid repeated file reads
+_lineage_data: Optional["LineageData"] = None
+
+
+@dataclass
+class Field:
+    name: str
+    path: str
+    isLineage: bool
+    relationship: Optional[Dict]
+
+
+@dataclass
+class Aspect:
+    name: str
+    fields: List[Field]
+
+
+@dataclass
+class Entity:
+    name: str
+    aspects: Dict[str, Aspect]
+
+
+@dataclass
+class LineageData:
+    # entity name -> aspect
+    entities: Dict[str, Entity]
+    generated_by: str
+    generated_at: str
+
+
+def get_lineage_data() -> LineageData:
+    """
+    This is experimental internal API subject to breaking changes without prior notice.
+    """
+    global _lineage_data
+
+    if _lineage_data is not None:
+        return _lineage_data
+
+    raw_data = _load_lineage_data()
+    _entities = raw_data.get("entities", {})
+    for entity_name, entity_data in _entities.items():
+        entity = Entity(
+            name=entity_name,
+            aspects={},
+        )
+        for aspect_name, aspect_data in entity_data.items():
+            entity.aspects[aspect_name] = Aspect(
+                name=aspect_name,
+                fields=[
+                    Field(
+                        name=field["name"],
+                        path=field["path"],
+                        isLineage=field["isLineage"],
+                        relationship=field.get("relationship", None),
+                    )
+                    for field in aspect_data.get("fields", [])
+                ],
+            )
+        _entities[entity_name] = entity
+
+    _lineage_data = LineageData(
+        entities=_entities,
+        generated_by=raw_data.get("generated_by", ""),
+        generated_at=raw_data.get("generated_at", ""),
+    )
+    return _lineage_data
+
+
+def get_all_aspect_names() -> List[str]:
+    """
+    This is experimental internal API subject to breaking changes without prior notice.
+    """
+    entities = get_lineage_data().entities
+    if not entities:
+        return []
+    first_entity = next(iter(entities.values()))
+    return list(first_entity.aspects.keys())
+
+
+def _load_lineage_data() -> Dict:
+    """
+    This is experimental internal API subject to breaking changes without prior notice.
+
+    Load lineage data from the autogenerated lineage.json file.
+
+    Returns:
+        Dict containing the lineage information, or empty dict if file doesn't exist
+
+    Raises:
+        json.JSONDecodeError: If lineage.json is malformed
+    """
+    # Get the path to lineage.json relative to this file
+    current_file = Path(__file__)
+    lineage_file = current_file.parent / "lineage.json"
+
+    if not lineage_file.exists():
+        logger.warning(
+            f"Lineage file not found: {lineage_file}. "
+            "This may indicate a packaging issue. Lineage detection will be disabled."
+        )
+        return {}
+
+    try:
+        with open(lineage_file, "r") as f:
+            return json.load(f)
+    except json.JSONDecodeError as e:
+        logger.error(
+            f"Failed to parse lineage.json: {e}. Lineage detection will be disabled."
+        )
+        return {}
+
+
+def _get_fields(entity_type: str, aspect_name: str) -> List[Dict]:
+    """
+    This is experimental internal API subject to breaking changes without prior notice.
+    """
+    lineage_data = get_lineage_data()
+    entity = lineage_data.entities.get(entity_type)
+    if not entity:
+        return []
+
+    aspect = entity.aspects.get(aspect_name)
+    if not aspect:
+        return []
+
+    return [
+        {
+            "name": field.name,
+            "path": field.path,
+            "isLineage": field.isLineage,
+            "relationship": field.relationship,
+        }
+        for field in aspect.fields
+    ]
+
+
+def _get_lineage_fields(entity_type: str, aspect_name: str) -> List[Dict]:
+    """
+    This is experimental internal API subject to breaking changes without prior notice.
+    """
+    return [
+        field
+        for field in _get_fields(entity_type, aspect_name)
+        if field.get("isLineage", False)
+    ]
+
+
+@lru_cache(maxsize=128)
+def is_lineage_aspect(entity_type: str, aspect_name: str) -> bool:
+    """
+    This is experimental internal API subject to breaking changes without prior notice.
+    """
+    return len(_get_lineage_fields(entity_type, aspect_name)) > 0
+
+
+def clear_cache() -> None:
+    """
+    This is experimental internal API subject to breaking changes without prior notice.
+
+    Clear the internal cache of lineage data.
+
+    This is useful for testing or when the lineage.json file has been updated.
+    """
+    global _lineage_data
+    _lineage_data = None

datahub/ingestion/extractor/schema_util.py

@@ -125,7 +125,7 @@ class AvroToMceSchemaConverter:
         self._prefix_name_stack: PrefixNameStack = [self.version_string]
         # Tracks the fields on the current path.
         self._fields_stack: FieldStack = []
-        # Tracks the record types seen so far. Used to prevent infinite recursion with recursive types.
+        # Stack of record types currently being processed. Used to prevent infinite recursion with recursive types.
         self._record_types_seen: List[str] = []
         # If part of the key-schema or value-schema.
         self._is_key_schema = is_key_schema
@@ -290,6 +290,12 @@ class AvroToMceSchemaConverter:
           This way we can use the type/description of the non-null type if needed.
         """
 
+        # props to skip when building jsonProps
+        json_props_to_skip = [
+            "_nullable",
+            "native_data_type",
+        ]
+
         def __init__(
             self,
             schema: SchemaOrField,
@@ -362,6 +368,7 @@ class AvroToMceSchemaConverter:
                 merged_props: Dict[str, Any] = {}
                 merged_props.update(self._schema.other_props)
                 merged_props.update(schema.other_props)
+                merged_props.update(actual_schema.other_props)
 
                 # Parse meta_mapping
                 meta_aspects: Dict[str, Any] = {}
@@ -406,6 +413,16 @@ class AvroToMceSchemaConverter:
                     or self._actual_schema.props.get("logicalType"),
                 )
 
+                json_props: Optional[Dict[str, Any]] = (
+                    {
+                        k: v
+                        for k, v in merged_props.items()
+                        if k not in self.json_props_to_skip
+                    }
+                    if merged_props
+                    else None
+                )
+
                 field = SchemaField(
                     fieldPath=field_path,
                     # Populate it with the simple native type for now.
@@ -420,7 +437,7 @@ class AvroToMceSchemaConverter:
                     isPartOfKey=self._converter._is_key_schema,
                     globalTags=tags_aspect,
                     glossaryTerms=meta_terms_aspect,
-                    jsonProps=json.dumps(merged_props) if merged_props else None,
+                    jsonProps=json.dumps(json_props) if json_props else None,
                 )
                 yield field
 
@@ -505,10 +522,12 @@ class AvroToMceSchemaConverter:
         # Handle recursive record definitions
         recurse: bool = True
         if isinstance(schema, avro.schema.RecordSchema):
-            if schema.fullname not in self._record_types_seen:
-                self._record_types_seen.append(schema.fullname)
-            else:
+            # Only prevent recursion if we're currently processing this record type (true recursion)
+            # Allow reuse of the same record type in different contexts
+            if schema.fullname in self._record_types_seen:
                 recurse = False
+            else:
+                self._record_types_seen.append(schema.fullname)
 
         # Adjust actual schema if needed
         actual_schema = self._get_underlying_type_if_option_as_union(schema, schema)
@@ -542,6 +561,13 @@ class AvroToMceSchemaConverter:
                     for sub_schema in self._get_sub_schemas(actual_schema):
                         yield from self._to_mce_fields(sub_schema)
 
+        # Clean up the processing stack
+        if (
+            isinstance(schema, avro.schema.RecordSchema)
+            and schema.fullname in self._record_types_seen
+        ):
+            self._record_types_seen.remove(schema.fullname)
+
     def _gen_non_nested_to_mce_fields(
         self, schema: SchemaOrField
     ) -> Iterable[SchemaField]:

datahub/ingestion/glossary/classification_mixin.py

@@ -90,6 +90,11 @@ class ClassificationHandler:
 
     def get_classifiers(self) -> List[Classifier]:
         classifiers = []
+        if (
+            not isinstance(self.config, ClassificationSourceConfigMixin)
+            or self.config.classification is None
+        ):
+            return classifiers
 
         for classifier in self.config.classification.classifiers:
             classifier_class = classifier_registry.get(classifier.type)
@@ -319,8 +324,10 @@ def classification_workunit_processor(
                         partial(
                             data_reader.get_sample_data_for_table,
                             table_id,
-                            classification_handler.config.classification.sample_size
-                            * SAMPLE_SIZE_MULTIPLIER,
+                            int(
+                                classification_handler.config.classification.sample_size
+                                * SAMPLE_SIZE_MULTIPLIER
+                            ),
                             **(data_reader_kwargs or {}),
                         )
                         if data_reader