acryl-datahub-cloud 0.3.12.4rc3__py3-none-any.whl → 0.3.13rc2__py3-none-any.whl

acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.12.4rc3",
+  "version": "0.3.13rc2",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

acryl_datahub_cloud/lineage_features/source.py

@@ -45,6 +45,7 @@ SYSTEM_ACTOR = "urn:li:corpuser:__datahub_system"
 
 class LineageFeaturesSourceConfig(ConfigModel):
     enabled: bool = True
+    materialize_entities: bool = False
     search_index: ElasticSearchClientConfig = ElasticSearchClientConfig()
     query_timeout: int = 30
     extract_batch_size: int = 3000
@@ -77,6 +78,7 @@ class LineageExtractGraphSourceReport(SourceReport, IngestionStageReport):
     upstream_count: int = 0
     downstream_count: int = 0
     edges_scanned: int = 0
+    skipped_materialized_urns_count: int = 0
 
 
 @platform_name(id="datahub", platform_name="DataHub")
@@ -321,15 +323,17 @@ class DataHubLineageFeaturesSource(Source):
                         f"Failed to cleanup PIT after error: {cleanup_error}"
                     )
                 raise
-        # So previous stage's calculations are done
-        self.report.new_stage("Extract lineage features End")
         self._update_report()
         self._delete_pit_with_retry(server, pit)
 
+        self.report.new_stage("start emission of lineage features")
         # In Python 3.9, can be replaced by `self.self.upstream_counts.keys() | self.downstream_counts.keys()`
         for urn in set(self.upstream_counts.keys()).union(
             self.downstream_counts.keys()
         ):
+            if (not self.config.materialize_entities) and urn not in self.valid_urns:
+                self.report.skipped_materialized_urns_count += 1
+                continue
             logger.debug(
                 f"{urn}: {self.upstream_counts[urn]}, {self.downstream_counts[urn]}"
             )
@@ -346,6 +350,8 @@ class DataHubLineageFeaturesSource(Source):
             ).as_workunit()
             self.report.report_workunit(wu)
             yield wu
+        # So previous stage's calculations are done
+        self.report.new_stage("end emission of lineage features")
 
     def get_report(self) -> SourceReport:
         return self.report

acryl_datahub_cloud/metadata/_urns/urn_defs.py

@@ -1882,6 +1882,62 @@ class ChartUrn(_SpecificUrn):
     def chart_id(self) -> str:
         return self._entity_ids[1]
 
+if TYPE_CHECKING:
+    from datahub.metadata.schema_classes import DataHubPageTemplateKeyClass
+
+class DataHubPageTemplateUrn(_SpecificUrn):
+    ENTITY_TYPE: ClassVar[Literal["dataHubPageTemplate"]] = "dataHubPageTemplate"
+    _URN_PARTS: ClassVar[int] = 1
+
+    def __init__(self, id: Union["DataHubPageTemplateUrn", str], *, _allow_coercion: bool = True) -> None:
+        if _allow_coercion:
+            # Field coercion logic (if any is required).
+            if isinstance(id, str):
+                if id.startswith('urn:li:'):
+                    try:
+                        id = DataHubPageTemplateUrn.from_string(id)
+                    except InvalidUrnError:
+                        raise InvalidUrnError(f'Expecting a DataHubPageTemplateUrn but got {id}')
+                else:
+                    id = UrnEncoder.encode_string(id)
+
+        # Validation logic.
+        if not id:
+            raise InvalidUrnError("DataHubPageTemplateUrn id cannot be empty")
+        if isinstance(id, DataHubPageTemplateUrn):
+            id = id.id
+        elif isinstance(id, Urn):
+            raise InvalidUrnError(f'Expecting a DataHubPageTemplateUrn but got {id}')
+        if UrnEncoder.contains_reserved_char(id):
+            raise InvalidUrnError(f'DataHubPageTemplateUrn id contains reserved characters')
+
+        super().__init__(self.ENTITY_TYPE, [id])
+
+    @classmethod
+    def _parse_ids(cls, entity_ids: List[str]) -> "DataHubPageTemplateUrn":
+        if len(entity_ids) != cls._URN_PARTS:
+            raise InvalidUrnError(f"DataHubPageTemplateUrn should have {cls._URN_PARTS} parts, got {len(entity_ids)}: {entity_ids}")
+        return cls(id=entity_ids[0], _allow_coercion=False)
+
+    @classmethod
+    def underlying_key_aspect_type(cls) -> Type["DataHubPageTemplateKeyClass"]:
+        from datahub.metadata.schema_classes import DataHubPageTemplateKeyClass
+
+        return DataHubPageTemplateKeyClass
+
+    def to_key_aspect(self) -> "DataHubPageTemplateKeyClass":
+        from datahub.metadata.schema_classes import DataHubPageTemplateKeyClass
+
+        return DataHubPageTemplateKeyClass(id=self.id)
+
+    @classmethod
+    def from_key_aspect(cls, key_aspect: "DataHubPageTemplateKeyClass") -> "DataHubPageTemplateUrn":
+        return cls(id=key_aspect.id)
+
+    @property
+    def id(self) -> str:
+        return self._entity_ids[0]
+
 if TYPE_CHECKING:
     from datahub.metadata.schema_classes import OwnershipTypeKeyClass
 
@@ -3830,6 +3886,62 @@ class ContainerUrn(_SpecificUrn):
     def guid(self) -> str:
         return self._entity_ids[0]
 
+if TYPE_CHECKING:
+    from datahub.metadata.schema_classes import DataHubPageModuleKeyClass
+
+class DataHubPageModuleUrn(_SpecificUrn):
+    ENTITY_TYPE: ClassVar[Literal["dataHubPageModule"]] = "dataHubPageModule"
+    _URN_PARTS: ClassVar[int] = 1
+
+    def __init__(self, id: Union["DataHubPageModuleUrn", str], *, _allow_coercion: bool = True) -> None:
+        if _allow_coercion:
+            # Field coercion logic (if any is required).
+            if isinstance(id, str):
+                if id.startswith('urn:li:'):
+                    try:
+                        id = DataHubPageModuleUrn.from_string(id)
+                    except InvalidUrnError:
+                        raise InvalidUrnError(f'Expecting a DataHubPageModuleUrn but got {id}')
+                else:
+                    id = UrnEncoder.encode_string(id)
+
+        # Validation logic.
+        if not id:
+            raise InvalidUrnError("DataHubPageModuleUrn id cannot be empty")
+        if isinstance(id, DataHubPageModuleUrn):
+            id = id.id
+        elif isinstance(id, Urn):
+            raise InvalidUrnError(f'Expecting a DataHubPageModuleUrn but got {id}')
+        if UrnEncoder.contains_reserved_char(id):
+            raise InvalidUrnError(f'DataHubPageModuleUrn id contains reserved characters')
+
+        super().__init__(self.ENTITY_TYPE, [id])
+
+    @classmethod
+    def _parse_ids(cls, entity_ids: List[str]) -> "DataHubPageModuleUrn":
+        if len(entity_ids) != cls._URN_PARTS:
+            raise InvalidUrnError(f"DataHubPageModuleUrn should have {cls._URN_PARTS} parts, got {len(entity_ids)}: {entity_ids}")
+        return cls(id=entity_ids[0], _allow_coercion=False)
+
+    @classmethod
+    def underlying_key_aspect_type(cls) -> Type["DataHubPageModuleKeyClass"]:
+        from datahub.metadata.schema_classes import DataHubPageModuleKeyClass
+
+        return DataHubPageModuleKeyClass
+
+    def to_key_aspect(self) -> "DataHubPageModuleKeyClass":
+        from datahub.metadata.schema_classes import DataHubPageModuleKeyClass
+
+        return DataHubPageModuleKeyClass(id=self.id)
+
+    @classmethod
+    def from_key_aspect(cls, key_aspect: "DataHubPageModuleKeyClass") -> "DataHubPageModuleUrn":
+        return cls(id=key_aspect.id)
+
+    @property
+    def id(self) -> str:
+        return self._entity_ids[0]
+
 if TYPE_CHECKING:
     from datahub.metadata.schema_classes import EntityTypeKeyClass
 

acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/identity/__init__.py

@@ -13,6 +13,7 @@ from .....schema_classes import CorpGroupSettingsClass
 from .....schema_classes import CorpUserAppearanceSettingsClass
 from .....schema_classes import CorpUserCredentialsClass
 from .....schema_classes import CorpUserEditableInfoClass
+from .....schema_classes import CorpUserHomePageSettingsClass
 from .....schema_classes import CorpUserInfoClass
 from .....schema_classes import CorpUserSettingsClass
 from .....schema_classes import CorpUserStatusClass
@@ -29,6 +30,7 @@ CorpGroupSettings = CorpGroupSettingsClass
 CorpUserAppearanceSettings = CorpUserAppearanceSettingsClass
 CorpUserCredentials = CorpUserCredentialsClass
 CorpUserEditableInfo = CorpUserEditableInfoClass
+CorpUserHomePageSettings = CorpUserHomePageSettingsClass
 CorpUserInfo = CorpUserInfoClass
 CorpUserSettings = CorpUserSettingsClass
 CorpUserStatus = CorpUserStatusClass

acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py

@@ -24,6 +24,8 @@ from ......schema_classes import DataHubConnectionKeyClass
 from ......schema_classes import DataHubIngestionSourceKeyClass
 from ......schema_classes import DataHubMetricCubeKeyClass
 from ......schema_classes import DataHubOpenAPISchemaKeyClass
+from ......schema_classes import DataHubPageModuleKeyClass
+from ......schema_classes import DataHubPageTemplateKeyClass
 from ......schema_classes import DataHubPersonaKeyClass
 from ......schema_classes import DataHubPolicyKeyClass
 from ......schema_classes import DataHubRetentionKeyClass
@@ -91,6 +93,8 @@ DataHubConnectionKey = DataHubConnectionKeyClass
 DataHubIngestionSourceKey = DataHubIngestionSourceKeyClass
 DataHubMetricCubeKey = DataHubMetricCubeKeyClass
 DataHubOpenAPISchemaKey = DataHubOpenAPISchemaKeyClass
+DataHubPageModuleKey = DataHubPageModuleKeyClass
+DataHubPageTemplateKey = DataHubPageTemplateKeyClass
 DataHubPersonaKey = DataHubPersonaKeyClass
 DataHubPolicyKey = DataHubPolicyKeyClass
 DataHubRetentionKey = DataHubRetentionKeyClass

acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/module/__init__.py

@@ -0,0 +1,27 @@
+# mypy: ignore-errors
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# pylint: skip-file
+# fmt: off
+# isort: skip_file
+from .....schema_classes import DataHubPageModuleParamsClass
+from .....schema_classes import DataHubPageModulePropertiesClass
+from .....schema_classes import DataHubPageModuleTypeClass
+from .....schema_classes import DataHubPageModuleVisibilityClass
+from .....schema_classes import LinkModuleParamsClass
+from .....schema_classes import PageModuleScopeClass
+from .....schema_classes import RichTextModuleParamsClass
+
+
+DataHubPageModuleParams = DataHubPageModuleParamsClass
+DataHubPageModuleProperties = DataHubPageModulePropertiesClass
+DataHubPageModuleType = DataHubPageModuleTypeClass
+DataHubPageModuleVisibility = DataHubPageModuleVisibilityClass
+LinkModuleParams = LinkModuleParamsClass
+PageModuleScope = PageModuleScopeClass
+RichTextModuleParams = RichTextModuleParamsClass
+
+# fmt: on

acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py

@@ -7,9 +7,11 @@
 # pylint: skip-file
 # fmt: off
 # isort: skip_file
+from ......schema_classes import ApplicationsSettingsClass
 from ......schema_classes import DocPropagationFeatureSettingsClass
 from ......schema_classes import DocumentationAiSettingsClass
 from ......schema_classes import EmailIntegrationSettingsClass
+from ......schema_classes import GlobalHomePageSettingsClass
 from ......schema_classes import GlobalIncidentsSettingsClass
 from ......schema_classes import GlobalIntegrationSettingsClass
 from ......schema_classes import GlobalNotificationSettingsClass
@@ -22,9 +24,11 @@ from ......schema_classes import SlackIntegrationSettingsClass
 from ......schema_classes import SsoSettingsClass
 
 
+ApplicationsSettings = ApplicationsSettingsClass
 DocPropagationFeatureSettings = DocPropagationFeatureSettingsClass
 DocumentationAiSettings = DocumentationAiSettingsClass
 EmailIntegrationSettings = EmailIntegrationSettingsClass
+GlobalHomePageSettings = GlobalHomePageSettingsClass
 GlobalIncidentsSettings = GlobalIncidentsSettingsClass
 GlobalIntegrationSettings = GlobalIntegrationSettingsClass
 GlobalNotificationSettings = GlobalNotificationSettingsClass

acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/template/__init__.py

@@ -0,0 +1,25 @@
+# mypy: ignore-errors
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# pylint: skip-file
+# fmt: off
+# isort: skip_file
+from .....schema_classes import DataHubPageTemplatePropertiesClass
+from .....schema_classes import DataHubPageTemplateRowClass
+from .....schema_classes import DataHubPageTemplateSurfaceClass
+from .....schema_classes import DataHubPageTemplateVisibilityClass
+from .....schema_classes import PageTemplateScopeClass
+from .....schema_classes import PageTemplateSurfaceTypeClass
+
+
+DataHubPageTemplateProperties = DataHubPageTemplatePropertiesClass
+DataHubPageTemplateRow = DataHubPageTemplateRowClass
+DataHubPageTemplateSurface = DataHubPageTemplateSurfaceClass
+DataHubPageTemplateVisibility = DataHubPageTemplateVisibilityClass
+PageTemplateScope = PageTemplateScopeClass
+PageTemplateSurfaceType = PageTemplateSurfaceTypeClass
+
+# fmt: on