acryl-datahub 1.2.0__py3-none-any.whl → 1.2.0.1rc1__py3-none-any.whl

{acryl_datahub-1.2.0.dist-info → acryl_datahub-1.2.0.1rc1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.2.0.1rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=K2CkU-_AQgzgug1v9mpqxvf5t_O-NFUOD0V82i150go,318
+datahub/_version.py,sha256=lFvgok6Ukyjs_TiTTBHr0Swy25dhf9vLGNMLEKQ9Szg,323
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -110,7 +110,7 @@ datahub/configuration/json_loader.py,sha256=vIDnjwXWi9yHDO8KW64EupOzOb_sspehGCD7
 datahub/configuration/kafka.py,sha256=qj4qNBzeXeS-mUtf441B1jj_22wPO6Eho_stErMD-GY,2586
 datahub/configuration/kafka_consumer_config.py,sha256=LivsObTt9yC3WoGnslJbF_x4ojfNdxMIMEhb8vvJfcA,2133
 datahub/configuration/pattern_utils.py,sha256=Q5IB9RfWOOo5FvRVBU7XkhiwHCxSQ1NTMfUlWtWI9qc,699
-datahub/configuration/pydantic_migration_helpers.py,sha256=4C_COAVZ5iJ8yxcWNgXZNWsY7ULogICNZ368oNF7zWg,1462
+datahub/configuration/pydantic_migration_helpers.py,sha256=7HbLtMl95ERWminslXFxaEliXEpbTzDINspkiP_jT2c,1672
 datahub/configuration/source_common.py,sha256=t7rLiV76dKQUCqdRb6BpFnDqeNW_NvoE0ZehhrUSCOI,2605
 datahub/configuration/time_window_config.py,sha256=c4mbrgmTobt4t_j6unDeYvmGSlbRB2hAgAst6yq4nHA,5412
 datahub/configuration/toml.py,sha256=Ohc5sAWLPoAinPYL8njyheZ3ak81fC2Sp8IbBbESPGg,380
@@ -391,9 +391,9 @@ datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 datahub/ingestion/source/metadata/business_glossary.py,sha256=T_RJHst6iQRghJNmLLPeSBMEDsbEKf3yBldOAgMcGuo,19666
 datahub/ingestion/source/metadata/lineage.py,sha256=PA4JwSeQ-30XFMN4O5tPwIu-hZF1e-xMZ_CnEUE2c-Q,9595
 datahub/ingestion/source/mock_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=qq05NhOWY1tB_Th528HT9WhVlhMdsIu5Cr9wmXvpkFw,17398
+datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=JHR6NZnR15qoVZ9R4CZl61nmZXmpsX55PaoP0ykJ_Ns,18810
 datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
-datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=oIC1vcOx76Vl63O9kcjP_iInBHyS-ATdN3Y932TCCZg,3283
+datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=zJtEBSJGDvVr-kiKjK7LbHAifK3sfE786M3yO--Bn2o,3493
 datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/neo4j/neo4j_source.py,sha256=JqKCwxBJfOrC8SF7CmDG0cseWxHk_7E2v4Diw3Q0-WM,14181
 datahub/ingestion/source/powerbi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -618,8 +618,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
 datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
 datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
-datahub/metadata/_internal_schema_classes.py,sha256=Hp7LXq9qVe9KI1MDnIHgwaiJNbgbkI344AjfhwIUpBw,1044482
-datahub/metadata/schema.avsc,sha256=0u6V_sIgWTFJldnsY9ZD7N_GyOyxufnAUsR_Yc_Nxws,733968
+datahub/metadata/_internal_schema_classes.py,sha256=V43yFMGVycJf0UV_b-ZAaOigWedg_iItFj3rOoIx3Ro,1046990
+datahub/metadata/schema.avsc,sha256=NlFTKx_U18Je4-BV3GAKS6wlEZrBTRUt-UF9gUH-5z0,735189
 datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -673,7 +673,7 @@ datahub/metadata/com/linkedin/pegasus2avro/metadata/query/filter/__init__.py,sha
 datahub/metadata/com/linkedin/pegasus2avro/metadata/snapshot/__init__.py,sha256=OPboF8SV11wGnjvWQB-rxtB0otMdCsE7Tcy7xkOUgz8,2358
 datahub/metadata/com/linkedin/pegasus2avro/ml/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/pegasus2avro/ml/metadata/__init__.py,sha256=qefB0n1xilQHCPla80b39wdjHOYoVtzBJT2jGc2szkM,3309
-datahub/metadata/com/linkedin/pegasus2avro/module/__init__.py,sha256=Q64XyonAWViy4FlbjTz2yLWLB1gkAF3I9UtDkVAjlJM,980
+datahub/metadata/com/linkedin/pegasus2avro/module/__init__.py,sha256=Np5zhmoMzMEIioN7bPwPZkXBh_0aTvcEaVAdL0qrzSE,1108
 datahub/metadata/com/linkedin/pegasus2avro/mxe/__init__.py,sha256=LqGp9QTLk_tiSsbHMGSUH7uPG00Bf_qQIMiU7vtO4Tk,973
 datahub/metadata/com/linkedin/pegasus2avro/notebook/__init__.py,sha256=BcjOsz4YeHQbLLBb4Im4uJ7ux1hGHquQDmiIOiDXVtE,901
 datahub/metadata/com/linkedin/pegasus2avro/ownership/__init__.py,sha256=r813MW_bkP1ZpC2NJf7uCHEOapjebl611c90vryKX4A,302
@@ -752,7 +752,7 @@ datahub/metadata/schemas/DataHubIngestionSourceInfo.avsc,sha256=4wac7sluRIq-0ZjO
 datahub/metadata/schemas/DataHubIngestionSourceKey.avsc,sha256=TGmm9WEGTaABs7kt5Uc-N-kbc5Sd-2sQwx-JpfAptvw,545
 datahub/metadata/schemas/DataHubOpenAPISchemaKey.avsc,sha256=q6ZyMoxInwmrkrXkUgMe-i-WZzAxbjcvJ-EI99SnEp8,599
 datahub/metadata/schemas/DataHubPageModuleKey.avsc,sha256=NyFN8cVO6s6rtgoLGJJGfcPfpGr5PfmZlIhM6ajldfQ,460
-datahub/metadata/schemas/DataHubPageModuleProperties.avsc,sha256=eiCZBsLMgpkGtwSw-nn6BpumdwHAsHh1AiCa_hj5bHk,6471
+datahub/metadata/schemas/DataHubPageModuleProperties.avsc,sha256=NldfplvG_NKnbu1x0A1T6oTYKoTcGf_saa9AYFrcsTs,7618
 datahub/metadata/schemas/DataHubPageTemplateKey.avsc,sha256=0sVqwL97Rp8YHPytp2RqUP5hIW048hmT2hPNP5k6arc,472
 datahub/metadata/schemas/DataHubPageTemplateProperties.avsc,sha256=0ndN64UNAADL6G_GVjJLHbe_dBnWhVRjtI3MilOlHQc,5651
 datahub/metadata/schemas/DataHubPersonaInfo.avsc,sha256=OUvbTgPQsBtzkDDb9pxHXpQ6A7dkL77ZnCXZ-MLEG14,227
@@ -954,7 +954,7 @@ datahub/sdk/mlmodel.py,sha256=cO5R8BYVljmQ0w33RIOuZmj4nq8OJCDVAZGTQI6YFS8,12628
 datahub/sdk/mlmodelgroup.py,sha256=wlZZHny0UORpF0fRYuVkWLSQwIHX_fWl5lPb1NKR6dM,8194
 datahub/sdk/resolver_client.py,sha256=nKMAZJt2tRSGfKSzoREIh43PXqjM3umLiYkYHJjo1io,3243
 datahub/sdk/search_client.py,sha256=hlk40VnD3eT88hMgwXAUv31-ENbDe50P-gsXUnGSNeo,3512
-datahub/sdk/search_filters.py,sha256=BcMhvG5hGYAATtLPLz4WLRjKApX2oLYrrcGn-CG__ek,12901
+datahub/sdk/search_filters.py,sha256=xk19K7V6Y3YflNqNXgMTn_BpaRuoFhlub7w4tLjQlc8,15619
 datahub/secret/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/secret/datahub_secret_store.py,sha256=xyNAZY62d6KSz_kYF9wN7RDMLvNhu2ayOzcYvubOX1E,2519
 datahub/secret/datahub_secrets_client.py,sha256=nDmhziKdvseJHlaDVUcAwK8Fv8maeAaG-ktZtWG2b70,1316
@@ -1003,7 +1003,7 @@ datahub/testing/mcp_diff.py,sha256=1BpQ3hST46cOQi1SmKdsto3j6x6Sk6yHm0vG1w9IDL0,1
 datahub/testing/pytest_hooks.py,sha256=eifmj0M68AIfjTn_-0vtaBkKl75vNKMjsbYX-pJqmGY,1417
 datahub/testing/sdk_v2_helpers.py,sha256=FooqGn5PfdJJrCFm3x_uh02IMhDdLjqEf64W16WdvE0,424
 datahub/upgrade/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/upgrade/upgrade.py,sha256=-4_kDLk0buuCkqftlE1entT1duqIiyZUVsyncstiibA,18130
+datahub/upgrade/upgrade.py,sha256=bxGjfLU-hSQXS9Q1RlhWeJMETTGKqLNGFDPtcheCO4o,18474
 datahub/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/utilities/_custom_package_loader.py,sha256=9kgPE7Y77E-hNee8l4sKtVby-btUNum3dBfDixMzcVA,2059
 datahub/utilities/_markupsafe_compat.py,sha256=QX7c9KiHs56ASl7bJlgR4FAf3CGiY94zIr0h6Ak15To,444
@@ -1093,8 +1093,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.dist-info/METADATA,sha256=TMsSn0UWNaLqwWCsFIEr21hSQASsjXiMbR1nCPrB-5U,181878
-acryl_datahub-1.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
-acryl_datahub-1.2.0.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.dist-info/RECORD,,
+acryl_datahub-1.2.0.1rc1.dist-info/METADATA,sha256=Xybyu1aEfknH9PuIeVzawfwQ-xAlvMoBFASyzB83Mfo,181893
+acryl_datahub-1.2.0.1rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.2.0.1rc1.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
+acryl_datahub-1.2.0.1rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.2.0.1rc1.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0"
+__version__ = "1.2.0.1rc1"
 
 
 def is_dev_mode() -> bool:

datahub/configuration/pydantic_migration_helpers.py

@@ -1,12 +1,13 @@
 import pydantic.version
 from packaging.version import Version
 
-PYDANTIC_VERSION_2: bool
-if Version(pydantic.version.VERSION) >= Version("2.0"):
-    PYDANTIC_VERSION_2 = True
-else:
-    PYDANTIC_VERSION_2 = False
+_pydantic_version = Version(pydantic.version.VERSION)
+
+PYDANTIC_VERSION_2 = _pydantic_version >= Version("2.0")
 
+# The pydantic.Discriminator type was added in v2.5.0.
+# https://docs.pydantic.dev/latest/changelog/#v250-2023-11-13
+PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR = _pydantic_version >= Version("2.5.0")
 
 # This can be used to silence deprecation warnings while we migrate.
 if PYDANTIC_VERSION_2:
@@ -50,6 +51,7 @@ class v1_ConfigModel(v1_BaseModel):
 
 __all__ = [
     "PYDANTIC_VERSION_2",
+    "PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR",
     "PydanticDeprecatedSince20",
     "GenericModel",
     "v1_ConfigModel",

datahub/ingestion/source/mock_data/datahub_mock_data.py

@@ -75,6 +75,11 @@ class LineageConfigGen1(ConfigModel):
         description="Whether this source is enabled",
     )
 
+    table_name_prefix: Optional[str] = Field(
+        default=None,
+        description="Prefix to add to the table name. This is useful for testing purposes.",
+    )
+
     emit_lineage: bool = Field(
         default=True,
         description="Whether to emit lineage data for testing purposes. When False, no lineage data is generated regardless of other settings.",
@@ -104,6 +109,11 @@ class LineageConfigGen1(ConfigModel):
         description="Pattern for determining SubTypes. Options: 'alternating', 'all_table', 'all_view', 'level_based'",
     )
 
+    subtype_types: List[str] = Field(
+        default=["Table", "View"],
+        description="List of types to use in alternating pattern. Defaults to ['Table', 'View'].",
+    )
+
     level_subtypes: Dict[int, str] = Field(
         default={0: "Table", 1: "View", 2: "Table"},
         description="Mapping of level to subtype for level_based pattern",
@@ -240,38 +250,45 @@ class DataHubMockDataSource(Source):
             return fan_out_after_first if fan_out_after_first is not None else fan_out
 
     def _determine_subtype(
-        self, table_name: str, table_level: int, table_index: int
+        self,
+        table_level: int,
+        table_index: int,
+        subtype_pattern: SubTypePattern,
+        subtype_types: List[str],
+        level_subtypes: Dict[int, str],
     ) -> str:
         """
         Determine subtype based on configured pattern.
 
         Args:
-            table_name: Name of the table
             table_level: Level of the table in the lineage graph
             table_index: Index of the table within its level
+            subtype_pattern: Pattern for determining subtypes
+            subtype_types: List of types to use in alternating pattern
+            level_subtypes: Mapping of level to subtype for level_based pattern
 
         Returns:
-            The determined subtype ("Table" or "View")
+            The determined subtype from the configured types
         """
-        pattern = self.config.gen_1.subtype_pattern
-
-        if pattern == SubTypePattern.ALTERNATING:
-            return (
-                DatasetSubTypes.TABLE if table_index % 2 == 0 else DatasetSubTypes.VIEW
-            )
-        elif pattern == SubTypePattern.LEVEL_BASED:
-            return self.config.gen_1.level_subtypes.get(
-                table_level, DatasetSubTypes.TABLE
-            )
-        elif pattern == SubTypePattern.ALL_TABLE:
+        if subtype_pattern == SubTypePattern.ALTERNATING:
+            return subtype_types[table_index % len(subtype_types)]
+        elif subtype_pattern == SubTypePattern.LEVEL_BASED:
+            return level_subtypes.get(table_level, DatasetSubTypes.TABLE)
+        elif subtype_pattern == SubTypePattern.ALL_TABLE:
             return DatasetSubTypes.TABLE
-        elif pattern == SubTypePattern.ALL_VIEW:
+        elif subtype_pattern == SubTypePattern.ALL_VIEW:
             return DatasetSubTypes.VIEW
         else:
             return DatasetSubTypes.TABLE  # default
 
     def _get_subtypes_aspect(
-        self, table_name: str, table_level: int, table_index: int
+        self,
+        table_name: str,
+        table_level: int,
+        table_index: int,
+        subtype_pattern: SubTypePattern,
+        subtype_types: List[str],
+        level_subtypes: Dict[int, str],
     ) -> MetadataWorkUnit:
         """
         Create a SubTypes aspect for a table based on deterministic pattern.
@@ -280,12 +297,17 @@ class DataHubMockDataSource(Source):
             table_name: Name of the table
             table_level: Level of the table in the lineage graph
             table_index: Index of the table within its level
+            subtype_pattern: Pattern for determining subtypes
+            subtype_types: List of types to use in alternating pattern
+            level_subtypes: Mapping of level to subtype for level_based pattern
 
         Returns:
             MetadataWorkUnit containing the SubTypes aspect
         """
         # Determine subtype based on pattern
-        subtype = self._determine_subtype(table_name, table_level, table_index)
+        subtype = self._determine_subtype(
+            table_level, table_index, subtype_pattern, subtype_types, level_subtypes
+        )
 
         urn = make_dataset_urn(platform="fake", name=table_name)
         mcp = MetadataChangeProposalWrapper(
@@ -316,11 +338,20 @@ class DataHubMockDataSource(Source):
             tables_at_level = tables_at_levels[i]
 
             for j in range(tables_at_level):
-                table_name = TableNamingHelper.generate_table_name(hops, fan_out, i, j)
+                table_name = TableNamingHelper.generate_table_name(
+                    hops, fan_out, i, j, gen_1.table_name_prefix
+                )
 
                 yield self._get_status_aspect(table_name)
 
-                yield self._get_subtypes_aspect(table_name, i, j)
+                yield self._get_subtypes_aspect(
+                    table_name,
+                    i,
+                    j,
+                    gen_1.subtype_pattern,
+                    gen_1.subtype_types,
+                    gen_1.level_subtypes,
+                )
 
                 yield self._get_profile_aspect(table_name)
 
@@ -336,6 +367,7 @@ class DataHubMockDataSource(Source):
                         fan_out=fan_out,
                         fan_out_after_first=fan_out_after_first,
                         tables_at_levels=tables_at_levels,
+                        table_name_prefix=gen_1.table_name_prefix,
                     )
 
     def _generate_lineage_for_table(
@@ -347,6 +379,7 @@ class DataHubMockDataSource(Source):
         fan_out: int,
         fan_out_after_first: Optional[int],
         tables_at_levels: List[int],
+        table_name_prefix: Optional[str],
     ) -> Iterable[MetadataWorkUnit]:
         """Generate lineage relationships for a specific table."""
         # Only generate lineage if there are downstream levels
@@ -365,6 +398,7 @@ class DataHubMockDataSource(Source):
             hops=hops,
             fan_out=fan_out,
             tables_at_levels=tables_at_levels,
+            table_name_prefix=table_name_prefix,
         )
 
     def _generate_downstream_lineage(
@@ -376,6 +410,7 @@ class DataHubMockDataSource(Source):
         hops: int,
         fan_out: int,
         tables_at_levels: List[int],
+        table_name_prefix: Optional[str],
     ) -> Iterable[MetadataWorkUnit]:
         """Generate lineage relationships to downstream tables."""
         downstream_level = upstream_table_level + 1
@@ -389,7 +424,7 @@ class DataHubMockDataSource(Source):
 
         for downstream_index in range(start_downstream, end_downstream):
             downstream_table_name = TableNamingHelper.generate_table_name(
-                hops, fan_out, downstream_level, downstream_index
+                hops, fan_out, downstream_level, downstream_index, table_name_prefix
             )
             yield self._get_upstream_aspect(
                 upstream_table=upstream_table_name,

datahub/ingestion/source/mock_data/table_naming_helper.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Optional
 
 
 class TableNamingHelper:
@@ -10,7 +10,11 @@ class TableNamingHelper:
 
     @staticmethod
     def generate_table_name(
-        lineage_hops: int, lineage_fan_out: int, level: int, table_index: int
+        lineage_hops: int,
+        lineage_fan_out: int,
+        level: int,
+        table_index: int,
+        prefix: Optional[str] = None,
     ) -> str:
         """
         Generate a table name following the standard naming convention.
@@ -20,11 +24,13 @@ class TableNamingHelper:
             lineage_fan_out: Number of downstream tables per upstream table
             level: Level of the table in the lineage graph (0-based)
             table_index: Index of the table within its level (0-based)
+            prefix: Optional prefix to add to the table name
 
         Returns:
-            Table name following the pattern: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+            Table name following the pattern: "{prefix}hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
         """
-        return f"hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+        base_name = f"hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+        return f"{prefix}{base_name}" if prefix else base_name
 
     @staticmethod
     def parse_table_name(table_name: str) -> Dict[str, int]:

datahub/metadata/_internal_schema_classes.py

@@ -19931,6 +19931,31 @@ class TrainingDataClass(_Aspect):
         self._inner_dict['trainingData'] = value
     
     
+class AssetCollectionModuleParamsClass(DictWrapper):
+    """The params required if the module is type ASSET_COLLECTION"""
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.module.AssetCollectionModuleParams")
+    def __init__(self,
+        assetUrns: List[str],
+    ):
+        super().__init__()
+        
+        self.assetUrns = assetUrns
+    
+    def _restore_defaults(self) -> None:
+        self.assetUrns = list()
+    
+    
+    @property
+    def assetUrns(self) -> List[str]:
+        # No docs available.
+        return self._inner_dict.get('assetUrns')  # type: ignore
+    
+    @assetUrns.setter
+    def assetUrns(self, value: List[str]) -> None:
+        self._inner_dict['assetUrns'] = value
+    
+    
 class DataHubPageModuleParamsClass(DictWrapper):
     """The specific parameters stored for a module"""
     
@@ -19938,15 +19963,18 @@ class DataHubPageModuleParamsClass(DictWrapper):
     def __init__(self,
         linkParams: Union[None, "LinkModuleParamsClass"]=None,
         richTextParams: Union[None, "RichTextModuleParamsClass"]=None,
+        assetCollectionParams: Union[None, "AssetCollectionModuleParamsClass"]=None,
     ):
         super().__init__()
         
         self.linkParams = linkParams
         self.richTextParams = richTextParams
+        self.assetCollectionParams = assetCollectionParams
     
     def _restore_defaults(self) -> None:
         self.linkParams = self.RECORD_SCHEMA.fields_dict["linkParams"].default
         self.richTextParams = self.RECORD_SCHEMA.fields_dict["richTextParams"].default
+        self.assetCollectionParams = self.RECORD_SCHEMA.fields_dict["assetCollectionParams"].default
     
     
     @property
@@ -19969,6 +19997,16 @@ class DataHubPageModuleParamsClass(DictWrapper):
         self._inner_dict['richTextParams'] = value
     
     
+    @property
+    def assetCollectionParams(self) -> Union[None, "AssetCollectionModuleParamsClass"]:
+        """The params required if the module is type ASSET_COLLECTION"""
+        return self._inner_dict.get('assetCollectionParams')  # type: ignore
+    
+    @assetCollectionParams.setter
+    def assetCollectionParams(self, value: Union[None, "AssetCollectionModuleParamsClass"]) -> None:
+        self._inner_dict['assetCollectionParams'] = value
+    
+    
 class DataHubPageModulePropertiesClass(_Aspect):
     """The main properties of a DataHub page module"""
 
@@ -20116,24 +20154,50 @@ class LinkModuleParamsClass(DictWrapper):
     
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.module.LinkModuleParams")
     def __init__(self,
-        linkUrn: str,
+        linkUrl: str,
+        imageUrl: Union[None, str]=None,
+        description: Union[None, str]=None,
     ):
         super().__init__()
         
-        self.linkUrn = linkUrn
+        self.linkUrl = linkUrl
+        self.imageUrl = imageUrl
+        self.description = description
     
     def _restore_defaults(self) -> None:
-        self.linkUrn = str()
+        self.linkUrl = str()
+        self.imageUrl = self.RECORD_SCHEMA.fields_dict["imageUrl"].default
+        self.description = self.RECORD_SCHEMA.fields_dict["description"].default
+    
+    
+    @property
+    def linkUrl(self) -> str:
+        # No docs available.
+        return self._inner_dict.get('linkUrl')  # type: ignore
+    
+    @linkUrl.setter
+    def linkUrl(self, value: str) -> None:
+        self._inner_dict['linkUrl'] = value
+    
+    
+    @property
+    def imageUrl(self) -> Union[None, str]:
+        # No docs available.
+        return self._inner_dict.get('imageUrl')  # type: ignore
+    
+    @imageUrl.setter
+    def imageUrl(self, value: Union[None, str]) -> None:
+        self._inner_dict['imageUrl'] = value
     
     
     @property
-    def linkUrn(self) -> str:
+    def description(self) -> Union[None, str]:
         # No docs available.
-        return self._inner_dict.get('linkUrn')  # type: ignore
+        return self._inner_dict.get('description')  # type: ignore
     
-    @linkUrn.setter
-    def linkUrn(self, value: str) -> None:
-        self._inner_dict['linkUrn'] = value
+    @description.setter
+    def description(self, value: Union[None, str]) -> None:
+        self._inner_dict['description'] = value
     
     
 class PageModuleScopeClass(object):
@@ -27102,6 +27166,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.ml.metadata.SourceCodeUrl': SourceCodeUrlClass,
     'com.linkedin.pegasus2avro.ml.metadata.SourceCodeUrlType': SourceCodeUrlTypeClass,
     'com.linkedin.pegasus2avro.ml.metadata.TrainingData': TrainingDataClass,
+    'com.linkedin.pegasus2avro.module.AssetCollectionModuleParams': AssetCollectionModuleParamsClass,
     'com.linkedin.pegasus2avro.module.DataHubPageModuleParams': DataHubPageModuleParamsClass,
     'com.linkedin.pegasus2avro.module.DataHubPageModuleProperties': DataHubPageModulePropertiesClass,
     'com.linkedin.pegasus2avro.module.DataHubPageModuleType': DataHubPageModuleTypeClass,
@@ -27610,6 +27675,7 @@ __SCHEMA_TYPES = {
     'SourceCodeUrl': SourceCodeUrlClass,
     'SourceCodeUrlType': SourceCodeUrlTypeClass,
     'TrainingData': TrainingDataClass,
+    'AssetCollectionModuleParams': AssetCollectionModuleParamsClass,
     'DataHubPageModuleParams': DataHubPageModuleParamsClass,
     'DataHubPageModuleProperties': DataHubPageModulePropertiesClass,
     'DataHubPageModuleType': DataHubPageModuleTypeClass,

datahub/metadata/com/linkedin/pegasus2avro/module/__init__.py

@@ -7,6 +7,7 @@
 # pylint: skip-file
 # fmt: off
 # isort: skip_file
+from .....schema_classes import AssetCollectionModuleParamsClass
 from .....schema_classes import DataHubPageModuleParamsClass
 from .....schema_classes import DataHubPageModulePropertiesClass
 from .....schema_classes import DataHubPageModuleTypeClass
@@ -16,6 +17,7 @@ from .....schema_classes import PageModuleScopeClass
 from .....schema_classes import RichTextModuleParamsClass
 
 
+AssetCollectionModuleParams = AssetCollectionModuleParamsClass
 DataHubPageModuleParams = DataHubPageModuleParamsClass
 DataHubPageModuleProperties = DataHubPageModulePropertiesClass
 DataHubPageModuleType = DataHubPageModuleTypeClass

datahub/metadata/schema.avsc

@@ -17723,12 +17723,24 @@
                   "namespace": "com.linkedin.pegasus2avro.module",
                   "fields": [
                     {
-                      "java": {
-                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                      },
-                      "Urn": "Urn",
                       "type": "string",
-                      "name": "linkUrn"
+                      "name": "linkUrl"
+                    },
+                    {
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "imageUrl",
+                      "default": null
+                    },
+                    {
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "description",
+                      "default": null
                     }
                   ]
                 }
@@ -17755,6 +17767,31 @@
               "name": "richTextParams",
               "default": null,
               "doc": "The params required if the module is type RICH_TEXT"
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "AssetCollectionModuleParams",
+                  "namespace": "com.linkedin.pegasus2avro.module",
+                  "fields": [
+                    {
+                      "Urn": "Urn",
+                      "urn_is_array": true,
+                      "type": {
+                        "type": "array",
+                        "items": "string"
+                      },
+                      "name": "assetUrns"
+                    }
+                  ],
+                  "doc": "The params required if the module is type ASSET_COLLECTION"
+                }
+              ],
+              "name": "assetCollectionParams",
+              "default": null,
+              "doc": "The params required if the module is type ASSET_COLLECTION"
             }
           ],
           "doc": "The specific parameters stored for a module"

datahub/metadata/schemas/DataHubPageModuleProperties.avsc

@@ -87,12 +87,24 @@
                 "namespace": "com.linkedin.pegasus2avro.module",
                 "fields": [
                   {
-                    "java": {
-                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                    },
                     "type": "string",
-                    "name": "linkUrn",
-                    "Urn": "Urn"
+                    "name": "linkUrl"
+                  },
+                  {
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "imageUrl",
+                    "default": null
+                  },
+                  {
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "description",
+                    "default": null
                   }
                 ]
               }
@@ -119,6 +131,31 @@
             "name": "richTextParams",
             "default": null,
             "doc": "The params required if the module is type RICH_TEXT"
+          },
+          {
+            "type": [
+              "null",
+              {
+                "type": "record",
+                "name": "AssetCollectionModuleParams",
+                "namespace": "com.linkedin.pegasus2avro.module",
+                "fields": [
+                  {
+                    "type": {
+                      "type": "array",
+                      "items": "string"
+                    },
+                    "name": "assetUrns",
+                    "Urn": "Urn",
+                    "urn_is_array": true
+                  }
+                ],
+                "doc": "The params required if the module is type ASSET_COLLECTION"
+              }
+            ],
+            "name": "assetCollectionParams",
+            "default": null,
+            "doc": "The params required if the module is type ASSET_COLLECTION"
           }
         ],
         "doc": "The specific parameters stored for a module"