acryl-datahub 1.0.0rc18__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

{acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.0.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/entry_points.txt

@@ -36,8 +36,10 @@ csv-enricher = datahub.ingestion.source.csv_enricher:CSVEnricherSource
 datahub = datahub.ingestion.source.datahub.datahub_source:DataHubSource
 datahub-apply = datahub.ingestion.source.apply.datahub_apply:DataHubApplySource
 datahub-business-glossary = datahub.ingestion.source.metadata.business_glossary:BusinessGlossaryFileSource
+datahub-debug = datahub.ingestion.source.debug.datahub_debug:DataHubDebugSource
 datahub-gc = datahub.ingestion.source.gc.datahub_gc:DataHubGcSource
 datahub-lineage-file = datahub.ingestion.source.metadata.lineage:LineageFileSource
+datahub-mock-data = datahub.ingestion.source.mock_data.datahub_mock_data:DataHubMockDataSource
 dbt = datahub.ingestion.source.dbt.dbt_core:DBTCoreSource
 dbt-cloud = datahub.ingestion.source.dbt.dbt_cloud:DBTCloudSource
 delta-lake = datahub.ingestion.source.delta_lake:DeltaLakeSource
@@ -46,6 +48,7 @@ dremio = datahub.ingestion.source.dremio.dremio_source:DremioSource
 druid = datahub.ingestion.source.sql.druid:DruidSource
 dynamodb = datahub.ingestion.source.dynamodb.dynamodb:DynamoDBSource
 elasticsearch = datahub.ingestion.source.elastic_search:ElasticsearchSource
+excel = datahub.ingestion.source.excel.source:ExcelSource
 feast = datahub.ingestion.source.feast:FeastRepositorySource
 file = datahub.ingestion.source.file:GenericFileSource
 fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource
@@ -53,6 +56,7 @@ gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource
 glue = datahub.ingestion.source.aws.glue:GlueSource
 grafana = datahub.ingestion.source.grafana.grafana_source:GrafanaSource
 hana = datahub.ingestion.source.sql.hana:HanaSource
+hex = datahub.ingestion.source.hex.hex:HexSource
 hive = datahub.ingestion.source.sql.hive:HiveSource
 hive-metastore = datahub.ingestion.source.sql.hive_metastore:HiveMetastoreSource
 iceberg = datahub.ingestion.source.iceberg.iceberg:IcebergSource
@@ -90,6 +94,7 @@ sagemaker = datahub.ingestion.source.aws.sagemaker:SagemakerSource
 salesforce = datahub.ingestion.source.salesforce:SalesforceSource
 sigma = datahub.ingestion.source.sigma.sigma:SigmaSource
 slack = datahub.ingestion.source.slack.slack:SlackSource
+snaplogic = datahub.ingestion.source.snaplogic.snaplogic:SnaplogicSource
 snowflake = datahub.ingestion.source.snowflake.snowflake_v2:SnowflakeV2Source
 snowflake-queries = datahub.ingestion.source.snowflake.snowflake_queries:SnowflakeQueriesSource
 snowflake-summary = datahub.ingestion.source.snowflake.snowflake_summary:SnowflakeSummarySource
@@ -101,7 +106,7 @@ tableau = datahub.ingestion.source.tableau.tableau:TableauSource
 teradata = datahub.ingestion.source.sql.teradata:TeradataSource
 trino = datahub.ingestion.source.sql.trino:TrinoSource
 unity-catalog = datahub.ingestion.source.unity.source:UnityCatalogSource
-vertexai = datahub.ingestion.source.vertexai:VertexAISource
+vertexai = datahub.ingestion.source.vertexai.vertexai:VertexAISource
 vertica = datahub.ingestion.source.sql.vertica:VerticaSource
 
 [datahub.ingestion.transformer.plugins]
@@ -126,6 +131,7 @@ pattern_cleanup_dataset_usage_user = datahub.ingestion.transformer.pattern_clean
 pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership
 replace_external_url = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlDataset
 replace_external_url_container = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlContainer
+set_browse_path = datahub.ingestion.transformer.set_browse_path:SetBrowsePathTransformer
 set_dataset_browse_path = datahub.ingestion.transformer.add_dataset_browse_path:AddDatasetBrowsePathTransformer
 simple_add_dataset_dataproduct = datahub.ingestion.transformer.add_dataset_dataproduct:SimpleAddDatasetDataProduct
 simple_add_dataset_domain = datahub.ingestion.transformer.dataset_domain:SimpleAddDatasetDomain

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.0.0rc18"
+__version__ = "1.3.0.1rc9"
 
 
 def is_dev_mode() -> bool:

datahub/api/circuit_breaker/operation_circuit_breaker.py

@@ -55,9 +55,9 @@ class OperationCircuitBreaker(AbstractCircuitBreaker):
             which is set as Airflow connection.
         :param partition: The partition to check the operation.
         :param source_type: The source type to filter on. If not set it will accept any source type.
-            See valid types here: https://datahubproject.io/docs/graphql/enums#operationsourcetype
+            See valid types here: https://docs.datahub.com/docs/graphql/enums#operationsourcetype
         :param operation_type: The operation type to filter on. If not set it will accept any source type.
-            See valid types here: https://datahubproject.io/docs/graphql/enums/#operationtype
+            See valid types here: https://docs.datahub.com/docs/graphql/enums/#operationtype
         """
 
         start_time_millis: int = int(

datahub/api/entities/assertion/assertion.py

@@ -53,5 +53,5 @@ class BaseEntityAssertion(BaseAssertion):
     )
 
     trigger: Optional[AssertionTrigger] = v1_Field(
-        description="The trigger schedule for assertion", alias="schedule"
+        default=None, description="The trigger schedule for assertion", alias="schedule"
     )

datahub/api/entities/common/serialized_value.py

@@ -131,7 +131,7 @@ class SerializedResourceValue(BaseModel):
         elif isinstance(object, BaseModel):
             return SerializedResourceValue(
                 content_type=models.SerializedValueContentTypeClass.JSON,
-                blob=json.dumps(object.dict()).encode("utf-8"),
+                blob=json.dumps(object.dict(), sort_keys=True).encode("utf-8"),
                 schema_type=models.SerializedValueSchemaTypeClass.JSON,
                 schema_ref=object.__class__.__name__,
             )

datahub/api/entities/corpgroup/corpgroup.py

@@ -71,7 +71,7 @@ class CorpGroup(BaseModel):
     _rename_admins_to_owners = pydantic_renamed_field("admins", "owners")
 
     @pydantic.validator("owners", "members", each_item=True)
-    def make_urn_if_needed(v):
+    def make_urn_if_needed(cls, v):
         if isinstance(v, str):
             return builder.make_user_urn(v)
         return v

datahub/api/entities/datacontract/datacontract.py

@@ -1,5 +1,5 @@
 import collections
-from typing import Iterable, List, Optional, Tuple
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 
 from ruamel.yaml import YAML
 from typing_extensions import Literal
@@ -25,6 +25,8 @@ from datahub.metadata.schema_classes import (
     FreshnessContractClass,
     SchemaContractClass,
     StatusClass,
+    StructuredPropertiesClass,
+    StructuredPropertyValueAssignmentClass,
 )
 from datahub.utilities.urns.urn import guess_entity_type
 
@@ -47,8 +49,12 @@ class DataContract(v1_ConfigModel):
     entity: str = v1_Field(
         description="The entity urn that the Data Contract is associated with"
     )
-    # TODO: add support for properties
-    # properties: Optional[Dict[str, str]] = None
+    properties: Optional[Dict[str, Union[str, float, List[Union[str, float]]]]] = (
+        v1_Field(
+            default=None,
+            description="Structured properties associated with the data contract.",
+        )
+    )
 
     schema_field: Optional[SchemaAssertion] = v1_Field(default=None, alias="schema")
 
@@ -172,6 +178,30 @@ class DataContract(v1_ConfigModel):
         )
         yield from dq_assertion_mcps
 
+        # Construct the structured properties aspect if properties are defined
+        structured_properties_aspect: Optional[StructuredPropertiesClass] = None
+        if self.properties:
+            property_assignments: List[StructuredPropertyValueAssignmentClass] = []
+            for key, value in self.properties.items():
+                # Use f-string formatting for the property URN, like in dataset.py
+                prop_urn = f"urn:li:structuredProperty:{key}"
+                # Ensure value is a list for StructuredPropertyValueAssignmentClass
+                values_list = value if isinstance(value, list) else [value]
+                property_assignments.append(
+                    StructuredPropertyValueAssignmentClass(
+                        propertyUrn=prop_urn,
+                        values=[
+                            str(v) for v in values_list
+                        ],  # Ensure all values are strings
+                    )
+                )
+            if (
+                property_assignments
+            ):  # Only create aspect if there are valid assignments
+                structured_properties_aspect = StructuredPropertiesClass(
+                    properties=property_assignments
+                )
+
         # Now that we've generated the assertions, we can generate
         # the actual data contract.
         yield from MetadataChangeProposalWrapper.construct_many(
@@ -202,6 +232,8 @@ class DataContract(v1_ConfigModel):
                     if True
                     else None
                 ),
+                # Add structured properties aspect if defined
+                structured_properties_aspect,
             ],
         )
 

datahub/api/entities/datajob/dataflow.py

@@ -9,6 +9,7 @@ from datahub.metadata.schema_classes import (
     AuditStampClass,
     DataFlowInfoClass,
     DataFlowSnapshotClass,
+    DataPlatformInstanceClass,
     GlobalTagsClass,
     MetadataChangeEventClass,
     OwnerClass,
@@ -29,7 +30,7 @@ class DataFlow:
     """The DataHub representation of data-flow.
 
     Args:
-        urn (int): Unique identifier of the DataFlow in DataHub. For more detail refer https://datahubproject.io/docs/what/urn/.
+        urn (int): Unique identifier of the DataFlow in DataHub. For more detail refer https://docs.datahub.com/docs/what/urn/.
         id (str): Identifier of DataFlow in orchestrator.
         orchestrator (str): orchestrator. for example airflow.
         cluster (Optional[str]): [deprecated] Please use env.
@@ -39,8 +40,8 @@ class DataFlow:
         url (Optional[str]): URL pointing to DataFlow.
         tags (Set[str]): tags that need to be apply on DataFlow.
         owners (Set[str]): owners that need to be apply on DataFlow.
-        platform_instance (Optional[str]): The instance of the platform that all assets produced by this orchestrator belong to. For more detail refer https://datahubproject.io/docs/platform-instances/.
-        env (Optional[str]): The environment that all assets produced by this orchestrator belong to. For more detail and possible values refer https://datahubproject.io/docs/graphql/enums/#fabrictype.
+        platform_instance (Optional[str]): The instance of the platform that all assets produced by this orchestrator belong to. For more detail refer https://docs.datahub.com/docs/platform-instances/.
+        env (Optional[str]): The environment that all assets produced by this orchestrator belong to. For more detail and possible values refer https://docs.datahub.com/docs/graphql/enums/#fabrictype.
     """
 
     urn: DataFlowUrn = field(init=False)
@@ -164,6 +165,20 @@ class DataFlow:
         )
         yield mcp
 
+        if self.platform_instance:
+            instance = builder.make_dataplatform_instance_urn(
+                platform=self.orchestrator,
+                instance=self.platform_instance,
+            )
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=str(self.urn),
+                aspect=DataPlatformInstanceClass(
+                    platform=builder.make_data_platform_urn(self.orchestrator),
+                    instance=instance,
+                ),
+            )
+            yield mcp
+
         for owner in self.generate_ownership_aspect():
             mcp = MetadataChangeProposalWrapper(
                 entityUrn=str(self.urn),

datahub/api/entities/datajob/datajob.py

@@ -10,6 +10,7 @@ from datahub.metadata.schema_classes import (
     AzkabanJobTypeClass,
     DataJobInfoClass,
     DataJobInputOutputClass,
+    DataPlatformInstanceClass,
     FineGrainedLineageClass,
     GlobalTagsClass,
     OwnerClass,
@@ -45,6 +46,7 @@ class DataJob:
         outlets (List[str]): List of urns the DataProcessInstance produces
         fine_grained_lineages: Column lineage for the inlets and outlets
         upstream_urns: List[DataJobUrn] = field(default_factory=list)
+        platform_instance (Optional[str]): The instance of the platform that all assets produced by this orchestrator belong to.
     """
 
     id: str
@@ -61,6 +63,7 @@ class DataJob:
     outlets: List[DatasetUrn] = field(default_factory=list)
     fine_grained_lineages: List[FineGrainedLineageClass] = field(default_factory=list)
     upstream_urns: List[DataJobUrn] = field(default_factory=list)
+    platform_instance: Optional[str] = None
 
     def __post_init__(self):
         job_flow_urn = DataFlowUrn.create_from_ids(
@@ -105,7 +108,9 @@ class DataJob:
         return [tags]
 
     def generate_mcp(
-        self, materialize_iolets: bool = True
+        self,
+        generate_lineage: bool = True,
+        materialize_iolets: bool = True,
     ) -> Iterable[MetadataChangeProposalWrapper]:
         env: Optional[str] = None
         if self.flow_urn.cluster.upper() in builder.ALL_ENV_TYPES:
@@ -127,6 +132,20 @@ class DataJob:
         )
         yield mcp
 
+        if self.platform_instance:
+            instance = builder.make_dataplatform_instance_urn(
+                platform=self.flow_urn.orchestrator,
+                instance=self.platform_instance,
+            )
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=str(self.urn),
+                aspect=DataPlatformInstanceClass(
+                    platform=builder.make_data_platform_urn(self.flow_urn.orchestrator),
+                    instance=instance,
+                ),
+            )
+            yield mcp
+
         mcp = MetadataChangeProposalWrapper(
             entityUrn=str(self.urn),
             aspect=StatusClass(
@@ -135,9 +154,10 @@ class DataJob:
         )
         yield mcp
 
-        yield from self.generate_data_input_output_mcp(
-            materialize_iolets=materialize_iolets
-        )
+        if generate_lineage:
+            yield from self.generate_data_input_output_mcp(
+                materialize_iolets=materialize_iolets
+            )
 
         for owner in self.generate_ownership_aspect():
             mcp = MetadataChangeProposalWrapper(

datahub/api/entities/dataprocess/dataprocess_instance.py

@@ -159,6 +159,7 @@ class DataProcessInstance:
                         env=self.template_urn.get_env(),
                         orchestrator=self.template_urn.get_orchestrator_name(),
                         id=self.template_urn.get_flow_id(),
+                        platform_instance=self.data_platform_instance,
                     )
                     for mcp in template_object.generate_mcp():
                         self._emit_mcp(mcp, emitter, callback)
@@ -168,6 +169,7 @@ class DataProcessInstance:
                         id=self.template_urn.get_job_id(),
                         upstream_urns=input_datajob_urns,
                         flow_urn=self.template_urn.get_data_flow_urn(),
+                        platform_instance=self.data_platform_instance,
                     )
                     for mcp in template_object.generate_mcp():
                         self._emit_mcp(mcp, emitter, callback)
@@ -382,6 +384,7 @@ class DataProcessInstance:
             cluster=datajob.flow_urn.cluster,
             template_urn=datajob.urn,
             id=id,
+            data_platform_instance=datajob.platform_instance,
         )
         dpi._template_object = datajob
 
@@ -438,6 +441,7 @@ class DataProcessInstance:
             orchestrator=dataflow.orchestrator,
             cluster=cast(str, dataflow.env),
             template_urn=dataflow.urn,
+            data_platform_instance=dataflow.platform_instance,
         )
         dpi._template_object = dataflow
         return dpi

datahub/api/entities/dataproduct/dataproduct.py

@@ -6,9 +6,10 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import pydantic
 from ruamel.yaml import YAML
+from typing_extensions import assert_never
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import ConfigModel
+from datahub.configuration.common import ConfigModel, LaxStr
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
@@ -110,8 +111,9 @@ class DataProduct(ConfigModel):
     description: Optional[str] = None
     tags: Optional[List[str]] = None
     terms: Optional[List[str]] = None
-    properties: Optional[Dict[str, str]] = None
+    properties: Optional[Dict[str, LaxStr]] = None
     external_url: Optional[str] = None
+    output_ports: Optional[List[str]] = None
     _original_yaml_dict: Optional[dict] = None
 
     @pydantic.validator("assets", each_item=True)
@@ -123,6 +125,22 @@ class DataProduct(ConfigModel):
 
         return v
 
+    @pydantic.validator("output_ports", each_item=True)
+    def output_ports_must_be_urns(cls, v: str) -> str:
+        try:
+            Urn.create_from_string(v)
+        except Exception as e:
+            raise ValueError(f"Output port {v} is not an urn: {e}") from e
+
+        return v
+
+    @pydantic.validator("output_ports", each_item=True)
+    def output_ports_must_be_from_asset_list(cls, v: str, values: dict) -> str:
+        assets = values.get("assets", [])
+        if v not in assets:
+            raise ValueError(f"Output port {v} is not in asset list")
+        return v
+
     @property
     def urn(self) -> str:
         if self.id.startswith("urn:li:dataProduct:"):
@@ -180,6 +198,7 @@ class DataProduct(ConfigModel):
                         DataProductAssociationClass(
                             destinationUrn=asset,
                             created=self._mint_auditstamp("yaml"),
+                            outputPort=asset in (self.output_ports or []),
                         )
                         for asset in self.assets
                     ]
@@ -203,6 +222,7 @@ class DataProduct(ConfigModel):
                         DataProductAssociationClass(
                             destinationUrn=asset,
                             created=self._mint_auditstamp("yaml"),
+                            outputPort=asset in (self.output_ports or []),
                         )
                         for asset in self.assets or []
                     ],
@@ -368,6 +388,13 @@ class DataProduct(ConfigModel):
             external_url=(
                 data_product_properties.externalUrl if data_product_properties else None
             ),
+            output_ports=[
+                e.destinationUrn
+                for e in (data_product_properties.assets or [])
+                if e.outputPort
+            ]
+            if data_product_properties
+            else None,
         )
 
     def _patch_ownership(
@@ -414,7 +441,9 @@ class DataProduct(ConfigModel):
                                 "type": new_owner_type_map[owner_urn],
                             }
                     else:
-                        patches_drop[i] = o
+                        patches_drop[i] = o.model_dump()
+                else:
+                    assert_never(o)
 
         # Figure out what if any are new owners to add
         new_owners_to_add = {o for o in new_owner_type_map} - set(owners_matched)