acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

{acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/entry_points.txt

@@ -36,8 +36,10 @@ csv-enricher = datahub.ingestion.source.csv_enricher:CSVEnricherSource
 datahub = datahub.ingestion.source.datahub.datahub_source:DataHubSource
 datahub-apply = datahub.ingestion.source.apply.datahub_apply:DataHubApplySource
 datahub-business-glossary = datahub.ingestion.source.metadata.business_glossary:BusinessGlossaryFileSource
+datahub-debug = datahub.ingestion.source.debug.datahub_debug:DataHubDebugSource
 datahub-gc = datahub.ingestion.source.gc.datahub_gc:DataHubGcSource
 datahub-lineage-file = datahub.ingestion.source.metadata.lineage:LineageFileSource
+datahub-mock-data = datahub.ingestion.source.mock_data.datahub_mock_data:DataHubMockDataSource
 dbt = datahub.ingestion.source.dbt.dbt_core:DBTCoreSource
 dbt-cloud = datahub.ingestion.source.dbt.dbt_cloud:DBTCloudSource
 delta-lake = datahub.ingestion.source.delta_lake:DeltaLakeSource
@@ -46,6 +48,7 @@ dremio = datahub.ingestion.source.dremio.dremio_source:DremioSource
 druid = datahub.ingestion.source.sql.druid:DruidSource
 dynamodb = datahub.ingestion.source.dynamodb.dynamodb:DynamoDBSource
 elasticsearch = datahub.ingestion.source.elastic_search:ElasticsearchSource
+excel = datahub.ingestion.source.excel.source:ExcelSource
 feast = datahub.ingestion.source.feast:FeastRepositorySource
 file = datahub.ingestion.source.file:GenericFileSource
 fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource
@@ -91,6 +94,7 @@ sagemaker = datahub.ingestion.source.aws.sagemaker:SagemakerSource
 salesforce = datahub.ingestion.source.salesforce:SalesforceSource
 sigma = datahub.ingestion.source.sigma.sigma:SigmaSource
 slack = datahub.ingestion.source.slack.slack:SlackSource
+snaplogic = datahub.ingestion.source.snaplogic.snaplogic:SnaplogicSource
 snowflake = datahub.ingestion.source.snowflake.snowflake_v2:SnowflakeV2Source
 snowflake-queries = datahub.ingestion.source.snowflake.snowflake_queries:SnowflakeQueriesSource
 snowflake-summary = datahub.ingestion.source.snowflake.snowflake_summary:SnowflakeSummarySource
@@ -127,6 +131,7 @@ pattern_cleanup_dataset_usage_user = datahub.ingestion.transformer.pattern_clean
 pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership
 replace_external_url = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlDataset
 replace_external_url_container = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlContainer
+set_browse_path = datahub.ingestion.transformer.set_browse_path:SetBrowsePathTransformer
 set_dataset_browse_path = datahub.ingestion.transformer.add_dataset_browse_path:AddDatasetBrowsePathTransformer
 simple_add_dataset_dataproduct = datahub.ingestion.transformer.add_dataset_dataproduct:SimpleAddDatasetDataProduct
 simple_add_dataset_domain = datahub.ingestion.transformer.dataset_domain:SimpleAddDatasetDomain

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.1.1rc4"
+__version__ = "1.3.0.1rc9"
 
 
 def is_dev_mode() -> bool:

datahub/api/entities/assertion/assertion.py

@@ -53,5 +53,5 @@ class BaseEntityAssertion(BaseAssertion):
     )
 
     trigger: Optional[AssertionTrigger] = v1_Field(
-        description="The trigger schedule for assertion", alias="schedule"
+        default=None, description="The trigger schedule for assertion", alias="schedule"
     )

datahub/api/entities/common/serialized_value.py

@@ -131,7 +131,7 @@ class SerializedResourceValue(BaseModel):
         elif isinstance(object, BaseModel):
             return SerializedResourceValue(
                 content_type=models.SerializedValueContentTypeClass.JSON,
-                blob=json.dumps(object.dict()).encode("utf-8"),
+                blob=json.dumps(object.dict(), sort_keys=True).encode("utf-8"),
                 schema_type=models.SerializedValueSchemaTypeClass.JSON,
                 schema_ref=object.__class__.__name__,
             )

datahub/api/entities/corpgroup/corpgroup.py

@@ -71,7 +71,7 @@ class CorpGroup(BaseModel):
     _rename_admins_to_owners = pydantic_renamed_field("admins", "owners")
 
     @pydantic.validator("owners", "members", each_item=True)
-    def make_urn_if_needed(v):
+    def make_urn_if_needed(cls, v):
         if isinstance(v, str):
             return builder.make_user_urn(v)
         return v

datahub/api/entities/dataproduct/dataproduct.py

@@ -6,9 +6,10 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import pydantic
 from ruamel.yaml import YAML
+from typing_extensions import assert_never
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import ConfigModel
+from datahub.configuration.common import ConfigModel, LaxStr
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
@@ -110,8 +111,9 @@ class DataProduct(ConfigModel):
     description: Optional[str] = None
     tags: Optional[List[str]] = None
     terms: Optional[List[str]] = None
-    properties: Optional[Dict[str, str]] = None
+    properties: Optional[Dict[str, LaxStr]] = None
     external_url: Optional[str] = None
+    output_ports: Optional[List[str]] = None
     _original_yaml_dict: Optional[dict] = None
 
     @pydantic.validator("assets", each_item=True)
@@ -123,6 +125,22 @@ class DataProduct(ConfigModel):
 
         return v
 
+    @pydantic.validator("output_ports", each_item=True)
+    def output_ports_must_be_urns(cls, v: str) -> str:
+        try:
+            Urn.create_from_string(v)
+        except Exception as e:
+            raise ValueError(f"Output port {v} is not an urn: {e}") from e
+
+        return v
+
+    @pydantic.validator("output_ports", each_item=True)
+    def output_ports_must_be_from_asset_list(cls, v: str, values: dict) -> str:
+        assets = values.get("assets", [])
+        if v not in assets:
+            raise ValueError(f"Output port {v} is not in asset list")
+        return v
+
     @property
     def urn(self) -> str:
         if self.id.startswith("urn:li:dataProduct:"):
@@ -180,6 +198,7 @@ class DataProduct(ConfigModel):
                         DataProductAssociationClass(
                             destinationUrn=asset,
                             created=self._mint_auditstamp("yaml"),
+                            outputPort=asset in (self.output_ports or []),
                         )
                         for asset in self.assets
                     ]
@@ -203,6 +222,7 @@ class DataProduct(ConfigModel):
                         DataProductAssociationClass(
                             destinationUrn=asset,
                             created=self._mint_auditstamp("yaml"),
+                            outputPort=asset in (self.output_ports or []),
                         )
                         for asset in self.assets or []
                     ],
@@ -368,6 +388,13 @@ class DataProduct(ConfigModel):
             external_url=(
                 data_product_properties.externalUrl if data_product_properties else None
             ),
+            output_ports=[
+                e.destinationUrn
+                for e in (data_product_properties.assets or [])
+                if e.outputPort
+            ]
+            if data_product_properties
+            else None,
         )
 
     def _patch_ownership(
@@ -414,7 +441,9 @@ class DataProduct(ConfigModel):
                                 "type": new_owner_type_map[owner_urn],
                             }
                     else:
-                        patches_drop[i] = o
+                        patches_drop[i] = o.model_dump()
+                else:
+                    assert_never(o)
 
         # Figure out what if any are new owners to add
         new_owners_to_add = {o for o in new_owner_type_map} - set(owners_matched)

datahub/api/entities/dataset/dataset.py

@@ -27,10 +27,11 @@ from typing_extensions import TypeAlias
 
 import datahub.metadata.schema_classes as models
 from datahub.api.entities.structuredproperties.structuredproperties import AllowedTypes
-from datahub.configuration.common import ConfigModel
+from datahub.configuration.common import ConfigModel, LaxStr
 from datahub.emitter.mce_builder import (
     make_data_platform_urn,
     make_dataset_urn,
+    make_domain_urn,
     make_schema_field_urn,
     make_tag_urn,
     make_term_urn,
@@ -43,6 +44,7 @@ from datahub.ingestion.graph.client import DataHubGraph
 from datahub.metadata.schema_classes import (
     AuditStampClass,
     DatasetPropertiesClass,
+    DomainsClass,
     GlobalTagsClass,
     GlossaryTermAssociationClass,
     GlossaryTermsClass,
@@ -134,14 +136,13 @@ class StructuredPropertiesHelper:
 
 class SchemaFieldSpecification(StrictModel):
     id: Optional[str] = None
-    urn: Optional[str] = None
+    urn: Optional[str] = Field(None, validate_default=True)
     structured_properties: Optional[StructuredProperties] = None
     type: Optional[str] = None
     nativeDataType: Optional[str] = None
     jsonPath: Union[None, str] = None
     nullable: bool = False
     description: Union[None, str] = None
-    doc: Union[None, str] = None  # doc is an alias for description
     label: Optional[str] = None
     created: Optional[dict] = None
     lastModified: Optional[dict] = None
@@ -219,14 +220,14 @@ class SchemaFieldSpecification(StrictModel):
         return v
 
     @root_validator(pre=True)
-    def sync_description_and_doc(cls, values: Dict) -> Dict:
-        """Synchronize doc and description fields if one is provided but not the other."""
+    def sync_doc_into_description(cls, values: Dict) -> Dict:
+        """Synchronize doc into description field if doc is provided."""
         description = values.get("description")
-        doc = values.get("doc")
+        doc = values.pop("doc", None)
 
-        if description is not None and doc is None:
-            values["doc"] = description
-        elif doc is not None and description is None:
+        if doc is not None:
+            if description is not None:
+                raise ValueError("doc and description cannot both be provided")
             values["description"] = doc
 
         return values
@@ -294,10 +295,6 @@ class SchemaFieldSpecification(StrictModel):
             """Custom dict method for Pydantic v1 to handle YAML serialization properly."""
             exclude = kwargs.pop("exclude", None) or set()
 
-            # If description and doc are identical, exclude doc from the output
-            if self.description == self.doc and self.description is not None:
-                exclude.add("doc")
-
             # if nativeDataType and type are identical, exclude nativeDataType from the output
             if self.nativeDataType == self.type and self.nativeDataType is not None:
                 exclude.add("nativeDataType")
@@ -325,10 +322,6 @@ class SchemaFieldSpecification(StrictModel):
             """Custom model_dump method for Pydantic v2 to handle YAML serialization properly."""
             exclude = kwargs.pop("exclude", None) or set()
 
-            # If description and doc are identical, exclude doc from the output
-            if self.description == self.doc and self.description is not None:
-                exclude.add("doc")
-
             # if nativeDataType and type are identical, exclude nativeDataType from the output
             if self.nativeDataType == self.type and self.nativeDataType is not None:
                 exclude.add("nativeDataType")
@@ -380,12 +373,12 @@ class Dataset(StrictModel):
     id: Optional[str] = None
     platform: Optional[str] = None
     env: str = "PROD"
-    urn: Optional[str] = None
+    urn: Optional[str] = Field(None, validate_default=True)
     description: Optional[str] = None
-    name: Optional[str] = None
-    schema_metadata: Optional[SchemaSpecification] = Field(alias="schema")
+    name: Optional[str] = Field(None, validate_default=True)
+    schema_metadata: Optional[SchemaSpecification] = Field(default=None, alias="schema")
     downstreams: Optional[List[str]] = None
-    properties: Optional[Dict[str, str]] = None
+    properties: Optional[Dict[str, LaxStr]] = None
     subtype: Optional[str] = None
     subtypes: Optional[List[str]] = None
     tags: Optional[List[str]] = None
@@ -393,6 +386,7 @@ class Dataset(StrictModel):
     owners: Optional[List[Union[str, Ownership]]] = None
     structured_properties: Optional[StructuredProperties] = None
     external_url: Optional[str] = None
+    domains: Optional[List[str]] = None
 
     @property
     def platform_urn(self) -> str:
@@ -602,7 +596,7 @@ class Dataset(StrictModel):
                         ],
                         platformSchema=OtherSchemaClass(
                             rawSchema=yaml.dump(
-                                self.schema_metadata.dict(
+                                self.schema_metadata.model_dump(
                                     exclude_none=True, exclude_unset=True
                                 )
                             )
@@ -735,7 +729,14 @@ class Dataset(StrictModel):
                     )
                 )
                 yield from patch_builder.build()
-
+        if self.domains:
+            mcp = MetadataChangeProposalWrapper(
+                entityUrn=self.urn,
+                aspect=DomainsClass(
+                    [make_domain_urn(domain) for domain in self.domains]
+                ),
+            )
+            yield mcp
         logger.info(f"Created dataset {self.urn}")
 
     @staticmethod
@@ -897,6 +898,7 @@ class Dataset(StrictModel):
                     structured_properties_map[sp.propertyUrn].extend(sp.values)  # type: ignore[arg-type,union-attr]
                 else:
                     structured_properties_map[sp.propertyUrn] = sp.values
+        domains: Optional[DomainsClass] = graph.get_aspect(urn, DomainsClass)
 
         if config.include_downstreams:
             related_downstreams = graph.get_related_entities(
@@ -937,6 +939,7 @@ class Dataset(StrictModel):
             structured_properties=(
                 structured_properties_map if structured_properties else None
             ),
+            domains=[domain for domain in domains.domains] if domains else None,
             downstreams=downstreams if config.include_downstreams else None,
         )