acryl-datahub 1.3.0.1rc5__py3-none-any.whl → 1.3.0.1rc6__py3-none-any.whl

datahub/metadata/_internal_schema_classes.py

@@ -13088,6 +13088,185 @@ class StructuredExecutionReportClass(DictWrapper):
         self._inner_dict['contentType'] = value
     
     
+class BucketStorageLocationClass(DictWrapper):
+    """Information where a file is stored"""
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.file.BucketStorageLocation")
+    def __init__(self,
+        storageBucket: str,
+        storageKey: str,
+    ):
+        super().__init__()
+        
+        self.storageBucket = storageBucket
+        self.storageKey = storageKey
+    
+    def _restore_defaults(self) -> None:
+        self.storageBucket = str()
+        self.storageKey = str()
+    
+    
+    @property
+    def storageBucket(self) -> str:
+        """The storage bucket this file is stored in"""
+        return self._inner_dict.get('storageBucket')  # type: ignore
+    
+    @storageBucket.setter
+    def storageBucket(self, value: str) -> None:
+        self._inner_dict['storageBucket'] = value
+    
+    
+    @property
+    def storageKey(self) -> str:
+        """The key for where this file is stored inside of the given bucket"""
+        return self._inner_dict.get('storageKey')  # type: ignore
+    
+    @storageKey.setter
+    def storageKey(self, value: str) -> None:
+        self._inner_dict['storageKey'] = value
+    
+    
+class DataHubFileInfoClass(_Aspect):
+    """Information about a DataHub file - a file stored in S3 for use within DataHub platform features like documentation, home pages, and announcements."""
+
+
+    ASPECT_NAME = 'dataHubFileInfo'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.file.DataHubFileInfo")
+
+    def __init__(self,
+        bucketStorageLocation: "BucketStorageLocationClass",
+        originalFileName: str,
+        mimeType: str,
+        sizeInBytes: int,
+        scenario: Union[str, "FileUploadScenarioClass"],
+        created: "AuditStampClass",
+        referencedByAsset: Union[None, str]=None,
+        schemaField: Union[None, str]=None,
+        contentHash: Union[None, str]=None,
+    ):
+        super().__init__()
+        
+        self.bucketStorageLocation = bucketStorageLocation
+        self.originalFileName = originalFileName
+        self.mimeType = mimeType
+        self.sizeInBytes = sizeInBytes
+        self.scenario = scenario
+        self.referencedByAsset = referencedByAsset
+        self.schemaField = schemaField
+        self.created = created
+        self.contentHash = contentHash
+    
+    def _restore_defaults(self) -> None:
+        self.bucketStorageLocation = BucketStorageLocationClass._construct_with_defaults()
+        self.originalFileName = str()
+        self.mimeType = str()
+        self.sizeInBytes = int()
+        self.scenario = FileUploadScenarioClass.ASSET_DOCUMENTATION
+        self.referencedByAsset = self.RECORD_SCHEMA.fields_dict["referencedByAsset"].default
+        self.schemaField = self.RECORD_SCHEMA.fields_dict["schemaField"].default
+        self.created = AuditStampClass._construct_with_defaults()
+        self.contentHash = self.RECORD_SCHEMA.fields_dict["contentHash"].default
+    
+    
+    @property
+    def bucketStorageLocation(self) -> "BucketStorageLocationClass":
+        """Info about where a file is stored"""
+        return self._inner_dict.get('bucketStorageLocation')  # type: ignore
+    
+    @bucketStorageLocation.setter
+    def bucketStorageLocation(self, value: "BucketStorageLocationClass") -> None:
+        self._inner_dict['bucketStorageLocation'] = value
+    
+    
+    @property
+    def originalFileName(self) -> str:
+        """The original filename as uploaded by the user"""
+        return self._inner_dict.get('originalFileName')  # type: ignore
+    
+    @originalFileName.setter
+    def originalFileName(self, value: str) -> None:
+        self._inner_dict['originalFileName'] = value
+    
+    
+    @property
+    def mimeType(self) -> str:
+        """MIME type of the file (e.g., image/png, application/pdf)"""
+        return self._inner_dict.get('mimeType')  # type: ignore
+    
+    @mimeType.setter
+    def mimeType(self, value: str) -> None:
+        self._inner_dict['mimeType'] = value
+    
+    
+    @property
+    def sizeInBytes(self) -> int:
+        """Size of the file in bytes"""
+        return self._inner_dict.get('sizeInBytes')  # type: ignore
+    
+    @sizeInBytes.setter
+    def sizeInBytes(self, value: int) -> None:
+        self._inner_dict['sizeInBytes'] = value
+    
+    
+    @property
+    def scenario(self) -> Union[str, "FileUploadScenarioClass"]:
+        """The scenario/context in which this file was uploaded"""
+        return self._inner_dict.get('scenario')  # type: ignore
+    
+    @scenario.setter
+    def scenario(self, value: Union[str, "FileUploadScenarioClass"]) -> None:
+        self._inner_dict['scenario'] = value
+    
+    
+    @property
+    def referencedByAsset(self) -> Union[None, str]:
+        """Optional URN of the entity this file is associated with (e.g., the dataset whose docs contain this file)"""
+        return self._inner_dict.get('referencedByAsset')  # type: ignore
+    
+    @referencedByAsset.setter
+    def referencedByAsset(self, value: Union[None, str]) -> None:
+        self._inner_dict['referencedByAsset'] = value
+    
+    
+    @property
+    def schemaField(self) -> Union[None, str]:
+        """The dataset schema field urn this file is referenced by"""
+        return self._inner_dict.get('schemaField')  # type: ignore
+    
+    @schemaField.setter
+    def schemaField(self, value: Union[None, str]) -> None:
+        self._inner_dict['schemaField'] = value
+    
+    
+    @property
+    def created(self) -> "AuditStampClass":
+        """Timestamp when this file was created and by whom"""
+        return self._inner_dict.get('created')  # type: ignore
+    
+    @created.setter
+    def created(self, value: "AuditStampClass") -> None:
+        self._inner_dict['created'] = value
+    
+    
+    @property
+    def contentHash(self) -> Union[None, str]:
+        """SHA-256 hash of file contents"""
+        return self._inner_dict.get('contentHash')  # type: ignore
+    
+    @contentHash.setter
+    def contentHash(self, value: Union[None, str]) -> None:
+        self._inner_dict['contentHash'] = value
+    
+    
+class FileUploadScenarioClass(object):
+    # No docs available.
+    
+    ASSET_DOCUMENTATION = "ASSET_DOCUMENTATION"
+    """File uploaded for entity documentation"""
+    
+    
+    
 class DynamicFormAssignmentClass(_Aspect):
     """Information about how a form is assigned to entities dynamically. Provide a filter to
     match a set of entities instead of explicitly applying a form to specific entities."""
@@ -15712,6 +15891,35 @@ class DataHubConnectionKeyClass(_Aspect):
         self._inner_dict['id'] = value
     
     
+class DataHubFileKeyClass(_Aspect):
+    """Key for a DataHubFile"""
+
+
+    ASPECT_NAME = 'dataHubFileKey'
+    ASPECT_INFO = {'keyForEntity': 'dataHubFile', 'entityCategory': 'core', 'entityAspects': ['dataHubFileInfo']}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DataHubFileKey")
+
+    def __init__(self,
+        id: str,
+    ):
+        super().__init__()
+        
+        self.id = id
+    
+    def _restore_defaults(self) -> None:
+        self.id = str()
+    
+    
+    @property
+    def id(self) -> str:
+        """Unique id for the file."""
+        return self._inner_dict.get('id')  # type: ignore
+    
+    @id.setter
+    def id(self, value: str) -> None:
+        self._inner_dict['id'] = value
+    
+    
 class DataHubIngestionSourceKeyClass(_Aspect):
     """Key for a DataHub ingestion source"""
 
@@ -27751,6 +27959,9 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.execution.ExecutionRequestSignal': ExecutionRequestSignalClass,
     'com.linkedin.pegasus2avro.execution.ExecutionRequestSource': ExecutionRequestSourceClass,
     'com.linkedin.pegasus2avro.execution.StructuredExecutionReport': StructuredExecutionReportClass,
+    'com.linkedin.pegasus2avro.file.BucketStorageLocation': BucketStorageLocationClass,
+    'com.linkedin.pegasus2avro.file.DataHubFileInfo': DataHubFileInfoClass,
+    'com.linkedin.pegasus2avro.file.FileUploadScenario': FileUploadScenarioClass,
     'com.linkedin.pegasus2avro.form.DynamicFormAssignment': DynamicFormAssignmentClass,
     'com.linkedin.pegasus2avro.form.FormActorAssignment': FormActorAssignmentClass,
     'com.linkedin.pegasus2avro.form.FormInfo': FormInfoClass,
@@ -27800,6 +28011,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.metadata.key.DataHubAccessTokenKey': DataHubAccessTokenKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubActionKey': DataHubActionKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubConnectionKey': DataHubConnectionKeyClass,
+    'com.linkedin.pegasus2avro.metadata.key.DataHubFileKey': DataHubFileKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubIngestionSourceKey': DataHubIngestionSourceKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubOpenAPISchemaKey': DataHubOpenAPISchemaKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubPageModuleKey': DataHubPageModuleKeyClass,
@@ -28273,6 +28485,9 @@ __SCHEMA_TYPES = {
     'ExecutionRequestSignal': ExecutionRequestSignalClass,
     'ExecutionRequestSource': ExecutionRequestSourceClass,
     'StructuredExecutionReport': StructuredExecutionReportClass,
+    'BucketStorageLocation': BucketStorageLocationClass,
+    'DataHubFileInfo': DataHubFileInfoClass,
+    'FileUploadScenario': FileUploadScenarioClass,
     'DynamicFormAssignment': DynamicFormAssignmentClass,
     'FormActorAssignment': FormActorAssignmentClass,
     'FormInfo': FormInfoClass,
@@ -28322,6 +28537,7 @@ __SCHEMA_TYPES = {
     'DataHubAccessTokenKey': DataHubAccessTokenKeyClass,
     'DataHubActionKey': DataHubActionKeyClass,
     'DataHubConnectionKey': DataHubConnectionKeyClass,
+    'DataHubFileKey': DataHubFileKeyClass,
     'DataHubIngestionSourceKey': DataHubIngestionSourceKeyClass,
     'DataHubOpenAPISchemaKey': DataHubOpenAPISchemaKeyClass,
     'DataHubPageModuleKey': DataHubPageModuleKeyClass,
@@ -28585,6 +28801,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     DashboardInfoClass,
     EditableSchemaMetadataClass,
     SchemaMetadataClass,
+    DataHubFileInfoClass,
     AssertionActionsClass,
     AssertionRunEventClass,
     AssertionInfoClass,
@@ -28666,6 +28883,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     DataHubAccessTokenKeyClass,
     DataHubActionKeyClass,
     MLPrimaryKeyKeyClass,
+    DataHubFileKeyClass,
     TestKeyClass,
     GlossaryTermKeyClass,
     InviteTokenKeyClass,
@@ -28819,6 +29037,7 @@ class AspectBag(TypedDict, total=False):
     dashboardInfo: DashboardInfoClass
     editableSchemaMetadata: EditableSchemaMetadataClass
     schemaMetadata: SchemaMetadataClass
+    dataHubFileInfo: DataHubFileInfoClass
     assertionActions: AssertionActionsClass
     assertionRunEvent: AssertionRunEventClass
     assertionInfo: AssertionInfoClass
@@ -28900,6 +29119,7 @@ class AspectBag(TypedDict, total=False):
     dataHubAccessTokenKey: DataHubAccessTokenKeyClass
     dataHubActionKey: DataHubActionKeyClass
     mlPrimaryKeyKey: MLPrimaryKeyKeyClass
+    dataHubFileKey: DataHubFileKeyClass
     testKey: TestKeyClass
     glossaryTermKey: GlossaryTermKeyClass
     inviteTokenKey: InviteTokenKeyClass
@@ -29069,6 +29289,7 @@ KEY_ASPECTS: Dict[str, Type[_Aspect]] = {
     'dataHubAccessToken': DataHubAccessTokenKeyClass,
     'dataHubAction': DataHubActionKeyClass,
     'mlPrimaryKey': MLPrimaryKeyKeyClass,
+    'dataHubFile': DataHubFileKeyClass,
     'test': TestKeyClass,
     'glossaryTerm': GlossaryTermKeyClass,
     'inviteToken': InviteTokenKeyClass,
@@ -29135,6 +29356,7 @@ ENTITY_TYPE_NAMES: List[str] = [
     'dataHubAccessToken',
     'dataHubAction',
     'mlPrimaryKey',
+    'dataHubFile',
     'test',
     'glossaryTerm',
     'inviteToken',
@@ -29198,6 +29420,7 @@ EntityTypeName = Literal[
     'dataHubAccessToken',
     'dataHubAction',
     'mlPrimaryKey',
+    'dataHubFile',
     'test',
     'glossaryTerm',
     'inviteToken',

datahub/metadata/_urns/urn_defs.py

@@ -1885,6 +1885,62 @@ class MlPrimaryKeyUrn(_SpecificUrn):
     def name(self) -> str:
         return self._entity_ids[1]
 
+if TYPE_CHECKING:
+    from datahub.metadata.schema_classes import DataHubFileKeyClass
+
+class DataHubFileUrn(_SpecificUrn):
+    ENTITY_TYPE: ClassVar[Literal["dataHubFile"]] = "dataHubFile"
+    _URN_PARTS: ClassVar[int] = 1
+
+    def __init__(self, id: Union["DataHubFileUrn", str], *, _allow_coercion: bool = True) -> None:
+        if _allow_coercion:
+            # Field coercion logic (if any is required).
+            if isinstance(id, str):
+                if id.startswith('urn:li:'):
+                    try:
+                        id = DataHubFileUrn.from_string(id)
+                    except InvalidUrnError:
+                        raise InvalidUrnError(f'Expecting a DataHubFileUrn but got {id}')
+                else:
+                    id = UrnEncoder.encode_string(id)
+
+        # Validation logic.
+        if not id:
+            raise InvalidUrnError("DataHubFileUrn id cannot be empty")
+        if isinstance(id, DataHubFileUrn):
+            id = id.id
+        elif isinstance(id, Urn):
+            raise InvalidUrnError(f'Expecting a DataHubFileUrn but got {id}')
+        if UrnEncoder.contains_reserved_char(id):
+            raise InvalidUrnError(f'DataHubFileUrn id contains reserved characters')
+
+        super().__init__(self.ENTITY_TYPE, [id])
+
+    @classmethod
+    def _parse_ids(cls, entity_ids: List[str]) -> "DataHubFileUrn":
+        if len(entity_ids) != cls._URN_PARTS:
+            raise InvalidUrnError(f"DataHubFileUrn should have {cls._URN_PARTS} parts, got {len(entity_ids)}: {entity_ids}")
+        return cls(id=entity_ids[0], _allow_coercion=False)
+
+    @classmethod
+    def underlying_key_aspect_type(cls) -> Type["DataHubFileKeyClass"]:
+        from datahub.metadata.schema_classes import DataHubFileKeyClass
+
+        return DataHubFileKeyClass
+
+    def to_key_aspect(self) -> "DataHubFileKeyClass":
+        from datahub.metadata.schema_classes import DataHubFileKeyClass
+
+        return DataHubFileKeyClass(id=self.id)
+
+    @classmethod
+    def from_key_aspect(cls, key_aspect: "DataHubFileKeyClass") -> "DataHubFileUrn":
+        return cls(id=key_aspect.id)
+
+    @property
+    def id(self) -> str:
+        return self._entity_ids[0]
+
 if TYPE_CHECKING:
     from datahub.metadata.schema_classes import TestKeyClass
 

datahub/metadata/com/linkedin/pegasus2avro/file/__init__.py

@@ -0,0 +1,19 @@
+# mypy: ignore-errors
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# pylint: skip-file
+# fmt: off
+# isort: skip_file
+from .....schema_classes import BucketStorageLocationClass
+from .....schema_classes import DataHubFileInfoClass
+from .....schema_classes import FileUploadScenarioClass
+
+
+BucketStorageLocation = BucketStorageLocationClass
+DataHubFileInfo = DataHubFileInfoClass
+FileUploadScenario = FileUploadScenarioClass
+
+# fmt: on

datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py

@@ -18,6 +18,7 @@ from ......schema_classes import DataFlowKeyClass
 from ......schema_classes import DataHubAccessTokenKeyClass
 from ......schema_classes import DataHubActionKeyClass
 from ......schema_classes import DataHubConnectionKeyClass
+from ......schema_classes import DataHubFileKeyClass
 from ......schema_classes import DataHubIngestionSourceKeyClass
 from ......schema_classes import DataHubOpenAPISchemaKeyClass
 from ......schema_classes import DataHubPageModuleKeyClass
@@ -74,6 +75,7 @@ DataFlowKey = DataFlowKeyClass
 DataHubAccessTokenKey = DataHubAccessTokenKeyClass
 DataHubActionKey = DataHubActionKeyClass
 DataHubConnectionKey = DataHubConnectionKeyClass
+DataHubFileKey = DataHubFileKeyClass
 DataHubIngestionSourceKey = DataHubIngestionSourceKeyClass
 DataHubOpenAPISchemaKey = DataHubOpenAPISchemaKeyClass
 DataHubPageModuleKey = DataHubPageModuleKeyClass

datahub/metadata/schema.avsc

@@ -1903,6 +1903,191 @@
     ],
     "doc": "SchemaMetadata to describe metadata related to store schema"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataHubFileInfo"
+    },
+    "name": "DataHubFileInfo",
+    "namespace": "com.linkedin.pegasus2avro.file",
+    "fields": [
+      {
+        "type": {
+          "type": "record",
+          "name": "BucketStorageLocation",
+          "namespace": "com.linkedin.pegasus2avro.file",
+          "fields": [
+            {
+              "Searchable": {
+                "fieldType": "KEYWORD"
+              },
+              "type": "string",
+              "name": "storageBucket",
+              "doc": "The storage bucket this file is stored in"
+            },
+            {
+              "Searchable": {
+                "fieldType": "KEYWORD"
+              },
+              "type": "string",
+              "name": "storageKey",
+              "doc": "The key for where this file is stored inside of the given bucket"
+            }
+          ],
+          "doc": "Information where a file is stored"
+        },
+        "name": "bucketStorageLocation",
+        "doc": "Info about where a file is stored"
+      },
+      {
+        "Searchable": {
+          "fieldType": "TEXT_PARTIAL"
+        },
+        "type": "string",
+        "name": "originalFileName",
+        "doc": "The original filename as uploaded by the user"
+      },
+      {
+        "Searchable": {
+          "fieldType": "KEYWORD"
+        },
+        "type": "string",
+        "name": "mimeType",
+        "doc": "MIME type of the file (e.g., image/png, application/pdf)"
+      },
+      {
+        "type": "long",
+        "name": "sizeInBytes",
+        "doc": "Size of the file in bytes"
+      },
+      {
+        "Searchable": {
+          "fieldType": "KEYWORD"
+        },
+        "type": {
+          "type": "enum",
+          "symbolDocs": {
+            "ASSET_DOCUMENTATION": "File uploaded for entity documentation"
+          },
+          "name": "FileUploadScenario",
+          "namespace": "com.linkedin.pegasus2avro.file",
+          "symbols": [
+            "ASSET_DOCUMENTATION"
+          ]
+        },
+        "name": "scenario",
+        "doc": "The scenario/context in which this file was uploaded"
+      },
+      {
+        "Relationship": {
+          "entityTypes": [
+            "dataset",
+            "chart",
+            "container",
+            "dashboard",
+            "dataFlow",
+            "dataJob",
+            "glossaryTerm",
+            "glossaryNode",
+            "mlModel",
+            "mlFeature",
+            "notebook",
+            "mlFeatureTable",
+            "mlPrimaryKey",
+            "mlModelGroup",
+            "domain",
+            "dataProduct"
+          ],
+          "name": "ReferencedBy"
+        },
+        "Searchable": {
+          "fieldType": "URN"
+        },
+        "java": {
+          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+        },
+        "Urn": "Urn",
+        "entityTypes": [
+          "dataset",
+          "chart",
+          "container",
+          "dashboard",
+          "dataFlow",
+          "dataJob",
+          "glossaryTerm",
+          "glossaryNode",
+          "mlModel",
+          "mlFeature",
+          "notebook",
+          "mlFeatureTable",
+          "mlPrimaryKey",
+          "mlModelGroup",
+          "domain",
+          "dataProduct"
+        ],
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "referencedByAsset",
+        "default": null,
+        "doc": "Optional URN of the entity this file is associated with (e.g., the dataset whose docs contain this file)"
+      },
+      {
+        "Relationship": {
+          "entityTypes": [
+            "schemaField"
+          ],
+          "name": "ReferencedBy"
+        },
+        "Searchable": {
+          "fieldType": "URN"
+        },
+        "java": {
+          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+        },
+        "Urn": "Urn",
+        "entityTypes": [
+          "schemaField"
+        ],
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "schemaField",
+        "default": null,
+        "doc": "The dataset schema field urn this file is referenced by"
+      },
+      {
+        "Searchable": {
+          "/actor": {
+            "fieldName": "createdBy",
+            "fieldType": "URN"
+          },
+          "/time": {
+            "fieldName": "createdAt",
+            "fieldType": "DATETIME"
+          }
+        },
+        "type": "com.linkedin.pegasus2avro.common.AuditStamp",
+        "name": "created",
+        "doc": "Timestamp when this file was created and by whom"
+      },
+      {
+        "Searchable": {
+          "fieldType": "KEYWORD"
+        },
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "contentHash",
+        "default": null,
+        "doc": "SHA-256 hash of file contents"
+      }
+    ],
+    "doc": "Information about a DataHub file - a file stored in S3 for use within DataHub platform features like documentation, home pages, and announcements."
+  },
   {
     "type": "record",
     "Aspect": {
@@ -15005,6 +15190,27 @@
     "doc": "Key for a DataHub Action Pipeline"
   },
   "com.linkedin.pegasus2avro.metadata.key.MLPrimaryKeyKey",
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataHubFileKey",
+      "keyForEntity": "dataHubFile",
+      "entityCategory": "core",
+      "entityAspects": [
+        "dataHubFileInfo"
+      ]
+    },
+    "name": "DataHubFileKey",
+    "namespace": "com.linkedin.pegasus2avro.metadata.key",
+    "fields": [
+      {
+        "type": "string",
+        "name": "id",
+        "doc": "Unique id for the file."
+      }
+    ],
+    "doc": "Key for a DataHubFile"
+  },
   {
     "type": "record",
     "Aspect": {