acryl-datahub 0.15.0rc25__py3-none-any.whl → 0.15.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (120) hide show
  1. {acryl_datahub-0.15.0rc25.dist-info → acryl_datahub-0.15.0.1.dist-info}/METADATA +2236 -2240
  2. {acryl_datahub-0.15.0rc25.dist-info → acryl_datahub-0.15.0.1.dist-info}/RECORD +116 -106
  3. {acryl_datahub-0.15.0rc25.dist-info → acryl_datahub-0.15.0.1.dist-info}/WHEEL +1 -1
  4. {acryl_datahub-0.15.0rc25.dist-info → acryl_datahub-0.15.0.1.dist-info}/entry_points.txt +1 -1
  5. datahub/__init__.py +1 -1
  6. datahub/api/circuit_breaker/assertion_circuit_breaker.py +5 -4
  7. datahub/api/entities/structuredproperties/structuredproperties.py +20 -8
  8. datahub/configuration/common.py +2 -5
  9. datahub/configuration/source_common.py +13 -0
  10. datahub/emitter/mce_builder.py +20 -4
  11. datahub/emitter/mcp_builder.py +2 -7
  12. datahub/emitter/mcp_patch_builder.py +37 -13
  13. datahub/emitter/rest_emitter.py +25 -3
  14. datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py +98 -0
  15. datahub/ingestion/api/closeable.py +3 -3
  16. datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py +4 -7
  17. datahub/ingestion/api/report.py +4 -1
  18. datahub/ingestion/api/sink.py +4 -3
  19. datahub/ingestion/api/source.py +4 -0
  20. datahub/ingestion/api/source_helpers.py +2 -6
  21. datahub/ingestion/glossary/classifier.py +2 -3
  22. datahub/ingestion/graph/client.py +6 -3
  23. datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +44 -1
  24. datahub/ingestion/source/aws/aws_common.py +231 -27
  25. datahub/ingestion/source/aws/glue.py +12 -2
  26. datahub/ingestion/source/bigquery_v2/bigquery.py +10 -18
  27. datahub/ingestion/source/bigquery_v2/bigquery_config.py +3 -9
  28. datahub/ingestion/source/bigquery_v2/bigquery_schema.py +5 -20
  29. datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +11 -17
  30. datahub/ingestion/source/bigquery_v2/lineage.py +9 -22
  31. datahub/ingestion/source/datahub/config.py +22 -1
  32. datahub/ingestion/source/datahub/datahub_database_reader.py +3 -17
  33. datahub/ingestion/source/datahub/datahub_kafka_reader.py +2 -1
  34. datahub/ingestion/source/datahub/datahub_source.py +1 -1
  35. datahub/ingestion/source/dbt/dbt_cloud.py +10 -3
  36. datahub/ingestion/source/gc/datahub_gc.py +21 -5
  37. datahub/ingestion/source/gc/dataprocess_cleanup.py +23 -10
  38. datahub/ingestion/source/gc/execution_request_cleanup.py +61 -16
  39. datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +178 -83
  40. datahub/ingestion/source/iceberg/iceberg.py +27 -1
  41. datahub/ingestion/source/iceberg/iceberg_common.py +4 -0
  42. datahub/ingestion/source/kafka_connect/__init__.py +0 -0
  43. datahub/ingestion/source/kafka_connect/common.py +202 -0
  44. datahub/ingestion/source/kafka_connect/kafka_connect.py +367 -0
  45. datahub/ingestion/source/kafka_connect/sink_connectors.py +341 -0
  46. datahub/ingestion/source/kafka_connect/source_connectors.py +570 -0
  47. datahub/ingestion/source/looker/looker_common.py +63 -2
  48. datahub/ingestion/source/looker/looker_dataclasses.py +7 -9
  49. datahub/ingestion/source/looker/looker_lib_wrapper.py +13 -1
  50. datahub/ingestion/source/looker/looker_source.py +31 -4
  51. datahub/ingestion/source/looker/looker_usage.py +23 -17
  52. datahub/ingestion/source/mlflow.py +30 -5
  53. datahub/ingestion/source/mode.py +40 -27
  54. datahub/ingestion/source/powerbi/config.py +1 -14
  55. datahub/ingestion/source/powerbi/dataplatform_instance_resolver.py +1 -1
  56. datahub/ingestion/source/powerbi/m_query/pattern_handler.py +1 -1
  57. datahub/ingestion/source/powerbi/powerbi-lexical-grammar.rule +16 -2
  58. datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +16 -15
  59. datahub/ingestion/source/s3/source.py +1 -1
  60. datahub/ingestion/source/snowflake/snowflake_config.py +13 -34
  61. datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +14 -64
  62. datahub/ingestion/source/snowflake/snowflake_queries.py +44 -14
  63. datahub/ingestion/source/snowflake/snowflake_query.py +5 -10
  64. datahub/ingestion/source/snowflake/snowflake_schema_gen.py +53 -7
  65. datahub/ingestion/source/snowflake/snowflake_shares.py +1 -1
  66. datahub/ingestion/source/snowflake/snowflake_usage_v2.py +2 -5
  67. datahub/ingestion/source/snowflake/snowflake_utils.py +22 -18
  68. datahub/ingestion/source/snowflake/snowflake_v2.py +38 -34
  69. datahub/ingestion/source/sql/hive.py +621 -8
  70. datahub/ingestion/source/sql/hive_metastore.py +7 -0
  71. datahub/ingestion/source/sql/mssql/job_models.py +30 -1
  72. datahub/ingestion/source/sql/mssql/source.py +15 -1
  73. datahub/ingestion/source/sql/sql_common.py +41 -102
  74. datahub/ingestion/source/sql/sql_generic_profiler.py +5 -6
  75. datahub/ingestion/source/sql/sql_report.py +2 -0
  76. datahub/ingestion/source/state/checkpoint.py +2 -1
  77. datahub/ingestion/source/tableau/tableau.py +122 -45
  78. datahub/ingestion/source/tableau/tableau_common.py +18 -0
  79. datahub/ingestion/source/tableau/tableau_constant.py +3 -1
  80. datahub/ingestion/source/tableau/tableau_server_wrapper.py +6 -2
  81. datahub/ingestion/source/tableau/tableau_validation.py +1 -1
  82. datahub/ingestion/source/unity/proxy.py +8 -27
  83. datahub/ingestion/source/usage/usage_common.py +15 -1
  84. datahub/ingestion/source_report/ingestion_stage.py +3 -0
  85. datahub/metadata/_schema_classes.py +256 -3
  86. datahub/metadata/_urns/urn_defs.py +168 -168
  87. datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +4 -0
  88. datahub/metadata/com/linkedin/pegasus2avro/ml/metadata/__init__.py +2 -0
  89. datahub/metadata/schema.avsc +252 -33
  90. datahub/metadata/schemas/DataJobKey.avsc +2 -1
  91. datahub/metadata/schemas/DataProcessInstanceKey.avsc +5 -1
  92. datahub/metadata/schemas/DataProcessInstanceOutput.avsc +2 -1
  93. datahub/metadata/schemas/DataTransformLogic.avsc +63 -0
  94. datahub/metadata/schemas/MLModelGroupProperties.avsc +82 -0
  95. datahub/metadata/schemas/MLModelProperties.avsc +62 -2
  96. datahub/metadata/schemas/MLTrainingRunProperties.avsc +171 -0
  97. datahub/metadata/schemas/MetadataChangeEvent.avsc +94 -2
  98. datahub/specific/aspect_helpers/__init__.py +0 -0
  99. datahub/specific/aspect_helpers/custom_properties.py +79 -0
  100. datahub/specific/aspect_helpers/ownership.py +67 -0
  101. datahub/specific/aspect_helpers/structured_properties.py +72 -0
  102. datahub/specific/aspect_helpers/tags.py +42 -0
  103. datahub/specific/aspect_helpers/terms.py +43 -0
  104. datahub/specific/chart.py +28 -184
  105. datahub/specific/dashboard.py +31 -196
  106. datahub/specific/datajob.py +34 -189
  107. datahub/specific/dataproduct.py +24 -86
  108. datahub/specific/dataset.py +48 -133
  109. datahub/specific/form.py +12 -32
  110. datahub/specific/structured_property.py +9 -9
  111. datahub/sql_parsing/sql_parsing_aggregator.py +10 -9
  112. datahub/sql_parsing/sqlglot_lineage.py +15 -5
  113. datahub/sql_parsing/tool_meta_extractor.py +119 -5
  114. datahub/utilities/time.py +8 -3
  115. datahub/utilities/urns/_urn_base.py +5 -7
  116. datahub/ingestion/source/kafka/kafka_connect.py +0 -1468
  117. datahub/specific/custom_properties.py +0 -37
  118. datahub/specific/ownership.py +0 -48
  119. datahub/specific/structured_properties.py +0 -53
  120. {acryl_datahub-0.15.0rc25.dist-info → acryl_datahub-0.15.0.1.dist-info}/top_level.txt +0 -0
@@ -1,37 +0,0 @@
1
- from typing import Generic, TypeVar
2
-
3
- from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
4
-
5
- _Parent = TypeVar("_Parent", bound=MetadataPatchProposal)
6
-
7
-
8
- class CustomPropertiesPatchHelper(Generic[_Parent]):
9
- def __init__(
10
- self,
11
- parent: _Parent,
12
- aspect_name: str,
13
- ) -> None:
14
- self.aspect_name = aspect_name
15
- self._parent = parent
16
- self.aspect_field = "customProperties"
17
-
18
- def parent(self) -> _Parent:
19
- return self._parent
20
-
21
- def add_property(self, key: str, value: str) -> "CustomPropertiesPatchHelper":
22
- self._parent._add_patch(
23
- self.aspect_name,
24
- "add",
25
- path=f"/{self.aspect_field}/{key}",
26
- value=value,
27
- )
28
- return self
29
-
30
- def remove_property(self, key: str) -> "CustomPropertiesPatchHelper":
31
- self._parent._add_patch(
32
- self.aspect_name,
33
- "remove",
34
- path=f"/{self.aspect_field}/{key}",
35
- value={},
36
- )
37
- return self
@@ -1,48 +0,0 @@
1
- from typing import Generic, List, Optional, TypeVar
2
-
3
- from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
4
- from datahub.metadata.schema_classes import (
5
- OwnerClass,
6
- OwnershipClass,
7
- OwnershipTypeClass,
8
- )
9
-
10
- _Parent = TypeVar("_Parent", bound=MetadataPatchProposal)
11
-
12
-
13
- class OwnershipPatchHelper(Generic[_Parent]):
14
- def __init__(self, parent: _Parent) -> None:
15
- self._parent = parent
16
- self.aspect_field = OwnershipClass.ASPECT_NAME
17
-
18
- def parent(self) -> _Parent:
19
- return self._parent
20
-
21
- def add_owner(self, owner: OwnerClass) -> "OwnershipPatchHelper":
22
- self._parent._add_patch(
23
- OwnershipClass.ASPECT_NAME,
24
- "add",
25
- path=f"/owners/{owner.owner}/{owner.type}",
26
- value=owner,
27
- )
28
- return self
29
-
30
- def remove_owner(
31
- self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
32
- ) -> "OwnershipPatchHelper":
33
- """
34
- param: owner_type is optional
35
- """
36
- self._parent._add_patch(
37
- OwnershipClass.ASPECT_NAME,
38
- "remove",
39
- path=f"/owners/{owner}" + (f"/{owner_type}" if owner_type else ""),
40
- value=owner,
41
- )
42
- return self
43
-
44
- def set_owners(self, owners: List[OwnerClass]) -> "OwnershipPatchHelper":
45
- self._parent._add_patch(
46
- OwnershipClass.ASPECT_NAME, "add", path="/owners", value=owners
47
- )
48
- return self
@@ -1,53 +0,0 @@
1
- from typing import Generic, List, TypeVar, Union
2
-
3
- from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
4
- from datahub.metadata.schema_classes import StructuredPropertyValueAssignmentClass
5
- from datahub.utilities.urns.structured_properties_urn import (
6
- make_structured_property_urn,
7
- )
8
-
9
- _Parent = TypeVar("_Parent", bound=MetadataPatchProposal)
10
-
11
-
12
- class StructuredPropertiesPatchHelper(Generic[_Parent]):
13
- def __init__(
14
- self,
15
- parent: _Parent,
16
- aspect_name: str = "structuredProperties",
17
- ) -> None:
18
- self.aspect_name = aspect_name
19
- self._parent = parent
20
- self.aspect_field = "properties"
21
-
22
- def parent(self) -> _Parent:
23
- return self._parent
24
-
25
- def set_property(
26
- self, key: str, value: Union[str, float, List[Union[str, float]]]
27
- ) -> "StructuredPropertiesPatchHelper":
28
- self.remove_property(key)
29
- self.add_property(key, value)
30
- return self
31
-
32
- def remove_property(self, key: str) -> "StructuredPropertiesPatchHelper":
33
- self._parent._add_patch(
34
- self.aspect_name,
35
- "remove",
36
- path=(self.aspect_field, make_structured_property_urn(key)),
37
- value={},
38
- )
39
- return self
40
-
41
- def add_property(
42
- self, key: str, value: Union[str, float, List[Union[str, float]]]
43
- ) -> "StructuredPropertiesPatchHelper":
44
- self._parent._add_patch(
45
- self.aspect_name,
46
- "add",
47
- path=(self.aspect_field, make_structured_property_urn(key)),
48
- value=StructuredPropertyValueAssignmentClass(
49
- propertyUrn=make_structured_property_urn(key),
50
- values=value if isinstance(value, list) else [value],
51
- ),
52
- )
53
- return self