acryl-datahub 1.2.0.10rc3__py3-none-any.whl → 1.2.0.10rc5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (94) hide show
  1. {acryl_datahub-1.2.0.10rc3.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/METADATA +2513 -2571
  2. {acryl_datahub-1.2.0.10rc3.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/RECORD +94 -87
  3. {acryl_datahub-1.2.0.10rc3.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/entry_points.txt +2 -0
  4. datahub/_version.py +1 -1
  5. datahub/api/entities/assertion/assertion.py +1 -1
  6. datahub/api/entities/corpgroup/corpgroup.py +1 -1
  7. datahub/api/entities/dataproduct/dataproduct.py +6 -3
  8. datahub/api/entities/dataset/dataset.py +9 -18
  9. datahub/api/entities/structuredproperties/structuredproperties.py +2 -2
  10. datahub/api/graphql/operation.py +10 -6
  11. datahub/cli/docker_check.py +2 -2
  12. datahub/configuration/common.py +29 -1
  13. datahub/configuration/connection_resolver.py +5 -2
  14. datahub/configuration/import_resolver.py +7 -4
  15. datahub/configuration/pydantic_migration_helpers.py +0 -9
  16. datahub/configuration/source_common.py +3 -2
  17. datahub/configuration/validate_field_deprecation.py +5 -2
  18. datahub/configuration/validate_field_removal.py +5 -2
  19. datahub/configuration/validate_field_rename.py +6 -5
  20. datahub/configuration/validate_multiline_string.py +5 -2
  21. datahub/ingestion/autogenerated/capability_summary.json +33 -1
  22. datahub/ingestion/run/pipeline_config.py +2 -2
  23. datahub/ingestion/source/azure/azure_common.py +1 -1
  24. datahub/ingestion/source/bigquery_v2/bigquery_config.py +28 -14
  25. datahub/ingestion/source/bigquery_v2/queries_extractor.py +4 -5
  26. datahub/ingestion/source/common/gcp_credentials_config.py +3 -1
  27. datahub/ingestion/source/data_lake_common/path_spec.py +16 -16
  28. datahub/ingestion/source/datahub/config.py +8 -9
  29. datahub/ingestion/source/delta_lake/config.py +1 -1
  30. datahub/ingestion/source/dremio/dremio_config.py +3 -4
  31. datahub/ingestion/source/feast.py +8 -10
  32. datahub/ingestion/source/fivetran/config.py +1 -1
  33. datahub/ingestion/source/ge_profiling_config.py +26 -22
  34. datahub/ingestion/source/grafana/grafana_config.py +2 -2
  35. datahub/ingestion/source/grafana/models.py +12 -14
  36. datahub/ingestion/source/hex/hex.py +6 -1
  37. datahub/ingestion/source/iceberg/iceberg_profiler.py +4 -2
  38. datahub/ingestion/source/kafka_connect/common.py +2 -2
  39. datahub/ingestion/source/looker/looker_common.py +1 -1
  40. datahub/ingestion/source/looker/looker_config.py +15 -4
  41. datahub/ingestion/source/looker/looker_source.py +52 -3
  42. datahub/ingestion/source/looker/lookml_config.py +1 -1
  43. datahub/ingestion/source/metadata/business_glossary.py +7 -7
  44. datahub/ingestion/source/metadata/lineage.py +1 -1
  45. datahub/ingestion/source/mode.py +13 -5
  46. datahub/ingestion/source/nifi.py +1 -1
  47. datahub/ingestion/source/powerbi/config.py +14 -21
  48. datahub/ingestion/source/preset.py +1 -1
  49. datahub/ingestion/source/qlik_sense/data_classes.py +28 -8
  50. datahub/ingestion/source/redshift/config.py +6 -3
  51. datahub/ingestion/source/salesforce.py +13 -9
  52. datahub/ingestion/source/schema/json_schema.py +14 -14
  53. datahub/ingestion/source/sigma/data_classes.py +3 -0
  54. datahub/ingestion/source/snaplogic/__init__.py +0 -0
  55. datahub/ingestion/source/snaplogic/snaplogic.py +355 -0
  56. datahub/ingestion/source/snaplogic/snaplogic_config.py +37 -0
  57. datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py +107 -0
  58. datahub/ingestion/source/snaplogic/snaplogic_parser.py +168 -0
  59. datahub/ingestion/source/snaplogic/snaplogic_utils.py +31 -0
  60. datahub/ingestion/source/snowflake/snowflake_config.py +12 -15
  61. datahub/ingestion/source/snowflake/snowflake_connection.py +8 -3
  62. datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +15 -2
  63. datahub/ingestion/source/snowflake/snowflake_queries.py +4 -5
  64. datahub/ingestion/source/sql/athena.py +2 -1
  65. datahub/ingestion/source/sql/clickhouse.py +12 -7
  66. datahub/ingestion/source/sql/cockroachdb.py +5 -3
  67. datahub/ingestion/source/sql/druid.py +2 -2
  68. datahub/ingestion/source/sql/hive.py +4 -3
  69. datahub/ingestion/source/sql/hive_metastore.py +7 -9
  70. datahub/ingestion/source/sql/mssql/source.py +2 -2
  71. datahub/ingestion/source/sql/mysql.py +2 -2
  72. datahub/ingestion/source/sql/oracle.py +3 -3
  73. datahub/ingestion/source/sql/presto.py +2 -1
  74. datahub/ingestion/source/sql/teradata.py +4 -4
  75. datahub/ingestion/source/sql/trino.py +2 -1
  76. datahub/ingestion/source/sql/two_tier_sql_source.py +2 -3
  77. datahub/ingestion/source/sql/vertica.py +1 -1
  78. datahub/ingestion/source/sql_queries.py +6 -6
  79. datahub/ingestion/source/state/checkpoint.py +5 -1
  80. datahub/ingestion/source/state/entity_removal_state.py +5 -2
  81. datahub/ingestion/source/state/stateful_ingestion_base.py +5 -8
  82. datahub/ingestion/source/superset.py +1 -2
  83. datahub/ingestion/source/tableau/tableau.py +20 -6
  84. datahub/ingestion/source/unity/config.py +7 -3
  85. datahub/ingestion/source/usage/usage_common.py +3 -3
  86. datahub/ingestion/source_config/pulsar.py +3 -1
  87. datahub/ingestion/transformer/set_browse_path.py +112 -0
  88. datahub/sdk/_shared.py +126 -0
  89. datahub/sdk/chart.py +87 -30
  90. datahub/sdk/dashboard.py +79 -32
  91. datahub/sdk/search_filters.py +1 -7
  92. {acryl_datahub-1.2.0.10rc3.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/WHEEL +0 -0
  93. {acryl_datahub-1.2.0.10rc3.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/licenses/LICENSE +0 -0
  94. {acryl_datahub-1.2.0.10rc3.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/top_level.txt +0 -0
datahub/sdk/dashboard.py CHANGED
@@ -9,6 +9,8 @@ from typing_extensions import Self
9
9
  import datahub.metadata.schema_classes as models
10
10
  from datahub.metadata.urns import ChartUrn, DashboardUrn, DatasetUrn, Urn
11
11
  from datahub.sdk._shared import (
12
+ ActorUrnOrStr,
13
+ ChangeAuditStampsMixin,
12
14
  ChartUrnOrStr,
13
15
  DashboardUrnOrStr,
14
16
  DataPlatformInstanceUrnOrStr,
@@ -36,6 +38,7 @@ from datahub.utilities.sentinels import Unset, unset
36
38
 
37
39
 
38
40
  class Dashboard(
41
+ ChangeAuditStampsMixin,
39
42
  HasPlatformInstance,
40
43
  HasSubtype,
41
44
  HasOwnership,
@@ -72,6 +75,11 @@ class Dashboard(
72
75
  dashboard_url: Optional[str] = None,
73
76
  custom_properties: Optional[Dict[str, str]] = None,
74
77
  last_modified: Optional[datetime] = None,
78
+ last_modified_by: Optional[ActorUrnOrStr] = None,
79
+ created_at: Optional[datetime] = None,
80
+ created_by: Optional[ActorUrnOrStr] = None,
81
+ deleted_on: Optional[datetime] = None,
82
+ deleted_by: Optional[ActorUrnOrStr] = None,
75
83
  last_refreshed: Optional[datetime] = None,
76
84
  input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]] = None,
77
85
  charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]] = None,
@@ -96,17 +104,48 @@ class Dashboard(
96
104
  self._set_extra_aspects(extra_aspects)
97
105
 
98
106
  self._set_platform_instance(platform, platform_instance)
107
+ self._ensure_dashboard_props(display_name=display_name)
108
+
109
+ self._init_dashboard_properties(
110
+ description,
111
+ display_name,
112
+ external_url,
113
+ dashboard_url,
114
+ custom_properties,
115
+ last_modified,
116
+ last_modified_by,
117
+ created_at,
118
+ created_by,
119
+ last_refreshed,
120
+ deleted_on,
121
+ deleted_by,
122
+ input_datasets,
123
+ charts,
124
+ dashboards,
125
+ )
126
+ self._init_standard_aspects(
127
+ parent_container, subtype, owners, links, tags, terms, domain
128
+ )
99
129
 
100
- # Initialize DashboardInfoClass with default values
101
- dashboard_info = self._ensure_dashboard_props(display_name=display_name)
102
- if last_modified:
103
- dashboard_info.lastModified = models.ChangeAuditStampsClass(
104
- lastModified=models.AuditStampClass(
105
- time=int(last_modified.timestamp()),
106
- actor="urn:li:corpuser:datahub",
107
- ),
108
- )
109
-
130
+ def _init_dashboard_properties(
131
+ self,
132
+ description: Optional[str],
133
+ display_name: Optional[str],
134
+ external_url: Optional[str],
135
+ dashboard_url: Optional[str],
136
+ custom_properties: Optional[Dict[str, str]],
137
+ last_modified: Optional[datetime],
138
+ last_modified_by: Optional[ActorUrnOrStr],
139
+ created_at: Optional[datetime],
140
+ created_by: Optional[ActorUrnOrStr],
141
+ last_refreshed: Optional[datetime],
142
+ deleted_on: Optional[datetime],
143
+ deleted_by: Optional[ActorUrnOrStr],
144
+ input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]],
145
+ charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]],
146
+ dashboards: Optional[Sequence[Union[DashboardUrnOrStr, Dashboard]]],
147
+ ) -> None:
148
+ """Initialize dashboard-specific properties."""
110
149
  if description is not None:
111
150
  self.set_description(description)
112
151
  if display_name is not None:
@@ -119,6 +158,16 @@ class Dashboard(
119
158
  self.set_custom_properties(custom_properties)
120
159
  if last_modified is not None:
121
160
  self.set_last_modified(last_modified)
161
+ if last_modified_by is not None:
162
+ self.set_last_modified_by(last_modified_by)
163
+ if created_at is not None:
164
+ self.set_created_at(created_at)
165
+ if created_by is not None:
166
+ self.set_created_by(created_by)
167
+ if deleted_on is not None:
168
+ self.set_deleted_on(deleted_on)
169
+ if deleted_by is not None:
170
+ self.set_deleted_by(deleted_by)
122
171
  if last_refreshed is not None:
123
172
  self.set_last_refreshed(last_refreshed)
124
173
  if input_datasets is not None:
@@ -128,6 +177,17 @@ class Dashboard(
128
177
  if dashboards is not None:
129
178
  self.set_dashboards(dashboards)
130
179
 
180
+ def _init_standard_aspects(
181
+ self,
182
+ parent_container: ParentContainerInputType | Unset,
183
+ subtype: Optional[str],
184
+ owners: Optional[OwnersInputType],
185
+ links: Optional[LinksInputType],
186
+ tags: Optional[TagsInputType],
187
+ terms: Optional[TermsInputType],
188
+ domain: Optional[DomainInputType],
189
+ ) -> None:
190
+ """Initialize standard aspects."""
131
191
  if parent_container is not unset:
132
192
  self._set_container(parent_container)
133
193
  if subtype is not None:
@@ -165,16 +225,20 @@ class Dashboard(
165
225
  models.DashboardInfoClass(
166
226
  title=display_name or self.urn.dashboard_id,
167
227
  description="",
168
- lastModified=models.ChangeAuditStampsClass(
169
- lastModified=models.AuditStampClass(
170
- time=0, actor="urn:li:corpuser:unknown"
171
- )
172
- ),
228
+ lastModified=models.ChangeAuditStampsClass(),
173
229
  customProperties={},
174
230
  dashboards=[],
175
231
  )
176
232
  )
177
233
 
234
+ def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
235
+ """Get the audit stamps from the dashboard properties."""
236
+ return self._ensure_dashboard_props().lastModified
237
+
238
+ def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
239
+ """Set the audit stamps on the dashboard properties."""
240
+ self._ensure_dashboard_props().lastModified = audit_stamps
241
+
178
242
  @property
179
243
  def name(self) -> str:
180
244
  """Get the name of the dashboard."""
@@ -238,23 +302,6 @@ class Dashboard(
238
302
  """Set the custom properties of the dashboard."""
239
303
  self._ensure_dashboard_props().customProperties = custom_properties
240
304
 
241
- @property
242
- def last_modified(self) -> Optional[datetime]:
243
- """Get the last modification timestamp of the dashboard."""
244
- props = self._ensure_dashboard_props()
245
- if props.lastModified.lastModified.time == 0:
246
- return None
247
- return datetime.fromtimestamp(props.lastModified.lastModified.time)
248
-
249
- def set_last_modified(self, last_modified: datetime) -> None:
250
- """Set the last modification timestamp of the dashboard."""
251
- self._ensure_dashboard_props().lastModified = models.ChangeAuditStampsClass(
252
- lastModified=models.AuditStampClass(
253
- time=int(last_modified.timestamp()),
254
- actor="urn:li:corpuser:datahub",
255
- ),
256
- )
257
-
258
305
  @property
259
306
  def last_refreshed(self) -> Optional[datetime]:
260
307
  """Get the last refresh timestamp of the dashboard."""
@@ -39,13 +39,7 @@ _OrFilters = List[_AndSearchFilterRule]
39
39
 
40
40
 
41
41
  class _BaseFilter(ConfigModel):
42
- class Config:
43
- # We can't wrap this in a TYPE_CHECKING block because the pydantic plugin
44
- # doesn't recognize it properly. So unfortunately we'll need to live
45
- # with the deprecation warning w/ pydantic v2.
46
- allow_population_by_field_name = True
47
- if PYDANTIC_VERSION_2:
48
- populate_by_name = True
42
+ model_config = pydantic.ConfigDict(populate_by_name=True)
49
43
 
50
44
  @abc.abstractmethod
51
45
  def compile(self) -> _OrFilters: ...