acryl-datahub 1.1.0.5rc7__py3-none-any.whl → 1.1.0.5rc9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (59) hide show
  1. {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/METADATA +2620 -2622
  2. {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/RECORD +59 -59
  3. datahub/_version.py +1 -1
  4. datahub/cli/check_cli.py +0 -7
  5. datahub/cli/cli_utils.py +73 -0
  6. datahub/cli/delete_cli.py +0 -6
  7. datahub/cli/docker_check.py +107 -12
  8. datahub/cli/docker_cli.py +148 -228
  9. datahub/cli/exists_cli.py +0 -4
  10. datahub/cli/get_cli.py +0 -4
  11. datahub/cli/ingest_cli.py +1 -20
  12. datahub/cli/put_cli.py +0 -6
  13. datahub/cli/quickstart_versioning.py +50 -5
  14. datahub/cli/specific/assertions_cli.py +0 -6
  15. datahub/cli/specific/datacontract_cli.py +0 -6
  16. datahub/cli/specific/dataproduct_cli.py +0 -22
  17. datahub/cli/specific/dataset_cli.py +0 -11
  18. datahub/cli/specific/forms_cli.py +0 -6
  19. datahub/cli/specific/group_cli.py +0 -4
  20. datahub/cli/specific/structuredproperties_cli.py +0 -7
  21. datahub/cli/specific/user_cli.py +0 -4
  22. datahub/cli/state_cli.py +0 -4
  23. datahub/cli/timeline_cli.py +0 -4
  24. datahub/entrypoints.py +4 -3
  25. datahub/ingestion/autogenerated/capability_summary.json +88 -23
  26. datahub/ingestion/extractor/schema_util.py +13 -4
  27. datahub/ingestion/graph/client.py +2 -2
  28. datahub/ingestion/run/pipeline.py +43 -0
  29. datahub/ingestion/source/bigquery_v2/bigquery.py +9 -1
  30. datahub/ingestion/source/datahub/datahub_database_reader.py +1 -2
  31. datahub/ingestion/source/dremio/dremio_source.py +1 -4
  32. datahub/ingestion/source/gcs/gcs_source.py +9 -1
  33. datahub/ingestion/source/identity/okta.py +0 -13
  34. datahub/ingestion/source/powerbi/powerbi.py +0 -5
  35. datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +0 -1
  36. datahub/ingestion/source/powerbi_report_server/report_server.py +0 -23
  37. datahub/ingestion/source/sigma/sigma.py +6 -1
  38. datahub/ingestion/source/snowflake/snowflake_config.py +11 -0
  39. datahub/ingestion/source/snowflake/snowflake_queries.py +100 -58
  40. datahub/ingestion/source/snowflake/snowflake_v2.py +11 -1
  41. datahub/ingestion/source/snowflake/stored_proc_lineage.py +1 -1
  42. datahub/ingestion/source/sql/hive_metastore.py +0 -10
  43. datahub/ingestion/source/sql/sql_common.py +8 -0
  44. datahub/ingestion/source/sql/teradata.py +993 -234
  45. datahub/ingestion/source/sql/vertica.py +0 -4
  46. datahub/ingestion/source/sql_queries.py +2 -2
  47. datahub/ingestion/source/superset.py +56 -1
  48. datahub/ingestion/source/tableau/tableau.py +40 -34
  49. datahub/ingestion/source/tableau/tableau_constant.py +0 -2
  50. datahub/ingestion/source/unity/source.py +9 -1
  51. datahub/sdk/lineage_client.py +2 -2
  52. datahub/sql_parsing/sql_parsing_aggregator.py +21 -12
  53. datahub/sql_parsing/sqlglot_lineage.py +40 -15
  54. datahub/upgrade/upgrade.py +46 -13
  55. datahub/utilities/server_config_util.py +8 -0
  56. {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/WHEEL +0 -0
  57. {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/entry_points.txt +0 -0
  58. {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/licenses/LICENSE +0 -0
  59. {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/top_level.txt +0 -0
@@ -32,6 +32,7 @@ class ServerVersionStats(BaseModel):
32
32
  current: VersionStats
33
33
  latest: Optional[VersionStats] = None
34
34
  current_server_type: Optional[str] = None
35
+ current_server_default_cli_version: Optional[VersionStats] = None
35
36
 
36
37
 
37
38
  class ClientVersionStats(BaseModel):
@@ -44,7 +45,7 @@ class DataHubVersionStats(BaseModel):
44
45
  client: ClientVersionStats
45
46
 
46
47
 
47
- async def get_client_version_stats():
48
+ async def get_client_version_stats() -> ClientVersionStats:
48
49
  import aiohttp
49
50
 
50
51
  current_version_string = __version__
@@ -52,6 +53,7 @@ async def get_client_version_stats():
52
53
  client_version_stats: ClientVersionStats = ClientVersionStats(
53
54
  current=VersionStats(version=current_version, release_date=None), latest=None
54
55
  )
56
+
55
57
  async with aiohttp.ClientSession() as session:
56
58
  pypi_url = "https://pypi.org/pypi/acryl_datahub/json"
57
59
  async with session.get(pypi_url) as resp:
@@ -131,7 +133,7 @@ async def get_server_config(gms_url: str, token: Optional[str]) -> RestServiceCo
131
133
 
132
134
  async def get_server_version_stats(
133
135
  server: Optional[DataHubGraph] = None,
134
- ) -> Tuple[Optional[str], Optional[Version], Optional[datetime]]:
136
+ ) -> Tuple[Optional[str], Optional[Version], Optional[str], Optional[datetime]]:
135
137
  import aiohttp
136
138
 
137
139
  server_config: Optional[RestServiceConfig] = None
@@ -151,12 +153,13 @@ async def get_server_version_stats(
151
153
 
152
154
  server_type = None
153
155
  server_version: Optional[Version] = None
156
+ current_server_default_cli_version = None
154
157
  current_server_release_date = None
155
158
  if server_config:
156
159
  server_version_string = server_config.service_version
157
160
  commit_hash = server_config.commit_hash
158
161
  server_type = server_config.server_type
159
-
162
+ current_server_default_cli_version = server_config.default_cli_version
160
163
  if server_type == "quickstart" and commit_hash:
161
164
  async with aiohttp.ClientSession(
162
165
  headers={"Accept": "application/vnd.github.v3+json"}
@@ -171,7 +174,12 @@ async def get_server_version_stats(
171
174
  if server_version_string and server_version_string.startswith("v"):
172
175
  server_version = Version(server_version_string[1:])
173
176
 
174
- return (server_type, server_version, current_server_release_date)
177
+ return (
178
+ server_type,
179
+ server_version,
180
+ current_server_default_cli_version,
181
+ current_server_release_date,
182
+ )
175
183
 
176
184
 
177
185
  def retrieve_version_stats(
@@ -214,6 +222,7 @@ async def _retrieve_version_stats(
214
222
  (
215
223
  current_server_type,
216
224
  current_server_version,
225
+ current_server_default_cli_version,
217
226
  current_server_release_date,
218
227
  ) = results[2]
219
228
 
@@ -223,6 +232,14 @@ async def _retrieve_version_stats(
223
232
  current=VersionStats(
224
233
  version=current_server_version, release_date=current_server_release_date
225
234
  ),
235
+ current_server_default_cli_version=(
236
+ VersionStats(
237
+ version=Version(current_server_default_cli_version),
238
+ release_date=None,
239
+ )
240
+ if current_server_default_cli_version
241
+ else None
242
+ ),
226
243
  latest=(
227
244
  VersionStats(version=last_server_version, release_date=last_server_date)
228
245
  if last_server_version
@@ -255,21 +272,14 @@ def valid_client_version(version: Version) -> bool:
255
272
  """Only version strings like 0.4.5 and 0.6.7.8 are valid. 0.8.6.7rc1 is not"""
256
273
  if version.is_prerelease or version.is_postrelease or version.is_devrelease:
257
274
  return False
258
- if version.major == 0 and version.minor in [8, 9, 10, 11]:
259
- return True
260
-
261
- return False
275
+ return True
262
276
 
263
277
 
264
278
  def valid_server_version(version: Version) -> bool:
265
279
  """Only version strings like 0.8.x, 0.9.x or 0.10.x are valid. 0.1.x is not"""
266
280
  if version.is_prerelease or version.is_postrelease or version.is_devrelease:
267
281
  return False
268
-
269
- if version.major == 0 and version.minor in [8, 9, 10]:
270
- return True
271
-
272
- return False
282
+ return True
273
283
 
274
284
 
275
285
  def is_client_server_compatible(client: VersionStats, server: VersionStats) -> int:
@@ -291,6 +301,27 @@ def is_client_server_compatible(client: VersionStats, server: VersionStats) -> i
291
301
  return server.version.micro - client.version.micro
292
302
 
293
303
 
304
+ def is_server_default_cli_ahead(version_stats: DataHubVersionStats) -> bool:
305
+ """
306
+ Check if the server default CLI version is ahead of the current CLI version.
307
+ Returns True if server default CLI is newer and both versions are valid.
308
+ """
309
+ if not version_stats.server.current_server_default_cli_version:
310
+ return False
311
+
312
+ current_cli = version_stats.client.current
313
+ server_default_cli = version_stats.server.current_server_default_cli_version
314
+
315
+ is_valid_client_version = valid_client_version(current_cli.version)
316
+ is_valid_server_version = valid_client_version(server_default_cli.version)
317
+
318
+ if not (is_valid_client_version and is_valid_server_version):
319
+ return False
320
+
321
+ compatibility_result = is_client_server_compatible(current_cli, server_default_cli)
322
+ return compatibility_result > 0
323
+
324
+
294
325
  def _maybe_print_upgrade_message(
295
326
  version_stats: Optional[DataHubVersionStats],
296
327
  ) -> None:
@@ -429,6 +460,8 @@ def check_upgrade_post(
429
460
 
430
461
 
431
462
  def check_upgrade(func: Callable[..., T]) -> Callable[..., T]:
463
+ log.debug(f"Checking upgrade for {func.__module__}.{func.__name__}")
464
+
432
465
  @wraps(func)
433
466
  def async_wrapper(*args: Any, **kwargs: Any) -> Any:
434
467
  with PerfTimer() as timer:
@@ -183,6 +183,14 @@ class RestServiceConfig:
183
183
  managed_ingestion = self.raw_config.get("managedIngestion") or {}
184
184
  return managed_ingestion.get("enabled", False)
185
185
 
186
+ @property
187
+ def default_cli_version(self) -> Optional[str]:
188
+ """
189
+ Get the default CLI version.
190
+ """
191
+ managed_ingestion = self.raw_config.get("managedIngestion") or {}
192
+ return managed_ingestion.get("defaultCliVersion")
193
+
186
194
  @property
187
195
  def is_datahub_cloud(self) -> bool:
188
196
  """