acryl-datahub 1.1.0.5rc7__py3-none-any.whl → 1.1.0.5rc9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/METADATA +2620 -2622
- {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/RECORD +59 -59
- datahub/_version.py +1 -1
- datahub/cli/check_cli.py +0 -7
- datahub/cli/cli_utils.py +73 -0
- datahub/cli/delete_cli.py +0 -6
- datahub/cli/docker_check.py +107 -12
- datahub/cli/docker_cli.py +148 -228
- datahub/cli/exists_cli.py +0 -4
- datahub/cli/get_cli.py +0 -4
- datahub/cli/ingest_cli.py +1 -20
- datahub/cli/put_cli.py +0 -6
- datahub/cli/quickstart_versioning.py +50 -5
- datahub/cli/specific/assertions_cli.py +0 -6
- datahub/cli/specific/datacontract_cli.py +0 -6
- datahub/cli/specific/dataproduct_cli.py +0 -22
- datahub/cli/specific/dataset_cli.py +0 -11
- datahub/cli/specific/forms_cli.py +0 -6
- datahub/cli/specific/group_cli.py +0 -4
- datahub/cli/specific/structuredproperties_cli.py +0 -7
- datahub/cli/specific/user_cli.py +0 -4
- datahub/cli/state_cli.py +0 -4
- datahub/cli/timeline_cli.py +0 -4
- datahub/entrypoints.py +4 -3
- datahub/ingestion/autogenerated/capability_summary.json +88 -23
- datahub/ingestion/extractor/schema_util.py +13 -4
- datahub/ingestion/graph/client.py +2 -2
- datahub/ingestion/run/pipeline.py +43 -0
- datahub/ingestion/source/bigquery_v2/bigquery.py +9 -1
- datahub/ingestion/source/datahub/datahub_database_reader.py +1 -2
- datahub/ingestion/source/dremio/dremio_source.py +1 -4
- datahub/ingestion/source/gcs/gcs_source.py +9 -1
- datahub/ingestion/source/identity/okta.py +0 -13
- datahub/ingestion/source/powerbi/powerbi.py +0 -5
- datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +0 -1
- datahub/ingestion/source/powerbi_report_server/report_server.py +0 -23
- datahub/ingestion/source/sigma/sigma.py +6 -1
- datahub/ingestion/source/snowflake/snowflake_config.py +11 -0
- datahub/ingestion/source/snowflake/snowflake_queries.py +100 -58
- datahub/ingestion/source/snowflake/snowflake_v2.py +11 -1
- datahub/ingestion/source/snowflake/stored_proc_lineage.py +1 -1
- datahub/ingestion/source/sql/hive_metastore.py +0 -10
- datahub/ingestion/source/sql/sql_common.py +8 -0
- datahub/ingestion/source/sql/teradata.py +993 -234
- datahub/ingestion/source/sql/vertica.py +0 -4
- datahub/ingestion/source/sql_queries.py +2 -2
- datahub/ingestion/source/superset.py +56 -1
- datahub/ingestion/source/tableau/tableau.py +40 -34
- datahub/ingestion/source/tableau/tableau_constant.py +0 -2
- datahub/ingestion/source/unity/source.py +9 -1
- datahub/sdk/lineage_client.py +2 -2
- datahub/sql_parsing/sql_parsing_aggregator.py +21 -12
- datahub/sql_parsing/sqlglot_lineage.py +40 -15
- datahub/upgrade/upgrade.py +46 -13
- datahub/utilities/server_config_util.py +8 -0
- {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.1.0.5rc7.dist-info → acryl_datahub-1.1.0.5rc9.dist-info}/top_level.txt +0 -0
datahub/upgrade/upgrade.py
CHANGED
|
@@ -32,6 +32,7 @@ class ServerVersionStats(BaseModel):
|
|
|
32
32
|
current: VersionStats
|
|
33
33
|
latest: Optional[VersionStats] = None
|
|
34
34
|
current_server_type: Optional[str] = None
|
|
35
|
+
current_server_default_cli_version: Optional[VersionStats] = None
|
|
35
36
|
|
|
36
37
|
|
|
37
38
|
class ClientVersionStats(BaseModel):
|
|
@@ -44,7 +45,7 @@ class DataHubVersionStats(BaseModel):
|
|
|
44
45
|
client: ClientVersionStats
|
|
45
46
|
|
|
46
47
|
|
|
47
|
-
async def get_client_version_stats():
|
|
48
|
+
async def get_client_version_stats() -> ClientVersionStats:
|
|
48
49
|
import aiohttp
|
|
49
50
|
|
|
50
51
|
current_version_string = __version__
|
|
@@ -52,6 +53,7 @@ async def get_client_version_stats():
|
|
|
52
53
|
client_version_stats: ClientVersionStats = ClientVersionStats(
|
|
53
54
|
current=VersionStats(version=current_version, release_date=None), latest=None
|
|
54
55
|
)
|
|
56
|
+
|
|
55
57
|
async with aiohttp.ClientSession() as session:
|
|
56
58
|
pypi_url = "https://pypi.org/pypi/acryl_datahub/json"
|
|
57
59
|
async with session.get(pypi_url) as resp:
|
|
@@ -131,7 +133,7 @@ async def get_server_config(gms_url: str, token: Optional[str]) -> RestServiceCo
|
|
|
131
133
|
|
|
132
134
|
async def get_server_version_stats(
|
|
133
135
|
server: Optional[DataHubGraph] = None,
|
|
134
|
-
) -> Tuple[Optional[str], Optional[Version], Optional[datetime]]:
|
|
136
|
+
) -> Tuple[Optional[str], Optional[Version], Optional[str], Optional[datetime]]:
|
|
135
137
|
import aiohttp
|
|
136
138
|
|
|
137
139
|
server_config: Optional[RestServiceConfig] = None
|
|
@@ -151,12 +153,13 @@ async def get_server_version_stats(
|
|
|
151
153
|
|
|
152
154
|
server_type = None
|
|
153
155
|
server_version: Optional[Version] = None
|
|
156
|
+
current_server_default_cli_version = None
|
|
154
157
|
current_server_release_date = None
|
|
155
158
|
if server_config:
|
|
156
159
|
server_version_string = server_config.service_version
|
|
157
160
|
commit_hash = server_config.commit_hash
|
|
158
161
|
server_type = server_config.server_type
|
|
159
|
-
|
|
162
|
+
current_server_default_cli_version = server_config.default_cli_version
|
|
160
163
|
if server_type == "quickstart" and commit_hash:
|
|
161
164
|
async with aiohttp.ClientSession(
|
|
162
165
|
headers={"Accept": "application/vnd.github.v3+json"}
|
|
@@ -171,7 +174,12 @@ async def get_server_version_stats(
|
|
|
171
174
|
if server_version_string and server_version_string.startswith("v"):
|
|
172
175
|
server_version = Version(server_version_string[1:])
|
|
173
176
|
|
|
174
|
-
return (
|
|
177
|
+
return (
|
|
178
|
+
server_type,
|
|
179
|
+
server_version,
|
|
180
|
+
current_server_default_cli_version,
|
|
181
|
+
current_server_release_date,
|
|
182
|
+
)
|
|
175
183
|
|
|
176
184
|
|
|
177
185
|
def retrieve_version_stats(
|
|
@@ -214,6 +222,7 @@ async def _retrieve_version_stats(
|
|
|
214
222
|
(
|
|
215
223
|
current_server_type,
|
|
216
224
|
current_server_version,
|
|
225
|
+
current_server_default_cli_version,
|
|
217
226
|
current_server_release_date,
|
|
218
227
|
) = results[2]
|
|
219
228
|
|
|
@@ -223,6 +232,14 @@ async def _retrieve_version_stats(
|
|
|
223
232
|
current=VersionStats(
|
|
224
233
|
version=current_server_version, release_date=current_server_release_date
|
|
225
234
|
),
|
|
235
|
+
current_server_default_cli_version=(
|
|
236
|
+
VersionStats(
|
|
237
|
+
version=Version(current_server_default_cli_version),
|
|
238
|
+
release_date=None,
|
|
239
|
+
)
|
|
240
|
+
if current_server_default_cli_version
|
|
241
|
+
else None
|
|
242
|
+
),
|
|
226
243
|
latest=(
|
|
227
244
|
VersionStats(version=last_server_version, release_date=last_server_date)
|
|
228
245
|
if last_server_version
|
|
@@ -255,21 +272,14 @@ def valid_client_version(version: Version) -> bool:
|
|
|
255
272
|
"""Only version strings like 0.4.5 and 0.6.7.8 are valid. 0.8.6.7rc1 is not"""
|
|
256
273
|
if version.is_prerelease or version.is_postrelease or version.is_devrelease:
|
|
257
274
|
return False
|
|
258
|
-
|
|
259
|
-
return True
|
|
260
|
-
|
|
261
|
-
return False
|
|
275
|
+
return True
|
|
262
276
|
|
|
263
277
|
|
|
264
278
|
def valid_server_version(version: Version) -> bool:
|
|
265
279
|
"""Only version strings like 0.8.x, 0.9.x or 0.10.x are valid. 0.1.x is not"""
|
|
266
280
|
if version.is_prerelease or version.is_postrelease or version.is_devrelease:
|
|
267
281
|
return False
|
|
268
|
-
|
|
269
|
-
if version.major == 0 and version.minor in [8, 9, 10]:
|
|
270
|
-
return True
|
|
271
|
-
|
|
272
|
-
return False
|
|
282
|
+
return True
|
|
273
283
|
|
|
274
284
|
|
|
275
285
|
def is_client_server_compatible(client: VersionStats, server: VersionStats) -> int:
|
|
@@ -291,6 +301,27 @@ def is_client_server_compatible(client: VersionStats, server: VersionStats) -> i
|
|
|
291
301
|
return server.version.micro - client.version.micro
|
|
292
302
|
|
|
293
303
|
|
|
304
|
+
def is_server_default_cli_ahead(version_stats: DataHubVersionStats) -> bool:
|
|
305
|
+
"""
|
|
306
|
+
Check if the server default CLI version is ahead of the current CLI version.
|
|
307
|
+
Returns True if server default CLI is newer and both versions are valid.
|
|
308
|
+
"""
|
|
309
|
+
if not version_stats.server.current_server_default_cli_version:
|
|
310
|
+
return False
|
|
311
|
+
|
|
312
|
+
current_cli = version_stats.client.current
|
|
313
|
+
server_default_cli = version_stats.server.current_server_default_cli_version
|
|
314
|
+
|
|
315
|
+
is_valid_client_version = valid_client_version(current_cli.version)
|
|
316
|
+
is_valid_server_version = valid_client_version(server_default_cli.version)
|
|
317
|
+
|
|
318
|
+
if not (is_valid_client_version and is_valid_server_version):
|
|
319
|
+
return False
|
|
320
|
+
|
|
321
|
+
compatibility_result = is_client_server_compatible(current_cli, server_default_cli)
|
|
322
|
+
return compatibility_result > 0
|
|
323
|
+
|
|
324
|
+
|
|
294
325
|
def _maybe_print_upgrade_message(
|
|
295
326
|
version_stats: Optional[DataHubVersionStats],
|
|
296
327
|
) -> None:
|
|
@@ -429,6 +460,8 @@ def check_upgrade_post(
|
|
|
429
460
|
|
|
430
461
|
|
|
431
462
|
def check_upgrade(func: Callable[..., T]) -> Callable[..., T]:
|
|
463
|
+
log.debug(f"Checking upgrade for {func.__module__}.{func.__name__}")
|
|
464
|
+
|
|
432
465
|
@wraps(func)
|
|
433
466
|
def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
434
467
|
with PerfTimer() as timer:
|
|
@@ -183,6 +183,14 @@ class RestServiceConfig:
|
|
|
183
183
|
managed_ingestion = self.raw_config.get("managedIngestion") or {}
|
|
184
184
|
return managed_ingestion.get("enabled", False)
|
|
185
185
|
|
|
186
|
+
@property
|
|
187
|
+
def default_cli_version(self) -> Optional[str]:
|
|
188
|
+
"""
|
|
189
|
+
Get the default CLI version.
|
|
190
|
+
"""
|
|
191
|
+
managed_ingestion = self.raw_config.get("managedIngestion") or {}
|
|
192
|
+
return managed_ingestion.get("defaultCliVersion")
|
|
193
|
+
|
|
186
194
|
@property
|
|
187
195
|
def is_datahub_cloud(self) -> bool:
|
|
188
196
|
"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|