snowflake-cli 3.0.2__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli/__about__.py +1 -1
- snowflake/cli/_app/cli_app.py +3 -0
- snowflake/cli/_app/dev/docs/templates/overview.rst.jinja2 +1 -1
- snowflake/cli/_app/dev/docs/templates/usage.rst.jinja2 +2 -2
- snowflake/cli/_app/telemetry.py +69 -4
- snowflake/cli/_plugins/connection/commands.py +152 -99
- snowflake/cli/_plugins/connection/util.py +54 -9
- snowflake/cli/_plugins/cortex/manager.py +1 -1
- snowflake/cli/_plugins/git/commands.py +6 -3
- snowflake/cli/_plugins/git/manager.py +9 -4
- snowflake/cli/_plugins/nativeapp/artifacts.py +77 -13
- snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/compiler.py +7 -0
- snowflake/cli/_plugins/nativeapp/codegen/sandbox.py +10 -10
- snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/extension_function_utils.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +8 -8
- snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +5 -3
- snowflake/cli/_plugins/nativeapp/commands.py +144 -188
- snowflake/cli/_plugins/nativeapp/constants.py +1 -0
- snowflake/cli/_plugins/nativeapp/entities/application.py +564 -351
- snowflake/cli/_plugins/nativeapp/entities/application_package.py +583 -929
- snowflake/cli/_plugins/nativeapp/entities/models/event_sharing_telemetry.py +58 -0
- snowflake/cli/_plugins/nativeapp/exceptions.py +12 -0
- snowflake/cli/_plugins/nativeapp/same_account_install_method.py +0 -2
- snowflake/cli/_plugins/nativeapp/sf_facade.py +30 -0
- snowflake/cli/_plugins/nativeapp/sf_facade_constants.py +25 -0
- snowflake/cli/_plugins/nativeapp/sf_facade_exceptions.py +117 -0
- snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +525 -0
- snowflake/cli/_plugins/nativeapp/v2_conversions/{v2_to_v1_decorator.py → compat.py} +88 -117
- snowflake/cli/_plugins/nativeapp/version/commands.py +36 -32
- snowflake/cli/_plugins/notebook/manager.py +2 -2
- snowflake/cli/_plugins/object/commands.py +10 -1
- snowflake/cli/_plugins/object/manager.py +13 -5
- snowflake/cli/_plugins/snowpark/common.py +63 -21
- snowflake/cli/_plugins/snowpark/package/anaconda_packages.py +3 -3
- snowflake/cli/_plugins/spcs/common.py +29 -0
- snowflake/cli/_plugins/spcs/compute_pool/manager.py +7 -9
- snowflake/cli/_plugins/spcs/image_registry/manager.py +2 -2
- snowflake/cli/_plugins/spcs/image_repository/commands.py +4 -37
- snowflake/cli/_plugins/spcs/image_repository/manager.py +4 -1
- snowflake/cli/_plugins/spcs/services/commands.py +100 -17
- snowflake/cli/_plugins/spcs/services/manager.py +108 -16
- snowflake/cli/_plugins/sql/commands.py +9 -1
- snowflake/cli/_plugins/sql/manager.py +9 -4
- snowflake/cli/_plugins/stage/commands.py +28 -19
- snowflake/cli/_plugins/stage/diff.py +17 -17
- snowflake/cli/_plugins/stage/manager.py +304 -84
- snowflake/cli/_plugins/stage/md5.py +1 -1
- snowflake/cli/_plugins/streamlit/manager.py +5 -5
- snowflake/cli/_plugins/workspace/commands.py +27 -4
- snowflake/cli/_plugins/workspace/context.py +38 -0
- snowflake/cli/_plugins/workspace/manager.py +23 -13
- snowflake/cli/api/cli_global_context.py +4 -3
- snowflake/cli/api/commands/flags.py +23 -7
- snowflake/cli/api/config.py +30 -9
- snowflake/cli/api/connections.py +12 -1
- snowflake/cli/api/console/console.py +4 -19
- snowflake/cli/api/entities/common.py +4 -2
- snowflake/cli/api/entities/utils.py +36 -69
- snowflake/cli/api/errno.py +2 -0
- snowflake/cli/api/exceptions.py +41 -0
- snowflake/cli/api/identifiers.py +8 -0
- snowflake/cli/api/metrics.py +223 -7
- snowflake/cli/api/output/types.py +1 -1
- snowflake/cli/api/project/definition_conversion.py +293 -77
- snowflake/cli/api/project/schemas/entities/common.py +11 -0
- snowflake/cli/api/project/schemas/project_definition.py +30 -25
- snowflake/cli/api/rest_api.py +26 -4
- snowflake/cli/api/secure_utils.py +1 -1
- snowflake/cli/api/sql_execution.py +40 -29
- snowflake/cli/api/stage_path.py +244 -0
- snowflake/cli/api/utils/definition_rendering.py +3 -5
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/METADATA +14 -15
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/RECORD +78 -77
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/WHEEL +1 -1
- snowflake/cli/_plugins/nativeapp/manager.py +0 -415
- snowflake/cli/_plugins/nativeapp/project_model.py +0 -211
- snowflake/cli/_plugins/nativeapp/run_processor.py +0 -184
- snowflake/cli/_plugins/nativeapp/teardown_processor.py +0 -70
- snowflake/cli/_plugins/nativeapp/version/version_processor.py +0 -98
- snowflake/cli/_plugins/workspace/action_context.py +0 -18
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/entry_points.txt +0 -0
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -17,16 +17,21 @@ from __future__ import annotations
|
|
|
17
17
|
import fnmatch
|
|
18
18
|
import glob
|
|
19
19
|
import logging
|
|
20
|
+
import os
|
|
20
21
|
import re
|
|
22
|
+
import shutil
|
|
21
23
|
import sys
|
|
24
|
+
import time
|
|
25
|
+
from collections import deque
|
|
22
26
|
from contextlib import nullcontext
|
|
23
27
|
from dataclasses import dataclass
|
|
24
28
|
from os import path
|
|
25
29
|
from pathlib import Path
|
|
30
|
+
from tempfile import TemporaryDirectory
|
|
26
31
|
from textwrap import dedent
|
|
27
|
-
from typing import Dict, List, Optional, Union
|
|
32
|
+
from typing import Deque, Dict, Generator, List, Optional, Union
|
|
28
33
|
|
|
29
|
-
from click import ClickException
|
|
34
|
+
from click import ClickException, UsageError
|
|
30
35
|
from snowflake.cli._plugins.snowpark.package_utils import parse_requirements
|
|
31
36
|
from snowflake.cli.api.commands.common import (
|
|
32
37
|
OnErrorType,
|
|
@@ -39,6 +44,7 @@ from snowflake.cli.api.identifiers import FQN
|
|
|
39
44
|
from snowflake.cli.api.project.util import to_string_literal
|
|
40
45
|
from snowflake.cli.api.secure_path import SecurePath
|
|
41
46
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin
|
|
47
|
+
from snowflake.cli.api.stage_path import StagePath
|
|
42
48
|
from snowflake.cli.api.utils.path_utils import path_resolver
|
|
43
49
|
from snowflake.connector import DictCursor, ProgrammingError
|
|
44
50
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
@@ -90,12 +96,12 @@ class StagePathParts:
|
|
|
90
96
|
raise NotImplementedError
|
|
91
97
|
|
|
92
98
|
def get_full_stage_path(self, path: str):
|
|
93
|
-
if prefix := FQN.
|
|
99
|
+
if prefix := FQN.from_stage_path(self.stage).prefix:
|
|
94
100
|
return prefix + "." + path
|
|
95
101
|
return path
|
|
96
102
|
|
|
97
103
|
def get_standard_stage_path(self) -> str:
|
|
98
|
-
path = self.path
|
|
104
|
+
path = self.get_full_stage_path(self.path)
|
|
99
105
|
return f"@{path}{'/'if self.is_directory and not path.endswith('/') else ''}"
|
|
100
106
|
|
|
101
107
|
def get_standard_stage_directory_path(self) -> str:
|
|
@@ -104,6 +110,9 @@ class StagePathParts:
|
|
|
104
110
|
return path + "/"
|
|
105
111
|
return path
|
|
106
112
|
|
|
113
|
+
def strip_stage_prefix(self, path: str):
|
|
114
|
+
raise NotImplementedError
|
|
115
|
+
|
|
107
116
|
|
|
108
117
|
@dataclass
|
|
109
118
|
class DefaultStagePathParts(StagePathParts):
|
|
@@ -141,6 +150,13 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
141
150
|
file_path_without_prefix = Path(file_path).parts[OMIT_FIRST]
|
|
142
151
|
return f"{stage}/{'/'.join(file_path_without_prefix)}"
|
|
143
152
|
|
|
153
|
+
def strip_stage_prefix(self, file_path: str) -> str:
|
|
154
|
+
if file_path.startswith("@"):
|
|
155
|
+
file_path = file_path[OMIT_FIRST]
|
|
156
|
+
if file_path.startswith(self.stage_name):
|
|
157
|
+
return file_path[len(self.stage_name) :]
|
|
158
|
+
return file_path
|
|
159
|
+
|
|
144
160
|
def add_stage_prefix(self, file_path: str) -> str:
|
|
145
161
|
stage = self.stage.rstrip("/")
|
|
146
162
|
return f"{stage}/{file_path.lstrip('/')}"
|
|
@@ -197,6 +213,10 @@ class StageManager(SqlExecutionMixin):
|
|
|
197
213
|
super().__init__()
|
|
198
214
|
self._python_exe_procedure = None
|
|
199
215
|
|
|
216
|
+
@staticmethod
|
|
217
|
+
def build_path(stage_path: str) -> StagePath:
|
|
218
|
+
return StagePath.from_stage_str(stage_path)
|
|
219
|
+
|
|
200
220
|
@staticmethod
|
|
201
221
|
def get_standard_stage_prefix(name: str | FQN) -> str:
|
|
202
222
|
if isinstance(name, FQN):
|
|
@@ -234,12 +254,17 @@ class StageManager(SqlExecutionMixin):
|
|
|
234
254
|
return uri
|
|
235
255
|
return to_string_literal(uri)
|
|
236
256
|
|
|
237
|
-
def list_files(
|
|
238
|
-
stage_name =
|
|
239
|
-
|
|
257
|
+
def list_files(
|
|
258
|
+
self, stage_name: str | StagePath, pattern: str | None = None
|
|
259
|
+
) -> DictCursor:
|
|
260
|
+
if not isinstance(stage_name, StagePath):
|
|
261
|
+
stage_path = self.build_path(stage_name).path_for_sql()
|
|
262
|
+
else:
|
|
263
|
+
stage_path = stage_name.path_for_sql()
|
|
264
|
+
query = f"ls {stage_path}"
|
|
240
265
|
if pattern is not None:
|
|
241
266
|
query += f" pattern = '{pattern}'"
|
|
242
|
-
return self.
|
|
267
|
+
return self.execute_query(query, cursor_class=DictCursor)
|
|
243
268
|
|
|
244
269
|
@staticmethod
|
|
245
270
|
def _assure_is_existing_directory(path: Path) -> None:
|
|
@@ -251,27 +276,27 @@ class StageManager(SqlExecutionMixin):
|
|
|
251
276
|
def get(
|
|
252
277
|
self, stage_path: str, dest_path: Path, parallel: int = 4
|
|
253
278
|
) -> SnowflakeCursor:
|
|
254
|
-
|
|
279
|
+
spath = self.build_path(stage_path)
|
|
255
280
|
self._assure_is_existing_directory(dest_path)
|
|
256
281
|
dest_directory = f"{dest_path}/"
|
|
257
|
-
return self.
|
|
258
|
-
f"get {
|
|
282
|
+
return self.execute_query(
|
|
283
|
+
f"get {spath.path_for_sql()} {self._to_uri(dest_directory)} parallel={parallel}"
|
|
259
284
|
)
|
|
260
285
|
|
|
261
286
|
def get_recursive(
|
|
262
287
|
self, stage_path: str, dest_path: Path, parallel: int = 4
|
|
263
288
|
) -> List[SnowflakeCursor]:
|
|
264
|
-
|
|
289
|
+
stage_root = self.build_path(stage_path)
|
|
265
290
|
|
|
266
291
|
results = []
|
|
267
|
-
for file_path in self.iter_stage(
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
self._assure_is_existing_directory(
|
|
272
|
-
|
|
273
|
-
result = self.
|
|
274
|
-
f"get {
|
|
292
|
+
for file_path in self.iter_stage(stage_root):
|
|
293
|
+
local_dir = file_path.get_local_target_path(
|
|
294
|
+
target_dir=dest_path, stage_root=stage_root
|
|
295
|
+
)
|
|
296
|
+
self._assure_is_existing_directory(local_dir)
|
|
297
|
+
|
|
298
|
+
result = self.execute_query(
|
|
299
|
+
f"get {file_path.path_for_sql()} {self._to_uri(f'{local_dir}/')} parallel={parallel}"
|
|
275
300
|
)
|
|
276
301
|
results.append(result)
|
|
277
302
|
|
|
@@ -285,6 +310,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
285
310
|
overwrite: bool = False,
|
|
286
311
|
role: Optional[str] = None,
|
|
287
312
|
auto_compress: bool = False,
|
|
313
|
+
use_dict_cursor: bool = False,
|
|
288
314
|
) -> SnowflakeCursor:
|
|
289
315
|
"""
|
|
290
316
|
This method will take a file path from the user's system and put it into a Snowflake stage,
|
|
@@ -292,30 +318,157 @@ class StageManager(SqlExecutionMixin):
|
|
|
292
318
|
If provided with a role, then temporarily use this role to perform the operation above,
|
|
293
319
|
and switch back to the original role for the next commands to run.
|
|
294
320
|
"""
|
|
321
|
+
if "*" not in str(local_path):
|
|
322
|
+
local_path = (
|
|
323
|
+
os.path.join(local_path, "*")
|
|
324
|
+
if Path(local_path).is_dir()
|
|
325
|
+
else str(local_path)
|
|
326
|
+
)
|
|
295
327
|
with self.use_role(role) if role else nullcontext():
|
|
296
|
-
|
|
328
|
+
spath = self.build_path(stage_path)
|
|
297
329
|
local_resolved_path = path_resolver(str(local_path))
|
|
298
330
|
log.info("Uploading %s to %s", local_resolved_path, stage_path)
|
|
299
|
-
cursor = self.
|
|
300
|
-
f"put {self._to_uri(local_resolved_path)} {
|
|
301
|
-
f"auto_compress={str(auto_compress).lower()} parallel={parallel} overwrite={overwrite}"
|
|
331
|
+
cursor = self.execute_query(
|
|
332
|
+
f"put {self._to_uri(local_resolved_path)} {spath.path_for_sql()} "
|
|
333
|
+
f"auto_compress={str(auto_compress).lower()} parallel={parallel} overwrite={overwrite}",
|
|
334
|
+
cursor_class=DictCursor if use_dict_cursor else SnowflakeCursor,
|
|
302
335
|
)
|
|
303
336
|
return cursor
|
|
304
337
|
|
|
338
|
+
@staticmethod
|
|
339
|
+
def _symlink_or_copy(source_root: Path, source_file_or_dir: Path, dest_dir: Path):
|
|
340
|
+
from snowflake.cli._plugins.nativeapp.artifacts import resolve_without_follow
|
|
341
|
+
|
|
342
|
+
absolute_src = resolve_without_follow(source_file_or_dir)
|
|
343
|
+
dest_path = dest_dir / source_file_or_dir.relative_to(source_root)
|
|
344
|
+
|
|
345
|
+
if absolute_src.is_file():
|
|
346
|
+
try:
|
|
347
|
+
os.symlink(absolute_src, dest_path)
|
|
348
|
+
except OSError:
|
|
349
|
+
if not dest_path.parent.exists():
|
|
350
|
+
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
351
|
+
shutil.copyfile(absolute_src, dest_path)
|
|
352
|
+
else:
|
|
353
|
+
dest_path.mkdir(exist_ok=True, parents=True)
|
|
354
|
+
|
|
355
|
+
def put_recursive(
|
|
356
|
+
self,
|
|
357
|
+
local_path: Path,
|
|
358
|
+
stage_path: str,
|
|
359
|
+
parallel: int = 4,
|
|
360
|
+
overwrite: bool = False,
|
|
361
|
+
role: Optional[str] = None,
|
|
362
|
+
auto_compress: bool = False,
|
|
363
|
+
) -> Generator[dict, None, None]:
|
|
364
|
+
if local_path.is_file():
|
|
365
|
+
raise UsageError("Cannot use recursive upload with a single file.")
|
|
366
|
+
|
|
367
|
+
if local_path.is_dir():
|
|
368
|
+
root = local_path
|
|
369
|
+
glob_pattern = str(local_path / "**/*")
|
|
370
|
+
else:
|
|
371
|
+
root = Path([p for p in local_path.parents if p.is_dir()][0])
|
|
372
|
+
glob_pattern = str(local_path)
|
|
373
|
+
|
|
374
|
+
with TemporaryDirectory() as tmp:
|
|
375
|
+
temp_dir_with_copy = Path(tmp)
|
|
376
|
+
|
|
377
|
+
# Create a symlink or copy the file to the temp directory
|
|
378
|
+
for file_or_dir in glob.iglob(glob_pattern, recursive=True):
|
|
379
|
+
self._symlink_or_copy(
|
|
380
|
+
source_root=root,
|
|
381
|
+
source_file_or_dir=Path(file_or_dir),
|
|
382
|
+
dest_dir=temp_dir_with_copy,
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
# Find the deepest directories, we will be iterating from bottom to top
|
|
386
|
+
deepest_dirs_list = self._find_deepest_directories(temp_dir_with_copy)
|
|
387
|
+
|
|
388
|
+
while deepest_dirs_list:
|
|
389
|
+
# Remove as visited
|
|
390
|
+
directory = deepest_dirs_list.pop(0)
|
|
391
|
+
|
|
392
|
+
# We reached root but there are still directories to process
|
|
393
|
+
if directory == temp_dir_with_copy and deepest_dirs_list:
|
|
394
|
+
continue
|
|
395
|
+
|
|
396
|
+
# Upload the directory content, at this moment the directory has only files
|
|
397
|
+
if list(directory.iterdir()):
|
|
398
|
+
destination = StagePath.from_stage_str(
|
|
399
|
+
stage_path
|
|
400
|
+
) / directory.relative_to(temp_dir_with_copy)
|
|
401
|
+
results: list[dict] = self.put(
|
|
402
|
+
local_path=directory,
|
|
403
|
+
stage_path=destination,
|
|
404
|
+
parallel=parallel,
|
|
405
|
+
overwrite=overwrite,
|
|
406
|
+
role=role,
|
|
407
|
+
auto_compress=auto_compress,
|
|
408
|
+
use_dict_cursor=True,
|
|
409
|
+
).fetchall()
|
|
410
|
+
|
|
411
|
+
# Rewrite results to have resolved paths for better UX
|
|
412
|
+
for item in results:
|
|
413
|
+
item["source"] = (directory / item["source"]).relative_to(
|
|
414
|
+
temp_dir_with_copy
|
|
415
|
+
)
|
|
416
|
+
item["target"] = str(destination / item["target"])
|
|
417
|
+
yield item
|
|
418
|
+
|
|
419
|
+
# We end if we reach the root directory
|
|
420
|
+
if directory == temp_dir_with_copy:
|
|
421
|
+
break
|
|
422
|
+
|
|
423
|
+
# Add parent directory to the list if it's not already there
|
|
424
|
+
if directory.parent not in deepest_dirs_list:
|
|
425
|
+
deepest_dirs_list.append(directory.parent)
|
|
426
|
+
|
|
427
|
+
# Remove the directory so the parent directory will contain only files
|
|
428
|
+
shutil.rmtree(directory)
|
|
429
|
+
|
|
430
|
+
@staticmethod
|
|
431
|
+
def _find_deepest_directories(root_directory: Path) -> list[Path]:
|
|
432
|
+
"""
|
|
433
|
+
BFS to find the deepest directories. Build a tree of directories
|
|
434
|
+
structure and return leaves.
|
|
435
|
+
"""
|
|
436
|
+
deepest_dirs: list[Path] = list()
|
|
437
|
+
|
|
438
|
+
queue: Deque[Path] = deque()
|
|
439
|
+
queue.append(root_directory)
|
|
440
|
+
while queue:
|
|
441
|
+
current_dir = queue.popleft()
|
|
442
|
+
# Sorted to have deterministic order
|
|
443
|
+
children_directories = sorted(
|
|
444
|
+
list(d for d in current_dir.iterdir() if d.is_dir())
|
|
445
|
+
)
|
|
446
|
+
if not children_directories and current_dir not in deepest_dirs:
|
|
447
|
+
deepest_dirs.append(current_dir)
|
|
448
|
+
else:
|
|
449
|
+
queue.extend([c for c in children_directories if c not in deepest_dirs])
|
|
450
|
+
deepest_dirs_list = sorted(
|
|
451
|
+
list(deepest_dirs), key=lambda d: len(d.parts), reverse=True
|
|
452
|
+
)
|
|
453
|
+
return deepest_dirs_list
|
|
454
|
+
|
|
305
455
|
def copy_files(self, source_path: str, destination_path: str) -> SnowflakeCursor:
|
|
306
|
-
|
|
307
|
-
|
|
456
|
+
source_stage_path = self.build_path(source_path)
|
|
457
|
+
# We copy only into stage
|
|
458
|
+
destination_stage_path = StagePath.from_stage_str(destination_path)
|
|
308
459
|
|
|
309
|
-
if
|
|
460
|
+
if destination_stage_path.is_user_stage():
|
|
310
461
|
raise ClickException(
|
|
311
462
|
"Destination path cannot be a user stage. Please provide a named stage."
|
|
312
463
|
)
|
|
313
464
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
465
|
+
log.info(
|
|
466
|
+
"Copying files from %s to %s", source_stage_path, destination_stage_path
|
|
467
|
+
)
|
|
468
|
+
# Destination needs to end with /
|
|
469
|
+
dest = destination_stage_path.absolute_path().rstrip("/") + "/"
|
|
470
|
+
query = f"copy files into {dest} from {source_stage_path}"
|
|
471
|
+
return self.execute_query(query)
|
|
319
472
|
|
|
320
473
|
def remove(
|
|
321
474
|
self, stage_name: str, path: str, role: Optional[str] = None
|
|
@@ -327,29 +480,48 @@ class StageManager(SqlExecutionMixin):
|
|
|
327
480
|
and switch back to the original role for the next commands to run.
|
|
328
481
|
"""
|
|
329
482
|
with self.use_role(role) if role else nullcontext():
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
quoted_stage_name = self.quote_stage_name(f"{stage_name}{path}")
|
|
333
|
-
return self._execute_query(f"remove {quoted_stage_name}")
|
|
483
|
+
stage_path = self.build_path(stage_name) / path
|
|
484
|
+
return self.execute_query(f"remove {stage_path.path_for_sql()}")
|
|
334
485
|
|
|
335
|
-
def create(
|
|
336
|
-
|
|
486
|
+
def create(
|
|
487
|
+
self, fqn: FQN, comment: Optional[str] = None, temporary: bool = False
|
|
488
|
+
) -> SnowflakeCursor:
|
|
489
|
+
temporary_str = "temporary " if temporary else ""
|
|
490
|
+
query = f"create {temporary_str}stage if not exists {fqn.sql_identifier}"
|
|
337
491
|
if comment:
|
|
338
492
|
query += f" comment='{comment}'"
|
|
339
|
-
return self.
|
|
493
|
+
return self.execute_query(query)
|
|
340
494
|
|
|
341
|
-
def iter_stage(self, stage_path:
|
|
342
|
-
for file in self.list_files(stage_path).fetchall():
|
|
343
|
-
|
|
495
|
+
def iter_stage(self, stage_path: StagePath):
|
|
496
|
+
for file in self.list_files(stage_path.absolute_path()).fetchall():
|
|
497
|
+
if stage_path.is_user_stage():
|
|
498
|
+
path = StagePath.get_user_stage() / file["name"]
|
|
499
|
+
else:
|
|
500
|
+
path = self.build_path(file["name"])
|
|
501
|
+
yield path
|
|
344
502
|
|
|
345
503
|
def execute(
|
|
346
504
|
self,
|
|
347
|
-
|
|
505
|
+
stage_path_str: str,
|
|
348
506
|
on_error: OnErrorType,
|
|
349
507
|
variables: Optional[List[str]] = None,
|
|
508
|
+
requires_temporary_stage: bool = False,
|
|
350
509
|
):
|
|
351
|
-
|
|
352
|
-
|
|
510
|
+
if requires_temporary_stage:
|
|
511
|
+
(
|
|
512
|
+
stage_path_parts,
|
|
513
|
+
original_path_parts,
|
|
514
|
+
) = self._create_temporary_copy_of_stage(stage_path_str)
|
|
515
|
+
stage_path = StagePath.from_stage_str(
|
|
516
|
+
stage_path_parts.get_standard_stage_path()
|
|
517
|
+
)
|
|
518
|
+
else:
|
|
519
|
+
stage_path_parts = self._stage_path_part_factory(stage_path_str)
|
|
520
|
+
stage_path = self.build_path(stage_path_str)
|
|
521
|
+
|
|
522
|
+
all_files_list = self._get_files_list_from_stage(stage_path.root_path())
|
|
523
|
+
if not all_files_list:
|
|
524
|
+
raise ClickException(f"No files found on stage '{stage_path}'")
|
|
353
525
|
|
|
354
526
|
all_files_with_stage_name_prefix = [
|
|
355
527
|
stage_path_parts.get_directory(file) for file in all_files_list
|
|
@@ -370,42 +542,75 @@ class StageManager(SqlExecutionMixin):
|
|
|
370
542
|
|
|
371
543
|
parsed_variables = parse_key_value_variables(variables)
|
|
372
544
|
sql_variables = self._parse_execute_variables(parsed_variables)
|
|
373
|
-
python_variables =
|
|
545
|
+
python_variables = self._parse_python_variables(parsed_variables)
|
|
374
546
|
results = []
|
|
375
547
|
|
|
376
548
|
if any(file.endswith(".py") for file in sorted_file_path_list):
|
|
377
549
|
self._python_exe_procedure = self._bootstrap_snowpark_execution_environment(
|
|
378
|
-
|
|
550
|
+
stage_path
|
|
379
551
|
)
|
|
380
552
|
|
|
381
553
|
for file_path in sorted_file_path_list:
|
|
382
554
|
file_stage_path = stage_path_parts.add_stage_prefix(file_path)
|
|
555
|
+
|
|
556
|
+
# For better reporting push down the information about original
|
|
557
|
+
# path if execution happens from temporary stage
|
|
558
|
+
if requires_temporary_stage:
|
|
559
|
+
original_path = original_path_parts.add_stage_prefix(file_path)
|
|
560
|
+
else:
|
|
561
|
+
original_path = file_stage_path
|
|
562
|
+
|
|
383
563
|
if file_path.endswith(".py"):
|
|
384
564
|
result = self._execute_python(
|
|
385
565
|
file_stage_path=file_stage_path,
|
|
386
566
|
on_error=on_error,
|
|
387
567
|
variables=python_variables,
|
|
568
|
+
original_file=original_path,
|
|
388
569
|
)
|
|
389
570
|
else:
|
|
390
571
|
result = self._call_execute_immediate(
|
|
391
572
|
file_stage_path=file_stage_path,
|
|
392
573
|
variables=sql_variables,
|
|
393
574
|
on_error=on_error,
|
|
575
|
+
original_file=original_path,
|
|
394
576
|
)
|
|
395
577
|
results.append(result)
|
|
396
578
|
|
|
397
579
|
return results
|
|
398
580
|
|
|
399
|
-
def
|
|
400
|
-
self,
|
|
401
|
-
) ->
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
581
|
+
def _create_temporary_copy_of_stage(
|
|
582
|
+
self, stage_path: str
|
|
583
|
+
) -> tuple[StagePathParts, StagePathParts]:
|
|
584
|
+
sm = StageManager()
|
|
585
|
+
|
|
586
|
+
# Rewrite stage paths to temporary stage paths. Git paths become stage paths
|
|
587
|
+
original_path_parts = self._stage_path_part_factory(stage_path) # noqa: SLF001
|
|
405
588
|
|
|
406
|
-
|
|
407
|
-
|
|
589
|
+
tmp_stage_name = f"snowflake_cli_tmp_stage_{int(time.time())}"
|
|
590
|
+
tmp_stage_fqn = FQN.from_stage(tmp_stage_name).using_connection(conn=self._conn)
|
|
591
|
+
tmp_stage = tmp_stage_fqn.identifier
|
|
592
|
+
stage_path_parts = sm._stage_path_part_factory( # noqa: SLF001
|
|
593
|
+
tmp_stage + "/" + original_path_parts.directory
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
# Create temporary stage, it will be dropped with end of session
|
|
597
|
+
sm.create(tmp_stage_fqn, temporary=True)
|
|
598
|
+
|
|
599
|
+
# Copy the content
|
|
600
|
+
self.copy_files(
|
|
601
|
+
source_path=original_path_parts.get_full_stage_path(
|
|
602
|
+
original_path_parts.stage_name
|
|
603
|
+
),
|
|
604
|
+
destination_path=stage_path_parts.get_full_stage_path(
|
|
605
|
+
stage_path_parts.stage_name
|
|
606
|
+
),
|
|
607
|
+
)
|
|
608
|
+
return stage_path_parts, original_path_parts
|
|
408
609
|
|
|
610
|
+
def _get_files_list_from_stage(
|
|
611
|
+
self, stage_path: StagePath, pattern: str | None = None
|
|
612
|
+
) -> List[str]:
|
|
613
|
+
files_list_result = self.list_files(stage_path, pattern=pattern).fetchall()
|
|
409
614
|
return [f["name"] for f in files_list_result]
|
|
410
615
|
|
|
411
616
|
def _filter_files_list(
|
|
@@ -444,6 +649,17 @@ class StageManager(SqlExecutionMixin):
|
|
|
444
649
|
query_parameters = [f"{v.key}=>{v.value}" for v in variables]
|
|
445
650
|
return f" using ({', '.join(query_parameters)})"
|
|
446
651
|
|
|
652
|
+
@staticmethod
|
|
653
|
+
def _parse_python_variables(variables: List[Variable]) -> Dict:
|
|
654
|
+
def _unwrap(s: str):
|
|
655
|
+
if s.startswith("'") and s.endswith("'"):
|
|
656
|
+
return s[1:-1]
|
|
657
|
+
if s.startswith('"') and s.endswith('"'):
|
|
658
|
+
return s[1:-1]
|
|
659
|
+
return s
|
|
660
|
+
|
|
661
|
+
return {str(v.key): _unwrap(v.value) for v in variables}
|
|
662
|
+
|
|
447
663
|
@staticmethod
|
|
448
664
|
def _success_result(file: str):
|
|
449
665
|
cli_console.warning(f"SUCCESS - {file}")
|
|
@@ -464,16 +680,17 @@ class StageManager(SqlExecutionMixin):
|
|
|
464
680
|
file_stage_path: str,
|
|
465
681
|
variables: Optional[str],
|
|
466
682
|
on_error: OnErrorType,
|
|
683
|
+
original_file: str,
|
|
467
684
|
) -> Dict:
|
|
468
685
|
try:
|
|
469
686
|
query = f"execute immediate from {self.quote_stage_name(file_stage_path)}"
|
|
470
687
|
if variables:
|
|
471
688
|
query += variables
|
|
472
|
-
self.
|
|
473
|
-
return StageManager._success_result(file=
|
|
689
|
+
self.execute_query(query)
|
|
690
|
+
return StageManager._success_result(file=original_file)
|
|
474
691
|
except ProgrammingError as e:
|
|
475
692
|
StageManager._handle_execution_exception(on_error=on_error, exception=e)
|
|
476
|
-
return StageManager._error_result(file=
|
|
693
|
+
return StageManager._error_result(file=original_file, msg=e.msg)
|
|
477
694
|
|
|
478
695
|
@staticmethod
|
|
479
696
|
def _stage_path_part_factory(stage_path: str) -> StagePathParts:
|
|
@@ -482,32 +699,34 @@ class StageManager(SqlExecutionMixin):
|
|
|
482
699
|
return UserStagePathParts(stage_path)
|
|
483
700
|
return DefaultStagePathParts(stage_path)
|
|
484
701
|
|
|
485
|
-
def _check_for_requirements_file(
|
|
486
|
-
self, stage_path_parts: StagePathParts
|
|
487
|
-
) -> List[str]:
|
|
702
|
+
def _check_for_requirements_file(self, stage_path: StagePath) -> List[str]:
|
|
488
703
|
"""Looks for requirements.txt file on stage."""
|
|
704
|
+
current_dir = stage_path.parent if stage_path.is_file() else stage_path
|
|
489
705
|
req_files_on_stage = self._get_files_list_from_stage(
|
|
490
|
-
|
|
706
|
+
current_dir, pattern=r".*requirements\.txt$"
|
|
491
707
|
)
|
|
492
708
|
if not req_files_on_stage:
|
|
493
709
|
return []
|
|
494
710
|
|
|
495
711
|
# Construct all possible path for requirements file for this context
|
|
496
|
-
# We don't use os.path or pathlib to preserve compatibility on Windows
|
|
497
712
|
req_file_name = "requirements.txt"
|
|
498
|
-
path_parts = stage_path_parts.path.split("/")
|
|
499
713
|
possible_req_files = []
|
|
714
|
+
while not current_dir.is_root():
|
|
715
|
+
current_file = current_dir / req_file_name
|
|
716
|
+
possible_req_files.append(current_file)
|
|
717
|
+
current_dir = current_dir.parent
|
|
500
718
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
possible_req_files.append(str(current_file))
|
|
504
|
-
path_parts = path_parts[:-1]
|
|
719
|
+
current_file = current_dir / req_file_name
|
|
720
|
+
possible_req_files.append(current_file)
|
|
505
721
|
|
|
506
722
|
# Now for every possible path check if the file exists on stage,
|
|
507
723
|
# if yes break, we use the first possible file
|
|
508
|
-
requirements_file = None
|
|
724
|
+
requirements_file: StagePath | None = None
|
|
509
725
|
for req_file in possible_req_files:
|
|
510
|
-
if
|
|
726
|
+
if (
|
|
727
|
+
req_file.absolute_path(no_fqn=True, at_prefix=False)
|
|
728
|
+
in req_files_on_stage
|
|
729
|
+
):
|
|
511
730
|
requirements_file = req_file
|
|
512
731
|
break
|
|
513
732
|
|
|
@@ -516,37 +735,34 @@ class StageManager(SqlExecutionMixin):
|
|
|
516
735
|
return []
|
|
517
736
|
|
|
518
737
|
# req_file at this moment is the first found requirements file
|
|
738
|
+
requirements_path = requirements_file.with_stage(stage_path.stage)
|
|
519
739
|
with SecurePath.temporary_directory() as tmp_dir:
|
|
520
|
-
self.get(
|
|
521
|
-
stage_path_parts.get_full_stage_path(requirements_file), tmp_dir.path
|
|
522
|
-
)
|
|
740
|
+
self.get(str(requirements_path), tmp_dir.path)
|
|
523
741
|
requirements = parse_requirements(
|
|
524
742
|
requirements_file=tmp_dir / "requirements.txt"
|
|
525
743
|
)
|
|
526
744
|
|
|
527
745
|
return [req.package_name for req in requirements]
|
|
528
746
|
|
|
529
|
-
def _bootstrap_snowpark_execution_environment(
|
|
530
|
-
self, stage_path_parts: StagePathParts
|
|
531
|
-
):
|
|
747
|
+
def _bootstrap_snowpark_execution_environment(self, stage_path: StagePath):
|
|
532
748
|
"""Prepares Snowpark session for executing Python code remotely."""
|
|
533
749
|
if sys.version_info >= PYTHON_3_12:
|
|
534
750
|
raise ClickException(
|
|
535
|
-
f"Executing
|
|
751
|
+
f"Executing Python files is not supported in Python >= 3.12. Current version: {sys.version}"
|
|
536
752
|
)
|
|
537
753
|
|
|
538
754
|
from snowflake.snowpark.functions import sproc
|
|
539
755
|
|
|
540
756
|
self.snowpark_session.add_packages("snowflake-snowpark-python")
|
|
541
757
|
self.snowpark_session.add_packages("snowflake.core")
|
|
542
|
-
requirements = self._check_for_requirements_file(
|
|
758
|
+
requirements = self._check_for_requirements_file(stage_path)
|
|
543
759
|
self.snowpark_session.add_packages(*requirements)
|
|
544
760
|
|
|
545
|
-
@sproc(is_permanent=False)
|
|
761
|
+
@sproc(is_permanent=False, session=self.snowpark_session)
|
|
546
762
|
def _python_execution_procedure(
|
|
547
763
|
_: Session, file_path: str, variables: Dict | None = None
|
|
548
764
|
) -> None:
|
|
549
|
-
"""Snowpark session-scoped stored procedure to execute content of provided
|
|
765
|
+
"""Snowpark session-scoped stored procedure to execute content of provided Python file."""
|
|
550
766
|
import json
|
|
551
767
|
|
|
552
768
|
from snowflake.snowpark.files import SnowflakeFile
|
|
@@ -566,7 +782,11 @@ class StageManager(SqlExecutionMixin):
|
|
|
566
782
|
return _python_execution_procedure
|
|
567
783
|
|
|
568
784
|
def _execute_python(
|
|
569
|
-
self,
|
|
785
|
+
self,
|
|
786
|
+
file_stage_path: str,
|
|
787
|
+
on_error: OnErrorType,
|
|
788
|
+
variables: Dict,
|
|
789
|
+
original_file: str,
|
|
570
790
|
):
|
|
571
791
|
"""
|
|
572
792
|
Executes Python file from stage using a Snowpark temporary procedure.
|
|
@@ -575,8 +795,8 @@ class StageManager(SqlExecutionMixin):
|
|
|
575
795
|
from snowflake.snowpark.exceptions import SnowparkSQLException
|
|
576
796
|
|
|
577
797
|
try:
|
|
578
|
-
self._python_exe_procedure(self.get_standard_stage_prefix(file_stage_path), variables) # type: ignore
|
|
579
|
-
return StageManager._success_result(file=
|
|
798
|
+
self._python_exe_procedure(self.get_standard_stage_prefix(file_stage_path), variables, session=self.snowpark_session) # type: ignore
|
|
799
|
+
return StageManager._success_result(file=original_file)
|
|
580
800
|
except SnowparkSQLException as e:
|
|
581
801
|
StageManager._handle_execution_exception(on_error=on_error, exception=e)
|
|
582
|
-
return StageManager._error_result(file=
|
|
802
|
+
return StageManager._error_result(file=original_file, msg=e.message)
|
|
@@ -110,7 +110,7 @@ def file_matches_md5sum(local_file: Path, remote_md5: str | None) -> bool:
|
|
|
110
110
|
to a file that has a given remote md5sum.
|
|
111
111
|
|
|
112
112
|
Handles the multi-part md5sums generated by e.g. AWS S3, using values
|
|
113
|
-
from the
|
|
113
|
+
from the Python connector to make educated guesses on chunk size.
|
|
114
114
|
|
|
115
115
|
Assumes that upload time would dominate local hashing time.
|
|
116
116
|
"""
|
|
@@ -45,10 +45,10 @@ log = logging.getLogger(__name__)
|
|
|
45
45
|
class StreamlitManager(SqlExecutionMixin):
|
|
46
46
|
def execute(self, app_name: FQN):
|
|
47
47
|
query = f"EXECUTE STREAMLIT {app_name.sql_identifier}()"
|
|
48
|
-
return self.
|
|
48
|
+
return self.execute_query(query=query)
|
|
49
49
|
|
|
50
50
|
def share(self, streamlit_name: FQN, to_role: str) -> SnowflakeCursor:
|
|
51
|
-
return self.
|
|
51
|
+
return self.execute_query(
|
|
52
52
|
f"grant usage on streamlit {streamlit_name.sql_identifier} to role {to_role}"
|
|
53
53
|
)
|
|
54
54
|
|
|
@@ -118,7 +118,7 @@ class StreamlitManager(SqlExecutionMixin):
|
|
|
118
118
|
if streamlit.secrets:
|
|
119
119
|
query.append(streamlit.get_secrets_sql())
|
|
120
120
|
|
|
121
|
-
self.
|
|
121
|
+
self.execute_query("\n".join(query))
|
|
122
122
|
|
|
123
123
|
def deploy(self, streamlit: StreamlitEntityModel, replace: bool = False):
|
|
124
124
|
streamlit_id = streamlit.fqn.using_connection(self._conn)
|
|
@@ -152,11 +152,11 @@ class StreamlitManager(SqlExecutionMixin):
|
|
|
152
152
|
)
|
|
153
153
|
try:
|
|
154
154
|
if use_versioned_stage:
|
|
155
|
-
self.
|
|
155
|
+
self.execute_query(
|
|
156
156
|
f"ALTER STREAMLIT {streamlit_id.identifier} ADD LIVE VERSION FROM LAST"
|
|
157
157
|
)
|
|
158
158
|
elif not FeatureFlag.ENABLE_STREAMLIT_NO_CHECKOUTS.is_enabled():
|
|
159
|
-
self.
|
|
159
|
+
self.execute_query(
|
|
160
160
|
f"ALTER streamlit {streamlit_id.identifier} CHECKOUT"
|
|
161
161
|
)
|
|
162
162
|
except ProgrammingError as e:
|