snowflake-cli 3.0.2__py3-none-any.whl → 3.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. snowflake/cli/__about__.py +1 -1
  2. snowflake/cli/_app/cli_app.py +3 -0
  3. snowflake/cli/_app/dev/docs/templates/overview.rst.jinja2 +1 -1
  4. snowflake/cli/_app/dev/docs/templates/usage.rst.jinja2 +2 -2
  5. snowflake/cli/_app/telemetry.py +69 -4
  6. snowflake/cli/_plugins/connection/commands.py +152 -99
  7. snowflake/cli/_plugins/connection/util.py +54 -9
  8. snowflake/cli/_plugins/cortex/manager.py +1 -1
  9. snowflake/cli/_plugins/git/commands.py +6 -3
  10. snowflake/cli/_plugins/git/manager.py +9 -4
  11. snowflake/cli/_plugins/nativeapp/artifacts.py +77 -13
  12. snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py +1 -1
  13. snowflake/cli/_plugins/nativeapp/codegen/compiler.py +7 -0
  14. snowflake/cli/_plugins/nativeapp/codegen/sandbox.py +10 -10
  15. snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +2 -2
  16. snowflake/cli/_plugins/nativeapp/codegen/snowpark/extension_function_utils.py +1 -1
  17. snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +8 -8
  18. snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +5 -3
  19. snowflake/cli/_plugins/nativeapp/commands.py +144 -188
  20. snowflake/cli/_plugins/nativeapp/constants.py +1 -0
  21. snowflake/cli/_plugins/nativeapp/entities/application.py +564 -351
  22. snowflake/cli/_plugins/nativeapp/entities/application_package.py +583 -929
  23. snowflake/cli/_plugins/nativeapp/entities/models/event_sharing_telemetry.py +58 -0
  24. snowflake/cli/_plugins/nativeapp/exceptions.py +12 -0
  25. snowflake/cli/_plugins/nativeapp/same_account_install_method.py +0 -2
  26. snowflake/cli/_plugins/nativeapp/sf_facade.py +30 -0
  27. snowflake/cli/_plugins/nativeapp/sf_facade_constants.py +25 -0
  28. snowflake/cli/_plugins/nativeapp/sf_facade_exceptions.py +117 -0
  29. snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +525 -0
  30. snowflake/cli/_plugins/nativeapp/v2_conversions/{v2_to_v1_decorator.py → compat.py} +88 -117
  31. snowflake/cli/_plugins/nativeapp/version/commands.py +36 -32
  32. snowflake/cli/_plugins/notebook/manager.py +2 -2
  33. snowflake/cli/_plugins/object/commands.py +10 -1
  34. snowflake/cli/_plugins/object/manager.py +13 -5
  35. snowflake/cli/_plugins/snowpark/common.py +63 -21
  36. snowflake/cli/_plugins/snowpark/package/anaconda_packages.py +3 -3
  37. snowflake/cli/_plugins/spcs/common.py +29 -0
  38. snowflake/cli/_plugins/spcs/compute_pool/manager.py +7 -9
  39. snowflake/cli/_plugins/spcs/image_registry/manager.py +2 -2
  40. snowflake/cli/_plugins/spcs/image_repository/commands.py +4 -37
  41. snowflake/cli/_plugins/spcs/image_repository/manager.py +4 -1
  42. snowflake/cli/_plugins/spcs/services/commands.py +100 -17
  43. snowflake/cli/_plugins/spcs/services/manager.py +108 -16
  44. snowflake/cli/_plugins/sql/commands.py +9 -1
  45. snowflake/cli/_plugins/sql/manager.py +9 -4
  46. snowflake/cli/_plugins/stage/commands.py +28 -19
  47. snowflake/cli/_plugins/stage/diff.py +17 -17
  48. snowflake/cli/_plugins/stage/manager.py +304 -84
  49. snowflake/cli/_plugins/stage/md5.py +1 -1
  50. snowflake/cli/_plugins/streamlit/manager.py +5 -5
  51. snowflake/cli/_plugins/workspace/commands.py +27 -4
  52. snowflake/cli/_plugins/workspace/context.py +38 -0
  53. snowflake/cli/_plugins/workspace/manager.py +23 -13
  54. snowflake/cli/api/cli_global_context.py +4 -3
  55. snowflake/cli/api/commands/flags.py +23 -7
  56. snowflake/cli/api/config.py +30 -9
  57. snowflake/cli/api/connections.py +12 -1
  58. snowflake/cli/api/console/console.py +4 -19
  59. snowflake/cli/api/entities/common.py +4 -2
  60. snowflake/cli/api/entities/utils.py +36 -69
  61. snowflake/cli/api/errno.py +2 -0
  62. snowflake/cli/api/exceptions.py +41 -0
  63. snowflake/cli/api/identifiers.py +8 -0
  64. snowflake/cli/api/metrics.py +223 -7
  65. snowflake/cli/api/output/types.py +1 -1
  66. snowflake/cli/api/project/definition_conversion.py +293 -77
  67. snowflake/cli/api/project/schemas/entities/common.py +11 -0
  68. snowflake/cli/api/project/schemas/project_definition.py +30 -25
  69. snowflake/cli/api/rest_api.py +26 -4
  70. snowflake/cli/api/secure_utils.py +1 -1
  71. snowflake/cli/api/sql_execution.py +40 -29
  72. snowflake/cli/api/stage_path.py +244 -0
  73. snowflake/cli/api/utils/definition_rendering.py +3 -5
  74. {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/METADATA +14 -15
  75. {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/RECORD +78 -77
  76. {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/WHEEL +1 -1
  77. snowflake/cli/_plugins/nativeapp/manager.py +0 -415
  78. snowflake/cli/_plugins/nativeapp/project_model.py +0 -211
  79. snowflake/cli/_plugins/nativeapp/run_processor.py +0 -184
  80. snowflake/cli/_plugins/nativeapp/teardown_processor.py +0 -70
  81. snowflake/cli/_plugins/nativeapp/version/version_processor.py +0 -98
  82. snowflake/cli/_plugins/workspace/action_context.py +0 -18
  83. {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/entry_points.txt +0 -0
  84. {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/licenses/LICENSE +0 -0
@@ -17,16 +17,21 @@ from __future__ import annotations
17
17
  import fnmatch
18
18
  import glob
19
19
  import logging
20
+ import os
20
21
  import re
22
+ import shutil
21
23
  import sys
24
+ import time
25
+ from collections import deque
22
26
  from contextlib import nullcontext
23
27
  from dataclasses import dataclass
24
28
  from os import path
25
29
  from pathlib import Path
30
+ from tempfile import TemporaryDirectory
26
31
  from textwrap import dedent
27
- from typing import Dict, List, Optional, Union
32
+ from typing import Deque, Dict, Generator, List, Optional, Union
28
33
 
29
- from click import ClickException
34
+ from click import ClickException, UsageError
30
35
  from snowflake.cli._plugins.snowpark.package_utils import parse_requirements
31
36
  from snowflake.cli.api.commands.common import (
32
37
  OnErrorType,
@@ -39,6 +44,7 @@ from snowflake.cli.api.identifiers import FQN
39
44
  from snowflake.cli.api.project.util import to_string_literal
40
45
  from snowflake.cli.api.secure_path import SecurePath
41
46
  from snowflake.cli.api.sql_execution import SqlExecutionMixin
47
+ from snowflake.cli.api.stage_path import StagePath
42
48
  from snowflake.cli.api.utils.path_utils import path_resolver
43
49
  from snowflake.connector import DictCursor, ProgrammingError
44
50
  from snowflake.connector.cursor import SnowflakeCursor
@@ -90,12 +96,12 @@ class StagePathParts:
90
96
  raise NotImplementedError
91
97
 
92
98
  def get_full_stage_path(self, path: str):
93
- if prefix := FQN.from_stage(self.stage).prefix:
99
+ if prefix := FQN.from_stage_path(self.stage).prefix:
94
100
  return prefix + "." + path
95
101
  return path
96
102
 
97
103
  def get_standard_stage_path(self) -> str:
98
- path = self.path
104
+ path = self.get_full_stage_path(self.path)
99
105
  return f"@{path}{'/'if self.is_directory and not path.endswith('/') else ''}"
100
106
 
101
107
  def get_standard_stage_directory_path(self) -> str:
@@ -104,6 +110,9 @@ class StagePathParts:
104
110
  return path + "/"
105
111
  return path
106
112
 
113
+ def strip_stage_prefix(self, path: str):
114
+ raise NotImplementedError
115
+
107
116
 
108
117
  @dataclass
109
118
  class DefaultStagePathParts(StagePathParts):
@@ -141,6 +150,13 @@ class DefaultStagePathParts(StagePathParts):
141
150
  file_path_without_prefix = Path(file_path).parts[OMIT_FIRST]
142
151
  return f"{stage}/{'/'.join(file_path_without_prefix)}"
143
152
 
153
+ def strip_stage_prefix(self, file_path: str) -> str:
154
+ if file_path.startswith("@"):
155
+ file_path = file_path[OMIT_FIRST]
156
+ if file_path.startswith(self.stage_name):
157
+ return file_path[len(self.stage_name) :]
158
+ return file_path
159
+
144
160
  def add_stage_prefix(self, file_path: str) -> str:
145
161
  stage = self.stage.rstrip("/")
146
162
  return f"{stage}/{file_path.lstrip('/')}"
@@ -197,6 +213,10 @@ class StageManager(SqlExecutionMixin):
197
213
  super().__init__()
198
214
  self._python_exe_procedure = None
199
215
 
216
+ @staticmethod
217
+ def build_path(stage_path: str) -> StagePath:
218
+ return StagePath.from_stage_str(stage_path)
219
+
200
220
  @staticmethod
201
221
  def get_standard_stage_prefix(name: str | FQN) -> str:
202
222
  if isinstance(name, FQN):
@@ -234,12 +254,17 @@ class StageManager(SqlExecutionMixin):
234
254
  return uri
235
255
  return to_string_literal(uri)
236
256
 
237
- def list_files(self, stage_name: str, pattern: str | None = None) -> DictCursor:
238
- stage_name = self.get_standard_stage_prefix(stage_name)
239
- query = f"ls {self.quote_stage_name(stage_name)}"
257
+ def list_files(
258
+ self, stage_name: str | StagePath, pattern: str | None = None
259
+ ) -> DictCursor:
260
+ if not isinstance(stage_name, StagePath):
261
+ stage_path = self.build_path(stage_name).path_for_sql()
262
+ else:
263
+ stage_path = stage_name.path_for_sql()
264
+ query = f"ls {stage_path}"
240
265
  if pattern is not None:
241
266
  query += f" pattern = '{pattern}'"
242
- return self._execute_query(query, cursor_class=DictCursor)
267
+ return self.execute_query(query, cursor_class=DictCursor)
243
268
 
244
269
  @staticmethod
245
270
  def _assure_is_existing_directory(path: Path) -> None:
@@ -251,27 +276,27 @@ class StageManager(SqlExecutionMixin):
251
276
  def get(
252
277
  self, stage_path: str, dest_path: Path, parallel: int = 4
253
278
  ) -> SnowflakeCursor:
254
- stage_path = self.get_standard_stage_prefix(stage_path)
279
+ spath = self.build_path(stage_path)
255
280
  self._assure_is_existing_directory(dest_path)
256
281
  dest_directory = f"{dest_path}/"
257
- return self._execute_query(
258
- f"get {self.quote_stage_name(stage_path)} {self._to_uri(dest_directory)} parallel={parallel}"
282
+ return self.execute_query(
283
+ f"get {spath.path_for_sql()} {self._to_uri(dest_directory)} parallel={parallel}"
259
284
  )
260
285
 
261
286
  def get_recursive(
262
287
  self, stage_path: str, dest_path: Path, parallel: int = 4
263
288
  ) -> List[SnowflakeCursor]:
264
- stage_path_parts = self._stage_path_part_factory(stage_path)
289
+ stage_root = self.build_path(stage_path)
265
290
 
266
291
  results = []
267
- for file_path in self.iter_stage(stage_path):
268
- dest_directory = dest_path
269
- for path_part in stage_path_parts.get_directory_from_file_path(file_path):
270
- dest_directory = dest_directory / path_part
271
- self._assure_is_existing_directory(dest_directory)
272
-
273
- result = self._execute_query(
274
- f"get {self.quote_stage_name(stage_path_parts.replace_stage_prefix(file_path))} {self._to_uri(f'{dest_directory}/')} parallel={parallel}"
292
+ for file_path in self.iter_stage(stage_root):
293
+ local_dir = file_path.get_local_target_path(
294
+ target_dir=dest_path, stage_root=stage_root
295
+ )
296
+ self._assure_is_existing_directory(local_dir)
297
+
298
+ result = self.execute_query(
299
+ f"get {file_path.path_for_sql()} {self._to_uri(f'{local_dir}/')} parallel={parallel}"
275
300
  )
276
301
  results.append(result)
277
302
 
@@ -285,6 +310,7 @@ class StageManager(SqlExecutionMixin):
285
310
  overwrite: bool = False,
286
311
  role: Optional[str] = None,
287
312
  auto_compress: bool = False,
313
+ use_dict_cursor: bool = False,
288
314
  ) -> SnowflakeCursor:
289
315
  """
290
316
  This method will take a file path from the user's system and put it into a Snowflake stage,
@@ -292,30 +318,157 @@ class StageManager(SqlExecutionMixin):
292
318
  If provided with a role, then temporarily use this role to perform the operation above,
293
319
  and switch back to the original role for the next commands to run.
294
320
  """
321
+ if "*" not in str(local_path):
322
+ local_path = (
323
+ os.path.join(local_path, "*")
324
+ if Path(local_path).is_dir()
325
+ else str(local_path)
326
+ )
295
327
  with self.use_role(role) if role else nullcontext():
296
- stage_path = self.get_standard_stage_prefix(stage_path)
328
+ spath = self.build_path(stage_path)
297
329
  local_resolved_path = path_resolver(str(local_path))
298
330
  log.info("Uploading %s to %s", local_resolved_path, stage_path)
299
- cursor = self._execute_query(
300
- f"put {self._to_uri(local_resolved_path)} {self.quote_stage_name(stage_path)} "
301
- f"auto_compress={str(auto_compress).lower()} parallel={parallel} overwrite={overwrite}"
331
+ cursor = self.execute_query(
332
+ f"put {self._to_uri(local_resolved_path)} {spath.path_for_sql()} "
333
+ f"auto_compress={str(auto_compress).lower()} parallel={parallel} overwrite={overwrite}",
334
+ cursor_class=DictCursor if use_dict_cursor else SnowflakeCursor,
302
335
  )
303
336
  return cursor
304
337
 
338
+ @staticmethod
339
+ def _symlink_or_copy(source_root: Path, source_file_or_dir: Path, dest_dir: Path):
340
+ from snowflake.cli._plugins.nativeapp.artifacts import resolve_without_follow
341
+
342
+ absolute_src = resolve_without_follow(source_file_or_dir)
343
+ dest_path = dest_dir / source_file_or_dir.relative_to(source_root)
344
+
345
+ if absolute_src.is_file():
346
+ try:
347
+ os.symlink(absolute_src, dest_path)
348
+ except OSError:
349
+ if not dest_path.parent.exists():
350
+ dest_path.parent.mkdir(parents=True, exist_ok=True)
351
+ shutil.copyfile(absolute_src, dest_path)
352
+ else:
353
+ dest_path.mkdir(exist_ok=True, parents=True)
354
+
355
+ def put_recursive(
356
+ self,
357
+ local_path: Path,
358
+ stage_path: str,
359
+ parallel: int = 4,
360
+ overwrite: bool = False,
361
+ role: Optional[str] = None,
362
+ auto_compress: bool = False,
363
+ ) -> Generator[dict, None, None]:
364
+ if local_path.is_file():
365
+ raise UsageError("Cannot use recursive upload with a single file.")
366
+
367
+ if local_path.is_dir():
368
+ root = local_path
369
+ glob_pattern = str(local_path / "**/*")
370
+ else:
371
+ root = Path([p for p in local_path.parents if p.is_dir()][0])
372
+ glob_pattern = str(local_path)
373
+
374
+ with TemporaryDirectory() as tmp:
375
+ temp_dir_with_copy = Path(tmp)
376
+
377
+ # Create a symlink or copy the file to the temp directory
378
+ for file_or_dir in glob.iglob(glob_pattern, recursive=True):
379
+ self._symlink_or_copy(
380
+ source_root=root,
381
+ source_file_or_dir=Path(file_or_dir),
382
+ dest_dir=temp_dir_with_copy,
383
+ )
384
+
385
+ # Find the deepest directories, we will be iterating from bottom to top
386
+ deepest_dirs_list = self._find_deepest_directories(temp_dir_with_copy)
387
+
388
+ while deepest_dirs_list:
389
+ # Remove as visited
390
+ directory = deepest_dirs_list.pop(0)
391
+
392
+ # We reached root but there are still directories to process
393
+ if directory == temp_dir_with_copy and deepest_dirs_list:
394
+ continue
395
+
396
+ # Upload the directory content, at this moment the directory has only files
397
+ if list(directory.iterdir()):
398
+ destination = StagePath.from_stage_str(
399
+ stage_path
400
+ ) / directory.relative_to(temp_dir_with_copy)
401
+ results: list[dict] = self.put(
402
+ local_path=directory,
403
+ stage_path=destination,
404
+ parallel=parallel,
405
+ overwrite=overwrite,
406
+ role=role,
407
+ auto_compress=auto_compress,
408
+ use_dict_cursor=True,
409
+ ).fetchall()
410
+
411
+ # Rewrite results to have resolved paths for better UX
412
+ for item in results:
413
+ item["source"] = (directory / item["source"]).relative_to(
414
+ temp_dir_with_copy
415
+ )
416
+ item["target"] = str(destination / item["target"])
417
+ yield item
418
+
419
+ # We end if we reach the root directory
420
+ if directory == temp_dir_with_copy:
421
+ break
422
+
423
+ # Add parent directory to the list if it's not already there
424
+ if directory.parent not in deepest_dirs_list:
425
+ deepest_dirs_list.append(directory.parent)
426
+
427
+ # Remove the directory so the parent directory will contain only files
428
+ shutil.rmtree(directory)
429
+
430
+ @staticmethod
431
+ def _find_deepest_directories(root_directory: Path) -> list[Path]:
432
+ """
433
+ BFS to find the deepest directories. Build a tree of directories
434
+ structure and return leaves.
435
+ """
436
+ deepest_dirs: list[Path] = list()
437
+
438
+ queue: Deque[Path] = deque()
439
+ queue.append(root_directory)
440
+ while queue:
441
+ current_dir = queue.popleft()
442
+ # Sorted to have deterministic order
443
+ children_directories = sorted(
444
+ list(d for d in current_dir.iterdir() if d.is_dir())
445
+ )
446
+ if not children_directories and current_dir not in deepest_dirs:
447
+ deepest_dirs.append(current_dir)
448
+ else:
449
+ queue.extend([c for c in children_directories if c not in deepest_dirs])
450
+ deepest_dirs_list = sorted(
451
+ list(deepest_dirs), key=lambda d: len(d.parts), reverse=True
452
+ )
453
+ return deepest_dirs_list
454
+
305
455
  def copy_files(self, source_path: str, destination_path: str) -> SnowflakeCursor:
306
- source_path_parts = self._stage_path_part_factory(source_path)
307
- destination_path_parts = self._stage_path_part_factory(destination_path)
456
+ source_stage_path = self.build_path(source_path)
457
+ # We copy only into stage
458
+ destination_stage_path = StagePath.from_stage_str(destination_path)
308
459
 
309
- if isinstance(destination_path_parts, UserStagePathParts):
460
+ if destination_stage_path.is_user_stage():
310
461
  raise ClickException(
311
462
  "Destination path cannot be a user stage. Please provide a named stage."
312
463
  )
313
464
 
314
- source = source_path_parts.get_standard_stage_path()
315
- destination = destination_path_parts.get_standard_stage_directory_path()
316
- log.info("Copying files from %s to %s", source, destination)
317
- query = f"copy files into {destination} from {source}"
318
- return self._execute_query(query)
465
+ log.info(
466
+ "Copying files from %s to %s", source_stage_path, destination_stage_path
467
+ )
468
+ # Destination needs to end with /
469
+ dest = destination_stage_path.absolute_path().rstrip("/") + "/"
470
+ query = f"copy files into {dest} from {source_stage_path}"
471
+ return self.execute_query(query)
319
472
 
320
473
  def remove(
321
474
  self, stage_name: str, path: str, role: Optional[str] = None
@@ -327,29 +480,48 @@ class StageManager(SqlExecutionMixin):
327
480
  and switch back to the original role for the next commands to run.
328
481
  """
329
482
  with self.use_role(role) if role else nullcontext():
330
- stage_name = self.get_standard_stage_prefix(stage_name)
331
- path = path if path.startswith("/") else "/" + path
332
- quoted_stage_name = self.quote_stage_name(f"{stage_name}{path}")
333
- return self._execute_query(f"remove {quoted_stage_name}")
483
+ stage_path = self.build_path(stage_name) / path
484
+ return self.execute_query(f"remove {stage_path.path_for_sql()}")
334
485
 
335
- def create(self, fqn: FQN, comment: Optional[str] = None) -> SnowflakeCursor:
336
- query = f"create stage if not exists {fqn.sql_identifier}"
486
+ def create(
487
+ self, fqn: FQN, comment: Optional[str] = None, temporary: bool = False
488
+ ) -> SnowflakeCursor:
489
+ temporary_str = "temporary " if temporary else ""
490
+ query = f"create {temporary_str}stage if not exists {fqn.sql_identifier}"
337
491
  if comment:
338
492
  query += f" comment='{comment}'"
339
- return self._execute_query(query)
493
+ return self.execute_query(query)
340
494
 
341
- def iter_stage(self, stage_path: str):
342
- for file in self.list_files(stage_path).fetchall():
343
- yield file["name"]
495
+ def iter_stage(self, stage_path: StagePath):
496
+ for file in self.list_files(stage_path.absolute_path()).fetchall():
497
+ if stage_path.is_user_stage():
498
+ path = StagePath.get_user_stage() / file["name"]
499
+ else:
500
+ path = self.build_path(file["name"])
501
+ yield path
344
502
 
345
503
  def execute(
346
504
  self,
347
- stage_path: str,
505
+ stage_path_str: str,
348
506
  on_error: OnErrorType,
349
507
  variables: Optional[List[str]] = None,
508
+ requires_temporary_stage: bool = False,
350
509
  ):
351
- stage_path_parts = self._stage_path_part_factory(stage_path)
352
- all_files_list = self._get_files_list_from_stage(stage_path_parts)
510
+ if requires_temporary_stage:
511
+ (
512
+ stage_path_parts,
513
+ original_path_parts,
514
+ ) = self._create_temporary_copy_of_stage(stage_path_str)
515
+ stage_path = StagePath.from_stage_str(
516
+ stage_path_parts.get_standard_stage_path()
517
+ )
518
+ else:
519
+ stage_path_parts = self._stage_path_part_factory(stage_path_str)
520
+ stage_path = self.build_path(stage_path_str)
521
+
522
+ all_files_list = self._get_files_list_from_stage(stage_path.root_path())
523
+ if not all_files_list:
524
+ raise ClickException(f"No files found on stage '{stage_path}'")
353
525
 
354
526
  all_files_with_stage_name_prefix = [
355
527
  stage_path_parts.get_directory(file) for file in all_files_list
@@ -370,42 +542,75 @@ class StageManager(SqlExecutionMixin):
370
542
 
371
543
  parsed_variables = parse_key_value_variables(variables)
372
544
  sql_variables = self._parse_execute_variables(parsed_variables)
373
- python_variables = {str(v.key): v.value for v in parsed_variables}
545
+ python_variables = self._parse_python_variables(parsed_variables)
374
546
  results = []
375
547
 
376
548
  if any(file.endswith(".py") for file in sorted_file_path_list):
377
549
  self._python_exe_procedure = self._bootstrap_snowpark_execution_environment(
378
- stage_path_parts
550
+ stage_path
379
551
  )
380
552
 
381
553
  for file_path in sorted_file_path_list:
382
554
  file_stage_path = stage_path_parts.add_stage_prefix(file_path)
555
+
556
+ # For better reporting push down the information about original
557
+ # path if execution happens from temporary stage
558
+ if requires_temporary_stage:
559
+ original_path = original_path_parts.add_stage_prefix(file_path)
560
+ else:
561
+ original_path = file_stage_path
562
+
383
563
  if file_path.endswith(".py"):
384
564
  result = self._execute_python(
385
565
  file_stage_path=file_stage_path,
386
566
  on_error=on_error,
387
567
  variables=python_variables,
568
+ original_file=original_path,
388
569
  )
389
570
  else:
390
571
  result = self._call_execute_immediate(
391
572
  file_stage_path=file_stage_path,
392
573
  variables=sql_variables,
393
574
  on_error=on_error,
575
+ original_file=original_path,
394
576
  )
395
577
  results.append(result)
396
578
 
397
579
  return results
398
580
 
399
- def _get_files_list_from_stage(
400
- self, stage_path_parts: StagePathParts, pattern: str | None = None
401
- ) -> List[str]:
402
- files_list_result = self.list_files(
403
- stage_path_parts.stage, pattern=pattern
404
- ).fetchall()
581
+ def _create_temporary_copy_of_stage(
582
+ self, stage_path: str
583
+ ) -> tuple[StagePathParts, StagePathParts]:
584
+ sm = StageManager()
585
+
586
+ # Rewrite stage paths to temporary stage paths. Git paths become stage paths
587
+ original_path_parts = self._stage_path_part_factory(stage_path) # noqa: SLF001
405
588
 
406
- if not files_list_result:
407
- raise ClickException(f"No files found on stage '{stage_path_parts.stage}'")
589
+ tmp_stage_name = f"snowflake_cli_tmp_stage_{int(time.time())}"
590
+ tmp_stage_fqn = FQN.from_stage(tmp_stage_name).using_connection(conn=self._conn)
591
+ tmp_stage = tmp_stage_fqn.identifier
592
+ stage_path_parts = sm._stage_path_part_factory( # noqa: SLF001
593
+ tmp_stage + "/" + original_path_parts.directory
594
+ )
595
+
596
+ # Create temporary stage, it will be dropped with end of session
597
+ sm.create(tmp_stage_fqn, temporary=True)
598
+
599
+ # Copy the content
600
+ self.copy_files(
601
+ source_path=original_path_parts.get_full_stage_path(
602
+ original_path_parts.stage_name
603
+ ),
604
+ destination_path=stage_path_parts.get_full_stage_path(
605
+ stage_path_parts.stage_name
606
+ ),
607
+ )
608
+ return stage_path_parts, original_path_parts
408
609
 
610
+ def _get_files_list_from_stage(
611
+ self, stage_path: StagePath, pattern: str | None = None
612
+ ) -> List[str]:
613
+ files_list_result = self.list_files(stage_path, pattern=pattern).fetchall()
409
614
  return [f["name"] for f in files_list_result]
410
615
 
411
616
  def _filter_files_list(
@@ -444,6 +649,17 @@ class StageManager(SqlExecutionMixin):
444
649
  query_parameters = [f"{v.key}=>{v.value}" for v in variables]
445
650
  return f" using ({', '.join(query_parameters)})"
446
651
 
652
+ @staticmethod
653
+ def _parse_python_variables(variables: List[Variable]) -> Dict:
654
+ def _unwrap(s: str):
655
+ if s.startswith("'") and s.endswith("'"):
656
+ return s[1:-1]
657
+ if s.startswith('"') and s.endswith('"'):
658
+ return s[1:-1]
659
+ return s
660
+
661
+ return {str(v.key): _unwrap(v.value) for v in variables}
662
+
447
663
  @staticmethod
448
664
  def _success_result(file: str):
449
665
  cli_console.warning(f"SUCCESS - {file}")
@@ -464,16 +680,17 @@ class StageManager(SqlExecutionMixin):
464
680
  file_stage_path: str,
465
681
  variables: Optional[str],
466
682
  on_error: OnErrorType,
683
+ original_file: str,
467
684
  ) -> Dict:
468
685
  try:
469
686
  query = f"execute immediate from {self.quote_stage_name(file_stage_path)}"
470
687
  if variables:
471
688
  query += variables
472
- self._execute_query(query)
473
- return StageManager._success_result(file=file_stage_path)
689
+ self.execute_query(query)
690
+ return StageManager._success_result(file=original_file)
474
691
  except ProgrammingError as e:
475
692
  StageManager._handle_execution_exception(on_error=on_error, exception=e)
476
- return StageManager._error_result(file=file_stage_path, msg=e.msg)
693
+ return StageManager._error_result(file=original_file, msg=e.msg)
477
694
 
478
695
  @staticmethod
479
696
  def _stage_path_part_factory(stage_path: str) -> StagePathParts:
@@ -482,32 +699,34 @@ class StageManager(SqlExecutionMixin):
482
699
  return UserStagePathParts(stage_path)
483
700
  return DefaultStagePathParts(stage_path)
484
701
 
485
- def _check_for_requirements_file(
486
- self, stage_path_parts: StagePathParts
487
- ) -> List[str]:
702
+ def _check_for_requirements_file(self, stage_path: StagePath) -> List[str]:
488
703
  """Looks for requirements.txt file on stage."""
704
+ current_dir = stage_path.parent if stage_path.is_file() else stage_path
489
705
  req_files_on_stage = self._get_files_list_from_stage(
490
- stage_path_parts, pattern=r".*requirements\.txt$"
706
+ current_dir, pattern=r".*requirements\.txt$"
491
707
  )
492
708
  if not req_files_on_stage:
493
709
  return []
494
710
 
495
711
  # Construct all possible path for requirements file for this context
496
- # We don't use os.path or pathlib to preserve compatibility on Windows
497
712
  req_file_name = "requirements.txt"
498
- path_parts = stage_path_parts.path.split("/")
499
713
  possible_req_files = []
714
+ while not current_dir.is_root():
715
+ current_file = current_dir / req_file_name
716
+ possible_req_files.append(current_file)
717
+ current_dir = current_dir.parent
500
718
 
501
- while path_parts:
502
- current_file = "/".join([*path_parts, req_file_name])
503
- possible_req_files.append(str(current_file))
504
- path_parts = path_parts[:-1]
719
+ current_file = current_dir / req_file_name
720
+ possible_req_files.append(current_file)
505
721
 
506
722
  # Now for every possible path check if the file exists on stage,
507
723
  # if yes break, we use the first possible file
508
- requirements_file = None
724
+ requirements_file: StagePath | None = None
509
725
  for req_file in possible_req_files:
510
- if req_file in req_files_on_stage:
726
+ if (
727
+ req_file.absolute_path(no_fqn=True, at_prefix=False)
728
+ in req_files_on_stage
729
+ ):
511
730
  requirements_file = req_file
512
731
  break
513
732
 
@@ -516,37 +735,34 @@ class StageManager(SqlExecutionMixin):
516
735
  return []
517
736
 
518
737
  # req_file at this moment is the first found requirements file
738
+ requirements_path = requirements_file.with_stage(stage_path.stage)
519
739
  with SecurePath.temporary_directory() as tmp_dir:
520
- self.get(
521
- stage_path_parts.get_full_stage_path(requirements_file), tmp_dir.path
522
- )
740
+ self.get(str(requirements_path), tmp_dir.path)
523
741
  requirements = parse_requirements(
524
742
  requirements_file=tmp_dir / "requirements.txt"
525
743
  )
526
744
 
527
745
  return [req.package_name for req in requirements]
528
746
 
529
- def _bootstrap_snowpark_execution_environment(
530
- self, stage_path_parts: StagePathParts
531
- ):
747
+ def _bootstrap_snowpark_execution_environment(self, stage_path: StagePath):
532
748
  """Prepares Snowpark session for executing Python code remotely."""
533
749
  if sys.version_info >= PYTHON_3_12:
534
750
  raise ClickException(
535
- f"Executing python files is not supported in Python >= 3.12. Current version: {sys.version}"
751
+ f"Executing Python files is not supported in Python >= 3.12. Current version: {sys.version}"
536
752
  )
537
753
 
538
754
  from snowflake.snowpark.functions import sproc
539
755
 
540
756
  self.snowpark_session.add_packages("snowflake-snowpark-python")
541
757
  self.snowpark_session.add_packages("snowflake.core")
542
- requirements = self._check_for_requirements_file(stage_path_parts)
758
+ requirements = self._check_for_requirements_file(stage_path)
543
759
  self.snowpark_session.add_packages(*requirements)
544
760
 
545
- @sproc(is_permanent=False)
761
+ @sproc(is_permanent=False, session=self.snowpark_session)
546
762
  def _python_execution_procedure(
547
763
  _: Session, file_path: str, variables: Dict | None = None
548
764
  ) -> None:
549
- """Snowpark session-scoped stored procedure to execute content of provided python file."""
765
+ """Snowpark session-scoped stored procedure to execute content of provided Python file."""
550
766
  import json
551
767
 
552
768
  from snowflake.snowpark.files import SnowflakeFile
@@ -566,7 +782,11 @@ class StageManager(SqlExecutionMixin):
566
782
  return _python_execution_procedure
567
783
 
568
784
  def _execute_python(
569
- self, file_stage_path: str, on_error: OnErrorType, variables: Dict
785
+ self,
786
+ file_stage_path: str,
787
+ on_error: OnErrorType,
788
+ variables: Dict,
789
+ original_file: str,
570
790
  ):
571
791
  """
572
792
  Executes Python file from stage using a Snowpark temporary procedure.
@@ -575,8 +795,8 @@ class StageManager(SqlExecutionMixin):
575
795
  from snowflake.snowpark.exceptions import SnowparkSQLException
576
796
 
577
797
  try:
578
- self._python_exe_procedure(self.get_standard_stage_prefix(file_stage_path), variables) # type: ignore
579
- return StageManager._success_result(file=file_stage_path)
798
+ self._python_exe_procedure(self.get_standard_stage_prefix(file_stage_path), variables, session=self.snowpark_session) # type: ignore
799
+ return StageManager._success_result(file=original_file)
580
800
  except SnowparkSQLException as e:
581
801
  StageManager._handle_execution_exception(on_error=on_error, exception=e)
582
- return StageManager._error_result(file=file_stage_path, msg=e.message)
802
+ return StageManager._error_result(file=original_file, msg=e.message)
@@ -110,7 +110,7 @@ def file_matches_md5sum(local_file: Path, remote_md5: str | None) -> bool:
110
110
  to a file that has a given remote md5sum.
111
111
 
112
112
  Handles the multi-part md5sums generated by e.g. AWS S3, using values
113
- from the python connector to make educated guesses on chunk size.
113
+ from the Python connector to make educated guesses on chunk size.
114
114
 
115
115
  Assumes that upload time would dominate local hashing time.
116
116
  """
@@ -45,10 +45,10 @@ log = logging.getLogger(__name__)
45
45
  class StreamlitManager(SqlExecutionMixin):
46
46
  def execute(self, app_name: FQN):
47
47
  query = f"EXECUTE STREAMLIT {app_name.sql_identifier}()"
48
- return self._execute_query(query=query)
48
+ return self.execute_query(query=query)
49
49
 
50
50
  def share(self, streamlit_name: FQN, to_role: str) -> SnowflakeCursor:
51
- return self._execute_query(
51
+ return self.execute_query(
52
52
  f"grant usage on streamlit {streamlit_name.sql_identifier} to role {to_role}"
53
53
  )
54
54
 
@@ -118,7 +118,7 @@ class StreamlitManager(SqlExecutionMixin):
118
118
  if streamlit.secrets:
119
119
  query.append(streamlit.get_secrets_sql())
120
120
 
121
- self._execute_query("\n".join(query))
121
+ self.execute_query("\n".join(query))
122
122
 
123
123
  def deploy(self, streamlit: StreamlitEntityModel, replace: bool = False):
124
124
  streamlit_id = streamlit.fqn.using_connection(self._conn)
@@ -152,11 +152,11 @@ class StreamlitManager(SqlExecutionMixin):
152
152
  )
153
153
  try:
154
154
  if use_versioned_stage:
155
- self._execute_query(
155
+ self.execute_query(
156
156
  f"ALTER STREAMLIT {streamlit_id.identifier} ADD LIVE VERSION FROM LAST"
157
157
  )
158
158
  elif not FeatureFlag.ENABLE_STREAMLIT_NO_CHECKOUTS.is_enabled():
159
- self._execute_query(
159
+ self.execute_query(
160
160
  f"ALTER streamlit {streamlit_id.identifier} CHECKOUT"
161
161
  )
162
162
  except ProgrammingError as e: