snowflake-cli 3.9.1__py3-none-any.whl → 3.10.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli/__about__.py +1 -1
- snowflake/cli/_app/commands_registration/builtin_plugins.py +2 -2
- snowflake/cli/_app/printing.py +53 -13
- snowflake/cli/_app/snow_connector.py +1 -0
- snowflake/cli/_app/telemetry.py +2 -0
- snowflake/cli/_app/version_check.py +73 -6
- snowflake/cli/_plugins/cortex/commands.py +8 -3
- snowflake/cli/_plugins/cortex/manager.py +24 -20
- snowflake/cli/_plugins/dbt/commands.py +5 -2
- snowflake/cli/_plugins/dbt/manager.py +9 -7
- snowflake/cli/_plugins/{project → dcm}/commands.py +95 -48
- snowflake/cli/_plugins/{project/project_entity_model.py → dcm/dcm_project_entity_model.py} +5 -5
- snowflake/cli/_plugins/{project → dcm}/manager.py +35 -14
- snowflake/cli/_plugins/{project → dcm}/plugin_spec.py +1 -1
- snowflake/cli/_plugins/git/manager.py +1 -11
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +4 -0
- snowflake/cli/_plugins/nativeapp/commands.py +3 -4
- snowflake/cli/_plugins/nativeapp/entities/application_package.py +1 -1
- snowflake/cli/_plugins/nativeapp/release_channel/commands.py +1 -2
- snowflake/cli/_plugins/nativeapp/version/commands.py +1 -2
- snowflake/cli/_plugins/snowpark/common.py +23 -11
- snowflake/cli/_plugins/snowpark/snowpark_entity.py +13 -5
- snowflake/cli/_plugins/snowpark/snowpark_entity_model.py +10 -2
- snowflake/cli/_plugins/sql/commands.py +49 -1
- snowflake/cli/_plugins/sql/manager.py +14 -4
- snowflake/cli/_plugins/sql/repl.py +4 -0
- snowflake/cli/_plugins/stage/commands.py +30 -11
- snowflake/cli/_plugins/stage/diff.py +2 -0
- snowflake/cli/_plugins/stage/manager.py +79 -55
- snowflake/cli/_plugins/streamlit/streamlit_entity.py +17 -30
- snowflake/cli/api/artifacts/upload.py +1 -1
- snowflake/cli/api/cli_global_context.py +5 -14
- snowflake/cli/api/commands/decorators.py +7 -0
- snowflake/cli/api/commands/flags.py +12 -0
- snowflake/cli/api/commands/snow_typer.py +23 -2
- snowflake/cli/api/config.py +9 -5
- snowflake/cli/api/connections.py +1 -0
- snowflake/cli/api/constants.py +2 -2
- snowflake/cli/api/entities/common.py +16 -13
- snowflake/cli/api/entities/utils.py +15 -9
- snowflake/cli/api/feature_flags.py +2 -5
- snowflake/cli/api/output/formats.py +6 -0
- snowflake/cli/api/output/types.py +48 -2
- snowflake/cli/api/project/schemas/entities/entities.py +6 -6
- snowflake/cli/api/rendering/sql_templates.py +67 -11
- snowflake/cli/api/rest_api.py +1 -0
- snowflake/cli/api/stage_path.py +41 -5
- {snowflake_cli-3.9.1.dist-info → snowflake_cli-3.10.1.dist-info}/METADATA +46 -13
- {snowflake_cli-3.9.1.dist-info → snowflake_cli-3.10.1.dist-info}/RECORD +53 -54
- snowflake/cli/_plugins/project/feature_flags.py +0 -22
- /snowflake/cli/_plugins/{project → dcm}/__init__.py +0 -0
- {snowflake_cli-3.9.1.dist-info → snowflake_cli-3.10.1.dist-info}/WHEEL +0 -0
- {snowflake_cli-3.9.1.dist-info → snowflake_cli-3.10.1.dist-info}/entry_points.txt +0 -0
- {snowflake_cli-3.9.1.dist-info → snowflake_cli-3.10.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -173,20 +173,28 @@ class SnowparkEntity(EntityBase[Generic[T]]):
|
|
|
173
173
|
if self.model.secrets:
|
|
174
174
|
query.append(self.model.get_secrets_sql())
|
|
175
175
|
|
|
176
|
-
if self.model.
|
|
177
|
-
|
|
176
|
+
if self.model.artifact_repository and (
|
|
177
|
+
self.model.artifact_repository_packages or self.model.packages
|
|
178
|
+
):
|
|
179
|
+
if self.model.artifact_repository_packages:
|
|
180
|
+
packages = [
|
|
181
|
+
f"'{item}'" for item in self.model.artifact_repository_packages
|
|
182
|
+
]
|
|
183
|
+
else:
|
|
184
|
+
packages = [f"'{item}'" for item in self.model.packages]
|
|
178
185
|
|
|
179
|
-
if self.model.artifact_repository and self.model.artifact_repository_packages:
|
|
180
|
-
packages = [f"'{item}'" for item in self.model.artifact_repository_packages]
|
|
181
186
|
query.extend(
|
|
182
187
|
[
|
|
183
188
|
f"ARTIFACT_REPOSITORY= {self.model.artifact_repository} ",
|
|
184
|
-
f"
|
|
189
|
+
f"PACKAGES=({','.join(packages)})",
|
|
185
190
|
]
|
|
186
191
|
)
|
|
187
192
|
if self.model.resource_constraint:
|
|
188
193
|
query.append(self._get_resource_constraints_sql())
|
|
189
194
|
|
|
195
|
+
if self.model.type == "procedure" and self.model.execute_as_caller:
|
|
196
|
+
query.append("EXECUTE AS CALLER")
|
|
197
|
+
|
|
190
198
|
return "\n".join(query)
|
|
191
199
|
|
|
192
200
|
def get_execute_sql(self, execution_arguments: List[str] | None = None):
|
|
@@ -54,6 +54,9 @@ class SnowparkEntityModel(
|
|
|
54
54
|
default=None, title="Artifact repository to be used"
|
|
55
55
|
)
|
|
56
56
|
artifact_repository_packages: Optional[List[str]] = Field(
|
|
57
|
+
default=None, title="Alias for packages"
|
|
58
|
+
)
|
|
59
|
+
packages: Optional[List[str]] = Field(
|
|
57
60
|
default=None, title="Packages to be installed from artifact repository"
|
|
58
61
|
)
|
|
59
62
|
|
|
@@ -91,9 +94,14 @@ class SnowparkEntityModel(
|
|
|
91
94
|
def check_artifact_repository(cls, values: dict) -> dict:
|
|
92
95
|
artifact_repository = values.get("artifact_repository")
|
|
93
96
|
artifact_repository_packages = values.get("artifact_repository_packages")
|
|
94
|
-
|
|
97
|
+
packages = values.get("packages")
|
|
98
|
+
if artifact_repository_packages and packages:
|
|
99
|
+
raise ValueError(
|
|
100
|
+
"You cannot specify both artifact_repository_packages and packages.",
|
|
101
|
+
)
|
|
102
|
+
if (artifact_repository_packages or packages) and not artifact_repository:
|
|
95
103
|
raise ValueError(
|
|
96
|
-
"You specified
|
|
104
|
+
"You specified packages / artifact_repository_packages without setting artifact_repository.",
|
|
97
105
|
)
|
|
98
106
|
return values
|
|
99
107
|
|
|
@@ -15,11 +15,13 @@
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
17
|
import sys
|
|
18
|
+
from enum import Enum
|
|
18
19
|
from logging import getLogger
|
|
19
20
|
from pathlib import Path
|
|
20
21
|
from typing import List, Optional
|
|
21
22
|
|
|
22
23
|
import typer
|
|
24
|
+
from click import UsageError
|
|
23
25
|
from snowflake.cli._plugins.sql.manager import SqlManager
|
|
24
26
|
from snowflake.cli.api.commands.decorators import with_project_definition
|
|
25
27
|
from snowflake.cli.api.commands.flags import (
|
|
@@ -34,6 +36,7 @@ from snowflake.cli.api.output.types import (
|
|
|
34
36
|
MultipleResults,
|
|
35
37
|
QueryResult,
|
|
36
38
|
)
|
|
39
|
+
from snowflake.cli.api.rendering.sql_templates import SQLTemplateSyntaxConfig
|
|
37
40
|
|
|
38
41
|
logger = getLogger(__name__)
|
|
39
42
|
# simple Typer with defaults because it won't become a command group as it contains only one command
|
|
@@ -46,6 +49,37 @@ SourceOption = OverrideableOption(
|
|
|
46
49
|
)
|
|
47
50
|
|
|
48
51
|
|
|
52
|
+
class _EnabledTemplating(str, Enum):
|
|
53
|
+
LEGACY = "LEGACY"
|
|
54
|
+
STANDARD = "STANDARD"
|
|
55
|
+
JINJA = "JINJA"
|
|
56
|
+
ALL = "ALL"
|
|
57
|
+
NONE = "NONE"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _parse_template_syntax_config(
|
|
61
|
+
enabled_syntaxes: List[_EnabledTemplating],
|
|
62
|
+
) -> SQLTemplateSyntaxConfig:
|
|
63
|
+
if (
|
|
64
|
+
_EnabledTemplating.ALL in enabled_syntaxes
|
|
65
|
+
or _EnabledTemplating.NONE in enabled_syntaxes
|
|
66
|
+
) and len(enabled_syntaxes) > 1:
|
|
67
|
+
raise UsageError(
|
|
68
|
+
"ALL and NONE template syntax options should not be used with other options."
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
if _EnabledTemplating.ALL in enabled_syntaxes:
|
|
72
|
+
return SQLTemplateSyntaxConfig(True, True, True)
|
|
73
|
+
if _EnabledTemplating.NONE in enabled_syntaxes:
|
|
74
|
+
return SQLTemplateSyntaxConfig(False, False, False)
|
|
75
|
+
|
|
76
|
+
result = SQLTemplateSyntaxConfig()
|
|
77
|
+
result.enable_legacy_syntax = _EnabledTemplating.LEGACY in enabled_syntaxes
|
|
78
|
+
result.enable_standard_syntax = _EnabledTemplating.STANDARD in enabled_syntaxes
|
|
79
|
+
result.enable_jinja_syntax = _EnabledTemplating.JINJA in enabled_syntaxes
|
|
80
|
+
return result
|
|
81
|
+
|
|
82
|
+
|
|
49
83
|
@app.command(name="sql", requires_connection=True, no_args_is_help=False)
|
|
50
84
|
@with_project_definition(is_optional=True)
|
|
51
85
|
def execute_sql(
|
|
@@ -83,6 +117,12 @@ def execute_sql(
|
|
|
83
117
|
flag_value=False,
|
|
84
118
|
is_flag=True,
|
|
85
119
|
),
|
|
120
|
+
enabled_templating: List[_EnabledTemplating] = typer.Option(
|
|
121
|
+
[_EnabledTemplating.LEGACY, _EnabledTemplating.STANDARD],
|
|
122
|
+
"--enable-templating",
|
|
123
|
+
help="Syntax used to resolve variables before passing queries to Snowflake.",
|
|
124
|
+
case_sensitive=False,
|
|
125
|
+
),
|
|
86
126
|
**options,
|
|
87
127
|
) -> CommandResult:
|
|
88
128
|
"""
|
|
@@ -100,6 +140,8 @@ def execute_sql(
|
|
|
100
140
|
if data_override:
|
|
101
141
|
data = {v.key: v.value for v in parse_key_value_variables(data_override)}
|
|
102
142
|
|
|
143
|
+
template_syntax_config = _parse_template_syntax_config(enabled_templating)
|
|
144
|
+
|
|
103
145
|
retain_comments = bool(retain_comments)
|
|
104
146
|
single_transaction = bool(single_transaction)
|
|
105
147
|
std_in = bool(std_in)
|
|
@@ -116,7 +158,12 @@ def execute_sql(
|
|
|
116
158
|
raise CliArgumentError("single transaction cannot be used with REPL")
|
|
117
159
|
from snowflake.cli._plugins.sql.repl import Repl
|
|
118
160
|
|
|
119
|
-
Repl(
|
|
161
|
+
Repl(
|
|
162
|
+
SqlManager(),
|
|
163
|
+
data=data,
|
|
164
|
+
retain_comments=retain_comments,
|
|
165
|
+
template_syntax_config=template_syntax_config,
|
|
166
|
+
).run()
|
|
120
167
|
sys.exit(0)
|
|
121
168
|
|
|
122
169
|
manager = SqlManager()
|
|
@@ -128,6 +175,7 @@ def execute_sql(
|
|
|
128
175
|
data=data,
|
|
129
176
|
retain_comments=retain_comments,
|
|
130
177
|
single_transaction=single_transaction,
|
|
178
|
+
template_syntax_config=template_syntax_config,
|
|
131
179
|
)
|
|
132
180
|
if expected_results_cnt == 0:
|
|
133
181
|
# case expected if input only scheduled async queries
|
|
@@ -32,7 +32,10 @@ from snowflake.cli.api.cli_global_context import get_cli_context
|
|
|
32
32
|
from snowflake.cli.api.console import cli_console
|
|
33
33
|
from snowflake.cli.api.exceptions import CliArgumentError, CliSqlError
|
|
34
34
|
from snowflake.cli.api.output.types import CollectionResult
|
|
35
|
-
from snowflake.cli.api.rendering.sql_templates import
|
|
35
|
+
from snowflake.cli.api.rendering.sql_templates import (
|
|
36
|
+
SQLTemplateSyntaxConfig,
|
|
37
|
+
snowflake_sql_jinja_render,
|
|
38
|
+
)
|
|
36
39
|
from snowflake.cli.api.secure_path import SecurePath
|
|
37
40
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin, VerboseCursor
|
|
38
41
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
@@ -51,6 +54,7 @@ class SqlManager(SqlExecutionMixin):
|
|
|
51
54
|
data: Dict | None = None,
|
|
52
55
|
retain_comments: bool = False,
|
|
53
56
|
single_transaction: bool = False,
|
|
57
|
+
template_syntax_config: SQLTemplateSyntaxConfig = SQLTemplateSyntaxConfig(),
|
|
54
58
|
) -> Tuple[ExpectedResultsCount, Iterable[SnowflakeCursor]]:
|
|
55
59
|
"""Reads, transforms and execute statements from input.
|
|
56
60
|
|
|
@@ -62,9 +66,15 @@ class SqlManager(SqlExecutionMixin):
|
|
|
62
66
|
"""
|
|
63
67
|
query = sys.stdin.read() if std_in else query
|
|
64
68
|
|
|
65
|
-
stmt_operators =
|
|
66
|
-
|
|
67
|
-
|
|
69
|
+
stmt_operators = []
|
|
70
|
+
if template_syntax_config.enable_legacy_syntax:
|
|
71
|
+
stmt_operators.append(transpile_snowsql_templates)
|
|
72
|
+
stmt_operators.append(
|
|
73
|
+
partial(
|
|
74
|
+
snowflake_sql_jinja_render,
|
|
75
|
+
template_syntax_config=template_syntax_config,
|
|
76
|
+
data=data,
|
|
77
|
+
)
|
|
68
78
|
)
|
|
69
79
|
remove_comments = not retain_comments
|
|
70
80
|
|
|
@@ -13,6 +13,7 @@ from snowflake.cli._plugins.sql.manager import SqlManager
|
|
|
13
13
|
from snowflake.cli.api.cli_global_context import get_cli_context_manager
|
|
14
14
|
from snowflake.cli.api.console import cli_console
|
|
15
15
|
from snowflake.cli.api.output.types import MultipleResults, QueryResult
|
|
16
|
+
from snowflake.cli.api.rendering.sql_templates import SQLTemplateSyntaxConfig
|
|
16
17
|
from snowflake.cli.api.secure_path import SecurePath
|
|
17
18
|
from snowflake.connector.config_manager import CONFIG_MANAGER
|
|
18
19
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
@@ -35,6 +36,7 @@ class Repl:
|
|
|
35
36
|
sql_manager: SqlManager,
|
|
36
37
|
data: dict | None = None,
|
|
37
38
|
retain_comments: bool = False,
|
|
39
|
+
template_syntax_config: SQLTemplateSyntaxConfig = SQLTemplateSyntaxConfig(),
|
|
38
40
|
):
|
|
39
41
|
"""Requires a `SqlManager` instance to execute queries.
|
|
40
42
|
|
|
@@ -46,6 +48,7 @@ class Repl:
|
|
|
46
48
|
setattr(get_cli_context_manager(), "is_repl", True)
|
|
47
49
|
self._data = data or {}
|
|
48
50
|
self._retain_comments = retain_comments
|
|
51
|
+
self._template_syntax_config = template_syntax_config
|
|
49
52
|
self._history = FileHistory(HISTORY_FILE)
|
|
50
53
|
self._lexer = PygmentsLexer(CliLexer)
|
|
51
54
|
self._completer = cli_completer
|
|
@@ -155,6 +158,7 @@ class Repl:
|
|
|
155
158
|
std_in=False,
|
|
156
159
|
data=self._data,
|
|
157
160
|
retain_comments=self._retain_comments,
|
|
161
|
+
template_syntax_config=self._template_syntax_config,
|
|
158
162
|
)
|
|
159
163
|
return cursors
|
|
160
164
|
|
|
@@ -48,7 +48,6 @@ from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
|
|
|
48
48
|
from snowflake.cli.api.console import cli_console
|
|
49
49
|
from snowflake.cli.api.constants import ObjectType
|
|
50
50
|
from snowflake.cli.api.identifiers import FQN
|
|
51
|
-
from snowflake.cli.api.output.formats import OutputFormat
|
|
52
51
|
from snowflake.cli.api.output.types import (
|
|
53
52
|
CollectionResult,
|
|
54
53
|
CommandResult,
|
|
@@ -74,7 +73,7 @@ add_object_command_aliases(
|
|
|
74
73
|
object_type=ObjectType.STAGE,
|
|
75
74
|
name_argument=StageNameArgument,
|
|
76
75
|
like_option=like_option(
|
|
77
|
-
help_example='`list --like "my%"` lists all stages that begin with
|
|
76
|
+
help_example='`list --like "my%"` lists all stages that begin with "my"',
|
|
78
77
|
),
|
|
79
78
|
scope_option=scope_option(help_example="`list --in database my_db`"),
|
|
80
79
|
)
|
|
@@ -98,7 +97,7 @@ def copy(
|
|
|
98
97
|
show_default=False,
|
|
99
98
|
),
|
|
100
99
|
destination_path: str = typer.Argument(
|
|
101
|
-
help="Target directory path for copy operation.
|
|
100
|
+
help="Target directory path for copy operation.",
|
|
102
101
|
show_default=False,
|
|
103
102
|
),
|
|
104
103
|
overwrite: bool = typer.Option(
|
|
@@ -117,19 +116,24 @@ def copy(
|
|
|
117
116
|
default=False,
|
|
118
117
|
help="Specifies whether Snowflake uses gzip to compress files during upload. Ignored when downloading.",
|
|
119
118
|
),
|
|
119
|
+
refresh: bool = typer.Option(
|
|
120
|
+
default=False,
|
|
121
|
+
help="Specifies whether ALTER STAGE {name} REFRESH should be executed after uploading.",
|
|
122
|
+
),
|
|
120
123
|
**options,
|
|
121
124
|
) -> CommandResult:
|
|
122
125
|
"""
|
|
123
|
-
Copies all files from
|
|
124
|
-
to and downloading files from the stage.
|
|
126
|
+
Copies all files from source path to target directory. This works for uploading
|
|
127
|
+
to and downloading files from the stage, and copying between named stages.
|
|
125
128
|
"""
|
|
126
129
|
is_get = is_stage_path(source_path)
|
|
127
130
|
is_put = is_stage_path(destination_path)
|
|
128
131
|
|
|
129
132
|
if is_get and is_put:
|
|
130
|
-
|
|
131
|
-
|
|
133
|
+
cursor = StageManager().copy_files(
|
|
134
|
+
source_path=source_path, destination_path=destination_path
|
|
132
135
|
)
|
|
136
|
+
return QueryResult(cursor)
|
|
133
137
|
if not is_get and not is_put:
|
|
134
138
|
raise click.ClickException(
|
|
135
139
|
"Both source and target path are local. This operation is not supported."
|
|
@@ -149,6 +153,7 @@ def copy(
|
|
|
149
153
|
parallel=parallel,
|
|
150
154
|
overwrite=overwrite,
|
|
151
155
|
auto_compress=auto_compress,
|
|
156
|
+
refresh=refresh,
|
|
152
157
|
)
|
|
153
158
|
|
|
154
159
|
|
|
@@ -160,12 +165,19 @@ def stage_create(
|
|
|
160
165
|
"--encryption",
|
|
161
166
|
help="Type of encryption supported for all files stored on the stage.",
|
|
162
167
|
),
|
|
168
|
+
enable_directory: bool = typer.Option(
|
|
169
|
+
False,
|
|
170
|
+
"--enable-directory",
|
|
171
|
+
help="Specifies whether directory support is enabled for the stage.",
|
|
172
|
+
),
|
|
163
173
|
**options,
|
|
164
174
|
) -> CommandResult:
|
|
165
175
|
"""
|
|
166
176
|
Creates a named stage if it does not already exist.
|
|
167
177
|
"""
|
|
168
|
-
cursor = StageManager().create(
|
|
178
|
+
cursor = StageManager().create(
|
|
179
|
+
fqn=stage_name, encryption=encryption, enable_directory=enable_directory
|
|
180
|
+
)
|
|
169
181
|
return SingleQueryResult(cursor)
|
|
170
182
|
|
|
171
183
|
|
|
@@ -206,7 +218,7 @@ def stage_diff(
|
|
|
206
218
|
local_root=Path(folder_name),
|
|
207
219
|
stage_path=StageManager.stage_path_parts_from_str(stage_name), # noqa: SLF001
|
|
208
220
|
)
|
|
209
|
-
if get_cli_context().output_format
|
|
221
|
+
if get_cli_context().output_format.is_json:
|
|
210
222
|
return ObjectResult(diff.to_dict())
|
|
211
223
|
else:
|
|
212
224
|
print_diff_to_console(diff)
|
|
@@ -264,6 +276,7 @@ def _put(
|
|
|
264
276
|
parallel: int,
|
|
265
277
|
overwrite: bool,
|
|
266
278
|
auto_compress: bool,
|
|
279
|
+
refresh: bool,
|
|
267
280
|
):
|
|
268
281
|
if recursive and not source_path.is_file():
|
|
269
282
|
cursor_generator = StageManager().put_recursive(
|
|
@@ -273,7 +286,7 @@ def _put(
|
|
|
273
286
|
parallel=parallel,
|
|
274
287
|
auto_compress=auto_compress,
|
|
275
288
|
)
|
|
276
|
-
|
|
289
|
+
output = CollectionResult(cursor_generator)
|
|
277
290
|
else:
|
|
278
291
|
cursor = StageManager().put(
|
|
279
292
|
local_path=source_path.resolve(),
|
|
@@ -282,4 +295,10 @@ def _put(
|
|
|
282
295
|
parallel=parallel,
|
|
283
296
|
auto_compress=auto_compress,
|
|
284
297
|
)
|
|
285
|
-
|
|
298
|
+
output = QueryResult(cursor)
|
|
299
|
+
|
|
300
|
+
if refresh:
|
|
301
|
+
StageManager().refresh(
|
|
302
|
+
StageManager.stage_path_parts_from_str(destination_path).stage_name
|
|
303
|
+
)
|
|
304
|
+
return output
|
|
@@ -239,6 +239,7 @@ def sync_local_diff_with_stage(
|
|
|
239
239
|
deploy_root_path: Path,
|
|
240
240
|
diff_result: DiffResult,
|
|
241
241
|
stage_full_path: str,
|
|
242
|
+
force_overwrite: bool = False,
|
|
242
243
|
):
|
|
243
244
|
"""
|
|
244
245
|
Syncs a given local directory's contents with a Snowflake stage, including removing old files, and re-uploading modified and new files.
|
|
@@ -267,6 +268,7 @@ def sync_local_diff_with_stage(
|
|
|
267
268
|
deploy_root_path=deploy_root_path,
|
|
268
269
|
stage_paths=diff_result.only_local,
|
|
269
270
|
role=role,
|
|
271
|
+
overwrite=force_overwrite,
|
|
270
272
|
)
|
|
271
273
|
except Exception as err:
|
|
272
274
|
# Could be ProgrammingError or IntegrityError from SnowflakeCursor
|
|
@@ -32,7 +32,7 @@ from tempfile import TemporaryDirectory
|
|
|
32
32
|
from textwrap import dedent
|
|
33
33
|
from typing import Deque, Dict, Generator, List, Optional, Union
|
|
34
34
|
|
|
35
|
-
from click import
|
|
35
|
+
from click import UsageError
|
|
36
36
|
from snowflake.cli._plugins.snowpark.package_utils import parse_requirements
|
|
37
37
|
from snowflake.cli.api.commands.common import (
|
|
38
38
|
OnErrorType,
|
|
@@ -41,6 +41,7 @@ from snowflake.cli.api.commands.common import (
|
|
|
41
41
|
from snowflake.cli.api.commands.utils import parse_key_value_variables
|
|
42
42
|
from snowflake.cli.api.console import cli_console
|
|
43
43
|
from snowflake.cli.api.constants import PYTHON_3_12
|
|
44
|
+
from snowflake.cli.api.exceptions import CliError
|
|
44
45
|
from snowflake.cli.api.identifiers import FQN
|
|
45
46
|
from snowflake.cli.api.project.util import VALID_IDENTIFIER_REGEX, to_string_literal
|
|
46
47
|
from snowflake.cli.api.secure_path import SecurePath
|
|
@@ -58,7 +59,10 @@ log = logging.getLogger(__name__)
|
|
|
58
59
|
|
|
59
60
|
|
|
60
61
|
UNQUOTED_FILE_URI_REGEX = r"[\w/*?\-.=&{}$#[\]\"\\!@%^+:]+"
|
|
62
|
+
AT_PREFIX = "@"
|
|
61
63
|
USER_STAGE_PREFIX = "@~"
|
|
64
|
+
SNOW_PREFIX = "snow://"
|
|
65
|
+
|
|
62
66
|
EXECUTE_SUPPORTED_FILES_FORMATS = (
|
|
63
67
|
".sql",
|
|
64
68
|
".py",
|
|
@@ -68,6 +72,17 @@ EXECUTE_SUPPORTED_FILES_FORMATS = (
|
|
|
68
72
|
OMIT_FIRST = slice(1, None)
|
|
69
73
|
STAGE_PATH_REGEX = rf"(?P<prefix>(@|{re.escape('snow://')}))?(?:(?P<first_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?:(?P<second_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?P<name>{VALID_IDENTIFIER_REGEX})/?(?P<directory>([^/]*/?)*)?"
|
|
70
74
|
|
|
75
|
+
# Define supported VSTAGE resource types
|
|
76
|
+
VSTAGE_RESOURCE_TYPE_REGEX = r"[a-zA-Z0-9\-]+"
|
|
77
|
+
VSTAGE_PATH_REGEX = (
|
|
78
|
+
rf"(?P<prefix>{re.escape(SNOW_PREFIX)})"
|
|
79
|
+
rf"(?P<resource_type>{VSTAGE_RESOURCE_TYPE_REGEX})/"
|
|
80
|
+
rf"(?:(?P<first_qualifier>{VALID_IDENTIFIER_REGEX})\.)?"
|
|
81
|
+
rf"(?:(?P<second_qualifier>{VALID_IDENTIFIER_REGEX})\.)?"
|
|
82
|
+
rf"(?P<name>{VALID_IDENTIFIER_REGEX})/?"
|
|
83
|
+
rf"(?P<directory>([^/]*/?)*)?"
|
|
84
|
+
)
|
|
85
|
+
|
|
71
86
|
|
|
72
87
|
class InternalStageEncryptionType(Enum):
|
|
73
88
|
SNOWFLAKE_FULL = "SNOWFLAKE_FULL"
|
|
@@ -80,6 +95,7 @@ class StagePathParts:
|
|
|
80
95
|
stage: str
|
|
81
96
|
stage_name: str
|
|
82
97
|
is_directory: bool
|
|
98
|
+
is_vstage: bool = False
|
|
83
99
|
|
|
84
100
|
@classmethod
|
|
85
101
|
def get_directory(cls, stage_path: str) -> str:
|
|
@@ -97,15 +113,9 @@ class StagePathParts:
|
|
|
97
113
|
def schema(self) -> str | None:
|
|
98
114
|
raise NotImplementedError
|
|
99
115
|
|
|
100
|
-
def replace_stage_prefix(self, file_path: str) -> str:
|
|
101
|
-
raise NotImplementedError
|
|
102
|
-
|
|
103
116
|
def add_stage_prefix(self, file_path: str) -> str:
|
|
104
117
|
raise NotImplementedError
|
|
105
118
|
|
|
106
|
-
def get_directory_from_file_path(self, file_path: str) -> List[str]:
|
|
107
|
-
raise NotImplementedError
|
|
108
|
-
|
|
109
119
|
def get_full_stage_path(self, path: str):
|
|
110
120
|
if prefix := FQN.from_stage_path(self.stage).prefix:
|
|
111
121
|
return prefix + "." + path
|
|
@@ -113,7 +123,7 @@ class StagePathParts:
|
|
|
113
123
|
|
|
114
124
|
def get_standard_stage_path(self) -> str:
|
|
115
125
|
path = self.get_full_stage_path(self.path)
|
|
116
|
-
return f"
|
|
126
|
+
return f"{AT_PREFIX}{path}{'/'if self.is_directory and not path.endswith('/') else ''}"
|
|
117
127
|
|
|
118
128
|
def get_standard_stage_directory_path(self) -> str:
|
|
119
129
|
path = self.get_standard_stage_path()
|
|
@@ -121,17 +131,6 @@ class StagePathParts:
|
|
|
121
131
|
return path + "/"
|
|
122
132
|
return path
|
|
123
133
|
|
|
124
|
-
def strip_stage_prefix(self, path: str):
|
|
125
|
-
raise NotImplementedError
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
def _strip_standard_stage_prefix(path: str) -> str:
|
|
129
|
-
"""Removes '@' or 'snow://' prefix from given string"""
|
|
130
|
-
for prefix in ["@", "snow://"]:
|
|
131
|
-
if path.startswith(prefix):
|
|
132
|
-
path = path.removeprefix(prefix)
|
|
133
|
-
return path
|
|
134
|
-
|
|
135
134
|
|
|
136
135
|
@dataclass
|
|
137
136
|
class DefaultStagePathParts(StagePathParts):
|
|
@@ -149,10 +148,10 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
149
148
|
def __init__(self, stage_path: str):
|
|
150
149
|
match = re.fullmatch(STAGE_PATH_REGEX, stage_path)
|
|
151
150
|
if match is None:
|
|
152
|
-
raise
|
|
151
|
+
raise CliError("Invalid stage path")
|
|
153
152
|
self.directory = match.group("directory")
|
|
154
153
|
self._schema = match.group("second_qualifier") or match.group("first_qualifier")
|
|
155
|
-
self._prefix = match.group("prefix") or
|
|
154
|
+
self._prefix = match.group("prefix") or AT_PREFIX
|
|
156
155
|
self.stage = stage_path.removesuffix(self.directory).rstrip("/")
|
|
157
156
|
|
|
158
157
|
stage_name = FQN.from_stage(self.stage).name
|
|
@@ -180,24 +179,47 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
180
179
|
def schema(self) -> str | None:
|
|
181
180
|
return self._schema
|
|
182
181
|
|
|
183
|
-
def replace_stage_prefix(self, file_path: str) -> str:
|
|
184
|
-
file_path = _strip_standard_stage_prefix(file_path)
|
|
185
|
-
file_path_without_prefix = Path(file_path).parts[OMIT_FIRST]
|
|
186
|
-
return f"{self.stage}/{'/'.join(file_path_without_prefix)}"
|
|
187
|
-
|
|
188
|
-
def strip_stage_prefix(self, file_path: str) -> str:
|
|
189
|
-
file_path = _strip_standard_stage_prefix(file_path)
|
|
190
|
-
if file_path.startswith(self.stage_name):
|
|
191
|
-
return file_path[len(self.stage_name) :]
|
|
192
|
-
return file_path
|
|
193
|
-
|
|
194
182
|
def add_stage_prefix(self, file_path: str) -> str:
|
|
195
183
|
stage = self.stage.rstrip("/")
|
|
196
184
|
return f"{stage}/{file_path.lstrip('/')}"
|
|
197
185
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
186
|
+
|
|
187
|
+
@dataclass
|
|
188
|
+
class VStagePathParts(StagePathParts):
|
|
189
|
+
def __init__(self, stage_path: str):
|
|
190
|
+
match = re.fullmatch(VSTAGE_PATH_REGEX, stage_path)
|
|
191
|
+
if match is None or not match.group("resource_type") or not match.group("name"):
|
|
192
|
+
raise CliError(f"Invalid vstage path: {stage_path}.")
|
|
193
|
+
self.resource_type = match.group("resource_type")
|
|
194
|
+
self.directory = match.group("directory")
|
|
195
|
+
self._schema = match.group("second_qualifier") or match.group("first_qualifier")
|
|
196
|
+
self._prefix = match.group("prefix")
|
|
197
|
+
self.stage = stage_path.removesuffix(self.directory).rstrip("/")
|
|
198
|
+
self.stage_name = self.stage.removeprefix(self._prefix)
|
|
199
|
+
self.is_directory = True if stage_path.endswith("/") else False
|
|
200
|
+
self.is_vstage = True
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def path(self) -> str:
|
|
204
|
+
return f"{self._prefix}{self.stage_name.rstrip('/')}/{self.directory}".rstrip(
|
|
205
|
+
"/"
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
@property
|
|
209
|
+
def full_path(self) -> str:
|
|
210
|
+
return f"{self._prefix}{self.stage_name.rstrip('/')}/{self.directory}".rstrip(
|
|
211
|
+
"/"
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
@property
|
|
215
|
+
def schema(self) -> str | None:
|
|
216
|
+
return self._schema
|
|
217
|
+
|
|
218
|
+
def add_stage_prefix(self, file_path: str) -> str:
|
|
219
|
+
return self.full_path
|
|
220
|
+
|
|
221
|
+
def get_standard_stage_path(self) -> str:
|
|
222
|
+
return self.full_path
|
|
201
223
|
|
|
202
224
|
|
|
203
225
|
@dataclass
|
|
@@ -229,37 +251,29 @@ class UserStagePathParts(StagePathParts):
|
|
|
229
251
|
def full_path(self) -> str:
|
|
230
252
|
return f"{self.stage}/{self.directory}".rstrip("/")
|
|
231
253
|
|
|
232
|
-
def replace_stage_prefix(self, file_path: str) -> str:
|
|
233
|
-
if Path(file_path).parts[0] == self.stage_name:
|
|
234
|
-
return file_path
|
|
235
|
-
return f"{self.stage}/{file_path}"
|
|
236
|
-
|
|
237
254
|
def add_stage_prefix(self, file_path: str) -> str:
|
|
238
255
|
return f"{self.stage}/{file_path}"
|
|
239
256
|
|
|
240
|
-
def get_directory_from_file_path(self, file_path: str) -> List[str]:
|
|
241
|
-
stage_path_length = len(Path(self.directory).parts)
|
|
242
|
-
return list(Path(file_path).parts[stage_path_length:-1])
|
|
243
|
-
|
|
244
257
|
|
|
245
258
|
class StageManager(SqlExecutionMixin):
|
|
246
259
|
def __init__(self):
|
|
247
260
|
super().__init__()
|
|
248
261
|
self._python_exe_procedure = None
|
|
249
262
|
|
|
250
|
-
|
|
251
|
-
|
|
263
|
+
def build_path(self, stage_path: Union[str, StagePath]) -> StagePath:
|
|
264
|
+
if isinstance(stage_path, StagePath):
|
|
265
|
+
return stage_path
|
|
252
266
|
return StagePath.from_stage_str(stage_path)
|
|
253
267
|
|
|
254
268
|
@staticmethod
|
|
255
269
|
def get_standard_stage_prefix(name: str | FQN) -> str:
|
|
256
270
|
if isinstance(name, FQN):
|
|
257
271
|
name = name.identifier
|
|
258
|
-
# Handle
|
|
259
|
-
if name.startswith(
|
|
272
|
+
# Handle vstages
|
|
273
|
+
if name.startswith(SNOW_PREFIX) or name.startswith(AT_PREFIX):
|
|
260
274
|
return name
|
|
261
275
|
|
|
262
|
-
return f"
|
|
276
|
+
return f"{AT_PREFIX}{name}"
|
|
263
277
|
|
|
264
278
|
@staticmethod
|
|
265
279
|
def get_stage_from_path(path: str):
|
|
@@ -275,7 +289,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
275
289
|
return name # already quoted
|
|
276
290
|
|
|
277
291
|
standard_name = StageManager.get_standard_stage_prefix(name)
|
|
278
|
-
if standard_name.startswith(
|
|
292
|
+
if standard_name.startswith(AT_PREFIX) and not re.fullmatch(
|
|
279
293
|
r"@([\w./$])+", standard_name
|
|
280
294
|
):
|
|
281
295
|
return to_string_literal(standard_name)
|
|
@@ -503,7 +517,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
503
517
|
destination_stage_path = StagePath.from_stage_str(destination_path)
|
|
504
518
|
|
|
505
519
|
if destination_stage_path.is_user_stage():
|
|
506
|
-
raise
|
|
520
|
+
raise CliError(
|
|
507
521
|
"Destination path cannot be a user stage. Please provide a named stage."
|
|
508
522
|
)
|
|
509
523
|
|
|
@@ -534,11 +548,14 @@ class StageManager(SqlExecutionMixin):
|
|
|
534
548
|
comment: Optional[str] = None,
|
|
535
549
|
temporary: bool = False,
|
|
536
550
|
encryption: InternalStageEncryptionType | None = None,
|
|
551
|
+
enable_directory: bool = False,
|
|
537
552
|
) -> SnowflakeCursor:
|
|
538
553
|
temporary_str = "temporary " if temporary else ""
|
|
539
554
|
query = f"create {temporary_str}stage if not exists {fqn.sql_identifier}"
|
|
540
555
|
if encryption:
|
|
541
556
|
query += f" encryption = (type = '{encryption.value}')"
|
|
557
|
+
if enable_directory:
|
|
558
|
+
query += f" directory = (enable = true)"
|
|
542
559
|
if comment:
|
|
543
560
|
query += f" comment='{comment}'"
|
|
544
561
|
return self.execute_query(query)
|
|
@@ -572,7 +589,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
572
589
|
|
|
573
590
|
all_files_list = self._get_files_list_from_stage(stage_path.root_path())
|
|
574
591
|
if not all_files_list:
|
|
575
|
-
raise
|
|
592
|
+
raise CliError(f"No files found on stage '{stage_path}'")
|
|
576
593
|
|
|
577
594
|
all_files_with_stage_name_prefix = [
|
|
578
595
|
stage_path_parts.get_directory(file) for file in all_files_list
|
|
@@ -584,7 +601,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
584
601
|
)
|
|
585
602
|
|
|
586
603
|
if not filtered_file_list:
|
|
587
|
-
raise
|
|
604
|
+
raise CliError(f"No files matched pattern '{stage_path}'")
|
|
588
605
|
|
|
589
606
|
# sort filtered files in alphabetical order with directories at the end
|
|
590
607
|
sorted_file_path_list = sorted(
|
|
@@ -678,7 +695,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
678
695
|
if filtered_files:
|
|
679
696
|
return filtered_files
|
|
680
697
|
else:
|
|
681
|
-
raise
|
|
698
|
+
raise CliError(
|
|
682
699
|
f"Invalid file extension, only {', '.join(EXECUTE_SUPPORTED_FILES_FORMATS)} files are allowed."
|
|
683
700
|
)
|
|
684
701
|
# Filter with fnmatch if contains `*` or `?`
|
|
@@ -750,8 +767,15 @@ class StageManager(SqlExecutionMixin):
|
|
|
750
767
|
stage_path = StageManager.get_standard_stage_prefix(stage_path)
|
|
751
768
|
if stage_path.startswith(USER_STAGE_PREFIX):
|
|
752
769
|
return UserStagePathParts(stage_path)
|
|
770
|
+
elif stage_path.startswith(SNOW_PREFIX):
|
|
771
|
+
return VStagePathParts(stage_path)
|
|
753
772
|
return DefaultStagePathParts(stage_path)
|
|
754
773
|
|
|
774
|
+
def refresh(self, stage_name):
|
|
775
|
+
sql = f"ALTER STAGE {stage_name} REFRESH"
|
|
776
|
+
log.info("Refreshing stage %s", stage_name)
|
|
777
|
+
return self.execute_query(sql)
|
|
778
|
+
|
|
755
779
|
def _check_for_requirements_file(self, stage_path: StagePath) -> List[str]:
|
|
756
780
|
"""Looks for requirements.txt file on stage."""
|
|
757
781
|
current_dir = stage_path.parent if stage_path.is_file() else stage_path
|
|
@@ -800,7 +824,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
800
824
|
def _bootstrap_snowpark_execution_environment(self, stage_path: StagePath):
|
|
801
825
|
"""Prepares Snowpark session for executing Python code remotely."""
|
|
802
826
|
if sys.version_info >= PYTHON_3_12:
|
|
803
|
-
raise
|
|
827
|
+
raise CliError(
|
|
804
828
|
f"Executing Python files is not supported in Python >= 3.12. Current version: {sys.version}"
|
|
805
829
|
)
|
|
806
830
|
|