snowflake-cli 3.6.0__py3-none-any.whl → 3.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli/__about__.py +1 -1
- snowflake/cli/_app/commands_registration/builtin_plugins.py +2 -0
- snowflake/cli/_app/loggers.py +2 -2
- snowflake/cli/_app/snow_connector.py +2 -2
- snowflake/cli/_plugins/connection/commands.py +2 -1
- snowflake/cli/_plugins/helpers/commands.py +25 -1
- snowflake/cli/_plugins/helpers/snowsl_vars_reader.py +133 -0
- snowflake/cli/_plugins/init/commands.py +9 -6
- snowflake/cli/_plugins/logs/__init__.py +0 -0
- snowflake/cli/_plugins/logs/commands.py +105 -0
- snowflake/cli/_plugins/logs/manager.py +107 -0
- snowflake/cli/_plugins/logs/plugin_spec.py +16 -0
- snowflake/cli/_plugins/logs/utils.py +60 -0
- snowflake/cli/_plugins/notebook/commands.py +3 -0
- snowflake/cli/_plugins/notebook/notebook_entity.py +16 -27
- snowflake/cli/_plugins/project/commands.py +73 -48
- snowflake/cli/_plugins/project/manager.py +57 -23
- snowflake/cli/_plugins/project/project_entity_model.py +22 -3
- snowflake/cli/_plugins/snowpark/commands.py +15 -2
- snowflake/cli/_plugins/spcs/image_registry/manager.py +15 -6
- snowflake/cli/_plugins/sql/manager.py +4 -4
- snowflake/cli/_plugins/stage/manager.py +17 -10
- snowflake/cli/_plugins/streamlit/commands.py +3 -0
- snowflake/cli/_plugins/streamlit/manager.py +19 -15
- snowflake/cli/api/artifacts/upload.py +30 -34
- snowflake/cli/api/artifacts/utils.py +8 -6
- snowflake/cli/api/cli_global_context.py +7 -2
- snowflake/cli/api/commands/decorators.py +11 -2
- snowflake/cli/api/commands/flags.py +23 -2
- snowflake/cli/api/commands/snow_typer.py +20 -2
- snowflake/cli/api/config.py +5 -3
- snowflake/cli/api/entities/utils.py +29 -16
- snowflake/cli/api/exceptions.py +69 -28
- snowflake/cli/api/identifiers.py +2 -0
- snowflake/cli/api/plugins/plugin_config.py +2 -2
- snowflake/cli/api/project/schemas/template.py +3 -3
- snowflake/cli/api/rendering/project_templates.py +3 -3
- snowflake/cli/api/rendering/sql_templates.py +2 -2
- snowflake/cli/api/sql_execution.py +1 -1
- snowflake/cli/api/utils/definition_rendering.py +14 -8
- snowflake/cli/api/utils/templating_functions.py +4 -4
- {snowflake_cli-3.6.0.dist-info → snowflake_cli-3.7.1.dist-info}/METADATA +9 -8
- {snowflake_cli-3.6.0.dist-info → snowflake_cli-3.7.1.dist-info}/RECORD +46 -40
- {snowflake_cli-3.6.0.dist-info → snowflake_cli-3.7.1.dist-info}/WHEEL +0 -0
- {snowflake_cli-3.6.0.dist-info → snowflake_cli-3.7.1.dist-info}/entry_points.txt +0 -0
- {snowflake_cli-3.6.0.dist-info → snowflake_cli-3.7.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -4,9 +4,8 @@ from click import ClickException
|
|
|
4
4
|
from snowflake.cli._plugins.connection.util import make_snowsight_url
|
|
5
5
|
from snowflake.cli._plugins.notebook.notebook_entity_model import NotebookEntityModel
|
|
6
6
|
from snowflake.cli._plugins.notebook.notebook_project_paths import NotebookProjectPaths
|
|
7
|
-
from snowflake.cli._plugins.stage.manager import StageManager
|
|
8
7
|
from snowflake.cli._plugins.workspace.context import ActionContext
|
|
9
|
-
from snowflake.cli.api.artifacts.
|
|
8
|
+
from snowflake.cli.api.artifacts.upload import sync_artifacts_with_stage
|
|
10
9
|
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
11
10
|
from snowflake.cli.api.console.console import cli_console
|
|
12
11
|
from snowflake.cli.api.entities.common import EntityBase
|
|
@@ -22,12 +21,15 @@ class NotebookEntity(EntityBase[NotebookEntityModel]):
|
|
|
22
21
|
A notebook.
|
|
23
22
|
"""
|
|
24
23
|
|
|
24
|
+
@property
|
|
25
|
+
def _stage_path_from_model(self) -> str:
|
|
26
|
+
if self.model.stage_path is None:
|
|
27
|
+
return f"{_DEFAULT_NOTEBOOK_STAGE_NAME}/{self.fqn.name}"
|
|
28
|
+
return self.model.stage_path
|
|
29
|
+
|
|
25
30
|
@functools.cached_property
|
|
26
31
|
def _stage_path(self) -> StagePath:
|
|
27
|
-
|
|
28
|
-
if stage_path is None:
|
|
29
|
-
stage_path = f"{_DEFAULT_NOTEBOOK_STAGE_NAME}/{self.fqn.name}"
|
|
30
|
-
return StagePath.from_stage_str(stage_path)
|
|
32
|
+
return StagePath.from_stage_str(self._stage_path_from_model)
|
|
31
33
|
|
|
32
34
|
@functools.cached_property
|
|
33
35
|
def _project_paths(self):
|
|
@@ -41,26 +43,6 @@ class NotebookEntity(EntityBase[NotebookEntityModel]):
|
|
|
41
43
|
except ProgrammingError:
|
|
42
44
|
return False
|
|
43
45
|
|
|
44
|
-
def _upload_artifacts(self):
|
|
45
|
-
stage_fqn = self._stage_path.stage_fqn
|
|
46
|
-
stage_manager = StageManager()
|
|
47
|
-
cli_console.step(f"Creating stage {stage_fqn} if not exists")
|
|
48
|
-
stage_manager.create(fqn=stage_fqn)
|
|
49
|
-
|
|
50
|
-
cli_console.step("Uploading artifacts")
|
|
51
|
-
|
|
52
|
-
# creating bundle map to handle glob patterns logic
|
|
53
|
-
bundle_map = bundle_artifacts(self._project_paths, self.model.artifacts)
|
|
54
|
-
for absolute_src, absolute_dest in bundle_map.all_mappings(
|
|
55
|
-
absolute=True, expand_directories=True
|
|
56
|
-
):
|
|
57
|
-
artifact_stage_path = self._stage_path / (
|
|
58
|
-
absolute_dest.relative_to(self._project_paths.bundle_root).parent
|
|
59
|
-
)
|
|
60
|
-
stage_manager.put(
|
|
61
|
-
local_path=absolute_src, stage_path=artifact_stage_path, overwrite=True
|
|
62
|
-
)
|
|
63
|
-
|
|
64
46
|
def get_create_sql(self, replace: bool) -> str:
|
|
65
47
|
main_file_stage_path = self._stage_path / (
|
|
66
48
|
self.model.notebook_file.absolute().relative_to(
|
|
@@ -99,6 +81,7 @@ class NotebookEntity(EntityBase[NotebookEntityModel]):
|
|
|
99
81
|
self,
|
|
100
82
|
action_ctx: ActionContext,
|
|
101
83
|
replace: bool,
|
|
84
|
+
prune: bool,
|
|
102
85
|
*args,
|
|
103
86
|
**kwargs,
|
|
104
87
|
) -> str:
|
|
@@ -108,7 +91,13 @@ class NotebookEntity(EntityBase[NotebookEntityModel]):
|
|
|
108
91
|
f"Notebook {self.fqn.name} already exists. Consider using --replace."
|
|
109
92
|
)
|
|
110
93
|
with cli_console.phase(f"Uploading artifacts to {self._stage_path}"):
|
|
111
|
-
|
|
94
|
+
sync_artifacts_with_stage(
|
|
95
|
+
project_paths=self._project_paths,
|
|
96
|
+
stage_root=self._stage_path_from_model,
|
|
97
|
+
prune=prune,
|
|
98
|
+
artifacts=self.model.artifacts,
|
|
99
|
+
)
|
|
100
|
+
|
|
112
101
|
with cli_console.phase(f"Creating notebook {self.fqn}"):
|
|
113
102
|
return self.action_create(replace=replace)
|
|
114
103
|
|
|
@@ -15,18 +15,20 @@
|
|
|
15
15
|
from typing import List, Optional
|
|
16
16
|
|
|
17
17
|
import typer
|
|
18
|
+
from click import ClickException
|
|
18
19
|
from snowflake.cli._plugins.object.command_aliases import add_object_command_aliases
|
|
19
20
|
from snowflake.cli._plugins.object.commands import scope_option
|
|
21
|
+
from snowflake.cli._plugins.object.manager import ObjectManager
|
|
20
22
|
from snowflake.cli._plugins.project.feature_flags import FeatureFlag
|
|
21
23
|
from snowflake.cli._plugins.project.manager import ProjectManager
|
|
22
24
|
from snowflake.cli._plugins.project.project_entity_model import (
|
|
23
25
|
ProjectEntityModel,
|
|
24
26
|
)
|
|
25
|
-
from snowflake.cli._plugins.stage.manager import StageManager
|
|
26
|
-
from snowflake.cli.api.artifacts.upload import put_files
|
|
27
27
|
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
28
28
|
from snowflake.cli.api.commands.decorators import with_project_definition
|
|
29
29
|
from snowflake.cli.api.commands.flags import (
|
|
30
|
+
OverrideableOption,
|
|
31
|
+
PruneOption,
|
|
30
32
|
entity_argument,
|
|
31
33
|
identifier_argument,
|
|
32
34
|
like_option,
|
|
@@ -37,8 +39,7 @@ from snowflake.cli.api.commands.utils import get_entity_for_operation
|
|
|
37
39
|
from snowflake.cli.api.console.console import cli_console
|
|
38
40
|
from snowflake.cli.api.constants import ObjectType
|
|
39
41
|
from snowflake.cli.api.identifiers import FQN
|
|
40
|
-
from snowflake.cli.api.output.types import MessageResult, SingleQueryResult
|
|
41
|
-
from snowflake.cli.api.project.project_paths import ProjectPaths
|
|
42
|
+
from snowflake.cli.api.output.types import MessageResult, QueryResult, SingleQueryResult
|
|
42
43
|
|
|
43
44
|
app = SnowTyperFactory(
|
|
44
45
|
name="project",
|
|
@@ -48,11 +49,20 @@ app = SnowTyperFactory(
|
|
|
48
49
|
|
|
49
50
|
project_identifier = identifier_argument(sf_object="project", example="MY_PROJECT")
|
|
50
51
|
version_flag = typer.Option(
|
|
51
|
-
|
|
52
|
+
None,
|
|
53
|
+
"--version",
|
|
54
|
+
help="Version of the project to use. If not specified default version is used",
|
|
55
|
+
show_default=False,
|
|
52
56
|
)
|
|
53
57
|
variables_flag = variables_option(
|
|
54
58
|
'Variables for the execution context; for example: `-D "<key>=<value>"`.'
|
|
55
59
|
)
|
|
60
|
+
from_option = OverrideableOption(
|
|
61
|
+
None,
|
|
62
|
+
"--from",
|
|
63
|
+
help="Create a new version using given stage instead of uploading local files.",
|
|
64
|
+
show_default=False,
|
|
65
|
+
)
|
|
56
66
|
|
|
57
67
|
|
|
58
68
|
add_object_command_aliases(
|
|
@@ -70,7 +80,7 @@ add_object_command_aliases(
|
|
|
70
80
|
@app.command(requires_connection=True)
|
|
71
81
|
def execute(
|
|
72
82
|
identifier: FQN = project_identifier,
|
|
73
|
-
version: str = version_flag,
|
|
83
|
+
version: Optional[str] = version_flag,
|
|
74
84
|
variables: Optional[List[str]] = variables_flag,
|
|
75
85
|
**options,
|
|
76
86
|
):
|
|
@@ -86,7 +96,7 @@ def execute(
|
|
|
86
96
|
@app.command(requires_connection=True)
|
|
87
97
|
def dry_run(
|
|
88
98
|
identifier: FQN = project_identifier,
|
|
89
|
-
version: str = version_flag,
|
|
99
|
+
version: Optional[str] = version_flag,
|
|
90
100
|
variables: Optional[List[str]] = variables_flag,
|
|
91
101
|
**options,
|
|
92
102
|
):
|
|
@@ -101,12 +111,18 @@ def dry_run(
|
|
|
101
111
|
|
|
102
112
|
@app.command(requires_connection=True)
|
|
103
113
|
@with_project_definition()
|
|
104
|
-
def
|
|
114
|
+
def create(
|
|
105
115
|
entity_id: str = entity_argument("project"),
|
|
116
|
+
no_version: bool = typer.Option(
|
|
117
|
+
False,
|
|
118
|
+
"--no-version",
|
|
119
|
+
help="Do not initialize project with a new version, only create the snowflake object.",
|
|
120
|
+
),
|
|
106
121
|
**options,
|
|
107
122
|
):
|
|
108
123
|
"""
|
|
109
|
-
|
|
124
|
+
Creates a project in Snowflake.
|
|
125
|
+
By default, the project is initialized with a new version created from local files.
|
|
110
126
|
"""
|
|
111
127
|
cli_context = get_cli_context()
|
|
112
128
|
project: ProjectEntityModel = get_entity_for_operation(
|
|
@@ -115,59 +131,68 @@ def create_version(
|
|
|
115
131
|
project_definition=cli_context.project_definition,
|
|
116
132
|
entity_type="project",
|
|
117
133
|
)
|
|
134
|
+
om = ObjectManager()
|
|
135
|
+
if om.object_exists(object_type="project", fqn=project.fqn):
|
|
136
|
+
raise ClickException(f"Project '{project.fqn}' already exists.")
|
|
137
|
+
if not no_version and om.object_exists(
|
|
138
|
+
object_type="stage", fqn=FQN.from_stage(project.stage)
|
|
139
|
+
):
|
|
140
|
+
raise ClickException(f"Stage '{project.stage}' already exists.")
|
|
118
141
|
|
|
119
|
-
|
|
120
|
-
with cli_console.phase("
|
|
121
|
-
|
|
122
|
-
sm = StageManager()
|
|
123
|
-
|
|
124
|
-
cli_console.step(f"Creating stage {stage_name}")
|
|
125
|
-
sm.create(fqn=stage_name)
|
|
126
|
-
|
|
127
|
-
put_files(
|
|
128
|
-
project_paths=ProjectPaths(project_root=cli_context.project_root),
|
|
129
|
-
stage_root=project.stage,
|
|
130
|
-
artifacts=project.artifacts,
|
|
131
|
-
)
|
|
132
|
-
|
|
133
|
-
# Create project and version
|
|
134
|
-
with cli_console.phase("Creating project and version"):
|
|
135
|
-
pm = ProjectManager()
|
|
136
|
-
cli_console.step(f"Creating project {project.fqn}")
|
|
137
|
-
pm.create(project_name=project.fqn)
|
|
142
|
+
pm = ProjectManager()
|
|
143
|
+
with cli_console.phase(f"Creating project '{project.fqn}'"):
|
|
144
|
+
pm.create(project=project, initialize_version_from_local_files=not no_version)
|
|
138
145
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
return MessageResult(
|
|
146
|
+
if no_version:
|
|
147
|
+
return MessageResult(f"Project '{project.fqn}' successfully created.")
|
|
148
|
+
return MessageResult(
|
|
149
|
+
f"Project '{project.fqn}' successfully created and initial version is added."
|
|
150
|
+
)
|
|
142
151
|
|
|
143
152
|
|
|
144
153
|
@app.command(requires_connection=True)
|
|
145
154
|
@with_project_definition()
|
|
146
155
|
def add_version(
|
|
147
156
|
entity_id: str = entity_argument("project"),
|
|
148
|
-
_from: str =
|
|
149
|
-
|
|
150
|
-
"--from",
|
|
151
|
-
help="Source stage to create the version from.",
|
|
152
|
-
show_default=False,
|
|
153
|
-
),
|
|
154
|
-
alias: str
|
|
155
|
-
| None = typer.Option(
|
|
157
|
+
_from: Optional[str] = from_option(mutually_exclusive=["prune"]),
|
|
158
|
+
_alias: Optional[str] = typer.Option(
|
|
156
159
|
None, "--alias", help="Alias for the version.", show_default=False
|
|
157
160
|
),
|
|
158
|
-
comment: str
|
|
159
|
-
| None = typer.Option(
|
|
161
|
+
comment: Optional[str] = typer.Option(
|
|
160
162
|
None, "--comment", help="Version comment.", show_default=False
|
|
161
163
|
),
|
|
164
|
+
prune: bool = PruneOption(mutually_exclusive=["_from"]),
|
|
162
165
|
**options,
|
|
163
166
|
):
|
|
164
|
-
"""
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
"""Uploads local files to Snowflake and cerates a new project version."""
|
|
168
|
+
cli_context = get_cli_context()
|
|
169
|
+
project: ProjectEntityModel = get_entity_for_operation(
|
|
170
|
+
cli_context=cli_context,
|
|
171
|
+
entity_id=entity_id,
|
|
172
|
+
project_definition=cli_context.project_definition,
|
|
173
|
+
entity_type="project",
|
|
174
|
+
)
|
|
175
|
+
ProjectManager().add_version(
|
|
176
|
+
project=project,
|
|
177
|
+
prune=prune,
|
|
169
178
|
from_stage=_from,
|
|
170
|
-
alias=
|
|
179
|
+
alias=_alias,
|
|
171
180
|
comment=comment,
|
|
172
181
|
)
|
|
173
|
-
|
|
182
|
+
alias_str = "" if _alias is None else f"'{_alias}' "
|
|
183
|
+
return MessageResult(
|
|
184
|
+
f"New project version {alias_str}added to project '{project.fqn}'"
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@app.command(requires_connection=True)
|
|
189
|
+
def list_versions(
|
|
190
|
+
identifier: FQN = project_identifier,
|
|
191
|
+
**options,
|
|
192
|
+
):
|
|
193
|
+
"""
|
|
194
|
+
Lists versions of given project.
|
|
195
|
+
"""
|
|
196
|
+
pm = ProjectManager()
|
|
197
|
+
results = pm.list_versions(project_name=identifier)
|
|
198
|
+
return QueryResult(results)
|
|
@@ -12,13 +12,19 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
from textwrap import dedent
|
|
15
|
-
from typing import List
|
|
15
|
+
from typing import List, Optional
|
|
16
16
|
|
|
17
|
+
from snowflake.cli._plugins.project.project_entity_model import ProjectEntityModel
|
|
17
18
|
from snowflake.cli._plugins.stage.manager import StageManager
|
|
19
|
+
from snowflake.cli.api.artifacts.upload import sync_artifacts_with_stage
|
|
20
|
+
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
18
21
|
from snowflake.cli.api.commands.utils import parse_key_value_variables
|
|
22
|
+
from snowflake.cli.api.console.console import cli_console
|
|
19
23
|
from snowflake.cli.api.identifiers import FQN
|
|
24
|
+
from snowflake.cli.api.project.project_paths import ProjectPaths
|
|
20
25
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin
|
|
21
26
|
from snowflake.cli.api.stage_path import StagePath
|
|
27
|
+
from snowflake.connector.cursor import SnowflakeCursor
|
|
22
28
|
|
|
23
29
|
|
|
24
30
|
class ProjectManager(SqlExecutionMixin):
|
|
@@ -40,37 +46,65 @@ class ProjectManager(SqlExecutionMixin):
|
|
|
40
46
|
query += " DRY_RUN=TRUE"
|
|
41
47
|
return self.execute_query(query=query)
|
|
42
48
|
|
|
43
|
-
def
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
) -> str:
|
|
47
|
-
queries = dedent(f"CREATE PROJECT IF NOT EXISTS {project_name.sql_identifier}")
|
|
48
|
-
return self.execute_query(query=queries)
|
|
49
|
+
def _create_object(self, project_name: FQN) -> SnowflakeCursor:
|
|
50
|
+
query = dedent(f"CREATE PROJECT {project_name.sql_identifier}")
|
|
51
|
+
return self.execute_query(query)
|
|
49
52
|
|
|
50
|
-
def
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
)
|
|
56
|
-
)
|
|
53
|
+
def create(
|
|
54
|
+
self, project: ProjectEntityModel, initialize_version_from_local_files: bool
|
|
55
|
+
) -> None:
|
|
56
|
+
self._create_object(project.fqn)
|
|
57
|
+
if initialize_version_from_local_files:
|
|
58
|
+
self.add_version(project=project)
|
|
57
59
|
|
|
58
|
-
def
|
|
60
|
+
def _create_version(
|
|
59
61
|
self,
|
|
60
|
-
project_name:
|
|
62
|
+
project_name: FQN,
|
|
61
63
|
from_stage: str,
|
|
62
64
|
alias: str | None = None,
|
|
63
65
|
comment: str | None = None,
|
|
64
66
|
):
|
|
65
|
-
|
|
66
|
-
project_name
|
|
67
|
-
if isinstance(project_name, FQN)
|
|
68
|
-
else FQN.from_string(project_name)
|
|
69
|
-
)
|
|
67
|
+
stage_path = StagePath.from_stage_str(from_stage)
|
|
70
68
|
query = f"ALTER PROJECT {project_name.identifier} ADD VERSION"
|
|
71
69
|
if alias:
|
|
72
|
-
query += f
|
|
73
|
-
query += f" FROM {
|
|
70
|
+
query += f' IF NOT EXISTS "{alias}"'
|
|
71
|
+
query += f" FROM {stage_path.absolute_path(at_prefix=True)}"
|
|
74
72
|
if comment:
|
|
75
73
|
query += f" COMMENT = '{comment}'"
|
|
76
74
|
return self.execute_query(query=query)
|
|
75
|
+
|
|
76
|
+
def add_version(
|
|
77
|
+
self,
|
|
78
|
+
project: ProjectEntityModel,
|
|
79
|
+
prune: bool = False,
|
|
80
|
+
from_stage: Optional[str] = None,
|
|
81
|
+
alias: Optional[str] = None,
|
|
82
|
+
comment: Optional[str] = None,
|
|
83
|
+
):
|
|
84
|
+
"""
|
|
85
|
+
Adds a version to project. If [from_stage] is not defined,
|
|
86
|
+
uploads local files to the stage defined in project definition.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
if not from_stage:
|
|
90
|
+
cli_context = get_cli_context()
|
|
91
|
+
from_stage = project.stage
|
|
92
|
+
with cli_console.phase("Uploading artifacts"):
|
|
93
|
+
sync_artifacts_with_stage(
|
|
94
|
+
project_paths=ProjectPaths(project_root=cli_context.project_root),
|
|
95
|
+
stage_root=from_stage,
|
|
96
|
+
artifacts=project.artifacts,
|
|
97
|
+
prune=prune,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
with cli_console.phase(f"Creating project version from stage {from_stage}"):
|
|
101
|
+
return self._create_version(
|
|
102
|
+
project_name=project.fqn,
|
|
103
|
+
from_stage=from_stage, # type:ignore
|
|
104
|
+
alias=alias,
|
|
105
|
+
comment=comment,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def list_versions(self, project_name: FQN):
|
|
109
|
+
query = f"SHOW VERSIONS IN PROJECT {project_name.identifier}"
|
|
110
|
+
return self.execute_query(query=query)
|
|
@@ -13,26 +13,45 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
-
from typing import Literal, Optional, TypeVar
|
|
16
|
+
from typing import List, Literal, Optional, TypeVar
|
|
17
17
|
|
|
18
|
-
from pydantic import Field
|
|
18
|
+
from pydantic import Field, field_validator
|
|
19
|
+
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
19
20
|
from snowflake.cli.api.entities.common import EntityBase, attach_spans_to_entity_actions
|
|
21
|
+
from snowflake.cli.api.exceptions import CliError
|
|
20
22
|
from snowflake.cli.api.project.schemas.entities.common import (
|
|
23
|
+
Artifacts,
|
|
21
24
|
EntityModelBaseWithArtifacts,
|
|
25
|
+
PathMapping,
|
|
22
26
|
)
|
|
23
27
|
from snowflake.cli.api.project.schemas.updatable_model import (
|
|
24
28
|
DiscriminatorField,
|
|
25
29
|
)
|
|
30
|
+
from snowflake.cli.api.secure_path import SecurePath
|
|
26
31
|
|
|
27
32
|
T = TypeVar("T")
|
|
28
33
|
|
|
29
34
|
|
|
35
|
+
MANIFEST_FILE_NAME = "manifest.yml"
|
|
36
|
+
|
|
37
|
+
|
|
30
38
|
class ProjectEntityModel(EntityModelBaseWithArtifacts):
|
|
31
39
|
type: Literal["project"] = DiscriminatorField() # noqa: A003
|
|
32
40
|
stage: Optional[str] = Field(
|
|
33
41
|
title="Stage in which the project artifacts will be stored", default=None
|
|
34
42
|
)
|
|
35
|
-
|
|
43
|
+
|
|
44
|
+
@field_validator("artifacts")
|
|
45
|
+
@classmethod
|
|
46
|
+
def transform_artifacts(cls, orig_artifacts: Artifacts) -> List[PathMapping]:
|
|
47
|
+
if not (
|
|
48
|
+
SecurePath(get_cli_context().project_root) / MANIFEST_FILE_NAME
|
|
49
|
+
).exists():
|
|
50
|
+
raise CliError(
|
|
51
|
+
f"{MANIFEST_FILE_NAME} was not found in project root directory"
|
|
52
|
+
)
|
|
53
|
+
orig_artifacts.append(PathMapping(src=MANIFEST_FILE_NAME))
|
|
54
|
+
return super().transform_artifacts(orig_artifacts)
|
|
36
55
|
|
|
37
56
|
|
|
38
57
|
@attach_spans_to_entity_actions(entity_name="project")
|
|
@@ -71,6 +71,7 @@ from snowflake.cli.api.commands.decorators import (
|
|
|
71
71
|
)
|
|
72
72
|
from snowflake.cli.api.commands.flags import (
|
|
73
73
|
ForceReplaceOption,
|
|
74
|
+
PruneOption,
|
|
74
75
|
ReplaceOption,
|
|
75
76
|
execution_identifier_argument,
|
|
76
77
|
identifier_argument,
|
|
@@ -133,6 +134,9 @@ def deploy(
|
|
|
133
134
|
"overwrites existing files, but does not remove any files already on the stage."
|
|
134
135
|
),
|
|
135
136
|
force_replace: bool = ForceReplaceOption(),
|
|
137
|
+
prune: bool = PruneOption(
|
|
138
|
+
help="Remove contents of the stage before uploading artifacts."
|
|
139
|
+
),
|
|
136
140
|
**options,
|
|
137
141
|
) -> CommandResult:
|
|
138
142
|
"""
|
|
@@ -174,7 +178,7 @@ def deploy(
|
|
|
174
178
|
snowpark_entities=snowpark_entities,
|
|
175
179
|
)
|
|
176
180
|
|
|
177
|
-
create_stages_and_upload_artifacts(stages_to_artifact_map)
|
|
181
|
+
create_stages_and_upload_artifacts(stages_to_artifact_map, prune=prune)
|
|
178
182
|
|
|
179
183
|
# Create snowpark entities
|
|
180
184
|
with cli_console.phase("Creating Snowpark entities"):
|
|
@@ -242,8 +246,17 @@ def build_artifacts_mappings(
|
|
|
242
246
|
return entities_to_imports_map, stages_to_artifact_map
|
|
243
247
|
|
|
244
248
|
|
|
245
|
-
def create_stages_and_upload_artifacts(
|
|
249
|
+
def create_stages_and_upload_artifacts(
|
|
250
|
+
stages_to_artifact_map: StageToArtifactMapping, prune: bool
|
|
251
|
+
):
|
|
246
252
|
stage_manager = StageManager()
|
|
253
|
+
if prune:
|
|
254
|
+
# snowflake.cli._plugins.snowpark.snowpark_project_paths.Artifact class assumes that "stage"
|
|
255
|
+
# is a stage object, not path on stage - whole stage is managed by snowpark - it can be removed
|
|
256
|
+
for stage in stages_to_artifact_map.keys():
|
|
257
|
+
cli_console.step(f"Removing contents of stage {stage}")
|
|
258
|
+
stage_manager.remove(stage, path="")
|
|
259
|
+
|
|
247
260
|
for stage, artifacts in stages_to_artifact_map.items():
|
|
248
261
|
cli_console.step(f"Creating (if not exists) stage: {stage}")
|
|
249
262
|
stage = FQN.from_stage(stage).using_context()
|
|
@@ -74,12 +74,21 @@ class RegistryManager(SqlExecutionMixin):
|
|
|
74
74
|
return re.fullmatch(r"^.*//.+", url) is not None
|
|
75
75
|
|
|
76
76
|
def get_registry_url(self) -> str:
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
77
|
+
images_query = "show image repositories in schema snowflake.images;"
|
|
78
|
+
images_result = self.execute_query(images_query, cursor_class=DictCursor)
|
|
79
|
+
|
|
80
|
+
results = images_result.fetchone()
|
|
81
|
+
|
|
82
|
+
if not results:
|
|
83
|
+
# fallback to account level query - slower one, so we try to avoid it if possible
|
|
84
|
+
repositories_query = "show image repositories in account"
|
|
85
|
+
result_set = self.execute_query(repositories_query, cursor_class=DictCursor)
|
|
86
|
+
results = result_set.fetchone()
|
|
87
|
+
|
|
88
|
+
if not results:
|
|
89
|
+
raise NoImageRepositoriesFoundError()
|
|
90
|
+
|
|
91
|
+
sample_repository_url = results["repository_url"]
|
|
83
92
|
if not self._has_url_scheme(sample_repository_url):
|
|
84
93
|
sample_repository_url = f"//{sample_repository_url}"
|
|
85
94
|
return urlparse(sample_repository_url).netloc
|
|
@@ -20,7 +20,6 @@ from functools import partial
|
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
from typing import Dict, Iterable, List, Tuple
|
|
22
22
|
|
|
23
|
-
from click import ClickException, UsageError
|
|
24
23
|
from snowflake.cli._plugins.sql.snowsql_templating import transpile_snowsql_templates
|
|
25
24
|
from snowflake.cli._plugins.sql.source_reader import (
|
|
26
25
|
compile_statements,
|
|
@@ -28,6 +27,7 @@ from snowflake.cli._plugins.sql.source_reader import (
|
|
|
28
27
|
query_reader,
|
|
29
28
|
)
|
|
30
29
|
from snowflake.cli.api.console import cli_console
|
|
30
|
+
from snowflake.cli.api.exceptions import CliArgumentError, CliSqlError
|
|
31
31
|
from snowflake.cli.api.rendering.sql_templates import snowflake_sql_jinja_render
|
|
32
32
|
from snowflake.cli.api.secure_path import SecurePath
|
|
33
33
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin, VerboseCursor
|
|
@@ -69,17 +69,17 @@ class SqlManager(SqlExecutionMixin):
|
|
|
69
69
|
secured_files = [SecurePath(f) for f in files]
|
|
70
70
|
stmt_reader = files_reader(secured_files, stmt_operators, remove_comments)
|
|
71
71
|
else:
|
|
72
|
-
raise
|
|
72
|
+
raise CliArgumentError("Use either query, filename or input option.")
|
|
73
73
|
|
|
74
74
|
errors, stmt_count, compiled_statements = compile_statements(stmt_reader)
|
|
75
75
|
if not any((errors, stmt_count, compiled_statements)):
|
|
76
|
-
raise
|
|
76
|
+
raise CliArgumentError("Use either query, filename or input option.")
|
|
77
77
|
|
|
78
78
|
if errors:
|
|
79
79
|
for error in errors:
|
|
80
80
|
logger.info("Statement compilation error: %s", error)
|
|
81
81
|
cli_console.warning(error)
|
|
82
|
-
raise
|
|
82
|
+
raise CliSqlError("SQL rendering error")
|
|
83
83
|
|
|
84
84
|
is_single_statement = not (stmt_count > 1)
|
|
85
85
|
return is_single_statement, self.execute_string(
|
|
@@ -65,7 +65,7 @@ EXECUTE_SUPPORTED_FILES_FORMATS = (
|
|
|
65
65
|
|
|
66
66
|
# Replace magic numbers with constants
|
|
67
67
|
OMIT_FIRST = slice(1, None)
|
|
68
|
-
STAGE_PATH_REGEX = rf"(?P<prefix
|
|
68
|
+
STAGE_PATH_REGEX = rf"(?P<prefix>(@|{re.escape('snow://')}))?(?:(?P<first_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?:(?P<second_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?P<name>{VALID_IDENTIFIER_REGEX})/?(?P<directory>([^/]*/?)*)?"
|
|
69
69
|
|
|
70
70
|
|
|
71
71
|
@dataclass
|
|
@@ -119,6 +119,14 @@ class StagePathParts:
|
|
|
119
119
|
raise NotImplementedError
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
def _strip_standard_stage_prefix(path: str) -> str:
|
|
123
|
+
"""Removes '@' or 'snow://' prefix from given string"""
|
|
124
|
+
for prefix in ["@", "snow://"]:
|
|
125
|
+
if path.startswith(prefix):
|
|
126
|
+
path = path.removeprefix(prefix)
|
|
127
|
+
return path
|
|
128
|
+
|
|
129
|
+
|
|
122
130
|
@dataclass
|
|
123
131
|
class DefaultStagePathParts(StagePathParts):
|
|
124
132
|
"""
|
|
@@ -126,8 +134,8 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
126
134
|
directory = dir
|
|
127
135
|
stage = @db.schema.stage
|
|
128
136
|
stage_name = stage
|
|
129
|
-
For
|
|
130
|
-
stage ->
|
|
137
|
+
For `snow://stage/dir` to
|
|
138
|
+
stage -> snow://stage
|
|
131
139
|
stage_name -> stage
|
|
132
140
|
directory -> dir
|
|
133
141
|
"""
|
|
@@ -138,12 +146,12 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
138
146
|
raise ClickException("Invalid stage path")
|
|
139
147
|
self.directory = match.group("directory")
|
|
140
148
|
self._schema = match.group("second_qualifier") or match.group("first_qualifier")
|
|
149
|
+
self._prefix = match.group("prefix") or "@"
|
|
141
150
|
self.stage = stage_path.removesuffix(self.directory).rstrip("/")
|
|
142
151
|
|
|
143
152
|
stage_name = FQN.from_stage(self.stage).name
|
|
144
|
-
stage_name
|
|
145
|
-
stage_name
|
|
146
|
-
)
|
|
153
|
+
if stage_name.startswith(self._prefix):
|
|
154
|
+
stage_name = stage_name.removeprefix(self._prefix)
|
|
147
155
|
self.stage_name = stage_name
|
|
148
156
|
self.is_directory = True if stage_path.endswith("/") else False
|
|
149
157
|
|
|
@@ -167,13 +175,12 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
167
175
|
return self._schema
|
|
168
176
|
|
|
169
177
|
def replace_stage_prefix(self, file_path: str) -> str:
|
|
170
|
-
|
|
178
|
+
file_path = _strip_standard_stage_prefix(file_path)
|
|
171
179
|
file_path_without_prefix = Path(file_path).parts[OMIT_FIRST]
|
|
172
|
-
return f"{stage}/{'/'.join(file_path_without_prefix)}"
|
|
180
|
+
return f"{self.stage}/{'/'.join(file_path_without_prefix)}"
|
|
173
181
|
|
|
174
182
|
def strip_stage_prefix(self, file_path: str) -> str:
|
|
175
|
-
|
|
176
|
-
file_path = file_path[OMIT_FIRST]
|
|
183
|
+
file_path = _strip_standard_stage_prefix(file_path)
|
|
177
184
|
if file_path.startswith(self.stage_name):
|
|
178
185
|
return file_path[len(self.stage_name) :]
|
|
179
186
|
return file_path
|
|
@@ -37,6 +37,7 @@ from snowflake.cli.api.commands.decorators import (
|
|
|
37
37
|
with_project_definition,
|
|
38
38
|
)
|
|
39
39
|
from snowflake.cli.api.commands.flags import (
|
|
40
|
+
PruneOption,
|
|
40
41
|
ReplaceOption,
|
|
41
42
|
entity_argument,
|
|
42
43
|
identifier_argument,
|
|
@@ -136,6 +137,7 @@ def streamlit_deploy(
|
|
|
136
137
|
help="Replaces the Streamlit app if it already exists. It only uploads new and overwrites existing files, "
|
|
137
138
|
"but does not remove any files already on the stage."
|
|
138
139
|
),
|
|
140
|
+
prune: bool = PruneOption(),
|
|
139
141
|
entity_id: str = entity_argument("streamlit"),
|
|
140
142
|
open_: bool = OpenOption,
|
|
141
143
|
**options,
|
|
@@ -168,6 +170,7 @@ def streamlit_deploy(
|
|
|
168
170
|
streamlit=streamlit,
|
|
169
171
|
streamlit_project_paths=streamlit_project_paths,
|
|
170
172
|
replace=replace,
|
|
173
|
+
prune=prune,
|
|
171
174
|
)
|
|
172
175
|
|
|
173
176
|
if open_:
|