snowflake-cli 3.2.2__py3-none-any.whl → 3.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli/__about__.py +1 -1
- snowflake/cli/_app/__main__.py +2 -2
- snowflake/cli/_app/cli_app.py +224 -192
- snowflake/cli/_app/commands_registration/commands_registration_with_callbacks.py +1 -27
- snowflake/cli/_app/constants.py +4 -0
- snowflake/cli/_app/snow_connector.py +12 -0
- snowflake/cli/_app/telemetry.py +10 -3
- snowflake/cli/_plugins/connection/util.py +12 -19
- snowflake/cli/_plugins/cortex/commands.py +2 -4
- snowflake/cli/_plugins/git/manager.py +1 -1
- snowflake/cli/_plugins/helpers/commands.py +207 -1
- snowflake/cli/_plugins/nativeapp/artifacts.py +16 -628
- snowflake/cli/_plugins/nativeapp/bundle_context.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/compiler.py +42 -20
- snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +9 -2
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +6 -3
- snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +44 -34
- snowflake/cli/_plugins/nativeapp/commands.py +113 -21
- snowflake/cli/_plugins/nativeapp/constants.py +5 -0
- snowflake/cli/_plugins/nativeapp/entities/application.py +226 -296
- snowflake/cli/_plugins/nativeapp/entities/application_package.py +911 -141
- snowflake/cli/_plugins/nativeapp/entities/application_package_child_interface.py +43 -0
- snowflake/cli/_plugins/nativeapp/feature_flags.py +5 -1
- snowflake/cli/_plugins/nativeapp/release_channel/__init__.py +13 -0
- snowflake/cli/_plugins/nativeapp/release_channel/commands.py +246 -0
- snowflake/cli/_plugins/nativeapp/release_directive/__init__.py +13 -0
- snowflake/cli/_plugins/nativeapp/release_directive/commands.py +243 -0
- snowflake/cli/_plugins/nativeapp/same_account_install_method.py +9 -17
- snowflake/cli/_plugins/nativeapp/sf_facade_exceptions.py +80 -0
- snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +1184 -80
- snowflake/cli/_plugins/nativeapp/utils.py +11 -0
- snowflake/cli/_plugins/nativeapp/v2_conversions/compat.py +7 -3
- snowflake/cli/_plugins/nativeapp/version/commands.py +32 -5
- snowflake/cli/_plugins/notebook/commands.py +55 -2
- snowflake/cli/_plugins/notebook/exceptions.py +1 -1
- snowflake/cli/_plugins/notebook/manager.py +7 -5
- snowflake/cli/_plugins/notebook/notebook_entity.py +120 -0
- snowflake/cli/_plugins/notebook/notebook_entity_model.py +42 -0
- snowflake/cli/_plugins/notebook/notebook_project_paths.py +15 -0
- snowflake/cli/_plugins/notebook/types.py +3 -0
- snowflake/cli/_plugins/snowpark/commands.py +48 -30
- snowflake/cli/_plugins/snowpark/common.py +47 -2
- snowflake/cli/_plugins/snowpark/snowpark_entity.py +247 -4
- snowflake/cli/_plugins/snowpark/snowpark_entity_model.py +18 -30
- snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +156 -23
- snowflake/cli/_plugins/snowpark/zipper.py +33 -1
- snowflake/cli/_plugins/spcs/common.py +129 -0
- snowflake/cli/_plugins/spcs/services/commands.py +131 -14
- snowflake/cli/_plugins/spcs/services/manager.py +169 -1
- snowflake/cli/_plugins/stage/commands.py +2 -1
- snowflake/cli/_plugins/stage/diff.py +60 -39
- snowflake/cli/_plugins/stage/manager.py +34 -13
- snowflake/cli/_plugins/stage/utils.py +1 -1
- snowflake/cli/_plugins/streamlit/commands.py +10 -1
- snowflake/cli/_plugins/streamlit/manager.py +70 -22
- snowflake/cli/_plugins/streamlit/streamlit_entity.py +131 -1
- snowflake/cli/_plugins/streamlit/streamlit_entity_model.py +14 -24
- snowflake/cli/_plugins/streamlit/streamlit_project_paths.py +30 -0
- snowflake/cli/_plugins/workspace/commands.py +6 -5
- snowflake/cli/_plugins/workspace/manager.py +9 -5
- snowflake/cli/api/artifacts/__init__.py +13 -0
- snowflake/cli/api/artifacts/bundle_map.py +500 -0
- snowflake/cli/api/artifacts/common.py +78 -0
- snowflake/cli/api/artifacts/utils.py +82 -0
- snowflake/cli/api/cli_global_context.py +36 -2
- snowflake/cli/api/commands/flags.py +10 -4
- snowflake/cli/api/commands/utils.py +28 -2
- snowflake/cli/api/config.py +6 -2
- snowflake/cli/api/connections.py +12 -1
- snowflake/cli/api/constants.py +10 -1
- snowflake/cli/api/entities/common.py +81 -14
- snowflake/cli/api/entities/resolver.py +160 -0
- snowflake/cli/api/entities/utils.py +65 -23
- snowflake/cli/api/errno.py +63 -3
- snowflake/cli/api/feature_flags.py +19 -4
- snowflake/cli/api/metrics.py +21 -27
- snowflake/cli/api/project/definition_conversion.py +4 -4
- snowflake/cli/api/project/project_paths.py +28 -0
- snowflake/cli/api/project/schemas/entities/common.py +130 -1
- snowflake/cli/api/project/schemas/entities/entities.py +4 -0
- snowflake/cli/api/project/schemas/project_definition.py +54 -6
- snowflake/cli/api/project/schemas/updatable_model.py +2 -2
- snowflake/cli/api/project/schemas/v1/native_app/native_app.py +5 -7
- snowflake/cli/api/project/schemas/v1/streamlit/streamlit.py +1 -1
- snowflake/cli/api/project/util.py +45 -0
- snowflake/cli/api/secure_path.py +6 -0
- snowflake/cli/api/sql_execution.py +5 -1
- snowflake/cli/api/stage_path.py +7 -2
- snowflake/cli/api/utils/graph.py +3 -0
- snowflake/cli/api/utils/path_utils.py +24 -0
- {snowflake_cli-3.2.2.dist-info → snowflake_cli-3.4.1.dist-info}/METADATA +14 -15
- {snowflake_cli-3.2.2.dist-info → snowflake_cli-3.4.1.dist-info}/RECORD +96 -82
- {snowflake_cli-3.2.2.dist-info → snowflake_cli-3.4.1.dist-info}/WHEEL +1 -1
- snowflake/cli/api/project/schemas/v1/native_app/path_mapping.py +0 -65
- {snowflake_cli-3.2.2.dist-info → snowflake_cli-3.4.1.dist-info}/entry_points.txt +0 -0
- {snowflake_cli-3.2.2.dist-info → snowflake_cli-3.4.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -96,3 +96,14 @@ def verify_no_directories(paths_to_sync: Iterable[Path]):
|
|
|
96
96
|
def verify_exists(path: Path):
|
|
97
97
|
if not path.exists():
|
|
98
98
|
raise ClickException(f"The following path does not exist: {path}")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def sanitize_dir_name(dir_name: str) -> str:
|
|
102
|
+
"""
|
|
103
|
+
Returns a string that is safe to use as a directory name.
|
|
104
|
+
For simplicity, this function is over restricitive: it strips non alphanumeric characters,
|
|
105
|
+
unless listed in the allow list. Additional characters can be allowed in the future, but
|
|
106
|
+
we need to be careful to consider both Unix/Windows directory naming rules.
|
|
107
|
+
"""
|
|
108
|
+
allowed_chars = [" ", "_"]
|
|
109
|
+
return "".join(char for char in dir_name if char in allowed_chars or char.isalnum())
|
|
@@ -46,7 +46,7 @@ APP_AND_PACKAGE_OPTIONS = [
|
|
|
46
46
|
annotation=Optional[str],
|
|
47
47
|
default=typer.Option(
|
|
48
48
|
default="",
|
|
49
|
-
help="The ID of the package entity on which to operate when definition_version is 2 or higher.",
|
|
49
|
+
help="The ID of the package entity on which to operate when the definition_version is 2 or higher.",
|
|
50
50
|
),
|
|
51
51
|
),
|
|
52
52
|
inspect.Parameter(
|
|
@@ -55,7 +55,7 @@ APP_AND_PACKAGE_OPTIONS = [
|
|
|
55
55
|
annotation=Optional[str],
|
|
56
56
|
default=typer.Option(
|
|
57
57
|
default="",
|
|
58
|
-
help="The ID of the application entity on which to operate when definition_version is 2 or higher.",
|
|
58
|
+
help="The ID of the application entity on which to operate when the definition_version is 2 or higher.",
|
|
59
59
|
),
|
|
60
60
|
),
|
|
61
61
|
]
|
|
@@ -217,7 +217,11 @@ def force_project_definition_v2(
|
|
|
217
217
|
entities_to_keep.add(app_definition.entity_id)
|
|
218
218
|
kwargs["app_entity_id"] = app_definition.entity_id
|
|
219
219
|
for entity_id in list(original_pdf.entities):
|
|
220
|
-
|
|
220
|
+
entity_type = original_pdf.entities[entity_id].type.lower()
|
|
221
|
+
if (
|
|
222
|
+
entity_type in ["application", "application package"]
|
|
223
|
+
and entity_id not in entities_to_keep
|
|
224
|
+
):
|
|
221
225
|
# This happens after templates are rendered,
|
|
222
226
|
# so we can safely remove the entity
|
|
223
227
|
del original_pdf.entities[entity_id]
|
|
@@ -18,6 +18,7 @@ import logging
|
|
|
18
18
|
from typing import Optional
|
|
19
19
|
|
|
20
20
|
import typer
|
|
21
|
+
from snowflake.cli._plugins.nativeapp.artifacts import VersionInfo
|
|
21
22
|
from snowflake.cli._plugins.nativeapp.common_flags import ForceOption, InteractiveOption
|
|
22
23
|
from snowflake.cli._plugins.nativeapp.v2_conversions.compat import (
|
|
23
24
|
force_project_definition_v2,
|
|
@@ -28,8 +29,15 @@ from snowflake.cli.api.commands.decorators import (
|
|
|
28
29
|
with_project_definition,
|
|
29
30
|
)
|
|
30
31
|
from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
|
|
31
|
-
from snowflake.cli.api.entities.
|
|
32
|
-
from snowflake.cli.api.output.
|
|
32
|
+
from snowflake.cli.api.entities.utils import EntityActions
|
|
33
|
+
from snowflake.cli.api.output.formats import OutputFormat
|
|
34
|
+
from snowflake.cli.api.output.types import (
|
|
35
|
+
CollectionResult,
|
|
36
|
+
CommandResult,
|
|
37
|
+
MessageResult,
|
|
38
|
+
ObjectResult,
|
|
39
|
+
)
|
|
40
|
+
from snowflake.cli.api.project.util import to_identifier
|
|
33
41
|
|
|
34
42
|
app = SnowTyperFactory(
|
|
35
43
|
name="version",
|
|
@@ -64,6 +72,12 @@ def create(
|
|
|
64
72
|
help="When enabled, the Snowflake CLI skips checking if your project has any untracked or stages files in git. Default: unset.",
|
|
65
73
|
is_flag=True,
|
|
66
74
|
),
|
|
75
|
+
from_stage: bool = typer.Option(
|
|
76
|
+
False,
|
|
77
|
+
"--from-stage",
|
|
78
|
+
help="When enabled, the Snowflake CLI creates a version from the current application package stage without syncing to the stage first.",
|
|
79
|
+
is_flag=True,
|
|
80
|
+
),
|
|
67
81
|
interactive: bool = InteractiveOption,
|
|
68
82
|
force: Optional[bool] = ForceOption,
|
|
69
83
|
**options,
|
|
@@ -78,7 +92,7 @@ def create(
|
|
|
78
92
|
project_root=cli_context.project_root,
|
|
79
93
|
)
|
|
80
94
|
package_id = options["package_entity_id"]
|
|
81
|
-
ws.perform_action(
|
|
95
|
+
result: VersionInfo = ws.perform_action(
|
|
82
96
|
package_id,
|
|
83
97
|
EntityActions.VERSION_CREATE,
|
|
84
98
|
version=version,
|
|
@@ -87,8 +101,21 @@ def create(
|
|
|
87
101
|
force=force,
|
|
88
102
|
interactive=interactive,
|
|
89
103
|
skip_git_check=skip_git_check,
|
|
104
|
+
from_stage=from_stage,
|
|
90
105
|
)
|
|
91
|
-
|
|
106
|
+
|
|
107
|
+
message = "Version create is now complete."
|
|
108
|
+
if cli_context.output_format == OutputFormat.JSON:
|
|
109
|
+
return ObjectResult(
|
|
110
|
+
{
|
|
111
|
+
"message": message,
|
|
112
|
+
"version": to_identifier(result.version_name),
|
|
113
|
+
"patch": result.patch_number,
|
|
114
|
+
"label": result.label,
|
|
115
|
+
}
|
|
116
|
+
)
|
|
117
|
+
else:
|
|
118
|
+
return MessageResult(message)
|
|
92
119
|
|
|
93
120
|
|
|
94
121
|
@app.command("list", requires_connection=True)
|
|
@@ -110,7 +137,7 @@ def version_list(
|
|
|
110
137
|
package_id,
|
|
111
138
|
EntityActions.VERSION_LIST,
|
|
112
139
|
)
|
|
113
|
-
return
|
|
140
|
+
return CollectionResult(cursor)
|
|
114
141
|
|
|
115
142
|
|
|
116
143
|
@app.command(requires_connection=True)
|
|
@@ -15,12 +15,28 @@
|
|
|
15
15
|
import logging
|
|
16
16
|
|
|
17
17
|
import typer
|
|
18
|
+
from click import UsageError
|
|
18
19
|
from snowflake.cli._plugins.notebook.manager import NotebookManager
|
|
20
|
+
from snowflake.cli._plugins.notebook.notebook_entity_model import NotebookEntityModel
|
|
19
21
|
from snowflake.cli._plugins.notebook.types import NotebookStagePath
|
|
20
|
-
from snowflake.cli.
|
|
22
|
+
from snowflake.cli._plugins.workspace.manager import WorkspaceManager
|
|
23
|
+
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
24
|
+
from snowflake.cli.api.commands.decorators import (
|
|
25
|
+
with_project_definition,
|
|
26
|
+
)
|
|
27
|
+
from snowflake.cli.api.commands.flags import (
|
|
28
|
+
ReplaceOption,
|
|
29
|
+
entity_argument,
|
|
30
|
+
identifier_argument,
|
|
31
|
+
)
|
|
21
32
|
from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
|
|
33
|
+
from snowflake.cli.api.commands.utils import get_entity_for_operation
|
|
34
|
+
from snowflake.cli.api.entities.common import EntityActions
|
|
22
35
|
from snowflake.cli.api.identifiers import FQN
|
|
23
|
-
from snowflake.cli.api.output.types import
|
|
36
|
+
from snowflake.cli.api.output.types import (
|
|
37
|
+
CommandResult,
|
|
38
|
+
MessageResult,
|
|
39
|
+
)
|
|
24
40
|
from typing_extensions import Annotated
|
|
25
41
|
|
|
26
42
|
app = SnowTyperFactory(
|
|
@@ -84,3 +100,40 @@ def create(
|
|
|
84
100
|
notebook_file=notebook_file,
|
|
85
101
|
)
|
|
86
102
|
return MessageResult(message=notebook_url)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@app.command(requires_connection=True)
|
|
106
|
+
@with_project_definition()
|
|
107
|
+
def deploy(
|
|
108
|
+
entity_id: str = entity_argument("notebook"),
|
|
109
|
+
replace: bool = ReplaceOption(
|
|
110
|
+
help="Replace notebook object if it already exists.",
|
|
111
|
+
),
|
|
112
|
+
**options,
|
|
113
|
+
) -> CommandResult:
|
|
114
|
+
"""Uploads a notebook and required files to a stage and creates a Snowflake notebook."""
|
|
115
|
+
cli_context = get_cli_context()
|
|
116
|
+
pd = cli_context.project_definition
|
|
117
|
+
if not pd.meets_version_requirement("2"):
|
|
118
|
+
raise UsageError(
|
|
119
|
+
"This command requires project definition of version at least 2."
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
notebook: NotebookEntityModel = get_entity_for_operation(
|
|
123
|
+
cli_context=cli_context,
|
|
124
|
+
entity_id=entity_id,
|
|
125
|
+
project_definition=pd,
|
|
126
|
+
entity_type="notebook",
|
|
127
|
+
)
|
|
128
|
+
ws = WorkspaceManager(
|
|
129
|
+
project_definition=cli_context.project_definition,
|
|
130
|
+
project_root=cli_context.project_root,
|
|
131
|
+
)
|
|
132
|
+
notebook_url = ws.perform_action(
|
|
133
|
+
notebook.entity_id,
|
|
134
|
+
EntityActions.DEPLOY,
|
|
135
|
+
replace=replace,
|
|
136
|
+
)
|
|
137
|
+
return MessageResult(
|
|
138
|
+
f"Notebook successfully deployed and available under {notebook_url}"
|
|
139
|
+
)
|
|
@@ -16,11 +16,12 @@ from pathlib import Path
|
|
|
16
16
|
from textwrap import dedent
|
|
17
17
|
|
|
18
18
|
from snowflake.cli._plugins.connection.util import make_snowsight_url
|
|
19
|
-
from snowflake.cli._plugins.notebook.exceptions import
|
|
19
|
+
from snowflake.cli._plugins.notebook.exceptions import NotebookFilePathError
|
|
20
20
|
from snowflake.cli._plugins.notebook.types import NotebookStagePath
|
|
21
21
|
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
22
22
|
from snowflake.cli.api.identifiers import FQN
|
|
23
23
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin
|
|
24
|
+
from snowflake.cli.api.stage_path import StagePath
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
class NotebookManager(SqlExecutionMixin):
|
|
@@ -39,10 +40,11 @@ class NotebookManager(SqlExecutionMixin):
|
|
|
39
40
|
def parse_stage_as_path(notebook_file: str) -> Path:
|
|
40
41
|
"""Parses notebook file path to pathlib.Path."""
|
|
41
42
|
if not notebook_file.endswith(".ipynb"):
|
|
42
|
-
raise
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
raise NotebookFilePathError(notebook_file)
|
|
44
|
+
# we don't perform any operations on the path, so we don't need to differentiate git repository paths
|
|
45
|
+
stage_path = StagePath.from_stage_str(notebook_file)
|
|
46
|
+
if len(stage_path.parts) < 1:
|
|
47
|
+
raise NotebookFilePathError(notebook_file)
|
|
46
48
|
|
|
47
49
|
return stage_path
|
|
48
50
|
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
|
|
3
|
+
from click import ClickException
|
|
4
|
+
from snowflake.cli._plugins.connection.util import make_snowsight_url
|
|
5
|
+
from snowflake.cli._plugins.notebook.notebook_entity_model import NotebookEntityModel
|
|
6
|
+
from snowflake.cli._plugins.notebook.notebook_project_paths import NotebookProjectPaths
|
|
7
|
+
from snowflake.cli._plugins.stage.manager import StageManager
|
|
8
|
+
from snowflake.cli._plugins.workspace.context import ActionContext
|
|
9
|
+
from snowflake.cli.api.artifacts.utils import bundle_artifacts
|
|
10
|
+
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
11
|
+
from snowflake.cli.api.console.console import cli_console
|
|
12
|
+
from snowflake.cli.api.entities.common import EntityBase
|
|
13
|
+
from snowflake.cli.api.stage_path import StagePath
|
|
14
|
+
from snowflake.connector import ProgrammingError
|
|
15
|
+
from snowflake.connector.cursor import SnowflakeCursor
|
|
16
|
+
|
|
17
|
+
_DEFAULT_NOTEBOOK_STAGE_NAME = "@notebooks"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class NotebookEntity(EntityBase[NotebookEntityModel]):
|
|
21
|
+
"""
|
|
22
|
+
A notebook.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
@functools.cached_property
|
|
26
|
+
def _stage_path(self) -> StagePath:
|
|
27
|
+
stage_path = self.model.stage_path
|
|
28
|
+
if stage_path is None:
|
|
29
|
+
stage_path = f"{_DEFAULT_NOTEBOOK_STAGE_NAME}/{self.fqn.name}"
|
|
30
|
+
return StagePath.from_stage_str(stage_path)
|
|
31
|
+
|
|
32
|
+
@functools.cached_property
|
|
33
|
+
def _project_paths(self):
|
|
34
|
+
return NotebookProjectPaths(get_cli_context().project_root)
|
|
35
|
+
|
|
36
|
+
def _object_exists(self) -> bool:
|
|
37
|
+
# currently notebook objects are not supported by object manager - re-implementing "exists"
|
|
38
|
+
try:
|
|
39
|
+
self.action_describe()
|
|
40
|
+
return True
|
|
41
|
+
except ProgrammingError:
|
|
42
|
+
return False
|
|
43
|
+
|
|
44
|
+
def _upload_artifacts(self):
|
|
45
|
+
stage_fqn = self._stage_path.stage_fqn
|
|
46
|
+
stage_manager = StageManager()
|
|
47
|
+
cli_console.step(f"Creating stage {stage_fqn} if not exists")
|
|
48
|
+
stage_manager.create(fqn=stage_fqn)
|
|
49
|
+
|
|
50
|
+
cli_console.step("Uploading artifacts")
|
|
51
|
+
|
|
52
|
+
# creating bundle map to handle glob patterns logic
|
|
53
|
+
bundle_map = bundle_artifacts(self._project_paths, self.model.artifacts)
|
|
54
|
+
for absolute_src, absolute_dest in bundle_map.all_mappings(
|
|
55
|
+
absolute=True, expand_directories=True
|
|
56
|
+
):
|
|
57
|
+
artifact_stage_path = self._stage_path / (
|
|
58
|
+
absolute_dest.relative_to(self._project_paths.bundle_root).parent
|
|
59
|
+
)
|
|
60
|
+
stage_manager.put(
|
|
61
|
+
local_path=absolute_src, stage_path=artifact_stage_path, overwrite=True
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def get_create_sql(self, replace: bool) -> str:
|
|
65
|
+
main_file_stage_path = self._stage_path / (
|
|
66
|
+
self.model.notebook_file.absolute().relative_to(
|
|
67
|
+
self._project_paths.project_root
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
query = "CREATE OR REPLACE " if replace else "CREATE "
|
|
71
|
+
query += (
|
|
72
|
+
f"NOTEBOOK {self.fqn.sql_identifier}\n"
|
|
73
|
+
f"FROM '{main_file_stage_path.stage_with_at}'\n"
|
|
74
|
+
f"QUERY_WAREHOUSE = '{self.model.query_warehouse}'\n"
|
|
75
|
+
f"MAIN_FILE = '{main_file_stage_path.path}'"
|
|
76
|
+
)
|
|
77
|
+
if self.model.compute_pool:
|
|
78
|
+
query += f"\nCOMPUTE_POOL = '{self.model.compute_pool}'"
|
|
79
|
+
if self.model.runtime_name:
|
|
80
|
+
query += f"\nRUNTIME_NAME = '{self.model.runtime_name}'"
|
|
81
|
+
|
|
82
|
+
query += (
|
|
83
|
+
";\n// Cannot use IDENTIFIER(...)"
|
|
84
|
+
f"\nALTER NOTEBOOK {self.fqn.identifier} ADD LIVE VERSION FROM LAST;"
|
|
85
|
+
)
|
|
86
|
+
return query
|
|
87
|
+
|
|
88
|
+
def action_describe(self) -> SnowflakeCursor:
|
|
89
|
+
return self._sql_executor.execute_query(self.get_describe_sql())
|
|
90
|
+
|
|
91
|
+
def action_create(self, replace: bool) -> str:
|
|
92
|
+
self._sql_executor.execute_query(self.get_create_sql(replace))
|
|
93
|
+
return make_snowsight_url(
|
|
94
|
+
self._conn,
|
|
95
|
+
f"/#/notebooks/{self.fqn.using_connection(self._conn).url_identifier}",
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
def action_deploy(
|
|
99
|
+
self,
|
|
100
|
+
action_ctx: ActionContext,
|
|
101
|
+
replace: bool,
|
|
102
|
+
*args,
|
|
103
|
+
**kwargs,
|
|
104
|
+
) -> str:
|
|
105
|
+
if self._object_exists():
|
|
106
|
+
if not replace:
|
|
107
|
+
raise ClickException(
|
|
108
|
+
f"Notebook {self.fqn.name} already exists. Consider using --replace."
|
|
109
|
+
)
|
|
110
|
+
with cli_console.phase(f"Uploading artifacts to {self._stage_path}"):
|
|
111
|
+
self._upload_artifacts()
|
|
112
|
+
with cli_console.phase(f"Creating notebook {self.fqn}"):
|
|
113
|
+
return self.action_create(replace=replace)
|
|
114
|
+
|
|
115
|
+
# complementary actions, currently not used - to be implemented in future
|
|
116
|
+
def action_drop(self, *args, **kwargs):
|
|
117
|
+
raise ClickException("action DROP not supported by NOTEBOOK entity")
|
|
118
|
+
|
|
119
|
+
def action_teardown(self, *args, **kwargs):
|
|
120
|
+
raise ClickException("action TEARDOWN not supported by NOTEBOOK entity")
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Literal, Optional
|
|
5
|
+
|
|
6
|
+
from pydantic import Field, model_validator
|
|
7
|
+
from snowflake.cli._plugins.notebook.exceptions import NotebookFilePathError
|
|
8
|
+
from snowflake.cli.api.project.schemas.entities.common import (
|
|
9
|
+
EntityModelBaseWithArtifacts,
|
|
10
|
+
)
|
|
11
|
+
from snowflake.cli.api.project.schemas.updatable_model import (
|
|
12
|
+
DiscriminatorField,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class NotebookEntityModel(EntityModelBaseWithArtifacts):
|
|
17
|
+
type: Literal["notebook"] = DiscriminatorField() # noqa: A003
|
|
18
|
+
stage_path: Optional[str] = Field(
|
|
19
|
+
title="Stage directory in which the notebook file will be stored", default=None
|
|
20
|
+
)
|
|
21
|
+
notebook_file: Path = Field(title="Notebook file")
|
|
22
|
+
query_warehouse: str = Field(title="Snowflake warehouse to execute the notebook")
|
|
23
|
+
compute_pool: Optional[str] = Field(
|
|
24
|
+
title="Compute pool to run the notebook in", default=None
|
|
25
|
+
)
|
|
26
|
+
runtime_name: Optional[str] = Field(title="Container Runtime for ML", default=None)
|
|
27
|
+
|
|
28
|
+
@model_validator(mode="after")
|
|
29
|
+
def validate_notebook_file(self):
|
|
30
|
+
if not self.notebook_file.exists():
|
|
31
|
+
raise ValueError(f"Notebook file {self.notebook_file} does not exist")
|
|
32
|
+
if self.notebook_file.suffix.lower() != ".ipynb":
|
|
33
|
+
raise NotebookFilePathError(str(self.notebook_file))
|
|
34
|
+
return self
|
|
35
|
+
|
|
36
|
+
@model_validator(mode="after")
|
|
37
|
+
def validate_container_setup(self):
|
|
38
|
+
if self.compute_pool and not self.runtime_name:
|
|
39
|
+
raise ValueError("compute_pool is specified without runtime_name")
|
|
40
|
+
if self.runtime_name and not self.compute_pool and not self:
|
|
41
|
+
raise ValueError("runtime_name is specified without compute_pool")
|
|
42
|
+
return self
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class NotebookProjectPaths(ProjectPaths):
|
|
9
|
+
"""
|
|
10
|
+
This class allows you to manage files paths related to given project.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
@property
|
|
14
|
+
def bundle_root(self) -> Path:
|
|
15
|
+
return bundle_root(self.project_root, "notebook")
|
|
@@ -39,7 +39,9 @@ from snowflake.cli._plugins.snowpark.common import (
|
|
|
39
39
|
SnowparkEntities,
|
|
40
40
|
SnowparkObject,
|
|
41
41
|
SnowparkObjectManager,
|
|
42
|
-
|
|
42
|
+
StageToArtifactMapping,
|
|
43
|
+
map_path_mapping_to_artifact,
|
|
44
|
+
zip_and_copy_artifacts_to_deploy,
|
|
43
45
|
)
|
|
44
46
|
from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
|
|
45
47
|
AnacondaPackages,
|
|
@@ -68,6 +70,7 @@ from snowflake.cli.api.commands.decorators import (
|
|
|
68
70
|
with_project_definition,
|
|
69
71
|
)
|
|
70
72
|
from snowflake.cli.api.commands.flags import (
|
|
73
|
+
ForceReplaceOption,
|
|
71
74
|
ReplaceOption,
|
|
72
75
|
execution_identifier_argument,
|
|
73
76
|
identifier_argument,
|
|
@@ -81,6 +84,7 @@ from snowflake.cli.api.constants import (
|
|
|
81
84
|
from snowflake.cli.api.exceptions import (
|
|
82
85
|
SecretsWithoutExternalAccessIntegrationError,
|
|
83
86
|
)
|
|
87
|
+
from snowflake.cli.api.feature_flags import FeatureFlag
|
|
84
88
|
from snowflake.cli.api.identifiers import FQN
|
|
85
89
|
from snowflake.cli.api.output.types import (
|
|
86
90
|
CollectionResult,
|
|
@@ -125,8 +129,9 @@ LikeOption = like_option(
|
|
|
125
129
|
@with_project_definition()
|
|
126
130
|
def deploy(
|
|
127
131
|
replace: bool = ReplaceOption(
|
|
128
|
-
help="Replaces procedure or function
|
|
132
|
+
help="Replaces procedure or function if there were changes in the definition."
|
|
129
133
|
),
|
|
134
|
+
force_replace: bool = ForceReplaceOption(),
|
|
130
135
|
**options,
|
|
131
136
|
) -> CommandResult:
|
|
132
137
|
"""
|
|
@@ -156,7 +161,11 @@ def deploy(
|
|
|
156
161
|
with cli_console.phase("Checking remote state"):
|
|
157
162
|
om = ObjectManager()
|
|
158
163
|
_check_if_all_defined_integrations_exists(om, snowpark_entities)
|
|
159
|
-
existing_objects =
|
|
164
|
+
existing_objects = (
|
|
165
|
+
{}
|
|
166
|
+
if force_replace
|
|
167
|
+
else check_for_existing_objects(om, replace, snowpark_entities)
|
|
168
|
+
)
|
|
160
169
|
|
|
161
170
|
with cli_console.phase("Preparing required stages and artifacts"):
|
|
162
171
|
entities_to_imports_map, stages_to_artifact_map = build_artifacts_mappings(
|
|
@@ -189,11 +198,11 @@ def validate_all_artifacts_exists(
|
|
|
189
198
|
project_paths: SnowparkProjectPaths, snowpark_entities: SnowparkEntities
|
|
190
199
|
):
|
|
191
200
|
for key, entity in snowpark_entities.items():
|
|
192
|
-
for
|
|
193
|
-
path = project_paths.
|
|
201
|
+
for artifact in entity.artifacts:
|
|
202
|
+
path = project_paths.get_artifact_dto(artifact).post_build_path
|
|
194
203
|
if not path.exists():
|
|
195
204
|
raise UsageError(
|
|
196
|
-
f"
|
|
205
|
+
f"Artifact {path} required for {entity.type} {key} does not exist."
|
|
197
206
|
)
|
|
198
207
|
|
|
199
208
|
|
|
@@ -213,38 +222,39 @@ def check_for_existing_objects(
|
|
|
213
222
|
|
|
214
223
|
def build_artifacts_mappings(
|
|
215
224
|
project_paths: SnowparkProjectPaths, snowpark_entities: SnowparkEntities
|
|
216
|
-
) -> Tuple[EntityToImportPathsMapping,
|
|
217
|
-
stages_to_artifact_map:
|
|
225
|
+
) -> Tuple[EntityToImportPathsMapping, StageToArtifactMapping]:
|
|
226
|
+
stages_to_artifact_map: StageToArtifactMapping = defaultdict(set)
|
|
218
227
|
entities_to_imports_map: EntityToImportPathsMapping = defaultdict(set)
|
|
219
|
-
for
|
|
228
|
+
for name, entity in snowpark_entities.items():
|
|
220
229
|
stage = entity.stage
|
|
221
230
|
required_artifacts = set()
|
|
222
|
-
for
|
|
223
|
-
|
|
224
|
-
required_artifacts.add(
|
|
225
|
-
entities_to_imports_map[
|
|
231
|
+
for artifact in entity.artifacts:
|
|
232
|
+
artifact_dto = project_paths.get_artifact_dto(artifact)
|
|
233
|
+
required_artifacts.add(artifact_dto)
|
|
234
|
+
entities_to_imports_map[name].add(artifact_dto.import_path(stage))
|
|
226
235
|
stages_to_artifact_map[stage].update(required_artifacts)
|
|
227
236
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
stages_to_artifact_map[stage].add(
|
|
231
|
-
entities_to_imports_map[
|
|
237
|
+
deps_artifact = project_paths.get_dependencies_artifact()
|
|
238
|
+
if deps_artifact.post_build_path.exists():
|
|
239
|
+
stages_to_artifact_map[stage].add(deps_artifact)
|
|
240
|
+
entities_to_imports_map[name].add(deps_artifact.import_path(stage))
|
|
232
241
|
return entities_to_imports_map, stages_to_artifact_map
|
|
233
242
|
|
|
234
243
|
|
|
235
|
-
def create_stages_and_upload_artifacts(stages_to_artifact_map:
|
|
244
|
+
def create_stages_and_upload_artifacts(stages_to_artifact_map: StageToArtifactMapping):
|
|
236
245
|
stage_manager = StageManager()
|
|
237
246
|
for stage, artifacts in stages_to_artifact_map.items():
|
|
238
247
|
cli_console.step(f"Creating (if not exists) stage: {stage}")
|
|
239
248
|
stage = FQN.from_stage(stage).using_context()
|
|
240
249
|
stage_manager.create(fqn=stage, comment="deployments managed by Snowflake CLI")
|
|
241
|
-
for
|
|
250
|
+
for artifact in artifacts:
|
|
251
|
+
post_build_path = artifact.post_build_path
|
|
242
252
|
cli_console.step(
|
|
243
|
-
f"Uploading {
|
|
253
|
+
f"Uploading {post_build_path.name} to {artifact.upload_path(stage)}"
|
|
244
254
|
)
|
|
245
255
|
stage_manager.put(
|
|
246
|
-
local_path=
|
|
247
|
-
stage_path=
|
|
256
|
+
local_path=post_build_path,
|
|
257
|
+
stage_path=artifact.upload_path(stage),
|
|
248
258
|
overwrite=True,
|
|
249
259
|
)
|
|
250
260
|
|
|
@@ -324,6 +334,9 @@ def build(
|
|
|
324
334
|
|
|
325
335
|
anaconda_packages_manager = AnacondaPackagesManager()
|
|
326
336
|
|
|
337
|
+
# Clean up bundle root
|
|
338
|
+
project_paths.remove_up_bundle_root()
|
|
339
|
+
|
|
327
340
|
# Resolve dependencies
|
|
328
341
|
if project_paths.requirements.exists():
|
|
329
342
|
with (
|
|
@@ -362,22 +375,27 @@ def build(
|
|
|
362
375
|
)
|
|
363
376
|
|
|
364
377
|
if any(temp_deps_dir.path.iterdir()):
|
|
365
|
-
|
|
378
|
+
dep_artifact = project_paths.get_dependencies_artifact()
|
|
379
|
+
cli_console.step(f"Creating {dep_artifact.path.name}")
|
|
366
380
|
zip_dir(
|
|
367
381
|
source=temp_deps_dir.path,
|
|
368
|
-
dest_zip=
|
|
382
|
+
dest_zip=dep_artifact.post_build_path,
|
|
369
383
|
)
|
|
370
384
|
else:
|
|
371
385
|
cli_console.step(f"No external dependencies.")
|
|
372
386
|
|
|
373
387
|
artifacts = set()
|
|
374
|
-
for entity in get_snowpark_entities(pd).values():
|
|
375
|
-
artifacts.update(entity.artifacts)
|
|
376
|
-
|
|
377
388
|
with cli_console.phase("Preparing artifacts for source code"):
|
|
378
|
-
for
|
|
379
|
-
|
|
380
|
-
|
|
389
|
+
for entity in get_snowpark_entities(pd).values():
|
|
390
|
+
artifacts.update(
|
|
391
|
+
map_path_mapping_to_artifact(project_paths, entity.artifacts)
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
|
|
395
|
+
zip_and_copy_artifacts_to_deploy(artifacts, project_paths.bundle_root)
|
|
396
|
+
else:
|
|
397
|
+
for artifact in artifacts:
|
|
398
|
+
artifact.build()
|
|
381
399
|
|
|
382
400
|
return MessageResult(f"Build done.")
|
|
383
401
|
|
|
@@ -17,6 +17,7 @@ from __future__ import annotations
|
|
|
17
17
|
import logging
|
|
18
18
|
import re
|
|
19
19
|
from enum import Enum
|
|
20
|
+
from pathlib import Path
|
|
20
21
|
from typing import Dict, List, Set
|
|
21
22
|
|
|
22
23
|
from click import UsageError
|
|
@@ -25,7 +26,13 @@ from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
|
|
|
25
26
|
ProcedureEntityModel,
|
|
26
27
|
SnowparkEntityModel,
|
|
27
28
|
)
|
|
28
|
-
from snowflake.cli._plugins.snowpark.snowpark_project_paths import
|
|
29
|
+
from snowflake.cli._plugins.snowpark.snowpark_project_paths import (
|
|
30
|
+
Artifact,
|
|
31
|
+
SnowparkProjectPaths,
|
|
32
|
+
)
|
|
33
|
+
from snowflake.cli._plugins.snowpark.zipper import zip_dir_using_bundle_map
|
|
34
|
+
from snowflake.cli.api.artifacts.bundle_map import BundleMap
|
|
35
|
+
from snowflake.cli.api.artifacts.utils import symlink_or_copy
|
|
29
36
|
from snowflake.cli.api.console import cli_console
|
|
30
37
|
from snowflake.cli.api.constants import (
|
|
31
38
|
INIT_TEMPLATE_VARIABLE_CLOSING,
|
|
@@ -34,13 +41,14 @@ from snowflake.cli.api.constants import (
|
|
|
34
41
|
PROJECT_TEMPLATE_VARIABLE_OPENING,
|
|
35
42
|
ObjectType,
|
|
36
43
|
)
|
|
44
|
+
from snowflake.cli.api.project.schemas.entities.common import PathMapping
|
|
37
45
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin
|
|
38
46
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
39
47
|
|
|
40
48
|
log = logging.getLogger(__name__)
|
|
41
49
|
|
|
42
50
|
SnowparkEntities = Dict[str, SnowparkEntityModel]
|
|
43
|
-
|
|
51
|
+
StageToArtifactMapping = Dict[str, set[Artifact]]
|
|
44
52
|
EntityToImportPathsMapping = Dict[str, set[str]]
|
|
45
53
|
|
|
46
54
|
DEFAULT_RUNTIME = "3.10"
|
|
@@ -214,6 +222,43 @@ def _snowflake_dependencies_differ(
|
|
|
214
222
|
return _standardize(old_dependencies) != _standardize(new_dependencies)
|
|
215
223
|
|
|
216
224
|
|
|
225
|
+
def map_path_mapping_to_artifact(
|
|
226
|
+
project_paths: SnowparkProjectPaths, artifacts: List[PathMapping]
|
|
227
|
+
) -> List[Artifact]:
|
|
228
|
+
return [project_paths.get_artifact_dto(artifact) for artifact in artifacts]
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def zip_and_copy_artifacts_to_deploy(
|
|
232
|
+
artifacts: Set[Artifact] | List[Artifact], bundle_root: Path
|
|
233
|
+
) -> List[Path]:
|
|
234
|
+
copied_files = []
|
|
235
|
+
for artifact in artifacts:
|
|
236
|
+
bundle_map = BundleMap(
|
|
237
|
+
project_root=artifact.project_root,
|
|
238
|
+
deploy_root=bundle_root,
|
|
239
|
+
)
|
|
240
|
+
bundle_map.add(PathMapping(src=str(artifact.path), dest=artifact.dest))
|
|
241
|
+
|
|
242
|
+
if artifact.path.is_file():
|
|
243
|
+
for (absolute_src, absolute_dest) in bundle_map.all_mappings(
|
|
244
|
+
absolute=True, expand_directories=False
|
|
245
|
+
):
|
|
246
|
+
symlink_or_copy(
|
|
247
|
+
absolute_src,
|
|
248
|
+
absolute_dest,
|
|
249
|
+
deploy_root=bundle_map.deploy_root(),
|
|
250
|
+
)
|
|
251
|
+
copied_files.append(absolute_dest)
|
|
252
|
+
else:
|
|
253
|
+
post_build_path = artifact.post_build_path
|
|
254
|
+
zip_dir_using_bundle_map(
|
|
255
|
+
bundle_map=bundle_map,
|
|
256
|
+
dest_zip=post_build_path,
|
|
257
|
+
)
|
|
258
|
+
copied_files.append(post_build_path)
|
|
259
|
+
return copied_files
|
|
260
|
+
|
|
261
|
+
|
|
217
262
|
def same_type(sf_type: str, local_type: str) -> bool:
|
|
218
263
|
sf_type, local_type = sf_type.upper(), local_type.upper()
|
|
219
264
|
|