snowflake-cli 3.3.0__py3-none-any.whl → 3.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli/__about__.py +1 -1
- snowflake/cli/_app/__main__.py +2 -2
- snowflake/cli/_app/cli_app.py +224 -192
- snowflake/cli/_app/commands_registration/commands_registration_with_callbacks.py +1 -27
- snowflake/cli/_plugins/cortex/commands.py +2 -4
- snowflake/cli/_plugins/git/manager.py +1 -1
- snowflake/cli/_plugins/nativeapp/artifacts.py +6 -624
- snowflake/cli/_plugins/nativeapp/bundle_context.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/compiler.py +1 -3
- snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/commands.py +21 -19
- snowflake/cli/_plugins/nativeapp/entities/application.py +16 -19
- snowflake/cli/_plugins/nativeapp/entities/application_package.py +142 -55
- snowflake/cli/_plugins/nativeapp/release_channel/commands.py +37 -3
- snowflake/cli/_plugins/nativeapp/release_directive/commands.py +80 -2
- snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +224 -44
- snowflake/cli/_plugins/nativeapp/v2_conversions/compat.py +2 -2
- snowflake/cli/_plugins/nativeapp/version/commands.py +1 -1
- snowflake/cli/_plugins/notebook/commands.py +55 -2
- snowflake/cli/_plugins/notebook/exceptions.py +1 -1
- snowflake/cli/_plugins/notebook/manager.py +3 -3
- snowflake/cli/_plugins/notebook/notebook_entity.py +120 -0
- snowflake/cli/_plugins/notebook/notebook_entity_model.py +42 -0
- snowflake/cli/_plugins/notebook/notebook_project_paths.py +15 -0
- snowflake/cli/_plugins/notebook/types.py +3 -0
- snowflake/cli/_plugins/snowpark/commands.py +48 -30
- snowflake/cli/_plugins/snowpark/common.py +47 -2
- snowflake/cli/_plugins/snowpark/snowpark_entity.py +38 -25
- snowflake/cli/_plugins/snowpark/snowpark_entity_model.py +18 -30
- snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +156 -23
- snowflake/cli/_plugins/snowpark/zipper.py +33 -1
- snowflake/cli/_plugins/spcs/services/commands.py +0 -3
- snowflake/cli/_plugins/stage/commands.py +2 -1
- snowflake/cli/_plugins/stage/diff.py +60 -39
- snowflake/cli/_plugins/stage/manager.py +24 -11
- snowflake/cli/_plugins/stage/utils.py +1 -1
- snowflake/cli/_plugins/streamlit/commands.py +10 -1
- snowflake/cli/_plugins/streamlit/manager.py +62 -21
- snowflake/cli/_plugins/streamlit/streamlit_entity.py +20 -41
- snowflake/cli/_plugins/streamlit/streamlit_entity_model.py +14 -24
- snowflake/cli/_plugins/streamlit/streamlit_project_paths.py +30 -0
- snowflake/cli/_plugins/workspace/commands.py +3 -3
- snowflake/cli/_plugins/workspace/manager.py +1 -1
- snowflake/cli/api/artifacts/__init__.py +13 -0
- snowflake/cli/api/artifacts/bundle_map.py +500 -0
- snowflake/cli/api/artifacts/common.py +78 -0
- snowflake/cli/api/artifacts/utils.py +82 -0
- snowflake/cli/api/cli_global_context.py +14 -1
- snowflake/cli/api/commands/flags.py +10 -4
- snowflake/cli/api/commands/utils.py +28 -2
- snowflake/cli/api/constants.py +1 -0
- snowflake/cli/api/entities/common.py +14 -32
- snowflake/cli/api/entities/resolver.py +160 -0
- snowflake/cli/api/entities/utils.py +56 -15
- snowflake/cli/api/errno.py +3 -0
- snowflake/cli/api/feature_flags.py +1 -2
- snowflake/cli/api/project/definition_conversion.py +3 -2
- snowflake/cli/api/project/project_paths.py +28 -0
- snowflake/cli/api/project/schemas/entities/common.py +130 -1
- snowflake/cli/api/project/schemas/entities/entities.py +4 -0
- snowflake/cli/api/project/schemas/project_definition.py +27 -0
- snowflake/cli/api/project/schemas/updatable_model.py +2 -2
- snowflake/cli/api/project/schemas/v1/native_app/native_app.py +5 -7
- snowflake/cli/api/secure_path.py +6 -0
- snowflake/cli/api/sql_execution.py +5 -1
- snowflake/cli/api/stage_path.py +7 -2
- snowflake/cli/api/utils/graph.py +3 -0
- snowflake/cli/api/utils/path_utils.py +24 -0
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.4.1.dist-info}/METADATA +8 -9
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.4.1.dist-info}/RECORD +76 -67
- snowflake/cli/api/project/schemas/v1/native_app/path_mapping.py +0 -65
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.4.1.dist-info}/WHEEL +0 -0
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.4.1.dist-info}/entry_points.txt +0 -0
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.4.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -13,38 +13,60 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
+
import glob
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
16
19
|
from dataclasses import dataclass
|
|
17
20
|
from pathlib import Path, PurePosixPath
|
|
21
|
+
from typing import Optional
|
|
18
22
|
|
|
19
|
-
from snowflake.cli._plugins.snowpark.snowpark_entity_model import PathMapping
|
|
20
23
|
from snowflake.cli._plugins.snowpark.zipper import zip_dir
|
|
21
24
|
from snowflake.cli.api.console import cli_console
|
|
22
25
|
from snowflake.cli.api.constants import DEPLOYMENT_STAGE
|
|
26
|
+
from snowflake.cli.api.feature_flags import FeatureFlag
|
|
23
27
|
from snowflake.cli.api.identifiers import FQN
|
|
28
|
+
from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root
|
|
29
|
+
from snowflake.cli.api.project.schemas.entities.common import PathMapping
|
|
24
30
|
from snowflake.cli.api.secure_path import SecurePath
|
|
25
31
|
|
|
26
32
|
|
|
27
33
|
@dataclass
|
|
28
|
-
class SnowparkProjectPaths:
|
|
34
|
+
class SnowparkProjectPaths(ProjectPaths):
|
|
29
35
|
"""
|
|
30
|
-
This class
|
|
36
|
+
This class allows you to manage files paths related to given project.
|
|
31
37
|
"""
|
|
32
38
|
|
|
33
|
-
project_root: Path
|
|
34
|
-
|
|
35
39
|
def path_relative_to_root(self, artifact_path: Path) -> Path:
|
|
36
40
|
if artifact_path.is_absolute():
|
|
37
41
|
return artifact_path
|
|
38
42
|
return (self.project_root / artifact_path).resolve()
|
|
39
43
|
|
|
40
|
-
def
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
def get_artifact_dto(self, artifact_path: PathMapping) -> Artifact:
|
|
45
|
+
if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
|
|
46
|
+
return Artifact(
|
|
47
|
+
project_root=self.project_root,
|
|
48
|
+
bundle_root=self.bundle_root,
|
|
49
|
+
dest=artifact_path.dest,
|
|
50
|
+
path=Path(artifact_path.src),
|
|
51
|
+
)
|
|
52
|
+
else:
|
|
53
|
+
return ArtifactOldBuild(
|
|
54
|
+
dest=artifact_path.dest,
|
|
55
|
+
path=self.path_relative_to_root(Path(artifact_path.src)),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def get_dependencies_artifact(self) -> Artifact:
|
|
59
|
+
if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
|
|
60
|
+
return Artifact(
|
|
61
|
+
project_root=self.project_root,
|
|
62
|
+
bundle_root=self.bundle_root,
|
|
63
|
+
dest=None,
|
|
64
|
+
path=Path("dependencies.zip"),
|
|
65
|
+
)
|
|
66
|
+
else:
|
|
67
|
+
return ArtifactOldBuild(
|
|
68
|
+
dest=None, path=self.path_relative_to_root(Path("dependencies.zip"))
|
|
69
|
+
)
|
|
48
70
|
|
|
49
71
|
@property
|
|
50
72
|
def snowflake_requirements(self) -> SecurePath:
|
|
@@ -57,19 +79,130 @@ class SnowparkProjectPaths:
|
|
|
57
79
|
return SecurePath(self.path_relative_to_root(Path("requirements.txt")))
|
|
58
80
|
|
|
59
81
|
@property
|
|
60
|
-
def
|
|
61
|
-
return self.
|
|
82
|
+
def bundle_root(self) -> Path:
|
|
83
|
+
return bundle_root(self.project_root, "snowpark")
|
|
62
84
|
|
|
63
85
|
|
|
64
86
|
@dataclass(unsafe_hash=True)
|
|
65
|
-
class
|
|
66
|
-
"""Helper for getting paths related to given
|
|
87
|
+
class Artifact:
|
|
88
|
+
"""Helper for getting paths related to given artifact."""
|
|
67
89
|
|
|
90
|
+
project_root: Path
|
|
91
|
+
bundle_root: Path
|
|
68
92
|
path: Path
|
|
69
93
|
dest: str | None = None
|
|
70
94
|
|
|
95
|
+
def __init__(
|
|
96
|
+
self,
|
|
97
|
+
project_root: Path,
|
|
98
|
+
bundle_root: Path,
|
|
99
|
+
path: Path,
|
|
100
|
+
dest: Optional[str] = None,
|
|
101
|
+
) -> None:
|
|
102
|
+
self.project_root = project_root
|
|
103
|
+
self.bundle_root = bundle_root
|
|
104
|
+
self.path = path
|
|
105
|
+
self.dest = dest
|
|
106
|
+
if self.dest and not self._is_dest_a_file() and not self.dest.endswith("/"):
|
|
107
|
+
self.dest = self.dest + "/"
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def _artifact_name(self) -> str:
|
|
111
|
+
"""
|
|
112
|
+
Returns artifact name. Directories are mapped to corresponding .zip files.
|
|
113
|
+
For paths with glob patterns, the last part of the path is used.
|
|
114
|
+
For files, the file name is used.
|
|
115
|
+
"""
|
|
116
|
+
if glob.has_magic(str(self.path)):
|
|
117
|
+
last_part = None
|
|
118
|
+
for part in self.path.parts:
|
|
119
|
+
if glob.has_magic(part):
|
|
120
|
+
break
|
|
121
|
+
else:
|
|
122
|
+
last_part = part
|
|
123
|
+
if not last_part:
|
|
124
|
+
last_part = os.path.commonpath(
|
|
125
|
+
[str(self.path), str(self.path.absolute())]
|
|
126
|
+
)
|
|
127
|
+
return last_part + ".zip"
|
|
128
|
+
if (self.project_root / self.path).is_dir():
|
|
129
|
+
return self.path.stem + ".zip"
|
|
130
|
+
if (self.project_root / self.path).is_file() and self._is_dest_a_file():
|
|
131
|
+
return Path(self.dest).name # type: ignore
|
|
132
|
+
return self.path.name
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def post_build_path(self) -> Path:
|
|
136
|
+
"""
|
|
137
|
+
Returns post-build artifact path. Directories are mapped to corresponding .zip files.
|
|
138
|
+
"""
|
|
139
|
+
bundle_root = self.bundle_root
|
|
140
|
+
path = (
|
|
141
|
+
self._path_until_asterisk()
|
|
142
|
+
if glob.has_magic(str(self.path))
|
|
143
|
+
else self.path.parent
|
|
144
|
+
)
|
|
145
|
+
if self._is_dest_a_file():
|
|
146
|
+
return bundle_root / self.dest # type: ignore
|
|
147
|
+
return bundle_root / (self.dest or path) / self._artifact_name
|
|
148
|
+
|
|
149
|
+
def upload_path(self, stage: FQN | str | None) -> str:
|
|
150
|
+
"""
|
|
151
|
+
Path on stage to which the artifact should be uploaded.
|
|
152
|
+
"""
|
|
153
|
+
stage = stage or DEPLOYMENT_STAGE
|
|
154
|
+
if isinstance(stage, str):
|
|
155
|
+
stage = FQN.from_stage(stage).using_context()
|
|
156
|
+
|
|
157
|
+
stage_path = PurePosixPath(f"@{stage}")
|
|
158
|
+
if self.dest:
|
|
159
|
+
stage_path /= (
|
|
160
|
+
PurePosixPath(self.dest).parent if self._is_dest_a_file() else self.dest
|
|
161
|
+
)
|
|
162
|
+
else:
|
|
163
|
+
stage_path /= (
|
|
164
|
+
self._path_until_asterisk()
|
|
165
|
+
if glob.has_magic(str(self.path))
|
|
166
|
+
else PurePosixPath(self.path).parent
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
return str(stage_path) + "/"
|
|
170
|
+
|
|
171
|
+
def import_path(self, stage: FQN | str | None) -> str:
|
|
172
|
+
"""Path for UDF/sproc imports clause."""
|
|
173
|
+
return self.upload_path(stage) + self._artifact_name
|
|
174
|
+
|
|
175
|
+
def _is_dest_a_file(self) -> bool:
|
|
176
|
+
if not self.dest:
|
|
177
|
+
return False
|
|
178
|
+
return re.search(r"\.[a-zA-Z0-9]{2,4}$", self.dest) is not None
|
|
179
|
+
|
|
180
|
+
def _path_until_asterisk(self) -> Path:
|
|
181
|
+
path = []
|
|
182
|
+
for part in self.path.parts:
|
|
183
|
+
if glob.has_magic(part):
|
|
184
|
+
break
|
|
185
|
+
else:
|
|
186
|
+
path.append(part)
|
|
187
|
+
return Path(*path[:-1])
|
|
188
|
+
|
|
189
|
+
# Can be removed after removing ENABLE_SNOWPARK_GLOB_SUPPORT feature flag.
|
|
190
|
+
def build(self) -> None:
|
|
191
|
+
raise NotImplementedError("Not implemented in Artifact class.")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@dataclass(unsafe_hash=True)
|
|
195
|
+
class ArtifactOldBuild(Artifact):
|
|
196
|
+
"""Helper for getting paths related to given artifact."""
|
|
197
|
+
|
|
198
|
+
path: Path
|
|
199
|
+
dest: str | None = None
|
|
200
|
+
|
|
201
|
+
def __init__(self, path: Path, dest: Optional[str] = None) -> None:
|
|
202
|
+
super().__init__(project_root=Path(), bundle_root=Path(), path=path, dest=dest)
|
|
203
|
+
|
|
71
204
|
@property
|
|
72
|
-
def
|
|
205
|
+
def _artifact_name(self) -> str:
|
|
73
206
|
if self.path.is_dir():
|
|
74
207
|
return self.path.stem + ".zip"
|
|
75
208
|
return self.path.name
|
|
@@ -77,13 +210,13 @@ class Artefact:
|
|
|
77
210
|
@property
|
|
78
211
|
def post_build_path(self) -> Path:
|
|
79
212
|
"""
|
|
80
|
-
Returns post-build
|
|
213
|
+
Returns post-build artifact path. Directories are mapped to corresponding .zip files.
|
|
81
214
|
"""
|
|
82
|
-
return self.path.parent / self.
|
|
215
|
+
return self.path.parent / self._artifact_name
|
|
83
216
|
|
|
84
217
|
def upload_path(self, stage: FQN | str | None) -> str:
|
|
85
218
|
"""
|
|
86
|
-
Path on stage to which the
|
|
219
|
+
Path on stage to which the artifact should be uploaded.
|
|
87
220
|
"""
|
|
88
221
|
stage = stage or DEPLOYMENT_STAGE
|
|
89
222
|
if isinstance(stage, str):
|
|
@@ -96,10 +229,10 @@ class Artefact:
|
|
|
96
229
|
|
|
97
230
|
def import_path(self, stage: FQN | str | None) -> str:
|
|
98
231
|
"""Path for UDF/sproc imports clause."""
|
|
99
|
-
return self.upload_path(stage) + self.
|
|
232
|
+
return self.upload_path(stage) + self._artifact_name
|
|
100
233
|
|
|
101
234
|
def build(self) -> None:
|
|
102
|
-
"""Build the
|
|
235
|
+
"""Build the artifact. Applies only to directories. Files are untouched."""
|
|
103
236
|
if not self.path.is_dir():
|
|
104
237
|
return
|
|
105
238
|
cli_console.step(f"Creating: {self.post_build_path.name}")
|
|
@@ -20,6 +20,10 @@ from pathlib import Path
|
|
|
20
20
|
from typing import Dict, List, Literal
|
|
21
21
|
from zipfile import ZIP_DEFLATED, ZipFile
|
|
22
22
|
|
|
23
|
+
from snowflake.cli.api.artifacts.bundle_map import BundleMap
|
|
24
|
+
from snowflake.cli.api.console import cli_console
|
|
25
|
+
from snowflake.cli.api.secure_path import SecurePath
|
|
26
|
+
|
|
23
27
|
log = logging.getLogger(__name__)
|
|
24
28
|
|
|
25
29
|
IGNORED_FILES = [
|
|
@@ -64,7 +68,10 @@ def zip_dir(
|
|
|
64
68
|
mode: Literal["r", "w", "x", "a"] = "w",
|
|
65
69
|
) -> None:
|
|
66
70
|
|
|
67
|
-
if
|
|
71
|
+
if not dest_zip.parent.exists():
|
|
72
|
+
SecurePath(dest_zip).parent.mkdir(parents=True)
|
|
73
|
+
|
|
74
|
+
if isinstance(source, Path) or isinstance(source, SecurePath):
|
|
68
75
|
source = [source]
|
|
69
76
|
|
|
70
77
|
files_to_pack: Dict[Path, List[Path]] = {
|
|
@@ -74,11 +81,36 @@ def zip_dir(
|
|
|
74
81
|
|
|
75
82
|
with ZipFile(dest_zip, mode, ZIP_DEFLATED, allowZip64=True) as package_zip:
|
|
76
83
|
for src, files in files_to_pack.items():
|
|
84
|
+
if isinstance(src, SecurePath):
|
|
85
|
+
src = src.path
|
|
77
86
|
for file in files:
|
|
78
87
|
log.debug("Adding %s to %s", file, dest_zip)
|
|
79
88
|
package_zip.write(file, arcname=file.relative_to(src))
|
|
80
89
|
|
|
81
90
|
|
|
91
|
+
def zip_dir_using_bundle_map(
|
|
92
|
+
bundle_map: BundleMap,
|
|
93
|
+
dest_zip: Path,
|
|
94
|
+
mode: Literal["r", "w", "x", "a"] = "w",
|
|
95
|
+
) -> None:
|
|
96
|
+
if not dest_zip.parent.exists():
|
|
97
|
+
SecurePath(dest_zip).parent.mkdir(parents=True)
|
|
98
|
+
|
|
99
|
+
with ZipFile(dest_zip, mode, ZIP_DEFLATED, allowZip64=True) as package_zip:
|
|
100
|
+
cli_console.step(f"Creating: {dest_zip}")
|
|
101
|
+
for src, _ in bundle_map.all_mappings(expand_directories=True):
|
|
102
|
+
if src.is_file():
|
|
103
|
+
log.debug("Adding %s to %s", src, dest_zip)
|
|
104
|
+
package_zip.write(src, arcname=_path_without_top_level_directory(src))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _path_without_top_level_directory(path: Path) -> str:
|
|
108
|
+
path_parts = path.parts
|
|
109
|
+
if len(path_parts) > 1:
|
|
110
|
+
return str(Path(*path_parts[1:]))
|
|
111
|
+
return str(path)
|
|
112
|
+
|
|
113
|
+
|
|
82
114
|
def _to_be_zipped(file: Path) -> bool:
|
|
83
115
|
for pattern in IGNORED_FILES:
|
|
84
116
|
# This has to be a string because of fnmatch
|
|
@@ -40,7 +40,6 @@ from snowflake.cli.api.constants import ObjectType
|
|
|
40
40
|
from snowflake.cli.api.exceptions import (
|
|
41
41
|
IncompatibleParametersError,
|
|
42
42
|
)
|
|
43
|
-
from snowflake.cli.api.feature_flags import FeatureFlag
|
|
44
43
|
from snowflake.cli.api.identifiers import FQN
|
|
45
44
|
from snowflake.cli.api.output.types import (
|
|
46
45
|
CollectionResult,
|
|
@@ -321,7 +320,6 @@ def logs(
|
|
|
321
320
|
|
|
322
321
|
@app.command(
|
|
323
322
|
requires_connection=True,
|
|
324
|
-
is_enabled=FeatureFlag.ENABLE_SPCS_SERVICE_EVENTS.is_enabled,
|
|
325
323
|
)
|
|
326
324
|
def events(
|
|
327
325
|
name: FQN = ServiceNameArgument,
|
|
@@ -369,7 +367,6 @@ def events(
|
|
|
369
367
|
|
|
370
368
|
@app.command(
|
|
371
369
|
requires_connection=True,
|
|
372
|
-
is_enabled=FeatureFlag.ENABLE_SPCS_SERVICE_METRICS.is_enabled,
|
|
373
370
|
)
|
|
374
371
|
def metrics(
|
|
375
372
|
name: FQN = ServiceNameArgument,
|
|
@@ -192,7 +192,8 @@ def stage_diff(
|
|
|
192
192
|
Diffs a stage with a local folder.
|
|
193
193
|
"""
|
|
194
194
|
diff: DiffResult = compute_stage_diff(
|
|
195
|
-
local_root=Path(folder_name),
|
|
195
|
+
local_root=Path(folder_name),
|
|
196
|
+
stage_path=StageManager.stage_path_parts_from_str(stage_name), # noqa: SLF001
|
|
196
197
|
)
|
|
197
198
|
if get_cli_context().output_format == OutputFormat.JSON:
|
|
198
199
|
return ObjectResult(diff.to_dict())
|
|
@@ -19,13 +19,14 @@ from dataclasses import dataclass, field
|
|
|
19
19
|
from pathlib import Path, PurePosixPath
|
|
20
20
|
from typing import Collection, Dict, List, Optional, Tuple
|
|
21
21
|
|
|
22
|
-
from snowflake.cli.
|
|
22
|
+
from snowflake.cli.api.artifacts.bundle_map import BundleMap
|
|
23
23
|
from snowflake.cli.api.exceptions import (
|
|
24
24
|
SnowflakeSQLExecutionError,
|
|
25
25
|
)
|
|
26
|
+
from snowflake.cli.api.project.util import unquote_identifier
|
|
26
27
|
from snowflake.connector.cursor import DictCursor
|
|
27
28
|
|
|
28
|
-
from .manager import StageManager
|
|
29
|
+
from .manager import StageManager, StagePathParts
|
|
29
30
|
from .md5 import UnknownMD5FormatError, file_matches_md5sum
|
|
30
31
|
|
|
31
32
|
log = logging.getLogger(__name__)
|
|
@@ -83,18 +84,31 @@ def enumerate_files(path: Path) -> List[Path]:
|
|
|
83
84
|
return paths
|
|
84
85
|
|
|
85
86
|
|
|
86
|
-
def
|
|
87
|
-
"""
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def build_md5_map(list_stage_cursor: DictCursor) -> Dict[StagePathType, Optional[str]]:
|
|
87
|
+
def relative_to_stage_path(path: str, stage_path: StagePathParts) -> StagePathType:
|
|
88
|
+
"""
|
|
89
|
+
@param path: file path on the stage.
|
|
90
|
+
@param stage_path: stage path object.
|
|
91
|
+
@return: path of the file relative to the stage and subdirectory
|
|
92
92
|
"""
|
|
93
|
-
|
|
93
|
+
# path is returned from a SQL call so it's unquoted. Unquote stage_path identifiers to match.
|
|
94
|
+
# Stage is always returned in lower-case from ls SQL request
|
|
95
|
+
stage_name = unquote_identifier(stage_path.stage_name).lower()
|
|
96
|
+
stage_subdirectory = stage_path.directory
|
|
97
|
+
path_wo_stage_name = path.removeprefix(stage_name).lstrip("/")
|
|
98
|
+
relative_path = path_wo_stage_name.removeprefix(stage_subdirectory).lstrip("/")
|
|
99
|
+
return StagePathType(relative_path)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def build_md5_map(
|
|
103
|
+
list_stage_cursor: DictCursor, stage_path: StagePathParts
|
|
104
|
+
) -> Dict[StagePathType, Optional[str]]:
|
|
94
105
|
"""
|
|
106
|
+
Returns a mapping of file paths to their md5sums. File paths are relative to the stage and subdirectory.
|
|
107
|
+
"""
|
|
108
|
+
all_files = list_stage_cursor.fetchall()
|
|
95
109
|
return {
|
|
96
|
-
|
|
97
|
-
for file in
|
|
110
|
+
relative_to_stage_path(file["name"], stage_path): file["md5"]
|
|
111
|
+
for file in all_files
|
|
98
112
|
}
|
|
99
113
|
|
|
100
114
|
|
|
@@ -115,50 +129,50 @@ def preserve_from_diff(
|
|
|
115
129
|
return preserved_diff
|
|
116
130
|
|
|
117
131
|
|
|
118
|
-
def compute_stage_diff(
|
|
119
|
-
local_root: Path,
|
|
120
|
-
stage_fqn: str,
|
|
121
|
-
) -> DiffResult:
|
|
132
|
+
def compute_stage_diff(local_root: Path, stage_path: StagePathParts) -> DiffResult:
|
|
122
133
|
"""
|
|
123
|
-
Diffs the files in
|
|
134
|
+
Diffs the files in the local_root with files in the stage path that is stage_path's full_path.
|
|
124
135
|
"""
|
|
125
136
|
stage_manager = StageManager()
|
|
126
137
|
local_files = enumerate_files(local_root)
|
|
127
|
-
|
|
138
|
+
remote_files = stage_manager.list_files(stage_path.full_path)
|
|
139
|
+
|
|
140
|
+
# Create a mapping from remote_file path to file's md5sum. Path is relative to stage_name/directory.
|
|
141
|
+
remote_md5 = build_md5_map(remote_files, stage_path)
|
|
128
142
|
|
|
129
143
|
result: DiffResult = DiffResult()
|
|
130
144
|
|
|
131
145
|
for local_file in local_files:
|
|
132
146
|
relpath = local_file.relative_to(local_root)
|
|
133
|
-
|
|
134
|
-
if
|
|
147
|
+
rel_stage_path = to_stage_path(relpath)
|
|
148
|
+
if rel_stage_path not in remote_md5:
|
|
135
149
|
# doesn't exist on the stage
|
|
136
|
-
result.only_local.append(
|
|
150
|
+
result.only_local.append(rel_stage_path)
|
|
137
151
|
else:
|
|
138
152
|
# N.B. file size on stage is not always accurate, so cannot fail fast
|
|
139
153
|
try:
|
|
140
|
-
if file_matches_md5sum(local_file, remote_md5[
|
|
154
|
+
if file_matches_md5sum(local_file, remote_md5[rel_stage_path]):
|
|
141
155
|
# We are assuming that we will not get accidental collisions here due to the
|
|
142
156
|
# large space of the md5sum (32 * 4 = 128 bits means 1-in-9-trillion chance)
|
|
143
157
|
# combined with the fact that the file name + path must also match elsewhere.
|
|
144
|
-
result.identical.append(
|
|
158
|
+
result.identical.append(rel_stage_path)
|
|
145
159
|
else:
|
|
146
160
|
# either the file has changed, or we can't tell if it has
|
|
147
|
-
result.different.append(
|
|
161
|
+
result.different.append(rel_stage_path)
|
|
148
162
|
except UnknownMD5FormatError:
|
|
149
163
|
log.warning(
|
|
150
164
|
"Could not compare md5 for %s, assuming file has changed",
|
|
151
165
|
local_file,
|
|
152
166
|
exc_info=True,
|
|
153
167
|
)
|
|
154
|
-
result.different.append(
|
|
168
|
+
result.different.append(rel_stage_path)
|
|
155
169
|
|
|
156
170
|
# mark this file as seen
|
|
157
|
-
del remote_md5[
|
|
171
|
+
del remote_md5[rel_stage_path]
|
|
158
172
|
|
|
159
173
|
# every entry here is a file we never saw locally
|
|
160
|
-
for
|
|
161
|
-
result.only_on_stage.append(
|
|
174
|
+
for rel_stage_path in remote_md5.keys():
|
|
175
|
+
result.only_on_stage.append(rel_stage_path)
|
|
162
176
|
|
|
163
177
|
return result
|
|
164
178
|
|
|
@@ -185,7 +199,7 @@ def to_local_path(stage_path: StagePathType) -> Path:
|
|
|
185
199
|
|
|
186
200
|
def delete_only_on_stage_files(
|
|
187
201
|
stage_manager: StageManager,
|
|
188
|
-
|
|
202
|
+
stage_root: str,
|
|
189
203
|
only_on_stage: List[StagePathType],
|
|
190
204
|
role: Optional[str] = None,
|
|
191
205
|
):
|
|
@@ -193,12 +207,12 @@ def delete_only_on_stage_files(
|
|
|
193
207
|
Deletes all files from a Snowflake stage according to the input list of filenames, using a custom role.
|
|
194
208
|
"""
|
|
195
209
|
for _stage_path in only_on_stage:
|
|
196
|
-
stage_manager.remove(stage_name=
|
|
210
|
+
stage_manager.remove(stage_name=stage_root, path=str(_stage_path), role=role)
|
|
197
211
|
|
|
198
212
|
|
|
199
213
|
def put_files_on_stage(
|
|
200
214
|
stage_manager: StageManager,
|
|
201
|
-
|
|
215
|
+
stage_root: str,
|
|
202
216
|
deploy_root_path: Path,
|
|
203
217
|
stage_paths: List[StagePathType],
|
|
204
218
|
role: Optional[str] = None,
|
|
@@ -210,7 +224,7 @@ def put_files_on_stage(
|
|
|
210
224
|
for _stage_path in stage_paths:
|
|
211
225
|
stage_sub_path = get_stage_subpath(_stage_path)
|
|
212
226
|
full_stage_path = (
|
|
213
|
-
f"{
|
|
227
|
+
f"{stage_root}/{stage_sub_path}" if stage_sub_path else stage_root
|
|
214
228
|
)
|
|
215
229
|
stage_manager.put(
|
|
216
230
|
local_path=deploy_root_path / to_local_path(_stage_path),
|
|
@@ -221,7 +235,10 @@ def put_files_on_stage(
|
|
|
221
235
|
|
|
222
236
|
|
|
223
237
|
def sync_local_diff_with_stage(
|
|
224
|
-
role: str | None,
|
|
238
|
+
role: str | None,
|
|
239
|
+
deploy_root_path: Path,
|
|
240
|
+
diff_result: DiffResult,
|
|
241
|
+
stage_full_path: str,
|
|
225
242
|
):
|
|
226
243
|
"""
|
|
227
244
|
Syncs a given local directory's contents with a Snowflake stage, including removing old files, and re-uploading modified and new files.
|
|
@@ -234,18 +251,22 @@ def sync_local_diff_with_stage(
|
|
|
234
251
|
|
|
235
252
|
try:
|
|
236
253
|
delete_only_on_stage_files(
|
|
237
|
-
stage_manager,
|
|
254
|
+
stage_manager, stage_full_path, diff_result.only_on_stage, role
|
|
238
255
|
)
|
|
239
256
|
put_files_on_stage(
|
|
240
|
-
stage_manager,
|
|
241
|
-
|
|
242
|
-
deploy_root_path,
|
|
243
|
-
diff_result.different,
|
|
244
|
-
role,
|
|
257
|
+
stage_manager=stage_manager,
|
|
258
|
+
stage_root=stage_full_path,
|
|
259
|
+
deploy_root_path=deploy_root_path,
|
|
260
|
+
stage_paths=diff_result.different,
|
|
261
|
+
role=role,
|
|
245
262
|
overwrite=True,
|
|
246
263
|
)
|
|
247
264
|
put_files_on_stage(
|
|
248
|
-
stage_manager,
|
|
265
|
+
stage_manager=stage_manager,
|
|
266
|
+
stage_root=stage_full_path,
|
|
267
|
+
deploy_root_path=deploy_root_path,
|
|
268
|
+
stage_paths=diff_result.only_local,
|
|
269
|
+
role=role,
|
|
249
270
|
)
|
|
250
271
|
except Exception as err:
|
|
251
272
|
# Could be ProgrammingError or IntegrityError from SnowflakeCursor
|
|
@@ -41,11 +41,11 @@ from snowflake.cli.api.commands.utils import parse_key_value_variables
|
|
|
41
41
|
from snowflake.cli.api.console import cli_console
|
|
42
42
|
from snowflake.cli.api.constants import PYTHON_3_12
|
|
43
43
|
from snowflake.cli.api.identifiers import FQN
|
|
44
|
-
from snowflake.cli.api.project.util import
|
|
44
|
+
from snowflake.cli.api.project.util import VALID_IDENTIFIER_REGEX, to_string_literal
|
|
45
45
|
from snowflake.cli.api.secure_path import SecurePath
|
|
46
46
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin
|
|
47
47
|
from snowflake.cli.api.stage_path import StagePath
|
|
48
|
-
from snowflake.cli.api.utils.path_utils import path_resolver
|
|
48
|
+
from snowflake.cli.api.utils.path_utils import path_resolver, resolve_without_follow
|
|
49
49
|
from snowflake.connector import DictCursor, ProgrammingError
|
|
50
50
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
51
51
|
|
|
@@ -65,6 +65,7 @@ EXECUTE_SUPPORTED_FILES_FORMATS = (
|
|
|
65
65
|
|
|
66
66
|
# Replace magic numbers with constants
|
|
67
67
|
OMIT_FIRST = slice(1, None)
|
|
68
|
+
STAGE_PATH_REGEX = rf"(?P<prefix>@)?(?:(?P<first_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?:(?P<second_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?P<name>{VALID_IDENTIFIER_REGEX})/?(?P<directory>([^/]*/?)*)?"
|
|
68
69
|
|
|
69
70
|
|
|
70
71
|
@dataclass
|
|
@@ -132,15 +133,27 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
132
133
|
"""
|
|
133
134
|
|
|
134
135
|
def __init__(self, stage_path: str):
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
136
|
+
match = re.fullmatch(STAGE_PATH_REGEX, stage_path)
|
|
137
|
+
if match is None:
|
|
138
|
+
raise ClickException("Invalid stage path")
|
|
139
|
+
self.directory = match.group("directory")
|
|
140
|
+
self._schema = match.group("second_qualifier") or match.group("first_qualifier")
|
|
141
|
+
self.stage = stage_path.removesuffix(self.directory).rstrip("/")
|
|
142
|
+
|
|
143
|
+
stage_name = FQN.from_stage(self.stage).name
|
|
138
144
|
stage_name = (
|
|
139
145
|
stage_name[OMIT_FIRST] if stage_name.startswith("@") else stage_name
|
|
140
146
|
)
|
|
141
147
|
self.stage_name = stage_name
|
|
142
148
|
self.is_directory = True if stage_path.endswith("/") else False
|
|
143
149
|
|
|
150
|
+
@classmethod
|
|
151
|
+
def from_fqn(
|
|
152
|
+
cls, stage_fqn: str, subdir: str | None = None
|
|
153
|
+
) -> DefaultStagePathParts:
|
|
154
|
+
full_path = f"{stage_fqn}/{subdir}" if subdir else stage_fqn
|
|
155
|
+
return cls(full_path)
|
|
156
|
+
|
|
144
157
|
@property
|
|
145
158
|
def path(self) -> str:
|
|
146
159
|
return f"{self.stage_name.rstrip('/')}/{self.directory}".rstrip("/")
|
|
@@ -151,7 +164,7 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
151
164
|
|
|
152
165
|
@property
|
|
153
166
|
def schema(self) -> str | None:
|
|
154
|
-
return
|
|
167
|
+
return self._schema
|
|
155
168
|
|
|
156
169
|
def replace_stage_prefix(self, file_path: str) -> str:
|
|
157
170
|
stage = Path(self.stage).parts[0]
|
|
@@ -345,7 +358,6 @@ class StageManager(SqlExecutionMixin):
|
|
|
345
358
|
|
|
346
359
|
@staticmethod
|
|
347
360
|
def _symlink_or_copy(source_root: Path, source_file_or_dir: Path, dest_dir: Path):
|
|
348
|
-
from snowflake.cli._plugins.nativeapp.artifacts import resolve_without_follow
|
|
349
361
|
|
|
350
362
|
absolute_src = resolve_without_follow(source_file_or_dir)
|
|
351
363
|
dest_path = dest_dir / source_file_or_dir.relative_to(source_root)
|
|
@@ -524,7 +536,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
524
536
|
stage_path_parts.get_standard_stage_path()
|
|
525
537
|
)
|
|
526
538
|
else:
|
|
527
|
-
stage_path_parts = self.
|
|
539
|
+
stage_path_parts = self.stage_path_parts_from_str(stage_path_str)
|
|
528
540
|
stage_path = self.build_path(stage_path_str)
|
|
529
541
|
|
|
530
542
|
all_files_list = self._get_files_list_from_stage(stage_path.root_path())
|
|
@@ -592,12 +604,12 @@ class StageManager(SqlExecutionMixin):
|
|
|
592
604
|
sm = StageManager()
|
|
593
605
|
|
|
594
606
|
# Rewrite stage paths to temporary stage paths. Git paths become stage paths
|
|
595
|
-
original_path_parts = self.
|
|
607
|
+
original_path_parts = self.stage_path_parts_from_str(stage_path) # noqa: SLF001
|
|
596
608
|
|
|
597
609
|
tmp_stage_name = f"snowflake_cli_tmp_stage_{int(time.time())}"
|
|
598
610
|
tmp_stage_fqn = FQN.from_stage(tmp_stage_name).using_connection(conn=self._conn)
|
|
599
611
|
tmp_stage = tmp_stage_fqn.identifier
|
|
600
|
-
stage_path_parts = sm.
|
|
612
|
+
stage_path_parts = sm.stage_path_parts_from_str( # noqa: SLF001
|
|
601
613
|
tmp_stage + "/" + original_path_parts.directory
|
|
602
614
|
)
|
|
603
615
|
|
|
@@ -701,7 +713,8 @@ class StageManager(SqlExecutionMixin):
|
|
|
701
713
|
return StageManager._error_result(file=original_file, msg=e.msg)
|
|
702
714
|
|
|
703
715
|
@staticmethod
|
|
704
|
-
def
|
|
716
|
+
def stage_path_parts_from_str(stage_path: str) -> StagePathParts:
|
|
717
|
+
"""Create StagePathParts object from stage path string."""
|
|
705
718
|
stage_path = StageManager.get_standard_stage_prefix(stage_path)
|
|
706
719
|
if stage_path.startswith(USER_STAGE_PREFIX):
|
|
707
720
|
return UserStagePathParts(stage_path)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
from snowflake.cli._plugins.nativeapp.artifacts import BundleMap
|
|
4
3
|
from snowflake.cli._plugins.stage.diff import (
|
|
5
4
|
DiffResult,
|
|
6
5
|
_to_diff_line,
|
|
7
6
|
_to_src_dest_pair,
|
|
8
7
|
)
|
|
8
|
+
from snowflake.cli.api.artifacts.bundle_map import BundleMap
|
|
9
9
|
from snowflake.cli.api.console import cli_console as cc
|
|
10
10
|
|
|
11
11
|
|
|
@@ -29,6 +29,9 @@ from snowflake.cli._plugins.streamlit.manager import StreamlitManager
|
|
|
29
29
|
from snowflake.cli._plugins.streamlit.streamlit_entity_model import (
|
|
30
30
|
StreamlitEntityModel,
|
|
31
31
|
)
|
|
32
|
+
from snowflake.cli._plugins.streamlit.streamlit_project_paths import (
|
|
33
|
+
StreamlitProjectPaths,
|
|
34
|
+
)
|
|
32
35
|
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
33
36
|
from snowflake.cli.api.commands.decorators import (
|
|
34
37
|
with_experimental_behaviour,
|
|
@@ -156,6 +159,8 @@ def streamlit_deploy(
|
|
|
156
159
|
entity_type="streamlit"
|
|
157
160
|
)
|
|
158
161
|
|
|
162
|
+
streamlit_project_paths = StreamlitProjectPaths(cli_context.project_root)
|
|
163
|
+
|
|
159
164
|
if not streamlits:
|
|
160
165
|
raise NoProjectDefinitionError(
|
|
161
166
|
project_type="streamlit", project_root=cli_context.project_root
|
|
@@ -174,7 +179,11 @@ def streamlit_deploy(
|
|
|
174
179
|
|
|
175
180
|
# Get first streamlit
|
|
176
181
|
streamlit: StreamlitEntityModel = streamlits[entity_id]
|
|
177
|
-
url = StreamlitManager().deploy(
|
|
182
|
+
url = StreamlitManager().deploy(
|
|
183
|
+
streamlit=streamlit,
|
|
184
|
+
streamlit_project_paths=streamlit_project_paths,
|
|
185
|
+
replace=replace,
|
|
186
|
+
)
|
|
178
187
|
|
|
179
188
|
if open_:
|
|
180
189
|
typer.launch(url)
|