snowflake-cli 3.3.0__py3-none-any.whl → 3.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli/__about__.py +1 -1
- snowflake/cli/_app/__main__.py +2 -2
- snowflake/cli/_app/cli_app.py +220 -197
- snowflake/cli/_app/commands_registration/builtin_plugins.py +5 -1
- snowflake/cli/_app/commands_registration/command_plugins_loader.py +3 -1
- snowflake/cli/_app/commands_registration/commands_registration_with_callbacks.py +4 -30
- snowflake/cli/_app/printing.py +2 -2
- snowflake/cli/_plugins/connection/commands.py +2 -4
- snowflake/cli/_plugins/cortex/commands.py +2 -4
- snowflake/cli/_plugins/git/manager.py +1 -1
- snowflake/cli/_plugins/helpers/commands.py +3 -4
- snowflake/cli/_plugins/nativeapp/artifacts.py +6 -624
- snowflake/cli/_plugins/nativeapp/bundle_context.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/compiler.py +1 -3
- snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/commands.py +21 -19
- snowflake/cli/_plugins/nativeapp/entities/application.py +16 -19
- snowflake/cli/_plugins/nativeapp/entities/application_package.py +142 -55
- snowflake/cli/_plugins/nativeapp/release_channel/commands.py +37 -3
- snowflake/cli/_plugins/nativeapp/release_directive/commands.py +80 -2
- snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +224 -44
- snowflake/cli/_plugins/nativeapp/v2_conversions/compat.py +2 -2
- snowflake/cli/_plugins/nativeapp/version/commands.py +1 -1
- snowflake/cli/_plugins/notebook/commands.py +54 -2
- snowflake/cli/_plugins/notebook/exceptions.py +1 -1
- snowflake/cli/_plugins/notebook/manager.py +3 -3
- snowflake/cli/_plugins/notebook/notebook_entity.py +120 -0
- snowflake/cli/_plugins/notebook/notebook_entity_model.py +42 -0
- snowflake/cli/_plugins/notebook/notebook_project_paths.py +15 -0
- snowflake/cli/_plugins/notebook/types.py +3 -0
- snowflake/cli/_plugins/plugin/commands.py +79 -0
- snowflake/cli/_plugins/plugin/manager.py +74 -0
- snowflake/cli/_plugins/plugin/plugin_spec.py +30 -0
- snowflake/cli/_plugins/project/__init__.py +0 -0
- snowflake/cli/_plugins/project/commands.py +157 -0
- snowflake/cli/_plugins/project/feature_flags.py +22 -0
- snowflake/cli/_plugins/project/manager.py +76 -0
- snowflake/cli/_plugins/project/plugin_spec.py +30 -0
- snowflake/cli/_plugins/project/project_entity_model.py +40 -0
- snowflake/cli/_plugins/snowpark/commands.py +49 -30
- snowflake/cli/_plugins/snowpark/common.py +47 -2
- snowflake/cli/_plugins/snowpark/snowpark_entity.py +38 -25
- snowflake/cli/_plugins/snowpark/snowpark_entity_model.py +18 -30
- snowflake/cli/_plugins/snowpark/snowpark_project_paths.py +156 -23
- snowflake/cli/_plugins/snowpark/zipper.py +33 -1
- snowflake/cli/_plugins/spcs/compute_pool/commands.py +53 -5
- snowflake/cli/_plugins/spcs/compute_pool/compute_pool_entity.py +8 -0
- snowflake/cli/_plugins/spcs/compute_pool/compute_pool_entity_model.py +37 -0
- snowflake/cli/_plugins/spcs/compute_pool/manager.py +45 -0
- snowflake/cli/_plugins/spcs/image_repository/commands.py +29 -0
- snowflake/cli/_plugins/spcs/image_repository/image_repository_entity.py +8 -0
- snowflake/cli/_plugins/spcs/image_repository/image_repository_entity_model.py +8 -0
- snowflake/cli/_plugins/spcs/image_repository/manager.py +1 -1
- snowflake/cli/_plugins/spcs/services/commands.py +51 -1
- snowflake/cli/_plugins/spcs/services/manager.py +114 -0
- snowflake/cli/_plugins/spcs/services/service_entity.py +6 -0
- snowflake/cli/_plugins/spcs/services/service_entity_model.py +45 -0
- snowflake/cli/_plugins/spcs/services/service_project_paths.py +15 -0
- snowflake/cli/_plugins/stage/commands.py +2 -1
- snowflake/cli/_plugins/stage/diff.py +60 -39
- snowflake/cli/_plugins/stage/manager.py +26 -13
- snowflake/cli/_plugins/stage/utils.py +1 -1
- snowflake/cli/_plugins/streamlit/commands.py +18 -24
- snowflake/cli/_plugins/streamlit/manager.py +37 -27
- snowflake/cli/_plugins/streamlit/streamlit_entity.py +20 -41
- snowflake/cli/_plugins/streamlit/streamlit_entity_model.py +14 -24
- snowflake/cli/_plugins/streamlit/streamlit_project_paths.py +30 -0
- snowflake/cli/_plugins/workspace/commands.py +3 -3
- snowflake/cli/_plugins/workspace/manager.py +1 -1
- snowflake/cli/api/artifacts/bundle_map.py +500 -0
- snowflake/cli/api/artifacts/common.py +78 -0
- snowflake/cli/api/artifacts/upload.py +51 -0
- snowflake/cli/api/artifacts/utils.py +82 -0
- snowflake/cli/api/cli_global_context.py +14 -1
- snowflake/cli/api/commands/flags.py +34 -13
- snowflake/cli/api/commands/snow_typer.py +12 -0
- snowflake/cli/api/commands/utils.py +30 -2
- snowflake/cli/api/config.py +15 -10
- snowflake/cli/api/constants.py +1 -0
- snowflake/cli/api/entities/common.py +14 -32
- snowflake/cli/api/entities/resolver.py +160 -0
- snowflake/cli/api/entities/utils.py +56 -15
- snowflake/cli/api/errno.py +3 -0
- snowflake/cli/api/exceptions.py +8 -1
- snowflake/cli/api/feature_flags.py +1 -1
- snowflake/cli/api/plugins/plugin_config.py +43 -4
- snowflake/cli/api/project/definition_conversion.py +3 -2
- snowflake/cli/api/project/definition_helper.py +31 -0
- snowflake/cli/api/project/project_paths.py +28 -0
- snowflake/cli/api/project/schemas/entities/common.py +130 -1
- snowflake/cli/api/project/schemas/entities/entities.py +30 -0
- snowflake/cli/api/project/schemas/project_definition.py +27 -0
- snowflake/cli/api/project/schemas/updatable_model.py +2 -2
- snowflake/cli/api/project/schemas/v1/native_app/native_app.py +5 -7
- snowflake/cli/api/secure_path.py +6 -0
- snowflake/cli/api/sql_execution.py +5 -1
- snowflake/cli/api/stage_path.py +7 -2
- snowflake/cli/api/utils/graph.py +3 -0
- snowflake/cli/api/utils/path_utils.py +24 -0
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.5.0.dist-info}/METADATA +12 -13
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.5.0.dist-info}/RECORD +109 -85
- snowflake/cli/_app/api_impl/plugin/plugin_config_provider_impl.py +0 -66
- snowflake/cli/api/__init__.py +0 -48
- snowflake/cli/api/project/schemas/v1/native_app/path_mapping.py +0 -65
- /snowflake/cli/{_app/api_impl → _plugins/plugin}/__init__.py +0 -0
- /snowflake/cli/{_app/api_impl/plugin → api/artifacts}/__init__.py +0 -0
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.5.0.dist-info}/WHEEL +0 -0
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.5.0.dist-info}/entry_points.txt +0 -0
- {snowflake_cli-3.3.0.dist-info → snowflake_cli-3.5.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -35,9 +35,17 @@ from snowflake.cli._plugins.spcs.common import (
|
|
|
35
35
|
new_logs_only,
|
|
36
36
|
strip_empty_lines,
|
|
37
37
|
)
|
|
38
|
+
from snowflake.cli._plugins.spcs.services.service_project_paths import (
|
|
39
|
+
ServiceProjectPaths,
|
|
40
|
+
)
|
|
41
|
+
from snowflake.cli._plugins.stage.manager import StageManager
|
|
42
|
+
from snowflake.cli.api.artifacts.utils import bundle_artifacts
|
|
38
43
|
from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB, ObjectType
|
|
44
|
+
from snowflake.cli.api.identifiers import FQN
|
|
45
|
+
from snowflake.cli.api.project.schemas.entities.common import Artifacts
|
|
39
46
|
from snowflake.cli.api.secure_path import SecurePath
|
|
40
47
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin
|
|
48
|
+
from snowflake.cli.api.stage_path import StagePath
|
|
41
49
|
from snowflake.connector.cursor import DictCursor, SnowflakeCursor
|
|
42
50
|
from snowflake.connector.errors import ProgrammingError
|
|
43
51
|
|
|
@@ -95,6 +103,112 @@ class ServiceManager(SqlExecutionMixin):
|
|
|
95
103
|
except ProgrammingError as e:
|
|
96
104
|
handle_object_already_exists(e, ObjectType.SERVICE, service_name)
|
|
97
105
|
|
|
106
|
+
def deploy(
|
|
107
|
+
self,
|
|
108
|
+
service_name: str,
|
|
109
|
+
stage: str,
|
|
110
|
+
artifacts: List[str],
|
|
111
|
+
compute_pool: str,
|
|
112
|
+
spec_path: Path,
|
|
113
|
+
min_instances: int,
|
|
114
|
+
max_instances: int,
|
|
115
|
+
auto_resume: bool,
|
|
116
|
+
external_access_integrations: Optional[List[str]],
|
|
117
|
+
query_warehouse: Optional[str],
|
|
118
|
+
tags: Optional[List[Tag]],
|
|
119
|
+
comment: Optional[str],
|
|
120
|
+
service_project_paths: ServiceProjectPaths,
|
|
121
|
+
upgrade: bool,
|
|
122
|
+
) -> SnowflakeCursor:
|
|
123
|
+
stage_manager = StageManager()
|
|
124
|
+
stage_manager.create(fqn=FQN.from_stage(stage))
|
|
125
|
+
|
|
126
|
+
stage = stage_manager.get_standard_stage_prefix(stage)
|
|
127
|
+
self._upload_artifacts(
|
|
128
|
+
stage_manager=stage_manager,
|
|
129
|
+
service_project_paths=service_project_paths,
|
|
130
|
+
artifacts=artifacts,
|
|
131
|
+
stage=stage,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
if upgrade:
|
|
135
|
+
self.set_property(
|
|
136
|
+
service_name=service_name,
|
|
137
|
+
min_instances=min_instances,
|
|
138
|
+
max_instances=max_instances,
|
|
139
|
+
query_warehouse=query_warehouse,
|
|
140
|
+
auto_resume=auto_resume,
|
|
141
|
+
external_access_integrations=external_access_integrations,
|
|
142
|
+
comment=comment,
|
|
143
|
+
)
|
|
144
|
+
query = [
|
|
145
|
+
f"ALTER SERVICE {service_name}",
|
|
146
|
+
f"FROM {stage}",
|
|
147
|
+
f"SPECIFICATION_FILE = '{spec_path}'",
|
|
148
|
+
]
|
|
149
|
+
return self.execute_query(strip_empty_lines(query))
|
|
150
|
+
else:
|
|
151
|
+
query = [
|
|
152
|
+
f"CREATE SERVICE {service_name}",
|
|
153
|
+
f"IN COMPUTE POOL {compute_pool}",
|
|
154
|
+
f"FROM {stage}",
|
|
155
|
+
f"SPECIFICATION_FILE = '{spec_path}'",
|
|
156
|
+
f"AUTO_RESUME = {auto_resume}",
|
|
157
|
+
]
|
|
158
|
+
|
|
159
|
+
if min_instances:
|
|
160
|
+
query.append(f"MIN_INSTANCES = {min_instances}")
|
|
161
|
+
|
|
162
|
+
if max_instances:
|
|
163
|
+
query.append(f"MAX_INSTANCES = {max_instances}")
|
|
164
|
+
|
|
165
|
+
if query_warehouse:
|
|
166
|
+
query.append(f"QUERY_WAREHOUSE = {query_warehouse}")
|
|
167
|
+
|
|
168
|
+
if external_access_integrations:
|
|
169
|
+
external_access_integration_list = ",".join(
|
|
170
|
+
f"{e}" for e in external_access_integrations
|
|
171
|
+
)
|
|
172
|
+
query.append(
|
|
173
|
+
f"EXTERNAL_ACCESS_INTEGRATIONS = ({external_access_integration_list})"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
if comment:
|
|
177
|
+
query.append(f"COMMENT = {comment}")
|
|
178
|
+
|
|
179
|
+
if tags:
|
|
180
|
+
tag_list = ",".join(
|
|
181
|
+
f"{t.name}={t.value_string_literal()}" for t in tags
|
|
182
|
+
)
|
|
183
|
+
query.append(f"WITH TAG ({tag_list})")
|
|
184
|
+
|
|
185
|
+
try:
|
|
186
|
+
return self.execute_query(strip_empty_lines(query))
|
|
187
|
+
except ProgrammingError as e:
|
|
188
|
+
handle_object_already_exists(e, ObjectType.SERVICE, service_name)
|
|
189
|
+
|
|
190
|
+
@staticmethod
|
|
191
|
+
def _upload_artifacts(
|
|
192
|
+
stage_manager: StageManager,
|
|
193
|
+
service_project_paths: ServiceProjectPaths,
|
|
194
|
+
artifacts: Artifacts,
|
|
195
|
+
stage: str,
|
|
196
|
+
):
|
|
197
|
+
if not artifacts:
|
|
198
|
+
raise ValueError("Service needs to have artifacts to deploy")
|
|
199
|
+
|
|
200
|
+
bundle_map = bundle_artifacts(service_project_paths, artifacts)
|
|
201
|
+
for absolute_src, absolute_dest in bundle_map.all_mappings(
|
|
202
|
+
absolute=True, expand_directories=True
|
|
203
|
+
):
|
|
204
|
+
# We treat the bundle/service root as deploy root
|
|
205
|
+
stage_path = StagePath.from_stage_str(stage) / (
|
|
206
|
+
absolute_dest.relative_to(service_project_paths.bundle_root).parent
|
|
207
|
+
)
|
|
208
|
+
stage_manager.put(
|
|
209
|
+
local_path=absolute_dest, stage_path=stage_path, overwrite=True
|
|
210
|
+
)
|
|
211
|
+
|
|
98
212
|
def execute_job(
|
|
99
213
|
self,
|
|
100
214
|
job_service_name: str,
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import List, Literal, Optional
|
|
3
|
+
|
|
4
|
+
from pydantic import Field, field_validator
|
|
5
|
+
from snowflake.cli._plugins.object.common import Tag
|
|
6
|
+
from snowflake.cli.api.project.schemas.entities.common import (
|
|
7
|
+
EntityModelBaseWithArtifacts,
|
|
8
|
+
ExternalAccessBaseModel,
|
|
9
|
+
)
|
|
10
|
+
from snowflake.cli.api.project.schemas.updatable_model import DiscriminatorField
|
|
11
|
+
from snowflake.cli.api.project.util import to_string_literal
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ServiceEntityModel(EntityModelBaseWithArtifacts, ExternalAccessBaseModel):
|
|
15
|
+
type: Literal["service"] = DiscriminatorField() # noqa: A003
|
|
16
|
+
stage: str = Field(
|
|
17
|
+
title="Stage where the service specification file is located", default=None
|
|
18
|
+
)
|
|
19
|
+
compute_pool: str = Field(title="Compute pool to run the service on", default=None)
|
|
20
|
+
spec_file: Path = Field(
|
|
21
|
+
title="Path to service specification file on stage", default=None
|
|
22
|
+
)
|
|
23
|
+
min_instances: Optional[int] = Field(
|
|
24
|
+
title="Minimum number of instances", default=1, ge=1
|
|
25
|
+
)
|
|
26
|
+
max_instances: Optional[int] = Field(
|
|
27
|
+
title="Maximum number of instances", default=None, ge=1
|
|
28
|
+
)
|
|
29
|
+
auto_resume: bool = Field(
|
|
30
|
+
title="The service will automatically resume when a service function or ingress is called.",
|
|
31
|
+
default=True,
|
|
32
|
+
)
|
|
33
|
+
query_warehouse: Optional[str] = Field(
|
|
34
|
+
title="Warehouse to use if a service container connects to Snowflake to execute a query without explicitly specifying a warehouse to use",
|
|
35
|
+
default=None,
|
|
36
|
+
)
|
|
37
|
+
tags: Optional[List[Tag]] = Field(title="Tag for the service", default=None)
|
|
38
|
+
comment: Optional[str] = Field(title="Comment for the service", default=None)
|
|
39
|
+
|
|
40
|
+
@field_validator("comment")
|
|
41
|
+
@classmethod
|
|
42
|
+
def _convert_artifacts(cls, comment: Optional[str]):
|
|
43
|
+
if comment:
|
|
44
|
+
return to_string_literal(comment)
|
|
45
|
+
return comment
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from snowflake.cli.api.project.project_paths import ProjectPaths, bundle_root
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class ServiceProjectPaths(ProjectPaths):
|
|
9
|
+
"""
|
|
10
|
+
This class allows you to manage files paths related to given project.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
@property
|
|
14
|
+
def bundle_root(self) -> Path:
|
|
15
|
+
return bundle_root(self.project_root, "service")
|
|
@@ -192,7 +192,8 @@ def stage_diff(
|
|
|
192
192
|
Diffs a stage with a local folder.
|
|
193
193
|
"""
|
|
194
194
|
diff: DiffResult = compute_stage_diff(
|
|
195
|
-
local_root=Path(folder_name),
|
|
195
|
+
local_root=Path(folder_name),
|
|
196
|
+
stage_path=StageManager.stage_path_parts_from_str(stage_name), # noqa: SLF001
|
|
196
197
|
)
|
|
197
198
|
if get_cli_context().output_format == OutputFormat.JSON:
|
|
198
199
|
return ObjectResult(diff.to_dict())
|
|
@@ -19,13 +19,14 @@ from dataclasses import dataclass, field
|
|
|
19
19
|
from pathlib import Path, PurePosixPath
|
|
20
20
|
from typing import Collection, Dict, List, Optional, Tuple
|
|
21
21
|
|
|
22
|
-
from snowflake.cli.
|
|
22
|
+
from snowflake.cli.api.artifacts.bundle_map import BundleMap
|
|
23
23
|
from snowflake.cli.api.exceptions import (
|
|
24
24
|
SnowflakeSQLExecutionError,
|
|
25
25
|
)
|
|
26
|
+
from snowflake.cli.api.project.util import unquote_identifier
|
|
26
27
|
from snowflake.connector.cursor import DictCursor
|
|
27
28
|
|
|
28
|
-
from .manager import StageManager
|
|
29
|
+
from .manager import StageManager, StagePathParts
|
|
29
30
|
from .md5 import UnknownMD5FormatError, file_matches_md5sum
|
|
30
31
|
|
|
31
32
|
log = logging.getLogger(__name__)
|
|
@@ -83,18 +84,31 @@ def enumerate_files(path: Path) -> List[Path]:
|
|
|
83
84
|
return paths
|
|
84
85
|
|
|
85
86
|
|
|
86
|
-
def
|
|
87
|
-
"""
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def build_md5_map(list_stage_cursor: DictCursor) -> Dict[StagePathType, Optional[str]]:
|
|
87
|
+
def relative_to_stage_path(path: str, stage_path: StagePathParts) -> StagePathType:
|
|
88
|
+
"""
|
|
89
|
+
@param path: file path on the stage.
|
|
90
|
+
@param stage_path: stage path object.
|
|
91
|
+
@return: path of the file relative to the stage and subdirectory
|
|
92
92
|
"""
|
|
93
|
-
|
|
93
|
+
# path is returned from a SQL call so it's unquoted. Unquote stage_path identifiers to match.
|
|
94
|
+
# Stage is always returned in lower-case from ls SQL request
|
|
95
|
+
stage_name = unquote_identifier(stage_path.stage_name).lower()
|
|
96
|
+
stage_subdirectory = stage_path.directory
|
|
97
|
+
path_wo_stage_name = path.removeprefix(stage_name).lstrip("/")
|
|
98
|
+
relative_path = path_wo_stage_name.removeprefix(stage_subdirectory).lstrip("/")
|
|
99
|
+
return StagePathType(relative_path)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def build_md5_map(
|
|
103
|
+
list_stage_cursor: DictCursor, stage_path: StagePathParts
|
|
104
|
+
) -> Dict[StagePathType, Optional[str]]:
|
|
94
105
|
"""
|
|
106
|
+
Returns a mapping of file paths to their md5sums. File paths are relative to the stage and subdirectory.
|
|
107
|
+
"""
|
|
108
|
+
all_files = list_stage_cursor.fetchall()
|
|
95
109
|
return {
|
|
96
|
-
|
|
97
|
-
for file in
|
|
110
|
+
relative_to_stage_path(file["name"], stage_path): file["md5"]
|
|
111
|
+
for file in all_files
|
|
98
112
|
}
|
|
99
113
|
|
|
100
114
|
|
|
@@ -115,50 +129,50 @@ def preserve_from_diff(
|
|
|
115
129
|
return preserved_diff
|
|
116
130
|
|
|
117
131
|
|
|
118
|
-
def compute_stage_diff(
|
|
119
|
-
local_root: Path,
|
|
120
|
-
stage_fqn: str,
|
|
121
|
-
) -> DiffResult:
|
|
132
|
+
def compute_stage_diff(local_root: Path, stage_path: StagePathParts) -> DiffResult:
|
|
122
133
|
"""
|
|
123
|
-
Diffs the files in
|
|
134
|
+
Diffs the files in the local_root with files in the stage path that is stage_path's full_path.
|
|
124
135
|
"""
|
|
125
136
|
stage_manager = StageManager()
|
|
126
137
|
local_files = enumerate_files(local_root)
|
|
127
|
-
|
|
138
|
+
remote_files = stage_manager.list_files(stage_path.full_path)
|
|
139
|
+
|
|
140
|
+
# Create a mapping from remote_file path to file's md5sum. Path is relative to stage_name/directory.
|
|
141
|
+
remote_md5 = build_md5_map(remote_files, stage_path)
|
|
128
142
|
|
|
129
143
|
result: DiffResult = DiffResult()
|
|
130
144
|
|
|
131
145
|
for local_file in local_files:
|
|
132
146
|
relpath = local_file.relative_to(local_root)
|
|
133
|
-
|
|
134
|
-
if
|
|
147
|
+
rel_stage_path = to_stage_path(relpath)
|
|
148
|
+
if rel_stage_path not in remote_md5:
|
|
135
149
|
# doesn't exist on the stage
|
|
136
|
-
result.only_local.append(
|
|
150
|
+
result.only_local.append(rel_stage_path)
|
|
137
151
|
else:
|
|
138
152
|
# N.B. file size on stage is not always accurate, so cannot fail fast
|
|
139
153
|
try:
|
|
140
|
-
if file_matches_md5sum(local_file, remote_md5[
|
|
154
|
+
if file_matches_md5sum(local_file, remote_md5[rel_stage_path]):
|
|
141
155
|
# We are assuming that we will not get accidental collisions here due to the
|
|
142
156
|
# large space of the md5sum (32 * 4 = 128 bits means 1-in-9-trillion chance)
|
|
143
157
|
# combined with the fact that the file name + path must also match elsewhere.
|
|
144
|
-
result.identical.append(
|
|
158
|
+
result.identical.append(rel_stage_path)
|
|
145
159
|
else:
|
|
146
160
|
# either the file has changed, or we can't tell if it has
|
|
147
|
-
result.different.append(
|
|
161
|
+
result.different.append(rel_stage_path)
|
|
148
162
|
except UnknownMD5FormatError:
|
|
149
163
|
log.warning(
|
|
150
164
|
"Could not compare md5 for %s, assuming file has changed",
|
|
151
165
|
local_file,
|
|
152
166
|
exc_info=True,
|
|
153
167
|
)
|
|
154
|
-
result.different.append(
|
|
168
|
+
result.different.append(rel_stage_path)
|
|
155
169
|
|
|
156
170
|
# mark this file as seen
|
|
157
|
-
del remote_md5[
|
|
171
|
+
del remote_md5[rel_stage_path]
|
|
158
172
|
|
|
159
173
|
# every entry here is a file we never saw locally
|
|
160
|
-
for
|
|
161
|
-
result.only_on_stage.append(
|
|
174
|
+
for rel_stage_path in remote_md5.keys():
|
|
175
|
+
result.only_on_stage.append(rel_stage_path)
|
|
162
176
|
|
|
163
177
|
return result
|
|
164
178
|
|
|
@@ -185,7 +199,7 @@ def to_local_path(stage_path: StagePathType) -> Path:
|
|
|
185
199
|
|
|
186
200
|
def delete_only_on_stage_files(
|
|
187
201
|
stage_manager: StageManager,
|
|
188
|
-
|
|
202
|
+
stage_root: str,
|
|
189
203
|
only_on_stage: List[StagePathType],
|
|
190
204
|
role: Optional[str] = None,
|
|
191
205
|
):
|
|
@@ -193,12 +207,12 @@ def delete_only_on_stage_files(
|
|
|
193
207
|
Deletes all files from a Snowflake stage according to the input list of filenames, using a custom role.
|
|
194
208
|
"""
|
|
195
209
|
for _stage_path in only_on_stage:
|
|
196
|
-
stage_manager.remove(stage_name=
|
|
210
|
+
stage_manager.remove(stage_name=stage_root, path=str(_stage_path), role=role)
|
|
197
211
|
|
|
198
212
|
|
|
199
213
|
def put_files_on_stage(
|
|
200
214
|
stage_manager: StageManager,
|
|
201
|
-
|
|
215
|
+
stage_root: str,
|
|
202
216
|
deploy_root_path: Path,
|
|
203
217
|
stage_paths: List[StagePathType],
|
|
204
218
|
role: Optional[str] = None,
|
|
@@ -210,7 +224,7 @@ def put_files_on_stage(
|
|
|
210
224
|
for _stage_path in stage_paths:
|
|
211
225
|
stage_sub_path = get_stage_subpath(_stage_path)
|
|
212
226
|
full_stage_path = (
|
|
213
|
-
f"{
|
|
227
|
+
f"{stage_root}/{stage_sub_path}" if stage_sub_path else stage_root
|
|
214
228
|
)
|
|
215
229
|
stage_manager.put(
|
|
216
230
|
local_path=deploy_root_path / to_local_path(_stage_path),
|
|
@@ -221,7 +235,10 @@ def put_files_on_stage(
|
|
|
221
235
|
|
|
222
236
|
|
|
223
237
|
def sync_local_diff_with_stage(
|
|
224
|
-
role: str | None,
|
|
238
|
+
role: str | None,
|
|
239
|
+
deploy_root_path: Path,
|
|
240
|
+
diff_result: DiffResult,
|
|
241
|
+
stage_full_path: str,
|
|
225
242
|
):
|
|
226
243
|
"""
|
|
227
244
|
Syncs a given local directory's contents with a Snowflake stage, including removing old files, and re-uploading modified and new files.
|
|
@@ -234,18 +251,22 @@ def sync_local_diff_with_stage(
|
|
|
234
251
|
|
|
235
252
|
try:
|
|
236
253
|
delete_only_on_stage_files(
|
|
237
|
-
stage_manager,
|
|
254
|
+
stage_manager, stage_full_path, diff_result.only_on_stage, role
|
|
238
255
|
)
|
|
239
256
|
put_files_on_stage(
|
|
240
|
-
stage_manager,
|
|
241
|
-
|
|
242
|
-
deploy_root_path,
|
|
243
|
-
diff_result.different,
|
|
244
|
-
role,
|
|
257
|
+
stage_manager=stage_manager,
|
|
258
|
+
stage_root=stage_full_path,
|
|
259
|
+
deploy_root_path=deploy_root_path,
|
|
260
|
+
stage_paths=diff_result.different,
|
|
261
|
+
role=role,
|
|
245
262
|
overwrite=True,
|
|
246
263
|
)
|
|
247
264
|
put_files_on_stage(
|
|
248
|
-
stage_manager,
|
|
265
|
+
stage_manager=stage_manager,
|
|
266
|
+
stage_root=stage_full_path,
|
|
267
|
+
deploy_root_path=deploy_root_path,
|
|
268
|
+
stage_paths=diff_result.only_local,
|
|
269
|
+
role=role,
|
|
249
270
|
)
|
|
250
271
|
except Exception as err:
|
|
251
272
|
# Could be ProgrammingError or IntegrityError from SnowflakeCursor
|
|
@@ -41,11 +41,11 @@ from snowflake.cli.api.commands.utils import parse_key_value_variables
|
|
|
41
41
|
from snowflake.cli.api.console import cli_console
|
|
42
42
|
from snowflake.cli.api.constants import PYTHON_3_12
|
|
43
43
|
from snowflake.cli.api.identifiers import FQN
|
|
44
|
-
from snowflake.cli.api.project.util import
|
|
44
|
+
from snowflake.cli.api.project.util import VALID_IDENTIFIER_REGEX, to_string_literal
|
|
45
45
|
from snowflake.cli.api.secure_path import SecurePath
|
|
46
46
|
from snowflake.cli.api.sql_execution import SqlExecutionMixin
|
|
47
47
|
from snowflake.cli.api.stage_path import StagePath
|
|
48
|
-
from snowflake.cli.api.utils.path_utils import path_resolver
|
|
48
|
+
from snowflake.cli.api.utils.path_utils import path_resolver, resolve_without_follow
|
|
49
49
|
from snowflake.connector import DictCursor, ProgrammingError
|
|
50
50
|
from snowflake.connector.cursor import SnowflakeCursor
|
|
51
51
|
|
|
@@ -65,6 +65,7 @@ EXECUTE_SUPPORTED_FILES_FORMATS = (
|
|
|
65
65
|
|
|
66
66
|
# Replace magic numbers with constants
|
|
67
67
|
OMIT_FIRST = slice(1, None)
|
|
68
|
+
STAGE_PATH_REGEX = rf"(?P<prefix>@)?(?:(?P<first_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?:(?P<second_qualifier>{VALID_IDENTIFIER_REGEX})\.)?(?P<name>{VALID_IDENTIFIER_REGEX})/?(?P<directory>([^/]*/?)*)?"
|
|
68
69
|
|
|
69
70
|
|
|
70
71
|
@dataclass
|
|
@@ -132,15 +133,27 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
132
133
|
"""
|
|
133
134
|
|
|
134
135
|
def __init__(self, stage_path: str):
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
136
|
+
match = re.fullmatch(STAGE_PATH_REGEX, stage_path)
|
|
137
|
+
if match is None:
|
|
138
|
+
raise ClickException("Invalid stage path")
|
|
139
|
+
self.directory = match.group("directory")
|
|
140
|
+
self._schema = match.group("second_qualifier") or match.group("first_qualifier")
|
|
141
|
+
self.stage = stage_path.removesuffix(self.directory).rstrip("/")
|
|
142
|
+
|
|
143
|
+
stage_name = FQN.from_stage(self.stage).name
|
|
138
144
|
stage_name = (
|
|
139
145
|
stage_name[OMIT_FIRST] if stage_name.startswith("@") else stage_name
|
|
140
146
|
)
|
|
141
147
|
self.stage_name = stage_name
|
|
142
148
|
self.is_directory = True if stage_path.endswith("/") else False
|
|
143
149
|
|
|
150
|
+
@classmethod
|
|
151
|
+
def from_fqn(
|
|
152
|
+
cls, stage_fqn: str, subdir: str | None = None
|
|
153
|
+
) -> DefaultStagePathParts:
|
|
154
|
+
full_path = f"{stage_fqn}/{subdir}" if subdir else stage_fqn
|
|
155
|
+
return cls(full_path)
|
|
156
|
+
|
|
144
157
|
@property
|
|
145
158
|
def path(self) -> str:
|
|
146
159
|
return f"{self.stage_name.rstrip('/')}/{self.directory}".rstrip("/")
|
|
@@ -151,7 +164,7 @@ class DefaultStagePathParts(StagePathParts):
|
|
|
151
164
|
|
|
152
165
|
@property
|
|
153
166
|
def schema(self) -> str | None:
|
|
154
|
-
return
|
|
167
|
+
return self._schema
|
|
155
168
|
|
|
156
169
|
def replace_stage_prefix(self, file_path: str) -> str:
|
|
157
170
|
stage = Path(self.stage).parts[0]
|
|
@@ -345,7 +358,6 @@ class StageManager(SqlExecutionMixin):
|
|
|
345
358
|
|
|
346
359
|
@staticmethod
|
|
347
360
|
def _symlink_or_copy(source_root: Path, source_file_or_dir: Path, dest_dir: Path):
|
|
348
|
-
from snowflake.cli._plugins.nativeapp.artifacts import resolve_without_follow
|
|
349
361
|
|
|
350
362
|
absolute_src = resolve_without_follow(source_file_or_dir)
|
|
351
363
|
dest_path = dest_dir / source_file_or_dir.relative_to(source_root)
|
|
@@ -524,7 +536,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
524
536
|
stage_path_parts.get_standard_stage_path()
|
|
525
537
|
)
|
|
526
538
|
else:
|
|
527
|
-
stage_path_parts = self.
|
|
539
|
+
stage_path_parts = self.stage_path_parts_from_str(stage_path_str)
|
|
528
540
|
stage_path = self.build_path(stage_path_str)
|
|
529
541
|
|
|
530
542
|
all_files_list = self._get_files_list_from_stage(stage_path.root_path())
|
|
@@ -549,7 +561,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
549
561
|
)
|
|
550
562
|
|
|
551
563
|
parsed_variables = parse_key_value_variables(variables)
|
|
552
|
-
sql_variables = self.
|
|
564
|
+
sql_variables = self.parse_execute_variables(parsed_variables)
|
|
553
565
|
python_variables = self._parse_python_variables(parsed_variables)
|
|
554
566
|
results = []
|
|
555
567
|
|
|
@@ -592,12 +604,12 @@ class StageManager(SqlExecutionMixin):
|
|
|
592
604
|
sm = StageManager()
|
|
593
605
|
|
|
594
606
|
# Rewrite stage paths to temporary stage paths. Git paths become stage paths
|
|
595
|
-
original_path_parts = self.
|
|
607
|
+
original_path_parts = self.stage_path_parts_from_str(stage_path) # noqa: SLF001
|
|
596
608
|
|
|
597
609
|
tmp_stage_name = f"snowflake_cli_tmp_stage_{int(time.time())}"
|
|
598
610
|
tmp_stage_fqn = FQN.from_stage(tmp_stage_name).using_connection(conn=self._conn)
|
|
599
611
|
tmp_stage = tmp_stage_fqn.identifier
|
|
600
|
-
stage_path_parts = sm.
|
|
612
|
+
stage_path_parts = sm.stage_path_parts_from_str( # noqa: SLF001
|
|
601
613
|
tmp_stage + "/" + original_path_parts.directory
|
|
602
614
|
)
|
|
603
615
|
|
|
@@ -651,7 +663,7 @@ class StageManager(SqlExecutionMixin):
|
|
|
651
663
|
return [f for f in files if Path(f).suffix in EXECUTE_SUPPORTED_FILES_FORMATS]
|
|
652
664
|
|
|
653
665
|
@staticmethod
|
|
654
|
-
def
|
|
666
|
+
def parse_execute_variables(variables: List[Variable]) -> Optional[str]:
|
|
655
667
|
if not variables:
|
|
656
668
|
return None
|
|
657
669
|
query_parameters = [f"{v.key}=>{v.value}" for v in variables]
|
|
@@ -701,7 +713,8 @@ class StageManager(SqlExecutionMixin):
|
|
|
701
713
|
return StageManager._error_result(file=original_file, msg=e.msg)
|
|
702
714
|
|
|
703
715
|
@staticmethod
|
|
704
|
-
def
|
|
716
|
+
def stage_path_parts_from_str(stage_path: str) -> StagePathParts:
|
|
717
|
+
"""Create StagePathParts object from stage path string."""
|
|
705
718
|
stage_path = StageManager.get_standard_stage_prefix(stage_path)
|
|
706
719
|
if stage_path.startswith(USER_STAGE_PREFIX):
|
|
707
720
|
return UserStagePathParts(stage_path)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
from snowflake.cli._plugins.nativeapp.artifacts import BundleMap
|
|
4
3
|
from snowflake.cli._plugins.stage.diff import (
|
|
5
4
|
DiffResult,
|
|
6
5
|
_to_diff_line,
|
|
7
6
|
_to_src_dest_pair,
|
|
8
7
|
)
|
|
8
|
+
from snowflake.cli.api.artifacts.bundle_map import BundleMap
|
|
9
9
|
from snowflake.cli.api.console import cli_console as cc
|
|
10
10
|
|
|
11
11
|
|
|
@@ -16,11 +16,10 @@ from __future__ import annotations
|
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
18
|
from pathlib import Path
|
|
19
|
-
from typing import Dict
|
|
20
19
|
|
|
21
20
|
import click
|
|
22
21
|
import typer
|
|
23
|
-
from click import ClickException
|
|
22
|
+
from click import ClickException
|
|
24
23
|
from snowflake.cli._plugins.object.command_aliases import (
|
|
25
24
|
add_object_command_aliases,
|
|
26
25
|
scope_option,
|
|
@@ -29,6 +28,9 @@ from snowflake.cli._plugins.streamlit.manager import StreamlitManager
|
|
|
29
28
|
from snowflake.cli._plugins.streamlit.streamlit_entity_model import (
|
|
30
29
|
StreamlitEntityModel,
|
|
31
30
|
)
|
|
31
|
+
from snowflake.cli._plugins.streamlit.streamlit_project_paths import (
|
|
32
|
+
StreamlitProjectPaths,
|
|
33
|
+
)
|
|
32
34
|
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
33
35
|
from snowflake.cli.api.commands.decorators import (
|
|
34
36
|
with_experimental_behaviour,
|
|
@@ -41,6 +43,7 @@ from snowflake.cli.api.commands.flags import (
|
|
|
41
43
|
like_option,
|
|
42
44
|
)
|
|
43
45
|
from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
|
|
46
|
+
from snowflake.cli.api.commands.utils import get_entity_for_operation
|
|
44
47
|
from snowflake.cli.api.constants import ObjectType
|
|
45
48
|
from snowflake.cli.api.exceptions import NoProjectDefinitionError
|
|
46
49
|
from snowflake.cli.api.identifiers import FQN
|
|
@@ -130,7 +133,8 @@ def _default_file_callback(param_name: str):
|
|
|
130
133
|
@with_experimental_behaviour()
|
|
131
134
|
def streamlit_deploy(
|
|
132
135
|
replace: bool = ReplaceOption(
|
|
133
|
-
help="
|
|
136
|
+
help="Replaces the Streamlit app if it already exists. It only uploads new and overwrites existing files, "
|
|
137
|
+
"but does not remove any files already on the stage."
|
|
134
138
|
),
|
|
135
139
|
entity_id: str = entity_argument("streamlit"),
|
|
136
140
|
open_: bool = OpenOption,
|
|
@@ -152,29 +156,19 @@ def streamlit_deploy(
|
|
|
152
156
|
)
|
|
153
157
|
pd = convert_project_definition_to_v2(cli_context.project_root, pd)
|
|
154
158
|
|
|
155
|
-
|
|
156
|
-
|
|
159
|
+
streamlit: StreamlitEntityModel = get_entity_for_operation(
|
|
160
|
+
cli_context=cli_context,
|
|
161
|
+
entity_id=entity_id,
|
|
162
|
+
project_definition=pd,
|
|
163
|
+
entity_type="streamlit",
|
|
157
164
|
)
|
|
158
165
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
raise UsageError(f"No '{entity_id}' entity in project definition file.")
|
|
166
|
-
|
|
167
|
-
if len(streamlits.keys()) == 1:
|
|
168
|
-
entity_id = list(streamlits.keys())[0]
|
|
169
|
-
|
|
170
|
-
if entity_id is None:
|
|
171
|
-
raise UsageError(
|
|
172
|
-
"Multiple Streamlit apps found. Please provide entity id for the operation."
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
# Get first streamlit
|
|
176
|
-
streamlit: StreamlitEntityModel = streamlits[entity_id]
|
|
177
|
-
url = StreamlitManager().deploy(streamlit=streamlit, replace=replace)
|
|
166
|
+
streamlit_project_paths = StreamlitProjectPaths(cli_context.project_root)
|
|
167
|
+
url = StreamlitManager().deploy(
|
|
168
|
+
streamlit=streamlit,
|
|
169
|
+
streamlit_project_paths=streamlit_project_paths,
|
|
170
|
+
replace=replace,
|
|
171
|
+
)
|
|
178
172
|
|
|
179
173
|
if open_:
|
|
180
174
|
typer.launch(url)
|