snowflake-cli 3.0.2__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli/__about__.py +1 -1
- snowflake/cli/_app/cli_app.py +3 -0
- snowflake/cli/_app/dev/docs/templates/overview.rst.jinja2 +1 -1
- snowflake/cli/_app/dev/docs/templates/usage.rst.jinja2 +2 -2
- snowflake/cli/_app/telemetry.py +69 -4
- snowflake/cli/_plugins/connection/commands.py +152 -99
- snowflake/cli/_plugins/connection/util.py +54 -9
- snowflake/cli/_plugins/cortex/manager.py +1 -1
- snowflake/cli/_plugins/git/commands.py +6 -3
- snowflake/cli/_plugins/git/manager.py +9 -4
- snowflake/cli/_plugins/nativeapp/artifacts.py +77 -13
- snowflake/cli/_plugins/nativeapp/codegen/artifact_processor.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/compiler.py +7 -0
- snowflake/cli/_plugins/nativeapp/codegen/sandbox.py +10 -10
- snowflake/cli/_plugins/nativeapp/codegen/setup/native_app_setup_processor.py +2 -2
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/extension_function_utils.py +1 -1
- snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +8 -8
- snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +5 -3
- snowflake/cli/_plugins/nativeapp/commands.py +144 -188
- snowflake/cli/_plugins/nativeapp/constants.py +1 -0
- snowflake/cli/_plugins/nativeapp/entities/application.py +564 -351
- snowflake/cli/_plugins/nativeapp/entities/application_package.py +583 -929
- snowflake/cli/_plugins/nativeapp/entities/models/event_sharing_telemetry.py +58 -0
- snowflake/cli/_plugins/nativeapp/exceptions.py +12 -0
- snowflake/cli/_plugins/nativeapp/same_account_install_method.py +0 -2
- snowflake/cli/_plugins/nativeapp/sf_facade.py +30 -0
- snowflake/cli/_plugins/nativeapp/sf_facade_constants.py +25 -0
- snowflake/cli/_plugins/nativeapp/sf_facade_exceptions.py +117 -0
- snowflake/cli/_plugins/nativeapp/sf_sql_facade.py +525 -0
- snowflake/cli/_plugins/nativeapp/v2_conversions/{v2_to_v1_decorator.py → compat.py} +88 -117
- snowflake/cli/_plugins/nativeapp/version/commands.py +36 -32
- snowflake/cli/_plugins/notebook/manager.py +2 -2
- snowflake/cli/_plugins/object/commands.py +10 -1
- snowflake/cli/_plugins/object/manager.py +13 -5
- snowflake/cli/_plugins/snowpark/common.py +63 -21
- snowflake/cli/_plugins/snowpark/package/anaconda_packages.py +3 -3
- snowflake/cli/_plugins/spcs/common.py +29 -0
- snowflake/cli/_plugins/spcs/compute_pool/manager.py +7 -9
- snowflake/cli/_plugins/spcs/image_registry/manager.py +2 -2
- snowflake/cli/_plugins/spcs/image_repository/commands.py +4 -37
- snowflake/cli/_plugins/spcs/image_repository/manager.py +4 -1
- snowflake/cli/_plugins/spcs/services/commands.py +100 -17
- snowflake/cli/_plugins/spcs/services/manager.py +108 -16
- snowflake/cli/_plugins/sql/commands.py +9 -1
- snowflake/cli/_plugins/sql/manager.py +9 -4
- snowflake/cli/_plugins/stage/commands.py +28 -19
- snowflake/cli/_plugins/stage/diff.py +17 -17
- snowflake/cli/_plugins/stage/manager.py +304 -84
- snowflake/cli/_plugins/stage/md5.py +1 -1
- snowflake/cli/_plugins/streamlit/manager.py +5 -5
- snowflake/cli/_plugins/workspace/commands.py +27 -4
- snowflake/cli/_plugins/workspace/context.py +38 -0
- snowflake/cli/_plugins/workspace/manager.py +23 -13
- snowflake/cli/api/cli_global_context.py +4 -3
- snowflake/cli/api/commands/flags.py +23 -7
- snowflake/cli/api/config.py +30 -9
- snowflake/cli/api/connections.py +12 -1
- snowflake/cli/api/console/console.py +4 -19
- snowflake/cli/api/entities/common.py +4 -2
- snowflake/cli/api/entities/utils.py +36 -69
- snowflake/cli/api/errno.py +2 -0
- snowflake/cli/api/exceptions.py +41 -0
- snowflake/cli/api/identifiers.py +8 -0
- snowflake/cli/api/metrics.py +223 -7
- snowflake/cli/api/output/types.py +1 -1
- snowflake/cli/api/project/definition_conversion.py +293 -77
- snowflake/cli/api/project/schemas/entities/common.py +11 -0
- snowflake/cli/api/project/schemas/project_definition.py +30 -25
- snowflake/cli/api/rest_api.py +26 -4
- snowflake/cli/api/secure_utils.py +1 -1
- snowflake/cli/api/sql_execution.py +40 -29
- snowflake/cli/api/stage_path.py +244 -0
- snowflake/cli/api/utils/definition_rendering.py +3 -5
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/METADATA +14 -15
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/RECORD +78 -77
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/WHEEL +1 -1
- snowflake/cli/_plugins/nativeapp/manager.py +0 -415
- snowflake/cli/_plugins/nativeapp/project_model.py +0 -211
- snowflake/cli/_plugins/nativeapp/run_processor.py +0 -184
- snowflake/cli/_plugins/nativeapp/teardown_processor.py +0 -70
- snowflake/cli/_plugins/nativeapp/version/version_processor.py +0 -98
- snowflake/cli/_plugins/workspace/action_context.py +0 -18
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/entry_points.txt +0 -0
- {snowflake_cli-3.0.2.dist-info → snowflake_cli-3.2.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,14 +1,28 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
import shutil
|
|
5
|
+
import tempfile
|
|
6
|
+
from copy import deepcopy
|
|
4
7
|
from pathlib import Path
|
|
8
|
+
from tempfile import TemporaryDirectory, mkstemp
|
|
5
9
|
from typing import Any, Dict, Literal, Optional
|
|
6
10
|
|
|
7
11
|
from click import ClickException
|
|
8
12
|
from snowflake.cli._plugins.nativeapp.artifacts import (
|
|
9
|
-
|
|
13
|
+
bundle_artifacts,
|
|
14
|
+
)
|
|
15
|
+
from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext
|
|
16
|
+
from snowflake.cli._plugins.nativeapp.codegen.compiler import TEMPLATES_PROCESSOR
|
|
17
|
+
from snowflake.cli._plugins.nativeapp.codegen.templates.templates_processor import (
|
|
18
|
+
TemplatesProcessor,
|
|
19
|
+
)
|
|
20
|
+
from snowflake.cli._plugins.nativeapp.entities.application_package import (
|
|
21
|
+
ApplicationPackageEntityModel,
|
|
10
22
|
)
|
|
11
23
|
from snowflake.cli._plugins.snowpark.common import is_name_a_templated_one
|
|
24
|
+
from snowflake.cli.api.cli_global_context import get_cli_context
|
|
25
|
+
from snowflake.cli.api.console import cli_console
|
|
12
26
|
from snowflake.cli.api.constants import (
|
|
13
27
|
DEFAULT_ENV_FILE,
|
|
14
28
|
DEFAULT_PAGES_DIR,
|
|
@@ -17,7 +31,9 @@ from snowflake.cli.api.constants import (
|
|
|
17
31
|
SNOWPARK_SHARED_MIXIN,
|
|
18
32
|
)
|
|
19
33
|
from snowflake.cli.api.entities.utils import render_script_template
|
|
34
|
+
from snowflake.cli.api.metrics import CLICounterField
|
|
20
35
|
from snowflake.cli.api.project.schemas.entities.common import (
|
|
36
|
+
MetaField,
|
|
21
37
|
SqlScriptHookType,
|
|
22
38
|
)
|
|
23
39
|
from snowflake.cli.api.project.schemas.project_definition import (
|
|
@@ -37,9 +53,26 @@ from snowflake.cli.api.project.schemas.v1.snowpark.callable import (
|
|
|
37
53
|
from snowflake.cli.api.project.schemas.v1.snowpark.snowpark import Snowpark
|
|
38
54
|
from snowflake.cli.api.project.schemas.v1.streamlit.streamlit import Streamlit
|
|
39
55
|
from snowflake.cli.api.rendering.jinja import get_basic_jinja_env
|
|
56
|
+
from snowflake.cli.api.utils.definition_rendering import render_definition_template
|
|
57
|
+
from snowflake.cli.api.utils.dict_utils import deep_merge_dicts
|
|
40
58
|
|
|
41
59
|
log = logging.getLogger(__name__)
|
|
42
60
|
|
|
61
|
+
# A directory to hold temporary files created during in-memory definition conversion
|
|
62
|
+
# We need a global reference to this directory to prevent the object from being
|
|
63
|
+
# garbage collected before the files in the directory are used by other parts
|
|
64
|
+
# of the CLI. The directory will then be deleted on interpreter exit
|
|
65
|
+
_IN_MEMORY_CONVERSION_TEMP_DIR: TemporaryDirectory | None = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _get_temp_dir() -> TemporaryDirectory:
|
|
69
|
+
global _IN_MEMORY_CONVERSION_TEMP_DIR
|
|
70
|
+
if _IN_MEMORY_CONVERSION_TEMP_DIR is None:
|
|
71
|
+
_IN_MEMORY_CONVERSION_TEMP_DIR = TemporaryDirectory(
|
|
72
|
+
suffix="_pdf_conversion", ignore_cleanup_errors=True
|
|
73
|
+
)
|
|
74
|
+
return _IN_MEMORY_CONVERSION_TEMP_DIR
|
|
75
|
+
|
|
43
76
|
|
|
44
77
|
def _is_field_defined(template_context: Optional[Dict[str, Any]], *path: str) -> bool:
|
|
45
78
|
"""
|
|
@@ -66,20 +99,29 @@ def _is_field_defined(template_context: Optional[Dict[str, Any]], *path: str) ->
|
|
|
66
99
|
|
|
67
100
|
def convert_project_definition_to_v2(
|
|
68
101
|
project_root: Path,
|
|
69
|
-
|
|
102
|
+
definition_v1: ProjectDefinition,
|
|
70
103
|
accept_templates: bool = False,
|
|
71
104
|
template_context: Optional[Dict[str, Any]] = None,
|
|
105
|
+
in_memory: bool = False,
|
|
72
106
|
) -> ProjectDefinitionV2:
|
|
73
|
-
_check_if_project_definition_meets_requirements(
|
|
107
|
+
_check_if_project_definition_meets_requirements(definition_v1, accept_templates)
|
|
74
108
|
|
|
75
|
-
snowpark_data =
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
convert_native_app_to_v2_data(project_root, pd.native_app, template_context)
|
|
79
|
-
if pd.native_app
|
|
109
|
+
snowpark_data = (
|
|
110
|
+
convert_snowpark_to_v2_data(definition_v1.snowpark)
|
|
111
|
+
if definition_v1.snowpark
|
|
80
112
|
else {}
|
|
81
113
|
)
|
|
82
|
-
|
|
114
|
+
streamlit_data = (
|
|
115
|
+
convert_streamlit_to_v2_data(definition_v1.streamlit)
|
|
116
|
+
if definition_v1.streamlit
|
|
117
|
+
else {}
|
|
118
|
+
)
|
|
119
|
+
native_app_data, native_app_template_context = (
|
|
120
|
+
convert_native_app_to_v2_data(definition_v1.native_app, template_context)
|
|
121
|
+
if definition_v1.native_app
|
|
122
|
+
else ({}, {})
|
|
123
|
+
)
|
|
124
|
+
envs = convert_envs_to_v2(definition_v1)
|
|
83
125
|
|
|
84
126
|
data = {
|
|
85
127
|
"definition_version": "2",
|
|
@@ -89,10 +131,31 @@ def convert_project_definition_to_v2(
|
|
|
89
131
|
native_app_data.get("entities", {}),
|
|
90
132
|
),
|
|
91
133
|
"mixins": snowpark_data.get("mixins", None),
|
|
92
|
-
"env": envs,
|
|
93
134
|
}
|
|
135
|
+
if envs is not None:
|
|
136
|
+
data["env"] = envs
|
|
137
|
+
|
|
138
|
+
if in_memory:
|
|
139
|
+
# If this is an in-memory conversion, we need to evaluate templates right away
|
|
140
|
+
# since the file won't be re-read as it would be for a permanent conversion
|
|
141
|
+
definition_v2 = render_definition_template(data, {}).project_definition
|
|
142
|
+
else:
|
|
143
|
+
definition_v2 = ProjectDefinitionV2(**data)
|
|
144
|
+
|
|
145
|
+
# If the user's files have any template tags in them, they
|
|
146
|
+
# also need to be migrated to point to the v2 entities
|
|
147
|
+
replacement_template_context = deepcopy(template_context) or {}
|
|
148
|
+
deep_merge_dicts(replacement_template_context, native_app_template_context)
|
|
149
|
+
if replacement_template_context:
|
|
150
|
+
_convert_templates_in_files(
|
|
151
|
+
project_root,
|
|
152
|
+
definition_v1,
|
|
153
|
+
definition_v2,
|
|
154
|
+
in_memory,
|
|
155
|
+
replacement_template_context,
|
|
156
|
+
)
|
|
94
157
|
|
|
95
|
-
return
|
|
158
|
+
return definition_v2
|
|
96
159
|
|
|
97
160
|
|
|
98
161
|
def convert_snowpark_to_v2_data(snowpark: Snowpark) -> Dict[str, Any]:
|
|
@@ -196,10 +259,9 @@ def convert_streamlit_to_v2_data(streamlit: Streamlit) -> Dict[str, Any]:
|
|
|
196
259
|
|
|
197
260
|
|
|
198
261
|
def convert_native_app_to_v2_data(
|
|
199
|
-
project_root,
|
|
200
262
|
native_app: NativeApp,
|
|
201
263
|
template_context: Optional[Dict[str, Any]] = None,
|
|
202
|
-
) ->
|
|
264
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
203
265
|
def _make_meta(obj: Application | Package):
|
|
204
266
|
meta = {}
|
|
205
267
|
if obj.role:
|
|
@@ -210,62 +272,6 @@ def convert_native_app_to_v2_data(
|
|
|
210
272
|
meta["post_deploy"] = obj.post_deploy
|
|
211
273
|
return meta
|
|
212
274
|
|
|
213
|
-
def _find_manifest():
|
|
214
|
-
# We don't know which file in the project directory is the actual manifest,
|
|
215
|
-
# and we can't iterate through the artifacts property since the src can contain
|
|
216
|
-
# glob patterns. The simplest solution is to bundle the app and find the
|
|
217
|
-
# manifest file from the resultant BundleMap, since the bundle process ensures
|
|
218
|
-
# that only a single source path can map to the corresponding destination path
|
|
219
|
-
bundle_map = BundleMap(
|
|
220
|
-
project_root=project_root, deploy_root=Path(native_app.deploy_root)
|
|
221
|
-
)
|
|
222
|
-
for artifact in native_app.artifacts:
|
|
223
|
-
bundle_map.add(artifact)
|
|
224
|
-
|
|
225
|
-
manifest_path = next(
|
|
226
|
-
(
|
|
227
|
-
src
|
|
228
|
-
for src, dest in bundle_map.all_mappings(
|
|
229
|
-
absolute=True, expand_directories=True
|
|
230
|
-
)
|
|
231
|
-
if dest.name == "manifest.yml"
|
|
232
|
-
),
|
|
233
|
-
None,
|
|
234
|
-
)
|
|
235
|
-
if not manifest_path:
|
|
236
|
-
# The manifest field is required, so we can't gracefully handle it being missing
|
|
237
|
-
raise ClickException(
|
|
238
|
-
"manifest.yml file not found in any Native App artifact sources, "
|
|
239
|
-
"unable to perform migration"
|
|
240
|
-
)
|
|
241
|
-
|
|
242
|
-
# Use a POSIX path to be consistent with other migrated fields
|
|
243
|
-
# which use POSIX paths as default values
|
|
244
|
-
return manifest_path.relative_to(project_root).as_posix()
|
|
245
|
-
|
|
246
|
-
def _make_template(template: str) -> str:
|
|
247
|
-
return f"{PROJECT_TEMPLATE_VARIABLE_OPENING} {template} {PROJECT_TEMPLATE_VARIABLE_CLOSING}"
|
|
248
|
-
|
|
249
|
-
def _convert_package_script_files(package_scripts: list[str]):
|
|
250
|
-
# PDFv2 doesn't support package scripts, only post-deploy scripts, so we
|
|
251
|
-
# need to convert the Jinja syntax from {{ }} to <% %>
|
|
252
|
-
# Luckily, package scripts only support {{ package_name }}, so let's convert that tag
|
|
253
|
-
# to v2 template syntax by running it though the template process with a fake
|
|
254
|
-
# package name that's actually a valid v2 template, which will be evaluated
|
|
255
|
-
# when the script is used as a post-deploy script
|
|
256
|
-
fake_package_replacement_template = _make_template(
|
|
257
|
-
f"ctx.entities.{package_entity_name}.identifier"
|
|
258
|
-
)
|
|
259
|
-
jinja_context = dict(package_name=fake_package_replacement_template)
|
|
260
|
-
post_deploy_hooks = []
|
|
261
|
-
for script_file in package_scripts:
|
|
262
|
-
new_contents = render_script_template(
|
|
263
|
-
project_root, jinja_context, script_file, get_basic_jinja_env()
|
|
264
|
-
)
|
|
265
|
-
(project_root / script_file).write_text(new_contents)
|
|
266
|
-
post_deploy_hooks.append(SqlScriptHookType(sql_script=script_file))
|
|
267
|
-
return post_deploy_hooks
|
|
268
|
-
|
|
269
275
|
package_entity_name = "pkg"
|
|
270
276
|
if (
|
|
271
277
|
native_app.package
|
|
@@ -281,7 +287,6 @@ def convert_native_app_to_v2_data(
|
|
|
281
287
|
package = {
|
|
282
288
|
"type": "application package",
|
|
283
289
|
"identifier": package_identifier,
|
|
284
|
-
"manifest": _find_manifest(),
|
|
285
290
|
"artifacts": native_app.artifacts,
|
|
286
291
|
}
|
|
287
292
|
|
|
@@ -303,12 +308,11 @@ def convert_native_app_to_v2_data(
|
|
|
303
308
|
package["distribution"] = native_app.package.distribution
|
|
304
309
|
package_meta = _make_meta(native_app.package)
|
|
305
310
|
if native_app.package.scripts:
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
)
|
|
311
|
+
# Package scripts are not supported in PDFv2 but we
|
|
312
|
+
# don't convert them here, conversion is deferred until
|
|
313
|
+
# the final v2 Pydantic model is available
|
|
314
|
+
# (see _convert_templates_in_files())
|
|
315
|
+
pass
|
|
312
316
|
if package_meta:
|
|
313
317
|
package["meta"] = package_meta
|
|
314
318
|
|
|
@@ -337,12 +341,63 @@ def convert_native_app_to_v2_data(
|
|
|
337
341
|
):
|
|
338
342
|
app["debug"] = native_app.application.debug
|
|
339
343
|
|
|
340
|
-
|
|
344
|
+
pdfv2_yml = {
|
|
341
345
|
"entities": {
|
|
342
346
|
package_entity_name: package,
|
|
343
347
|
app_entity_name: app,
|
|
344
348
|
}
|
|
345
349
|
}
|
|
350
|
+
template_replacements = {
|
|
351
|
+
"ctx": {
|
|
352
|
+
"native_app": {
|
|
353
|
+
"name": native_app.name, # This is a literal since there's no equivalent in v2
|
|
354
|
+
# omitting "artifacts" since lists are not supported in templates
|
|
355
|
+
"bundle_root": _make_template(
|
|
356
|
+
f"ctx.entities.{package_entity_name}.bundle_root"
|
|
357
|
+
),
|
|
358
|
+
"deploy_root": _make_template(
|
|
359
|
+
f"ctx.entities.{package_entity_name}.deploy_root"
|
|
360
|
+
),
|
|
361
|
+
"generated_root": _make_template(
|
|
362
|
+
f"ctx.entities.{package_entity_name}.generated_root"
|
|
363
|
+
),
|
|
364
|
+
"source_stage": _make_template(
|
|
365
|
+
f"ctx.entities.{package_entity_name}.stage"
|
|
366
|
+
),
|
|
367
|
+
"scratch_stage": _make_template(
|
|
368
|
+
f"ctx.entities.{package_entity_name}.scratch_stage"
|
|
369
|
+
),
|
|
370
|
+
"package": {
|
|
371
|
+
# omitting "scripts" since lists are not supported in templates
|
|
372
|
+
"role": _make_template(
|
|
373
|
+
f"ctx.entities.{package_entity_name}.meta.role"
|
|
374
|
+
),
|
|
375
|
+
"name": _make_template(
|
|
376
|
+
f"ctx.entities.{package_entity_name}.identifier"
|
|
377
|
+
),
|
|
378
|
+
"warehouse": _make_template(
|
|
379
|
+
f"ctx.entities.{package_entity_name}.meta.warehouse"
|
|
380
|
+
),
|
|
381
|
+
"distribution": _make_template(
|
|
382
|
+
f"ctx.entities.{package_entity_name}.distribution"
|
|
383
|
+
),
|
|
384
|
+
# omitting "post_deploy" since lists are not supported in templates
|
|
385
|
+
},
|
|
386
|
+
"application": {
|
|
387
|
+
"role": _make_template(f"ctx.entities.{app_entity_name}.meta.role"),
|
|
388
|
+
"name": _make_template(
|
|
389
|
+
f"ctx.entities.{app_entity_name}.identifier"
|
|
390
|
+
),
|
|
391
|
+
"warehouse": _make_template(
|
|
392
|
+
f"ctx.entities.{app_entity_name}.meta.warehouse"
|
|
393
|
+
),
|
|
394
|
+
"debug": _make_template(f"ctx.entities.{app_entity_name}.debug"),
|
|
395
|
+
# omitting "post_deploy" since lists are not supported in templates
|
|
396
|
+
},
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
return pdfv2_yml, template_replacements
|
|
346
401
|
|
|
347
402
|
|
|
348
403
|
def convert_envs_to_v2(pd: ProjectDefinition):
|
|
@@ -352,6 +407,167 @@ def convert_envs_to_v2(pd: ProjectDefinition):
|
|
|
352
407
|
return None
|
|
353
408
|
|
|
354
409
|
|
|
410
|
+
def _convert_templates_in_files(
|
|
411
|
+
project_root: Path,
|
|
412
|
+
definition_v1: ProjectDefinition,
|
|
413
|
+
definition_v2: ProjectDefinitionV2,
|
|
414
|
+
in_memory: bool,
|
|
415
|
+
replacement_template_context: dict[str, Any],
|
|
416
|
+
):
|
|
417
|
+
"""Converts templates in other files to the new format"""
|
|
418
|
+
# Set up fakers so that references to ctx.env. and fn.
|
|
419
|
+
# get templated to the same literal, since those references
|
|
420
|
+
# are the same in v1 and v2
|
|
421
|
+
replacement_template_context["ctx"]["env"] = _EnvFaker()
|
|
422
|
+
replacement_template_context["fn"] = _FnFaker()
|
|
423
|
+
|
|
424
|
+
metrics = get_cli_context().metrics
|
|
425
|
+
|
|
426
|
+
if na := definition_v1.native_app:
|
|
427
|
+
# If the v1 definition has a Native App, we know
|
|
428
|
+
# that the v2 definition will have exactly one application package entity
|
|
429
|
+
pkg_model: ApplicationPackageEntityModel = list(
|
|
430
|
+
definition_v2.get_entities_by_type(
|
|
431
|
+
ApplicationPackageEntityModel.get_type()
|
|
432
|
+
).values()
|
|
433
|
+
)[0]
|
|
434
|
+
|
|
435
|
+
# Convert templates in artifacts by passing them through the TemplatesProcessor
|
|
436
|
+
# but providing a context that maps v1 template references to the equivalent v2
|
|
437
|
+
# references instead of resolving to literals
|
|
438
|
+
# For example, replacement_template_context might look like
|
|
439
|
+
# {
|
|
440
|
+
# "ctx": {
|
|
441
|
+
# "native_app": {
|
|
442
|
+
# "bundle_root": "<% ctx.entities.pkg.bundle_root %>",
|
|
443
|
+
# "deploy_root": "<% ctx.entities.pkg.deploy_root %>",
|
|
444
|
+
# "application": {
|
|
445
|
+
# "name": "<% ctx.entities.app.identifier %>",
|
|
446
|
+
# }
|
|
447
|
+
# and so on...
|
|
448
|
+
# }
|
|
449
|
+
# }
|
|
450
|
+
# }
|
|
451
|
+
# We only convert files on-disk if the "templates" processor is used in the artifacts
|
|
452
|
+
# and if we're doing a permanent conversion. If we're doing an in-memory conversion,
|
|
453
|
+
# the CLI global template context is already populated with the v1 definition, so
|
|
454
|
+
# we don't want to convert the v1 template references in artifact files
|
|
455
|
+
metrics.set_counter_default(CLICounterField.TEMPLATES_PROCESSOR, 0)
|
|
456
|
+
artifacts_to_template = [
|
|
457
|
+
artifact
|
|
458
|
+
for artifact in pkg_model.artifacts
|
|
459
|
+
for processor in artifact.processors
|
|
460
|
+
if processor.name == TEMPLATES_PROCESSOR
|
|
461
|
+
]
|
|
462
|
+
if not in_memory and artifacts_to_template:
|
|
463
|
+
metrics.set_counter(CLICounterField.TEMPLATES_PROCESSOR, 1)
|
|
464
|
+
|
|
465
|
+
# Create a temporary directory to hold the expanded templates,
|
|
466
|
+
# as if a bundle step had been run but without affecting any
|
|
467
|
+
# files on disk outside of the artifacts we want to convert
|
|
468
|
+
with tempfile.TemporaryDirectory() as d:
|
|
469
|
+
deploy_root = Path(d)
|
|
470
|
+
bundle_ctx = BundleContext(
|
|
471
|
+
package_name=pkg_model.identifier,
|
|
472
|
+
artifacts=pkg_model.artifacts,
|
|
473
|
+
project_root=project_root,
|
|
474
|
+
bundle_root=project_root / pkg_model.bundle_root,
|
|
475
|
+
deploy_root=deploy_root,
|
|
476
|
+
generated_root=(
|
|
477
|
+
project_root / deploy_root / pkg_model.generated_root
|
|
478
|
+
),
|
|
479
|
+
)
|
|
480
|
+
template_processor = TemplatesProcessor(bundle_ctx)
|
|
481
|
+
bundle_map = bundle_artifacts(
|
|
482
|
+
project_root, deploy_root, artifacts_to_template
|
|
483
|
+
)
|
|
484
|
+
for src, dest in bundle_map.all_mappings(
|
|
485
|
+
absolute=True, expand_directories=True
|
|
486
|
+
):
|
|
487
|
+
if src.is_dir():
|
|
488
|
+
continue
|
|
489
|
+
# We call the implementation directly instead of calling process()
|
|
490
|
+
# since we need access to the BundleMap to copy files anyways
|
|
491
|
+
template_processor.expand_templates_in_file(
|
|
492
|
+
src, dest, replacement_template_context
|
|
493
|
+
)
|
|
494
|
+
# Copy the expanded file back to its original source location if it was modified
|
|
495
|
+
if not dest.is_symlink():
|
|
496
|
+
shutil.copyfile(dest, src)
|
|
497
|
+
|
|
498
|
+
# Convert package script files to post-deploy hooks
|
|
499
|
+
metrics.set_counter_default(CLICounterField.PACKAGE_SCRIPTS, 0)
|
|
500
|
+
if (pkg := na.package) and pkg.scripts:
|
|
501
|
+
metrics.set_counter(CLICounterField.PACKAGE_SCRIPTS, 1)
|
|
502
|
+
cli_console.warning(
|
|
503
|
+
"WARNING: native_app.package.scripts is deprecated. "
|
|
504
|
+
"Please migrate to using native_app.package.post_deploy."
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
converted_post_deploy_hooks = _convert_package_script_files(
|
|
508
|
+
project_root, pkg.scripts, pkg_model, in_memory
|
|
509
|
+
)
|
|
510
|
+
if pkg_model.meta is None:
|
|
511
|
+
pkg_model.meta = MetaField()
|
|
512
|
+
if pkg_model.meta.post_deploy is None:
|
|
513
|
+
pkg_model.meta.post_deploy = []
|
|
514
|
+
pkg_model.meta.post_deploy += converted_post_deploy_hooks
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _convert_package_script_files(
|
|
518
|
+
project_root: Path,
|
|
519
|
+
package_scripts: list[str],
|
|
520
|
+
pkg_model: ApplicationPackageEntityModel,
|
|
521
|
+
in_memory: bool,
|
|
522
|
+
):
|
|
523
|
+
# PDFv2 doesn't support package scripts, only post-deploy scripts, so we
|
|
524
|
+
# need to convert the Jinja syntax from {{ }} to <% %>
|
|
525
|
+
# Luckily, package scripts only support {{ package_name }}, so let's convert that tag
|
|
526
|
+
# to v2 template syntax by running it though the template process with a fake
|
|
527
|
+
# package name that's actually a valid v2 template, which will be evaluated
|
|
528
|
+
# when the script is used as a post-deploy script
|
|
529
|
+
# If we're doing an in-memory conversion, we can just hardcode the converted
|
|
530
|
+
# package name directly into the script since it's being written to a temporary file
|
|
531
|
+
package_name_replacement = (
|
|
532
|
+
pkg_model.fqn.name
|
|
533
|
+
if in_memory
|
|
534
|
+
else _make_template(f"ctx.entities.{pkg_model.entity_id}.identifier")
|
|
535
|
+
)
|
|
536
|
+
jinja_context = dict(package_name=package_name_replacement)
|
|
537
|
+
post_deploy_hooks = []
|
|
538
|
+
for script_file in package_scripts:
|
|
539
|
+
original_script_file = script_file
|
|
540
|
+
new_contents = render_script_template(
|
|
541
|
+
project_root, jinja_context, script_file, get_basic_jinja_env()
|
|
542
|
+
)
|
|
543
|
+
if in_memory:
|
|
544
|
+
# If we're converting the definition in-memory, we can't touch
|
|
545
|
+
# the package scripts on disk, so we'll write them to a temporary file
|
|
546
|
+
d = _get_temp_dir().name
|
|
547
|
+
_, script_file = mkstemp(dir=d, suffix="_converted.sql", text=True)
|
|
548
|
+
(project_root / script_file).write_text(new_contents)
|
|
549
|
+
hook = SqlScriptHookType(sql_script=script_file)
|
|
550
|
+
hook._display_path = original_script_file # noqa: SLF001
|
|
551
|
+
post_deploy_hooks.append(hook)
|
|
552
|
+
return post_deploy_hooks
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
class _EnvFaker:
|
|
556
|
+
def __getitem__(self, item):
|
|
557
|
+
return _make_template(f"ctx.env.{item}")
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
class _FnFaker:
|
|
561
|
+
def __getitem__(self, item):
|
|
562
|
+
return lambda *args: _make_template(
|
|
563
|
+
f"fn.{item}({', '.join(repr(a) for a in args)})"
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _make_template(template: str) -> str:
|
|
568
|
+
return f"{PROJECT_TEMPLATE_VARIABLE_OPENING} {template} {PROJECT_TEMPLATE_VARIABLE_CLOSING}"
|
|
569
|
+
|
|
570
|
+
|
|
355
571
|
def _check_if_project_definition_meets_requirements(
|
|
356
572
|
pd: ProjectDefinition, accept_templates: bool
|
|
357
573
|
):
|
|
@@ -28,6 +28,17 @@ from snowflake.cli.api.project.schemas.updatable_model import (
|
|
|
28
28
|
class SqlScriptHookType(UpdatableModel):
|
|
29
29
|
sql_script: str = Field(title="SQL file path relative to the project root")
|
|
30
30
|
|
|
31
|
+
# Used to store a user-friendly path for this script, when the
|
|
32
|
+
# value of `sql_script` is a path to a different file
|
|
33
|
+
# This is used in the UI to display the path relative to the
|
|
34
|
+
# project root when `sql_script` is a actually path to a temp file
|
|
35
|
+
# generated by the in-memory PDF v1 to v2 conversion
|
|
36
|
+
_display_path: str = PrivateAttr(default="")
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def display_path(self):
|
|
40
|
+
return self._display_path or self.sql_script
|
|
41
|
+
|
|
31
42
|
|
|
32
43
|
# Currently sql_script is the only supported hook type. Change to a Union once other hook types are added
|
|
33
44
|
PostDeployHook = SqlScriptHookType
|
|
@@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Union
|
|
|
19
19
|
|
|
20
20
|
from packaging.version import Version
|
|
21
21
|
from pydantic import Field, ValidationError, field_validator, model_validator
|
|
22
|
+
from pydantic_core.core_schema import ValidationInfo
|
|
22
23
|
from snowflake.cli._plugins.nativeapp.entities.application import ApplicationEntityModel
|
|
23
24
|
from snowflake.cli.api.project.errors import SchemaValidationError
|
|
24
25
|
from snowflake.cli.api.project.schemas.entities.common import (
|
|
@@ -115,7 +116,17 @@ class DefinitionV11(DefinitionV10):
|
|
|
115
116
|
|
|
116
117
|
|
|
117
118
|
class DefinitionV20(_ProjectDefinitionBase):
|
|
118
|
-
entities: Dict[str, AnnotatedEntity] = Field(
|
|
119
|
+
entities: Dict[str, AnnotatedEntity] = Field(
|
|
120
|
+
title="Entity definitions.", default={}
|
|
121
|
+
)
|
|
122
|
+
env: Optional[Dict[str, Union[str, int, bool]]] = Field(
|
|
123
|
+
title="Default environment specification for this project.",
|
|
124
|
+
default=None,
|
|
125
|
+
)
|
|
126
|
+
mixins: Optional[Dict[str, Dict]] = Field(
|
|
127
|
+
title="Mixins to apply to entities",
|
|
128
|
+
default=None,
|
|
129
|
+
)
|
|
119
130
|
|
|
120
131
|
@model_validator(mode="after")
|
|
121
132
|
def validate_entities_identifiers(self):
|
|
@@ -163,38 +174,32 @@ class DefinitionV20(_ProjectDefinitionBase):
|
|
|
163
174
|
f"Target type mismatch. Expected {target_type.__name__}, got {actual_target_type.__name__}"
|
|
164
175
|
)
|
|
165
176
|
|
|
166
|
-
env: Optional[Dict[str, Union[str, int, bool]]] = Field(
|
|
167
|
-
title="Default environment specification for this project.",
|
|
168
|
-
default=None,
|
|
169
|
-
)
|
|
170
|
-
|
|
171
|
-
mixins: Optional[Dict[str, Dict]] = Field(
|
|
172
|
-
title="Mixins to apply to entities",
|
|
173
|
-
default=None,
|
|
174
|
-
)
|
|
175
|
-
|
|
176
177
|
@model_validator(mode="before")
|
|
177
178
|
@classmethod
|
|
178
|
-
def apply_mixins(cls, data: Dict) -> Dict:
|
|
179
|
+
def apply_mixins(cls, data: Dict, info: ValidationInfo) -> Dict:
|
|
179
180
|
"""
|
|
180
181
|
Applies mixins to those entities, whose meta field contains the mixin name.
|
|
181
182
|
"""
|
|
182
183
|
if "mixins" not in data or "entities" not in data:
|
|
183
184
|
return data
|
|
184
185
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
186
|
+
duplicated_run = (
|
|
187
|
+
info.context.get("is_duplicated_run", False) if info.context else False
|
|
188
|
+
)
|
|
189
|
+
if not duplicated_run:
|
|
190
|
+
entities = data["entities"]
|
|
191
|
+
for entity_name, entity in entities.items():
|
|
192
|
+
entity_mixins = entity_mixins_to_list(
|
|
193
|
+
entity.get("meta", {}).get("use_mixins")
|
|
194
|
+
)
|
|
190
195
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
196
|
+
merged_values = cls._merge_mixins_with_entity(
|
|
197
|
+
entity_id=entity_name,
|
|
198
|
+
entity=entity,
|
|
199
|
+
entity_mixins_names=entity_mixins,
|
|
200
|
+
mixin_defs=data["mixins"],
|
|
201
|
+
)
|
|
202
|
+
entities[entity_name] = merged_values
|
|
198
203
|
return data
|
|
199
204
|
|
|
200
205
|
@classmethod
|
|
@@ -325,6 +330,6 @@ def get_allowed_fields_for_entity(entity: Dict[str, Any]) -> List[str]:
|
|
|
325
330
|
def _unique_extend(list_a: List, list_b: List) -> List:
|
|
326
331
|
new_list = list(list_a)
|
|
327
332
|
for item in list_b:
|
|
328
|
-
if item
|
|
333
|
+
if all(item != x for x in list_a):
|
|
329
334
|
new_list.append(item)
|
|
330
335
|
return new_list
|
snowflake/cli/api/rest_api.py
CHANGED
|
@@ -17,6 +17,7 @@ from __future__ import annotations
|
|
|
17
17
|
import json
|
|
18
18
|
import logging
|
|
19
19
|
from typing import Any, Dict, Optional
|
|
20
|
+
from urllib.parse import parse_qsl, urlencode, urlparse
|
|
20
21
|
|
|
21
22
|
from click import ClickException
|
|
22
23
|
from snowflake.cli.api.constants import SF_REST_API_URL_PREFIX
|
|
@@ -107,7 +108,9 @@ class RestApi:
|
|
|
107
108
|
url = f"{SF_REST_API_URL_PREFIX}/databases/{db_name}/schemas/{schema_name}"
|
|
108
109
|
return self._fetch_endpoint_exists(url)
|
|
109
110
|
|
|
110
|
-
def determine_url_for_create_query(
|
|
111
|
+
def determine_url_for_create_query(
|
|
112
|
+
self, object_type: str, replace: bool = False, if_not_exists: bool = False
|
|
113
|
+
) -> str:
|
|
111
114
|
"""
|
|
112
115
|
Determine an url for creating an object of given type via REST API.
|
|
113
116
|
If URL cannot be determined, the function throws CannotDetermineCreateURLException exception.
|
|
@@ -124,10 +127,16 @@ class RestApi:
|
|
|
124
127
|
"""
|
|
125
128
|
plural_object_type = _pluralize_object_type(object_type)
|
|
126
129
|
|
|
130
|
+
query_params = {}
|
|
131
|
+
|
|
132
|
+
if replace or if_not_exists:
|
|
133
|
+
param = "ifNotExists" if if_not_exists else "orReplace"
|
|
134
|
+
query_params = {"createMode": param}
|
|
135
|
+
|
|
127
136
|
if self.get_endpoint_exists(
|
|
128
137
|
url := f"{SF_REST_API_URL_PREFIX}/{plural_object_type}/"
|
|
129
138
|
):
|
|
130
|
-
return url
|
|
139
|
+
return self._add_query_parameters_to_url(url, query_params)
|
|
131
140
|
|
|
132
141
|
db = self.conn.database
|
|
133
142
|
if not db:
|
|
@@ -139,7 +148,7 @@ class RestApi:
|
|
|
139
148
|
if self.get_endpoint_exists(
|
|
140
149
|
url := f"{SF_REST_API_URL_PREFIX}/databases/{db}/{plural_object_type}/"
|
|
141
150
|
):
|
|
142
|
-
return url
|
|
151
|
+
return self._add_query_parameters_to_url(url, query_params)
|
|
143
152
|
|
|
144
153
|
schema = self.conn.schema
|
|
145
154
|
if not schema:
|
|
@@ -151,12 +160,25 @@ class RestApi:
|
|
|
151
160
|
if self.get_endpoint_exists(
|
|
152
161
|
url := f"{SF_REST_API_URL_PREFIX}/databases/{self.conn.database}/schemas/{self.conn.schema}/{plural_object_type}/"
|
|
153
162
|
):
|
|
154
|
-
return url
|
|
163
|
+
return self._add_query_parameters_to_url(url, query_params)
|
|
155
164
|
|
|
156
165
|
raise CannotDetermineCreateURLException(
|
|
157
166
|
f"Create operation for type {object_type} is not supported. Try using `sql -q 'CREATE ...'` command."
|
|
158
167
|
)
|
|
159
168
|
|
|
169
|
+
@staticmethod
|
|
170
|
+
def _add_query_parameters_to_url(url: str, query_params: Dict[str, Any]) -> str:
|
|
171
|
+
"""
|
|
172
|
+
Updates existing url with new query parameters.
|
|
173
|
+
They should be passed as key-value pairs in query_params dict.
|
|
174
|
+
"""
|
|
175
|
+
if not query_params:
|
|
176
|
+
return url
|
|
177
|
+
url_parts = urlparse(url)
|
|
178
|
+
query = dict(parse_qsl(url_parts.query))
|
|
179
|
+
query.update(query_params)
|
|
180
|
+
return url_parts._replace(query=urlencode(query)).geturl()
|
|
181
|
+
|
|
160
182
|
|
|
161
183
|
class DatabaseNotDefinedException(ClickException):
|
|
162
184
|
pass
|
|
@@ -108,7 +108,7 @@ def _windows_restrict_file_permissions(path: Path) -> None:
|
|
|
108
108
|
|
|
109
109
|
for user in windows_get_not_whitelisted_users_with_access(path):
|
|
110
110
|
log.info("Removing permissions of user %s from file %s", user, path)
|
|
111
|
-
subprocess.run(["icacls", str(path), "/
|
|
111
|
+
subprocess.run(["icacls", str(path), "/remove:g", f"{user}"])
|
|
112
112
|
|
|
113
113
|
|
|
114
114
|
def restrict_file_permissions(file_path: Path) -> None:
|