snowflake-cli-labs 2.5.0rc3__py3-none-any.whl → 2.6.0__py3-none-any.whl

snowflake/cli/api/project/errors.py

@@ -14,15 +14,16 @@
 
 from textwrap import dedent
 
+from click import ClickException
 from pydantic import ValidationError
 
 
-class SchemaValidationError(Exception):
-    generic_message = "For field {loc} you provided '{loc}'. This caused: {msg}"
+class SchemaValidationError(ClickException):
+    generic_message = "For field {location} you provided '{input}'. This caused: {msg}"
     message_templates = {
-        "string_type": "{msg} for field '{loc}', you provided '{input}'.",
-        "extra_forbidden": "{msg}. You provided field '{loc}' with value '{input}' that is not supported in given version.",
-        "missing": "Your project definition is missing following fields: {loc}",
+        "string_type": "{msg} for field '{location}', you provided '{input}'.",
+        "extra_forbidden": "{msg}. You provided field '{location}' with value '{input}' that is not supported in given version.",
+        "missing": "Your project definition is missing the following field: '{location}'",
     }
 
     def __init__(self, error: ValidationError):
@@ -30,7 +31,9 @@ class SchemaValidationError(Exception):
         message = f"During evaluation of {error.title} in project definition following errors were encountered:\n"
         message += "\n".join(
             [
-                self.message_templates.get(e["type"], self.generic_message).format(**e)
+                self.message_templates.get(e["type"], self.generic_message).format(
+                    **e, location=".".join(e["loc"]) if e["loc"] is not None else None
+                )
                 for e in errors
             ]
         )

snowflake/cli/api/project/schemas/identifier_model.py

@@ -36,7 +36,7 @@ def ObjectIdentifierModel(object_name: str) -> ObjectIdentifierBaseModel:  # noq
     """Generates ObjectIdentifierBaseModel but with object specific descriptions."""
 
     class _ObjectIdentifierModel(ObjectIdentifierBaseModel):
-        name: str = Field(title=f"{object_name} name")
+        name: str = Field(title=f"{object_name.capitalize()} name")
         database: Optional[str] = IdentifierField(
             title=f"Name of the database for the {object_name}", default=None
         )

snowflake/cli/api/project/schemas/native_app/application.py

@@ -14,7 +14,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import Field
 from snowflake.cli.api.project.schemas.updatable_model import (
@@ -23,6 +23,14 @@ from snowflake.cli.api.project.schemas.updatable_model import (
 )
 
 
+class SqlScriptHookType(UpdatableModel):
+    sql_script: str = Field(title="SQL file path relative to the project root")
+
+
+# Currently sql_script is the only supported hook type. Change to a Union once other hook types are added
+ApplicationPostDeployHook = SqlScriptHookType
+
+
 class Application(UpdatableModel):
     role: Optional[str] = Field(
         title="Role to use when creating the application object and consumer-side objects",
@@ -37,6 +45,10 @@ class Application(UpdatableModel):
         default=None,
     )
     debug: Optional[bool] = Field(
-        title="Whether to enable debug mode when using a named stage to create an application object",
-        default=True,
+        title="When set, forces debug_mode on/off for the deployed application object",
+        default=None,
+    )
+    post_deploy: Optional[List[ApplicationPostDeployHook]] = Field(
+        title="Actions that will be executed after the application object is created/upgraded",
+        default=None,
     )

snowflake/cli/api/project/schemas/native_app/native_app.py

@@ -34,8 +34,12 @@ class NativeApp(UpdatableModel):
     artifacts: List[Union[PathMapping, str]] = Field(
         title="List of file source and destination pairs to add to the deploy root",
     )
+    bundle_root: Optional[str] = Field(
+        title="Folder at the root of your project where artifacts necessary to perform the bundle step are stored.",
+        default="output/bundle/",
+    )
     deploy_root: Optional[str] = Field(
-        title="Folder at the root of your project where the build step copies the artifacts.",
+        title="Folder at the root of your project where the bundle step copies the artifacts.",
         default="output/deploy/",
     )
     generated_root: Optional[str] = Field(

snowflake/cli/api/project/schemas/native_app/path_mapping.py

@@ -31,9 +31,20 @@ class ProcessorMapping(UpdatableModel):
 
 
 class PathMapping(UpdatableModel):
-    src: str
-    dest: Optional[str] = None
-    processors: Optional[List[Union[str, ProcessorMapping]]] = []
+    src: str = Field(
+        title="Source path or glob pattern (relative to project root)", default=None
+    )
+
+    dest: Optional[str] = Field(
+        title="Destination path on stage",
+        description="Paths are relative to stage root; paths ending with a slash indicate that the destination is a directory which source files should be copied into.",
+        default=None,
+    )
+
+    processors: Optional[List[Union[str, ProcessorMapping]]] = Field(
+        title="List of processors to apply to matching source files during bundling.",
+        default=[],
+    )
 
     @field_validator("processors")
     @classmethod

snowflake/cli/api/project/schemas/project_definition.py

@@ -14,18 +14,45 @@
 
 from __future__ import annotations
 
+from dataclasses import dataclass
 from typing import Any, Dict, Optional, Union
 
 from packaging.version import Version
-from pydantic import Field, field_validator
+from pydantic import Field, ValidationError, field_validator
+from snowflake.cli.api.project.errors import SchemaValidationError
 from snowflake.cli.api.project.schemas.native_app.native_app import NativeApp
 from snowflake.cli.api.project.schemas.snowpark.snowpark import Snowpark
 from snowflake.cli.api.project.schemas.streamlit.streamlit import Streamlit
 from snowflake.cli.api.project.schemas.updatable_model import UpdatableModel
-from snowflake.cli.api.utils.models import EnvironWithDefinedDictFallback
+from snowflake.cli.api.utils.models import ProjectEnvironment
+from snowflake.cli.api.utils.types import Context
+
+
+@dataclass
+class ProjectProperties:
+    """
+    This class stores 2 objects representing the snowflake project:
+
+    The project_context object:
+    - Used as the context for templating when users reference variables in the project definition file.
+
+    The project_definition object:
+    - This is a transformed object type through Pydantic, which has been normalized.
+    - This object could have slightly different structure than what the users see in their yaml project definition files.
+    - This should be used for the business logic of snow CLI modules.
+    """
+
+    project_definition: ProjectDefinition
+    project_context: Context
 
 
 class _BaseDefinition(UpdatableModel):
+    def __init__(self, *args, **kwargs):
+        try:
+            super().__init__(**kwargs)
+        except ValidationError as e:
+            raise SchemaValidationError(e) from e
+
     definition_version: Union[str, int] = Field(
         title="Version of the project definition schema, which is currently 1",
     )
@@ -58,17 +85,21 @@ class _DefinitionV10(_BaseDefinition):
 
 
 class _DefinitionV11(_DefinitionV10):
-    env: Optional[Dict] = Field(
+    env: Union[Dict[str, str], ProjectEnvironment, None] = Field(
         title="Environment specification for this project.",
         default=None,
         validation_alias="env",
+        union_mode="smart",
     )
 
     @field_validator("env")
     @classmethod
-    def _convert_env(cls, env: Optional[Dict]) -> EnvironWithDefinedDictFallback:
-        variables = EnvironWithDefinedDictFallback(env if env else {})
-        return variables
+    def _convert_env(
+        cls, env: Union[Dict, ProjectEnvironment, None]
+    ) -> ProjectEnvironment:
+        if isinstance(env, ProjectEnvironment):
+            return env
+        return ProjectEnvironment(default_env=(env or {}), override_env={})
 
 
 class ProjectDefinition(_DefinitionV11):

snowflake/cli/api/project/schemas/streamlit/streamlit.py

@@ -40,3 +40,6 @@ class Streamlit(UpdatableModel, ObjectIdentifierModel(object_name="Streamlit")):
         title="List of additional files which should be included into deployment artifacts",
         default=None,
     )
+    title: Optional[str] = Field(
+        title="Human-readable title for the Streamlit dashboard", default=None
+    )

snowflake/cli/api/project/schemas/updatable_model.py

@@ -16,8 +16,7 @@ from __future__ import annotations
 
 from typing import Any, Dict
 
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
-from snowflake.cli.api.project.errors import SchemaValidationError
+from pydantic import BaseModel, ConfigDict, Field
 from snowflake.cli.api.project.util import IDENTIFIER_NO_LENGTH
 
 
@@ -25,10 +24,7 @@ class UpdatableModel(BaseModel):
     model_config = ConfigDict(validate_assignment=True, extra="forbid")
 
     def __init__(self, *args, **kwargs):
-        try:
-            super().__init__(**kwargs)
-        except ValidationError as e:
-            raise SchemaValidationError(e)
+        super().__init__(**kwargs)
 
     def update_from_dict(self, update_values: Dict[str, Any]):
         """

snowflake/cli/api/rest_api.py

@@ -0,0 +1,113 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+from snowflake.cli.api.constants import SF_REST_API_URL_PREFIX
+from snowflake.connector.connection import SnowflakeConnection
+from snowflake.connector.errors import InterfaceError
+from snowflake.connector.network import SnowflakeRestful
+
+log = logging.getLogger(__name__)
+
+
+class RestApi:
+    def __init__(self, connection: SnowflakeConnection):
+        self.conn = connection
+        self.rest: SnowflakeRestful = connection.rest
+
+    def get_endpoint_exists(self, url: str) -> bool:
+        """
+        Check whether [get] endpoint exists under given URL.
+        """
+        try:
+            result = self.send_rest_request(url, method="get")
+            return bool(result) or result == []
+        except InterfaceError as err:
+            if "404 Not Found" in str(err):
+                return False
+            raise err
+
+    def send_rest_request(
+        self, url: str, method: str, data: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Executes rest request via snowflake.connector.network.SnowflakeRestful
+        """
+        # SnowflakeRestful.request assumes that API response is always a dict,
+        # which is not true in case of this API, so we need to do this workaround:
+        from snowflake.connector.network import (
+            CONTENT_TYPE_APPLICATION_JSON,
+            HTTP_HEADER_ACCEPT,
+            HTTP_HEADER_CONTENT_TYPE,
+            HTTP_HEADER_USER_AGENT,
+            PYTHON_CONNECTOR_USER_AGENT,
+        )
+
+        log.debug("Sending %s request to %s", method, url)
+        full_url = f"{self.rest.server_url}{url}"
+        headers = {
+            HTTP_HEADER_CONTENT_TYPE: CONTENT_TYPE_APPLICATION_JSON,
+            HTTP_HEADER_ACCEPT: CONTENT_TYPE_APPLICATION_JSON,
+            HTTP_HEADER_USER_AGENT: PYTHON_CONNECTOR_USER_AGENT,
+        }
+        return self.rest.fetch(
+            method=method,
+            full_url=full_url,
+            headers=headers,
+            token=self.rest.token,
+            data=json.dumps(data if data else {}),
+            no_retry=True,
+        )
+
+    def determine_url_for_create_query(
+        self, *, plural_object_type: str
+    ) -> Optional[str]:
+        """
+        Determine an url for creating an object of given type via REST API.
+        The function returns None if URL cannot be determined.
+
+        URLs we check:
+         * /api/v2/<type>/
+         * /api/v2/databases/<database>/<type>/
+         * /api/v2/databases/<database>/schemas/<schema>/<type>/
+
+         We assume that the URLs for CREATE and LIST are the same for every type of object
+         (endpoints differ by method: POST vs GET, accordingly).
+         To check whether an URL exists, we send read-only GET request (LIST endpoint,
+         which should imply CREATE endpoint).
+        """
+        urls_to_be_checked: List[Optional[str]] = [
+            f"{SF_REST_API_URL_PREFIX}/{plural_object_type}/",
+            (
+                f"{SF_REST_API_URL_PREFIX}/databases/{self.conn.database}/{plural_object_type}/"
+                if self.conn.database
+                else None
+            ),
+            (
+                f"{SF_REST_API_URL_PREFIX}/databases/{self.conn.database}/schemas/{self.conn.schema}/{plural_object_type}/"
+                if self.conn.database and self.conn.schema
+                else None
+            ),
+        ]
+
+        for url in urls_to_be_checked:
+            if url and self.get_endpoint_exists(url):
+                return url
+
+        return None

snowflake/cli/api/sanitizers.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import re
+
+# 7-bit C1 ANSI sequences
+_ANSI_ESCAPE = re.compile(
+    r"""
+    \x1B  # ESC
+    (?:   # 7-bit C1 Fe (except CSI)
+        [@-Z\\-_]
+    |     # or [ for CSI, followed by a control sequence
+        \[
+        [0-?]*  # Parameter bytes
+        [ -/]*  # Intermediate bytes
+        [@-~]   # Final byte
+    )
+""",
+    re.VERBOSE,
+)
+
+
+def sanitize_for_terminal(text: str) -> str | None:
+    """
+    Escape ASCII escape codes in string. This should be always used
+    when printing output to terminal.
+    """
+    if text is None:
+        return None
+    return _ANSI_ESCAPE.sub("", text)

snowflake/cli/api/sql_execution.py

@@ -22,6 +22,7 @@ from textwrap import dedent
 from typing import Iterable, Optional, Tuple
 
 from snowflake.cli.api.cli_global_context import cli_context
+from snowflake.cli.api.console import cli_console
 from snowflake.cli.api.constants import ObjectType
 from snowflake.cli.api.exceptions import (
     DatabaseNotProvidedError,
@@ -212,3 +213,9 @@ class SqlExecutionMixin:
             lambda row: row[name_col] == unquote_identifier(unqualified_name),
         )
         return show_obj_row
+
+
+class VerboseCursor(SnowflakeCursor):
+    def execute(self, command: str, *args, **kwargs):
+        cli_console.message(command)
+        super().execute(command, *args, **kwargs)

snowflake/cli/api/utils/definition_rendering.py

@@ -15,15 +15,19 @@
 from __future__ import annotations
 
 import copy
-import os
 from typing import Any, Optional
 
-from jinja2 import Environment, nodes
+from jinja2 import Environment, TemplateSyntaxError, nodes
 from packaging.version import Version
+from snowflake.cli.api.console import cli_console as cc
 from snowflake.cli.api.exceptions import CycleDetectedError, InvalidTemplate
+from snowflake.cli.api.project.schemas.project_definition import (
+    ProjectDefinition,
+    ProjectProperties,
+)
 from snowflake.cli.api.utils.dict_utils import traverse
 from snowflake.cli.api.utils.graph import Graph, Node
-from snowflake.cli.api.utils.models import EnvironWithDefinedDictFallback
+from snowflake.cli.api.utils.models import ProjectEnvironment
 from snowflake.cli.api.utils.rendering import CONTEXT_KEY, get_snowflake_cli_jinja_env
 from snowflake.cli.api.utils.types import Context, Definition
 
@@ -46,7 +50,13 @@ class TemplatedEnvironment:
 
     def get_referenced_vars(self, template_value: Any) -> set[TemplateVar]:
         template_str = str(template_value)
-        ast = self._jinja_env.parse(template_str)
+        try:
+            ast = self._jinja_env.parse(template_str)
+        except TemplateSyntaxError as e:
+            raise InvalidTemplate(
+                f"Error parsing template from project definition file. Value: '{template_str}'. Error: {e}"
+            ) from e
+
         return self._get_referenced_vars(ast, template_str)
 
     def _get_referenced_vars(
@@ -159,9 +169,13 @@ class TemplateVar:
             current_dict_level = current_dict_level[key]
 
         value = current_dict_level
-        if value is None or isinstance(value, (dict, list)):
+
+        if value is None:
+            raise InvalidTemplate(f"Template variable {self.key} does not have a value")
+
+        if isinstance(value, (dict, list)):
             raise InvalidTemplate(
-                f"Template variable {self.key} does not contain a valid value"
+                f"Template variable {self.key} does not have a scalar value"
             )
 
         return value
@@ -174,17 +188,20 @@ class TemplateVar:
 
 
 def _build_dependency_graph(
-    env: TemplatedEnvironment, all_vars: set[TemplateVar], context: Context
+    env: TemplatedEnvironment,
+    all_vars: set[TemplateVar],
+    context: Context,
+    environment_overrides: ProjectEnvironment,
 ) -> Graph[TemplateVar]:
     dependencies_graph = Graph[TemplateVar]()
     for variable in all_vars:
         dependencies_graph.add(Node[TemplateVar](key=variable.key, data=variable))
 
     for variable in all_vars:
-        if variable.is_env_var and variable.get_env_var_name() in os.environ:
-            # If variable is found in os.environ, then use the value as is
-            # skip rendering by pre-setting the rendered_value attribute
-            env_value = os.environ.get(variable.get_env_var_name())
+        # If variable is found in os.environ or from cli override, then use the value as is
+        # skip rendering by pre-setting the rendered_value attribute
+        if variable.is_env_var and variable.get_env_var_name() in environment_overrides:
+            env_value = environment_overrides.get(variable.get_env_var_name())
             variable.rendered_value = env_value
             variable.templated_value = env_value
         else:
@@ -210,7 +227,41 @@ def _render_graph_node(env: TemplatedEnvironment, node: Node[TemplateVar]) -> No
     node.data.rendered_value = env.render(node.data.templated_value, current_context)
 
 
-def render_definition_template(original_definition: Definition) -> Definition:
+def _validate_env_section(env_section: dict):
+    if not isinstance(env_section, dict):
+        raise InvalidTemplate(
+            "env section in project definition file should be a mapping"
+        )
+    for variable, value in env_section.items():
+        if value is None or isinstance(value, (dict, list)):
+            raise InvalidTemplate(
+                f"Variable {variable} in env section of project definition file should be a scalar"
+            )
+
+
+def _get_referenced_vars_in_definition(
+    template_env: TemplatedEnvironment, definition: Definition
+):
+    referenced_vars = set()
+
+    def find_any_template_vars(element):
+        referenced_vars.update(template_env.get_referenced_vars(element))
+
+    traverse(definition, visit_action=find_any_template_vars)
+
+    return referenced_vars
+
+
+def _template_version_warning():
+    cc.warning(
+        "Ignoring template pattern in project definition file. "
+        "Update 'definition_version' to 1.1 or later in snowflake.yml to enable template expansion."
+    )
+
+
+def render_definition_template(
+    original_definition: Optional[Definition], context_overrides: Context
+) -> ProjectProperties:
     """
     Takes a definition file as input. An arbitrary structure containing dict|list|scalars,
     with the top level being a dictionary.
@@ -225,23 +276,42 @@ def render_definition_template(original_definition: Definition) -> Definition:
     # protect input from update
     definition = copy.deepcopy(original_definition)
 
+    # start with an environment from overrides and environment variables:
+    override_env = context_overrides.get(CONTEXT_KEY, {}).get("env", {})
+    environment_overrides = ProjectEnvironment(
+        default_env={}, override_env=override_env
+    )
+
+    if definition is None:
+        return ProjectProperties(None, {CONTEXT_KEY: {"env": environment_overrides}})
+
+    project_context = {CONTEXT_KEY: definition}
+    template_env = TemplatedEnvironment(get_snowflake_cli_jinja_env())
+
     if "definition_version" not in definition or Version(
         definition["definition_version"]
     ) < Version("1.1"):
-        return definition
+        try:
+            referenced_vars = _get_referenced_vars_in_definition(
+                template_env, definition
+            )
+            if referenced_vars:
+                _template_version_warning()
+        except Exception:
+            # also warn on Exception, as it means the user is incorrectly attempting to use templating
+            _template_version_warning()
 
-    template_env = TemplatedEnvironment(get_snowflake_cli_jinja_env())
-    project_context = {CONTEXT_KEY: definition}
+        project_definition = ProjectDefinition(**original_definition)
+        project_context[CONTEXT_KEY]["env"] = environment_overrides
+        return ProjectProperties(project_definition, project_context)
 
-    referenced_vars = set()
+    default_env = definition.get("env", {})
+    _validate_env_section(default_env)
 
-    def find_any_template_vars(element):
-        referenced_vars.update(template_env.get_referenced_vars(element))
-
-    traverse(definition, visit_action=find_any_template_vars)
+    referenced_vars = _get_referenced_vars_in_definition(template_env, definition)
 
     dependencies_graph = _build_dependency_graph(
-        template_env, referenced_vars, project_context
+        template_env, referenced_vars, project_context, environment_overrides
     )
 
     def on_cycle_action(node: Node[TemplateVar]):
@@ -265,7 +335,7 @@ def render_definition_template(original_definition: Definition) -> Definition:
         update_action=lambda val: template_env.render(val, final_context),
     )
 
-    current_env = definition.setdefault("env", {})
-    definition["env"] = EnvironWithDefinedDictFallback(current_env)
-
-    return definition
+    definition["env"] = ProjectEnvironment(default_env, override_env)
+    project_context[CONTEXT_KEY] = definition
+    project_definition = ProjectDefinition(**definition)
+    return ProjectProperties(project_definition, project_context)

snowflake/cli/api/utils/models.py

@@ -15,41 +15,46 @@
 from __future__ import annotations
 
 import os
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
-from snowflake.cli.api.exceptions import InvalidTemplate
+from snowflake.cli.api.project.schemas.updatable_model import UpdatableModel
 
 
-def _validate_env(current_env: dict):
-    if not isinstance(current_env, dict):
-        raise InvalidTemplate(
-            "env section in project definition file should be a mapping"
-        )
-    for variable, value in current_env.items():
-        if value is None or isinstance(value, (dict, list)):
-            raise InvalidTemplate(
-                f"Variable {variable} in env section or project definition file should be a scalar"
-            )
+class ProjectEnvironment(UpdatableModel):
+    """
+    This class handles retrieval of project env variables.
+    These env variables can be accessed through templating, as ctx.env.<var_name>
 
+    This class checks for env values in the following order:
+    - Check for overrides values from the command line. Use these values first.
+    - Check if these variables are available as environment variables and return them if found.
+    - Check for default values from the project definition file.
+    """
 
-class EnvironWithDefinedDictFallback(Dict):
-    def __init__(self, dict_input: dict):
-        _validate_env(dict_input)
-        super().__init__(dict_input)
+    override_env: Dict[str, Any] = {}
+    default_env: Dict[str, Any] = {}
 
-    def __getattr__(self, item):
-        try:
-            return self[item]
-        except KeyError as e:
-            raise AttributeError(e)
+    def __init__(
+        self, default_env: Dict[str, Any], override_env: Optional[Dict[str, Any]] = None
+    ):
+        super().__init__(self, default_env=default_env, override_env=override_env or {})
 
     def __getitem__(self, item):
+        if item in self.override_env:
+            return self.override_env.get(item)
         if item in os.environ:
             return os.environ[item]
-        return super().__getitem__(item)
+        return self.default_env[item]
 
-    def __contains__(self, item):
-        return item in os.environ or super().__contains__(item)
+    def get(self, item, default=None):
+        try:
+            return self[item]
+        except KeyError:
+            return default
 
-    def update_from_dict(self, update_values: Dict[str, Any]):
-        return super().update(update_values)
+    def __contains__(self, item) -> bool:
+        try:
+            self[item]
+            return True
+        except KeyError:
+            return False