snowflake-cli-labs 2.7.0rc3__py3-none-any.whl → 2.8.0__py3-none-any.whl

snowflake/cli/api/project/util.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 import codecs
 import os
 import re
-from typing import Optional
+from typing import List, Optional
 from urllib.parse import quote
 
 IDENTIFIER = r'((?:"[^"]*(?:""[^"]*)*")|(?:[A-Za-z_][\w$]{0,254}))'
@@ -42,12 +42,20 @@ def encode_uri_component(s: str) -> str:
     return quote(s, safe="!~*'()")
 
 
-def clean_identifier(input_: str):
+def sanitize_identifier(input_: str):
     """
-    Removes characters that cannot be used in an unquoted identifier,
-    converting to lowercase as well.
+    Removes characters that cannot be used in an unquoted identifier.
+    If the identifier does not start with a letter or underscore, prefix it with an underscore.
+    Limits the identifier to 255 characters.
     """
-    return re.sub(r"[^a-z0-9_$]", "", f"{input_}".lower())
+    value = re.sub(r"[^a-zA-Z0-9_$]", "", f"{input_}")
+
+    # if it does not start with a letter or underscore, prefix it with an underscore
+    if not value or not re.match(r"[a-zA-Z_]", value[0]):
+        value = f"_{value}"
+
+    # limit it to 255 characters
+    return value[:255]
 
 
 def is_valid_unquoted_identifier(identifier: str) -> bool:
@@ -88,6 +96,18 @@ def is_valid_object_name(name: str, max_depth=2, allow_quoted=True) -> bool:
     return re.fullmatch(pattern, name) is not None
 
 
+def to_quoted_identifier(input_value: str) -> str:
+    """
+    Turn the input into a valid quoted identifier.
+    If it is already a valid quoted identifier,
+    return it as is.
+    """
+    if is_valid_quoted_identifier(input_value):
+        return input_value
+
+    return '"' + input_value.replace('"', '""') + '"'
+
+
 def to_identifier(name: str) -> str:
     """
     Converts a name to a valid Snowflake identifier. If the name is already a valid
@@ -96,8 +116,15 @@ def to_identifier(name: str) -> str:
     if is_valid_identifier(name):
         return name
 
-    # double quote the identifier
-    return '"' + name.replace('"', '""') + '"'
+    return to_quoted_identifier(name)
+
+
+def identifier_to_str(identifier: str) -> str:
+    if is_valid_quoted_identifier(identifier):
+        unquoted_id = identifier[1:-1]
+        return unquoted_id.replace('""', '"')
+
+    return identifier
 
 
 def append_to_identifier(identifier: str, suffix: str) -> str:
@@ -183,6 +210,27 @@ def get_env_username() -> Optional[str]:
     return first_set_env("USER", "USERNAME", "LOGNAME")
 
 
+def concat_identifiers(identifiers: list[str]) -> str:
+    """
+    Concatenate multiple identifiers.
+    If any of them is quoted identifier or contains unsafe characters, quote the result.
+    Otherwise, the resulting identifier will be unquoted.
+    """
+    quotes_found = False
+    stringified_identifiers: List[str] = []
+
+    for identifier in identifiers:
+        if is_valid_quoted_identifier(identifier):
+            quotes_found = True
+        stringified_identifiers.append(identifier_to_str(identifier))
+
+    concatenated_ids_str = "".join(stringified_identifiers)
+    if quotes_found:
+        return to_quoted_identifier(concatenated_ids_str)
+
+    return to_identifier(concatenated_ids_str)
+
+
 SUPPORTED_VERSIONS = [1]
 
 

snowflake/cli/api/rendering/jinja.py

@@ -24,6 +24,7 @@ from jinja2 import Environment, StrictUndefined, loaders
 from snowflake.cli.api.secure_path import UNLIMITED, SecurePath
 
 CONTEXT_KEY = "ctx"
+FUNCTION_KEY = "fn"
 
 
 def read_file_content(file_name: str):

snowflake/cli/api/rendering/project_templates.py

@@ -28,8 +28,10 @@ from snowflake.cli.api.exceptions import InvalidTemplate
 from snowflake.cli.api.rendering.jinja import IgnoreAttrEnvironment, env_bootstrap
 from snowflake.cli.api.secure_path import SecurePath
 
-_PROJECT_TEMPLATE_START = "<!"
-_PROJECT_TEMPLATE_END = "!>"
+_VARIABLE_TEMPLATE_START = "<!"
+_VARIABLE_TEMPLATE_END = "!>"
+_BLOCK_TEMPLATE_START = "<!!"
+_BLOCK_TEMPLATE_END = "!!>"
 
 
 def to_snowflake_identifier(value: Optional[str]) -> Optional[str]:
@@ -60,15 +62,14 @@ PROJECT_TEMPLATE_FILTERS = [to_snowflake_identifier]
 
 
 def get_template_cli_jinja_env(template_root: SecurePath) -> Environment:
-    _random_block = "___very___unique___block___to___disable___logic___blocks___"
     env = env_bootstrap(
         IgnoreAttrEnvironment(
             loader=loaders.FileSystemLoader(searchpath=template_root.path),
             keep_trailing_newline=True,
-            variable_start_string=_PROJECT_TEMPLATE_START,
-            variable_end_string=_PROJECT_TEMPLATE_END,
-            block_start_string=_random_block,
-            block_end_string=_random_block,
+            variable_start_string=_VARIABLE_TEMPLATE_START,
+            variable_end_string=_VARIABLE_TEMPLATE_END,
+            block_start_string=_BLOCK_TEMPLATE_START,
+            block_end_string=_BLOCK_TEMPLATE_END,
             undefined=StrictUndefined,
         )
     )

snowflake/cli/api/rendering/sql_templates.py

@@ -21,12 +21,14 @@ from jinja2 import StrictUndefined, loaders
 from snowflake.cli.api.cli_global_context import cli_context
 from snowflake.cli.api.rendering.jinja import (
     CONTEXT_KEY,
+    FUNCTION_KEY,
     IgnoreAttrEnvironment,
     env_bootstrap,
 )
 
 _SQL_TEMPLATE_START = "&{"
 _SQL_TEMPLATE_END = "}"
+RESERVED_KEYS = [CONTEXT_KEY, FUNCTION_KEY]
 
 
 def get_sql_cli_jinja_env(*, loader: Optional[loaders.BaseLoader] = None):
@@ -46,10 +48,12 @@ def get_sql_cli_jinja_env(*, loader: Optional[loaders.BaseLoader] = None):
 
 def snowflake_sql_jinja_render(content: str, data: Dict | None = None) -> str:
     data = data or {}
-    if CONTEXT_KEY in data:
-        raise ClickException(
-            f"{CONTEXT_KEY} in user defined data. The `{CONTEXT_KEY}` variable is reserved for CLI usage."
-        )
+
+    for reserved_key in RESERVED_KEYS:
+        if reserved_key in data:
+            raise ClickException(
+                f"{reserved_key} in user defined data. The `{reserved_key}` variable is reserved for CLI usage."
+            )
 
     context_data = cli_context.template_context
     context_data.update(data)

snowflake/cli/api/utils/definition_rendering.py

@@ -25,13 +25,15 @@ from snowflake.cli.api.project.schemas.project_definition import (
     ProjectProperties,
     build_project_definition,
 )
+from snowflake.cli.api.project.schemas.updatable_model import context
 from snowflake.cli.api.rendering.jinja import CONTEXT_KEY
 from snowflake.cli.api.rendering.project_definition_templates import (
     get_project_definition_cli_jinja_env,
 )
-from snowflake.cli.api.utils.dict_utils import traverse
+from snowflake.cli.api.utils.dict_utils import deep_merge_dicts, traverse
 from snowflake.cli.api.utils.graph import Graph, Node
 from snowflake.cli.api.utils.models import ProjectEnvironment
+from snowflake.cli.api.utils.templating_functions import get_templating_functions
 from snowflake.cli.api.utils.types import Context, Definition
 
 
@@ -81,7 +83,17 @@ class TemplatedEnvironment:
             all_referenced_vars.add(TemplateVar(current_attr_chain))
             current_attr_chain = None
         elif (
-            not isinstance(ast_node, (nodes.Template, nodes.TemplateData, nodes.Output))
+            not isinstance(
+                ast_node,
+                (
+                    nodes.Template,
+                    nodes.TemplateData,
+                    nodes.Output,
+                    nodes.Call,
+                    nodes.Const,
+                    nodes.Filter,
+                ),
+            )
             or current_attr_chain is not None
         ):
             raise InvalidTemplate(f"Unexpected templating syntax in {template_value}")
@@ -199,7 +211,6 @@ def _build_dependency_graph(
     dependencies_graph = Graph[TemplateVar]()
     for variable in all_vars:
         dependencies_graph.add(Node[TemplateVar](key=variable.key, data=variable))
-
     for variable in all_vars:
         # If variable is found in os.environ or from cli override, then use the value as is
         # skip rendering by pre-setting the rendered_value attribute
@@ -262,6 +273,22 @@ def _template_version_warning():
     )
 
 
+def _add_defaults_to_definition(original_definition: Definition) -> Definition:
+    with context({"skip_validation_on_templates": True}):
+        # pass a flag to Pydantic to skip validation for templated scalars
+        # populate the defaults
+        project_definition = build_project_definition(**original_definition)
+
+    definition_with_defaults = project_definition.model_dump(
+        exclude_none=True, warnings=False, by_alias=True
+    )
+    # The main purpose of the above operation was to populate defaults from Pydantic.
+    # By merging the original definition back in, we ensure that any transformations
+    # that Pydantic would have performed are undone.
+    deep_merge_dicts(definition_with_defaults, original_definition)
+    return definition_with_defaults
+
+
 def render_definition_template(
     original_definition: Optional[Definition], context_overrides: Context
 ) -> ProjectProperties:
@@ -276,11 +303,14 @@ def render_definition_template(
     Environment variables take precedence during the rendering process.
     """
 
-    # protect input from update
+    # copy input to protect it from update
     definition = copy.deepcopy(original_definition)
 
-    # start with an environment from overrides and environment variables:
+    # collect all the override --env variables passed through CLI input
     override_env = context_overrides.get(CONTEXT_KEY, {}).get("env", {})
+
+    # set up Project Environment with empty default_env because
+    # default env section from project definition file is still templated at this time
     environment_overrides = ProjectEnvironment(
         default_env={}, override_env=override_env
     )
@@ -288,7 +318,6 @@ def render_definition_template(
     if definition is None:
         return ProjectProperties(None, {CONTEXT_KEY: {"env": environment_overrides}})
 
-    project_context = {CONTEXT_KEY: definition}
     template_env = TemplatedEnvironment(get_project_definition_cli_jinja_env())
 
     if "definition_version" not in definition or Version(
@@ -304,12 +333,18 @@ def render_definition_template(
             # also warn on Exception, as it means the user is incorrectly attempting to use templating
             _template_version_warning()
 
-        project_definition = build_project_definition(**original_definition)
+        project_definition = build_project_definition(**definition)
+        project_context = {CONTEXT_KEY: definition}
         project_context[CONTEXT_KEY]["env"] = environment_overrides
         return ProjectProperties(project_definition, project_context)
 
-    default_env = definition.get("env", {})
-    _validate_env_section(default_env)
+    definition = _add_defaults_to_definition(definition)
+    project_context = {CONTEXT_KEY: definition}
+
+    _validate_env_section(definition.get("env", {}))
+
+    # add available templating functions
+    project_context["fn"] = get_templating_functions()
 
     referenced_vars = _get_referenced_vars_in_definition(template_env, definition)
 
@@ -338,7 +373,11 @@ def render_definition_template(
         update_action=lambda val: template_env.render(val, final_context),
     )
 
-    definition["env"] = ProjectEnvironment(default_env, override_env)
-    project_context[CONTEXT_KEY] = definition
     project_definition = build_project_definition(**definition)
+    project_context[CONTEXT_KEY] = definition
+    # Use `ProjectEnvironment` in project context in order to
+    # handle env variables overrides from OS env and from CLI arguments.
+    project_context[CONTEXT_KEY]["env"] = ProjectEnvironment(
+        default_env=project_context[CONTEXT_KEY].get("env"), override_env=override_env
+    )
     return ProjectProperties(project_definition, project_context)

snowflake/cli/api/utils/models.py

@@ -15,12 +15,12 @@
 from __future__ import annotations
 
 import os
+from dataclasses import dataclass
 from typing import Any, Dict, Optional
 
-from snowflake.cli.api.project.schemas.updatable_model import UpdatableModel
 
-
-class ProjectEnvironment(UpdatableModel):
+@dataclass
+class ProjectEnvironment:
     """
     This class handles retrieval of project env variables.
     These env variables can be accessed through templating, as ctx.env.<var_name>
@@ -31,13 +31,16 @@ class ProjectEnvironment(UpdatableModel):
     - Check for default values from the project definition file.
     """
 
-    override_env: Dict[str, Any] = {}
-    default_env: Dict[str, Any] = {}
+    override_env: Dict[str, Any]
+    default_env: Dict[str, Any]
 
     def __init__(
-        self, default_env: Dict[str, Any], override_env: Optional[Dict[str, Any]] = None
+        self,
+        default_env: Optional[Dict[str, Any]] = None,
+        override_env: Optional[Dict[str, Any]] = None,
     ):
-        super().__init__(self, default_env=default_env, override_env=override_env or {})
+        self.override_env = override_env or {}
+        self.default_env = default_env or {}
 
     def __getitem__(self, item):
         if item in self.override_env:

snowflake/cli/api/utils/templating_functions.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Any, List, Optional
+
+from snowflake.cli.api.exceptions import InvalidTemplate
+from snowflake.cli.api.project.util import (
+    concat_identifiers,
+    get_env_username,
+    identifier_to_str,
+    sanitize_identifier,
+    to_identifier,
+)
+
+
+class TemplatingFunctions:
+    """
+    This class contains all the functions available for templating.
+    Any callable not starting with '_' will automatically be available for users to use.
+    """
+
+    @staticmethod
+    def _verify_str_arguments(
+        func_name: str,
+        args: List[Any],
+        *,
+        min_count: Optional[int] = None,
+        max_count: Optional[int] = None,
+    ):
+        if min_count is not None and len(args) < min_count:
+            raise InvalidTemplate(
+                f"{func_name} requires at least {min_count} argument(s)"
+            )
+
+        if max_count is not None and len(args) > max_count:
+            raise InvalidTemplate(
+                f"{func_name} supports at most {max_count} argument(s)"
+            )
+
+        for arg in args:
+            if not isinstance(arg, str):
+                raise InvalidTemplate(f"{func_name} only accepts String values")
+
+    @staticmethod
+    def concat_ids(*args):
+        """
+        input: one or more string arguments (SQL ID or plain String).
+        output: a valid SQL ID (quoted or unquoted)
+
+        Takes on multiple String arguments and concatenate them into one String.
+        If any of the Strings is a valid quoted ID, it will be unescaped for the concatenation process.
+        The resulting String is then escaped and quoted if:
+        - It contains non SQL safe characters
+        - Any of the input was a valid quoted identifier.
+        """
+        TemplatingFunctions._verify_str_arguments("concat_ids", args, min_count=1)
+        return concat_identifiers(args)
+
+    @staticmethod
+    def str_to_id(*args):
+        """
+        input: one string argument. (SQL ID or plain String)
+        output: a valid SQL ID (quoted or unquoted)
+
+        If the input is a valid quoted or valid unquoted identifier, return it as is.
+        Otherwise, if the input contains unsafe characters and is not properly quoted,
+        then escape it and quote it.
+        """
+        TemplatingFunctions._verify_str_arguments(
+            "str_to_id", args, min_count=1, max_count=1
+        )
+        return to_identifier(args[0])
+
+    @staticmethod
+    def id_to_str(*args):
+        """
+        input: one string argument (SQL ID or plain String).
+        output: a plain string
+
+        If the input is a valid SQL ID, then unescape it and return the plain String version.
+        Otherwise, return the input as is.
+        """
+        TemplatingFunctions._verify_str_arguments(
+            "id_to_str", args, min_count=1, max_count=1
+        )
+        return identifier_to_str(args[0])
+
+    @staticmethod
+    def get_username(*args):
+        """
+        input: one optional string containing the fallback value
+        output: current username detected from the Operating System
+
+        If the current username is not found or is blank, return blank
+        or use the fallback value if provided.
+        """
+        TemplatingFunctions._verify_str_arguments(
+            "get_username", args, min_count=0, max_count=1
+        )
+        fallback_username = args[0] if len(args) > 0 else ""
+        return get_env_username() or fallback_username
+
+    @staticmethod
+    def sanitize_id(*args):
+        """
+        input: one string argument
+        output: a valid non-quoted SQL ID
+
+        Removes any unsafe SQL characters from the input,
+        prepend it with an underscore if it does not start with a valid character,
+        and limit the result to 255 characters.
+        The result is a valid unquoted SQL ID.
+        """
+        TemplatingFunctions._verify_str_arguments(
+            "sanitize_id", args, min_count=1, max_count=1
+        )
+
+        return sanitize_identifier(args[0])
+
+
+def get_templating_functions():
+    """
+    Returns a dictionary with all the functions available for templating
+    """
+    templating_functions = {
+        func: getattr(TemplatingFunctions, func)
+        for func in dir(TemplatingFunctions)
+        if callable(getattr(TemplatingFunctions, func)) and not func.startswith("_")
+    }
+
+    return templating_functions

snowflake/cli/app/build_and_push.sh

@@ -0,0 +1,8 @@
+set -e
+export SF_REGISTRY="$(snow spcs image-registry url -c integration)"
+DATABASE=$(echo "${SNOWFLAKE_CONNECTIONS_INTEGRATION_DATABASE}" | tr '[:upper:]' '[:lower:]')
+
+echo "Using registry: ${SF_REGISTRY}"
+docker build --platform linux/amd64 -t "${SF_REGISTRY}/${DATABASE}/public/snowcli_repository/test_counter" .
+snow spcs image-registry token --format=json -c integration | docker login "${SF_REGISTRY}/${DATABASE}/public/snowcli_repository" -u 0sessiontoken --password-stdin
+docker push "${SF_REGISTRY}/${DATABASE}/public/snowcli_repository/test_counter"

snowflake/cli/app/snow_connector.py

@@ -97,7 +97,7 @@ def connect_to_snowflake(
         k: v for k, v in connection_parameters.items() if v is not None
     }
 
-    connection_parameters = _update_connection_details_with_private_key(
+    connection_parameters = update_connection_details_with_private_key(
         connection_parameters
     )
 
@@ -163,7 +163,7 @@ def _raise_errors_related_to_session_token(
         )
 
 
-def _update_connection_details_with_private_key(connection_parameters: Dict):
+def update_connection_details_with_private_key(connection_parameters: Dict):
     if "private_key_path" in connection_parameters:
         if connection_parameters.get("authenticator") == "SNOWFLAKE_JWT":
             private_key = _load_pem_to_der(connection_parameters["private_key_path"])
@@ -189,13 +189,6 @@ def _load_pem_to_der(private_key_path: str) -> bytes:
     Given a private key file path (in PEM format), decode key data into DER
     format
     """
-    from cryptography.hazmat.backends import default_backend
-    from cryptography.hazmat.primitives.serialization import (
-        Encoding,
-        NoEncryption,
-        PrivateFormat,
-        load_pem_private_key,
-    )
 
     with SecurePath(private_key_path).open(
         "rb", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB
@@ -222,6 +215,18 @@ def _load_pem_to_der(private_key_path: str) -> bytes:
     if private_key_pem.startswith(UNENCRYPTED_PKCS8_PK_HEADER):
         private_key_passphrase = None
 
+    return prepare_private_key(private_key_pem, private_key_passphrase)
+
+
+def prepare_private_key(private_key_pem, private_key_passphrase=None):
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives.serialization import (
+        Encoding,
+        NoEncryption,
+        PrivateFormat,
+        load_pem_private_key,
+    )
+
     private_key = load_pem_private_key(
         private_key_pem,
         (
@@ -231,7 +236,6 @@ def _load_pem_to_der(private_key_path: str) -> bytes:
         ),
         default_backend(),
     )
-
     return private_key.private_bytes(
         encoding=Encoding.DER,
         format=PrivateFormat.PKCS8,

snowflake/cli/plugins/init/commands.py

@@ -20,6 +20,7 @@ from typing import Any, Dict, List, Optional
 import typer
 import yaml
 from click import ClickException
+from snowflake.cli.__about__ import VERSION
 from snowflake.cli.api.commands.flags import (
     NoInteractiveOption,
     parse_key_value_variables,
@@ -67,7 +68,8 @@ TemplateOption = typer.Option(
     show_default=False,
 )
 SourceOption = typer.Option(
-    default=DEFAULT_SOURCE,
+    DEFAULT_SOURCE,
+    "--template-source",
     help=f"local path to template directory or URL to git repository with templates.",
 )
 VariablesOption = variables_option(
@@ -131,13 +133,13 @@ def _fetch_remote_template(
     return template_root
 
 
-def _read_template_metadata(template_root: SecurePath) -> Template:
+def _read_template_metadata(template_root: SecurePath, args_error_msg: str) -> Template:
     """Parse template.yml file."""
     template_metadata_path = template_root / TEMPLATE_METADATA_FILE_NAME
     log.debug("Reading template metadata from %s", template_metadata_path.path)
     if not template_metadata_path.exists():
         raise InvalidTemplate(
-            f"Template does not have {TEMPLATE_METADATA_FILE_NAME} file."
+            f"File {TEMPLATE_METADATA_FILE_NAME} not found. {args_error_msg}"
         )
     with template_metadata_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd:
         yaml_contents = yaml.safe_load(fd) or {}
@@ -176,7 +178,6 @@ def _determine_variable_values(
 
 def _validate_cli_version(required_version: str) -> None:
     from packaging.version import parse
-    from snowflake.cli.__about__ import VERSION
 
     if parse(required_version) > parse(VERSION):
         raise ClickException(
@@ -203,6 +204,7 @@ def init(
     is_remote = any(
         template_source.startswith(prefix) for prefix in ["git@", "http://", "https://"]  # type: ignore
     )
+    args_error_msg = f"Check whether {TemplateOption.param_decls[0]} and {SourceOption.param_decls[0]} arguments are correct."
 
     # copy/download template into tmpdir, so it is going to be removed in case command ends with an error
     with SecurePath.temporary_directory() as tmpdir:
@@ -217,7 +219,9 @@ def init(
                 destination=tmpdir,
             )
 
-        template_metadata = _read_template_metadata(template_root)
+        template_metadata = _read_template_metadata(
+            template_root, args_error_msg=args_error_msg
+        )
         if template_metadata.minimum_cli_version:
             _validate_cli_version(template_metadata.minimum_cli_version)
 
@@ -225,8 +229,10 @@ def init(
             variables_metadata=template_metadata.variables,
             variables_from_flags=variables_from_flags,
             no_interactive=no_interactive,
-        )
-        variable_values["project_dir_name"] = SecurePath(path).name
+        ) | {
+            "project_dir_name": SecurePath(path).name,
+            "snowflake_cli_version": VERSION,
+        }
         log.debug(
             "Rendering template files: %s", ", ".join(template_metadata.files_to_render)
         )