snowflake-cli-labs 2.4.1__py3-none-any.whl → 2.5.0rc1__py3-none-any.whl

snowflake/cli/api/secure_path.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from __future__ import annotations
 
 import errno
@@ -50,7 +64,7 @@ class SecurePath:
         """
         When the path points to a directory, yield path objects of the directory contents.
         Otherwise, NotADirectoryError is raised.
-        If the locartion does not exists, FileNotFoundError is raised.
+        If the location does not exist, FileNotFoundError is raised.
 
         For details, check pathlib.Path.iterdir()
         """
@@ -64,6 +78,25 @@ class SecurePath:
         """
         return self._path.exists()
 
+    def is_dir(self) -> bool:
+        """
+        Return True if the path points to a directory (or a symbolic link pointing to a directory),
+        False if it points to another kind of file.
+        """
+        return self._path.is_dir()
+
+    def is_file(self) -> bool:
+        """
+        Return True if the path points to a regular file (or a symbolic link pointing to a regular file),
+        False if it points to another kind of file.
+        """
+        return self._path.is_file()
+
+    @property
+    def name(self) -> str:
+        """A string representing the final path component."""
+        return self._path.name
+
     def chmod(self, permissions_mask: int) -> None:
         """
         Change the file mode and permissions, like os.chmod().
@@ -73,6 +106,20 @@ class SecurePath:
         )
         self._path.chmod(permissions_mask)
 
+    def restrict_permissions(self) -> None:
+        """
+        Restrict file/directory permissions to owner-only.
+        """
+        import stat
+
+        owner_permissions = (
+            # https://docs.python.org/3/library/stat.html
+            stat.S_IRUSR  # readable by owner
+            | stat.S_IWUSR  # writeable by owner
+            | stat.S_IXUSR  # executable by owner
+        )
+        self.chmod(self._path.stat().st_mode & owner_permissions)
+
     def touch(self, permissions_mask: int = 0o600, exist_ok: bool = True) -> None:
         """
         Create a file at this given path. For details, check pathlib.Path.touch()
@@ -90,9 +137,13 @@ class SecurePath:
         """
         Create a directory at this given path. For details, check pathlib.Path.mkdir()
         """
+        if parents and not self.parent.exists():
+            self.parent.mkdir(
+                permissions_mask=permissions_mask, exist_ok=exist_ok, parents=True
+            )
         if not self.exists():
             log.info("Creating directory %s", str(self._path))
-        self._path.mkdir(mode=permissions_mask, parents=parents, exist_ok=exist_ok)
+        self._path.mkdir(mode=permissions_mask, exist_ok=exist_ok)
 
     def read_text(self, file_size_limit_mb: int, *args, **kwargs) -> str:
         """
@@ -203,19 +254,20 @@ class SecurePath:
                 for child in src.iterdir():
                     _recursive_check_for_conflicts(child, dst / child.name)
 
-        def _recursive_copy(src: Path, dst: Path):
+        def _recursive_copy(src: SecurePath, dst: SecurePath):
             if src.is_file():
-                log.info("Copying file %s into %s", src, dst)
+                log.info("Copying file %s into %s", src.path, dst.path)
                 if dst.exists():
                     dst.unlink()
-                shutil.copyfile(src, dst)
+                shutil.copyfile(src.path, dst.path)
+                dst.restrict_permissions()
             if src.is_dir():
-                self.__class__(dst).mkdir(exist_ok=True)
+                dst.mkdir(exist_ok=True)
                 for child in src.iterdir():
                     _recursive_copy(child, dst / child.name)
 
         _recursive_check_for_conflicts(self._path, destination)
-        _recursive_copy(self._path, destination)
+        _recursive_copy(self, self.__class__(destination))
 
         return SecurePath(destination)
 

snowflake/cli/api/secure_utils.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import stat
 from pathlib import Path
 

snowflake/cli/api/sql_execution.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from __future__ import annotations
 
 import logging

snowflake/cli/api/utils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

snowflake/cli/api/utils/cursor.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from __future__ import annotations
 
 from typing import Callable, List, Optional

snowflake/cli/api/utils/definition_rendering.py

@@ -0,0 +1,268 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import copy
+import os
+from typing import Any, Optional
+
+from jinja2 import Environment, UndefinedError, nodes
+from packaging.version import Version
+from snowflake.cli.api.exceptions import CycleDetectedError, InvalidTemplate
+from snowflake.cli.api.utils.dict_utils import deep_merge_dicts, traverse
+from snowflake.cli.api.utils.graph import Graph, Node
+from snowflake.cli.api.utils.rendering import CONTEXT_KEY, get_snowflake_cli_jinja_env
+from snowflake.cli.api.utils.types import Context, Definition
+
+
+class TemplatedEnvironment:
+    """
+    This class is a utility class
+    that encapsulates some of the Jinja Templating functionality.
+    """
+
+    def __init__(self, env: Environment):
+        self._jinja_env: Environment = env
+
+    def render(self, template_value: Any, context: Context) -> Any:
+        if not self.get_referenced_vars(template_value):
+            return template_value
+
+        jinja_template = self._jinja_env.from_string(str(template_value))
+        return jinja_template.render(context)
+
+    def get_referenced_vars(self, template_value: Any) -> set[TemplateVar]:
+        template_str = str(template_value)
+        ast = self._jinja_env.parse(template_str)
+        return self._get_referenced_vars(ast, template_str)
+
+    def _get_referenced_vars(
+        self,
+        ast_node: nodes.Template,
+        template_value: str,
+        current_attr_chain: Optional[list[str]] = None,
+    ) -> set[TemplateVar]:
+        """
+        Traverse Jinja AST to find the variable chain referenced by the template.
+        A variable like ctx.env.test is internally represented in the AST tree as
+        Getattr Node (attr='test') -> Getattr Node (attr='env') -> Name Node (name='ctx')
+        """
+        all_referenced_vars: set[TemplateVar] = set()
+        if isinstance(ast_node, nodes.Getattr):
+            current_attr_chain = [ast_node.attr] + (current_attr_chain or [])  # type: ignore[attr-defined]
+        elif isinstance(ast_node, nodes.Name):
+            current_attr_chain = [ast_node.name] + (current_attr_chain or [])  # type: ignore[attr-defined]
+            all_referenced_vars.add(TemplateVar(current_attr_chain))
+            current_attr_chain = None
+        elif (
+            not isinstance(ast_node, (nodes.Template, nodes.TemplateData, nodes.Output))
+            or current_attr_chain is not None
+        ):
+            raise InvalidTemplate(f"Unexpected templating syntax in {template_value}")
+
+        for child_node in ast_node.iter_child_nodes():
+            all_referenced_vars.update(
+                self._get_referenced_vars(
+                    child_node, template_value, current_attr_chain
+                )
+            )
+
+        return all_referenced_vars
+
+
+class TemplateVar:
+    """
+    This class tracks template variable information.
+    For a variable like ctx.env.var, this class will track
+    the chain of keys referenced by this variable (ctx, env, var),
+    as well as the value of this variable. (e.g. ctx.env.var = "hello_<% ctx.definition_version %>")
+
+    The value of this variable is divided into 2 parts.
+    The templated value (e.g. "hello_<% ctx.definition %>"),
+    as well as the rendered_value (e.g. "hello_1.1")
+    """
+
+    def __init__(self, vars_chain):
+        self._vars_chain: list[str] = list(vars_chain)
+        self.templated_value: Optional[Any] = None
+        self.rendered_value: Optional[Any] = None
+
+    @property
+    def key(self) -> str:
+        return ".".join(self._vars_chain)
+
+    @property
+    def is_env_var(self) -> bool:
+        return (
+            len(self._vars_chain) == 3
+            and self._vars_chain[0] == CONTEXT_KEY
+            and self._vars_chain[1] == "env"
+        )
+
+    def get_env_var_name(self) -> str:
+        if not self.is_env_var:
+            raise KeyError(
+                f"Referenced variable {self.key} is not an environment variable"
+            )
+        return self._vars_chain[2]
+
+    def add_to_context(self, context: Context) -> None:
+        """
+        Takes a multi-level context dict as input. Modifies the context dict with the rendered value of this variable.
+
+        If the variable has multi-levels (e.g. ctx.env), recursively traverse the dictionary
+        to set the inner level's key to the rendered value of this variable.
+
+        Example: vars chain contains ['ctx', 'env', 'x'], and context is {}, and rendered_value is 'val'.
+        At the end of this call, context content will be: {'ctx': {'env': {'x': 'val'}}}
+        """
+        current_dict_level = context
+        last_element_index = len(self._vars_chain) - 1
+        for index, var in enumerate(self._vars_chain):
+            if index == last_element_index:
+                current_dict_level[var] = self.rendered_value
+            else:
+                current_dict_level.setdefault(var, {})
+                current_dict_level = current_dict_level[var]
+
+    def read_from_context(self, context: Context) -> Any:
+        """
+        Takes a multi-level context dict as input.
+
+        If the variable has multi-levels (e.g. ctx.env), recursively traverse the dictionary
+        to find the key that the variable points to.
+
+        Returns the value in that location.
+
+        Raise UndefinedError if the variable is None or not found.
+        """
+        current_dict_level = context
+        for key in self._vars_chain:
+            if (
+                not isinstance(current_dict_level, dict)
+                or key not in current_dict_level
+            ):
+                raise UndefinedError(f"Could not find template variable {self.key}")
+            current_dict_level = current_dict_level[key]
+
+        value = current_dict_level
+        if value is None or isinstance(value, (dict, list)):
+            raise UndefinedError(
+                f"Template variable {self.key} does not contain a valid value"
+            )
+
+        return value
+
+    def __hash__(self):
+        return hash(self.key)
+
+    def __eq__(self, other):
+        return self.key == other.key
+
+
+def _build_dependency_graph(
+    env: TemplatedEnvironment, all_vars: set[TemplateVar], context: Context
+) -> Graph[TemplateVar]:
+    dependencies_graph = Graph[TemplateVar]()
+    for variable in all_vars:
+        dependencies_graph.add(Node[TemplateVar](key=variable.key, data=variable))
+
+    for variable in all_vars:
+        if variable.is_env_var and variable.get_env_var_name() in os.environ:
+            # If variable is found in os.environ, then use the value as is
+            # skip rendering by pre-setting the rendered_value attribute
+            env_value = os.environ.get(variable.get_env_var_name())
+            variable.rendered_value = env_value
+            variable.templated_value = env_value
+        else:
+            variable.templated_value = variable.read_from_context(context)
+            dependencies_vars = env.get_referenced_vars(variable.templated_value)
+
+            for referenced_var in dependencies_vars:
+                dependencies_graph.add_directed_edge(variable.key, referenced_var.key)
+
+    return dependencies_graph
+
+
+def _render_graph_node(env: TemplatedEnvironment, node: Node[TemplateVar]) -> None:
+    if node.data.rendered_value is not None:
+        # Do not re-evaluate resolved nodes like env variable nodes
+        # which might contain template-like values, or non-string nodes
+        return
+
+    current_context: Context = {}
+    for dep_node in node.neighbors:
+        dep_node.data.add_to_context(current_context)
+
+    node.data.rendered_value = env.render(node.data.templated_value, current_context)
+
+
+def render_definition_template(original_definition: Definition) -> Definition:
+    """
+    Takes a definition file as input. An arbitrary structure containing dict|list|scalars,
+    with the top level being a dictionary.
+    Requires item 'definition_version' to be set to a version of 1.1 and higher.
+
+    Searches for any templating in all of the scalar fields, and attempts to resolve them
+    from the definition structure itself or from the environment variable.
+
+    Environment variables take precedence during the rendering process.
+    """
+
+    # protect input from update
+    definition = copy.deepcopy(original_definition)
+
+    if "definition_version" not in definition or Version(
+        definition["definition_version"]
+    ) < Version("1.1"):
+        return definition
+
+    template_env = TemplatedEnvironment(get_snowflake_cli_jinja_env())
+    project_context = {CONTEXT_KEY: definition}
+
+    referenced_vars = set()
+
+    def find_any_template_vars(element):
+        referenced_vars.update(template_env.get_referenced_vars(element))
+
+    traverse(definition, visit_action=find_any_template_vars)
+
+    dependencies_graph = _build_dependency_graph(
+        template_env, referenced_vars, project_context
+    )
+
+    def on_cycle_action(node: Node[TemplateVar]):
+        raise CycleDetectedError(
+            f"Cycle detected in templating variable {node.data.key}"
+        )
+
+    dependencies_graph.dfs(
+        visit_action=lambda node: _render_graph_node(template_env, node),
+        on_cycle_action=on_cycle_action,
+    )
+
+    # now that we determined the values of all templated vars,
+    # use these resolved values as a fresh context to resolve definition
+    final_context: Context = {}
+    for node in dependencies_graph.get_all_nodes():
+        node.data.add_to_context(final_context)
+
+    traverse(
+        definition,
+        update_action=lambda val: template_env.render(val, final_context),
+    )
+    deep_merge_dicts(definition, {"env": dict(os.environ)})
+
+    return definition

snowflake/cli/api/utils/dict_utils.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+
+def deep_merge_dicts(
+    original_values: dict[Any, Any], override_values: dict[Any, Any]
+) -> None:
+    """
+    Takes 2 dictionaries as input: original and override. The original dictionary is modified.
+
+    For every key in the override dictionary, override the same key
+    in the original dictionary, or create a new one if the key is not present.
+
+    If the override value and the original value are both dictionaries,
+    instead of overriding, this function recursively calls itself to merge the keys of the sub-dictionaries.
+    """
+    if not isinstance(override_values, dict) or not isinstance(original_values, dict):
+        raise ValueError("Arguments are not of type dict")
+
+    for field, value in override_values.items():
+        if (
+            field in original_values
+            and isinstance(original_values[field], dict)
+            and isinstance(value, dict)
+        ):
+            deep_merge_dicts(original_values[field], value)
+        else:
+            original_values[field] = value
+
+
+def traverse(
+    element: Any,
+    visit_action: Callable[[Any], None] = lambda element: None,
+    update_action: Callable[[Any], Any] = lambda element: element,
+) -> Any:
+    """
+    Traverse a nested structure (lists, dicts, scalars).
+
+    On traversal, it allows for actions or updates on each visit.
+
+    visit_action: caller can provide a function to execute on each scalar element in the structure (leaves of the tree).
+    visit_action accepts an element (scalar) as input. Return value is ignored.
+
+    update_action: caller can provide a function to update each scalar element in the structure.
+    update_action accepts an element (scalar) as input, and returns the modified value.
+
+    """
+    if isinstance(element, dict):
+        for key, value in element.items():
+            element[key] = traverse(value, visit_action, update_action)
+        return element
+    elif isinstance(element, list):
+        for index, value in enumerate(element):
+            element[index] = traverse(value, visit_action, update_action)
+        return element
+    else:
+        visit_action(element)
+        return update_action(element)

snowflake/cli/api/utils/error_handling.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from contextlib import contextmanager
 
 

snowflake/cli/api/utils/graph.py

@@ -0,0 +1,97 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Callable, Generic, TypeVar
+
+T = TypeVar("T")
+
+
+class VisitStatus(Enum):
+    VISITING = 1
+    VISITED = 2
+
+
+@dataclass
+class Node(Generic[T]):
+    key: str
+    data: T
+    neighbors: set[Node[T]] = field(default_factory=set)
+
+    def __hash__(self):
+        return hash(self.key)
+
+    def __eq__(self, other):
+        return self.key == other.key
+
+
+class Graph(Generic[T]):
+    def __init__(self):
+        self._graph_nodes_map: dict[str, Node[T]] = {}
+
+    def get(self, key: str) -> Node[T]:
+        if key in self._graph_nodes_map:
+            return self._graph_nodes_map[key]
+        raise KeyError(f"Node with key {key} not found")
+
+    def get_all_nodes(self) -> set[Node[T]]:
+        return set(self._graph_nodes_map.values())
+
+    def add(self, node: Node[T]) -> None:
+        if node.key in self._graph_nodes_map:
+            raise KeyError(f"Node key {node.key} already exists")
+        self._graph_nodes_map[node.key] = node
+
+    def add_directed_edge(self, from_node_key: str, to_node_key: str) -> None:
+        from_node = self.get(from_node_key)
+        to_node = self.get(to_node_key)
+        from_node.neighbors.add(to_node)
+
+    @staticmethod
+    def _dfs_visit(
+        nodes_status: dict[str, VisitStatus],
+        node: Node[T],
+        visit_action: Callable[[Node[T]], None],
+        on_cycle_action: Callable[[Node[T]], None],
+    ) -> None:
+        if nodes_status.get(node.key) == VisitStatus.VISITED:
+            return
+
+        nodes_status[node.key] = VisitStatus.VISITING
+        for neighbor_node in node.neighbors:
+            if nodes_status.get(neighbor_node.key) == VisitStatus.VISITING:
+                on_cycle_action(node)
+            else:
+                Graph._dfs_visit(
+                    nodes_status, neighbor_node, visit_action, on_cycle_action
+                )
+
+        visit_action(node)
+
+        nodes_status[node.key] = VisitStatus.VISITED
+
+    def dfs(
+        self,
+        visit_action: Callable[[Node[T]], None] = lambda node: None,
+        on_cycle_action: Callable[[Node[T]], None] = lambda node: None,
+    ) -> None:
+        nodes_status: dict[str, VisitStatus] = {}
+        for node in self._graph_nodes_map.values():
+            Graph._dfs_visit(nodes_status, node, visit_action, on_cycle_action)
+
+    def __contains__(self, key: str) -> bool:
+        return key in self._graph_nodes_map

snowflake/cli/api/utils/models.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from __future__ import annotations
 
 import os

snowflake/cli/api/utils/naming_utils.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.