snowflake-cli-labs 2.8.0rc0__py3-none-any.whl → 3.0.0rc0__py3-none-any.whl

snowflake/cli/api/entities/utils.py

@@ -0,0 +1,321 @@
+import os
+from pathlib import Path
+from textwrap import dedent
+from typing import Any, Callable, Dict, List, NoReturn, Optional
+
+import jinja2
+from click import ClickException
+from snowflake.cli._plugins.nativeapp.artifacts import (
+    BundleMap,
+    resolve_without_follow,
+)
+from snowflake.cli._plugins.nativeapp.constants import OWNER_COL
+from snowflake.cli._plugins.nativeapp.exceptions import (
+    InvalidScriptError,
+    MissingScriptError,
+    UnexpectedOwnerError,
+)
+from snowflake.cli._plugins.nativeapp.utils import verify_exists, verify_no_directories
+from snowflake.cli._plugins.stage.diff import (
+    DiffResult,
+    StagePath,
+    compute_stage_diff,
+    preserve_from_diff,
+    sync_local_diff_with_stage,
+    to_stage_path,
+)
+from snowflake.cli._plugins.stage.utils import print_diff_to_console
+from snowflake.cli.api.console.abc import AbstractConsole
+from snowflake.cli.api.entities.common import get_sql_executor
+from snowflake.cli.api.errno import (
+    DOES_NOT_EXIST_OR_CANNOT_BE_PERFORMED,
+    NO_WAREHOUSE_SELECTED_IN_SESSION,
+)
+from snowflake.cli.api.project.schemas.entities.common import PostDeployHook
+from snowflake.cli.api.project.util import unquote_identifier
+from snowflake.cli.api.rendering.sql_templates import (
+    snowflake_sql_jinja_render,
+)
+from snowflake.cli.api.secure_path import UNLIMITED, SecurePath
+from snowflake.connector import ProgrammingError
+
+
+def generic_sql_error_handler(
+    err: ProgrammingError, role: Optional[str] = None, warehouse: Optional[str] = None
+) -> NoReturn:
+    # Potential refactor: If moving away from Python 3.8 and 3.9 to >= 3.10, use match ... case
+    if err.errno == DOES_NOT_EXIST_OR_CANNOT_BE_PERFORMED:
+        raise ProgrammingError(
+            msg=dedent(
+                f"""\
+                Received error message '{err.msg}' while executing SQL statement.
+                '{role}' may not have access to warehouse '{warehouse}'.
+                Please grant usage privilege on warehouse to this role.
+                """
+            ),
+            errno=err.errno,
+        )
+    elif err.errno == NO_WAREHOUSE_SELECTED_IN_SESSION:
+        raise ProgrammingError(
+            msg=dedent(
+                f"""\
+                Received error message '{err.msg}' while executing SQL statement.
+                Please provide a warehouse for the active session role in your project definition file, config.toml file, or via command line.
+                """
+            ),
+            errno=err.errno,
+        )
+    elif "does not exist or not authorized" in err.msg:
+        raise ProgrammingError(
+            msg=dedent(
+                f"""\
+                Received error message '{err.msg}' while executing SQL statement.
+                Please check the name of the resource you are trying to query or the permissions of the role you are using to run the query.
+                """
+            )
+        )
+    raise err
+
+
+def ensure_correct_owner(row: dict, role: str, obj_name: str) -> None:
+    """
+    Check if an object has the right owner role
+    """
+    actual_owner = row[
+        OWNER_COL
+    ].upper()  # Because unquote_identifier() always returns uppercase str
+    if actual_owner != unquote_identifier(role):
+        raise UnexpectedOwnerError(obj_name, role, actual_owner)
+
+
+def _get_stage_paths_to_sync(
+    local_paths_to_sync: List[Path], deploy_root: Path
+) -> List[StagePath]:
+    """
+    Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root.
+    """
+
+    stage_paths = []
+    for path in local_paths_to_sync:
+        if path.is_dir():
+            for current_dir, _dirs, files in os.walk(path):
+                for file in files:
+                    deploy_path = Path(current_dir, file).relative_to(deploy_root)
+                    stage_paths.append(to_stage_path(deploy_path))
+        else:
+            stage_paths.append(to_stage_path(path.relative_to(deploy_root)))
+    return stage_paths
+
+
+def sync_deploy_root_with_stage(
+    console: AbstractConsole,
+    deploy_root: Path,
+    package_name: str,
+    stage_schema: str,
+    bundle_map: BundleMap,
+    role: str,
+    prune: bool,
+    recursive: bool,
+    stage_fqn: str,
+    local_paths_to_sync: List[Path] | None = None,
+    print_diff: bool = True,
+) -> DiffResult:
+    """
+    Ensures that the files on our remote stage match the artifacts we have in
+    the local filesystem.
+
+    Args:
+        bundle_map (BundleMap): The artifact mapping computed by the `build_bundle` function.
+        role (str): The name of the role to use for queries and commands.
+        prune (bool): Whether to prune artifacts from the stage that don't exist locally.
+        recursive (bool): Whether to traverse directories recursively.
+        stage_fqn (str): The name of the stage to diff against and upload to.
+        local_paths_to_sync (List[Path], optional): List of local paths to sync. Defaults to None to sync all
+         local paths. Note that providing an empty list here is equivalent to None.
+        print_diff (bool): Whether to print the diff between the local files and the remote stage. Defaults to True
+
+    Returns:
+        A `DiffResult` instance describing the changes that were performed.
+    """
+
+    sql_executor = get_sql_executor()
+    # Does a stage already exist within the application package, or we need to create one?
+    # Using "if not exists" should take care of either case.
+    console.step(
+        f"Checking if stage {stage_fqn} exists, or creating a new one if none exists."
+    )
+    with sql_executor.use_role(role):
+        sql_executor.execute_query(
+            f"create schema if not exists {package_name}.{stage_schema}"
+        )
+        sql_executor.execute_query(
+            f"""
+                    create stage if not exists {stage_fqn}
+                    encryption = (TYPE = 'SNOWFLAKE_SSE')
+                    DIRECTORY = (ENABLE = TRUE)"""
+        )
+
+    # Perform a diff operation and display results to the user for informational purposes
+    if print_diff:
+        console.step(
+            "Performing a diff between the Snowflake stage and your local deploy_root ('%s') directory."
+            % deploy_root.resolve()
+        )
+    diff: DiffResult = compute_stage_diff(deploy_root, stage_fqn)
+
+    if local_paths_to_sync:
+        # Deploying specific files/directories
+        resolved_paths_to_sync = [
+            resolve_without_follow(p) for p in local_paths_to_sync
+        ]
+        if not recursive:
+            verify_no_directories(resolved_paths_to_sync)
+
+        deploy_paths_to_sync = []
+        for resolved_path in resolved_paths_to_sync:
+            verify_exists(resolved_path)
+            deploy_paths = bundle_map.to_deploy_paths(resolved_path)
+            if not deploy_paths:
+                if resolved_path.is_dir() and recursive:
+                    # No direct artifact mapping found for this path. Check to see
+                    # if there are subpaths of this directory that are matches. We
+                    # loop over sources because it's likely a much smaller list
+                    # than the project directory.
+                    for src in bundle_map.all_sources(absolute=True):
+                        if resolved_path in src.parents:
+                            # There is a source that contains this path, get its dest path(s)
+                            deploy_paths.extend(bundle_map.to_deploy_paths(src))
+
+            if not deploy_paths:
+                raise ClickException(f"No artifact found for {resolved_path}")
+            deploy_paths_to_sync.extend(deploy_paths)
+
+        stage_paths_to_sync = _get_stage_paths_to_sync(
+            deploy_paths_to_sync, resolve_without_follow(deploy_root)
+        )
+        diff = preserve_from_diff(diff, stage_paths_to_sync)
+    else:
+        # Full deploy
+        if not recursive:
+            verify_no_directories(deploy_root.resolve().iterdir())
+
+    if not prune:
+        files_not_removed = [str(path) for path in diff.only_on_stage]
+        diff.only_on_stage = []
+
+        if len(files_not_removed) > 0:
+            files_not_removed_str = "\n".join(files_not_removed)
+            console.warning(
+                f"The following files exist only on the stage:\n{files_not_removed_str}\n\nUse the --prune flag to delete them from the stage."
+            )
+
+    if print_diff:
+        print_diff_to_console(diff, bundle_map)
+
+    # Upload diff-ed files to application package stage
+    if diff.has_changes():
+        console.step(
+            "Updating the Snowflake stage from your local %s directory."
+            % deploy_root.resolve(),
+        )
+        sync_local_diff_with_stage(
+            role=role,
+            deploy_root_path=deploy_root,
+            diff_result=diff,
+            stage_fqn=stage_fqn,
+        )
+    return diff
+
+
+def _execute_sql_script(
+    script_content: str,
+    database_name: Optional[str] = None,
+) -> None:
+    """
+    Executing the provided SQL script content.
+    This assumes that a relevant warehouse is already active.
+    If database_name is passed in, it will be used first.
+    """
+    try:
+        sql_executor = get_sql_executor()
+        if database_name is not None:
+            sql_executor.execute_query(f"use database {database_name}")
+        sql_executor.execute_queries(script_content)
+    except ProgrammingError as err:
+        generic_sql_error_handler(err)
+
+
+def execute_post_deploy_hooks(
+    console: AbstractConsole,
+    project_root: Path,
+    post_deploy_hooks: Optional[List[PostDeployHook]],
+    deployed_object_type: str,
+    database_name: str,
+) -> None:
+    """
+    Executes post-deploy hooks for the given object type.
+    While executing SQL post deploy hooks, it first switches to the database provided in the input.
+    All post deploy scripts templates will first be expanded using the global template context.
+    """
+    if not post_deploy_hooks:
+        return
+
+    with console.phase(f"Executing {deployed_object_type} post-deploy actions"):
+        sql_scripts_paths = []
+        for hook in post_deploy_hooks:
+            if hook.sql_script:
+                sql_scripts_paths.append(hook.sql_script)
+            else:
+                raise ValueError(
+                    f"Unsupported {deployed_object_type} post-deploy hook type: {hook}"
+                )
+
+        scripts_content_list = render_script_templates(
+            project_root,
+            snowflake_sql_jinja_render,
+            {},
+            sql_scripts_paths,
+        )
+
+        for index, sql_script_path in enumerate(sql_scripts_paths):
+            console.step(f"Executing SQL script: {sql_script_path}")
+            _execute_sql_script(
+                script_content=scripts_content_list[index],
+                database_name=database_name,
+            )
+
+
+def render_script_templates(
+    project_root: Path,
+    render_from_str: Callable[[str, Dict[str, Any]], str],
+    jinja_context: dict[str, Any],
+    scripts: List[str],
+) -> List[str]:
+    """
+    Input:
+    - project_root: path to project root
+    - render_from_str: function which renders a jinja template from a string and jinja context
+    - jinja_context: a dictionary with the jinja context
+    - scripts: list of script paths relative to the project root
+    Returns:
+    - List of rendered scripts content
+    Size of the return list is the same as the size of the input scripts list.
+    """
+    scripts_contents = []
+    for relpath in scripts:
+        script_full_path = SecurePath(project_root) / relpath
+        try:
+            template_content = script_full_path.read_text(file_size_limit_mb=UNLIMITED)
+            result = render_from_str(template_content, jinja_context)
+            scripts_contents.append(result)
+
+        except FileNotFoundError as e:
+            raise MissingScriptError(relpath) from e
+
+        except jinja2.TemplateSyntaxError as e:
+            raise InvalidScriptError(relpath, e, e.lineno) from e
+
+        except jinja2.UndefinedError as e:
+            raise InvalidScriptError(relpath, e) from e
+
+    return scripts_contents

snowflake/cli/api/exceptions.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Optional
 
-from click.exceptions import ClickException
+from click.exceptions import ClickException, UsageError
 from snowflake.cli.api.constants import ObjectType
 
 
@@ -101,9 +101,16 @@ class ObjectAlreadyExistsError(ClickException):
 
 
 class NoProjectDefinitionError(ClickException):
-    def __init__(self, project_type: str, project_file: str):
+    def __init__(self, project_type: str, project_root: str | Path):
         super().__init__(
-            f"No {project_type} project definition found in {project_file}"
+            f"No {project_type} project definition found in {project_root}"
+        )
+
+
+class InvalidProjectDefinitionVersionError(ClickException):
+    def __init__(self, expected_version: str, actual_version: str):
+        super().__init__(
+            f"This command only supports definition version {expected_version}, got {actual_version}."
         )
 
 
@@ -162,3 +169,12 @@ class FQNInconsistencyError(ClickException):
         super().__init__(
             f"{part.capitalize()} provided but name '{name}' is fully qualified name."
         )
+
+
+class IncompatibleParametersError(UsageError):
+    def __init__(self, options: list[str]):
+        options_with_quotes = [f"'{option}'" for option in options]
+        comma_separated_options = ", ".join(options_with_quotes[:-1])
+        super().__init__(
+            f"Parameters {comma_separated_options} and {options_with_quotes[-1]} are incompatible and cannot be used simultaneously."
+        )

snowflake/cli/api/feature_flags.py

@@ -52,4 +52,5 @@ class FeatureFlag(FeatureFlagMixin):
     ENABLE_STREAMLIT_VERSIONED_STAGE = BooleanFlag(
         "ENABLE_STREAMLIT_VERSIONED_STAGE", False
     )
-    ENABLE_PROJECT_DEFINITION_V2 = BooleanFlag("ENABLE_PROJECT_DEFINITION_V2", False)
+    # TODO: remove in 3.0
+    ENABLE_PROJECT_DEFINITION_V2 = BooleanFlag("ENABLE_PROJECT_DEFINITION_V2", True)

snowflake/cli/api/identifiers.py

@@ -17,9 +17,11 @@ from __future__ import annotations
 import re
 
 from click import ClickException
-from snowflake.cli.api.cli_global_context import cli_context
 from snowflake.cli.api.exceptions import FQNInconsistencyError, FQNNameError
-from snowflake.cli.api.project.schemas.identifier_model import ObjectIdentifierBaseModel
+from snowflake.cli.api.project.schemas.identifier_model import (
+    Identifier,
+    ObjectIdentifierBaseModel,
+)
 from snowflake.cli.api.project.util import VALID_IDENTIFIER_REGEX, identifier_for_url
 
 
@@ -35,10 +37,17 @@ class FQN:
     fqn = FQN.from_string("my_name").set_database("db").set_schema("foo")
     """
 
-    def __init__(self, database: str | None, schema: str | None, name: str):
+    def __init__(
+        self,
+        database: str | None,
+        schema: str | None,
+        name: str,
+        signature: str | None = None,
+    ):
         self._database = database
         self._schema = schema
         self._name = name
+        self.signature = signature
 
     @property
     def database(self) -> str | None:
@@ -72,6 +81,8 @@ class FQN:
 
     @property
     def sql_identifier(self) -> str:
+        if self.signature:
+            return f"IDENTIFIER('{self.identifier}'){self.signature}"
         return f"IDENTIFIER('{self.identifier}')"
 
     def __str__(self):
@@ -98,9 +109,13 @@ class FQN:
         else:
             database = None
             schema = result.group("first_qualifier")
-        if signature := result.group("signature"):
-            unqualified_name = unqualified_name + signature
-        return cls(name=unqualified_name, schema=schema, database=database)
+
+        signature = None
+        if result.group("signature"):
+            signature = result.group("signature")
+        return cls(
+            name=unqualified_name, schema=schema, database=database, signature=signature
+        )
 
     @classmethod
     def from_stage(cls, stage: str) -> "FQN":
@@ -110,11 +125,11 @@ class FQN:
         return cls.from_string(name)
 
     @classmethod
-    def from_identifier_model(cls, model: ObjectIdentifierBaseModel) -> "FQN":
+    def from_identifier_model_v1(cls, model: ObjectIdentifierBaseModel) -> "FQN":
         """Create an instance from object model."""
         if not isinstance(model, ObjectIdentifierBaseModel):
             raise ClickException(
-                f"Expected {type(ObjectIdentifierBaseModel)}, got {model}."
+                f"Expected {type(ObjectIdentifierBaseModel).__name__}, got {model}."
             )
 
         fqn = cls.from_string(model.name)
@@ -126,6 +141,21 @@ class FQN:
 
         return fqn.set_database(model.database).set_schema(model.schema_name)
 
+    @classmethod
+    def from_identifier_model_v2(cls, model: Identifier) -> "FQN":
+        """Create an instance from object model."""
+        if not isinstance(model, Identifier):
+            raise ClickException(f"Expected {type(Identifier).__name__}, got {model}.")
+
+        fqn = cls.from_string(model.name)
+
+        if fqn.database and model.database:
+            raise FQNInconsistencyError("database", model.name)
+        if fqn.schema and model.schema_:
+            raise FQNInconsistencyError("schema", model.name)
+
+        return fqn.set_database(model.database).set_schema(model.schema_)
+
     def set_database(self, database: str | None) -> "FQN":
         if database:
             self._database = database
@@ -151,4 +181,6 @@ class FQN:
 
     def using_context(self) -> "FQN":
         """Update the instance with database and schema from connection in current cli context."""
-        return self.using_connection(cli_context.connection)
+        from snowflake.cli.api.cli_global_context import get_cli_context
+
+        return self.using_connection(get_cli_context().connection)

snowflake/cli/api/project/definition.py

@@ -18,7 +18,7 @@ from pathlib import Path
 from typing import List, Optional
 
 import yaml
-from snowflake.cli.api.cli_global_context import cli_context
+from snowflake.cli.api.cli_global_context import get_cli_context
 from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
 from snowflake.cli.api.project.schemas.project_definition import (
     ProjectProperties,
@@ -30,7 +30,10 @@ from snowflake.cli.api.project.util import (
     to_identifier,
 )
 from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.api.utils.definition_rendering import render_definition_template
+from snowflake.cli.api.utils.definition_rendering import (
+    raw_project_properties,
+    render_definition_template,
+)
 from snowflake.cli.api.utils.dict_utils import deep_merge_dicts
 from snowflake.cli.api.utils.types import Context, Definition
 
@@ -58,7 +61,9 @@ def _get_merged_definitions(paths: List[Path]) -> Optional[Definition]:
 
 
 def load_project(
-    paths: List[Path], context_overrides: Optional[Context] = None
+    paths: List[Path],
+    context_overrides: Optional[Context] = None,
+    render_templates: bool = True,
 ) -> ProjectProperties:
     """
     Loads project definition, optionally overriding values. Definition values
@@ -66,7 +71,10 @@ def load_project(
     Templating is also applied after the merging process.
     """
     merged_definitions = _get_merged_definitions(paths)
-    return render_definition_template(merged_definitions, context_overrides or {})
+    if render_templates:
+        return render_definition_template(merged_definitions, context_overrides or {})
+    else:
+        return raw_project_properties(merged_definitions)
 
 
 def default_app_package(project_name: str):
@@ -75,7 +83,7 @@ def default_app_package(project_name: str):
 
 
 def default_role():
-    conn = cli_context.connection
+    conn = get_cli_context().connection
     return conn.role
 
 

snowflake/cli/api/project/definition_manager.py

@@ -20,7 +20,10 @@ from pathlib import Path
 from typing import List, Optional
 
 from snowflake.cli.api.project.definition import ProjectProperties, load_project
-from snowflake.cli.api.project.schemas.project_definition import ProjectDefinitionV1
+from snowflake.cli.api.project.schemas.project_definition import (
+    ProjectDefinition,
+    ProjectDefinitionV1,
+)
 from snowflake.cli.api.utils.types import Context
 
 
@@ -125,10 +128,18 @@ class DefinitionManager:
     def _project_properties(self) -> ProjectProperties:
         return load_project(self._project_config_paths, self._context_overrides)
 
+    @functools.cached_property
+    def _raw_project_data(self) -> ProjectProperties:
+        return load_project(self._project_config_paths, {}, False)
+
     @functools.cached_property
     def project_definition(self) -> ProjectDefinitionV1:
         return self._project_properties.project_definition
 
+    @functools.cached_property
+    def unrendered_project_definition(self) -> ProjectDefinition:
+        return self._raw_project_data.project_definition
+
     @functools.cached_property
     def template_context(self) -> Context:
         return self._project_properties.project_context

snowflake/cli/api/project/errors.py

@@ -29,10 +29,25 @@ class SchemaValidationError(ClickException):
     def __init__(self, error: ValidationError):
         errors = error.errors()
         message = f"During evaluation of {error.title} in project definition following errors were encountered:\n"
+
+        def calculate_location(e):
+            if e["loc"] is None:
+                return None
+
+            # show numbers as list indexes and strings as dictionary keys. Example: key1[0].key2
+            result = "".join(
+                f"[{item}]" if isinstance(item, int) else f".{item}"
+                for item in e["loc"]
+            )
+
+            # remove leading dot from the string if any:
+            return result[1:] if result.startswith(".") else result
+
         message += "\n".join(
             [
                 self.message_templates.get(e["type"], self.generic_message).format(
-                    **e, location=".".join(e["loc"]) if e["loc"] is not None else None
+                    **e,
+                    location=calculate_location(e),
                 )
                 for e in errors
             ]

snowflake/cli/api/project/project_verification.py

@@ -11,13 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from snowflake.cli.api.cli_global_context import cli_context
+from snowflake.cli.api.cli_global_context import get_cli_context
 from snowflake.cli.api.exceptions import NoProjectDefinitionError
 
 
 def assert_project_type(project_type: str):
+    cli_context = get_cli_context()
     if not getattr(cli_context.project_definition, project_type, None):
         raise NoProjectDefinitionError(
-            project_type=project_type, project_file=cli_context.project_root
+            project_type=project_type, project_root=cli_context.project_root
         )

snowflake/cli/api/project/schemas/entities/{application_entity.py → application_entity_model.py}

@@ -16,25 +16,24 @@ from __future__ import annotations
 
 from typing import Literal, Optional
 
-from pydantic import Field
-from snowflake.cli.api.project.schemas.entities.application_package_entity import (
-    ApplicationPackageEntity,
+from pydantic import Field, field_validator
+from snowflake.cli.api.project.schemas.entities.application_package_entity_model import (
+    ApplicationPackageEntityModel,
 )
 from snowflake.cli.api.project.schemas.entities.common import (
-    EntityBase,
+    EntityModelBase,
     TargetField,
 )
+from snowflake.cli.api.project.schemas.identifier_model import Identifier
 from snowflake.cli.api.project.schemas.updatable_model import (
     DiscriminatorField,
 )
+from snowflake.cli.api.project.util import append_test_resource_suffix
 
 
-class ApplicationEntity(EntityBase):
+class ApplicationEntityModel(EntityModelBase):
     type: Literal["application"] = DiscriminatorField()  # noqa A003
-    name: str = Field(
-        title="Name of the application created when this entity is deployed"
-    )
-    from_: TargetField[ApplicationPackageEntity] = Field(
+    from_: TargetField[ApplicationPackageEntityModel] = Field(
         alias="from",
         title="An application package this entity should be created from",
     )
@@ -42,3 +41,16 @@ class ApplicationEntity(EntityBase):
         title="Whether to enable debug mode when using a named stage to create an application object",
         default=None,
     )
+
+    @field_validator("identifier")
+    @classmethod
+    def append_test_resource_suffix_to_identifier(
+        cls, input_value: Identifier | str
+    ) -> Identifier | str:
+        identifier = (
+            input_value.name if isinstance(input_value, Identifier) else input_value
+        )
+        with_suffix = append_test_resource_suffix(identifier)
+        if isinstance(input_value, Identifier):
+            return input_value.model_copy(update=dict(name=with_suffix))
+        return with_suffix