snowflake-cli-labs 2.8.0rc1__py3-none-any.whl → 3.0.0__py3-none-any.whl

snowflake/cli/api/metrics.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Optional
+
+
+class _TypePrefix:
+    FEATURES = "features"
+
+
+class _DomainPrefix:
+    GLOBAL = "global"
+    APP = "app"
+    SQL = "sql"
+
+
+class CLICounterField:
+    """
+    for each counter field we're adopting a convention of
+    <type>.<domain>.<name>
+    for example, if we're tracking a global feature, then the field name would be
+    features.global.feature_name
+
+    The metrics API is implemented to be generic, but we are adopting a convention
+    for feature tracking with the following model for a given command execution:
+    * counter not present -> feature is not available
+    * counter == 0 -> feature is available, but not used
+    * counter == 1 -> feature is used
+    this makes it easy to compute percentages for feature dashboards in Snowsight
+    """
+
+    TEMPLATES_PROCESSOR = (
+        f"{_TypePrefix.FEATURES}.{_DomainPrefix.GLOBAL}.templates_processor"
+    )
+    SQL_TEMPLATES = f"{_TypePrefix.FEATURES}.{_DomainPrefix.SQL}.sql_templates"
+    PDF_TEMPLATES = f"{_TypePrefix.FEATURES}.{_DomainPrefix.GLOBAL}.pdf_templates"
+    SNOWPARK_PROCESSOR = (
+        f"{_TypePrefix.FEATURES}.{_DomainPrefix.APP}.snowpark_processor"
+    )
+    POST_DEPLOY_SCRIPTS = (
+        f"{_TypePrefix.FEATURES}.{_DomainPrefix.APP}.post_deploy_scripts"
+    )
+    PACKAGE_SCRIPTS = f"{_TypePrefix.FEATURES}.{_DomainPrefix.APP}.package_scripts"
+
+
+class CLIMetrics:
+    """
+    Class to track various metrics across the execution of a command
+    """
+
+    def __init__(self):
+        self._counters: Dict[str, int] = {}
+
+    def __eq__(self, other):
+        if isinstance(other, CLIMetrics):
+            return self._counters == other._counters
+        return False
+
+    def get_counter(self, name: str) -> Optional[int]:
+        return self._counters.get(name)
+
+    def set_counter(self, name: str, value: int) -> None:
+        self._counters[name] = value
+
+    def set_counter_default(self, name: str, value: int) -> None:
+        """
+        sets the counter if it does not already exist
+        """
+        if name not in self._counters:
+            self.set_counter(name, value)
+
+    def increment_counter(self, name: str, value: int = 1) -> None:
+        if name not in self._counters:
+            self.set_counter(name, value)
+        else:
+            self._counters[name] += value
+
+    @property
+    def counters(self) -> Dict[str, int]:
+        # return a copy of the original dict to avoid mutating the original
+        return self._counters.copy()

snowflake/cli/api/project/definition.py

@@ -18,10 +18,12 @@ from pathlib import Path
 from typing import List, Optional
 
 import yaml
-from snowflake.cli.api.cli_global_context import cli_context
+from click import ClickException
+from snowflake.cli.api.cli_global_context import get_cli_context
 from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
 from snowflake.cli.api.project.schemas.project_definition import (
     ProjectProperties,
+    YamlOverride,
 )
 from snowflake.cli.api.project.util import (
     append_to_identifier,
@@ -30,9 +32,13 @@ from snowflake.cli.api.project.util import (
     to_identifier,
 )
 from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.api.utils.definition_rendering import render_definition_template
+from snowflake.cli.api.utils.definition_rendering import (
+    raw_project_properties,
+    render_definition_template,
+)
 from snowflake.cli.api.utils.dict_utils import deep_merge_dicts
 from snowflake.cli.api.utils.types import Context, Definition
+from yaml import MappingNode, SequenceNode
 
 DEFAULT_USERNAME = "unknown_user"
 
@@ -42,8 +48,14 @@ def _get_merged_definitions(paths: List[Path]) -> Optional[Definition]:
     if len(spaths) == 0:
         return None
 
+    loader = yaml.BaseLoader
+    loader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _no_duplicates_constructor
+    )
+    loader.add_constructor("!override", _override_tag)
+
     with spaths[0].open("r", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as base_yml:
-        definition = yaml.load(base_yml.read(), Loader=yaml.loader.BaseLoader) or {}
+        definition = yaml.load(base_yml.read(), Loader=loader) or {}
 
     for override_path in spaths[1:]:
         with override_path.open(
@@ -58,7 +70,9 @@ def _get_merged_definitions(paths: List[Path]) -> Optional[Definition]:
 
 
 def load_project(
-    paths: List[Path], context_overrides: Optional[Context] = None
+    paths: List[Path],
+    context_overrides: Optional[Context] = None,
+    render_templates: bool = True,
 ) -> ProjectProperties:
     """
     Loads project definition, optionally overriding values. Definition values
@@ -66,7 +80,10 @@ def load_project(
     Templating is also applied after the merging process.
     """
     merged_definitions = _get_merged_definitions(paths)
-    return render_definition_template(merged_definitions, context_overrides or {})
+    if render_templates:
+        return render_definition_template(merged_definitions, context_overrides or {})
+    else:
+        return raw_project_properties(merged_definitions)
 
 
 def default_app_package(project_name: str):
@@ -75,10 +92,35 @@ def default_app_package(project_name: str):
 
 
 def default_role():
-    conn = cli_context.connection
+    conn = get_cli_context().connection
     return conn.role
 
 
 def default_application(project_name: str):
     user = sanitize_identifier(get_env_username() or DEFAULT_USERNAME).lower()
     return append_to_identifier(to_identifier(project_name), f"_{user}")
+
+
+def _no_duplicates_constructor(loader, node, deep=False):
+    """
+    Raises error it there are duplicated keys on the same level in the yaml file
+    """
+    mapping = {}
+
+    for key_node, value_node in node.value:
+        key = loader.construct_object(key_node, deep=deep)
+        value = loader.construct_object(value_node, deep=deep)
+        if key in mapping.keys():
+            raise ClickException(
+                f"While loading the project definition file, duplicate key was found: {key}"
+            )
+        mapping[key] = value
+    return loader.construct_mapping(node, deep)
+
+
+def _override_tag(loader, node, deep=False):
+    if isinstance(node, SequenceNode):
+        return YamlOverride(data=loader.construct_sequence(node, deep))
+    if isinstance(node, MappingNode):
+        return YamlOverride(data=loader.construct_mapping(node, deep))
+    return node.value

snowflake/cli/api/project/definition_conversion.py

@@ -0,0 +1,400 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, Literal, Optional
+
+from click import ClickException
+from snowflake.cli._plugins.nativeapp.artifacts import (
+    BundleMap,
+)
+from snowflake.cli._plugins.snowpark.common import is_name_a_templated_one
+from snowflake.cli.api.constants import (
+    DEFAULT_ENV_FILE,
+    DEFAULT_PAGES_DIR,
+    PROJECT_TEMPLATE_VARIABLE_CLOSING,
+    PROJECT_TEMPLATE_VARIABLE_OPENING,
+    SNOWPARK_SHARED_MIXIN,
+)
+from snowflake.cli.api.entities.utils import render_script_template
+from snowflake.cli.api.project.schemas.entities.common import (
+    SqlScriptHookType,
+)
+from snowflake.cli.api.project.schemas.project_definition import (
+    ProjectDefinition,
+    ProjectDefinitionV2,
+)
+from snowflake.cli.api.project.schemas.v1.native_app.application import (
+    Application,
+    ApplicationV11,
+)
+from snowflake.cli.api.project.schemas.v1.native_app.native_app import NativeApp
+from snowflake.cli.api.project.schemas.v1.native_app.package import Package, PackageV11
+from snowflake.cli.api.project.schemas.v1.snowpark.callable import (
+    FunctionSchema,
+    ProcedureSchema,
+)
+from snowflake.cli.api.project.schemas.v1.snowpark.snowpark import Snowpark
+from snowflake.cli.api.project.schemas.v1.streamlit.streamlit import Streamlit
+from snowflake.cli.api.rendering.jinja import get_basic_jinja_env
+
+log = logging.getLogger(__name__)
+
+
+def _is_field_defined(template_context: Optional[Dict[str, Any]], *path: str) -> bool:
+    """
+    Determines if a field is defined in the provided template context. For example,
+
+    _is_field_defined({"ctx": {"native_app": {"bundle_root": "my_root"}}}, "ctx", "native_app", "bundle_root")
+
+    returns True. If the provided template context is None, this function returns True for all paths.
+
+    """
+    if template_context is None:
+        return True  # No context, so assume that all variables are defined
+
+    current_dict = template_context
+    for key in path:
+        if not isinstance(current_dict, dict):
+            return False
+        if key not in current_dict:
+            return False
+        current_dict = current_dict[key]
+
+    return True
+
+
+def convert_project_definition_to_v2(
+    project_root: Path,
+    pd: ProjectDefinition,
+    accept_templates: bool = False,
+    template_context: Optional[Dict[str, Any]] = None,
+) -> ProjectDefinitionV2:
+    _check_if_project_definition_meets_requirements(pd, accept_templates)
+
+    snowpark_data = convert_snowpark_to_v2_data(pd.snowpark) if pd.snowpark else {}
+    streamlit_data = convert_streamlit_to_v2_data(pd.streamlit) if pd.streamlit else {}
+    native_app_data = (
+        convert_native_app_to_v2_data(project_root, pd.native_app, template_context)
+        if pd.native_app
+        else {}
+    )
+    envs = convert_envs_to_v2(pd)
+
+    data = {
+        "definition_version": "2",
+        "entities": get_list_of_all_entities(
+            snowpark_data.get("entities", {}),
+            streamlit_data.get("entities", {}),
+            native_app_data.get("entities", {}),
+        ),
+        "mixins": snowpark_data.get("mixins", None),
+        "env": envs,
+    }
+
+    return ProjectDefinitionV2(**data)
+
+
+def convert_snowpark_to_v2_data(snowpark: Snowpark) -> Dict[str, Any]:
+    artifact_mapping = {"src": snowpark.src}
+    if snowpark.project_name:
+        artifact_mapping["dest"] = snowpark.project_name
+
+    data: dict = {
+        "mixins": {
+            SNOWPARK_SHARED_MIXIN: {
+                "stage": snowpark.stage_name,
+                "artifacts": [artifact_mapping],
+            }
+        },
+        "entities": {},
+    }
+
+    for index, entity in enumerate([*snowpark.procedures, *snowpark.functions]):
+        identifier = {"name": entity.name}
+        if entity.database is not None:
+            identifier["database"] = entity.database
+        if entity.schema_name is not None:
+            identifier["schema"] = entity.schema_name
+
+        entity_name = (
+            f"snowpark_entity_{index}"
+            if is_name_a_templated_one(entity.name)
+            else entity.name
+        )
+
+        if entity_name in data["entities"]:
+            raise ClickException(
+                f"Entity with name {entity_name} seems to be duplicated. Please rename it and try again."
+            )
+
+        v2_entity = {
+            "type": "function" if isinstance(entity, FunctionSchema) else "procedure",
+            "stage": snowpark.stage_name,
+            "handler": entity.handler,
+            "returns": entity.returns,
+            "signature": entity.signature,
+            "runtime": entity.runtime,
+            "external_access_integrations": entity.external_access_integrations,
+            "secrets": entity.secrets,
+            "imports": entity.imports,
+            "identifier": identifier,
+            "meta": {"use_mixins": [SNOWPARK_SHARED_MIXIN]},
+        }
+        if isinstance(entity, ProcedureSchema):
+            v2_entity["execute_as_caller"] = entity.execute_as_caller
+
+        data["entities"][entity_name] = v2_entity
+
+    return data
+
+
+def convert_streamlit_to_v2_data(streamlit: Streamlit) -> Dict[str, Any]:
+    # Process env file and pages dir
+    environment_file = _process_streamlit_files(streamlit.env_file, "environment")
+    pages_dir = _process_streamlit_files(streamlit.pages_dir, "pages")
+
+    # Build V2 definition
+    artifacts = [
+        streamlit.main_file,
+        environment_file,
+        pages_dir,
+    ]
+    artifacts = [a for a in artifacts if a is not None]
+
+    if streamlit.additional_source_files:
+        artifacts.extend(streamlit.additional_source_files)
+
+    identifier = {"name": streamlit.name}
+    if streamlit.schema_name:
+        identifier["schema"] = streamlit.schema_name
+    if streamlit.database:
+        identifier["database"] = streamlit.database
+
+    streamlit_name = (
+        "streamlit_entity_1"
+        if is_name_a_templated_one(streamlit.name)
+        else streamlit.name
+    )
+
+    data = {
+        "entities": {
+            streamlit_name: {
+                "type": "streamlit",
+                "identifier": identifier,
+                "title": streamlit.title,
+                "comment": streamlit.comment,
+                "query_warehouse": streamlit.query_warehouse,
+                "main_file": str(streamlit.main_file),
+                "pages_dir": str(streamlit.pages_dir),
+                "stage": streamlit.stage,
+                "artifacts": artifacts,
+            }
+        }
+    }
+    return data
+
+
+def convert_native_app_to_v2_data(
+    project_root,
+    native_app: NativeApp,
+    template_context: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    def _make_meta(obj: Application | Package):
+        meta = {}
+        if obj.role:
+            meta["role"] = obj.role
+        if obj.warehouse:
+            meta["warehouse"] = obj.warehouse
+        if obj.post_deploy:
+            meta["post_deploy"] = obj.post_deploy
+        return meta
+
+    def _find_manifest():
+        # We don't know which file in the project directory is the actual manifest,
+        # and we can't iterate through the artifacts property since the src can contain
+        # glob patterns. The simplest solution is to bundle the app and find the
+        # manifest file from the resultant BundleMap, since the bundle process ensures
+        # that only a single source path can map to the corresponding destination path
+        bundle_map = BundleMap(
+            project_root=project_root, deploy_root=Path(native_app.deploy_root)
+        )
+        for artifact in native_app.artifacts:
+            bundle_map.add(artifact)
+
+        manifest_path = next(
+            (
+                src
+                for src, dest in bundle_map.all_mappings(
+                    absolute=True, expand_directories=True
+                )
+                if dest.name == "manifest.yml"
+            ),
+            None,
+        )
+        if not manifest_path:
+            # The manifest field is required, so we can't gracefully handle it being missing
+            raise ClickException(
+                "manifest.yml file not found in any Native App artifact sources, "
+                "unable to perform migration"
+            )
+
+        # Use a POSIX path to be consistent with other migrated fields
+        # which use POSIX paths as default values
+        return manifest_path.relative_to(project_root).as_posix()
+
+    def _make_template(template: str) -> str:
+        return f"{PROJECT_TEMPLATE_VARIABLE_OPENING} {template} {PROJECT_TEMPLATE_VARIABLE_CLOSING}"
+
+    def _convert_package_script_files(package_scripts: list[str]):
+        # PDFv2 doesn't support package scripts, only post-deploy scripts, so we
+        # need to convert the Jinja syntax from {{ }} to <% %>
+        # Luckily, package scripts only support {{ package_name }}, so let's convert that tag
+        # to v2 template syntax by running it though the template process with a fake
+        # package name that's actually a valid v2 template, which will be evaluated
+        # when the script is used as a post-deploy script
+        fake_package_replacement_template = _make_template(
+            f"ctx.entities.{package_entity_name}.identifier"
+        )
+        jinja_context = dict(package_name=fake_package_replacement_template)
+        post_deploy_hooks = []
+        for script_file in package_scripts:
+            new_contents = render_script_template(
+                project_root, jinja_context, script_file, get_basic_jinja_env()
+            )
+            (project_root / script_file).write_text(new_contents)
+            post_deploy_hooks.append(SqlScriptHookType(sql_script=script_file))
+        return post_deploy_hooks
+
+    package_entity_name = "pkg"
+    if (
+        native_app.package
+        and native_app.package.name
+        and native_app.package.name != PackageV11.model_fields["name"].default
+    ):
+        package_identifier = native_app.package.name
+    else:
+        # Backport the PackageV11 default name template, updated for PDFv2
+        package_identifier = _make_template(
+            f"fn.concat_ids('{native_app.name}', '_pkg_', fn.sanitize_id(fn.get_username('unknown_user')) | lower)"
+        )
+    package = {
+        "type": "application package",
+        "identifier": package_identifier,
+        "manifest": _find_manifest(),
+        "artifacts": native_app.artifacts,
+    }
+
+    if _is_field_defined(template_context, "ctx", "native_app", "bundle_root"):
+        package["bundle_root"] = native_app.bundle_root
+    if _is_field_defined(template_context, "ctx", "native_app", "generated_root"):
+        package["generated_root"] = native_app.generated_root
+    if _is_field_defined(template_context, "ctx", "native_app", "deploy_root"):
+        package["deploy_root"] = native_app.deploy_root
+    if _is_field_defined(template_context, "ctx", "native_app", "source_stage"):
+        package["stage"] = native_app.source_stage
+    if _is_field_defined(template_context, "ctx", "native_app", "scratch_stage"):
+        package["scratch_stage"] = native_app.scratch_stage
+
+    if native_app.package:
+        if _is_field_defined(
+            template_context, "ctx", "native_app", "package", "distribution"
+        ):
+            package["distribution"] = native_app.package.distribution
+        package_meta = _make_meta(native_app.package)
+        if native_app.package.scripts:
+            converted_post_deploy_hooks = _convert_package_script_files(
+                native_app.package.scripts
+            )
+            package_meta["post_deploy"] = (
+                package_meta.get("post_deploy", []) + converted_post_deploy_hooks
+            )
+        if package_meta:
+            package["meta"] = package_meta
+
+    app_entity_name = "app"
+    if (
+        native_app.application
+        and native_app.application.name
+        and native_app.application.name != ApplicationV11.model_fields["name"].default
+    ):
+        app_identifier = native_app.application.name
+    else:
+        # Backport the ApplicationV11 default name template, updated for PDFv2
+        app_identifier = _make_template(
+            f"fn.concat_ids('{native_app.name}', '_', fn.sanitize_id(fn.get_username('unknown_user')) | lower)"
+        )
+    app = {
+        "type": "application",
+        "identifier": app_identifier,
+        "from": {"target": package_entity_name},
+    }
+    if native_app.application:
+        if app_meta := _make_meta(native_app.application):
+            app["meta"] = app_meta
+        if _is_field_defined(
+            template_context, "ctx", "native_app", "application", "debug"
+        ):
+            app["debug"] = native_app.application.debug
+
+    return {
+        "entities": {
+            package_entity_name: package,
+            app_entity_name: app,
+        }
+    }
+
+
+def convert_envs_to_v2(pd: ProjectDefinition):
+    if hasattr(pd, "env") and pd.env:
+        data = {k: v for k, v in pd.env.items()}
+        return data
+    return None
+
+
+def _check_if_project_definition_meets_requirements(
+    pd: ProjectDefinition, accept_templates: bool
+):
+    if pd.meets_version_requirement("2"):
+        raise ClickException("Project definition is already at version 2.")
+
+    if PROJECT_TEMPLATE_VARIABLE_OPENING in str(pd):
+        if not accept_templates:
+            raise ClickException(
+                "Project definition contains templates. They may not be migrated correctly, and require manual migration."
+                "You can try again with --accept-templates  option, to attempt automatic migration."
+            )
+        log.warning(
+            "Your V1 definition contains templates. We cannot guarantee the correctness of the migration."
+        )
+
+
+def _process_streamlit_files(
+    file_name: Optional[str], file_type: Literal["pages", "environment"]
+):
+    default = DEFAULT_PAGES_DIR if file_type == "pages" else DEFAULT_ENV_FILE
+
+    if file_name and not Path(file_name).exists():
+        raise ClickException(f"Provided file {file_name} does not exist")
+    elif file_name is None and Path(default).exists():
+        file_name = default
+    return file_name
+
+
+def get_list_of_all_entities(
+    snowpark_entities: Dict[str, Any],
+    streamlit_entities: Dict[str, Any],
+    native_app_entities: Dict[str, Any],
+):
+    # Check all combinations of entity types for overlapping names
+    # (No need to use itertools here, PDFv1 only supports these three types)
+    for types, first, second in [
+        ("streamlit and snowpark", streamlit_entities, snowpark_entities),
+        ("streamlit and native app", streamlit_entities, native_app_entities),
+        ("native app and snowpark", native_app_entities, snowpark_entities),
+    ]:
+        if first.keys() & second.keys():
+            raise ClickException(
+                f"In your project, {types} entities share the same name. Please rename them and try again."
+            )
+    return snowpark_entities | streamlit_entities | native_app_entities

snowflake/cli/api/project/definition_manager.py

@@ -20,7 +20,10 @@ from pathlib import Path
 from typing import List, Optional
 
 from snowflake.cli.api.project.definition import ProjectProperties, load_project
-from snowflake.cli.api.project.schemas.project_definition import ProjectDefinitionV1
+from snowflake.cli.api.project.schemas.project_definition import (
+    ProjectDefinition,
+    ProjectDefinitionV1,
+)
 from snowflake.cli.api.utils.types import Context
 
 
@@ -38,7 +41,7 @@ class DefinitionManager:
     USER_DEFINITION_FILENAME = "snowflake.local.yml"
 
     project_root: Path
-    _project_config_paths: List[Path]
+    project_config_paths: List[Path]
 
     def __init__(
         self,
@@ -50,12 +53,12 @@ class DefinitionManager:
         )
 
         self.project_root = project_root
-        self._project_config_paths = self._find_definition_files(self.project_root)
+        self.project_config_paths = self._find_definition_files(self.project_root)
         self._context_overrides = context_overrides
 
     @functools.cached_property
     def has_definition_file(self):
-        return len(self._project_config_paths) > 0
+        return len(self.project_config_paths) > 0
 
     @staticmethod
     def _find_definition_files(project_root: Path) -> List[Path]:
@@ -123,12 +126,20 @@ class DefinitionManager:
 
     @functools.cached_property
     def _project_properties(self) -> ProjectProperties:
-        return load_project(self._project_config_paths, self._context_overrides)
+        return load_project(self.project_config_paths, self._context_overrides)
+
+    @functools.cached_property
+    def _raw_project_data(self) -> ProjectProperties:
+        return load_project(self.project_config_paths, {}, False)
 
     @functools.cached_property
     def project_definition(self) -> ProjectDefinitionV1:
         return self._project_properties.project_definition
 
+    @functools.cached_property
+    def unrendered_project_definition(self) -> ProjectDefinition:
+        return self._raw_project_data.project_definition
+
     @functools.cached_property
     def template_context(self) -> Context:
         return self._project_properties.project_context

snowflake/cli/api/project/project_verification.py

@@ -11,13 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from snowflake.cli.api.cli_global_context import cli_context
+from snowflake.cli.api.cli_global_context import get_cli_context
 from snowflake.cli.api.exceptions import NoProjectDefinitionError
 
 
 def assert_project_type(project_type: str):
+    cli_context = get_cli_context()
     if not getattr(cli_context.project_definition, project_type, None):
         raise NoProjectDefinitionError(
-            project_type=project_type, project_file=cli_context.project_root
+            project_type=project_type, project_root=cli_context.project_root
         )