snowflake-cli-labs 2.8.1__py3-none-any.whl → 3.0.0__py3-none-any.whl

snowflake/cli/_plugins/snowpark/common.py

@@ -0,0 +1,268 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+import re
+from enum import Enum
+from typing import Dict, List, Set
+
+from click import UsageError
+from snowflake.cli._plugins.snowpark.models import Requirement
+from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
+    ProcedureEntityModel,
+    SnowparkEntityModel,
+)
+from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact
+from snowflake.cli.api.console import cli_console
+from snowflake.cli.api.constants import (
+    INIT_TEMPLATE_VARIABLE_CLOSING,
+    INIT_TEMPLATE_VARIABLE_OPENING,
+    PROJECT_TEMPLATE_VARIABLE_CLOSING,
+    PROJECT_TEMPLATE_VARIABLE_OPENING,
+    ObjectType,
+)
+from snowflake.cli.api.sql_execution import SqlExecutionMixin
+from snowflake.connector.cursor import SnowflakeCursor
+
+log = logging.getLogger(__name__)
+
+SnowparkEntities = Dict[str, SnowparkEntityModel]
+StageToArtefactMapping = Dict[str, set[Artefact]]
+EntityToImportPathsMapping = Dict[str, set[str]]
+
+DEFAULT_RUNTIME = "3.10"
+
+
+class SnowparkObject(Enum):
+    """This clas is used only for Snowpark execute where choice is limited."""
+
+    PROCEDURE = str(ObjectType.PROCEDURE)
+    FUNCTION = str(ObjectType.FUNCTION)
+
+
+class SnowparkObjectManager(SqlExecutionMixin):
+    def execute(
+        self, execution_identifier: str, object_type: SnowparkObject
+    ) -> SnowflakeCursor:
+        if object_type == SnowparkObject.FUNCTION:
+            return self._execute_query(f"select {execution_identifier}")
+        if object_type == SnowparkObject.PROCEDURE:
+            return self._execute_query(f"call {execution_identifier}")
+        raise UsageError(f"Unknown object type: {object_type}.")
+
+    def create_or_replace(
+        self,
+        entity: SnowparkEntityModel,
+        artifact_files: set[str],
+        snowflake_dependencies: list[str],
+    ) -> str:
+        entity.imports.extend(artifact_files)
+        imports = [f"'{x}'" for x in entity.imports]
+        packages_list = ",".join(f"'{p}'" for p in snowflake_dependencies)
+
+        object_type = entity.get_type()
+
+        query = [
+            f"create or replace {object_type} {entity.udf_sproc_identifier.identifier_for_sql}",
+            f"copy grants",
+            f"returns {entity.returns}",
+            "language python",
+            f"runtime_version={entity.runtime or DEFAULT_RUNTIME}",
+            f"imports=({', '.join(imports)})",
+            f"handler='{entity.handler}'",
+            f"packages=({packages_list})",
+        ]
+
+        if entity.external_access_integrations:
+            query.append(entity.get_external_access_integrations_sql())
+
+        if entity.secrets:
+            query.append(entity.get_secrets_sql())
+
+        if isinstance(entity, ProcedureEntityModel) and entity.execute_as_caller:
+            query.append("execute as caller")
+
+        return self._execute_query("\n".join(query))
+
+    def deploy_entity(
+        self,
+        entity: SnowparkEntityModel,
+        existing_objects: Dict[str, SnowflakeCursor],
+        snowflake_dependencies: List[str],
+        entities_to_artifact_map: EntityToImportPathsMapping,
+    ):
+        cli_console.step(f"Creating {entity.type} {entity.fqn}")
+        object_exists = entity.entity_id in existing_objects
+        replace_object = False
+        if object_exists:
+            replace_object = _check_if_replace_is_required(
+                entity=entity,
+                current_state=existing_objects[entity.entity_id],
+                snowflake_dependencies=snowflake_dependencies,
+                stage_artifact_files=entities_to_artifact_map[entity.entity_id],
+            )
+
+        state = {
+            "object": entity.udf_sproc_identifier.identifier_with_arg_names_types_defaults,
+            "type": entity.get_type(),
+        }
+        if object_exists and not replace_object:
+            return {**state, "status": "packages updated"}
+
+        self.create_or_replace(
+            entity=entity,
+            artifact_files=entities_to_artifact_map[entity.entity_id],
+            snowflake_dependencies=snowflake_dependencies,
+        )
+        return {
+            **state,
+            "status": "created" if not object_exists else "definition updated",
+        }
+
+
+def _check_if_replace_is_required(
+    entity: SnowparkEntityModel,
+    current_state,
+    snowflake_dependencies: List[str],
+    stage_artifact_files: set[str],
+) -> bool:
+    object_type = entity.get_type()
+    resource_json = _convert_resource_details_to_dict(current_state)
+    old_dependencies = resource_json["packages"]
+
+    if _snowflake_dependencies_differ(old_dependencies, snowflake_dependencies):
+        log.info(
+            "Found difference of package requirements. Replacing the %s.", object_type
+        )
+        return True
+
+    if set(entity.external_access_integrations) != set(
+        resource_json.get("external_access_integrations", [])
+    ):
+        log.info(
+            "Found difference of external access integrations. Replacing the %s.",
+            object_type,
+        )
+        return True
+
+    if (
+        resource_json["handler"].lower() != entity.handler.lower()
+        or _sql_to_python_return_type_mapper(resource_json["returns"]).lower()
+        != entity.returns.lower()
+    ):
+        log.info(
+            "Return type or handler types do not match. Replacing the %s.", object_type
+        )
+        return True
+
+    if _compare_imports(resource_json, entity.imports, stage_artifact_files):
+        log.info("Imports do not match. Replacing the %s", object_type)
+        return True
+
+    if entity.runtime is not None and entity.runtime != resource_json.get(
+        "runtime_version", "RUNTIME_NOT_SET"
+    ):
+        log.info("Runtime versions do not match. Replacing the %s", object_type)
+        return True
+
+    if isinstance(entity, ProcedureEntityModel):
+        if resource_json.get("execute as", "OWNER") != (
+            "CALLER" if entity.execute_as_caller else "OWNER"
+        ):
+            log.info(
+                "Execute as caller settings do not match. Replacing the %s", object_type
+            )
+            return True
+
+    return False
+
+
+def _convert_resource_details_to_dict(function_details: SnowflakeCursor) -> dict:
+    import json
+
+    function_dict = {}
+    json_properties = ["packages", "installed_packages"]
+    for function in function_details:
+        if function[0] in json_properties:
+            function_dict[function[0]] = json.loads(
+                function[1].replace("'", '"'),
+            )
+        else:
+            function_dict[function[0]] = function[1]
+    return function_dict
+
+
+def _snowflake_dependencies_differ(
+    old_dependencies: List[str], new_dependencies: List[str]
+) -> bool:
+    def _standardize(packages: List[str]) -> Set[str]:
+        return set(
+            Requirement.parse_line(package).name_and_version for package in packages
+        )
+
+    return _standardize(old_dependencies) != _standardize(new_dependencies)
+
+
+def _sql_to_python_return_type_mapper(resource_return_type: str) -> str:
+    """
+    Some of the Python data types get converted to SQL types, when function/procedure is created.
+    So, to properly compare types, we use mapping based on:
+    https://docs.snowflake.com/en/developer-guide/udf-stored-procedure-data-type-mapping#sql-python-data-type-mappings
+
+    Mind you, this only applies to cases, in which Snowflake accepts Python type as return.
+    Ie. if function returns list, it has to be declared as 'array' during creation,
+    therefore any conversion is not necessary
+    """
+    mapping = {
+        "number(38,0)": "int",
+        "timestamp_ntz(9)": "datetime",
+        "timestamp_tz(9)": "datetime",
+        "varchar(16777216)": "string",
+    }
+
+    return mapping.get(resource_return_type.lower(), resource_return_type.lower())
+
+
+def _compare_imports(
+    resource_json: dict, imports: List[str], artifact_files: set[str]
+) -> bool:
+    pattern = re.compile(r"(?:\[@?\w+_\w+\.)?(\w+(?:/\w+)+\.\w+)(?:\])?")
+
+    project_imports = {
+        imp
+        for import_string in [*imports, *artifact_files]
+        for imp in pattern.findall(import_string.lower())
+    }
+
+    if "imports" not in resource_json.keys():
+        object_imports = set()
+    else:
+        object_imports = {
+            imp.lower()
+            for imp in pattern.findall(resource_json.get("imports", "").lower())
+        }
+
+    return project_imports != object_imports
+
+
+def is_name_a_templated_one(name: str) -> bool:
+    return (
+        PROJECT_TEMPLATE_VARIABLE_OPENING in name
+        and PROJECT_TEMPLATE_VARIABLE_CLOSING in name
+    ) or (
+        INIT_TEMPLATE_VARIABLE_OPENING in name
+        and INIT_TEMPLATE_VARIABLE_CLOSING in name
+    )

snowflake/cli/{plugins → _plugins}/snowpark/models.py

@@ -17,19 +17,12 @@ from __future__ import annotations
 import re
 import zipfile
 from dataclasses import dataclass
-from enum import Enum
 from pathlib import Path
 from typing import List
 
 from requirements import requirement
 
 
-class YesNoAsk(Enum):
-    YES = "yes"
-    NO = "no"
-    ASK = "ask"
-
-
 class Requirement(requirement.Requirement):
     extra_pattern = re.compile("'([^']*)'")
 

snowflake/cli/{plugins → _plugins}/snowpark/package/anaconda_packages.py

@@ -18,16 +18,13 @@ import logging
 from dataclasses import dataclass
 from typing import Dict, List, Set
 
-import requests
-from click import ClickException
 from packaging.requirements import InvalidRequirement
 from packaging.requirements import Requirement as PkgRequirement
 from packaging.version import InvalidVersion, parse
-from requests import HTTPError
+from snowflake.cli._plugins.snowpark.models import Requirement
 from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError
 from snowflake.cli.api.secure_path import SecurePath
 from snowflake.cli.api.sql_execution import SqlExecutionMixin
-from snowflake.cli.plugins.snowpark.models import Requirement
 from snowflake.connector import DictCursor
 
 log = logging.getLogger(__name__)
@@ -170,23 +167,13 @@ class AnacondaPackagesManager(SqlExecutionMixin):
         "https://repo.anaconda.com/pkgs/snowflake/channeldata.json"
     )
 
-    # TODO in v3.0: Keep only SQL query, remove fallback to JSON with channel's metadata
     def find_packages_available_in_snowflake_anaconda(self) -> AnacondaPackages:
         """
         Finds python packages available in Snowflake to use in functions and stored procedures.
         It tries to get the list of packages using SQL query
         but if the try fails then the fallback is to parse JSON containing info about Snowflake's Anaconda channel.
         """
-        try:
-            packages = self._query_snowflake_for_available_packages()
-        except Exception as ex:
-            log.warning(
-                "Cannot fetch available packages information from Snowflake. "
-                "Please check your connection configuration. "
-                "Fallback to Anaconda channel metadata."
-            )
-            log.debug("Available packages query failure: %s", ex.__str__(), exc_info=ex)
-            packages = self._get_available_packages_from_anaconda_channel_info()
+        packages = self._query_snowflake_for_available_packages()
         return AnacondaPackages(packages)
 
     def _query_snowflake_for_available_packages(self) -> dict[str, AvailablePackage]:
@@ -210,24 +197,3 @@ class AnacondaPackagesManager(SqlExecutionMixin):
                     snowflake_name=package_name, versions={version}
                 )
         return packages
-
-    def _get_available_packages_from_anaconda_channel_info(
-        self,
-    ) -> dict[str, AvailablePackage]:
-        try:
-            response = requests.get(self._snowflake_channel_url)
-            response.raise_for_status()
-            packages = {}
-            for key, package in response.json()["packages"].items():
-                if not (version := package.get("version")):
-                    continue
-                package_name = package.get("name", key)
-                standardized_name = Requirement.standardize_name(package_name)
-                packages[standardized_name] = AvailablePackage(
-                    snowflake_name=package_name, versions={version}
-                )
-            return packages
-        except HTTPError as err:
-            raise ClickException(
-                f"Accessing Snowflake Anaconda channel failed. Reason {err}"
-            )

snowflake/cli/{plugins → _plugins}/snowpark/package/commands.py

@@ -21,35 +21,29 @@ from typing import Optional
 
 import typer
 from click import ClickException
-from snowflake.cli.api.commands.flags import (
-    deprecated_flag_callback,
-)
-from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
-from snowflake.cli.api.output.types import CommandResult, MessageResult
-from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.models import (
+from snowflake.cli._plugins.snowpark.models import (
     Requirement,
-    YesNoAsk,
 )
-from snowflake.cli.plugins.snowpark.package.anaconda_packages import (
+from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
     AnacondaPackages,
     AnacondaPackagesManager,
 )
-from snowflake.cli.plugins.snowpark.package.manager import upload
-from snowflake.cli.plugins.snowpark.package_utils import (
+from snowflake.cli._plugins.snowpark.package.manager import upload
+from snowflake.cli._plugins.snowpark.package_utils import (
     detect_and_log_shared_libraries,
     download_unavailable_packages,
     get_package_name_from_pip_wheel,
 )
-from snowflake.cli.plugins.snowpark.snowpark_shared import (
+from snowflake.cli._plugins.snowpark.snowpark_shared import (
     AllowSharedLibrariesOption,
     IgnoreAnacondaOption,
     IndexUrlOption,
     SkipVersionCheckOption,
-    deprecated_allow_native_libraries_option,
-    resolve_allow_shared_libraries_yes_no_ask,
 )
-from snowflake.cli.plugins.snowpark.zipper import zip_dir
+from snowflake.cli._plugins.snowpark.zipper import zip_dir
+from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
+from snowflake.cli.api.output.types import CommandResult, MessageResult
+from snowflake.cli.api.secure_path import SecurePath
 
 app = SnowTyperFactory(
     name="package",
@@ -58,36 +52,11 @@ app = SnowTyperFactory(
 log = logging.getLogger(__name__)
 
 
-lookup_install_option = typer.Option(
-    False,
-    "--pypi-download",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "Using --pypi-download is deprecated. Lookup command no longer checks for package in PyPi."
-    ),
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-)
-
-lookup_deprecated_install_option = typer.Option(
-    False,
-    "--yes",
-    "-y",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "Using --yes is deprecated. Lookup command no longer checks for package in PyPi."
-    ),
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-)
-
-
 @app.command("lookup", requires_connection=True)
 def package_lookup(
     package_name: str = typer.Argument(
         ..., help="Name of the package.", show_default=False
     ),
-    # todo: remove with 3.0
-    _: bool = lookup_install_option,
-    __: bool = lookup_deprecated_install_option,
     **options,
 ) -> CommandResult:
     """
@@ -125,12 +94,14 @@ def package_upload(
         "-f",
         help="Path to the file to upload.",
         exists=False,
+        show_default=False,
     ),
     stage: str = typer.Option(
         ...,
         "--stage",
         "-s",
         help="Name of the stage in which to upload the file, not including the @ symbol.",
+        show_default=False,
     ),
     overwrite: bool = typer.Option(
         False,
@@ -146,43 +117,17 @@ def package_upload(
     return MessageResult(upload(file=file, stage=stage, overwrite=overwrite))
 
 
-deprecated_pypi_download_option = typer.Option(
-    False,
-    "--pypi-download",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "Using --pypi-download is deprecated. Create command always checks for package in PyPi."
-    ),
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-)
-
-deprecated_install_option = typer.Option(
-    False,
-    "--yes",
-    "-y",
-    hidden=True,
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-    callback=deprecated_flag_callback(
-        "Using --yes is deprecated. Create command always checks for package in PyPi."
-    ),
-)
-
-
 @app.command("create", requires_connection=True)
 def package_create(
     name: str = typer.Argument(
         ...,
         help="Name of the package to create.",
+        show_default=False,
     ),
     ignore_anaconda: bool = IgnoreAnacondaOption,
     index_url: Optional[str] = IndexUrlOption,
     skip_version_check: bool = SkipVersionCheckOption,
     allow_shared_libraries: bool = AllowSharedLibrariesOption,
-    deprecated_allow_native_libraries: YesNoAsk = deprecated_allow_native_libraries_option(
-        "--allow-native-libraries"
-    ),
-    _deprecated_install_option: bool = deprecated_install_option,
-    _deprecated_install_packages: bool = deprecated_pypi_download_option,
     **options,
 ) -> CommandResult:
     """
@@ -217,13 +162,7 @@ def package_create(
         # The package is not in anaconda, so we have to pack it
         log.info("Checking to see if packages have shared (.so/.dll) libraries...")
         if detect_and_log_shared_libraries(download_result.downloaded_packages_details):
-            # TODO: yes/no/ask logic should be removed in 3.0
-            if not (
-                allow_shared_libraries
-                or resolve_allow_shared_libraries_yes_no_ask(
-                    deprecated_allow_native_libraries
-                )
-            ):
+            if not allow_shared_libraries:
                 raise ClickException(
                     "Some packages contain shared (.so/.dll) libraries. "
                     "Try again with --allow-shared-libraries."

snowflake/cli/{plugins → _plugins}/snowpark/package/manager.py

@@ -17,10 +17,10 @@ from __future__ import annotations
 import logging
 from pathlib import Path
 
+from snowflake.cli._plugins.snowpark.package.utils import prepare_app_zip
+from snowflake.cli._plugins.stage.manager import StageManager
 from snowflake.cli.api.identifiers import FQN
 from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.package.utils import prepare_app_zip
-from snowflake.cli.plugins.stage.manager import StageManager
 
 log = logging.getLogger(__name__)
 

snowflake/cli/{plugins → _plugins}/snowpark/package_utils.py

@@ -25,17 +25,17 @@ from textwrap import dedent
 from typing import Dict, List, Optional
 
 from click import ClickException
-from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
-from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.models import (
+from snowflake.cli._plugins.snowpark.models import (
     Requirement,
     RequirementWithFiles,
     RequirementWithWheel,
     WheelMetadata,
 )
-from snowflake.cli.plugins.snowpark.package.anaconda_packages import (
+from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
     AnacondaPackages,
 )
+from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
+from snowflake.cli.api.secure_path import SecurePath
 
 log = logging.getLogger(__name__)
 
@@ -61,7 +61,7 @@ def parse_requirements(
         ).splitlines():
             line = re.sub(r"\s*#.*", "", line).strip()
             if line:
-                reqs.append(Requirement.parse(line))
+                reqs.append(Requirement.parse_line(line))
     return reqs
 
 

snowflake/cli/_plugins/snowpark/plugin_spec.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from snowflake.cli._plugins.snowpark import commands
+from snowflake.cli.api.plugins.command import (
+    SNOWCLI_ROOT_COMMAND_PATH,
+    CommandSpec,
+    CommandType,
+    plugin_hook_impl,
+)
+
+
+@plugin_hook_impl
+def command_spec():
+    return CommandSpec(
+        parent_command_path=SNOWCLI_ROOT_COMMAND_PATH,
+        command_type=CommandType.COMMAND_GROUP,
+        typer_instance=commands.app.create_instance(),
+    )

snowflake/cli/_plugins/snowpark/snowpark_entity.py

@@ -0,0 +1,29 @@
+from typing import Generic, TypeVar
+
+from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
+    FunctionEntityModel,
+    ProcedureEntityModel,
+)
+from snowflake.cli.api.entities.common import EntityBase
+
+T = TypeVar("T")
+
+
+class SnowparkEntity(EntityBase[Generic[T]]):
+    pass
+
+
+class FunctionEntity(SnowparkEntity[FunctionEntityModel]):
+    """
+    A single UDF
+    """
+
+    pass
+
+
+class ProcedureEntity(SnowparkEntity[ProcedureEntityModel]):
+    """
+    A stored procedure
+    """
+
+    pass