snowflake-cli-labs 2.8.0rc0__py3-none-any.whl → 3.0.0rc0__py3-none-any.whl

snowflake/cli/_plugins/snowpark/common.py

@@ -0,0 +1,252 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+import re
+from enum import Enum
+from typing import Dict, List, Set
+
+from click import UsageError
+from snowflake.cli._plugins.snowpark.models import Requirement
+from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact
+from snowflake.cli.api.console import cli_console
+from snowflake.cli.api.constants import ObjectType
+from snowflake.cli.api.project.schemas.entities.snowpark_entity import (
+    ProcedureEntityModel,
+    SnowparkEntityModel,
+)
+from snowflake.cli.api.sql_execution import SqlExecutionMixin
+from snowflake.connector.cursor import SnowflakeCursor
+
+log = logging.getLogger(__name__)
+
+SnowparkEntities = Dict[str, SnowparkEntityModel]
+StageToArtefactMapping = Dict[str, set[Artefact]]
+EntityToImportPathsMapping = Dict[str, set[str]]
+
+DEFAULT_RUNTIME = "3.10"
+
+
+class SnowparkObject(Enum):
+    """This clas is used only for Snowpark execute where choice is limited."""
+
+    PROCEDURE = str(ObjectType.PROCEDURE)
+    FUNCTION = str(ObjectType.FUNCTION)
+
+
+class SnowparkObjectManager(SqlExecutionMixin):
+    def execute(
+        self, execution_identifier: str, object_type: SnowparkObject
+    ) -> SnowflakeCursor:
+        if object_type == SnowparkObject.FUNCTION:
+            return self._execute_query(f"select {execution_identifier}")
+        if object_type == SnowparkObject.PROCEDURE:
+            return self._execute_query(f"call {execution_identifier}")
+        raise UsageError(f"Unknown object type: {object_type}.")
+
+    def create_or_replace(
+        self,
+        entity: SnowparkEntityModel,
+        artifact_files: set[str],
+        snowflake_dependencies: list[str],
+    ) -> str:
+        entity.imports.extend(artifact_files)
+        imports = [f"'{x}'" for x in entity.imports]
+        packages_list = ",".join(f"'{p}'" for p in snowflake_dependencies)
+
+        object_type = entity.get_type()
+
+        query = [
+            f"create or replace {object_type} {entity.udf_sproc_identifier.identifier_for_sql}",
+            f"copy grants",
+            f"returns {entity.returns}",
+            "language python",
+            f"runtime_version={entity.runtime or DEFAULT_RUNTIME}",
+            f"imports=({', '.join(imports)})",
+            f"handler='{entity.handler}'",
+            f"packages=({packages_list})",
+        ]
+
+        if entity.external_access_integrations:
+            query.append(entity.get_external_access_integrations_sql())
+
+        if entity.secrets:
+            query.append(entity.get_secrets_sql())
+
+        if isinstance(entity, ProcedureEntityModel) and entity.execute_as_caller:
+            query.append("execute as caller")
+
+        return self._execute_query("\n".join(query))
+
+    def deploy_entity(
+        self,
+        entity: SnowparkEntityModel,
+        existing_objects: Dict[str, SnowflakeCursor],
+        snowflake_dependencies: List[str],
+        entities_to_artifact_map: EntityToImportPathsMapping,
+    ):
+        cli_console.step(f"Creating {entity.type} {entity.fqn}")
+        object_exists = entity.entity_id in existing_objects
+        replace_object = False
+        if object_exists:
+            replace_object = _check_if_replace_is_required(
+                entity=entity,
+                current_state=existing_objects[entity.entity_id],
+                snowflake_dependencies=snowflake_dependencies,
+                stage_artifact_files=entities_to_artifact_map[entity.entity_id],
+            )
+
+        state = {
+            "object": entity.udf_sproc_identifier.identifier_with_arg_names_types_defaults,
+            "type": entity.get_type(),
+        }
+        if object_exists and not replace_object:
+            return {**state, "status": "packages updated"}
+
+        self.create_or_replace(
+            entity=entity,
+            artifact_files=entities_to_artifact_map[entity.entity_id],
+            snowflake_dependencies=snowflake_dependencies,
+        )
+        return {
+            **state,
+            "status": "created" if not object_exists else "definition updated",
+        }
+
+
+def _check_if_replace_is_required(
+    entity: SnowparkEntityModel,
+    current_state,
+    snowflake_dependencies: List[str],
+    stage_artifact_files: set[str],
+) -> bool:
+    object_type = entity.get_type()
+    resource_json = _convert_resource_details_to_dict(current_state)
+    old_dependencies = resource_json["packages"]
+
+    if _snowflake_dependencies_differ(old_dependencies, snowflake_dependencies):
+        log.info(
+            "Found difference of package requirements. Replacing the %s.", object_type
+        )
+        return True
+
+    if set(entity.external_access_integrations) != set(
+        resource_json.get("external_access_integrations", [])
+    ):
+        log.info(
+            "Found difference of external access integrations. Replacing the %s.",
+            object_type,
+        )
+        return True
+
+    if (
+        resource_json["handler"].lower() != entity.handler.lower()
+        or _sql_to_python_return_type_mapper(resource_json["returns"]).lower()
+        != entity.returns.lower()
+    ):
+        log.info(
+            "Return type or handler types do not match. Replacing the %s.", object_type
+        )
+        return True
+
+    if _compare_imports(resource_json, entity.imports, stage_artifact_files):
+        log.info("Imports do not match. Replacing the %s", object_type)
+        return True
+
+    if entity.runtime is not None and entity.runtime != resource_json.get(
+        "runtime_version", "RUNTIME_NOT_SET"
+    ):
+        log.info("Runtime versions do not match. Replacing the %s", object_type)
+        return True
+
+    if isinstance(entity, ProcedureEntityModel):
+        if resource_json.get("execute as", "OWNER") != (
+            "CALLER" if entity.execute_as_caller else "OWNER"
+        ):
+            log.info(
+                "Execute as caller settings do not match. Replacing the %s", object_type
+            )
+            return True
+
+    return False
+
+
+def _convert_resource_details_to_dict(function_details: SnowflakeCursor) -> dict:
+    import json
+
+    function_dict = {}
+    json_properties = ["packages", "installed_packages"]
+    for function in function_details:
+        if function[0] in json_properties:
+            function_dict[function[0]] = json.loads(
+                function[1].replace("'", '"'),
+            )
+        else:
+            function_dict[function[0]] = function[1]
+    return function_dict
+
+
+def _snowflake_dependencies_differ(
+    old_dependencies: List[str], new_dependencies: List[str]
+) -> bool:
+    def _standardize(packages: List[str]) -> Set[str]:
+        return set(
+            Requirement.parse_line(package).name_and_version for package in packages
+        )
+
+    return _standardize(old_dependencies) != _standardize(new_dependencies)
+
+
+def _sql_to_python_return_type_mapper(resource_return_type: str) -> str:
+    """
+    Some of the Python data types get converted to SQL types, when function/procedure is created.
+    So, to properly compare types, we use mapping based on:
+    https://docs.snowflake.com/en/developer-guide/udf-stored-procedure-data-type-mapping#sql-python-data-type-mappings
+
+    Mind you, this only applies to cases, in which Snowflake accepts Python type as return.
+    Ie. if function returns list, it has to be declared as 'array' during creation,
+    therefore any conversion is not necessary
+    """
+    mapping = {
+        "number(38,0)": "int",
+        "timestamp_ntz(9)": "datetime",
+        "timestamp_tz(9)": "datetime",
+        "varchar(16777216)": "string",
+    }
+
+    return mapping.get(resource_return_type.lower(), resource_return_type.lower())
+
+
+def _compare_imports(
+    resource_json: dict, imports: List[str], artifact_files: set[str]
+) -> bool:
+    pattern = re.compile(r"(?:\[@?\w+_\w+\.)?(\w+(?:/\w+)+\.\w+)(?:\])?")
+
+    project_imports = {
+        imp
+        for import_string in [*imports, *artifact_files]
+        for imp in pattern.findall(import_string.lower())
+    }
+
+    if "imports" not in resource_json.keys():
+        object_imports = set()
+    else:
+        object_imports = {
+            imp.lower()
+            for imp in pattern.findall(resource_json.get("imports", "").lower())
+        }
+
+    return project_imports != object_imports

snowflake/cli/{plugins → _plugins}/snowpark/models.py

@@ -17,19 +17,12 @@ from __future__ import annotations
 import re
 import zipfile
 from dataclasses import dataclass
-from enum import Enum
 from pathlib import Path
 from typing import List
 
 from requirements import requirement
 
 
-class YesNoAsk(Enum):
-    YES = "yes"
-    NO = "no"
-    ASK = "ask"
-
-
 class Requirement(requirement.Requirement):
     extra_pattern = re.compile("'([^']*)'")
 

snowflake/cli/{plugins → _plugins}/snowpark/package/anaconda_packages.py

@@ -24,10 +24,10 @@ from packaging.requirements import InvalidRequirement
 from packaging.requirements import Requirement as PkgRequirement
 from packaging.version import InvalidVersion, parse
 from requests import HTTPError
+from snowflake.cli._plugins.snowpark.models import Requirement
 from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError
 from snowflake.cli.api.secure_path import SecurePath
 from snowflake.cli.api.sql_execution import SqlExecutionMixin
-from snowflake.cli.plugins.snowpark.models import Requirement
 from snowflake.connector import DictCursor
 
 log = logging.getLogger(__name__)

snowflake/cli/{plugins → _plugins}/snowpark/package/commands.py

@@ -21,35 +21,29 @@ from typing import Optional
 
 import typer
 from click import ClickException
-from snowflake.cli.api.commands.flags import (
-    deprecated_flag_callback,
-)
-from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
-from snowflake.cli.api.output.types import CommandResult, MessageResult
-from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.models import (
+from snowflake.cli._plugins.snowpark.models import (
     Requirement,
-    YesNoAsk,
 )
-from snowflake.cli.plugins.snowpark.package.anaconda_packages import (
+from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
     AnacondaPackages,
     AnacondaPackagesManager,
 )
-from snowflake.cli.plugins.snowpark.package.manager import upload
-from snowflake.cli.plugins.snowpark.package_utils import (
+from snowflake.cli._plugins.snowpark.package.manager import upload
+from snowflake.cli._plugins.snowpark.package_utils import (
     detect_and_log_shared_libraries,
     download_unavailable_packages,
     get_package_name_from_pip_wheel,
 )
-from snowflake.cli.plugins.snowpark.snowpark_shared import (
+from snowflake.cli._plugins.snowpark.snowpark_shared import (
     AllowSharedLibrariesOption,
     IgnoreAnacondaOption,
     IndexUrlOption,
     SkipVersionCheckOption,
-    deprecated_allow_native_libraries_option,
-    resolve_allow_shared_libraries_yes_no_ask,
 )
-from snowflake.cli.plugins.snowpark.zipper import zip_dir
+from snowflake.cli._plugins.snowpark.zipper import zip_dir
+from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
+from snowflake.cli.api.output.types import CommandResult, MessageResult
+from snowflake.cli.api.secure_path import SecurePath
 
 app = SnowTyperFactory(
     name="package",
@@ -58,36 +52,11 @@ app = SnowTyperFactory(
 log = logging.getLogger(__name__)
 
 
-lookup_install_option = typer.Option(
-    False,
-    "--pypi-download",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "Using --pypi-download is deprecated. Lookup command no longer checks for package in PyPi."
-    ),
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-)
-
-lookup_deprecated_install_option = typer.Option(
-    False,
-    "--yes",
-    "-y",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "Using --yes is deprecated. Lookup command no longer checks for package in PyPi."
-    ),
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-)
-
-
 @app.command("lookup", requires_connection=True)
 def package_lookup(
     package_name: str = typer.Argument(
         ..., help="Name of the package.", show_default=False
     ),
-    # todo: remove with 3.0
-    _: bool = lookup_install_option,
-    __: bool = lookup_deprecated_install_option,
     **options,
 ) -> CommandResult:
     """
@@ -125,12 +94,14 @@ def package_upload(
         "-f",
         help="Path to the file to upload.",
         exists=False,
+        show_default=False,
     ),
     stage: str = typer.Option(
         ...,
         "--stage",
         "-s",
         help="Name of the stage in which to upload the file, not including the @ symbol.",
+        show_default=False,
     ),
     overwrite: bool = typer.Option(
         False,
@@ -146,43 +117,17 @@ def package_upload(
     return MessageResult(upload(file=file, stage=stage, overwrite=overwrite))
 
 
-deprecated_pypi_download_option = typer.Option(
-    False,
-    "--pypi-download",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "Using --pypi-download is deprecated. Create command always checks for package in PyPi."
-    ),
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-)
-
-deprecated_install_option = typer.Option(
-    False,
-    "--yes",
-    "-y",
-    hidden=True,
-    help="Installs packages that are not available on the Snowflake Anaconda channel.",
-    callback=deprecated_flag_callback(
-        "Using --yes is deprecated. Create command always checks for package in PyPi."
-    ),
-)
-
-
 @app.command("create", requires_connection=True)
 def package_create(
     name: str = typer.Argument(
         ...,
         help="Name of the package to create.",
+        show_default=False,
     ),
     ignore_anaconda: bool = IgnoreAnacondaOption,
     index_url: Optional[str] = IndexUrlOption,
     skip_version_check: bool = SkipVersionCheckOption,
     allow_shared_libraries: bool = AllowSharedLibrariesOption,
-    deprecated_allow_native_libraries: YesNoAsk = deprecated_allow_native_libraries_option(
-        "--allow-native-libraries"
-    ),
-    _deprecated_install_option: bool = deprecated_install_option,
-    _deprecated_install_packages: bool = deprecated_pypi_download_option,
     **options,
 ) -> CommandResult:
     """
@@ -217,13 +162,7 @@ def package_create(
         # The package is not in anaconda, so we have to pack it
         log.info("Checking to see if packages have shared (.so/.dll) libraries...")
         if detect_and_log_shared_libraries(download_result.downloaded_packages_details):
-            # TODO: yes/no/ask logic should be removed in 3.0
-            if not (
-                allow_shared_libraries
-                or resolve_allow_shared_libraries_yes_no_ask(
-                    deprecated_allow_native_libraries
-                )
-            ):
+            if not allow_shared_libraries:
                 raise ClickException(
                     "Some packages contain shared (.so/.dll) libraries. "
                     "Try again with --allow-shared-libraries."

snowflake/cli/{plugins → _plugins}/snowpark/package/manager.py

@@ -17,9 +17,10 @@ from __future__ import annotations
 import logging
 from pathlib import Path
 
+from snowflake.cli._plugins.snowpark.package.utils import prepare_app_zip
+from snowflake.cli._plugins.stage.manager import StageManager
+from snowflake.cli.api.identifiers import FQN
 from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.package.utils import prepare_app_zip
-from snowflake.cli.plugins.stage.manager import StageManager
 
 log = logging.getLogger(__name__)
 
@@ -30,7 +31,7 @@ def upload(file: Path, stage: str, overwrite: bool):
         temp_app_zip_path = prepare_app_zip(SecurePath(file), temp_dir)
         sm = StageManager()
 
-        sm.create(sm.get_stage_from_path(stage))
+        sm.create(FQN.from_string(sm.get_stage_from_path(stage)))
         put_response = sm.put(
             temp_app_zip_path.path, stage, overwrite=overwrite
         ).fetchone()

snowflake/cli/{plugins → _plugins}/snowpark/package_utils.py

@@ -25,17 +25,17 @@ from textwrap import dedent
 from typing import Dict, List, Optional
 
 from click import ClickException
-from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
-from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.models import (
+from snowflake.cli._plugins.snowpark.models import (
     Requirement,
     RequirementWithFiles,
     RequirementWithWheel,
     WheelMetadata,
 )
-from snowflake.cli.plugins.snowpark.package.anaconda_packages import (
+from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
     AnacondaPackages,
 )
+from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
+from snowflake.cli.api.secure_path import SecurePath
 
 log = logging.getLogger(__name__)
 
@@ -61,7 +61,7 @@ def parse_requirements(
         ).splitlines():
             line = re.sub(r"\s*#.*", "", line).strip()
             if line:
-                reqs.append(Requirement.parse(line))
+                reqs.append(Requirement.parse_line(line))
     return reqs
 
 

snowflake/cli/{plugins/nativeapp → _plugins/snowpark}/plugin_spec.py

@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from snowflake.cli._plugins.snowpark import commands
 from snowflake.cli.api.plugins.command import (
     SNOWCLI_ROOT_COMMAND_PATH,
     CommandSpec,
     CommandType,
     plugin_hook_impl,
 )
-from snowflake.cli.plugins.nativeapp import commands
 
 
 @plugin_hook_impl

snowflake/cli/_plugins/snowpark/snowpark_project_paths.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+from snowflake.cli._plugins.snowpark.zipper import zip_dir
+from snowflake.cli.api.console import cli_console
+from snowflake.cli.api.constants import DEPLOYMENT_STAGE
+from snowflake.cli.api.identifiers import FQN
+from snowflake.cli.api.project.schemas.entities.snowpark_entity import PathMapping
+from snowflake.cli.api.secure_path import SecurePath
+
+
+@dataclass
+class SnowparkProjectPaths:
+    """
+    This class represents allows you to manage files paths related to given project.
+    """
+
+    project_root: Path
+
+    def path_relative_to_root(self, artifact_path: Path) -> Path:
+        if artifact_path.is_absolute():
+            return artifact_path
+        return (self.project_root / artifact_path).resolve()
+
+    def get_artefact_dto(self, artifact_path: PathMapping) -> Artefact:
+        return Artefact(
+            dest=artifact_path.dest,
+            path=self.path_relative_to_root(artifact_path.src),
+        )
+
+    def get_dependencies_artefact(self) -> Artefact:
+        return Artefact(dest=None, path=self.dependencies)
+
+    @property
+    def snowflake_requirements(self) -> SecurePath:
+        return SecurePath(
+            self.path_relative_to_root(Path("requirements.snowflake.txt"))
+        )
+
+    @property
+    def requirements(self) -> SecurePath:
+        return SecurePath(self.path_relative_to_root(Path("requirements.txt")))
+
+    @property
+    def dependencies(self) -> Path:
+        return self.path_relative_to_root(Path("dependencies.zip"))
+
+
+@dataclass(unsafe_hash=True)
+class Artefact:
+    """Helper for getting paths related to given artefact."""
+
+    path: Path
+    dest: str | None = None
+
+    @property
+    def _artefact_name(self) -> str:
+        if self.path.is_dir():
+            return self.path.stem + ".zip"
+        return self.path.name
+
+    @property
+    def post_build_path(self) -> Path:
+        """
+        Returns post-build artefact path. Directories are mapped to corresponding .zip files.
+        """
+        return self.path.parent / self._artefact_name
+
+    def upload_path(self, stage: FQN | str | None) -> str:
+        """
+        Path on stage to which the artefact should be uploaded.
+        """
+        stage = stage or DEPLOYMENT_STAGE
+        if isinstance(stage, str):
+            stage = FQN.from_stage(stage).using_context()
+
+        stage_path = PurePosixPath(f"@{stage}")
+        if self.dest:
+            stage_path = stage_path / self.dest
+        return str(stage_path) + "/"
+
+    def import_path(self, stage: FQN | str | None) -> str:
+        """Path for UDF/sproc imports clause."""
+        return self.upload_path(stage) + self._artefact_name
+
+    def build(self) -> None:
+        """Build the artefact. Applies only to directories. Files are untouched."""
+        if not self.path.is_dir():
+            return
+        cli_console.step(f"Creating: {self.post_build_path.name}")
+        zip_dir(
+            source=self.path,
+            dest_zip=self.post_build_path,
+        )

snowflake/cli/{plugins → _plugins}/snowpark/snowpark_shared.py

@@ -14,33 +14,7 @@
 
 from __future__ import annotations
 
-import click
 import typer
-from snowflake.cli.api.commands.flags import (
-    deprecated_flag_callback,
-    deprecated_flag_callback_enum,
-)
-from snowflake.cli.plugins.snowpark.models import YesNoAsk
-
-
-def deprecated_allow_native_libraries_option(old_flag_name: str):
-    return typer.Option(
-        YesNoAsk.NO.value,
-        old_flag_name,
-        help="Allows native libraries, when using packages installed through PIP",
-        hidden=True,
-        callback=deprecated_flag_callback_enum(
-            f"{old_flag_name} flag is deprecated. Use --allow-shared-libraries flag instead."
-        ),
-    )
-
-
-def resolve_allow_shared_libraries_yes_no_ask(allow_shared_libraries: YesNoAsk) -> bool:
-    if allow_shared_libraries == YesNoAsk.ASK:
-        return click.confirm("Continue with package installation?", default=False)
-    else:
-        return allow_shared_libraries == YesNoAsk.YES
-
 
 AllowSharedLibrariesOption: bool = typer.Option(
     False,
@@ -48,16 +22,6 @@ AllowSharedLibrariesOption: bool = typer.Option(
     help="Allows shared (.so) libraries, when using packages installed through PIP.",
 )
 
-DeprecatedCheckAnacondaForPyPiDependencies: bool = typer.Option(
-    True,
-    "--check-anaconda-for-pypi-deps/--no-check-anaconda-for-pypi-deps",
-    "-a",
-    help="""Checks if any of missing Anaconda packages dependencies can be imported directly from Anaconda. Valid values include: `true`, `false`, Default: `true`.""",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "--check-anaconda-for-pypi-deps flag is deprecated. Use --ignore-anaconda flag instead."
-    ),
-)
 
 IgnoreAnacondaOption: bool = typer.Option(
     False,