snowflake-cli-labs 3.0.0rc4__py3-none-any.whl → 3.0.1__py3-none-any.whl

snowflake/cli/_plugins/snowpark/common.py

@@ -1,268 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import logging
-import re
-from enum import Enum
-from typing import Dict, List, Set
-
-from click import UsageError
-from snowflake.cli._plugins.snowpark.models import Requirement
-from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
-    ProcedureEntityModel,
-    SnowparkEntityModel,
-)
-from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact
-from snowflake.cli.api.console import cli_console
-from snowflake.cli.api.constants import (
-    INIT_TEMPLATE_VARIABLE_CLOSING,
-    INIT_TEMPLATE_VARIABLE_OPENING,
-    PROJECT_TEMPLATE_VARIABLE_CLOSING,
-    PROJECT_TEMPLATE_VARIABLE_OPENING,
-    ObjectType,
-)
-from snowflake.cli.api.sql_execution import SqlExecutionMixin
-from snowflake.connector.cursor import SnowflakeCursor
-
-log = logging.getLogger(__name__)
-
-SnowparkEntities = Dict[str, SnowparkEntityModel]
-StageToArtefactMapping = Dict[str, set[Artefact]]
-EntityToImportPathsMapping = Dict[str, set[str]]
-
-DEFAULT_RUNTIME = "3.10"
-
-
-class SnowparkObject(Enum):
-    """This clas is used only for Snowpark execute where choice is limited."""
-
-    PROCEDURE = str(ObjectType.PROCEDURE)
-    FUNCTION = str(ObjectType.FUNCTION)
-
-
-class SnowparkObjectManager(SqlExecutionMixin):
-    def execute(
-        self, execution_identifier: str, object_type: SnowparkObject
-    ) -> SnowflakeCursor:
-        if object_type == SnowparkObject.FUNCTION:
-            return self._execute_query(f"select {execution_identifier}")
-        if object_type == SnowparkObject.PROCEDURE:
-            return self._execute_query(f"call {execution_identifier}")
-        raise UsageError(f"Unknown object type: {object_type}.")
-
-    def create_or_replace(
-        self,
-        entity: SnowparkEntityModel,
-        artifact_files: set[str],
-        snowflake_dependencies: list[str],
-    ) -> str:
-        entity.imports.extend(artifact_files)
-        imports = [f"'{x}'" for x in entity.imports]
-        packages_list = ",".join(f"'{p}'" for p in snowflake_dependencies)
-
-        object_type = entity.get_type()
-
-        query = [
-            f"create or replace {object_type} {entity.udf_sproc_identifier.identifier_for_sql}",
-            f"copy grants",
-            f"returns {entity.returns}",
-            "language python",
-            f"runtime_version={entity.runtime or DEFAULT_RUNTIME}",
-            f"imports=({', '.join(imports)})",
-            f"handler='{entity.handler}'",
-            f"packages=({packages_list})",
-        ]
-
-        if entity.external_access_integrations:
-            query.append(entity.get_external_access_integrations_sql())
-
-        if entity.secrets:
-            query.append(entity.get_secrets_sql())
-
-        if isinstance(entity, ProcedureEntityModel) and entity.execute_as_caller:
-            query.append("execute as caller")
-
-        return self._execute_query("\n".join(query))
-
-    def deploy_entity(
-        self,
-        entity: SnowparkEntityModel,
-        existing_objects: Dict[str, SnowflakeCursor],
-        snowflake_dependencies: List[str],
-        entities_to_artifact_map: EntityToImportPathsMapping,
-    ):
-        cli_console.step(f"Creating {entity.type} {entity.fqn}")
-        object_exists = entity.entity_id in existing_objects
-        replace_object = False
-        if object_exists:
-            replace_object = _check_if_replace_is_required(
-                entity=entity,
-                current_state=existing_objects[entity.entity_id],
-                snowflake_dependencies=snowflake_dependencies,
-                stage_artifact_files=entities_to_artifact_map[entity.entity_id],
-            )
-
-        state = {
-            "object": entity.udf_sproc_identifier.identifier_with_arg_names_types_defaults,
-            "type": entity.get_type(),
-        }
-        if object_exists and not replace_object:
-            return {**state, "status": "packages updated"}
-
-        self.create_or_replace(
-            entity=entity,
-            artifact_files=entities_to_artifact_map[entity.entity_id],
-            snowflake_dependencies=snowflake_dependencies,
-        )
-        return {
-            **state,
-            "status": "created" if not object_exists else "definition updated",
-        }
-
-
-def _check_if_replace_is_required(
-    entity: SnowparkEntityModel,
-    current_state,
-    snowflake_dependencies: List[str],
-    stage_artifact_files: set[str],
-) -> bool:
-    object_type = entity.get_type()
-    resource_json = _convert_resource_details_to_dict(current_state)
-    old_dependencies = resource_json["packages"]
-
-    if _snowflake_dependencies_differ(old_dependencies, snowflake_dependencies):
-        log.info(
-            "Found difference of package requirements. Replacing the %s.", object_type
-        )
-        return True
-
-    if set(entity.external_access_integrations) != set(
-        resource_json.get("external_access_integrations", [])
-    ):
-        log.info(
-            "Found difference of external access integrations. Replacing the %s.",
-            object_type,
-        )
-        return True
-
-    if (
-        resource_json["handler"].lower() != entity.handler.lower()
-        or _sql_to_python_return_type_mapper(resource_json["returns"]).lower()
-        != entity.returns.lower()
-    ):
-        log.info(
-            "Return type or handler types do not match. Replacing the %s.", object_type
-        )
-        return True
-
-    if _compare_imports(resource_json, entity.imports, stage_artifact_files):
-        log.info("Imports do not match. Replacing the %s", object_type)
-        return True
-
-    if entity.runtime is not None and entity.runtime != resource_json.get(
-        "runtime_version", "RUNTIME_NOT_SET"
-    ):
-        log.info("Runtime versions do not match. Replacing the %s", object_type)
-        return True
-
-    if isinstance(entity, ProcedureEntityModel):
-        if resource_json.get("execute as", "OWNER") != (
-            "CALLER" if entity.execute_as_caller else "OWNER"
-        ):
-            log.info(
-                "Execute as caller settings do not match. Replacing the %s", object_type
-            )
-            return True
-
-    return False
-
-
-def _convert_resource_details_to_dict(function_details: SnowflakeCursor) -> dict:
-    import json
-
-    function_dict = {}
-    json_properties = ["packages", "installed_packages"]
-    for function in function_details:
-        if function[0] in json_properties:
-            function_dict[function[0]] = json.loads(
-                function[1].replace("'", '"'),
-            )
-        else:
-            function_dict[function[0]] = function[1]
-    return function_dict
-
-
-def _snowflake_dependencies_differ(
-    old_dependencies: List[str], new_dependencies: List[str]
-) -> bool:
-    def _standardize(packages: List[str]) -> Set[str]:
-        return set(
-            Requirement.parse_line(package).name_and_version for package in packages
-        )
-
-    return _standardize(old_dependencies) != _standardize(new_dependencies)
-
-
-def _sql_to_python_return_type_mapper(resource_return_type: str) -> str:
-    """
-    Some of the Python data types get converted to SQL types, when function/procedure is created.
-    So, to properly compare types, we use mapping based on:
-    https://docs.snowflake.com/en/developer-guide/udf-stored-procedure-data-type-mapping#sql-python-data-type-mappings
-
-    Mind you, this only applies to cases, in which Snowflake accepts Python type as return.
-    Ie. if function returns list, it has to be declared as 'array' during creation,
-    therefore any conversion is not necessary
-    """
-    mapping = {
-        "number(38,0)": "int",
-        "timestamp_ntz(9)": "datetime",
-        "timestamp_tz(9)": "datetime",
-        "varchar(16777216)": "string",
-    }
-
-    return mapping.get(resource_return_type.lower(), resource_return_type.lower())
-
-
-def _compare_imports(
-    resource_json: dict, imports: List[str], artifact_files: set[str]
-) -> bool:
-    pattern = re.compile(r"(?:\[@?\w+_\w+\.)?(\w+(?:/\w+)+\.\w+)(?:\])?")
-
-    project_imports = {
-        imp
-        for import_string in [*imports, *artifact_files]
-        for imp in pattern.findall(import_string.lower())
-    }
-
-    if "imports" not in resource_json.keys():
-        object_imports = set()
-    else:
-        object_imports = {
-            imp.lower()
-            for imp in pattern.findall(resource_json.get("imports", "").lower())
-        }
-
-    return project_imports != object_imports
-
-
-def is_name_a_templated_one(name: str) -> bool:
-    return (
-        PROJECT_TEMPLATE_VARIABLE_OPENING in name
-        and PROJECT_TEMPLATE_VARIABLE_CLOSING in name
-    ) or (
-        INIT_TEMPLATE_VARIABLE_OPENING in name
-        and INIT_TEMPLATE_VARIABLE_CLOSING in name
-    )

snowflake/cli/_plugins/snowpark/models.py

@@ -1,150 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import re
-import zipfile
-from dataclasses import dataclass
-from pathlib import Path
-from typing import List
-
-from requirements import requirement
-
-
-class Requirement(requirement.Requirement):
-    extra_pattern = re.compile("'([^']*)'")
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.package_name = None
-
-    @classmethod
-    def parse_line(cls, line: str) -> Requirement:
-        if len(line_elements := line.split(";")) > 1:
-            line = line_elements[0]
-        result = super().parse_line(line)
-
-        if len(line_elements) > 1:
-            for element in line_elements[1:]:
-                if "extra" in element and (extras := cls.extra_pattern.search(element)):
-                    result.extras.extend(extras.groups())
-
-        result.package_name = result.name
-
-        if result.uri and not result.name:
-            result.name = get_package_name(result.uri)
-        result.name = cls.standardize_name(result.name)
-
-        return result
-
-    @staticmethod
-    def standardize_name(name: str) -> str:
-        return WheelMetadata.to_wheel_name_format(name.lower())
-
-    @property
-    def formatted_specs(self):
-        return ",".join(sorted(spec[0] + spec[1] for spec in self.specs))
-
-    @property
-    def name_and_version(self):
-        return self.name + self.formatted_specs
-
-
-@dataclass
-class RequirementWithFiles:
-    """A dataclass to hold a requirement and the path to the
-    downloaded files/folders that belong to it"""
-
-    requirement: Requirement
-    files: List[str]
-
-
-@dataclass
-class RequirementWithWheel:
-    """A dataclass to hold a requirement and corresponding .whl file."""
-
-    requirement: Requirement
-    wheel_path: Path | None
-
-    def extract_files(self, destination: Path) -> None:
-        if self.wheel_path is not None:
-            zipfile.ZipFile(self.wheel_path).extractall(destination)
-
-    def namelist(self) -> List[str]:
-        if self.wheel_path is None:
-            return []
-        return zipfile.ZipFile(self.wheel_path).namelist()
-
-
-@dataclass
-class WheelMetadata:
-    """A dataclass to hold metadata from .whl file.
-    [name] is the name of the package standardized according to
-    https://peps.python.org/pep-0491/#escaping-and-unicode
-    """
-
-    name: str
-    wheel_path: Path
-    dependencies: List[str]
-
-    @classmethod
-    def from_wheel(cls, wheel_path: Path):
-        """Parses wheel metadata according to
-        https://peps.python.org/pep-0491/#file-contents"""
-        with zipfile.ZipFile(wheel_path, "r") as whl:
-            metadata_path = [
-                path for path in whl.namelist() if path.endswith(".dist-info/METADATA")
-            ]
-            if len(metadata_path) != 1:
-                # malformatted wheel package
-                return None
-
-            root = zipfile.Path(whl)
-            metadata = (root / metadata_path[0]).read_text(encoding="utf-8")
-
-            dep_keyword = "Requires-Dist:"
-            dependencies = [
-                line[len(dep_keyword) :].strip()
-                for line in metadata.splitlines()
-                if line.startswith(dep_keyword)
-            ]
-            name = cls._get_name_from_wheel_filename(wheel_path.name)
-
-            return cls(name=name, wheel_path=wheel_path, dependencies=dependencies)
-
-    @staticmethod
-    def _get_name_from_wheel_filename(wheel_filename: str) -> str:
-        # wheel filename is in format {name}-{version}[-{extra info}]
-        # https://peps.python.org/pep-0491/#file-name-convention
-        return wheel_filename.split("-")[0].lower()
-
-    @staticmethod
-    def to_wheel_name_format(package_name: str) -> str:
-        # https://peps.python.org/pep-0491/#escaping-and-unicode
-        return re.sub(r"[^\w\d.]+", "_", package_name, re.UNICODE)
-
-
-def get_package_name(name: str) -> str:
-    if name.lower().startswith(("git+", "http")):
-        pattern = re.compile(r"github\.com\/[^\/]+\/([^\/][^.@$/]+)")
-        if match := pattern.search(name):
-            return match.group(1)
-        else:
-            return name
-
-    elif name.endswith(".zip"):
-        return name.replace(".zip", "")
-    else:
-        return name

snowflake/cli/_plugins/snowpark/package/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

snowflake/cli/_plugins/snowpark/package/anaconda_packages.py

@@ -1,199 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import logging
-from dataclasses import dataclass
-from typing import Dict, List, Set
-
-from packaging.requirements import InvalidRequirement
-from packaging.requirements import Requirement as PkgRequirement
-from packaging.version import InvalidVersion, parse
-from snowflake.cli._plugins.snowpark.models import Requirement
-from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError
-from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.api.sql_execution import SqlExecutionMixin
-from snowflake.connector import DictCursor
-
-log = logging.getLogger(__name__)
-
-
-@dataclass
-class FilterRequirementsResult:
-    """A dataclass to hold the results of parsing requirements files and dividing them into
-    snowflake-supported vs other packages.
-    """
-
-    in_snowflake: List[Requirement]
-    unavailable: List[Requirement]
-
-
-@dataclass
-class AvailablePackage:
-    snowflake_name: str
-    versions: Set[str]
-
-    def iter_versions(self):
-        for version in self.versions:
-            yield parse(version)
-
-    def is_required_version_available(self, requirement: Requirement) -> bool:
-        try:
-            package_specifiers = PkgRequirement(requirement.line).specifier
-            return any(
-                version in package_specifiers for version in self.iter_versions()
-            )
-        except (InvalidVersion, InvalidRequirement):
-            # fail-safe for non-pep508 formats
-            return False
-
-
-class AnacondaPackages:
-    def __init__(self, packages: Dict[str, AvailablePackage]):
-        """
-        [packages] should be a dictionary mapping package name to AnacondaPackageData object.
-        All package names should be provided in wheel escape format:
-        https://peps.python.org/pep-0491/#escaping-and-unicode
-        """
-        self._packages = packages
-
-    @classmethod
-    def empty(cls):
-        return cls({})
-
-    def is_package_available(
-        self, package: Requirement, skip_version_check: bool = False
-    ) -> bool:
-        """
-        Checks of a requirement is available in the Snowflake Anaconda Channel.
-
-        As Snowflake currently doesn't support extra syntax (ex. `jinja2[diagrams]`), if such
-        extra is present in the dependency, we mark it as unavailable.
-        """
-        if not package.name or package.extras:
-            return False
-        if package.name not in self._packages:
-            return False
-        if skip_version_check or not package.specs:
-            return True
-        return self._packages[package.name].is_required_version_available(package)
-
-    def package_latest_version(self, package: Requirement) -> str | None:
-        """Returns the latest version of the package or None if the latest version can't be determined."""
-        if package.name not in self._packages:
-            return None
-        try:
-            return str(max(self._packages[package.name].iter_versions()))
-        except InvalidVersion:
-            # fail-safe for non-pep8 versions
-            return None
-
-    def package_versions(self, package: Requirement) -> List[str]:
-        """Returns list of available versions of the package."""
-        if package.name not in self._packages:
-            return []
-        package_data = self._packages[package.name]
-        try:
-            return list(
-                str(x) for x in sorted(package_data.iter_versions(), reverse=True)
-            )
-        except InvalidVersion:
-            return list(sorted(package_data.versions, reverse=True))
-
-    def filter_available_packages(
-        self, packages: List[Requirement], skip_version_check: bool = False
-    ) -> FilterRequirementsResult:
-        """
-        Checks if a list of packages are available in the Snowflake Anaconda channel.
-        Returns an object with two attributes: 'snowflake' and 'other'.
-        Each key contains a list of Requirement object.
-
-        Parameters:
-            packages (List[Requirement]) - list of requirements to be checked
-            skip_version_check (bool) - skip comparing versions of packages
-
-        Returns:
-            result (FilterRequirementsResult) - object containing two arguments:
-              - in_snowflake - packages available in conda
-              - unavailable - packages not available in conda
-        """
-        result = FilterRequirementsResult([], [])
-        for package in packages:
-            if self.is_package_available(
-                package, skip_version_check=skip_version_check
-            ):
-                result.in_snowflake.append(package)
-            else:
-                log.info(
-                    "'%s' not found in Snowflake Anaconda channel (or ignored)...",
-                    package.name,
-                )
-                result.unavailable.append(package)
-        return result
-
-    def write_requirements_file_in_snowflake_format(
-        self,
-        file_path: SecurePath,
-        requirements: List[Requirement],
-    ):
-        """Saves requirements to a file in format accepted by Snowflake SQL commands."""
-        log.info("Writing requirements into file %s", file_path.path)
-        formatted_requirements = []
-        for requirement in requirements:
-            if requirement.name and requirement.name in self._packages:
-                snowflake_name = self._packages[requirement.name].snowflake_name
-                formatted_requirements.append(
-                    snowflake_name + requirement.formatted_specs
-                )
-
-        if formatted_requirements:
-            file_path.write_text("\n".join(formatted_requirements))
-
-
-class AnacondaPackagesManager(SqlExecutionMixin):
-    _snowflake_channel_url: str = (
-        "https://repo.anaconda.com/pkgs/snowflake/channeldata.json"
-    )
-
-    def find_packages_available_in_snowflake_anaconda(self) -> AnacondaPackages:
-        """
-        Finds python packages available in Snowflake to use in functions and stored procedures.
-        It tries to get the list of packages using SQL query
-        but if the try fails then the fallback is to parse JSON containing info about Snowflake's Anaconda channel.
-        """
-        packages = self._query_snowflake_for_available_packages()
-        return AnacondaPackages(packages)
-
-    def _query_snowflake_for_available_packages(self) -> dict[str, AvailablePackage]:
-        cursor = self._execute_query(
-            "select package_name, version from information_schema.packages where language = 'python'",
-            cursor_class=DictCursor,
-        )
-        if cursor.rowcount is None or cursor.rowcount == 0:
-            raise SnowflakeSQLExecutionError()
-        packages: dict[str, AvailablePackage] = {}
-        for row in cursor:
-            if not (package_name := row["PACKAGE_NAME"]):
-                continue
-            if not (version := row["VERSION"]):
-                continue
-            standardized_name = Requirement.standardize_name(package_name)
-            if standardized_name in packages:
-                packages[standardized_name].versions.add(version)
-            else:
-                packages[standardized_name] = AvailablePackage(
-                    snowflake_name=package_name, versions={version}
-                )
-        return packages