snowflake-cli-labs 3.0.0rc5__py3-none-any.whl → 3.0.2__py3-none-any.whl

snowflake/cli/_plugins/snowpark/package/commands.py

@@ -1,195 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import logging
-from pathlib import Path
-from textwrap import dedent
-from typing import Optional
-
-import typer
-from click import ClickException
-from snowflake.cli._plugins.snowpark.models import (
-    Requirement,
-)
-from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
-    AnacondaPackages,
-    AnacondaPackagesManager,
-)
-from snowflake.cli._plugins.snowpark.package.manager import upload
-from snowflake.cli._plugins.snowpark.package_utils import (
-    detect_and_log_shared_libraries,
-    download_unavailable_packages,
-    get_package_name_from_pip_wheel,
-)
-from snowflake.cli._plugins.snowpark.snowpark_shared import (
-    AllowSharedLibrariesOption,
-    IgnoreAnacondaOption,
-    IndexUrlOption,
-    SkipVersionCheckOption,
-)
-from snowflake.cli._plugins.snowpark.zipper import zip_dir
-from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
-from snowflake.cli.api.output.types import CommandResult, MessageResult
-from snowflake.cli.api.secure_path import SecurePath
-
-app = SnowTyperFactory(
-    name="package",
-    help="Manages custom Python packages for Snowpark",
-)
-log = logging.getLogger(__name__)
-
-
-@app.command("lookup", requires_connection=True)
-def package_lookup(
-    package_name: str = typer.Argument(
-        ..., help="Name of the package.", show_default=False
-    ),
-    **options,
-) -> CommandResult:
-    """
-    Checks if a package is available on the Snowflake Anaconda channel.
-    """
-    anaconda_packages_manager = AnacondaPackagesManager()
-    anaconda_packages = (
-        anaconda_packages_manager.find_packages_available_in_snowflake_anaconda()
-    )
-
-    package = Requirement.parse(package_name)
-    if anaconda_packages.is_package_available(package=package):
-        msg = f"Package `{package_name}` is available in Anaconda"
-        if version := anaconda_packages.package_latest_version(package=package):
-            msg += f". Latest available version: {version}."
-        elif versions := anaconda_packages.package_versions(package=package):
-            msg += f" in versions: {', '.join(versions)}."
-        return MessageResult(msg)
-
-    return MessageResult(
-        dedent(
-            f"""
-        Package `{package_name}` is not available in Anaconda. To prepare Snowpark compatible package run:
-        snow snowpark package create {package_name}
-        """
-        )
-    )
-
-
-@app.command("upload", requires_connection=True)
-def package_upload(
-    file: Path = typer.Option(
-        ...,
-        "--file",
-        "-f",
-        help="Path to the file to upload.",
-        exists=False,
-        show_default=False,
-    ),
-    stage: str = typer.Option(
-        ...,
-        "--stage",
-        "-s",
-        help="Name of the stage in which to upload the file, not including the @ symbol.",
-        show_default=False,
-    ),
-    overwrite: bool = typer.Option(
-        False,
-        "--overwrite",
-        "-o",
-        help="Overwrites the file if it already exists.",
-    ),
-    **options,
-) -> CommandResult:
-    """
-    Uploads a Python package zip file to a Snowflake stage so it can be referenced in the imports of a procedure or function.
-    """
-    return MessageResult(upload(file=file, stage=stage, overwrite=overwrite))
-
-
-@app.command("create", requires_connection=True)
-def package_create(
-    name: str = typer.Argument(
-        ...,
-        help="Name of the package to create.",
-        show_default=False,
-    ),
-    ignore_anaconda: bool = IgnoreAnacondaOption,
-    index_url: Optional[str] = IndexUrlOption,
-    skip_version_check: bool = SkipVersionCheckOption,
-    allow_shared_libraries: bool = AllowSharedLibrariesOption,
-    **options,
-) -> CommandResult:
-    """
-    Creates a Python package as a zip file that can be uploaded to a stage and imported for a Snowpark Python app.
-    """
-    with SecurePath.temporary_directory() as packages_dir:
-        package = Requirement.parse(name)
-        anaconda_packages_manager = AnacondaPackagesManager()
-        download_result = download_unavailable_packages(
-            requirements=[package],
-            target_dir=packages_dir,
-            anaconda_packages=(
-                AnacondaPackages.empty()
-                if ignore_anaconda
-                else anaconda_packages_manager.find_packages_available_in_snowflake_anaconda()
-            ),
-            skip_version_check=skip_version_check,
-            pip_index_url=index_url,
-        )
-        if not download_result.succeeded:
-            raise ClickException(download_result.error_message)
-
-        # check if package was detected as available
-        package_available_in_conda = any(
-            p.line == package.line for p in download_result.anaconda_packages
-        )
-        if package_available_in_conda:
-            return MessageResult(
-                f"Package {name} is already available in Snowflake Anaconda Channel."
-            )
-
-        # The package is not in anaconda, so we have to pack it
-        log.info("Checking to see if packages have shared (.so/.dll) libraries...")
-        if detect_and_log_shared_libraries(download_result.downloaded_packages_details):
-            if not allow_shared_libraries:
-                raise ClickException(
-                    "Some packages contain shared (.so/.dll) libraries. "
-                    "Try again with --allow-shared-libraries."
-                )
-
-        # The package is not in anaconda, so we have to pack it
-        # the package was downloaded once, pip wheel should use cache
-        zip_file = f"{get_package_name_from_pip_wheel(name, index_url=index_url)}.zip"
-        zip_dir(dest_zip=Path(zip_file), source=packages_dir.path)
-        message = dedent(
-            f"""
-        Package {zip_file} created. You can now upload it to a stage using
-        snow snowpark package upload -f {zip_file} -s <stage-name>`
-        and reference it in your procedure or function.
-        Remember to add it to imports in the procedure or function definition.
-        """
-        )
-        if download_result.anaconda_packages:
-            message += dedent(
-                f"""
-                The package {name} is successfully created, but depends on the following
-                Anaconda libraries. They need to be included in project requirements,
-                as their are not included in .zip.
-                """
-            )
-            message += "\n".join(
-                (req.line for req in download_result.anaconda_packages)
-            )
-
-        return MessageResult(message)

snowflake/cli/_plugins/snowpark/package/manager.py

@@ -1,44 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import logging
-from pathlib import Path
-
-from snowflake.cli._plugins.snowpark.package.utils import prepare_app_zip
-from snowflake.cli._plugins.stage.manager import StageManager
-from snowflake.cli.api.identifiers import FQN
-from snowflake.cli.api.secure_path import SecurePath
-
-log = logging.getLogger(__name__)
-
-
-def upload(file: Path, stage: str, overwrite: bool):
-    log.info("Uploading %s to Snowflake @%s/%s...", file, stage, file)
-    with SecurePath.temporary_directory() as temp_dir:
-        temp_app_zip_path = prepare_app_zip(SecurePath(file), temp_dir)
-        sm = StageManager()
-
-        sm.create(FQN.from_string(sm.get_stage_from_path(stage)))
-        put_response = sm.put(
-            temp_app_zip_path.path, stage, overwrite=overwrite
-        ).fetchone()
-
-    message = f"Package {file} {put_response[6]} to Snowflake @{stage}/{file}."
-
-    if put_response[6] == "SKIPPED":
-        message = "Package already exists on stage. Consider using --overwrite to overwrite the file."
-
-    return message

snowflake/cli/_plugins/snowpark/package/utils.py

@@ -1,26 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-from snowflake.cli.api.secure_path import SecurePath
-
-
-def prepare_app_zip(file_path: SecurePath, temp_dir: SecurePath) -> SecurePath:
-    # get filename from file path (e.g. app.zip from /path/to/app.zip)
-    # TODO: think if no file exceptions are handled correctly
-    file_name = file_path.path.name
-    temp_path = temp_dir / file_name
-    file_path.copy(temp_path.path)
-    return temp_path

snowflake/cli/_plugins/snowpark/package_utils.py

@@ -1,354 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import dataclasses
-import locale
-import logging
-import os
-import re
-import subprocess
-from pathlib import Path
-from textwrap import dedent
-from typing import Dict, List, Optional
-
-from click import ClickException
-from snowflake.cli._plugins.snowpark.models import (
-    Requirement,
-    RequirementWithFiles,
-    RequirementWithWheel,
-    WheelMetadata,
-)
-from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
-    AnacondaPackages,
-)
-from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
-from snowflake.cli.api.secure_path import SecurePath
-
-log = logging.getLogger(__name__)
-
-PIP_PATH = os.environ.get("SNOWCLI_PIP_PATH", "pip")
-
-
-def parse_requirements(
-    requirements_file: SecurePath = SecurePath("requirements.txt"),
-) -> List[Requirement]:
-    """Reads and parses a Python requirements.txt file.
-
-    Args:
-        requirements_file (str, optional): The name of the file.
-        Defaults to 'requirements.txt'.
-
-    Returns:
-        list[Requirement]: A flat list of necessary packages
-    """
-    reqs = []
-    if requirements_file.exists():
-        for line in requirements_file.read_text(
-            file_size_limit_mb=DEFAULT_SIZE_LIMIT_MB
-        ).splitlines():
-            line = re.sub(r"\s*#.*", "", line).strip()
-            if line:
-                reqs.append(Requirement.parse_line(line))
-    return reqs
-
-
-def generate_deploy_stage_name(identifier: str) -> str:
-    return (
-        identifier.replace("()", "")
-        .replace(
-            "(",
-            "_",
-        )
-        .replace(
-            ")",
-            "",
-        )
-        .replace(
-            " ",
-            "_",
-        )
-        .replace(
-            ",",
-            "",
-        )
-    )
-
-
-def get_package_name_from_pip_wheel(package: str, index_url: str | None = None) -> str:
-    """Downloads the package using pip and returns the package name.
-    If the package name cannot be determined, it returns the [package]."""
-    with SecurePath.temporary_directory() as tmp_dir:
-        pip_result = pip_wheel(
-            package_name=package,
-            requirements_file=None,
-            download_dir=tmp_dir.path,
-            index_url=index_url,
-            dependencies=False,
-        )
-        file_list = [
-            f.path.name for f in tmp_dir.iterdir() if f.path.name.endswith(".whl")
-        ]
-
-        if pip_result != 0 or len(file_list) != 1:
-            # cannot determine package name
-            return package
-        return WheelMetadata.from_wheel((tmp_dir / file_list[0]).path).name
-
-
-def _write_requirements_file(file_path: SecurePath, requirements: List[Requirement]):
-    log.info("Writing %s file", file_path.path)
-    with file_path.open("w", encoding="utf-8") as f:
-        for req in requirements:
-            f.write(f"{req.line}\n")
-
-
-@dataclasses.dataclass
-class DownloadUnavailablePackagesResult:
-    succeeded: bool
-    error_message: str | None = None
-    anaconda_packages: List[Requirement] = dataclasses.field(default_factory=list)
-    downloaded_packages_details: List[RequirementWithFiles] = dataclasses.field(
-        default_factory=list
-    )
-
-
-def download_unavailable_packages(
-    requirements: List[Requirement],
-    target_dir: SecurePath,
-    # available packages lookup specs
-    anaconda_packages: AnacondaPackages,
-    skip_version_check: bool = False,
-    # pip lookup specs
-    pip_index_url: str | None = None,
-) -> DownloadUnavailablePackagesResult:
-    """Download packages unavailable on Snowflake Anaconda Channel to target directory.
-
-    Returns an object with fields:
-    - download_successful - whether packages were successfully downloaded
-    - error_message - error message if download was not successful
-    - anaconda_packages - list of omitted packages
-    - downloaded_packages - list of downloaded packages details
-    """
-    # pre-check of available packages to avoid potentially heavy downloads
-    split_requirements = anaconda_packages.filter_available_packages(
-        requirements, skip_version_check=skip_version_check
-    )
-    packages_in_snowflake = split_requirements.in_snowflake
-    requirements = split_requirements.unavailable
-    if not requirements:
-        # all packages are available in Snowflake
-        return DownloadUnavailablePackagesResult(
-            succeeded=True,
-            anaconda_packages=packages_in_snowflake,
-        )
-
-    # download all packages with their dependencies
-    with SecurePath.temporary_directory() as downloads_dir:
-        # This is a Windows workaround where use TemporaryDirectory instead of NamedTemporaryFile
-        requirements_file = downloads_dir / "requirements.txt"
-        _write_requirements_file(requirements_file, requirements)  # type: ignore
-        pip_wheel_result = pip_wheel(
-            package_name=None,
-            requirements_file=requirements_file.path,  # type: ignore
-            download_dir=downloads_dir.path,
-            index_url=pip_index_url,
-            dependencies=True,
-        )
-        if pip_wheel_result != 0:
-            log.info(_pip_failed_log_msg(pip_wheel_result))
-            return DownloadUnavailablePackagesResult(
-                succeeded=False,
-                error_message=_pip_failed_log_msg(pip_wheel_result),
-            )
-
-        # scan all downloaded packages and filter out ones available on Anaconda
-        dependencies = split_downloaded_dependencies(
-            requirements_file,
-            downloads_dir=downloads_dir.path,
-            anaconda_packages=anaconda_packages,
-            skip_version_check=skip_version_check,
-        )
-        log.info(
-            "Downloaded packages: %s",
-            ", ".join(
-                dep.requirement.name
-                for dep in dependencies.unavailable_dependencies_wheels
-            ),
-        )
-        _log_dependencies_found_in_conda(dependencies.snowflake_dependencies)
-        packages_in_snowflake += dependencies.snowflake_dependencies
-
-        # move filtered packages to target directory
-        target_dir.mkdir(exist_ok=True)
-        for package in dependencies.unavailable_dependencies_wheels:
-            package.extract_files(target_dir.path)
-        return DownloadUnavailablePackagesResult(
-            succeeded=True,
-            anaconda_packages=packages_in_snowflake,
-            downloaded_packages_details=[
-                RequirementWithFiles(requirement=dep.requirement, files=dep.namelist())
-                for dep in dependencies.unavailable_dependencies_wheels
-            ],
-        )
-
-
-def pip_wheel(
-    requirements_file: Optional[str],
-    package_name: Optional[str],
-    download_dir: Path,
-    index_url: Optional[str],
-    dependencies: bool = True,
-):
-    command = ["-m", "pip", "wheel", "-w", str(download_dir)]
-    if package_name:
-        command.append(package_name)
-    if requirements_file:
-        command.extend(["-r", str(requirements_file)])
-    if index_url:
-        command.extend(["-i", index_url])
-    if not dependencies:
-        command.append("--no-deps")
-
-    try:
-        log.info(
-            "Running pip wheel with command: %s",
-            " ".join([str(com) for com in command]),
-        )
-        process = subprocess.run(
-            ["python", *command],
-            capture_output=True,
-            text=True,
-            encoding=locale.getpreferredencoding(),
-        )
-    except subprocess.CalledProcessError as e:
-        log.error("Encountered error %s", e.stderr)
-        raise ClickException(f"Encountered error while running pip wheel.")
-
-    log.info("Pip wheel command executed successfully")
-    return process.returncode
-
-
-@dataclasses.dataclass
-class SplitDownloadedDependenciesResult:
-    snowflake_dependencies: List[Requirement] = dataclasses.field(default_factory=list)
-    unavailable_dependencies_wheels: List[RequirementWithWheel] = dataclasses.field(
-        default_factory=list
-    )
-
-
-def split_downloaded_dependencies(
-    requirements_file: SecurePath,
-    downloads_dir: Path,
-    anaconda_packages: AnacondaPackages,
-    skip_version_check: bool,
-) -> SplitDownloadedDependenciesResult:
-    packages_metadata: Dict[str, WheelMetadata] = {
-        meta.name: meta
-        for meta in (
-            WheelMetadata.from_wheel(wheel_path)
-            for wheel_path in downloads_dir.glob("*.whl")
-        )
-        if meta is not None
-    }
-    available_in_snowflake_dependencies: Dict = {}
-    unavailable_dependencies: Dict = {}
-
-    def _get_dependencies(package: Requirement):
-        if (
-            package.name not in available_in_snowflake_dependencies
-            and package.name not in unavailable_dependencies
-        ):
-            if anaconda_packages.is_package_available(
-                package, skip_version_check=skip_version_check
-            ):
-                available_in_snowflake_dependencies[package.name] = package
-            else:
-                meta = packages_metadata.get(
-                    WheelMetadata.to_wheel_name_format(package.name)
-                )
-                wheel_path = meta.wheel_path if meta else None
-                requires = meta.dependencies if meta else []
-                unavailable_dependencies[package.name] = RequirementWithWheel(
-                    requirement=package,
-                    wheel_path=wheel_path,
-                )
-
-                log.debug(
-                    "Checking package %s, with dependencies: %s", package.name, requires
-                )
-
-                for package in requires:
-                    _get_dependencies(Requirement.parse_line(package))
-
-    with requirements_file.open("r", read_file_limit_mb=512) as req_file:
-        for line in req_file:
-            _get_dependencies(Requirement.parse_line(line))
-
-    return SplitDownloadedDependenciesResult(
-        snowflake_dependencies=list(available_in_snowflake_dependencies.values()),
-        unavailable_dependencies_wheels=list(unavailable_dependencies.values()),
-    )
-
-
-def detect_and_log_shared_libraries(dependencies: List[RequirementWithFiles]):
-    shared_libraries = [
-        dependency.requirement.name
-        for dependency in dependencies
-        if any(
-            file.endswith(".so") or file.endswith(".dll") for file in dependency.files
-        )
-    ]
-    if shared_libraries:
-        _log_shared_libraries(shared_libraries)
-        return True
-    else:
-        log.info("Unsupported native libraries not found in packages (Good news!)...")
-        return False
-
-
-def _log_shared_libraries(
-    shared_libraries: List[str],
-) -> None:
-    log.error(
-        "Following dependencies utilise shared libraries, not supported by Conda:"
-    )
-    log.error("\n".join(set(shared_libraries)))
-    log.error(
-        "You may still try to create your package with --allow-shared-libraries, but the might not work."
-    )
-    log.error("You may also request adding the package to Snowflake Conda channel")
-    log.error("at https://support.anaconda.com/")
-
-
-def _log_dependencies_found_in_conda(available_dependencies: List[Requirement]) -> None:
-    if len(available_dependencies) > 0:
-        log.info(
-            "Good news! Packages available in Anaconda and excluded from the zip archive: %s",
-            format(", ".join(dep.name for dep in available_dependencies)),
-        )
-    else:
-        log.info("None of the package dependencies were found on Anaconda")
-
-
-def _pip_failed_log_msg(return_code: int) -> str:
-    return dedent(
-        f"""
-        pip failed with return code {return_code}. Most likely reasons:
-         * incorrect package name or version
-         * package isn't compatible with host architecture (most probably due to .so libraries)
-         * pip is not installed correctly
-        """
-    )

snowflake/cli/_plugins/snowpark/plugin_spec.py

@@ -1,30 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from snowflake.cli._plugins.snowpark import commands
-from snowflake.cli.api.plugins.command import (
-    SNOWCLI_ROOT_COMMAND_PATH,
-    CommandSpec,
-    CommandType,
-    plugin_hook_impl,
-)
-
-
-@plugin_hook_impl
-def command_spec():
-    return CommandSpec(
-        parent_command_path=SNOWCLI_ROOT_COMMAND_PATH,
-        command_type=CommandType.COMMAND_GROUP,
-        typer_instance=commands.app.create_instance(),
-    )

snowflake/cli/_plugins/snowpark/snowpark_entity.py

@@ -1,29 +0,0 @@
-from typing import Generic, TypeVar
-
-from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
-    FunctionEntityModel,
-    ProcedureEntityModel,
-)
-from snowflake.cli.api.entities.common import EntityBase
-
-T = TypeVar("T")
-
-
-class SnowparkEntity(EntityBase[Generic[T]]):
-    pass
-
-
-class FunctionEntity(SnowparkEntity[FunctionEntityModel]):
-    """
-    A single UDF
-    """
-
-    pass
-
-
-class ProcedureEntity(SnowparkEntity[ProcedureEntityModel]):
-    """
-    A stored procedure
-    """
-
-    pass