snowflake-cli-labs 2.8.0rc1__py3-none-any.whl → 2.8.2__py3-none-any.whl

snowflake/cli/plugins/snowpark/package_utils.py

@@ -1,354 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import dataclasses
-import locale
-import logging
-import os
-import re
-import subprocess
-from pathlib import Path
-from textwrap import dedent
-from typing import Dict, List, Optional
-
-from click import ClickException
-from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
-from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.models import (
-    Requirement,
-    RequirementWithFiles,
-    RequirementWithWheel,
-    WheelMetadata,
-)
-from snowflake.cli.plugins.snowpark.package.anaconda_packages import (
-    AnacondaPackages,
-)
-
-log = logging.getLogger(__name__)
-
-PIP_PATH = os.environ.get("SNOWCLI_PIP_PATH", "pip")
-
-
-def parse_requirements(
-    requirements_file: SecurePath = SecurePath("requirements.txt"),
-) -> List[Requirement]:
-    """Reads and parses a Python requirements.txt file.
-
-    Args:
-        requirements_file (str, optional): The name of the file.
-        Defaults to 'requirements.txt'.
-
-    Returns:
-        list[Requirement]: A flat list of necessary packages
-    """
-    reqs = []
-    if requirements_file.exists():
-        for line in requirements_file.read_text(
-            file_size_limit_mb=DEFAULT_SIZE_LIMIT_MB
-        ).splitlines():
-            line = re.sub(r"\s*#.*", "", line).strip()
-            if line:
-                reqs.append(Requirement.parse(line))
-    return reqs
-
-
-def generate_deploy_stage_name(identifier: str) -> str:
-    return (
-        identifier.replace("()", "")
-        .replace(
-            "(",
-            "_",
-        )
-        .replace(
-            ")",
-            "",
-        )
-        .replace(
-            " ",
-            "_",
-        )
-        .replace(
-            ",",
-            "",
-        )
-    )
-
-
-def get_package_name_from_pip_wheel(package: str, index_url: str | None = None) -> str:
-    """Downloads the package using pip and returns the package name.
-    If the package name cannot be determined, it returns the [package]."""
-    with SecurePath.temporary_directory() as tmp_dir:
-        pip_result = pip_wheel(
-            package_name=package,
-            requirements_file=None,
-            download_dir=tmp_dir.path,
-            index_url=index_url,
-            dependencies=False,
-        )
-        file_list = [
-            f.path.name for f in tmp_dir.iterdir() if f.path.name.endswith(".whl")
-        ]
-
-        if pip_result != 0 or len(file_list) != 1:
-            # cannot determine package name
-            return package
-        return WheelMetadata.from_wheel((tmp_dir / file_list[0]).path).name
-
-
-def _write_requirements_file(file_path: SecurePath, requirements: List[Requirement]):
-    log.info("Writing %s file", file_path.path)
-    with file_path.open("w", encoding="utf-8") as f:
-        for req in requirements:
-            f.write(f"{req.line}\n")
-
-
-@dataclasses.dataclass
-class DownloadUnavailablePackagesResult:
-    succeeded: bool
-    error_message: str | None = None
-    anaconda_packages: List[Requirement] = dataclasses.field(default_factory=list)
-    downloaded_packages_details: List[RequirementWithFiles] = dataclasses.field(
-        default_factory=list
-    )
-
-
-def download_unavailable_packages(
-    requirements: List[Requirement],
-    target_dir: SecurePath,
-    # available packages lookup specs
-    anaconda_packages: AnacondaPackages,
-    skip_version_check: bool = False,
-    # pip lookup specs
-    pip_index_url: str | None = None,
-) -> DownloadUnavailablePackagesResult:
-    """Download packages unavailable on Snowflake Anaconda Channel to target directory.
-
-    Returns an object with fields:
-    - download_successful - whether packages were successfully downloaded
-    - error_message - error message if download was not successful
-    - anaconda_packages - list of omitted packages
-    - downloaded_packages - list of downloaded packages details
-    """
-    # pre-check of available packages to avoid potentially heavy downloads
-    split_requirements = anaconda_packages.filter_available_packages(
-        requirements, skip_version_check=skip_version_check
-    )
-    packages_in_snowflake = split_requirements.in_snowflake
-    requirements = split_requirements.unavailable
-    if not requirements:
-        # all packages are available in Snowflake
-        return DownloadUnavailablePackagesResult(
-            succeeded=True,
-            anaconda_packages=packages_in_snowflake,
-        )
-
-    # download all packages with their dependencies
-    with SecurePath.temporary_directory() as downloads_dir:
-        # This is a Windows workaround where use TemporaryDirectory instead of NamedTemporaryFile
-        requirements_file = downloads_dir / "requirements.txt"
-        _write_requirements_file(requirements_file, requirements)  # type: ignore
-        pip_wheel_result = pip_wheel(
-            package_name=None,
-            requirements_file=requirements_file.path,  # type: ignore
-            download_dir=downloads_dir.path,
-            index_url=pip_index_url,
-            dependencies=True,
-        )
-        if pip_wheel_result != 0:
-            log.info(_pip_failed_log_msg(pip_wheel_result))
-            return DownloadUnavailablePackagesResult(
-                succeeded=False,
-                error_message=_pip_failed_log_msg(pip_wheel_result),
-            )
-
-        # scan all downloaded packages and filter out ones available on Anaconda
-        dependencies = split_downloaded_dependencies(
-            requirements_file,
-            downloads_dir=downloads_dir.path,
-            anaconda_packages=anaconda_packages,
-            skip_version_check=skip_version_check,
-        )
-        log.info(
-            "Downloaded packages: %s",
-            ", ".join(
-                dep.requirement.name
-                for dep in dependencies.unavailable_dependencies_wheels
-            ),
-        )
-        _log_dependencies_found_in_conda(dependencies.snowflake_dependencies)
-        packages_in_snowflake += dependencies.snowflake_dependencies
-
-        # move filtered packages to target directory
-        target_dir.mkdir(exist_ok=True)
-        for package in dependencies.unavailable_dependencies_wheels:
-            package.extract_files(target_dir.path)
-        return DownloadUnavailablePackagesResult(
-            succeeded=True,
-            anaconda_packages=packages_in_snowflake,
-            downloaded_packages_details=[
-                RequirementWithFiles(requirement=dep.requirement, files=dep.namelist())
-                for dep in dependencies.unavailable_dependencies_wheels
-            ],
-        )
-
-
-def pip_wheel(
-    requirements_file: Optional[str],
-    package_name: Optional[str],
-    download_dir: Path,
-    index_url: Optional[str],
-    dependencies: bool = True,
-):
-    command = ["-m", "pip", "wheel", "-w", str(download_dir)]
-    if package_name:
-        command.append(package_name)
-    if requirements_file:
-        command.extend(["-r", str(requirements_file)])
-    if index_url:
-        command.extend(["-i", index_url])
-    if not dependencies:
-        command.append("--no-deps")
-
-    try:
-        log.info(
-            "Running pip wheel with command: %s",
-            " ".join([str(com) for com in command]),
-        )
-        process = subprocess.run(
-            ["python", *command],
-            capture_output=True,
-            text=True,
-            encoding=locale.getpreferredencoding(),
-        )
-    except subprocess.CalledProcessError as e:
-        log.error("Encountered error %s", e.stderr)
-        raise ClickException(f"Encountered error while running pip wheel.")
-
-    log.info("Pip wheel command executed successfully")
-    return process.returncode
-
-
-@dataclasses.dataclass
-class SplitDownloadedDependenciesResult:
-    snowflake_dependencies: List[Requirement] = dataclasses.field(default_factory=list)
-    unavailable_dependencies_wheels: List[RequirementWithWheel] = dataclasses.field(
-        default_factory=list
-    )
-
-
-def split_downloaded_dependencies(
-    requirements_file: SecurePath,
-    downloads_dir: Path,
-    anaconda_packages: AnacondaPackages,
-    skip_version_check: bool,
-) -> SplitDownloadedDependenciesResult:
-    packages_metadata: Dict[str, WheelMetadata] = {
-        meta.name: meta
-        for meta in (
-            WheelMetadata.from_wheel(wheel_path)
-            for wheel_path in downloads_dir.glob("*.whl")
-        )
-        if meta is not None
-    }
-    available_in_snowflake_dependencies: Dict = {}
-    unavailable_dependencies: Dict = {}
-
-    def _get_dependencies(package: Requirement):
-        if (
-            package.name not in available_in_snowflake_dependencies
-            and package.name not in unavailable_dependencies
-        ):
-            if anaconda_packages.is_package_available(
-                package, skip_version_check=skip_version_check
-            ):
-                available_in_snowflake_dependencies[package.name] = package
-            else:
-                meta = packages_metadata.get(
-                    WheelMetadata.to_wheel_name_format(package.name)
-                )
-                wheel_path = meta.wheel_path if meta else None
-                requires = meta.dependencies if meta else []
-                unavailable_dependencies[package.name] = RequirementWithWheel(
-                    requirement=package,
-                    wheel_path=wheel_path,
-                )
-
-                log.debug(
-                    "Checking package %s, with dependencies: %s", package.name, requires
-                )
-
-                for package in requires:
-                    _get_dependencies(Requirement.parse_line(package))
-
-    with requirements_file.open("r", read_file_limit_mb=512) as req_file:
-        for line in req_file:
-            _get_dependencies(Requirement.parse_line(line))
-
-    return SplitDownloadedDependenciesResult(
-        snowflake_dependencies=list(available_in_snowflake_dependencies.values()),
-        unavailable_dependencies_wheels=list(unavailable_dependencies.values()),
-    )
-
-
-def detect_and_log_shared_libraries(dependencies: List[RequirementWithFiles]):
-    shared_libraries = [
-        dependency.requirement.name
-        for dependency in dependencies
-        if any(
-            file.endswith(".so") or file.endswith(".dll") for file in dependency.files
-        )
-    ]
-    if shared_libraries:
-        _log_shared_libraries(shared_libraries)
-        return True
-    else:
-        log.info("Unsupported native libraries not found in packages (Good news!)...")
-        return False
-
-
-def _log_shared_libraries(
-    shared_libraries: List[str],
-) -> None:
-    log.error(
-        "Following dependencies utilise shared libraries, not supported by Conda:"
-    )
-    log.error("\n".join(set(shared_libraries)))
-    log.error(
-        "You may still try to create your package with --allow-shared-libraries, but the might not work."
-    )
-    log.error("You may also request adding the package to Snowflake Conda channel")
-    log.error("at https://support.anaconda.com/")
-
-
-def _log_dependencies_found_in_conda(available_dependencies: List[Requirement]) -> None:
-    if len(available_dependencies) > 0:
-        log.info(
-            "Good news! Packages available in Anaconda and excluded from the zip archive: %s",
-            format(", ".join(dep.name for dep in available_dependencies)),
-        )
-    else:
-        log.info("None of the package dependencies were found on Anaconda")
-
-
-def _pip_failed_log_msg(return_code: int) -> str:
-    return dedent(
-        f"""
-        pip failed with return code {return_code}. Most likely reasons:
-         * incorrect package name or version
-         * package isn't compatible with host architecture (most probably due to .so libraries)
-         * pip is not installed correctly
-        """
-    )

snowflake/cli/plugins/snowpark/plugin_spec.py

@@ -1,30 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from snowflake.cli.api.plugins.command import (
-    SNOWCLI_ROOT_COMMAND_PATH,
-    CommandSpec,
-    CommandType,
-    plugin_hook_impl,
-)
-from snowflake.cli.plugins.snowpark import commands
-
-
-@plugin_hook_impl
-def command_spec():
-    return CommandSpec(
-        parent_command_path=SNOWCLI_ROOT_COMMAND_PATH,
-        command_type=CommandType.COMMAND_GROUP,
-        typer_instance=commands.app.create_instance(),
-    )

snowflake/cli/plugins/snowpark/snowpark_package_paths.py

@@ -1,65 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from dataclasses import dataclass
-
-from snowflake.cli.api.project.schemas.snowpark.snowpark import Snowpark
-from snowflake.cli.api.secure_path import SecurePath
-
-_DEFINED_REQUIREMENTS = "requirements.txt"
-_REQUIREMENTS_SNOWFLAKE = "requirements.snowflake.txt"
-
-
-@dataclass
-class SnowparkPackagePaths:
-    source: SecurePath
-    artifact_file: SecurePath
-    defined_requirements_file: SecurePath = SecurePath(_DEFINED_REQUIREMENTS)
-    snowflake_requirements_file: SecurePath = SecurePath(_REQUIREMENTS_SNOWFLAKE)
-
-    @classmethod
-    def for_snowpark_project(
-        cls, project_root: SecurePath, snowpark_project_definition: Snowpark
-    ) -> "SnowparkPackagePaths":
-        defined_source_path = SecurePath(snowpark_project_definition.src)
-        return cls(
-            source=cls._get_snowpark_project_source_absolute_path(
-                project_root=project_root,
-                defined_source_path=defined_source_path,
-            ),
-            artifact_file=cls._get_snowpark_project_artifact_absolute_path(
-                project_root=project_root,
-                defined_source_path=defined_source_path,
-            ),
-            defined_requirements_file=project_root / _DEFINED_REQUIREMENTS,
-            snowflake_requirements_file=project_root / _REQUIREMENTS_SNOWFLAKE,
-        )
-
-    @classmethod
-    def _get_snowpark_project_source_absolute_path(
-        cls, project_root: SecurePath, defined_source_path: SecurePath
-    ) -> SecurePath:
-        if defined_source_path.path.is_absolute():
-            return defined_source_path
-        return SecurePath((project_root / defined_source_path.path).path.resolve())
-
-    @classmethod
-    def _get_snowpark_project_artifact_absolute_path(
-        cls, project_root: SecurePath, defined_source_path: SecurePath
-    ) -> SecurePath:
-        source_path = cls._get_snowpark_project_source_absolute_path(
-            project_root=project_root, defined_source_path=defined_source_path
-        )
-        artifact_file = project_root / (source_path.path.name + ".zip")
-        return artifact_file

snowflake/cli/plugins/snowpark/snowpark_shared.py

@@ -1,95 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import click
-import typer
-from snowflake.cli.api.commands.flags import (
-    deprecated_flag_callback,
-    deprecated_flag_callback_enum,
-)
-from snowflake.cli.plugins.snowpark.models import YesNoAsk
-
-
-def deprecated_allow_native_libraries_option(old_flag_name: str):
-    return typer.Option(
-        YesNoAsk.NO.value,
-        old_flag_name,
-        help="Allows native libraries, when using packages installed through PIP",
-        hidden=True,
-        callback=deprecated_flag_callback_enum(
-            f"{old_flag_name} flag is deprecated. Use --allow-shared-libraries flag instead."
-        ),
-    )
-
-
-def resolve_allow_shared_libraries_yes_no_ask(allow_shared_libraries: YesNoAsk) -> bool:
-    if allow_shared_libraries == YesNoAsk.ASK:
-        return click.confirm("Continue with package installation?", default=False)
-    else:
-        return allow_shared_libraries == YesNoAsk.YES
-
-
-AllowSharedLibrariesOption: bool = typer.Option(
-    False,
-    "--allow-shared-libraries",
-    help="Allows shared (.so) libraries, when using packages installed through PIP.",
-)
-
-DeprecatedCheckAnacondaForPyPiDependencies: bool = typer.Option(
-    True,
-    "--check-anaconda-for-pypi-deps/--no-check-anaconda-for-pypi-deps",
-    "-a",
-    help="""Checks if any of missing Anaconda packages dependencies can be imported directly from Anaconda. Valid values include: `true`, `false`, Default: `true`.""",
-    hidden=True,
-    callback=deprecated_flag_callback(
-        "--check-anaconda-for-pypi-deps flag is deprecated. Use --ignore-anaconda flag instead."
-    ),
-)
-
-IgnoreAnacondaOption: bool = typer.Option(
-    False,
-    "--ignore-anaconda",
-    help="Does not lookup packages on Snowflake Anaconda channel.",
-)
-
-SkipVersionCheckOption: bool = typer.Option(
-    False,
-    "--skip-version-check",
-    help="Skip comparing versions of dependencies between requirements and Anaconda.",
-)
-
-IndexUrlOption: str | None = typer.Option(
-    None,
-    "--index-url",
-    help="Base URL of the Python Package Index to use for package lookup. This should point to "
-    " a repository compliant with PEP 503 (the simple repository API) or a local directory laid"
-    " out in the same format.",
-    show_default=False,
-)
-
-ReturnsOption: str = typer.Option(
-    ...,
-    "--returns",
-    "-r",
-    help="Data type for the procedure to return.",
-)
-
-OverwriteOption: bool = typer.Option(
-    False,
-    "--overwrite",
-    "-o",
-    help="Replaces an existing procedure with this one.",
-)

snowflake/cli/plugins/snowpark/zipper.py

@@ -1,81 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import annotations
-
-import fnmatch
-import logging
-from pathlib import Path
-from typing import Iterator, Literal
-from zipfile import ZIP_DEFLATED, ZipFile
-
-log = logging.getLogger(__name__)
-
-IGNORED_FILES = [
-    "**/.DS_Store",
-    "**/.git",
-    "**/.git/*",
-    "**/.gitignore",
-    "**/.env",
-    "**/.env/*",
-    "**/.venv",
-    "**/.venv/*",
-    "**/__pycache__",
-    "**/*.zip",
-    "**/*.pyc",
-    "**/env",
-    "**/env/*",
-    "**/ENV",
-    "**/ENV/*",
-    "**/venv",
-    "**/venv/*",
-    "**/requirements.txt",
-    "**/requirements.snowflake.txt",
-    "**/requirements.other.txt",
-    "**/snowflake.yml",
-]
-
-
-def add_file_to_existing_zip(zip_file: str, file: str):
-    """Adds another file to an existing zip file
-
-    Args:
-        zip_file (str): The existing zip file
-        file (str): The new file to add
-    """
-    with ZipFile(zip_file, mode="a") as myzip:
-        myzip.write(file, Path(file).name)
-
-
-def zip_dir(
-    source: Path, dest_zip: Path, mode: Literal["r", "w", "x", "a"] = "w"
-) -> None:
-    files_to_pack: Iterator[Path] = filter(
-        _to_be_zipped, map(lambda f: f.absolute(), source.glob("**/*"))
-    )
-
-    with ZipFile(dest_zip, mode, ZIP_DEFLATED, allowZip64=True) as package_zip:
-        for file in files_to_pack:
-            log.debug("Adding %s to %s", file, dest_zip)
-            package_zip.write(file, arcname=file.relative_to(source.absolute()))
-
-
-def _to_be_zipped(file: Path) -> bool:
-    for pattern in IGNORED_FILES:
-        # This has to be a string because of fnmatch
-        file_as_str = str(file)
-        if file_as_str == pattern or fnmatch.fnmatch(file_as_str, pattern):
-            return False
-
-    return True

snowflake/cli/plugins/spcs/__init__.py

@@ -1,35 +0,0 @@
-# Copyright (c) 2024 Snowflake Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
-from snowflake.cli.plugins.spcs.compute_pool.commands import (
-    app as compute_pools_app,
-)
-from snowflake.cli.plugins.spcs.image_registry.commands import app as registry_app
-from snowflake.cli.plugins.spcs.image_repository.commands import (
-    app as image_repository_app,
-)
-from snowflake.cli.plugins.spcs.jobs.commands import app as jobs_app
-from snowflake.cli.plugins.spcs.services.commands import app as services_app
-
-app = SnowTyperFactory(
-    name="spcs",
-    help="Manages Snowpark Container Services compute pools, services, image registries, and image repositories.",
-)
-
-app.add_typer(compute_pools_app)
-app.add_typer(services_app)
-app.add_typer(jobs_app)
-app.add_typer(registry_app)
-app.add_typer(image_repository_app)