snowflake-cli 3.3.0__py3-none-any.whl → 3.5.0__py3-none-any.whl

snowflake/cli/_plugins/snowpark/commands.py

@@ -39,7 +39,9 @@ from snowflake.cli._plugins.snowpark.common import (
     SnowparkEntities,
     SnowparkObject,
     SnowparkObjectManager,
-    StageToArtefactMapping,
+    StageToArtifactMapping,
+    map_path_mapping_to_artifact,
+    zip_and_copy_artifacts_to_deploy,
 )
 from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
     AnacondaPackages,
@@ -68,6 +70,7 @@ from snowflake.cli.api.commands.decorators import (
     with_project_definition,
 )
 from snowflake.cli.api.commands.flags import (
+    ForceReplaceOption,
     ReplaceOption,
     execution_identifier_argument,
     identifier_argument,
@@ -81,6 +84,7 @@ from snowflake.cli.api.constants import (
 from snowflake.cli.api.exceptions import (
     SecretsWithoutExternalAccessIntegrationError,
 )
+from snowflake.cli.api.feature_flags import FeatureFlag
 from snowflake.cli.api.identifiers import FQN
 from snowflake.cli.api.output.types import (
     CollectionResult,
@@ -125,8 +129,10 @@ LikeOption = like_option(
 @with_project_definition()
 def deploy(
     replace: bool = ReplaceOption(
-        help="Replaces procedure or function, even if no detected changes to metadata"
+        help="Replaces procedure or function if there were changes in the definition. It only uploads new and "
+        "overwrites existing files, but does not remove any files already on the stage."
     ),
+    force_replace: bool = ForceReplaceOption(),
     **options,
 ) -> CommandResult:
     """
@@ -156,7 +162,11 @@ def deploy(
     with cli_console.phase("Checking remote state"):
         om = ObjectManager()
         _check_if_all_defined_integrations_exists(om, snowpark_entities)
-        existing_objects = check_for_existing_objects(om, replace, snowpark_entities)
+        existing_objects = (
+            {}
+            if force_replace
+            else check_for_existing_objects(om, replace, snowpark_entities)
+        )
 
     with cli_console.phase("Preparing required stages and artifacts"):
         entities_to_imports_map, stages_to_artifact_map = build_artifacts_mappings(
@@ -189,11 +199,11 @@ def validate_all_artifacts_exists(
     project_paths: SnowparkProjectPaths, snowpark_entities: SnowparkEntities
 ):
     for key, entity in snowpark_entities.items():
-        for artefact in entity.artifacts:
-            path = project_paths.get_artefact_dto(artefact).post_build_path
+        for artifact in entity.artifacts:
+            path = project_paths.get_artifact_dto(artifact).post_build_path
             if not path.exists():
                 raise UsageError(
-                    f"Artefact {path} required for {entity.type} {key} does not exist."
+                    f"Artifact {path} required for {entity.type} {key} does not exist."
                 )
 
 
@@ -213,38 +223,39 @@ def check_for_existing_objects(
 
 def build_artifacts_mappings(
     project_paths: SnowparkProjectPaths, snowpark_entities: SnowparkEntities
-) -> Tuple[EntityToImportPathsMapping, StageToArtefactMapping]:
-    stages_to_artifact_map: StageToArtefactMapping = defaultdict(set)
+) -> Tuple[EntityToImportPathsMapping, StageToArtifactMapping]:
+    stages_to_artifact_map: StageToArtifactMapping = defaultdict(set)
     entities_to_imports_map: EntityToImportPathsMapping = defaultdict(set)
-    for entity_id, entity in snowpark_entities.items():
+    for name, entity in snowpark_entities.items():
         stage = entity.stage
         required_artifacts = set()
-        for artefact in entity.artifacts:
-            artefact_dto = project_paths.get_artefact_dto(artefact)
-            required_artifacts.add(artefact_dto)
-            entities_to_imports_map[entity_id].add(artefact_dto.import_path(stage))
+        for artifact in entity.artifacts:
+            artifact_dto = project_paths.get_artifact_dto(artifact)
+            required_artifacts.add(artifact_dto)
+            entities_to_imports_map[name].add(artifact_dto.import_path(stage))
         stages_to_artifact_map[stage].update(required_artifacts)
 
-        if project_paths.dependencies.exists():
-            deps_artefact = project_paths.get_dependencies_artefact()
-            stages_to_artifact_map[stage].add(deps_artefact)
-            entities_to_imports_map[entity_id].add(deps_artefact.import_path(stage))
+        deps_artifact = project_paths.get_dependencies_artifact()
+        if deps_artifact.post_build_path.exists():
+            stages_to_artifact_map[stage].add(deps_artifact)
+            entities_to_imports_map[name].add(deps_artifact.import_path(stage))
     return entities_to_imports_map, stages_to_artifact_map
 
 
-def create_stages_and_upload_artifacts(stages_to_artifact_map: StageToArtefactMapping):
+def create_stages_and_upload_artifacts(stages_to_artifact_map: StageToArtifactMapping):
     stage_manager = StageManager()
     for stage, artifacts in stages_to_artifact_map.items():
         cli_console.step(f"Creating (if not exists) stage: {stage}")
         stage = FQN.from_stage(stage).using_context()
         stage_manager.create(fqn=stage, comment="deployments managed by Snowflake CLI")
-        for artefact in artifacts:
+        for artifact in artifacts:
+            post_build_path = artifact.post_build_path
             cli_console.step(
-                f"Uploading {artefact.post_build_path.name} to {artefact.upload_path(stage)}"
+                f"Uploading {post_build_path.name} to {artifact.upload_path(stage)}"
             )
             stage_manager.put(
-                local_path=artefact.post_build_path,
-                stage_path=artefact.upload_path(stage),
+                local_path=post_build_path,
+                stage_path=artifact.upload_path(stage),
                 overwrite=True,
             )
 
@@ -324,6 +335,9 @@ def build(
 
     anaconda_packages_manager = AnacondaPackagesManager()
 
+    # Clean up bundle root
+    project_paths.remove_up_bundle_root()
+
     # Resolve dependencies
     if project_paths.requirements.exists():
         with (
@@ -362,22 +376,27 @@ def build(
                 )
 
             if any(temp_deps_dir.path.iterdir()):
-                cli_console.step(f"Creating {project_paths.dependencies.name}")
+                dep_artifact = project_paths.get_dependencies_artifact()
+                cli_console.step(f"Creating {dep_artifact.path.name}")
                 zip_dir(
                     source=temp_deps_dir.path,
-                    dest_zip=project_paths.dependencies,
+                    dest_zip=dep_artifact.post_build_path,
                 )
             else:
                 cli_console.step(f"No external dependencies.")
 
     artifacts = set()
-    for entity in get_snowpark_entities(pd).values():
-        artifacts.update(entity.artifacts)
-
     with cli_console.phase("Preparing artifacts for source code"):
-        for artefact in artifacts:
-            artefact_dto = project_paths.get_artefact_dto(artefact)
-            artefact_dto.build()
+        for entity in get_snowpark_entities(pd).values():
+            artifacts.update(
+                map_path_mapping_to_artifact(project_paths, entity.artifacts)
+            )
+
+        if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
+            zip_and_copy_artifacts_to_deploy(artifacts, project_paths.bundle_root)
+        else:
+            for artifact in artifacts:
+                artifact.build()
 
     return MessageResult(f"Build done.")
 

snowflake/cli/_plugins/snowpark/common.py

@@ -17,6 +17,7 @@ from __future__ import annotations
 import logging
 import re
 from enum import Enum
+from pathlib import Path
 from typing import Dict, List, Set
 
 from click import UsageError
@@ -25,7 +26,13 @@ from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
     ProcedureEntityModel,
     SnowparkEntityModel,
 )
-from snowflake.cli._plugins.snowpark.snowpark_project_paths import Artefact
+from snowflake.cli._plugins.snowpark.snowpark_project_paths import (
+    Artifact,
+    SnowparkProjectPaths,
+)
+from snowflake.cli._plugins.snowpark.zipper import zip_dir_using_bundle_map
+from snowflake.cli.api.artifacts.bundle_map import BundleMap
+from snowflake.cli.api.artifacts.utils import symlink_or_copy
 from snowflake.cli.api.console import cli_console
 from snowflake.cli.api.constants import (
     INIT_TEMPLATE_VARIABLE_CLOSING,
@@ -34,13 +41,14 @@ from snowflake.cli.api.constants import (
     PROJECT_TEMPLATE_VARIABLE_OPENING,
     ObjectType,
 )
+from snowflake.cli.api.project.schemas.entities.common import PathMapping
 from snowflake.cli.api.sql_execution import SqlExecutionMixin
 from snowflake.connector.cursor import SnowflakeCursor
 
 log = logging.getLogger(__name__)
 
 SnowparkEntities = Dict[str, SnowparkEntityModel]
-StageToArtefactMapping = Dict[str, set[Artefact]]
+StageToArtifactMapping = Dict[str, set[Artifact]]
 EntityToImportPathsMapping = Dict[str, set[str]]
 
 DEFAULT_RUNTIME = "3.10"
@@ -214,6 +222,43 @@ def _snowflake_dependencies_differ(
     return _standardize(old_dependencies) != _standardize(new_dependencies)
 
 
+def map_path_mapping_to_artifact(
+    project_paths: SnowparkProjectPaths, artifacts: List[PathMapping]
+) -> List[Artifact]:
+    return [project_paths.get_artifact_dto(artifact) for artifact in artifacts]
+
+
+def zip_and_copy_artifacts_to_deploy(
+    artifacts: Set[Artifact] | List[Artifact], bundle_root: Path
+) -> List[Path]:
+    copied_files = []
+    for artifact in artifacts:
+        bundle_map = BundleMap(
+            project_root=artifact.project_root,
+            deploy_root=bundle_root,
+        )
+        bundle_map.add(PathMapping(src=str(artifact.path), dest=artifact.dest))
+
+        if artifact.path.is_file():
+            for (absolute_src, absolute_dest) in bundle_map.all_mappings(
+                absolute=True, expand_directories=False
+            ):
+                symlink_or_copy(
+                    absolute_src,
+                    absolute_dest,
+                    deploy_root=bundle_map.deploy_root(),
+                )
+                copied_files.append(absolute_dest)
+        else:
+            post_build_path = artifact.post_build_path
+            zip_dir_using_bundle_map(
+                bundle_map=bundle_map,
+                dest_zip=post_build_path,
+            )
+            copied_files.append(post_build_path)
+    return copied_files
+
+
 def same_type(sf_type: str, local_type: str) -> bool:
     sf_type, local_type = sf_type.upper(), local_type.upper()
 

snowflake/cli/_plugins/snowpark/snowpark_entity.py

@@ -5,7 +5,11 @@ from typing import Generic, List, Optional, TypeVar
 from click import ClickException
 from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag
 from snowflake.cli._plugins.snowpark import package_utils
-from snowflake.cli._plugins.snowpark.common import DEFAULT_RUNTIME
+from snowflake.cli._plugins.snowpark.common import (
+    DEFAULT_RUNTIME,
+    map_path_mapping_to_artifact,
+    zip_and_copy_artifacts_to_deploy,
+)
 from snowflake.cli._plugins.snowpark.package.anaconda_packages import (
     AnacondaPackages,
     AnacondaPackagesManager,
@@ -17,6 +21,7 @@ from snowflake.cli._plugins.snowpark.snowpark_entity_model import (
     FunctionEntityModel,
     ProcedureEntityModel,
 )
+from snowflake.cli._plugins.snowpark.snowpark_project_paths import SnowparkProjectPaths
 from snowflake.cli._plugins.snowpark.zipper import zip_dir
 from snowflake.cli._plugins.workspace.context import ActionContext
 from snowflake.cli.api.entities.common import EntityBase
@@ -44,27 +49,31 @@ class SnowparkEntity(EntityBase[Generic[T]]):
     def action_bundle(
         self,
         action_ctx: ActionContext,
-        output_dir: Path | None,
         ignore_anaconda: bool,
         skip_version_check: bool,
+        output_dir: Path | None = None,
         index_url: str | None = None,
         allow_shared_libraries: bool = False,
         *args,
         **kwargs,
     ) -> List[Path]:
         return self.bundle(
-            output_dir,
             ignore_anaconda,
             skip_version_check,
+            output_dir,
             index_url,
             allow_shared_libraries,
         )
 
     def action_deploy(
-        self, action_ctx: ActionContext, mode: CreateMode, *args, **kwargs
+        self,
+        action_ctx: ActionContext,
+        mode: CreateMode = CreateMode.create,
+        *args,
+        **kwargs,
     ):
         # TODO: After introducing bundle map, we should introduce file copying part here
-        return self._execute_query(self.get_deploy_sql(mode))
+        return self.deploy(mode, *args, **kwargs)
 
     def action_drop(self, action_ctx: ActionContext, *args, **kwargs):
         return self._execute_query(self.get_drop_sql())
@@ -83,9 +92,9 @@ class SnowparkEntity(EntityBase[Generic[T]]):
 
     def bundle(
         self,
-        output_dir: Path | None,
         ignore_anaconda: bool,
         skip_version_check: bool,
+        output_dir: Path | None = None,
         index_url: str | None = None,
         allow_shared_libraries: bool = False,
     ) -> List[Path]:
@@ -93,18 +102,20 @@ class SnowparkEntity(EntityBase[Generic[T]]):
         Bundles the entity artifacts and dependencies into a directory.
         Parameters:
             output_dir: The directory to output the bundled artifacts to. Defaults to output dir in project root
-            ignore_anaconda: If True, ignores anaconda chceck and tries to download all packages using pip
+            ignore_anaconda: If True, ignores anaconda check and tries to download all packages using pip
             skip_version_check: If True, skips version check when downloading packages
             index_url: The index URL to use when downloading packages, if none set - default pip index is used (in most cases- Pypi)
             allow_shared_libraries: If not set to True, using dependency with .so/.dll files will raise an exception
         Returns:
         """
         # 0 Create a directory for the entity
+        project_paths = SnowparkProjectPaths(
+            project_root=self.root.absolute(),
+        )
         if not output_dir:
-            output_dir = self.root / "output" / "bundle" / "snowpark"
-        output_dir.mkdir(parents=True, exist_ok=True)  # type: ignore
-
-        output_files = []
+            output_dir = project_paths.bundle_root
+        if not output_dir.exists():  # type: ignore[union-attr]
+            SecurePath(output_dir).mkdir(parents=True)
 
         # 1 Check if requirements exits
         if (self.root / "requirements.txt").exists():
@@ -118,21 +129,20 @@ class SnowparkEntity(EntityBase[Generic[T]]):
                 allow_shared_libraries=allow_shared_libraries,
             )
 
-        # 3 get the artifacts list
-        artifacts = self.model.artifacts
-
-        for artifact in artifacts:
-            output_file = output_dir / artifact.dest / artifact.src.name
-
-            if artifact.src.is_file():
-                output_file.mkdir(parents=True, exist_ok=True)
-                SecurePath(artifact.src).copy(output_file)
-            elif artifact.is_dir():
-                output_file.mkdir(parents=True, exist_ok=True)
+        # 2 get the artifacts list
+        artifacts = map_path_mapping_to_artifact(project_paths, self.model.artifacts)
+        from snowflake.cli.api.feature_flags import FeatureFlag
 
-            output_files.append(output_file)
-
-        return output_files
+        if FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_enabled():
+            return zip_and_copy_artifacts_to_deploy(
+                artifacts, project_paths.bundle_root
+            )
+        else:
+            copied_files = []
+            for artifact in artifacts:
+                artifact.build()
+                copied_files.append(artifact.post_build_path)
+            return copied_files
 
     def check_if_exists(
         self, action_ctx: ActionContext
@@ -143,6 +153,9 @@ class SnowparkEntity(EntityBase[Generic[T]]):
         except ProgrammingError:
             return False
 
+    def deploy(self, mode: CreateMode = CreateMode.create, *args, **kwargs):
+        return self._execute_query(self.get_deploy_sql(mode))
+
     def get_deploy_sql(self, mode: CreateMode):
         query = [
             f"{mode.value} {self.model.type.upper()} {self.identifier}",

snowflake/cli/_plugins/snowpark/snowpark_entity_model.py

@@ -14,37 +14,27 @@
 
 from __future__ import annotations
 
-from pathlib import Path
+import glob
 from typing import List, Literal, Optional, Union
 
 from pydantic import Field, field_validator
+from snowflake.cli.api.feature_flags import FeatureFlag
 from snowflake.cli.api.identifiers import FQN
 from snowflake.cli.api.project.schemas.entities.common import (
-    EntityModelBase,
+    EntityModelBaseWithArtifacts,
     ExternalAccessBaseModel,
     ImportsBaseModel,
+    PathMapping,
 )
 from snowflake.cli.api.project.schemas.updatable_model import (
     DiscriminatorField,
-    UpdatableModel,
 )
 from snowflake.cli.api.project.schemas.v1.snowpark.argument import Argument
 
 
-class PathMapping(UpdatableModel):
-    class Config:
-        frozen = True
-
-    src: Path = Field(title="Source path (relative to project root)", default=None)
-
-    dest: Optional[str] = Field(
-        title="Destination path on stage",
-        description="Paths are relative to stage root; paths ending with a slash indicate that the destination is a directory which source files should be copied into.",
-        default=None,
-    )
-
-
-class SnowparkEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseModel):
+class SnowparkEntityModel(
+    EntityModelBaseWithArtifacts, ExternalAccessBaseModel, ImportsBaseModel
+):
     handler: str = Field(
         title="Function’s or procedure’s implementation of the object inside source module",
         examples=["functions.hello_function"],
@@ -59,17 +49,23 @@ class SnowparkEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseM
         title="Python version to use when executing ", default=None
     )
     stage: str = Field(title="Stage in which artifacts will be stored")
-    artifacts: List[Union[PathMapping, str]] = Field(title="List of required sources")
 
     @field_validator("artifacts")
     @classmethod
     def _convert_artifacts(cls, artifacts: Union[dict, str]):
         _artifacts = []
-        for artefact in artifacts:
-            if isinstance(artefact, PathMapping):
-                _artifacts.append(artefact)
+        for artifact in artifacts:
+            if (
+                (isinstance(artifact, str) and glob.has_magic(artifact))
+                or (isinstance(artifact, PathMapping) and glob.has_magic(artifact.src))
+            ) and FeatureFlag.ENABLE_SNOWPARK_GLOB_SUPPORT.is_disabled():
+                raise ValueError(
+                    "If you want to use glob patterns in artifacts, you need to enable the Snowpark new build feature flag (enable_snowpark_glob_support=true)"
+                )
+            if isinstance(artifact, PathMapping):
+                _artifacts.append(artifact)
             else:
-                _artifacts.append(PathMapping(src=artefact))
+                _artifacts.append(PathMapping(src=artifact))
         return _artifacts
 
     @field_validator("runtime")
@@ -79,14 +75,6 @@ class SnowparkEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBaseM
             return str(runtime_input)
         return runtime_input
 
-    @field_validator("artifacts")
-    @classmethod
-    def validate_artifacts(cls, artifacts: List[Path]) -> List[Path]:
-        for artefact in artifacts:
-            if "*" in str(artefact):
-                raise ValueError("Glob patterns not supported for Snowpark artifacts.")
-        return artifacts
-
     @property
     def udf_sproc_identifier(self) -> UdfSprocIdentifier:
         return UdfSprocIdentifier.from_definition(self)