snowflake-cli-labs 2.8.0rc1__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

snowflake/cli/api/project/definition_conversion.py

@@ -0,0 +1,194 @@
+import logging
+from pathlib import Path
+from typing import Any, Dict, Literal, Optional
+
+from click import ClickException
+from snowflake.cli._plugins.snowpark.common import is_name_a_templated_one
+from snowflake.cli.api.constants import (
+    DEFAULT_ENV_FILE,
+    DEFAULT_PAGES_DIR,
+    PROJECT_TEMPLATE_VARIABLE_OPENING,
+    SNOWPARK_SHARED_MIXIN,
+)
+from snowflake.cli.api.project.schemas.project_definition import (
+    ProjectDefinition,
+    ProjectDefinitionV2,
+)
+from snowflake.cli.api.project.schemas.snowpark.callable import (
+    FunctionSchema,
+    ProcedureSchema,
+)
+from snowflake.cli.api.project.schemas.snowpark.snowpark import Snowpark
+from snowflake.cli.api.project.schemas.streamlit.streamlit import Streamlit
+
+log = logging.getLogger(__name__)
+
+
+def convert_project_definition_to_v2(
+    pd: ProjectDefinition, accept_templates: bool = False
+) -> ProjectDefinitionV2:
+    _check_if_project_definition_meets_requirements(pd, accept_templates)
+
+    snowpark_data = convert_snowpark_to_v2_data(pd.snowpark) if pd.snowpark else {}
+    streamlit_data = convert_streamlit_to_v2_data(pd.streamlit) if pd.streamlit else {}
+    envs = convert_envs_to_v2(pd)
+
+    data = {
+        "definition_version": "2",
+        "entities": get_list_of_all_entities(
+            snowpark_data.get("entities", {}), streamlit_data.get("entities", {})
+        ),
+        "mixins": snowpark_data.get("mixins", None),
+        "env": envs,
+    }
+
+    return ProjectDefinitionV2(**data)
+
+
+def convert_snowpark_to_v2_data(snowpark: Snowpark) -> Dict[str, Any]:
+    artifact_mapping = {"src": snowpark.src}
+    if snowpark.project_name:
+        artifact_mapping["dest"] = snowpark.project_name
+
+    data: dict = {
+        "mixins": {
+            SNOWPARK_SHARED_MIXIN: {
+                "stage": snowpark.stage_name,
+                "artifacts": [artifact_mapping],
+            }
+        },
+        "entities": {},
+    }
+
+    for index, entity in enumerate([*snowpark.procedures, *snowpark.functions]):
+        identifier = {"name": entity.name}
+        if entity.database is not None:
+            identifier["database"] = entity.database
+        if entity.schema_name is not None:
+            identifier["schema"] = entity.schema_name
+
+        entity_name = (
+            f"snowpark_entity_{index}"
+            if is_name_a_templated_one(entity.name)
+            else entity.name
+        )
+
+        if entity_name in data["entities"]:
+            raise ClickException(
+                f"Entity with name {entity_name} seems to be duplicated. Please rename it and try again."
+            )
+
+        v2_entity = {
+            "type": "function" if isinstance(entity, FunctionSchema) else "procedure",
+            "stage": snowpark.stage_name,
+            "handler": entity.handler,
+            "returns": entity.returns,
+            "signature": entity.signature,
+            "runtime": entity.runtime,
+            "external_access_integrations": entity.external_access_integrations,
+            "secrets": entity.secrets,
+            "imports": entity.imports,
+            "identifier": identifier,
+            "meta": {"use_mixins": [SNOWPARK_SHARED_MIXIN]},
+        }
+        if isinstance(entity, ProcedureSchema):
+            v2_entity["execute_as_caller"] = entity.execute_as_caller
+
+        data["entities"][entity_name] = v2_entity
+
+    return data
+
+
+def convert_streamlit_to_v2_data(streamlit: Streamlit):
+    # Process env file and pages dir
+    environment_file = _process_streamlit_files(streamlit.env_file, "environment")
+    pages_dir = _process_streamlit_files(streamlit.pages_dir, "pages")
+
+    # Build V2 definition
+    artifacts = [
+        streamlit.main_file,
+        environment_file,
+        pages_dir,
+    ]
+    artifacts = [a for a in artifacts if a is not None]
+
+    if streamlit.additional_source_files:
+        artifacts.extend(streamlit.additional_source_files)
+
+    identifier = {"name": streamlit.name}
+    if streamlit.schema_name:
+        identifier["schema"] = streamlit.schema_name
+    if streamlit.database:
+        identifier["database"] = streamlit.database
+
+    streamlit_name = (
+        "streamlit_entity_1"
+        if is_name_a_templated_one(streamlit.name)
+        else streamlit.name
+    )
+
+    data = {
+        "entities": {
+            streamlit_name: {
+                "type": "streamlit",
+                "identifier": identifier,
+                "title": streamlit.title,
+                "query_warehouse": streamlit.query_warehouse,
+                "main_file": str(streamlit.main_file),
+                "pages_dir": str(streamlit.pages_dir),
+                "stage": streamlit.stage,
+                "artifacts": artifacts,
+            }
+        }
+    }
+    return data
+
+
+def convert_envs_to_v2(pd: ProjectDefinition):
+    if hasattr(pd, "env") and pd.env:
+        data = {k: v for k, v in pd.env.items()}
+        return data
+    return None
+
+
+def _check_if_project_definition_meets_requirements(
+    pd: ProjectDefinition, accept_templates: bool
+):
+    if pd.meets_version_requirement("2"):
+        raise ClickException("Project definition is already at version 2.")
+
+    if PROJECT_TEMPLATE_VARIABLE_OPENING in str(pd):
+        if not accept_templates:
+            raise ClickException(
+                "Project definition contains templates. They may not be migrated correctly, and require manual migration."
+                "You can try again with --accept-templates  option, to attempt automatic migration."
+            )
+        log.warning(
+            "Your V1 definition contains templates. We cannot guarantee the correctness of the migration."
+        )
+    if pd.native_app:
+        raise ClickException(
+            "Your project file contains a native app definition. Conversion of Native apps is not yet supported"
+        )
+
+
+def _process_streamlit_files(
+    file_name: Optional[str], file_type: Literal["pages", "environment"]
+):
+    default = DEFAULT_PAGES_DIR if file_type == "pages" else DEFAULT_ENV_FILE
+
+    if file_name and not Path(file_name).exists():
+        raise ClickException(f"Provided file {file_name} does not exist")
+    elif file_name is None and Path(default).exists():
+        file_name = default
+    return file_name
+
+
+def get_list_of_all_entities(
+    snowpark_entities: Dict[str, Any], streamlit_entities: Dict[str, Any]
+):
+    if snowpark_entities.keys() & streamlit_entities.keys():
+        raise ClickException(
+            "In your project, streamlit and snowpark entities share the same name. Please rename them and try again."
+        )
+    return snowpark_entities | streamlit_entities

snowflake/cli/api/project/definition_manager.py

@@ -20,7 +20,10 @@ from pathlib import Path
 from typing import List, Optional
 
 from snowflake.cli.api.project.definition import ProjectProperties, load_project
-from snowflake.cli.api.project.schemas.project_definition import ProjectDefinitionV1
+from snowflake.cli.api.project.schemas.project_definition import (
+    ProjectDefinition,
+    ProjectDefinitionV1,
+)
 from snowflake.cli.api.utils.types import Context
 
 
@@ -125,10 +128,18 @@ class DefinitionManager:
     def _project_properties(self) -> ProjectProperties:
         return load_project(self._project_config_paths, self._context_overrides)
 
+    @functools.cached_property
+    def _raw_project_data(self) -> ProjectProperties:
+        return load_project(self._project_config_paths, {}, False)
+
     @functools.cached_property
     def project_definition(self) -> ProjectDefinitionV1:
         return self._project_properties.project_definition
 
+    @functools.cached_property
+    def unrendered_project_definition(self) -> ProjectDefinition:
+        return self._raw_project_data.project_definition
+
     @functools.cached_property
     def template_context(self) -> Context:
         return self._project_properties.project_context

snowflake/cli/api/project/project_verification.py

@@ -11,13 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from snowflake.cli.api.cli_global_context import cli_context
+from snowflake.cli.api.cli_global_context import get_cli_context
 from snowflake.cli.api.exceptions import NoProjectDefinitionError
 
 
 def assert_project_type(project_type: str):
+    cli_context = get_cli_context()
     if not getattr(cli_context.project_definition, project_type, None):
         raise NoProjectDefinitionError(
-            project_type=project_type, project_file=cli_context.project_root
+            project_type=project_type, project_root=cli_context.project_root
         )

snowflake/cli/api/project/schemas/entities/{application_entity.py → application_entity_model.py}

@@ -16,25 +16,24 @@ from __future__ import annotations
 
 from typing import Literal, Optional
 
-from pydantic import Field
-from snowflake.cli.api.project.schemas.entities.application_package_entity import (
-    ApplicationPackageEntity,
+from pydantic import Field, field_validator
+from snowflake.cli.api.project.schemas.entities.application_package_entity_model import (
+    ApplicationPackageEntityModel,
 )
 from snowflake.cli.api.project.schemas.entities.common import (
-    EntityBase,
+    EntityModelBase,
     TargetField,
 )
+from snowflake.cli.api.project.schemas.identifier_model import Identifier
 from snowflake.cli.api.project.schemas.updatable_model import (
     DiscriminatorField,
 )
+from snowflake.cli.api.project.util import append_test_resource_suffix
 
 
-class ApplicationEntity(EntityBase):
+class ApplicationEntityModel(EntityModelBase):
     type: Literal["application"] = DiscriminatorField()  # noqa A003
-    name: str = Field(
-        title="Name of the application created when this entity is deployed"
-    )
-    from_: TargetField[ApplicationPackageEntity] = Field(
+    from_: TargetField[ApplicationPackageEntityModel] = Field(
         alias="from",
         title="An application package this entity should be created from",
     )
@@ -42,3 +41,16 @@ class ApplicationEntity(EntityBase):
         title="Whether to enable debug mode when using a named stage to create an application object",
         default=None,
     )
+
+    @field_validator("identifier")
+    @classmethod
+    def append_test_resource_suffix_to_identifier(
+        cls, input_value: Identifier | str
+    ) -> Identifier | str:
+        identifier = (
+            input_value.name if isinstance(input_value, Identifier) else input_value
+        )
+        with_suffix = append_test_resource_suffix(identifier)
+        if isinstance(input_value, Identifier):
+            return input_value.model_copy(update=dict(name=with_suffix))
+        return with_suffix

snowflake/cli/api/project/schemas/entities/{application_package_entity.py → application_package_entity_model.py}

@@ -14,40 +14,38 @@
 
 from __future__ import annotations
 
-from pathlib import Path
 from typing import List, Literal, Optional, Union
 
-from pydantic import Field
+from pydantic import Field, field_validator
 from snowflake.cli.api.project.schemas.entities.common import (
-    EntityBase,
+    EntityModelBase,
 )
+from snowflake.cli.api.project.schemas.identifier_model import Identifier
 from snowflake.cli.api.project.schemas.native_app.package import DistributionOptions
 from snowflake.cli.api.project.schemas.native_app.path_mapping import PathMapping
 from snowflake.cli.api.project.schemas.updatable_model import (
     DiscriminatorField,
     IdentifierField,
 )
+from snowflake.cli.api.project.util import append_test_resource_suffix
 
 
-class ApplicationPackageEntity(EntityBase):
+class ApplicationPackageEntityModel(EntityModelBase):
     type: Literal["application package"] = DiscriminatorField()  # noqa: A003
-    name: str = Field(
-        title="Name of the application package created when this entity is deployed"
-    )
-    artifacts: List[Union[PathMapping, Path]] = Field(
+    artifacts: List[Union[PathMapping, str]] = Field(
         title="List of paths or file source/destination pairs to add to the deploy root",
     )
-    bundle_root: Optional[Path] = Field(
+    bundle_root: Optional[str] = Field(
         title="Folder at the root of your project where artifacts necessary to perform the bundle step are stored.",
-        default=Path("output/bundle/"),
+        default="output/bundle/",
     )
-    deploy_root: Optional[Path] = Field(
+    deploy_root: Optional[str] = Field(
         title="Folder at the root of your project where the build step copies the artifacts",
-        default=Path("output/deploy/"),
+        default="output/deploy/",
     )
-    generated_root: Optional[Path] = Field(
+    generated_root: Optional[str] = Field(
         title="Subdirectory of the deploy root where files generated by the Snowflake CLI will be written.",
-        default=Path("__generated/"),
+        default="__generated/",
     )
     stage: Optional[str] = IdentifierField(
         title="Identifier of the stage that stores the application artifacts.",
@@ -61,6 +59,36 @@ class ApplicationPackageEntity(EntityBase):
         title="Distribution of the application package created by the Snowflake CLI",
         default="internal",
     )
-    manifest: Path = Field(
+    manifest: str = Field(
         title="Path to manifest.yml",
     )
+
+    @field_validator("identifier")
+    @classmethod
+    def append_test_resource_suffix_to_identifier(
+        cls, input_value: Identifier | str
+    ) -> Identifier | str:
+        identifier = (
+            input_value.name if isinstance(input_value, Identifier) else input_value
+        )
+        with_suffix = append_test_resource_suffix(identifier)
+        if isinstance(input_value, Identifier):
+            return input_value.model_copy(update=dict(name=with_suffix))
+        return with_suffix
+
+    @field_validator("artifacts")
+    @classmethod
+    def transform_artifacts(
+        cls, orig_artifacts: List[Union[PathMapping, str]]
+    ) -> List[PathMapping]:
+        transformed_artifacts = []
+        if orig_artifacts is None:
+            return transformed_artifacts
+
+        for artifact in orig_artifacts:
+            if isinstance(artifact, PathMapping):
+                transformed_artifacts.append(artifact)
+            else:
+                transformed_artifacts.append(PathMapping(src=artifact))
+
+        return transformed_artifacts

snowflake/cli/api/project/schemas/entities/common.py

@@ -15,18 +15,25 @@
 from __future__ import annotations
 
 from abc import ABC
-from typing import Generic, List, Optional, TypeVar
+from typing import Generic, List, Optional, TypeVar, Union
 
-from pydantic import Field
-from snowflake.cli.api.project.schemas.native_app.application import (
-    PostDeployHook,
-)
+from pydantic import Field, PrivateAttr, field_validator
+from snowflake.cli.api.identifiers import FQN
+from snowflake.cli.api.project.schemas.identifier_model import Identifier
 from snowflake.cli.api.project.schemas.updatable_model import (
     IdentifierField,
     UpdatableModel,
 )
 
 
+class SqlScriptHookType(UpdatableModel):
+    sql_script: str = Field(title="SQL file path relative to the project root")
+
+
+# Currently sql_script is the only supported hook type. Change to a Union once other hook types are added
+PostDeployHook = SqlScriptHookType
+
+
 class MetaField(UpdatableModel):
     warehouse: Optional[str] = IdentifierField(
         title="Warehouse used to run the scripts", default=None
@@ -39,6 +46,19 @@ class MetaField(UpdatableModel):
         title="Actions that will be executed after the application object is created/upgraded",
         default=None,
     )
+    use_mixins: Optional[List[str]] = Field(
+        title="Name of the mixin used to fill the entity fields",
+        default=None,
+    )
+
+    @field_validator("use_mixins", mode="before")
+    @classmethod
+    def ensure_use_mixins_is_a_list(
+        cls, mixins: Optional[str | List[str]]
+    ) -> Optional[List[str]]:
+        if isinstance(mixins, str):
+            return [mixins]
+        return mixins
 
 
 class DefaultsField(UpdatableModel):
@@ -53,12 +73,38 @@ class DefaultsField(UpdatableModel):
     )
 
 
-class EntityBase(ABC, UpdatableModel):
+class EntityModelBase(ABC, UpdatableModel):
     @classmethod
     def get_type(cls) -> str:
         return cls.model_fields["type"].annotation.__args__[0]
 
     meta: Optional[MetaField] = Field(title="Meta fields", default=None)
+    identifier: Optional[Union[Identifier | str]] = Field(
+        title="Entity identifier", default=None
+    )
+    # Set by parent model in post validation. To reference it use `entity_id`.
+    _entity_id: str = PrivateAttr(default=None)
+
+    @property
+    def entity_id(self):
+        return self._entity_id
+
+    def set_entity_id(self, value: str):
+        self._entity_id = value
+
+    def validate_identifier(self):
+        """Helper that's used by ProjectDefinition validator."""
+        if not self._entity_id and not self.identifier:
+            raise ValueError("Missing entity identifier")
+
+    @property
+    def fqn(self) -> FQN:
+        if isinstance(self.identifier, str):
+            return FQN.from_string(self.identifier)
+        if isinstance(self.identifier, Identifier):
+            return FQN.from_identifier_model_v2(self.identifier)
+        if self.entity_id:
+            return FQN.from_string(self.entity_id)
 
 
 TargetType = TypeVar("TargetType")
@@ -76,3 +122,31 @@ class TargetField(UpdatableModel, Generic[TargetType]):
         them in __pydantic_generic_metadata__
         """
         return self.__pydantic_generic_metadata__["args"][0]
+
+
+from typing import Dict, List, Optional
+
+from pydantic import Field
+
+
+class ExternalAccessBaseModel:
+    external_access_integrations: Optional[List[str]] = Field(
+        title="Names of external access integrations needed for this entity to access external networks",
+        default=[],
+    )
+    secrets: Optional[Dict[str, str]] = Field(
+        title="Assigns the names of secrets to variables so that you can use the variables to reference the secrets",
+        default={},
+    )
+
+    def get_external_access_integrations_sql(self) -> str | None:
+        if not self.external_access_integrations:
+            return None
+        external_access_integration_name = ", ".join(self.external_access_integrations)
+        return f"external_access_integrations=({external_access_integration_name})"
+
+    def get_secrets_sql(self) -> str | None:
+        if not self.secrets:
+            return None
+        secrets = ", ".join(f"'{key}'={value}" for key, value in self.secrets.items())
+        return f"secrets=({secrets})"

snowflake/cli/api/project/schemas/entities/entities.py

@@ -14,17 +14,47 @@
 
 from __future__ import annotations
 
-from typing import Union, get_args
+from typing import Dict, List, Union, get_args
 
-from snowflake.cli.api.project.schemas.entities.application_entity import (
-    ApplicationEntity,
-)
-from snowflake.cli.api.project.schemas.entities.application_package_entity import (
+from snowflake.cli.api.entities.application_entity import ApplicationEntity
+from snowflake.cli.api.entities.application_package_entity import (
     ApplicationPackageEntity,
 )
+from snowflake.cli.api.entities.snowpark_entity import FunctionEntity, ProcedureEntity
+from snowflake.cli.api.entities.streamlit_entity import StreamlitEntity
+from snowflake.cli.api.project.schemas.entities.application_entity_model import (
+    ApplicationEntityModel,
+)
+from snowflake.cli.api.project.schemas.entities.application_package_entity_model import (
+    ApplicationPackageEntityModel,
+)
+from snowflake.cli.api.project.schemas.entities.snowpark_entity import (
+    FunctionEntityModel,
+    ProcedureEntityModel,
+)
+from snowflake.cli.api.project.schemas.entities.streamlit_entity_model import (
+    StreamlitEntityModel,
+)
 
-Entity = Union[ApplicationEntity, ApplicationPackageEntity]
+Entity = Union[
+    ApplicationEntity,
+    ApplicationPackageEntity,
+    StreamlitEntity,
+    ProcedureEntity,
+    FunctionEntity,
+]
+EntityModel = Union[
+    ApplicationEntityModel,
+    ApplicationPackageEntityModel,
+    StreamlitEntityModel,
+    FunctionEntityModel,
+    ProcedureEntityModel,
+]
 
-ALL_ENTITIES = [*get_args(Entity)]
+ALL_ENTITIES: List[Entity] = [*get_args(Entity)]
+ALL_ENTITY_MODELS: List[EntityModel] = [*get_args(EntityModel)]
 
-v2_entity_types_map = {e.get_type(): e for e in ALL_ENTITIES}
+v2_entity_model_types_map = {e.get_type(): e for e in ALL_ENTITY_MODELS}
+v2_entity_model_to_entity_map: Dict[EntityModel, Entity] = {
+    e.get_entity_model_type(): e for e in ALL_ENTITIES
+}