snowflake-ml-python 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

snowflake/ml/_internal/env_utils.py

@@ -13,12 +13,8 @@ from packaging import requirements, specifiers, utils as packaging_utils, versio
 
 import snowflake.connector
 from snowflake.ml._internal import env as snowml_env
-from snowflake.ml._internal.exceptions import (
-    error_codes,
-    exceptions as snowml_exceptions,
-)
 from snowflake.ml._internal.utils import query_result_checker
-from snowflake.snowpark import session
+from snowflake.snowpark import context, exceptions, session
 from snowflake.snowpark._internal import utils as snowpark_utils
 
 
@@ -237,6 +233,72 @@ def get_local_installed_version_of_pip_package(pip_req: requirements.Requirement
     return new_pip_req
 
 
+class IncorrectLocalEnvironmentError(Exception):
+    ...
+
+
+def validate_local_installed_version_of_pip_package(pip_req: requirements.Requirement) -> None:
+    """Validate if the package is locally installed, and the local version meet the specifier of the requirements.
+
+    Args:
+        pip_req: A requirements.Requirement object showing the requirement.
+
+    Raises:
+        IncorrectLocalEnvironmentError: Raised when cannot find the local installation of the requested package.
+        IncorrectLocalEnvironmentError: Raised when the local installed version cannot meet the requirement.
+    """
+    try:
+        local_dist = importlib_metadata.distribution(pip_req.name)
+        local_dist_version = version.parse(local_dist.version)
+    except importlib_metadata.PackageNotFoundError:
+        raise IncorrectLocalEnvironmentError(f"Cannot find the local installation of the requested package {pip_req}.")
+
+    if not pip_req.specifier.contains(local_dist_version):
+        raise IncorrectLocalEnvironmentError(
+            f"The local installed version {local_dist_version} cannot meet the requirement {pip_req}."
+        )
+
+
+CONDA_PKG_NAME_TO_PYPI_MAP = {"pytorch": "torch"}
+
+
+def try_convert_conda_requirement_to_pip(conda_req: requirements.Requirement) -> requirements.Requirement:
+    """Return a new requirements.Requirement object whose name has been attempted to convert to name in pypi from conda.
+
+    Args:
+        conda_req: A requirements.Requirement object showing the requirement in conda.
+
+    Returns:
+        A new requirements.Requirement object showing the requirement in pypi.
+    """
+    pip_req = copy.deepcopy(conda_req)
+    pip_req.name = CONDA_PKG_NAME_TO_PYPI_MAP.get(conda_req.name, conda_req.name)
+    return pip_req
+
+
+def validate_py_runtime_version(provided_py_version_str: str) -> None:
+    """Validate the provided python version string with python version in current runtime.
+        If the major or minor is different, errors out.
+
+    Args:
+        provided_py_version_str: the provided python version string.
+
+    Raises:
+        IncorrectLocalEnvironmentError: Raised when the provided python version has different major or minor.
+    """
+    if provided_py_version_str != snowml_env.PYTHON_VERSION:
+        provided_py_version = version.parse(provided_py_version_str)
+        current_py_version = version.parse(snowml_env.PYTHON_VERSION)
+        if (
+            provided_py_version.major != current_py_version.major
+            or provided_py_version.minor != current_py_version.minor
+        ):
+            raise IncorrectLocalEnvironmentError(
+                f"Requested python version is {provided_py_version_str} "
+                f"while current Python version is {snowml_env.PYTHON_VERSION}. "
+            )
+
+
 def get_package_spec_with_supported_ops_only(req: requirements.Requirement) -> requirements.Requirement:
     """Get the package spec with supported ops only including ==, >=, <=, > and <
 
@@ -332,6 +394,16 @@ def get_matched_package_versions_in_snowflake_conda_channel(
     return matched_versions
 
 
+def get_matched_package_versions_in_information_schema_with_active_session(
+    reqs: List[requirements.Requirement], python_version: str
+) -> Dict[str, List[version.Version]]:
+    try:
+        session = context.get_active_session()
+    except exceptions.SnowparkSessionException:
+        return {}
+    return get_matched_package_versions_in_information_schema(session, reqs, python_version)
+
+
 def get_matched_package_versions_in_information_schema(
     session: session.Session, reqs: List[requirements.Requirement], python_version: str
 ) -> Dict[str, List[version.Version]]:
@@ -558,33 +630,6 @@ def parse_python_version_string(dep: str) -> Optional[str]:
     return None
 
 
-def validate_py_runtime_version(provided_py_version_str: str) -> None:
-    """Validate the provided python version string with python version in current runtime.
-        If the major or minor is different, errors out.
-
-    Args:
-        provided_py_version_str: the provided python version string.
-
-    Raises:
-        SnowflakeMLException: Raised when the provided python version has different major or minor.
-    """
-    if provided_py_version_str != snowml_env.PYTHON_VERSION:
-        provided_py_version = version.parse(provided_py_version_str)
-        current_py_version = version.parse(snowml_env.PYTHON_VERSION)
-        if (
-            provided_py_version.major != current_py_version.major
-            or provided_py_version.minor != current_py_version.minor
-        ):
-            raise snowml_exceptions.SnowflakeMLException(
-                error_code=error_codes.LOCAL_ENVIRONMENT_ERROR,
-                original_exception=RuntimeError(
-                    f"Unable to load model which is saved with Python {provided_py_version_str} "
-                    f"while current Python version is {snowml_env.PYTHON_VERSION}. "
-                    "To load model metadata only, set meta_only to True."
-                ),
-            )
-
-
 def _find_conda_dep_spec(
     conda_chan_deps: DefaultDict[str, List[requirements.Requirement]], pkg_name: str
 ) -> Optional[Tuple[str, requirements.Requirement]]:

snowflake/ml/_internal/exceptions/dataset_error_messages.py

@@ -0,0 +1,5 @@
+DATASET_ALREADY_EXISTS = "Dataset {} already exists."
+DATASET_VERSION_ALREADY_EXISTS = "Dataset {} version {} already exists."
+
+DATASET_NOT_EXIST = "Dataset {} does not exist or is inaccessible."
+DATASET_VERSION_NOT_EXIST = "Dataset {} version '{}' does not exist or is inaccessible."

snowflake/ml/_internal/exceptions/dataset_errors.py

@@ -0,0 +1,24 @@
+# Error code from Snowflake Python Connector.
+ERRNO_OBJECT_ALREADY_EXISTS = "002002"
+ERRNO_OBJECT_NOT_EXIST = "002043"
+ERRNO_FILES_ALREADY_EXISTING = "001030"
+ERRNO_VERSION_ALREADY_EXISTS = "092917"
+ERRNO_DATASET_NOT_EXIST = "399019"
+ERRNO_DATASET_VERSION_NOT_EXIST = "399012"
+ERRNO_DATASET_VERSION_ALREADY_EXISTS = "399020"
+
+
+class DatasetError(Exception):
+    """Base class for other exceptions."""
+
+
+class DatasetNotExistError(DatasetError):
+    """Raised when the requested Dataset does not exist."""
+
+
+class DatasetExistError(DatasetError):
+    """Raised when there is already an existing Dataset with the same name and version in selected schema."""
+
+
+class DatasetCannotDeleteError(DatasetError):
+    """Raised when a Dataset is unable to get deleted."""

snowflake/ml/_internal/exceptions/error_codes.py

@@ -105,3 +105,6 @@ UNFEASIBLE_ENVIRONMENT_ERROR = "2502"
 
 # Missing required client side dependency.
 CLIENT_DEPENDENCY_MISSING_ERROR = "2511"
+
+# Current client side snowpark-ml-python version is outdated and may have forward compatibility issue
+SNOWML_PACKAGE_OUTDATED = "2700"

snowflake/ml/_internal/lineage/data_source.py

@@ -0,0 +1,10 @@
+import dataclasses
+from typing import List, Optional
+
+
+@dataclasses.dataclass(frozen=True)
+class DataSource:
+    fully_qualified_name: str
+    version: str
+    url: str
+    exclude_cols: Optional[List[str]] = None

snowflake/ml/_internal/lineage/dataset_dataframe.py

@@ -0,0 +1,44 @@
+import copy
+from typing import List
+
+from snowflake import snowpark
+from snowflake.ml._internal.lineage import data_source
+
+
+class DatasetDataFrame(snowpark.DataFrame):
+    """
+    Represents a lazily-evaluated dataset. It extends :class:`snowpark.DataFrame` so all
+    :class:`snowpark.DataFrame` operations can be applied to it. It holds additional information
+    related to the :class`Dataset`.
+
+    It will be created by dataset.read.to_snowpark_dataframe() API and by the transformations
+    that produce a new dataframe.
+    """
+
+    @staticmethod
+    def from_dataframe(
+        df: snowpark.DataFrame, data_sources: List[data_source.DataSource], inplace: bool = False
+    ) -> "DatasetDataFrame":
+        """
+        Create a new DatasetDataFrame instance from a snowpark.DataFrame instance with
+        additional source information.
+
+        Args:
+            df (snowpark.DataFrame): The Snowpark DataFrame to be converted.
+            data_sources (List[DataSource]): A list of data sources to associate with the DataFrame.
+            inplace (bool): If True, modifies the DataFrame in place; otherwise, returns a new DatasetDataFrame.
+
+        Returns:
+            DatasetDataFrame: A new or modified DatasetDataFrame depending on the 'inplace' argument.
+        """
+        if not inplace:
+            df = copy.deepcopy(df)
+        df.__class__ = DatasetDataFrame
+        df._data_sources = data_sources  # type:ignore[attr-defined]
+        return df  # type: ignore[return-value]
+
+    def _get_sources(self) -> List[data_source.DataSource]:
+        """
+        Returns the data sources associated with the DataFrame.
+        """
+        return self._data_sources  # type: ignore[no-any-return]

snowflake/ml/_internal/utils/identifier.py

@@ -338,4 +338,6 @@ def resolve_identifier(name: str) -> str:
     elif UNQUOTED_CASE_INSENSITIVE_RE.match(name):
         return name.upper()
     else:
-        raise ValueError(f"Invalid name {name} passed. ID is not quoted and cannot normalized.")
+        raise ValueError(
+            f"{name} is not a valid SQL identifier: https://docs.snowflake.com/en/sql-reference/identifiers-syntax"
+        )

snowflake/ml/_internal/utils/sql_identifier.py

@@ -27,15 +27,11 @@ class SqlIdentifier(str):
                 automatically added if necessary to make sure the original input's cases are preserved.
                 Default to False.
 
-        Raises:
-            ValueError: input name is not a valid identifier.
-
         Returns:
             Returns new instance created.
         """
-        # TODO (wezhou) add stronger validation to recognize a valid snowflake identifier.
-        if not name:
-            raise ValueError(f"name:`{name}` is not a valid identifier.")
+        assert name is not None
+
         if case_sensitive:
             return super().__new__(cls, identifier.get_inferred_name(name))
         else:

snowflake/ml/dataset/__init__.py

@@ -0,0 +1,10 @@
+from .dataset import Dataset
+from .dataset_factory import create_from_dataframe, load_dataset
+from .dataset_reader import DatasetReader
+
+__all__ = [
+    "Dataset",
+    "DatasetReader",
+    "create_from_dataframe",
+    "load_dataset",
+]