snowflake-ml-python 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

snowflake/ml/_internal/file_utils.py

@@ -157,14 +157,14 @@ def zip_python_package(zipfile_path: str, package_name: str, ignore_generated_py
                 arcname = base_arcname / path_info.name
                 if not _able_ascii_encode(str(arcname)):
                     raise ValueError(f"File name {arcname} cannot be encoded using ASCII. Please rename.")
-                zf.writestr(str(arcname), path_info.read_bytes())  # type: ignore[no-untyped-call]
+                zf.writestr(str(arcname), path_info.read_bytes())
             elif path_info.is_dir():
                 arcname = base_arcname / path_info.name
                 zf.writestr(str(arcname) + "/", "")
-                for sub_path_info in path_info.iterdir():  # type: ignore[no-untyped-call]
+                for sub_path_info in path_info.iterdir():
                     _add_to_zip(zf, sub_path_info, arcname)
 
-        for sub_path_info in importlib_resources.files(package_name).iterdir():  # type: ignore[no-untyped-call]
+        for sub_path_info in importlib_resources.files(package_name).iterdir():
             _add_to_zip(zf, sub_path_info, base_arcname)
 
 

snowflake/ml/_internal/human_readable_id/adjectives.txt

@@ -0,0 +1,128 @@
+afraid
+ancient
+angry
+average
+bad
+big
+bitter
+black
+blue
+brave
+breezy
+bright
+brown
+calm
+chatty
+chilly
+clever
+cold
+cowardly
+cuddly
+curly
+curvy
+dangerous
+dry
+dull
+empty
+evil
+fast
+fat
+fluffy
+foolish
+fresh
+friendly
+funny
+gentle
+giant
+good
+great
+green
+grumpy
+happy
+hard
+heavy
+helpless
+honest
+horrible
+hot
+hungry
+itchy
+jolly
+kind
+lazy
+light
+little
+loud
+lovely
+lucky
+massive
+mean
+mighty
+modern
+moody
+nasty
+neat
+nervous
+new
+nice
+odd
+old
+orange
+ordinary
+perfect
+pink
+plastic
+polite
+popular
+pretty
+proud
+purple
+quick
+quiet
+rare
+red
+rotten
+rude
+selfish
+serious
+shaggy
+sharp
+short
+shy
+silent
+silly
+slimy
+slippery
+smart
+smooth
+soft
+sour
+spicy
+splendid
+spotty
+stale
+strange
+strong
+stupid
+sweet
+swift
+tall
+tame
+tasty
+tender
+terrible
+thin
+tidy
+tiny
+tough
+tricky
+ugly
+warm
+weak
+wet
+wicked
+wise
+witty
+wonderful
+yellow
+young

snowflake/ml/_internal/human_readable_id/animals.txt

@@ -0,0 +1,128 @@
+anaconda
+ant
+ape
+baboon
+badger
+bat
+bear
+bird
+bobcat
+bulldog
+bullfrog
+camel
+canary
+capybara
+cat
+catfish
+cheetah
+chicken
+chipmunk
+cobra
+cougar
+cow
+crab
+deer
+dingo
+dodo
+dog
+dolphin
+donkey
+dragon
+dragonfly
+duck
+eagle
+earwig
+eel
+egret
+elephant
+emu
+falcon
+fireant
+firefox
+fish
+fly
+fox
+frog
+gazelle
+gecko
+gibbon
+giraffe
+goat
+goose
+gorilla
+grasshopper
+horse
+hound
+husky
+impala
+insect
+jackal
+jaguar
+jellyfish
+kangaroo
+kiwi
+ladybug
+leech
+leopard
+llama
+liger
+lion
+lionfish
+lizard
+lobster
+mayfly
+mamba
+mole
+monkey
+moose
+moth
+mouse
+mule
+newt
+octopus
+otter
+owl
+panda
+panther
+parrot
+penguin
+pig
+puma
+pug
+python
+quail
+rabbit
+ram
+rat
+ray
+rattlesnake
+robin
+salmon
+seahorse
+seal
+shark
+sheep
+shrimp
+skunk
+sloth
+snail
+snake
+squid
+starfish
+stingray
+swan
+termite
+tiger
+treefrog
+turkey
+turtle
+vampirebat
+walrus
+warthog
+wasp
+wolverine
+wombat
+worm
+yak
+yeti
+zebra

snowflake/ml/_internal/human_readable_id/hrid_generator.py

@@ -0,0 +1,40 @@
+"""Implement a generator for human readable ID (HRID).
+
+The original idea for this comes from Asana where it is documented on their
+blog:
+    http://blog.asana.com/2011/09/6-sad-squid-snuggle-softly/
+
+There are other partial implementations of this and can be found here:
+    Node.js: https://github.com/linus/greg
+    Java: https://github.com/PerWiklander/IdentifierSentence
+
+In this module you will find:
+
+    HRID16: An implementation of HRIDBase for 16 bit integers.
+
+The list used here is coming from:
+    https://git.coolaj86.com/coolaj86/human-readable-ids.js
+"""
+
+import random
+
+import importlib_resources
+
+from snowflake.ml._internal import human_readable_id
+from snowflake.ml._internal.human_readable_id import hrid_generator_base
+
+
+class HRID16(hrid_generator_base.HRIDBase):
+    """An implementation of HRIDBase for 16 bit integers."""
+
+    def __id_generator__(self) -> int:
+        return int(random.getrandbits(16))
+
+    __hrid_structure__ = ("adjective", "animal", "number")
+    __hrid_words__ = dict(
+        number=tuple(str(x) for x in range(1, 5)),
+        adjective=tuple(
+            importlib_resources.files(human_readable_id).joinpath("adjectives.txt").read_text("utf-8").split()
+        ),
+        animal=tuple(importlib_resources.files(human_readable_id).joinpath("animals.txt").read_text("utf-8").split()),
+    )

snowflake/ml/_internal/human_readable_id/hrid_generator_base.py

@@ -0,0 +1,135 @@
+"""Implement a generator for human readable ID (HRID).
+
+The original idea for this comes from Asana where it is documented on their
+blog:
+    http://blog.asana.com/2011/09/6-sad-squid-snuggle-softly/
+
+There are other partial implementations of this and can be found here:
+    Node.js: https://github.com/linus/greg
+    Java: https://github.com/PerWiklander/IdentifierSentence
+
+In this module you will find:
+
+    HRIDBase: The base class for all human readable id.
+"""
+
+import math
+from abc import ABC, abstractmethod
+from typing import Dict, List, Tuple
+
+
+class HRIDBase(ABC):
+    """The base class for all all human readable id.
+
+    This provides all of the necessary helper functionality to turn IDs into
+    HRIDs and HRIDs into IDs. ID typically is a random int, while HRID is a corresponding short string.
+    """
+
+    @abstractmethod
+    def __id_generator__(self) -> int:
+        """The generator to use to generate new IDs. The implementer needs to provide this."""
+        pass
+
+    __hrid_structure__: Tuple[str, ...]
+    """The HRID structure to be generated. The implementer needs to provide this."""
+
+    __hrid_words__: Dict[str, Tuple[str, ...]]
+    """The mapping between the HRID parts and the words to use. The implementer needs to provide this."""
+
+    __separator__ = "_"
+
+    def __init__(self) -> None:
+        self._part_n_words = dict()
+        self._part_bits = dict()
+        for part in self.__hrid_structure__:
+            n_words = len(self.__hrid_words__[part])
+            self._part_n_words[part] = n_words
+            if not (n_words > 0 and ((n_words & (n_words - 1)) == 0)):
+                raise ValueError(f"{part} part has {n_words} words, which is not a power of 2")
+            self._part_bits[part] = int(math.log(self._part_n_words[part], 2))
+        self.__total_bits__ = sum(v for v in self._part_bits.values())
+
+    def hrid_to_id(self, hrid: str) -> int:
+        """Take the HRID and convert it the ID.
+
+        Args:
+            hrid: The HRID to convert into an ID
+
+        Returns:
+            The ID represented by the HRID
+        """
+        idxs = self._hrid_to_idxs(hrid)
+        id = 0
+        for i in range(len(idxs)):
+            part = self.__hrid_structure__[i]
+            id = (id << self._part_bits[part]) + idxs[i]
+        return id
+
+    def id_to_hrid(self, id: int) -> str:
+        """Take the ID and convert it a HRID.
+
+        Args:
+            id: The ID to convert into a HRID
+
+        Returns:
+            The HRID represented by the ID
+        """
+        idxs = self._id_to_idxs(id)
+        hrid = []
+        for i in range(len(self.__hrid_structure__)):
+            part = self.__hrid_structure__[i]
+            values = self.__hrid_words__[part]
+            hrid.append(str(values[idxs[i]]))
+        return self.__separator__.join(hrid)
+
+    def generate(self) -> Tuple[int, str]:
+        """Generate an ID and the corresponding HRID.
+
+        Returns:
+            A tuple containing the id and the HRID
+        """
+        id = self.__id_generator__()
+        hrid = self.id_to_hrid(id)
+        return (id, hrid)
+
+    def _id_to_idxs(self, id: int) -> List[int]:
+        """Take the ID and convert it to indices into the HRID words.
+
+        Args:
+            id: The ID to convert into indices
+
+        Returns:
+            A list of indices into the HRID words
+        """
+        shift = self.__total_bits__
+        idxs = []
+        for part in self.__hrid_structure__:
+            shift -= self._part_bits[part]
+            mask = (self._part_n_words[part] - 1) << shift
+            idxs.append((id & mask) >> shift)
+        return idxs
+
+    def _hrid_to_idxs(self, hrid: str) -> List[int]:
+        """Take the HRID and convert it to indices into the HRID words.
+
+        Args:
+            hrid: The HRID to convert into indices
+
+        Raises:
+            ValueError: Raised when the input does not meet the structure.
+
+        Returns:
+            A list of indices into the HRID words
+        """
+        split_hrid = hrid.split(self.__separator__)
+        if len(split_hrid) != len(self.__hrid_structure__):
+            raise ValueError(
+                ("The hrid must have {} parts and be of the form {}").format(
+                    len(self.__hrid_structure__), self.__hrid_structure__
+                )
+            )
+        idxs = []
+        for i in range(len(self.__hrid_structure__)):
+            part = self.__hrid_structure__[i]
+            idxs.append(self.__hrid_words__[part].index(split_hrid[i]))
+        return idxs

snowflake/ml/_internal/telemetry.py

@@ -32,6 +32,13 @@ from snowflake.snowpark._internal import utils
 _log_counter = 0
 _FLUSH_SIZE = 10
 
+# Prepopulate allowed connection types for type checking later since getattr is slow on large modules
+_CONNECTION_TYPES = {
+    conn_type: getattr(connector, conn_type)
+    for conn_type in ["SnowflakeConnection", "StoredProcConnection"]
+    if hasattr(connector, conn_type)
+}
+
 _Args = ParamSpec("_Args")
 _ReturnValue = TypeVar("_ReturnValue")
 
@@ -321,8 +328,10 @@ def send_api_usage_telemetry(
             if conn_attr_name:
                 # raise AttributeError if conn attribute does not exist in `self`
                 conn = operator.attrgetter(conn_attr_name)(args[0])
-                if not isinstance(conn, connector.SnowflakeConnection):
-                    raise TypeError(f"Expected a conn object of type SnowflakeConnection, but got {type(conn)}")
+                if not isinstance(conn, _CONNECTION_TYPES.get(type(conn).__name__, connector.SnowflakeConnection)):
+                    raise TypeError(
+                        f"Expected a conn object of type {' or '.join(_CONNECTION_TYPES.keys())} but got {type(conn)}"
+                    )
             # get an active session
             else:
                 try:

snowflake/ml/_internal/utils/formatting.py

@@ -1,7 +1,7 @@
 """String formatting utilities for general use in the SnowML Reposiory.
 
 This file contains a collection of utilities that help with formatting strings. Functionality is not limited to tests
-only. Anything that is re-usable across different modules and related to string formatting should go here.
+only. Anything that is reusable across different modules and related to string formatting should go here.
 """
 
 import re

snowflake/ml/feature_store/feature_store.py

@@ -233,13 +233,16 @@ class FeatureStore:
         self._default_warehouse = warehouse
 
     @dispatch_decorator(prpr_version="1.0.8")
-    def register_entity(self, entity: Entity) -> None:
+    def register_entity(self, entity: Entity) -> Entity:
         """
         Register Entity in the FeatureStore.
 
         Args:
             entity: Entity object to register.
 
+        Returns:
+            A registered entity object.
+
         Raises:
             SnowflakeMLException: [ValueError] Entity with same name is already registered.
             SnowflakeMLException: [RuntimeError] Failed to find resources.
@@ -269,15 +272,18 @@ class FeatureStore:
                 error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
                 original_exception=RuntimeError(f"Failed to register entity `{entity.name}`: {e}."),
             ) from e
+
         logger.info(f"Registered Entity {entity}.")
 
+        return self.get_entity(entity.name)
+
     # TODO: add support to update column desc once SNOW-894249 is fixed
     @dispatch_decorator(prpr_version="1.0.8")
     def register_feature_view(
         self,
         feature_view: FeatureView,
         version: str,
-        block: bool = False,
+        block: bool = True,
         override: bool = False,
     ) -> FeatureView:
         """
@@ -297,7 +303,7 @@ class FeatureStore:
             version: version of the registered FeatureView.
                 NOTE: Version only accepts letters, numbers and underscore. Also version will be capitalized.
             block: Specify whether the FeatureView backend materialization should be blocking or not. If blocking then
-                the API will wait until the initial FeatureView data is generated.
+                the API will wait until the initial FeatureView data is generated. Default to true.
             override: Override the existing FeatureView with same version. This is the same as dropping the FeatureView
                 first then recreate. NOTE: there will be backfill cost associated if the FeatureView is being
                 continuously maintained.
@@ -525,104 +531,6 @@ class FeatureStore:
 
         return self._compose_feature_view(results[0], self.list_entities().collect())
 
-    @dispatch_decorator(prpr_version="1.0.8")
-    def merge_features(
-        self,
-        features: List[Union[FeatureView, FeatureViewSlice]],
-        name: str,
-        desc: str = "",
-    ) -> FeatureView:
-        """
-        Merge multiple registered FeatureView or FeatureViewSlice to form a new FeatureView.
-        This is typically used to add new features to existing FeatureViews since registered FeatureView is immutable.
-        The FeatureViews or FeatureViewSlices to merge should have same Entity and timestamp column setup.
-
-        Args:
-            features: List of FeatureViews or FeatureViewSlices to merge
-            name: name of the new constructed FeatureView
-            desc: description of the new constructed FeatureView
-
-        Returns:
-            a new FeatureView with features merged.
-
-        Raises:
-            SnowflakeMLException: [ValueError] Features length is not valid or if Entitis and timestamp_col is
-                inconsistent.
-            SnowflakeMLException: [ValueError] FeatureView has not been registered.
-            SnowflakeMLException: [ValueError] FeatureView merge failed.
-        """
-        name = SqlIdentifier(name)
-
-        if len(features) < 2:
-            raise snowml_exceptions.SnowflakeMLException(
-                error_code=error_codes.INVALID_ARGUMENT,
-                original_exception=ValueError("features should have at least two entries"),
-            )
-
-        left = features[0]
-        left_columns = None
-        if isinstance(left, FeatureViewSlice):
-            left_columns = ", ".join(left.names)
-            left = left.feature_view_ref
-
-        if left.status == FeatureViewStatus.DRAFT:
-            raise snowml_exceptions.SnowflakeMLException(
-                error_code=error_codes.NOT_FOUND,
-                original_exception=ValueError(f"FeatureView {left.name} has not been registered."),
-            )
-
-        join_keys = [k for e in left.entities for k in e.join_keys]
-
-        ts_col_expr = "" if left.timestamp_col is None else f" , {left.timestamp_col}"
-        left_columns = "*" if left_columns is None else f"{', '.join(join_keys)}, {left_columns}{ts_col_expr}"
-        left_df = self._session.sql(f"SELECT {left_columns} FROM {left.fully_qualified_name()}")
-
-        for right in features[1:]:
-            right_columns = None
-            if isinstance(right, FeatureViewSlice):
-                right_columns = ", ".join(right.names)
-                right = right.feature_view_ref
-
-            if left.entities != right.entities:
-                raise snowml_exceptions.SnowflakeMLException(
-                    error_code=error_codes.INVALID_ARGUMENT,
-                    original_exception=ValueError(
-                        f"Cannot merge FeatureView {left.name} and {right.name} with different Entities: "
-                        f"{left.entities} vs {right.entities}"  # noqa: E501
-                    ),
-                )
-            if left.timestamp_col != right.timestamp_col:
-                raise snowml_exceptions.SnowflakeMLException(
-                    error_code=error_codes.INVALID_ARGUMENT,
-                    original_exception=ValueError(
-                        f"Cannot merge FeatureView {left.name} and {right.name} with different timestamp_col: "
-                        f"{left.timestamp_col} vs {right.timestamp_col}"  # noqa: E501
-                    ),
-                )
-            if right.status == FeatureViewStatus.DRAFT:
-                raise snowml_exceptions.SnowflakeMLException(
-                    error_code=error_codes.NOT_FOUND,
-                    original_exception=ValueError(f"FeatureView {right.name} has not been registered."),
-                )
-
-            right_columns = "*" if right_columns is None else f"{', '.join(join_keys)}, {right_columns}"
-            exclude_ts_expr = (
-                "" if right.timestamp_col is None or right_columns != "*" else f"EXCLUDE {right.timestamp_col}"
-            )
-            right_df = self._session.sql(
-                f"SELECT {right_columns} {exclude_ts_expr} FROM {right.fully_qualified_name()}"
-            )
-
-            left_df = left_df.join(right=right_df, on=join_keys)
-
-        return FeatureView(
-            name=name,
-            entities=left.entities,
-            feature_df=left_df,
-            timestamp_col=left.timestamp_col,
-            desc=desc,
-        )
-
     @dispatch_decorator(prpr_version="1.0.8")
     def resume_feature_view(self, feature_view: FeatureView) -> FeatureView:
         """
@@ -1056,10 +964,7 @@ class FeatureStore:
                 WAREHOUSE = {warehouse}
                 AS {feature_view.query}
             """
-            self._session.sql(query).collect(statement_params=self._telemetry_stmp)
-            self._session.sql(f"ALTER DYNAMIC TABLE {fully_qualified_name} REFRESH").collect(
-                block=block, statement_params=self._telemetry_stmp
-            )
+            self._session.sql(query).collect(block=block, statement_params=self._telemetry_stmp)
 
             if schedule_task:
                 try:
@@ -1092,6 +997,10 @@ class FeatureStore:
                 ),
             ) from e
 
+        if block:
+            self._check_dynamic_table_refresh_mode(feature_view_name)
+
+    def _check_dynamic_table_refresh_mode(self, feature_view_name: SqlIdentifier) -> None:
         found_dts = self._find_object("DYNAMIC TABLES", feature_view_name)
         if len(found_dts) != 1:
             raise snowml_exceptions.SnowflakeMLException(
@@ -1161,7 +1070,7 @@ class FeatureStore:
     def _validate_entity_exists(self, name: SqlIdentifier) -> bool:
         full_entity_tag_name = self._get_entity_name(name)
         found_rows = self._find_object("TAGS", full_entity_tag_name)
-        return len(found_rows) > 0
+        return len(found_rows) == 1
 
     def _join_features(
         self,

snowflake/ml/fileset/sfcfs.py

@@ -90,11 +90,12 @@ class SFFileSystem(fsspec.AbstractFileSystem):
                 ) from e
 
         if sf_connection:
-            self._conn = sf_connection
+            self._session = snowpark.Session.builder.config("connection", sf_connection).create()
         elif snowpark_session:
-            self._conn = snowpark_session._conn._conn
+            self._session = snowpark_session
         else:
             raise ValueError("Either sf_connection or snowpark_session has to be non-empty!")
+        self._conn = self._session._conn._conn  # Telemetry wrappers expect connection under `conn_attr_name="_conn"``
         self._kwargs = kwargs
         self._stage_fs_set: Dict[Tuple[str, str, str], stage_fs.SFStageFileSystem] = {}
 
@@ -168,7 +169,7 @@ class SFFileSystem(fsspec.AbstractFileSystem):
         stage_fs_key = (sf_file_path.database, sf_file_path.schema, sf_file_path.stage)
         if stage_fs_key not in self._stage_fs_set:
             cnt_stage_fs = stage_fs.SFStageFileSystem(
-                sf_connection=self._conn,
+                snowpark_session=self._session,
                 db=sf_file_path.database,
                 schema=sf_file_path.schema,
                 stage=sf_file_path.stage,