snowflake-ml-python 1.5.1__py3-none-any.whl → 1.5.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/_sentiment.py +7 -4
- snowflake/ml/_internal/utils/temp_file_utils.py +5 -2
- snowflake/ml/feature_store/access_manager.py +34 -30
- snowflake/ml/feature_store/feature_store.py +1 -1
- snowflake/ml/feature_store/feature_view.py +12 -11
- snowflake/ml/fileset/snowfs.py +2 -31
- snowflake/ml/model/_client/ops/model_ops.py +43 -0
- snowflake/ml/model/_client/sql/model_version.py +53 -1
- snowflake/ml/model/_model_composer/model_composer.py +6 -2
- snowflake/ml/model/_packager/model_meta/model_meta.py +1 -3
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +3 -27
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +58 -139
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_search_udf_file.py +159 -0
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +8 -1
- snowflake/ml/modeling/cluster/affinity_propagation.py +8 -1
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +8 -1
- snowflake/ml/modeling/cluster/birch.py +8 -1
- snowflake/ml/modeling/cluster/bisecting_k_means.py +8 -1
- snowflake/ml/modeling/cluster/dbscan.py +8 -1
- snowflake/ml/modeling/cluster/feature_agglomeration.py +8 -1
- snowflake/ml/modeling/cluster/k_means.py +8 -1
- snowflake/ml/modeling/cluster/mean_shift.py +8 -1
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +8 -1
- snowflake/ml/modeling/cluster/optics.py +8 -1
- snowflake/ml/modeling/cluster/spectral_biclustering.py +8 -1
- snowflake/ml/modeling/cluster/spectral_clustering.py +8 -1
- snowflake/ml/modeling/cluster/spectral_coclustering.py +8 -1
- snowflake/ml/modeling/compose/column_transformer.py +8 -1
- snowflake/ml/modeling/compose/transformed_target_regressor.py +8 -1
- snowflake/ml/modeling/covariance/elliptic_envelope.py +8 -1
- snowflake/ml/modeling/covariance/empirical_covariance.py +8 -1
- snowflake/ml/modeling/covariance/graphical_lasso.py +8 -1
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +8 -1
- snowflake/ml/modeling/covariance/ledoit_wolf.py +8 -1
- snowflake/ml/modeling/covariance/min_cov_det.py +8 -1
- snowflake/ml/modeling/covariance/oas.py +8 -1
- snowflake/ml/modeling/covariance/shrunk_covariance.py +8 -1
- snowflake/ml/modeling/decomposition/dictionary_learning.py +8 -1
- snowflake/ml/modeling/decomposition/factor_analysis.py +8 -1
- snowflake/ml/modeling/decomposition/fast_ica.py +8 -1
- snowflake/ml/modeling/decomposition/incremental_pca.py +8 -1
- snowflake/ml/modeling/decomposition/kernel_pca.py +8 -1
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +8 -1
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +8 -1
- snowflake/ml/modeling/decomposition/pca.py +8 -1
- snowflake/ml/modeling/decomposition/sparse_pca.py +8 -1
- snowflake/ml/modeling/decomposition/truncated_svd.py +8 -1
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +8 -1
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +8 -1
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +8 -1
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +8 -1
- snowflake/ml/modeling/ensemble/bagging_classifier.py +8 -1
- snowflake/ml/modeling/ensemble/bagging_regressor.py +8 -1
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +8 -1
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +8 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +8 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +8 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +8 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +8 -1
- snowflake/ml/modeling/ensemble/isolation_forest.py +8 -1
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +8 -1
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +8 -1
- snowflake/ml/modeling/ensemble/stacking_regressor.py +8 -1
- snowflake/ml/modeling/ensemble/voting_classifier.py +8 -1
- snowflake/ml/modeling/ensemble/voting_regressor.py +8 -1
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +8 -1
- snowflake/ml/modeling/feature_selection/select_fdr.py +8 -1
- snowflake/ml/modeling/feature_selection/select_fpr.py +8 -1
- snowflake/ml/modeling/feature_selection/select_fwe.py +8 -1
- snowflake/ml/modeling/feature_selection/select_k_best.py +8 -1
- snowflake/ml/modeling/feature_selection/select_percentile.py +8 -1
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +8 -1
- snowflake/ml/modeling/feature_selection/variance_threshold.py +8 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +8 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +8 -1
- snowflake/ml/modeling/impute/iterative_imputer.py +8 -1
- snowflake/ml/modeling/impute/knn_imputer.py +8 -1
- snowflake/ml/modeling/impute/missing_indicator.py +8 -1
- snowflake/ml/modeling/impute/simple_imputer.py +21 -2
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +8 -1
- snowflake/ml/modeling/kernel_approximation/nystroem.py +8 -1
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +8 -1
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +8 -1
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +8 -1
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +8 -1
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +8 -1
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/ard_regression.py +8 -1
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +8 -1
- snowflake/ml/modeling/linear_model/elastic_net.py +8 -1
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +8 -1
- snowflake/ml/modeling/linear_model/gamma_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/huber_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/lars.py +8 -1
- snowflake/ml/modeling/linear_model/lars_cv.py +8 -1
- snowflake/ml/modeling/linear_model/lasso.py +8 -1
- snowflake/ml/modeling/linear_model/lasso_cv.py +8 -1
- snowflake/ml/modeling/linear_model/lasso_lars.py +8 -1
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +8 -1
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +8 -1
- snowflake/ml/modeling/linear_model/linear_regression.py +8 -1
- snowflake/ml/modeling/linear_model/logistic_regression.py +8 -1
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +8 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +8 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +8 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +8 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +8 -1
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +8 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +8 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/perceptron.py +8 -1
- snowflake/ml/modeling/linear_model/poisson_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/ransac_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/ridge.py +8 -1
- snowflake/ml/modeling/linear_model/ridge_classifier.py +8 -1
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +8 -1
- snowflake/ml/modeling/linear_model/ridge_cv.py +8 -1
- snowflake/ml/modeling/linear_model/sgd_classifier.py +8 -1
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +8 -1
- snowflake/ml/modeling/linear_model/sgd_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +8 -1
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +8 -1
- snowflake/ml/modeling/manifold/isomap.py +8 -1
- snowflake/ml/modeling/manifold/mds.py +8 -1
- snowflake/ml/modeling/manifold/spectral_embedding.py +8 -1
- snowflake/ml/modeling/manifold/tsne.py +8 -1
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +8 -1
- snowflake/ml/modeling/mixture/gaussian_mixture.py +8 -1
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +8 -1
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +8 -1
- snowflake/ml/modeling/multiclass/output_code_classifier.py +8 -1
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +8 -1
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +8 -1
- snowflake/ml/modeling/naive_bayes/complement_nb.py +8 -1
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +8 -1
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +8 -1
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +8 -1
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +8 -1
- snowflake/ml/modeling/neighbors/kernel_density.py +8 -1
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +8 -1
- snowflake/ml/modeling/neighbors/nearest_centroid.py +8 -1
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +8 -1
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +8 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +8 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +8 -1
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +8 -1
- snowflake/ml/modeling/neural_network/mlp_classifier.py +8 -1
- snowflake/ml/modeling/neural_network/mlp_regressor.py +8 -1
- snowflake/ml/modeling/parameters/enable_anonymous_sproc.py +5 -0
- snowflake/ml/modeling/preprocessing/polynomial_features.py +8 -1
- snowflake/ml/modeling/semi_supervised/label_propagation.py +8 -1
- snowflake/ml/modeling/semi_supervised/label_spreading.py +8 -1
- snowflake/ml/modeling/svm/linear_svc.py +8 -1
- snowflake/ml/modeling/svm/linear_svr.py +8 -1
- snowflake/ml/modeling/svm/nu_svc.py +8 -1
- snowflake/ml/modeling/svm/nu_svr.py +8 -1
- snowflake/ml/modeling/svm/svc.py +8 -1
- snowflake/ml/modeling/svm/svr.py +8 -1
- snowflake/ml/modeling/tree/decision_tree_classifier.py +8 -1
- snowflake/ml/modeling/tree/decision_tree_regressor.py +8 -1
- snowflake/ml/modeling/tree/extra_tree_classifier.py +8 -1
- snowflake/ml/modeling/tree/extra_tree_regressor.py +8 -1
- snowflake/ml/modeling/xgboost/xgb_classifier.py +8 -1
- snowflake/ml/modeling/xgboost/xgb_regressor.py +8 -1
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +8 -1
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +8 -1
- snowflake/ml/registry/_manager/model_manager.py +59 -1
- snowflake/ml/registry/registry.py +10 -1
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.5.1.dist-info → snowflake_ml_python-1.5.2.dist-info}/METADATA +13 -1
- {snowflake_ml_python-1.5.1.dist-info → snowflake_ml_python-1.5.2.dist-info}/RECORD +174 -172
- {snowflake_ml_python-1.5.1.dist-info → snowflake_ml_python-1.5.2.dist-info}/LICENSE.txt +0 -0
- {snowflake_ml_python-1.5.1.dist-info → snowflake_ml_python-1.5.2.dist-info}/WHEEL +0 -0
- {snowflake_ml_python-1.5.1.dist-info → snowflake_ml_python-1.5.2.dist-info}/top_level.txt +0 -0
snowflake/cortex/_sentiment.py
CHANGED
@@ -11,7 +11,7 @@ from snowflake.ml._internal import telemetry
|
|
11
11
|
)
|
12
12
|
def Sentiment(
|
13
13
|
text: Union[str, snowpark.Column], session: Optional[snowpark.Session] = None
|
14
|
-
) -> Union[
|
14
|
+
) -> Union[float, snowpark.Column]:
|
15
15
|
"""Sentiment calls into the LLM inference service to perform sentiment analysis on the input text.
|
16
16
|
|
17
17
|
Args:
|
@@ -21,11 +21,14 @@ def Sentiment(
|
|
21
21
|
Returns:
|
22
22
|
A column of floats. 1 represents positive sentiment, -1 represents negative sentiment.
|
23
23
|
"""
|
24
|
-
|
25
24
|
return _sentiment_impl("snowflake.cortex.sentiment", text, session=session)
|
26
25
|
|
27
26
|
|
28
27
|
def _sentiment_impl(
|
29
28
|
function: str, text: Union[str, snowpark.Column], session: Optional[snowpark.Session] = None
|
30
|
-
) -> Union[
|
31
|
-
|
29
|
+
) -> Union[float, snowpark.Column]:
|
30
|
+
|
31
|
+
output = call_sql_function(function, session, text)
|
32
|
+
if isinstance(output, snowpark.Column):
|
33
|
+
return output
|
34
|
+
return float(output)
|
@@ -8,14 +8,17 @@ from absl.logging import logging
|
|
8
8
|
logger = logging.getLogger(__name__)
|
9
9
|
|
10
10
|
|
11
|
-
def get_temp_file_path() -> str:
|
11
|
+
def get_temp_file_path(prefix: str = "") -> str:
|
12
12
|
"""Returns a new random temp file path.
|
13
13
|
|
14
|
+
Args:
|
15
|
+
prefix: A prefix to the temp file path, this can help add stored file information. Defaults to None.
|
16
|
+
|
14
17
|
Returns:
|
15
18
|
A new temp file path.
|
16
19
|
"""
|
17
20
|
# TODO(snandamuri): Use in-memory filesystem for temp files.
|
18
|
-
local_file = tempfile.NamedTemporaryFile(delete=True)
|
21
|
+
local_file = tempfile.NamedTemporaryFile(prefix=prefix, delete=True)
|
19
22
|
local_file_name = local_file.name
|
20
23
|
local_file.close()
|
21
24
|
return local_file_name
|
@@ -42,6 +42,8 @@ class _SessionInfo:
|
|
42
42
|
# Lists of permissions as tuples of (OBJECT_TYPE, [PRIVILEGES, ...])
|
43
43
|
_PRE_INIT_PRIVILEGES: Dict[_FeatureStoreRole, List[_Privilege]] = {
|
44
44
|
_FeatureStoreRole.PRODUCER: [
|
45
|
+
_Privilege("DATABASE", "{database}", ["USAGE"]),
|
46
|
+
_Privilege("SCHEMA", "{database}.{schema}", ["USAGE"]),
|
45
47
|
_Privilege(
|
46
48
|
"SCHEMA",
|
47
49
|
"{database}.{schema}",
|
@@ -69,8 +71,7 @@ _PRE_INIT_PRIVILEGES: Dict[_FeatureStoreRole, List[_Privilege]] = {
|
|
69
71
|
_Privilege("DYNAMIC TABLE", _ALL_OBJECTS, ["SELECT", "MONITOR"], "SCHEMA {database}.{schema}"),
|
70
72
|
_Privilege("VIEW", _ALL_OBJECTS, ["SELECT", "REFERENCES"], "SCHEMA {database}.{schema}"),
|
71
73
|
_Privilege("TABLE", _ALL_OBJECTS, ["SELECT", "REFERENCES"], "SCHEMA {database}.{schema}"),
|
72
|
-
|
73
|
-
# _Privilege("DATASET", _ALL_OBJECTS, ["USAGE"], "SCHEMA {database}.{schema}"),
|
74
|
+
_Privilege("DATASET", _ALL_OBJECTS, ["USAGE"], "SCHEMA {database}.{schema}"),
|
74
75
|
# User should decide whether they want to grant warehouse usage to CONSUMER
|
75
76
|
# _Privilege("WAREHOUSE", "{warehouse}", ["USAGE"]),
|
76
77
|
],
|
@@ -128,8 +129,7 @@ def _grant_privileges(
|
|
128
129
|
def _configure_pre_init_privileges(
|
129
130
|
session: Session,
|
130
131
|
session_info: _SessionInfo,
|
131
|
-
|
132
|
-
consumer_role: str = "SNOWML_FEATURE_STORE_CONSUMER_RL",
|
132
|
+
roles_to_create: Dict[_FeatureStoreRole, str],
|
133
133
|
) -> None:
|
134
134
|
"""
|
135
135
|
Configure Feature Store role privileges. Must be run with ACCOUNTADMIN
|
@@ -141,8 +141,7 @@ def _configure_pre_init_privileges(
|
|
141
141
|
Args:
|
142
142
|
session: Snowpark Session to interact with Snowflake backend.
|
143
143
|
session_info: Session info like database and schema for the FeatureStore instance.
|
144
|
-
|
145
|
-
consumer_role: Name of consumer role to be configured.
|
144
|
+
roles_to_create: Producer and optional consumer roles to create.
|
146
145
|
"""
|
147
146
|
|
148
147
|
# Create schema if not already exists
|
@@ -159,29 +158,30 @@ def _configure_pre_init_privileges(
|
|
159
158
|
|
160
159
|
# Pass schema ownership from admin to PRODUCER
|
161
160
|
if schema_created:
|
161
|
+
# TODO: we are missing a test case for this code path
|
162
162
|
session.sql(
|
163
|
-
f"GRANT OWNERSHIP ON SCHEMA {session_info.database}.{session_info.schema}
|
163
|
+
f"GRANT OWNERSHIP ON SCHEMA {session_info.database}.{session_info.schema} "
|
164
|
+
f"TO ROLE {roles_to_create[_FeatureStoreRole.PRODUCER]}"
|
164
165
|
).collect()
|
165
166
|
|
166
167
|
# Grant privileges to roles
|
167
|
-
|
168
|
-
|
168
|
+
for role_type, role in roles_to_create.items():
|
169
|
+
_grant_privileges(session, role, _PRE_INIT_PRIVILEGES[role_type], session_info)
|
169
170
|
|
170
171
|
|
171
172
|
def _configure_post_init_privileges(
|
172
173
|
session: Session,
|
173
174
|
session_info: _SessionInfo,
|
174
|
-
|
175
|
-
consumer_role: str = "FS_CONSUMER",
|
175
|
+
roles_to_create: Dict[_FeatureStoreRole, str],
|
176
176
|
) -> None:
|
177
|
-
|
178
|
-
|
177
|
+
for role_type, role in roles_to_create.items():
|
178
|
+
_grant_privileges(session, role, _POST_INIT_PRIVILEGES[role_type], session_info)
|
179
179
|
|
180
180
|
|
181
181
|
def _configure_role_hierarchy(
|
182
182
|
session: Session,
|
183
183
|
producer_role: str,
|
184
|
-
consumer_role: str,
|
184
|
+
consumer_role: Optional[str],
|
185
185
|
) -> None:
|
186
186
|
"""
|
187
187
|
Create Feature Store roles and configure role hierarchy hierarchy. Must be run with
|
@@ -195,18 +195,17 @@ def _configure_role_hierarchy(
|
|
195
195
|
producer_role: Name of producer role to be configured.
|
196
196
|
consumer_role: Name of consumer role to be configured.
|
197
197
|
"""
|
198
|
+
# Create the necessary roles and build role hierarchy
|
198
199
|
producer_role = SqlIdentifier(producer_role)
|
199
|
-
consumer_role = SqlIdentifier(consumer_role)
|
200
|
-
|
201
|
-
# Create the necessary roles
|
202
200
|
session.sql(f"CREATE ROLE IF NOT EXISTS {producer_role}").collect()
|
203
|
-
session.sql(f"CREATE ROLE IF NOT EXISTS {consumer_role}").collect()
|
204
|
-
|
205
|
-
# Build role hierarchy
|
206
|
-
session.sql(f"GRANT ROLE {consumer_role} TO ROLE {producer_role}").collect()
|
207
201
|
session.sql(f"GRANT ROLE {producer_role} TO ROLE SYSADMIN").collect()
|
208
202
|
session.sql(f"GRANT ROLE {producer_role} TO ROLE {session.get_current_role()}").collect()
|
209
203
|
|
204
|
+
if consumer_role is not None:
|
205
|
+
consumer_role = SqlIdentifier(consumer_role)
|
206
|
+
session.sql(f"CREATE ROLE IF NOT EXISTS {consumer_role}").collect()
|
207
|
+
session.sql(f"GRANT ROLE {consumer_role} TO ROLE {producer_role}").collect()
|
208
|
+
|
210
209
|
|
211
210
|
@telemetry.send_api_usage_telemetry(project=_PROJECT)
|
212
211
|
def setup_feature_store(
|
@@ -215,7 +214,7 @@ def setup_feature_store(
|
|
215
214
|
schema: str,
|
216
215
|
warehouse: str,
|
217
216
|
producer_role: str = "FS_PRODUCER",
|
218
|
-
consumer_role: str =
|
217
|
+
consumer_role: Optional[str] = None,
|
219
218
|
) -> FeatureStore:
|
220
219
|
"""
|
221
220
|
Sets up a new Feature Store including role/privilege setup. Must be run with ACCOUNTADMIN
|
@@ -230,7 +229,7 @@ def setup_feature_store(
|
|
230
229
|
schema: Schema to create the FeatureStore instance.
|
231
230
|
warehouse: Default warehouse for Feature Store compute.
|
232
231
|
producer_role: Name of producer role to be configured.
|
233
|
-
consumer_role: Name of consumer role to be configured.
|
232
|
+
consumer_role: Name of consumer role to be configured. If not specified, consumer role won't be created.
|
234
233
|
|
235
234
|
Returns:
|
236
235
|
Feature Store instance.
|
@@ -249,20 +248,25 @@ def setup_feature_store(
|
|
249
248
|
)
|
250
249
|
|
251
250
|
try:
|
251
|
+
roles_to_create = {_FeatureStoreRole.PRODUCER: producer_role}
|
252
|
+
if consumer_role is not None:
|
253
|
+
roles_to_create.update({_FeatureStoreRole.CONSUMER: consumer_role})
|
252
254
|
_configure_role_hierarchy(session, producer_role=producer_role, consumer_role=consumer_role)
|
253
255
|
except exceptions.SnowparkSQLException:
|
254
256
|
# Error can be safely ignored if roles already exist and hierarchy is already built
|
255
|
-
for role in (
|
257
|
+
for _, role in roles_to_create.items():
|
256
258
|
# Ensure roles already exist
|
257
259
|
if session.sql(f"SHOW ROLES LIKE '{role}' STARTS WITH '{role}'").count() == 0:
|
258
260
|
raise
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
261
|
+
|
262
|
+
if consumer_role is not None:
|
263
|
+
# Ensure hierarchy already configured
|
264
|
+
consumer_grants = session.sql(f"SHOW GRANTS ON ROLE {consumer_role}").collect()
|
265
|
+
if not any(r["granted_to"] == "ROLE" and r["grantee_name"] == producer_role for r in consumer_grants):
|
266
|
+
raise
|
263
267
|
|
264
268
|
# Do any pre-FeatureStore.__init__() privilege setup
|
265
|
-
_configure_pre_init_privileges(session, session_info,
|
269
|
+
_configure_pre_init_privileges(session, session_info, roles_to_create)
|
266
270
|
|
267
271
|
# Use PRODUCER role to create and operate new Feature Store
|
268
272
|
current_role = session.get_current_role()
|
@@ -274,6 +278,6 @@ def setup_feature_store(
|
|
274
278
|
session.use_role(current_role)
|
275
279
|
|
276
280
|
# Do any post-FeatureStore.__init__() privilege setup
|
277
|
-
_configure_post_init_privileges(session, session_info,
|
281
|
+
_configure_post_init_privileges(session, session_info, roles_to_create)
|
278
282
|
|
279
283
|
return fs
|
@@ -1761,7 +1761,7 @@ class FeatureStore:
|
|
1761
1761
|
self._session.sql(
|
1762
1762
|
f"""
|
1763
1763
|
SELECT * FROM TABLE(
|
1764
|
-
INFORMATION_SCHEMA.TAG_REFERENCES_INTERNAL(
|
1764
|
+
{self._config.database}.INFORMATION_SCHEMA.TAG_REFERENCES_INTERNAL(
|
1765
1765
|
TAG_NAME => '{_FEATURE_STORE_OBJECT_TAG}'
|
1766
1766
|
)
|
1767
1767
|
) LIMIT 1;
|
@@ -7,10 +7,6 @@ from dataclasses import asdict, dataclass
|
|
7
7
|
from enum import Enum
|
8
8
|
from typing import Any, Dict, List, Optional
|
9
9
|
|
10
|
-
from snowflake.ml._internal.exceptions import (
|
11
|
-
error_codes,
|
12
|
-
exceptions as snowml_exceptions,
|
13
|
-
)
|
14
10
|
from snowflake.ml._internal.utils.identifier import concat_names
|
15
11
|
from snowflake.ml._internal.utils.sql_identifier import (
|
16
12
|
SqlIdentifier,
|
@@ -34,6 +30,11 @@ _FEATURE_OBJ_TYPE = "FEATURE_OBJ_TYPE"
|
|
34
30
|
_FEATURE_VIEW_VERSION_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.\-]*$")
|
35
31
|
_FEATURE_VIEW_VERSION_MAX_LENGTH = 128
|
36
32
|
|
33
|
+
_RESULT_SCAN_QUERY_PATTERN = re.compile(
|
34
|
+
r".*FROM\s*TABLE\s*\(\s*RESULT_SCAN\s*\(.*",
|
35
|
+
flags=re.DOTALL | re.IGNORECASE | re.X,
|
36
|
+
)
|
37
|
+
|
37
38
|
|
38
39
|
@dataclass(frozen=True)
|
39
40
|
class _FeatureViewMetadata:
|
@@ -54,13 +55,10 @@ class _FeatureViewMetadata:
|
|
54
55
|
class FeatureViewVersion(str):
|
55
56
|
def __new__(cls, version: str) -> FeatureViewVersion:
|
56
57
|
if not _FEATURE_VIEW_VERSION_RE.match(version) or len(version) > _FEATURE_VIEW_VERSION_MAX_LENGTH:
|
57
|
-
raise
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
"It must start with letter or digit, and followed by letter, digit, '_', '-' or '.'. "
|
62
|
-
f"The length limit is {_FEATURE_VIEW_VERSION_MAX_LENGTH}."
|
63
|
-
),
|
58
|
+
raise ValueError(
|
59
|
+
f"`{version}` is not a valid feature view version. "
|
60
|
+
"It must start with letter or digit, and followed by letter, digit, '_', '-' or '.'. "
|
61
|
+
f"The length limit is {_FEATURE_VIEW_VERSION_MAX_LENGTH}."
|
64
62
|
)
|
65
63
|
return super().__new__(cls, version)
|
66
64
|
|
@@ -352,6 +350,9 @@ Got {len(self._feature_df.queries['queries'])}: {self._feature_df.queries['queri
|
|
352
350
|
if not isinstance(col_type, (DateType, TimeType, TimestampType, _NumericType)):
|
353
351
|
raise ValueError(f"Invalid data type for timestamp_col {ts_col}: {col_type}.")
|
354
352
|
|
353
|
+
if re.match(_RESULT_SCAN_QUERY_PATTERN, self._query) is not None:
|
354
|
+
raise ValueError(f"feature_df should not be reading from RESULT_SCAN. Invalid query: {self._query}")
|
355
|
+
|
355
356
|
def _get_feature_names(self) -> List[SqlIdentifier]:
|
356
357
|
join_keys = [k for e in self._entities for k in e.join_keys]
|
357
358
|
ts_col = [self._timestamp_col] if self._timestamp_col is not None else []
|
snowflake/ml/fileset/snowfs.py
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
import collections
|
2
2
|
import logging
|
3
3
|
import re
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, Optional
|
5
5
|
|
6
6
|
import fsspec
|
7
|
-
import packaging.version as pkg_version
|
8
7
|
|
9
8
|
from snowflake import snowpark
|
10
9
|
from snowflake.connector import connection
|
@@ -12,7 +11,7 @@ from snowflake.ml._internal.exceptions import (
|
|
12
11
|
error_codes,
|
13
12
|
exceptions as snowml_exceptions,
|
14
13
|
)
|
15
|
-
from snowflake.ml._internal.utils import identifier
|
14
|
+
from snowflake.ml._internal.utils import identifier
|
16
15
|
from snowflake.ml.fileset import embedded_stage_fs, sfcfs
|
17
16
|
|
18
17
|
PROTOCOL_NAME = "snow"
|
@@ -28,10 +27,6 @@ _SNOWURL_PATTERN = re.compile(
|
|
28
27
|
r"(?P<path>versions/(?:(?P<version>[^/]+)(?:/(?P<relpath>.*))?)?)"
|
29
28
|
)
|
30
29
|
|
31
|
-
# FIXME(dhung): Temporary fix for bug in GS version 8.17
|
32
|
-
_BUG_VERSION_MIN = pkg_version.Version("8.17") # Inclusive minimum version with bugged behavior
|
33
|
-
_BUG_VERSION_MAX = pkg_version.Version("8.18") # Exclusive maximum version with bugged behavior
|
34
|
-
|
35
30
|
|
36
31
|
class SnowFileSystem(sfcfs.SFFileSystem):
|
37
32
|
"""A filesystem that allows user to access Snowflake embedded stage files with valid Snowflake locations.
|
@@ -54,21 +49,6 @@ class SnowFileSystem(sfcfs.SFFileSystem):
|
|
54
49
|
) -> None:
|
55
50
|
super().__init__(sf_connection=sf_connection, snowpark_session=snowpark_session, **kwargs)
|
56
51
|
|
57
|
-
# FIXME(dhung): Temporary fix for bug in GS version 8.17
|
58
|
-
if SnowFileSystem._IS_BUGGED_VERSION is None:
|
59
|
-
try:
|
60
|
-
sf_version = snowflake_env.get_current_snowflake_version(self._session)
|
61
|
-
SnowFileSystem._IS_BUGGED_VERSION = _BUG_VERSION_MIN <= sf_version < _BUG_VERSION_MAX
|
62
|
-
except Exception:
|
63
|
-
SnowFileSystem._IS_BUGGED_VERSION = False
|
64
|
-
|
65
|
-
def info(self, path: str, **kwargs: Any) -> Dict[str, Any]:
|
66
|
-
# FIXME(dhung): Temporary fix for bug in GS version 8.17
|
67
|
-
res: Dict[str, Any] = super().info(path, **kwargs)
|
68
|
-
if res.get("type") == "directory" and not res["name"].endswith("/"):
|
69
|
-
res["name"] += "/"
|
70
|
-
return res
|
71
|
-
|
72
52
|
def _get_stage_fs(
|
73
53
|
self, sf_file_path: _SFFileEntityPath # type: ignore[override]
|
74
54
|
) -> embedded_stage_fs.SFEmbeddedStageFileSystem:
|
@@ -100,12 +80,6 @@ class SnowFileSystem(sfcfs.SFFileSystem):
|
|
100
80
|
if stage_name.startswith(protocol):
|
101
81
|
stage_name = stage_name[len(protocol) :]
|
102
82
|
abs_path = stage_name + "/" + path
|
103
|
-
# FIXME(dhung): Temporary fix for bug in GS version 8.17
|
104
|
-
if self._IS_BUGGED_VERSION:
|
105
|
-
match = _SNOWURL_PATTERN.fullmatch(abs_path)
|
106
|
-
assert match is not None
|
107
|
-
if match.group("relpath"):
|
108
|
-
abs_path = abs_path.replace(match.group("relpath"), match.group("relpath").lstrip("/"))
|
109
83
|
return abs_path
|
110
84
|
|
111
85
|
@classmethod
|
@@ -144,9 +118,6 @@ class SnowFileSystem(sfcfs.SFFileSystem):
|
|
144
118
|
version = snowurl_match.group("version")
|
145
119
|
relative_path = snowurl_match.group("relpath") or ""
|
146
120
|
logging.debug(f"Parsed snow URL: {snowurl_match.groups()}")
|
147
|
-
# FIXME(dhung): Temporary fix for bug in GS version 8.17
|
148
|
-
if cls._IS_BUGGED_VERSION:
|
149
|
-
filepath = f"versions/{version}//{relative_path}"
|
150
121
|
return _SFFileEntityPath(
|
151
122
|
domain=domain, name=name, version=version, relative_path=relative_path, filepath=filepath
|
152
123
|
)
|
@@ -140,6 +140,49 @@ class ModelOperator:
|
|
140
140
|
statement_params=statement_params,
|
141
141
|
)
|
142
142
|
|
143
|
+
def create_from_model_version(
|
144
|
+
self,
|
145
|
+
*,
|
146
|
+
source_database_name: Optional[sql_identifier.SqlIdentifier],
|
147
|
+
source_schema_name: Optional[sql_identifier.SqlIdentifier],
|
148
|
+
source_model_name: sql_identifier.SqlIdentifier,
|
149
|
+
source_version_name: sql_identifier.SqlIdentifier,
|
150
|
+
database_name: Optional[sql_identifier.SqlIdentifier],
|
151
|
+
schema_name: Optional[sql_identifier.SqlIdentifier],
|
152
|
+
model_name: sql_identifier.SqlIdentifier,
|
153
|
+
version_name: sql_identifier.SqlIdentifier,
|
154
|
+
statement_params: Optional[Dict[str, Any]] = None,
|
155
|
+
) -> None:
|
156
|
+
if self.validate_existence(
|
157
|
+
database_name=database_name,
|
158
|
+
schema_name=schema_name,
|
159
|
+
model_name=model_name,
|
160
|
+
statement_params=statement_params,
|
161
|
+
):
|
162
|
+
return self._model_version_client.add_version_from_model_version(
|
163
|
+
source_database_name=source_database_name,
|
164
|
+
source_schema_name=source_schema_name,
|
165
|
+
source_model_name=source_model_name,
|
166
|
+
source_version_name=source_version_name,
|
167
|
+
database_name=database_name,
|
168
|
+
schema_name=schema_name,
|
169
|
+
model_name=model_name,
|
170
|
+
version_name=version_name,
|
171
|
+
statement_params=statement_params,
|
172
|
+
)
|
173
|
+
else:
|
174
|
+
return self._model_version_client.create_from_model_version(
|
175
|
+
source_database_name=source_database_name,
|
176
|
+
source_schema_name=source_schema_name,
|
177
|
+
source_model_name=source_model_name,
|
178
|
+
source_version_name=source_version_name,
|
179
|
+
database_name=database_name,
|
180
|
+
schema_name=schema_name,
|
181
|
+
model_name=model_name,
|
182
|
+
version_name=version_name,
|
183
|
+
statement_params=statement_params,
|
184
|
+
)
|
185
|
+
|
143
186
|
def show_models_or_versions(
|
144
187
|
self,
|
145
188
|
*,
|
@@ -44,6 +44,32 @@ class ModelVersionSQLClient(_base._BaseSQLClient):
|
|
44
44
|
statement_params=statement_params,
|
45
45
|
).has_dimensions(expected_rows=1, expected_cols=1).validate()
|
46
46
|
|
47
|
+
def create_from_model_version(
|
48
|
+
self,
|
49
|
+
*,
|
50
|
+
source_database_name: Optional[sql_identifier.SqlIdentifier],
|
51
|
+
source_schema_name: Optional[sql_identifier.SqlIdentifier],
|
52
|
+
source_model_name: sql_identifier.SqlIdentifier,
|
53
|
+
source_version_name: sql_identifier.SqlIdentifier,
|
54
|
+
database_name: Optional[sql_identifier.SqlIdentifier],
|
55
|
+
schema_name: Optional[sql_identifier.SqlIdentifier],
|
56
|
+
model_name: sql_identifier.SqlIdentifier,
|
57
|
+
version_name: sql_identifier.SqlIdentifier,
|
58
|
+
statement_params: Optional[Dict[str, Any]] = None,
|
59
|
+
) -> None:
|
60
|
+
fq_source_model_name = self.fully_qualified_object_name(
|
61
|
+
source_database_name, source_schema_name, source_model_name
|
62
|
+
)
|
63
|
+
fq_model_name = self.fully_qualified_object_name(database_name, schema_name, model_name)
|
64
|
+
query_result_checker.SqlResultValidator(
|
65
|
+
self._session,
|
66
|
+
(
|
67
|
+
f"CREATE MODEL {fq_model_name} WITH VERSION {version_name} FROM MODEL {fq_source_model_name}"
|
68
|
+
f" VERSION {source_version_name}"
|
69
|
+
),
|
70
|
+
statement_params=statement_params,
|
71
|
+
).has_dimensions(expected_rows=1, expected_cols=1).validate()
|
72
|
+
|
47
73
|
# TODO(SNOW-987381): Merge with above when we have `create or alter module m [with] version v1 ...`
|
48
74
|
def add_version_from_stage(
|
49
75
|
self,
|
@@ -64,6 +90,32 @@ class ModelVersionSQLClient(_base._BaseSQLClient):
|
|
64
90
|
statement_params=statement_params,
|
65
91
|
).has_dimensions(expected_rows=1, expected_cols=1).validate()
|
66
92
|
|
93
|
+
def add_version_from_model_version(
|
94
|
+
self,
|
95
|
+
*,
|
96
|
+
source_database_name: Optional[sql_identifier.SqlIdentifier],
|
97
|
+
source_schema_name: Optional[sql_identifier.SqlIdentifier],
|
98
|
+
source_model_name: sql_identifier.SqlIdentifier,
|
99
|
+
source_version_name: sql_identifier.SqlIdentifier,
|
100
|
+
database_name: Optional[sql_identifier.SqlIdentifier],
|
101
|
+
schema_name: Optional[sql_identifier.SqlIdentifier],
|
102
|
+
model_name: sql_identifier.SqlIdentifier,
|
103
|
+
version_name: sql_identifier.SqlIdentifier,
|
104
|
+
statement_params: Optional[Dict[str, Any]] = None,
|
105
|
+
) -> None:
|
106
|
+
fq_source_model_name = self.fully_qualified_object_name(
|
107
|
+
source_database_name, source_schema_name, source_model_name
|
108
|
+
)
|
109
|
+
fq_model_name = self.fully_qualified_object_name(database_name, schema_name, model_name)
|
110
|
+
query_result_checker.SqlResultValidator(
|
111
|
+
self._session,
|
112
|
+
(
|
113
|
+
f"ALTER MODEL {fq_model_name} ADD VERSION {version_name} FROM MODEL {fq_source_model_name}"
|
114
|
+
f" VERSION {source_version_name}"
|
115
|
+
),
|
116
|
+
statement_params=statement_params,
|
117
|
+
).has_dimensions(expected_rows=1, expected_cols=1).validate()
|
118
|
+
|
67
119
|
def set_default_version(
|
68
120
|
self,
|
69
121
|
*,
|
@@ -145,7 +197,7 @@ class ModelVersionSQLClient(_base._BaseSQLClient):
|
|
145
197
|
if snowpark_utils.is_in_stored_procedure(): # type: ignore[no-untyped-call]
|
146
198
|
options = {"parallel": 10}
|
147
199
|
cursor = self._session._conn._cursor
|
148
|
-
cursor._download(stage_location_url, str(target_path), options) # type: ignore[attr
|
200
|
+
cursor._download(stage_location_url, str(target_path), options) # type: ignore[union-attr]
|
149
201
|
cursor.fetchall()
|
150
202
|
else:
|
151
203
|
query_result_checker.SqlResultValidator(
|
@@ -136,7 +136,7 @@ class ModelComposer:
|
|
136
136
|
model_meta=self.packager.meta,
|
137
137
|
model_file_rel_path=pathlib.PurePosixPath(self.model_file_rel_path),
|
138
138
|
options=options,
|
139
|
-
data_sources=self._get_data_sources(model),
|
139
|
+
data_sources=self._get_data_sources(model, sample_input_data),
|
140
140
|
)
|
141
141
|
|
142
142
|
file_utils.upload_directory_to_stage(
|
@@ -179,8 +179,12 @@ class ModelComposer:
|
|
179
179
|
mp.load(meta_only=meta_only, options=options)
|
180
180
|
return mp
|
181
181
|
|
182
|
-
def _get_data_sources(
|
182
|
+
def _get_data_sources(
|
183
|
+
self, model: model_types.SupportedModelType, sample_input_data: Optional[model_types.SupportedDataType] = None
|
184
|
+
) -> Optional[List[data_source.DataSource]]:
|
183
185
|
data_sources = getattr(model, lineage_utils.DATA_SOURCES_ATTR, None)
|
186
|
+
if not data_sources and sample_input_data is not None:
|
187
|
+
data_sources = getattr(sample_input_data, lineage_utils.DATA_SOURCES_ATTR, None)
|
184
188
|
if isinstance(data_sources, list) and all(isinstance(item, data_source.DataSource) for item in data_sources):
|
185
189
|
return data_sources
|
186
190
|
return None
|
@@ -281,9 +281,7 @@ class ModelMetadata:
|
|
281
281
|
"cpu": model_runtime.ModelRuntime("cpu", self.env),
|
282
282
|
}
|
283
283
|
if self.env.cuda_version:
|
284
|
-
runtimes.update(
|
285
|
-
{"gpu": model_runtime.ModelRuntime("gpu", self.env, is_gpu=True, server_availability_source="conda")}
|
286
|
-
)
|
284
|
+
runtimes.update({"gpu": model_runtime.ModelRuntime("gpu", self.env, is_gpu=True)})
|
287
285
|
return runtimes
|
288
286
|
|
289
287
|
def save(self, model_dir_path: str) -> None:
|
@@ -1,11 +1,11 @@
|
|
1
1
|
import copy
|
2
2
|
import pathlib
|
3
3
|
import warnings
|
4
|
-
from typing import List,
|
4
|
+
from typing import List, Optional
|
5
5
|
|
6
6
|
from packaging import requirements
|
7
7
|
|
8
|
-
from snowflake.ml._internal import
|
8
|
+
from snowflake.ml._internal import env_utils, file_utils
|
9
9
|
from snowflake.ml.model._packager.model_env import model_env
|
10
10
|
from snowflake.ml.model._packager.model_meta import model_meta_schema
|
11
11
|
from snowflake.ml.model._packager.model_runtime import (
|
@@ -37,7 +37,6 @@ class ModelRuntime:
|
|
37
37
|
env: model_env.ModelEnv,
|
38
38
|
imports: Optional[List[pathlib.PurePosixPath]] = None,
|
39
39
|
is_gpu: bool = False,
|
40
|
-
server_availability_source: Literal["snowflake", "conda"] = "snowflake",
|
41
40
|
loading_from_file: bool = False,
|
42
41
|
) -> None:
|
43
42
|
self.name = name
|
@@ -48,30 +47,7 @@ class ModelRuntime:
|
|
48
47
|
return
|
49
48
|
|
50
49
|
snowml_pkg_spec = f"{env_utils.SNOWPARK_ML_PKG_NAME}=={self.runtime_env.snowpark_ml_version}"
|
51
|
-
|
52
|
-
self.embed_local_ml_library = True
|
53
|
-
else:
|
54
|
-
if server_availability_source == "snowflake":
|
55
|
-
snowml_server_availability = (
|
56
|
-
len(
|
57
|
-
env_utils.get_matched_package_versions_in_information_schema_with_active_session(
|
58
|
-
reqs=[requirements.Requirement(snowml_pkg_spec)],
|
59
|
-
python_version=snowml_env.PYTHON_VERSION,
|
60
|
-
).get(env_utils.SNOWPARK_ML_PKG_NAME, [])
|
61
|
-
)
|
62
|
-
>= 1
|
63
|
-
)
|
64
|
-
else:
|
65
|
-
snowml_server_availability = (
|
66
|
-
len(
|
67
|
-
env_utils.get_matched_package_versions_in_snowflake_conda_channel(
|
68
|
-
req=requirements.Requirement(snowml_pkg_spec),
|
69
|
-
python_version=snowml_env.PYTHON_VERSION,
|
70
|
-
)
|
71
|
-
)
|
72
|
-
>= 1
|
73
|
-
)
|
74
|
-
self.embed_local_ml_library = not snowml_server_availability
|
50
|
+
self.embed_local_ml_library = self.runtime_env._snowpark_ml_version.local
|
75
51
|
|
76
52
|
additional_package = (
|
77
53
|
_SNOWML_INFERENCE_ALTERNATIVE_DEPENDENCIES if self.embed_local_ml_library else [snowml_pkg_spec]
|