snowflake-ml-python 1.4.1__py3-none-any.whl → 1.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. snowflake/ml/_internal/env_utils.py +66 -31
  2. snowflake/ml/_internal/exceptions/dataset_error_messages.py +5 -0
  3. snowflake/ml/_internal/exceptions/dataset_errors.py +24 -0
  4. snowflake/ml/_internal/exceptions/error_codes.py +3 -0
  5. snowflake/ml/_internal/lineage/data_source.py +10 -0
  6. snowflake/ml/_internal/lineage/dataset_dataframe.py +44 -0
  7. snowflake/ml/dataset/__init__.py +10 -0
  8. snowflake/ml/dataset/dataset.py +454 -129
  9. snowflake/ml/dataset/dataset_factory.py +53 -0
  10. snowflake/ml/dataset/dataset_metadata.py +103 -0
  11. snowflake/ml/dataset/dataset_reader.py +202 -0
  12. snowflake/ml/feature_store/feature_store.py +408 -282
  13. snowflake/ml/feature_store/feature_view.py +37 -8
  14. snowflake/ml/fileset/embedded_stage_fs.py +146 -0
  15. snowflake/ml/fileset/sfcfs.py +0 -4
  16. snowflake/ml/fileset/snowfs.py +159 -0
  17. snowflake/ml/fileset/stage_fs.py +1 -4
  18. snowflake/ml/model/__init__.py +2 -2
  19. snowflake/ml/model/_api.py +16 -1
  20. snowflake/ml/model/_client/model/model_impl.py +27 -0
  21. snowflake/ml/model/_client/model/model_version_impl.py +135 -0
  22. snowflake/ml/model/_client/ops/model_ops.py +137 -67
  23. snowflake/ml/model/_client/sql/model.py +16 -14
  24. snowflake/ml/model/_client/sql/model_version.py +109 -1
  25. snowflake/ml/model/_deploy_client/image_builds/server_image_builder.py +5 -1
  26. snowflake/ml/model/_deploy_client/image_builds/templates/dockerfile_template +1 -0
  27. snowflake/ml/model/_deploy_client/snowservice/deploy.py +2 -0
  28. snowflake/ml/model/_deploy_client/utils/constants.py +0 -5
  29. snowflake/ml/model/_deploy_client/utils/snowservice_client.py +21 -50
  30. snowflake/ml/model/_model_composer/model_composer.py +22 -1
  31. snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +22 -0
  32. snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py +11 -0
  33. snowflake/ml/model/_packager/model_env/model_env.py +41 -0
  34. snowflake/ml/model/_packager/model_meta/model_meta.py +1 -5
  35. snowflake/ml/model/_packager/model_packager.py +0 -3
  36. snowflake/ml/modeling/_internal/local_implementations/pandas_trainer.py +55 -3
  37. snowflake/ml/modeling/_internal/ml_runtime_implementations/ml_runtime_handlers.py +34 -18
  38. snowflake/ml/modeling/_internal/model_trainer.py +7 -0
  39. snowflake/ml/modeling/_internal/model_trainer_builder.py +42 -9
  40. snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_handlers.py +24 -2
  41. snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +261 -16
  42. snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +51 -52
  43. snowflake/ml/modeling/cluster/affinity_propagation.py +51 -52
  44. snowflake/ml/modeling/cluster/agglomerative_clustering.py +51 -52
  45. snowflake/ml/modeling/cluster/birch.py +53 -52
  46. snowflake/ml/modeling/cluster/bisecting_k_means.py +53 -52
  47. snowflake/ml/modeling/cluster/dbscan.py +51 -52
  48. snowflake/ml/modeling/cluster/feature_agglomeration.py +53 -52
  49. snowflake/ml/modeling/cluster/k_means.py +53 -52
  50. snowflake/ml/modeling/cluster/mean_shift.py +51 -52
  51. snowflake/ml/modeling/cluster/mini_batch_k_means.py +53 -52
  52. snowflake/ml/modeling/cluster/optics.py +51 -52
  53. snowflake/ml/modeling/cluster/spectral_biclustering.py +51 -52
  54. snowflake/ml/modeling/cluster/spectral_clustering.py +51 -52
  55. snowflake/ml/modeling/cluster/spectral_coclustering.py +51 -52
  56. snowflake/ml/modeling/compose/column_transformer.py +53 -52
  57. snowflake/ml/modeling/compose/transformed_target_regressor.py +51 -52
  58. snowflake/ml/modeling/covariance/elliptic_envelope.py +51 -52
  59. snowflake/ml/modeling/covariance/empirical_covariance.py +51 -52
  60. snowflake/ml/modeling/covariance/graphical_lasso.py +51 -52
  61. snowflake/ml/modeling/covariance/graphical_lasso_cv.py +51 -52
  62. snowflake/ml/modeling/covariance/ledoit_wolf.py +51 -52
  63. snowflake/ml/modeling/covariance/min_cov_det.py +51 -52
  64. snowflake/ml/modeling/covariance/oas.py +51 -52
  65. snowflake/ml/modeling/covariance/shrunk_covariance.py +51 -52
  66. snowflake/ml/modeling/decomposition/dictionary_learning.py +53 -52
  67. snowflake/ml/modeling/decomposition/factor_analysis.py +53 -52
  68. snowflake/ml/modeling/decomposition/fast_ica.py +53 -52
  69. snowflake/ml/modeling/decomposition/incremental_pca.py +53 -52
  70. snowflake/ml/modeling/decomposition/kernel_pca.py +53 -52
  71. snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +53 -52
  72. snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +53 -52
  73. snowflake/ml/modeling/decomposition/pca.py +53 -52
  74. snowflake/ml/modeling/decomposition/sparse_pca.py +53 -52
  75. snowflake/ml/modeling/decomposition/truncated_svd.py +53 -52
  76. snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +53 -52
  77. snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +51 -52
  78. snowflake/ml/modeling/ensemble/ada_boost_classifier.py +51 -52
  79. snowflake/ml/modeling/ensemble/ada_boost_regressor.py +51 -52
  80. snowflake/ml/modeling/ensemble/bagging_classifier.py +51 -52
  81. snowflake/ml/modeling/ensemble/bagging_regressor.py +51 -52
  82. snowflake/ml/modeling/ensemble/extra_trees_classifier.py +51 -52
  83. snowflake/ml/modeling/ensemble/extra_trees_regressor.py +51 -52
  84. snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +51 -52
  85. snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +51 -52
  86. snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +51 -52
  87. snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +51 -52
  88. snowflake/ml/modeling/ensemble/isolation_forest.py +51 -52
  89. snowflake/ml/modeling/ensemble/random_forest_classifier.py +51 -52
  90. snowflake/ml/modeling/ensemble/random_forest_regressor.py +51 -52
  91. snowflake/ml/modeling/ensemble/stacking_regressor.py +53 -52
  92. snowflake/ml/modeling/ensemble/voting_classifier.py +53 -52
  93. snowflake/ml/modeling/ensemble/voting_regressor.py +53 -52
  94. snowflake/ml/modeling/feature_selection/generic_univariate_select.py +53 -52
  95. snowflake/ml/modeling/feature_selection/select_fdr.py +53 -52
  96. snowflake/ml/modeling/feature_selection/select_fpr.py +53 -52
  97. snowflake/ml/modeling/feature_selection/select_fwe.py +53 -52
  98. snowflake/ml/modeling/feature_selection/select_k_best.py +53 -52
  99. snowflake/ml/modeling/feature_selection/select_percentile.py +53 -52
  100. snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +53 -52
  101. snowflake/ml/modeling/feature_selection/variance_threshold.py +53 -52
  102. snowflake/ml/modeling/framework/base.py +63 -36
  103. snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +51 -52
  104. snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +51 -52
  105. snowflake/ml/modeling/impute/iterative_imputer.py +53 -52
  106. snowflake/ml/modeling/impute/knn_imputer.py +53 -52
  107. snowflake/ml/modeling/impute/missing_indicator.py +53 -52
  108. snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +53 -52
  109. snowflake/ml/modeling/kernel_approximation/nystroem.py +53 -52
  110. snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +53 -52
  111. snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +53 -52
  112. snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +53 -52
  113. snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +51 -52
  114. snowflake/ml/modeling/lightgbm/lgbm_classifier.py +51 -52
  115. snowflake/ml/modeling/lightgbm/lgbm_regressor.py +51 -52
  116. snowflake/ml/modeling/linear_model/ard_regression.py +51 -52
  117. snowflake/ml/modeling/linear_model/bayesian_ridge.py +51 -52
  118. snowflake/ml/modeling/linear_model/elastic_net.py +51 -52
  119. snowflake/ml/modeling/linear_model/elastic_net_cv.py +51 -52
  120. snowflake/ml/modeling/linear_model/gamma_regressor.py +51 -52
  121. snowflake/ml/modeling/linear_model/huber_regressor.py +51 -52
  122. snowflake/ml/modeling/linear_model/lars.py +51 -52
  123. snowflake/ml/modeling/linear_model/lars_cv.py +51 -52
  124. snowflake/ml/modeling/linear_model/lasso.py +51 -52
  125. snowflake/ml/modeling/linear_model/lasso_cv.py +51 -52
  126. snowflake/ml/modeling/linear_model/lasso_lars.py +51 -52
  127. snowflake/ml/modeling/linear_model/lasso_lars_cv.py +51 -52
  128. snowflake/ml/modeling/linear_model/lasso_lars_ic.py +51 -52
  129. snowflake/ml/modeling/linear_model/linear_regression.py +51 -52
  130. snowflake/ml/modeling/linear_model/logistic_regression.py +51 -52
  131. snowflake/ml/modeling/linear_model/logistic_regression_cv.py +51 -52
  132. snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +51 -52
  133. snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +51 -52
  134. snowflake/ml/modeling/linear_model/multi_task_lasso.py +51 -52
  135. snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +51 -52
  136. snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +51 -52
  137. snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +51 -52
  138. snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +51 -52
  139. snowflake/ml/modeling/linear_model/perceptron.py +51 -52
  140. snowflake/ml/modeling/linear_model/poisson_regressor.py +51 -52
  141. snowflake/ml/modeling/linear_model/ransac_regressor.py +51 -52
  142. snowflake/ml/modeling/linear_model/ridge.py +51 -52
  143. snowflake/ml/modeling/linear_model/ridge_classifier.py +51 -52
  144. snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +51 -52
  145. snowflake/ml/modeling/linear_model/ridge_cv.py +51 -52
  146. snowflake/ml/modeling/linear_model/sgd_classifier.py +51 -52
  147. snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +51 -52
  148. snowflake/ml/modeling/linear_model/sgd_regressor.py +51 -52
  149. snowflake/ml/modeling/linear_model/theil_sen_regressor.py +51 -52
  150. snowflake/ml/modeling/linear_model/tweedie_regressor.py +51 -52
  151. snowflake/ml/modeling/manifold/isomap.py +53 -52
  152. snowflake/ml/modeling/manifold/mds.py +53 -52
  153. snowflake/ml/modeling/manifold/spectral_embedding.py +53 -52
  154. snowflake/ml/modeling/manifold/tsne.py +53 -52
  155. snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +51 -52
  156. snowflake/ml/modeling/mixture/gaussian_mixture.py +51 -52
  157. snowflake/ml/modeling/model_selection/grid_search_cv.py +21 -23
  158. snowflake/ml/modeling/model_selection/randomized_search_cv.py +38 -20
  159. snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +51 -52
  160. snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +51 -52
  161. snowflake/ml/modeling/multiclass/output_code_classifier.py +51 -52
  162. snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +51 -52
  163. snowflake/ml/modeling/naive_bayes/categorical_nb.py +51 -52
  164. snowflake/ml/modeling/naive_bayes/complement_nb.py +51 -52
  165. snowflake/ml/modeling/naive_bayes/gaussian_nb.py +51 -52
  166. snowflake/ml/modeling/naive_bayes/multinomial_nb.py +51 -52
  167. snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +51 -52
  168. snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +51 -52
  169. snowflake/ml/modeling/neighbors/kernel_density.py +51 -52
  170. snowflake/ml/modeling/neighbors/local_outlier_factor.py +51 -52
  171. snowflake/ml/modeling/neighbors/nearest_centroid.py +51 -52
  172. snowflake/ml/modeling/neighbors/nearest_neighbors.py +51 -52
  173. snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +53 -52
  174. snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +51 -52
  175. snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +51 -52
  176. snowflake/ml/modeling/neural_network/bernoulli_rbm.py +53 -52
  177. snowflake/ml/modeling/neural_network/mlp_classifier.py +51 -52
  178. snowflake/ml/modeling/neural_network/mlp_regressor.py +51 -52
  179. snowflake/ml/modeling/pipeline/pipeline.py +514 -32
  180. snowflake/ml/modeling/preprocessing/one_hot_encoder.py +12 -0
  181. snowflake/ml/modeling/preprocessing/polynomial_features.py +53 -52
  182. snowflake/ml/modeling/semi_supervised/label_propagation.py +51 -52
  183. snowflake/ml/modeling/semi_supervised/label_spreading.py +51 -52
  184. snowflake/ml/modeling/svm/linear_svc.py +51 -52
  185. snowflake/ml/modeling/svm/linear_svr.py +51 -52
  186. snowflake/ml/modeling/svm/nu_svc.py +51 -52
  187. snowflake/ml/modeling/svm/nu_svr.py +51 -52
  188. snowflake/ml/modeling/svm/svc.py +51 -52
  189. snowflake/ml/modeling/svm/svr.py +51 -52
  190. snowflake/ml/modeling/tree/decision_tree_classifier.py +51 -52
  191. snowflake/ml/modeling/tree/decision_tree_regressor.py +51 -52
  192. snowflake/ml/modeling/tree/extra_tree_classifier.py +51 -52
  193. snowflake/ml/modeling/tree/extra_tree_regressor.py +51 -52
  194. snowflake/ml/modeling/xgboost/xgb_classifier.py +51 -52
  195. snowflake/ml/modeling/xgboost/xgb_regressor.py +51 -52
  196. snowflake/ml/modeling/xgboost/xgbrf_classifier.py +51 -52
  197. snowflake/ml/modeling/xgboost/xgbrf_regressor.py +51 -52
  198. snowflake/ml/registry/model_registry.py +3 -149
  199. snowflake/ml/version.py +1 -1
  200. {snowflake_ml_python-1.4.1.dist-info → snowflake_ml_python-1.5.0.dist-info}/METADATA +63 -2
  201. {snowflake_ml_python-1.4.1.dist-info → snowflake_ml_python-1.5.0.dist-info}/RECORD +204 -196
  202. snowflake/ml/registry/_artifact_manager.py +0 -156
  203. snowflake/ml/registry/artifact.py +0 -46
  204. {snowflake_ml_python-1.4.1.dist-info → snowflake_ml_python-1.5.0.dist-info}/LICENSE.txt +0 -0
  205. {snowflake_ml_python-1.4.1.dist-info → snowflake_ml_python-1.5.0.dist-info}/WHEEL +0 -0
  206. {snowflake_ml_python-1.4.1.dist-info → snowflake_ml_python-1.5.0.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,9 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import json
4
+ import re
4
5
  from collections import OrderedDict
5
- from dataclasses import dataclass
6
+ from dataclasses import asdict, dataclass
6
7
  from enum import Enum
7
8
  from typing import Dict, List, Optional
8
9
 
@@ -28,19 +29,42 @@ from snowflake.snowpark.types import (
28
29
  _FEATURE_VIEW_NAME_DELIMITER = "$"
29
30
  _TIMESTAMP_COL_PLACEHOLDER = "FS_TIMESTAMP_COL_PLACEHOLDER_VAL"
30
31
  _FEATURE_OBJ_TYPE = "FEATURE_OBJ_TYPE"
32
+ # Feature view version rule is aligned with dataset version rule in SQL.
33
+ _FEATURE_VIEW_VERSION_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.\-]*$")
34
+ _FEATURE_VIEW_VERSION_MAX_LENGTH = 128
31
35
 
32
36
 
33
- class FeatureViewVersion(SqlIdentifier):
37
+ @dataclass(frozen=True)
38
+ class _FeatureViewMetadata:
39
+ """Represent metadata tracked on top of FV backend object"""
40
+
41
+ entities: List[str]
42
+ timestamp_col: str
43
+
44
+ def to_json(self) -> str:
45
+ return json.dumps(asdict(self))
46
+
47
+ @classmethod
48
+ def from_json(cls, json_str: str) -> _FeatureViewMetadata:
49
+ state_dict = json.loads(json_str)
50
+ return cls(**state_dict)
51
+
52
+
53
+ class FeatureViewVersion(str):
34
54
  def __new__(cls, version: str) -> FeatureViewVersion:
35
- if _FEATURE_VIEW_NAME_DELIMITER in version:
55
+ if not _FEATURE_VIEW_VERSION_RE.match(version) or len(version) > _FEATURE_VIEW_VERSION_MAX_LENGTH:
36
56
  raise snowml_exceptions.SnowflakeMLException(
37
57
  error_code=error_codes.INVALID_ARGUMENT,
38
- original_exception=ValueError(f"{_FEATURE_VIEW_NAME_DELIMITER} is not allowed in version: {version}."),
58
+ original_exception=ValueError(
59
+ f"`{version}` is not a valid feature view version. "
60
+ "It must start with letter or digit, and followed by letter, digit, '_', '-' or '.'. "
61
+ f"The length limit is {_FEATURE_VIEW_VERSION_MAX_LENGTH}."
62
+ ),
39
63
  )
40
- return super().__new__(cls, version) # type: ignore[return-value]
64
+ return super().__new__(cls, version)
41
65
 
42
66
  def __init__(self, version: str) -> None:
43
- super().__init__(version)
67
+ super().__init__()
44
68
 
45
69
 
46
70
  class FeatureViewStatus(Enum):
@@ -285,6 +309,11 @@ class FeatureView:
285
309
  def owner(self) -> Optional[str]:
286
310
  return self._owner
287
311
 
312
+ def _metadata(self) -> _FeatureViewMetadata:
313
+ entity_names = [e.name.identifier() for e in self.entities]
314
+ ts_col = self.timestamp_col.identifier() if self.timestamp_col is not None else _TIMESTAMP_COL_PLACEHOLDER
315
+ return _FeatureViewMetadata(entity_names, ts_col)
316
+
288
317
  def _get_query(self) -> str:
289
318
  if len(self._feature_df.queries["queries"]) != 1:
290
319
  raise ValueError(
@@ -436,8 +465,8 @@ Got {len(self._feature_df.queries['queries'])}: {self._feature_df.queries['queri
436
465
  status: FeatureViewStatus,
437
466
  feature_descs: Dict[str, str],
438
467
  refresh_freq: Optional[str],
439
- database: Optional[str],
440
- schema: Optional[str],
468
+ database: str,
469
+ schema: str,
441
470
  warehouse: Optional[str],
442
471
  refresh_mode: Optional[str],
443
472
  refresh_mode_reason: Optional[str],
@@ -0,0 +1,146 @@
1
+ import re
2
+ from collections import defaultdict
3
+ from typing import Any, List, Optional, Tuple
4
+
5
+ from snowflake import snowpark
6
+ from snowflake.connector import connection
7
+ from snowflake.ml._internal import telemetry
8
+ from snowflake.ml._internal.exceptions import (
9
+ error_codes,
10
+ exceptions as snowml_exceptions,
11
+ fileset_errors,
12
+ )
13
+ from snowflake.ml._internal.utils import identifier
14
+ from snowflake.snowpark import exceptions as snowpark_exceptions
15
+
16
+ from . import stage_fs
17
+
18
+ _SNOWURL_PATH_RE = re.compile(r"versions/(?P<version>[^/]+)(?:/+(?P<filepath>.*))?")
19
+
20
+
21
+ class SFEmbeddedStageFileSystem(stage_fs.SFStageFileSystem):
22
+ def __init__(
23
+ self,
24
+ *,
25
+ domain: str,
26
+ name: str,
27
+ snowpark_session: Optional[snowpark.Session] = None,
28
+ sf_connection: Optional[connection.SnowflakeConnection] = None,
29
+ **kwargs: Any,
30
+ ) -> None:
31
+
32
+ (db, schema, object_name, _) = identifier.parse_schema_level_object_identifier(name)
33
+ self._name = name # TODO: Require or resolve FQN
34
+ self._domain = domain
35
+
36
+ super().__init__(
37
+ db=db,
38
+ schema=schema,
39
+ stage=object_name,
40
+ snowpark_session=snowpark_session,
41
+ sf_connection=sf_connection,
42
+ **kwargs,
43
+ )
44
+
45
+ @property
46
+ def stage_name(self) -> str:
47
+ """Get the Snowflake path to this stage.
48
+
49
+ Returns:
50
+ A string in the format of snow://<domain>/<name>
51
+ Example: snow://dataset/my_dataset
52
+
53
+ # noqa: DAR203
54
+ """
55
+ return f"snow://{self._domain}/{self._name}"
56
+
57
+ def _stage_path_to_relative_path(self, stage_path: str) -> str:
58
+ """Convert a stage file path which comes from the LIST query to a relative file path in that stage.
59
+
60
+ The file path returned by LIST query always has the format "versions/<version>/<relative_file_path>".
61
+ The full "versions/<version>/<relative_file_path>" is returned
62
+
63
+ Args:
64
+ stage_path: A string started with the name of the stage.
65
+
66
+ Returns:
67
+ A string of the relative stage path.
68
+ """
69
+ return stage_path
70
+
71
+ def _fetch_presigned_urls(
72
+ self, files: List[str], url_lifetime: float = stage_fs._PRESIGNED_URL_LIFETIME_SEC
73
+ ) -> List[Tuple[str, str]]:
74
+ """Fetch presigned urls for the given files."""
75
+ # SnowURL requires full snow://<domain>/<entity>/versions/<version> as the stage path arg to get_presigned_Url
76
+ versions_dict = defaultdict(list)
77
+ for file in files:
78
+ match = _SNOWURL_PATH_RE.fullmatch(file)
79
+ assert match is not None and match.group("filepath") is not None
80
+ versions_dict[match.group("version")].append(match.group("filepath"))
81
+ presigned_urls: List[Tuple[str, str]] = []
82
+ try:
83
+ for version, version_files in versions_dict.items():
84
+ for file in version_files:
85
+ stage_loc = f"{self.stage_name}/versions/{version}"
86
+ presigned_urls.extend(
87
+ self._session.sql(
88
+ f"select '{version}/{file}' as name,"
89
+ f" get_presigned_url('{stage_loc}', '{file}', {url_lifetime}) as url"
90
+ ).collect(
91
+ statement_params=telemetry.get_function_usage_statement_params(
92
+ project=stage_fs._PROJECT,
93
+ api_calls=[snowpark.DataFrame.collect],
94
+ ),
95
+ )
96
+ )
97
+ except snowpark_exceptions.SnowparkClientException as e:
98
+ if e.message.startswith(fileset_errors.ERRNO_DOMAIN_NOT_EXIST) or e.message.startswith(
99
+ fileset_errors.ERRNO_STAGE_NOT_EXIST
100
+ ):
101
+ raise snowml_exceptions.SnowflakeMLException(
102
+ error_code=error_codes.SNOWML_NOT_FOUND,
103
+ original_exception=fileset_errors.StageNotFoundError(
104
+ f"Stage {self.stage_name} does not exist or is not authorized."
105
+ ),
106
+ )
107
+ else:
108
+ raise snowml_exceptions.SnowflakeMLException(
109
+ error_code=error_codes.INTERNAL_SNOWML_ERROR,
110
+ original_exception=fileset_errors.FileSetError(str(e)),
111
+ )
112
+ return presigned_urls
113
+
114
+ @classmethod
115
+ def _parent(cls, path: str) -> str:
116
+ """Get parent of specified path up to minimally valid root path.
117
+
118
+ For SnowURL, the minimum valid path is snow://<domain>/<entity>/versions/<version>
119
+
120
+ Args:
121
+ path: File or directory path
122
+
123
+ Returns:
124
+ Parent path
125
+
126
+ Examples:
127
+ ----
128
+ >>> fs._parent("snow://dataset/my_ds/versions/my_version/file.ext")
129
+ "snow://dataset/my_ds/versions/my_version/"
130
+ >>> fs._parent("snow://dataset/my_ds/versions/my_version/subdir/file.ext")
131
+ "snow://dataset/my_ds/versions/my_version/subdir/"
132
+ >>> fs._parent("snow://dataset/my_ds/versions/my_version/")
133
+ "snow://dataset/my_ds/versions/my_version/"
134
+ >>> fs._parent("snow://dataset/my_ds/versions/my_version")
135
+ "snow://dataset/my_ds/versions/my_version"
136
+ """
137
+ path_match = _SNOWURL_PATH_RE.fullmatch(path)
138
+ if not path_match:
139
+ return super()._parent(path) # type: ignore[no-any-return]
140
+ filepath: str = path_match.group("filepath") or ""
141
+ root: str = path[: path_match.start("filepath")] if filepath else path
142
+ if "/" in filepath:
143
+ parent = filepath.rsplit("/", 1)[0]
144
+ return root + parent
145
+ else:
146
+ return root
@@ -185,7 +185,6 @@ class SFFileSystem(fsspec.AbstractFileSystem):
185
185
  func_params_to_log=["detail"],
186
186
  conn_attr_name="_conn",
187
187
  )
188
- @snowpark._internal.utils.private_preview(version="0.2.0")
189
188
  def ls(self, path: str, detail: bool = False, **kwargs: Any) -> Union[List[str], List[Dict[str, Any]]]:
190
189
  """Override fsspec `ls` method. List single "directory" with or without details.
191
190
 
@@ -216,7 +215,6 @@ class SFFileSystem(fsspec.AbstractFileSystem):
216
215
  project=_PROJECT,
217
216
  conn_attr_name="_conn",
218
217
  )
219
- @snowpark._internal.utils.private_preview(version="0.2.0")
220
218
  def optimize_read(self, files: Optional[List[str]] = None) -> None:
221
219
  """Prefetch and cache the presigned urls for all the given files to speed up the file opening.
222
220
 
@@ -242,7 +240,6 @@ class SFFileSystem(fsspec.AbstractFileSystem):
242
240
  project=_PROJECT,
243
241
  conn_attr_name="_conn",
244
242
  )
245
- @snowpark._internal.utils.private_preview(version="0.2.0")
246
243
  def _open(self, path: str, **kwargs: Any) -> fsspec.spec.AbstractBufferedFile:
247
244
  """Override fsspec `_open` method. Open a file for reading in 'rb' mode.
248
245
 
@@ -268,7 +265,6 @@ class SFFileSystem(fsspec.AbstractFileSystem):
268
265
  project=_PROJECT,
269
266
  conn_attr_name="_conn",
270
267
  )
271
- @snowpark._internal.utils.private_preview(version="0.2.0")
272
268
  def info(self, path: str, **kwargs: Any) -> Dict[str, Any]:
273
269
  """Override fsspec `info` method. Give details of entry at path."""
274
270
  file_path = self._parse_file_path(path)
@@ -0,0 +1,159 @@
1
+ import collections
2
+ import logging
3
+ import re
4
+ from typing import Any, Dict, Optional
5
+
6
+ import fsspec
7
+ import packaging.version as pkg_version
8
+
9
+ from snowflake import snowpark
10
+ from snowflake.connector import connection
11
+ from snowflake.ml._internal.exceptions import (
12
+ error_codes,
13
+ exceptions as snowml_exceptions,
14
+ )
15
+ from snowflake.ml._internal.utils import identifier, snowflake_env
16
+ from snowflake.ml.fileset import embedded_stage_fs, sfcfs
17
+
18
+ PROTOCOL_NAME = "snow"
19
+
20
+ _SFFileEntityPath = collections.namedtuple(
21
+ "_SFFileEntityPath", ["domain", "name", "filepath", "version", "relative_path"]
22
+ )
23
+ _PROJECT = "FileSet"
24
+ _SNOWURL_PATTERN = re.compile(
25
+ f"({PROTOCOL_NAME}://)?"
26
+ r"(?<!@)(?P<domain>\w+)/"
27
+ rf"(?P<name>(?:{identifier._SF_IDENTIFIER}\.){{,2}}{identifier._SF_IDENTIFIER})/"
28
+ r"(?P<path>versions/(?:(?P<version>[^/]+)(?:/(?P<relpath>.*))?)?)"
29
+ )
30
+
31
+ # FIXME(dhung): Temporary fix for bug in GS version 8.17
32
+ _BUG_VERSION_MIN = pkg_version.Version("8.17") # Inclusive minimum version with bugged behavior
33
+ _BUG_VERSION_MAX = pkg_version.Version("8.18") # Exclusive maximum version with bugged behavior
34
+
35
+
36
+ class SnowFileSystem(sfcfs.SFFileSystem):
37
+ """A filesystem that allows user to access Snowflake embedded stage files with valid Snowflake locations.
38
+
39
+ The file system is is based on fsspec (https://filesystem-spec.readthedocs.io/). It is a file system wrapper
40
+ built on top of SFStageFileSystem. It takes Snowflake embedded stage path as the input and supports read operation.
41
+ A valid Snowflake location will have the form "snow://{domain}/{entity_name}/versions/{version}/{path_to_file}".
42
+
43
+ See `sfcfs.SFFileSystem` documentation for example usage patterns.
44
+ """
45
+
46
+ protocol = PROTOCOL_NAME
47
+ _IS_BUGGED_VERSION = None
48
+
49
+ def __init__(
50
+ self,
51
+ sf_connection: Optional[connection.SnowflakeConnection] = None,
52
+ snowpark_session: Optional[snowpark.Session] = None,
53
+ **kwargs: Any,
54
+ ) -> None:
55
+ super().__init__(sf_connection=sf_connection, snowpark_session=snowpark_session, **kwargs)
56
+
57
+ # FIXME(dhung): Temporary fix for bug in GS version 8.17
58
+ if SnowFileSystem._IS_BUGGED_VERSION is None:
59
+ try:
60
+ sf_version = snowflake_env.get_current_snowflake_version(self._session)
61
+ SnowFileSystem._IS_BUGGED_VERSION = _BUG_VERSION_MIN <= sf_version < _BUG_VERSION_MAX
62
+ except Exception:
63
+ SnowFileSystem._IS_BUGGED_VERSION = False
64
+
65
+ def info(self, path: str, **kwargs: Any) -> Dict[str, Any]:
66
+ # FIXME(dhung): Temporary fix for bug in GS version 8.17
67
+ res: Dict[str, Any] = super().info(path, **kwargs)
68
+ if res.get("type") == "directory" and not res["name"].endswith("/"):
69
+ res["name"] += "/"
70
+ return res
71
+
72
+ def _get_stage_fs(
73
+ self, sf_file_path: _SFFileEntityPath # type: ignore[override]
74
+ ) -> embedded_stage_fs.SFEmbeddedStageFileSystem:
75
+ """Get the stage file system for the given snowflake location.
76
+
77
+ Args:
78
+ sf_file_path: The Snowflake path information.
79
+
80
+ Returns:
81
+ A SFEmbeddedStageFileSystem object which supports readonly file operations on Snowflake embedded stages.
82
+ """
83
+ stage_fs_key = (sf_file_path.domain, sf_file_path.name, sf_file_path.version)
84
+ if stage_fs_key not in self._stage_fs_set:
85
+ cnt_stage_fs = embedded_stage_fs.SFEmbeddedStageFileSystem(
86
+ snowpark_session=self._session,
87
+ domain=sf_file_path.domain,
88
+ name=sf_file_path.name,
89
+ **self._kwargs,
90
+ )
91
+ self._stage_fs_set[stage_fs_key] = cnt_stage_fs
92
+ return self._stage_fs_set[stage_fs_key]
93
+
94
+ def _stage_path_to_absolute_path(self, stage_fs: embedded_stage_fs.SFEmbeddedStageFileSystem, path: str) -> str:
95
+ """Convert the relative path in a stage to an absolute path starts with the location of the stage."""
96
+ # Strip protocol from absolute path, since backend needs snow:// prefix to resolve correctly
97
+ # but fsspec logic strips protocol when doing any searching and globbing
98
+ stage_name = stage_fs.stage_name
99
+ protocol = f"{PROTOCOL_NAME}://"
100
+ if stage_name.startswith(protocol):
101
+ stage_name = stage_name[len(protocol) :]
102
+ abs_path = stage_name + "/" + path
103
+ # FIXME(dhung): Temporary fix for bug in GS version 8.17
104
+ if self._IS_BUGGED_VERSION:
105
+ match = _SNOWURL_PATTERN.fullmatch(abs_path)
106
+ assert match is not None
107
+ abs_path = abs_path.replace(match.group("relpath"), match.group("relpath").lstrip("/"))
108
+ return abs_path
109
+
110
+ @classmethod
111
+ def _parse_file_path(cls, path: str) -> _SFFileEntityPath: # type: ignore[override]
112
+ """Parse a snowflake location path.
113
+
114
+ The following properties will be extracted from the path input:
115
+ - embedded stage domain
116
+ - entity name
117
+ - path (in format `versions/{version}/{relative_path}`)
118
+ - entity version (optional)
119
+ - relative file path (optional)
120
+
121
+ Args:
122
+ path: A string in the format of "snow://{domain}/{entity_name}/versions/{version}/{path_to_file}".
123
+
124
+ Returns:
125
+ A namedtuple consists of domain, entity name, filepath, version, and relative path, where
126
+ filepath = "versions/{version}/{relative_path}"
127
+
128
+ Raises:
129
+ SnowflakeMLException: An error occurred when invalid path is given.
130
+ """
131
+ snowurl_match = _SNOWURL_PATTERN.fullmatch(path)
132
+ if not snowurl_match:
133
+ raise snowml_exceptions.SnowflakeMLException(
134
+ error_code=error_codes.SNOWML_INVALID_STAGE,
135
+ original_exception=ValueError(f"Invalid Snow URL: {path}"),
136
+ )
137
+
138
+ try:
139
+ domain = snowurl_match.group("domain")
140
+ parsed_name = identifier.parse_schema_level_object_identifier(snowurl_match.group("name"))
141
+ name = identifier.get_schema_level_object_identifier(*parsed_name)
142
+ filepath = snowurl_match.group("path")
143
+ version = snowurl_match.group("version")
144
+ relative_path = snowurl_match.group("relpath") or ""
145
+ logging.debug(f"Parsed snow URL: {snowurl_match.groups()}")
146
+ # FIXME(dhung): Temporary fix for bug in GS version 8.17
147
+ if cls._IS_BUGGED_VERSION:
148
+ filepath = filepath.replace(f"{version}/", f"{version}//")
149
+ return _SFFileEntityPath(
150
+ domain=domain, name=name, version=version, relative_path=relative_path, filepath=filepath
151
+ )
152
+ except ValueError as e:
153
+ raise snowml_exceptions.SnowflakeMLException(
154
+ error_code=error_codes.SNOWML_INVALID_STAGE,
155
+ original_exception=e,
156
+ )
157
+
158
+
159
+ fsspec.register_implementation(PROTOCOL_NAME, SnowFileSystem)
@@ -144,7 +144,6 @@ class SFStageFileSystem(fsspec.AbstractFileSystem):
144
144
  project=_PROJECT,
145
145
  func_params_to_log=["detail"],
146
146
  )
147
- @snowpark._internal.utils.private_preview(version="0.2.0")
148
147
  def ls(self, path: str, detail: bool = False) -> Union[List[str], List[Dict[str, Any]]]:
149
148
  """Override fsspec `ls` method. List single "directory" with or without details.
150
149
 
@@ -168,7 +167,7 @@ class SFStageFileSystem(fsspec.AbstractFileSystem):
168
167
  try:
169
168
  loc = self.stage_name
170
169
  path = path.lstrip("/")
171
- objects = self._session.sql(f"LIST {loc}/{path}").collect()
170
+ objects = self._session.sql(f"LIST '{loc}/{path}'").collect()
172
171
  except snowpark_exceptions.SnowparkClientException as e:
173
172
  if e.message.startswith(fileset_errors.ERRNO_DOMAIN_NOT_EXIST):
174
173
  raise snowml_exceptions.SnowflakeMLException(
@@ -191,7 +190,6 @@ class SFStageFileSystem(fsspec.AbstractFileSystem):
191
190
  @telemetry.send_api_usage_telemetry(
192
191
  project=_PROJECT,
193
192
  )
194
- @snowpark._internal.utils.private_preview(version="0.2.0")
195
193
  def optimize_read(self, files: Optional[List[str]] = None) -> None:
196
194
  """Prefetch and cache the presigned urls for all the given files to speed up the read performance.
197
195
 
@@ -218,7 +216,6 @@ class SFStageFileSystem(fsspec.AbstractFileSystem):
218
216
  @telemetry.send_api_usage_telemetry(
219
217
  project=_PROJECT,
220
218
  )
221
- @snowpark._internal.utils.private_preview(version="0.2.0")
222
219
  def _open(self, path: str, mode: str = "rb", **kwargs: Any) -> fsspec.spec.AbstractBufferedFile:
223
220
  """Override fsspec `_open` method. Open a file for reading.
224
221
 
@@ -1,6 +1,6 @@
1
1
  from snowflake.ml.model._client.model.model_impl import Model
2
- from snowflake.ml.model._client.model.model_version_impl import ModelVersion
2
+ from snowflake.ml.model._client.model.model_version_impl import ExportMode, ModelVersion
3
3
  from snowflake.ml.model.models.huggingface_pipeline import HuggingFacePipelineModel
4
4
  from snowflake.ml.model.models.llm import LLM, LLMOptions
5
5
 
6
- __all__ = ["Model", "ModelVersion", "HuggingFacePipelineModel", "LLM", "LLMOptions"]
6
+ __all__ = ["Model", "ModelVersion", "ExportMode", "HuggingFacePipelineModel", "LLM", "LLMOptions"]
@@ -2,6 +2,7 @@ from types import ModuleType
2
2
  from typing import Any, Dict, List, Literal, Optional, Union, cast, overload
3
3
 
4
4
  import pandas as pd
5
+ from typing_extensions import deprecated
5
6
 
6
7
  from snowflake.ml._internal.exceptions import (
7
8
  error_codes,
@@ -23,6 +24,7 @@ from snowflake.ml.model._signatures import snowpark_handler
23
24
  from snowflake.snowpark import DataFrame as SnowparkDataFrame, Session, functions as F
24
25
 
25
26
 
27
+ @deprecated("Only used by PrPr model registry.")
26
28
  @overload
27
29
  def save_model(
28
30
  *,
@@ -61,6 +63,7 @@ def save_model(
61
63
  ...
62
64
 
63
65
 
66
+ @deprecated("Only used by PrPr model registry.")
64
67
  @overload
65
68
  def save_model(
66
69
  *,
@@ -101,6 +104,7 @@ def save_model(
101
104
  ...
102
105
 
103
106
 
107
+ @deprecated("Only used by PrPr model registry.")
104
108
  @overload
105
109
  def save_model(
106
110
  *,
@@ -142,6 +146,7 @@ def save_model(
142
146
  ...
143
147
 
144
148
 
149
+ @deprecated("Only used by PrPr model registry.")
145
150
  def save_model(
146
151
  *,
147
152
  name: str,
@@ -208,6 +213,7 @@ def save_model(
208
213
  return m
209
214
 
210
215
 
216
+ @deprecated("Only used by PrPr model registry.")
211
217
  @overload
212
218
  def load_model(*, session: Session, stage_path: str) -> model_composer.ModelComposer:
213
219
  """Load the model into memory from a zip file in the stage.
@@ -219,6 +225,7 @@ def load_model(*, session: Session, stage_path: str) -> model_composer.ModelComp
219
225
  ...
220
226
 
221
227
 
228
+ @deprecated("Only used by PrPr model registry.")
222
229
  @overload
223
230
  def load_model(*, session: Session, stage_path: str, meta_only: Literal[False]) -> model_composer.ModelComposer:
224
231
  """Load the model into memory from a zip file in the stage.
@@ -231,6 +238,7 @@ def load_model(*, session: Session, stage_path: str, meta_only: Literal[False])
231
238
  ...
232
239
 
233
240
 
241
+ @deprecated("Only used by PrPr model registry.")
234
242
  @overload
235
243
  def load_model(*, session: Session, stage_path: str, meta_only: Literal[True]) -> model_composer.ModelComposer:
236
244
  """Load the model into memory from a zip file in the stage with metadata only.
@@ -243,6 +251,7 @@ def load_model(*, session: Session, stage_path: str, meta_only: Literal[True]) -
243
251
  ...
244
252
 
245
253
 
254
+ @deprecated("Only used by PrPr model registry.")
246
255
  def load_model(
247
256
  *,
248
257
  session: Session,
@@ -261,10 +270,11 @@ def load_model(
261
270
  Loaded model.
262
271
  """
263
272
  m = model_composer.ModelComposer(session=session, stage_path=stage_path)
264
- m.load(meta_only=meta_only)
273
+ m.legacy_load(meta_only=meta_only)
265
274
  return m
266
275
 
267
276
 
277
+ @deprecated("Only used by PrPr model registry.")
268
278
  @overload
269
279
  def deploy(
270
280
  session: Session,
@@ -290,6 +300,7 @@ def deploy(
290
300
  ...
291
301
 
292
302
 
303
+ @deprecated("Only used by PrPr model registry.")
293
304
  @overload
294
305
  def deploy(
295
306
  session: Session,
@@ -319,6 +330,7 @@ def deploy(
319
330
  ...
320
331
 
321
332
 
333
+ @deprecated("Only used by PrPr model registry.")
322
334
  def deploy(
323
335
  session: Session,
324
336
  *,
@@ -423,6 +435,7 @@ def deploy(
423
435
  return info
424
436
 
425
437
 
438
+ @deprecated("Only used by PrPr model registry.")
426
439
  @overload
427
440
  def predict(
428
441
  session: Session,
@@ -443,6 +456,7 @@ def predict(
443
456
  ...
444
457
 
445
458
 
459
+ @deprecated("Only used by PrPr model registry.")
446
460
  @overload
447
461
  def predict(
448
462
  session: Session,
@@ -462,6 +476,7 @@ def predict(
462
476
  ...
463
477
 
464
478
 
479
+ @deprecated("Only used by PrPr model registry.")
465
480
  def predict(
466
481
  session: Session,
467
482
  *,
@@ -350,3 +350,30 @@ class Model:
350
350
  tag_name=tag_name_id,
351
351
  statement_params=statement_params,
352
352
  )
353
+
354
+ @telemetry.send_api_usage_telemetry(
355
+ project=_TELEMETRY_PROJECT,
356
+ subproject=_TELEMETRY_SUBPROJECT,
357
+ )
358
+ def rename(self, model_name: str) -> None:
359
+ """Rename a model. Can be used to move a model when a fully qualified name is provided.
360
+
361
+ Args:
362
+ model_name: The new model name.
363
+ """
364
+ statement_params = telemetry.get_statement_params(
365
+ project=_TELEMETRY_PROJECT,
366
+ subproject=_TELEMETRY_SUBPROJECT,
367
+ )
368
+ db, schema, model, _ = identifier.parse_schema_level_object_identifier(model_name)
369
+ new_model_db = sql_identifier.SqlIdentifier(db) if db else None
370
+ new_model_schema = sql_identifier.SqlIdentifier(schema) if schema else None
371
+ new_model_id = sql_identifier.SqlIdentifier(model)
372
+ self._model_ops.rename(
373
+ model_name=self._model_name,
374
+ new_model_db=new_model_db,
375
+ new_model_schema=new_model_schema,
376
+ new_model_name=new_model_id,
377
+ statement_params=statement_params,
378
+ )
379
+ self._model_name = new_model_id