snowflake-ml-python 1.6.2__py3-none-any.whl → 1.6.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. snowflake/cortex/__init__.py +4 -0
  2. snowflake/cortex/_classify_text.py +2 -2
  3. snowflake/cortex/_embed_text_1024.py +37 -0
  4. snowflake/cortex/_embed_text_768.py +37 -0
  5. snowflake/cortex/_extract_answer.py +2 -2
  6. snowflake/cortex/_sentiment.py +2 -2
  7. snowflake/cortex/_summarize.py +2 -2
  8. snowflake/cortex/_translate.py +2 -2
  9. snowflake/cortex/_util.py +4 -4
  10. snowflake/ml/_internal/env_utils.py +5 -5
  11. snowflake/ml/_internal/exceptions/error_codes.py +2 -0
  12. snowflake/ml/_internal/utils/db_utils.py +50 -0
  13. snowflake/ml/_internal/utils/service_logger.py +63 -0
  14. snowflake/ml/_internal/utils/sql_identifier.py +25 -1
  15. snowflake/ml/data/_internal/arrow_ingestor.py +1 -11
  16. snowflake/ml/data/ingestor_utils.py +20 -10
  17. snowflake/ml/feature_store/access_manager.py +3 -3
  18. snowflake/ml/feature_store/feature_store.py +19 -2
  19. snowflake/ml/feature_store/feature_view.py +82 -28
  20. snowflake/ml/fileset/stage_fs.py +2 -1
  21. snowflake/ml/lineage/lineage_node.py +7 -2
  22. snowflake/ml/model/__init__.py +1 -2
  23. snowflake/ml/model/_client/model/model_version_impl.py +78 -9
  24. snowflake/ml/model/_client/ops/model_ops.py +89 -7
  25. snowflake/ml/model/_client/ops/service_ops.py +200 -91
  26. snowflake/ml/model/_client/service/model_deployment_spec.py +4 -0
  27. snowflake/ml/model/_client/service/model_deployment_spec_schema.py +1 -0
  28. snowflake/ml/model/_client/sql/_base.py +5 -0
  29. snowflake/ml/model/_client/sql/model.py +1 -0
  30. snowflake/ml/model/_client/sql/model_version.py +9 -5
  31. snowflake/ml/model/_client/sql/service.py +35 -13
  32. snowflake/ml/model/_model_composer/model_composer.py +11 -41
  33. snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +29 -4
  34. snowflake/ml/model/_packager/model_env/model_env.py +4 -38
  35. snowflake/ml/model/_packager/model_handlers/_utils.py +106 -32
  36. snowflake/ml/model/_packager/model_handlers/catboost.py +26 -27
  37. snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py +3 -3
  38. snowflake/ml/model/_packager/model_handlers/lightgbm.py +21 -6
  39. snowflake/ml/model/_packager/model_handlers/mlflow.py +3 -5
  40. snowflake/ml/model/_packager/model_handlers/model_objective_utils.py +111 -58
  41. snowflake/ml/model/_packager/model_handlers/sentence_transformers.py +15 -8
  42. snowflake/ml/model/_packager/model_handlers/sklearn.py +50 -66
  43. snowflake/ml/model/_packager/model_handlers/snowmlmodel.py +36 -17
  44. snowflake/ml/model/_packager/model_handlers/xgboost.py +22 -7
  45. snowflake/ml/model/_packager/model_meta/model_meta.py +16 -45
  46. snowflake/ml/model/_packager/model_meta/model_meta_schema.py +1 -6
  47. snowflake/ml/model/_packager/model_packager.py +14 -10
  48. snowflake/ml/model/_packager/model_runtime/model_runtime.py +11 -0
  49. snowflake/ml/model/_signatures/snowpark_handler.py +3 -2
  50. snowflake/ml/model/type_hints.py +11 -152
  51. snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +0 -2
  52. snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +17 -6
  53. snowflake/ml/modeling/_internal/snowpark_implementations/xgboost_external_memory_trainer.py +0 -1
  54. snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +1 -0
  55. snowflake/ml/modeling/cluster/affinity_propagation.py +1 -0
  56. snowflake/ml/modeling/cluster/agglomerative_clustering.py +1 -0
  57. snowflake/ml/modeling/cluster/birch.py +1 -0
  58. snowflake/ml/modeling/cluster/bisecting_k_means.py +1 -0
  59. snowflake/ml/modeling/cluster/dbscan.py +1 -0
  60. snowflake/ml/modeling/cluster/feature_agglomeration.py +1 -0
  61. snowflake/ml/modeling/cluster/k_means.py +1 -0
  62. snowflake/ml/modeling/cluster/mean_shift.py +1 -0
  63. snowflake/ml/modeling/cluster/mini_batch_k_means.py +1 -0
  64. snowflake/ml/modeling/cluster/optics.py +1 -0
  65. snowflake/ml/modeling/cluster/spectral_biclustering.py +1 -0
  66. snowflake/ml/modeling/cluster/spectral_clustering.py +1 -0
  67. snowflake/ml/modeling/cluster/spectral_coclustering.py +1 -0
  68. snowflake/ml/modeling/compose/column_transformer.py +1 -0
  69. snowflake/ml/modeling/compose/transformed_target_regressor.py +1 -0
  70. snowflake/ml/modeling/covariance/elliptic_envelope.py +1 -0
  71. snowflake/ml/modeling/covariance/empirical_covariance.py +1 -0
  72. snowflake/ml/modeling/covariance/graphical_lasso.py +1 -0
  73. snowflake/ml/modeling/covariance/graphical_lasso_cv.py +1 -0
  74. snowflake/ml/modeling/covariance/ledoit_wolf.py +1 -0
  75. snowflake/ml/modeling/covariance/min_cov_det.py +1 -0
  76. snowflake/ml/modeling/covariance/oas.py +1 -0
  77. snowflake/ml/modeling/covariance/shrunk_covariance.py +1 -0
  78. snowflake/ml/modeling/decomposition/dictionary_learning.py +1 -0
  79. snowflake/ml/modeling/decomposition/factor_analysis.py +1 -0
  80. snowflake/ml/modeling/decomposition/fast_ica.py +1 -0
  81. snowflake/ml/modeling/decomposition/incremental_pca.py +1 -0
  82. snowflake/ml/modeling/decomposition/kernel_pca.py +1 -0
  83. snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +1 -0
  84. snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +1 -0
  85. snowflake/ml/modeling/decomposition/pca.py +1 -0
  86. snowflake/ml/modeling/decomposition/sparse_pca.py +1 -0
  87. snowflake/ml/modeling/decomposition/truncated_svd.py +1 -0
  88. snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +1 -0
  89. snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +1 -0
  90. snowflake/ml/modeling/ensemble/ada_boost_classifier.py +1 -0
  91. snowflake/ml/modeling/ensemble/ada_boost_regressor.py +1 -0
  92. snowflake/ml/modeling/ensemble/bagging_classifier.py +1 -0
  93. snowflake/ml/modeling/ensemble/bagging_regressor.py +1 -0
  94. snowflake/ml/modeling/ensemble/extra_trees_classifier.py +1 -0
  95. snowflake/ml/modeling/ensemble/extra_trees_regressor.py +1 -0
  96. snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +1 -0
  97. snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +1 -0
  98. snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +1 -0
  99. snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +1 -0
  100. snowflake/ml/modeling/ensemble/isolation_forest.py +1 -0
  101. snowflake/ml/modeling/ensemble/random_forest_classifier.py +1 -0
  102. snowflake/ml/modeling/ensemble/random_forest_regressor.py +1 -0
  103. snowflake/ml/modeling/ensemble/stacking_regressor.py +1 -0
  104. snowflake/ml/modeling/ensemble/voting_classifier.py +1 -0
  105. snowflake/ml/modeling/ensemble/voting_regressor.py +1 -0
  106. snowflake/ml/modeling/feature_selection/generic_univariate_select.py +1 -0
  107. snowflake/ml/modeling/feature_selection/select_fdr.py +1 -0
  108. snowflake/ml/modeling/feature_selection/select_fpr.py +1 -0
  109. snowflake/ml/modeling/feature_selection/select_fwe.py +1 -0
  110. snowflake/ml/modeling/feature_selection/select_k_best.py +1 -0
  111. snowflake/ml/modeling/feature_selection/select_percentile.py +1 -0
  112. snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +1 -0
  113. snowflake/ml/modeling/feature_selection/variance_threshold.py +1 -0
  114. snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +1 -0
  115. snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +1 -0
  116. snowflake/ml/modeling/impute/iterative_imputer.py +1 -0
  117. snowflake/ml/modeling/impute/knn_imputer.py +1 -0
  118. snowflake/ml/modeling/impute/missing_indicator.py +1 -0
  119. snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +1 -0
  120. snowflake/ml/modeling/kernel_approximation/nystroem.py +1 -0
  121. snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +1 -0
  122. snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +1 -0
  123. snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +1 -0
  124. snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +1 -0
  125. snowflake/ml/modeling/lightgbm/lgbm_classifier.py +1 -0
  126. snowflake/ml/modeling/lightgbm/lgbm_regressor.py +1 -0
  127. snowflake/ml/modeling/linear_model/ard_regression.py +1 -0
  128. snowflake/ml/modeling/linear_model/bayesian_ridge.py +1 -0
  129. snowflake/ml/modeling/linear_model/elastic_net.py +1 -0
  130. snowflake/ml/modeling/linear_model/elastic_net_cv.py +1 -0
  131. snowflake/ml/modeling/linear_model/gamma_regressor.py +1 -0
  132. snowflake/ml/modeling/linear_model/huber_regressor.py +1 -0
  133. snowflake/ml/modeling/linear_model/lars.py +1 -0
  134. snowflake/ml/modeling/linear_model/lars_cv.py +1 -0
  135. snowflake/ml/modeling/linear_model/lasso.py +1 -0
  136. snowflake/ml/modeling/linear_model/lasso_cv.py +1 -0
  137. snowflake/ml/modeling/linear_model/lasso_lars.py +1 -0
  138. snowflake/ml/modeling/linear_model/lasso_lars_cv.py +1 -0
  139. snowflake/ml/modeling/linear_model/lasso_lars_ic.py +1 -0
  140. snowflake/ml/modeling/linear_model/linear_regression.py +1 -0
  141. snowflake/ml/modeling/linear_model/logistic_regression.py +1 -0
  142. snowflake/ml/modeling/linear_model/logistic_regression_cv.py +1 -0
  143. snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +1 -0
  144. snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +1 -0
  145. snowflake/ml/modeling/linear_model/multi_task_lasso.py +1 -0
  146. snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +1 -0
  147. snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +1 -0
  148. snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +1 -0
  149. snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +1 -0
  150. snowflake/ml/modeling/linear_model/perceptron.py +1 -0
  151. snowflake/ml/modeling/linear_model/poisson_regressor.py +1 -0
  152. snowflake/ml/modeling/linear_model/ransac_regressor.py +1 -0
  153. snowflake/ml/modeling/linear_model/ridge.py +1 -0
  154. snowflake/ml/modeling/linear_model/ridge_classifier.py +1 -0
  155. snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +1 -0
  156. snowflake/ml/modeling/linear_model/ridge_cv.py +1 -0
  157. snowflake/ml/modeling/linear_model/sgd_classifier.py +1 -0
  158. snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +1 -0
  159. snowflake/ml/modeling/linear_model/sgd_regressor.py +1 -0
  160. snowflake/ml/modeling/linear_model/theil_sen_regressor.py +1 -0
  161. snowflake/ml/modeling/linear_model/tweedie_regressor.py +1 -0
  162. snowflake/ml/modeling/manifold/isomap.py +1 -0
  163. snowflake/ml/modeling/manifold/mds.py +1 -0
  164. snowflake/ml/modeling/manifold/spectral_embedding.py +1 -0
  165. snowflake/ml/modeling/manifold/tsne.py +1 -0
  166. snowflake/ml/modeling/metrics/metrics_utils.py +2 -2
  167. snowflake/ml/modeling/metrics/ranking.py +0 -3
  168. snowflake/ml/modeling/metrics/regression.py +0 -3
  169. snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +1 -0
  170. snowflake/ml/modeling/mixture/gaussian_mixture.py +1 -0
  171. snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +1 -0
  172. snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +1 -0
  173. snowflake/ml/modeling/multiclass/output_code_classifier.py +1 -0
  174. snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +1 -0
  175. snowflake/ml/modeling/naive_bayes/categorical_nb.py +1 -0
  176. snowflake/ml/modeling/naive_bayes/complement_nb.py +1 -0
  177. snowflake/ml/modeling/naive_bayes/gaussian_nb.py +1 -0
  178. snowflake/ml/modeling/naive_bayes/multinomial_nb.py +1 -0
  179. snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +1 -0
  180. snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +1 -0
  181. snowflake/ml/modeling/neighbors/kernel_density.py +1 -0
  182. snowflake/ml/modeling/neighbors/local_outlier_factor.py +1 -0
  183. snowflake/ml/modeling/neighbors/nearest_centroid.py +1 -0
  184. snowflake/ml/modeling/neighbors/nearest_neighbors.py +1 -0
  185. snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +1 -0
  186. snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +1 -0
  187. snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +1 -0
  188. snowflake/ml/modeling/neural_network/bernoulli_rbm.py +1 -0
  189. snowflake/ml/modeling/neural_network/mlp_classifier.py +1 -0
  190. snowflake/ml/modeling/neural_network/mlp_regressor.py +1 -0
  191. snowflake/ml/modeling/pipeline/pipeline.py +0 -1
  192. snowflake/ml/modeling/preprocessing/polynomial_features.py +1 -0
  193. snowflake/ml/modeling/semi_supervised/label_propagation.py +1 -0
  194. snowflake/ml/modeling/semi_supervised/label_spreading.py +1 -0
  195. snowflake/ml/modeling/svm/linear_svc.py +1 -0
  196. snowflake/ml/modeling/svm/linear_svr.py +1 -0
  197. snowflake/ml/modeling/svm/nu_svc.py +1 -0
  198. snowflake/ml/modeling/svm/nu_svr.py +1 -0
  199. snowflake/ml/modeling/svm/svc.py +1 -0
  200. snowflake/ml/modeling/svm/svr.py +1 -0
  201. snowflake/ml/modeling/tree/decision_tree_classifier.py +1 -0
  202. snowflake/ml/modeling/tree/decision_tree_regressor.py +1 -0
  203. snowflake/ml/modeling/tree/extra_tree_classifier.py +1 -0
  204. snowflake/ml/modeling/tree/extra_tree_regressor.py +1 -0
  205. snowflake/ml/modeling/xgboost/xgb_classifier.py +1 -0
  206. snowflake/ml/modeling/xgboost/xgb_regressor.py +1 -0
  207. snowflake/ml/modeling/xgboost/xgbrf_classifier.py +1 -0
  208. snowflake/ml/modeling/xgboost/xgbrf_regressor.py +1 -0
  209. snowflake/ml/monitoring/_client/model_monitor.py +126 -0
  210. snowflake/ml/monitoring/_client/model_monitor_manager.py +361 -0
  211. snowflake/ml/monitoring/_client/model_monitor_version.py +1 -0
  212. snowflake/ml/monitoring/_client/monitor_sql_client.py +1335 -0
  213. snowflake/ml/monitoring/_client/queries/record_count.ssql +14 -0
  214. snowflake/ml/monitoring/_client/queries/rmse.ssql +28 -0
  215. snowflake/ml/monitoring/entities/model_monitor_config.py +28 -0
  216. snowflake/ml/monitoring/entities/model_monitor_interval.py +46 -0
  217. snowflake/ml/monitoring/entities/output_score_type.py +90 -0
  218. snowflake/ml/registry/_manager/model_manager.py +4 -4
  219. snowflake/ml/registry/registry.py +165 -6
  220. snowflake/ml/version.py +1 -1
  221. {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/METADATA +30 -9
  222. {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/RECORD +225 -249
  223. {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/WHEEL +1 -1
  224. snowflake/ml/_internal/container_services/image_registry/credential.py +0 -84
  225. snowflake/ml/_internal/container_services/image_registry/http_client.py +0 -127
  226. snowflake/ml/_internal/container_services/image_registry/imagelib.py +0 -400
  227. snowflake/ml/_internal/container_services/image_registry/registry_client.py +0 -212
  228. snowflake/ml/_internal/utils/log_stream_processor.py +0 -30
  229. snowflake/ml/_internal/utils/session_token_manager.py +0 -46
  230. snowflake/ml/_internal/utils/spcs_attribution_utils.py +0 -122
  231. snowflake/ml/_internal/utils/uri.py +0 -77
  232. snowflake/ml/model/_api.py +0 -568
  233. snowflake/ml/model/_deploy_client/image_builds/base_image_builder.py +0 -12
  234. snowflake/ml/model/_deploy_client/image_builds/client_image_builder.py +0 -249
  235. snowflake/ml/model/_deploy_client/image_builds/docker_context.py +0 -130
  236. snowflake/ml/model/_deploy_client/image_builds/gunicorn_run.sh +0 -36
  237. snowflake/ml/model/_deploy_client/image_builds/inference_server/main.py +0 -268
  238. snowflake/ml/model/_deploy_client/image_builds/server_image_builder.py +0 -215
  239. snowflake/ml/model/_deploy_client/image_builds/templates/dockerfile_template +0 -53
  240. snowflake/ml/model/_deploy_client/image_builds/templates/image_build_job_spec_template +0 -38
  241. snowflake/ml/model/_deploy_client/image_builds/templates/kaniko_shell_script_template +0 -105
  242. snowflake/ml/model/_deploy_client/snowservice/deploy.py +0 -611
  243. snowflake/ml/model/_deploy_client/snowservice/deploy_options.py +0 -116
  244. snowflake/ml/model/_deploy_client/snowservice/instance_types.py +0 -10
  245. snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template +0 -28
  246. snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template_with_model +0 -21
  247. snowflake/ml/model/_deploy_client/utils/constants.py +0 -48
  248. snowflake/ml/model/_deploy_client/utils/snowservice_client.py +0 -280
  249. snowflake/ml/model/_deploy_client/warehouse/deploy.py +0 -202
  250. snowflake/ml/model/_deploy_client/warehouse/infer_template.py +0 -99
  251. snowflake/ml/model/_packager/model_handlers/llm.py +0 -269
  252. snowflake/ml/model/_packager/model_meta/_core_requirements.py +0 -11
  253. snowflake/ml/model/deploy_platforms.py +0 -6
  254. snowflake/ml/model/models/llm.py +0 -106
  255. snowflake/ml/monitoring/monitor.py +0 -203
  256. snowflake/ml/registry/_initial_schema.py +0 -142
  257. snowflake/ml/registry/_schema.py +0 -82
  258. snowflake/ml/registry/_schema_upgrade_plans.py +0 -116
  259. snowflake/ml/registry/_schema_version_manager.py +0 -163
  260. snowflake/ml/registry/model_registry.py +0 -2048
  261. {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/LICENSE.txt +0 -0
  262. {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/top_level.txt +0 -0
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (74.1.2)
2
+ Generator: setuptools (75.2.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,84 +0,0 @@
1
- # TODO[shchen]: Remove this file and use session_token_manager instead.
2
- import base64
3
- import contextlib
4
- import json
5
- from typing import Generator, TypedDict
6
-
7
- from snowflake import snowpark
8
- from snowflake.ml._internal.utils import query_result_checker
9
-
10
-
11
- class SessionToken(TypedDict):
12
- token: str
13
- expires_in: str
14
-
15
-
16
- @contextlib.contextmanager
17
- def generate_image_registry_credential(session: snowpark.Session) -> Generator[str, None, None]:
18
- """Construct basic auth credential that is specific to SPCS image registry. For image registry authentication, we
19
- will use a session token obtained from the Snowpark session object. The token authentication mechanism is
20
- automatically used when the username is set to "0sessiontoken" according to the registry implementation.
21
-
22
- As a workaround for SNOW-841699: Fail to authenticate to image registry with session token generated from
23
- Snowpark. We need to temporarily set the json query format in order to process GS token response. Note that we
24
- should set the format back only after registry authentication is complete, otherwise authentication will fail.
25
-
26
- Args:
27
- session: snowpark session
28
-
29
- Yields:
30
- base64-encoded credentials.
31
- """
32
-
33
- query_result = (
34
- query_result_checker.SqlResultValidator(
35
- session,
36
- query="SHOW PARAMETERS LIKE 'PYTHON_CONNECTOR_QUERY_RESULT_FORMAT' IN SESSION",
37
- )
38
- .has_dimensions(expected_rows=1)
39
- .validate()
40
- )
41
- prev_format = query_result[0].value
42
- try:
43
- session.sql("ALTER SESSION SET PYTHON_CONNECTOR_QUERY_RESULT_FORMAT = 'json'").collect()
44
- token = _get_session_token(session)
45
- yield _get_base64_encoded_credentials(username="0sessiontoken", password=json.dumps(token))
46
- finally:
47
- session.sql(f"ALTER SESSION SET PYTHON_CONNECTOR_QUERY_RESULT_FORMAT = '{prev_format}'").collect()
48
-
49
-
50
- def _get_session_token(session: snowpark.Session) -> SessionToken:
51
- """
52
- This function retrieves the session token from a given Snowpark session object.
53
-
54
- Args:
55
- session: snowpark session.
56
-
57
- Returns:
58
- The session token string value.
59
- """
60
- ctx = session._conn._conn
61
- assert ctx._rest, "SnowflakeRestful is not set in session"
62
- token_data = ctx._rest._token_request("ISSUE")
63
- session_token = token_data["data"]["sessionToken"]
64
- validity_in_seconds = token_data["data"]["validityInSecondsST"]
65
- assert session_token, "session_token is not obtained successfully from the session object"
66
- assert validity_in_seconds, "validityInSecondsST is not obtained successfully from the session object"
67
- return {"token": session_token, "expires_in": validity_in_seconds}
68
-
69
-
70
- def _get_base64_encoded_credentials(username: str, password: str) -> str:
71
- """This function returns the base64 encoded username:password, which is compatible with registry, such as
72
- SnowService image registry, that uses Docker credential helper.
73
-
74
- Args:
75
- username: username for authentication.
76
- password: password for authentication.
77
-
78
- Returns:
79
- base64 encoded credential string.
80
-
81
- """
82
- credentials = f"{username}:{password}"
83
- encoded_credentials = base64.b64encode(credentials.encode("utf-8")).decode("utf-8")
84
- return encoded_credentials
@@ -1,127 +0,0 @@
1
- import http
2
- import json
3
- import logging
4
- import time
5
- from typing import Any, Callable, Dict, FrozenSet, Optional
6
- from urllib.parse import urlparse, urlunparse
7
-
8
- import requests
9
-
10
- from snowflake import snowpark
11
- from snowflake.ml._internal.exceptions import (
12
- error_codes,
13
- exceptions as snowml_exceptions,
14
- )
15
- from snowflake.ml._internal.utils import retryable_http, session_token_manager
16
-
17
- logger = logging.getLogger(__name__)
18
-
19
- _MAX_RETRIES = 5
20
- _RETRY_DELAY_SECONDS = 1
21
- _RETRYABLE_HTTP_CODE = frozenset([http.HTTPStatus.UNAUTHORIZED])
22
-
23
-
24
- def retry_on_error(
25
- http_call_function: Callable[..., requests.Response],
26
- retryable_http_code: FrozenSet[http.HTTPStatus] = _RETRYABLE_HTTP_CODE,
27
- ) -> Callable[..., requests.Response]:
28
- def wrapper(*args: Any, **kwargs: Any) -> Any:
29
- retry_delay_seconds = _RETRY_DELAY_SECONDS
30
- for attempt in range(1, _MAX_RETRIES + 1):
31
- resp = http_call_function(*args, **kwargs)
32
- if resp.status_code in retryable_http_code:
33
- logger.warning(
34
- f"Received {resp.status_code} status code. Retrying " f"(attempt {attempt}/{_MAX_RETRIES})..."
35
- )
36
- time.sleep(retry_delay_seconds)
37
- retry_delay_seconds *= 2 # Increase the retry delay exponentially
38
- if attempt < _MAX_RETRIES:
39
- assert isinstance(args[0], ImageRegistryHttpClient)
40
- args[0]._fetch_bearer_token()
41
- else:
42
- return resp
43
-
44
- if attempt == _MAX_RETRIES:
45
- raise snowml_exceptions.SnowflakeMLException(
46
- error_code=error_codes.INTERNAL_SNOWFLAKE_IMAGE_REGISTRY_ERROR,
47
- original_exception=RuntimeError(
48
- f"Failed to authenticate to registry after max retries {attempt} \n"
49
- f"Status {resp.status_code},"
50
- f"{str(resp.text)}"
51
- ),
52
- )
53
-
54
- return wrapper
55
-
56
-
57
- class ImageRegistryHttpClient:
58
- """
59
- An image registry HTTP client utilizes a retryable HTTP client underneath. Its primary function is to facilitate
60
- re-authentication with the image registry by obtaining a new GS token, which is then used to acquire a new bearer
61
- token for subsequent HTTP request authentication.
62
-
63
- Ideally you should not use this client directly. Please use ImageRegistryClient for image registry-specific
64
- operations. For general use of a retryable HTTP client, consider using the "retryable_http" module.
65
- """
66
-
67
- def __init__(self, *, repo_url: str, session: Optional[snowpark.Session] = None, no_cred: bool = False) -> None:
68
- self._repo_url = repo_url
69
- self._retryable_http = retryable_http.get_http_client()
70
- self._no_cred = no_cred
71
-
72
- if not self._no_cred:
73
- self._bearer_token = ""
74
- assert session is not None
75
- self._session_token_manager = session_token_manager.SessionTokenManager(session)
76
-
77
- def _with_bearer_token_header(self, headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
78
- if self._no_cred:
79
- return {} if not headers else headers.copy()
80
-
81
- if not self._bearer_token:
82
- self._fetch_bearer_token()
83
- assert self._bearer_token
84
- new_headers = {} if not headers else headers.copy()
85
- new_headers["Authorization"] = f"Bearer {self._bearer_token}"
86
- return new_headers
87
-
88
- def _fetch_bearer_token(self) -> None:
89
- resp = self._login()
90
- self._bearer_token = str(json.loads(resp.text)["token"])
91
-
92
- def _login(self) -> requests.Response:
93
- """Log in to image registry. repo_url is expected to set when _login function is invoked.
94
-
95
- Returns:
96
- Bearer token when login succeeded.
97
- """
98
- parsed_url = urlparse(self._repo_url)
99
- scheme = parsed_url.scheme
100
- host = parsed_url.netloc
101
-
102
- login_path = "/login" # Construct the login path
103
- url_tuple = (scheme, host, login_path, "", "", "")
104
- login_url = urlunparse(url_tuple)
105
-
106
- base64_encoded_token = self._session_token_manager.get_base64_encoded_token()
107
- return self._retryable_http.get(login_url, headers={"Authorization": f"Basic {base64_encoded_token}"})
108
-
109
- @retry_on_error
110
- def head(self, api_url: str, *, headers: Optional[Dict[str, str]] = None) -> requests.Response:
111
- return self._retryable_http.head(api_url, headers=self._with_bearer_token_header(headers))
112
-
113
- @retry_on_error
114
- def get(self, api_url: str, *, headers: Optional[Dict[str, str]] = None) -> requests.Response:
115
- return self._retryable_http.get(api_url, headers=self._with_bearer_token_header(headers))
116
-
117
- @retry_on_error
118
- def put(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
119
- return self._retryable_http.put(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
120
-
121
- @retry_on_error
122
- def post(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
123
- return self._retryable_http.post(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
124
-
125
- @retry_on_error
126
- def patch(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
127
- return self._retryable_http.patch(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
@@ -1,400 +0,0 @@
1
- """
2
- A minimal pure python library to copy images between two remote registries.
3
-
4
- This library only supports a limited set of features:
5
- - Works only with docker and OCI manifests and manifest lists for multiarch images (most newer images)
6
- - Supported OCI manifest type: application/vnd.oci.image.manifest.v1+json
7
- - Supported Docker manifest type: application/vnd.docker.distribution.manifest.v2+json
8
- - Supports only pulling a single architecture from a multiarch image. Does not support pulling all architectures.
9
- - Supports only schemaVersion 2.
10
- - Streams images from source to destination without any intermediate disk storage in chunks.
11
- - Does not support copying in parallel.
12
-
13
- It's recommended to use this library to copy previously tested images using sha256 to avoid surprises
14
- with respect to compatibility.
15
- """
16
-
17
- import dataclasses
18
- import hashlib
19
- import io
20
- import json
21
- import logging
22
- from collections import namedtuple
23
- from typing import Dict, List, Optional, Tuple
24
-
25
- import requests
26
-
27
- from snowflake.ml._internal.container_services.image_registry import (
28
- http_client as image_registry_http_client,
29
- )
30
-
31
- # Common HTTP headers
32
- _CONTENT_LENGTH_HEADER = "content-length"
33
- _CONTENT_TYPE_HEADER = "content-type"
34
- _CONTENT_RANGE_HEADER = "content-range"
35
- _LOCATION_HEADER = "location"
36
- _AUTHORIZATION_HEADER = "Authorization"
37
- _ACCEPT_HEADER = "accept"
38
-
39
- _OCI_MANIFEST_LIST_TYPE = "application/vnd.oci.image.index.v1+json"
40
- _DOCKER_MANIFEST_LIST_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json"
41
-
42
- _OCI_MANIFEST_TYPE = "application/vnd.oci.image.manifest.v1+json"
43
- _DOCKER_MANIFEST_TYPE = "application/vnd.docker.distribution.manifest.v2+json"
44
-
45
- ALL_SUPPORTED_MEDIA_TYPES = [
46
- _OCI_MANIFEST_LIST_TYPE,
47
- _DOCKER_MANIFEST_LIST_TYPE,
48
- _OCI_MANIFEST_TYPE,
49
- _DOCKER_MANIFEST_TYPE,
50
- ]
51
- _MANIFEST_SUPPORTED_KEYS = {"schemaVersion", "mediaType", "config", "layers"}
52
-
53
- # Architecture descriptor as a named tuple
54
- _Arch = namedtuple("_Arch", ["arch_name", "os"])
55
-
56
- logger = logging.getLogger(__name__)
57
-
58
-
59
- @dataclasses.dataclass
60
- class ImageDescriptor:
61
- """
62
- Create an image descriptor.
63
-
64
- registry_name: the name of the registry like gcr.io
65
- repository_name: the name of the repository like kaniko-project/executor
66
- tag: the tag of the image like v1.6.0
67
- digest: the sha256 digest of the image like sha256:b8c0...
68
- protocol: the protocol to use, defaults to https
69
-
70
- Only a tag or a digest must be specified, not both.
71
- """
72
-
73
- registry_name: str
74
- repository_name: str
75
- tag: Optional[str] = None
76
- digest: Optional[str] = None
77
- protocol: str = "https"
78
-
79
- def __baseurl(self) -> str:
80
- return f"{self.protocol}://{self.registry_name}/v2/"
81
-
82
- def manifest_link(self) -> str:
83
- return f"{self.__baseurl()}{self.repository_name}/manifests/{self.tag or self.digest}"
84
-
85
- def blob_link(self, digest: str) -> str:
86
- return f"{self.__baseurl()}{self.repository_name}/blobs/{digest}"
87
-
88
- def blob_upload_link(self) -> str:
89
- return f"{self.__baseurl()}{self.repository_name}/blobs/uploads/"
90
-
91
- def manifest_upload_link(self, tag: str) -> str:
92
- return f"{self.__baseurl()}{self.repository_name}/manifests/{tag}"
93
-
94
- def __str__(self) -> str:
95
- return f"{self.registry_name}/{self.repository_name}@{self.tag or self.digest}"
96
-
97
-
98
- class Manifest:
99
- def __init__(self, manifest_bytes: bytes, manifest_digest: str) -> None:
100
- """Create a manifest object from the manifest JSON dict.
101
-
102
- Args:
103
- manifest_bytes: manifest content in bytes.
104
- manifest_digest: SHA256 digest.
105
- """
106
- self.manifest_bytes = manifest_bytes
107
- self.manifest = json.loads(manifest_bytes.decode("utf-8"))
108
- self.__validate(self.manifest)
109
-
110
- self.manifest_digest = manifest_digest
111
- self.media_type = self.manifest["mediaType"]
112
-
113
- def get_blob_digests(self) -> List[str]:
114
- """
115
- Get the list of blob digests from the manifest including config and layers.
116
- """
117
- blobs = []
118
- blobs.extend([x["digest"] for x in self.manifest["layers"]])
119
- blobs.append(self.manifest["config"]["digest"])
120
-
121
- return blobs
122
-
123
- def __validate(self, manifest: Dict[str, str]) -> None:
124
- """
125
- Validate the manifest.
126
- """
127
- assert (
128
- manifest.keys() == _MANIFEST_SUPPORTED_KEYS
129
- ), f"Manifest must contain all keys and no more {_MANIFEST_SUPPORTED_KEYS}"
130
- assert int(manifest["schemaVersion"]) == 2, "Only manifest schemaVersion 2 is supported"
131
- assert manifest["mediaType"] in [
132
- _OCI_MANIFEST_TYPE,
133
- _DOCKER_MANIFEST_TYPE,
134
- ], f'Unsupported mediaType {manifest["mediaType"]}'
135
-
136
- def __str__(self) -> str:
137
- """
138
- Return the manifest as a string.
139
- """
140
- return json.dumps(self.manifest, indent=4)
141
-
142
-
143
- @dataclasses.dataclass
144
- class BlobTransfer:
145
- """
146
- Helper class to transfer a blob from one registry to another
147
- in small chunks using in-memory buffering.
148
- """
149
-
150
- # Uploads in chunks of 1MB
151
- chunk_size_bytes = 1024 * 1024
152
-
153
- src_image: ImageDescriptor
154
- dest_image: ImageDescriptor
155
- manifest: Manifest
156
- src_image_registry_http_client: image_registry_http_client.ImageRegistryHttpClient
157
- dest_image_registry_http_client: image_registry_http_client.ImageRegistryHttpClient
158
-
159
- def upload_all_blobs(self) -> None:
160
- blob_digests = self.manifest.get_blob_digests()
161
- logger.debug(f"Found {len(blob_digests)} blobs for {self.src_image}")
162
-
163
- for blob_digest in blob_digests:
164
- logger.debug(f"Transferring blob {blob_digest} from {self.src_image} to {self.dest_image}")
165
- if self._should_upload(blob_digest):
166
- self._transfer(blob_digest)
167
- else:
168
- logger.debug(f"Blob {blob_digest} already exists in {self.dest_image}")
169
-
170
- def _should_upload(self, blob_digest: str) -> bool:
171
- """
172
- Check if the blob already exists in the destination registry.
173
- """
174
- resp = self.dest_image_registry_http_client.head(self.dest_image.blob_link(blob_digest), headers={})
175
- return resp.status_code != 200
176
-
177
- def _fetch_blob(self, blob_digest: str) -> Tuple[io.BytesIO, int]:
178
- """
179
- Fetch a stream to the blob from the source registry.
180
- """
181
- src_blob_link = self.src_image.blob_link(blob_digest)
182
- headers = {_CONTENT_LENGTH_HEADER: "0"}
183
- resp = self.src_image_registry_http_client.get(src_blob_link, headers=headers)
184
-
185
- assert resp.status_code == 200, f"Blob GET failed with code {resp.status_code}"
186
- assert _CONTENT_LENGTH_HEADER in resp.headers, f"Blob does not contain {_CONTENT_LENGTH_HEADER}"
187
-
188
- return io.BytesIO(resp.content), int(resp.headers[_CONTENT_LENGTH_HEADER])
189
-
190
- def _get_upload_url(self) -> str:
191
- """
192
- Obtain the upload URL from the destination registry.
193
- """
194
- response = self.dest_image_registry_http_client.post(self.dest_image.blob_upload_link())
195
- assert (
196
- response.status_code == 202
197
- ), f"Failed to get the upload URL to destination. Status {response.status_code}. {str(response.content)}"
198
- return str(response.headers[_LOCATION_HEADER])
199
-
200
- def _upload_blob(self, blob_digest: str, blob_data: io.BytesIO, content_length: int) -> None:
201
- """
202
- Upload a blob to the destination registry.
203
- """
204
- upload_url = self._get_upload_url()
205
- headers = {
206
- _CONTENT_TYPE_HEADER: "application/octet-stream",
207
- }
208
-
209
- # Use chunked transfer
210
- # This can be optimized to use a single PUT request for small blobs
211
- next_loc = upload_url
212
- start_byte = 0
213
- while start_byte < content_length:
214
- chunk = blob_data.read(self.chunk_size_bytes)
215
- chunk_length = len(chunk)
216
- end_byte = start_byte + chunk_length - 1
217
-
218
- headers[_CONTENT_RANGE_HEADER] = f"{start_byte}-{end_byte}"
219
- headers[_CONTENT_LENGTH_HEADER] = str(chunk_length)
220
-
221
- resp = self.dest_image_registry_http_client.patch(next_loc, headers=headers, data=chunk)
222
- assert resp.status_code == 202, f"Blob PATCH failed with code {resp.status_code}"
223
-
224
- next_loc = resp.headers[_LOCATION_HEADER]
225
- start_byte += chunk_length
226
-
227
- # Finalize the upload
228
- resp = self.dest_image_registry_http_client.put(f"{next_loc}&digest={blob_digest}")
229
- assert resp.status_code == 201, f"Blob PUT failed with code {resp.status_code}"
230
-
231
- def _transfer(self, blob_digest: str) -> None:
232
- """
233
- Transfer a blob from the source registry to the destination registry.
234
- """
235
- blob_data, content_length = self._fetch_blob(blob_digest)
236
- self._upload_blob(blob_digest, blob_data, content_length)
237
-
238
-
239
- def get_bytes_with_sha_verification(resp: requests.Response, sha256_digest: str) -> Tuple[bytes, str]:
240
- """Get the bytes of a response and verify the sha256 digest.
241
-
242
- Args:
243
- resp: the response object
244
- sha256_digest: the expected sha256 digest in format "sha256:b8c0..."
245
-
246
- Returns:
247
- (res, sha256_digest)
248
-
249
- """
250
- digest = hashlib.sha256()
251
- chunks = []
252
- for chunk in resp.iter_content(chunk_size=8192):
253
- digest.update(chunk)
254
- chunks.append(chunk)
255
-
256
- calculated_digest = digest.hexdigest()
257
- assert not sha256_digest or sha256_digest.endswith(calculated_digest), "SHA256 digest does not match"
258
-
259
- content = b"".join(chunks) # Minimize allocations by joining chunks
260
- return content, calculated_digest
261
-
262
-
263
- def get_manifest(
264
- image_descriptor: ImageDescriptor, arch: _Arch, retryable_http: image_registry_http_client.ImageRegistryHttpClient
265
- ) -> Manifest:
266
- """Get the manifest of an image from the remote registry.
267
-
268
- Args:
269
- image_descriptor: the image descriptor
270
- arch: the architecture to filter for if it's a multi-arch image
271
- retryable_http: a retryable http client.
272
-
273
- Returns:
274
- Manifest object.
275
-
276
- """
277
- logger.debug(f"Getting manifest from {image_descriptor.manifest_link()}")
278
-
279
- headers = {_ACCEPT_HEADER: ",".join(ALL_SUPPORTED_MEDIA_TYPES)}
280
-
281
- response = retryable_http.get(image_descriptor.manifest_link(), headers=headers)
282
- assert response.status_code == 200, f"Manifest GET failed with code {response.status_code}, {response.text}"
283
-
284
- assert image_descriptor.digest
285
- manifest_bytes, manifest_digest = get_bytes_with_sha_verification(response, image_descriptor.digest)
286
- manifest_json = json.loads(manifest_bytes.decode("utf-8"))
287
-
288
- # If this is a manifest list, find the manifest for the specified architecture
289
- # and recurse till we find the real manifest
290
- if manifest_json["mediaType"] in [
291
- _OCI_MANIFEST_LIST_TYPE,
292
- _DOCKER_MANIFEST_LIST_TYPE,
293
- ]:
294
- logger.debug("Found a multiarch image. Following manifest reference.")
295
-
296
- assert "manifests" in manifest_json, "Manifest list does not contain manifests"
297
- qualified_manifests = [
298
- x
299
- for x in manifest_json["manifests"]
300
- if x["platform"]["architecture"] == arch.arch_name and x["platform"]["os"] == arch.os
301
- ]
302
- assert (
303
- len(qualified_manifests) == 1
304
- ), "Manifest list does not contain exactly one qualified manifest for this arch"
305
-
306
- manifest_object = qualified_manifests[0]
307
- manifest_digest = manifest_object["digest"]
308
-
309
- logger.debug(f"Found manifest reference for arch {arch}: {manifest_digest}")
310
-
311
- # Copy the image descriptor to fetch the arch-specific manifest
312
- descriptor_copy = ImageDescriptor(
313
- registry_name=image_descriptor.registry_name,
314
- repository_name=image_descriptor.repository_name,
315
- digest=manifest_digest,
316
- tag=None,
317
- )
318
-
319
- # Supports only one level of manifest list nesting to avoid infinite recursion
320
- return get_manifest(descriptor_copy, arch, retryable_http)
321
-
322
- return Manifest(manifest_bytes, manifest_digest)
323
-
324
-
325
- def put_manifest(
326
- image_descriptor: ImageDescriptor,
327
- manifest: Manifest,
328
- retryable_http: image_registry_http_client.ImageRegistryHttpClient,
329
- ) -> None:
330
- """
331
- Upload the given manifest to the destination registry.
332
- """
333
- assert image_descriptor.tag is not None, "Tag must be specified for manifest upload"
334
- headers = {_CONTENT_TYPE_HEADER: manifest.media_type}
335
- url = image_descriptor.manifest_upload_link(image_descriptor.tag)
336
- logger.debug(f"Uploading manifest to {url}")
337
- response = retryable_http.put(url, headers=headers, data=manifest.manifest_bytes)
338
- assert response.status_code == 201, f"Manifest PUT failed with code {response.status_code}"
339
-
340
-
341
- def copy_image(
342
- src_image: ImageDescriptor,
343
- dest_image: ImageDescriptor,
344
- arch: _Arch,
345
- src_retryable_http: image_registry_http_client.ImageRegistryHttpClient,
346
- dest_retryable_http: image_registry_http_client.ImageRegistryHttpClient,
347
- ) -> None:
348
- logger.debug(f"Pulling image manifest for {src_image}")
349
-
350
- # 1. Get the manifest
351
- manifest = get_manifest(src_image, arch, src_retryable_http)
352
- logger.debug(f"Manifest pulled for {src_image} with digest {manifest.manifest_digest}")
353
-
354
- # 2: Retrieve all blob digests from manifest; fetch blob based on blob digest, then upload blob.
355
- blob_transfer = BlobTransfer(
356
- src_image,
357
- dest_image,
358
- manifest,
359
- src_image_registry_http_client=src_retryable_http,
360
- dest_image_registry_http_client=dest_retryable_http,
361
- )
362
- blob_transfer.upload_all_blobs()
363
-
364
- # 3. Upload the manifest
365
- logger.debug(f"All blobs copied successfully. Copying manifest for {src_image} to {dest_image}")
366
- put_manifest(
367
- dest_image,
368
- manifest,
369
- dest_retryable_http,
370
- )
371
-
372
- logger.debug(f"Image {src_image} copied to {dest_image}")
373
-
374
-
375
- def convert_to_image_descriptor(
376
- image_name: str,
377
- with_digest: bool = False,
378
- with_tag: bool = False,
379
- ) -> ImageDescriptor:
380
- """Convert a full image name to a ImageDescriptor object.
381
-
382
- Args:
383
- image_name: name of image.
384
- with_digest: boolean to specify whether a digest is included in the image name
385
- with_tag: boolean to specify whether a tag is included in the image name.
386
-
387
- Returns:
388
- An ImageDescriptor instance
389
- """
390
- assert with_digest or with_tag, "image should contain either digest or tag"
391
- sep = "@" if with_digest else ":"
392
- parts = image_name.split("/")
393
- assert len(parts[-1].split(sep)) == 2, f"Image {image_name} missing digest/tag"
394
- tag_digest = parts[-1].split(sep)[1]
395
- return ImageDescriptor(
396
- registry_name=parts[0],
397
- repository_name="/".join(parts[1:-1] + [parts[-1].split(sep)[0]]),
398
- digest=tag_digest if with_digest else None,
399
- tag=tag_digest if with_tag else None,
400
- )