snowflake-ml-python 1.6.2__py3-none-any.whl → 1.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/__init__.py +4 -0
- snowflake/cortex/_classify_text.py +2 -2
- snowflake/cortex/_embed_text_1024.py +37 -0
- snowflake/cortex/_embed_text_768.py +37 -0
- snowflake/cortex/_extract_answer.py +2 -2
- snowflake/cortex/_sentiment.py +2 -2
- snowflake/cortex/_summarize.py +2 -2
- snowflake/cortex/_translate.py +2 -2
- snowflake/cortex/_util.py +4 -4
- snowflake/ml/_internal/env_utils.py +5 -5
- snowflake/ml/_internal/exceptions/error_codes.py +2 -0
- snowflake/ml/_internal/utils/db_utils.py +50 -0
- snowflake/ml/_internal/utils/service_logger.py +63 -0
- snowflake/ml/_internal/utils/sql_identifier.py +25 -1
- snowflake/ml/data/_internal/arrow_ingestor.py +1 -11
- snowflake/ml/data/ingestor_utils.py +20 -10
- snowflake/ml/feature_store/access_manager.py +3 -3
- snowflake/ml/feature_store/feature_store.py +19 -2
- snowflake/ml/feature_store/feature_view.py +82 -28
- snowflake/ml/fileset/stage_fs.py +2 -1
- snowflake/ml/lineage/lineage_node.py +7 -2
- snowflake/ml/model/__init__.py +1 -2
- snowflake/ml/model/_client/model/model_version_impl.py +78 -9
- snowflake/ml/model/_client/ops/model_ops.py +89 -7
- snowflake/ml/model/_client/ops/service_ops.py +200 -91
- snowflake/ml/model/_client/service/model_deployment_spec.py +4 -0
- snowflake/ml/model/_client/service/model_deployment_spec_schema.py +1 -0
- snowflake/ml/model/_client/sql/_base.py +5 -0
- snowflake/ml/model/_client/sql/model.py +1 -0
- snowflake/ml/model/_client/sql/model_version.py +9 -5
- snowflake/ml/model/_client/sql/service.py +35 -13
- snowflake/ml/model/_model_composer/model_composer.py +11 -41
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +29 -4
- snowflake/ml/model/_packager/model_env/model_env.py +4 -38
- snowflake/ml/model/_packager/model_handlers/_utils.py +106 -32
- snowflake/ml/model/_packager/model_handlers/catboost.py +26 -27
- snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py +3 -3
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +21 -6
- snowflake/ml/model/_packager/model_handlers/mlflow.py +3 -5
- snowflake/ml/model/_packager/model_handlers/model_objective_utils.py +111 -58
- snowflake/ml/model/_packager/model_handlers/sentence_transformers.py +15 -8
- snowflake/ml/model/_packager/model_handlers/sklearn.py +50 -66
- snowflake/ml/model/_packager/model_handlers/snowmlmodel.py +36 -17
- snowflake/ml/model/_packager/model_handlers/xgboost.py +22 -7
- snowflake/ml/model/_packager/model_meta/model_meta.py +16 -45
- snowflake/ml/model/_packager/model_meta/model_meta_schema.py +1 -6
- snowflake/ml/model/_packager/model_packager.py +14 -10
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +11 -0
- snowflake/ml/model/_signatures/snowpark_handler.py +3 -2
- snowflake/ml/model/type_hints.py +11 -152
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +0 -2
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +17 -6
- snowflake/ml/modeling/_internal/snowpark_implementations/xgboost_external_memory_trainer.py +0 -1
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +1 -0
- snowflake/ml/modeling/cluster/affinity_propagation.py +1 -0
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +1 -0
- snowflake/ml/modeling/cluster/birch.py +1 -0
- snowflake/ml/modeling/cluster/bisecting_k_means.py +1 -0
- snowflake/ml/modeling/cluster/dbscan.py +1 -0
- snowflake/ml/modeling/cluster/feature_agglomeration.py +1 -0
- snowflake/ml/modeling/cluster/k_means.py +1 -0
- snowflake/ml/modeling/cluster/mean_shift.py +1 -0
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +1 -0
- snowflake/ml/modeling/cluster/optics.py +1 -0
- snowflake/ml/modeling/cluster/spectral_biclustering.py +1 -0
- snowflake/ml/modeling/cluster/spectral_clustering.py +1 -0
- snowflake/ml/modeling/cluster/spectral_coclustering.py +1 -0
- snowflake/ml/modeling/compose/column_transformer.py +1 -0
- snowflake/ml/modeling/compose/transformed_target_regressor.py +1 -0
- snowflake/ml/modeling/covariance/elliptic_envelope.py +1 -0
- snowflake/ml/modeling/covariance/empirical_covariance.py +1 -0
- snowflake/ml/modeling/covariance/graphical_lasso.py +1 -0
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +1 -0
- snowflake/ml/modeling/covariance/ledoit_wolf.py +1 -0
- snowflake/ml/modeling/covariance/min_cov_det.py +1 -0
- snowflake/ml/modeling/covariance/oas.py +1 -0
- snowflake/ml/modeling/covariance/shrunk_covariance.py +1 -0
- snowflake/ml/modeling/decomposition/dictionary_learning.py +1 -0
- snowflake/ml/modeling/decomposition/factor_analysis.py +1 -0
- snowflake/ml/modeling/decomposition/fast_ica.py +1 -0
- snowflake/ml/modeling/decomposition/incremental_pca.py +1 -0
- snowflake/ml/modeling/decomposition/kernel_pca.py +1 -0
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +1 -0
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +1 -0
- snowflake/ml/modeling/decomposition/pca.py +1 -0
- snowflake/ml/modeling/decomposition/sparse_pca.py +1 -0
- snowflake/ml/modeling/decomposition/truncated_svd.py +1 -0
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +1 -0
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +1 -0
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +1 -0
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +1 -0
- snowflake/ml/modeling/ensemble/bagging_classifier.py +1 -0
- snowflake/ml/modeling/ensemble/bagging_regressor.py +1 -0
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +1 -0
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +1 -0
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +1 -0
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +1 -0
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +1 -0
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +1 -0
- snowflake/ml/modeling/ensemble/isolation_forest.py +1 -0
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +1 -0
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +1 -0
- snowflake/ml/modeling/ensemble/stacking_regressor.py +1 -0
- snowflake/ml/modeling/ensemble/voting_classifier.py +1 -0
- snowflake/ml/modeling/ensemble/voting_regressor.py +1 -0
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +1 -0
- snowflake/ml/modeling/feature_selection/select_fdr.py +1 -0
- snowflake/ml/modeling/feature_selection/select_fpr.py +1 -0
- snowflake/ml/modeling/feature_selection/select_fwe.py +1 -0
- snowflake/ml/modeling/feature_selection/select_k_best.py +1 -0
- snowflake/ml/modeling/feature_selection/select_percentile.py +1 -0
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +1 -0
- snowflake/ml/modeling/feature_selection/variance_threshold.py +1 -0
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +1 -0
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +1 -0
- snowflake/ml/modeling/impute/iterative_imputer.py +1 -0
- snowflake/ml/modeling/impute/knn_imputer.py +1 -0
- snowflake/ml/modeling/impute/missing_indicator.py +1 -0
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +1 -0
- snowflake/ml/modeling/kernel_approximation/nystroem.py +1 -0
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +1 -0
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +1 -0
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +1 -0
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +1 -0
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +1 -0
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/ard_regression.py +1 -0
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +1 -0
- snowflake/ml/modeling/linear_model/elastic_net.py +1 -0
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +1 -0
- snowflake/ml/modeling/linear_model/gamma_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/huber_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/lars.py +1 -0
- snowflake/ml/modeling/linear_model/lars_cv.py +1 -0
- snowflake/ml/modeling/linear_model/lasso.py +1 -0
- snowflake/ml/modeling/linear_model/lasso_cv.py +1 -0
- snowflake/ml/modeling/linear_model/lasso_lars.py +1 -0
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +1 -0
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +1 -0
- snowflake/ml/modeling/linear_model/linear_regression.py +1 -0
- snowflake/ml/modeling/linear_model/logistic_regression.py +1 -0
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +1 -0
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +1 -0
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +1 -0
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +1 -0
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +1 -0
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +1 -0
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +1 -0
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/perceptron.py +1 -0
- snowflake/ml/modeling/linear_model/poisson_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/ransac_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/ridge.py +1 -0
- snowflake/ml/modeling/linear_model/ridge_classifier.py +1 -0
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +1 -0
- snowflake/ml/modeling/linear_model/ridge_cv.py +1 -0
- snowflake/ml/modeling/linear_model/sgd_classifier.py +1 -0
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +1 -0
- snowflake/ml/modeling/linear_model/sgd_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +1 -0
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +1 -0
- snowflake/ml/modeling/manifold/isomap.py +1 -0
- snowflake/ml/modeling/manifold/mds.py +1 -0
- snowflake/ml/modeling/manifold/spectral_embedding.py +1 -0
- snowflake/ml/modeling/manifold/tsne.py +1 -0
- snowflake/ml/modeling/metrics/metrics_utils.py +2 -2
- snowflake/ml/modeling/metrics/ranking.py +0 -3
- snowflake/ml/modeling/metrics/regression.py +0 -3
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +1 -0
- snowflake/ml/modeling/mixture/gaussian_mixture.py +1 -0
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +1 -0
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +1 -0
- snowflake/ml/modeling/multiclass/output_code_classifier.py +1 -0
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +1 -0
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +1 -0
- snowflake/ml/modeling/naive_bayes/complement_nb.py +1 -0
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +1 -0
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +1 -0
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +1 -0
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +1 -0
- snowflake/ml/modeling/neighbors/kernel_density.py +1 -0
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +1 -0
- snowflake/ml/modeling/neighbors/nearest_centroid.py +1 -0
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +1 -0
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +1 -0
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +1 -0
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +1 -0
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +1 -0
- snowflake/ml/modeling/neural_network/mlp_classifier.py +1 -0
- snowflake/ml/modeling/neural_network/mlp_regressor.py +1 -0
- snowflake/ml/modeling/pipeline/pipeline.py +0 -1
- snowflake/ml/modeling/preprocessing/polynomial_features.py +1 -0
- snowflake/ml/modeling/semi_supervised/label_propagation.py +1 -0
- snowflake/ml/modeling/semi_supervised/label_spreading.py +1 -0
- snowflake/ml/modeling/svm/linear_svc.py +1 -0
- snowflake/ml/modeling/svm/linear_svr.py +1 -0
- snowflake/ml/modeling/svm/nu_svc.py +1 -0
- snowflake/ml/modeling/svm/nu_svr.py +1 -0
- snowflake/ml/modeling/svm/svc.py +1 -0
- snowflake/ml/modeling/svm/svr.py +1 -0
- snowflake/ml/modeling/tree/decision_tree_classifier.py +1 -0
- snowflake/ml/modeling/tree/decision_tree_regressor.py +1 -0
- snowflake/ml/modeling/tree/extra_tree_classifier.py +1 -0
- snowflake/ml/modeling/tree/extra_tree_regressor.py +1 -0
- snowflake/ml/modeling/xgboost/xgb_classifier.py +1 -0
- snowflake/ml/modeling/xgboost/xgb_regressor.py +1 -0
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +1 -0
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +1 -0
- snowflake/ml/monitoring/_client/model_monitor.py +126 -0
- snowflake/ml/monitoring/_client/model_monitor_manager.py +361 -0
- snowflake/ml/monitoring/_client/model_monitor_version.py +1 -0
- snowflake/ml/monitoring/_client/monitor_sql_client.py +1335 -0
- snowflake/ml/monitoring/_client/queries/record_count.ssql +14 -0
- snowflake/ml/monitoring/_client/queries/rmse.ssql +28 -0
- snowflake/ml/monitoring/entities/model_monitor_config.py +28 -0
- snowflake/ml/monitoring/entities/model_monitor_interval.py +46 -0
- snowflake/ml/monitoring/entities/output_score_type.py +90 -0
- snowflake/ml/registry/_manager/model_manager.py +4 -4
- snowflake/ml/registry/registry.py +165 -6
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/METADATA +30 -9
- {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/RECORD +225 -249
- {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/WHEEL +1 -1
- snowflake/ml/_internal/container_services/image_registry/credential.py +0 -84
- snowflake/ml/_internal/container_services/image_registry/http_client.py +0 -127
- snowflake/ml/_internal/container_services/image_registry/imagelib.py +0 -400
- snowflake/ml/_internal/container_services/image_registry/registry_client.py +0 -212
- snowflake/ml/_internal/utils/log_stream_processor.py +0 -30
- snowflake/ml/_internal/utils/session_token_manager.py +0 -46
- snowflake/ml/_internal/utils/spcs_attribution_utils.py +0 -122
- snowflake/ml/_internal/utils/uri.py +0 -77
- snowflake/ml/model/_api.py +0 -568
- snowflake/ml/model/_deploy_client/image_builds/base_image_builder.py +0 -12
- snowflake/ml/model/_deploy_client/image_builds/client_image_builder.py +0 -249
- snowflake/ml/model/_deploy_client/image_builds/docker_context.py +0 -130
- snowflake/ml/model/_deploy_client/image_builds/gunicorn_run.sh +0 -36
- snowflake/ml/model/_deploy_client/image_builds/inference_server/main.py +0 -268
- snowflake/ml/model/_deploy_client/image_builds/server_image_builder.py +0 -215
- snowflake/ml/model/_deploy_client/image_builds/templates/dockerfile_template +0 -53
- snowflake/ml/model/_deploy_client/image_builds/templates/image_build_job_spec_template +0 -38
- snowflake/ml/model/_deploy_client/image_builds/templates/kaniko_shell_script_template +0 -105
- snowflake/ml/model/_deploy_client/snowservice/deploy.py +0 -611
- snowflake/ml/model/_deploy_client/snowservice/deploy_options.py +0 -116
- snowflake/ml/model/_deploy_client/snowservice/instance_types.py +0 -10
- snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template +0 -28
- snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template_with_model +0 -21
- snowflake/ml/model/_deploy_client/utils/constants.py +0 -48
- snowflake/ml/model/_deploy_client/utils/snowservice_client.py +0 -280
- snowflake/ml/model/_deploy_client/warehouse/deploy.py +0 -202
- snowflake/ml/model/_deploy_client/warehouse/infer_template.py +0 -99
- snowflake/ml/model/_packager/model_handlers/llm.py +0 -269
- snowflake/ml/model/_packager/model_meta/_core_requirements.py +0 -11
- snowflake/ml/model/deploy_platforms.py +0 -6
- snowflake/ml/model/models/llm.py +0 -106
- snowflake/ml/monitoring/monitor.py +0 -203
- snowflake/ml/registry/_initial_schema.py +0 -142
- snowflake/ml/registry/_schema.py +0 -82
- snowflake/ml/registry/_schema_upgrade_plans.py +0 -116
- snowflake/ml/registry/_schema_version_manager.py +0 -163
- snowflake/ml/registry/model_registry.py +0 -2048
- {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/LICENSE.txt +0 -0
- {snowflake_ml_python-1.6.2.dist-info → snowflake_ml_python-1.6.4.dist-info}/top_level.txt +0 -0
@@ -1,84 +0,0 @@
|
|
1
|
-
# TODO[shchen]: Remove this file and use session_token_manager instead.
|
2
|
-
import base64
|
3
|
-
import contextlib
|
4
|
-
import json
|
5
|
-
from typing import Generator, TypedDict
|
6
|
-
|
7
|
-
from snowflake import snowpark
|
8
|
-
from snowflake.ml._internal.utils import query_result_checker
|
9
|
-
|
10
|
-
|
11
|
-
class SessionToken(TypedDict):
|
12
|
-
token: str
|
13
|
-
expires_in: str
|
14
|
-
|
15
|
-
|
16
|
-
@contextlib.contextmanager
|
17
|
-
def generate_image_registry_credential(session: snowpark.Session) -> Generator[str, None, None]:
|
18
|
-
"""Construct basic auth credential that is specific to SPCS image registry. For image registry authentication, we
|
19
|
-
will use a session token obtained from the Snowpark session object. The token authentication mechanism is
|
20
|
-
automatically used when the username is set to "0sessiontoken" according to the registry implementation.
|
21
|
-
|
22
|
-
As a workaround for SNOW-841699: Fail to authenticate to image registry with session token generated from
|
23
|
-
Snowpark. We need to temporarily set the json query format in order to process GS token response. Note that we
|
24
|
-
should set the format back only after registry authentication is complete, otherwise authentication will fail.
|
25
|
-
|
26
|
-
Args:
|
27
|
-
session: snowpark session
|
28
|
-
|
29
|
-
Yields:
|
30
|
-
base64-encoded credentials.
|
31
|
-
"""
|
32
|
-
|
33
|
-
query_result = (
|
34
|
-
query_result_checker.SqlResultValidator(
|
35
|
-
session,
|
36
|
-
query="SHOW PARAMETERS LIKE 'PYTHON_CONNECTOR_QUERY_RESULT_FORMAT' IN SESSION",
|
37
|
-
)
|
38
|
-
.has_dimensions(expected_rows=1)
|
39
|
-
.validate()
|
40
|
-
)
|
41
|
-
prev_format = query_result[0].value
|
42
|
-
try:
|
43
|
-
session.sql("ALTER SESSION SET PYTHON_CONNECTOR_QUERY_RESULT_FORMAT = 'json'").collect()
|
44
|
-
token = _get_session_token(session)
|
45
|
-
yield _get_base64_encoded_credentials(username="0sessiontoken", password=json.dumps(token))
|
46
|
-
finally:
|
47
|
-
session.sql(f"ALTER SESSION SET PYTHON_CONNECTOR_QUERY_RESULT_FORMAT = '{prev_format}'").collect()
|
48
|
-
|
49
|
-
|
50
|
-
def _get_session_token(session: snowpark.Session) -> SessionToken:
|
51
|
-
"""
|
52
|
-
This function retrieves the session token from a given Snowpark session object.
|
53
|
-
|
54
|
-
Args:
|
55
|
-
session: snowpark session.
|
56
|
-
|
57
|
-
Returns:
|
58
|
-
The session token string value.
|
59
|
-
"""
|
60
|
-
ctx = session._conn._conn
|
61
|
-
assert ctx._rest, "SnowflakeRestful is not set in session"
|
62
|
-
token_data = ctx._rest._token_request("ISSUE")
|
63
|
-
session_token = token_data["data"]["sessionToken"]
|
64
|
-
validity_in_seconds = token_data["data"]["validityInSecondsST"]
|
65
|
-
assert session_token, "session_token is not obtained successfully from the session object"
|
66
|
-
assert validity_in_seconds, "validityInSecondsST is not obtained successfully from the session object"
|
67
|
-
return {"token": session_token, "expires_in": validity_in_seconds}
|
68
|
-
|
69
|
-
|
70
|
-
def _get_base64_encoded_credentials(username: str, password: str) -> str:
|
71
|
-
"""This function returns the base64 encoded username:password, which is compatible with registry, such as
|
72
|
-
SnowService image registry, that uses Docker credential helper.
|
73
|
-
|
74
|
-
Args:
|
75
|
-
username: username for authentication.
|
76
|
-
password: password for authentication.
|
77
|
-
|
78
|
-
Returns:
|
79
|
-
base64 encoded credential string.
|
80
|
-
|
81
|
-
"""
|
82
|
-
credentials = f"{username}:{password}"
|
83
|
-
encoded_credentials = base64.b64encode(credentials.encode("utf-8")).decode("utf-8")
|
84
|
-
return encoded_credentials
|
@@ -1,127 +0,0 @@
|
|
1
|
-
import http
|
2
|
-
import json
|
3
|
-
import logging
|
4
|
-
import time
|
5
|
-
from typing import Any, Callable, Dict, FrozenSet, Optional
|
6
|
-
from urllib.parse import urlparse, urlunparse
|
7
|
-
|
8
|
-
import requests
|
9
|
-
|
10
|
-
from snowflake import snowpark
|
11
|
-
from snowflake.ml._internal.exceptions import (
|
12
|
-
error_codes,
|
13
|
-
exceptions as snowml_exceptions,
|
14
|
-
)
|
15
|
-
from snowflake.ml._internal.utils import retryable_http, session_token_manager
|
16
|
-
|
17
|
-
logger = logging.getLogger(__name__)
|
18
|
-
|
19
|
-
_MAX_RETRIES = 5
|
20
|
-
_RETRY_DELAY_SECONDS = 1
|
21
|
-
_RETRYABLE_HTTP_CODE = frozenset([http.HTTPStatus.UNAUTHORIZED])
|
22
|
-
|
23
|
-
|
24
|
-
def retry_on_error(
|
25
|
-
http_call_function: Callable[..., requests.Response],
|
26
|
-
retryable_http_code: FrozenSet[http.HTTPStatus] = _RETRYABLE_HTTP_CODE,
|
27
|
-
) -> Callable[..., requests.Response]:
|
28
|
-
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
29
|
-
retry_delay_seconds = _RETRY_DELAY_SECONDS
|
30
|
-
for attempt in range(1, _MAX_RETRIES + 1):
|
31
|
-
resp = http_call_function(*args, **kwargs)
|
32
|
-
if resp.status_code in retryable_http_code:
|
33
|
-
logger.warning(
|
34
|
-
f"Received {resp.status_code} status code. Retrying " f"(attempt {attempt}/{_MAX_RETRIES})..."
|
35
|
-
)
|
36
|
-
time.sleep(retry_delay_seconds)
|
37
|
-
retry_delay_seconds *= 2 # Increase the retry delay exponentially
|
38
|
-
if attempt < _MAX_RETRIES:
|
39
|
-
assert isinstance(args[0], ImageRegistryHttpClient)
|
40
|
-
args[0]._fetch_bearer_token()
|
41
|
-
else:
|
42
|
-
return resp
|
43
|
-
|
44
|
-
if attempt == _MAX_RETRIES:
|
45
|
-
raise snowml_exceptions.SnowflakeMLException(
|
46
|
-
error_code=error_codes.INTERNAL_SNOWFLAKE_IMAGE_REGISTRY_ERROR,
|
47
|
-
original_exception=RuntimeError(
|
48
|
-
f"Failed to authenticate to registry after max retries {attempt} \n"
|
49
|
-
f"Status {resp.status_code},"
|
50
|
-
f"{str(resp.text)}"
|
51
|
-
),
|
52
|
-
)
|
53
|
-
|
54
|
-
return wrapper
|
55
|
-
|
56
|
-
|
57
|
-
class ImageRegistryHttpClient:
|
58
|
-
"""
|
59
|
-
An image registry HTTP client utilizes a retryable HTTP client underneath. Its primary function is to facilitate
|
60
|
-
re-authentication with the image registry by obtaining a new GS token, which is then used to acquire a new bearer
|
61
|
-
token for subsequent HTTP request authentication.
|
62
|
-
|
63
|
-
Ideally you should not use this client directly. Please use ImageRegistryClient for image registry-specific
|
64
|
-
operations. For general use of a retryable HTTP client, consider using the "retryable_http" module.
|
65
|
-
"""
|
66
|
-
|
67
|
-
def __init__(self, *, repo_url: str, session: Optional[snowpark.Session] = None, no_cred: bool = False) -> None:
|
68
|
-
self._repo_url = repo_url
|
69
|
-
self._retryable_http = retryable_http.get_http_client()
|
70
|
-
self._no_cred = no_cred
|
71
|
-
|
72
|
-
if not self._no_cred:
|
73
|
-
self._bearer_token = ""
|
74
|
-
assert session is not None
|
75
|
-
self._session_token_manager = session_token_manager.SessionTokenManager(session)
|
76
|
-
|
77
|
-
def _with_bearer_token_header(self, headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
|
78
|
-
if self._no_cred:
|
79
|
-
return {} if not headers else headers.copy()
|
80
|
-
|
81
|
-
if not self._bearer_token:
|
82
|
-
self._fetch_bearer_token()
|
83
|
-
assert self._bearer_token
|
84
|
-
new_headers = {} if not headers else headers.copy()
|
85
|
-
new_headers["Authorization"] = f"Bearer {self._bearer_token}"
|
86
|
-
return new_headers
|
87
|
-
|
88
|
-
def _fetch_bearer_token(self) -> None:
|
89
|
-
resp = self._login()
|
90
|
-
self._bearer_token = str(json.loads(resp.text)["token"])
|
91
|
-
|
92
|
-
def _login(self) -> requests.Response:
|
93
|
-
"""Log in to image registry. repo_url is expected to set when _login function is invoked.
|
94
|
-
|
95
|
-
Returns:
|
96
|
-
Bearer token when login succeeded.
|
97
|
-
"""
|
98
|
-
parsed_url = urlparse(self._repo_url)
|
99
|
-
scheme = parsed_url.scheme
|
100
|
-
host = parsed_url.netloc
|
101
|
-
|
102
|
-
login_path = "/login" # Construct the login path
|
103
|
-
url_tuple = (scheme, host, login_path, "", "", "")
|
104
|
-
login_url = urlunparse(url_tuple)
|
105
|
-
|
106
|
-
base64_encoded_token = self._session_token_manager.get_base64_encoded_token()
|
107
|
-
return self._retryable_http.get(login_url, headers={"Authorization": f"Basic {base64_encoded_token}"})
|
108
|
-
|
109
|
-
@retry_on_error
|
110
|
-
def head(self, api_url: str, *, headers: Optional[Dict[str, str]] = None) -> requests.Response:
|
111
|
-
return self._retryable_http.head(api_url, headers=self._with_bearer_token_header(headers))
|
112
|
-
|
113
|
-
@retry_on_error
|
114
|
-
def get(self, api_url: str, *, headers: Optional[Dict[str, str]] = None) -> requests.Response:
|
115
|
-
return self._retryable_http.get(api_url, headers=self._with_bearer_token_header(headers))
|
116
|
-
|
117
|
-
@retry_on_error
|
118
|
-
def put(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
|
119
|
-
return self._retryable_http.put(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
|
120
|
-
|
121
|
-
@retry_on_error
|
122
|
-
def post(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
|
123
|
-
return self._retryable_http.post(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
|
124
|
-
|
125
|
-
@retry_on_error
|
126
|
-
def patch(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
|
127
|
-
return self._retryable_http.patch(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
|
@@ -1,400 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
A minimal pure python library to copy images between two remote registries.
|
3
|
-
|
4
|
-
This library only supports a limited set of features:
|
5
|
-
- Works only with docker and OCI manifests and manifest lists for multiarch images (most newer images)
|
6
|
-
- Supported OCI manifest type: application/vnd.oci.image.manifest.v1+json
|
7
|
-
- Supported Docker manifest type: application/vnd.docker.distribution.manifest.v2+json
|
8
|
-
- Supports only pulling a single architecture from a multiarch image. Does not support pulling all architectures.
|
9
|
-
- Supports only schemaVersion 2.
|
10
|
-
- Streams images from source to destination without any intermediate disk storage in chunks.
|
11
|
-
- Does not support copying in parallel.
|
12
|
-
|
13
|
-
It's recommended to use this library to copy previously tested images using sha256 to avoid surprises
|
14
|
-
with respect to compatibility.
|
15
|
-
"""
|
16
|
-
|
17
|
-
import dataclasses
|
18
|
-
import hashlib
|
19
|
-
import io
|
20
|
-
import json
|
21
|
-
import logging
|
22
|
-
from collections import namedtuple
|
23
|
-
from typing import Dict, List, Optional, Tuple
|
24
|
-
|
25
|
-
import requests
|
26
|
-
|
27
|
-
from snowflake.ml._internal.container_services.image_registry import (
|
28
|
-
http_client as image_registry_http_client,
|
29
|
-
)
|
30
|
-
|
31
|
-
# Common HTTP headers
|
32
|
-
_CONTENT_LENGTH_HEADER = "content-length"
|
33
|
-
_CONTENT_TYPE_HEADER = "content-type"
|
34
|
-
_CONTENT_RANGE_HEADER = "content-range"
|
35
|
-
_LOCATION_HEADER = "location"
|
36
|
-
_AUTHORIZATION_HEADER = "Authorization"
|
37
|
-
_ACCEPT_HEADER = "accept"
|
38
|
-
|
39
|
-
_OCI_MANIFEST_LIST_TYPE = "application/vnd.oci.image.index.v1+json"
|
40
|
-
_DOCKER_MANIFEST_LIST_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json"
|
41
|
-
|
42
|
-
_OCI_MANIFEST_TYPE = "application/vnd.oci.image.manifest.v1+json"
|
43
|
-
_DOCKER_MANIFEST_TYPE = "application/vnd.docker.distribution.manifest.v2+json"
|
44
|
-
|
45
|
-
ALL_SUPPORTED_MEDIA_TYPES = [
|
46
|
-
_OCI_MANIFEST_LIST_TYPE,
|
47
|
-
_DOCKER_MANIFEST_LIST_TYPE,
|
48
|
-
_OCI_MANIFEST_TYPE,
|
49
|
-
_DOCKER_MANIFEST_TYPE,
|
50
|
-
]
|
51
|
-
_MANIFEST_SUPPORTED_KEYS = {"schemaVersion", "mediaType", "config", "layers"}
|
52
|
-
|
53
|
-
# Architecture descriptor as a named tuple
|
54
|
-
_Arch = namedtuple("_Arch", ["arch_name", "os"])
|
55
|
-
|
56
|
-
logger = logging.getLogger(__name__)
|
57
|
-
|
58
|
-
|
59
|
-
@dataclasses.dataclass
|
60
|
-
class ImageDescriptor:
|
61
|
-
"""
|
62
|
-
Create an image descriptor.
|
63
|
-
|
64
|
-
registry_name: the name of the registry like gcr.io
|
65
|
-
repository_name: the name of the repository like kaniko-project/executor
|
66
|
-
tag: the tag of the image like v1.6.0
|
67
|
-
digest: the sha256 digest of the image like sha256:b8c0...
|
68
|
-
protocol: the protocol to use, defaults to https
|
69
|
-
|
70
|
-
Only a tag or a digest must be specified, not both.
|
71
|
-
"""
|
72
|
-
|
73
|
-
registry_name: str
|
74
|
-
repository_name: str
|
75
|
-
tag: Optional[str] = None
|
76
|
-
digest: Optional[str] = None
|
77
|
-
protocol: str = "https"
|
78
|
-
|
79
|
-
def __baseurl(self) -> str:
|
80
|
-
return f"{self.protocol}://{self.registry_name}/v2/"
|
81
|
-
|
82
|
-
def manifest_link(self) -> str:
|
83
|
-
return f"{self.__baseurl()}{self.repository_name}/manifests/{self.tag or self.digest}"
|
84
|
-
|
85
|
-
def blob_link(self, digest: str) -> str:
|
86
|
-
return f"{self.__baseurl()}{self.repository_name}/blobs/{digest}"
|
87
|
-
|
88
|
-
def blob_upload_link(self) -> str:
|
89
|
-
return f"{self.__baseurl()}{self.repository_name}/blobs/uploads/"
|
90
|
-
|
91
|
-
def manifest_upload_link(self, tag: str) -> str:
|
92
|
-
return f"{self.__baseurl()}{self.repository_name}/manifests/{tag}"
|
93
|
-
|
94
|
-
def __str__(self) -> str:
|
95
|
-
return f"{self.registry_name}/{self.repository_name}@{self.tag or self.digest}"
|
96
|
-
|
97
|
-
|
98
|
-
class Manifest:
|
99
|
-
def __init__(self, manifest_bytes: bytes, manifest_digest: str) -> None:
|
100
|
-
"""Create a manifest object from the manifest JSON dict.
|
101
|
-
|
102
|
-
Args:
|
103
|
-
manifest_bytes: manifest content in bytes.
|
104
|
-
manifest_digest: SHA256 digest.
|
105
|
-
"""
|
106
|
-
self.manifest_bytes = manifest_bytes
|
107
|
-
self.manifest = json.loads(manifest_bytes.decode("utf-8"))
|
108
|
-
self.__validate(self.manifest)
|
109
|
-
|
110
|
-
self.manifest_digest = manifest_digest
|
111
|
-
self.media_type = self.manifest["mediaType"]
|
112
|
-
|
113
|
-
def get_blob_digests(self) -> List[str]:
|
114
|
-
"""
|
115
|
-
Get the list of blob digests from the manifest including config and layers.
|
116
|
-
"""
|
117
|
-
blobs = []
|
118
|
-
blobs.extend([x["digest"] for x in self.manifest["layers"]])
|
119
|
-
blobs.append(self.manifest["config"]["digest"])
|
120
|
-
|
121
|
-
return blobs
|
122
|
-
|
123
|
-
def __validate(self, manifest: Dict[str, str]) -> None:
|
124
|
-
"""
|
125
|
-
Validate the manifest.
|
126
|
-
"""
|
127
|
-
assert (
|
128
|
-
manifest.keys() == _MANIFEST_SUPPORTED_KEYS
|
129
|
-
), f"Manifest must contain all keys and no more {_MANIFEST_SUPPORTED_KEYS}"
|
130
|
-
assert int(manifest["schemaVersion"]) == 2, "Only manifest schemaVersion 2 is supported"
|
131
|
-
assert manifest["mediaType"] in [
|
132
|
-
_OCI_MANIFEST_TYPE,
|
133
|
-
_DOCKER_MANIFEST_TYPE,
|
134
|
-
], f'Unsupported mediaType {manifest["mediaType"]}'
|
135
|
-
|
136
|
-
def __str__(self) -> str:
|
137
|
-
"""
|
138
|
-
Return the manifest as a string.
|
139
|
-
"""
|
140
|
-
return json.dumps(self.manifest, indent=4)
|
141
|
-
|
142
|
-
|
143
|
-
@dataclasses.dataclass
|
144
|
-
class BlobTransfer:
|
145
|
-
"""
|
146
|
-
Helper class to transfer a blob from one registry to another
|
147
|
-
in small chunks using in-memory buffering.
|
148
|
-
"""
|
149
|
-
|
150
|
-
# Uploads in chunks of 1MB
|
151
|
-
chunk_size_bytes = 1024 * 1024
|
152
|
-
|
153
|
-
src_image: ImageDescriptor
|
154
|
-
dest_image: ImageDescriptor
|
155
|
-
manifest: Manifest
|
156
|
-
src_image_registry_http_client: image_registry_http_client.ImageRegistryHttpClient
|
157
|
-
dest_image_registry_http_client: image_registry_http_client.ImageRegistryHttpClient
|
158
|
-
|
159
|
-
def upload_all_blobs(self) -> None:
|
160
|
-
blob_digests = self.manifest.get_blob_digests()
|
161
|
-
logger.debug(f"Found {len(blob_digests)} blobs for {self.src_image}")
|
162
|
-
|
163
|
-
for blob_digest in blob_digests:
|
164
|
-
logger.debug(f"Transferring blob {blob_digest} from {self.src_image} to {self.dest_image}")
|
165
|
-
if self._should_upload(blob_digest):
|
166
|
-
self._transfer(blob_digest)
|
167
|
-
else:
|
168
|
-
logger.debug(f"Blob {blob_digest} already exists in {self.dest_image}")
|
169
|
-
|
170
|
-
def _should_upload(self, blob_digest: str) -> bool:
|
171
|
-
"""
|
172
|
-
Check if the blob already exists in the destination registry.
|
173
|
-
"""
|
174
|
-
resp = self.dest_image_registry_http_client.head(self.dest_image.blob_link(blob_digest), headers={})
|
175
|
-
return resp.status_code != 200
|
176
|
-
|
177
|
-
def _fetch_blob(self, blob_digest: str) -> Tuple[io.BytesIO, int]:
|
178
|
-
"""
|
179
|
-
Fetch a stream to the blob from the source registry.
|
180
|
-
"""
|
181
|
-
src_blob_link = self.src_image.blob_link(blob_digest)
|
182
|
-
headers = {_CONTENT_LENGTH_HEADER: "0"}
|
183
|
-
resp = self.src_image_registry_http_client.get(src_blob_link, headers=headers)
|
184
|
-
|
185
|
-
assert resp.status_code == 200, f"Blob GET failed with code {resp.status_code}"
|
186
|
-
assert _CONTENT_LENGTH_HEADER in resp.headers, f"Blob does not contain {_CONTENT_LENGTH_HEADER}"
|
187
|
-
|
188
|
-
return io.BytesIO(resp.content), int(resp.headers[_CONTENT_LENGTH_HEADER])
|
189
|
-
|
190
|
-
def _get_upload_url(self) -> str:
|
191
|
-
"""
|
192
|
-
Obtain the upload URL from the destination registry.
|
193
|
-
"""
|
194
|
-
response = self.dest_image_registry_http_client.post(self.dest_image.blob_upload_link())
|
195
|
-
assert (
|
196
|
-
response.status_code == 202
|
197
|
-
), f"Failed to get the upload URL to destination. Status {response.status_code}. {str(response.content)}"
|
198
|
-
return str(response.headers[_LOCATION_HEADER])
|
199
|
-
|
200
|
-
def _upload_blob(self, blob_digest: str, blob_data: io.BytesIO, content_length: int) -> None:
|
201
|
-
"""
|
202
|
-
Upload a blob to the destination registry.
|
203
|
-
"""
|
204
|
-
upload_url = self._get_upload_url()
|
205
|
-
headers = {
|
206
|
-
_CONTENT_TYPE_HEADER: "application/octet-stream",
|
207
|
-
}
|
208
|
-
|
209
|
-
# Use chunked transfer
|
210
|
-
# This can be optimized to use a single PUT request for small blobs
|
211
|
-
next_loc = upload_url
|
212
|
-
start_byte = 0
|
213
|
-
while start_byte < content_length:
|
214
|
-
chunk = blob_data.read(self.chunk_size_bytes)
|
215
|
-
chunk_length = len(chunk)
|
216
|
-
end_byte = start_byte + chunk_length - 1
|
217
|
-
|
218
|
-
headers[_CONTENT_RANGE_HEADER] = f"{start_byte}-{end_byte}"
|
219
|
-
headers[_CONTENT_LENGTH_HEADER] = str(chunk_length)
|
220
|
-
|
221
|
-
resp = self.dest_image_registry_http_client.patch(next_loc, headers=headers, data=chunk)
|
222
|
-
assert resp.status_code == 202, f"Blob PATCH failed with code {resp.status_code}"
|
223
|
-
|
224
|
-
next_loc = resp.headers[_LOCATION_HEADER]
|
225
|
-
start_byte += chunk_length
|
226
|
-
|
227
|
-
# Finalize the upload
|
228
|
-
resp = self.dest_image_registry_http_client.put(f"{next_loc}&digest={blob_digest}")
|
229
|
-
assert resp.status_code == 201, f"Blob PUT failed with code {resp.status_code}"
|
230
|
-
|
231
|
-
def _transfer(self, blob_digest: str) -> None:
|
232
|
-
"""
|
233
|
-
Transfer a blob from the source registry to the destination registry.
|
234
|
-
"""
|
235
|
-
blob_data, content_length = self._fetch_blob(blob_digest)
|
236
|
-
self._upload_blob(blob_digest, blob_data, content_length)
|
237
|
-
|
238
|
-
|
239
|
-
def get_bytes_with_sha_verification(resp: requests.Response, sha256_digest: str) -> Tuple[bytes, str]:
|
240
|
-
"""Get the bytes of a response and verify the sha256 digest.
|
241
|
-
|
242
|
-
Args:
|
243
|
-
resp: the response object
|
244
|
-
sha256_digest: the expected sha256 digest in format "sha256:b8c0..."
|
245
|
-
|
246
|
-
Returns:
|
247
|
-
(res, sha256_digest)
|
248
|
-
|
249
|
-
"""
|
250
|
-
digest = hashlib.sha256()
|
251
|
-
chunks = []
|
252
|
-
for chunk in resp.iter_content(chunk_size=8192):
|
253
|
-
digest.update(chunk)
|
254
|
-
chunks.append(chunk)
|
255
|
-
|
256
|
-
calculated_digest = digest.hexdigest()
|
257
|
-
assert not sha256_digest or sha256_digest.endswith(calculated_digest), "SHA256 digest does not match"
|
258
|
-
|
259
|
-
content = b"".join(chunks) # Minimize allocations by joining chunks
|
260
|
-
return content, calculated_digest
|
261
|
-
|
262
|
-
|
263
|
-
def get_manifest(
|
264
|
-
image_descriptor: ImageDescriptor, arch: _Arch, retryable_http: image_registry_http_client.ImageRegistryHttpClient
|
265
|
-
) -> Manifest:
|
266
|
-
"""Get the manifest of an image from the remote registry.
|
267
|
-
|
268
|
-
Args:
|
269
|
-
image_descriptor: the image descriptor
|
270
|
-
arch: the architecture to filter for if it's a multi-arch image
|
271
|
-
retryable_http: a retryable http client.
|
272
|
-
|
273
|
-
Returns:
|
274
|
-
Manifest object.
|
275
|
-
|
276
|
-
"""
|
277
|
-
logger.debug(f"Getting manifest from {image_descriptor.manifest_link()}")
|
278
|
-
|
279
|
-
headers = {_ACCEPT_HEADER: ",".join(ALL_SUPPORTED_MEDIA_TYPES)}
|
280
|
-
|
281
|
-
response = retryable_http.get(image_descriptor.manifest_link(), headers=headers)
|
282
|
-
assert response.status_code == 200, f"Manifest GET failed with code {response.status_code}, {response.text}"
|
283
|
-
|
284
|
-
assert image_descriptor.digest
|
285
|
-
manifest_bytes, manifest_digest = get_bytes_with_sha_verification(response, image_descriptor.digest)
|
286
|
-
manifest_json = json.loads(manifest_bytes.decode("utf-8"))
|
287
|
-
|
288
|
-
# If this is a manifest list, find the manifest for the specified architecture
|
289
|
-
# and recurse till we find the real manifest
|
290
|
-
if manifest_json["mediaType"] in [
|
291
|
-
_OCI_MANIFEST_LIST_TYPE,
|
292
|
-
_DOCKER_MANIFEST_LIST_TYPE,
|
293
|
-
]:
|
294
|
-
logger.debug("Found a multiarch image. Following manifest reference.")
|
295
|
-
|
296
|
-
assert "manifests" in manifest_json, "Manifest list does not contain manifests"
|
297
|
-
qualified_manifests = [
|
298
|
-
x
|
299
|
-
for x in manifest_json["manifests"]
|
300
|
-
if x["platform"]["architecture"] == arch.arch_name and x["platform"]["os"] == arch.os
|
301
|
-
]
|
302
|
-
assert (
|
303
|
-
len(qualified_manifests) == 1
|
304
|
-
), "Manifest list does not contain exactly one qualified manifest for this arch"
|
305
|
-
|
306
|
-
manifest_object = qualified_manifests[0]
|
307
|
-
manifest_digest = manifest_object["digest"]
|
308
|
-
|
309
|
-
logger.debug(f"Found manifest reference for arch {arch}: {manifest_digest}")
|
310
|
-
|
311
|
-
# Copy the image descriptor to fetch the arch-specific manifest
|
312
|
-
descriptor_copy = ImageDescriptor(
|
313
|
-
registry_name=image_descriptor.registry_name,
|
314
|
-
repository_name=image_descriptor.repository_name,
|
315
|
-
digest=manifest_digest,
|
316
|
-
tag=None,
|
317
|
-
)
|
318
|
-
|
319
|
-
# Supports only one level of manifest list nesting to avoid infinite recursion
|
320
|
-
return get_manifest(descriptor_copy, arch, retryable_http)
|
321
|
-
|
322
|
-
return Manifest(manifest_bytes, manifest_digest)
|
323
|
-
|
324
|
-
|
325
|
-
def put_manifest(
|
326
|
-
image_descriptor: ImageDescriptor,
|
327
|
-
manifest: Manifest,
|
328
|
-
retryable_http: image_registry_http_client.ImageRegistryHttpClient,
|
329
|
-
) -> None:
|
330
|
-
"""
|
331
|
-
Upload the given manifest to the destination registry.
|
332
|
-
"""
|
333
|
-
assert image_descriptor.tag is not None, "Tag must be specified for manifest upload"
|
334
|
-
headers = {_CONTENT_TYPE_HEADER: manifest.media_type}
|
335
|
-
url = image_descriptor.manifest_upload_link(image_descriptor.tag)
|
336
|
-
logger.debug(f"Uploading manifest to {url}")
|
337
|
-
response = retryable_http.put(url, headers=headers, data=manifest.manifest_bytes)
|
338
|
-
assert response.status_code == 201, f"Manifest PUT failed with code {response.status_code}"
|
339
|
-
|
340
|
-
|
341
|
-
def copy_image(
|
342
|
-
src_image: ImageDescriptor,
|
343
|
-
dest_image: ImageDescriptor,
|
344
|
-
arch: _Arch,
|
345
|
-
src_retryable_http: image_registry_http_client.ImageRegistryHttpClient,
|
346
|
-
dest_retryable_http: image_registry_http_client.ImageRegistryHttpClient,
|
347
|
-
) -> None:
|
348
|
-
logger.debug(f"Pulling image manifest for {src_image}")
|
349
|
-
|
350
|
-
# 1. Get the manifest
|
351
|
-
manifest = get_manifest(src_image, arch, src_retryable_http)
|
352
|
-
logger.debug(f"Manifest pulled for {src_image} with digest {manifest.manifest_digest}")
|
353
|
-
|
354
|
-
# 2: Retrieve all blob digests from manifest; fetch blob based on blob digest, then upload blob.
|
355
|
-
blob_transfer = BlobTransfer(
|
356
|
-
src_image,
|
357
|
-
dest_image,
|
358
|
-
manifest,
|
359
|
-
src_image_registry_http_client=src_retryable_http,
|
360
|
-
dest_image_registry_http_client=dest_retryable_http,
|
361
|
-
)
|
362
|
-
blob_transfer.upload_all_blobs()
|
363
|
-
|
364
|
-
# 3. Upload the manifest
|
365
|
-
logger.debug(f"All blobs copied successfully. Copying manifest for {src_image} to {dest_image}")
|
366
|
-
put_manifest(
|
367
|
-
dest_image,
|
368
|
-
manifest,
|
369
|
-
dest_retryable_http,
|
370
|
-
)
|
371
|
-
|
372
|
-
logger.debug(f"Image {src_image} copied to {dest_image}")
|
373
|
-
|
374
|
-
|
375
|
-
def convert_to_image_descriptor(
|
376
|
-
image_name: str,
|
377
|
-
with_digest: bool = False,
|
378
|
-
with_tag: bool = False,
|
379
|
-
) -> ImageDescriptor:
|
380
|
-
"""Convert a full image name to a ImageDescriptor object.
|
381
|
-
|
382
|
-
Args:
|
383
|
-
image_name: name of image.
|
384
|
-
with_digest: boolean to specify whether a digest is included in the image name
|
385
|
-
with_tag: boolean to specify whether a tag is included in the image name.
|
386
|
-
|
387
|
-
Returns:
|
388
|
-
An ImageDescriptor instance
|
389
|
-
"""
|
390
|
-
assert with_digest or with_tag, "image should contain either digest or tag"
|
391
|
-
sep = "@" if with_digest else ":"
|
392
|
-
parts = image_name.split("/")
|
393
|
-
assert len(parts[-1].split(sep)) == 2, f"Image {image_name} missing digest/tag"
|
394
|
-
tag_digest = parts[-1].split(sep)[1]
|
395
|
-
return ImageDescriptor(
|
396
|
-
registry_name=parts[0],
|
397
|
-
repository_name="/".join(parts[1:-1] + [parts[-1].split(sep)[0]]),
|
398
|
-
digest=tag_digest if with_digest else None,
|
399
|
-
tag=tag_digest if with_tag else None,
|
400
|
-
)
|