snowflake-ml-python 1.6.1__py3-none-any.whl → 1.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/__init__.py +4 -0
- snowflake/cortex/_classify_text.py +2 -2
- snowflake/cortex/_embed_text_1024.py +37 -0
- snowflake/cortex/_embed_text_768.py +37 -0
- snowflake/cortex/_extract_answer.py +2 -2
- snowflake/cortex/_sentiment.py +2 -2
- snowflake/cortex/_summarize.py +2 -2
- snowflake/cortex/_translate.py +2 -2
- snowflake/cortex/_util.py +4 -4
- snowflake/ml/_internal/env_utils.py +5 -5
- snowflake/ml/_internal/exceptions/error_codes.py +2 -0
- snowflake/ml/_internal/telemetry.py +142 -20
- snowflake/ml/_internal/utils/db_utils.py +50 -0
- snowflake/ml/_internal/utils/identifier.py +48 -11
- snowflake/ml/_internal/utils/service_logger.py +63 -0
- snowflake/ml/_internal/utils/snowflake_env.py +23 -13
- snowflake/ml/_internal/utils/sql_identifier.py +26 -2
- snowflake/ml/_internal/utils/table_manager.py +19 -1
- snowflake/ml/data/_internal/arrow_ingestor.py +1 -11
- snowflake/ml/data/data_connector.py +33 -7
- snowflake/ml/data/ingestor_utils.py +20 -10
- snowflake/ml/data/torch_utils.py +68 -0
- snowflake/ml/dataset/dataset.py +1 -3
- snowflake/ml/feature_store/access_manager.py +3 -3
- snowflake/ml/feature_store/feature_store.py +60 -19
- snowflake/ml/feature_store/feature_view.py +84 -30
- snowflake/ml/fileset/embedded_stage_fs.py +1 -1
- snowflake/ml/fileset/fileset.py +1 -1
- snowflake/ml/fileset/sfcfs.py +9 -3
- snowflake/ml/fileset/stage_fs.py +2 -1
- snowflake/ml/lineage/lineage_node.py +7 -2
- snowflake/ml/model/__init__.py +1 -2
- snowflake/ml/model/_client/model/model_version_impl.py +96 -12
- snowflake/ml/model/_client/ops/model_ops.py +124 -6
- snowflake/ml/model/_client/ops/service_ops.py +309 -9
- snowflake/ml/model/_client/service/model_deployment_spec.py +8 -5
- snowflake/ml/model/_client/service/model_deployment_spec_schema.py +2 -2
- snowflake/ml/model/_client/sql/_base.py +5 -0
- snowflake/ml/model/_client/sql/model.py +1 -0
- snowflake/ml/model/_client/sql/model_version.py +9 -5
- snowflake/ml/model/_client/sql/service.py +121 -20
- snowflake/ml/model/_model_composer/model_composer.py +11 -39
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +31 -11
- snowflake/ml/model/_packager/model_env/model_env.py +4 -38
- snowflake/ml/model/_packager/model_handlers/_utils.py +134 -28
- snowflake/ml/model/_packager/model_handlers/catboost.py +31 -30
- snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py +26 -18
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +31 -58
- snowflake/ml/model/_packager/model_handlers/mlflow.py +3 -5
- snowflake/ml/model/_packager/model_handlers/model_objective_utils.py +169 -0
- snowflake/ml/model/_packager/model_handlers/sentence_transformers.py +15 -8
- snowflake/ml/model/_packager/model_handlers/sklearn.py +56 -60
- snowflake/ml/model/_packager/model_handlers/snowmlmodel.py +141 -9
- snowflake/ml/model/_packager/model_handlers/torchscript.py +2 -2
- snowflake/ml/model/_packager/model_handlers/xgboost.py +63 -48
- snowflake/ml/model/_packager/model_meta/model_meta.py +16 -42
- snowflake/ml/model/_packager/model_meta/model_meta_schema.py +1 -14
- snowflake/ml/model/_packager/model_packager.py +14 -8
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +11 -0
- snowflake/ml/model/_signatures/pytorch_handler.py +1 -1
- snowflake/ml/model/_signatures/snowpark_handler.py +3 -2
- snowflake/ml/model/_signatures/utils.py +9 -0
- snowflake/ml/model/type_hints.py +12 -145
- snowflake/ml/modeling/_internal/constants.py +1 -0
- snowflake/ml/modeling/_internal/local_implementations/pandas_handlers.py +5 -5
- snowflake/ml/modeling/_internal/local_implementations/pandas_trainer.py +9 -6
- snowflake/ml/modeling/_internal/model_specifications.py +2 -0
- snowflake/ml/modeling/_internal/model_trainer.py +1 -0
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +2 -4
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_handlers.py +5 -5
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +130 -166
- snowflake/ml/modeling/_internal/snowpark_implementations/xgboost_external_memory_trainer.py +0 -1
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +61 -21
- snowflake/ml/modeling/cluster/affinity_propagation.py +61 -21
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +61 -21
- snowflake/ml/modeling/cluster/birch.py +61 -21
- snowflake/ml/modeling/cluster/bisecting_k_means.py +61 -21
- snowflake/ml/modeling/cluster/dbscan.py +61 -21
- snowflake/ml/modeling/cluster/feature_agglomeration.py +61 -21
- snowflake/ml/modeling/cluster/k_means.py +61 -21
- snowflake/ml/modeling/cluster/mean_shift.py +61 -21
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +61 -21
- snowflake/ml/modeling/cluster/optics.py +61 -21
- snowflake/ml/modeling/cluster/spectral_biclustering.py +61 -21
- snowflake/ml/modeling/cluster/spectral_clustering.py +61 -21
- snowflake/ml/modeling/cluster/spectral_coclustering.py +61 -21
- snowflake/ml/modeling/compose/column_transformer.py +61 -21
- snowflake/ml/modeling/compose/transformed_target_regressor.py +61 -21
- snowflake/ml/modeling/covariance/elliptic_envelope.py +61 -21
- snowflake/ml/modeling/covariance/empirical_covariance.py +61 -21
- snowflake/ml/modeling/covariance/graphical_lasso.py +61 -21
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +61 -21
- snowflake/ml/modeling/covariance/ledoit_wolf.py +61 -21
- snowflake/ml/modeling/covariance/min_cov_det.py +61 -21
- snowflake/ml/modeling/covariance/oas.py +61 -21
- snowflake/ml/modeling/covariance/shrunk_covariance.py +61 -21
- snowflake/ml/modeling/decomposition/dictionary_learning.py +61 -21
- snowflake/ml/modeling/decomposition/factor_analysis.py +61 -21
- snowflake/ml/modeling/decomposition/fast_ica.py +61 -21
- snowflake/ml/modeling/decomposition/incremental_pca.py +61 -21
- snowflake/ml/modeling/decomposition/kernel_pca.py +61 -21
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +61 -21
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +61 -21
- snowflake/ml/modeling/decomposition/pca.py +61 -21
- snowflake/ml/modeling/decomposition/sparse_pca.py +61 -21
- snowflake/ml/modeling/decomposition/truncated_svd.py +61 -21
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +61 -21
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +61 -21
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/bagging_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/bagging_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/isolation_forest.py +61 -21
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/stacking_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/voting_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/voting_regressor.py +61 -21
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +61 -21
- snowflake/ml/modeling/feature_selection/select_fdr.py +61 -21
- snowflake/ml/modeling/feature_selection/select_fpr.py +61 -21
- snowflake/ml/modeling/feature_selection/select_fwe.py +61 -21
- snowflake/ml/modeling/feature_selection/select_k_best.py +61 -21
- snowflake/ml/modeling/feature_selection/select_percentile.py +61 -21
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +61 -21
- snowflake/ml/modeling/feature_selection/variance_threshold.py +61 -21
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +61 -21
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +61 -21
- snowflake/ml/modeling/impute/iterative_imputer.py +61 -21
- snowflake/ml/modeling/impute/knn_imputer.py +61 -21
- snowflake/ml/modeling/impute/missing_indicator.py +61 -21
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +61 -21
- snowflake/ml/modeling/kernel_approximation/nystroem.py +61 -21
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +61 -21
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +61 -21
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +61 -21
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +61 -21
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +61 -21
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/ard_regression.py +61 -21
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +61 -21
- snowflake/ml/modeling/linear_model/elastic_net.py +61 -21
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +61 -21
- snowflake/ml/modeling/linear_model/gamma_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/huber_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/lars.py +61 -21
- snowflake/ml/modeling/linear_model/lars_cv.py +61 -21
- snowflake/ml/modeling/linear_model/lasso.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_cv.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_lars.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +61 -21
- snowflake/ml/modeling/linear_model/linear_regression.py +61 -21
- snowflake/ml/modeling/linear_model/logistic_regression.py +61 -21
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +61 -21
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +61 -21
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +61 -21
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/perceptron.py +61 -21
- snowflake/ml/modeling/linear_model/poisson_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/ransac_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/ridge.py +61 -21
- snowflake/ml/modeling/linear_model/ridge_classifier.py +61 -21
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +61 -21
- snowflake/ml/modeling/linear_model/ridge_cv.py +61 -21
- snowflake/ml/modeling/linear_model/sgd_classifier.py +61 -21
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +61 -21
- snowflake/ml/modeling/linear_model/sgd_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +61 -21
- snowflake/ml/modeling/manifold/isomap.py +61 -21
- snowflake/ml/modeling/manifold/mds.py +61 -21
- snowflake/ml/modeling/manifold/spectral_embedding.py +61 -21
- snowflake/ml/modeling/manifold/tsne.py +61 -21
- snowflake/ml/modeling/metrics/metrics_utils.py +2 -2
- snowflake/ml/modeling/metrics/ranking.py +0 -3
- snowflake/ml/modeling/metrics/regression.py +0 -3
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +61 -21
- snowflake/ml/modeling/mixture/gaussian_mixture.py +61 -21
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +61 -21
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +61 -21
- snowflake/ml/modeling/multiclass/output_code_classifier.py +61 -21
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/complement_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +61 -21
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +61 -21
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +61 -21
- snowflake/ml/modeling/neighbors/kernel_density.py +61 -21
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +61 -21
- snowflake/ml/modeling/neighbors/nearest_centroid.py +61 -21
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +61 -21
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +61 -21
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +61 -21
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +61 -21
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +61 -21
- snowflake/ml/modeling/neural_network/mlp_classifier.py +61 -21
- snowflake/ml/modeling/neural_network/mlp_regressor.py +61 -21
- snowflake/ml/modeling/parameters/disable_model_tracer.py +5 -0
- snowflake/ml/modeling/pipeline/pipeline.py +1 -13
- snowflake/ml/modeling/preprocessing/polynomial_features.py +61 -21
- snowflake/ml/modeling/semi_supervised/label_propagation.py +61 -21
- snowflake/ml/modeling/semi_supervised/label_spreading.py +61 -21
- snowflake/ml/modeling/svm/linear_svc.py +61 -21
- snowflake/ml/modeling/svm/linear_svr.py +61 -21
- snowflake/ml/modeling/svm/nu_svc.py +61 -21
- snowflake/ml/modeling/svm/nu_svr.py +61 -21
- snowflake/ml/modeling/svm/svc.py +61 -21
- snowflake/ml/modeling/svm/svr.py +61 -21
- snowflake/ml/modeling/tree/decision_tree_classifier.py +61 -21
- snowflake/ml/modeling/tree/decision_tree_regressor.py +61 -21
- snowflake/ml/modeling/tree/extra_tree_classifier.py +61 -21
- snowflake/ml/modeling/tree/extra_tree_regressor.py +61 -21
- snowflake/ml/modeling/xgboost/xgb_classifier.py +64 -23
- snowflake/ml/modeling/xgboost/xgb_regressor.py +64 -23
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +64 -23
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +64 -23
- snowflake/ml/monitoring/_client/model_monitor.py +126 -0
- snowflake/ml/monitoring/_client/model_monitor_manager.py +361 -0
- snowflake/ml/monitoring/_client/model_monitor_version.py +1 -0
- snowflake/ml/monitoring/_client/monitor_sql_client.py +1335 -0
- snowflake/ml/monitoring/_client/queries/record_count.ssql +14 -0
- snowflake/ml/monitoring/_client/queries/rmse.ssql +28 -0
- snowflake/ml/monitoring/entities/model_monitor_config.py +28 -0
- snowflake/ml/monitoring/entities/model_monitor_interval.py +46 -0
- snowflake/ml/monitoring/entities/output_score_type.py +90 -0
- snowflake/ml/registry/_manager/model_manager.py +4 -0
- snowflake/ml/registry/registry.py +166 -8
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/METADATA +43 -9
- snowflake_ml_python-1.6.3.dist-info/RECORD +400 -0
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/WHEEL +1 -1
- snowflake/ml/_internal/container_services/image_registry/credential.py +0 -84
- snowflake/ml/_internal/container_services/image_registry/http_client.py +0 -127
- snowflake/ml/_internal/container_services/image_registry/imagelib.py +0 -400
- snowflake/ml/_internal/container_services/image_registry/registry_client.py +0 -212
- snowflake/ml/_internal/utils/log_stream_processor.py +0 -30
- snowflake/ml/_internal/utils/session_token_manager.py +0 -46
- snowflake/ml/_internal/utils/spcs_attribution_utils.py +0 -122
- snowflake/ml/_internal/utils/uri.py +0 -77
- snowflake/ml/data/torch_dataset.py +0 -33
- snowflake/ml/model/_api.py +0 -568
- snowflake/ml/model/_deploy_client/image_builds/base_image_builder.py +0 -12
- snowflake/ml/model/_deploy_client/image_builds/client_image_builder.py +0 -249
- snowflake/ml/model/_deploy_client/image_builds/docker_context.py +0 -130
- snowflake/ml/model/_deploy_client/image_builds/gunicorn_run.sh +0 -36
- snowflake/ml/model/_deploy_client/image_builds/inference_server/main.py +0 -268
- snowflake/ml/model/_deploy_client/image_builds/server_image_builder.py +0 -215
- snowflake/ml/model/_deploy_client/image_builds/templates/dockerfile_template +0 -53
- snowflake/ml/model/_deploy_client/image_builds/templates/image_build_job_spec_template +0 -38
- snowflake/ml/model/_deploy_client/image_builds/templates/kaniko_shell_script_template +0 -105
- snowflake/ml/model/_deploy_client/snowservice/deploy.py +0 -611
- snowflake/ml/model/_deploy_client/snowservice/deploy_options.py +0 -116
- snowflake/ml/model/_deploy_client/snowservice/instance_types.py +0 -10
- snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template +0 -28
- snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template_with_model +0 -21
- snowflake/ml/model/_deploy_client/utils/constants.py +0 -48
- snowflake/ml/model/_deploy_client/utils/snowservice_client.py +0 -280
- snowflake/ml/model/_deploy_client/warehouse/deploy.py +0 -202
- snowflake/ml/model/_deploy_client/warehouse/infer_template.py +0 -99
- snowflake/ml/model/_packager/model_handlers/llm.py +0 -267
- snowflake/ml/model/_packager/model_meta/_core_requirements.py +0 -11
- snowflake/ml/model/deploy_platforms.py +0 -6
- snowflake/ml/model/models/llm.py +0 -104
- snowflake/ml/monitoring/monitor.py +0 -203
- snowflake/ml/registry/_initial_schema.py +0 -142
- snowflake/ml/registry/_schema.py +0 -82
- snowflake/ml/registry/_schema_upgrade_plans.py +0 -116
- snowflake/ml/registry/_schema_version_manager.py +0 -163
- snowflake/ml/registry/model_registry.py +0 -2048
- snowflake_ml_python-1.6.1.dist-info/RECORD +0 -422
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/LICENSE.txt +0 -0
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/top_level.txt +0 -0
@@ -1,127 +0,0 @@
|
|
1
|
-
import http
|
2
|
-
import json
|
3
|
-
import logging
|
4
|
-
import time
|
5
|
-
from typing import Any, Callable, Dict, FrozenSet, Optional
|
6
|
-
from urllib.parse import urlparse, urlunparse
|
7
|
-
|
8
|
-
import requests
|
9
|
-
|
10
|
-
from snowflake import snowpark
|
11
|
-
from snowflake.ml._internal.exceptions import (
|
12
|
-
error_codes,
|
13
|
-
exceptions as snowml_exceptions,
|
14
|
-
)
|
15
|
-
from snowflake.ml._internal.utils import retryable_http, session_token_manager
|
16
|
-
|
17
|
-
logger = logging.getLogger(__name__)
|
18
|
-
|
19
|
-
_MAX_RETRIES = 5
|
20
|
-
_RETRY_DELAY_SECONDS = 1
|
21
|
-
_RETRYABLE_HTTP_CODE = frozenset([http.HTTPStatus.UNAUTHORIZED])
|
22
|
-
|
23
|
-
|
24
|
-
def retry_on_error(
|
25
|
-
http_call_function: Callable[..., requests.Response],
|
26
|
-
retryable_http_code: FrozenSet[http.HTTPStatus] = _RETRYABLE_HTTP_CODE,
|
27
|
-
) -> Callable[..., requests.Response]:
|
28
|
-
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
29
|
-
retry_delay_seconds = _RETRY_DELAY_SECONDS
|
30
|
-
for attempt in range(1, _MAX_RETRIES + 1):
|
31
|
-
resp = http_call_function(*args, **kwargs)
|
32
|
-
if resp.status_code in retryable_http_code:
|
33
|
-
logger.warning(
|
34
|
-
f"Received {resp.status_code} status code. Retrying " f"(attempt {attempt}/{_MAX_RETRIES})..."
|
35
|
-
)
|
36
|
-
time.sleep(retry_delay_seconds)
|
37
|
-
retry_delay_seconds *= 2 # Increase the retry delay exponentially
|
38
|
-
if attempt < _MAX_RETRIES:
|
39
|
-
assert isinstance(args[0], ImageRegistryHttpClient)
|
40
|
-
args[0]._fetch_bearer_token()
|
41
|
-
else:
|
42
|
-
return resp
|
43
|
-
|
44
|
-
if attempt == _MAX_RETRIES:
|
45
|
-
raise snowml_exceptions.SnowflakeMLException(
|
46
|
-
error_code=error_codes.INTERNAL_SNOWFLAKE_IMAGE_REGISTRY_ERROR,
|
47
|
-
original_exception=RuntimeError(
|
48
|
-
f"Failed to authenticate to registry after max retries {attempt} \n"
|
49
|
-
f"Status {resp.status_code},"
|
50
|
-
f"{str(resp.text)}"
|
51
|
-
),
|
52
|
-
)
|
53
|
-
|
54
|
-
return wrapper
|
55
|
-
|
56
|
-
|
57
|
-
class ImageRegistryHttpClient:
|
58
|
-
"""
|
59
|
-
An image registry HTTP client utilizes a retryable HTTP client underneath. Its primary function is to facilitate
|
60
|
-
re-authentication with the image registry by obtaining a new GS token, which is then used to acquire a new bearer
|
61
|
-
token for subsequent HTTP request authentication.
|
62
|
-
|
63
|
-
Ideally you should not use this client directly. Please use ImageRegistryClient for image registry-specific
|
64
|
-
operations. For general use of a retryable HTTP client, consider using the "retryable_http" module.
|
65
|
-
"""
|
66
|
-
|
67
|
-
def __init__(self, *, repo_url: str, session: Optional[snowpark.Session] = None, no_cred: bool = False) -> None:
|
68
|
-
self._repo_url = repo_url
|
69
|
-
self._retryable_http = retryable_http.get_http_client()
|
70
|
-
self._no_cred = no_cred
|
71
|
-
|
72
|
-
if not self._no_cred:
|
73
|
-
self._bearer_token = ""
|
74
|
-
assert session is not None
|
75
|
-
self._session_token_manager = session_token_manager.SessionTokenManager(session)
|
76
|
-
|
77
|
-
def _with_bearer_token_header(self, headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
|
78
|
-
if self._no_cred:
|
79
|
-
return {} if not headers else headers.copy()
|
80
|
-
|
81
|
-
if not self._bearer_token:
|
82
|
-
self._fetch_bearer_token()
|
83
|
-
assert self._bearer_token
|
84
|
-
new_headers = {} if not headers else headers.copy()
|
85
|
-
new_headers["Authorization"] = f"Bearer {self._bearer_token}"
|
86
|
-
return new_headers
|
87
|
-
|
88
|
-
def _fetch_bearer_token(self) -> None:
|
89
|
-
resp = self._login()
|
90
|
-
self._bearer_token = str(json.loads(resp.text)["token"])
|
91
|
-
|
92
|
-
def _login(self) -> requests.Response:
|
93
|
-
"""Log in to image registry. repo_url is expected to set when _login function is invoked.
|
94
|
-
|
95
|
-
Returns:
|
96
|
-
Bearer token when login succeeded.
|
97
|
-
"""
|
98
|
-
parsed_url = urlparse(self._repo_url)
|
99
|
-
scheme = parsed_url.scheme
|
100
|
-
host = parsed_url.netloc
|
101
|
-
|
102
|
-
login_path = "/login" # Construct the login path
|
103
|
-
url_tuple = (scheme, host, login_path, "", "", "")
|
104
|
-
login_url = urlunparse(url_tuple)
|
105
|
-
|
106
|
-
base64_encoded_token = self._session_token_manager.get_base64_encoded_token()
|
107
|
-
return self._retryable_http.get(login_url, headers={"Authorization": f"Basic {base64_encoded_token}"})
|
108
|
-
|
109
|
-
@retry_on_error
|
110
|
-
def head(self, api_url: str, *, headers: Optional[Dict[str, str]] = None) -> requests.Response:
|
111
|
-
return self._retryable_http.head(api_url, headers=self._with_bearer_token_header(headers))
|
112
|
-
|
113
|
-
@retry_on_error
|
114
|
-
def get(self, api_url: str, *, headers: Optional[Dict[str, str]] = None) -> requests.Response:
|
115
|
-
return self._retryable_http.get(api_url, headers=self._with_bearer_token_header(headers))
|
116
|
-
|
117
|
-
@retry_on_error
|
118
|
-
def put(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
|
119
|
-
return self._retryable_http.put(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
|
120
|
-
|
121
|
-
@retry_on_error
|
122
|
-
def post(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
|
123
|
-
return self._retryable_http.post(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
|
124
|
-
|
125
|
-
@retry_on_error
|
126
|
-
def patch(self, api_url: str, *, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> requests.Response:
|
127
|
-
return self._retryable_http.patch(api_url, headers=self._with_bearer_token_header(headers), **kwargs)
|
@@ -1,400 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
A minimal pure python library to copy images between two remote registries.
|
3
|
-
|
4
|
-
This library only supports a limited set of features:
|
5
|
-
- Works only with docker and OCI manifests and manifest lists for multiarch images (most newer images)
|
6
|
-
- Supported OCI manifest type: application/vnd.oci.image.manifest.v1+json
|
7
|
-
- Supported Docker manifest type: application/vnd.docker.distribution.manifest.v2+json
|
8
|
-
- Supports only pulling a single architecture from a multiarch image. Does not support pulling all architectures.
|
9
|
-
- Supports only schemaVersion 2.
|
10
|
-
- Streams images from source to destination without any intermediate disk storage in chunks.
|
11
|
-
- Does not support copying in parallel.
|
12
|
-
|
13
|
-
It's recommended to use this library to copy previously tested images using sha256 to avoid surprises
|
14
|
-
with respect to compatibility.
|
15
|
-
"""
|
16
|
-
|
17
|
-
import dataclasses
|
18
|
-
import hashlib
|
19
|
-
import io
|
20
|
-
import json
|
21
|
-
import logging
|
22
|
-
from collections import namedtuple
|
23
|
-
from typing import Dict, List, Optional, Tuple
|
24
|
-
|
25
|
-
import requests
|
26
|
-
|
27
|
-
from snowflake.ml._internal.container_services.image_registry import (
|
28
|
-
http_client as image_registry_http_client,
|
29
|
-
)
|
30
|
-
|
31
|
-
# Common HTTP headers
|
32
|
-
_CONTENT_LENGTH_HEADER = "content-length"
|
33
|
-
_CONTENT_TYPE_HEADER = "content-type"
|
34
|
-
_CONTENT_RANGE_HEADER = "content-range"
|
35
|
-
_LOCATION_HEADER = "location"
|
36
|
-
_AUTHORIZATION_HEADER = "Authorization"
|
37
|
-
_ACCEPT_HEADER = "accept"
|
38
|
-
|
39
|
-
_OCI_MANIFEST_LIST_TYPE = "application/vnd.oci.image.index.v1+json"
|
40
|
-
_DOCKER_MANIFEST_LIST_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json"
|
41
|
-
|
42
|
-
_OCI_MANIFEST_TYPE = "application/vnd.oci.image.manifest.v1+json"
|
43
|
-
_DOCKER_MANIFEST_TYPE = "application/vnd.docker.distribution.manifest.v2+json"
|
44
|
-
|
45
|
-
ALL_SUPPORTED_MEDIA_TYPES = [
|
46
|
-
_OCI_MANIFEST_LIST_TYPE,
|
47
|
-
_DOCKER_MANIFEST_LIST_TYPE,
|
48
|
-
_OCI_MANIFEST_TYPE,
|
49
|
-
_DOCKER_MANIFEST_TYPE,
|
50
|
-
]
|
51
|
-
_MANIFEST_SUPPORTED_KEYS = {"schemaVersion", "mediaType", "config", "layers"}
|
52
|
-
|
53
|
-
# Architecture descriptor as a named tuple
|
54
|
-
_Arch = namedtuple("_Arch", ["arch_name", "os"])
|
55
|
-
|
56
|
-
logger = logging.getLogger(__name__)
|
57
|
-
|
58
|
-
|
59
|
-
@dataclasses.dataclass
|
60
|
-
class ImageDescriptor:
|
61
|
-
"""
|
62
|
-
Create an image descriptor.
|
63
|
-
|
64
|
-
registry_name: the name of the registry like gcr.io
|
65
|
-
repository_name: the name of the repository like kaniko-project/executor
|
66
|
-
tag: the tag of the image like v1.6.0
|
67
|
-
digest: the sha256 digest of the image like sha256:b8c0...
|
68
|
-
protocol: the protocol to use, defaults to https
|
69
|
-
|
70
|
-
Only a tag or a digest must be specified, not both.
|
71
|
-
"""
|
72
|
-
|
73
|
-
registry_name: str
|
74
|
-
repository_name: str
|
75
|
-
tag: Optional[str] = None
|
76
|
-
digest: Optional[str] = None
|
77
|
-
protocol: str = "https"
|
78
|
-
|
79
|
-
def __baseurl(self) -> str:
|
80
|
-
return f"{self.protocol}://{self.registry_name}/v2/"
|
81
|
-
|
82
|
-
def manifest_link(self) -> str:
|
83
|
-
return f"{self.__baseurl()}{self.repository_name}/manifests/{self.tag or self.digest}"
|
84
|
-
|
85
|
-
def blob_link(self, digest: str) -> str:
|
86
|
-
return f"{self.__baseurl()}{self.repository_name}/blobs/{digest}"
|
87
|
-
|
88
|
-
def blob_upload_link(self) -> str:
|
89
|
-
return f"{self.__baseurl()}{self.repository_name}/blobs/uploads/"
|
90
|
-
|
91
|
-
def manifest_upload_link(self, tag: str) -> str:
|
92
|
-
return f"{self.__baseurl()}{self.repository_name}/manifests/{tag}"
|
93
|
-
|
94
|
-
def __str__(self) -> str:
|
95
|
-
return f"{self.registry_name}/{self.repository_name}@{self.tag or self.digest}"
|
96
|
-
|
97
|
-
|
98
|
-
class Manifest:
|
99
|
-
def __init__(self, manifest_bytes: bytes, manifest_digest: str) -> None:
|
100
|
-
"""Create a manifest object from the manifest JSON dict.
|
101
|
-
|
102
|
-
Args:
|
103
|
-
manifest_bytes: manifest content in bytes.
|
104
|
-
manifest_digest: SHA256 digest.
|
105
|
-
"""
|
106
|
-
self.manifest_bytes = manifest_bytes
|
107
|
-
self.manifest = json.loads(manifest_bytes.decode("utf-8"))
|
108
|
-
self.__validate(self.manifest)
|
109
|
-
|
110
|
-
self.manifest_digest = manifest_digest
|
111
|
-
self.media_type = self.manifest["mediaType"]
|
112
|
-
|
113
|
-
def get_blob_digests(self) -> List[str]:
|
114
|
-
"""
|
115
|
-
Get the list of blob digests from the manifest including config and layers.
|
116
|
-
"""
|
117
|
-
blobs = []
|
118
|
-
blobs.extend([x["digest"] for x in self.manifest["layers"]])
|
119
|
-
blobs.append(self.manifest["config"]["digest"])
|
120
|
-
|
121
|
-
return blobs
|
122
|
-
|
123
|
-
def __validate(self, manifest: Dict[str, str]) -> None:
|
124
|
-
"""
|
125
|
-
Validate the manifest.
|
126
|
-
"""
|
127
|
-
assert (
|
128
|
-
manifest.keys() == _MANIFEST_SUPPORTED_KEYS
|
129
|
-
), f"Manifest must contain all keys and no more {_MANIFEST_SUPPORTED_KEYS}"
|
130
|
-
assert int(manifest["schemaVersion"]) == 2, "Only manifest schemaVersion 2 is supported"
|
131
|
-
assert manifest["mediaType"] in [
|
132
|
-
_OCI_MANIFEST_TYPE,
|
133
|
-
_DOCKER_MANIFEST_TYPE,
|
134
|
-
], f'Unsupported mediaType {manifest["mediaType"]}'
|
135
|
-
|
136
|
-
def __str__(self) -> str:
|
137
|
-
"""
|
138
|
-
Return the manifest as a string.
|
139
|
-
"""
|
140
|
-
return json.dumps(self.manifest, indent=4)
|
141
|
-
|
142
|
-
|
143
|
-
@dataclasses.dataclass
|
144
|
-
class BlobTransfer:
|
145
|
-
"""
|
146
|
-
Helper class to transfer a blob from one registry to another
|
147
|
-
in small chunks using in-memory buffering.
|
148
|
-
"""
|
149
|
-
|
150
|
-
# Uploads in chunks of 1MB
|
151
|
-
chunk_size_bytes = 1024 * 1024
|
152
|
-
|
153
|
-
src_image: ImageDescriptor
|
154
|
-
dest_image: ImageDescriptor
|
155
|
-
manifest: Manifest
|
156
|
-
src_image_registry_http_client: image_registry_http_client.ImageRegistryHttpClient
|
157
|
-
dest_image_registry_http_client: image_registry_http_client.ImageRegistryHttpClient
|
158
|
-
|
159
|
-
def upload_all_blobs(self) -> None:
|
160
|
-
blob_digests = self.manifest.get_blob_digests()
|
161
|
-
logger.debug(f"Found {len(blob_digests)} blobs for {self.src_image}")
|
162
|
-
|
163
|
-
for blob_digest in blob_digests:
|
164
|
-
logger.debug(f"Transferring blob {blob_digest} from {self.src_image} to {self.dest_image}")
|
165
|
-
if self._should_upload(blob_digest):
|
166
|
-
self._transfer(blob_digest)
|
167
|
-
else:
|
168
|
-
logger.debug(f"Blob {blob_digest} already exists in {self.dest_image}")
|
169
|
-
|
170
|
-
def _should_upload(self, blob_digest: str) -> bool:
|
171
|
-
"""
|
172
|
-
Check if the blob already exists in the destination registry.
|
173
|
-
"""
|
174
|
-
resp = self.dest_image_registry_http_client.head(self.dest_image.blob_link(blob_digest), headers={})
|
175
|
-
return resp.status_code != 200
|
176
|
-
|
177
|
-
def _fetch_blob(self, blob_digest: str) -> Tuple[io.BytesIO, int]:
|
178
|
-
"""
|
179
|
-
Fetch a stream to the blob from the source registry.
|
180
|
-
"""
|
181
|
-
src_blob_link = self.src_image.blob_link(blob_digest)
|
182
|
-
headers = {_CONTENT_LENGTH_HEADER: "0"}
|
183
|
-
resp = self.src_image_registry_http_client.get(src_blob_link, headers=headers)
|
184
|
-
|
185
|
-
assert resp.status_code == 200, f"Blob GET failed with code {resp.status_code}"
|
186
|
-
assert _CONTENT_LENGTH_HEADER in resp.headers, f"Blob does not contain {_CONTENT_LENGTH_HEADER}"
|
187
|
-
|
188
|
-
return io.BytesIO(resp.content), int(resp.headers[_CONTENT_LENGTH_HEADER])
|
189
|
-
|
190
|
-
def _get_upload_url(self) -> str:
|
191
|
-
"""
|
192
|
-
Obtain the upload URL from the destination registry.
|
193
|
-
"""
|
194
|
-
response = self.dest_image_registry_http_client.post(self.dest_image.blob_upload_link())
|
195
|
-
assert (
|
196
|
-
response.status_code == 202
|
197
|
-
), f"Failed to get the upload URL to destination. Status {response.status_code}. {str(response.content)}"
|
198
|
-
return str(response.headers[_LOCATION_HEADER])
|
199
|
-
|
200
|
-
def _upload_blob(self, blob_digest: str, blob_data: io.BytesIO, content_length: int) -> None:
|
201
|
-
"""
|
202
|
-
Upload a blob to the destination registry.
|
203
|
-
"""
|
204
|
-
upload_url = self._get_upload_url()
|
205
|
-
headers = {
|
206
|
-
_CONTENT_TYPE_HEADER: "application/octet-stream",
|
207
|
-
}
|
208
|
-
|
209
|
-
# Use chunked transfer
|
210
|
-
# This can be optimized to use a single PUT request for small blobs
|
211
|
-
next_loc = upload_url
|
212
|
-
start_byte = 0
|
213
|
-
while start_byte < content_length:
|
214
|
-
chunk = blob_data.read(self.chunk_size_bytes)
|
215
|
-
chunk_length = len(chunk)
|
216
|
-
end_byte = start_byte + chunk_length - 1
|
217
|
-
|
218
|
-
headers[_CONTENT_RANGE_HEADER] = f"{start_byte}-{end_byte}"
|
219
|
-
headers[_CONTENT_LENGTH_HEADER] = str(chunk_length)
|
220
|
-
|
221
|
-
resp = self.dest_image_registry_http_client.patch(next_loc, headers=headers, data=chunk)
|
222
|
-
assert resp.status_code == 202, f"Blob PATCH failed with code {resp.status_code}"
|
223
|
-
|
224
|
-
next_loc = resp.headers[_LOCATION_HEADER]
|
225
|
-
start_byte += chunk_length
|
226
|
-
|
227
|
-
# Finalize the upload
|
228
|
-
resp = self.dest_image_registry_http_client.put(f"{next_loc}&digest={blob_digest}")
|
229
|
-
assert resp.status_code == 201, f"Blob PUT failed with code {resp.status_code}"
|
230
|
-
|
231
|
-
def _transfer(self, blob_digest: str) -> None:
|
232
|
-
"""
|
233
|
-
Transfer a blob from the source registry to the destination registry.
|
234
|
-
"""
|
235
|
-
blob_data, content_length = self._fetch_blob(blob_digest)
|
236
|
-
self._upload_blob(blob_digest, blob_data, content_length)
|
237
|
-
|
238
|
-
|
239
|
-
def get_bytes_with_sha_verification(resp: requests.Response, sha256_digest: str) -> Tuple[bytes, str]:
|
240
|
-
"""Get the bytes of a response and verify the sha256 digest.
|
241
|
-
|
242
|
-
Args:
|
243
|
-
resp: the response object
|
244
|
-
sha256_digest: the expected sha256 digest in format "sha256:b8c0..."
|
245
|
-
|
246
|
-
Returns:
|
247
|
-
(res, sha256_digest)
|
248
|
-
|
249
|
-
"""
|
250
|
-
digest = hashlib.sha256()
|
251
|
-
chunks = []
|
252
|
-
for chunk in resp.iter_content(chunk_size=8192):
|
253
|
-
digest.update(chunk)
|
254
|
-
chunks.append(chunk)
|
255
|
-
|
256
|
-
calculated_digest = digest.hexdigest()
|
257
|
-
assert not sha256_digest or sha256_digest.endswith(calculated_digest), "SHA256 digest does not match"
|
258
|
-
|
259
|
-
content = b"".join(chunks) # Minimize allocations by joining chunks
|
260
|
-
return content, calculated_digest
|
261
|
-
|
262
|
-
|
263
|
-
def get_manifest(
|
264
|
-
image_descriptor: ImageDescriptor, arch: _Arch, retryable_http: image_registry_http_client.ImageRegistryHttpClient
|
265
|
-
) -> Manifest:
|
266
|
-
"""Get the manifest of an image from the remote registry.
|
267
|
-
|
268
|
-
Args:
|
269
|
-
image_descriptor: the image descriptor
|
270
|
-
arch: the architecture to filter for if it's a multi-arch image
|
271
|
-
retryable_http: a retryable http client.
|
272
|
-
|
273
|
-
Returns:
|
274
|
-
Manifest object.
|
275
|
-
|
276
|
-
"""
|
277
|
-
logger.debug(f"Getting manifest from {image_descriptor.manifest_link()}")
|
278
|
-
|
279
|
-
headers = {_ACCEPT_HEADER: ",".join(ALL_SUPPORTED_MEDIA_TYPES)}
|
280
|
-
|
281
|
-
response = retryable_http.get(image_descriptor.manifest_link(), headers=headers)
|
282
|
-
assert response.status_code == 200, f"Manifest GET failed with code {response.status_code}, {response.text}"
|
283
|
-
|
284
|
-
assert image_descriptor.digest
|
285
|
-
manifest_bytes, manifest_digest = get_bytes_with_sha_verification(response, image_descriptor.digest)
|
286
|
-
manifest_json = json.loads(manifest_bytes.decode("utf-8"))
|
287
|
-
|
288
|
-
# If this is a manifest list, find the manifest for the specified architecture
|
289
|
-
# and recurse till we find the real manifest
|
290
|
-
if manifest_json["mediaType"] in [
|
291
|
-
_OCI_MANIFEST_LIST_TYPE,
|
292
|
-
_DOCKER_MANIFEST_LIST_TYPE,
|
293
|
-
]:
|
294
|
-
logger.debug("Found a multiarch image. Following manifest reference.")
|
295
|
-
|
296
|
-
assert "manifests" in manifest_json, "Manifest list does not contain manifests"
|
297
|
-
qualified_manifests = [
|
298
|
-
x
|
299
|
-
for x in manifest_json["manifests"]
|
300
|
-
if x["platform"]["architecture"] == arch.arch_name and x["platform"]["os"] == arch.os
|
301
|
-
]
|
302
|
-
assert (
|
303
|
-
len(qualified_manifests) == 1
|
304
|
-
), "Manifest list does not contain exactly one qualified manifest for this arch"
|
305
|
-
|
306
|
-
manifest_object = qualified_manifests[0]
|
307
|
-
manifest_digest = manifest_object["digest"]
|
308
|
-
|
309
|
-
logger.debug(f"Found manifest reference for arch {arch}: {manifest_digest}")
|
310
|
-
|
311
|
-
# Copy the image descriptor to fetch the arch-specific manifest
|
312
|
-
descriptor_copy = ImageDescriptor(
|
313
|
-
registry_name=image_descriptor.registry_name,
|
314
|
-
repository_name=image_descriptor.repository_name,
|
315
|
-
digest=manifest_digest,
|
316
|
-
tag=None,
|
317
|
-
)
|
318
|
-
|
319
|
-
# Supports only one level of manifest list nesting to avoid infinite recursion
|
320
|
-
return get_manifest(descriptor_copy, arch, retryable_http)
|
321
|
-
|
322
|
-
return Manifest(manifest_bytes, manifest_digest)
|
323
|
-
|
324
|
-
|
325
|
-
def put_manifest(
|
326
|
-
image_descriptor: ImageDescriptor,
|
327
|
-
manifest: Manifest,
|
328
|
-
retryable_http: image_registry_http_client.ImageRegistryHttpClient,
|
329
|
-
) -> None:
|
330
|
-
"""
|
331
|
-
Upload the given manifest to the destination registry.
|
332
|
-
"""
|
333
|
-
assert image_descriptor.tag is not None, "Tag must be specified for manifest upload"
|
334
|
-
headers = {_CONTENT_TYPE_HEADER: manifest.media_type}
|
335
|
-
url = image_descriptor.manifest_upload_link(image_descriptor.tag)
|
336
|
-
logger.debug(f"Uploading manifest to {url}")
|
337
|
-
response = retryable_http.put(url, headers=headers, data=manifest.manifest_bytes)
|
338
|
-
assert response.status_code == 201, f"Manifest PUT failed with code {response.status_code}"
|
339
|
-
|
340
|
-
|
341
|
-
def copy_image(
|
342
|
-
src_image: ImageDescriptor,
|
343
|
-
dest_image: ImageDescriptor,
|
344
|
-
arch: _Arch,
|
345
|
-
src_retryable_http: image_registry_http_client.ImageRegistryHttpClient,
|
346
|
-
dest_retryable_http: image_registry_http_client.ImageRegistryHttpClient,
|
347
|
-
) -> None:
|
348
|
-
logger.debug(f"Pulling image manifest for {src_image}")
|
349
|
-
|
350
|
-
# 1. Get the manifest
|
351
|
-
manifest = get_manifest(src_image, arch, src_retryable_http)
|
352
|
-
logger.debug(f"Manifest pulled for {src_image} with digest {manifest.manifest_digest}")
|
353
|
-
|
354
|
-
# 2: Retrieve all blob digests from manifest; fetch blob based on blob digest, then upload blob.
|
355
|
-
blob_transfer = BlobTransfer(
|
356
|
-
src_image,
|
357
|
-
dest_image,
|
358
|
-
manifest,
|
359
|
-
src_image_registry_http_client=src_retryable_http,
|
360
|
-
dest_image_registry_http_client=dest_retryable_http,
|
361
|
-
)
|
362
|
-
blob_transfer.upload_all_blobs()
|
363
|
-
|
364
|
-
# 3. Upload the manifest
|
365
|
-
logger.debug(f"All blobs copied successfully. Copying manifest for {src_image} to {dest_image}")
|
366
|
-
put_manifest(
|
367
|
-
dest_image,
|
368
|
-
manifest,
|
369
|
-
dest_retryable_http,
|
370
|
-
)
|
371
|
-
|
372
|
-
logger.debug(f"Image {src_image} copied to {dest_image}")
|
373
|
-
|
374
|
-
|
375
|
-
def convert_to_image_descriptor(
|
376
|
-
image_name: str,
|
377
|
-
with_digest: bool = False,
|
378
|
-
with_tag: bool = False,
|
379
|
-
) -> ImageDescriptor:
|
380
|
-
"""Convert a full image name to a ImageDescriptor object.
|
381
|
-
|
382
|
-
Args:
|
383
|
-
image_name: name of image.
|
384
|
-
with_digest: boolean to specify whether a digest is included in the image name
|
385
|
-
with_tag: boolean to specify whether a tag is included in the image name.
|
386
|
-
|
387
|
-
Returns:
|
388
|
-
An ImageDescriptor instance
|
389
|
-
"""
|
390
|
-
assert with_digest or with_tag, "image should contain either digest or tag"
|
391
|
-
sep = "@" if with_digest else ":"
|
392
|
-
parts = image_name.split("/")
|
393
|
-
assert len(parts[-1].split(sep)) == 2, f"Image {image_name} missing digest/tag"
|
394
|
-
tag_digest = parts[-1].split(sep)[1]
|
395
|
-
return ImageDescriptor(
|
396
|
-
registry_name=parts[0],
|
397
|
-
repository_name="/".join(parts[1:-1] + [parts[-1].split(sep)[0]]),
|
398
|
-
digest=tag_digest if with_digest else None,
|
399
|
-
tag=tag_digest if with_tag else None,
|
400
|
-
)
|