snowflake-ml-python 2.3.0__py3-none-any.whl → 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/_classify_text.py +51 -5
- snowflake/cortex/_complete.py +41 -9
- snowflake/cortex/_embed_text_1024.py +11 -5
- snowflake/cortex/_embed_text_768.py +11 -5
- snowflake/cortex/_extract_answer.py +54 -12
- snowflake/cortex/_sentiment.py +50 -10
- snowflake/cortex/_summarize.py +11 -5
- snowflake/cortex/_translate.py +11 -5
- snowflake/cortex/_util.py +35 -0
- snowflake/ml/experiment/_entities/run.py +14 -6
- snowflake/ml/experiment/callback/catboost.py +75 -0
- snowflake/ml/experiment/experiment_tracking.py +11 -7
- snowflake/ml/feature_store/feature_store.py +55 -0
- snowflake/ml/model/_client/model/batch_inference_job_specs.py +81 -1
- snowflake/ml/model/_client/model/batch_inference_task.py +24 -6
- snowflake/ml/model/_client/model/model_version_impl.py +97 -5
- snowflake/ml/model/_client/ops/service_ops.py +5 -4
- snowflake/ml/model/_client/service/inference_job_service_spec.py +1 -1
- snowflake/ml/model/_signatures/core.py +9 -4
- snowflake/ml/model/model_signature.py +85 -0
- snowflake/ml/modeling/_internal/model_specifications.py +10 -4
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +0 -2
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +2 -1
- snowflake/ml/modeling/cluster/affinity_propagation.py +2 -1
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +2 -1
- snowflake/ml/modeling/cluster/birch.py +2 -1
- snowflake/ml/modeling/cluster/bisecting_k_means.py +2 -1
- snowflake/ml/modeling/cluster/dbscan.py +2 -1
- snowflake/ml/modeling/cluster/feature_agglomeration.py +2 -1
- snowflake/ml/modeling/cluster/k_means.py +2 -1
- snowflake/ml/modeling/cluster/mean_shift.py +2 -1
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +2 -1
- snowflake/ml/modeling/cluster/optics.py +2 -1
- snowflake/ml/modeling/cluster/spectral_biclustering.py +2 -1
- snowflake/ml/modeling/cluster/spectral_clustering.py +2 -1
- snowflake/ml/modeling/cluster/spectral_coclustering.py +2 -1
- snowflake/ml/modeling/compose/column_transformer.py +2 -1
- snowflake/ml/modeling/compose/transformed_target_regressor.py +2 -1
- snowflake/ml/modeling/covariance/elliptic_envelope.py +2 -1
- snowflake/ml/modeling/covariance/empirical_covariance.py +2 -1
- snowflake/ml/modeling/covariance/graphical_lasso.py +2 -1
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +2 -1
- snowflake/ml/modeling/covariance/ledoit_wolf.py +2 -1
- snowflake/ml/modeling/covariance/min_cov_det.py +2 -1
- snowflake/ml/modeling/covariance/oas.py +2 -1
- snowflake/ml/modeling/covariance/shrunk_covariance.py +2 -1
- snowflake/ml/modeling/decomposition/dictionary_learning.py +2 -1
- snowflake/ml/modeling/decomposition/factor_analysis.py +2 -1
- snowflake/ml/modeling/decomposition/fast_ica.py +2 -1
- snowflake/ml/modeling/decomposition/incremental_pca.py +2 -1
- snowflake/ml/modeling/decomposition/kernel_pca.py +2 -1
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +2 -1
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +2 -1
- snowflake/ml/modeling/decomposition/pca.py +2 -1
- snowflake/ml/modeling/decomposition/sparse_pca.py +2 -1
- snowflake/ml/modeling/decomposition/truncated_svd.py +2 -1
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +2 -1
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +2 -1
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +2 -1
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +2 -1
- snowflake/ml/modeling/ensemble/bagging_classifier.py +2 -1
- snowflake/ml/modeling/ensemble/bagging_regressor.py +2 -1
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +2 -1
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +2 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +2 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +2 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +2 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +2 -1
- snowflake/ml/modeling/ensemble/isolation_forest.py +2 -1
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +2 -1
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +2 -1
- snowflake/ml/modeling/ensemble/stacking_regressor.py +2 -1
- snowflake/ml/modeling/ensemble/voting_classifier.py +2 -1
- snowflake/ml/modeling/ensemble/voting_regressor.py +2 -1
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +2 -1
- snowflake/ml/modeling/feature_selection/select_fdr.py +2 -1
- snowflake/ml/modeling/feature_selection/select_fpr.py +2 -1
- snowflake/ml/modeling/feature_selection/select_fwe.py +2 -1
- snowflake/ml/modeling/feature_selection/select_k_best.py +2 -1
- snowflake/ml/modeling/feature_selection/select_percentile.py +2 -1
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +2 -1
- snowflake/ml/modeling/feature_selection/variance_threshold.py +2 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +2 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +2 -1
- snowflake/ml/modeling/impute/iterative_imputer.py +2 -1
- snowflake/ml/modeling/impute/knn_imputer.py +2 -1
- snowflake/ml/modeling/impute/missing_indicator.py +2 -1
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +2 -1
- snowflake/ml/modeling/kernel_approximation/nystroem.py +2 -1
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +2 -1
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +2 -1
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +2 -1
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +2 -1
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +2 -1
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/ard_regression.py +2 -1
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +2 -1
- snowflake/ml/modeling/linear_model/elastic_net.py +2 -1
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +2 -1
- snowflake/ml/modeling/linear_model/gamma_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/huber_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/lars.py +2 -1
- snowflake/ml/modeling/linear_model/lars_cv.py +2 -1
- snowflake/ml/modeling/linear_model/lasso.py +2 -1
- snowflake/ml/modeling/linear_model/lasso_cv.py +2 -1
- snowflake/ml/modeling/linear_model/lasso_lars.py +2 -1
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +2 -1
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +2 -1
- snowflake/ml/modeling/linear_model/linear_regression.py +2 -1
- snowflake/ml/modeling/linear_model/logistic_regression.py +2 -1
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +2 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +2 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +2 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +2 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +2 -1
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +2 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +2 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/perceptron.py +2 -1
- snowflake/ml/modeling/linear_model/poisson_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/ransac_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/ridge.py +2 -1
- snowflake/ml/modeling/linear_model/ridge_classifier.py +2 -1
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +2 -1
- snowflake/ml/modeling/linear_model/ridge_cv.py +2 -1
- snowflake/ml/modeling/linear_model/sgd_classifier.py +2 -1
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +2 -1
- snowflake/ml/modeling/linear_model/sgd_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +2 -1
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +2 -1
- snowflake/ml/modeling/manifold/isomap.py +2 -1
- snowflake/ml/modeling/manifold/mds.py +2 -1
- snowflake/ml/modeling/manifold/spectral_embedding.py +2 -1
- snowflake/ml/modeling/manifold/tsne.py +2 -1
- snowflake/ml/modeling/metrics/classification.py +1 -0
- snowflake/ml/modeling/metrics/metrics_utils.py +6 -0
- snowflake/ml/modeling/metrics/ranking.py +3 -0
- snowflake/ml/modeling/metrics/regression.py +3 -0
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +2 -1
- snowflake/ml/modeling/mixture/gaussian_mixture.py +2 -1
- snowflake/ml/modeling/model_selection/grid_search_cv.py +2 -0
- snowflake/ml/modeling/model_selection/randomized_search_cv.py +2 -0
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +2 -1
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +2 -1
- snowflake/ml/modeling/multiclass/output_code_classifier.py +2 -1
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +2 -1
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +2 -1
- snowflake/ml/modeling/naive_bayes/complement_nb.py +2 -1
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +2 -1
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +2 -1
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +2 -1
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +2 -1
- snowflake/ml/modeling/neighbors/kernel_density.py +2 -1
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +2 -1
- snowflake/ml/modeling/neighbors/nearest_centroid.py +2 -1
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +2 -1
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +2 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +2 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +2 -1
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +2 -1
- snowflake/ml/modeling/neural_network/mlp_classifier.py +2 -1
- snowflake/ml/modeling/neural_network/mlp_regressor.py +2 -1
- snowflake/ml/modeling/pipeline/pipeline.py +6 -1
- snowflake/ml/modeling/preprocessing/polynomial_features.py +2 -1
- snowflake/ml/modeling/semi_supervised/label_propagation.py +2 -1
- snowflake/ml/modeling/semi_supervised/label_spreading.py +2 -1
- snowflake/ml/modeling/svm/linear_svc.py +2 -1
- snowflake/ml/modeling/svm/linear_svr.py +2 -1
- snowflake/ml/modeling/svm/nu_svc.py +2 -1
- snowflake/ml/modeling/svm/nu_svr.py +2 -1
- snowflake/ml/modeling/svm/svc.py +2 -1
- snowflake/ml/modeling/svm/svr.py +2 -1
- snowflake/ml/modeling/tree/decision_tree_classifier.py +2 -1
- snowflake/ml/modeling/tree/decision_tree_regressor.py +2 -1
- snowflake/ml/modeling/tree/extra_tree_classifier.py +2 -1
- snowflake/ml/modeling/tree/extra_tree_regressor.py +2 -1
- snowflake/ml/modeling/xgboost/xgb_classifier.py +2 -1
- snowflake/ml/modeling/xgboost/xgb_regressor.py +2 -1
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +2 -1
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +2 -1
- snowflake/ml/registry/_manager/model_parameter_reconciler.py +7 -7
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-2.3.0.dist-info → snowflake_ml_python-2.4.0.dist-info}/METADATA +44 -4
- {snowflake_ml_python-2.3.0.dist-info → snowflake_ml_python-2.4.0.dist-info}/RECORD +187 -186
- {snowflake_ml_python-2.3.0.dist-info → snowflake_ml_python-2.4.0.dist-info}/WHEEL +0 -0
- {snowflake_ml_python-2.3.0.dist-info → snowflake_ml_python-2.4.0.dist-info}/licenses/LICENSE.txt +0 -0
- {snowflake_ml_python-2.3.0.dist-info → snowflake_ml_python-2.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,10 +1,43 @@
|
|
|
1
|
-
|
|
1
|
+
import json
|
|
2
|
+
from typing import Any, cast
|
|
2
3
|
|
|
3
4
|
from typing_extensions import deprecated
|
|
4
5
|
|
|
5
6
|
from snowflake import snowpark
|
|
6
|
-
from snowflake.cortex._util import
|
|
7
|
+
from snowflake.cortex._util import (
|
|
8
|
+
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
9
|
+
call_sql_function,
|
|
10
|
+
warn_cortex_deprecated,
|
|
11
|
+
)
|
|
7
12
|
from snowflake.ml._internal import telemetry
|
|
13
|
+
from snowflake.snowpark import functions
|
|
14
|
+
|
|
15
|
+
_AI_CLASSIFY_FUNCTION_NAME = "AI_CLASSIFY"
|
|
16
|
+
# Legacy CLASSIFY_TEXT always returned exactly one label. AI_CLASSIFY defaults to single-label mode, but
|
|
17
|
+
# we set output_mode explicitly so the single-label contract can't drift and multi-label results can
|
|
18
|
+
# never leak through.
|
|
19
|
+
_AI_CLASSIFY_SINGLE_LABEL_CONFIG = {"output_mode": "single"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _adapt_ai_classify_result(result: Any) -> str:
|
|
23
|
+
"""Map an AI_CLASSIFY result to the legacy CLASSIFY_TEXT {"label": ...} JSON string."""
|
|
24
|
+
# Legacy classify_text returned the raw CLASSIFY_TEXT OBJECT, which the immediate (non-Column) path
|
|
25
|
+
# collects as a JSON string such as '{"label": "positive"}'. We reproduce that exact shape so
|
|
26
|
+
# existing callers that json.loads the result and read ["label"] keep working.
|
|
27
|
+
parsed: Any = result
|
|
28
|
+
if isinstance(result, str):
|
|
29
|
+
try:
|
|
30
|
+
parsed = json.loads(result)
|
|
31
|
+
except json.JSONDecodeError:
|
|
32
|
+
return result
|
|
33
|
+
label: Any = None
|
|
34
|
+
if isinstance(parsed, dict):
|
|
35
|
+
labels = parsed.get("labels")
|
|
36
|
+
if isinstance(labels, list) and labels:
|
|
37
|
+
label = labels[0]
|
|
38
|
+
elif parsed.get("label") is not None:
|
|
39
|
+
label = parsed.get("label")
|
|
40
|
+
return json.dumps({"label": label})
|
|
8
41
|
|
|
9
42
|
|
|
10
43
|
@telemetry.send_api_usage_telemetry(
|
|
@@ -25,17 +58,30 @@ def classify_text(
|
|
|
25
58
|
Returns:
|
|
26
59
|
A column of classification responses.
|
|
27
60
|
"""
|
|
28
|
-
|
|
29
|
-
|
|
61
|
+
warn_cortex_deprecated("classify_text", "ai_classify")
|
|
62
|
+
result = _classify_text_impl(
|
|
63
|
+
_AI_CLASSIFY_FUNCTION_NAME, str_input, categories, _AI_CLASSIFY_SINGLE_LABEL_CONFIG, session=session
|
|
64
|
+
)
|
|
65
|
+
if isinstance(result, snowpark.Column):
|
|
66
|
+
# AI_CLASSIFY returns {"labels": [...]}; re-wrap as the legacy CLASSIFY_TEXT {"label": ...} OBJECT
|
|
67
|
+
# so downstream SQL that reads col["label"] is unchanged. Subscript access (not functions.get with
|
|
68
|
+
# a bare string, which Snowpark treats as a column identifier) extracts the field by literal key.
|
|
69
|
+
label = functions.coalesce(result["labels"][0], result["label"])
|
|
70
|
+
return functions.object_construct(functions.lit("label"), label)
|
|
71
|
+
return _adapt_ai_classify_result(result)
|
|
30
72
|
|
|
31
73
|
|
|
32
74
|
def _classify_text_impl(
|
|
33
75
|
function: str,
|
|
34
76
|
str_input: str | snowpark.Column,
|
|
35
77
|
categories: list[str] | snowpark.Column,
|
|
78
|
+
config: dict[str, Any] | None = None,
|
|
36
79
|
session: snowpark.Session | None = None,
|
|
37
80
|
) -> str | snowpark.Column:
|
|
38
|
-
|
|
81
|
+
args: list[Any] = [str_input, categories]
|
|
82
|
+
if config is not None:
|
|
83
|
+
args.append(config)
|
|
84
|
+
return cast(str | snowpark.Column, call_sql_function(function, session, *args))
|
|
39
85
|
|
|
40
86
|
|
|
41
87
|
ClassifyText = deprecated(
|
snowflake/cortex/_complete.py
CHANGED
|
@@ -14,6 +14,7 @@ from snowflake.cortex._util import (
|
|
|
14
14
|
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
15
15
|
SnowflakeAuthenticationException,
|
|
16
16
|
SnowflakeConfigurationException,
|
|
17
|
+
warn_cortex_deprecated,
|
|
17
18
|
)
|
|
18
19
|
from snowflake.ml._internal import telemetry
|
|
19
20
|
from snowflake.snowpark import context, functions
|
|
@@ -331,12 +332,46 @@ def _return_stream_response(
|
|
|
331
332
|
pass
|
|
332
333
|
|
|
333
334
|
|
|
334
|
-
def _complete_call_sql_function_snowpark(
|
|
335
|
-
function: str, *args: str | snowpark.Column | CompleteOptions
|
|
336
|
-
) -> snowpark.Column:
|
|
335
|
+
def _complete_call_sql_function_snowpark(function: str, *args: Any) -> snowpark.Column:
|
|
337
336
|
return cast(snowpark.Column, functions.builtin(function)(*args))
|
|
338
337
|
|
|
339
338
|
|
|
339
|
+
_SQL_MODEL_PARAMETER_KEYS = ("max_tokens", "temperature", "top_p", "guardrails")
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _sql_ai_complete_args(
|
|
343
|
+
model: str | snowpark.Column,
|
|
344
|
+
prompt: str | list[ConversationMessage] | snowpark.Column,
|
|
345
|
+
options: CompleteOptions | snowpark.Column | None,
|
|
346
|
+
) -> tuple[Any, ...]:
|
|
347
|
+
"""Build positional SQL arguments for AI_COMPLETE.
|
|
348
|
+
|
|
349
|
+
Args:
|
|
350
|
+
model: Model name or Column of model names.
|
|
351
|
+
prompt: Prompt string, conversation history, or Column of prompts.
|
|
352
|
+
options: Complete options, a Column of options, or None.
|
|
353
|
+
|
|
354
|
+
Returns:
|
|
355
|
+
Positional AI_COMPLETE arguments with response format separate from
|
|
356
|
+
model parameters.
|
|
357
|
+
"""
|
|
358
|
+
if options is None:
|
|
359
|
+
return (model, prompt)
|
|
360
|
+
if isinstance(options, snowpark.Column):
|
|
361
|
+
# Can't introspect a Column's keys at build time, so split response_format out of the options
|
|
362
|
+
# object in SQL: model_parameters is the object minus response_format, response_format is read
|
|
363
|
+
# out separately (NULL when absent, which AI_COMPLETE treats as "no response format").
|
|
364
|
+
model_parameters_col = functions.builtin("object_delete")(options, functions.lit("response_format"))
|
|
365
|
+
response_format_col = functions.get(options, functions.lit("response_format"))
|
|
366
|
+
return (model, prompt, model_parameters_col, response_format_col)
|
|
367
|
+
option_values = cast(dict[str, Any], options)
|
|
368
|
+
model_parameters = {key: option_values[key] for key in _SQL_MODEL_PARAMETER_KEYS if key in option_values}
|
|
369
|
+
response_format = options.get("response_format")
|
|
370
|
+
if response_format is None:
|
|
371
|
+
return (model, prompt, model_parameters) if model_parameters else (model, prompt)
|
|
372
|
+
return (model, prompt, model_parameters, response_format)
|
|
373
|
+
|
|
374
|
+
|
|
340
375
|
def _complete_non_streaming_immediate(
|
|
341
376
|
snow_api_xp_request_handler: Callable[..., dict[str, Any]] | None,
|
|
342
377
|
model: str,
|
|
@@ -366,10 +401,7 @@ def _complete_non_streaming_impl(
|
|
|
366
401
|
deadline: float | None = None,
|
|
367
402
|
) -> str | snowpark.Column:
|
|
368
403
|
if isinstance(prompt, snowpark.Column):
|
|
369
|
-
|
|
370
|
-
return _complete_call_sql_function_snowpark(function, model, prompt, options)
|
|
371
|
-
else:
|
|
372
|
-
return _complete_call_sql_function_snowpark(function, model, prompt)
|
|
404
|
+
return _complete_call_sql_function_snowpark(function, *_sql_ai_complete_args(model, prompt, options))
|
|
373
405
|
if isinstance(model, snowpark.Column):
|
|
374
406
|
raise ValueError("'model' cannot be a snowpark.Column when 'prompt' is a string.")
|
|
375
407
|
if isinstance(options, snowpark.Column):
|
|
@@ -414,7 +446,7 @@ def _complete_impl(
|
|
|
414
446
|
model: str | snowpark.Column,
|
|
415
447
|
prompt: str | list[ConversationMessage] | snowpark.Column,
|
|
416
448
|
snow_api_xp_request_handler: Callable[..., dict[str, Any]] | None = None,
|
|
417
|
-
function: str = "
|
|
449
|
+
function: str = "AI_COMPLETE",
|
|
418
450
|
options: CompleteOptions | None = None,
|
|
419
451
|
session: snowpark.Session | None = None,
|
|
420
452
|
stream: bool = False,
|
|
@@ -480,7 +512,7 @@ def complete(
|
|
|
480
512
|
Returns:
|
|
481
513
|
A column of string responses.
|
|
482
514
|
"""
|
|
483
|
-
|
|
515
|
+
warn_cortex_deprecated("complete", "ai_complete")
|
|
484
516
|
# Set the XP snow api function, if available.
|
|
485
517
|
snow_api_xp_request_handler = None
|
|
486
518
|
if is_in_stored_procedure(): # type: ignore[no-untyped-call]
|
|
@@ -1,11 +1,17 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import cast
|
|
2
2
|
|
|
3
3
|
from typing_extensions import deprecated
|
|
4
4
|
|
|
5
5
|
from snowflake import snowpark
|
|
6
|
-
from snowflake.cortex._util import
|
|
6
|
+
from snowflake.cortex._util import (
|
|
7
|
+
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
8
|
+
call_sql_function,
|
|
9
|
+
warn_cortex_deprecated,
|
|
10
|
+
)
|
|
7
11
|
from snowflake.ml._internal import telemetry
|
|
8
12
|
|
|
13
|
+
_AI_EMBED_FUNCTION_NAME = "AI_EMBED"
|
|
14
|
+
|
|
9
15
|
|
|
10
16
|
@telemetry.send_api_usage_telemetry(
|
|
11
17
|
project=CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
@@ -26,8 +32,8 @@ def embed_text_1024(
|
|
|
26
32
|
Returns:
|
|
27
33
|
A column of vectors containing embeddings.
|
|
28
34
|
"""
|
|
29
|
-
|
|
30
|
-
return _embed_text_1024_impl(
|
|
35
|
+
warn_cortex_deprecated("embed_text_1024", "ai_embed")
|
|
36
|
+
return _embed_text_1024_impl(_AI_EMBED_FUNCTION_NAME, model, text, session=session)
|
|
31
37
|
|
|
32
38
|
|
|
33
39
|
def _embed_text_1024_impl(
|
|
@@ -36,7 +42,7 @@ def _embed_text_1024_impl(
|
|
|
36
42
|
text: str | snowpark.Column,
|
|
37
43
|
session: snowpark.Session | None = None,
|
|
38
44
|
) -> list[float] | snowpark.Column:
|
|
39
|
-
return cast(
|
|
45
|
+
return cast(list[float] | snowpark.Column, call_sql_function(function, session, model, text))
|
|
40
46
|
|
|
41
47
|
|
|
42
48
|
EmbedText1024 = deprecated(
|
|
@@ -1,11 +1,17 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import cast
|
|
2
2
|
|
|
3
3
|
from typing_extensions import deprecated
|
|
4
4
|
|
|
5
5
|
from snowflake import snowpark
|
|
6
|
-
from snowflake.cortex._util import
|
|
6
|
+
from snowflake.cortex._util import (
|
|
7
|
+
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
8
|
+
call_sql_function,
|
|
9
|
+
warn_cortex_deprecated,
|
|
10
|
+
)
|
|
7
11
|
from snowflake.ml._internal import telemetry
|
|
8
12
|
|
|
13
|
+
_AI_EMBED_FUNCTION_NAME = "AI_EMBED"
|
|
14
|
+
|
|
9
15
|
|
|
10
16
|
@telemetry.send_api_usage_telemetry(
|
|
11
17
|
project=CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
@@ -26,8 +32,8 @@ def embed_text_768(
|
|
|
26
32
|
Returns:
|
|
27
33
|
A column of vectors containing embeddings.
|
|
28
34
|
"""
|
|
29
|
-
|
|
30
|
-
return _embed_text_768_impl(
|
|
35
|
+
warn_cortex_deprecated("embed_text_768", "ai_embed")
|
|
36
|
+
return _embed_text_768_impl(_AI_EMBED_FUNCTION_NAME, model, text, session=session)
|
|
31
37
|
|
|
32
38
|
|
|
33
39
|
def _embed_text_768_impl(
|
|
@@ -36,7 +42,7 @@ def _embed_text_768_impl(
|
|
|
36
42
|
text: str | snowpark.Column,
|
|
37
43
|
session: snowpark.Session | None = None,
|
|
38
44
|
) -> list[float] | snowpark.Column:
|
|
39
|
-
return cast(
|
|
45
|
+
return cast(list[float] | snowpark.Column, call_sql_function(function, session, model, text))
|
|
40
46
|
|
|
41
47
|
|
|
42
48
|
EmbedText768 = deprecated(
|
|
@@ -1,10 +1,39 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import Any, cast
|
|
2
2
|
|
|
3
3
|
from typing_extensions import deprecated
|
|
4
4
|
|
|
5
5
|
from snowflake import snowpark
|
|
6
|
-
from snowflake.cortex._util import
|
|
6
|
+
from snowflake.cortex._util import (
|
|
7
|
+
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
8
|
+
SnowflakeAuthenticationException,
|
|
9
|
+
warn_cortex_deprecated,
|
|
10
|
+
)
|
|
7
11
|
from snowflake.ml._internal import telemetry
|
|
12
|
+
from snowflake.snowpark import context, functions
|
|
13
|
+
|
|
14
|
+
_AI_EXTRACT_FUNCTION_NAME = "AI_EXTRACT"
|
|
15
|
+
_ANSWER_KEY = "answer"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _legacy_extract_answer_column(
|
|
19
|
+
from_text: str | snowpark.Column,
|
|
20
|
+
response_format: dict[str, Any] | snowpark.Column,
|
|
21
|
+
) -> snowpark.Column:
|
|
22
|
+
"""Call AI_EXTRACT and reshape it to the legacy EXTRACT_ANSWER [{"answer": ..., "score": ...}] ARRAY."""
|
|
23
|
+
# AI_EXTRACT returns the score only when the `scores` argument is TRUE, and `scores` is a named-only
|
|
24
|
+
# argument that the Snowpark builtin wrapper can't pass positionally -- inject it as a raw SQL fragment.
|
|
25
|
+
# AI_EXTRACT(<text>, <response_format>, scores => TRUE) returns:
|
|
26
|
+
# {"response": {"answer": ...}, "scoring": {"scores": {"answer": {"score": ...}}}}
|
|
27
|
+
extracted = functions.builtin(_AI_EXTRACT_FUNCTION_NAME)(
|
|
28
|
+
from_text, response_format, functions.sql_expr("scores => TRUE")
|
|
29
|
+
)
|
|
30
|
+
# Subscript access (not functions.get with a bare string, which Snowpark treats as a column
|
|
31
|
+
# identifier) extracts fields by literal key.
|
|
32
|
+
answer = extracted["response"][_ANSWER_KEY]
|
|
33
|
+
score = extracted["scoring"]["scores"][_ANSWER_KEY]["score"]
|
|
34
|
+
return functions.array_construct(
|
|
35
|
+
functions.object_construct(functions.lit(_ANSWER_KEY), answer, functions.lit("score"), score)
|
|
36
|
+
)
|
|
8
37
|
|
|
9
38
|
|
|
10
39
|
@telemetry.send_api_usage_telemetry(
|
|
@@ -23,19 +52,32 @@ def extract_answer(
|
|
|
23
52
|
session: The snowpark session to use. Will be inferred by context if not specified.
|
|
24
53
|
|
|
25
54
|
Returns:
|
|
26
|
-
A column of
|
|
27
|
-
"""
|
|
55
|
+
A column of arrays, each [{"answer": ..., "score": ...}], matching legacy EXTRACT_ANSWER.
|
|
28
56
|
|
|
29
|
-
|
|
57
|
+
Raises:
|
|
58
|
+
SnowflakeAuthenticationException: if no session is provided and none is active.
|
|
59
|
+
"""
|
|
60
|
+
warn_cortex_deprecated("extract_answer", "ai_extract")
|
|
61
|
+
if isinstance(question, snowpark.Column):
|
|
62
|
+
response_format: dict[str, Any] | snowpark.Column = functions.object_construct(
|
|
63
|
+
functions.lit(_ANSWER_KEY), question
|
|
64
|
+
)
|
|
65
|
+
else:
|
|
66
|
+
response_format = {_ANSWER_KEY: question}
|
|
30
67
|
|
|
68
|
+
column = _legacy_extract_answer_column(from_text, response_format)
|
|
69
|
+
if isinstance(from_text, snowpark.Column) or isinstance(question, snowpark.Column):
|
|
70
|
+
return column
|
|
31
71
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
72
|
+
# Immediate (literal) path: evaluate the expression and return the collected ARRAY as a JSON string,
|
|
73
|
+
# exactly as the legacy EXTRACT_ANSWER immediate path did.
|
|
74
|
+
session = session or context.get_active_session()
|
|
75
|
+
if session is None:
|
|
76
|
+
raise SnowflakeAuthenticationException(
|
|
77
|
+
"""Session required. Provide the session through a session=... argument or ensure an active session is
|
|
78
|
+
available in your environment."""
|
|
79
|
+
)
|
|
80
|
+
return cast(str, session.create_dataframe([snowpark.Row()]).select(column).collect()[0][0])
|
|
39
81
|
|
|
40
82
|
|
|
41
83
|
ExtractAnswer = deprecated(
|
snowflake/cortex/_sentiment.py
CHANGED
|
@@ -3,8 +3,43 @@ from typing import cast
|
|
|
3
3
|
from typing_extensions import deprecated
|
|
4
4
|
|
|
5
5
|
from snowflake import snowpark
|
|
6
|
-
from snowflake.cortex._util import
|
|
6
|
+
from snowflake.cortex._util import (
|
|
7
|
+
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
8
|
+
SnowflakeAuthenticationException,
|
|
9
|
+
warn_cortex_deprecated,
|
|
10
|
+
)
|
|
7
11
|
from snowflake.ml._internal import telemetry
|
|
12
|
+
from snowflake.snowpark import context, functions, types
|
|
13
|
+
|
|
14
|
+
_AI_COMPLETE_FUNCTION_NAME = "AI_COMPLETE"
|
|
15
|
+
|
|
16
|
+
# AI_SENTIMENT returns a categorical label (positive/negative/neutral/mixed), not the legacy FLOAT in
|
|
17
|
+
# [-1, 1] that SNOWFLAKE.CORTEX.SENTIMENT produced. To keep the numeric return contract we score the
|
|
18
|
+
# text with AI_COMPLETE and a constrained prompt instead. The score is model-derived, so exact values
|
|
19
|
+
# differ from the legacy sentiment model.
|
|
20
|
+
_SENTIMENT_MODEL = "llama3.1-8b"
|
|
21
|
+
_SENTIMENT_PROMPT_PREFIX = (
|
|
22
|
+
"Rate the sentiment of the following text from -1.0 (very negative) to 1.0 (very positive). "
|
|
23
|
+
"Respond with only the score. Text: "
|
|
24
|
+
)
|
|
25
|
+
_SENTIMENT_MODEL_PARAMETERS = {"temperature": 0}
|
|
26
|
+
_SENTIMENT_RESPONSE_FORMAT = {
|
|
27
|
+
"type": "json",
|
|
28
|
+
"schema": {"type": "object", "properties": {"score": {"type": "number"}}, "required": ["score"]},
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _sentiment_score_column(text: snowpark.Column) -> snowpark.Column:
|
|
33
|
+
"""Build the AI_COMPLETE sentiment-score expression, clamped to the legacy [-1, 1] range."""
|
|
34
|
+
prompt = functions.concat(functions.lit(_SENTIMENT_PROMPT_PREFIX), text)
|
|
35
|
+
completion = functions.builtin(_AI_COMPLETE_FUNCTION_NAME)(
|
|
36
|
+
functions.lit(_SENTIMENT_MODEL),
|
|
37
|
+
prompt,
|
|
38
|
+
_SENTIMENT_MODEL_PARAMETERS,
|
|
39
|
+
_SENTIMENT_RESPONSE_FORMAT,
|
|
40
|
+
)
|
|
41
|
+
score = functions.get(functions.parse_json(completion), functions.lit("score")).cast(types.FloatType())
|
|
42
|
+
return functions.least(functions.lit(1.0), functions.greatest(functions.lit(-1.0), score))
|
|
8
43
|
|
|
9
44
|
|
|
10
45
|
@telemetry.send_api_usage_telemetry(
|
|
@@ -19,17 +54,22 @@ def sentiment(text: str | snowpark.Column, session: snowpark.Session | None = No
|
|
|
19
54
|
|
|
20
55
|
Returns:
|
|
21
56
|
A column of floats. 1 represents positive sentiment, -1 represents negative sentiment.
|
|
22
|
-
"""
|
|
23
|
-
return _sentiment_impl("snowflake.cortex.sentiment", text, session=session)
|
|
24
57
|
|
|
58
|
+
Raises:
|
|
59
|
+
SnowflakeAuthenticationException: if no session is provided and none is active.
|
|
60
|
+
"""
|
|
61
|
+
warn_cortex_deprecated("sentiment", "ai_complete")
|
|
62
|
+
if isinstance(text, snowpark.Column):
|
|
63
|
+
return _sentiment_score_column(text)
|
|
25
64
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
65
|
+
session = session or context.get_active_session()
|
|
66
|
+
if session is None:
|
|
67
|
+
raise SnowflakeAuthenticationException(
|
|
68
|
+
"""Session required. Provide the session through a session=... argument or ensure an active session is
|
|
69
|
+
available in your environment."""
|
|
70
|
+
)
|
|
71
|
+
score_column = _sentiment_score_column(functions.lit(text))
|
|
72
|
+
output = session.create_dataframe([snowpark.Row()]).select(score_column).collect()[0][0]
|
|
33
73
|
return float(cast(str, output))
|
|
34
74
|
|
|
35
75
|
|
snowflake/cortex/_summarize.py
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import cast
|
|
2
2
|
|
|
3
3
|
from typing_extensions import deprecated
|
|
4
4
|
|
|
5
5
|
from snowflake import snowpark
|
|
6
|
-
from snowflake.cortex._util import
|
|
6
|
+
from snowflake.cortex._util import (
|
|
7
|
+
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
8
|
+
call_sql_function,
|
|
9
|
+
warn_cortex_deprecated,
|
|
10
|
+
)
|
|
7
11
|
from snowflake.ml._internal import telemetry
|
|
8
12
|
|
|
13
|
+
_AI_SUMMARIZE_FUNCTION_NAME = "AI_SUMMARIZE"
|
|
14
|
+
|
|
9
15
|
|
|
10
16
|
@telemetry.send_api_usage_telemetry(
|
|
11
17
|
project=CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
@@ -23,8 +29,8 @@ def summarize(
|
|
|
23
29
|
Returns:
|
|
24
30
|
A column of string summaries.
|
|
25
31
|
"""
|
|
26
|
-
|
|
27
|
-
return _summarize_impl(
|
|
32
|
+
warn_cortex_deprecated("summarize", "ai_complete")
|
|
33
|
+
return _summarize_impl(_AI_SUMMARIZE_FUNCTION_NAME, text, session=session)
|
|
28
34
|
|
|
29
35
|
|
|
30
36
|
def _summarize_impl(
|
|
@@ -32,7 +38,7 @@ def _summarize_impl(
|
|
|
32
38
|
text: str | snowpark.Column,
|
|
33
39
|
session: snowpark.Session | None = None,
|
|
34
40
|
) -> str | snowpark.Column:
|
|
35
|
-
return cast(
|
|
41
|
+
return cast(str | snowpark.Column, call_sql_function(function, session, text))
|
|
36
42
|
|
|
37
43
|
|
|
38
44
|
Summarize = deprecated("Summarize() is deprecated and will be removed in a future release. Use summarize() instead")(
|
snowflake/cortex/_translate.py
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import cast
|
|
2
2
|
|
|
3
3
|
from typing_extensions import deprecated
|
|
4
4
|
|
|
5
5
|
from snowflake import snowpark
|
|
6
|
-
from snowflake.cortex._util import
|
|
6
|
+
from snowflake.cortex._util import (
|
|
7
|
+
CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
8
|
+
call_sql_function,
|
|
9
|
+
warn_cortex_deprecated,
|
|
10
|
+
)
|
|
7
11
|
from snowflake.ml._internal import telemetry
|
|
8
12
|
|
|
13
|
+
_AI_TRANSLATE_FUNCTION_NAME = "AI_TRANSLATE"
|
|
14
|
+
|
|
9
15
|
|
|
10
16
|
@telemetry.send_api_usage_telemetry(
|
|
11
17
|
project=CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
|
@@ -27,8 +33,8 @@ def translate(
|
|
|
27
33
|
Returns:
|
|
28
34
|
A column of string translations.
|
|
29
35
|
"""
|
|
30
|
-
|
|
31
|
-
return _translate_impl(
|
|
36
|
+
warn_cortex_deprecated("translate", "ai_translate")
|
|
37
|
+
return _translate_impl(_AI_TRANSLATE_FUNCTION_NAME, text, from_language, to_language, session=session)
|
|
32
38
|
|
|
33
39
|
|
|
34
40
|
def _translate_impl(
|
|
@@ -38,7 +44,7 @@ def _translate_impl(
|
|
|
38
44
|
to_language: str | snowpark.Column,
|
|
39
45
|
session: snowpark.Session | None = None,
|
|
40
46
|
) -> str | snowpark.Column:
|
|
41
|
-
return cast(
|
|
47
|
+
return cast(str | snowpark.Column, call_sql_function(function, session, text, from_language, to_language))
|
|
42
48
|
|
|
43
49
|
|
|
44
50
|
Translate = deprecated("Translate() is deprecated and will be removed in a future release. Use translate() instead")(
|
snowflake/cortex/_util.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from typing import Any, Union, cast
|
|
2
3
|
|
|
3
4
|
from snowflake import snowpark
|
|
@@ -8,6 +9,40 @@ from snowflake.snowpark import context, functions
|
|
|
8
9
|
CORTEX_FUNCTIONS_TELEMETRY_PROJECT = "CortexFunctions"
|
|
9
10
|
|
|
10
11
|
|
|
12
|
+
_SNOWPARK_FUNCTIONS_DOC_URL = (
|
|
13
|
+
"https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/latest/"
|
|
14
|
+
"snowpark/api/snowflake.snowpark.functions.{replacement}"
|
|
15
|
+
)
|
|
16
|
+
_MIGRATION_GUIDE_URL = "https://docs.snowflake.com/en/user-guide/snowflake-cortex/aisql-migrate-legacy-functions"
|
|
17
|
+
|
|
18
|
+
# Functions that have already emitted their deprecation warning this process. Guarding on this keeps a
|
|
19
|
+
# repeated call (e.g. inside a loop) from spamming the console -- each deprecated function warns once.
|
|
20
|
+
_emitted_deprecations: set[str] = set()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def warn_cortex_deprecated(name: str, replacement: str) -> None:
|
|
24
|
+
"""Emit the deprecation warning for the named deprecated snowflake.cortex function, once per process.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
name: The deprecated snowflake.cortex function name (e.g. "classify_text").
|
|
28
|
+
replacement: The snowflake.snowpark.functions replacement to point callers at (e.g. "ai_classify").
|
|
29
|
+
"""
|
|
30
|
+
# Warn on use (not on package import) so callers of functions that are NOT deprecated -- e.g.
|
|
31
|
+
# Finetune -- are never warned.
|
|
32
|
+
if name in _emitted_deprecations:
|
|
33
|
+
return
|
|
34
|
+
_emitted_deprecations.add(name)
|
|
35
|
+
warnings.warn(
|
|
36
|
+
f"snowflake.cortex.{name} is deprecated and will be removed in a future release. "
|
|
37
|
+
f"Its backend is now redirected to Snowflake AI Functions. "
|
|
38
|
+
f"Use snowflake.snowpark.functions.{replacement} instead. "
|
|
39
|
+
f"API: {_SNOWPARK_FUNCTIONS_DOC_URL.format(replacement=replacement)} "
|
|
40
|
+
f"Migration guide: {_MIGRATION_GUIDE_URL}",
|
|
41
|
+
DeprecationWarning,
|
|
42
|
+
stacklevel=3,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
11
46
|
class SnowflakeAuthenticationException(Exception):
|
|
12
47
|
"""This exception is raised when there is an issue with Snowflake's configuration."""
|
|
13
48
|
|
|
@@ -57,11 +57,19 @@ class Run:
|
|
|
57
57
|
run_name=self.name.identifier(),
|
|
58
58
|
)
|
|
59
59
|
|
|
60
|
-
def
|
|
60
|
+
def _handle_run_metadata_size_limit(self, sql_error_msg: str) -> None:
|
|
61
|
+
"""Report that run metadata exceeded the size limit.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
sql_error_msg: The error message returned by Snowflake.
|
|
65
|
+
|
|
66
|
+
Raises:
|
|
67
|
+
RuntimeWarning: If the experiment tracking instance was created with warnings_as_errors=True.
|
|
68
|
+
"""
|
|
69
|
+
message = f"{sql_error_msg}. {METADATA_SIZE_WARNING_MESSAGE}"
|
|
70
|
+
if self._experiment_tracking._warnings_as_errors:
|
|
71
|
+
raise RuntimeWarning(message)
|
|
72
|
+
# Once the limit is hit every subsequent log attempt fails the same way, so warn only once.
|
|
61
73
|
if not self._warned_about_metadata_size:
|
|
62
|
-
warnings.warn(
|
|
63
|
-
f"{sql_error_msg}. {METADATA_SIZE_WARNING_MESSAGE}",
|
|
64
|
-
RuntimeWarning,
|
|
65
|
-
stacklevel=2,
|
|
66
|
-
)
|
|
74
|
+
warnings.warn(message, RuntimeWarning, stacklevel=2)
|
|
67
75
|
self._warned_about_metadata_size = True
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, Any
|
|
2
|
+
|
|
3
|
+
from snowflake.ml.experiment import utils
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from snowflake.ml.experiment.experiment_tracking import ExperimentTracking
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SnowflakeCatboostCallback:
|
|
10
|
+
"""CatBoost callback that logs training metrics and parameters to a Snowflake ML Experiment.
|
|
11
|
+
|
|
12
|
+
CatBoost's callback protocol only exposes iteration number and metrics, not the model itself.
|
|
13
|
+
This callback logs metrics during training and, optionally, model parameters. To log the trained
|
|
14
|
+
model, call ``ExperimentTracking.log_model`` after ``fit`` returns::
|
|
15
|
+
|
|
16
|
+
callback = SnowflakeCatboostCallback(
|
|
17
|
+
exp,
|
|
18
|
+
params=model.get_params(),
|
|
19
|
+
)
|
|
20
|
+
model.fit(X, y, callbacks=[callback])
|
|
21
|
+
exp.log_model(model, model_name="my_model", signatures={"predict": sig})
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
experiment_tracking: "ExperimentTracking",
|
|
27
|
+
*,
|
|
28
|
+
params: dict[str, Any] | None = None,
|
|
29
|
+
log_every_n_epochs: int = 1,
|
|
30
|
+
) -> None:
|
|
31
|
+
"""
|
|
32
|
+
Initialize the callback.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
experiment_tracking: The Experiment Tracking instance to use for logging.
|
|
36
|
+
params: Model parameters to log at the start of training. Pass
|
|
37
|
+
``model.get_params()`` before calling ``fit``. If None, no parameters are logged.
|
|
38
|
+
log_every_n_epochs: Frequency with which to log metrics. Must be a positive integer.
|
|
39
|
+
Default is 1, logging after every iteration. A metric is only logged for iterations
|
|
40
|
+
where CatBoost recomputed it, so a model trained with ``metric_period`` greater than 1
|
|
41
|
+
logs less often than this setting alone implies.
|
|
42
|
+
|
|
43
|
+
Raises:
|
|
44
|
+
ValueError: When ``log_every_n_epochs`` is not a positive integer.
|
|
45
|
+
"""
|
|
46
|
+
self._experiment_tracking = experiment_tracking
|
|
47
|
+
self._params = params
|
|
48
|
+
if not (utils.is_integer(log_every_n_epochs) and log_every_n_epochs > 0):
|
|
49
|
+
raise ValueError("`log_every_n_epochs` must be a positive integer.")
|
|
50
|
+
self.log_every_n_epochs = log_every_n_epochs
|
|
51
|
+
self._params_logged = False
|
|
52
|
+
self._metric_value_counts: dict[str, int] = {}
|
|
53
|
+
|
|
54
|
+
def after_iteration(self, info: Any) -> bool:
|
|
55
|
+
# CatBoost numbers iterations from 1; normalize to 0-based to match the other callbacks.
|
|
56
|
+
epoch = info.iteration - 1
|
|
57
|
+
|
|
58
|
+
if self._params is not None and not self._params_logged:
|
|
59
|
+
self._experiment_tracking.log_params(utils.flatten_nested_params(self._params))
|
|
60
|
+
self._params_logged = True
|
|
61
|
+
|
|
62
|
+
should_log = epoch % self.log_every_n_epochs == 0
|
|
63
|
+
for dataset_name, metrics in info.metrics.items():
|
|
64
|
+
for metric_name, values in metrics.items():
|
|
65
|
+
metric_key = dataset_name + ":" + metric_name
|
|
66
|
+
# CatBoost runs callbacks on every iteration but only appends a metric value on the
|
|
67
|
+
# iterations where it recomputes one, which `metric_period` can make less frequent. A
|
|
68
|
+
# list that did not grow means the trailing value belongs to an earlier iteration, so
|
|
69
|
+
# logging it again would attribute a stale value to this step.
|
|
70
|
+
recomputed = len(values) > self._metric_value_counts.get(metric_key, 0)
|
|
71
|
+
self._metric_value_counts[metric_key] = len(values)
|
|
72
|
+
if should_log and recomputed:
|
|
73
|
+
self._experiment_tracking.log_metric(key=metric_key, value=values[-1], step=epoch)
|
|
74
|
+
|
|
75
|
+
return True
|