snowflake-ml-python 1.6.1__py3-none-any.whl → 1.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/__init__.py +4 -0
- snowflake/cortex/_classify_text.py +2 -2
- snowflake/cortex/_embed_text_1024.py +37 -0
- snowflake/cortex/_embed_text_768.py +37 -0
- snowflake/cortex/_extract_answer.py +2 -2
- snowflake/cortex/_sentiment.py +2 -2
- snowflake/cortex/_summarize.py +2 -2
- snowflake/cortex/_translate.py +2 -2
- snowflake/cortex/_util.py +4 -4
- snowflake/ml/_internal/env_utils.py +5 -5
- snowflake/ml/_internal/exceptions/error_codes.py +2 -0
- snowflake/ml/_internal/telemetry.py +142 -20
- snowflake/ml/_internal/utils/db_utils.py +50 -0
- snowflake/ml/_internal/utils/identifier.py +48 -11
- snowflake/ml/_internal/utils/service_logger.py +63 -0
- snowflake/ml/_internal/utils/snowflake_env.py +23 -13
- snowflake/ml/_internal/utils/sql_identifier.py +26 -2
- snowflake/ml/_internal/utils/table_manager.py +19 -1
- snowflake/ml/data/_internal/arrow_ingestor.py +1 -11
- snowflake/ml/data/data_connector.py +33 -7
- snowflake/ml/data/ingestor_utils.py +20 -10
- snowflake/ml/data/torch_utils.py +68 -0
- snowflake/ml/dataset/dataset.py +1 -3
- snowflake/ml/feature_store/access_manager.py +3 -3
- snowflake/ml/feature_store/feature_store.py +60 -19
- snowflake/ml/feature_store/feature_view.py +84 -30
- snowflake/ml/fileset/embedded_stage_fs.py +1 -1
- snowflake/ml/fileset/fileset.py +1 -1
- snowflake/ml/fileset/sfcfs.py +9 -3
- snowflake/ml/fileset/stage_fs.py +2 -1
- snowflake/ml/lineage/lineage_node.py +7 -2
- snowflake/ml/model/__init__.py +1 -2
- snowflake/ml/model/_client/model/model_version_impl.py +96 -12
- snowflake/ml/model/_client/ops/model_ops.py +124 -6
- snowflake/ml/model/_client/ops/service_ops.py +309 -9
- snowflake/ml/model/_client/service/model_deployment_spec.py +8 -5
- snowflake/ml/model/_client/service/model_deployment_spec_schema.py +2 -2
- snowflake/ml/model/_client/sql/_base.py +5 -0
- snowflake/ml/model/_client/sql/model.py +1 -0
- snowflake/ml/model/_client/sql/model_version.py +9 -5
- snowflake/ml/model/_client/sql/service.py +121 -20
- snowflake/ml/model/_model_composer/model_composer.py +11 -39
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +31 -11
- snowflake/ml/model/_packager/model_env/model_env.py +4 -38
- snowflake/ml/model/_packager/model_handlers/_utils.py +134 -28
- snowflake/ml/model/_packager/model_handlers/catboost.py +31 -30
- snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py +26 -18
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +31 -58
- snowflake/ml/model/_packager/model_handlers/mlflow.py +3 -5
- snowflake/ml/model/_packager/model_handlers/model_objective_utils.py +169 -0
- snowflake/ml/model/_packager/model_handlers/sentence_transformers.py +15 -8
- snowflake/ml/model/_packager/model_handlers/sklearn.py +56 -60
- snowflake/ml/model/_packager/model_handlers/snowmlmodel.py +141 -9
- snowflake/ml/model/_packager/model_handlers/torchscript.py +2 -2
- snowflake/ml/model/_packager/model_handlers/xgboost.py +63 -48
- snowflake/ml/model/_packager/model_meta/model_meta.py +16 -42
- snowflake/ml/model/_packager/model_meta/model_meta_schema.py +1 -14
- snowflake/ml/model/_packager/model_packager.py +14 -8
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +11 -0
- snowflake/ml/model/_signatures/pytorch_handler.py +1 -1
- snowflake/ml/model/_signatures/snowpark_handler.py +3 -2
- snowflake/ml/model/_signatures/utils.py +9 -0
- snowflake/ml/model/type_hints.py +12 -145
- snowflake/ml/modeling/_internal/constants.py +1 -0
- snowflake/ml/modeling/_internal/local_implementations/pandas_handlers.py +5 -5
- snowflake/ml/modeling/_internal/local_implementations/pandas_trainer.py +9 -6
- snowflake/ml/modeling/_internal/model_specifications.py +2 -0
- snowflake/ml/modeling/_internal/model_trainer.py +1 -0
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +2 -4
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_handlers.py +5 -5
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +130 -166
- snowflake/ml/modeling/_internal/snowpark_implementations/xgboost_external_memory_trainer.py +0 -1
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +61 -21
- snowflake/ml/modeling/cluster/affinity_propagation.py +61 -21
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +61 -21
- snowflake/ml/modeling/cluster/birch.py +61 -21
- snowflake/ml/modeling/cluster/bisecting_k_means.py +61 -21
- snowflake/ml/modeling/cluster/dbscan.py +61 -21
- snowflake/ml/modeling/cluster/feature_agglomeration.py +61 -21
- snowflake/ml/modeling/cluster/k_means.py +61 -21
- snowflake/ml/modeling/cluster/mean_shift.py +61 -21
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +61 -21
- snowflake/ml/modeling/cluster/optics.py +61 -21
- snowflake/ml/modeling/cluster/spectral_biclustering.py +61 -21
- snowflake/ml/modeling/cluster/spectral_clustering.py +61 -21
- snowflake/ml/modeling/cluster/spectral_coclustering.py +61 -21
- snowflake/ml/modeling/compose/column_transformer.py +61 -21
- snowflake/ml/modeling/compose/transformed_target_regressor.py +61 -21
- snowflake/ml/modeling/covariance/elliptic_envelope.py +61 -21
- snowflake/ml/modeling/covariance/empirical_covariance.py +61 -21
- snowflake/ml/modeling/covariance/graphical_lasso.py +61 -21
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +61 -21
- snowflake/ml/modeling/covariance/ledoit_wolf.py +61 -21
- snowflake/ml/modeling/covariance/min_cov_det.py +61 -21
- snowflake/ml/modeling/covariance/oas.py +61 -21
- snowflake/ml/modeling/covariance/shrunk_covariance.py +61 -21
- snowflake/ml/modeling/decomposition/dictionary_learning.py +61 -21
- snowflake/ml/modeling/decomposition/factor_analysis.py +61 -21
- snowflake/ml/modeling/decomposition/fast_ica.py +61 -21
- snowflake/ml/modeling/decomposition/incremental_pca.py +61 -21
- snowflake/ml/modeling/decomposition/kernel_pca.py +61 -21
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +61 -21
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +61 -21
- snowflake/ml/modeling/decomposition/pca.py +61 -21
- snowflake/ml/modeling/decomposition/sparse_pca.py +61 -21
- snowflake/ml/modeling/decomposition/truncated_svd.py +61 -21
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +61 -21
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +61 -21
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/bagging_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/bagging_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/isolation_forest.py +61 -21
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/stacking_regressor.py +61 -21
- snowflake/ml/modeling/ensemble/voting_classifier.py +61 -21
- snowflake/ml/modeling/ensemble/voting_regressor.py +61 -21
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +61 -21
- snowflake/ml/modeling/feature_selection/select_fdr.py +61 -21
- snowflake/ml/modeling/feature_selection/select_fpr.py +61 -21
- snowflake/ml/modeling/feature_selection/select_fwe.py +61 -21
- snowflake/ml/modeling/feature_selection/select_k_best.py +61 -21
- snowflake/ml/modeling/feature_selection/select_percentile.py +61 -21
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +61 -21
- snowflake/ml/modeling/feature_selection/variance_threshold.py +61 -21
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +61 -21
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +61 -21
- snowflake/ml/modeling/impute/iterative_imputer.py +61 -21
- snowflake/ml/modeling/impute/knn_imputer.py +61 -21
- snowflake/ml/modeling/impute/missing_indicator.py +61 -21
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +61 -21
- snowflake/ml/modeling/kernel_approximation/nystroem.py +61 -21
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +61 -21
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +61 -21
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +61 -21
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +61 -21
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +61 -21
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/ard_regression.py +61 -21
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +61 -21
- snowflake/ml/modeling/linear_model/elastic_net.py +61 -21
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +61 -21
- snowflake/ml/modeling/linear_model/gamma_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/huber_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/lars.py +61 -21
- snowflake/ml/modeling/linear_model/lars_cv.py +61 -21
- snowflake/ml/modeling/linear_model/lasso.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_cv.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_lars.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +61 -21
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +61 -21
- snowflake/ml/modeling/linear_model/linear_regression.py +61 -21
- snowflake/ml/modeling/linear_model/logistic_regression.py +61 -21
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +61 -21
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +61 -21
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +61 -21
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +61 -21
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/perceptron.py +61 -21
- snowflake/ml/modeling/linear_model/poisson_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/ransac_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/ridge.py +61 -21
- snowflake/ml/modeling/linear_model/ridge_classifier.py +61 -21
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +61 -21
- snowflake/ml/modeling/linear_model/ridge_cv.py +61 -21
- snowflake/ml/modeling/linear_model/sgd_classifier.py +61 -21
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +61 -21
- snowflake/ml/modeling/linear_model/sgd_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +61 -21
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +61 -21
- snowflake/ml/modeling/manifold/isomap.py +61 -21
- snowflake/ml/modeling/manifold/mds.py +61 -21
- snowflake/ml/modeling/manifold/spectral_embedding.py +61 -21
- snowflake/ml/modeling/manifold/tsne.py +61 -21
- snowflake/ml/modeling/metrics/metrics_utils.py +2 -2
- snowflake/ml/modeling/metrics/ranking.py +0 -3
- snowflake/ml/modeling/metrics/regression.py +0 -3
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +61 -21
- snowflake/ml/modeling/mixture/gaussian_mixture.py +61 -21
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +61 -21
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +61 -21
- snowflake/ml/modeling/multiclass/output_code_classifier.py +61 -21
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/complement_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +61 -21
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +61 -21
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +61 -21
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +61 -21
- snowflake/ml/modeling/neighbors/kernel_density.py +61 -21
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +61 -21
- snowflake/ml/modeling/neighbors/nearest_centroid.py +61 -21
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +61 -21
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +61 -21
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +61 -21
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +61 -21
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +61 -21
- snowflake/ml/modeling/neural_network/mlp_classifier.py +61 -21
- snowflake/ml/modeling/neural_network/mlp_regressor.py +61 -21
- snowflake/ml/modeling/parameters/disable_model_tracer.py +5 -0
- snowflake/ml/modeling/pipeline/pipeline.py +1 -13
- snowflake/ml/modeling/preprocessing/polynomial_features.py +61 -21
- snowflake/ml/modeling/semi_supervised/label_propagation.py +61 -21
- snowflake/ml/modeling/semi_supervised/label_spreading.py +61 -21
- snowflake/ml/modeling/svm/linear_svc.py +61 -21
- snowflake/ml/modeling/svm/linear_svr.py +61 -21
- snowflake/ml/modeling/svm/nu_svc.py +61 -21
- snowflake/ml/modeling/svm/nu_svr.py +61 -21
- snowflake/ml/modeling/svm/svc.py +61 -21
- snowflake/ml/modeling/svm/svr.py +61 -21
- snowflake/ml/modeling/tree/decision_tree_classifier.py +61 -21
- snowflake/ml/modeling/tree/decision_tree_regressor.py +61 -21
- snowflake/ml/modeling/tree/extra_tree_classifier.py +61 -21
- snowflake/ml/modeling/tree/extra_tree_regressor.py +61 -21
- snowflake/ml/modeling/xgboost/xgb_classifier.py +64 -23
- snowflake/ml/modeling/xgboost/xgb_regressor.py +64 -23
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +64 -23
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +64 -23
- snowflake/ml/monitoring/_client/model_monitor.py +126 -0
- snowflake/ml/monitoring/_client/model_monitor_manager.py +361 -0
- snowflake/ml/monitoring/_client/model_monitor_version.py +1 -0
- snowflake/ml/monitoring/_client/monitor_sql_client.py +1335 -0
- snowflake/ml/monitoring/_client/queries/record_count.ssql +14 -0
- snowflake/ml/monitoring/_client/queries/rmse.ssql +28 -0
- snowflake/ml/monitoring/entities/model_monitor_config.py +28 -0
- snowflake/ml/monitoring/entities/model_monitor_interval.py +46 -0
- snowflake/ml/monitoring/entities/output_score_type.py +90 -0
- snowflake/ml/registry/_manager/model_manager.py +4 -0
- snowflake/ml/registry/registry.py +166 -8
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/METADATA +43 -9
- snowflake_ml_python-1.6.3.dist-info/RECORD +400 -0
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/WHEEL +1 -1
- snowflake/ml/_internal/container_services/image_registry/credential.py +0 -84
- snowflake/ml/_internal/container_services/image_registry/http_client.py +0 -127
- snowflake/ml/_internal/container_services/image_registry/imagelib.py +0 -400
- snowflake/ml/_internal/container_services/image_registry/registry_client.py +0 -212
- snowflake/ml/_internal/utils/log_stream_processor.py +0 -30
- snowflake/ml/_internal/utils/session_token_manager.py +0 -46
- snowflake/ml/_internal/utils/spcs_attribution_utils.py +0 -122
- snowflake/ml/_internal/utils/uri.py +0 -77
- snowflake/ml/data/torch_dataset.py +0 -33
- snowflake/ml/model/_api.py +0 -568
- snowflake/ml/model/_deploy_client/image_builds/base_image_builder.py +0 -12
- snowflake/ml/model/_deploy_client/image_builds/client_image_builder.py +0 -249
- snowflake/ml/model/_deploy_client/image_builds/docker_context.py +0 -130
- snowflake/ml/model/_deploy_client/image_builds/gunicorn_run.sh +0 -36
- snowflake/ml/model/_deploy_client/image_builds/inference_server/main.py +0 -268
- snowflake/ml/model/_deploy_client/image_builds/server_image_builder.py +0 -215
- snowflake/ml/model/_deploy_client/image_builds/templates/dockerfile_template +0 -53
- snowflake/ml/model/_deploy_client/image_builds/templates/image_build_job_spec_template +0 -38
- snowflake/ml/model/_deploy_client/image_builds/templates/kaniko_shell_script_template +0 -105
- snowflake/ml/model/_deploy_client/snowservice/deploy.py +0 -611
- snowflake/ml/model/_deploy_client/snowservice/deploy_options.py +0 -116
- snowflake/ml/model/_deploy_client/snowservice/instance_types.py +0 -10
- snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template +0 -28
- snowflake/ml/model/_deploy_client/snowservice/templates/service_spec_template_with_model +0 -21
- snowflake/ml/model/_deploy_client/utils/constants.py +0 -48
- snowflake/ml/model/_deploy_client/utils/snowservice_client.py +0 -280
- snowflake/ml/model/_deploy_client/warehouse/deploy.py +0 -202
- snowflake/ml/model/_deploy_client/warehouse/infer_template.py +0 -99
- snowflake/ml/model/_packager/model_handlers/llm.py +0 -267
- snowflake/ml/model/_packager/model_meta/_core_requirements.py +0 -11
- snowflake/ml/model/deploy_platforms.py +0 -6
- snowflake/ml/model/models/llm.py +0 -104
- snowflake/ml/monitoring/monitor.py +0 -203
- snowflake/ml/registry/_initial_schema.py +0 -142
- snowflake/ml/registry/_schema.py +0 -82
- snowflake/ml/registry/_schema_upgrade_plans.py +0 -116
- snowflake/ml/registry/_schema_version_manager.py +0 -163
- snowflake/ml/registry/model_registry.py +0 -2048
- snowflake_ml_python-1.6.1.dist-info/RECORD +0 -422
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/LICENSE.txt +0 -0
- {snowflake_ml_python-1.6.1.dist-info → snowflake_ml_python-1.6.3.dist-info}/top_level.txt +0 -0
snowflake/cortex/__init__.py
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
from snowflake.cortex._classify_text import ClassifyText
|
2
2
|
from snowflake.cortex._complete import Complete, CompleteOptions
|
3
|
+
from snowflake.cortex._embed_text_768 import EmbedText768
|
4
|
+
from snowflake.cortex._embed_text_1024 import EmbedText1024
|
3
5
|
from snowflake.cortex._extract_answer import ExtractAnswer
|
4
6
|
from snowflake.cortex._sentiment import Sentiment
|
5
7
|
from snowflake.cortex._summarize import Summarize
|
@@ -9,6 +11,8 @@ __all__ = [
|
|
9
11
|
"ClassifyText",
|
10
12
|
"Complete",
|
11
13
|
"CompleteOptions",
|
14
|
+
"EmbedText768",
|
15
|
+
"EmbedText1024",
|
12
16
|
"ExtractAnswer",
|
13
17
|
"Sentiment",
|
14
18
|
"Summarize",
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import List, Optional, Union
|
1
|
+
from typing import List, Optional, Union, cast
|
2
2
|
|
3
3
|
from snowflake import snowpark
|
4
4
|
from snowflake.cortex._util import CORTEX_FUNCTIONS_TELEMETRY_PROJECT, call_sql_function
|
@@ -33,4 +33,4 @@ def _classify_text_impl(
|
|
33
33
|
categories: Union[List[str], snowpark.Column],
|
34
34
|
session: Optional[snowpark.Session] = None,
|
35
35
|
) -> Union[str, snowpark.Column]:
|
36
|
-
return call_sql_function(function, session, str_input, categories)
|
36
|
+
return cast(Union[str, snowpark.Column], call_sql_function(function, session, str_input, categories))
|
@@ -0,0 +1,37 @@
|
|
1
|
+
from typing import List, Optional, Union, cast
|
2
|
+
|
3
|
+
from snowflake import snowpark
|
4
|
+
from snowflake.cortex._util import CORTEX_FUNCTIONS_TELEMETRY_PROJECT, call_sql_function
|
5
|
+
from snowflake.ml._internal import telemetry
|
6
|
+
|
7
|
+
|
8
|
+
@telemetry.send_api_usage_telemetry(
|
9
|
+
project=CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
10
|
+
)
|
11
|
+
def EmbedText1024(
|
12
|
+
model: Union[str, snowpark.Column],
|
13
|
+
text: Union[str, snowpark.Column],
|
14
|
+
session: Optional[snowpark.Session] = None,
|
15
|
+
) -> Union[List[float], snowpark.Column]:
|
16
|
+
"""TextEmbed calls into the LLM inference service to embed the text.
|
17
|
+
|
18
|
+
Args:
|
19
|
+
model: A Column of strings representing the model to use for embedding. The value
|
20
|
+
of the strings must be within the SUPPORTED_MODELS list.
|
21
|
+
text: A Column of strings representing input text.
|
22
|
+
session: The snowpark session to use. Will be inferred by context if not specified.
|
23
|
+
|
24
|
+
Returns:
|
25
|
+
A column of vectors containing embeddings.
|
26
|
+
"""
|
27
|
+
|
28
|
+
return _embed_text_1024_impl("snowflake.cortex.embed_text_1024", model, text, session=session)
|
29
|
+
|
30
|
+
|
31
|
+
def _embed_text_1024_impl(
|
32
|
+
function: str,
|
33
|
+
model: Union[str, snowpark.Column],
|
34
|
+
text: Union[str, snowpark.Column],
|
35
|
+
session: Optional[snowpark.Session] = None,
|
36
|
+
) -> Union[List[float], snowpark.Column]:
|
37
|
+
return cast(Union[List[float], snowpark.Column], call_sql_function(function, session, model, text))
|
@@ -0,0 +1,37 @@
|
|
1
|
+
from typing import List, Optional, Union, cast
|
2
|
+
|
3
|
+
from snowflake import snowpark
|
4
|
+
from snowflake.cortex._util import CORTEX_FUNCTIONS_TELEMETRY_PROJECT, call_sql_function
|
5
|
+
from snowflake.ml._internal import telemetry
|
6
|
+
|
7
|
+
|
8
|
+
@telemetry.send_api_usage_telemetry(
|
9
|
+
project=CORTEX_FUNCTIONS_TELEMETRY_PROJECT,
|
10
|
+
)
|
11
|
+
def EmbedText768(
|
12
|
+
model: Union[str, snowpark.Column],
|
13
|
+
text: Union[str, snowpark.Column],
|
14
|
+
session: Optional[snowpark.Session] = None,
|
15
|
+
) -> Union[List[float], snowpark.Column]:
|
16
|
+
"""TextEmbed calls into the LLM inference service to embed the text.
|
17
|
+
|
18
|
+
Args:
|
19
|
+
model: A Column of strings representing the model to use for embedding. The value
|
20
|
+
of the strings must be within the SUPPORTED_MODELS list.
|
21
|
+
text: A Column of strings representing input text.
|
22
|
+
session: The snowpark session to use. Will be inferred by context if not specified.
|
23
|
+
|
24
|
+
Returns:
|
25
|
+
A column of vectors containing embeddings.
|
26
|
+
"""
|
27
|
+
|
28
|
+
return _embed_text_768_impl("snowflake.cortex.embed_text_768", model, text, session=session)
|
29
|
+
|
30
|
+
|
31
|
+
def _embed_text_768_impl(
|
32
|
+
function: str,
|
33
|
+
model: Union[str, snowpark.Column],
|
34
|
+
text: Union[str, snowpark.Column],
|
35
|
+
session: Optional[snowpark.Session] = None,
|
36
|
+
) -> Union[List[float], snowpark.Column]:
|
37
|
+
return cast(Union[List[float], snowpark.Column], call_sql_function(function, session, model, text))
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Optional, Union
|
1
|
+
from typing import Optional, Union, cast
|
2
2
|
|
3
3
|
from snowflake import snowpark
|
4
4
|
from snowflake.cortex._util import CORTEX_FUNCTIONS_TELEMETRY_PROJECT, call_sql_function
|
@@ -33,4 +33,4 @@ def _extract_answer_impl(
|
|
33
33
|
question: Union[str, snowpark.Column],
|
34
34
|
session: Optional[snowpark.Session] = None,
|
35
35
|
) -> Union[str, snowpark.Column]:
|
36
|
-
return call_sql_function(function, session, from_text, question)
|
36
|
+
return cast(Union[str, snowpark.Column], call_sql_function(function, session, from_text, question))
|
snowflake/cortex/_sentiment.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Optional, Union
|
1
|
+
from typing import Optional, Union, cast
|
2
2
|
|
3
3
|
from snowflake import snowpark
|
4
4
|
from snowflake.cortex._util import CORTEX_FUNCTIONS_TELEMETRY_PROJECT, call_sql_function
|
@@ -30,4 +30,4 @@ def _sentiment_impl(
|
|
30
30
|
output = call_sql_function(function, session, text)
|
31
31
|
if isinstance(output, snowpark.Column):
|
32
32
|
return output
|
33
|
-
return float(output)
|
33
|
+
return float(cast(str, output))
|
snowflake/cortex/_summarize.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Optional, Union
|
1
|
+
from typing import Optional, Union, cast
|
2
2
|
|
3
3
|
from snowflake import snowpark
|
4
4
|
from snowflake.cortex._util import CORTEX_FUNCTIONS_TELEMETRY_PROJECT, call_sql_function
|
@@ -30,4 +30,4 @@ def _summarize_impl(
|
|
30
30
|
text: Union[str, snowpark.Column],
|
31
31
|
session: Optional[snowpark.Session] = None,
|
32
32
|
) -> Union[str, snowpark.Column]:
|
33
|
-
return call_sql_function(function, session, text)
|
33
|
+
return cast(Union[str, snowpark.Column], call_sql_function(function, session, text))
|
snowflake/cortex/_translate.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Optional, Union
|
1
|
+
from typing import Optional, Union, cast
|
2
2
|
|
3
3
|
from snowflake import snowpark
|
4
4
|
from snowflake.cortex._util import CORTEX_FUNCTIONS_TELEMETRY_PROJECT, call_sql_function
|
@@ -36,4 +36,4 @@ def _translate_impl(
|
|
36
36
|
to_language: Union[str, snowpark.Column],
|
37
37
|
session: Optional[snowpark.Session] = None,
|
38
38
|
) -> Union[str, snowpark.Column]:
|
39
|
-
return call_sql_function(function, session, text, from_language, to_language)
|
39
|
+
return cast(Union[str, snowpark.Column], call_sql_function(function, session, text, from_language, to_language))
|
snowflake/cortex/_util.py
CHANGED
@@ -24,7 +24,7 @@ def call_sql_function(
|
|
24
24
|
function: str,
|
25
25
|
session: Optional[snowpark.Session],
|
26
26
|
*args: Union[str, List[str], snowpark.Column, Dict[str, Union[int, float]]],
|
27
|
-
) -> Union[str, snowpark.Column]:
|
27
|
+
) -> Union[str, List[float], snowpark.Column]:
|
28
28
|
handle_as_column = False
|
29
29
|
|
30
30
|
for arg in args:
|
@@ -32,9 +32,9 @@ def call_sql_function(
|
|
32
32
|
handle_as_column = True
|
33
33
|
|
34
34
|
if handle_as_column:
|
35
|
-
return cast(Union[str, snowpark.Column], _call_sql_function_column(function, *args))
|
35
|
+
return cast(Union[str, List[float], snowpark.Column], _call_sql_function_column(function, *args))
|
36
36
|
return cast(
|
37
|
-
Union[str, snowpark.Column],
|
37
|
+
Union[str, List[float], snowpark.Column],
|
38
38
|
_call_sql_function_immediate(function, session, *args),
|
39
39
|
)
|
40
40
|
|
@@ -49,7 +49,7 @@ def _call_sql_function_immediate(
|
|
49
49
|
function: str,
|
50
50
|
session: Optional[snowpark.Session],
|
51
51
|
*args: Union[str, List[str], snowpark.Column, Dict[str, Union[int, float]]],
|
52
|
-
) -> str:
|
52
|
+
) -> Union[str, List[float]]:
|
53
53
|
session = session or context.get_active_session()
|
54
54
|
if session is None:
|
55
55
|
raise SnowflakeAuthenticationException(
|
@@ -9,7 +9,7 @@ from importlib import metadata as importlib_metadata
|
|
9
9
|
from typing import Any, DefaultDict, Dict, List, Optional, Tuple
|
10
10
|
|
11
11
|
import yaml
|
12
|
-
from packaging import requirements, specifiers,
|
12
|
+
from packaging import requirements, specifiers, version
|
13
13
|
|
14
14
|
import snowflake.connector
|
15
15
|
from snowflake.ml._internal import env as snowml_env
|
@@ -54,15 +54,12 @@ def _validate_pip_requirement_string(req_str: str) -> requirements.Requirement:
|
|
54
54
|
"""
|
55
55
|
try:
|
56
56
|
r = requirements.Requirement(req_str)
|
57
|
-
r.name = packaging_utils.canonicalize_name(r.name)
|
58
57
|
|
59
58
|
if r.name == "python":
|
60
59
|
raise ValueError("Don't specify python as a dependency, use python version argument instead.")
|
61
60
|
except requirements.InvalidRequirement:
|
62
61
|
raise ValueError(f"Invalid package requirement {req_str} found.")
|
63
62
|
|
64
|
-
if r.marker:
|
65
|
-
raise ValueError("Markers is not supported in conda dependency.")
|
66
63
|
return r
|
67
64
|
|
68
65
|
|
@@ -84,6 +81,8 @@ def _validate_conda_dependency_string(dep_str: str) -> Tuple[str, requirements.R
|
|
84
81
|
channel_str, _, requirement_str = dep_str.rpartition("::")
|
85
82
|
r = _validate_pip_requirement_string(requirement_str)
|
86
83
|
if channel_str != "pip":
|
84
|
+
if r.marker:
|
85
|
+
raise ValueError("Markers is not supported in conda dependency.")
|
87
86
|
if r.extras:
|
88
87
|
raise ValueError("Extras is not supported in conda dependency.")
|
89
88
|
if r.url:
|
@@ -221,7 +220,7 @@ def get_local_installed_version_of_pip_package(pip_req: requirements.Requirement
|
|
221
220
|
else:
|
222
221
|
return pip_req
|
223
222
|
new_pip_req = copy.deepcopy(pip_req)
|
224
|
-
new_pip_req.specifier = specifiers.SpecifierSet(specifiers=f"=={local_dist_version}")
|
223
|
+
new_pip_req.specifier = specifiers.SpecifierSet(specifiers=f"=={version.parse(local_dist_version).base_version}")
|
225
224
|
if not pip_req.specifier.contains(local_dist_version):
|
226
225
|
warnings.warn(
|
227
226
|
f"Package requirement {str(pip_req)} specified, while version {local_dist_version} is installed. "
|
@@ -513,6 +512,7 @@ def save_conda_env_file(
|
|
513
512
|
)
|
514
513
|
|
515
514
|
with open(path, "w", encoding="utf-8") as f:
|
515
|
+
yaml.SafeDumper.ignore_aliases = lambda *args: True # type: ignore[method-assign]
|
516
516
|
yaml.safe_dump(env, stream=f, default_flow_style=False)
|
517
517
|
|
518
518
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
#!/usr/bin/env python3
|
2
|
+
import contextvars
|
2
3
|
import enum
|
3
4
|
import functools
|
4
5
|
import inspect
|
@@ -12,6 +13,7 @@ from typing import (
|
|
12
13
|
List,
|
13
14
|
Mapping,
|
14
15
|
Optional,
|
16
|
+
Set,
|
15
17
|
Tuple,
|
16
18
|
TypeVar,
|
17
19
|
Union,
|
@@ -28,7 +30,7 @@ from snowflake.ml._internal.exceptions import (
|
|
28
30
|
exceptions as snowml_exceptions,
|
29
31
|
)
|
30
32
|
from snowflake.snowpark import dataframe, exceptions as snowpark_exceptions, session
|
31
|
-
from snowflake.snowpark._internal import utils
|
33
|
+
from snowflake.snowpark._internal import server_connection, utils
|
32
34
|
|
33
35
|
_log_counter = 0
|
34
36
|
_FLUSH_SIZE = 10
|
@@ -85,6 +87,122 @@ class TelemetryField(enum.Enum):
|
|
85
87
|
FUNC_CAT_USAGE = "usage"
|
86
88
|
|
87
89
|
|
90
|
+
class _TelemetrySourceType(enum.Enum):
|
91
|
+
# Automatically inferred telemetry/statement parameters
|
92
|
+
AUTO_TELEMETRY = "SNOWML_AUTO_TELEMETRY"
|
93
|
+
# Mixture of manual and automatic telemetry/statement parameters
|
94
|
+
AUGMENT_TELEMETRY = "SNOWML_AUGMENT_TELEMETRY"
|
95
|
+
|
96
|
+
|
97
|
+
_statement_params_context_var: contextvars.ContextVar[Dict[str, str]] = contextvars.ContextVar("statement_params")
|
98
|
+
|
99
|
+
|
100
|
+
class _StatementParamsPatchManager:
|
101
|
+
def __init__(self) -> None:
|
102
|
+
self._patch_cache: Set[server_connection.ServerConnection] = set()
|
103
|
+
self._context_var: contextvars.ContextVar[Dict[str, str]] = _statement_params_context_var
|
104
|
+
|
105
|
+
def apply_patches(self) -> None:
|
106
|
+
try:
|
107
|
+
# Apply patching to all active sessions in case of multiple
|
108
|
+
for sess in session._get_active_sessions():
|
109
|
+
# Check patch cache here to avoid unnecessary context switches
|
110
|
+
if self._get_target(sess) not in self._patch_cache:
|
111
|
+
self._patch_session(sess)
|
112
|
+
except snowpark_exceptions.SnowparkSessionException:
|
113
|
+
pass
|
114
|
+
|
115
|
+
def set_statement_params(self, statement_params: Dict[str, str]) -> None:
|
116
|
+
# Only set value if not already set in context
|
117
|
+
if not self._context_var.get({}):
|
118
|
+
self._context_var.set(statement_params)
|
119
|
+
|
120
|
+
def _get_target(self, session: session.Session) -> server_connection.ServerConnection:
|
121
|
+
return cast(server_connection.ServerConnection, session._conn)
|
122
|
+
|
123
|
+
def _patch_session(self, session: session.Session, throw_on_patch_fail: bool = False) -> None:
|
124
|
+
# Extract target
|
125
|
+
try:
|
126
|
+
target = self._get_target(session)
|
127
|
+
except AttributeError:
|
128
|
+
if throw_on_patch_fail:
|
129
|
+
raise
|
130
|
+
# TODO: Log a warning, this probably means there was a breaking change in Snowpark/SnowflakeConnection
|
131
|
+
return
|
132
|
+
|
133
|
+
# Check if session has already been patched
|
134
|
+
if target in self._patch_cache:
|
135
|
+
return
|
136
|
+
self._patch_cache.add(target)
|
137
|
+
|
138
|
+
functions = [
|
139
|
+
("execute_and_notify_query_listener", "_statement_params"),
|
140
|
+
("execute_async_and_notify_query_listener", "_statement_params"),
|
141
|
+
]
|
142
|
+
|
143
|
+
for func, param_name in functions:
|
144
|
+
try:
|
145
|
+
self._patch_with_statement_params(target, func, param_name=param_name)
|
146
|
+
except AttributeError:
|
147
|
+
if throw_on_patch_fail: # primarily used for testing
|
148
|
+
raise
|
149
|
+
# TODO: Log a warning, this probably means there was a breaking change in Snowpark/SnowflakeConnection
|
150
|
+
pass
|
151
|
+
|
152
|
+
def _patch_with_statement_params(
|
153
|
+
self, target: object, function_name: str, param_name: str = "statement_params"
|
154
|
+
) -> None:
|
155
|
+
func = getattr(target, function_name)
|
156
|
+
assert callable(func)
|
157
|
+
|
158
|
+
@functools.wraps(func)
|
159
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
160
|
+
# Retrieve context level statement parameters
|
161
|
+
context_params = self._context_var.get(dict())
|
162
|
+
if not context_params:
|
163
|
+
# Exit early if not in SnowML (decorator) context
|
164
|
+
return func(*args, **kwargs)
|
165
|
+
|
166
|
+
# Extract any explicitly provided statement parameters
|
167
|
+
orig_kwargs = dict(kwargs)
|
168
|
+
in_params = kwargs.pop(param_name, None) or {}
|
169
|
+
|
170
|
+
# Inject a special flag to statement parameters so we can filter out these patched logs if necessary
|
171
|
+
# Calls that include SnowML telemetry are tagged with "SNOWML_AUGMENT_TELEMETRY"
|
172
|
+
# and calls without SnowML telemetry are tagged with "SNOWML_AUTO_TELEMETRY"
|
173
|
+
if TelemetryField.KEY_PROJECT.value in in_params:
|
174
|
+
context_params["snowml_telemetry_type"] = _TelemetrySourceType.AUGMENT_TELEMETRY.value
|
175
|
+
else:
|
176
|
+
context_params["snowml_telemetry_type"] = _TelemetrySourceType.AUTO_TELEMETRY.value
|
177
|
+
|
178
|
+
# Apply any explicitly provided statement parameters and result into function call
|
179
|
+
context_params.update(in_params)
|
180
|
+
kwargs[param_name] = context_params
|
181
|
+
|
182
|
+
try:
|
183
|
+
return func(*args, **kwargs)
|
184
|
+
except TypeError as e:
|
185
|
+
if str(e).endswith(f"unexpected keyword argument '{param_name}'"):
|
186
|
+
# TODO: Log warning that this patch is invalid
|
187
|
+
# Unwrap function for future invocations
|
188
|
+
setattr(target, function_name, func)
|
189
|
+
return func(*args, **orig_kwargs)
|
190
|
+
else:
|
191
|
+
raise
|
192
|
+
|
193
|
+
setattr(target, function_name, wrapper)
|
194
|
+
|
195
|
+
def __getstate__(self) -> Dict[str, Any]:
|
196
|
+
return {}
|
197
|
+
|
198
|
+
def __setstate__(self, state: Dict[str, Any]) -> None:
|
199
|
+
# unpickling does not call __init__ by default, do it manually here
|
200
|
+
self.__init__() # type: ignore[misc]
|
201
|
+
|
202
|
+
|
203
|
+
_patch_manager = _StatementParamsPatchManager()
|
204
|
+
|
205
|
+
|
88
206
|
def get_statement_params(
|
89
207
|
project: str, subproject: Optional[str] = None, class_name: Optional[str] = None
|
90
208
|
) -> Dict[str, Any]:
|
@@ -375,7 +493,18 @@ def send_api_usage_telemetry(
|
|
375
493
|
obj._statement_params = statement_params # type: ignore[assignment]
|
376
494
|
return obj
|
377
495
|
|
496
|
+
# Set up framework-level credit usage instrumentation
|
497
|
+
ctx = contextvars.copy_context()
|
498
|
+
_patch_manager.apply_patches()
|
499
|
+
|
500
|
+
# This function should be executed with ctx.run()
|
501
|
+
def execute_func_with_statement_params() -> _ReturnValue:
|
502
|
+
_patch_manager.set_statement_params(statement_params)
|
503
|
+
result = func(*args, **kwargs)
|
504
|
+
return update_stmt_params_if_snowpark_df(result, statement_params)
|
505
|
+
|
378
506
|
# prioritize `conn_attr_name` over the active session
|
507
|
+
telemetry_enabled = True
|
379
508
|
if conn_attr_name:
|
380
509
|
# raise AttributeError if conn attribute does not exist in `self`
|
381
510
|
conn = operator.attrgetter(conn_attr_name)(args[0])
|
@@ -387,22 +516,17 @@ def send_api_usage_telemetry(
|
|
387
516
|
else:
|
388
517
|
try:
|
389
518
|
active_session = next(iter(session._get_active_sessions()))
|
390
|
-
|
519
|
+
conn = active_session._conn._conn
|
520
|
+
telemetry_enabled = active_session.telemetry_enabled
|
391
521
|
except snowpark_exceptions.SnowparkSessionException:
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
conn = active_session._conn._conn
|
401
|
-
if (not active_session.telemetry_enabled) or (conn is None):
|
402
|
-
try:
|
403
|
-
return update_stmt_params_if_snowpark_df(func(*args, **kwargs), statement_params)
|
404
|
-
except snowml_exceptions.SnowflakeMLException as e:
|
405
|
-
raise e.original_exception from e
|
522
|
+
conn = None
|
523
|
+
|
524
|
+
if conn is None or not telemetry_enabled:
|
525
|
+
# Telemetry not enabled, just execute without our additional telemetry logic
|
526
|
+
try:
|
527
|
+
return ctx.run(execute_func_with_statement_params)
|
528
|
+
except snowml_exceptions.SnowflakeMLException as e:
|
529
|
+
raise e.original_exception from e
|
406
530
|
|
407
531
|
# TODO(hayu): [SNOW-750287] Optimize telemetry client to a singleton.
|
408
532
|
telemetry = _SourceTelemetryClient(conn=conn, project=project, subproject=subproject_name)
|
@@ -415,11 +539,11 @@ def send_api_usage_telemetry(
|
|
415
539
|
custom_tags=custom_tags,
|
416
540
|
)
|
417
541
|
try:
|
418
|
-
|
542
|
+
return ctx.run(execute_func_with_statement_params)
|
419
543
|
except Exception as e:
|
420
544
|
if not isinstance(e, snowml_exceptions.SnowflakeMLException):
|
421
545
|
# already handled via a nested decorated function
|
422
|
-
if
|
546
|
+
if getattr(e, "_snowflake_ml_handled", False):
|
423
547
|
raise e
|
424
548
|
if isinstance(e, snowpark_exceptions.SnowparkClientException):
|
425
549
|
me = snowml_exceptions.SnowflakeMLException(
|
@@ -438,8 +562,6 @@ def send_api_usage_telemetry(
|
|
438
562
|
raise me.original_exception from None
|
439
563
|
else:
|
440
564
|
raise me.original_exception from e
|
441
|
-
else:
|
442
|
-
return update_stmt_params_if_snowpark_df(res, statement_params)
|
443
565
|
finally:
|
444
566
|
telemetry.send_function_usage_telemetry(**telemetry_args)
|
445
567
|
global _log_counter
|
@@ -0,0 +1,50 @@
|
|
1
|
+
from enum import Enum
|
2
|
+
from typing import Any, Dict, Optional
|
3
|
+
|
4
|
+
from snowflake.ml._internal.utils import query_result_checker, sql_identifier
|
5
|
+
from snowflake.snowpark import session
|
6
|
+
|
7
|
+
MAX_IDENTIFIER_LENGTH = 255
|
8
|
+
|
9
|
+
|
10
|
+
class SnowflakeDbObjectType(Enum):
|
11
|
+
TABLE = "TABLE"
|
12
|
+
WAREHOUSE = "WAREHOUSE"
|
13
|
+
|
14
|
+
|
15
|
+
def db_object_exists(
|
16
|
+
session: session.Session,
|
17
|
+
object_type: SnowflakeDbObjectType,
|
18
|
+
object_name: sql_identifier.SqlIdentifier,
|
19
|
+
*,
|
20
|
+
database_name: Optional[sql_identifier.SqlIdentifier] = None,
|
21
|
+
schema_name: Optional[sql_identifier.SqlIdentifier] = None,
|
22
|
+
statement_params: Optional[Dict[str, Any]] = None,
|
23
|
+
) -> bool:
|
24
|
+
"""Check if object exists in database.
|
25
|
+
|
26
|
+
Args:
|
27
|
+
session: Active Snowpark Session.
|
28
|
+
object_type: Type of object to search for.
|
29
|
+
object_name: Name of object to search for.
|
30
|
+
database_name: Optional database name to search in. Only used if both schema is also provided.
|
31
|
+
schema_name: Optional schema to search in.
|
32
|
+
statement_params: Optional set of statement_params to include with queries.
|
33
|
+
|
34
|
+
Returns:
|
35
|
+
boolean indicating whether object exists.
|
36
|
+
"""
|
37
|
+
optional_in_clause = ""
|
38
|
+
if database_name and schema_name:
|
39
|
+
optional_in_clause = f" IN {database_name}.{schema_name}"
|
40
|
+
|
41
|
+
result = (
|
42
|
+
query_result_checker.SqlResultValidator(
|
43
|
+
session,
|
44
|
+
f"""SHOW {object_type.value}S LIKE '{object_name}'{optional_in_clause}""",
|
45
|
+
statement_params=statement_params,
|
46
|
+
)
|
47
|
+
.has_column("name", allow_empty=True) # TODO: Check this is actually what is returned from server
|
48
|
+
.validate()
|
49
|
+
)
|
50
|
+
return len(result) == 1
|
@@ -10,9 +10,11 @@ SF_QUOTED_IDENTIFIER = '"(?:[^"]|"")*"'
|
|
10
10
|
_SF_IDENTIFIER = f"({_SF_UNQUOTED_CASE_INSENSITIVE_IDENTIFIER}|{SF_QUOTED_IDENTIFIER})"
|
11
11
|
SF_IDENTIFIER_RE = re.compile(_SF_IDENTIFIER)
|
12
12
|
_SF_SCHEMA_LEVEL_OBJECT = (
|
13
|
-
rf"(?:(?:(?P<db>{_SF_IDENTIFIER})\.)?(?P<schema>{_SF_IDENTIFIER})\.)?(?P<object>{_SF_IDENTIFIER})
|
13
|
+
rf"(?:(?:(?P<db>{_SF_IDENTIFIER})\.)?(?P<schema>{_SF_IDENTIFIER})\.)?(?P<object>{_SF_IDENTIFIER})"
|
14
14
|
)
|
15
|
+
_SF_STAGE_PATH = rf"{_SF_SCHEMA_LEVEL_OBJECT}(?P<path>.*)"
|
15
16
|
_SF_SCHEMA_LEVEL_OBJECT_RE = re.compile(_SF_SCHEMA_LEVEL_OBJECT)
|
17
|
+
_SF_STAGE_PATH_RE = re.compile(_SF_STAGE_PATH)
|
16
18
|
|
17
19
|
UNQUOTED_CASE_INSENSITIVE_RE = re.compile(f"^({_SF_UNQUOTED_CASE_INSENSITIVE_IDENTIFIER})$")
|
18
20
|
UNQUOTED_CASE_SENSITIVE_RE = re.compile(f"^({_SF_UNQUOTED_CASE_SENSITIVE_IDENTIFIER})$")
|
@@ -139,29 +141,61 @@ def rename_to_valid_snowflake_identifier(name: str) -> str:
|
|
139
141
|
|
140
142
|
|
141
143
|
def parse_schema_level_object_identifier(
|
144
|
+
object_name: str,
|
145
|
+
) -> Tuple[Union[str, Any], Union[str, Any], Union[str, Any]]:
|
146
|
+
"""Parse a string which starts with schema level object.
|
147
|
+
|
148
|
+
Args:
|
149
|
+
object_name: A string starts with a schema level object path, which is in the format
|
150
|
+
'<db>.<schema>.<object_name>'. Here, '<db>', '<schema>' and '<object_name>' are all snowflake identifiers.
|
151
|
+
|
152
|
+
Returns:
|
153
|
+
A tuple of 3 strings in the form of (db, schema, object_name).
|
154
|
+
|
155
|
+
Raises:
|
156
|
+
ValueError: If the id is invalid.
|
157
|
+
"""
|
158
|
+
res = _SF_SCHEMA_LEVEL_OBJECT_RE.fullmatch(object_name)
|
159
|
+
if not res:
|
160
|
+
raise ValueError(
|
161
|
+
"Invalid identifier because it does not follow the pattern. "
|
162
|
+
f"It should start with [[database.]schema.]object. Getting {object_name}"
|
163
|
+
)
|
164
|
+
return (
|
165
|
+
res.group("db"),
|
166
|
+
res.group("schema"),
|
167
|
+
res.group("object"),
|
168
|
+
)
|
169
|
+
|
170
|
+
|
171
|
+
def parse_snowflake_stage_path(
|
142
172
|
path: str,
|
143
173
|
) -> Tuple[Union[str, Any], Union[str, Any], Union[str, Any], Union[str, Any]]:
|
144
|
-
"""Parse a string which
|
174
|
+
"""Parse a string which represents a snowflake stage path.
|
145
175
|
|
146
176
|
Args:
|
147
|
-
path: A string starts with a schema level object path, which is in the format
|
148
|
-
Here, '<db>', '<schema>' and '<object_name>' are all snowflake
|
177
|
+
path: A string starts with a schema level object path, which is in the format
|
178
|
+
'<db>.<schema>.<object_name><path>'. Here, '<db>', '<schema>' and '<object_name>' are all snowflake
|
179
|
+
identifiers.
|
149
180
|
|
150
181
|
Returns:
|
151
|
-
A tuple of 4 strings in the form of (db, schema, object_name,
|
152
|
-
from the schema level object and '
|
182
|
+
A tuple of 4 strings in the form of (db, schema, object_name, path). 'db', 'schema', 'object_name' are parsed
|
183
|
+
from the schema level object and 'path' are all the content post to the object.
|
153
184
|
|
154
185
|
Raises:
|
155
186
|
ValueError: If the id is invalid.
|
156
187
|
"""
|
157
|
-
res =
|
188
|
+
res = _SF_STAGE_PATH_RE.fullmatch(path)
|
158
189
|
if not res:
|
159
|
-
raise ValueError(
|
190
|
+
raise ValueError(
|
191
|
+
"Invalid identifier because it does not follow the pattern. "
|
192
|
+
f"It should start with [[database.]schema.]object. Getting {path}"
|
193
|
+
)
|
160
194
|
return (
|
161
195
|
res.group("db"),
|
162
196
|
res.group("schema"),
|
163
197
|
res.group("object"),
|
164
|
-
res.group("
|
198
|
+
res.group("path"),
|
165
199
|
)
|
166
200
|
|
167
201
|
|
@@ -175,8 +209,11 @@ def is_fully_qualified_name(name: str) -> bool:
|
|
175
209
|
Returns:
|
176
210
|
bool: True if the name is fully qualified, False otherwise.
|
177
211
|
"""
|
178
|
-
|
179
|
-
|
212
|
+
try:
|
213
|
+
res = parse_schema_level_object_identifier(name)
|
214
|
+
return all(res)
|
215
|
+
except ValueError:
|
216
|
+
return False
|
180
217
|
|
181
218
|
|
182
219
|
def get_schema_level_object_identifier(
|