snowflake-ml-python 1.8.2__py3-none-any.whl → 1.8.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/__init__.py +7 -1
- snowflake/cortex/_classify_text.py +3 -3
- snowflake/cortex/_complete.py +23 -24
- snowflake/cortex/_embed_text_1024.py +4 -4
- snowflake/cortex/_embed_text_768.py +4 -4
- snowflake/cortex/_finetune.py +8 -8
- snowflake/cortex/_util.py +8 -12
- snowflake/ml/_internal/env.py +4 -3
- snowflake/ml/_internal/env_utils.py +63 -34
- snowflake/ml/_internal/file_utils.py +10 -21
- snowflake/ml/_internal/human_readable_id/hrid_generator_base.py +5 -7
- snowflake/ml/_internal/init_utils.py +2 -3
- snowflake/ml/_internal/lineage/lineage_utils.py +6 -6
- snowflake/ml/_internal/platform_capabilities.py +18 -16
- snowflake/ml/_internal/telemetry.py +39 -52
- snowflake/ml/_internal/type_utils.py +3 -3
- snowflake/ml/_internal/utils/db_utils.py +2 -2
- snowflake/ml/_internal/utils/identifier.py +10 -10
- snowflake/ml/_internal/utils/import_utils.py +2 -2
- snowflake/ml/_internal/utils/parallelize.py +7 -7
- snowflake/ml/_internal/utils/pkg_version_utils.py +11 -11
- snowflake/ml/_internal/utils/query_result_checker.py +4 -4
- snowflake/ml/_internal/utils/snowflake_env.py +28 -6
- snowflake/ml/_internal/utils/snowpark_dataframe_utils.py +2 -2
- snowflake/ml/_internal/utils/sql_identifier.py +3 -3
- snowflake/ml/_internal/utils/table_manager.py +9 -9
- snowflake/ml/data/_internal/arrow_ingestor.py +7 -7
- snowflake/ml/data/data_connector.py +15 -36
- snowflake/ml/data/data_ingestor.py +4 -15
- snowflake/ml/data/data_source.py +2 -2
- snowflake/ml/data/ingestor_utils.py +3 -3
- snowflake/ml/data/torch_utils.py +5 -5
- snowflake/ml/dataset/dataset.py +11 -11
- snowflake/ml/dataset/dataset_metadata.py +8 -8
- snowflake/ml/dataset/dataset_reader.py +7 -7
- snowflake/ml/feature_store/__init__.py +1 -1
- snowflake/ml/feature_store/access_manager.py +7 -7
- snowflake/ml/feature_store/entity.py +6 -6
- snowflake/ml/feature_store/examples/airline_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/airline_features/features/plane_features.py +1 -3
- snowflake/ml/feature_store/examples/airline_features/features/weather_features.py +1 -3
- snowflake/ml/feature_store/examples/citibike_trip_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/citibike_trip_features/features/station_feature.py +1 -3
- snowflake/ml/feature_store/examples/citibike_trip_features/features/trip_feature.py +1 -3
- snowflake/ml/feature_store/examples/example_helper.py +16 -16
- snowflake/ml/feature_store/examples/new_york_taxi_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/new_york_taxi_features/features/location_features.py +1 -3
- snowflake/ml/feature_store/examples/new_york_taxi_features/features/trip_features.py +1 -3
- snowflake/ml/feature_store/examples/wine_quality_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/wine_quality_features/features/managed_wine_features.py +1 -3
- snowflake/ml/feature_store/examples/wine_quality_features/features/static_wine_features.py +1 -3
- snowflake/ml/feature_store/feature_store.py +52 -64
- snowflake/ml/feature_store/feature_view.py +24 -24
- snowflake/ml/fileset/embedded_stage_fs.py +5 -5
- snowflake/ml/fileset/fileset.py +5 -5
- snowflake/ml/fileset/sfcfs.py +13 -13
- snowflake/ml/fileset/stage_fs.py +15 -15
- snowflake/ml/jobs/_utils/constants.py +1 -1
- snowflake/ml/jobs/_utils/interop_utils.py +10 -10
- snowflake/ml/jobs/_utils/payload_utils.py +45 -46
- snowflake/ml/jobs/_utils/scripts/get_instance_ip.py +4 -4
- snowflake/ml/jobs/_utils/scripts/mljob_launcher.py +8 -5
- snowflake/ml/jobs/_utils/scripts/signal_workers.py +8 -8
- snowflake/ml/jobs/_utils/spec_utils.py +18 -29
- snowflake/ml/jobs/_utils/types.py +2 -2
- snowflake/ml/jobs/decorators.py +10 -5
- snowflake/ml/jobs/job.py +87 -30
- snowflake/ml/jobs/manager.py +86 -56
- snowflake/ml/lineage/lineage_node.py +5 -5
- snowflake/ml/model/_client/model/model_impl.py +3 -3
- snowflake/ml/model/_client/model/model_version_impl.py +103 -35
- snowflake/ml/model/_client/ops/metadata_ops.py +7 -7
- snowflake/ml/model/_client/ops/model_ops.py +41 -41
- snowflake/ml/model/_client/ops/service_ops.py +217 -32
- snowflake/ml/model/_client/service/model_deployment_spec.py +359 -65
- snowflake/ml/model/_client/service/model_deployment_spec_schema.py +69 -24
- snowflake/ml/model/_client/sql/model.py +8 -8
- snowflake/ml/model/_client/sql/model_version.py +26 -26
- snowflake/ml/model/_client/sql/service.py +17 -26
- snowflake/ml/model/_client/sql/stage.py +2 -2
- snowflake/ml/model/_client/sql/tag.py +6 -6
- snowflake/ml/model/_model_composer/model_composer.py +58 -32
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +20 -16
- snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py +14 -13
- snowflake/ml/model/_model_composer/model_method/model_method.py +3 -3
- snowflake/ml/model/_packager/model_env/model_env.py +28 -25
- snowflake/ml/model/_packager/model_handler.py +4 -4
- snowflake/ml/model/_packager/model_handlers/_base.py +2 -2
- snowflake/ml/model/_packager/model_handlers/_utils.py +47 -5
- snowflake/ml/model/_packager/model_handlers/catboost.py +5 -5
- snowflake/ml/model/_packager/model_handlers/custom.py +9 -5
- snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py +7 -21
- snowflake/ml/model/_packager/model_handlers/keras.py +4 -4
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +4 -14
- snowflake/ml/model/_packager/model_handlers/mlflow.py +3 -3
- snowflake/ml/model/_packager/model_handlers/pytorch.py +5 -6
- snowflake/ml/model/_packager/model_handlers/sentence_transformers.py +5 -5
- snowflake/ml/model/_packager/model_handlers/sklearn.py +104 -46
- snowflake/ml/model/_packager/model_handlers/snowmlmodel.py +3 -3
- snowflake/ml/model/_packager/model_handlers/tensorflow.py +11 -8
- snowflake/ml/model/_packager/model_handlers/torchscript.py +6 -6
- snowflake/ml/model/_packager/model_handlers/xgboost.py +21 -22
- snowflake/ml/model/_packager/model_meta/model_blob_meta.py +2 -2
- snowflake/ml/model/_packager/model_meta/model_meta.py +39 -38
- snowflake/ml/model/_packager/model_meta/model_meta_schema.py +14 -11
- snowflake/ml/model/_packager/model_meta_migrator/base_migrator.py +3 -3
- snowflake/ml/model/_packager/model_meta_migrator/migrator_plans.py +3 -3
- snowflake/ml/model/_packager/model_meta_migrator/migrator_v1.py +4 -4
- snowflake/ml/model/_packager/model_packager.py +11 -9
- snowflake/ml/model/_packager/model_runtime/_snowml_inference_alternative_requirements.py +32 -1
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +4 -2
- snowflake/ml/model/_signatures/core.py +16 -24
- snowflake/ml/model/_signatures/dmatrix_handler.py +17 -4
- snowflake/ml/model/_signatures/utils.py +6 -6
- snowflake/ml/model/custom_model.py +24 -11
- snowflake/ml/model/model_signature.py +12 -23
- snowflake/ml/model/models/huggingface_pipeline.py +7 -4
- snowflake/ml/model/type_hints.py +3 -3
- snowflake/ml/modeling/_internal/estimator_utils.py +7 -7
- snowflake/ml/modeling/_internal/local_implementations/pandas_handlers.py +6 -6
- snowflake/ml/modeling/_internal/local_implementations/pandas_trainer.py +7 -7
- snowflake/ml/modeling/_internal/model_specifications.py +8 -10
- snowflake/ml/modeling/_internal/model_trainer.py +5 -5
- snowflake/ml/modeling/_internal/model_trainer_builder.py +6 -6
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +30 -30
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_handlers.py +13 -13
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +31 -31
- snowflake/ml/modeling/_internal/snowpark_implementations/xgboost_external_memory_trainer.py +19 -19
- snowflake/ml/modeling/_internal/transformer_protocols.py +17 -17
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +9 -1
- snowflake/ml/modeling/cluster/affinity_propagation.py +9 -1
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +9 -1
- snowflake/ml/modeling/cluster/birch.py +9 -1
- snowflake/ml/modeling/cluster/bisecting_k_means.py +9 -1
- snowflake/ml/modeling/cluster/dbscan.py +9 -1
- snowflake/ml/modeling/cluster/feature_agglomeration.py +9 -1
- snowflake/ml/modeling/cluster/k_means.py +9 -1
- snowflake/ml/modeling/cluster/mean_shift.py +9 -1
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +9 -1
- snowflake/ml/modeling/cluster/optics.py +9 -1
- snowflake/ml/modeling/cluster/spectral_biclustering.py +9 -1
- snowflake/ml/modeling/cluster/spectral_clustering.py +9 -1
- snowflake/ml/modeling/cluster/spectral_coclustering.py +9 -1
- snowflake/ml/modeling/compose/column_transformer.py +9 -1
- snowflake/ml/modeling/compose/transformed_target_regressor.py +9 -1
- snowflake/ml/modeling/covariance/elliptic_envelope.py +9 -1
- snowflake/ml/modeling/covariance/empirical_covariance.py +9 -1
- snowflake/ml/modeling/covariance/graphical_lasso.py +9 -1
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +9 -1
- snowflake/ml/modeling/covariance/ledoit_wolf.py +9 -1
- snowflake/ml/modeling/covariance/min_cov_det.py +9 -1
- snowflake/ml/modeling/covariance/oas.py +9 -1
- snowflake/ml/modeling/covariance/shrunk_covariance.py +9 -1
- snowflake/ml/modeling/decomposition/dictionary_learning.py +9 -1
- snowflake/ml/modeling/decomposition/factor_analysis.py +9 -1
- snowflake/ml/modeling/decomposition/fast_ica.py +9 -1
- snowflake/ml/modeling/decomposition/incremental_pca.py +9 -1
- snowflake/ml/modeling/decomposition/kernel_pca.py +9 -1
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +9 -1
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +9 -1
- snowflake/ml/modeling/decomposition/pca.py +9 -1
- snowflake/ml/modeling/decomposition/sparse_pca.py +9 -1
- snowflake/ml/modeling/decomposition/truncated_svd.py +9 -1
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +9 -1
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +9 -1
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +9 -1
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +9 -1
- snowflake/ml/modeling/ensemble/bagging_classifier.py +9 -1
- snowflake/ml/modeling/ensemble/bagging_regressor.py +9 -1
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +9 -1
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +9 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +9 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +9 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +9 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +9 -1
- snowflake/ml/modeling/ensemble/isolation_forest.py +9 -1
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +9 -1
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +9 -1
- snowflake/ml/modeling/ensemble/stacking_regressor.py +9 -1
- snowflake/ml/modeling/ensemble/voting_classifier.py +9 -1
- snowflake/ml/modeling/ensemble/voting_regressor.py +9 -1
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +9 -1
- snowflake/ml/modeling/feature_selection/select_fdr.py +9 -1
- snowflake/ml/modeling/feature_selection/select_fpr.py +9 -1
- snowflake/ml/modeling/feature_selection/select_fwe.py +9 -1
- snowflake/ml/modeling/feature_selection/select_k_best.py +9 -1
- snowflake/ml/modeling/feature_selection/select_percentile.py +9 -1
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +9 -1
- snowflake/ml/modeling/feature_selection/variance_threshold.py +9 -1
- snowflake/ml/modeling/framework/_utils.py +10 -10
- snowflake/ml/modeling/framework/base.py +32 -32
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +9 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +9 -1
- snowflake/ml/modeling/impute/__init__.py +1 -1
- snowflake/ml/modeling/impute/iterative_imputer.py +9 -1
- snowflake/ml/modeling/impute/knn_imputer.py +9 -1
- snowflake/ml/modeling/impute/missing_indicator.py +9 -1
- snowflake/ml/modeling/impute/simple_imputer.py +5 -5
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +9 -1
- snowflake/ml/modeling/kernel_approximation/nystroem.py +9 -1
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +9 -1
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +9 -1
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +9 -1
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +9 -1
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +9 -1
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/ard_regression.py +9 -1
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +9 -1
- snowflake/ml/modeling/linear_model/elastic_net.py +9 -1
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +9 -1
- snowflake/ml/modeling/linear_model/gamma_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/huber_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/lars.py +9 -1
- snowflake/ml/modeling/linear_model/lars_cv.py +9 -1
- snowflake/ml/modeling/linear_model/lasso.py +9 -1
- snowflake/ml/modeling/linear_model/lasso_cv.py +9 -1
- snowflake/ml/modeling/linear_model/lasso_lars.py +9 -1
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +9 -1
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +9 -1
- snowflake/ml/modeling/linear_model/linear_regression.py +9 -1
- snowflake/ml/modeling/linear_model/logistic_regression.py +9 -1
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +9 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +9 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +9 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +9 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +9 -1
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +9 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +9 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/perceptron.py +9 -1
- snowflake/ml/modeling/linear_model/poisson_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/ransac_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/ridge.py +9 -1
- snowflake/ml/modeling/linear_model/ridge_classifier.py +9 -1
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +9 -1
- snowflake/ml/modeling/linear_model/ridge_cv.py +9 -1
- snowflake/ml/modeling/linear_model/sgd_classifier.py +9 -1
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +9 -1
- snowflake/ml/modeling/linear_model/sgd_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +9 -1
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +9 -1
- snowflake/ml/modeling/manifold/isomap.py +9 -1
- snowflake/ml/modeling/manifold/mds.py +9 -1
- snowflake/ml/modeling/manifold/spectral_embedding.py +9 -1
- snowflake/ml/modeling/manifold/tsne.py +9 -1
- snowflake/ml/modeling/metrics/__init__.py +1 -1
- snowflake/ml/modeling/metrics/classification.py +39 -39
- snowflake/ml/modeling/metrics/metrics_utils.py +12 -12
- snowflake/ml/modeling/metrics/ranking.py +7 -7
- snowflake/ml/modeling/metrics/regression.py +13 -13
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +9 -1
- snowflake/ml/modeling/mixture/gaussian_mixture.py +9 -1
- snowflake/ml/modeling/model_selection/__init__.py +1 -1
- snowflake/ml/modeling/model_selection/grid_search_cv.py +7 -7
- snowflake/ml/modeling/model_selection/randomized_search_cv.py +7 -7
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +9 -1
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +9 -1
- snowflake/ml/modeling/multiclass/output_code_classifier.py +9 -1
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +9 -1
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +9 -1
- snowflake/ml/modeling/naive_bayes/complement_nb.py +9 -1
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +9 -1
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +9 -1
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +9 -1
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +9 -1
- snowflake/ml/modeling/neighbors/kernel_density.py +9 -1
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +9 -1
- snowflake/ml/modeling/neighbors/nearest_centroid.py +9 -1
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +9 -1
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +9 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +9 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +9 -1
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +9 -1
- snowflake/ml/modeling/neural_network/mlp_classifier.py +9 -1
- snowflake/ml/modeling/neural_network/mlp_regressor.py +9 -1
- snowflake/ml/modeling/pipeline/__init__.py +1 -1
- snowflake/ml/modeling/pipeline/pipeline.py +18 -18
- snowflake/ml/modeling/preprocessing/__init__.py +1 -1
- snowflake/ml/modeling/preprocessing/k_bins_discretizer.py +13 -13
- snowflake/ml/modeling/preprocessing/max_abs_scaler.py +4 -4
- snowflake/ml/modeling/preprocessing/min_max_scaler.py +8 -8
- snowflake/ml/modeling/preprocessing/normalizer.py +0 -1
- snowflake/ml/modeling/preprocessing/one_hot_encoder.py +28 -28
- snowflake/ml/modeling/preprocessing/ordinal_encoder.py +9 -9
- snowflake/ml/modeling/preprocessing/polynomial_features.py +9 -1
- snowflake/ml/modeling/preprocessing/robust_scaler.py +7 -7
- snowflake/ml/modeling/preprocessing/standard_scaler.py +5 -5
- snowflake/ml/modeling/semi_supervised/label_propagation.py +9 -1
- snowflake/ml/modeling/semi_supervised/label_spreading.py +9 -1
- snowflake/ml/modeling/svm/linear_svc.py +9 -1
- snowflake/ml/modeling/svm/linear_svr.py +9 -1
- snowflake/ml/modeling/svm/nu_svc.py +9 -1
- snowflake/ml/modeling/svm/nu_svr.py +9 -1
- snowflake/ml/modeling/svm/svc.py +9 -1
- snowflake/ml/modeling/svm/svr.py +9 -1
- snowflake/ml/modeling/tree/decision_tree_classifier.py +9 -1
- snowflake/ml/modeling/tree/decision_tree_regressor.py +9 -1
- snowflake/ml/modeling/tree/extra_tree_classifier.py +9 -1
- snowflake/ml/modeling/tree/extra_tree_regressor.py +9 -1
- snowflake/ml/modeling/xgboost/xgb_classifier.py +9 -1
- snowflake/ml/modeling/xgboost/xgb_regressor.py +9 -1
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +9 -1
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +9 -1
- snowflake/ml/monitoring/_client/model_monitor_sql_client.py +26 -26
- snowflake/ml/monitoring/_manager/model_monitor_manager.py +5 -5
- snowflake/ml/monitoring/entities/model_monitor_config.py +6 -6
- snowflake/ml/monitoring/explain_visualize.py +286 -0
- snowflake/ml/registry/_manager/model_manager.py +55 -32
- snowflake/ml/registry/registry.py +39 -31
- snowflake/ml/utils/authentication.py +2 -2
- snowflake/ml/utils/connection_params.py +5 -5
- snowflake/ml/utils/sparse.py +5 -4
- snowflake/ml/utils/sql_client.py +1 -2
- snowflake/ml/version.py +2 -1
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.4.dist-info}/METADATA +55 -14
- snowflake_ml_python-1.8.4.dist-info/RECORD +419 -0
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.4.dist-info}/WHEEL +1 -1
- snowflake/ml/model/_packager/model_meta/_packaging_requirements.py +0 -1
- snowflake/ml/modeling/_internal/constants.py +0 -2
- snowflake_ml_python-1.8.2.dist-info/RECORD +0 -420
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.4.dist-info}/licenses/LICENSE.txt +0 -0
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.4.dist-info}/top_level.txt +0 -0
snowflake/ml/data/torch_utils.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Any,
|
1
|
+
from typing import Any, Iterator, Optional, Union
|
2
2
|
|
3
3
|
import numpy as np
|
4
4
|
import numpy.typing as npt
|
@@ -7,7 +7,7 @@ import torch.utils.data
|
|
7
7
|
from snowflake.ml.data import data_ingestor
|
8
8
|
|
9
9
|
|
10
|
-
class TorchDatasetWrapper(torch.utils.data.IterableDataset[
|
10
|
+
class TorchDatasetWrapper(torch.utils.data.IterableDataset[dict[str, Any]]):
|
11
11
|
"""Wrap a DataIngestor into a PyTorch IterableDataset"""
|
12
12
|
|
13
13
|
def __init__(
|
@@ -32,7 +32,7 @@ class TorchDatasetWrapper(torch.utils.data.IterableDataset[Dict[str, Any]]):
|
|
32
32
|
self._squeeze_outputs = squeeze
|
33
33
|
self._expand_dims = expand_dims
|
34
34
|
|
35
|
-
def __iter__(self) -> Iterator[
|
35
|
+
def __iter__(self) -> Iterator[dict[str, Union[npt.NDArray[Any], list[Any]]]]:
|
36
36
|
max_idx = 0
|
37
37
|
filter_idx = 0
|
38
38
|
worker_info = torch.utils.data.get_worker_info()
|
@@ -59,7 +59,7 @@ class TorchDatasetWrapper(torch.utils.data.IterableDataset[Dict[str, Any]]):
|
|
59
59
|
counter = 0
|
60
60
|
|
61
61
|
|
62
|
-
class TorchDataPipeWrapper(TorchDatasetWrapper, torch.utils.data.IterDataPipe[
|
62
|
+
class TorchDataPipeWrapper(TorchDatasetWrapper, torch.utils.data.IterDataPipe[dict[str, Any]]):
|
63
63
|
"""Wrap a DataIngestor into a PyTorch IterDataPipe"""
|
64
64
|
|
65
65
|
def __init__(
|
@@ -77,7 +77,7 @@ class TorchDataPipeWrapper(TorchDatasetWrapper, torch.utils.data.IterDataPipe[Di
|
|
77
77
|
|
78
78
|
def _preprocess_array(
|
79
79
|
arr: npt.NDArray[Any], squeeze: bool = False, expand_dims: bool = True
|
80
|
-
) -> Union[npt.NDArray[Any],
|
80
|
+
) -> Union[npt.NDArray[Any], list[np.object_]]:
|
81
81
|
"""Preprocesses batch column values."""
|
82
82
|
single_dimensional = arr.ndim < 2 and not arr.dtype == np.object_
|
83
83
|
|
snowflake/ml/dataset/dataset.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import json
|
2
2
|
import warnings
|
3
3
|
from datetime import datetime
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, Optional, Union
|
5
5
|
|
6
6
|
from snowflake import snowpark
|
7
7
|
from snowflake.ml._internal import telemetry
|
@@ -46,8 +46,8 @@ class DatasetVersion:
|
|
46
46
|
self._version = version
|
47
47
|
self._session: snowpark.Session = self._parent._session
|
48
48
|
|
49
|
-
self._properties: Optional[
|
50
|
-
self._raw_metadata: Optional[
|
49
|
+
self._properties: Optional[dict[str, Any]] = None
|
50
|
+
self._raw_metadata: Optional[dict[str, Any]] = None
|
51
51
|
self._metadata: Optional[dataset_metadata.DatasetMetadata] = None
|
52
52
|
|
53
53
|
@property
|
@@ -66,14 +66,14 @@ class DatasetVersion:
|
|
66
66
|
return comment
|
67
67
|
|
68
68
|
@property
|
69
|
-
def label_cols(self) ->
|
69
|
+
def label_cols(self) -> list[str]:
|
70
70
|
metadata = self._get_metadata()
|
71
71
|
if metadata is None or metadata.label_cols is None:
|
72
72
|
return []
|
73
73
|
return metadata.label_cols
|
74
74
|
|
75
75
|
@property
|
76
|
-
def exclude_cols(self) ->
|
76
|
+
def exclude_cols(self) -> list[str]:
|
77
77
|
metadata = self._get_metadata()
|
78
78
|
if metadata is None or metadata.exclude_cols is None:
|
79
79
|
return []
|
@@ -115,7 +115,7 @@ class DatasetVersion:
|
|
115
115
|
return path
|
116
116
|
|
117
117
|
@telemetry.send_api_usage_telemetry(project=_PROJECT)
|
118
|
-
def list_files(self, subdir: Optional[str] = None) ->
|
118
|
+
def list_files(self, subdir: Optional[str] = None) -> list[snowpark.Row]:
|
119
119
|
"""Get the list of remote file paths for the current DatasetVersion."""
|
120
120
|
return self._session.sql(f"LIST {self.url()}{subdir or ''}").collect(
|
121
121
|
statement_params=_TELEMETRY_STATEMENT_PARAMS
|
@@ -244,7 +244,7 @@ class Dataset(lineage_node.LineageNode):
|
|
244
244
|
raise
|
245
245
|
|
246
246
|
@telemetry.send_api_usage_telemetry(project=_PROJECT)
|
247
|
-
def list_versions(self, detailed: bool = False) -> Union[
|
247
|
+
def list_versions(self, detailed: bool = False) -> Union[list[str], list[snowpark.Row]]:
|
248
248
|
"""Return list of versions"""
|
249
249
|
versions = self._list_versions()
|
250
250
|
versions.sort(key=lambda r: r[_DATASET_VERSION_NAME_COL])
|
@@ -271,8 +271,8 @@ class Dataset(lineage_node.LineageNode):
|
|
271
271
|
version: str,
|
272
272
|
input_dataframe: snowpark.DataFrame,
|
273
273
|
shuffle: bool = False,
|
274
|
-
exclude_cols: Optional[
|
275
|
-
label_cols: Optional[
|
274
|
+
exclude_cols: Optional[list[str]] = None,
|
275
|
+
label_cols: Optional[list[str]] = None,
|
276
276
|
properties: Optional[dataset_metadata.DatasetPropertiesType] = None,
|
277
277
|
partition_by: Optional[str] = None,
|
278
278
|
comment: Optional[str] = None,
|
@@ -423,7 +423,7 @@ class Dataset(lineage_node.LineageNode):
|
|
423
423
|
statement_params=_TELEMETRY_STATEMENT_PARAMS
|
424
424
|
)
|
425
425
|
|
426
|
-
def _list_versions(self, pattern: Optional[str] = None) ->
|
426
|
+
def _list_versions(self, pattern: Optional[str] = None) -> list[snowpark.Row]:
|
427
427
|
"""Return list of versions"""
|
428
428
|
try:
|
429
429
|
pattern_clause = f" LIKE '{pattern}'" if pattern else ""
|
@@ -469,7 +469,7 @@ lineage_node.DOMAIN_LINEAGE_REGISTRY["dataset"] = Dataset
|
|
469
469
|
# Utility methods
|
470
470
|
|
471
471
|
|
472
|
-
def _get_schema_level_identifier(session: snowpark.Session, dataset_name: str) ->
|
472
|
+
def _get_schema_level_identifier(session: snowpark.Session, dataset_name: str) -> tuple[str, str, str]:
|
473
473
|
"""Resolve a dataset name into a validated schema-level location identifier"""
|
474
474
|
db, schema, object_name = identifier.parse_schema_level_object_identifier(dataset_name)
|
475
475
|
db = db or session.get_current_database()
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import dataclasses
|
2
2
|
import json
|
3
3
|
import typing
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, Optional, Union
|
5
5
|
|
6
6
|
_PROPERTY_TYPE_KEY = "$proptype$"
|
7
7
|
DATASET_SCHEMA_VERSION = "1"
|
@@ -20,15 +20,15 @@ class FeatureStoreMetadata:
|
|
20
20
|
"""
|
21
21
|
|
22
22
|
spine_query: str
|
23
|
-
serialized_feature_views: Optional[
|
24
|
-
compact_feature_views: Optional[
|
23
|
+
serialized_feature_views: Optional[list[str]] = None
|
24
|
+
compact_feature_views: Optional[list[str]] = None
|
25
25
|
spine_timestamp_col: Optional[str] = None
|
26
26
|
|
27
27
|
def to_json(self) -> str:
|
28
28
|
return json.dumps(dataclasses.asdict(self))
|
29
29
|
|
30
30
|
@classmethod
|
31
|
-
def from_json(cls, input_json: Union[
|
31
|
+
def from_json(cls, input_json: Union[dict[str, Any], str, bytes]) -> "FeatureStoreMetadata":
|
32
32
|
if isinstance(input_json, dict):
|
33
33
|
return cls(**input_json)
|
34
34
|
return cls(**json.loads(input_json))
|
@@ -61,8 +61,8 @@ class DatasetMetadata:
|
|
61
61
|
|
62
62
|
source_query: str
|
63
63
|
owner: str
|
64
|
-
exclude_cols: Optional[
|
65
|
-
label_cols: Optional[
|
64
|
+
exclude_cols: Optional[list[str]] = None
|
65
|
+
label_cols: Optional[list[str]] = None
|
66
66
|
properties: Optional[DatasetPropertiesType] = None
|
67
67
|
schema_version: str = dataclasses.field(default=DATASET_SCHEMA_VERSION, init=False)
|
68
68
|
|
@@ -78,11 +78,11 @@ class DatasetMetadata:
|
|
78
78
|
return json.dumps(state_dict)
|
79
79
|
|
80
80
|
@classmethod
|
81
|
-
def from_json(cls, input_json: Union[
|
81
|
+
def from_json(cls, input_json: Union[dict[str, Any], str, bytes]) -> "DatasetMetadata":
|
82
82
|
if not input_json:
|
83
83
|
raise ValueError("json_str was empty or None")
|
84
84
|
try:
|
85
|
-
state_dict:
|
85
|
+
state_dict: dict[str, Any] = (
|
86
86
|
input_json if isinstance(input_json, dict) else json.loads(input_json, strict=False)
|
87
87
|
)
|
88
88
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Any,
|
1
|
+
from typing import Any, Optional
|
2
2
|
|
3
3
|
from snowflake import snowpark
|
4
4
|
from snowflake.ml._internal import telemetry
|
@@ -25,21 +25,21 @@ class DatasetReader(data_connector.DataConnector):
|
|
25
25
|
|
26
26
|
self._session: snowpark.Session = snowpark_session
|
27
27
|
self._fs: snowfs.SnowFileSystem = ingestor_utils.get_dataset_filesystem(self._session)
|
28
|
-
self._files: Optional[
|
28
|
+
self._files: Optional[list[str]] = None
|
29
29
|
|
30
30
|
@classmethod
|
31
31
|
def from_dataframe(
|
32
|
-
cls, df: snowpark.DataFrame, ingestor_class: Optional[
|
32
|
+
cls, df: snowpark.DataFrame, ingestor_class: Optional[type[data_ingestor.DataIngestor]] = None, **kwargs: Any
|
33
33
|
) -> "DatasetReader":
|
34
34
|
# Block superclass constructor from Snowpark DataFrames
|
35
35
|
raise RuntimeError("Creating DatasetReader from DataFrames not supported")
|
36
36
|
|
37
|
-
def _list_files(self) ->
|
37
|
+
def _list_files(self) -> list[str]:
|
38
38
|
"""Private helper function that lists all files in this DatasetVersion and caches the results."""
|
39
39
|
if self._files:
|
40
40
|
return self._files
|
41
41
|
|
42
|
-
files:
|
42
|
+
files: list[str] = []
|
43
43
|
for source in self.data_sources:
|
44
44
|
assert isinstance(source, data_source.DatasetInfo)
|
45
45
|
files.extend(ingestor_utils.get_dataset_files(self._session, source, filesystem=self._fs))
|
@@ -49,7 +49,7 @@ class DatasetReader(data_connector.DataConnector):
|
|
49
49
|
return self._files
|
50
50
|
|
51
51
|
@telemetry.send_api_usage_telemetry(project=_PROJECT, subproject=_SUBPROJECT)
|
52
|
-
def files(self) ->
|
52
|
+
def files(self) -> list[str]:
|
53
53
|
"""Get the list of remote file paths for the current DatasetVersion.
|
54
54
|
|
55
55
|
The file paths follows the snow protocol.
|
@@ -92,7 +92,7 @@ class DatasetReader(data_connector.DataConnector):
|
|
92
92
|
For example, an OBJECT column may be scanned back as a STRING column.
|
93
93
|
"""
|
94
94
|
file_path_pattern = ".*data_.*[.]parquet"
|
95
|
-
dfs:
|
95
|
+
dfs: list[snowpark.DataFrame] = []
|
96
96
|
for source in self.data_sources:
|
97
97
|
assert isinstance(source, data_source.DatasetInfo) and source.url is not None
|
98
98
|
stage_reader = self._session.read.option("pattern", file_path_pattern)
|
@@ -4,7 +4,7 @@ from snowflake.ml._internal import init_utils
|
|
4
4
|
|
5
5
|
from .access_manager import setup_feature_store
|
6
6
|
|
7
|
-
pkg_dir = os.path.dirname(
|
7
|
+
pkg_dir = os.path.dirname(__file__)
|
8
8
|
pkg_name = __name__
|
9
9
|
exportable_classes = init_utils.fetch_classes_from_modules_in_pkg_dir(pkg_dir=pkg_dir, pkg_name=pkg_name)
|
10
10
|
for k, v in exportable_classes.items():
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from dataclasses import asdict, dataclass
|
2
2
|
from enum import Enum
|
3
|
-
from typing import
|
3
|
+
from typing import Optional
|
4
4
|
from warnings import warn
|
5
5
|
|
6
6
|
from snowflake.ml._internal import telemetry
|
@@ -28,7 +28,7 @@ class _FeatureStoreRole(Enum):
|
|
28
28
|
class _Privilege:
|
29
29
|
object_type: str
|
30
30
|
object_name: str
|
31
|
-
privileges:
|
31
|
+
privileges: list[str]
|
32
32
|
scope: Optional[str] = None
|
33
33
|
optional: bool = False
|
34
34
|
|
@@ -41,7 +41,7 @@ class _SessionInfo:
|
|
41
41
|
|
42
42
|
|
43
43
|
# Lists of permissions as tuples of (OBJECT_TYPE, [PRIVILEGES, ...])
|
44
|
-
_PRE_INIT_PRIVILEGES:
|
44
|
+
_PRE_INIT_PRIVILEGES: dict[_FeatureStoreRole, list[_Privilege]] = {
|
45
45
|
_FeatureStoreRole.PRODUCER: [
|
46
46
|
_Privilege("DATABASE", "{database}", ["USAGE"]),
|
47
47
|
_Privilege("SCHEMA", "{database}.{schema}", ["USAGE"]),
|
@@ -78,7 +78,7 @@ _PRE_INIT_PRIVILEGES: Dict[_FeatureStoreRole, List[_Privilege]] = {
|
|
78
78
|
_FeatureStoreRole.NONE: [],
|
79
79
|
}
|
80
80
|
|
81
|
-
_POST_INIT_PRIVILEGES:
|
81
|
+
_POST_INIT_PRIVILEGES: dict[_FeatureStoreRole, list[_Privilege]] = {
|
82
82
|
_FeatureStoreRole.PRODUCER: [
|
83
83
|
_Privilege("TAG", f"{{database}}.{{schema}}.{_FEATURE_VIEW_METADATA_TAG}", ["APPLY"]),
|
84
84
|
_Privilege("TAG", f"{{database}}.{{schema}}.{_FEATURE_STORE_OBJECT_TAG}", ["APPLY"]),
|
@@ -89,7 +89,7 @@ _POST_INIT_PRIVILEGES: Dict[_FeatureStoreRole, List[_Privilege]] = {
|
|
89
89
|
|
90
90
|
|
91
91
|
def _grant_privileges(
|
92
|
-
session: Session, role_name: str, privileges:
|
92
|
+
session: Session, role_name: str, privileges: list[_Privilege], session_info: _SessionInfo
|
93
93
|
) -> None:
|
94
94
|
session_info_dict = asdict(session_info)
|
95
95
|
for p in privileges:
|
@@ -129,7 +129,7 @@ def _grant_privileges(
|
|
129
129
|
def _configure_pre_init_privileges(
|
130
130
|
session: Session,
|
131
131
|
session_info: _SessionInfo,
|
132
|
-
roles_to_create:
|
132
|
+
roles_to_create: dict[_FeatureStoreRole, str],
|
133
133
|
) -> None:
|
134
134
|
"""
|
135
135
|
Configure Feature Store role privileges. Must be run with ACCOUNTADMIN
|
@@ -172,7 +172,7 @@ def _configure_pre_init_privileges(
|
|
172
172
|
def _configure_post_init_privileges(
|
173
173
|
session: Session,
|
174
174
|
session_info: _SessionInfo,
|
175
|
-
roles_to_create:
|
175
|
+
roles_to_create: dict[_FeatureStoreRole, str],
|
176
176
|
) -> None:
|
177
177
|
for role_type, role in roles_to_create.items():
|
178
178
|
_grant_privileges(session, role, _POST_INIT_PRIVILEGES[role_type], session_info)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import
|
1
|
+
from typing import Optional
|
2
2
|
|
3
3
|
from snowflake.ml._internal.utils.sql_identifier import (
|
4
4
|
SqlIdentifier,
|
@@ -22,7 +22,7 @@ class Entity:
|
|
22
22
|
It can also be used for FeatureView search and lineage tracking.
|
23
23
|
"""
|
24
24
|
|
25
|
-
def __init__(self, name: str, join_keys:
|
25
|
+
def __init__(self, name: str, join_keys: list[str], *, desc: str = "") -> None:
|
26
26
|
"""
|
27
27
|
Creates an Entity instance.
|
28
28
|
|
@@ -51,11 +51,11 @@ class Entity:
|
|
51
51
|
self._validate(name, join_keys)
|
52
52
|
|
53
53
|
self.name: SqlIdentifier = SqlIdentifier(name)
|
54
|
-
self.join_keys:
|
54
|
+
self.join_keys: list[SqlIdentifier] = to_sql_identifiers(join_keys)
|
55
55
|
self.owner: Optional[str] = None
|
56
56
|
self.desc: str = desc
|
57
57
|
|
58
|
-
def _validate(self, name: str, join_keys:
|
58
|
+
def _validate(self, name: str, join_keys: list[str]) -> None:
|
59
59
|
if len(name) > _ENTITY_NAME_LENGTH_LIMIT:
|
60
60
|
raise ValueError(f"Entity name `{name}` exceeds maximum length: {_ENTITY_NAME_LENGTH_LIMIT}")
|
61
61
|
if _FEATURE_VIEW_ENTITY_TAG_DELIMITER in name:
|
@@ -73,7 +73,7 @@ class Entity:
|
|
73
73
|
if len(k) > _ENTITY_JOIN_KEY_LENGTH_LIMIT:
|
74
74
|
raise ValueError(f"Join key: {k} exceeds length limit {_ENTITY_JOIN_KEY_LENGTH_LIMIT}.")
|
75
75
|
|
76
|
-
def _to_dict(self) ->
|
76
|
+
def _to_dict(self) -> dict[str, str]:
|
77
77
|
entity_dict = self.__dict__.copy()
|
78
78
|
for k, v in entity_dict.items():
|
79
79
|
if isinstance(v, SqlIdentifier):
|
@@ -81,7 +81,7 @@ class Entity:
|
|
81
81
|
return entity_dict
|
82
82
|
|
83
83
|
@staticmethod
|
84
|
-
def _construct_entity(name: str, join_keys:
|
84
|
+
def _construct_entity(name: str, join_keys: list[str], desc: str, owner: str) -> "Entity":
|
85
85
|
e = Entity(name, join_keys, desc=desc)
|
86
86
|
e.owner = owner
|
87
87
|
return e
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import Entity
|
4
2
|
|
5
3
|
zipcode_entity = Entity(
|
@@ -12,5 +10,5 @@ plane_entity = Entity(name="PLANE_MODEL", join_keys=["PLANE_MODEL"], desc="The m
|
|
12
10
|
|
13
11
|
|
14
12
|
# This will be invoked by example_helper.py. Do not change function name.
|
15
|
-
def get_all_entities() ->
|
13
|
+
def get_all_entities() -> list[Entity]:
|
16
14
|
return [zipcode_entity, plane_entity]
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.airline_features.entities import plane_entity
|
5
3
|
from snowflake.snowpark import DataFrame, Session
|
@@ -7,7 +5,7 @@ from snowflake.snowpark import DataFrame, Session
|
|
7
5
|
|
8
6
|
# This function will be invoked by example_helper.py. Do not change the name.
|
9
7
|
def create_draft_feature_view(
|
10
|
-
session: Session, source_dfs:
|
8
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
11
9
|
) -> FeatureView:
|
12
10
|
"""Create a feature view about airplane model."""
|
13
11
|
query = session.sql(
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.airline_features.entities import zipcode_entity
|
5
3
|
from snowflake.snowpark import DataFrame, Session
|
@@ -7,7 +5,7 @@ from snowflake.snowpark import DataFrame, Session
|
|
7
5
|
|
8
6
|
# This function will be invoked by example_helper.py. Do not change the name.
|
9
7
|
def create_draft_feature_view(
|
10
|
-
session: Session, source_dfs:
|
8
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
11
9
|
) -> FeatureView:
|
12
10
|
"""Create a feature view about airport weather."""
|
13
11
|
query = session.sql(
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import Entity
|
4
2
|
|
5
3
|
end_station_id = Entity(
|
@@ -16,5 +14,5 @@ trip_id = Entity(
|
|
16
14
|
|
17
15
|
|
18
16
|
# This will be invoked by example_helper.py. Do not change function name.
|
19
|
-
def get_all_entities() ->
|
17
|
+
def get_all_entities() -> list[Entity]:
|
20
18
|
return [end_station_id, trip_id]
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.citibike_trip_features.entities import (
|
5
3
|
end_station_id,
|
@@ -9,7 +7,7 @@ from snowflake.snowpark import DataFrame, Session
|
|
9
7
|
|
10
8
|
# This function will be invoked by example_helper.py. Do not change the name.
|
11
9
|
def create_draft_feature_view(
|
12
|
-
session: Session, source_dfs:
|
10
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
13
11
|
) -> FeatureView:
|
14
12
|
"""Create a feature view about trip station."""
|
15
13
|
query = session.sql(
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.citibike_trip_features.entities import trip_id
|
5
3
|
from snowflake.snowpark import DataFrame, Session, functions as F
|
@@ -7,7 +5,7 @@ from snowflake.snowpark import DataFrame, Session, functions as F
|
|
7
5
|
|
8
6
|
# This function will be invoked by example_helper.py. Do not change the name.
|
9
7
|
def create_draft_feature_view(
|
10
|
-
session: Session, source_dfs:
|
8
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
11
9
|
) -> FeatureView:
|
12
10
|
"""Create a feature view about trip."""
|
13
11
|
feature_df = source_dfs[0].select(
|
@@ -3,7 +3,7 @@ import logging
|
|
3
3
|
import os
|
4
4
|
import re
|
5
5
|
from pathlib import Path
|
6
|
-
from typing import Any,
|
6
|
+
from typing import Any, Optional
|
7
7
|
|
8
8
|
import yaml
|
9
9
|
|
@@ -32,12 +32,12 @@ class ExampleHelper:
|
|
32
32
|
|
33
33
|
def _clear(self) -> None:
|
34
34
|
self._selected_example = None
|
35
|
-
self._source_tables:
|
36
|
-
self._source_dfs:
|
37
|
-
self._excluded_columns:
|
38
|
-
self._label_columns:
|
35
|
+
self._source_tables: list[str] = []
|
36
|
+
self._source_dfs: list[DataFrame] = []
|
37
|
+
self._excluded_columns: list[sql_identifier.SqlIdentifier] = []
|
38
|
+
self._label_columns: list[sql_identifier.SqlIdentifier] = []
|
39
39
|
self._timestamp_column: Optional[sql_identifier.SqlIdentifier] = None
|
40
|
-
self._epoch_to_timestamp_cols:
|
40
|
+
self._epoch_to_timestamp_cols: list[str] = []
|
41
41
|
self._add_id_column: Optional[sql_identifier.SqlIdentifier] = None
|
42
42
|
self._training_spine_table: str = ""
|
43
43
|
|
@@ -53,7 +53,7 @@ class ExampleHelper:
|
|
53
53
|
rows.append((f_name, source_dict["model_category"], source_dict["desc"], source_dict["label_columns"]))
|
54
54
|
return self._session.create_dataframe(rows, schema=["NAME", "MODEL_CATEGORY", "DESC", "LABEL_COLS"])
|
55
55
|
|
56
|
-
def load_draft_feature_views(self) ->
|
56
|
+
def load_draft_feature_views(self) -> list[FeatureView]:
|
57
57
|
"""Return all feature views in an example.
|
58
58
|
|
59
59
|
Returns:
|
@@ -74,7 +74,7 @@ class ExampleHelper:
|
|
74
74
|
|
75
75
|
return fvs
|
76
76
|
|
77
|
-
def load_entities(self) ->
|
77
|
+
def load_entities(self) -> list[Entity]:
|
78
78
|
"""Return all entities in an example.
|
79
79
|
|
80
80
|
Returns:
|
@@ -88,7 +88,7 @@ class ExampleHelper:
|
|
88
88
|
with open(file_path) as fs:
|
89
89
|
return yaml.safe_load(fs)
|
90
90
|
|
91
|
-
def _create_file_format(self, format_dict:
|
91
|
+
def _create_file_format(self, format_dict: dict[str, str], format_name: str) -> None:
|
92
92
|
"""Create a file name with given name."""
|
93
93
|
self._session.sql(
|
94
94
|
f"""
|
@@ -110,7 +110,7 @@ class ExampleHelper:
|
|
110
110
|
"""
|
111
111
|
).collect()
|
112
112
|
|
113
|
-
def _load_csv(self, schema_dict:
|
113
|
+
def _load_csv(self, schema_dict: dict[str, str], temp_stage_name: str) -> list[str]:
|
114
114
|
# create temp file format
|
115
115
|
file_format_name = f"{self._database_name}.{self._dataset_schema}.feature_store_temp_format"
|
116
116
|
format_str = ""
|
@@ -145,7 +145,7 @@ class ExampleHelper:
|
|
145
145
|
|
146
146
|
return [schema_dict["destination_table_name"]]
|
147
147
|
|
148
|
-
def _load_parquet(self, schema_dict:
|
148
|
+
def _load_parquet(self, schema_dict: dict[str, str], temp_stage_name: str) -> list[str]:
|
149
149
|
regex_pattern = schema_dict["load_files_pattern"]
|
150
150
|
all_files = self._session.sql(f"list @{temp_stage_name}").collect()
|
151
151
|
filtered_files = [item["name"] for item in all_files if re.match(regex_pattern, item["name"])]
|
@@ -187,7 +187,7 @@ class ExampleHelper:
|
|
187
187
|
|
188
188
|
return result
|
189
189
|
|
190
|
-
def _load_source_data(self, schema_yaml_file: str) ->
|
190
|
+
def _load_source_data(self, schema_yaml_file: str) -> list[str]:
|
191
191
|
"""Parse a yaml schema file and load data into Snowflake.
|
192
192
|
|
193
193
|
Args:
|
@@ -210,7 +210,7 @@ class ExampleHelper:
|
|
210
210
|
else:
|
211
211
|
return self._load_csv(schema_dict, temp_stage_name)
|
212
212
|
|
213
|
-
def load_example(self, example_name: str) ->
|
213
|
+
def load_example(self, example_name: str) -> list[str]:
|
214
214
|
"""Select the active example and load its datasets to Snowflake.
|
215
215
|
|
216
216
|
Args:
|
@@ -247,7 +247,7 @@ class ExampleHelper:
|
|
247
247
|
|
248
248
|
return self.load_source_data(source_yaml_data)
|
249
249
|
|
250
|
-
def load_source_data(self, source_data_name: str) ->
|
250
|
+
def load_source_data(self, source_data_name: str) -> list[str]:
|
251
251
|
"""Load source data into Snowflake.
|
252
252
|
|
253
253
|
Args:
|
@@ -269,10 +269,10 @@ class ExampleHelper:
|
|
269
269
|
def get_current_schema(self) -> str:
|
270
270
|
return self._dataset_schema
|
271
271
|
|
272
|
-
def get_label_cols(self) ->
|
272
|
+
def get_label_cols(self) -> list[str]:
|
273
273
|
return [item.resolved() for item in self._label_columns]
|
274
274
|
|
275
|
-
def get_excluded_cols(self) ->
|
275
|
+
def get_excluded_cols(self) -> list[str]:
|
276
276
|
return [item.resolved() for item in self._excluded_columns]
|
277
277
|
|
278
278
|
def get_training_data_timestamp_col(self) -> Optional[str]:
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import Entity
|
4
2
|
|
5
3
|
trip_id = Entity(name="TRIP_ID", join_keys=["TRIP_ID"], desc="Trip id.")
|
@@ -8,5 +6,5 @@ location_id = Entity(name="DOLOCATIONID", join_keys=["DOLOCATIONID"], desc="Drop
|
|
8
6
|
|
9
7
|
|
10
8
|
# This will be invoked by example_helper.py. Do not change function name.
|
11
|
-
def get_all_entities() ->
|
9
|
+
def get_all_entities() -> list[Entity]:
|
12
10
|
return [trip_id, location_id]
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.new_york_taxi_features.entities import (
|
5
3
|
location_id,
|
@@ -9,7 +7,7 @@ from snowflake.snowpark import DataFrame, Session
|
|
9
7
|
|
10
8
|
# This function will be invoked by example_helper.py. Do not change the name.
|
11
9
|
def create_draft_feature_view(
|
12
|
-
session: Session, source_dfs:
|
10
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
13
11
|
) -> FeatureView:
|
14
12
|
"""Create a draft feature view."""
|
15
13
|
feature_df = session.sql(
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.new_york_taxi_features.entities import trip_id
|
5
3
|
from snowflake.snowpark import DataFrame, Session
|
@@ -7,7 +5,7 @@ from snowflake.snowpark import DataFrame, Session
|
|
7
5
|
|
8
6
|
# This function will be invoked by example_helper.py. Do not change the name.
|
9
7
|
def create_draft_feature_view(
|
10
|
-
session: Session, source_dfs:
|
8
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
11
9
|
) -> FeatureView:
|
12
10
|
"""Create a draft feature view."""
|
13
11
|
feature_df = session.sql(
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import Entity
|
4
2
|
|
5
3
|
wine_id = Entity(
|
@@ -10,5 +8,5 @@ wine_id = Entity(
|
|
10
8
|
|
11
9
|
|
12
10
|
# This will be invoked by example_helper.py. Do not change function name.
|
13
|
-
def get_all_entities() ->
|
11
|
+
def get_all_entities() -> list[Entity]:
|
14
12
|
return [wine_id]
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.wine_quality_features.entities import wine_id
|
5
3
|
from snowflake.snowpark import DataFrame, Session, functions as F
|
@@ -7,7 +5,7 @@ from snowflake.snowpark import DataFrame, Session, functions as F
|
|
7
5
|
|
8
6
|
# This function will be invoked by example_helper.py. Do not change the name.
|
9
7
|
def create_draft_feature_view(
|
10
|
-
session: Session, source_dfs:
|
8
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
11
9
|
) -> FeatureView:
|
12
10
|
"""Create a feature view about trip station."""
|
13
11
|
feature_df = source_dfs[0].select(
|
@@ -1,5 +1,3 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
1
|
from snowflake.ml.feature_store import FeatureView
|
4
2
|
from snowflake.ml.feature_store.examples.wine_quality_features.entities import wine_id
|
5
3
|
from snowflake.snowpark import DataFrame, Session
|
@@ -7,7 +5,7 @@ from snowflake.snowpark import DataFrame, Session
|
|
7
5
|
|
8
6
|
# This function will be invoked by example_helper.py. Do not change the name.
|
9
7
|
def create_draft_feature_view(
|
10
|
-
session: Session, source_dfs:
|
8
|
+
session: Session, source_dfs: list[DataFrame], source_tables: list[str], database: str, schema: str
|
11
9
|
) -> FeatureView:
|
12
10
|
"""Create a feature view about trip station."""
|
13
11
|
feature_df = source_dfs[0].select("WINE_ID", "SULPHATES", "ALCOHOL")
|