snowflake-ml-python 1.10.0__py3-none-any.whl → 1.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/_complete.py +3 -2
- snowflake/ml/_internal/utils/service_logger.py +26 -1
- snowflake/ml/experiment/_client/artifact.py +76 -0
- snowflake/ml/experiment/_client/experiment_tracking_sql_client.py +64 -1
- snowflake/ml/experiment/callback/keras.py +63 -0
- snowflake/ml/experiment/callback/lightgbm.py +5 -1
- snowflake/ml/experiment/callback/xgboost.py +5 -1
- snowflake/ml/experiment/experiment_tracking.py +89 -4
- snowflake/ml/feature_store/feature_store.py +1150 -131
- snowflake/ml/feature_store/feature_view.py +122 -0
- snowflake/ml/jobs/_utils/__init__.py +0 -0
- snowflake/ml/jobs/_utils/constants.py +9 -14
- snowflake/ml/jobs/_utils/feature_flags.py +16 -0
- snowflake/ml/jobs/_utils/payload_utils.py +61 -19
- snowflake/ml/jobs/_utils/query_helper.py +5 -1
- snowflake/ml/jobs/_utils/runtime_env_utils.py +63 -0
- snowflake/ml/jobs/_utils/scripts/get_instance_ip.py +18 -7
- snowflake/ml/jobs/_utils/scripts/mljob_launcher.py +15 -7
- snowflake/ml/jobs/_utils/spec_utils.py +44 -13
- snowflake/ml/jobs/_utils/stage_utils.py +22 -9
- snowflake/ml/jobs/_utils/types.py +7 -8
- snowflake/ml/jobs/job.py +34 -18
- snowflake/ml/jobs/manager.py +107 -24
- snowflake/ml/model/__init__.py +6 -1
- snowflake/ml/model/_client/model/batch_inference_specs.py +27 -0
- snowflake/ml/model/_client/model/model_version_impl.py +225 -73
- snowflake/ml/model/_client/ops/service_ops.py +128 -174
- snowflake/ml/model/_client/service/model_deployment_spec.py +123 -64
- snowflake/ml/model/_client/service/model_deployment_spec_schema.py +25 -9
- snowflake/ml/model/_model_composer/model_composer.py +1 -70
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +2 -43
- snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py +207 -2
- snowflake/ml/model/_packager/model_handlers/sklearn.py +3 -1
- snowflake/ml/model/_packager/model_runtime/_snowml_inference_alternative_requirements.py +3 -3
- snowflake/ml/model/_signatures/snowpark_handler.py +1 -1
- snowflake/ml/model/_signatures/utils.py +4 -2
- snowflake/ml/model/inference_engine.py +5 -0
- snowflake/ml/model/models/huggingface_pipeline.py +4 -3
- snowflake/ml/model/openai_signatures.py +57 -0
- snowflake/ml/modeling/_internal/estimator_utils.py +43 -1
- snowflake/ml/modeling/_internal/local_implementations/pandas_trainer.py +14 -3
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +17 -6
- snowflake/ml/modeling/calibration/calibrated_classifier_cv.py +1 -1
- snowflake/ml/modeling/cluster/affinity_propagation.py +1 -1
- snowflake/ml/modeling/cluster/agglomerative_clustering.py +1 -1
- snowflake/ml/modeling/cluster/birch.py +1 -1
- snowflake/ml/modeling/cluster/bisecting_k_means.py +1 -1
- snowflake/ml/modeling/cluster/dbscan.py +1 -1
- snowflake/ml/modeling/cluster/feature_agglomeration.py +1 -1
- snowflake/ml/modeling/cluster/k_means.py +1 -1
- snowflake/ml/modeling/cluster/mean_shift.py +1 -1
- snowflake/ml/modeling/cluster/mini_batch_k_means.py +1 -1
- snowflake/ml/modeling/cluster/optics.py +1 -1
- snowflake/ml/modeling/cluster/spectral_biclustering.py +1 -1
- snowflake/ml/modeling/cluster/spectral_clustering.py +1 -1
- snowflake/ml/modeling/cluster/spectral_coclustering.py +1 -1
- snowflake/ml/modeling/compose/column_transformer.py +1 -1
- snowflake/ml/modeling/compose/transformed_target_regressor.py +1 -1
- snowflake/ml/modeling/covariance/elliptic_envelope.py +1 -1
- snowflake/ml/modeling/covariance/empirical_covariance.py +1 -1
- snowflake/ml/modeling/covariance/graphical_lasso.py +1 -1
- snowflake/ml/modeling/covariance/graphical_lasso_cv.py +1 -1
- snowflake/ml/modeling/covariance/ledoit_wolf.py +1 -1
- snowflake/ml/modeling/covariance/min_cov_det.py +1 -1
- snowflake/ml/modeling/covariance/oas.py +1 -1
- snowflake/ml/modeling/covariance/shrunk_covariance.py +1 -1
- snowflake/ml/modeling/decomposition/dictionary_learning.py +1 -1
- snowflake/ml/modeling/decomposition/factor_analysis.py +1 -1
- snowflake/ml/modeling/decomposition/fast_ica.py +1 -1
- snowflake/ml/modeling/decomposition/incremental_pca.py +1 -1
- snowflake/ml/modeling/decomposition/kernel_pca.py +1 -1
- snowflake/ml/modeling/decomposition/mini_batch_dictionary_learning.py +1 -1
- snowflake/ml/modeling/decomposition/mini_batch_sparse_pca.py +1 -1
- snowflake/ml/modeling/decomposition/pca.py +1 -1
- snowflake/ml/modeling/decomposition/sparse_pca.py +1 -1
- snowflake/ml/modeling/decomposition/truncated_svd.py +1 -1
- snowflake/ml/modeling/discriminant_analysis/linear_discriminant_analysis.py +1 -1
- snowflake/ml/modeling/discriminant_analysis/quadratic_discriminant_analysis.py +1 -1
- snowflake/ml/modeling/ensemble/ada_boost_classifier.py +1 -1
- snowflake/ml/modeling/ensemble/ada_boost_regressor.py +1 -1
- snowflake/ml/modeling/ensemble/bagging_classifier.py +1 -1
- snowflake/ml/modeling/ensemble/bagging_regressor.py +1 -1
- snowflake/ml/modeling/ensemble/extra_trees_classifier.py +1 -1
- snowflake/ml/modeling/ensemble/extra_trees_regressor.py +1 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_classifier.py +1 -1
- snowflake/ml/modeling/ensemble/gradient_boosting_regressor.py +1 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_classifier.py +1 -1
- snowflake/ml/modeling/ensemble/hist_gradient_boosting_regressor.py +1 -1
- snowflake/ml/modeling/ensemble/isolation_forest.py +1 -1
- snowflake/ml/modeling/ensemble/random_forest_classifier.py +1 -1
- snowflake/ml/modeling/ensemble/random_forest_regressor.py +1 -1
- snowflake/ml/modeling/ensemble/stacking_regressor.py +1 -1
- snowflake/ml/modeling/ensemble/voting_classifier.py +1 -1
- snowflake/ml/modeling/ensemble/voting_regressor.py +1 -1
- snowflake/ml/modeling/feature_selection/generic_univariate_select.py +1 -1
- snowflake/ml/modeling/feature_selection/select_fdr.py +1 -1
- snowflake/ml/modeling/feature_selection/select_fpr.py +1 -1
- snowflake/ml/modeling/feature_selection/select_fwe.py +1 -1
- snowflake/ml/modeling/feature_selection/select_k_best.py +1 -1
- snowflake/ml/modeling/feature_selection/select_percentile.py +1 -1
- snowflake/ml/modeling/feature_selection/sequential_feature_selector.py +1 -1
- snowflake/ml/modeling/feature_selection/variance_threshold.py +1 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_classifier.py +1 -1
- snowflake/ml/modeling/gaussian_process/gaussian_process_regressor.py +1 -1
- snowflake/ml/modeling/impute/iterative_imputer.py +1 -1
- snowflake/ml/modeling/impute/knn_imputer.py +1 -1
- snowflake/ml/modeling/impute/missing_indicator.py +1 -1
- snowflake/ml/modeling/kernel_approximation/additive_chi2_sampler.py +1 -1
- snowflake/ml/modeling/kernel_approximation/nystroem.py +1 -1
- snowflake/ml/modeling/kernel_approximation/polynomial_count_sketch.py +1 -1
- snowflake/ml/modeling/kernel_approximation/rbf_sampler.py +1 -1
- snowflake/ml/modeling/kernel_approximation/skewed_chi2_sampler.py +1 -1
- snowflake/ml/modeling/kernel_ridge/kernel_ridge.py +1 -1
- snowflake/ml/modeling/lightgbm/lgbm_classifier.py +1 -1
- snowflake/ml/modeling/lightgbm/lgbm_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/ard_regression.py +1 -1
- snowflake/ml/modeling/linear_model/bayesian_ridge.py +1 -1
- snowflake/ml/modeling/linear_model/elastic_net.py +1 -1
- snowflake/ml/modeling/linear_model/elastic_net_cv.py +1 -1
- snowflake/ml/modeling/linear_model/gamma_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/huber_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/lars.py +1 -1
- snowflake/ml/modeling/linear_model/lars_cv.py +1 -1
- snowflake/ml/modeling/linear_model/lasso.py +1 -1
- snowflake/ml/modeling/linear_model/lasso_cv.py +1 -1
- snowflake/ml/modeling/linear_model/lasso_lars.py +1 -1
- snowflake/ml/modeling/linear_model/lasso_lars_cv.py +1 -1
- snowflake/ml/modeling/linear_model/lasso_lars_ic.py +1 -1
- snowflake/ml/modeling/linear_model/linear_regression.py +1 -1
- snowflake/ml/modeling/linear_model/logistic_regression.py +1 -1
- snowflake/ml/modeling/linear_model/logistic_regression_cv.py +1 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net.py +1 -1
- snowflake/ml/modeling/linear_model/multi_task_elastic_net_cv.py +1 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso.py +1 -1
- snowflake/ml/modeling/linear_model/multi_task_lasso_cv.py +1 -1
- snowflake/ml/modeling/linear_model/orthogonal_matching_pursuit.py +1 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_classifier.py +1 -1
- snowflake/ml/modeling/linear_model/passive_aggressive_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/perceptron.py +1 -1
- snowflake/ml/modeling/linear_model/poisson_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/ransac_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/ridge.py +1 -1
- snowflake/ml/modeling/linear_model/ridge_classifier.py +1 -1
- snowflake/ml/modeling/linear_model/ridge_classifier_cv.py +1 -1
- snowflake/ml/modeling/linear_model/ridge_cv.py +1 -1
- snowflake/ml/modeling/linear_model/sgd_classifier.py +1 -1
- snowflake/ml/modeling/linear_model/sgd_one_class_svm.py +1 -1
- snowflake/ml/modeling/linear_model/sgd_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/theil_sen_regressor.py +1 -1
- snowflake/ml/modeling/linear_model/tweedie_regressor.py +1 -1
- snowflake/ml/modeling/manifold/isomap.py +1 -1
- snowflake/ml/modeling/manifold/mds.py +1 -1
- snowflake/ml/modeling/manifold/spectral_embedding.py +1 -1
- snowflake/ml/modeling/manifold/tsne.py +1 -1
- snowflake/ml/modeling/mixture/bayesian_gaussian_mixture.py +1 -1
- snowflake/ml/modeling/mixture/gaussian_mixture.py +1 -1
- snowflake/ml/modeling/multiclass/one_vs_one_classifier.py +1 -1
- snowflake/ml/modeling/multiclass/one_vs_rest_classifier.py +1 -1
- snowflake/ml/modeling/multiclass/output_code_classifier.py +1 -1
- snowflake/ml/modeling/naive_bayes/bernoulli_nb.py +1 -1
- snowflake/ml/modeling/naive_bayes/categorical_nb.py +1 -1
- snowflake/ml/modeling/naive_bayes/complement_nb.py +1 -1
- snowflake/ml/modeling/naive_bayes/gaussian_nb.py +1 -1
- snowflake/ml/modeling/naive_bayes/multinomial_nb.py +1 -1
- snowflake/ml/modeling/neighbors/k_neighbors_classifier.py +1 -1
- snowflake/ml/modeling/neighbors/k_neighbors_regressor.py +1 -1
- snowflake/ml/modeling/neighbors/kernel_density.py +1 -1
- snowflake/ml/modeling/neighbors/local_outlier_factor.py +1 -1
- snowflake/ml/modeling/neighbors/nearest_centroid.py +1 -1
- snowflake/ml/modeling/neighbors/nearest_neighbors.py +1 -1
- snowflake/ml/modeling/neighbors/neighborhood_components_analysis.py +1 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_classifier.py +1 -1
- snowflake/ml/modeling/neighbors/radius_neighbors_regressor.py +1 -1
- snowflake/ml/modeling/neural_network/bernoulli_rbm.py +1 -1
- snowflake/ml/modeling/neural_network/mlp_classifier.py +1 -1
- snowflake/ml/modeling/neural_network/mlp_regressor.py +1 -1
- snowflake/ml/modeling/preprocessing/polynomial_features.py +1 -1
- snowflake/ml/modeling/semi_supervised/label_propagation.py +1 -1
- snowflake/ml/modeling/semi_supervised/label_spreading.py +1 -1
- snowflake/ml/modeling/svm/linear_svc.py +1 -1
- snowflake/ml/modeling/svm/linear_svr.py +1 -1
- snowflake/ml/modeling/svm/nu_svc.py +1 -1
- snowflake/ml/modeling/svm/nu_svr.py +1 -1
- snowflake/ml/modeling/svm/svc.py +1 -1
- snowflake/ml/modeling/svm/svr.py +1 -1
- snowflake/ml/modeling/tree/decision_tree_classifier.py +1 -1
- snowflake/ml/modeling/tree/decision_tree_regressor.py +1 -1
- snowflake/ml/modeling/tree/extra_tree_classifier.py +1 -1
- snowflake/ml/modeling/tree/extra_tree_regressor.py +1 -1
- snowflake/ml/modeling/xgboost/xgb_classifier.py +1 -1
- snowflake/ml/modeling/xgboost/xgb_regressor.py +1 -1
- snowflake/ml/modeling/xgboost/xgbrf_classifier.py +1 -1
- snowflake/ml/modeling/xgboost/xgbrf_regressor.py +1 -1
- snowflake/ml/monitoring/_client/model_monitor_sql_client.py +91 -6
- snowflake/ml/monitoring/_manager/model_monitor_manager.py +3 -0
- snowflake/ml/monitoring/entities/model_monitor_config.py +3 -0
- snowflake/ml/monitoring/model_monitor.py +26 -0
- snowflake/ml/registry/_manager/model_manager.py +7 -35
- snowflake/ml/registry/_manager/model_parameter_reconciler.py +194 -5
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.10.0.dist-info → snowflake_ml_python-1.12.0.dist-info}/METADATA +87 -7
- {snowflake_ml_python-1.10.0.dist-info → snowflake_ml_python-1.12.0.dist-info}/RECORD +205 -197
- {snowflake_ml_python-1.10.0.dist-info → snowflake_ml_python-1.12.0.dist-info}/WHEEL +0 -0
- {snowflake_ml_python-1.10.0.dist-info → snowflake_ml_python-1.12.0.dist-info}/licenses/LICENSE.txt +0 -0
- {snowflake_ml_python-1.10.0.dist-info → snowflake_ml_python-1.12.0.dist-info}/top_level.txt +0 -0
|
@@ -14,6 +14,7 @@ import packaging.version as pkg_version
|
|
|
14
14
|
from pytimeparse.timeparse import timeparse
|
|
15
15
|
from typing_extensions import Concatenate, ParamSpec
|
|
16
16
|
|
|
17
|
+
import snowflake.ml.feature_store.feature_view as fv_mod
|
|
17
18
|
import snowflake.ml.version as snowml_version
|
|
18
19
|
from snowflake.ml import dataset
|
|
19
20
|
from snowflake.ml._internal import telemetry
|
|
@@ -89,6 +90,7 @@ class _FeatureStoreObjTypes(Enum):
|
|
|
89
90
|
EXTERNAL_FEATURE_VIEW = "EXTERNAL_FEATURE_VIEW"
|
|
90
91
|
FEATURE_VIEW_REFRESH_TASK = "FEATURE_VIEW_REFRESH_TASK"
|
|
91
92
|
TRAINING_DATA = "TRAINING_DATA"
|
|
93
|
+
ONLINE_FEATURE_TABLE = "ONLINE_FEATURE_TABLE"
|
|
92
94
|
|
|
93
95
|
@classmethod
|
|
94
96
|
def parse(cls, val: str) -> _FeatureStoreObjTypes:
|
|
@@ -133,6 +135,7 @@ _LIST_FEATURE_VIEW_SCHEMA = StructType(
|
|
|
133
135
|
StructField("scheduling_state", StringType()),
|
|
134
136
|
StructField("warehouse", StringType()),
|
|
135
137
|
StructField("cluster_by", StringType()),
|
|
138
|
+
StructField("online_config", StringType()),
|
|
136
139
|
]
|
|
137
140
|
)
|
|
138
141
|
|
|
@@ -268,6 +271,7 @@ class FeatureStore:
|
|
|
268
271
|
"DATASETS": (self._config.full_schema_path, "DATASET"),
|
|
269
272
|
"DYNAMIC TABLES": (self._config.full_schema_path, "TABLE"),
|
|
270
273
|
"VIEWS": (self._config.full_schema_path, "TABLE"),
|
|
274
|
+
"ONLINE FEATURE TABLES": (self._config.full_schema_path, "TABLE"),
|
|
271
275
|
"SCHEMAS": (f"DATABASE {self._config.database}", "SCHEMA"),
|
|
272
276
|
"TAGS": (self._config.full_schema_path, None),
|
|
273
277
|
"TASKS": (self._config.full_schema_path, "TASK"),
|
|
@@ -484,6 +488,7 @@ class FeatureStore:
|
|
|
484
488
|
SnowflakeMLException: [ValueError] Warehouse or default warehouse is not specified.
|
|
485
489
|
SnowflakeMLException: [RuntimeError] Failed to create dynamic table, task, or view.
|
|
486
490
|
SnowflakeMLException: [RuntimeError] Failed to find resources.
|
|
491
|
+
Exception: Unexpected error during registration.
|
|
487
492
|
|
|
488
493
|
Example::
|
|
489
494
|
|
|
@@ -542,65 +547,72 @@ class FeatureStore:
|
|
|
542
547
|
except Exception:
|
|
543
548
|
pass
|
|
544
549
|
|
|
545
|
-
|
|
546
|
-
|
|
550
|
+
created_resources = []
|
|
551
|
+
try:
|
|
552
|
+
fully_qualified_name = self._get_fully_qualified_name(feature_view_name)
|
|
553
|
+
refresh_freq = feature_view.refresh_freq
|
|
547
554
|
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
555
|
+
if refresh_freq is None:
|
|
556
|
+
obj_info = _FeatureStoreObjInfo(_FeatureStoreObjTypes.EXTERNAL_FEATURE_VIEW, snowml_version.VERSION)
|
|
557
|
+
else:
|
|
558
|
+
obj_info = _FeatureStoreObjInfo(_FeatureStoreObjTypes.MANAGED_FEATURE_VIEW, snowml_version.VERSION)
|
|
552
559
|
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
560
|
+
tagging_clause = [
|
|
561
|
+
f"{self._get_fully_qualified_name(_FEATURE_STORE_OBJECT_TAG)} = '{obj_info.to_json()}'",
|
|
562
|
+
f"{self._get_fully_qualified_name(_FEATURE_VIEW_METADATA_TAG)} = '"
|
|
563
|
+
f"{feature_view._metadata().to_json()}'",
|
|
564
|
+
]
|
|
565
|
+
for e in feature_view.entities:
|
|
566
|
+
join_keys = [f"{key.resolved()}" for key in e.join_keys]
|
|
567
|
+
tagging_clause.append(
|
|
568
|
+
f"{self._get_fully_qualified_name(self._get_entity_name(e.name))} = '{','.join(join_keys)}'"
|
|
569
|
+
)
|
|
570
|
+
tagging_clause_str = ",\n".join(tagging_clause)
|
|
563
571
|
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
572
|
+
def create_col_desc(col: StructField) -> str:
|
|
573
|
+
desc = feature_view.feature_descs.get(SqlIdentifier(col.name), None) # type: ignore[union-attr]
|
|
574
|
+
desc = "" if desc is None else f"COMMENT '{desc}'"
|
|
575
|
+
return f"{col.name} {desc}"
|
|
568
576
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
577
|
+
column_descs = (
|
|
578
|
+
", ".join([f"{create_col_desc(col)}" for col in feature_view.output_schema.fields])
|
|
579
|
+
if feature_view.feature_descs is not None
|
|
580
|
+
else ""
|
|
581
|
+
)
|
|
574
582
|
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
overwrite,
|
|
583
|
+
# Step 1: Create offline feature view (Dynamic Table or View)
|
|
584
|
+
created_resources.extend(
|
|
585
|
+
self._create_offline_feature_view(
|
|
586
|
+
feature_view=feature_view,
|
|
587
|
+
feature_view_name=feature_view_name,
|
|
588
|
+
fully_qualified_name=fully_qualified_name,
|
|
589
|
+
column_descs=column_descs,
|
|
590
|
+
tagging_clause_str=tagging_clause_str,
|
|
591
|
+
block=block,
|
|
592
|
+
overwrite=overwrite,
|
|
593
|
+
)
|
|
587
594
|
)
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
)
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
595
|
+
|
|
596
|
+
# Step 2: Create online feature table if requested
|
|
597
|
+
if feature_view.online:
|
|
598
|
+
online_table_name = self._create_online_feature_table(
|
|
599
|
+
feature_view, feature_view_name, overwrite=overwrite
|
|
600
|
+
)
|
|
601
|
+
created_resources.append(
|
|
602
|
+
(_FeatureStoreObjTypes.ONLINE_FEATURE_TABLE, self._get_fully_qualified_name(online_table_name))
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
except Exception as e:
|
|
606
|
+
# We can't rollback in case of overwrite.
|
|
607
|
+
if not overwrite:
|
|
608
|
+
self._rollback_created_resources(created_resources)
|
|
609
|
+
|
|
610
|
+
if isinstance(e, snowml_exceptions.SnowflakeMLException):
|
|
611
|
+
raise
|
|
612
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
613
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
614
|
+
original_exception=RuntimeError(f"Failed to register feature view {feature_view.name}/{version}: {e}"),
|
|
615
|
+
) from e
|
|
604
616
|
|
|
605
617
|
logger.info(f"Registered FeatureView {feature_view.name}/{version} successfully.")
|
|
606
618
|
return self.get_feature_view(feature_view.name, str(version))
|
|
@@ -614,6 +626,7 @@ class FeatureStore:
|
|
|
614
626
|
refresh_freq: Optional[str] = None,
|
|
615
627
|
warehouse: Optional[str] = None,
|
|
616
628
|
desc: Optional[str] = None,
|
|
629
|
+
online_config: Optional[fv_mod.OnlineConfig] = None,
|
|
617
630
|
) -> FeatureView:
|
|
618
631
|
...
|
|
619
632
|
|
|
@@ -626,6 +639,7 @@ class FeatureStore:
|
|
|
626
639
|
refresh_freq: Optional[str] = None,
|
|
627
640
|
warehouse: Optional[str] = None,
|
|
628
641
|
desc: Optional[str] = None,
|
|
642
|
+
online_config: Optional[fv_mod.OnlineConfig] = None,
|
|
629
643
|
) -> FeatureView:
|
|
630
644
|
...
|
|
631
645
|
|
|
@@ -638,6 +652,7 @@ class FeatureStore:
|
|
|
638
652
|
refresh_freq: Optional[str] = None,
|
|
639
653
|
warehouse: Optional[str] = None,
|
|
640
654
|
desc: Optional[str] = None,
|
|
655
|
+
online_config: Optional[fv_mod.OnlineConfig] = None,
|
|
641
656
|
) -> FeatureView:
|
|
642
657
|
"""Update a registered feature view.
|
|
643
658
|
Check feature_view.py for which fields are allowed to be updated after registration.
|
|
@@ -648,6 +663,11 @@ class FeatureStore:
|
|
|
648
663
|
refresh_freq: updated refresh frequency.
|
|
649
664
|
warehouse: updated warehouse.
|
|
650
665
|
desc: description of feature view.
|
|
666
|
+
online_config: updated online configuration for the online feature table.
|
|
667
|
+
If provided with enable=True, creates online feature table if absent.
|
|
668
|
+
If provided with enable=False, drops online feature table if present.
|
|
669
|
+
If None (default), no change to online status.
|
|
670
|
+
During update, only explicitly set fields in the OnlineConfig will be updated.
|
|
651
671
|
|
|
652
672
|
Returns:
|
|
653
673
|
Updated FeatureView.
|
|
@@ -681,78 +701,117 @@ class FeatureStore:
|
|
|
681
701
|
------------------------------------------------
|
|
682
702
|
|FOO |v1 |THAT IS NEW DESCRIPTION |
|
|
683
703
|
------------------------------------------------
|
|
704
|
+
<BLANKLINE>
|
|
705
|
+
>>> # Enable online storage with custom configuration
|
|
706
|
+
>>> config = OnlineConfig(enable=True, target_lag='15s')
|
|
707
|
+
>>> online_fv = fs.update_feature_view(
|
|
708
|
+
... name='foo',
|
|
709
|
+
... version='v1',
|
|
710
|
+
... online_config=config,
|
|
711
|
+
... )
|
|
712
|
+
>>> print(online_fv.online)
|
|
713
|
+
True
|
|
684
714
|
|
|
685
715
|
Raises:
|
|
686
716
|
SnowflakeMLException: [RuntimeError] If FeatureView is not managed and refresh_freq is defined.
|
|
687
717
|
SnowflakeMLException: [RuntimeError] Failed to update feature view.
|
|
688
718
|
"""
|
|
719
|
+
if online_config is not None:
|
|
720
|
+
logging.warning("'online_config' is in private preview since 1.12.0. Do not use it in production.")
|
|
721
|
+
|
|
722
|
+
# Step 1: Validate inputs
|
|
689
723
|
feature_view = self._validate_feature_view_name_and_version_input(name, version)
|
|
690
724
|
new_desc = desc if desc is not None else feature_view.desc
|
|
691
725
|
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
raise snowml_exceptions.SnowflakeMLException(
|
|
696
|
-
error_code=error_codes.INVALID_ARGUMENT,
|
|
697
|
-
original_exception=RuntimeError(
|
|
698
|
-
f"Static feature view '{full_name}' does not support refresh_freq and warehouse."
|
|
699
|
-
),
|
|
700
|
-
)
|
|
701
|
-
new_query = f"""
|
|
702
|
-
ALTER VIEW {feature_view.fully_qualified_name()} SET
|
|
703
|
-
COMMENT = '{new_desc}'
|
|
704
|
-
"""
|
|
705
|
-
else:
|
|
706
|
-
warehouse = SqlIdentifier(warehouse) if warehouse else feature_view.warehouse
|
|
707
|
-
# TODO(@wezhou): we need to properly handle cron expr
|
|
708
|
-
new_query = f"""
|
|
709
|
-
ALTER DYNAMIC TABLE {feature_view.fully_qualified_name()} SET
|
|
710
|
-
TARGET_LAG = '{refresh_freq or feature_view.refresh_freq}'
|
|
711
|
-
WAREHOUSE = {warehouse}
|
|
712
|
-
COMMENT = '{new_desc}'
|
|
713
|
-
"""
|
|
714
|
-
|
|
715
|
-
try:
|
|
716
|
-
self._session.sql(new_query).collect(statement_params=self._telemetry_stmp)
|
|
717
|
-
except Exception as e:
|
|
726
|
+
# Validate static feature view constraints
|
|
727
|
+
if feature_view.status == FeatureViewStatus.STATIC and (refresh_freq or warehouse):
|
|
728
|
+
full_name = f"{feature_view.name}/{feature_view.version}"
|
|
718
729
|
raise snowml_exceptions.SnowflakeMLException(
|
|
719
|
-
error_code=error_codes.
|
|
730
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
720
731
|
original_exception=RuntimeError(
|
|
721
|
-
f"
|
|
732
|
+
f"Static feature view '{full_name}' does not support refresh_freq and warehouse."
|
|
722
733
|
),
|
|
723
|
-
)
|
|
734
|
+
)
|
|
735
|
+
|
|
736
|
+
# Step 2: Plan all operations
|
|
737
|
+
rollback_operations: list[Any] = []
|
|
738
|
+
try:
|
|
739
|
+
operations, rollback_operations = self._plan_feature_view_update_operations(
|
|
740
|
+
feature_view, refresh_freq, warehouse, new_desc, online_config
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
# Step 3: Execute atomically
|
|
744
|
+
self._execute_atomic_operations(operations)
|
|
745
|
+
|
|
746
|
+
except Exception as e:
|
|
747
|
+
# Step 4: Rollback on failure
|
|
748
|
+
self._handle_update_failure(e, rollback_operations, feature_view)
|
|
749
|
+
|
|
724
750
|
return self.get_feature_view(name=feature_view.name, version=str(feature_view.version))
|
|
725
751
|
|
|
726
752
|
@overload
|
|
727
|
-
def read_feature_view(
|
|
753
|
+
def read_feature_view(
|
|
754
|
+
self,
|
|
755
|
+
feature_view: str,
|
|
756
|
+
version: str,
|
|
757
|
+
*,
|
|
758
|
+
keys: Optional[list[list[str]]] = None,
|
|
759
|
+
feature_names: Optional[list[str]] = None,
|
|
760
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
761
|
+
) -> DataFrame:
|
|
728
762
|
...
|
|
729
763
|
|
|
730
764
|
@overload
|
|
731
|
-
def read_feature_view(
|
|
765
|
+
def read_feature_view(
|
|
766
|
+
self,
|
|
767
|
+
feature_view: FeatureView,
|
|
768
|
+
*,
|
|
769
|
+
keys: Optional[list[list[str]]] = None,
|
|
770
|
+
feature_names: Optional[list[str]] = None,
|
|
771
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
772
|
+
) -> DataFrame:
|
|
732
773
|
...
|
|
733
774
|
|
|
734
775
|
@dispatch_decorator() # type: ignore[misc]
|
|
735
|
-
def read_feature_view(
|
|
776
|
+
def read_feature_view(
|
|
777
|
+
self,
|
|
778
|
+
feature_view: Union[FeatureView, str],
|
|
779
|
+
version: Optional[str] = None,
|
|
780
|
+
*,
|
|
781
|
+
keys: Optional[list[list[str]]] = None,
|
|
782
|
+
feature_names: Optional[list[str]] = None,
|
|
783
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
784
|
+
) -> DataFrame:
|
|
736
785
|
"""
|
|
737
|
-
Read values from a FeatureView.
|
|
786
|
+
Read values from a FeatureView from either offline or online store.
|
|
738
787
|
|
|
739
788
|
Args:
|
|
740
789
|
feature_view: A FeatureView object to read from, or the name of feature view.
|
|
741
790
|
If name is provided then version also must be provided.
|
|
742
791
|
version: Optional version of feature view. Must set when argument feature_view is a str.
|
|
792
|
+
keys: Optional list of primary key value lists to filter by. Each inner list should contain
|
|
793
|
+
values in the same order as the entity join_keys. Works for both offline and online stores.
|
|
794
|
+
Example: [["user1"], ["user2"]] for single key,
|
|
795
|
+
[["user1", "item1"], ["user2", "item2"]] for composite keys.
|
|
796
|
+
If None, returns all data.
|
|
797
|
+
feature_names: Optional list of feature names to return. If None, returns all features.
|
|
798
|
+
Works consistently for both offline and online stores.
|
|
799
|
+
store_type: Store to read from - StoreType.ONLINE or StoreType.OFFLINE (default).
|
|
743
800
|
|
|
744
801
|
Returns:
|
|
745
|
-
Snowpark DataFrame
|
|
802
|
+
Snowpark DataFrame containing the FeatureView data.
|
|
746
803
|
|
|
747
804
|
Raises:
|
|
748
805
|
SnowflakeMLException: [ValueError] version argument is missing when argument feature_view is a str.
|
|
749
806
|
SnowflakeMLException: [ValueError] FeatureView is not registered.
|
|
807
|
+
SnowflakeMLException: [ValueError] Online store is not enabled for this feature view.
|
|
808
|
+
SnowflakeMLException: [ValueError] Invalid store type.
|
|
750
809
|
|
|
751
810
|
Example::
|
|
752
811
|
|
|
753
812
|
>>> fs = FeatureStore(...)
|
|
754
|
-
>>> # Read
|
|
755
|
-
>>> fs.read_feature_view('foo', 'v1').show()
|
|
813
|
+
>>> # Read all data from offline store
|
|
814
|
+
>>> fs.read_feature_view('foo', 'v1', store_type=StoreType.OFFLINE).show()
|
|
756
815
|
------------------------------------------
|
|
757
816
|
|"NAME" |"ID" |"TITLE" |"AGE" |"TS" |
|
|
758
817
|
------------------------------------------
|
|
@@ -760,15 +819,31 @@ class FeatureStore:
|
|
|
760
819
|
|porter |2 |manager |30 |200 |
|
|
761
820
|
------------------------------------------
|
|
762
821
|
<BLANKLINE>
|
|
763
|
-
>>> #
|
|
764
|
-
>>>
|
|
765
|
-
>>> fs.read_feature_view(fv).show()
|
|
822
|
+
>>> # Filter by keys in offline store
|
|
823
|
+
>>> fs.read_feature_view('foo', 'v1', keys=[["1"], ["2"]], store_type=StoreType.OFFLINE).show()
|
|
766
824
|
------------------------------------------
|
|
767
825
|
|"NAME" |"ID" |"TITLE" |"AGE" |"TS" |
|
|
768
826
|
------------------------------------------
|
|
769
827
|
|jonh |1 |boss |20 |100 |
|
|
770
828
|
|porter |2 |manager |30 |200 |
|
|
771
829
|
------------------------------------------
|
|
830
|
+
<BLANKLINE>
|
|
831
|
+
>>> # Read from online store with specific keys (same API)
|
|
832
|
+
>>> fs.read_feature_view('foo', 'v1', keys=[["1"], ["2"]], store_type=StoreType.ONLINE).show()
|
|
833
|
+
--------------------------------
|
|
834
|
+
|"ID" |"TITLE" |"AGE" |
|
|
835
|
+
--------------------------------
|
|
836
|
+
|1 |boss |20 |
|
|
837
|
+
|2 |manager |30 |
|
|
838
|
+
--------------------------------
|
|
839
|
+
<BLANKLINE>
|
|
840
|
+
>>> # Select specific features (works for both stores)
|
|
841
|
+
>>> fs.read_feature_view('foo', 'v1', keys=[["1"]], feature_names=["TITLE", "AGE"]).show()
|
|
842
|
+
----------------------
|
|
843
|
+
|"TITLE" |"AGE" |
|
|
844
|
+
----------------------
|
|
845
|
+
|boss |20 |
|
|
846
|
+
----------------------
|
|
772
847
|
|
|
773
848
|
"""
|
|
774
849
|
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
|
@@ -779,7 +854,17 @@ class FeatureStore:
|
|
|
779
854
|
original_exception=ValueError(f"FeatureView {feature_view.name} has not been registered."),
|
|
780
855
|
)
|
|
781
856
|
|
|
782
|
-
|
|
857
|
+
store_type = self._get_store_type(store_type)
|
|
858
|
+
|
|
859
|
+
if store_type == fv_mod.StoreType.ONLINE:
|
|
860
|
+
return self._read_from_online_store(feature_view, keys, feature_names)
|
|
861
|
+
elif store_type == fv_mod.StoreType.OFFLINE:
|
|
862
|
+
return self._read_from_offline_store(feature_view, keys, feature_names)
|
|
863
|
+
else:
|
|
864
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
865
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
866
|
+
original_exception=ValueError(f"Invalid store type: {store_type}"),
|
|
867
|
+
)
|
|
783
868
|
|
|
784
869
|
@dispatch_decorator()
|
|
785
870
|
def list_feature_views(
|
|
@@ -884,20 +969,42 @@ class FeatureStore:
|
|
|
884
969
|
)
|
|
885
970
|
|
|
886
971
|
@overload
|
|
887
|
-
def refresh_feature_view(
|
|
972
|
+
def refresh_feature_view(
|
|
973
|
+
self, feature_view: str, version: str, *, store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE
|
|
974
|
+
) -> None:
|
|
888
975
|
...
|
|
889
976
|
|
|
890
977
|
@overload
|
|
891
|
-
def refresh_feature_view(
|
|
978
|
+
def refresh_feature_view(
|
|
979
|
+
self,
|
|
980
|
+
feature_view: FeatureView,
|
|
981
|
+
version: Optional[str] = None,
|
|
982
|
+
*,
|
|
983
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
984
|
+
) -> None:
|
|
892
985
|
...
|
|
893
986
|
|
|
894
987
|
@dispatch_decorator() # type: ignore[misc]
|
|
895
|
-
def refresh_feature_view(
|
|
988
|
+
def refresh_feature_view(
|
|
989
|
+
self,
|
|
990
|
+
feature_view: Union[FeatureView, str],
|
|
991
|
+
version: Optional[str] = None,
|
|
992
|
+
*,
|
|
993
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
994
|
+
) -> None:
|
|
896
995
|
"""Manually refresh a feature view.
|
|
897
996
|
|
|
898
997
|
Args:
|
|
899
998
|
feature_view: A registered feature view object, or the name of feature view.
|
|
900
999
|
version: Optional version of feature view. Must set when argument feature_view is a str.
|
|
1000
|
+
store_type: Specify which storage to refresh. Can be StoreType.OFFLINE or StoreType.ONLINE.
|
|
1001
|
+
- StoreType.OFFLINE (default): Refreshes the offline feature view.
|
|
1002
|
+
- StoreType.ONLINE: Refreshes the online feature table for real-time serving.
|
|
1003
|
+
Only available for feature views with online=True.
|
|
1004
|
+
Defaults to StoreType.OFFLINE.
|
|
1005
|
+
|
|
1006
|
+
Raises:
|
|
1007
|
+
SnowflakeMLException: [ValueError] Invalid store type.
|
|
901
1008
|
|
|
902
1009
|
Example::
|
|
903
1010
|
|
|
@@ -926,43 +1033,89 @@ class FeatureStore:
|
|
|
926
1033
|
"""
|
|
927
1034
|
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
|
928
1035
|
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
1036
|
+
store_type = self._get_store_type(store_type)
|
|
1037
|
+
|
|
1038
|
+
if store_type == fv_mod.StoreType.ONLINE:
|
|
1039
|
+
# Refresh online feature table only
|
|
1040
|
+
if not feature_view.online:
|
|
1041
|
+
warnings.warn(
|
|
1042
|
+
f"Feature view {feature_view.name}/{feature_view.version} does not have online storage enabled.",
|
|
1043
|
+
stacklevel=2,
|
|
1044
|
+
category=UserWarning,
|
|
1045
|
+
)
|
|
1046
|
+
return
|
|
1047
|
+
|
|
1048
|
+
# Use the unified method but specify online-only refresh
|
|
1049
|
+
self._update_feature_view_status(feature_view, "REFRESH", store_type=fv_mod.StoreType.ONLINE)
|
|
1050
|
+
elif store_type == fv_mod.StoreType.OFFLINE:
|
|
1051
|
+
# Refresh offline feature view only
|
|
1052
|
+
if feature_view.status == FeatureViewStatus.STATIC:
|
|
1053
|
+
warnings.warn(
|
|
1054
|
+
"Static feature view can't be refreshed. You must set refresh_freq when register_feature_view().",
|
|
1055
|
+
stacklevel=2,
|
|
1056
|
+
category=UserWarning,
|
|
1057
|
+
)
|
|
1058
|
+
return
|
|
1059
|
+
self._update_feature_view_status(feature_view, "REFRESH", store_type=fv_mod.StoreType.OFFLINE)
|
|
1060
|
+
else:
|
|
1061
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
1062
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
1063
|
+
original_exception=ValueError(f"Invalid store type: {store_type}"),
|
|
934
1064
|
)
|
|
935
|
-
return
|
|
936
|
-
self._update_feature_view_status(feature_view, "REFRESH")
|
|
937
1065
|
|
|
938
1066
|
@overload
|
|
939
1067
|
def get_refresh_history(
|
|
940
|
-
self,
|
|
1068
|
+
self,
|
|
1069
|
+
feature_view: FeatureView,
|
|
1070
|
+
version: Optional[str] = None,
|
|
1071
|
+
*,
|
|
1072
|
+
verbose: bool = False,
|
|
1073
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
941
1074
|
) -> DataFrame:
|
|
942
1075
|
...
|
|
943
1076
|
|
|
944
1077
|
@overload
|
|
945
|
-
def get_refresh_history(
|
|
1078
|
+
def get_refresh_history(
|
|
1079
|
+
self,
|
|
1080
|
+
feature_view: str,
|
|
1081
|
+
version: str,
|
|
1082
|
+
*,
|
|
1083
|
+
verbose: bool = False,
|
|
1084
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
1085
|
+
) -> DataFrame:
|
|
946
1086
|
...
|
|
947
1087
|
|
|
948
1088
|
def get_refresh_history(
|
|
949
|
-
self,
|
|
1089
|
+
self,
|
|
1090
|
+
feature_view: Union[FeatureView, str],
|
|
1091
|
+
version: Optional[str] = None,
|
|
1092
|
+
*,
|
|
1093
|
+
verbose: bool = False,
|
|
1094
|
+
store_type: Union[fv_mod.StoreType, str] = fv_mod.StoreType.OFFLINE,
|
|
950
1095
|
) -> DataFrame:
|
|
951
|
-
"""Get refresh
|
|
1096
|
+
"""Get refresh history statistics about a feature view.
|
|
952
1097
|
|
|
953
1098
|
Args:
|
|
954
1099
|
feature_view: A registered feature view object, or the name of feature view.
|
|
955
1100
|
version: Optional version of feature view. Must set when argument feature_view is a str.
|
|
956
1101
|
verbose: Return more detailed history when set true.
|
|
1102
|
+
store_type: Store to get refresh history from - StoreType.ONLINE or StoreType.OFFLINE (default).
|
|
1103
|
+
- StoreType.OFFLINE (default): Returns refresh history for the offline feature view (dynamic table).
|
|
1104
|
+
- StoreType.ONLINE: Returns refresh history for the online feature table.
|
|
1105
|
+
Only available for feature views with online=True.
|
|
957
1106
|
|
|
958
1107
|
Returns:
|
|
959
1108
|
A dataframe contains the refresh history information.
|
|
960
1109
|
|
|
1110
|
+
Raises:
|
|
1111
|
+
SnowflakeMLException: [ValueError]
|
|
1112
|
+
If store_type is ONLINE but feature view doesn't have online storage enabled.
|
|
1113
|
+
|
|
961
1114
|
Example::
|
|
962
1115
|
|
|
963
1116
|
>>> fs = FeatureStore(...)
|
|
964
1117
|
>>> fv = fs.get_feature_view(name='MY_FV', version='v1')
|
|
965
|
-
>>> #
|
|
1118
|
+
>>> # Get offline refresh history (default)
|
|
966
1119
|
>>> fs.refresh_feature_view('MY_FV', 'v1')
|
|
967
1120
|
>>> fs.get_refresh_history('MY_FV', 'v1').show()
|
|
968
1121
|
-----------------------------------------------------------------------------------------------------
|
|
@@ -971,19 +1124,23 @@ class FeatureStore:
|
|
|
971
1124
|
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:53:58.504000 |2024-07-10 14:53:59.088000 |INCREMENTAL |
|
|
972
1125
|
-----------------------------------------------------------------------------------------------------
|
|
973
1126
|
<BLANKLINE>
|
|
974
|
-
>>> # refresh
|
|
975
|
-
>>> fs.
|
|
976
|
-
>>> fs.get_refresh_history(fv).show()
|
|
1127
|
+
>>> # Get online refresh history (for feature views with online storage)
|
|
1128
|
+
>>> fs.get_refresh_history('MY_FV', 'v1', store_type=StoreType.ONLINE).show()
|
|
977
1129
|
-----------------------------------------------------------------------------------------------------
|
|
978
|
-
|"NAME"
|
|
1130
|
+
|"NAME" |"STATE" |"REFRESH_START_TIME" |"REFRESH_END_TIME" |"REFRESH_ACTION" |
|
|
979
1131
|
-----------------------------------------------------------------------------------------------------
|
|
980
|
-
|MY_FV$v1
|
|
981
|
-
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:53:58.504000 |2024-07-10 14:53:59.088000 |INCREMENTAL |
|
|
1132
|
+
|MY_FV$v1$ONLINE |SUCCEEDED |2024-07-10 14:54:01.200000 |2024-07-10 14:54:02.100000 |INCREMENTAL |
|
|
982
1133
|
-----------------------------------------------------------------------------------------------------
|
|
1134
|
+
<BLANKLINE>
|
|
1135
|
+
>>> # Verbose mode works for both storage types
|
|
1136
|
+
>>> fs.get_refresh_history(fv, verbose=True, store_type=StoreType.OFFLINE).show()
|
|
1137
|
+
>>> fs.get_refresh_history(fv, verbose=True, store_type=StoreType.ONLINE).show()
|
|
983
1138
|
|
|
984
1139
|
"""
|
|
985
1140
|
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
|
986
1141
|
|
|
1142
|
+
store_type = self._get_store_type(store_type)
|
|
1143
|
+
|
|
987
1144
|
if feature_view.status == FeatureViewStatus.STATIC:
|
|
988
1145
|
warnings.warn(
|
|
989
1146
|
"Static feature view never refreshes.",
|
|
@@ -1000,6 +1157,27 @@ class FeatureStore:
|
|
|
1000
1157
|
)
|
|
1001
1158
|
return self._session.create_dataframe([Row()])
|
|
1002
1159
|
|
|
1160
|
+
# Validate online store request
|
|
1161
|
+
if store_type == fv_mod.StoreType.ONLINE:
|
|
1162
|
+
if not feature_view.online:
|
|
1163
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
1164
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
1165
|
+
original_exception=ValueError(
|
|
1166
|
+
f"Feature view '{feature_view.name}' version '{feature_view.version}' "
|
|
1167
|
+
"does not have online storage enabled. Cannot retrieve online refresh history."
|
|
1168
|
+
),
|
|
1169
|
+
)
|
|
1170
|
+
return self._get_online_refresh_history(feature_view, verbose)
|
|
1171
|
+
elif store_type == fv_mod.StoreType.OFFLINE:
|
|
1172
|
+
return self._get_offline_refresh_history(feature_view, verbose)
|
|
1173
|
+
else:
|
|
1174
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
1175
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
1176
|
+
original_exception=ValueError(f"Invalid store type: {store_type}"),
|
|
1177
|
+
)
|
|
1178
|
+
|
|
1179
|
+
def _get_offline_refresh_history(self, feature_view: FeatureView, verbose: bool) -> DataFrame:
|
|
1180
|
+
"""Get refresh history for offline feature view (dynamic table)."""
|
|
1003
1181
|
fv_resolved_name = FeatureView._get_physical_name(
|
|
1004
1182
|
feature_view.name,
|
|
1005
1183
|
feature_view.version, # type: ignore[arg-type]
|
|
@@ -1010,13 +1188,35 @@ class FeatureStore:
|
|
|
1010
1188
|
SELECT
|
|
1011
1189
|
{select_cols}
|
|
1012
1190
|
FROM TABLE (
|
|
1013
|
-
{self._config.database}.INFORMATION_SCHEMA.DYNAMIC_TABLE_REFRESH_HISTORY ()
|
|
1191
|
+
{self._config.database}.INFORMATION_SCHEMA.DYNAMIC_TABLE_REFRESH_HISTORY (RESULT_LIMIT => 10000)
|
|
1014
1192
|
)
|
|
1015
1193
|
WHERE NAME = '{fv_resolved_name}'
|
|
1016
1194
|
AND SCHEMA_NAME = '{self._config.schema}'
|
|
1017
1195
|
"""
|
|
1018
1196
|
)
|
|
1019
1197
|
|
|
1198
|
+
def _get_online_refresh_history(self, feature_view: FeatureView, verbose: bool) -> DataFrame:
|
|
1199
|
+
"""Get refresh history for online feature table."""
|
|
1200
|
+
online_table_name = FeatureView._get_online_table_name(feature_view.name, feature_view.version)
|
|
1201
|
+
select_cols = "*" if verbose else "name, state, refresh_start_time, refresh_end_time, refresh_action"
|
|
1202
|
+
prefix = (
|
|
1203
|
+
f"{self._config.database.resolved()}."
|
|
1204
|
+
f"{self._config.schema.resolved()}."
|
|
1205
|
+
f"{online_table_name.resolved()}"
|
|
1206
|
+
)
|
|
1207
|
+
return self._session.sql(
|
|
1208
|
+
f"""
|
|
1209
|
+
SELECT
|
|
1210
|
+
{select_cols}
|
|
1211
|
+
FROM TABLE (
|
|
1212
|
+
{self._config.database}.INFORMATION_SCHEMA.ONLINE_FEATURE_TABLE_REFRESH_HISTORY (
|
|
1213
|
+
NAME_PREFIX => '{prefix}'
|
|
1214
|
+
)
|
|
1215
|
+
|
|
1216
|
+
)
|
|
1217
|
+
"""
|
|
1218
|
+
)
|
|
1219
|
+
|
|
1020
1220
|
@overload
|
|
1021
1221
|
def resume_feature_view(self, feature_view: FeatureView) -> FeatureView:
|
|
1022
1222
|
...
|
|
@@ -1030,6 +1230,9 @@ class FeatureStore:
|
|
|
1030
1230
|
"""
|
|
1031
1231
|
Resume a previously suspended FeatureView.
|
|
1032
1232
|
|
|
1233
|
+
This operation resumes both the offline feature view (dynamic table and associated task)
|
|
1234
|
+
and the online feature table (if it exists) to ensure consistent state across all storage types.
|
|
1235
|
+
|
|
1033
1236
|
Args:
|
|
1034
1237
|
feature_view: FeatureView object or name to resume.
|
|
1035
1238
|
version: Optional version of feature view. Must set when argument feature_view is a str.
|
|
@@ -1060,7 +1263,19 @@ class FeatureStore:
|
|
|
1060
1263
|
|
|
1061
1264
|
"""
|
|
1062
1265
|
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
|
1063
|
-
|
|
1266
|
+
|
|
1267
|
+
# Plan atomic resume operations
|
|
1268
|
+
operations, rollback_operations = self._plan_feature_view_status_operations(feature_view, "RESUME")
|
|
1269
|
+
|
|
1270
|
+
try:
|
|
1271
|
+
# Execute all operations atomically
|
|
1272
|
+
self._execute_atomic_operations(operations)
|
|
1273
|
+
logger.info(f"Successfully RESUME FeatureView {feature_view.name}/{feature_view.version}.")
|
|
1274
|
+
except Exception as e:
|
|
1275
|
+
# Handle failure with rollback
|
|
1276
|
+
self._handle_status_operation_failure(e, rollback_operations, feature_view, "RESUME")
|
|
1277
|
+
|
|
1278
|
+
return self.get_feature_view(feature_view.name, str(feature_view.version))
|
|
1064
1279
|
|
|
1065
1280
|
@overload
|
|
1066
1281
|
def suspend_feature_view(self, feature_view: FeatureView) -> FeatureView:
|
|
@@ -1075,6 +1290,9 @@ class FeatureStore:
|
|
|
1075
1290
|
"""
|
|
1076
1291
|
Suspend an active FeatureView.
|
|
1077
1292
|
|
|
1293
|
+
This operation suspends both the offline feature view (dynamic table and associated task)
|
|
1294
|
+
and the online feature table (if it exists).
|
|
1295
|
+
|
|
1078
1296
|
Args:
|
|
1079
1297
|
feature_view: FeatureView object or name to suspend.
|
|
1080
1298
|
version: Optional version of feature view. Must set when argument feature_view is a str.
|
|
@@ -1105,7 +1323,19 @@ class FeatureStore:
|
|
|
1105
1323
|
|
|
1106
1324
|
"""
|
|
1107
1325
|
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
|
1108
|
-
|
|
1326
|
+
|
|
1327
|
+
# Plan atomic suspend operations
|
|
1328
|
+
operations, rollback_operations = self._plan_feature_view_status_operations(feature_view, "SUSPEND")
|
|
1329
|
+
|
|
1330
|
+
try:
|
|
1331
|
+
# Execute all operations atomically
|
|
1332
|
+
self._execute_atomic_operations(operations)
|
|
1333
|
+
logger.info(f"Successfully suspended FeatureView {feature_view.name}/{feature_view.version}.")
|
|
1334
|
+
except Exception as e:
|
|
1335
|
+
# Handle failure with rollback
|
|
1336
|
+
self._handle_status_operation_failure(e, rollback_operations, feature_view, "SUSPEND")
|
|
1337
|
+
|
|
1338
|
+
return self.get_feature_view(feature_view.name, str(feature_view.version))
|
|
1109
1339
|
|
|
1110
1340
|
@overload
|
|
1111
1341
|
def delete_feature_view(self, feature_view: FeatureView) -> None:
|
|
@@ -1184,6 +1414,21 @@ class FeatureStore:
|
|
|
1184
1414
|
statement_params=self._telemetry_stmp
|
|
1185
1415
|
)
|
|
1186
1416
|
|
|
1417
|
+
# Delete online feature table if it exists
|
|
1418
|
+
if feature_view.online:
|
|
1419
|
+
fully_qualified_online_name = feature_view.fully_qualified_online_table_name()
|
|
1420
|
+
try:
|
|
1421
|
+
self._session.sql(f"DROP ONLINE FEATURE TABLE IF EXISTS {fully_qualified_online_name}").collect(
|
|
1422
|
+
statement_params=self._telemetry_stmp
|
|
1423
|
+
)
|
|
1424
|
+
except Exception as e:
|
|
1425
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
1426
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
1427
|
+
original_exception=RuntimeError(
|
|
1428
|
+
f"Failed to delete online feature table {fully_qualified_online_name}: {e}"
|
|
1429
|
+
),
|
|
1430
|
+
)
|
|
1431
|
+
|
|
1187
1432
|
logger.info(f"Deleted FeatureView {feature_view.name}/{feature_view.version}.")
|
|
1188
1433
|
|
|
1189
1434
|
@dispatch_decorator()
|
|
@@ -1732,6 +1977,393 @@ class FeatureStore:
|
|
|
1732
1977
|
else:
|
|
1733
1978
|
return self._load_compact_feature_views(properties.compact_feature_views) # type: ignore[arg-type]
|
|
1734
1979
|
|
|
1980
|
+
def _rollback_created_resources(self, created_resources: list[tuple[_FeatureStoreObjTypes, str]]) -> None:
|
|
1981
|
+
"""Rollback created resources in reverse order.
|
|
1982
|
+
|
|
1983
|
+
Args:
|
|
1984
|
+
created_resources: List of (resource_type, resource_name) tuples to clean up
|
|
1985
|
+
"""
|
|
1986
|
+
for resource_type, resource_name in reversed(created_resources):
|
|
1987
|
+
try:
|
|
1988
|
+
if resource_type == _FeatureStoreObjTypes.MANAGED_FEATURE_VIEW:
|
|
1989
|
+
self._session.sql(f"DROP DYNAMIC TABLE IF EXISTS {resource_name}").collect(
|
|
1990
|
+
statement_params=self._telemetry_stmp
|
|
1991
|
+
)
|
|
1992
|
+
elif resource_type == _FeatureStoreObjTypes.EXTERNAL_FEATURE_VIEW:
|
|
1993
|
+
self._session.sql(f"DROP VIEW IF EXISTS {resource_name}").collect(
|
|
1994
|
+
statement_params=self._telemetry_stmp
|
|
1995
|
+
)
|
|
1996
|
+
elif resource_type == _FeatureStoreObjTypes.FEATURE_VIEW_REFRESH_TASK:
|
|
1997
|
+
self._session.sql(f"DROP TASK IF EXISTS {resource_name}").collect(
|
|
1998
|
+
statement_params=self._telemetry_stmp
|
|
1999
|
+
)
|
|
2000
|
+
elif resource_type == _FeatureStoreObjTypes.ONLINE_FEATURE_TABLE:
|
|
2001
|
+
self._session.sql(f"DROP ONLINE FEATURE TABLE IF EXISTS {resource_name}").collect(
|
|
2002
|
+
statement_params=self._telemetry_stmp
|
|
2003
|
+
)
|
|
2004
|
+
logger.info(f"Rollback: Successfully dropped {resource_type.value} {resource_name}")
|
|
2005
|
+
except Exception as rollback_error:
|
|
2006
|
+
# Log but don't fail the rollback process
|
|
2007
|
+
logger.warning(f"Rollback: Failed to drop {resource_type.value} {resource_name}: {rollback_error}")
|
|
2008
|
+
|
|
2009
|
+
@telemetry.send_api_usage_telemetry(project=_PROJECT)
|
|
2010
|
+
def _create_updated_feature_view(
|
|
2011
|
+
self, base_fv: FeatureView, online_config: Optional[fv_mod.OnlineConfig] = None
|
|
2012
|
+
) -> FeatureView:
|
|
2013
|
+
"""Create an updated FeatureView with new online configuration."""
|
|
2014
|
+
assert base_fv.version is not None
|
|
2015
|
+
assert base_fv.database is not None
|
|
2016
|
+
assert base_fv.schema is not None
|
|
2017
|
+
|
|
2018
|
+
feature_descs_str: Optional[dict[str, str]] = (
|
|
2019
|
+
{k.identifier(): v for k, v in base_fv.feature_descs.items()} if base_fv.feature_descs is not None else None
|
|
2020
|
+
)
|
|
2021
|
+
cluster_by_str: Optional[list[str]] = (
|
|
2022
|
+
[col.identifier() for col in base_fv.cluster_by] if base_fv.cluster_by is not None else None
|
|
2023
|
+
)
|
|
2024
|
+
|
|
2025
|
+
return FeatureView._construct_feature_view(
|
|
2026
|
+
name=base_fv.name.identifier(),
|
|
2027
|
+
entities=base_fv.entities,
|
|
2028
|
+
feature_df=base_fv.feature_df,
|
|
2029
|
+
timestamp_col=base_fv.timestamp_col.identifier() if base_fv.timestamp_col is not None else None,
|
|
2030
|
+
desc=base_fv.desc,
|
|
2031
|
+
version=str(base_fv.version),
|
|
2032
|
+
status=base_fv.status,
|
|
2033
|
+
feature_descs=feature_descs_str or {},
|
|
2034
|
+
refresh_freq=base_fv.refresh_freq,
|
|
2035
|
+
database=base_fv.database.identifier(),
|
|
2036
|
+
schema=base_fv.schema.identifier(),
|
|
2037
|
+
warehouse=base_fv.warehouse.identifier() if base_fv.warehouse is not None else None,
|
|
2038
|
+
refresh_mode=base_fv.refresh_mode,
|
|
2039
|
+
refresh_mode_reason=base_fv.refresh_mode_reason,
|
|
2040
|
+
initialize=base_fv.initialize,
|
|
2041
|
+
owner=base_fv.owner,
|
|
2042
|
+
infer_schema_df=base_fv._infer_schema_df,
|
|
2043
|
+
session=self._session,
|
|
2044
|
+
cluster_by=cluster_by_str,
|
|
2045
|
+
online_config=online_config,
|
|
2046
|
+
)
|
|
2047
|
+
|
|
2048
|
+
def _build_offline_update_queries(
|
|
2049
|
+
self, feature_view: FeatureView, refresh_freq: Optional[str], warehouse: Optional[str], desc: str
|
|
2050
|
+
) -> tuple[str, Optional[str]]:
|
|
2051
|
+
"""Build offline update query and its rollback query."""
|
|
2052
|
+
if feature_view.status == FeatureViewStatus.STATIC:
|
|
2053
|
+
update_query = f"""
|
|
2054
|
+
ALTER VIEW {feature_view.fully_qualified_name()} SET
|
|
2055
|
+
COMMENT = '{desc}'
|
|
2056
|
+
"""
|
|
2057
|
+
return update_query, None # No rollback needed for comment changes
|
|
2058
|
+
else:
|
|
2059
|
+
warehouse_id = SqlIdentifier(warehouse) if warehouse else feature_view.warehouse
|
|
2060
|
+
# TODO: SNOW-2260633 Handle cron expression updates for refresh_freq
|
|
2061
|
+
update_query = f"""
|
|
2062
|
+
ALTER DYNAMIC TABLE {feature_view.fully_qualified_name()} SET
|
|
2063
|
+
TARGET_LAG = '{refresh_freq or feature_view.refresh_freq}'
|
|
2064
|
+
WAREHOUSE = {warehouse_id}
|
|
2065
|
+
COMMENT = '{desc}'
|
|
2066
|
+
"""
|
|
2067
|
+
rollback_query = f"""ALTER DYNAMIC TABLE {feature_view.fully_qualified_name()} SET
|
|
2068
|
+
TARGET_LAG = '{feature_view.refresh_freq}'
|
|
2069
|
+
WAREHOUSE = {feature_view.warehouse}
|
|
2070
|
+
COMMENT = '{feature_view.desc}'
|
|
2071
|
+
"""
|
|
2072
|
+
return update_query, rollback_query
|
|
2073
|
+
|
|
2074
|
+
@dataclass(frozen=True)
|
|
2075
|
+
class _OnlineUpdateStrategy:
|
|
2076
|
+
"""Encapsulates online update operations and their rollbacks."""
|
|
2077
|
+
|
|
2078
|
+
operations: list[tuple[str, Union[str, FeatureView]]]
|
|
2079
|
+
rollback_operations: list[tuple[str, Union[str, FeatureView]]]
|
|
2080
|
+
final_config: Optional[fv_mod.OnlineConfig]
|
|
2081
|
+
|
|
2082
|
+
def _plan_online_update(
|
|
2083
|
+
self, feature_view: FeatureView, online_config: Optional[fv_mod.OnlineConfig]
|
|
2084
|
+
) -> _OnlineUpdateStrategy:
|
|
2085
|
+
"""Plan online update operations based on current state and target config."""
|
|
2086
|
+
if online_config is None:
|
|
2087
|
+
return self._OnlineUpdateStrategy([], [], None)
|
|
2088
|
+
|
|
2089
|
+
current_online = feature_view.online
|
|
2090
|
+
target_online = online_config.enable
|
|
2091
|
+
|
|
2092
|
+
# Enable online (create table)
|
|
2093
|
+
if target_online and not current_online:
|
|
2094
|
+
return self._plan_online_enable(feature_view, online_config)
|
|
2095
|
+
|
|
2096
|
+
# Disable online (drop table)
|
|
2097
|
+
elif not target_online and current_online:
|
|
2098
|
+
return self._plan_online_disable(feature_view)
|
|
2099
|
+
|
|
2100
|
+
# Update existing online table
|
|
2101
|
+
elif target_online and current_online:
|
|
2102
|
+
return self._plan_online_update_existing(feature_view, online_config)
|
|
2103
|
+
|
|
2104
|
+
# No change needed
|
|
2105
|
+
else:
|
|
2106
|
+
return self._OnlineUpdateStrategy([], [], online_config)
|
|
2107
|
+
|
|
2108
|
+
def _plan_online_enable(
|
|
2109
|
+
self, feature_view: FeatureView, online_config: fv_mod.OnlineConfig
|
|
2110
|
+
) -> _OnlineUpdateStrategy:
|
|
2111
|
+
"""Plan operations to enable online storage."""
|
|
2112
|
+
# Get default target_lag from existing config or use default
|
|
2113
|
+
default_target_lag = (
|
|
2114
|
+
feature_view.online_config.target_lag
|
|
2115
|
+
if feature_view.online_config and feature_view.online_config.target_lag
|
|
2116
|
+
else fv_mod._DEFAULT_TARGET_LAG
|
|
2117
|
+
)
|
|
2118
|
+
final_config = fv_mod.OnlineConfig(
|
|
2119
|
+
enable=True,
|
|
2120
|
+
target_lag=online_config.target_lag if online_config.target_lag is not None else default_target_lag,
|
|
2121
|
+
)
|
|
2122
|
+
|
|
2123
|
+
temp_fv = self._create_updated_feature_view(feature_view, final_config)
|
|
2124
|
+
|
|
2125
|
+
operations: list[tuple[str, Union[str, FeatureView]]] = [("CREATE_ONLINE", temp_fv)]
|
|
2126
|
+
rollback_ops: list[tuple[str, Union[str, FeatureView]]] = [
|
|
2127
|
+
("DELETE_ONLINE", temp_fv.fully_qualified_online_table_name())
|
|
2128
|
+
]
|
|
2129
|
+
|
|
2130
|
+
return self._OnlineUpdateStrategy(operations, rollback_ops, final_config)
|
|
2131
|
+
|
|
2132
|
+
def _plan_online_disable(self, feature_view: FeatureView) -> _OnlineUpdateStrategy:
|
|
2133
|
+
"""Plan operations to disable online storage."""
|
|
2134
|
+
table_name = feature_view.fully_qualified_online_table_name()
|
|
2135
|
+
|
|
2136
|
+
operations: list[tuple[str, Union[str, FeatureView]]] = [("DELETE_ONLINE", table_name)]
|
|
2137
|
+
rollback_ops: list[tuple[str, Union[str, FeatureView]]] = [
|
|
2138
|
+
("CREATE_ONLINE", self._create_updated_feature_view(feature_view, feature_view.online_config))
|
|
2139
|
+
]
|
|
2140
|
+
|
|
2141
|
+
# Create disabled config to properly represent the new state
|
|
2142
|
+
disabled_config = fv_mod.OnlineConfig(enable=False)
|
|
2143
|
+
|
|
2144
|
+
return self._OnlineUpdateStrategy(operations, rollback_ops, disabled_config)
|
|
2145
|
+
|
|
2146
|
+
def _plan_online_update_existing(
|
|
2147
|
+
self, feature_view: FeatureView, online_config: fv_mod.OnlineConfig
|
|
2148
|
+
) -> _OnlineUpdateStrategy:
|
|
2149
|
+
"""Plan operations to update existing online table configuration."""
|
|
2150
|
+
existing_config = feature_view.online_config or fv_mod.OnlineConfig(
|
|
2151
|
+
enable=True, target_lag=fv_mod._DEFAULT_TARGET_LAG
|
|
2152
|
+
)
|
|
2153
|
+
if online_config.target_lag is None or online_config.target_lag == existing_config.target_lag:
|
|
2154
|
+
return self._OnlineUpdateStrategy([], [], existing_config)
|
|
2155
|
+
|
|
2156
|
+
table_name = feature_view.fully_qualified_online_table_name()
|
|
2157
|
+
update_query = f"ALTER ONLINE FEATURE TABLE {table_name} SET TARGET_LAG = '{online_config.target_lag}'"
|
|
2158
|
+
rollback_query = f"ALTER ONLINE FEATURE TABLE {table_name} SET TARGET_LAG = '{existing_config.target_lag}'"
|
|
2159
|
+
|
|
2160
|
+
operations: list[tuple[str, Union[str, FeatureView]]] = [("UPDATE_ONLINE", update_query)]
|
|
2161
|
+
rollback_ops: list[tuple[str, Union[str, FeatureView]]] = [("UPDATE_ONLINE", rollback_query)]
|
|
2162
|
+
|
|
2163
|
+
final_config = fv_mod.OnlineConfig(
|
|
2164
|
+
enable=True,
|
|
2165
|
+
target_lag=online_config.target_lag,
|
|
2166
|
+
)
|
|
2167
|
+
|
|
2168
|
+
return self._OnlineUpdateStrategy(operations, rollback_ops, final_config)
|
|
2169
|
+
|
|
2170
|
+
def _plan_feature_view_update_operations(
|
|
2171
|
+
self,
|
|
2172
|
+
feature_view: FeatureView,
|
|
2173
|
+
refresh_freq: Optional[str],
|
|
2174
|
+
warehouse: Optional[str],
|
|
2175
|
+
desc: str,
|
|
2176
|
+
online_config: Optional[fv_mod.OnlineConfig],
|
|
2177
|
+
) -> tuple[list[tuple[str, Union[str, FeatureView]]], list[tuple[str, Union[str, FeatureView]]]]:
|
|
2178
|
+
"""Plan all update operations and their rollbacks."""
|
|
2179
|
+
operations: list[tuple[str, Union[str, FeatureView]]] = []
|
|
2180
|
+
rollback_operations: list[tuple[str, Union[str, FeatureView]]] = []
|
|
2181
|
+
|
|
2182
|
+
# Plan offline updates
|
|
2183
|
+
offline_update, offline_rollback = self._build_offline_update_queries(
|
|
2184
|
+
feature_view, refresh_freq, warehouse, desc
|
|
2185
|
+
)
|
|
2186
|
+
operations.append(("OFFLINE_UPDATE", offline_update))
|
|
2187
|
+
if offline_rollback:
|
|
2188
|
+
rollback_operations.append(("OFFLINE_ROLLBACK", offline_rollback))
|
|
2189
|
+
|
|
2190
|
+
# Plan online updates
|
|
2191
|
+
online_strategy = self._plan_online_update(feature_view, online_config)
|
|
2192
|
+
operations.extend(online_strategy.operations)
|
|
2193
|
+
rollback_operations.extend(online_strategy.rollback_operations)
|
|
2194
|
+
|
|
2195
|
+
return operations, rollback_operations
|
|
2196
|
+
|
|
2197
|
+
def _plan_feature_view_status_operations(
|
|
2198
|
+
self, feature_view: FeatureView, operation: str
|
|
2199
|
+
) -> tuple[list[tuple[str, Union[str, FeatureView]]], list[tuple[str, Union[str, FeatureView]]]]:
|
|
2200
|
+
"""Plan atomic operations for suspend/resume operations.
|
|
2201
|
+
|
|
2202
|
+
Args:
|
|
2203
|
+
feature_view: The feature view to operate on
|
|
2204
|
+
operation: "SUSPEND" or "RESUME"
|
|
2205
|
+
|
|
2206
|
+
Returns:
|
|
2207
|
+
Tuple of (operations, rollback_operations)
|
|
2208
|
+
"""
|
|
2209
|
+
assert operation in ["SUSPEND", "RESUME"], f"Operation {operation} not supported"
|
|
2210
|
+
|
|
2211
|
+
operations: list[tuple[str, Union[str, FeatureView]]] = []
|
|
2212
|
+
rollback_operations: list[tuple[str, Union[str, FeatureView]]] = []
|
|
2213
|
+
|
|
2214
|
+
fully_qualified_name = feature_view.fully_qualified_name()
|
|
2215
|
+
|
|
2216
|
+
# Define the reverse operation for rollback
|
|
2217
|
+
reverse_operation = "RESUME" if operation == "SUSPEND" else "SUSPEND"
|
|
2218
|
+
|
|
2219
|
+
# Plan offline operations (dynamic table + task)
|
|
2220
|
+
offline_sql = f"ALTER DYNAMIC TABLE {fully_qualified_name} {operation}"
|
|
2221
|
+
offline_rollback_sql = f"ALTER DYNAMIC TABLE {fully_qualified_name} {reverse_operation}"
|
|
2222
|
+
|
|
2223
|
+
task_sql = f"ALTER TASK IF EXISTS {fully_qualified_name} {operation}"
|
|
2224
|
+
task_rollback_sql = f"ALTER TASK IF EXISTS {fully_qualified_name} {reverse_operation}"
|
|
2225
|
+
|
|
2226
|
+
operations.append(("OFFLINE_STATUS", offline_sql))
|
|
2227
|
+
operations.append(("TASK_STATUS", task_sql))
|
|
2228
|
+
|
|
2229
|
+
# Rollback operations (in reverse order)
|
|
2230
|
+
rollback_operations.insert(0, ("TASK_STATUS", task_rollback_sql))
|
|
2231
|
+
rollback_operations.insert(0, ("OFFLINE_STATUS", offline_rollback_sql))
|
|
2232
|
+
|
|
2233
|
+
# Plan online operations if applicable
|
|
2234
|
+
if feature_view.online:
|
|
2235
|
+
fully_qualified_online_name = feature_view.fully_qualified_online_table_name()
|
|
2236
|
+
online_sql = f"ALTER ONLINE FEATURE TABLE {fully_qualified_online_name} {operation}"
|
|
2237
|
+
online_rollback_sql = f"ALTER ONLINE FEATURE TABLE {fully_qualified_online_name} {reverse_operation}"
|
|
2238
|
+
|
|
2239
|
+
operations.append(("ONLINE_STATUS", online_sql))
|
|
2240
|
+
# Add to front of rollback operations to maintain reverse order
|
|
2241
|
+
rollback_operations.insert(0, ("ONLINE_STATUS", online_rollback_sql))
|
|
2242
|
+
|
|
2243
|
+
return operations, rollback_operations
|
|
2244
|
+
|
|
2245
|
+
def _handle_update_failure(
|
|
2246
|
+
self,
|
|
2247
|
+
error: Exception,
|
|
2248
|
+
rollback_operations: list[tuple[str, Union[str, FeatureView]]],
|
|
2249
|
+
feature_view: FeatureView,
|
|
2250
|
+
) -> None:
|
|
2251
|
+
"""Handle update failure with rollback."""
|
|
2252
|
+
logger.warning(f"Update failed, attempting rollback: {error}")
|
|
2253
|
+
try:
|
|
2254
|
+
self._execute_atomic_operations(rollback_operations)
|
|
2255
|
+
logger.info("Rollback completed successfully")
|
|
2256
|
+
except Exception as rollback_error:
|
|
2257
|
+
logger.error(f"Rollback failed: {rollback_error}")
|
|
2258
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2259
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2260
|
+
original_exception=RuntimeError(
|
|
2261
|
+
f"Update failed and rollback failed. Original error: {error}. Rollback error: {rollback_error}"
|
|
2262
|
+
),
|
|
2263
|
+
) from error
|
|
2264
|
+
|
|
2265
|
+
# Re-raise original error
|
|
2266
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2267
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2268
|
+
original_exception=RuntimeError(
|
|
2269
|
+
f"Update feature view {feature_view.name}/{feature_view.version} failed: {error}"
|
|
2270
|
+
),
|
|
2271
|
+
) from error
|
|
2272
|
+
|
|
2273
|
+
def _handle_status_operation_failure(
|
|
2274
|
+
self,
|
|
2275
|
+
error: Exception,
|
|
2276
|
+
rollback_operations: list[tuple[str, Union[str, FeatureView]]],
|
|
2277
|
+
feature_view: FeatureView,
|
|
2278
|
+
operation: str,
|
|
2279
|
+
) -> None:
|
|
2280
|
+
"""Handle status operation failure (suspend/resume) with rollback."""
|
|
2281
|
+
logger.warning(f"{operation} failed, attempting rollback: {error}")
|
|
2282
|
+
try:
|
|
2283
|
+
self._execute_atomic_operations(rollback_operations)
|
|
2284
|
+
logger.info("Rollback completed successfully")
|
|
2285
|
+
except Exception as rollback_error:
|
|
2286
|
+
logger.error(f"Rollback failed: {rollback_error}")
|
|
2287
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2288
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2289
|
+
original_exception=RuntimeError(
|
|
2290
|
+
f"{operation} failed and rollback failed. "
|
|
2291
|
+
f"Operation error: {error}. "
|
|
2292
|
+
f"Rollback error: {rollback_error}"
|
|
2293
|
+
),
|
|
2294
|
+
) from error
|
|
2295
|
+
|
|
2296
|
+
# Re-raise original error
|
|
2297
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2298
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2299
|
+
original_exception=RuntimeError(
|
|
2300
|
+
f"{operation} feature view {feature_view.name}/{feature_view.version} failed: {error}"
|
|
2301
|
+
),
|
|
2302
|
+
) from error
|
|
2303
|
+
|
|
2304
|
+
def _execute_atomic_operations(self, operations: list[tuple[str, Union[str, FeatureView]]]) -> None:
|
|
2305
|
+
"""Execute a list of operations atomically.
|
|
2306
|
+
|
|
2307
|
+
Args:
|
|
2308
|
+
operations: List of (operation_type, operation_data) tuples
|
|
2309
|
+
"""
|
|
2310
|
+
for op_type, op_data in operations:
|
|
2311
|
+
if op_type in (
|
|
2312
|
+
"OFFLINE_UPDATE",
|
|
2313
|
+
"OFFLINE_ROLLBACK",
|
|
2314
|
+
"UPDATE_ONLINE",
|
|
2315
|
+
"OFFLINE_STATUS",
|
|
2316
|
+
"TASK_STATUS",
|
|
2317
|
+
"ONLINE_STATUS",
|
|
2318
|
+
):
|
|
2319
|
+
assert isinstance(op_data, str)
|
|
2320
|
+
self._session.sql(op_data).collect(statement_params=self._telemetry_stmp)
|
|
2321
|
+
elif op_type == "CREATE_ONLINE":
|
|
2322
|
+
assert isinstance(op_data, FeatureView)
|
|
2323
|
+
assert op_data.version is not None
|
|
2324
|
+
feature_view_name = FeatureView._get_physical_name(op_data.name, op_data.version)
|
|
2325
|
+
self._create_online_feature_table(op_data, feature_view_name)
|
|
2326
|
+
elif op_type == "DELETE_ONLINE":
|
|
2327
|
+
assert isinstance(op_data, str)
|
|
2328
|
+
self._session.sql(f"DROP ONLINE FEATURE TABLE IF EXISTS {op_data}").collect(
|
|
2329
|
+
statement_params=self._telemetry_stmp
|
|
2330
|
+
)
|
|
2331
|
+
|
|
2332
|
+
def _read_from_offline_store(
|
|
2333
|
+
self, feature_view: FeatureView, keys: Optional[list[list[str]]], feature_names: Optional[list[str]]
|
|
2334
|
+
) -> DataFrame:
|
|
2335
|
+
"""Read feature values from the offline store (main feature view table)."""
|
|
2336
|
+
table_name = feature_view.fully_qualified_name()
|
|
2337
|
+
|
|
2338
|
+
# Build SELECT and WHERE clauses using helper methods
|
|
2339
|
+
select_clause = self._build_select_clause_and_validate(feature_view, feature_names, include_join_keys=True)
|
|
2340
|
+
where_clause = self._build_where_clause_for_keys(feature_view, keys)
|
|
2341
|
+
|
|
2342
|
+
query = f"SELECT {select_clause} FROM {table_name}{where_clause}"
|
|
2343
|
+
return self._session.sql(query)
|
|
2344
|
+
|
|
2345
|
+
def _read_from_online_store(
|
|
2346
|
+
self, feature_view: FeatureView, keys: Optional[list[list[str]]], feature_names: Optional[list[str]]
|
|
2347
|
+
) -> DataFrame:
|
|
2348
|
+
"""Read feature values from the online store with optional key filtering."""
|
|
2349
|
+
# Check if online store is enabled
|
|
2350
|
+
if not feature_view.online:
|
|
2351
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2352
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
2353
|
+
original_exception=ValueError(
|
|
2354
|
+
f"Online store is not enabled for feature view {feature_view.name}/{feature_view.version}"
|
|
2355
|
+
),
|
|
2356
|
+
)
|
|
2357
|
+
|
|
2358
|
+
fully_qualified_online_name = feature_view.fully_qualified_online_table_name()
|
|
2359
|
+
|
|
2360
|
+
# Build SELECT and WHERE clauses using helper methods
|
|
2361
|
+
select_clause = self._build_select_clause_and_validate(feature_view, feature_names, include_join_keys=True)
|
|
2362
|
+
where_clause = self._build_where_clause_for_keys(feature_view, keys)
|
|
2363
|
+
|
|
2364
|
+
query = f"SELECT {select_clause} FROM {fully_qualified_online_name}{where_clause}"
|
|
2365
|
+
return self._session.sql(query)
|
|
2366
|
+
|
|
1735
2367
|
@dispatch_decorator()
|
|
1736
2368
|
def _clear(self, dryrun: bool = True) -> None:
|
|
1737
2369
|
"""
|
|
@@ -1810,6 +2442,7 @@ class FeatureStore:
|
|
|
1810
2442
|
override: bool,
|
|
1811
2443
|
) -> None:
|
|
1812
2444
|
# TODO: cluster by join keys once DT supports that
|
|
2445
|
+
query = ""
|
|
1813
2446
|
try:
|
|
1814
2447
|
override_clause = " OR REPLACE" if override else ""
|
|
1815
2448
|
query = f"""CREATE{override_clause} DYNAMIC TABLE {fully_qualified_name} ({column_descs})
|
|
@@ -1871,6 +2504,78 @@ class FeatureStore:
|
|
|
1871
2504
|
if block:
|
|
1872
2505
|
self._check_dynamic_table_refresh_mode(feature_view_name)
|
|
1873
2506
|
|
|
2507
|
+
def _create_offline_feature_view(
|
|
2508
|
+
self,
|
|
2509
|
+
feature_view: FeatureView,
|
|
2510
|
+
feature_view_name: SqlIdentifier,
|
|
2511
|
+
fully_qualified_name: str,
|
|
2512
|
+
column_descs: str,
|
|
2513
|
+
tagging_clause_str: str,
|
|
2514
|
+
block: bool,
|
|
2515
|
+
overwrite: bool,
|
|
2516
|
+
) -> list[tuple[_FeatureStoreObjTypes, str]]:
|
|
2517
|
+
"""Create the offline representation for a feature view.
|
|
2518
|
+
|
|
2519
|
+
Depending on `refresh_freq`, this creates either a Dynamic Table (managed feature view)
|
|
2520
|
+
or a View (external feature view). Returns a list of created resources for rollback.
|
|
2521
|
+
|
|
2522
|
+
Args:
|
|
2523
|
+
feature_view: The feature view definition to materialize.
|
|
2524
|
+
feature_view_name: The physical name object for the feature view.
|
|
2525
|
+
fully_qualified_name: Fully qualified name for the created view/dynamic table.
|
|
2526
|
+
column_descs: Column descriptions clause used in the CREATE statement.
|
|
2527
|
+
tagging_clause_str: Tagging clause used in the CREATE statement.
|
|
2528
|
+
block: Whether to block until the initial refresh completes when applicable.
|
|
2529
|
+
overwrite: Whether to replace existing objects if they already exist.
|
|
2530
|
+
|
|
2531
|
+
Returns:
|
|
2532
|
+
A list of tuples of the created object types and their fully qualified names,
|
|
2533
|
+
used for potential rollback.
|
|
2534
|
+
|
|
2535
|
+
Raises:
|
|
2536
|
+
SnowflakeMLException: [RuntimeError] If creating the view or dynamic table fails.
|
|
2537
|
+
"""
|
|
2538
|
+
created: list[tuple[_FeatureStoreObjTypes, str]] = []
|
|
2539
|
+
refresh_freq = feature_view.refresh_freq
|
|
2540
|
+
|
|
2541
|
+
# External feature view via View (no refresh schedule)
|
|
2542
|
+
if refresh_freq is None:
|
|
2543
|
+
try:
|
|
2544
|
+
overwrite_clause = " OR REPLACE" if overwrite else ""
|
|
2545
|
+
query = f"""CREATE{overwrite_clause} VIEW {fully_qualified_name} ({column_descs})
|
|
2546
|
+
COMMENT = '{feature_view.desc}'
|
|
2547
|
+
TAG (
|
|
2548
|
+
{tagging_clause_str}
|
|
2549
|
+
)
|
|
2550
|
+
AS {feature_view.query}
|
|
2551
|
+
"""
|
|
2552
|
+
self._session.sql(query).collect(statement_params=self._telemetry_stmp)
|
|
2553
|
+
created.append((_FeatureStoreObjTypes.EXTERNAL_FEATURE_VIEW, fully_qualified_name))
|
|
2554
|
+
return created
|
|
2555
|
+
except Exception as e:
|
|
2556
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2557
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2558
|
+
original_exception=RuntimeError(f"Create view {fully_qualified_name} failed: {e}"),
|
|
2559
|
+
) from e
|
|
2560
|
+
|
|
2561
|
+
# Managed feature view via Dynamic Table (and optional Task)
|
|
2562
|
+
schedule_task = refresh_freq != "DOWNSTREAM" and timeparse(refresh_freq) is None
|
|
2563
|
+
self._create_dynamic_table(
|
|
2564
|
+
feature_view_name,
|
|
2565
|
+
feature_view,
|
|
2566
|
+
fully_qualified_name,
|
|
2567
|
+
column_descs,
|
|
2568
|
+
tagging_clause_str,
|
|
2569
|
+
schedule_task,
|
|
2570
|
+
feature_view.warehouse if feature_view.warehouse is not None else self._default_warehouse,
|
|
2571
|
+
block,
|
|
2572
|
+
overwrite,
|
|
2573
|
+
)
|
|
2574
|
+
created.append((_FeatureStoreObjTypes.MANAGED_FEATURE_VIEW, fully_qualified_name))
|
|
2575
|
+
if schedule_task:
|
|
2576
|
+
created.append((_FeatureStoreObjTypes.FEATURE_VIEW_REFRESH_TASK, fully_qualified_name))
|
|
2577
|
+
return created
|
|
2578
|
+
|
|
1874
2579
|
def _check_dynamic_table_refresh_mode(self, feature_view_name: SqlIdentifier) -> None:
|
|
1875
2580
|
found_dts = self._find_object("DYNAMIC TABLES", feature_view_name)
|
|
1876
2581
|
if len(found_dts) != 1:
|
|
@@ -2173,7 +2878,9 @@ class FeatureStore:
|
|
|
2173
2878
|
]
|
|
2174
2879
|
return dynamic_table_results + view_results
|
|
2175
2880
|
|
|
2176
|
-
def _update_feature_view_status(
|
|
2881
|
+
def _update_feature_view_status(
|
|
2882
|
+
self, feature_view: FeatureView, operation: str, store_type: Optional[fv_mod.StoreType] = None
|
|
2883
|
+
) -> FeatureView:
|
|
2177
2884
|
assert operation in [
|
|
2178
2885
|
"RESUME",
|
|
2179
2886
|
"SUSPEND",
|
|
@@ -2186,21 +2893,51 @@ class FeatureStore:
|
|
|
2186
2893
|
)
|
|
2187
2894
|
|
|
2188
2895
|
fully_qualified_name = feature_view.fully_qualified_name()
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
self._session.sql(f"ALTER TASK IF EXISTS {fully_qualified_name} {operation}").collect(
|
|
2896
|
+
|
|
2897
|
+
# Handle offline feature view (default for suspend/resume, or when explicitly requested)
|
|
2898
|
+
if store_type is None or store_type == fv_mod.StoreType.OFFLINE:
|
|
2899
|
+
try:
|
|
2900
|
+
self._session.sql(f"ALTER DYNAMIC TABLE {fully_qualified_name} {operation}").collect(
|
|
2195
2901
|
statement_params=self._telemetry_stmp
|
|
2196
2902
|
)
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2903
|
+
if operation != "REFRESH":
|
|
2904
|
+
self._session.sql(f"ALTER TASK IF EXISTS {fully_qualified_name} {operation}").collect(
|
|
2905
|
+
statement_params=self._telemetry_stmp
|
|
2906
|
+
)
|
|
2907
|
+
except Exception as e:
|
|
2908
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2909
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2910
|
+
original_exception=RuntimeError(
|
|
2911
|
+
f"Failed to update feature view {fully_qualified_name}'s status: {e}"
|
|
2912
|
+
),
|
|
2913
|
+
) from e
|
|
2202
2914
|
|
|
2203
|
-
|
|
2915
|
+
elif store_type == fv_mod.StoreType.ONLINE and operation in ["SUSPEND", "RESUME", "REFRESH"]:
|
|
2916
|
+
if feature_view.online:
|
|
2917
|
+
fully_qualified_online_name = feature_view.fully_qualified_online_table_name()
|
|
2918
|
+
try:
|
|
2919
|
+
self._session.sql(f"ALTER ONLINE FEATURE TABLE {fully_qualified_online_name} {operation}").collect(
|
|
2920
|
+
statement_params=self._telemetry_stmp
|
|
2921
|
+
)
|
|
2922
|
+
logger.info(
|
|
2923
|
+
f"Successfully {operation.lower()}ed online feature table for "
|
|
2924
|
+
f"{feature_view.name}/{feature_view.version}"
|
|
2925
|
+
)
|
|
2926
|
+
except Exception as e:
|
|
2927
|
+
# For refresh operations, raise the exception; for suspend/resume, just log warning
|
|
2928
|
+
if operation == "REFRESH":
|
|
2929
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
2930
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2931
|
+
original_exception=RuntimeError(
|
|
2932
|
+
f"Failed to refresh online feature table {fully_qualified_online_name}: {e}"
|
|
2933
|
+
),
|
|
2934
|
+
) from e
|
|
2935
|
+
else:
|
|
2936
|
+
# Log warning but don't fail the entire operation if online table operation
|
|
2937
|
+
# fails for suspend/resume
|
|
2938
|
+
logger.warning(f"Failed to {operation} online feature table {fully_qualified_online_name}: {e}")
|
|
2939
|
+
|
|
2940
|
+
logger.info(f"Successfully {operation.lower()}ed FeatureView {feature_view.name}/{feature_view.version}.")
|
|
2204
2941
|
return self.get_feature_view(feature_view.name, feature_view.version)
|
|
2205
2942
|
|
|
2206
2943
|
def _optimized_find_feature_views(
|
|
@@ -2245,8 +2982,81 @@ class FeatureStore:
|
|
|
2245
2982
|
values.append(row["scheduling_state"] if "scheduling_state" in row else None)
|
|
2246
2983
|
values.append(row["warehouse"] if "warehouse" in row else None)
|
|
2247
2984
|
values.append(json.dumps(self._extract_cluster_by_columns(row["cluster_by"])) if "cluster_by" in row else None)
|
|
2985
|
+
|
|
2986
|
+
online_config_json = self._determine_online_config_from_oft(name, version, include_runtime_metadata=True)
|
|
2987
|
+
values.append(online_config_json)
|
|
2988
|
+
|
|
2248
2989
|
output_values.append(values)
|
|
2249
2990
|
|
|
2991
|
+
def _determine_online_config_from_oft(
|
|
2992
|
+
self, name: str, version: str, *, include_runtime_metadata: bool = False
|
|
2993
|
+
) -> str:
|
|
2994
|
+
"""Determine online configuration by checking for corresponding online feature table.
|
|
2995
|
+
|
|
2996
|
+
Args:
|
|
2997
|
+
name: Feature view name
|
|
2998
|
+
version: Feature view version
|
|
2999
|
+
include_runtime_metadata: If True, includes additional runtime metadata
|
|
3000
|
+
(refresh_mode, scheduling_state) in the JSON for display purposes.
|
|
3001
|
+
If False, returns only OnlineConfig-compatible JSON.
|
|
3002
|
+
|
|
3003
|
+
Returns:
|
|
3004
|
+
JSON string of OnlineConfig with enable=True and table's target_lag if online table exists,
|
|
3005
|
+
otherwise default config with enable=False. When include_runtime_metadata=True,
|
|
3006
|
+
may include additional fields not part of OnlineConfig.
|
|
3007
|
+
|
|
3008
|
+
Raises:
|
|
3009
|
+
SnowflakeMLException: If multiple online feature tables found for the given name/version,
|
|
3010
|
+
or if the online feature table is missing required 'target_lag' column.
|
|
3011
|
+
"""
|
|
3012
|
+
online_table_name = FeatureView._get_online_table_name(name, version)
|
|
3013
|
+
|
|
3014
|
+
online_tables = self._find_object(object_type="ONLINE FEATURE TABLES", object_name=online_table_name)
|
|
3015
|
+
|
|
3016
|
+
if online_tables:
|
|
3017
|
+
if len(online_tables) != 1:
|
|
3018
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
3019
|
+
error_code=error_codes.INTERNAL_SNOWML_ERROR,
|
|
3020
|
+
original_exception=RuntimeError(
|
|
3021
|
+
f"Expected exactly 1 online feature table for {online_table_name}, "
|
|
3022
|
+
f"but found {len(online_tables)}"
|
|
3023
|
+
),
|
|
3024
|
+
)
|
|
3025
|
+
|
|
3026
|
+
oft_row = online_tables[0]
|
|
3027
|
+
|
|
3028
|
+
def extract_field(row: Row, field_name: str) -> str:
|
|
3029
|
+
if field_name in row:
|
|
3030
|
+
return str(row[field_name])
|
|
3031
|
+
elif field_name.upper() in row:
|
|
3032
|
+
return str(row[field_name.upper()])
|
|
3033
|
+
else:
|
|
3034
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
3035
|
+
error_code=error_codes.INTERNAL_SNOWML_ERROR,
|
|
3036
|
+
original_exception=RuntimeError(
|
|
3037
|
+
f"Online feature table {online_table_name} missing required '{field_name}' column"
|
|
3038
|
+
),
|
|
3039
|
+
)
|
|
3040
|
+
|
|
3041
|
+
# Extract required fields using consistent pattern
|
|
3042
|
+
target_lag = extract_field(oft_row, "target_lag")
|
|
3043
|
+
|
|
3044
|
+
online_config = fv_mod.OnlineConfig(enable=True, target_lag=target_lag)
|
|
3045
|
+
|
|
3046
|
+
if include_runtime_metadata:
|
|
3047
|
+
display_data = json.loads(online_config.to_json())
|
|
3048
|
+
|
|
3049
|
+
display_data["refresh_mode"] = extract_field(oft_row, "refresh_mode")
|
|
3050
|
+
display_data["scheduling_state"] = extract_field(oft_row, "scheduling_state")
|
|
3051
|
+
|
|
3052
|
+
return json.dumps(display_data)
|
|
3053
|
+
else:
|
|
3054
|
+
return online_config.to_json()
|
|
3055
|
+
else:
|
|
3056
|
+
# No online feature table found - return default disabled config
|
|
3057
|
+
online_config = fv_mod.OnlineConfig(enable=False, target_lag=fv_mod._DEFAULT_TARGET_LAG)
|
|
3058
|
+
return online_config.to_json()
|
|
3059
|
+
|
|
2250
3060
|
def _lookup_feature_view_metadata(self, row: Row, fv_name: str) -> tuple[_FeatureViewMetadata, str]:
|
|
2251
3061
|
if len(row["text"]) == 0:
|
|
2252
3062
|
# NOTE: if this is a shared feature view, then text column will be empty due to privacy constraints.
|
|
@@ -2274,6 +3084,7 @@ class FeatureStore:
|
|
|
2274
3084
|
)
|
|
2275
3085
|
fv_metadata = _FeatureViewMetadata.from_json(m.group("fv_metadata"))
|
|
2276
3086
|
query = m.group("query")
|
|
3087
|
+
|
|
2277
3088
|
return (fv_metadata, query)
|
|
2278
3089
|
|
|
2279
3090
|
def _compose_feature_view(self, row: Row, obj_type: _FeatureStoreObjTypes, entity_list: list[Row]) -> FeatureView:
|
|
@@ -2296,6 +3107,9 @@ class FeatureStore:
|
|
|
2296
3107
|
infer_schema_df = self._session.sql(f"SELECT * FROM {self._get_fully_qualified_name(fv_name)}")
|
|
2297
3108
|
desc = row["comment"]
|
|
2298
3109
|
|
|
3110
|
+
online_config_json = self._determine_online_config_from_oft(name.identifier(), version)
|
|
3111
|
+
online_config = fv_mod.OnlineConfig.from_json(online_config_json)
|
|
3112
|
+
|
|
2299
3113
|
if obj_type == _FeatureStoreObjTypes.MANAGED_FEATURE_VIEW:
|
|
2300
3114
|
df = self._session.sql(query)
|
|
2301
3115
|
entities = [find_and_compose_entity(n) for n in fv_metadata.entities]
|
|
@@ -2332,6 +3146,7 @@ class FeatureStore:
|
|
|
2332
3146
|
infer_schema_df=infer_schema_df,
|
|
2333
3147
|
session=self._session,
|
|
2334
3148
|
cluster_by=self._extract_cluster_by_columns(row["cluster_by"]),
|
|
3149
|
+
online_config=online_config,
|
|
2335
3150
|
)
|
|
2336
3151
|
return fv
|
|
2337
3152
|
else:
|
|
@@ -2359,6 +3174,7 @@ class FeatureStore:
|
|
|
2359
3174
|
owner=row["owner"],
|
|
2360
3175
|
infer_schema_df=infer_schema_df,
|
|
2361
3176
|
session=self._session,
|
|
3177
|
+
online_config=online_config,
|
|
2362
3178
|
)
|
|
2363
3179
|
return fv
|
|
2364
3180
|
|
|
@@ -2373,6 +3189,103 @@ class FeatureStore:
|
|
|
2373
3189
|
descs[SqlIdentifier(r["name"], case_sensitive=True).identifier()] = r["comment"]
|
|
2374
3190
|
return descs
|
|
2375
3191
|
|
|
3192
|
+
@telemetry.send_api_usage_telemetry(project=_PROJECT)
|
|
3193
|
+
def _create_online_feature_table(
|
|
3194
|
+
self,
|
|
3195
|
+
feature_view: FeatureView,
|
|
3196
|
+
feature_view_name: SqlIdentifier,
|
|
3197
|
+
overwrite: bool = False,
|
|
3198
|
+
) -> str:
|
|
3199
|
+
"""Create online feature table for the feature view.
|
|
3200
|
+
|
|
3201
|
+
Args:
|
|
3202
|
+
feature_view: The FeatureView object for which to create the online feature table.
|
|
3203
|
+
feature_view_name: The name of the feature view.
|
|
3204
|
+
overwrite: Whether to overwrite existing online feature table. Defaults to False.
|
|
3205
|
+
|
|
3206
|
+
Returns:
|
|
3207
|
+
The name of the created online table (without schema qualification).
|
|
3208
|
+
|
|
3209
|
+
Raises:
|
|
3210
|
+
SnowflakeMLException: [ValueError] If OnlineConfig is required but not provided.
|
|
3211
|
+
SnowflakeMLException: If creating the online feature table fails.
|
|
3212
|
+
"""
|
|
3213
|
+
online_table_name = FeatureView._get_online_table_name(feature_view_name)
|
|
3214
|
+
|
|
3215
|
+
fully_qualified_online_name = self._get_fully_qualified_name(online_table_name)
|
|
3216
|
+
source_table_name = feature_view_name
|
|
3217
|
+
|
|
3218
|
+
# Extract join keys for PRIMARY KEY (preserve order and ensure unique)
|
|
3219
|
+
ordered_join_keys: list[str] = []
|
|
3220
|
+
seen_join_keys: set[str] = set()
|
|
3221
|
+
for entity in feature_view.entities:
|
|
3222
|
+
for join_key in entity.join_keys:
|
|
3223
|
+
resolved_key = join_key.resolved()
|
|
3224
|
+
if resolved_key not in seen_join_keys:
|
|
3225
|
+
seen_join_keys.add(resolved_key)
|
|
3226
|
+
ordered_join_keys.append(resolved_key)
|
|
3227
|
+
quoted_join_keys = [f'"{key}"' for key in ordered_join_keys]
|
|
3228
|
+
primary_key_clause = f"PRIMARY KEY ({', '.join(quoted_join_keys)})"
|
|
3229
|
+
|
|
3230
|
+
# Build online config clauses
|
|
3231
|
+
config = feature_view.online_config
|
|
3232
|
+
if not config:
|
|
3233
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
3234
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
3235
|
+
original_exception=ValueError("OnlineConfig is required to create online feature table"),
|
|
3236
|
+
)
|
|
3237
|
+
target_lag_value = config.target_lag if config.target_lag is not None else fv_mod._DEFAULT_TARGET_LAG
|
|
3238
|
+
target_lag_clause = f"TARGET_LAG='{target_lag_value}'"
|
|
3239
|
+
|
|
3240
|
+
warehouse_clause = ""
|
|
3241
|
+
if feature_view.warehouse:
|
|
3242
|
+
warehouse_clause = f"WAREHOUSE={feature_view.warehouse}"
|
|
3243
|
+
elif self._default_warehouse:
|
|
3244
|
+
warehouse_clause = f"WAREHOUSE={self._default_warehouse}"
|
|
3245
|
+
|
|
3246
|
+
refresh_mode_clause = ""
|
|
3247
|
+
if feature_view.refresh_mode:
|
|
3248
|
+
refresh_mode_clause = f"REFRESH_MODE='{feature_view.refresh_mode}'"
|
|
3249
|
+
|
|
3250
|
+
timestamp_clause = ""
|
|
3251
|
+
if feature_view.timestamp_col:
|
|
3252
|
+
timestamp_clause = f"TIMESTAMP_COLUMN='{feature_view.timestamp_col}'"
|
|
3253
|
+
|
|
3254
|
+
# Create online feature table
|
|
3255
|
+
try:
|
|
3256
|
+
overwrite_clause = "OR REPLACE " if overwrite else ""
|
|
3257
|
+
|
|
3258
|
+
query_parts = [
|
|
3259
|
+
f"CREATE {overwrite_clause}ONLINE FEATURE TABLE {fully_qualified_online_name}",
|
|
3260
|
+
primary_key_clause,
|
|
3261
|
+
refresh_mode_clause,
|
|
3262
|
+
timestamp_clause,
|
|
3263
|
+
warehouse_clause,
|
|
3264
|
+
target_lag_clause,
|
|
3265
|
+
f"FROM {source_table_name}",
|
|
3266
|
+
]
|
|
3267
|
+
|
|
3268
|
+
query = " ".join(part for part in query_parts if part)
|
|
3269
|
+
self._session.sql(query).collect(statement_params=self._telemetry_stmp)
|
|
3270
|
+
|
|
3271
|
+
oft_obj_info = _FeatureStoreObjInfo(_FeatureStoreObjTypes.ONLINE_FEATURE_TABLE, snowml_version.VERSION)
|
|
3272
|
+
tag_clause = f"""
|
|
3273
|
+
ALTER ONLINE FEATURE TABLE {fully_qualified_online_name} SET TAG
|
|
3274
|
+
{self._get_fully_qualified_name(_FEATURE_STORE_OBJECT_TAG)} = '{oft_obj_info.to_json()}'
|
|
3275
|
+
"""
|
|
3276
|
+
|
|
3277
|
+
self._session.sql(tag_clause).collect(statement_params=self._telemetry_stmp)
|
|
3278
|
+
except Exception as e:
|
|
3279
|
+
logger.error(f"Failed to create online feature table for {feature_view.name}: {e}")
|
|
3280
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
3281
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
3282
|
+
original_exception=RuntimeError(
|
|
3283
|
+
f"Create online feature table {fully_qualified_online_name} failed: {e}"
|
|
3284
|
+
),
|
|
3285
|
+
) from e
|
|
3286
|
+
|
|
3287
|
+
return online_table_name
|
|
3288
|
+
|
|
2376
3289
|
def _find_object(
|
|
2377
3290
|
self,
|
|
2378
3291
|
object_type: str,
|
|
@@ -2410,13 +3323,21 @@ class FeatureStore:
|
|
|
2410
3323
|
all_rows = self._session.sql(f"SHOW {object_type} LIKE '{match_name}' {search_scope}").collect(
|
|
2411
3324
|
statement_params=self._telemetry_stmp
|
|
2412
3325
|
)
|
|
2413
|
-
# There could be
|
|
3326
|
+
# There could be non-FS objects under FS schema, thus filter on objects with FS special tag.
|
|
2414
3327
|
if object_type not in tag_free_object_types and len(all_rows) > 0:
|
|
2415
3328
|
fs_obj_rows = self._lookup_tagged_objects(
|
|
2416
3329
|
_FEATURE_STORE_OBJECT_TAG, [lambda d: d["domain"] == obj_domain]
|
|
2417
3330
|
)
|
|
2418
3331
|
fs_tag_objects = [row["entityName"] for row in fs_obj_rows]
|
|
2419
3332
|
except Exception as e:
|
|
3333
|
+
# ONLINE FEATURE TABLE preview feature may raise SQL error if not enabled
|
|
3334
|
+
# Return empty list for discovery flows in this case
|
|
3335
|
+
if (
|
|
3336
|
+
object_type == "ONLINE FEATURE TABLES"
|
|
3337
|
+
and isinstance(e, SnowparkSQLException)
|
|
3338
|
+
and ("unexpected 'online'" in str(e).lower())
|
|
3339
|
+
):
|
|
3340
|
+
return []
|
|
2420
3341
|
raise snowml_exceptions.SnowflakeMLException(
|
|
2421
3342
|
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
|
2422
3343
|
original_exception=RuntimeError(f"Failed to find object : {e}"),
|
|
@@ -2631,3 +3552,101 @@ class FeatureStore:
|
|
|
2631
3552
|
# Handle both quoted and unquoted column names.
|
|
2632
3553
|
return re.findall(identifier.SF_IDENTIFIER_RE, match.group(1))
|
|
2633
3554
|
return []
|
|
3555
|
+
|
|
3556
|
+
def _build_select_clause_and_validate(
|
|
3557
|
+
self, feature_view: FeatureView, feature_names: Optional[list[str]], include_join_keys: bool = True
|
|
3558
|
+
) -> str:
|
|
3559
|
+
"""Build SELECT clause for feature view queries and validate feature names.
|
|
3560
|
+
|
|
3561
|
+
Args:
|
|
3562
|
+
feature_view: The feature view to build the clause for
|
|
3563
|
+
feature_names: Optional list of feature names to include
|
|
3564
|
+
include_join_keys: Whether to include join keys in the select clause
|
|
3565
|
+
|
|
3566
|
+
Returns:
|
|
3567
|
+
SELECT clause string
|
|
3568
|
+
|
|
3569
|
+
Raises:
|
|
3570
|
+
SnowflakeMLException: If requested feature names don't exist
|
|
3571
|
+
"""
|
|
3572
|
+
if feature_names:
|
|
3573
|
+
# Validate feature names exist
|
|
3574
|
+
available_features = [f.name for f in feature_view.output_schema.fields]
|
|
3575
|
+
for feature_name in feature_names:
|
|
3576
|
+
if feature_name not in available_features:
|
|
3577
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
3578
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
3579
|
+
original_exception=ValueError(
|
|
3580
|
+
f"Feature '{feature_name}' not found in feature view. "
|
|
3581
|
+
f"Available features: {available_features}"
|
|
3582
|
+
),
|
|
3583
|
+
)
|
|
3584
|
+
|
|
3585
|
+
# Build select clause with join keys and requested features
|
|
3586
|
+
select_columns = []
|
|
3587
|
+
if include_join_keys:
|
|
3588
|
+
all_join_keys = []
|
|
3589
|
+
for entity in feature_view.entities:
|
|
3590
|
+
all_join_keys.extend([key.resolved() for key in entity.join_keys])
|
|
3591
|
+
select_columns.extend([f'"{key}"' for key in all_join_keys])
|
|
3592
|
+
|
|
3593
|
+
select_columns.extend([f'"{name}"' for name in feature_names])
|
|
3594
|
+
return ", ".join(select_columns)
|
|
3595
|
+
else:
|
|
3596
|
+
# Select all columns
|
|
3597
|
+
return "*"
|
|
3598
|
+
|
|
3599
|
+
def _build_where_clause_for_keys(self, feature_view: FeatureView, keys: Optional[list[list[str]]]) -> str:
|
|
3600
|
+
"""Build WHERE clause for key filtering.
|
|
3601
|
+
|
|
3602
|
+
Args:
|
|
3603
|
+
feature_view: The feature view to build the clause for
|
|
3604
|
+
keys: Optional list of key value lists to filter by
|
|
3605
|
+
|
|
3606
|
+
Returns:
|
|
3607
|
+
WHERE clause string (empty if no keys provided)
|
|
3608
|
+
|
|
3609
|
+
Raises:
|
|
3610
|
+
SnowflakeMLException: If key structure is invalid
|
|
3611
|
+
"""
|
|
3612
|
+
if not keys:
|
|
3613
|
+
return ""
|
|
3614
|
+
|
|
3615
|
+
# Get join keys from entities for key filtering
|
|
3616
|
+
all_join_keys = []
|
|
3617
|
+
for entity in feature_view.entities:
|
|
3618
|
+
all_join_keys.extend([key.resolved() for key in entity.join_keys])
|
|
3619
|
+
|
|
3620
|
+
# Validate key structure
|
|
3621
|
+
for key_values in keys:
|
|
3622
|
+
if len(key_values) != len(all_join_keys):
|
|
3623
|
+
raise snowml_exceptions.SnowflakeMLException(
|
|
3624
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
|
3625
|
+
original_exception=ValueError(
|
|
3626
|
+
f"Each key must have {len(all_join_keys)} values for join keys {all_join_keys}, "
|
|
3627
|
+
f"got {len(key_values)} values"
|
|
3628
|
+
),
|
|
3629
|
+
)
|
|
3630
|
+
|
|
3631
|
+
where_conditions = []
|
|
3632
|
+
for key_values in keys:
|
|
3633
|
+
key_conditions = []
|
|
3634
|
+
for join_key, value in zip(all_join_keys, key_values):
|
|
3635
|
+
safe_value = str(value).replace("'", "''")
|
|
3636
|
+
key_conditions.append(f"\"{join_key}\" = '{safe_value}'")
|
|
3637
|
+
where_conditions.append(f"({' AND '.join(key_conditions)})")
|
|
3638
|
+
|
|
3639
|
+
return f" WHERE {' OR '.join(where_conditions)}"
|
|
3640
|
+
|
|
3641
|
+
def _get_store_type(self, store_type: Union[fv_mod.StoreType, str]) -> fv_mod.StoreType:
|
|
3642
|
+
"""Return a StoreType enum from a Union[StoreType, str].
|
|
3643
|
+
|
|
3644
|
+
Args:
|
|
3645
|
+
store_type: Store type enum or string value.
|
|
3646
|
+
|
|
3647
|
+
Returns:
|
|
3648
|
+
StoreType enum value.
|
|
3649
|
+
"""
|
|
3650
|
+
if isinstance(store_type, str):
|
|
3651
|
+
return fv_mod.StoreType(store_type.lower())
|
|
3652
|
+
return store_type
|