snowflake-ml-python 1.5.4__py3-none-any.whl → 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/__init__.py +2 -0
- snowflake/cortex/_classify_text.py +36 -0
- snowflake/cortex/_complete.py +67 -10
- snowflake/cortex/_util.py +4 -4
- snowflake/ml/_internal/lineage/lineage_utils.py +4 -4
- snowflake/ml/_internal/telemetry.py +12 -2
- snowflake/ml/data/_internal/arrow_ingestor.py +228 -0
- snowflake/ml/data/_internal/ingestor_utils.py +58 -0
- snowflake/ml/data/data_connector.py +133 -0
- snowflake/ml/data/data_ingestor.py +28 -0
- snowflake/ml/data/data_source.py +23 -0
- snowflake/ml/dataset/dataset.py +1 -13
- snowflake/ml/dataset/dataset_reader.py +18 -118
- snowflake/ml/feature_store/access_manager.py +7 -1
- snowflake/ml/feature_store/entity.py +19 -2
- snowflake/ml/feature_store/examples/citibike_trip_features/entities.py +20 -0
- snowflake/ml/feature_store/examples/citibike_trip_features/features/station_feature.py +31 -0
- snowflake/ml/feature_store/examples/citibike_trip_features/features/trip_feature.py +24 -0
- snowflake/ml/feature_store/examples/citibike_trip_features/source.yaml +4 -0
- snowflake/ml/feature_store/examples/example_helper.py +240 -0
- snowflake/ml/feature_store/examples/new_york_taxi_features/entities.py +12 -0
- snowflake/ml/feature_store/examples/new_york_taxi_features/features/dropoff_features.py +39 -0
- snowflake/ml/feature_store/examples/new_york_taxi_features/features/pickup_features.py +58 -0
- snowflake/ml/feature_store/examples/new_york_taxi_features/source.yaml +5 -0
- snowflake/ml/feature_store/examples/source_data/citibike_trips.yaml +36 -0
- snowflake/ml/feature_store/examples/source_data/fraud_transactions.yaml +29 -0
- snowflake/ml/feature_store/examples/source_data/nyc_yellow_trips.yaml +4 -0
- snowflake/ml/feature_store/examples/source_data/winequality_red.yaml +32 -0
- snowflake/ml/feature_store/examples/wine_quality_features/entities.py +14 -0
- snowflake/ml/feature_store/examples/wine_quality_features/features/managed_wine_features.py +29 -0
- snowflake/ml/feature_store/examples/wine_quality_features/features/static_wine_features.py +21 -0
- snowflake/ml/feature_store/examples/wine_quality_features/source.yaml +5 -0
- snowflake/ml/feature_store/feature_store.py +579 -53
- snowflake/ml/feature_store/feature_view.py +168 -5
- snowflake/ml/fileset/stage_fs.py +18 -10
- snowflake/ml/lineage/lineage_node.py +1 -1
- snowflake/ml/model/_deploy_client/image_builds/inference_server/main.py +2 -3
- snowflake/ml/model/_model_composer/model_composer.py +11 -14
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +24 -16
- snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py +2 -1
- snowflake/ml/model/_model_composer/model_method/function_generator.py +3 -3
- snowflake/ml/model/_model_composer/model_method/infer_function.py_template +3 -32
- snowflake/ml/model/_model_composer/model_method/infer_partitioned.py_template +3 -27
- snowflake/ml/model/_model_composer/model_method/infer_table_function.py_template +3 -32
- snowflake/ml/model/_model_composer/model_method/model_method.py +5 -2
- snowflake/ml/model/_packager/model_handlers/_base.py +11 -1
- snowflake/ml/model/_packager/model_handlers/_utils.py +58 -1
- snowflake/ml/model/_packager/model_handlers/catboost.py +42 -0
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +68 -0
- snowflake/ml/model/_packager/model_handlers/xgboost.py +59 -0
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +3 -5
- snowflake/ml/model/model_signature.py +4 -4
- snowflake/ml/model/type_hints.py +4 -0
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +1 -1
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_search_udf_file.py +13 -1
- snowflake/ml/modeling/impute/simple_imputer.py +26 -0
- snowflake/ml/modeling/pipeline/pipeline.py +4 -4
- snowflake/ml/registry/registry.py +100 -13
- snowflake/ml/version.py +1 -1
- {snowflake_ml_python-1.5.4.dist-info → snowflake_ml_python-1.6.0.dist-info}/METADATA +48 -2
- {snowflake_ml_python-1.5.4.dist-info → snowflake_ml_python-1.6.0.dist-info}/RECORD +64 -42
- {snowflake_ml_python-1.5.4.dist-info → snowflake_ml_python-1.6.0.dist-info}/WHEEL +1 -1
- snowflake/ml/_internal/lineage/data_source.py +0 -10
- {snowflake_ml_python-1.5.4.dist-info → snowflake_ml_python-1.6.0.dist-info}/LICENSE.txt +0 -0
- {snowflake_ml_python-1.5.4.dist-info → snowflake_ml_python-1.6.0.dist-info}/top_level.txt +0 -0
@@ -135,6 +135,7 @@ _LIST_FEATURE_VIEW_SCHEMA = StructType(
|
|
135
135
|
StructField("refresh_freq", StringType()),
|
136
136
|
StructField("refresh_mode", StringType()),
|
137
137
|
StructField("scheduling_state", StringType()),
|
138
|
+
StructField("warehouse", StringType()),
|
138
139
|
]
|
139
140
|
)
|
140
141
|
|
@@ -205,6 +206,7 @@ class FeatureStore:
|
|
205
206
|
database: str,
|
206
207
|
name: str,
|
207
208
|
default_warehouse: str,
|
209
|
+
*,
|
208
210
|
creation_mode: CreationMode = CreationMode.FAIL_IF_NOT_EXIST,
|
209
211
|
) -> None:
|
210
212
|
"""
|
@@ -224,6 +226,32 @@ class FeatureStore:
|
|
224
226
|
SnowflakeMLException: [ValueError] Required resources not exist when mode is FAIL_IF_NOT_EXIST.
|
225
227
|
SnowflakeMLException: [RuntimeError] Failed to find resources.
|
226
228
|
SnowflakeMLException: [RuntimeError] Failed to create feature store.
|
229
|
+
|
230
|
+
Example::
|
231
|
+
|
232
|
+
>>> from snowflake.ml.feature_store import (
|
233
|
+
... FeatureStore,
|
234
|
+
... CreationMode,
|
235
|
+
... )
|
236
|
+
<BLANKLINE>
|
237
|
+
>>> # Create a new Feature Store:
|
238
|
+
>>> fs = FeatureStore(
|
239
|
+
... session=session,
|
240
|
+
... database="MYDB",
|
241
|
+
... name="MYSCHEMA",
|
242
|
+
... default_warehouse="MYWH",
|
243
|
+
... creation_mode=CreationMode.CREATE_IF_NOT_EXIST
|
244
|
+
... )
|
245
|
+
<BLANKLINE>
|
246
|
+
>>> # Connect to an existing Feature Store:
|
247
|
+
>>> fs = FeatureStore(
|
248
|
+
... session=session,
|
249
|
+
... database="MYDB",
|
250
|
+
... name="MYSCHEMA",
|
251
|
+
... default_warehouse="MYWH",
|
252
|
+
... creation_mode=CreationMode.FAIL_IF_NOT_EXIST
|
253
|
+
... )
|
254
|
+
|
227
255
|
"""
|
228
256
|
|
229
257
|
database = SqlIdentifier(database)
|
@@ -284,6 +312,16 @@ class FeatureStore:
|
|
284
312
|
|
285
313
|
Raises:
|
286
314
|
SnowflakeMLException: If warehouse does not exists.
|
315
|
+
|
316
|
+
Example::
|
317
|
+
|
318
|
+
>>> fs = FeatureStore(...)
|
319
|
+
>>> fs.update_default_warehouse("MYWH_2")
|
320
|
+
>>> draft_fv = FeatureView("my_fv", ...)
|
321
|
+
>>> registered_fv = fs.register_feature_view(draft_fv, '2.0')
|
322
|
+
>>> print(registered_fv.warehouse)
|
323
|
+
MYWH_2
|
324
|
+
|
287
325
|
"""
|
288
326
|
warehouse = SqlIdentifier(warehouse_name)
|
289
327
|
warehouse_result = self._find_object("WAREHOUSES", warehouse)
|
@@ -301,15 +339,27 @@ class FeatureStore:
|
|
301
339
|
Register Entity in the FeatureStore.
|
302
340
|
|
303
341
|
Args:
|
304
|
-
entity: Entity object to
|
342
|
+
entity: Entity object to be registered.
|
305
343
|
|
306
344
|
Returns:
|
307
345
|
A registered entity object.
|
308
346
|
|
309
347
|
Raises:
|
310
348
|
SnowflakeMLException: [RuntimeError] Failed to find resources.
|
311
|
-
"""
|
312
349
|
|
350
|
+
Example::
|
351
|
+
|
352
|
+
>>> fs = FeatureStore(...)
|
353
|
+
>>> e = Entity('BAR', ['A'], desc='entity bar')
|
354
|
+
>>> fs.register_entity(e)
|
355
|
+
>>> fs.list_entities().show()
|
356
|
+
--------------------------------------------------
|
357
|
+
|"NAME" |"JOIN_KEYS" |"DESC" |"OWNER" |
|
358
|
+
--------------------------------------------------
|
359
|
+
|BAR |["A"] |entity bar |REGTEST_RL |
|
360
|
+
--------------------------------------------------
|
361
|
+
|
362
|
+
"""
|
313
363
|
tag_name = self._get_entity_name(entity.name)
|
314
364
|
found_rows = self._find_object("TAGS", tag_name)
|
315
365
|
if len(found_rows) > 0:
|
@@ -341,12 +391,74 @@ class FeatureStore:
|
|
341
391
|
|
342
392
|
return self.get_entity(entity.name)
|
343
393
|
|
394
|
+
def update_entity(self, name: str, *, desc: Optional[str] = None) -> Optional[Entity]:
|
395
|
+
"""Update a registered entity with provided information.
|
396
|
+
|
397
|
+
Args:
|
398
|
+
name: Name of entity to update.
|
399
|
+
desc: Optional new description to apply. Default to None.
|
400
|
+
|
401
|
+
Raises:
|
402
|
+
SnowflakeMLException: Error happen when updating.
|
403
|
+
|
404
|
+
Returns:
|
405
|
+
A new entity with updated information or None if the entity doesn't exist.
|
406
|
+
|
407
|
+
Example::
|
408
|
+
|
409
|
+
>>> fs = FeatureStore(...)
|
410
|
+
<BLANKLINE>
|
411
|
+
>>> e = Entity(name='foo', join_keys=['COL_1'], desc='old desc')
|
412
|
+
>>> fs.list_entities().show()
|
413
|
+
------------------------------------------------
|
414
|
+
|"NAME" |"JOIN_KEYS" |"DESC" |"OWNER" |
|
415
|
+
------------------------------------------------
|
416
|
+
|FOO |["COL_1"] |old desc |REGTEST_RL |
|
417
|
+
------------------------------------------------
|
418
|
+
<BLANKLINE>
|
419
|
+
>>> fs.update_entity('foo', desc='NEW DESC')
|
420
|
+
>>> fs.list_entities().show()
|
421
|
+
------------------------------------------------
|
422
|
+
|"NAME" |"JOIN_KEYS" |"DESC" |"OWNER" |
|
423
|
+
------------------------------------------------
|
424
|
+
|FOO |["COL_1"] |NEW DESC |REGTEST_RL |
|
425
|
+
------------------------------------------------
|
426
|
+
|
427
|
+
"""
|
428
|
+
name = SqlIdentifier(name)
|
429
|
+
found_rows = self.list_entities().filter(F.col("NAME") == name.resolved()).collect()
|
430
|
+
|
431
|
+
if len(found_rows) == 0:
|
432
|
+
warnings.warn(
|
433
|
+
f"Entity {name} does not exist.",
|
434
|
+
stacklevel=2,
|
435
|
+
category=UserWarning,
|
436
|
+
)
|
437
|
+
return None
|
438
|
+
|
439
|
+
new_desc = desc if desc is not None else found_rows[0]["DESC"]
|
440
|
+
|
441
|
+
try:
|
442
|
+
full_name = f"{self._config.full_schema_path}.{self._get_entity_name(name)}"
|
443
|
+
self._session.sql(f"ALTER TAG {full_name} SET COMMENT = '{new_desc}'").collect(
|
444
|
+
statement_params=self._telemetry_stmp
|
445
|
+
)
|
446
|
+
except Exception as e:
|
447
|
+
raise snowml_exceptions.SnowflakeMLException(
|
448
|
+
error_code=error_codes.INTERNAL_SNOWPARK_ERROR,
|
449
|
+
original_exception=RuntimeError(f"Failed to update entity `{name}`: {e}."),
|
450
|
+
) from e
|
451
|
+
|
452
|
+
logger.info(f"Successfully updated Entity {name}.")
|
453
|
+
return self.get_entity(name)
|
454
|
+
|
344
455
|
# TODO: add support to update column desc once SNOW-894249 is fixed
|
345
456
|
@dispatch_decorator()
|
346
457
|
def register_feature_view(
|
347
458
|
self,
|
348
459
|
feature_view: FeatureView,
|
349
460
|
version: str,
|
461
|
+
*,
|
350
462
|
block: bool = True,
|
351
463
|
overwrite: bool = False,
|
352
464
|
) -> FeatureView:
|
@@ -356,12 +468,6 @@ class FeatureStore:
|
|
356
468
|
NOTE: Each new materialization will trigger a full FeatureView history refresh for the data included in the
|
357
469
|
FeatureView.
|
358
470
|
|
359
|
-
Examples:
|
360
|
-
...
|
361
|
-
draft_fv = FeatureView(name="my_fv", entities=[entities], feature_df)
|
362
|
-
registered_fv = fs.register_feature_view(feature_view=draft_fv, version="v1")
|
363
|
-
...
|
364
|
-
|
365
471
|
Args:
|
366
472
|
feature_view: FeatureView instance to materialize.
|
367
473
|
version: version of the registered FeatureView.
|
@@ -380,6 +486,35 @@ class FeatureStore:
|
|
380
486
|
SnowflakeMLException: [ValueError] Warehouse or default warehouse is not specified.
|
381
487
|
SnowflakeMLException: [RuntimeError] Failed to create dynamic table, task, or view.
|
382
488
|
SnowflakeMLException: [RuntimeError] Failed to find resources.
|
489
|
+
|
490
|
+
Example::
|
491
|
+
|
492
|
+
>>> fs = FeatureStore(...)
|
493
|
+
>>> # draft_fv is a local object that hasn't materiaized to Snowflake backend yet.
|
494
|
+
>>> feature_df = session.sql("select f_1, f_2 from source_table")
|
495
|
+
>>> draft_fv = FeatureView("my_fv", [entities], feature_df)
|
496
|
+
>>> print(draft_fv.status)
|
497
|
+
FeatureViewStatus.DRAFT
|
498
|
+
<BLANKLINE>
|
499
|
+
>>> fs.list_feature_views().select("NAME", "VERSION", "SCHEDULING_STATE").show()
|
500
|
+
-------------------------------------------
|
501
|
+
|"NAME" |"VERSION" |"SCHEDULING_STATE" |
|
502
|
+
-------------------------------------------
|
503
|
+
| | | |
|
504
|
+
-------------------------------------------
|
505
|
+
<BLANKLINE>
|
506
|
+
>>> # registered_fv is a local object that maps to a Snowflake backend object.
|
507
|
+
>>> registered_fv = fs.register_feature_view(draft_fv, "v1")
|
508
|
+
>>> print(registered_fv.status)
|
509
|
+
FeatureViewStatus.ACTIVE
|
510
|
+
<BLANKLINE>
|
511
|
+
>>> fs.list_feature_views().select("NAME", "VERSION", "SCHEDULING_STATE").show()
|
512
|
+
-------------------------------------------
|
513
|
+
|"NAME" |"VERSION" |"SCHEDULING_STATE" |
|
514
|
+
-------------------------------------------
|
515
|
+
|MY_FV |v1 |ACTIVE |
|
516
|
+
-------------------------------------------
|
517
|
+
|
383
518
|
"""
|
384
519
|
version = FeatureViewVersion(version)
|
385
520
|
|
@@ -444,7 +579,7 @@ class FeatureStore:
|
|
444
579
|
column_descs,
|
445
580
|
tagging_clause_str,
|
446
581
|
schedule_task,
|
447
|
-
self._default_warehouse,
|
582
|
+
feature_view.warehouse if feature_view.warehouse is not None else self._default_warehouse,
|
448
583
|
block,
|
449
584
|
overwrite,
|
450
585
|
)
|
@@ -473,6 +608,7 @@ class FeatureStore:
|
|
473
608
|
self,
|
474
609
|
name: str,
|
475
610
|
version: str,
|
611
|
+
*,
|
476
612
|
refresh_freq: Optional[str] = None,
|
477
613
|
warehouse: Optional[str] = None,
|
478
614
|
desc: Optional[str] = None,
|
@@ -492,27 +628,33 @@ class FeatureStore:
|
|
492
628
|
|
493
629
|
Example::
|
494
630
|
|
495
|
-
>>> fs = FeatureStore(
|
496
|
-
... ...,
|
497
|
-
... default_warehouse='ORIGINAL_WH',
|
498
|
-
... )
|
631
|
+
>>> fs = FeatureStore(...)
|
499
632
|
>>> fv = FeatureView(
|
500
633
|
... name='foo',
|
501
634
|
... entities=[e1, e2],
|
502
635
|
... feature_df=session.sql('...'),
|
503
|
-
...
|
504
|
-
... refresh_freq='1d',
|
505
|
-
... desc='this is old description'
|
636
|
+
... desc='this is old description',
|
506
637
|
... )
|
507
638
|
>>> fv = fs.register_feature_view(feature_view=fv, version='v1')
|
639
|
+
>>> fs.list_feature_views().select("name", "version", "desc").show()
|
640
|
+
------------------------------------------------
|
641
|
+
|"NAME" |"VERSION" |"DESC" |
|
642
|
+
------------------------------------------------
|
643
|
+
|FOO |v1 |this is old description |
|
644
|
+
------------------------------------------------
|
645
|
+
<BLANKLINE>
|
508
646
|
>>> # update_feature_view will apply new arguments to the registered feature view.
|
509
647
|
>>> new_fv = fs.update_feature_view(
|
510
648
|
... name='foo',
|
511
649
|
... version='v1',
|
512
|
-
... refresh_freq='2d',
|
513
|
-
... warehouse='MY_NEW_WH',
|
514
650
|
... desc='that is new descption',
|
515
651
|
... )
|
652
|
+
>>> fs.list_feature_views().select("name", "version", "desc").show()
|
653
|
+
------------------------------------------------
|
654
|
+
|"NAME" |"VERSION" |"DESC" |
|
655
|
+
------------------------------------------------
|
656
|
+
|FOO |v1 |THAT IS NEW DESCRIPTION |
|
657
|
+
------------------------------------------------
|
516
658
|
|
517
659
|
Raises:
|
518
660
|
SnowflakeMLException: [RuntimeError] If FeatureView is not managed and refresh_freq is defined.
|
@@ -555,20 +697,56 @@ class FeatureStore:
|
|
555
697
|
) from e
|
556
698
|
return self.get_feature_view(name=name, version=version)
|
557
699
|
|
558
|
-
@
|
700
|
+
@overload
|
701
|
+
def read_feature_view(self, feature_view: str, version: str) -> DataFrame:
|
702
|
+
...
|
703
|
+
|
704
|
+
@overload
|
559
705
|
def read_feature_view(self, feature_view: FeatureView) -> DataFrame:
|
706
|
+
...
|
707
|
+
|
708
|
+
@dispatch_decorator() # type: ignore[misc]
|
709
|
+
def read_feature_view(self, feature_view: Union[FeatureView, str], version: Optional[str] = None) -> DataFrame:
|
560
710
|
"""
|
561
|
-
Read FeatureView
|
711
|
+
Read values from a FeatureView.
|
562
712
|
|
563
713
|
Args:
|
564
|
-
feature_view: FeatureView to
|
714
|
+
feature_view: A FeatureView object to read from, or the name of feature view.
|
715
|
+
If name is provided then version also must be provided.
|
716
|
+
version: Optional version of feature view. Must set when argument feature_view is a str.
|
565
717
|
|
566
718
|
Returns:
|
567
719
|
Snowpark DataFrame(lazy mode) containing the FeatureView data.
|
568
720
|
|
569
721
|
Raises:
|
722
|
+
SnowflakeMLException: [ValueError] version argument is missing when argument feature_view is a str.
|
570
723
|
SnowflakeMLException: [ValueError] FeatureView is not registered.
|
724
|
+
|
725
|
+
Example::
|
726
|
+
|
727
|
+
>>> fs = FeatureStore(...)
|
728
|
+
>>> # Read from feature view name and version.
|
729
|
+
>>> fs.read_feature_view('foo', 'v1').show()
|
730
|
+
------------------------------------------
|
731
|
+
|"NAME" |"ID" |"TITLE" |"AGE" |"TS" |
|
732
|
+
------------------------------------------
|
733
|
+
|jonh |1 |boss |20 |100 |
|
734
|
+
|porter |2 |manager |30 |200 |
|
735
|
+
------------------------------------------
|
736
|
+
<BLANKLINE>
|
737
|
+
>>> # Read from feature view object.
|
738
|
+
>>> fv = fs.get_feature_view('foo', 'v1')
|
739
|
+
>>> fs.read_feature_view(fv).show()
|
740
|
+
------------------------------------------
|
741
|
+
|"NAME" |"ID" |"TITLE" |"AGE" |"TS" |
|
742
|
+
------------------------------------------
|
743
|
+
|jonh |1 |boss |20 |100 |
|
744
|
+
|porter |2 |manager |30 |200 |
|
745
|
+
------------------------------------------
|
746
|
+
|
571
747
|
"""
|
748
|
+
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
749
|
+
|
572
750
|
if feature_view.status == FeatureViewStatus.DRAFT or feature_view.version is None:
|
573
751
|
raise snowml_exceptions.SnowflakeMLException(
|
574
752
|
error_code=error_codes.NOT_FOUND,
|
@@ -580,6 +758,7 @@ class FeatureStore:
|
|
580
758
|
@dispatch_decorator()
|
581
759
|
def list_feature_views(
|
582
760
|
self,
|
761
|
+
*,
|
583
762
|
entity_name: Optional[str] = None,
|
584
763
|
feature_view_name: Optional[str] = None,
|
585
764
|
) -> DataFrame:
|
@@ -594,6 +773,24 @@ class FeatureStore:
|
|
594
773
|
|
595
774
|
Returns:
|
596
775
|
FeatureViews information as a Snowpark DataFrame.
|
776
|
+
|
777
|
+
Example::
|
778
|
+
|
779
|
+
>>> fs = FeatureStore(...)
|
780
|
+
>>> draft_fv = FeatureView(
|
781
|
+
... name='foo',
|
782
|
+
... entities=[e1, e2],
|
783
|
+
... feature_df=session.sql('...'),
|
784
|
+
... desc='this is description',
|
785
|
+
... )
|
786
|
+
>>> fs.register_feature_view(feature_view=draft_fv, version='v1')
|
787
|
+
>>> fs.list_feature_views().select("name", "version", "desc").show()
|
788
|
+
--------------------------------------------
|
789
|
+
|"NAME" |"VERSION" |"DESC" |
|
790
|
+
--------------------------------------------
|
791
|
+
|FOO |v1 |this is description |
|
792
|
+
--------------------------------------------
|
793
|
+
|
597
794
|
"""
|
598
795
|
if feature_view_name is not None:
|
599
796
|
feature_view_name = SqlIdentifier(feature_view_name)
|
@@ -622,6 +819,28 @@ class FeatureStore:
|
|
622
819
|
Raises:
|
623
820
|
SnowflakeMLException: [ValueError] FeatureView with name and version is not found,
|
624
821
|
or incurred exception when reconstructing the FeatureView object.
|
822
|
+
|
823
|
+
Example::
|
824
|
+
|
825
|
+
>>> fs = FeatureStore(...)
|
826
|
+
>>> # draft_fv is a local object that hasn't materiaized to Snowflake backend yet.
|
827
|
+
>>> draft_fv = FeatureView(
|
828
|
+
... name='foo',
|
829
|
+
... entities=[e1],
|
830
|
+
... feature_df=session.sql('...'),
|
831
|
+
... desc='this is description',
|
832
|
+
... )
|
833
|
+
>>> fs.register_feature_view(feature_view=draft_fv, version='v1')
|
834
|
+
<BLANKLINE>
|
835
|
+
>>> # fv is a local object that maps to a Snowflake backend object.
|
836
|
+
>>> fv = fs.get_feature_view('foo', 'v1')
|
837
|
+
>>> print(f"name: {fv.name}")
|
838
|
+
>>> print(f"version:{fv.version}")
|
839
|
+
>>> print(f"desc:{fv.desc}")
|
840
|
+
name: FOO
|
841
|
+
version:v1
|
842
|
+
desc:this is description
|
843
|
+
|
625
844
|
"""
|
626
845
|
name = SqlIdentifier(name)
|
627
846
|
version = FeatureViewVersion(version)
|
@@ -636,25 +855,49 @@ class FeatureStore:
|
|
636
855
|
|
637
856
|
return self._compose_feature_view(results[0][0], results[0][1], self.list_entities().collect())
|
638
857
|
|
639
|
-
@
|
858
|
+
@overload
|
640
859
|
def refresh_feature_view(self, feature_view: FeatureView) -> None:
|
860
|
+
...
|
861
|
+
|
862
|
+
@overload
|
863
|
+
def refresh_feature_view(self, feature_view: str, version: str) -> None:
|
864
|
+
...
|
865
|
+
|
866
|
+
@dispatch_decorator() # type: ignore[misc]
|
867
|
+
def refresh_feature_view(self, feature_view: Union[FeatureView, str], version: Optional[str] = None) -> None:
|
641
868
|
"""Manually refresh a feature view.
|
642
869
|
|
643
870
|
Args:
|
644
|
-
feature_view: A registered feature view.
|
871
|
+
feature_view: A registered feature view object, or the name of feature view.
|
872
|
+
version: Optional version of feature view. Must set when argument feature_view is a str.
|
645
873
|
|
646
874
|
Example::
|
647
875
|
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
876
|
+
>>> fs = FeatureStore(...)
|
877
|
+
>>> fv = fs.get_feature_view(name='MY_FV', version='v1')
|
878
|
+
<BLANKLINE>
|
879
|
+
>>> # refresh with name and version
|
880
|
+
>>> fs.refresh_feature_view('MY_FV', 'v1')
|
881
|
+
>>> fs.get_refresh_history('MY_FV', 'v1').show()
|
882
|
+
-----------------------------------------------------------------------------------------------------
|
883
|
+
|"NAME" |"STATE" |"REFRESH_START_TIME" |"REFRESH_END_TIME" |"REFRESH_ACTION" |
|
884
|
+
-----------------------------------------------------------------------------------------------------
|
885
|
+
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:53:58.504000 |2024-07-10 14:53:59.088000 |INCREMENTAL |
|
886
|
+
-----------------------------------------------------------------------------------------------------
|
887
|
+
<BLANKLINE>
|
888
|
+
>>> # refresh with feature view object
|
889
|
+
>>> fs.refresh_feature_view(fv)
|
890
|
+
>>> fs.get_refresh_history(fv).show()
|
891
|
+
-----------------------------------------------------------------------------------------------------
|
892
|
+
|"NAME" |"STATE" |"REFRESH_START_TIME" |"REFRESH_END_TIME" |"REFRESH_ACTION" |
|
893
|
+
-----------------------------------------------------------------------------------------------------
|
894
|
+
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:54:06.680000 |2024-07-10 14:54:07.226000 |INCREMENTAL |
|
895
|
+
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:53:58.504000 |2024-07-10 14:53:59.088000 |INCREMENTAL |
|
896
|
+
-----------------------------------------------------------------------------------------------------
|
897
|
+
|
657
898
|
"""
|
899
|
+
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
900
|
+
|
658
901
|
if feature_view.status == FeatureViewStatus.STATIC:
|
659
902
|
warnings.warn(
|
660
903
|
"Static feature view can't be refreshed. You must set refresh_freq when register_feature_view().",
|
@@ -664,11 +907,24 @@ class FeatureStore:
|
|
664
907
|
return
|
665
908
|
self._update_feature_view_status(feature_view, "REFRESH")
|
666
909
|
|
667
|
-
|
910
|
+
@overload
|
911
|
+
def get_refresh_history(
|
912
|
+
self, feature_view: FeatureView, version: Optional[str] = None, *, verbose: bool = False
|
913
|
+
) -> DataFrame:
|
914
|
+
...
|
915
|
+
|
916
|
+
@overload
|
917
|
+
def get_refresh_history(self, feature_view: str, version: str, *, verbose: bool = False) -> DataFrame:
|
918
|
+
...
|
919
|
+
|
920
|
+
def get_refresh_history(
|
921
|
+
self, feature_view: Union[FeatureView, str], version: Optional[str] = None, *, verbose: bool = False
|
922
|
+
) -> DataFrame:
|
668
923
|
"""Get refresh hisotry statistics about a feature view.
|
669
924
|
|
670
925
|
Args:
|
671
|
-
feature_view: A registered feature view.
|
926
|
+
feature_view: A registered feature view object, or the name of feature view.
|
927
|
+
version: Optional version of feature view. Must set when argument feature_view is a str.
|
672
928
|
verbose: Return more detailed history when set true.
|
673
929
|
|
674
930
|
Returns:
|
@@ -676,16 +932,30 @@ class FeatureStore:
|
|
676
932
|
|
677
933
|
Example::
|
678
934
|
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
935
|
+
>>> fs = FeatureStore(...)
|
936
|
+
>>> fv = fs.get_feature_view(name='MY_FV', version='v1')
|
937
|
+
>>> # refresh with name and version
|
938
|
+
>>> fs.refresh_feature_view('MY_FV', 'v1')
|
939
|
+
>>> fs.get_refresh_history('MY_FV', 'v1').show()
|
940
|
+
-----------------------------------------------------------------------------------------------------
|
941
|
+
|"NAME" |"STATE" |"REFRESH_START_TIME" |"REFRESH_END_TIME" |"REFRESH_ACTION" |
|
942
|
+
-----------------------------------------------------------------------------------------------------
|
943
|
+
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:53:58.504000 |2024-07-10 14:53:59.088000 |INCREMENTAL |
|
944
|
+
-----------------------------------------------------------------------------------------------------
|
945
|
+
<BLANKLINE>
|
946
|
+
>>> # refresh with feature view object
|
947
|
+
>>> fs.refresh_feature_view(fv)
|
948
|
+
>>> fs.get_refresh_history(fv).show()
|
949
|
+
-----------------------------------------------------------------------------------------------------
|
950
|
+
|"NAME" |"STATE" |"REFRESH_START_TIME" |"REFRESH_END_TIME" |"REFRESH_ACTION" |
|
951
|
+
-----------------------------------------------------------------------------------------------------
|
952
|
+
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:54:06.680000 |2024-07-10 14:54:07.226000 |INCREMENTAL |
|
953
|
+
|MY_FV$v1 |SUCCEEDED |2024-07-10 14:53:58.504000 |2024-07-10 14:53:59.088000 |INCREMENTAL |
|
954
|
+
-----------------------------------------------------------------------------------------------------
|
955
|
+
|
688
956
|
"""
|
957
|
+
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
958
|
+
|
689
959
|
if feature_view.status == FeatureViewStatus.STATIC:
|
690
960
|
warnings.warn(
|
691
961
|
"Static feature view never refreshes.",
|
@@ -719,43 +989,151 @@ class FeatureStore:
|
|
719
989
|
"""
|
720
990
|
)
|
721
991
|
|
722
|
-
@
|
992
|
+
@overload
|
723
993
|
def resume_feature_view(self, feature_view: FeatureView) -> FeatureView:
|
994
|
+
...
|
995
|
+
|
996
|
+
@overload
|
997
|
+
def resume_feature_view(self, feature_view: str, version: str) -> FeatureView:
|
998
|
+
...
|
999
|
+
|
1000
|
+
@dispatch_decorator() # type: ignore[misc]
|
1001
|
+
def resume_feature_view(self, feature_view: Union[FeatureView, str], version: Optional[str] = None) -> FeatureView:
|
724
1002
|
"""
|
725
1003
|
Resume a previously suspended FeatureView.
|
726
1004
|
|
727
1005
|
Args:
|
728
|
-
feature_view: FeatureView to resume.
|
1006
|
+
feature_view: FeatureView object or name to resume.
|
1007
|
+
version: Optional version of feature view. Must set when argument feature_view is a str.
|
729
1008
|
|
730
1009
|
Returns:
|
731
1010
|
A new feature view with updated status.
|
1011
|
+
|
1012
|
+
Example::
|
1013
|
+
|
1014
|
+
>>> fs = FeatureStore(...)
|
1015
|
+
>>> # you must already have feature views registered
|
1016
|
+
>>> fv = fs.get_feature_view(name='MY_FV', version='v1')
|
1017
|
+
>>> fs.suspend_feature_view('MY_FV', 'v1')
|
1018
|
+
>>> fs.list_feature_views().select("NAME", "VERSION", "SCHEDULING_STATE").show()
|
1019
|
+
-------------------------------------------
|
1020
|
+
|"NAME" |"VERSION" |"SCHEDULING_STATE" |
|
1021
|
+
-------------------------------------------
|
1022
|
+
|MY_FV |v1 |SUSPENDED |
|
1023
|
+
-------------------------------------------
|
1024
|
+
<BLANKLINE>
|
1025
|
+
>>> fs.resume_feature_view('MY_FV', 'v1')
|
1026
|
+
>>> fs.list_feature_views().select("NAME", "VERSION", "SCHEDULING_STATE").show()
|
1027
|
+
-------------------------------------------
|
1028
|
+
|"NAME" |"VERSION" |"SCHEDULING_STATE" |
|
1029
|
+
-------------------------------------------
|
1030
|
+
|MY_FV |v1 |ACTIVE |
|
1031
|
+
-------------------------------------------
|
1032
|
+
|
732
1033
|
"""
|
1034
|
+
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
733
1035
|
return self._update_feature_view_status(feature_view, "RESUME")
|
734
1036
|
|
735
|
-
@
|
1037
|
+
@overload
|
736
1038
|
def suspend_feature_view(self, feature_view: FeatureView) -> FeatureView:
|
1039
|
+
...
|
1040
|
+
|
1041
|
+
@overload
|
1042
|
+
def suspend_feature_view(self, feature_view: str, version: str) -> FeatureView:
|
1043
|
+
...
|
1044
|
+
|
1045
|
+
@dispatch_decorator() # type: ignore[misc]
|
1046
|
+
def suspend_feature_view(self, feature_view: Union[FeatureView, str], version: Optional[str] = None) -> FeatureView:
|
737
1047
|
"""
|
738
1048
|
Suspend an active FeatureView.
|
739
1049
|
|
740
1050
|
Args:
|
741
|
-
feature_view: FeatureView to suspend.
|
1051
|
+
feature_view: FeatureView object or name to suspend.
|
1052
|
+
version: Optional version of feature view. Must set when argument feature_view is a str.
|
742
1053
|
|
743
1054
|
Returns:
|
744
1055
|
A new feature view with updated status.
|
1056
|
+
|
1057
|
+
Example::
|
1058
|
+
|
1059
|
+
>>> fs = FeatureStore(...)
|
1060
|
+
>>> # assume you already have feature views registered
|
1061
|
+
>>> fv = fs.get_feature_view(name='MY_FV', version='v1')
|
1062
|
+
>>> fs.suspend_feature_view('MY_FV', 'v1')
|
1063
|
+
>>> fs.list_feature_views().select("NAME", "VERSION", "SCHEDULING_STATE").show()
|
1064
|
+
-------------------------------------------
|
1065
|
+
|"NAME" |"VERSION" |"SCHEDULING_STATE" |
|
1066
|
+
-------------------------------------------
|
1067
|
+
|MY_FV |v1 |SUSPENDED |
|
1068
|
+
-------------------------------------------
|
1069
|
+
<BLANKLINE>
|
1070
|
+
>>> fs.resume_feature_view('MY_FV', 'v1')
|
1071
|
+
>>> fs.list_feature_views().select("NAME", "VERSION", "SCHEDULING_STATE").show()
|
1072
|
+
-------------------------------------------
|
1073
|
+
|"NAME" |"VERSION" |"SCHEDULING_STATE" |
|
1074
|
+
-------------------------------------------
|
1075
|
+
|MY_FV |v1 |ACTIVE |
|
1076
|
+
-------------------------------------------
|
1077
|
+
|
745
1078
|
"""
|
1079
|
+
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
746
1080
|
return self._update_feature_view_status(feature_view, "SUSPEND")
|
747
1081
|
|
748
|
-
@
|
1082
|
+
@overload
|
749
1083
|
def delete_feature_view(self, feature_view: FeatureView) -> None:
|
1084
|
+
...
|
1085
|
+
|
1086
|
+
@overload
|
1087
|
+
def delete_feature_view(self, feature_view: str, version: str) -> None:
|
1088
|
+
...
|
1089
|
+
|
1090
|
+
@dispatch_decorator() # type: ignore[misc]
|
1091
|
+
def delete_feature_view(self, feature_view: Union[FeatureView, str], version: Optional[str] = None) -> None:
|
750
1092
|
"""
|
751
1093
|
Delete a FeatureView.
|
752
1094
|
|
753
1095
|
Args:
|
754
|
-
feature_view: FeatureView to delete.
|
1096
|
+
feature_view: FeatureView object or name to delete.
|
1097
|
+
version: Optional version of feature view. Must set when argument feature_view is a str.
|
755
1098
|
|
756
1099
|
Raises:
|
757
1100
|
SnowflakeMLException: [ValueError] FeatureView is not registered.
|
1101
|
+
|
1102
|
+
Example::
|
1103
|
+
|
1104
|
+
>>> fs = FeatureStore(...)
|
1105
|
+
>>> fv = FeatureView('FV0', ...)
|
1106
|
+
>>> fv1 = fs.register_feature_view(fv, 'FIRST')
|
1107
|
+
>>> fv2 = fs.register_feature_view(fv, 'SECOND')
|
1108
|
+
>>> fs.list_feature_views().select('NAME', 'VERSION').show()
|
1109
|
+
----------------------
|
1110
|
+
|"NAME" |"VERSION" |
|
1111
|
+
----------------------
|
1112
|
+
|FV0 |SECOND |
|
1113
|
+
|FV0 |FIRST |
|
1114
|
+
----------------------
|
1115
|
+
<BLANKLINE>
|
1116
|
+
>>> # delete with name and version
|
1117
|
+
>>> fs.delete_feature_view('FV0', 'FIRST')
|
1118
|
+
>>> fs.list_feature_views().select('NAME', 'VERSION').show()
|
1119
|
+
----------------------
|
1120
|
+
|"NAME" |"VERSION" |
|
1121
|
+
----------------------
|
1122
|
+
|FV0 |SECOND |
|
1123
|
+
----------------------
|
1124
|
+
<BLANKLINE>
|
1125
|
+
>>> # delete with feature view object
|
1126
|
+
>>> fs.delete_feature_view(fv2)
|
1127
|
+
>>> fs.list_feature_views().select('NAME', 'VERSION').show()
|
1128
|
+
----------------------
|
1129
|
+
|"NAME" |"VERSION" |
|
1130
|
+
----------------------
|
1131
|
+
| | |
|
1132
|
+
----------------------
|
1133
|
+
|
758
1134
|
"""
|
1135
|
+
feature_view = self._validate_feature_view_name_and_version_input(feature_view, version)
|
1136
|
+
|
759
1137
|
# TODO: we should leverage lineage graph to check downstream deps, and block the deletion
|
760
1138
|
# if there're other FVs depending on this
|
761
1139
|
if feature_view.status == FeatureViewStatus.DRAFT or feature_view.version is None:
|
@@ -787,6 +1165,19 @@ class FeatureStore:
|
|
787
1165
|
|
788
1166
|
Returns:
|
789
1167
|
Snowpark DataFrame containing the results.
|
1168
|
+
|
1169
|
+
Example::
|
1170
|
+
|
1171
|
+
>>> fs = FeatureStore(...)
|
1172
|
+
>>> e_1 = Entity("my_entity", ['col_1'], desc='My first entity.')
|
1173
|
+
>>> fs.register_entity(e_1)
|
1174
|
+
>>> fs.list_entities().show()
|
1175
|
+
-----------------------------------------------------------
|
1176
|
+
|"NAME" |"JOIN_KEYS" |"DESC" |"OWNER" |
|
1177
|
+
-----------------------------------------------------------
|
1178
|
+
|MY_ENTITY |["COL_1"] |My first entity. |REGTEST_RL |
|
1179
|
+
-----------------------------------------------------------
|
1180
|
+
|
790
1181
|
"""
|
791
1182
|
prefix_len = len(_ENTITY_TAG_PREFIX) + 1
|
792
1183
|
return cast(
|
@@ -816,6 +1207,19 @@ class FeatureStore:
|
|
816
1207
|
SnowflakeMLException: [ValueError] Entity is not found.
|
817
1208
|
SnowflakeMLException: [RuntimeError] Failed to retrieve tag reference information.
|
818
1209
|
SnowflakeMLException: [RuntimeError] Failed to find resources.
|
1210
|
+
|
1211
|
+
Example::
|
1212
|
+
|
1213
|
+
>>> fs = FeatureStore(...)
|
1214
|
+
>>> # e_1 is a local object that hasn't registered to Snowflake backend yet.
|
1215
|
+
>>> e_1 = Entity("my_entity", ['col_1'], desc='My first entity.')
|
1216
|
+
>>> fs.register_entity(e_1)
|
1217
|
+
<BLANKLINE>
|
1218
|
+
>>> # e_2 is a local object that points a backend object in Snowflake.
|
1219
|
+
>>> e_2 = fs.get_entity("my_entity")
|
1220
|
+
>>> print(e_2)
|
1221
|
+
Entity(name=MY_ENTITY, join_keys=['COL_1'], owner=REGTEST_RL, desc=My first entity.)
|
1222
|
+
|
819
1223
|
"""
|
820
1224
|
name = SqlIdentifier(name)
|
821
1225
|
try:
|
@@ -846,12 +1250,33 @@ class FeatureStore:
|
|
846
1250
|
Delete a previously registered Entity.
|
847
1251
|
|
848
1252
|
Args:
|
849
|
-
name:
|
1253
|
+
name: Name of entity to be deleted.
|
850
1254
|
|
851
1255
|
Raises:
|
852
1256
|
SnowflakeMLException: [ValueError] Entity with given name not exists.
|
853
1257
|
SnowflakeMLException: [RuntimeError] Failed to alter schema or drop tag.
|
854
1258
|
SnowflakeMLException: [RuntimeError] Failed to find resources.
|
1259
|
+
|
1260
|
+
Example::
|
1261
|
+
|
1262
|
+
>>> fs = FeatureStore(...)
|
1263
|
+
>>> e_1 = Entity("my_entity", ['col_1'], desc='My first entity.')
|
1264
|
+
>>> fs.register_entity(e_1)
|
1265
|
+
>>> fs.list_entities().show()
|
1266
|
+
-----------------------------------------------------------
|
1267
|
+
|"NAME" |"JOIN_KEYS" |"DESC" |"OWNER" |
|
1268
|
+
-----------------------------------------------------------
|
1269
|
+
|MY_ENTITY |["COL_1"] |My first entity. |REGTEST_RL |
|
1270
|
+
-----------------------------------------------------------
|
1271
|
+
<BLANKLINE>
|
1272
|
+
>>> fs.delete_entity("my_entity")
|
1273
|
+
>>> fs.list_entities().show()
|
1274
|
+
-------------------------------------------
|
1275
|
+
|"NAME" |"JOIN_KEYS" |"DESC" |"OWNER" |
|
1276
|
+
-------------------------------------------
|
1277
|
+
| | | | |
|
1278
|
+
-------------------------------------------
|
1279
|
+
|
855
1280
|
"""
|
856
1281
|
name = SqlIdentifier(name)
|
857
1282
|
|
@@ -885,6 +1310,7 @@ class FeatureStore:
|
|
885
1310
|
self,
|
886
1311
|
spine_df: DataFrame,
|
887
1312
|
features: Union[List[Union[FeatureView, FeatureViewSlice]], List[str]],
|
1313
|
+
*,
|
888
1314
|
spine_timestamp_col: Optional[str] = None,
|
889
1315
|
exclude_columns: Optional[List[str]] = None,
|
890
1316
|
include_feature_view_timestamp_col: bool = False,
|
@@ -907,6 +1333,23 @@ class FeatureStore:
|
|
907
1333
|
|
908
1334
|
Raises:
|
909
1335
|
ValueError: if features is empty.
|
1336
|
+
|
1337
|
+
Example::
|
1338
|
+
|
1339
|
+
>>> fs = FeatureStore(...)
|
1340
|
+
>>> # Assume you already have feature view registered.
|
1341
|
+
>>> fv = fs.get_feature_view('my_fv', 'v1')
|
1342
|
+
>>> # Spine dataframe has same join keys as the entity of fv.
|
1343
|
+
>>> spine_df = session.create_dataframe(["1", "2"], schema=["id"])
|
1344
|
+
>>> fs.retrieve_feature_values(spine_df, [fv]).show()
|
1345
|
+
--------------------
|
1346
|
+
|"END_STATION_ID" |
|
1347
|
+
--------------------
|
1348
|
+
|505 |
|
1349
|
+
|347 |
|
1350
|
+
|466 |
|
1351
|
+
--------------------
|
1352
|
+
|
910
1353
|
"""
|
911
1354
|
if spine_timestamp_col is not None:
|
912
1355
|
spine_timestamp_col = SqlIdentifier(spine_timestamp_col)
|
@@ -933,6 +1376,7 @@ class FeatureStore:
|
|
933
1376
|
self,
|
934
1377
|
spine_df: DataFrame,
|
935
1378
|
features: List[Union[FeatureView, FeatureViewSlice]],
|
1379
|
+
*,
|
936
1380
|
save_as: Optional[str] = None,
|
937
1381
|
spine_timestamp_col: Optional[str] = None,
|
938
1382
|
spine_label_cols: Optional[List[str]] = None,
|
@@ -966,8 +1410,10 @@ class FeatureStore:
|
|
966
1410
|
Example::
|
967
1411
|
|
968
1412
|
>>> fs = FeatureStore(session, ...)
|
1413
|
+
>>> # Assume you already have feature view registered.
|
969
1414
|
>>> fv = fs.get_feature_view("MY_FV", "1")
|
970
|
-
>>>
|
1415
|
+
>>> # Spine dataframe has same join keys as the entity of fv.
|
1416
|
+
>>> spine_df = session.create_dataframe(["1", "2"], schema=["id"])
|
971
1417
|
>>> training_set = fs.generate_training_set(
|
972
1418
|
... spine_df,
|
973
1419
|
... [fv],
|
@@ -975,6 +1421,7 @@ class FeatureStore:
|
|
975
1421
|
... )
|
976
1422
|
>>> print(type(training_set))
|
977
1423
|
<class 'snowflake.snowpark.table.Table'>
|
1424
|
+
<BLANKLINE>
|
978
1425
|
>>> print(training_set.queries)
|
979
1426
|
{'queries': ['SELECT * FROM (my_training_set)'], 'post_actions': []}
|
980
1427
|
|
@@ -1014,6 +1461,7 @@ class FeatureStore:
|
|
1014
1461
|
name: str,
|
1015
1462
|
spine_df: DataFrame,
|
1016
1463
|
features: List[Union[FeatureView, FeatureViewSlice]],
|
1464
|
+
*,
|
1017
1465
|
version: Optional[str] = None,
|
1018
1466
|
spine_timestamp_col: Optional[str] = None,
|
1019
1467
|
spine_label_cols: Optional[List[str]] = None,
|
@@ -1030,6 +1478,7 @@ class FeatureStore:
|
|
1030
1478
|
name: str,
|
1031
1479
|
spine_df: DataFrame,
|
1032
1480
|
features: List[Union[FeatureView, FeatureViewSlice]],
|
1481
|
+
*,
|
1033
1482
|
output_type: Literal["table"],
|
1034
1483
|
version: Optional[str] = None,
|
1035
1484
|
spine_timestamp_col: Optional[str] = None,
|
@@ -1046,6 +1495,7 @@ class FeatureStore:
|
|
1046
1495
|
name: str,
|
1047
1496
|
spine_df: DataFrame,
|
1048
1497
|
features: List[Union[FeatureView, FeatureViewSlice]],
|
1498
|
+
*,
|
1049
1499
|
version: Optional[str] = None,
|
1050
1500
|
spine_timestamp_col: Optional[str] = None,
|
1051
1501
|
spine_label_cols: Optional[List[str]] = None,
|
@@ -1082,6 +1532,33 @@ class FeatureStore:
|
|
1082
1532
|
SnowflakeMLException: [ValueError] Invalid output_type specified.
|
1083
1533
|
SnowflakeMLException: [RuntimeError] Dataset name/version already exists.
|
1084
1534
|
SnowflakeMLException: [RuntimeError] Failed to find resources.
|
1535
|
+
|
1536
|
+
Example::
|
1537
|
+
|
1538
|
+
>>> fs = FeatureStore(session, ...)
|
1539
|
+
>>> # Assume you already have feature view registered.
|
1540
|
+
>>> fv = fs.get_feature_view("MY_FV", "1")
|
1541
|
+
>>> # Spine dataframe has same join keys as the entity of fv.
|
1542
|
+
>>> spine_df = session.create_dataframe(["1", "2"], schema=["id"])
|
1543
|
+
>>> my_dataset = fs.generate_dataset(
|
1544
|
+
... "my_dataset"
|
1545
|
+
... spine_df,
|
1546
|
+
... [fv],
|
1547
|
+
... )
|
1548
|
+
>>> # Current timestamp will be used as default version name.
|
1549
|
+
>>> # You can explicitly overwrite by setting a version.
|
1550
|
+
>>> my_dataset.list_versions()
|
1551
|
+
['2024_07_12_11_26_22']
|
1552
|
+
<BLANKLINE>
|
1553
|
+
>>> my_dataset.read.to_snowpark_dataframe().show(n=3)
|
1554
|
+
-------------------------------------------------------
|
1555
|
+
|"QUALITY" |"FIXED_ACIDITY" |"VOLATILE_ACIDITY" |
|
1556
|
+
-------------------------------------------------------
|
1557
|
+
|3 |11.600000381469727 |0.5799999833106995 |
|
1558
|
+
|3 |8.300000190734863 |1.0199999809265137 |
|
1559
|
+
|3 |7.400000095367432 |1.184999942779541 |
|
1560
|
+
-------------------------------------------------------
|
1561
|
+
|
1085
1562
|
"""
|
1086
1563
|
if output_type not in {"table", "dataset"}:
|
1087
1564
|
raise snowml_exceptions.SnowflakeMLException(
|
@@ -1166,6 +1643,32 @@ class FeatureStore:
|
|
1166
1643
|
|
1167
1644
|
Raises:
|
1168
1645
|
ValueError: if dataset object is not generated from feature store.
|
1646
|
+
|
1647
|
+
Example::
|
1648
|
+
|
1649
|
+
>>> fs = FeatureStore(session, ...)
|
1650
|
+
>>> # Assume you already have feature view registered.
|
1651
|
+
>>> fv = fs.get_feature_view("MY_FV", "1.0")
|
1652
|
+
>>> # Spine dataframe has same join keys as the entity of fv.
|
1653
|
+
>>> spine_df = session.create_dataframe(["1", "2"], schema=["id"])
|
1654
|
+
>>> my_dataset = fs.generate_dataset(
|
1655
|
+
... "my_dataset"
|
1656
|
+
... spine_df,
|
1657
|
+
... [fv],
|
1658
|
+
... )
|
1659
|
+
>>> fvs = fs.load_feature_views_from_dataset(my_dataset)
|
1660
|
+
>>> print(len(fvs))
|
1661
|
+
1
|
1662
|
+
<BLANKLINE>
|
1663
|
+
>>> print(type(fvs[0]))
|
1664
|
+
<class 'snowflake.ml.feature_store.feature_view.FeatureView'>
|
1665
|
+
<BLANKLINE>
|
1666
|
+
>>> print(fvs[0].name)
|
1667
|
+
MY_FV
|
1668
|
+
<BLANKLINE>
|
1669
|
+
>>> print(fvs[0].version)
|
1670
|
+
1.0
|
1671
|
+
|
1169
1672
|
"""
|
1170
1673
|
assert ds.selected_version is not None
|
1171
1674
|
source_meta = ds.selected_version._get_metadata()
|
@@ -1203,11 +1706,11 @@ class FeatureStore:
|
|
1203
1706
|
if dryrun:
|
1204
1707
|
logger.info(
|
1205
1708
|
"Following feature views and entities will be deleted."
|
1206
|
-
+ " Set 'dryrun=False' to perform the actual deletion."
|
1709
|
+
+ " Set 'dryrun=False' to perform the actual deletion.",
|
1207
1710
|
)
|
1208
1711
|
logger.info(f"Total {len(all_fvs_rows)} Feature views to be deleted:")
|
1209
1712
|
all_fvs_df.show(n=len(all_fvs_rows))
|
1210
|
-
logger.info(f"\nTotal {len(all_entities_rows)}
|
1713
|
+
logger.info(f"\nTotal {len(all_entities_rows)} Entities to be deleted:")
|
1211
1714
|
all_entities_df.show(n=len(all_entities_rows))
|
1212
1715
|
return
|
1213
1716
|
|
@@ -1686,6 +2189,7 @@ class FeatureStore:
|
|
1686
2189
|
values.append(row["target_lag"] if "target_lag" in row else None)
|
1687
2190
|
values.append(row["refresh_mode"] if "refresh_mode" in row else None)
|
1688
2191
|
values.append(row["scheduling_state"] if "scheduling_state" in row else None)
|
2192
|
+
values.append(row["warehouse"] if "warehouse" in row else None)
|
1689
2193
|
output_values.append(values)
|
1690
2194
|
|
1691
2195
|
def _lookup_feature_view_metadata(self, row: Row, fv_name: str) -> Tuple[_FeatureViewMetadata, str]:
|
@@ -1942,6 +2446,7 @@ class FeatureStore:
|
|
1942
2446
|
SnowflakeMLException: [RuntimeError] Failed to lookup tags.
|
1943
2447
|
|
1944
2448
|
Example::
|
2449
|
+
|
1945
2450
|
self._lookup_tags("TABLE", "MY_FV", [lambda d: d["tagName"] == "TARGET_TAG_NAME"])
|
1946
2451
|
|
1947
2452
|
"""
|
@@ -1979,6 +2484,7 @@ class FeatureStore:
|
|
1979
2484
|
SnowflakeMLException: [RuntimeError] Failed to lookup tagged objects.
|
1980
2485
|
|
1981
2486
|
Example::
|
2487
|
+
|
1982
2488
|
self._lookup_tagged_objects("TARGET_TAG_NAME", [lambda d: d["entityName"] == "MY_FV"])
|
1983
2489
|
|
1984
2490
|
"""
|
@@ -2024,3 +2530,23 @@ class FeatureStore:
|
|
2024
2530
|
),
|
2025
2531
|
)
|
2026
2532
|
return sorted_versions
|
2533
|
+
|
2534
|
+
def _validate_feature_view_name_and_version_input(
|
2535
|
+
self, feature_view: Union[FeatureView, str], version: Optional[str] = None
|
2536
|
+
) -> FeatureView:
|
2537
|
+
if isinstance(feature_view, str):
|
2538
|
+
if version is None:
|
2539
|
+
raise snowml_exceptions.SnowflakeMLException(
|
2540
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
2541
|
+
original_exception=ValueError("Version must be provided when argument feature_view is a str."),
|
2542
|
+
)
|
2543
|
+
feature_view = self.get_feature_view(feature_view, version)
|
2544
|
+
elif not isinstance(feature_view, FeatureView):
|
2545
|
+
raise snowml_exceptions.SnowflakeMLException(
|
2546
|
+
error_code=error_codes.INVALID_ARGUMENT,
|
2547
|
+
original_exception=ValueError(
|
2548
|
+
"Invalid type of argument feature_view. It must be either str or FeatureView type."
|
2549
|
+
),
|
2550
|
+
)
|
2551
|
+
|
2552
|
+
return feature_view
|