mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0__py3-none-any.whl

mlrun/feature_store/api.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import copy
 import importlib.util
 import pathlib
@@ -19,7 +20,6 @@ from datetime import datetime
 from typing import Any, Optional, Union
 
 import pandas as pd
-from deprecated import deprecated
 
 import mlrun
 import mlrun.errors
@@ -47,362 +47,20 @@ from .common import (
     get_feature_vector_by_uri,
     verify_feature_set_exists,
     verify_feature_set_permissions,
-    verify_feature_vector_permissions,
 )
 from .feature_set import FeatureSet
-from .feature_vector import (
-    FeatureVector,
-    FixedWindowType,
-    OfflineVectorResponse,
-    OnlineVectorService,
-)
 from .ingestion import (
     context_to_ingestion_params,
     init_featureset_graph,
     run_ingestion_job,
     run_spark_graph,
 )
-from .retrieval import RemoteVectorResponse, get_merger, run_merge_job
 
 _v3iofs = None
 spark_transform_handler = "transform"
 _TRANS_TABLE = str.maketrans({" ": "_", "(": "", ")": ""})
 
 
-def _features_to_vector_and_check_permissions(features, update_stats):
-    if isinstance(features, str):
-        vector = get_feature_vector_by_uri(features, update=update_stats)
-    elif isinstance(features, FeatureVector):
-        vector = features
-        if not vector.metadata.name:
-            raise mlrun.errors.MLRunInvalidArgumentError(
-                "feature vector name must be specified"
-            )
-        verify_feature_vector_permissions(
-            vector, mlrun.common.schemas.AuthorizationAction.update
-        )
-
-        vector.save()
-    else:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"illegal features value/type ({type(features)})"
-        )
-    return vector
-
-
-@deprecated(
-    version="1.6.0",
-    reason="get_offline_features() will be removed in 1.8.0, please instead use "
-    "get_feature_vector('store://feature_vector_name').get_offline_features()",
-    category=FutureWarning,
-)
-def get_offline_features(
-    feature_vector: Union[str, FeatureVector],
-    entity_rows=None,
-    entity_timestamp_column: str = None,
-    target: DataTargetBase = None,
-    run_config: RunConfig = None,
-    drop_columns: list[str] = None,
-    start_time: Union[str, datetime] = None,
-    end_time: Union[str, datetime] = None,
-    with_indexes: bool = False,
-    update_stats: bool = False,
-    engine: str = None,
-    engine_args: dict = None,
-    query: str = None,
-    order_by: Union[str, list[str]] = None,
-    spark_service: str = None,
-    timestamp_for_filtering: Union[str, dict[str, str]] = None,
-    additional_filters: list = None,
-):
-    """retrieve offline feature vector results
-
-    specify a feature vector object/uri and retrieve the desired features, their metadata
-    and statistics. returns :py:class:`~mlrun.feature_store.OfflineVectorResponse`,
-    results can be returned as a dataframe or written to a target
-
-    The start_time and end_time attributes allow filtering the data to a given time range, they accept
-    string values or pandas `Timestamp` objects, string values can also be relative, for example:
-    "now", "now - 1d2h", "now+5m", where a valid pandas Timedelta string follows the verb "now",
-    for time alignment you can use the verb "floor" e.g. "now -1d floor 1H" will align the time to the last hour
-    (the floor string is passed to pandas.Timestamp.floor(), can use D, H, T, S for day, hour, min, sec alignment).
-    Another option to filter the data is by the `query` argument - can be seen in the example.
-    example::
-
-        features = [
-            "stock-quotes.bid",
-            "stock-quotes.asks_sum_5h",
-            "stock-quotes.ask as mycol",
-            "stocks.*",
-        ]
-        vector = FeatureVector(features=features)
-        resp = get_offline_features(
-            vector,
-            entity_rows=trades,
-            entity_timestamp_column="time",
-            query="ticker in ['GOOG'] and bid>100",
-        )
-        print(resp.to_dataframe())
-        print(vector.get_stats_table())
-        resp.to_parquet("./out.parquet")
-
-    :param feature_vector:          feature vector uri or FeatureVector object. passing feature vector obj requires
-                                    update permissions
-    :param entity_rows:             dataframe with entity rows to join with
-    :param target:                  where to write the results to
-    :param drop_columns:            list of columns to drop from the final result
-    :param entity_timestamp_column: timestamp column name in the entity rows dataframe. can be specified
-                                    only if param entity_rows was specified.
-    :param run_config:              function and/or run configuration
-                                    see :py:class:`~mlrun.feature_store.RunConfig`
-    :param start_time:              datetime, low limit of time needed to be filtered. Optional.
-    :param end_time:                datetime, high limit of time needed to be filtered. Optional.
-    :param with_indexes:            Return vector with/without the entities and the timestamp_key of the feature sets
-                                    and with/without entity_timestamp_column and timestamp_for_filtering columns.
-                                    This property can be specified also in the feature vector spec
-                                    (feature_vector.spec.with_indexes)
-                                    (default False)
-    :param update_stats:            update features statistics from the requested feature sets on the vector.
-                                    (default False).
-    :param engine:                  processing engine kind ("local", "dask", or "spark")
-    :param engine_args:             kwargs for the processing engine
-    :param query:                   The query string used to filter rows on the output
-    :param spark_service:           Name of the spark service to be used (when using a remote-spark runtime)
-    :param order_by:                Name or list of names to order by. The name or the names in the list can be the
-                                    feature name or the alias of the feature you pass in the feature list.
-    :param timestamp_for_filtering: name of the column to filter by, can be str for all the feature sets or a
-                                    dictionary ({<feature set name>: <timestamp column name>, ...})
-                                    that indicates the timestamp column name for each feature set. Optional.
-                                    By default, the filter executes on the timestamp_key of each feature set.
-                                    Note: the time filtering is performed on each feature set before the
-                                    merge process using start_time and end_time params.
-    :param additional_filters: List of additional_filter conditions as tuples.
-                                Each tuple should be in the format (column_name, operator, value).
-                                Supported operators: "=", ">=", "<=", ">", "<".
-                                Example: [("Product", "=", "Computer")]
-                                For all supported filters, please see:
-                                https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetDataset.html
-
-
-    """
-    return _get_offline_features(
-        feature_vector,
-        entity_rows,
-        entity_timestamp_column,
-        target,
-        run_config,
-        drop_columns,
-        start_time,
-        end_time,
-        with_indexes,
-        update_stats,
-        engine,
-        engine_args,
-        query,
-        order_by,
-        spark_service,
-        timestamp_for_filtering,
-        additional_filters,
-    )
-
-
-def _get_offline_features(
-    feature_vector: Union[str, FeatureVector],
-    entity_rows=None,
-    entity_timestamp_column: str = None,
-    target: DataTargetBase = None,
-    run_config: RunConfig = None,
-    drop_columns: list[str] = None,
-    start_time: Union[str, datetime] = None,
-    end_time: Union[str, datetime] = None,
-    with_indexes: bool = False,
-    update_stats: bool = False,
-    engine: str = None,
-    engine_args: dict = None,
-    query: str = None,
-    order_by: Union[str, list[str]] = None,
-    spark_service: str = None,
-    timestamp_for_filtering: Union[str, dict[str, str]] = None,
-    additional_filters=None,
-) -> Union[OfflineVectorResponse, RemoteVectorResponse]:
-    if entity_rows is None and entity_timestamp_column is not None:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            "entity_timestamp_column param "
-            "can not be specified without entity_rows param"
-        )
-    if isinstance(target, BaseStoreTarget) and not target.support_pandas:
-        raise mlrun.errors.MLRunInvalidArgumentError(
-            f"get_offline_features does not support targets that do not support pandas engine."
-            f" Target kind: {target.kind}"
-        )
-
-    if isinstance(feature_vector, FeatureVector):
-        update_stats = True
-
-    feature_vector = _features_to_vector_and_check_permissions(
-        feature_vector, update_stats
-    )
-
-    entity_timestamp_column = (
-        entity_timestamp_column or feature_vector.spec.timestamp_field
-    )
-
-    merger_engine = get_merger(engine)
-
-    if run_config and not run_config.local:
-        return run_merge_job(
-            feature_vector,
-            target,
-            merger_engine,
-            engine,
-            engine_args,
-            spark_service,
-            entity_rows,
-            entity_timestamp_column=entity_timestamp_column,
-            run_config=run_config,
-            drop_columns=drop_columns,
-            with_indexes=with_indexes,
-            query=query,
-            order_by=order_by,
-            start_time=start_time,
-            end_time=end_time,
-            timestamp_for_filtering=timestamp_for_filtering,
-            additional_filters=additional_filters,
-        )
-
-    merger = merger_engine(feature_vector, **(engine_args or {}))
-    return merger.start(
-        entity_rows,
-        entity_timestamp_column,
-        target=target,
-        drop_columns=drop_columns,
-        start_time=start_time,
-        end_time=end_time,
-        timestamp_for_filtering=timestamp_for_filtering,
-        with_indexes=with_indexes,
-        update_stats=update_stats,
-        query=query,
-        order_by=order_by,
-        additional_filters=additional_filters,
-    )
-
-
-@deprecated(
-    version="1.6.0",
-    reason="get_online_feature_service() will be removed in 1.8.0, please instead use "
-    "get_feature_vector('store://feature_vector_name').get_online_feature_service()",
-    category=FutureWarning,
-)
-def get_online_feature_service(
-    feature_vector: Union[str, FeatureVector],
-    run_config: RunConfig = None,
-    fixed_window_type: FixedWindowType = FixedWindowType.LastClosedWindow,
-    impute_policy: dict = None,
-    update_stats: bool = False,
-    entity_keys: list[str] = None,
-):
-    """initialize and return online feature vector service api,
-    returns :py:class:`~mlrun.feature_store.OnlineVectorService`
-
-    :**usage**:
-        There are two ways to use the function:
-
-        1. As context manager
-
-            Example::
-
-                with get_online_feature_service(vector_uri) as svc:
-                    resp = svc.get([{"ticker": "GOOG"}, {"ticker": "MSFT"}])
-                    print(resp)
-                    resp = svc.get([{"ticker": "AAPL"}], as_list=True)
-                    print(resp)
-
-            Example with imputing::
-
-                with get_online_feature_service(vector_uri, entity_keys=['id'],
-                                                impute_policy={"*": "$mean", "amount": 0)) as svc:
-                    resp = svc.get([{"id": "C123487"}])
-
-        2. as simple function, note that in that option you need to close the session.
-
-            Example::
-
-                svc = get_online_feature_service(vector_uri, entity_keys=["ticker"])
-                try:
-                    resp = svc.get([{"ticker": "GOOG"}, {"ticker": "MSFT"}])
-                    print(resp)
-                    resp = svc.get([{"ticker": "AAPL"}], as_list=True)
-                    print(resp)
-
-                finally:
-                    svc.close()
-
-            Example with imputing::
-
-                svc = get_online_feature_service(vector_uri, entity_keys=['id'],
-                                                 impute_policy={"*": "$mean", "amount": 0))
-                try:
-                    resp = svc.get([{"id": "C123487"}])
-                except Exception as e:
-                    handling exception...
-                finally:
-                    svc.close()
-
-    :param feature_vector:      feature vector uri or FeatureVector object. passing feature vector obj requires update
-                                permissions.
-    :param run_config:          function and/or run configuration for remote jobs/services
-    :param impute_policy:       a dict with `impute_policy` per feature, the dict key is the feature name and the dict
-                                value indicate which value will be used in case the feature is NaN/empty, the replaced
-                                value can be fixed number for constants or $mean, $max, $min, $std, $count
-                                for statistical
-                                values. "*" is used to specify the default for all features, example: `{"*": "$mean"}`
-    :param fixed_window_type:   determines how to query the fixed window values which were previously inserted by ingest
-    :param update_stats:        update features statistics from the requested feature sets on the vector.
-                                Default: False.
-    :param entity_keys:         Entity list of the first feature_set in the vector.
-                                The indexes that are used to query the online service.
-    :return:                    Initialize the `OnlineVectorService`.
-                                Will be used in subclasses where `support_online=True`.
-    """
-    return _get_online_feature_service(
-        feature_vector,
-        run_config,
-        fixed_window_type,
-        impute_policy,
-        update_stats,
-        entity_keys,
-    )
-
-
-def _get_online_feature_service(
-    feature_vector: Union[str, FeatureVector],
-    run_config: RunConfig = None,
-    fixed_window_type: FixedWindowType = FixedWindowType.LastClosedWindow,
-    impute_policy: dict = None,
-    update_stats: bool = False,
-    entity_keys: list[str] = None,
-) -> OnlineVectorService:
-    if isinstance(feature_vector, FeatureVector):
-        update_stats = True
-    feature_vector = _features_to_vector_and_check_permissions(
-        feature_vector, update_stats
-    )
-
-    # Impute policies rely on statistics in many cases, so verifying that the fvec has stats in it
-    if impute_policy and not feature_vector.status.stats:
-        update_stats = True
-
-    engine_args = {"impute_policy": impute_policy}
-    merger_engine = get_merger("storey")
-    # todo: support remote service (using remote nuclio/mlrun function if run_config)
-
-    merger = merger_engine(feature_vector, **engine_args)
-
-    return merger.init_online_vector_service(
-        entity_keys, fixed_window_type, update_stats=update_stats
-    )
-
-
 def norm_column_name(name: str) -> str:
     """
     Remove parentheses () and replace whitespaces with an underscore _.
@@ -448,14 +106,14 @@ def _get_namespace(run_config: RunConfig) -> dict[str, Any]:
 
 
 def ingest(
+    mlrun_context: Union["mlrun.MLrunProject", "mlrun.MLClientCtx"],
     featureset: Union[FeatureSet, str] = None,
     source=None,
-    targets: list[DataTargetBase] = None,
+    targets: Optional[list[DataTargetBase]] = None,
     namespace=None,
     return_df: bool = True,
     infer_options: InferOptions = InferOptions.default(),
     run_config: RunConfig = None,
-    mlrun_context=None,
     spark_context=None,
     overwrite=None,
 ) -> Optional[pd.DataFrame]:
@@ -484,6 +142,7 @@ def ingest(
         targets = [CSVTarget("mycsv", path="./mycsv.csv")]
         ingest(measurements, source, targets)
 
+    :param mlrun_context: mlrun context
     :param featureset:    feature set object or featureset.uri. (uri must be of a feature set that is in the DB,
                           call `.save()` if it's not)
     :param source:        source dataframe or other sources (e.g. parquet source see:
@@ -496,7 +155,6 @@ def ingest(
                           histogram and preview infer options (:py:class:`~mlrun.feature_store.InferOptions`)
     :param run_config:    function and/or run configuration for remote jobs,
                           see :py:class:`~mlrun.feature_store.RunConfig`
-    :param mlrun_context: mlrun context (when running as a job), for internal use !
     :param spark_context: local spark session for spark ingestion, example for creating the spark context:
                           `spark = SparkSession.builder.appName("Spark function").getOrCreate()`
                           For remote spark ingestion, this should contain the remote spark service name
@@ -505,12 +163,9 @@ def ingest(
                           False for scheduled ingest - does not delete the target)
     :return:              if return_df is True, a dataframe will be returned based on the graph
     """
-    if mlrun_context is None:
-        deprecated(
-            version="1.6.0",
-            reason="Calling 'ingest' with mlrun_context=None is deprecated and will be removed in 1.8.0,\
-            use 'FeatureSet.ingest()' instead",
-            category=FutureWarning,
+    if not mlrun_context:
+        raise mlrun.errors.MLRunValueError(
+            "mlrun_context must be defined when calling ingest()"
         )
 
     return _ingest(
@@ -530,7 +185,7 @@ def ingest(
 def _ingest(
     featureset: Union[FeatureSet, str] = None,
     source=None,
-    targets: list[DataTargetBase] = None,
+    targets: Optional[list[DataTargetBase]] = None,
     namespace=None,
     return_df: bool = True,
     infer_options: InferOptions = InferOptions.default(),
@@ -775,61 +430,14 @@ def _ingest(
         return df
 
 
-@deprecated(
-    version="1.6.0",
-    reason="'preview' will be removed in 1.8.0, use 'FeatureSet.preview()' instead",
-    category=FutureWarning,
-)
-def preview(
-    featureset: FeatureSet,
-    source,
-    entity_columns: list = None,
-    namespace=None,
-    options: InferOptions = None,
-    verbose: bool = False,
-    sample_size: int = None,
-) -> pd.DataFrame:
-    """run the ingestion pipeline with local DataFrame/file data and infer features schema and stats
-
-    example::
-
-        quotes_set = FeatureSet("stock-quotes", entities=[Entity("ticker")])
-        quotes_set.add_aggregation("ask", ["sum", "max"], ["1h", "5h"], "10m")
-        quotes_set.add_aggregation("bid", ["min", "max"], ["1h"], "10m")
-        df = preview(
-            quotes_set,
-            quotes_df,
-            entity_columns=["ticker"],
-        )
-
-    :param featureset:     feature set object or uri
-    :param source:         source dataframe or csv/parquet file path
-    :param entity_columns: list of entity (index) column names
-    :param namespace:      namespace or module containing graph classes
-    :param options:        schema (for discovery of entities, features in featureset), index, stats,
-                           histogram and preview infer options (:py:class:`~mlrun.feature_store.InferOptions`)
-    :param verbose:        verbose log
-    :param sample_size:    num of rows to sample from the dataset (for large datasets)
-    """
-    return _preview(
-        featureset,
-        source,
-        entity_columns,
-        namespace,
-        options,
-        verbose,
-        sample_size,
-    )
-
-
 def _preview(
     featureset: FeatureSet,
     source,
-    entity_columns: list = None,
+    entity_columns: Optional[list] = None,
     namespace=None,
     options: InferOptions = None,
     verbose: bool = False,
-    sample_size: int = None,
+    sample_size: Optional[int] = None,
 ) -> pd.DataFrame:
     if isinstance(source, pd.DataFrame):
         source = _rename_source_dataframe_columns(source)
@@ -895,8 +503,8 @@ def _preview(
 def _run_ingestion_job(
     featureset: Union[FeatureSet, str],
     source: DataSource = None,
-    targets: list[DataTargetBase] = None,
-    name: str = None,
+    targets: Optional[list[DataTargetBase]] = None,
+    name: Optional[str] = None,
     infer_options: InferOptions = InferOptions.default(),
     run_config: RunConfig = None,
 ):
@@ -911,60 +519,11 @@ def _run_ingestion_job(
     return run_ingestion_job(name, featureset, run_config, source.schedule)
 
 
-@deprecated(
-    version="1.6.0",
-    reason="'deploy_ingestion_service_v2' will be removed in 1.8.0, "
-    "use 'FeatureSet.deploy_ingestion_service()' instead",
-    category=FutureWarning,
-)
-def deploy_ingestion_service_v2(
-    featureset: Union[FeatureSet, str],
-    source: DataSource = None,
-    targets: list[DataTargetBase] = None,
-    name: str = None,
-    run_config: RunConfig = None,
-    verbose=False,
-) -> tuple[str, BaseRuntime]:
-    """Start real-time ingestion service using nuclio function
-
-    Deploy a real-time function implementing feature ingestion pipeline
-    the source maps to Nuclio event triggers (http, kafka, v3io stream, etc.)
-
-    the `run_config` parameter allow specifying the function and job configuration,
-    see: :py:class:`~mlrun.feature_store.RunConfig`
-
-    example::
-
-        source = HTTPSource()
-        func = mlrun.code_to_function("ingest", kind="serving").apply(mount_v3io())
-        config = RunConfig(function=func)
-        deploy_ingestion_service_v2(my_set, source, run_config=config)
-
-    :param featureset:    feature set object or uri
-    :param source:        data source object describing the online or offline source
-    :param targets:       list of data target objects
-    :param name:          name for the job/function
-    :param run_config:    service runtime configuration (function object/uri, resources, etc..)
-    :param verbose:       verbose log
-
-    :return: URL to access the deployed ingestion service, and the function that was deployed (which will
-             differ from the function passed in via the run_config parameter).
-    """
-    return _deploy_ingestion_service_v2(
-        featureset,
-        source,
-        targets,
-        name,
-        run_config,
-        verbose,
-    )
-
-
 def _deploy_ingestion_service_v2(
     featureset: Union[FeatureSet, str],
     source: DataSource = None,
-    targets: list[DataTargetBase] = None,
-    name: str = None,
+    targets: Optional[list[DataTargetBase]] = None,
+    name: Optional[str] = None,
     run_config: RunConfig = None,
     verbose=False,
 ) -> tuple[str, BaseRuntime]:
@@ -1010,6 +569,9 @@ def _deploy_ingestion_service_v2(
     function.metadata.name = function.metadata.name or name
 
     function.spec.graph = featureset.spec.graph
+    function.spec.graph.engine = (
+        "async" if featureset.spec.engine == "storey" else "sync"
+    )
     function.spec.parameters = run_config.parameters
     function.spec.graph_initializer = (
         "mlrun.feature_store.ingestion.featureset_initializer"
@@ -1026,7 +588,7 @@ def _ingest_with_spark(
     spark=None,
     featureset: Union[FeatureSet, str] = None,
     source: BaseSourceDriver = None,
-    targets: list[BaseStoreTarget] = None,
+    targets: Optional[list[BaseStoreTarget]] = None,
     infer_options: InferOptions = InferOptions.default(),
     mlrun_context=None,
     namespace=None,
@@ -1199,8 +761,8 @@ def _infer_from_static_df(
 def set_task_params(
     featureset: FeatureSet,
     source: DataSource = None,
-    targets: list[DataTargetBase] = None,
-    parameters: dict = None,
+    targets: Optional[list[DataTargetBase]] = None,
+    parameters: Optional[dict] = None,
     infer_options: InferOptions = InferOptions.Null,
     overwrite=None,
 ):

mlrun/feature_store/common.py

@@ -178,17 +178,17 @@ class RunConfig:
     def __init__(
         self,
         function: typing.Union[str, FunctionReference, BaseRuntime] = None,
-        local: bool = None,
-        image: str = None,
-        kind: str = None,
-        handler: str = None,
-        parameters: dict = None,
-        watch: bool = None,
+        local: typing.Optional[bool] = None,
+        image: typing.Optional[str] = None,
+        kind: typing.Optional[str] = None,
+        handler: typing.Optional[str] = None,
+        parameters: typing.Optional[dict] = None,
+        watch: typing.Optional[bool] = None,
         owner=None,
         credentials: typing.Optional[mlrun.model.Credentials] = None,
-        code: str = None,
-        requirements: typing.Union[str, list[str]] = None,
-        extra_spec: dict = None,
+        code: typing.Optional[str] = None,
+        requirements: typing.Optional[typing.Union[str, list[str]]] = None,
+        extra_spec: typing.Optional[dict] = None,
         auth_info=None,
     ):
         """class for holding function and run specs for jobs and serving functions