mlrun 1.6.0rc26__py3-none-any.whl → 1.6.3rc1__py3-none-any.whl

mlrun/feature_store/api.py

@@ -93,7 +93,8 @@ def _features_to_vector_and_check_permissions(features, update_stats):
 
 @deprecated(
     version="1.6.0",
-    reason="'get_offline_features' will be removed in 1.8.0, use 'FeatureVector.get_offline_features()' instead",
+    reason="get_offline_features() will be removed in 1.8.0, please instead use "
+    "get_feature_vector('store://feature_vector_name').get_offline_features()",
     category=FutureWarning,
 )
 def get_offline_features(
@@ -114,44 +115,6 @@ def get_offline_features(
     spark_service: str = None,
     timestamp_for_filtering: Union[str, Dict[str, str]] = None,
 ):
-    return _get_offline_features(
-        feature_vector,
-        entity_rows,
-        entity_timestamp_column,
-        target,
-        run_config,
-        drop_columns,
-        start_time,
-        end_time,
-        with_indexes,
-        update_stats,
-        engine,
-        engine_args,
-        query,
-        order_by,
-        spark_service,
-        timestamp_for_filtering,
-    )
-
-
-def _get_offline_features(
-    feature_vector: Union[str, FeatureVector],
-    entity_rows=None,
-    entity_timestamp_column: str = None,
-    target: DataTargetBase = None,
-    run_config: RunConfig = None,
-    drop_columns: List[str] = None,
-    start_time: Union[str, datetime] = None,
-    end_time: Union[str, datetime] = None,
-    with_indexes: bool = False,
-    update_stats: bool = False,
-    engine: str = None,
-    engine_args: dict = None,
-    query: str = None,
-    order_by: Union[str, List[str]] = None,
-    spark_service: str = None,
-    timestamp_for_filtering: Union[str, Dict[str, str]] = None,
-) -> Union[OfflineVectorResponse, RemoteVectorResponse]:
     """retrieve offline feature vector results
 
     specify a feature vector object/uri and retrieve the desired features, their metadata
@@ -212,6 +175,44 @@ def _get_offline_features(
                                     merge process using start_time and end_time params.
 
     """
+    return _get_offline_features(
+        feature_vector,
+        entity_rows,
+        entity_timestamp_column,
+        target,
+        run_config,
+        drop_columns,
+        start_time,
+        end_time,
+        with_indexes,
+        update_stats,
+        engine,
+        engine_args,
+        query,
+        order_by,
+        spark_service,
+        timestamp_for_filtering,
+    )
+
+
+def _get_offline_features(
+    feature_vector: Union[str, FeatureVector],
+    entity_rows=None,
+    entity_timestamp_column: str = None,
+    target: DataTargetBase = None,
+    run_config: RunConfig = None,
+    drop_columns: List[str] = None,
+    start_time: Union[str, datetime] = None,
+    end_time: Union[str, datetime] = None,
+    with_indexes: bool = False,
+    update_stats: bool = False,
+    engine: str = None,
+    engine_args: dict = None,
+    query: str = None,
+    order_by: Union[str, List[str]] = None,
+    spark_service: str = None,
+    timestamp_for_filtering: Union[str, Dict[str, str]] = None,
+) -> Union[OfflineVectorResponse, RemoteVectorResponse]:
     if entity_rows is None and entity_timestamp_column is not None:
         raise mlrun.errors.MLRunInvalidArgumentError(
             "entity_timestamp_column param "
@@ -269,8 +270,8 @@ def _get_offline_features(
 
 @deprecated(
     version="1.6.0",
-    reason="'get_online_feature_service' will be removed in 1.8.0, "
-    "use 'FeatureVector.get_online_feature_service()' instead",
+    reason="get_online_feature_service() will be removed in 1.8.0, please instead use "
+    "get_feature_vector('store://feature_vector_name').get_online_feature_service()",
     category=FutureWarning,
 )
 def get_online_feature_service(
@@ -281,24 +282,6 @@ def get_online_feature_service(
     update_stats: bool = False,
     entity_keys: List[str] = None,
 ):
-    return _get_online_feature_service(
-        feature_vector,
-        run_config,
-        fixed_window_type,
-        impute_policy,
-        update_stats,
-        entity_keys,
-    )
-
-
-def _get_online_feature_service(
-    feature_vector: Union[str, FeatureVector],
-    run_config: RunConfig = None,
-    fixed_window_type: FixedWindowType = FixedWindowType.LastClosedWindow,
-    impute_policy: dict = None,
-    update_stats: bool = False,
-    entity_keys: List[str] = None,
-) -> OnlineVectorService:
     """initialize and return online feature vector service api,
     returns :py:class:`~mlrun.feature_store.OnlineVectorService`
 
@@ -362,6 +345,24 @@ def _get_online_feature_service(
     :return:                    Initialize the `OnlineVectorService`.
                                 Will be used in subclasses where `support_online=True`.
     """
+    return _get_online_feature_service(
+        feature_vector,
+        run_config,
+        fixed_window_type,
+        impute_policy,
+        update_stats,
+        entity_keys,
+    )
+
+
+def _get_online_feature_service(
+    feature_vector: Union[str, FeatureVector],
+    run_config: RunConfig = None,
+    fixed_window_type: FixedWindowType = FixedWindowType.LastClosedWindow,
+    impute_policy: dict = None,
+    update_stats: bool = False,
+    entity_keys: List[str] = None,
+) -> OnlineVectorService:
     if isinstance(feature_vector, FeatureVector):
         update_stats = True
     feature_vector = _features_to_vector_and_check_permissions(
@@ -438,40 +439,6 @@ def ingest(
     mlrun_context=None,
     spark_context=None,
     overwrite=None,
-) -> Optional[pd.DataFrame]:
-    if mlrun_context is None:
-        deprecated(
-            version="1.6.0",
-            reason="Calling 'ingest' with mlrun_context=None is deprecated and will be removed in 1.8.0,\
-            use 'FeatureSet.ingest()' instead",
-            category=FutureWarning,
-        )
-
-    return _ingest(
-        featureset,
-        source,
-        targets,
-        namespace,
-        return_df,
-        infer_options,
-        run_config,
-        mlrun_context,
-        spark_context,
-        overwrite,
-    )
-
-
-def _ingest(
-    featureset: Union[FeatureSet, str] = None,
-    source=None,
-    targets: List[DataTargetBase] = None,
-    namespace=None,
-    return_df: bool = True,
-    infer_options: InferOptions = InferOptions.default(),
-    run_config: RunConfig = None,
-    mlrun_context=None,
-    spark_context=None,
-    overwrite=None,
 ) -> Optional[pd.DataFrame]:
     """Read local DataFrame, file, URL, or source into the feature store
     Ingest reads from the source, run the graph transformations, infers  metadata and stats
@@ -519,6 +486,40 @@ def _ingest(
                           False for scheduled ingest - does not delete the target)
     :return:              if return_df is True, a dataframe will be returned based on the graph
     """
+    if mlrun_context is None:
+        deprecated(
+            version="1.6.0",
+            reason="Calling 'ingest' with mlrun_context=None is deprecated and will be removed in 1.8.0,\
+            use 'FeatureSet.ingest()' instead",
+            category=FutureWarning,
+        )
+
+    return _ingest(
+        featureset,
+        source,
+        targets,
+        namespace,
+        return_df,
+        infer_options,
+        run_config,
+        mlrun_context,
+        spark_context,
+        overwrite,
+    )
+
+
+def _ingest(
+    featureset: Union[FeatureSet, str] = None,
+    source=None,
+    targets: List[DataTargetBase] = None,
+    namespace=None,
+    return_df: bool = True,
+    infer_options: InferOptions = InferOptions.default(),
+    run_config: RunConfig = None,
+    mlrun_context=None,
+    spark_context=None,
+    overwrite=None,
+) -> Optional[pd.DataFrame]:
     if isinstance(source, pd.DataFrame):
         source = _rename_source_dataframe_columns(source)
 
@@ -768,26 +769,6 @@ def preview(
     options: InferOptions = None,
     verbose: bool = False,
     sample_size: int = None,
-) -> pd.DataFrame:
-    return _preview(
-        featureset,
-        source,
-        entity_columns,
-        namespace,
-        options,
-        verbose,
-        sample_size,
-    )
-
-
-def _preview(
-    featureset: FeatureSet,
-    source,
-    entity_columns: list = None,
-    namespace=None,
-    options: InferOptions = None,
-    verbose: bool = False,
-    sample_size: int = None,
 ) -> pd.DataFrame:
     """run the ingestion pipeline with local DataFrame/file data and infer features schema and stats
 
@@ -811,6 +792,26 @@ def _preview(
     :param verbose:        verbose log
     :param sample_size:    num of rows to sample from the dataset (for large datasets)
     """
+    return _preview(
+        featureset,
+        source,
+        entity_columns,
+        namespace,
+        options,
+        verbose,
+        sample_size,
+    )
+
+
+def _preview(
+    featureset: FeatureSet,
+    source,
+    entity_columns: list = None,
+    namespace=None,
+    options: InferOptions = None,
+    verbose: bool = False,
+    sample_size: int = None,
+) -> pd.DataFrame:
     if isinstance(source, pd.DataFrame):
         source = _rename_source_dataframe_columns(source)
 
@@ -904,24 +905,6 @@ def deploy_ingestion_service_v2(
     name: str = None,
     run_config: RunConfig = None,
     verbose=False,
-) -> typing.Tuple[str, BaseRuntime]:
-    return _deploy_ingestion_service_v2(
-        featureset,
-        source,
-        targets,
-        name,
-        run_config,
-        verbose,
-    )
-
-
-def _deploy_ingestion_service_v2(
-    featureset: Union[FeatureSet, str],
-    source: DataSource = None,
-    targets: List[DataTargetBase] = None,
-    name: str = None,
-    run_config: RunConfig = None,
-    verbose=False,
 ) -> typing.Tuple[str, BaseRuntime]:
     """Start real-time ingestion service using nuclio function
 
@@ -936,7 +919,7 @@ def _deploy_ingestion_service_v2(
         source = HTTPSource()
         func = mlrun.code_to_function("ingest", kind="serving").apply(mount_v3io())
         config = RunConfig(function=func)
-        my_set.deploy_ingestion_service(source, run_config=config)
+        deploy_ingestion_service_v2(my_set, source, run_config=config)
 
     :param featureset:    feature set object or uri
     :param source:        data source object describing the online or offline source
@@ -948,6 +931,24 @@ def _deploy_ingestion_service_v2(
     :return: URL to access the deployed ingestion service, and the function that was deployed (which will
              differ from the function passed in via the run_config parameter).
     """
+    return _deploy_ingestion_service_v2(
+        featureset,
+        source,
+        targets,
+        name,
+        run_config,
+        verbose,
+    )
+
+
+def _deploy_ingestion_service_v2(
+    featureset: Union[FeatureSet, str],
+    source: DataSource = None,
+    targets: List[DataTargetBase] = None,
+    name: str = None,
+    run_config: RunConfig = None,
+    verbose=False,
+) -> typing.Tuple[str, BaseRuntime]:
     if isinstance(featureset, str):
         featureset = get_feature_set_by_uri(featureset)
 

mlrun/feature_store/feature_set.py

@@ -992,6 +992,50 @@ class FeatureSet(ModelObj):
         spark_context=None,
         overwrite=None,
     ) -> Optional[pd.DataFrame]:
+        """Read local DataFrame, file, URL, or source into the feature store
+        Ingest reads from the source, run the graph transformations, infers  metadata and stats
+        and writes the results to the default of specified targets
+
+        when targets are not specified data is stored in the configured default targets
+        (will usually be NoSQL for real-time and Parquet for offline).
+
+        the `run_config` parameter allow specifying the function and job configuration,
+        see: :py:class:`~mlrun.feature_store.RunConfig`
+
+        example::
+
+            stocks_set = FeatureSet("stocks", entities=[Entity("ticker")])
+            stocks = pd.read_csv("stocks.csv")
+            df = stocks_set.ingest(stocks, infer_options=fstore.InferOptions.default())
+
+            # for running as remote job
+            config = RunConfig(image='mlrun/mlrun')
+            df = ingest(stocks_set, stocks, run_config=config)
+
+            # specify source and targets
+            source = CSVSource("mycsv", path="measurements.csv")
+            targets = [CSVTarget("mycsv", path="./mycsv.csv")]
+            ingest(measurements, source, targets)
+
+        :param source:        source dataframe or other sources (e.g. parquet source see:
+                            :py:class:`~mlrun.datastore.ParquetSource` and other classes in mlrun.datastore with suffix
+                            Source)
+        :param targets:       optional list of data target objects
+        :param namespace:     namespace or module containing graph classes
+        :param return_df:     indicate if to return a dataframe with the graph results
+        :param infer_options: schema (for discovery of entities, features in featureset), index, stats,
+                            histogram and preview infer options (:py:class:`~mlrun.feature_store.InferOptions`)
+        :param run_config:    function and/or run configuration for remote jobs,
+                            see :py:class:`~mlrun.feature_store.RunConfig`
+        :param mlrun_context: mlrun context (when running as a job), for internal use !
+        :param spark_context: local spark session for spark ingestion, example for creating the spark context:
+                            `spark = SparkSession.builder.appName("Spark function").getOrCreate()`
+                            For remote spark ingestion, this should contain the remote spark service name
+        :param overwrite:     delete the targets' data prior to ingestion
+                            (default: True for non scheduled ingest - deletes the targets that are about to be ingested.
+                            False for scheduled ingest - does not delete the target)
+        :return:              if return_df is True, a dataframe will be returned based on the graph
+        """
         return mlrun.feature_store.api._ingest(
             self,
             source,
@@ -1014,6 +1058,26 @@ class FeatureSet(ModelObj):
         verbose: bool = False,
         sample_size: int = None,
     ) -> pd.DataFrame:
+        """run the ingestion pipeline with local DataFrame/file data and infer features schema and stats
+
+        example::
+
+            quotes_set = FeatureSet("stock-quotes", entities=[Entity("ticker")])
+            quotes_set.add_aggregation("ask", ["sum", "max"], ["1h", "5h"], "10m")
+            quotes_set.add_aggregation("bid", ["min", "max"], ["1h"], "10m")
+            df = quotes_set.preview(
+                quotes_df,
+                entity_columns=["ticker"],
+            )
+
+        :param source:         source dataframe or csv/parquet file path
+        :param entity_columns: list of entity (index) column names
+        :param namespace:      namespace or module containing graph classes
+        :param options:        schema (for discovery of entities, features in featureset), index, stats,
+                            histogram and preview infer options (:py:class:`~mlrun.feature_store.InferOptions`)
+        :param verbose:        verbose log
+        :param sample_size:    num of rows to sample from the dataset (for large datasets)
+        """
         return mlrun.feature_store.api._preview(
             self, source, entity_columns, namespace, options, verbose, sample_size
         )
@@ -1026,6 +1090,31 @@ class FeatureSet(ModelObj):
         run_config: RunConfig = None,
         verbose=False,
     ) -> Tuple[str, BaseRuntime]:
+        """Start real-time ingestion service using nuclio function
+
+        Deploy a real-time function implementing feature ingestion pipeline
+        the source maps to Nuclio event triggers (http, kafka, v3io stream, etc.)
+
+        the `run_config` parameter allow specifying the function and job configuration,
+        see: :py:class:`~mlrun.feature_store.RunConfig`
+
+        example::
+
+            source = HTTPSource()
+            func = mlrun.code_to_function("ingest", kind="serving").apply(mount_v3io())
+            config = RunConfig(function=func)
+            my_set.deploy_ingestion_service(source, run_config=config)
+
+        :param source:        data source object describing the online or offline source
+        :param targets:       list of data target objects
+        :param name:          name for the job/function
+        :param run_config:    service runtime configuration (function object/uri, resources, etc..)
+        :param verbose:       verbose log
+
+        :return: URL to access the deployed ingestion service, and the function that was deployed (which will
+                differ from the function passed in via the run_config parameter).
+        """
+
         return mlrun.feature_store.api._deploy_ingestion_service_v2(
             self, source, targets, name, run_config, verbose
         )

mlrun/feature_store/feature_vector.py

@@ -744,6 +744,64 @@ class FeatureVector(ModelObj):
         spark_service: str = None,
         timestamp_for_filtering: Union[str, Dict[str, str]] = None,
     ):
+        """retrieve offline feature vector results
+
+        specify a feature vector object/uri and retrieve the desired features, their metadata
+        and statistics. returns :py:class:`~mlrun.feature_store.OfflineVectorResponse`,
+        results can be returned as a dataframe or written to a target
+
+        The start_time and end_time attributes allow filtering the data to a given time range, they accept
+        string values or pandas `Timestamp` objects, string values can also be relative, for example:
+        "now", "now - 1d2h", "now+5m", where a valid pandas Timedelta string follows the verb "now",
+        for time alignment you can use the verb "floor" e.g. "now -1d floor 1H" will align the time to the last hour
+        (the floor string is passed to pandas.Timestamp.floor(), can use D, H, T, S for day, hour, min, sec alignment).
+        Another option to filter the data is by the `query` argument - can be seen in the example.
+        example::
+
+            features = [
+                "stock-quotes.bid",
+                "stock-quotes.asks_sum_5h",
+                "stock-quotes.ask as mycol",
+                "stocks.*",
+            ]
+            vector = FeatureVector(features=features)
+            vector.get_offline_features(entity_rows=trades, entity_timestamp_column="time", query="ticker in ['GOOG']
+              and bid>100")
+            print(resp.to_dataframe())
+            print(vector.get_stats_table())
+            resp.to_parquet("./out.parquet")
+
+        :param entity_rows:             dataframe with entity rows to join with
+        :param target:                  where to write the results to
+        :param drop_columns:            list of columns to drop from the final result
+        :param entity_timestamp_column: timestamp column name in the entity rows dataframe. can be specified
+                                        only if param entity_rows was specified.
+        :param run_config:              function and/or run configuration
+                                        see :py:class:`~mlrun.feature_store.RunConfig`
+        :param start_time:              datetime, low limit of time needed to be filtered. Optional.
+        :param end_time:                datetime, high limit of time needed to be filtered. Optional.
+        :param with_indexes:            Return vector with/without the entities and the timestamp_key of the feature
+                                        sets and with/without entity_timestamp_column and timestamp_for_filtering
+                                        columns. This property can be specified also in the feature vector spec
+                                        (feature_vector.spec.with_indexes)
+                                        (default False)
+        :param update_stats:            update features statistics from the requested feature sets on the vector.
+                                        (default False).
+        :param engine:                  processing engine kind ("local", "dask", or "spark")
+        :param engine_args:             kwargs for the processing engine
+        :param query:                   The query string used to filter rows on the output
+        :param spark_service:           Name of the spark service to be used (when using a remote-spark runtime)
+        :param order_by:                Name or list of names to order by. The name or the names in the list can be the
+                                        feature name or the alias of the feature you pass in the feature list.
+        :param timestamp_for_filtering: name of the column to filter by, can be str for all the feature sets or a
+                                        dictionary ({<feature set name>: <timestamp column name>, ...})
+                                        that indicates the timestamp column name for each feature set. Optional.
+                                        By default, the filter executes on the timestamp_key of each feature set.
+                                        Note: the time filtering is performed on each feature set before the
+                                        merge process using start_time and end_time params.
+
+        """
+
         return mlrun.feature_store.api._get_offline_features(
             self,
             entity_rows,
@@ -771,6 +829,68 @@ class FeatureVector(ModelObj):
         update_stats: bool = False,
         entity_keys: List[str] = None,
     ):
+        """initialize and return online feature vector service api,
+        returns :py:class:`~mlrun.feature_store.OnlineVectorService`
+
+        :**usage**:
+            There are two ways to use the function:
+
+            1. As context manager
+
+                Example::
+
+                    with vector_uri.get_online_feature_service() as svc:
+                        resp = svc.get([{"ticker": "GOOG"}, {"ticker": "MSFT"}])
+                        print(resp)
+                        resp = svc.get([{"ticker": "AAPL"}], as_list=True)
+                        print(resp)
+
+                Example with imputing::
+
+                    with vector_uri.get_online_feature_service(entity_keys=['id'],
+                                                    impute_policy={"*": "$mean", "amount": 0)) as svc:
+                        resp = svc.get([{"id": "C123487"}])
+
+            2. as simple function, note that in that option you need to close the session.
+
+                Example::
+
+                    svc = vector_uri.get_online_feature_service(entity_keys=['ticker'])
+                    try:
+                        resp = svc.get([{"ticker": "GOOG"}, {"ticker": "MSFT"}])
+                        print(resp)
+                        resp = svc.get([{"ticker": "AAPL"}], as_list=True)
+                        print(resp)
+
+                    finally:
+                        svc.close()
+
+                Example with imputing::
+
+                    svc = vector_uri.get_online_feature_service(entity_keys=['id'],
+                                                    impute_policy={"*": "$mean", "amount": 0))
+                    try:
+                        resp = svc.get([{"id": "C123487"}])
+                    except Exception as e:
+                        handling exception...
+                    finally:
+                        svc.close()
+
+        :param run_config:          function and/or run configuration for remote jobs/services
+        :param impute_policy:       a dict with `impute_policy` per feature, the dict key is the feature name and the
+                                    dict value indicate which value will be used in case the feature is NaN/empty, the
+                                    replaced value can be fixed number for constants or $mean, $max, $min, $std, $count
+                                    for statistical values.
+                                    "*" is used to specify the default for all features, example: `{"*": "$mean"}`
+        :param fixed_window_type:   determines how to query the fixed window values which were previously inserted by
+                                    ingest
+        :param update_stats:        update features statistics from the requested feature sets on the vector.
+                                    Default: False.
+        :param entity_keys:         Entity list of the first feature_set in the vector.
+                                    The indexes that are used to query the online service.
+        :return:                    Initialize the `OnlineVectorService`.
+                                    Will be used in subclasses where `support_online=True`.
+        """
         return mlrun.feature_store.api._get_online_feature_service(
             self,
             run_config,

mlrun/frameworks/tf_keras/callbacks/logging_callback.py

@@ -389,9 +389,9 @@ class LoggingCallback(Callback):
         ):
             try:
                 self._get_hyperparameter(key_chain=learning_rate_key_chain)
-                self._dynamic_hyperparameters_keys[
-                    learning_rate_key
-                ] = learning_rate_key_chain
+                self._dynamic_hyperparameters_keys[learning_rate_key] = (
+                    learning_rate_key_chain
+                )
             except (KeyError, IndexError, ValueError):
                 pass
 

mlrun/frameworks/tf_keras/model_handler.py

@@ -263,13 +263,13 @@ class TFKerasModelHandler(DLModelHandler):
         # Update the paths and log artifacts if context is available:
         if self._weights_file is not None:
             if self._context is not None:
-                artifacts[
-                    self._get_weights_file_artifact_name()
-                ] = self._context.log_artifact(
-                    self._weights_file,
-                    local_path=self._weights_file,
-                    artifact_path=output_path,
-                    db_key=False,
+                artifacts[self._get_weights_file_artifact_name()] = (
+                    self._context.log_artifact(
+                        self._weights_file,
+                        local_path=self._weights_file,
+                        artifact_path=output_path,
+                        db_key=False,
+                    )
                 )
 
         return artifacts if self._context is not None else None