scikit-learn-intelex 2024.5.0__py312-none-manylinux1_x86_64.whl → 2024.6.0__py312-none-manylinux1_x86_64.whl

sklearnex/tests/test_memory_usage.py

@@ -14,126 +14,113 @@
 # limitations under the License.
 # ==============================================================================
 
-
 import gc
 import logging
+import os
 import tracemalloc
 import types
+import warnings
+from inspect import isclass
 
 import numpy as np
 import pandas as pd
 import pytest
 from scipy.stats import pearsonr
-from sklearn.base import BaseEstimator
+from sklearn.base import BaseEstimator, clone
 from sklearn.datasets import make_classification
 from sklearn.model_selection import KFold
 
-from sklearnex import get_patch_map
-from sklearnex.metrics import pairwise_distances, roc_auc_score
-from sklearnex.model_selection import train_test_split
-from sklearnex.utils import _assert_all_finite
-
-
-class TrainTestSplitEstimator:
-    def __init__(self):
-        pass
-
-    def fit(self, x, y):
-        train_test_split(x, y)
-
-
-class FiniteCheckEstimator:
-    def __init__(self):
-        pass
-
-    def fit(self, x, y):
-        _assert_all_finite(x)
-        _assert_all_finite(y)
-
-
-class PairwiseDistancesEstimator:
-    def fit(self, x, y):
-        pairwise_distances(x, metric=self.metric)
-
-
-class CosineDistancesEstimator(PairwiseDistancesEstimator):
-    def __init__(self):
-        self.metric = "cosine"
-
-
-class CorrelationDistancesEstimator(PairwiseDistancesEstimator):
-    def __init__(self):
-        self.metric = "correlation"
-
-
-class RocAucEstimator:
-    def __init__(self):
-        pass
-
-    def fit(self, x, y):
-        print(roc_auc_score(y, np.zeros(shape=y.shape, dtype=np.int32)))
-
-
-# add all daal4py estimators enabled in patching (except banned)
-
-
-def get_patched_estimators(ban_list, output_list):
-    patched_estimators = get_patch_map().values()
-    for listing in patched_estimators:
-        estimator, name = listing[0][0][2], listing[0][0][1]
-        if not isinstance(estimator, types.FunctionType):
-            if name not in ban_list:
-                if issubclass(estimator, BaseEstimator):
-                    if hasattr(estimator, "fit"):
-                        output_list.append(estimator)
-
+from onedal import _is_dpc_backend
+from onedal.tests.utils._dataframes_support import (
+    _convert_to_dataframe,
+    get_dataframes_and_queues,
+)
+from onedal.tests.utils._device_selection import get_queues, is_dpctl_available
+from sklearnex import config_context
+from sklearnex.tests._utils import PATCHED_FUNCTIONS, PATCHED_MODELS, SPECIAL_INSTANCES
+from sklearnex.utils import get_namespace
 
-def remove_duplicated_estimators(estimators_list):
-    estimators_map = {}
-    for estimator in estimators_list:
-        full_name = f"{estimator.__module__}.{estimator.__name__}"
-        estimators_map[full_name] = estimator
-    return estimators_map.values()
+if _is_dpc_backend:
+    from onedal import _backend
 
 
-BANNED_ESTIMATORS = (
+CPU_SKIP_LIST = (
+    "TSNE",  # too slow for using in testing on common data size
+    "config_context",  # does not malloc
+    "get_config",  # does not malloc
+    "set_config",  # does not malloc
+    "SVC(probability=True)",  # memory leak fortran numpy (investigate _fit_proba)
+    "NuSVC(probability=True)",  # memory leak fortran numpy (investigate _fit_proba)
     "IncrementalEmpiricalCovariance",  # dataframe_f issues
     "IncrementalLinearRegression",  # TODO fix memory leak issue in private CI for data_shape = (1000, 100), data_transform_function = dataframe_f
-    "TSNE",  # too slow for using in testing on common data size
+    "IncrementalPCA",  # TODO fix memory leak issue in private CI for data_shape = (1000, 100), data_transform_function = dataframe_f
+    "LogisticRegression(solver='newton-cg')",  # memory leak fortran (1000, 100)
 )
-estimators = [
-    TrainTestSplitEstimator,
-    FiniteCheckEstimator,
-    CosineDistancesEstimator,
-    CorrelationDistancesEstimator,
-    RocAucEstimator,
-]
-get_patched_estimators(BANNED_ESTIMATORS, estimators)
-estimators = remove_duplicated_estimators(estimators)
 
+GPU_SKIP_LIST = (
+    "TSNE",  # too slow for using in testing on common data size
+    "RandomForestRegressor",  # too slow for using in testing on common data size
+    "KMeans",  # does not support GPU offloading
+    "config_context",  # does not malloc
+    "get_config",  # does not malloc
+    "set_config",  # does not malloc
+    "Ridge",  # does not support GPU offloading (fails silently)
+    "ElasticNet",  # does not support GPU offloading (fails silently)
+    "Lasso",  # does not support GPU offloading (fails silently)
+    "SVR",  # does not support GPU offloading (fails silently)
+    "NuSVR",  # does not support GPU offloading (fails silently)
+    "NuSVC",  # does not support GPU offloading (fails silently)
+    "LogisticRegression",  # default parameters not supported, see solver=newton-cg
+    "NuSVC(probability=True)",  # does not support GPU offloading (fails silently)
+    "IncrementalLinearRegression",  # issue with potrf with the specific dataset
+    "LinearRegression",  # issue with potrf with the specific dataset
+)
 
-def ndarray_c(x, y):
-    return np.ascontiguousarray(x), y
 
+def gen_functions(functions):
+    func_dict = functions.copy()
 
-def ndarray_f(x, y):
-    return np.asfortranarray(x), y
+    roc_auc_score = func_dict.pop("roc_auc_score")
+    func_dict["roc_auc_score"] = lambda x, y: roc_auc_score(y, y)
 
+    pairwise_distances = func_dict.pop("pairwise_distances")
+    func_dict["pairwise_distances(metric='cosine')"] = lambda x, y: pairwise_distances(
+        x, metric="cosine"
+    )
+    func_dict["pairwise_distances(metric='correlation')"] = (
+        lambda x, y: pairwise_distances(x, metric="correlation")
+    )
 
-def dataframe_c(x, y):
-    return pd.DataFrame(np.ascontiguousarray(x)), pd.Series(y)
+    _assert_all_finite = func_dict.pop("_assert_all_finite")
+    func_dict["_assert_all_finite"] = lambda x, y: [
+        _assert_all_finite(x),
+        _assert_all_finite(y),
+    ]
+    return func_dict
 
 
-def dataframe_f(x, y):
-    return pd.DataFrame(np.asfortranarray(x)), pd.Series(y)
+FUNCTIONS = gen_functions(PATCHED_FUNCTIONS)
 
+CPU_ESTIMATORS = {
+    k: v
+    for k, v in {**PATCHED_MODELS, **SPECIAL_INSTANCES, **FUNCTIONS}.items()
+    if not k in CPU_SKIP_LIST
+}
 
-data_transforms = [ndarray_c, ndarray_f, dataframe_c, dataframe_f]
+GPU_ESTIMATORS = {
+    k: v
+    for k, v in {**PATCHED_MODELS, **SPECIAL_INSTANCES}.items()
+    if not k in GPU_SKIP_LIST
+}
 
-data_shapes = [(1000, 100), (2000, 50)]
+data_shapes = [
+    pytest.param((1000, 100), id="(1000, 100)"),
+    pytest.param((2000, 50), id="(2000, 50)"),
+]
 
 EXTRA_MEMORY_THRESHOLD = 0.15
 N_SPLITS = 10
+ORDER_DICT = {"F": np.asfortranarray, "C": np.ascontiguousarray}
 
 
 def gen_clsf_data(n_samples, n_features):
@@ -147,45 +134,82 @@ def gen_clsf_data(n_samples, n_features):
     )
 
 
-def split_train_inference(kf, x, y, estimator):
+def get_traced_memory(queue=None):
+    if _is_dpc_backend and queue and queue.sycl_device.is_gpu:
+        return _backend.get_used_memory(queue)
+    else:
+        return tracemalloc.get_traced_memory()[0]
+
+
+def take(x, index, axis=0, queue=None):
+    xp, array_api = get_namespace(x)
+    if array_api:
+        return xp.take(x, xp.asarray(index, device=queue), axis=axis)
+    else:
+        return x.take(index, axis=axis)
+
+
+def split_train_inference(kf, x, y, estimator, queue=None):
     mem_tracks = []
     for train_index, test_index in kf.split(x):
-        if isinstance(x, np.ndarray):
-            x_train, x_test = x[train_index], x[test_index]
-            y_train, y_test = y[train_index], y[test_index]
-        elif isinstance(x, pd.core.frame.DataFrame):
-            x_train, x_test = x.iloc[train_index], x.iloc[test_index]
-            y_train, y_test = y.iloc[train_index], y.iloc[test_index]
-        # TODO: add parameters for all estimators to prevent
-        # fallback to stock scikit-learn with default parameters
-
-        alg = estimator()
-        alg.fit(x_train, y_train)
-        if hasattr(alg, "predict"):
-            alg.predict(x_test)
-        elif hasattr(alg, "transform"):
-            alg.transform(x_test)
-        elif hasattr(alg, "kneighbors"):
-            alg.kneighbors(x_test)
-        del alg, x_train, x_test, y_train, y_test
-        mem_tracks.append(tracemalloc.get_traced_memory()[0])
+        x_train = take(x, train_index, queue=queue)
+        y_train = take(y, train_index, queue=queue)
+        x_test = take(x, test_index, queue=queue)
+        y_test = take(y, test_index, queue=queue)
+
+        if isclass(estimator) and issubclass(estimator, BaseEstimator):
+            alg = estimator()
+            flag = True
+        elif isinstance(estimator, BaseEstimator):
+            alg = clone(estimator)
+            flag = True
+        else:
+            flag = False
+
+        if flag:
+            alg.fit(x_train, y_train)
+            if hasattr(alg, "predict"):
+                alg.predict(x_test)
+            elif hasattr(alg, "transform"):
+                alg.transform(x_test)
+            elif hasattr(alg, "kneighbors"):
+                alg.kneighbors(x_test)
+            del alg
+        else:
+            estimator(x_train, y_train)
+
+        del x_train, x_test, y_train, y_test, flag
+        mem_tracks.append(get_traced_memory(queue))
     return mem_tracks
 
 
-def _kfold_function_template(estimator, data_transform_function, data_shape):
+def _kfold_function_template(estimator, dataframe, data_shape, queue=None, func=None):
     tracemalloc.start()
 
     n_samples, n_features = data_shape
-    x, y, data_memory_size = gen_clsf_data(n_samples, n_features)
+    X, y, data_memory_size = gen_clsf_data(n_samples, n_features)
     kf = KFold(n_splits=N_SPLITS)
-    x, y = data_transform_function(x, y)
+    if func:
+        X = func(X)
+
+    X = _convert_to_dataframe(X, sycl_queue=queue, target_df=dataframe)
+    y = _convert_to_dataframe(y, sycl_queue=queue, target_df=dataframe)
 
-    mem_before, _ = tracemalloc.get_traced_memory()
-    mem_tracks = split_train_inference(kf, x, y, estimator)
+    mem_before = get_traced_memory(queue)
+    mem_tracks = split_train_inference(kf, X, y, estimator, queue=queue)
     mem_iter_diffs = np.array(mem_tracks[1:]) - np.array(mem_tracks[:-1])
     mem_incr_mean, mem_incr_std = mem_iter_diffs.mean(), mem_iter_diffs.std()
     mem_incr_mean, mem_incr_std = round(mem_incr_mean), round(mem_incr_std)
-    mem_iter_corr, _ = pearsonr(mem_tracks, list(range(len(mem_tracks))))
+    with warnings.catch_warnings():
+        # In the case that the memory usage is constant, this will raise
+        # a ConstantInputWarning error in pearsonr from scipy, this can
+        # be ignored.
+        warnings.filterwarnings(
+            "ignore",
+            message="An input array is constant; the correlation coefficient is not defined",
+        )
+        mem_iter_corr, _ = pearsonr(mem_tracks, list(range(len(mem_tracks))))
+
     if mem_iter_corr > 0.95:
         logging.warning(
             "Memory usage is steadily increasing with iterations "
@@ -194,12 +218,17 @@ def _kfold_function_template(estimator, data_transform_function, data_shape):
             "Memory usage increase per iteration: "
             f"{mem_incr_mean}±{mem_incr_std} bytes"
         )
-    mem_before_gc, _ = tracemalloc.get_traced_memory()
+    mem_before_gc = get_traced_memory(queue)
     mem_diff = mem_before_gc - mem_before
+    if isinstance(estimator, BaseEstimator):
+        name = str(estimator)
+    else:
+        name = estimator.__name__
+
     message = (
         "Size of extra allocated memory {} using garbage collector "
         f"is greater than {EXTRA_MEMORY_THRESHOLD * 100}% of input data"
-        f"\n\tAlgorithm: {estimator.__name__}"
+        f"\n\tAlgorithm: {name}"
         f"\n\tInput data size: {data_memory_size} bytes"
         "\n\tExtra allocated memory size: {} bytes"
         " / {} %"
@@ -211,21 +240,51 @@ def _kfold_function_template(estimator, data_transform_function, data_shape):
             )
         )
     gc.collect()
-    mem_after, _ = tracemalloc.get_traced_memory()
+    mem_after = get_traced_memory(queue)
     tracemalloc.stop()
     mem_diff = mem_after - mem_before
 
-    assert mem_diff < EXTRA_MEMORY_THRESHOLD * data_memory_size, message.format(
-        "after", mem_diff, round((mem_diff) / data_memory_size * 100, 2)
-    )
+    # GPU offloading with SYCL contains a program/kernel cache which should
+    # be controllable via a KernelProgramCache object in the SYCL context.
+    # The programs and kernels are stored on the GPU, but cannot be cleared
+    # as this class is not available for access in all oneDAL DPC++ runtimes.
+    # Therefore, until this is implemented this test must be skipped for gpu
+    # as it looks like a memory leak (at least there is no way to discern a
+    # leak on the first run).
+    if queue is None or queue.sycl_device.is_cpu:
+        assert mem_diff < EXTRA_MEMORY_THRESHOLD * data_memory_size, message.format(
+            "after", mem_diff, round((mem_diff) / data_memory_size * 100, 2)
+        )
 
 
-# disable fallback check as logging impacts memory use
+@pytest.mark.parametrize("order", ["F", "C"])
+@pytest.mark.parametrize(
+    "dataframe,queue", get_dataframes_and_queues("numpy,pandas,dpctl", "cpu")
+)
+@pytest.mark.parametrize("estimator", CPU_ESTIMATORS.keys())
+@pytest.mark.parametrize("data_shape", data_shapes)
+def test_memory_leaks(estimator, dataframe, queue, order, data_shape):
+    func = ORDER_DICT[order]
+    if estimator == "_assert_all_finite" and queue is not None:
+        pytest.skip(f"{estimator} is not designed for device offloading")
+
+    _kfold_function_template(
+        CPU_ESTIMATORS[estimator], dataframe, data_shape, queue, func
+    )
 
 
-@pytest.mark.allow_sklearn_fallback
-@pytest.mark.parametrize("data_transform_function", data_transforms)
-@pytest.mark.parametrize("estimator", estimators)
+@pytest.mark.skipif(
+    os.getenv("ZES_ENABLE_SYSMAN") is None or not is_dpctl_available("gpu"),
+    reason="SYCL device memory leak check requires the level zero sysman",
+)
+@pytest.mark.parametrize("queue", get_queues("gpu"))
+@pytest.mark.parametrize("estimator", GPU_ESTIMATORS.keys())
+@pytest.mark.parametrize("order", ["F", "C"])
 @pytest.mark.parametrize("data_shape", data_shapes)
-def test_memory_leaks(estimator, data_transform_function, data_shape):
-    _kfold_function_template(estimator, data_transform_function, data_shape)
+def test_gpu_memory_leaks(estimator, queue, order, data_shape):
+    func = ORDER_DICT[order]
+    if "ExtraTrees" in estimator and data_shape == (2000, 50):
+        pytest.skip("Avoid a segmentation fault in Extra Trees algorithms")
+
+    with config_context(target_offload=queue):
+        _kfold_function_template(GPU_ESTIMATORS[estimator], None, data_shape, queue, func)

sklearnex/tests/test_patching.py

@@ -148,23 +148,16 @@ def test_standard_estimator_patching(caplog, dataframe, queue, dtype, estimator,
             and dtype in [np.uint32, np.uint64]
         ):
             pytest.skip("Windows segmentation fault for Ridge.predict for unsigned ints")
-        elif estimator == "IncrementalLinearRegression" and dtype in [
-            np.int8,
-            np.int16,
-            np.int32,
-            np.int64,
-            np.uint8,
-            np.uint16,
-            np.uint32,
-            np.uint64,
-        ]:
+        elif estimator == "IncrementalLinearRegression" and np.issubdtype(
+            dtype, np.integer
+        ):
             pytest.skip(
                 "IncrementalLinearRegression fails on oneDAL side with int types because dataset is filled by zeroes"
             )
         elif method and not hasattr(est, method):
             pytest.skip(f"sklearn available_if prevents testing {estimator}.{method}")
 
-        X, y = gen_dataset(est, queue=queue, target_df=dataframe, dtype=dtype)
+        X, y = gen_dataset(est, queue=queue, target_df=dataframe, dtype=dtype)[0]
         est.fit(X, y)
 
         if method:
@@ -196,7 +189,7 @@ def test_special_estimator_patching(caplog, dataframe, queue, dtype, estimator,
         elif dtype == np.float64 and queue and not queue.sycl_device.has_aspect_fp64:
             pytest.skip("Hardware does not support fp64 SYCL testing")
 
-        X, y = gen_dataset(est, queue=queue, target_df=dataframe, dtype=dtype)
+        X, y = gen_dataset(est, queue=queue, target_df=dataframe, dtype=dtype)[0]
         est.fit(X, y)
 
         if method and not hasattr(est, method):