julearn 0.3.6.dev6__tar.gz → 0.3.6.dev17__tar.gz

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/.github/workflows/ci-docs.yml

@@ -53,7 +53,7 @@ jobs:
           echo '</coverage>' >> coverage.xml
         if: matrix.python-version == 3.12
       - name: Upload mock coverage to Codecov
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v6
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
     - name: Test with tox
       run: |
         tox
-    - uses: codecov/codecov-action@v5
+    - uses: codecov/codecov-action@v6
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: true

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/.github/workflows/publish.yml

@@ -32,7 +32,7 @@ jobs:
       run:
         python -m build
     - uses: pypa/gh-action-pypi-publish@release/v1
-    - uses: softprops/action-gh-release@v2
+    - uses: softprops/action-gh-release@v3
       with:
         files: |
           dist/*.tar.gz

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: julearn
-Version: 0.3.6.dev6
+Version: 0.3.6.dev17
 Summary: Juelich Machine Learning Library
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Sami Hamdan <s.hamdan@fz-juelich.de>
 Maintainer-email: Sami Hamdan <s.hamdan@fz-juelich.de>

julearn-0.3.6.dev17/docs/changes/newsfragments/303.feature

@@ -0,0 +1 @@
+Allow setting of ``context_func`` for ``joblib-htcondor`` by `Fede Raimondo`_

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/julearn/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.3.6.dev6'
-__version_tuple__ = version_tuple = (0, 3, 6, 'dev6')
+__version__ = version = '0.3.6.dev17'
+__version_tuple__ = version_tuple = (0, 3, 6, 'dev17')
 
-__commit_id__ = commit_id = 'gff341ff24'
+__commit_id__ = commit_id = 'g591f6791b'

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/julearn/api.py

@@ -10,6 +10,7 @@ from typing import Optional, Union
 import numpy as np
 import pandas as pd
 import sklearn
+from joblib.parallel import get_active_backend
 from sklearn.base import BaseEstimator
 from sklearn.model_selection import (
     cross_validate,
@@ -17,6 +18,7 @@ from sklearn.model_selection import (
 from sklearn.model_selection._search import BaseSearchCV
 from sklearn.pipeline import Pipeline
 
+from .config import _joblib_htcondor_context_func
 from .inspect import Inspector
 from .model_selection.utils import check_cv
 from .pipeline import PipelineCreator
@@ -382,6 +384,28 @@ def _validate_api_params(  # noqa: C901
     return out
 
 
+def _check_configure_joblib_backend() -> None:
+    """Check and configure if we are running in a joblib parallel backend."""
+    backend, _ = get_active_backend()
+
+    # If we are in an HTCondor backend, we need to set the context function to
+    # set the global configuration variables from both julearn and scikit-
+    # learn, as these are not automatically propagated to the workers in
+    # non-shared memory backends.
+    if backend.__class__.__name__ == "_HTCondorBackend":
+        if hasattr(backend, "_context_func"):
+            backend._context_func = _joblib_htcondor_context_func(
+                current_func=backend._context_func
+            )
+        else:
+            raise_error(
+                "Running in an HTCondor backend, but it does not have a "
+                "_context_func attribute. Global configuration variables "
+                "will not be propagated to the workers. Please update your "
+                "joblib-htcondor package to version 0.2.3 or higher."
+            )
+
+
 def run_cross_validation(
     X: list[str],  # noqa: N803
     y: str,
@@ -613,6 +637,8 @@ def run_cross_validation(
         else:
             _sklearn_deprec_fit_params["fit_params"] = fit_params
 
+    _check_configure_joblib_backend()
+
     scores = cross_validate(
         pipeline,
         df_X,
@@ -835,6 +861,8 @@ def run_fit(
     else:
         _sklearn_deprec_fit_params["fit_params"] = fit_params
 
+    _check_configure_joblib_backend()
+
     logger.info("Fitting final model")
     pipeline.fit(df_X, df_y, **fit_params)
 

julearn-0.3.6.dev17/julearn/config.py

@@ -0,0 +1,118 @@
+"""Global config module."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+# License: AGPL
+
+from typing import Any
+
+from .utils import logger, raise_error
+from .utils.logging import DelayedFmtMessage as __
+
+
+_global_config = {}
+_global_config["MAX_X_WARNS"] = 5000
+_global_config["max_x_logs"] = 100
+_global_config["disable_x_check"] = False
+_global_config["disable_xtypes_check"] = False
+_global_config["disable_x_verbose"] = False
+_global_config["disable_xtypes_verbose"] = False
+_global_config["enable_parallel_column_transformers"] = False
+_global_config["enable_auto_escape_parenthesis"] = True
+
+
+def set_config(key: str, value: Any) -> None:
+    """Set a global config value.
+
+    Parameters
+    ----------
+    key : str
+        The key to set.
+    value : Any
+        The value to set.
+
+    """
+    if key not in _global_config:
+        raise_error(f"Global config {key} does not exist")
+    logger.info(
+        __("Setting global config {key} to {value}", key=key, value=value)
+    )
+    _global_config[key] = value
+
+
+def get_config(key: str) -> Any:
+    """Get a global config value.
+
+    Parameters
+    ----------
+    key : str
+        The key to get.
+
+    Returns
+    -------
+    Any
+        The value of the key.
+
+    """
+    return _global_config.get(key, None)
+
+
+def _joblib_htcondor_context_func(current_func=None) -> None:
+    """Create a function to seet the config variables.
+
+    Parameters
+    ----------
+    current_func : function
+        The current function that was set in the joblib context. If not None,
+        it will be called before setting the config variables (default None).
+
+    Returns
+    -------
+    function
+        A function to set the config variables.
+
+    """
+    from copy import deepcopy
+    from functools import partial
+
+    import sklearn
+
+    # Copy the global config
+    _config = {}
+    _config["julearn_config"] = deepcopy(_global_config)
+
+    _config["sklearn_config"] = sklearn.get_config()
+
+    # add the logging configuration
+    level = logger.level
+    fmt = None
+    if len(logger.handlers) > 0:
+        fmt = logger.handlers[0].formatter._fmt  # type: ignore
+
+    _config["julearn_logging_config"] = {
+        "level": level,
+        "output_format": fmt,
+    }
+
+    # Log the current function, if any, to be called before setting the config
+    _config["jht_current_func"] = current_func
+
+    def _set_context_vars(**kwargs):
+        # First, call the previous function
+        jht_current_func = kwargs.pop("jht_current_func")
+        if jht_current_func is not None:
+            jht_current_func()
+
+        # Julearn then sets the config variables. If user has set any variable
+        # in the context function, current julearn state should override this
+        # setting.
+        for key, value in kwargs.items():
+            if key == "sklearn_config":
+                sklearn.set_config(**value)
+            elif key == "julearn_logging_config":
+                from julearn.utils import configure_logging
+                configure_logging(**value)
+            elif key == "julearn_config":
+                for k, v in value.items():
+                    set_config(k, v)
+
+    return partial(_set_context_vars, **_config)  # type: ignore

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/julearn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: julearn
-Version: 0.3.6.dev6
+Version: 0.3.6.dev17
 Summary: Juelich Machine Learning Library
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Sami Hamdan <s.hamdan@fz-juelich.de>
 Maintainer-email: Sami Hamdan <s.hamdan@fz-juelich.de>

{julearn-0.3.6.dev6 → julearn-0.3.6.dev17}/julearn.egg-info/SOURCES.txt

@@ -57,6 +57,7 @@ docs/api/viz.rst
 docs/changes/contributors.inc
 docs/changes/newsfragments/.gitignore
 docs/changes/newsfragments/301.misc
+docs/changes/newsfragments/303.feature
 docs/images/corrected_ttest.png
 docs/images/final_estimator.png
 docs/images/iris_X.png

julearn-0.3.6.dev6/julearn/config.py

@@ -1,56 +0,0 @@
-"""Global config module."""
-
-# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
-# License: AGPL
-
-from typing import Any
-
-from .utils import logger, raise_error
-from .utils.logging import DelayedFmtMessage as __
-
-
-_global_config = {}
-_global_config["MAX_X_WARNS"] = 5000
-_global_config["max_x_logs"] = 100
-_global_config["disable_x_check"] = False
-_global_config["disable_xtypes_check"] = False
-_global_config["disable_x_verbose"] = False
-_global_config["disable_xtypes_verbose"] = False
-_global_config["enable_parallel_column_transformers"] = False
-_global_config["enable_auto_escape_parenthesis"] = True
-
-
-def set_config(key: str, value: Any) -> None:
-    """Set a global config value.
-
-    Parameters
-    ----------
-    key : str
-        The key to set.
-    value : Any
-        The value to set.
-
-    """
-    if key not in _global_config:
-        raise_error(f"Global config {key} does not exist")
-    logger.info(
-        __("Setting global config {key} to {value}", key=key, value=value)
-    )
-    _global_config[key] = value
-
-
-def get_config(key: str) -> Any:
-    """Get a global config value.
-
-    Parameters
-    ----------
-    key : str
-        The key to get.
-
-    Returns
-    -------
-    Any
-        The value of the key.
-
-    """
-    return _global_config.get(key, None)