aucurriculum 0.1.0__py3-none-any.whl

aucurriculum/__init__.py

@@ -0,0 +1,36 @@
+from autrainer.core.constants import (
+    ExportConstants,
+    NamingConstants,
+)
+
+from aucurriculum.core.constants import CurriculumConstants
+from aucurriculum.core.wrappers import main
+
+
+# always overlay the autrainer constants
+NamingConstants().CONFIG_DIRS = (
+    NamingConstants().CONFIG_DIRS + CurriculumConstants().CONFIG_DIRS
+)
+NamingConstants().NAMING_CONVENTION = NamingConstants().NAMING_CONVENTION[
+    :-1
+] + [
+    "curriculum",
+    "curriculum.sampling",
+    "curriculum.scoring",
+    "curriculum.pacing",
+    "curriculum.pacing.initial_size",
+    "curriculum.pacing.final_iteration",
+    "seed",
+]
+NamingConstants().VALID_AGGREGATIONS = list(
+    set(NamingConstants().NAMING_CONVENTION)
+    - set(NamingConstants().INVALID_AGGREGATIONS)
+)
+ExportConstants().LOGGING_DEPTH = 3
+ExportConstants().IGNORE_PARAMS = ExportConstants().IGNORE_PARAMS + [
+    "curriculum.short",
+    "curriculum.sampling.short",
+]
+
+__version__ = "0.1.0"
+__all__ = ["main"]

aucurriculum/cli/__init__.py

@@ -0,0 +1,33 @@
+from autrainer.core.scripts import (
+    inference,
+    rm_failed,
+    rm_states,
+)
+
+from aucurriculum.core.scripts import (
+    create,
+    curriculum,
+    fetch,
+    group,
+    postprocess,
+    preprocess,
+    show,
+    train,
+)
+from aucurriculum.core.scripts import list_configs as list
+
+
+__all__ = [
+    "create",
+    "curriculum",
+    "fetch",
+    "group",
+    "inference",
+    "list",
+    "postprocess",
+    "preprocess",
+    "rm_failed",
+    "rm_states",
+    "show",
+    "train",
+]

aucurriculum/core/constants/__init__.py

@@ -0,0 +1,4 @@
+from .curriculum_constants import CurriculumConstants
+
+
+__all__ = ["CurriculumConstants"]

aucurriculum/core/constants/curriculum_constants.py

@@ -0,0 +1,26 @@
+from typing import List
+
+from autrainer.core.constants import AbstractConstants
+
+
+class CurriculumConstants(AbstractConstants):
+    """Singleton for managing the curriculum configurations of
+    `aucurriculum`."""
+
+    _config_dirs = [
+        "curriculum",
+        "curriculum/pacing",
+        "curriculum/sampling",
+        "curriculum/scoring",
+    ]
+
+    @property
+    def CONFIG_DIRS(self) -> List[str]:
+        """Get the curriculum configuration directories for Hydra
+        configurations. Defaults to :attr:`["curriculum", "curriculum/pacing",
+        "curriculum/sampling", "curriculum/scoring"]`.
+
+        Returns:
+            Curriculum configuration directories for Hydra configurations.
+        """
+        return self._config_dirs

aucurriculum/core/filters/__init__.py

@@ -0,0 +1,4 @@
+from .filters import FilterPartialCurriculum, FilterPlaceholderScoringFunction
+
+
+__all__ = ["FilterPartialCurriculum", "FilterPlaceholderScoringFunction"]

aucurriculum/core/filters/filters.py

@@ -0,0 +1,45 @@
+from hydra_filter_sweeper import AbstractFilter
+from omegaconf import DictConfig
+
+
+class FilterPlaceholderScoringFunction(AbstractFilter):  # pragma: no cover
+    def filter(self, config: DictConfig, directory: str) -> bool:
+        """Filter out run configurations with placeholder (None) scoring
+        functions.
+
+        Args:
+            config: Run configuration.
+            directory: Directory where the run configuration is stored.
+
+        Returns:
+            Whether to filter out the run configuration.
+        """
+        return config.curriculum.scoring.id == "None"
+
+
+class FilterPartialCurriculum(AbstractFilter):  # pragma: no cover
+    def filter(self, config: DictConfig, directory: str) -> bool:
+        """Filter out run configurations with partial curricula.
+        A partial curriculum is one where the id is not "None" but one or more
+        of the sampling, scoring, and pacing ids are "None".
+
+        Args:
+            config: Run configuration.
+            directory: Directory where the run configuration is stored.
+
+        Returns:
+            Whether to filter out the run configuration.
+        """
+        c = config.curriculum
+        if c.id == "None":
+            if any(
+                id != "None"
+                for id in [c.sampling.id, c.scoring.id, c.pacing.id]
+            ):
+                return True
+            return False
+        if any(
+            id == "None" for id in [c.sampling.id, c.scoring.id, c.pacing.id]
+        ):
+            return True
+        return False

aucurriculum/core/plotting/__init__.py

@@ -0,0 +1,4 @@
+from .plot_curriculum import CurriculumPlots
+
+
+__all__ = ["CurriculumPlots"]

aucurriculum/core/plotting/plot_curriculum.py

@@ -0,0 +1,274 @@
+import os
+import warnings
+
+from autrainer.core.plotting import PlotBase
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+
+
+class CurriculumPlots(PlotBase):
+    def __init__(
+        self,
+        output_directory: str,
+        training_type: str,
+        figsize: tuple,
+        latex: bool,
+        filetypes: list,
+        pickle: bool,
+        context: str,
+        palette: str,
+        replace_none: bool,
+        add_titles: bool,
+        add_xlabels: bool,
+        add_ylabels: bool,
+        rcParams: dict,
+    ) -> None:
+        """Plot the sample difficulty scores and pace of curricula.
+
+        Args:
+            output_directory: Output directory to save plots to.
+            training_type: Type of training in ["Epoch", "Step"].
+            figsize: Figure size in inches.
+            latex: Whether to use LaTeX in plots. Requires the `latex` package.
+                To install all necessary dependencies, run:
+                `pip install autrainer[latex]`.
+            filetypes: Filetypes to save plots as.
+            pickle: Whether to save additional pickle files of the plots.
+            context: Context for seaborn plots.
+            palette: Color palette for seaborn plots.
+            replace_none: Whether to replace "None" in labels with "~".
+            add_titles: Whether to add titles to plots.
+            add_xlabels: Whether to add x-labels to plots.
+            add_ylabels: Whether to add y-labels to plots.
+            rcParams: Additional Matplotlib rcParams to set.
+        """
+        super().__init__(
+            output_directory,
+            training_type,
+            figsize,
+            latex,
+            filetypes,
+            pickle,
+            context,
+            palette,
+            replace_none,
+            add_titles,
+            add_xlabels,
+            add_ylabels,
+            rcParams,
+        )
+
+    def plot_score(self, df: pd.DataFrame, score_id: str, num_bins=20) -> None:
+        """Plot the sample difficulty scores by creating histograms,
+        facet grids, and ridge plots visualizing the global and per class
+        sample difficulty distributions.
+
+        Args:
+            df: DataFrame containing the sample difficulty scores ("mean",
+            "ranks") and corresponding labels ("decoded").
+            score_id: ID of the scoring function.
+            num_bins: Number of bins for the histogram. Defaults to 20.
+        """
+        df["decoded"] = df["decoded"].astype(str)
+        order = sorted(df["decoded"].unique())
+        sample_counts = df["decoded"].value_counts().sort_index().values
+
+        self._plot_fn(
+            df=df,
+            score_id=score_id,
+            name="histogram",
+            x="mean",
+            y_label="count",
+            fn=sns.histplot,
+            fn_kargs={"bins": num_bins},
+        )
+        self._plot_facet_grid(
+            df=df,
+            score_id=score_id,
+            name="Count per Class",
+            y_label="count",
+            fn_map=sns.histplot,
+            fn_kargs={
+                "bins": np.arange(0, 1 + (1 / num_bins), 1 / num_bins),
+                "stat": "count",
+            },
+            col_order=order,
+        )
+        percent_char = "\\%" if self.latex else "%"
+        self._plot_facet_grid(
+            df=df,
+            score_id=score_id,
+            name="Percent per Class",
+            y_label=f"ratio [{percent_char}]",
+            fn_map=sns.histplot,
+            fn_kargs={
+                "bins": np.arange(0, 1 + (1 / num_bins), 1 / num_bins),
+                "stat": "percent",
+            },
+            col_order=order,
+            sample_counts=sample_counts,
+        )
+        self._plot_ridge(
+            df=df, score_id=score_id, name="Ridge per Class", row_order=order
+        )
+
+    def _plot_facet_grid(
+        self,
+        df: pd.DataFrame,
+        score_id: str,
+        name: str,
+        y_label: str,
+        fn_map: callable,
+        fn_kargs: dict,
+        col_order: list = None,
+        sample_counts: list = None,
+    ) -> None:
+        fg = sns.FacetGrid(
+            df,
+            col="decoded",
+            col_order=col_order,
+            hue="decoded",
+            col_wrap=5,
+            height=self.figsize[1] / 2,
+            aspect=self.figsize[0] / (self.figsize[1] * 2),
+        )
+        fg.set_titles(col_template="{col_name}")
+        if sample_counts is not None:
+            for ax, ct in zip(fg.axes.flat, sample_counts):
+                ax.set_title(f"{ax.get_title()} ({ct} samples)")
+        fg.map_dataframe(fn_map, x="ranks", **fn_kargs)
+        if self.add_xlabels:
+            fg.set_xlabels("ranks")
+        else:
+            fg.set_xlabels("")
+        if self.add_ylabels:
+            fg.set_ylabels(y_label)
+        else:
+            fg.set_ylabels("")
+        if self.add_titles:
+            fg.figure.suptitle(f"{score_id} {name}", y=0.975)
+        else:
+            fg.figure.suptitle("")
+        path = os.path.join("_plots", score_id)
+        self.save_plot(fg.figure, name.lower().replace(" ", "_"), path)
+
+    def _plot_fn(
+        self,
+        df: pd.DataFrame,
+        score_id: str,
+        name: str,
+        x: str,
+        y_label: str,
+        fn: callable,
+        fn_kargs: dict,
+    ) -> None:
+        fig = plt.figure(figsize=self.figsize)
+        fn(data=df, x=x, **fn_kargs)
+        self._add_label(fig.gca(), x, y_label)
+        path = os.path.join("_plots", score_id)
+        self.save_plot(fig, name.lower().replace(" ", "_"), path)
+
+    def _plot_ridge(
+        self, df: pd.DataFrame, score_id: str, name: str, row_order: list
+    ) -> None:
+        _old_warn = warnings.filters[:]
+        warnings.filterwarnings("ignore", category=UserWarning)
+        height = (self.figsize[1] / 10) * 1.25
+        fg = sns.FacetGrid(
+            df,
+            row="decoded",
+            row_order=row_order,
+            hue="decoded",
+            height=height,
+            aspect=(self.figsize[0] * 1.25) / height,
+        )
+        fg.map_dataframe(
+            sns.kdeplot,
+            x="ranks",
+            bw_adjust=0.5,
+            fill=True,
+        )
+        for ax in fg.axes.flat:
+            ax.patch.set_facecolor("none")
+            ax.xaxis.grid(False)
+        fg.set_titles("")
+        fg.set(yticks=[], ylabel="")
+
+        def label(x, color, label):
+            ax = plt.gca()
+            ax.text(
+                x=0.04,
+                y=0,
+                s=label,
+                ha="right",
+                va="bottom",
+                transform=ax.transAxes,
+            )
+
+        fg.map(label, "ranks")
+        fg.despine(left=True)
+        if self.add_titles:
+            fg.figure.suptitle(f"{score_id} {name}", y=0.975)
+        else:
+            fg.figure.suptitle("")
+
+        path = os.path.join("_plots", score_id)
+        self.save_plot(fg.figure, name.lower().replace(" ", "_"), path)
+        warnings.filters = _old_warn
+
+    def plot_correlation_matrix(self, df: pd.DataFrame, name: str) -> None:
+        """Create and plot the correlation matrix of the scoring functions.
+
+        Args:
+            df: DataFrame containing multiple scoring functions per column.
+            name: Name of the correlation matrix plot.
+        """
+        df.sort_index(axis="columns", inplace=True)
+        fig_height = int(1 + len(df.columns))
+        fig = plt.figure(
+            figsize=(max(self.figsize[0], fig_height), fig_height)
+        )
+        correlation_matrix = df.corr(method="spearman")
+        correlation_matrix.to_csv(
+            os.path.join(self.output_directory, f"{name}.csv")
+        )
+        percent_char = "\\%" if self.latex else "%"
+
+        def escape_percent(x):
+            return f"{(x*100):.2f}{percent_char}"
+
+        sns.heatmap(
+            correlation_matrix,
+            annot=correlation_matrix.map(escape_percent),
+            fmt="",
+            cbar=False,
+            cmap="crest",
+        )
+        fig.gca().set_xticklabels(df.columns, rotation=25)
+        self.save_plot(fig, name)
+
+    def plot_run(self, df: pd.DataFrame) -> None:
+        """Plot the dynamic dataset size determined by the curriculum.
+
+        Args:
+            df: DataFrame containing the run metrics and dataset sizes.
+        """
+        fig = plt.figure(figsize=self.figsize)
+        sns.lineplot(
+            data=df["dataset_size"], dashes=False, drawstyle="steps-post"
+        )
+        self._add_label(fig.gca(), self.training_type, "dataset size")
+        self.save_plot(fig, "pace")
+
+    def plot_pace(self, df: pd.DataFrame) -> None:
+        """Plot the dynamic dataset size of multiple runs.
+
+        Args:
+            df: DataFrame containing multiple runs and their dataset sizes.
+        """
+        fig = plt.figure(figsize=self.figsize)
+        sns.lineplot(data=df, dashes=False, drawstyle="steps-post")
+        self._add_label(fig.gca(), self.training_type, "dataset size")
+        self.save_plot(fig, "pace")

aucurriculum/core/scripts/__init__.py

@@ -0,0 +1,28 @@
+from .cli_wrapper import fetch, preprocess
+from .create_script import CreateCurriculumScript, create
+from .curriculum_script import CurriculumScript, curriculum
+from .group_script import GroupCurriculumScript, group
+from .list_script import ListCurriculumScript, list_configs
+from .postprocess_script import PostprocessCurriculumScript, postprocess
+from .show_script import ShowCurriculumScript, show
+from .train_script import TrainCurriculumScript, train
+
+
+__all__ = [
+    "create",
+    "CreateCurriculumScript",
+    "curriculum",
+    "CurriculumScript",
+    "fetch",
+    "group",
+    "GroupCurriculumScript",
+    "list_configs",
+    "ListCurriculumScript",
+    "postprocess",
+    "PostprocessCurriculumScript",
+    "preprocess",
+    "show",
+    "ShowCurriculumScript",
+    "train",
+    "TrainCurriculumScript",
+]

aucurriculum/core/scripts/cli.py

@@ -0,0 +1,79 @@
+import argparse
+from typing import List
+
+from autrainer.core.scripts import (
+    AbstractScript,
+    CommandLineError,
+    DeleteFailedScript,
+    DeleteStatesScript,
+    FetchScript,
+    InferenceScript,
+    PreprocessScript,
+)
+from autrainer.core.scripts.cli import CLI as AutrainerCLI
+
+import aucurriculum
+from aucurriculum.core.scripts import (
+    CreateCurriculumScript,
+    CurriculumScript,
+    GroupCurriculumScript,
+    ListCurriculumScript,
+    PostprocessCurriculumScript,
+    ShowCurriculumScript,
+    TrainCurriculumScript,
+)
+
+
+class CLI(AutrainerCLI):
+    def __init__(self) -> None:
+        super().__init__()
+        self._update_metadata()
+
+    def _init_scripts(self) -> List[AbstractScript]:
+        return [
+            # configuration management
+            CreateCurriculumScript(),
+            ListCurriculumScript(),
+            ShowCurriculumScript(),
+            # preprocessing
+            FetchScript(),
+            PreprocessScript(),
+            # training
+            TrainCurriculumScript(),
+            # inference
+            InferenceScript(),
+            # postprocessing
+            PostprocessCurriculumScript(),
+            DeleteFailedScript(),
+            DeleteStatesScript(),
+            GroupCurriculumScript(),
+            # curriculum learning
+            CurriculumScript(),
+        ]
+
+    def _update_metadata(self) -> None:
+        self.parser.prog = "aucurriculum"
+        self.parser.description = (
+            "A Curriculum Learning Toolkit for Deep Learning Tasks "
+            "built on top of autrainer."
+        )
+        for action in self.parser._actions:
+            if isinstance(action, argparse._VersionAction):
+                action.version = f"%(prog)s {aucurriculum.__version__}"
+
+
+def main() -> None:
+    cli = CLI()
+    try:
+        cli.main()
+    except CommandLineError as e:
+        e.handle()
+
+
+def get_parser() -> argparse.ArgumentParser:
+    cli = CLI()  # pragma: no cover
+    return cli.parser  # pragma: no cover
+
+
+if __name__ == "__main__":
+    main()

aucurriculum/core/scripts/cli_wrapper.py

@@ -0,0 +1,96 @@
+from typing import Optional
+
+from autrainer.core.scripts import FetchScript, PreprocessScript
+from autrainer.core.scripts.abstract_script import MockParser
+from autrainer.core.scripts.fetch_script import PreprocessArgs as FetchArgs
+from autrainer.core.scripts.preprocess_script import PreprocessArgs
+from autrainer.core.scripts.utils import (
+    add_hydra_args_to_sys,
+    catch_cli_errors,
+    run_hydra_cmd,
+    running_in_notebook,
+)
+
+
+@catch_cli_errors
+def fetch(
+    override_kwargs: Optional[dict] = None,
+    cfg_launcher: bool = False,
+    config_name: str = "config",
+    config_path: Optional[str] = None,
+) -> None:
+    """Fetch the datasets and models specified in a training configuration.
+
+    Args:
+        override_kwargs: Additional Hydra override arguments to pass to the
+            train script.
+        cfg_launcher: Use the launcher specified in the configuration instead
+            of the Hydra basic launcher. Defaults to False.
+        config_name: The name of the config (usually the file name without the
+            .yaml extension). Defaults to "config".
+        config_path: The config path, a directory where Hydra will search for
+            config files. If config_path is None no directory is added to the
+            search path. Defaults to None.
+    """
+    if running_in_notebook():
+        run_hydra_cmd(
+            "fetch -l" if cfg_launcher else "fetch",
+            override_kwargs,
+            config_name,
+            config_path,
+            cmd_prefix="aucurriculum",
+        )
+
+    else:
+        add_hydra_args_to_sys(override_kwargs, config_name, config_path)
+        script = FetchScript()
+        script.parser = MockParser()
+        script.main(FetchArgs(cfg_launcher))
+
+
+@catch_cli_errors
+def preprocess(
+    override_kwargs: Optional[dict] = None,
+    num_workers: int = 1,
+    update_frequency: int = 1,
+    cfg_launcher: bool = False,
+    config_name: str = "config",
+    config_path: Optional[str] = None,
+) -> None:
+    """Launch a data preprocessing configuration.
+
+    Args:
+        override_kwargs: Additional Hydra override arguments to pass to the
+            train script.
+        num_workers: Number of workers (threads) to use for preprocessing.
+            Defaults to 1.
+        update_frequency: Frequency of progress bar updates for each worker
+            (thread). If 0, the progress bar will be disabled. Defaults to 1.
+        cfg_launcher: Use the launcher specified in the configuration instead
+            of the Hydra basic launcher. Defaults to False.
+        config_name: The name of the config (usually the file name without the
+            .yaml extension). Defaults to "config".
+        config_path: The config path, a directory where Hydra will search for
+            config files. If config_path is None no directory is added to the
+            search path. Defaults to None.
+    """
+    if running_in_notebook():
+        cmd = "preprocess"
+        if cfg_launcher:
+            cmd += " -l"
+        cmd += f" -n {num_workers} -u {update_frequency}"
+        run_hydra_cmd(
+            cmd,
+            override_kwargs,
+            config_name,
+            config_path,
+            cmd_prefix="aucurriculum",
+        )
+
+    else:
+        add_hydra_args_to_sys(override_kwargs, config_name, config_path)
+        script = PreprocessScript()
+        script.parser = MockParser()
+        script.main(
+            PreprocessArgs(cfg_launcher, num_workers, update_frequency)
+        )