chronobench 0.1.1__py3-none-any.whl

chronobench/__init__.py

@@ -0,0 +1,16 @@
+"""
+Chronobench: a registry-driven framework for benchmarking machine learning experiments.
+"""
+
+from ._main import main
+from .context import Context
+from .registry import data_loaders, metrics, models, runner
+
+__all__ = [
+    "main",
+    "Context",
+    "data_loaders",
+    "models",
+    "metrics",
+    "runner",
+]

chronobench/_main.py

@@ -0,0 +1,61 @@
+"""
+Entry point that dispatches CLI arguments to the appropriate execution mode.
+"""
+
+from . import registry
+from .modes.dry_run import main as main_dry_run
+from .modes.run_many import main as main_run_many
+from .utility.loading import load_environment, load_experiment_dict
+from .utility.parsing_arguments import parse_args
+
+__all__ = ["main"]
+
+
+def main():
+    """
+    Run chronobench from the command line using the registered components.
+
+    This is the primary public API. Before calling it, register your data
+    loaders, models, and metrics on the module-level registries and mark your
+    training loop with :func:`chronobench.runner`::
+
+        import chronobench as cb
+        from src.loaders import IrisLoader
+
+        cb.data_loaders.register(iris=IrisLoader)
+
+        @cb.runner
+        def runner(data_loader, model, metrics):
+            ...
+
+        cb.main()
+
+    Experiment TOML entries select an implementation with either a registered
+    ``name`` or a dotted ``target`` import path, so stock library classes (e.g.
+    ``sklearn.svm.SVC``) can be used without a wrapper. ``main()`` reads the
+    registries and the registered runner directly; nothing is passed to it.
+    """
+    args = parse_args()
+    environment = load_environment(args.environment)
+    experiments = load_experiment_dict(environment, args.experiments)
+
+    if args.dry_run:
+        if args.debug:
+            print("Warning: --debug has no effect in dry-run mode.")
+        main_dry_run(
+            experiments=experiments,
+            data_loaders=registry.data_loaders,
+            models=registry.models,
+            metrics=registry.metrics,
+            environment=environment,
+        )
+    else:
+        main_run_many(
+            experiments=experiments,
+            debug=args.debug,
+            data_loaders=registry.data_loaders,
+            models=registry.models,
+            metrics=registry.metrics,
+            runner=registry.get_runner(),
+            environment=environment,
+        )

chronobench/context.py

@@ -0,0 +1,42 @@
+"""
+Context dataclass passed to user-supplied initializer callbacks.
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Context:
+    """
+    Cross-initializer context passed progressively to each callback.
+
+    The context is populated in stages as initialization proceeds:
+
+    - ``initialize_data_loaders`` receives ``Context()`` (all fields ``None``).
+    - ``initialize_models`` receives a ``Context`` with the data-loader fields set.
+    - ``initialize_metrics`` receives a fully populated ``Context``.
+
+    Attributes
+    ----------
+    data_loader : object, optional
+        The instantiated data loader for the current job.
+    data_loader_name : str, optional
+        The name of the current data loader as defined in the experiment
+        configuration.
+    data_loader_kwargs : dict, optional
+        The keyword arguments used to instantiate the current data loader.
+    model : object, optional
+        The instantiated model for the current job.
+    model_name : str, optional
+        The name of the current model as defined in the experiment
+        configuration.
+    model_kwargs : dict, optional
+        The keyword arguments used to instantiate the current model.
+    """
+
+    data_loader: object | None = None
+    data_loader_name: str | None = None
+    data_loader_kwargs: dict | None = None
+    model: object | None = None
+    model_name: str | None = None
+    model_kwargs: dict | None = None

chronobench/modes/dry_run.py

@@ -0,0 +1,220 @@
+"""
+Mode for validating experiment configurations without executing any jobs.
+
+The dry-run report is the primary output of this mode, so it is written
+directly to stdout with :func:`print` rather than through :mod:`logging`. This
+means it is always visible without the caller having to configure logging.
+"""
+
+import shutil
+
+from chronobench.context import Context
+from chronobench.registry import resolve
+from chronobench.utility.config_validation import (
+    validate_environment,
+    validate_experiment,
+)
+from chronobench.utility.handling_kwargs import (
+    expand_kwargs_data_loaders,
+    expand_kwargs_metrics,
+    expand_kwargs_models,
+)
+
+__all__ = ["main"]
+
+_PREFIX = "[dry-run] "
+
+
+def main(
+    experiments,
+    data_loaders,
+    models,
+    metrics,
+    environment,
+):
+    """
+    Validate all experiment configurations and report any failures.
+
+    Checks the environment configuration and each experiment's TOML
+    structure, then attempts to instantiate every data loader, model
+    variant, and metrics list. A summary of passed and failed checks is
+    printed at the end. No jobs are executed and no results are written.
+
+    Parameters
+    ----------
+    experiments : dict
+        Mapping from experiment name to its parsed configuration dict.
+    data_loaders : Registry
+        Registry used to resolve data-loader ``name`` selectors.
+    models : Registry
+        Registry used to resolve model ``name`` selectors.
+    metrics : Registry
+        Registry used to resolve metric ``name`` selectors.
+    environment : dict
+        Loaded environment configuration.
+
+    Notes
+    -----
+    Initializers are called in order with a progressively populated
+    ``Context``, mirroring the staged approach used during a real run.
+    The first successful result from each stage is used to build the
+    context for the next stage, so cross-initializer dependencies are
+    exercised during dry-run validation.
+    """
+    _full_line("=")
+    failures = []
+
+    try:
+        validate_environment(environment)
+    except Exception as e:
+        print(f"{_PREFIX}  FAILED Validating environment configuration")
+        failures.append(
+            {
+                "name": "Validation of environment",
+                "error": e,
+            }
+        )
+
+    for experiment_name, experiment in experiments.items():
+        _full_with_header(experiment_name, "=")
+
+        try:
+            validate_experiment(experiment_name, experiment)
+        except ValueError as e:
+            print(f"{_PREFIX}  FAILED Validating experiment configuration")
+            failures.append(
+                {
+                    "name": "Validation of toml file",
+                    "error": e,
+                }
+            )
+            continue
+
+        print(f"{_PREFIX}>>> Data")
+        dl_results = _check_and_print(
+            environment,
+            expand_kwargs_data_loaders(experiment),
+            data_loaders,
+            experiment_name,
+            "data",
+            failures,
+        )
+
+        dl_ctx = (
+            Context(
+                data_loader=dl_results[0][2],
+                data_loader_name=dl_results[0][0],
+                data_loader_kwargs=dl_results[0][1],
+            )
+            if dl_results
+            else Context()
+        )
+
+        print(f"{_PREFIX}>>> Models")
+        model_results = _check_and_print(
+            environment,
+            expand_kwargs_models(experiment),
+            models,
+            experiment_name,
+            "models",
+            failures,
+            context=dl_ctx,
+        )
+
+        if dl_results and model_results:
+            metric_ctx = Context(
+                data_loader=dl_results[0][2],
+                data_loader_name=dl_results[0][0],
+                data_loader_kwargs=dl_results[0][1],
+                model=model_results[0][2],
+                model_name=model_results[0][0],
+                model_kwargs=model_results[0][1],
+            )
+        else:
+            metric_ctx = Context()
+
+        print(f"{_PREFIX}>>> Metrics")
+        _check_and_print(
+            environment,
+            expand_kwargs_metrics(experiment),
+            metrics,
+            experiment_name,
+            "metrics",
+            failures,
+            context=metric_ctx,
+        )
+
+    _print_summary(failures)
+
+
+def _check_and_print(
+    environment,
+    pairs,
+    registry,
+    experiment_name,
+    selector_key,
+    failures,
+    context=None,
+):
+    if context is None:
+        context = Context()
+    results = []
+    defer = selector_key == "metrics"
+    for selector, kwargs in pairs:
+        try:
+            result = resolve(
+                registry,
+                selector,
+                environment,
+                context,
+                defer_if_incomplete=defer,
+                **kwargs,
+            )
+            instances = result if isinstance(result, list) else [result]
+            results.append((selector.label, kwargs, instances[0]))
+            print(f"{_PREFIX}  PASSED {selector.label} {kwargs}")
+        except Exception as e:
+            print(f"{_PREFIX}  FAILED {selector.label} {kwargs}")
+            failures.append(
+                {
+                    "name": f"{experiment_name} / {selector_key} / {selector.label} {kwargs}",
+                    "error": e,
+                }
+            )
+    return results
+
+
+def _print_summary(failures):
+    _full_with_header("SUMMARY", "=")
+
+    if not failures:
+        print(f"{_PREFIX}All configurations passed successfully!")
+        print(_PREFIX)
+
+        _full_line("=")
+        return
+
+    print(f"{_PREFIX}{len(failures)} configuration(s) failed:")
+    for i, failure in enumerate(failures, start=1):
+        print(f"{_PREFIX}  [{i}] {failure['name']}")
+        print(
+            f"{_PREFIX}      {failure['error'].__class__.__name__}: {failure['error']}"
+        )
+
+    _full_line("=")
+
+
+def _prefix_len() -> int:
+    return len(_PREFIX)
+
+
+def _full_with_header(name: str, symbol: str):
+    width = shutil.get_terminal_size().columns - _prefix_len()
+    print(_PREFIX)
+    print(f"{_PREFIX}{symbol * 3 + ' ' + name + ' ':{symbol}<{width}}")
+    print(_PREFIX)
+
+
+def _full_line(symbol: str):
+    width = shutil.get_terminal_size().columns - _prefix_len()
+    print(f"{_PREFIX}{symbol * width}")

chronobench/modes/run.py

@@ -0,0 +1,240 @@
+"""
+Mode for executing a single experiment and saving results to CSV.
+"""
+
+import csv
+import logging
+import multiprocessing
+import pathlib
+from datetime import datetime
+from functools import partial
+from typing import NamedTuple
+
+try:
+    from tqdm import tqdm
+except ImportError:  # tqdm is an optional dependency
+
+    class tqdm:  # type: ignore[no-redef]  # noqa: N801
+        """No-op stand-in used when tqdm is not installed."""
+
+        def __init__(self, iterable=None, *args, **kwargs):
+            self._iterable = iterable if iterable is not None else []
+
+        def __iter__(self):
+            return iter(self._iterable)
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            return False
+
+        def update(self, n=1):
+            pass
+
+
+from chronobench.context import Context
+from chronobench.utility.handling_kwargs import (
+    initialize_all_data_loaders,
+    initialize_all_metrics,
+    initialize_all_models,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class Job(NamedTuple):
+    """
+    One unit of work: a single (data loader, model) combination.
+
+    Produced by the Cartesian product of all data loader variants and model
+    variants before execution begins.
+    """
+
+    loader_name: str
+    loader_kwargs: dict
+    data_loader: object
+    model_name: str
+    model_kwargs: dict
+    model: object
+
+
+def main(
+    name,
+    experiment,
+    data_loaders,
+    models,
+    metrics,
+    runner,
+    debug,
+    environment,
+):
+    """
+    Execute all jobs for a single experiment and write results to CSV.
+
+    Jobs are the Cartesian product of all data loader variants and model
+    variants. When ``n-jobs > 1`` in the environment configuration, jobs
+    run in parallel via :class:`multiprocessing.Pool`. Results are
+    appended to a timestamped CSV file under ``path-to-results/raw/``.
+
+    Parameters
+    ----------
+    name : str
+        Experiment name, used as the output subdirectory.
+    experiment : dict
+        Parsed experiment configuration.
+    data_loaders : Registry
+        Registry used to resolve data-loader ``name`` selectors. A factory
+        that returns a list fans out into one variant per element.
+    models : Registry
+        Registry used to resolve model ``name`` selectors. Resolved once per
+        data loader so each job receives its own model instance and can read
+        from the data-loader context. A factory that returns a list fans out
+        into one variant per element.
+    metrics : Registry
+        Registry used to resolve metric ``name`` selectors. Resolved once per
+        job so each job receives fresh metric instances.
+    runner : callable
+        Callback with signature ``(data_loader, model, metrics)`` returning a
+        ``dict`` of computed metric values.
+    debug : bool
+        When ``True``, limit execution to the first three jobs.
+    environment : dict
+        Loaded environment configuration.
+    """
+    dataloaders = initialize_all_data_loaders(environment, experiment, data_loaders)
+
+    jobs = []
+    for loader_name, loader_kwargs, data_loader in dataloaders:
+        dl_context = Context(
+            data_loader=data_loader,
+            data_loader_name=loader_name,
+            data_loader_kwargs=loader_kwargs,
+        )
+        for model_name, model_kwargs, model in initialize_all_models(
+            environment, experiment, models, context=dl_context
+        ):
+            jobs.append(
+                Job(
+                    loader_name,
+                    loader_kwargs,
+                    data_loader,
+                    model_name,
+                    model_kwargs,
+                    model,
+                )
+            )
+    if debug:
+        jobs = jobs[:3]
+
+    if environment["n-jobs"] == 1:
+        result = [
+            _single_job(
+                job,
+                metrics=metrics,
+                environment=environment,
+                experiment=experiment,
+                runner=runner,
+            )
+            for job in tqdm(jobs)
+        ]
+
+    else:
+        _single_job_function = partial(
+            _single_job,
+            metrics=metrics,
+            environment=environment,
+            experiment=experiment,
+            runner=runner,
+        )
+
+        with multiprocessing.Pool(processes=environment["n-jobs"]) as pool:
+            with tqdm(total=len(jobs)) as pbar:
+                result = [
+                    pool.apply_async(
+                        _single_job_function,
+                        args=(job,),
+                        callback=lambda _: pbar.update(1),
+                    )
+                    for job in jobs
+                ]
+                pool.close()
+                pool.join()
+
+        result = [r.get() for r in result]
+
+    save_results(environment, name, debug, result)
+
+
+def _single_job(job, metrics, environment, experiment, runner):
+    context = Context(
+        data_loader=job.data_loader,
+        data_loader_name=job.loader_name,
+        data_loader_kwargs=job.loader_kwargs,
+        model=job.model,
+        model_name=job.model_name,
+        model_kwargs=job.model_kwargs,
+    )
+    metric_instances = initialize_all_metrics(
+        environment, experiment, metrics, context=context
+    )
+    computed_metrics = runner(job.data_loader, job.model, metric_instances)
+
+    conflicting_keys = job.loader_kwargs.keys() & job.model_kwargs.keys()
+    return {
+        "Dataset": job.loader_name,
+        **{
+            f"Dataset.{k}" if k in conflicting_keys else k: v
+            for k, v in job.loader_kwargs.items()
+        },
+        "Model": job.model_name,
+        **{
+            f"Model.{k}" if k in conflicting_keys else k: v
+            for k, v in job.model_kwargs.items()
+        },
+        **computed_metrics,
+    }
+
+
+def save_results(
+    environment,
+    name,
+    debug,
+    rows,
+) -> None:
+    """
+    Write experiment results to a timestamped CSV file.
+
+    The output directory ``path-to-results/raw/<name>/`` is created if it
+    does not exist. Column names are derived from the union of all keys
+    present in ``rows``. Missing values for any row are written as empty
+    strings.
+
+    Parameters
+    ----------
+    environment : dict
+        Loaded environment configuration. Must contain ``path-to-results``.
+    name : str
+        Experiment name, used as the output subdirectory under
+        ``path-to-results/raw/``.
+    debug : bool
+        When ``True``, the output filename is prefixed with ``DEBUG_``.
+    rows : list of dict
+        One dict per job; keys become CSV column headers.
+    """
+    base_path = pathlib.Path(environment["path-to-results"]).resolve() / "raw" / name
+    if not base_path.exists():
+        logger.info("Creating directory %s", base_path)
+        base_path.mkdir(parents=True)
+
+    results_path = (
+        base_path
+        / f"{'DEBUG_' if debug else ''}{datetime.now().strftime('%Y%m%d-%H%M%S')}.csv"
+    )
+
+    fieldnames = list(dict.fromkeys(k for row in rows for k in row.keys()))
+
+    with open(results_path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames, restval="")
+        writer.writeheader()
+        writer.writerows(rows)

chronobench/modes/run_many.py

@@ -0,0 +1,51 @@
+"""
+Mode for executing multiple experiments sequentially.
+"""
+
+from chronobench.modes.run import main as main_run
+
+
+def main(
+    experiments,
+    data_loaders,
+    models,
+    metrics,
+    runner,
+    debug,
+    environment,
+):
+    """
+    Execute each experiment in ``experiments`` in sequence.
+
+    Each experiment is delegated to :func:`chronobench.modes.run.main`.
+    Results are written to separate CSV files under ``path-to-results``.
+
+    Parameters
+    ----------
+    experiments : dict
+        Mapping from experiment name to its parsed configuration dict.
+    data_loaders : Registry
+        Registry used to resolve data-loader ``name`` selectors.
+    models : Registry
+        Registry used to resolve model ``name`` selectors.
+    metrics : Registry
+        Registry used to resolve metric ``name`` selectors.
+    runner : callable
+        Callback with signature ``(data_loader, model, metrics)`` returning a
+        ``dict`` of computed metric values.
+    debug : bool
+        When ``True``, limit each experiment to the first three jobs.
+    environment : dict
+        Loaded environment configuration.
+    """
+    for name, experiment in experiments.items():
+        main_run(
+            name=name,
+            experiment=experiment,
+            data_loaders=data_loaders,
+            models=models,
+            metrics=metrics,
+            runner=runner,
+            debug=debug,
+            environment=environment,
+        )