loopkit 0.0.1a1__tar.gz

loopkit-0.0.1a1/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2025 Kim Kunhee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

loopkit-0.0.1a1/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: loopkit
+Version: 0.0.1a1
+Author-email: Kunhee Kim <kunhee.kim@kaist.ac.kr>
+License: MIT
+Project-URL: Repository, https://github.com/kunheek/loopkit
+Keywords: machine-learning,deep-learning,experiment-tracking,pytorch,configuration,logging,reproducibility,research,cli
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: matplotlib>=3.5.0
+Requires-Dist: polars>=0.20.0
+Requires-Dist: psutil>=5.9.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: wandb
+Requires-Dist: wandb>=0.15.0; extra == "wandb"
+Provides-Extra: tensorboard
+Requires-Dist: tensorboard>=2.13.0; extra == "tensorboard"
+Provides-Extra: tracking
+Requires-Dist: wandb>=0.15.0; extra == "tracking"
+Requires-Dist: tensorboard>=2.13.0; extra == "tracking"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Dynamic: license-file
+
+# LoopKit

loopkit-0.0.1a1/README.md

@@ -0,0 +1 @@
+# LoopKit

loopkit-0.0.1a1/pyproject.toml

@@ -0,0 +1,68 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "loopkit"
+version = "0.0.1a1"
+description = ""
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+  {name = "Kunhee Kim", email = "kunhee.kim@kaist.ac.kr"}
+]
+keywords = [
+  "machine-learning",
+  "deep-learning",
+  "experiment-tracking",
+  "pytorch",
+  "configuration",
+  "logging",
+  "reproducibility",
+  "research",
+  "cli"
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">=3.8"
+dependencies = [
+  "numpy>=1.21.0",
+  "matplotlib>=3.5.0",
+  "polars>=0.20.0",
+  "psutil>=5.9.0",
+  "pyyaml>=6.0",
+]
+
+[project.urls]
+Repository = "https://github.com/kunheek/loopkit"
+
+[project.scripts]
+loopkit = "loopkit.cli:main"
+
+[project.optional-dependencies]
+wandb = ["wandb>=0.15.0"]
+tensorboard = ["tensorboard>=2.13.0"]
+tracking = ["wandb>=0.15.0", "tensorboard>=2.13.0"]
+dev = [
+  "pytest>=7.0.0",
+  "pytest-cov>=4.0.0",
+  "black>=23.0.0",
+  "ruff>=0.1.0",
+  "mypy>=1.0.0",
+  "pre-commit>=3.0.0"
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]

loopkit-0.0.1a1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

loopkit-0.0.1a1/src/loopkit/__init__.py

@@ -0,0 +1,85 @@
+import os
+
+__version__ = "0.0.1a1"
+
+# Import submodules to make them available
+from . import config, git, logger, monitor, tracking, utils
+
+# Import optional modules
+try:
+    from . import torch
+except ImportError:
+    torch = None  # type: ignore
+
+# Import commonly used classes and functions
+from .config import Config, override_config
+from .logger import ExperimentLogger
+from .utils import get_gpu_devices, set_seed, str2bool
+
+# Export public API
+__all__ = [
+    "__version__",
+    "Config",
+    "ExperimentLogger",
+    "config",
+    "debug",
+    "force_single_process",
+    "get_gpu_devices",
+    "git",
+    "log_level",
+    "logger",
+    "monitor",
+    "override_config",
+    "save_checkpoints",
+    "set_seed",
+    "str2bool",
+    "timer",
+    "torch",
+    "track_git",
+    "track_metrics",
+    "tracking",
+    "utils",
+    "verbose",
+]
+
+# Global configuration flags
+# These can be modified at runtime or set via environment variables
+
+# Debug mode - enables debug logging and additional checks
+# Set via: loopkit.debug = True or LK_DEBUG=1
+debug = str2bool(os.environ.get("LK_DEBUG", "0"))
+
+# Logging level - default log level for ExperimentLogger instances
+# Options: DEBUG, INFO, WARNING, ERROR, CRITICAL
+# Set via: loopkit.log_level = "DEBUG" or LK_LOG_LEVEL=DEBUG
+log_level = os.environ.get("LK_LOG_LEVEL", "INFO").upper()
+
+# Profiling - controls whether logger.timer() performs timing measurements
+# Disable for production to eliminate profiling overhead
+# Set via: loopkit.timer = False or LK_TIMER=0
+timer = str2bool(os.environ.get("LK_TIMER", "1"))
+
+# Verbose mode - controls console output verbosity
+# Disable for cleaner logs when running many experiments
+# Set via: loopkit.verbose = False or LK_VERBOSE=0
+verbose = str2bool(os.environ.get("LK_VERBOSE", "1"))
+
+# Checkpoint saving - controls whether to save model checkpoints
+# Disable for quick experiments or testing to skip I/O
+# Set via: loopkit.save_checkpoints = False or LK_SAVE_CHECKPOINTS=0
+save_checkpoints = str2bool(os.environ.get("LK_SAVE_CHECKPOINTS", "1"))
+
+# Metrics tracking - controls whether to log metrics to CSV
+# Disable for faster iteration when metrics aren't needed
+# Set via: loopkit.track_metrics = False or LK_TRACK_METRICS=0
+track_metrics = str2bool(os.environ.get("LK_TRACK_METRICS", "1"))
+
+# Git tracking - controls whether to collect git repository information
+# Disable to skip git operations (useful in CI/CD or when git is slow)
+# Set via: loopkit.track_git = False or LK_TRACK_GIT=0
+track_git = str2bool(os.environ.get("LK_TRACK_GIT", "1"))
+
+# Force single process - force single-process mode in distributed environments
+# Useful for debugging distributed code without multiple processes
+# Set via: loopkit.force_single_process = True or LK_FORCE_SINGLE=1
+force_single_process = str2bool(os.environ.get("LK_FORCE_SINGLE", "0"))

loopkit-0.0.1a1/src/loopkit/cli/__init__.py

@@ -0,0 +1,3 @@
+from .main import main
+
+__all__ = ["main"]

loopkit-0.0.1a1/src/loopkit/cli/compare.py

@@ -0,0 +1,274 @@
+import argparse
+import json
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import polars as pl
+import yaml
+
+
+def load_run_info(run_dir: Union[str, Path]) -> Dict:
+    """Load information about a single run.
+
+    Args:
+        run_dir: Path to run directory
+
+    Returns:
+        Dict with run information including config, metrics, git info
+    """
+    run_dir = Path(run_dir)
+    info = {"run_dir": str(run_dir), "run_id": run_dir.name}
+
+    # Load config
+    config_file = run_dir / "run.yaml"
+    if config_file.exists():
+        with open(config_file) as f:
+            info["config"] = yaml.safe_load(f)
+
+    # Load best metrics
+    best_file = run_dir / "best.json"
+    if best_file.exists():
+        with open(best_file) as f:
+            info["best_metrics"] = json.load(f)
+
+    # Load final metrics from CSV
+    metrics_file = run_dir / "metrics.csv"
+    if metrics_file.exists():
+        df = pl.read_csv(metrics_file)
+        if not df.is_empty():
+            # Get last values for each metric
+            final_metrics = {}
+            for name in df["name"].unique():
+                metric_df = df.filter(pl.col("name") == name)
+                if not metric_df.is_empty():
+                    final_metrics[name] = metric_df.tail(1)["value"][0]
+            info["final_metrics"] = final_metrics
+
+    # Load git info
+    git_file = run_dir / "git_info.json"
+    if git_file.exists():
+        with open(git_file) as f:
+            info["git"] = json.load(f)
+
+    return info
+
+
+def compare_runs(
+    run_dirs: List[Union[str, Path]],
+    metrics: Optional[List[str]] = None,
+    show_config: bool = True,
+    show_git: bool = False,
+) -> pl.DataFrame:
+    """Compare multiple experiment runs.
+
+    Args:
+        run_dirs: List of run directories to compare
+        metrics: Specific metrics to compare (None = all)
+        show_config: Whether to include config values
+        show_git: Whether to include git info
+
+    Returns:
+        DataFrame with comparison
+    """
+    runs_data = []
+
+    for run_dir in run_dirs:
+        info = load_run_info(run_dir)
+        row = {"run_id": info["run_id"], "run_dir": info["run_dir"]}
+
+        # Add metrics
+        if "best_metrics" in info:
+            for metric_name, metric_info in info["best_metrics"].items():
+                if metrics is None or metric_name in metrics:
+                    row[f"best_{metric_name}"] = metric_info.get("value")
+                    row[f"best_{metric_name}_step"] = metric_info.get("step")
+
+        if "final_metrics" in info:
+            for metric_name, value in info["final_metrics"].items():
+                if metrics is None or metric_name in metrics:
+                    row[f"final_{metric_name}"] = value
+
+        # Add config
+        if show_config and "config" in info:
+            config = info["config"]
+            # Flatten config
+            for key, value in _flatten_dict(config).items():
+                row[f"config.{key}"] = value
+
+        # Add git info
+        if show_git and "git" in info:
+            row["git_sha"] = info["git"]["sha_short"]
+            row["git_dirty"] = info["git"]["dirty"]
+
+        runs_data.append(row)
+
+    return pl.DataFrame(runs_data)
+
+
+def _flatten_dict(d: Dict, parent_key: str = "", sep: str = ".") -> Dict:
+    """Flatten nested dictionary."""
+    items = []
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(_flatten_dict(v, new_key, sep=sep).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
+def find_config_diff(run_dirs: List[Union[str, Path]]) -> Dict[str, List]:
+    """Find configuration differences between runs.
+
+    Args:
+        run_dirs: List of run directories
+
+    Returns:
+        Dict mapping config keys to their values across runs
+    """
+    all_configs = []
+    for run_dir in run_dirs:
+        info = load_run_info(run_dir)
+        if "config" in info:
+            all_configs.append(_flatten_dict(info["config"]))
+
+    if not all_configs:
+        return {}
+
+    # Find keys that differ
+    all_keys = set()
+    for config in all_configs:
+        all_keys.update(config.keys())
+
+    diffs = {}
+    for key in sorted(all_keys):
+        values = [config.get(key, None) for config in all_configs]
+        # Only include if values differ
+        if len(set(str(v) for v in values)) > 1:
+            diffs[key] = values
+
+    return diffs
+
+
+def print_comparison(
+    run_dirs: List[Union[str, Path]],
+    metrics: Optional[List[str]] = None,
+    show_all_config: bool = False,
+):
+    """Print a formatted comparison of runs.
+
+    Args:
+        run_dirs: List of run directories to compare
+        metrics: Specific metrics to compare (None = all)
+        show_all_config: If False, only show differing config values
+    """
+    print(f"\n{'=' * 80}")
+    print(f"Comparing {len(run_dirs)} runs")
+    print(f"{'=' * 80}\n")
+
+    # Load run info
+    runs_info = [load_run_info(rd) for rd in run_dirs]
+
+    # Print run IDs
+    print("Run IDs:")
+    for i, info in enumerate(runs_info):
+        print(f"  [{i}] {info['run_id']}")
+    print()
+
+    # Print config differences
+    if show_all_config:
+        print("Configuration:")
+        df = compare_runs(run_dirs, show_config=True, show_git=False)
+        config_cols = [c for c in df.columns if c.startswith("config.")]
+        if config_cols:
+            for col in sorted(config_cols):
+                print(f"  {col}:")
+                for i, val in enumerate(df[col]):
+                    print(f"    [{i}] {val}")
+        print()
+    else:
+        diffs = find_config_diff(run_dirs)
+        if diffs:
+            print("Configuration Differences:")
+            for key, values in diffs.items():
+                print(f"  {key}:")
+                for i, val in enumerate(values):
+                    print(f"    [{i}] {val}")
+            print()
+        else:
+            print("No configuration differences found.\n")
+
+    # Print metrics comparison
+    print("Metrics Comparison:")
+    df = compare_runs(run_dirs, metrics=metrics, show_config=False, show_git=False)
+
+    # Show best metrics
+    best_cols = [
+        c for c in df.columns if c.startswith("best_") and not c.endswith("_step")
+    ]
+    if best_cols:
+        print("\n  Best Metrics:")
+        for col in sorted(best_cols):
+            metric_name = col.replace("best_", "")
+            print(f"    {metric_name}:")
+            for i, val in enumerate(df[col]):
+                step_col = f"best_{metric_name}_step"
+                step = df[step_col].iloc[i] if step_col in df.columns else "?"
+                print(f"      [{i}] {val:.4f} (step {step})")
+
+    # Show final metrics
+    final_cols = [c for c in df.columns if c.startswith("final_")]
+    if final_cols:
+        print("\n  Final Metrics:")
+        for col in sorted(final_cols):
+            metric_name = col.replace("final_", "")
+            print(f"    {metric_name}:")
+            for i, val in enumerate(df[col]):
+                print(f"      [{i}] {val:.4f}")
+
+    print(f"\n{'=' * 80}\n")
+
+
+def setup_compare_parser(subparsers):
+    """Setup the compare subcommand parser."""
+    compare_parser = subparsers.add_parser(
+        "compare",
+        help="Compare multiple experiment runs",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Compare runs
+  loopkit compare runs/exp1 runs/exp2 runs/exp3
+
+  # Compare specific metrics
+  loopkit compare runs/exp_* --metrics loss accuracy
+
+  # Show all config (not just diffs)
+  loopkit compare runs/exp1 runs/exp2 --all-config
+
+  # Export to CSV
+  loopkit compare runs/exp_* --export comparison.csv
+        """,
+    )
+    compare_parser.add_argument(
+        "run_dirs", nargs="+", help="Run directories to compare"
+    )
+    compare_parser.add_argument(
+        "--metrics", "-m", nargs="+", help="Specific metrics to compare"
+    )
+    compare_parser.add_argument(
+        "--all-config", action="store_true", help="Show all config, not just diffs"
+    )
+    compare_parser.add_argument("--export", "-e", help="Export comparison to CSV file")
+
+
+def run_compare_command(args):
+    """Run the compare command."""
+    if args.export:
+        df = compare_runs(args.run_dirs, metrics=args.metrics)
+        df.write_csv(args.export)
+        print(f"✅ Comparison exported to {args.export}")
+    else:
+        print_comparison(
+            args.run_dirs, metrics=args.metrics, show_all_config=args.all_config
+        )

loopkit-0.0.1a1/src/loopkit/cli/main.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+
+
+def main():
+    """Main CLI entrypoint for LoopKit."""
+    parser = argparse.ArgumentParser(
+        prog="loopkit",
+        description="LoopKit CLI",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Commands:
+  sweep        Run hyperparameter sweeps
+  viz          Visualize experiment metrics
+  compare      Compare multiple experiment runs
+
+Examples:
+  loopkit sweep "python train.py {config}" --config config.yaml --sweep "lr=[0.001,0.01]"
+  loopkit viz runs/exp_* --metrics loss accuracy
+  loopkit compare runs/exp1 runs/exp2 runs/exp3
+        """,
+    )
+
+    parser.add_argument("--version", action="version", version="%(prog)s 0.1.0")
+
+    subparsers = parser.add_subparsers(dest="command", help="Command to run")
+
+    # Import command setup functions
+    from .sweep import setup_sweep_parser
+    from .visualize import setup_visualize_parser
+    from .compare import setup_compare_parser
+
+    # Setup subcommands
+    setup_sweep_parser(subparsers)
+    setup_visualize_parser(subparsers)
+    setup_compare_parser(subparsers)
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    # Dispatch to appropriate command
+    if args.command == "sweep":
+        from .sweep import run_sweep_command
+
+        run_sweep_command(args)
+    elif args.command in ("viz", "visualize"):
+        from .visualize import run_visualize_command
+
+        run_visualize_command(args)
+    elif args.command == "compare":
+        from .compare import run_compare_command
+
+        run_compare_command(args)
+
+
+if __name__ == "__main__":
+    main()