arclm 0.1.0__py3-none-any.whl

arclm/__init__.py

@@ -0,0 +1,154 @@
+"""ArcLM - compact PyTorch language-model training and fine-tuning."""
+
+import torch
+
+from .config import Config, create_config
+from .data import DataBundle, load_tokens, prepare_data, read_tokens, split_train_val
+from .dataset import TextDataset, create_dataloader
+from .diagnostics import (
+    ConceptBenchmarkCase,
+    ConceptBenchmarkResult,
+    DEFAULT_CONCEPT_BENCHMARKS,
+    LongContextResult,
+    MetricsReport,
+    TopKPrediction,
+    build_training_diagnostics_report,
+    calculate_metrics,
+    calculate_perplexity,
+    export_metrics_to_json,
+    export_metrics_to_markdown,
+    format_concept_benchmark_report,
+    format_long_context_results,
+    format_tokenizer_coverage_report,
+    format_top_k_predictions,
+    predict_top_k,
+    run_long_context_evaluation,
+    score_concept_relationships,
+)
+from .generator import Generator
+from .inference import DEFAULT_MODEL_PATH, LoadedModel, load_model, predict
+from .instruction_dataset import InstructionDataset, create_instruction_dataloader
+from .logics import (
+    And,
+    Biconditional,
+    Implication,
+    Not,
+    Or,
+    Sentence,
+    Symbol,
+    model_check,
+)
+from .model import ArcLM, MiniGPT
+from .pipeline import (
+    build_model,
+    build_trainer,
+    checkpoint_is_compatible_for_continue_training,
+    checkpoint_is_compatible_for_tuining,
+    create_epoch_checkpoint_callback,
+    load_compatible_checkpoint,
+    save_training_checkpoint,
+)
+from .pipeline_v2 import ModelAdapter, PreTrainedModelLoader, StoppingCriteria, UnifiedPipeline
+from .regularization import (
+    EarlyStopping,
+    GeneralizationMonitor,
+    L1Regularization,
+    L2Regularization,
+    LabelSmoothing,
+    LearningRateScheduler,
+    MixupAugmentation,
+)
+from .tokenizer import SentencePieceTokenizer, Tokenizer, create_tokenizer, get_tokenizer_from_config
+from .trainer import Trainer
+from .utils import format_duration
+
+__version__ = "0.1.0"
+__author__ = "ArcLM Contributors"
+
+__all__ = [
+    "ArcLM",
+    "MiniGPT",
+    "Config",
+    "create_config",
+    "Tokenizer",
+    "SentencePieceTokenizer",
+    "create_tokenizer",
+    "get_tokenizer_from_config",
+    "TextDataset",
+    "create_dataloader",
+    "DataBundle",
+    "load_tokens",
+    "prepare_data",
+    "read_tokens",
+    "split_train_val",
+    "Trainer",
+    "Generator",
+    "DEFAULT_MODEL_PATH",
+    "LoadedModel",
+    "load_model",
+    "predict",
+    "build_model",
+    "build_trainer",
+    "checkpoint_is_compatible_for_continue_training",
+    "checkpoint_is_compatible_for_tuining",
+    "create_epoch_checkpoint_callback",
+    "load_compatible_checkpoint",
+    "save_training_checkpoint",
+    "UnifiedPipeline",
+    "PreTrainedModelLoader",
+    "ModelAdapter",
+    "StoppingCriteria",
+    "build_training_diagnostics_report",
+    "calculate_metrics",
+    "calculate_perplexity",
+    "export_metrics_to_json",
+    "export_metrics_to_markdown",
+    "ConceptBenchmarkCase",
+    "ConceptBenchmarkResult",
+    "DEFAULT_CONCEPT_BENCHMARKS",
+    "LongContextResult",
+    "MetricsReport",
+    "TopKPrediction",
+    "format_concept_benchmark_report",
+    "format_long_context_results",
+    "format_tokenizer_coverage_report",
+    "format_top_k_predictions",
+    "predict_top_k",
+    "run_long_context_evaluation",
+    "score_concept_relationships",
+    "format_duration",
+    "L1Regularization",
+    "L2Regularization",
+    "EarlyStopping",
+    "LearningRateScheduler",
+    "GeneralizationMonitor",
+    "MixupAugmentation",
+    "LabelSmoothing",
+    "create_instruction_dataloader",
+    "InstructionDataset",
+    "Sentence",
+    "Symbol",
+    "Not",
+    "And",
+    "Or",
+    "Implication",
+    "Biconditional",
+    "model_check",
+]
+
+
+def get_version():
+    return __version__
+
+
+def list_available_models():
+    from pathlib import Path
+
+    models_dir = Path("models")
+    if not models_dir.exists():
+        return []
+    return sorted(path.stem for path in models_dir.glob("*.pth") if path.is_file())
+
+
+def load_model_checkpoint(path: str, device_type: str):
+    return torch.load(path, map_location=torch.device(device_type))

arclm/api.py

@@ -0,0 +1,167 @@
+"""
+ArcLM public API.
+
+Example:
+    >>> from arclm import Config, UnifiedPipeline, load_model
+    >>> config = Config(embed_dim=128, learning_rate=1e-3)
+    >>> pipeline = UnifiedPipeline(config, mode="pre_training")
+    >>> pipeline.build(vocab_size=10000)
+    >>> results = pipeline.train(train_loader, num_epochs=5)
+"""
+
+__version__ = "0.1.0"
+__author__ = "ArcLM Contributors"
+__all__ = [
+    # Core Configuration
+    "Config",
+    
+    # Training API (P4-1)
+    "UnifiedPipeline",
+    "StoppingCriteria",
+    "PreTrainedModelLoader",
+    "ModelAdapter",
+    
+    # Tokenizers (P4-2)
+    "create_tokenizer",
+    "get_tokenizer_from_config",
+    "Tokenizer",
+    "SentencePieceTokenizer",
+    
+    # Metrics & Evaluation (P4-3)
+    "calculate_metrics",
+    "calculate_perplexity",
+    "export_metrics_to_json",
+    "export_metrics_to_markdown",
+    "MetricsReport",
+    
+    # Model Loading (Inference)
+    "load_model",
+    "LoadedModel",
+    "predict",
+    
+    # Advanced: Config Files & Experiment Tracking (P4-4 Advanced)
+    "load_config",
+    "load_config_yaml",
+    "load_config_json",
+    "save_config",
+    "save_config_yaml",
+    "save_config_json",
+    "ExperimentTracker",
+    "create_experiment",
+    "list_experiments",
+    
+    # Utilities
+    "create_config",
+]
+
+# Lazy imports for faster module loading
+def __getattr__(name):
+    """Lazy load modules on first use"""
+    if name == "Config":
+        from .config import Config
+        return Config
+    elif name == "create_config":
+        from .config import create_config
+        return create_config
+    elif name == "UnifiedPipeline":
+        from .pipeline_v2 import UnifiedPipeline
+        return UnifiedPipeline
+    elif name == "StoppingCriteria":
+        from .pipeline_v2 import StoppingCriteria
+        return StoppingCriteria
+    elif name == "PreTrainedModelLoader":
+        from .pipeline_v2 import PreTrainedModelLoader
+        return PreTrainedModelLoader
+    elif name == "ModelAdapter":
+        from .pipeline_v2 import ModelAdapter
+        return ModelAdapter
+    elif name == "create_tokenizer":
+        from .tokenizer import create_tokenizer
+        return create_tokenizer
+    elif name == "get_tokenizer_from_config":
+        from .tokenizer import get_tokenizer_from_config
+        return get_tokenizer_from_config
+    elif name == "Tokenizer":
+        from .tokenizer import Tokenizer
+        return Tokenizer
+    elif name == "SentencePieceTokenizer":
+        from .tokenizer import SentencePieceTokenizer
+        return SentencePieceTokenizer
+    elif name == "calculate_metrics":
+        from .diagnostics import calculate_metrics
+        return calculate_metrics
+    elif name == "calculate_perplexity":
+        from .diagnostics import calculate_perplexity
+        return calculate_perplexity
+    elif name == "export_metrics_to_json":
+        from .diagnostics import export_metrics_to_json
+        return export_metrics_to_json
+    elif name == "export_metrics_to_markdown":
+        from .diagnostics import export_metrics_to_markdown
+        return export_metrics_to_markdown
+    elif name == "MetricsReport":
+        from .diagnostics import MetricsReport
+        return MetricsReport
+    elif name == "load_model":
+        from .inference import load_model
+        return load_model
+    elif name == "LoadedModel":
+        from .inference import LoadedModel
+        return LoadedModel
+    elif name == "predict":
+        from .inference import predict
+        return predict
+    elif name == "load_config":
+        from .config_loader import load_config
+        return load_config
+    elif name == "load_config_yaml":
+        from .config_loader import load_config_yaml
+        return load_config_yaml
+    elif name == "load_config_json":
+        from .config_loader import load_config_json
+        return load_config_json
+    elif name == "save_config":
+        from .config_loader import save_config
+        return save_config
+    elif name == "save_config_yaml":
+        from .config_loader import save_config_yaml
+        return save_config_yaml
+    elif name == "save_config_json":
+        from .config_loader import save_config_json
+        return save_config_json
+    elif name == "ExperimentTracker":
+        from .tracking import ExperimentTracker
+        return ExperimentTracker
+    elif name == "create_experiment":
+        from .tracking import create_experiment
+        return create_experiment
+    elif name == "list_experiments":
+        from .tracking import list_experiments
+        return list_experiments
+    else:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def get_version():
+    """Get library version"""
+    return __version__
+
+
+def list_available_models():
+    """
+    List available pre-trained model checkpoints.
+    
+    Returns:
+        list: Available model names
+    """
+    from pathlib import Path
+    models_dir = Path(__file__).parent.parent / "models"
+    
+    if not models_dir.exists():
+        return []
+    
+    models = [
+        f.stem for f in models_dir.glob("*.pth") 
+        if f.is_file()
+    ]
+    return sorted(models)

arclm/cli.py

@@ -0,0 +1,385 @@
+"""
+ArcLM Command-Line Interface (CLI)
+
+Provides command-line tools for training, evaluation, and generation.
+
+Available Commands:
+  - arclm train    : Train a model from scratch or fine-tune
+  - arclm eval     : Evaluate model performance
+  - arclm generate : Generate text from a trained model
+
+Usage:
+  arclm train --config config.yaml --data data.txt
+  arclm eval --model models/model.pth --data test.txt
+  arclm generate --model models/model.pth --prompt "Hello"
+"""
+
+import argparse
+import sys
+import json
+from pathlib import Path
+from typing import Optional
+import torch
+
+from arclm import (
+    Config,
+    UnifiedPipeline,
+    calculate_metrics,
+    export_metrics_to_json,
+    export_metrics_to_markdown,
+    load_model,
+    create_tokenizer,
+    TextDataset,
+    create_dataloader,
+)
+from arclm.config_loader import load_config_yaml, load_config_json
+
+
+def create_train_parser(subparsers):
+    """Create parser for arclm train command"""
+    parser = subparsers.add_parser(
+        "train",
+        help="Train a model from scratch or fine-tune",
+        description="Train ArcLM model with configuration file or CLI arguments"
+    )
+    
+    # Config input
+    parser.add_argument(
+        "--config", type=str, default=None,
+        help="Path to config file (YAML or JSON) - overrides CLI args"
+    )
+    
+    # Model config
+    parser.add_argument("--embed-dim", type=int, default=128, help="Embedding dimension")
+    parser.add_argument("--num-blocks", type=int, default=4, help="Number of transformer blocks")
+    parser.add_argument("--block-size", type=int, default=512, help="Context window size")
+    parser.add_argument("--num-heads", type=int, default=4, help="Number of attention heads")
+    parser.add_argument("--learning-rate", type=float, default=1e-3, help="Learning rate")
+    
+    # Training config
+    parser.add_argument("--batch-size", type=int, default=16, help="Batch size")
+    parser.add_argument("--epochs", type=int, default=5, help="Number of epochs")
+    parser.add_argument("--mode", type=str, default="pre_training",
+                        choices=["pre_training", "fine_tuning", "instruction_tuning"],
+                        help="Training mode")
+    
+    # Data & checkpoint
+    parser.add_argument("--data", type=str, required=True, help="Path to training data file")
+    parser.add_argument("--output", type=str, default="models/trained_model.pth",
+                        help="Output model checkpoint path")
+    parser.add_argument("--pretrained", type=str, default=None,
+                        help="Path to pre-trained model (for fine-tuning)")
+    
+    # Logging & tracking
+    parser.add_argument("--experiment", type=str, default=None,
+                        help="Experiment name (for tracking)")
+    parser.add_argument("--log-dir", type=str, default="logs",
+                        help="Directory for logs and reports")
+    
+    parser.set_defaults(func=train_command)
+    return parser
+
+
+def create_eval_parser(subparsers):
+    """Create parser for arclm eval command"""
+    parser = subparsers.add_parser(
+        "eval",
+        help="Evaluate model performance",
+        description="Evaluate trained model on validation/test data"
+    )
+    
+    parser.add_argument("--model", type=str, required=True,
+                        help="Path to trained model checkpoint")
+    parser.add_argument("--data", type=str, required=True,
+                        help="Path to evaluation data")
+    parser.add_argument("--config", type=str, default=None,
+                        help="Path to config file (if not in model checkpoint)")
+    parser.add_argument("--output", type=str, default="metrics_report.json",
+                        help="Output metrics file")
+    parser.add_argument("--batch-size", type=int, default=32,
+                        help="Batch size for evaluation")
+    parser.add_argument("--device", type=str, default="auto",
+                        choices=["cpu", "cuda", "auto"],
+                        help="Device to use for evaluation")
+    
+    parser.set_defaults(func=eval_command)
+    return parser
+
+
+def create_generate_parser(subparsers):
+    """Create parser for arclm generate command"""
+    parser = subparsers.add_parser(
+        "generate",
+        help="Generate text from trained model",
+        description="Use trained model to generate text"
+    )
+    
+    parser.add_argument("--model", type=str, required=True,
+                        help="Path to trained model checkpoint")
+    parser.add_argument("--prompt", type=str, required=True,
+                        help="Prompt to generate from")
+    parser.add_argument("--length", type=int, default=100,
+                        help="Number of tokens to generate")
+    parser.add_argument("--temperature", type=float, default=1.0,
+                        help="Sampling temperature (higher = more random)")
+    parser.add_argument("--num-samples", type=int, default=1,
+                        help="Number of samples to generate")
+    parser.add_argument("--device", type=str, default="auto",
+                        choices=["cpu", "cuda", "auto"],
+                        help="Device to use for generation")
+    
+    parser.set_defaults(func=generate_command)
+    return parser
+
+
+def train_command(args):
+    """Execute training"""
+    print("\n" + "="*70)
+    print("ArcLM Training")
+    print("="*70)
+    
+    # Load config
+    if args.config:
+        print(f"\n📂 Loading config from: {args.config}")
+        if args.config.endswith(".yaml") or args.config.endswith(".yml"):
+            config = load_config_yaml(args.config)
+        elif args.config.endswith(".json"):
+            config = load_config_json(args.config)
+        else:
+            print(f"❌ Unknown config format: {args.config}")
+            return 1
+    else:
+        print("\n⚙️ Creating config from CLI arguments")
+        config = Config(
+            embed_dim=args.embed_dim,
+            num_blocks=args.num_blocks,
+            block_size=args.block_size,
+            num_heads=args.num_heads,
+            learning_rate=args.learning_rate,
+            batch_size=args.batch_size,
+        )
+    
+    print(f"✓ Config loaded: {config.embed_dim}D, {config.num_blocks} blocks")
+    
+    # Load data
+    try:
+        print(f"\n📊 Loading data from: {args.data}")
+        with open(args.data, "r") as f:
+            text = f.read()
+        
+        tokenizer = create_tokenizer("word", max_vocab=10000)
+        dataset = TextDataset(text, tokenizer, seq_len=config.block_size)
+        train_loader = create_dataloader(dataset, batch_size=config.batch_size)
+        print(f"✓ Data loaded: {len(dataset)} samples")
+    except FileNotFoundError:
+        print(f"❌ Data file not found: {args.data}")
+        return 1
+    except Exception as e:
+        print(f"❌ Error loading data: {e}")
+        return 1
+    
+    # Create pipeline
+    print(f"\n🔧 Creating pipeline (mode: {args.mode})")
+    pipeline = UnifiedPipeline(config, mode=args.mode)
+    
+    # Load pretrained if needed
+    if args.pretrained:
+        print(f"📦 Loading pre-trained model: {args.pretrained}")
+        from arclm import PreTrainedModelLoader
+        loader = PreTrainedModelLoader(args.pretrained)
+        model, _ = loader.load()
+        pipeline.build(vocab_size=tokenizer.max_vocab, pretrained_model=model)
+    else:
+        pipeline.build(vocab_size=tokenizer.max_vocab)
+    
+    print("✓ Pipeline ready")
+    
+    # Train
+    print(f"\n🚀 Starting training ({args.epochs} epochs)...")
+    try:
+        results = pipeline.train(train_loader, num_epochs=args.epochs)
+        print(f"✓ Training complete!")
+        print(f"  - Final loss: {results['final_loss']:.4f}")
+    except Exception as e:
+        print(f"❌ Training error: {e}")
+        return 1
+    
+    # Save model
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    torch.save(pipeline.model.state_dict(), output_path)
+    print(f"\n💾 Model saved to: {output_path}")
+    
+    # Save config
+    config_path = output_path.parent / "config.json"
+    with open(config_path, "w") as f:
+        json.dump(config.to_dict(), f, indent=2)
+    print(f"💾 Config saved to: {config_path}")
+    
+    # Log experiment
+    if args.experiment:
+        log_dir = Path(args.log_dir) / args.experiment
+        log_dir.mkdir(parents=True, exist_ok=True)
+        
+        log_file = log_dir / "training_log.json"
+        with open(log_file, "w") as f:
+            json.dump({
+                "model_path": str(output_path),
+                "config_path": str(config_path),
+                "final_loss": float(results["final_loss"]),
+                "epochs": args.epochs,
+            }, f, indent=2)
+        print(f"📝 Experiment logged: {log_file}")
+    
+    print("\n" + "="*70 + "\n")
+    return 0
+
+
+def eval_command(args):
+    """Execute evaluation"""
+    print("\n" + "="*70)
+    print("ArcLM Evaluation")
+    print("="*70)
+    
+    # Load model
+    try:
+        print(f"\n📦 Loading model from: {args.model}")
+        model = load_model(args.model, device=args.device)
+        print("✓ Model loaded")
+    except FileNotFoundError:
+        print(f"❌ Model file not found: {args.model}")
+        return 1
+    except Exception as e:
+        print(f"❌ Error loading model: {e}")
+        return 1
+    
+    # Load config
+    if args.config:
+        if args.config.endswith(".yaml"):
+            config = load_config_yaml(args.config)
+        else:
+            config = load_config_json(args.config)
+    else:
+        config_path = Path(args.model).parent / "config.json"
+        if config_path.exists():
+            config = load_config_json(str(config_path))
+        else:
+            print("⚠️ Config not found, using defaults")
+            config = Config()
+    
+    # Load data
+    try:
+        print(f"\n📊 Loading data from: {args.data}")
+        with open(args.data, "r") as f:
+            text = f.read()
+        
+        tokenizer = create_tokenizer("word", max_vocab=10000)
+        dataset = TextDataset(text, tokenizer, seq_len=config.block_size)
+        val_loader = create_dataloader(dataset, batch_size=args.batch_size)
+        print(f"✓ Data loaded: {len(dataset)} samples")
+    except Exception as e:
+        print(f"❌ Error loading data: {e}")
+        return 1
+    
+    # Evaluate
+    print(f"\n📈 Evaluating...")
+    try:
+        device = torch.device("cuda" if args.device == "auto" and torch.cuda.is_available() else args.device)
+        metrics = calculate_metrics(model.model, val_loader, config, device)
+        
+        print(f"✓ Evaluation complete!")
+        print(f"  - Perplexity: {metrics.perplexity:.4f}")
+        print(f"  - Loss: {metrics.avg_loss:.4f}")
+        print(f"  - Accuracy: {metrics.accuracy:.4%}")
+        print(f"  - Tokens: {metrics.total_tokens}")
+    except Exception as e:
+        print(f"❌ Evaluation error: {e}")
+        return 1
+    
+    # Save metrics
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    export_metrics_to_json(metrics, str(output_path))
+    print(f"\n💾 Metrics saved to: {output_path}")
+    
+    # Also save markdown report
+    report_path = output_path.with_suffix(".md")
+    export_metrics_to_markdown(metrics, str(report_path))
+    print(f"📝 Report saved to: {report_path}")
+    
+    print("\n" + "="*70 + "\n")
+    return 0
+
+
+def generate_command(args):
+    """Execute text generation"""
+    print("\n" + "="*70)
+    print("ArcLM Text Generation")
+    print("="*70)
+    
+    # Load model
+    try:
+        print(f"\n📦 Loading model from: {args.model}")
+        model = load_model(args.model, device=args.device)
+        print("✓ Model loaded")
+    except FileNotFoundError:
+        print(f"❌ Model file not found: {args.model}")
+        return 1
+    except Exception as e:
+        print(f"❌ Error loading model: {e}")
+        return 1
+    
+    print(f"\n✍️ Prompt: {args.prompt}")
+    print(f"📝 Generating {args.length} tokens (temperature={args.temperature})...")
+    
+    try:
+        for i in range(args.num_samples):
+            result = model.predict(
+                args.prompt,
+                max_new_tokens=args.length,
+                temperature=args.temperature
+            )
+            
+            if args.num_samples > 1:
+                print(f"\n--- Sample {i+1} ---")
+            print(result)
+    except Exception as e:
+        print(f"❌ Generation error: {e}")
+        return 1
+    
+    print("\n" + "="*70 + "\n")
+    return 0
+
+
+def main():
+    """Main CLI entry point"""
+    parser = argparse.ArgumentParser(
+        description="ArcLM - command-line tools for training and inference",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  arclm train --config config.yaml --data data.txt --output models/model.pth
+  arclm eval --model models/model.pth --data test.txt
+  arclm generate --model models/model.pth --prompt "The future is"
+        """
+    )
+    
+    parser.add_argument("--version", action="version", version="arclm 0.1.0")
+    
+    subparsers = parser.add_subparsers(title="commands", dest="command")
+    
+    create_train_parser(subparsers)
+    create_eval_parser(subparsers)
+    create_generate_parser(subparsers)
+    
+    args = parser.parse_args()
+    
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return 0
+    
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())