gptmed 0.3.5__py3-none-any.whl → 0.4.0__py3-none-any.whl

gptmed/services/training_service.py

@@ -26,7 +26,7 @@ import torch
 import random
 import numpy as np
 from pathlib import Path
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 
 from gptmed.services.device_manager import DeviceManager
 from gptmed.model.architecture import GPTTransformer
@@ -35,6 +35,13 @@ from gptmed.configs.train_config import TrainingConfig
 from gptmed.training.dataset import create_dataloaders
 from gptmed.training.trainer import Trainer
 
+# Observability imports
+from gptmed.observability import (
+    TrainingObserver,
+    MetricsTracker,
+    ConsoleCallback,
+)
+
 
 class TrainingService:
     """
@@ -162,7 +169,8 @@ class TrainingService:
         optimizer,
         train_config: TrainingConfig,
         device: str,
-        model_config_dict: dict
+        model_config_dict: dict,
+        observers: Optional[List[TrainingObserver]] = None,
     ) -> Dict[str, Any]:
         """
         Execute the training process.
@@ -175,13 +183,20 @@ class TrainingService:
             train_config: Training configuration
             device: Device to use
             model_config_dict: Model configuration as dictionary
+            observers: Optional list of TrainingObserver instances.
+                       If None, default observers (MetricsTracker) will be used.
             
         Returns:
             Dictionary with training results
         """
-        # Create trainer
+        # Set up default observers if none provided
+        if observers is None:
+            observers = self._create_default_observers(train_config)
+        
+        # Create trainer with observers
         if self.verbose:
             print(f"\n🎯 Initializing trainer...")
+            print(f"   Observers: {len(observers)} ({', '.join(o.name for o in observers)})")
         
         trainer = Trainer(
             model=model,
@@ -190,6 +205,7 @@ class TrainingService:
             optimizer=optimizer,
             config=train_config,
             device=device,
+            observers=observers,
         )
         
         # Resume if requested
@@ -209,9 +225,11 @@ class TrainingService:
             print("🚀 Starting Training!")
             print(f"{'='*60}\n")
         
+        interrupted = False
         try:
             trainer.train()
         except KeyboardInterrupt:
+            interrupted = True
             if self.verbose:
                 print("\n\n⏸️  Training interrupted by user")
                 print("💾 Saving checkpoint...")
@@ -227,29 +245,165 @@ class TrainingService:
             if self.verbose:
                 print("✓ Checkpoint saved. Resume with resume_from in config.")
         
+        # Generate observability reports (on BOTH normal and abnormal exit)
+        self._generate_observability_reports(
+            observers=observers,
+            train_config=train_config,
+            trainer=trainer,
+            interrupted=interrupted,
+        )
+        
         # Return results
-        best_checkpoint = Path(train_config.checkpoint_dir) / "best_model.pt"
+        final_checkpoint = Path(train_config.checkpoint_dir) / "final_model.pt"
         
         results = {
-            'best_checkpoint': str(best_checkpoint),
+            'final_checkpoint': str(final_checkpoint),
+            'best_checkpoint': str(final_checkpoint),  # Alias for backward compatibility
             'final_val_loss': trainer.best_val_loss,
             'total_epochs': trainer.current_epoch,
+            'total_steps': trainer.global_step,
             'checkpoint_dir': train_config.checkpoint_dir,
             'log_dir': train_config.log_dir,
+            'interrupted': interrupted,
         }
         
+        # Get training issues from metrics tracker
+        metrics_tracker = self._get_metrics_tracker(observers)
+        if metrics_tracker:
+            results['training_issues'] = metrics_tracker.detect_issues()
+        
         if self.verbose:
+            status = "⏸️  Training Interrupted" if interrupted else "✅ Training Complete!"
             print(f"\n{'='*60}")
-            print("✅ Training Complete!")
+            print(status)
             print(f"{'='*60}")
             print(f"\n📁 Results:")
-            print(f"  Best checkpoint: {results['best_checkpoint']}")
+            print(f"  Final checkpoint: {results['final_checkpoint']}")
             print(f"  Best val loss: {results['final_val_loss']:.4f}")
+            print(f"  Total steps: {results['total_steps']}")
             print(f"  Total epochs: {results['total_epochs']}")
             print(f"  Logs: {results['log_dir']}")
         
         return results
     
+    def _generate_observability_reports(
+        self,
+        observers: List[TrainingObserver],
+        train_config: TrainingConfig,
+        trainer,
+        interrupted: bool = False,
+    ) -> None:
+        """
+        Generate observability reports from metrics tracker.
+        
+        Called on both normal completion and abnormal exit (Ctrl+C).
+        
+        Args:
+            observers: List of training observers
+            train_config: Training configuration
+            trainer: Trainer instance
+            interrupted: Whether training was interrupted
+        """
+        metrics_tracker = self._get_metrics_tracker(observers)
+        
+        if not metrics_tracker:
+            if self.verbose:
+                print("\n⚠️  No MetricsTracker found - skipping observability reports")
+            return
+        
+        if self.verbose:
+            print(f"\n{'='*60}")
+            print("📊 Generating Observability Reports")
+            print(f"{'='*60}")
+        
+        try:
+            # Export metrics to CSV
+            csv_path = metrics_tracker.export_to_csv()
+            if self.verbose:
+                print(f"   ✓ CSV exported: {csv_path}")
+        except Exception as e:
+            if self.verbose:
+                print(f"   ✗ CSV export failed: {e}")
+        
+        try:
+            # Export metrics to JSON
+            json_path = metrics_tracker.export_to_json()
+            if self.verbose:
+                print(f"   ✓ JSON exported: {json_path}")
+        except Exception as e:
+            if self.verbose:
+                print(f"   ✗ JSON export failed: {e}")
+        
+        try:
+            # Generate loss curve plots
+            plot_path = metrics_tracker.plot_loss_curves()
+            if plot_path and self.verbose:
+                print(f"   ✓ Loss curves plotted: {plot_path}")
+        except ImportError:
+            if self.verbose:
+                print(f"   ⚠️  Plotting skipped (matplotlib not installed)")
+        except Exception as e:
+            if self.verbose:
+                print(f"   ✗ Plotting failed: {e}")
+        
+        # Training health check
+        issues = metrics_tracker.detect_issues()
+        if self.verbose:
+            print(f"\n📋 Training Health Check:")
+            for issue in issues:
+                print(f"   {issue}")
+        
+        # Add interrupted notice if applicable
+        if interrupted and self.verbose:
+            print(f"\n⚠️  Note: Training was interrupted at step {trainer.global_step}")
+            print(f"   Reports reflect partial training data only.")
+    
+    def _create_default_observers(self, train_config: TrainingConfig) -> List[TrainingObserver]:
+        """
+        Create default observers for training.
+        
+        Args:
+            train_config: Training configuration
+            
+        Returns:
+            List of default TrainingObserver instances
+        """
+        observers = []
+        
+        # MetricsTracker - comprehensive metrics logging
+        metrics_tracker = MetricsTracker(
+            log_dir=train_config.log_dir,
+            experiment_name="gptmed_training",
+            moving_avg_window=100,
+            log_interval=train_config.log_interval,
+            verbose=self.verbose,
+        )
+        observers.append(metrics_tracker)
+        
+        # Note: ConsoleCallback is optional since Trainer already has console output
+        # Uncomment if you want additional formatted console output:
+        # console_callback = ConsoleCallback(log_interval=train_config.log_interval)
+        # observers.append(console_callback)
+        
+        return observers
+    
+    def _get_metrics_tracker(self, observers: List[TrainingObserver]) -> Optional[MetricsTracker]:
+        """
+        Get MetricsTracker from observers list if present.
+        
+        Args:
+            observers: List of observers
+            
+        Returns:
+            MetricsTracker instance or None
+        """
+        for obs in observers:
+            if isinstance(obs, MetricsTracker):
+                return obs
+        return None
+        
+        return results
+    
     def train(
         self,
         model_size: str,

gptmed/training/trainer.py

@@ -49,7 +49,7 @@ import torch.nn as nn
 from torch.utils.data import DataLoader
 import time
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List
 
 from gptmed.model.architecture import GPTTransformer
 from gptmed.training.utils import (
@@ -62,6 +62,15 @@ from gptmed.training.utils import (
 from gptmed.utils.logging import MetricsLogger, log_training_step, log_validation
 from gptmed.utils.checkpoints import CheckpointManager
 
+# New observability imports
+from gptmed.observability.base import (
+    TrainingObserver,
+    ObserverManager,
+    StepMetrics,
+    ValidationMetrics,
+    GradientMetrics,
+)
+
 
 class Trainer:
     """
@@ -83,6 +92,7 @@ class Trainer:
         optimizer: torch.optim.Optimizer,
         config,  # TrainingConfig
         device: str = "cuda",
+        observers: List[TrainingObserver] = None,
     ):
         """
         Args:
@@ -92,6 +102,7 @@ class Trainer:
             optimizer: Optimizer (e.g., AdamW)
             config: TrainingConfig object
             device: Device to train on
+            observers: List of TrainingObserver instances for monitoring
         """
         self.model = model.to(device)
         self.train_loader = train_loader
@@ -100,7 +111,13 @@ class Trainer:
         self.config = config
         self.device = device
 
-        # Initialize utilities
+        # Initialize observability
+        self.observer_manager = ObserverManager()
+        if observers:
+            for obs in observers:
+                self.observer_manager.add(obs)
+
+        # Initialize utilities (keep for backward compatibility)
         self.logger = MetricsLogger(log_dir=config.log_dir, experiment_name="gpt_training")
 
         self.checkpoint_manager = CheckpointManager(
@@ -124,17 +141,32 @@ class Trainer:
         print(f"  Total steps: {self.total_steps}")
         print(f"  Steps per epoch: {steps_per_epoch}")
         print(f"  Num epochs: {config.num_epochs}")
+        print(f"  Observers: {len(self.observer_manager.observers)}")
+
+    def add_observer(self, observer: TrainingObserver) -> None:
+        """
+        Add an observer for training monitoring.
+        
+        Args:
+            observer: TrainingObserver instance
+        """
+        self.observer_manager.add(observer)
+        print(f"  Added observer: {observer.name}")
 
-    def train_step(self, batch: tuple) -> dict:
+    def train_step(self, batch: tuple, step: int = 0, lr: float = 0.0) -> dict:
         """
         Single training step.
 
         Args:
             batch: (input_ids, target_ids) tuple
+            step: Current global step (for observer metrics)
+            lr: Current learning rate (for observer metrics)
 
         Returns:
             Dictionary with step metrics
         """
+        step_start_time = time.time()
+        
         # Move batch to device
         input_ids, target_ids = batch
         input_ids = input_ids.to(self.device)
@@ -163,14 +195,34 @@ class Trainer:
         # Optimizer step
         self.optimizer.step()
 
-        # Return metrics
-        return {
+        # Calculate tokens per second
+        step_time = time.time() - step_start_time
+        tokens_per_sec = (batch_size * seq_len) / step_time if step_time > 0 else 0
+
+        # Create metrics dict (for backward compatibility)
+        metrics_dict = {
             "loss": loss.item(),
             "grad_norm": grad_norm,
             "batch_size": batch_size,
             "seq_len": seq_len,
+            "tokens_per_sec": tokens_per_sec,
         }
 
+        # Notify observers with StepMetrics
+        step_metrics = StepMetrics(
+            step=step,
+            loss=loss.item(),
+            learning_rate=lr,
+            grad_norm=grad_norm,
+            batch_size=batch_size,
+            seq_len=seq_len,
+            tokens_per_sec=tokens_per_sec,
+        )
+        self.observer_manager.notify_step(step_metrics)
+
+        # Return metrics
+        return metrics_dict
+
     def evaluate(self) -> dict:
         """
         Evaluate on validation set.
@@ -188,6 +240,14 @@ class Trainer:
 
         log_validation(self.global_step, val_loss, val_perplexity)
 
+        # Notify observers
+        val_metrics = ValidationMetrics(
+            step=self.global_step,
+            val_loss=val_loss,
+            val_perplexity=val_perplexity,
+        )
+        self.observer_manager.notify_validation(val_metrics)
+
         return {"val_loss": val_loss, "val_perplexity": val_perplexity}
 
     def train(self):
@@ -200,17 +260,37 @@ class Trainer:
         print("Starting Training")
         print("=" * 60)
 
+        # Notify observers of training start
+        train_config = {
+            "model_size": getattr(self.model.config, 'model_size', 'unknown'),
+            "device": self.device,
+            "batch_size": self.config.batch_size,
+            "learning_rate": self.config.learning_rate,
+            "num_epochs": self.config.num_epochs,
+            "max_steps": self.config.max_steps,
+            "total_steps": self.total_steps,
+            "warmup_steps": self.config.warmup_steps,
+            "grad_clip": self.config.grad_clip,
+            "weight_decay": self.config.weight_decay,
+        }
+        self.observer_manager.notify_train_start(train_config)
+
         self.model.train()
 
         # Training loop
         for epoch in range(self.config.num_epochs):
             self.current_epoch = epoch
 
+            # Notify observers of epoch start
+            self.observer_manager.notify_epoch_start(epoch)
+
             print(f"\n{'='*60}")
             print(f"Epoch {epoch + 1}/{self.config.num_epochs}")
             print(f"{'='*60}")
 
             epoch_start_time = time.time()
+            epoch_loss_sum = 0.0
+            epoch_steps = 0
 
             for batch_idx, batch in enumerate(self.train_loader):
                 step_start_time = time.time()
@@ -226,8 +306,12 @@ class Trainer:
                 )
                 set_learning_rate(self.optimizer, lr)
 
-                # Training step
-                metrics = self.train_step(batch)
+                # Training step (now with step and lr for observers)
+                metrics = self.train_step(batch, step=self.global_step, lr=lr)
+
+                # Track epoch loss
+                epoch_loss_sum += metrics["loss"]
+                epoch_steps += 1
 
                 # Calculate tokens per second
                 step_time = time.time() - step_start_time
@@ -243,7 +327,7 @@ class Trainer:
                         tokens_per_sec=tokens_per_sec,
                     )
 
-                # Log metrics
+                # Log metrics (legacy logger)
                 self.logger.log(
                     self.global_step,
                     {
@@ -269,7 +353,7 @@ class Trainer:
                         is_best = False
 
                     # Save checkpoint
-                    self.checkpoint_manager.save_checkpoint(
+                    checkpoint_path = self.checkpoint_manager.save_checkpoint(
                         model=self.model,
                         optimizer=self.optimizer,
                         step=self.global_step,
@@ -280,8 +364,19 @@ class Trainer:
                         is_best=is_best,
                     )
 
+                    # Notify observers of checkpoint
+                    if checkpoint_path:
+                        self.observer_manager.notify_checkpoint(self.global_step, str(checkpoint_path))
+
                     self.model.train()  # Back to training mode
 
+                    # Check for early stopping (if any observer requests it)
+                    for obs in self.observer_manager.observers:
+                        if hasattr(obs, 'should_stop') and obs.should_stop:
+                            print(f"\nEarly stopping requested by {obs.name}")
+                            self._finish_training()
+                            return
+
                 # Save checkpoint periodically
                 if self.global_step % self.config.save_interval == 0 and self.global_step > 0:
                     self.checkpoint_manager.save_checkpoint(
@@ -299,17 +394,36 @@ class Trainer:
                 # Check if reached max steps
                 if self.config.max_steps > 0 and self.global_step >= self.config.max_steps:
                     print(f"\nReached max_steps ({self.config.max_steps}). Stopping training.")
+                    self._finish_training()
                     return
 
-            # End of epoch
+            # End of epoch - notify observers
             epoch_time = time.time() - epoch_start_time
+            epoch_avg_loss = epoch_loss_sum / epoch_steps if epoch_steps > 0 else 0
+            self.observer_manager.notify_epoch_end(epoch, {
+                "train_loss": epoch_avg_loss,
+                "epoch_time": epoch_time,
+            })
             print(f"\nEpoch {epoch + 1} completed in {epoch_time:.2f}s")
 
+        self._finish_training()
+
+    def _finish_training(self):
+        """Finalize training and notify observers."""
         print("\n" + "=" * 60)
         print("Training Complete!")
         print("=" * 60)
         print(f"Best validation loss: {self.best_val_loss:.4f}")
 
+        # Notify observers of training end
+        final_metrics = {
+            "best_val_loss": self.best_val_loss,
+            "total_steps": self.global_step,
+            "final_epoch": self.current_epoch,
+            "final_checkpoint": str(self.checkpoint_manager.checkpoint_dir / "final_model.pt"),
+        }
+        self.observer_manager.notify_train_end(final_metrics)
+
     def resume_from_checkpoint(self, checkpoint_path: Optional[Path] = None):
         """
         Resume training from a checkpoint.

gptmed/utils/checkpoints.py

@@ -108,7 +108,7 @@ class CheckpointManager:
         # Save as best if applicable
         if is_best or val_loss < self.best_val_loss:
             self.best_val_loss = val_loss
-            best_path = self.checkpoint_dir / "best_model.pt"
+            best_path = self.checkpoint_dir / "final_model.pt"
             torch.save(checkpoint, best_path)
             print(f"Best model saved: {best_path} (val_loss: {val_loss:.4f})")