modelforge-finetuning 2.1.3__tar.gz → 2.2.0__tar.gz

{modelforge_finetuning-2.1.3 → modelforge_finetuning-2.2.0}/ModelForge/cli.py

@@ -50,7 +50,31 @@ def check_huggingface_login():
 def main():
     """
     Main entry point for ModelForge CLI.
+
+    Subcommands:
+        modelforge        — start the web server
+        modelforge cli    — launch the interactive CLI wizard
     """
+    if len(sys.argv) > 1:
+        subcommand = sys.argv[1]
+        if subcommand == "cli":
+            try:
+                from .notebook_cli.wizard import main as cli_main
+            except ImportError as e:
+                logger.error(f"Failed to import CLI wizard or its dependencies: {e}")
+                print("\nThe interactive CLI requires optional dependencies that are not installed.")
+                print("Please install the CLI extras and try again, for example:")
+                print("  pip install \"modelforge-finetuning[cli]\"")
+                sys.exit(1)
+            cli_main()
+            return
+        else:
+            print(f"Unknown subcommand: '{subcommand}'")
+            print("Usage:")
+            print("  modelforge        Start the web server")
+            print("  modelforge cli    Launch the interactive CLI wizard")
+            sys.exit(1)
+
     print("\n" + "=" * 80)
     print("  __  __           _      _ _____                     ")
     print(" |  \\/  |         | |    | |  ___|                    ")

{modelforge_finetuning-2.1.3 → modelforge_finetuning-2.2.0}/ModelForge/evaluation/dataset_validator.py

@@ -87,9 +87,10 @@ class DatasetValidator:
             if len(required_fields) > 0:
                 for i, example in enumerate(dataset.select(range(min(10, len(dataset))))):
                     for field in required_fields:
-                        if not example.get(field):
-                            logger.warning(
-                                f"Example {i} has empty field '{field}'"
+                        value = example.get(field)
+                        if value is None or (isinstance(value, str) and not value.strip()):
+                            raise DatasetValidationError(
+                                f"Example {i} has empty required field '{field}'."
                             )
 
             logger.info(

{modelforge_finetuning-2.1.3 → modelforge_finetuning-2.2.0}/ModelForge/evaluation/metrics.py

@@ -29,20 +29,27 @@ class MetricsCalculator:
         """
         logger.info("Computing causal LM metrics")
 
+        import torch
+        import torch.nn.functional as F
+
         predictions, labels = eval_pred.predictions, eval_pred.label_ids
 
-        # For language modeling, predictions are logits
-        # Loss is computed by the trainer, we just calculate perplexity
-        if hasattr(eval_pred, 'loss') and eval_pred.loss is not None:
-            loss = eval_pred.loss
-        else:
-            # Fallback: estimate loss from predictions
-            loss = 0.0
+        logits = torch.from_numpy(np.array(predictions)).float()
+        lbl = torch.from_numpy(np.array(labels))
+
+        shift_logits = logits[..., :-1, :].contiguous()
+        shift_labels = lbl[..., 1:].contiguous()
 
-        perplexity = np.exp(loss) if loss > 0 else 0.0
+        loss = F.cross_entropy(
+            shift_logits.view(-1, shift_logits.size(-1)),
+            shift_labels.view(-1).long(),
+            ignore_index=-100,
+            reduction="mean",
+        )
+        perplexity = float(torch.exp(loss))
 
         metrics = {
-            "perplexity": float(perplexity),
+            "perplexity": perplexity,
             "eval_loss": float(loss),
         }
 
@@ -69,6 +76,9 @@ class MetricsCalculator:
 
         predictions, labels = eval_pred.predictions, eval_pred.label_ids
 
+        if predictions.ndim == 3:
+            predictions = np.argmax(predictions, axis=-1)
+
         # Decode predictions and labels
         if tokenizer is not None:
             # Replace -100 in labels (used for padding)
@@ -88,9 +98,9 @@ class MetricsCalculator:
                 )
 
                 metrics = {
-                    "rouge1": float(result["rouge1"].mid.fmeasure),
-                    "rouge2": float(result["rouge2"].mid.fmeasure),
-                    "rougeL": float(result["rougeL"].mid.fmeasure),
+                    "rouge1": float(result["rouge1"]),
+                    "rouge2": float(result["rouge2"]),
+                    "rougeL": float(result["rougeL"]),
                 }
 
                 logger.info(f"Seq2Seq metrics: {metrics}")

modelforge_finetuning-2.2.0/ModelForge/notebook_cli/__init__.py

@@ -0,0 +1,19 @@
+"""
+ModelForge Notebook CLI — public API.
+
+Provides an interactive wizard-style interface that calls the ModelForge
+Python APIs directly, without going through the REST/web layer.
+
+Install the required extras with:
+    pip install modelforge-finetuning[cli]
+
+Usage from a notebook cell:
+    from ModelForge.notebook_cli import run_cli
+    run_cli()
+
+Usage from a terminal:
+    modelforge-nb
+"""
+from .wizard import ModelForgeWizard, main as run_cli
+
+__all__ = ["ModelForgeWizard", "run_cli"]

modelforge_finetuning-2.2.0/ModelForge/notebook_cli/progress.py

@@ -0,0 +1,127 @@
+"""
+Notebook-aware progress display for ModelForge CLI.
+
+Automatically uses ipywidgets when running inside a Jupyter/Colab/VSCode
+notebook environment, and falls back to tqdm for plain terminal use.
+"""
+from __future__ import annotations
+
+
+def _is_notebook() -> bool:
+    """Return True if running inside a Jupyter-compatible notebook kernel."""
+    try:
+        from IPython import get_ipython
+        shell = get_ipython()
+        if shell is None:
+            return False
+        return shell.__class__.__name__ == "ZMQInteractiveShell"
+    except ImportError:
+        return False
+
+
+class ProgressDisplay:
+    """
+    Unified progress display that works in notebooks and terminals.
+
+    Usage::
+
+        display = ProgressDisplay()
+        display.start("Loading model...")
+        display.update(50, "Halfway there")
+        display.update(100, "Done!")
+        display.close()
+    """
+
+    def __init__(self):
+        self._notebook = _is_notebook()
+        self._bar = None
+        self._label = None
+        self._tqdm_bar = None
+        self._last_value = 0
+
+    def start(self, message: str = "Starting...") -> None:
+        """Initialise and display the progress bar."""
+        self._last_value = 0
+        if self._notebook:
+            self._start_widget(message)
+        else:
+            self._start_tqdm(message)
+
+    def _start_widget(self, message: str) -> None:
+        try:
+            import ipywidgets as widgets
+            from IPython.display import display
+
+            self._bar = widgets.IntProgress(
+                value=0,
+                min=0,
+                max=100,
+                description="",
+                bar_style="info",
+                layout=widgets.Layout(width="100%"),
+            )
+            self._label = widgets.Label(value=message)
+            box = widgets.VBox([self._label, self._bar])
+            display(box)
+        except ImportError:
+            # ipywidgets not available — fall through to tqdm
+            self._notebook = False
+            self._start_tqdm(message)
+
+    def _start_tqdm(self, message: str) -> None:
+        from tqdm import tqdm
+
+        self._tqdm_bar = tqdm(
+            total=100,
+            desc=message,
+            bar_format="{desc}: {percentage:3.0f}%|{bar}| [{elapsed}]",
+            dynamic_ncols=True,
+        )
+
+    def update(self, value: int, message: str = "") -> None:
+        """
+        Update progress.
+
+        Args:
+            value: Progress value 0-100.
+            message: Status message to display alongside the bar.
+        """
+        value = max(0, min(100, value))
+        if self._notebook:
+            if self._bar is not None:
+                self._bar.value = value
+                if value >= 100:
+                    self._bar.bar_style = "success"
+            if self._label is not None and message:
+                self._label.value = message
+        else:
+            if self._tqdm_bar is not None:
+                delta = value - self._last_value
+                if delta > 0:
+                    self._tqdm_bar.update(delta)
+                if message:
+                    self._tqdm_bar.set_description(message)
+        self._last_value = value
+
+    def close(self) -> None:
+        """Close/finalize the progress display."""
+        if self._notebook:
+            if self._bar is not None:
+                self._bar.bar_style = "success"
+        else:
+            if self._tqdm_bar is not None:
+                self._tqdm_bar.close()
+                self._tqdm_bar = None
+
+    def error(self) -> None:
+        """Mark the progress bar as errored."""
+        if self._notebook:
+            if self._bar is not None:
+                self._bar.bar_style = "danger"
+            if self._label is not None:
+                self._label.value = "Training failed."
+        else:
+            if self._tqdm_bar is not None:
+                self._tqdm_bar.set_description("Training failed")
+                self._tqdm_bar.close()
+                self._tqdm_bar = None

modelforge_finetuning-2.2.0/ModelForge/notebook_cli/prompts.py

@@ -0,0 +1,273 @@
+"""
+Interactive prompt steps for the ModelForge notebook CLI wizard.
+
+Each function handles one wizard step and returns a typed value.
+All prompts use questionary; display panels use rich.
+"""
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+import questionary
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+if TYPE_CHECKING:
+    from ..services.training_service import TrainingService
+
+console = Console()
+
+# ── Choices ────────────────────────────────────────────────────────────────
+
+_TASK_CHOICES = [
+    questionary.Choice("Text Generation  (causal LM fine-tuning)", value="text-generation"),
+    questionary.Choice("Summarization    (seq-to-seq fine-tuning)", value="summarization"),
+    questionary.Choice(
+        "Extractive QA    (question-answering fine-tuning)",
+        value="extractive-question-answering",
+    ),
+]
+
+_PROVIDER_CHOICES = [
+    questionary.Choice("HuggingFace  (default, broad compatibility)", value="huggingface"),
+    questionary.Choice("Unsloth      (faster training, single-GPU optimised)", value="unsloth"),
+]
+
+_STRATEGY_CHOICES = [
+    questionary.Choice("SFT    – Supervised Fine-Tuning (recommended for most tasks)", value="sft"),
+    questionary.Choice("QLoRA  – Quantised LoRA (memory-efficient fine-tuning)", value="qlora"),
+    questionary.Choice("DPO    – Direct Preference Optimisation", value="dpo"),
+    questionary.Choice("RLHF   – Preference alignment (DPO-based, no reward model needed)", value="rlhf"),
+]
+
+_TASK_FORMAT_HELP = {
+    "text-generation": 'JSON Lines with "input" and "output" fields.',
+    "summarization": 'JSON Lines with "document" and "summary" fields.',
+    "extractive-question-answering": 'JSON Lines with "context", "question", and "answers" fields.',
+}
+
+_STRATEGY_FORMAT_HELP = {
+    "dpo":  'JSON Lines with "prompt", "chosen", and "rejected" fields.',
+    "rlhf": 'JSON Lines with "prompt", "chosen", and "rejected" fields.',
+}
+
+
+# ── Step functions ──────────────────────────────────────────────────────────
+
+def prompt_task() -> str:
+    """Ask the user to select a fine-tuning task."""
+    return questionary.select(
+        "Select fine-tuning task:",
+        choices=_TASK_CHOICES,
+        style=questionary.Style([("selected", "fg:cyan bold")]),
+    ).ask()
+
+
+def prompt_model(recommended: str, alternatives: list[str]) -> str:
+    """
+    Ask the user to select or enter a model name.
+
+    Args:
+        recommended: Hardware-recommended primary model.
+        alternatives: List of alternative model suggestions.
+
+    Returns:
+        HuggingFace model ID string.
+    """
+    choices = [questionary.Choice(f"{recommended}  (recommended for your hardware)", value=recommended)]
+    for alt in alternatives:
+        choices.append(questionary.Choice(alt, value=alt))
+    choices.append(questionary.Choice("Enter a custom model ID…", value="__custom__"))
+
+    selection = questionary.select(
+        "Select a base model:",
+        choices=choices,
+        style=questionary.Style([("selected", "fg:cyan bold")]),
+    ).ask()
+
+    if selection == "__custom__":
+        selection = questionary.text(
+            "Enter HuggingFace model ID (e.g. meta-llama/Llama-3.2-1B):",
+            validate=lambda v: bool(v.strip()) or "Model ID cannot be empty",
+        ).ask()
+
+    return selection
+
+
+def prompt_dataset(task: str, strategy: str, training_service: "TrainingService") -> str:
+    """
+    Ask for a dataset path and validate it via TrainingService.
+
+    Args:
+        task: Selected task type.
+        strategy: Selected strategy.
+        training_service: TrainingService instance for validation.
+
+    Returns:
+        Validated dataset path string.
+    """
+    fmt = _STRATEGY_FORMAT_HELP.get(strategy) or _TASK_FORMAT_HELP.get(task, "JSON Lines file.")
+    console.print(f"\n[dim]Expected format:[/dim] {fmt}")
+
+    while True:
+        path = questionary.path(
+            "Path to dataset file (.jsonl / .json):",
+            validate=lambda v: os.path.isfile(v) or "File not found — please enter a valid path",
+        ).ask()
+
+        if path is None:
+            raise KeyboardInterrupt
+
+        console.print("  [dim]Validating dataset…[/dim]")
+        try:
+            info = training_service.validate_and_prepare_dataset(path, task, strategy)
+            console.print(
+                f"  [green]✓[/green] Dataset valid — "
+                f"[bold]{info['num_examples']}[/bold] examples, "
+                f"fields: {info['fields']}"
+            )
+            return path
+        except Exception as exc:
+            console.print(f"  [red]✗[/red] Validation failed: {exc}")
+            retry = questionary.confirm("Try a different file?", default=True).ask()
+            if not retry:
+                raise
+
+
+def prompt_provider() -> str:
+    """Ask the user to choose a training provider."""
+    return questionary.select(
+        "Select training provider:",
+        choices=_PROVIDER_CHOICES,
+        style=questionary.Style([("selected", "fg:cyan bold")]),
+    ).ask()
+
+
+def prompt_strategy() -> str:
+    """Ask the user to choose a training strategy."""
+    return questionary.select(
+        "Select training strategy:",
+        choices=_STRATEGY_CHOICES,
+        style=questionary.Style([("selected", "fg:cyan bold")]),
+    ).ask()
+
+
+def prompt_hyperparams(defaults: dict) -> dict:
+    """
+    Interactively configure hyperparameters, showing defaults.
+
+    Users can press Enter to keep each default value.
+
+    Args:
+        defaults: Dictionary of default hyperparameter values.
+
+    Returns:
+        Dictionary with final hyperparameter values.
+    """
+    console.print(
+        Panel(
+            "[bold]Hyperparameter Configuration[/bold]\n"
+            "[dim]Press Enter to accept the default for each setting.[/dim]",
+            expand=False,
+        )
+    )
+
+    params = {}
+
+    def _ask_int(key: str, question: str) -> int:
+        default = defaults.get(key)
+        raw = questionary.text(
+            f"{question} [default: {default}]:",
+            default=str(default),
+            validate=lambda v: v.isdigit() and int(v) >= 1 or f"Must be a positive integer",
+        ).ask()
+        return int(raw)
+
+    def _ask_float(key: str, question: str, min_val: float = 0.0) -> float:
+        default = defaults.get(key)
+        raw = questionary.text(
+            f"{question} [default: {default}]:",
+            default=str(default),
+            validate=lambda v: _valid_float(v, min_val),
+        ).ask()
+        return float(raw)
+
+    def _valid_float(v: str, min_val: float) -> bool | str:
+        try:
+            f = float(v)
+            return f > min_val or f"Must be > {min_val}"
+        except ValueError:
+            return "Must be a number"
+
+    params["num_train_epochs"] = _ask_int("num_train_epochs", "Training epochs")
+    params["per_device_train_batch_size"] = _ask_int(
+        "per_device_train_batch_size", "Batch size per device"
+    )
+    params["gradient_accumulation_steps"] = _ask_int(
+        "gradient_accumulation_steps", "Gradient accumulation steps"
+    )
+    params["learning_rate"] = _ask_float("learning_rate", "Learning rate", min_val=0.0)
+    params["lora_r"] = _ask_int("lora_r", "LoRA rank (r)")
+    params["lora_alpha"] = _ask_int("lora_alpha", "LoRA alpha")
+    params["max_seq_length"] = _ask_int("max_seq_length", "Max sequence length")
+
+    quant_choices = [
+        questionary.Choice("4-bit (QLoRA, saves VRAM)", value="4bit"),
+        questionary.Choice("8-bit (moderate savings)", value="8bit"),
+        questionary.Choice("None (full precision / no bitsandbytes required)", value="none"),
+    ]
+    default_quant = "4bit" if defaults.get("use_4bit") else ("8bit" if defaults.get("use_8bit") else "none")
+    quant = questionary.select(
+        f"Quantization [default: {default_quant}]:",
+        choices=quant_choices,
+        default=quant_choices[["4bit", "8bit", "none"].index(default_quant)],
+    ).ask()
+    params["use_4bit"] = quant == "4bit"
+    params["use_8bit"] = quant == "8bit"
+
+    return params
+
+
+def prompt_confirm_config(config: dict) -> bool:
+    """
+    Show a summary table of the final config and ask for confirmation.
+
+    Args:
+        config: Full training configuration dictionary.
+
+    Returns:
+        True if confirmed, False if cancelled.
+    """
+    table = Table(title="Training Configuration", show_header=True, header_style="bold cyan")
+    table.add_column("Setting", style="dim", width=30)
+    table.add_column("Value")
+
+    display_keys = [
+        ("task", "Task"),
+        ("model_name", "Base model"),
+        ("provider", "Provider"),
+        ("strategy", "Strategy"),
+        ("dataset", "Dataset"),
+        ("compute_specs", "Compute profile"),
+        ("num_train_epochs", "Epochs"),
+        ("per_device_train_batch_size", "Batch size"),
+        ("gradient_accumulation_steps", "Gradient accum. steps"),
+        ("learning_rate", "Learning rate"),
+        ("lora_r", "LoRA rank"),
+        ("lora_alpha", "LoRA alpha"),
+        ("max_seq_length", "Max seq length"),
+        ("use_4bit", "4-bit quantization"),
+        ("use_8bit", "8-bit quantization"),
+    ]
+
+    for key, label in display_keys:
+        if key in config:
+            table.add_row(label, str(config[key]))
+
+    console.print(table)
+
+    return questionary.confirm(
+        "Start training with this configuration?", default=True
+    ).ask()