transfer-llm 0.1.0__tar.gz

transfer_llm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Dedy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

transfer_llm-0.1.0/PKG-INFO

@@ -0,0 +1,247 @@
+Metadata-Version: 2.4
+Name: transfer-llm
+Version: 0.1.0
+Summary: A modular PyTorch framework for fine-tuning Hugging Face models.
+Author-email: Dedy <dedy.ariansyah@gmail.com>
+Project-URL: Homepage, https://github.com/deduu/transfer
+Project-URL: Issues, https://github.com/deduu/transfer/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.21.6
+Requires-Dist: accelerate>=1.0.0
+Requires-Dist: bitsandbytes>=0.44.0
+Requires-Dist: transformers
+Requires-Dist: datasets
+Requires-Dist: tqdm
+Requires-Dist: pyyaml
+Requires-Dist: peft
+Requires-Dist: sentence-transformers
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
+Requires-Dist: hf-xet>=1.2.0
+Requires-Dist: torch>=2.3.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Dynamic: license-file
+
+# Transfer
+
+A modular PyTorch framework for fine-tuning Hugging Face language models.
+
+Transfer simplifies applying techniques like Supervised Fine-Tuning (SFT) and Direct Preference Optimization (DPO) to any Hugging Face causal language model.
+
+## Features
+
+- **SFT and DPO** — two fine-tuning strategies out of the box
+- **Multi-turn conversation support** — point to a `messages_column` of chat-style message lists
+- **Completion-only loss masking** — train on assistant tokens only with `train_on_completions_only`
+- **LoRA / QLoRA** — 4-bit quantization (nf4 / fp4) via bitsandbytes, configurable LoRA rank and target modules
+- **Training loop controls** — gradient accumulation, LR scheduling (cosine, linear, constant, …), gradient clipping
+- **Checkpointing and logging** — intermediate checkpoints every N steps, CSV training logs, optional W&B integration
+- **Built-in evaluation** — perplexity and semantic entropy metrics
+- **CLI** — `transfer train` and `transfer infer` commands for training and inference without writing Python
+
+## Installation
+
+### From PyPI
+
+```bash
+pip install transfer-llm
+```
+
+### From source
+
+```bash
+git clone https://github.com/deduu/transfer.git
+cd transfer
+pip install .
+```
+
+## Quick Start
+
+### Single-turn SFT
+
+```python
+from datasets import Dataset
+from transfer import Trainer, SFTConfig
+
+# Prepare data
+dataset = Dataset.from_dict({
+    "prompt": ["What is the capital of France?", "Explain gravity."],
+    "response": ["The capital of France is Paris.", "Gravity is ..."],
+})
+
+# Configure
+config = SFTConfig(
+    model_name="google/gemma-2b",
+    num_epochs=3,
+    batch_size=2,
+    learning_rate=5e-5,
+    output_dir="./gemma-sft",
+)
+
+# Train
+trainer = Trainer(task="sft", config=config, train_dataset=dataset)
+trainer.train()
+trainer.save_model()
+```
+
+### Multi-turn SFT with completion masking and LoRA
+
+```python
+from datasets import Dataset
+from transfer import Trainer, SFTConfig
+
+# Each row contains a list of message dicts
+conversations = [
+    {"messages": [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "What is the weather in SF?"},
+        {"role": "assistant", "content": "It is 62 F and foggy."},
+    ]},
+    # ... more conversations
+]
+dataset = Dataset.from_list(conversations)
+
+config = SFTConfig(
+    model_name="meta-llama/Llama-3.2-3B-Instruct",
+    messages_column="messages",           # read from chat-style column
+    train_on_completions_only=True,       # loss on assistant tokens only
+    num_epochs=3,
+    batch_size=1,
+    learning_rate=2e-5,
+    max_length=1024,
+    gradient_accumulation_steps=4,
+    warmup_steps=10,
+    scheduler_type="cosine",
+    save_steps=50,
+    logging_steps=5,
+    use_lora=True,
+    lora_r=16,
+    lora_alpha=32,
+    lora_dropout=0.05,
+    output_dir="./agent-sft-output",
+)
+
+trainer = Trainer(task="sft", config=config, train_dataset=dataset)
+trainer.train()
+trainer.save_model()
+```
+
+### DPO
+
+```python
+from datasets import load_dataset
+from transfer import Trainer, DPOConfig
+
+dataset = load_dataset("Anthropic/hh-rlhf", split="train[:1%]")
+
+config = DPOConfig(
+    model_name="google/gemma-2b",
+    num_epochs=1,
+    batch_size=2,
+    beta=0.1,
+    output_dir="./gemma-dpo",
+)
+
+trainer = Trainer(task="dpo", config=config, train_dataset=dataset)
+trainer.train()
+trainer.save_model()
+```
+
+## CLI Usage
+
+The `transfer` command is installed automatically with the package.
+
+### Training
+
+```bash
+transfer train \
+  --task sft \
+  --model_name google/gemma-2b \
+  --dataset_path data.jsonl \
+  --output_dir ./my-sft-model \
+  --num_epochs 3 \
+  --batch_size 4 \
+  --learning_rate 2e-4 \
+  --use_lora \
+  --gradient_accumulation_steps 4 \
+  --scheduler_type cosine \
+  --warmup_steps 50 \
+  --save_steps 200 \
+  --logging_steps 10
+```
+
+Use `--messages_column messages --train_on_completions_only` for multi-turn datasets.
+
+### Inference
+
+```bash
+transfer infer \
+  --model_name google/gemma-2b \
+  --adapter_path ./my-sft-model \
+  --prompt "Explain quantum computing" \
+  --use_chat_template \
+  --max_new_tokens 256 \
+  --temperature 0.7 \
+  --do_sample
+```
+
+## Configuration
+
+### SFTConfig
+
+| Parameter | Default | Description |
+|---|---|---|
+| `model_name` | `"google/gemma-2b"` | Hugging Face model name or path |
+| `num_epochs` | `30` | Number of training epochs |
+| `batch_size` | `1` | Training batch size |
+| `learning_rate` | `5e-5` | Learning rate |
+| `max_length` | `256` | Maximum sequence length |
+| `output_dir` | `"./sft_finetuned_model"` | Output directory |
+| `prompt_column` | `"prompt"` | Column name for prompts |
+| `response_column` | `"response"` | Column name for responses |
+| `messages_column` | `None` | Column with list of message dicts (multi-turn) |
+| `train_on_completions_only` | `False` | Only compute loss on assistant tokens |
+| `system_prompt` | `None` | System prompt for single-turn mode |
+| `use_lora` | `False` | Enable LoRA |
+| `lora_r` | `8` | LoRA rank |
+| `lora_alpha` | `16` | LoRA alpha |
+| `lora_dropout` | `0.1` | LoRA dropout |
+| `quantize` | `True` | Enable 4-bit quantization |
+| `quantization_type` | `"nf4"` | Quantization type (`nf4` or `fp4`) |
+| `gradient_accumulation_steps` | `1` | Gradient accumulation steps |
+| `warmup_steps` | `0` | LR scheduler warmup steps |
+| `scheduler_type` | `"cosine"` | LR scheduler type |
+| `max_grad_norm` | `1.0` | Gradient clipping norm (0 to disable) |
+| `save_steps` | `0` | Save checkpoint every N steps (0 to disable) |
+| `logging_steps` | `10` | Log metrics every N steps |
+
+### DPOConfig
+
+Inherits all parameters from above, plus:
+
+| Parameter | Default | Description |
+|---|---|---|
+| `beta` | `0.1` | DPO temperature parameter |
+| `learning_rate` | `1e-6` | Learning rate (lower default for DPO) |
+| `max_length` | `512` | Maximum sequence length |
+| `chosen_column` | `"chosen"` | Column name for chosen responses |
+| `rejected_column` | `"rejected"` | Column name for rejected responses |
+
+## Development
+
+```bash
+git clone https://github.com/deduu/transfer.git
+cd transfer
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+## License
+
+MIT

transfer_llm-0.1.0/README.md

@@ -0,0 +1,217 @@
+# Transfer
+
+A modular PyTorch framework for fine-tuning Hugging Face language models.
+
+Transfer simplifies applying techniques like Supervised Fine-Tuning (SFT) and Direct Preference Optimization (DPO) to any Hugging Face causal language model.
+
+## Features
+
+- **SFT and DPO** — two fine-tuning strategies out of the box
+- **Multi-turn conversation support** — point to a `messages_column` of chat-style message lists
+- **Completion-only loss masking** — train on assistant tokens only with `train_on_completions_only`
+- **LoRA / QLoRA** — 4-bit quantization (nf4 / fp4) via bitsandbytes, configurable LoRA rank and target modules
+- **Training loop controls** — gradient accumulation, LR scheduling (cosine, linear, constant, …), gradient clipping
+- **Checkpointing and logging** — intermediate checkpoints every N steps, CSV training logs, optional W&B integration
+- **Built-in evaluation** — perplexity and semantic entropy metrics
+- **CLI** — `transfer train` and `transfer infer` commands for training and inference without writing Python
+
+## Installation
+
+### From PyPI
+
+```bash
+pip install transfer-llm
+```
+
+### From source
+
+```bash
+git clone https://github.com/deduu/transfer.git
+cd transfer
+pip install .
+```
+
+## Quick Start
+
+### Single-turn SFT
+
+```python
+from datasets import Dataset
+from transfer import Trainer, SFTConfig
+
+# Prepare data
+dataset = Dataset.from_dict({
+    "prompt": ["What is the capital of France?", "Explain gravity."],
+    "response": ["The capital of France is Paris.", "Gravity is ..."],
+})
+
+# Configure
+config = SFTConfig(
+    model_name="google/gemma-2b",
+    num_epochs=3,
+    batch_size=2,
+    learning_rate=5e-5,
+    output_dir="./gemma-sft",
+)
+
+# Train
+trainer = Trainer(task="sft", config=config, train_dataset=dataset)
+trainer.train()
+trainer.save_model()
+```
+
+### Multi-turn SFT with completion masking and LoRA
+
+```python
+from datasets import Dataset
+from transfer import Trainer, SFTConfig
+
+# Each row contains a list of message dicts
+conversations = [
+    {"messages": [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "What is the weather in SF?"},
+        {"role": "assistant", "content": "It is 62 F and foggy."},
+    ]},
+    # ... more conversations
+]
+dataset = Dataset.from_list(conversations)
+
+config = SFTConfig(
+    model_name="meta-llama/Llama-3.2-3B-Instruct",
+    messages_column="messages",           # read from chat-style column
+    train_on_completions_only=True,       # loss on assistant tokens only
+    num_epochs=3,
+    batch_size=1,
+    learning_rate=2e-5,
+    max_length=1024,
+    gradient_accumulation_steps=4,
+    warmup_steps=10,
+    scheduler_type="cosine",
+    save_steps=50,
+    logging_steps=5,
+    use_lora=True,
+    lora_r=16,
+    lora_alpha=32,
+    lora_dropout=0.05,
+    output_dir="./agent-sft-output",
+)
+
+trainer = Trainer(task="sft", config=config, train_dataset=dataset)
+trainer.train()
+trainer.save_model()
+```
+
+### DPO
+
+```python
+from datasets import load_dataset
+from transfer import Trainer, DPOConfig
+
+dataset = load_dataset("Anthropic/hh-rlhf", split="train[:1%]")
+
+config = DPOConfig(
+    model_name="google/gemma-2b",
+    num_epochs=1,
+    batch_size=2,
+    beta=0.1,
+    output_dir="./gemma-dpo",
+)
+
+trainer = Trainer(task="dpo", config=config, train_dataset=dataset)
+trainer.train()
+trainer.save_model()
+```
+
+## CLI Usage
+
+The `transfer` command is installed automatically with the package.
+
+### Training
+
+```bash
+transfer train \
+  --task sft \
+  --model_name google/gemma-2b \
+  --dataset_path data.jsonl \
+  --output_dir ./my-sft-model \
+  --num_epochs 3 \
+  --batch_size 4 \
+  --learning_rate 2e-4 \
+  --use_lora \
+  --gradient_accumulation_steps 4 \
+  --scheduler_type cosine \
+  --warmup_steps 50 \
+  --save_steps 200 \
+  --logging_steps 10
+```
+
+Use `--messages_column messages --train_on_completions_only` for multi-turn datasets.
+
+### Inference
+
+```bash
+transfer infer \
+  --model_name google/gemma-2b \
+  --adapter_path ./my-sft-model \
+  --prompt "Explain quantum computing" \
+  --use_chat_template \
+  --max_new_tokens 256 \
+  --temperature 0.7 \
+  --do_sample
+```
+
+## Configuration
+
+### SFTConfig
+
+| Parameter | Default | Description |
+|---|---|---|
+| `model_name` | `"google/gemma-2b"` | Hugging Face model name or path |
+| `num_epochs` | `30` | Number of training epochs |
+| `batch_size` | `1` | Training batch size |
+| `learning_rate` | `5e-5` | Learning rate |
+| `max_length` | `256` | Maximum sequence length |
+| `output_dir` | `"./sft_finetuned_model"` | Output directory |
+| `prompt_column` | `"prompt"` | Column name for prompts |
+| `response_column` | `"response"` | Column name for responses |
+| `messages_column` | `None` | Column with list of message dicts (multi-turn) |
+| `train_on_completions_only` | `False` | Only compute loss on assistant tokens |
+| `system_prompt` | `None` | System prompt for single-turn mode |
+| `use_lora` | `False` | Enable LoRA |
+| `lora_r` | `8` | LoRA rank |
+| `lora_alpha` | `16` | LoRA alpha |
+| `lora_dropout` | `0.1` | LoRA dropout |
+| `quantize` | `True` | Enable 4-bit quantization |
+| `quantization_type` | `"nf4"` | Quantization type (`nf4` or `fp4`) |
+| `gradient_accumulation_steps` | `1` | Gradient accumulation steps |
+| `warmup_steps` | `0` | LR scheduler warmup steps |
+| `scheduler_type` | `"cosine"` | LR scheduler type |
+| `max_grad_norm` | `1.0` | Gradient clipping norm (0 to disable) |
+| `save_steps` | `0` | Save checkpoint every N steps (0 to disable) |
+| `logging_steps` | `10` | Log metrics every N steps |
+
+### DPOConfig
+
+Inherits all parameters from above, plus:
+
+| Parameter | Default | Description |
+|---|---|---|
+| `beta` | `0.1` | DPO temperature parameter |
+| `learning_rate` | `1e-6` | Learning rate (lower default for DPO) |
+| `max_length` | `512` | Maximum sequence length |
+| `chosen_column` | `"chosen"` | Column name for chosen responses |
+| `rejected_column` | `"rejected"` | Column name for rejected responses |
+
+## Development
+
+```bash
+git clone https://github.com/deduu/transfer.git
+cd transfer
+pip install -e ".[dev]"
+pytest tests/ -v
+```
+
+## License
+
+MIT

transfer_llm-0.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "transfer-llm"
+version = "0.1.0"
+authors = [
+  { name="Dedy", email="dedy.ariansyah@gmail.com" },
+]
+description = "A modular PyTorch framework for fine-tuning Hugging Face models."
+readme = "README.md"
+requires-python = ">=3.11"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "numpy>=1.21.6",
+    "accelerate>=1.0.0",
+    "bitsandbytes>=0.44.0",
+    "transformers",
+    "datasets",
+    "tqdm",
+    "pyyaml",
+    "peft",
+    "sentence-transformers",
+    "scikit-learn",
+    "scipy",
+    "hf-xet>=1.2.0",
+    "torch>=2.3.0",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0"]
+
+[project.urls]
+Homepage = "https://github.com/deduu/transfer"
+Issues = "https://github.com/deduu/transfer/issues"
+
+[project.scripts]
+transfer = "transfer.cli:main"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.setuptools]
+packages = ["transfer", "transfer.strategies"]
+
+[tool.uv.sources]
+torch = { index = "pytorch-cu124" }
+
+[[tool.uv.index]]
+name = "pytorch-cu124"
+url = "https://download.pytorch.org/whl/cu124"
+explicit = true
+

transfer_llm-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

transfer_llm-0.1.0/test/unit/evaluate/test_perplex.py

@@ -0,0 +1,141 @@
+import torch
+import gc
+from math import exp
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel
+
+PROMPT_TO_TEST = [
+    "Bagaimana cara menjelaskan Kepo kepada orang asing?",
+    # "Tulis artikel mengenai budaya kepo di Indonesia.",
+    # "Apakah kepo itu muncul secara genetik?"
+]
+
+BASE_MODEL_NAME = "meta-llama/Llama-3.2-3B-Instruct"
+LORA_ADAPTER_PATH = "./finetuned_model/llama-3.2-3b-it-kepo-lora"
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True,
+)
+
+
+def compute_per_prompt(model, tokenizer, prompt, device, log_details=False):
+    enc = tokenizer(prompt, return_tensors="pt", truncation=True).to(device)
+
+    with torch.inference_mode():
+        outputs = model(**enc, labels=enc["input_ids"])
+        loss = outputs.loss.item()
+
+    if log_details:
+        logits = outputs.logits[:, :-1, :]
+        labels = enc["input_ids"][:, 1:]
+        log_probs = torch.log_softmax(logits, dim=-1)
+        token_log_probs = log_probs.gather(-1,
+                                           labels.unsqueeze(-1)).squeeze(-1)
+
+        # Convert each token's loss
+        token_losses = (-token_log_probs.squeeze()).tolist()
+
+        print(f"\n🧠 Token losses for: `{prompt}`")
+        tokens = tokenizer.convert_ids_to_tokens(enc["input_ids"].squeeze())
+        for t, l in zip(tokens[1:], token_losses):  # skip first token
+            print(f"{t:20s} loss={l:.3f}")
+
+    return exp(loss), loss
+
+
+def compute_token_weighted(model, tokenizer, device, log_details=False):
+    """Compute total token-weighted perplexity."""
+    total_loss = 0.0
+    total_tokens = 0
+
+    for prompt in tqdm(PROMPT_TO_TEST, desc="Token-weighted scoring"):
+        enc = tokenizer(prompt, return_tensors="pt",
+                        truncation=True).to(device)
+        with torch.inference_mode():
+            outputs = model(**enc, labels=enc["input_ids"])
+
+        logits = outputs.logits[:, :-1, :]
+        labels = enc["input_ids"][:, 1:]
+        log_probs = torch.log_softmax(logits, dim=-1)
+        token_log_probs = log_probs.gather(-1,
+                                           labels.unsqueeze(-1)).squeeze(-1)
+
+        token_losses = -token_log_probs
+        total_loss += token_losses.sum().item()
+        total_tokens += token_losses.numel()
+
+        if log_details:
+            print(f"\n📍 Prompt: {prompt}")
+            print(
+                f"    Tokens: {token_losses.numel()} | Sum Loss: {token_losses.sum().item():.3f}")
+
+    avg_loss = total_loss / total_tokens
+    return exp(avg_loss), avg_loss
+
+
+def evaluate(label, tokenizer, load_lora=False, method="prompt_avg", log_details=False):
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    print(f"\n🔄 Loading {label}...")
+    model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL_NAME,
+        quantization_config=bnb_config,
+        device_map="auto"
+    )
+
+    if load_lora:
+        print("🔌 Applying LoRA adapter...")
+        model = PeftModel.from_pretrained(model, LORA_ADAPTER_PATH)
+
+    model.eval()
+
+    if method == "prompt_avg":
+        ppl_list = []
+        for prompt in PROMPT_TO_TEST:
+            ppl, _ = compute_per_prompt(
+                model, tokenizer, prompt, device, log_details)
+            ppl_list.append(ppl)
+        score = sum(ppl_list) / len(ppl_list)
+
+    elif method == "token_weighted":
+        score, _ = compute_token_weighted(
+            model, tokenizer, device, log_details)
+
+    print(f"\n📍 Perplexity ({method}) for {label}: {score:.4f}")
+
+    del model
+    torch.cuda.empty_cache()
+    gc.collect()
+    print("🧹 GPU cache cleared")
+
+    return score
+
+
+def main():
+    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_NAME)
+    tokenizer.pad_token = tokenizer.eos_token
+
+    print("\n📦 4-bit quantized evaluation starting...\n")
+
+    # Toggle here:
+    method = "prompt_avg"  # or: "token_weighted"
+    log_details = True     # show token losses
+
+    base = evaluate("Base Model", tokenizer, load_lora=False,
+                    method=method, log_details=log_details)
+    lora = evaluate("LoRA Model", tokenizer, load_lora=True,
+                    method=method, log_details=log_details)
+
+    print("\n============== FINAL RESULTS ==============")
+    print(f"Base model PPL: {base:.4f}")
+    print(f"LoRA model PPL: {lora:.4f}")
+    print(f"Improvement: {((base - lora)/base) * 100:.2f}%")
+    print("===========================================")
+
+
+if __name__ == "__main__":
+    main()