codefinetuner 0.1.0__py3-none-any.whl

codefinetuner/convert/convert_hf_to_gguf.version

@@ -0,0 +1 @@
+0.18.0

codefinetuner/convert/run.py

@@ -0,0 +1,82 @@
+import subprocess
+import sys
+import logging
+from typing import Any
+
+from .config import Config
+
+
+logger = logging.getLogger(__name__)
+
+
+def _log_subprocess_output(process: Any, prefix: str = "llama.cpp") -> None:
+    """
+    Captures subprocess output, handles multi-line blocks,
+    and re-logs them with correct severity levels.
+    """
+    log_levels = ("INFO:", "WARNING:", "ERROR:", "DEBUG:")
+    message_buffer = []
+
+    def flush_buffer() -> None:
+        if message_buffer:
+            msg = "".join(message_buffer).strip()
+            if msg.startswith("ERROR:"):
+                logger.error(f"[{prefix}] {msg}")
+            elif msg.startswith("WARNING:"):
+                logger.warning(f"[{prefix}] {msg}")
+            else:
+                logger.info(f"[{prefix}] {msg}")
+            message_buffer.clear()
+
+    for line in process.stdout:
+        # check if the current line starts a new log entry
+        if any(line.startswith(level) for level in log_levels):
+            flush_buffer()
+        
+        message_buffer.append(line)
+
+    flush_buffer()
+
+
+def _convert_to_gguf(config: Config, precision: str = "auto"):
+    cmd = [
+        sys.executable, str(config.convert_hf_to_gguf_local_path),
+        str(config.lora_model_path), 
+        "--outfile", str(config.lora_model_gguf_path),
+        "--outtype", precision
+    ]
+    
+    logger.info(f"Executing conversion: {config.lora_model_path} -> {config.lora_model_gguf_path}")
+
+    # initialize subprocess 
+    process = subprocess.Popen(
+        cmd, 
+        stdout=subprocess.PIPE,  # catch output in a private memory buffer (pipe) instead of letting it hit the console
+        stderr=subprocess.STDOUT,  # merge error messages into that same private memory buffer (pipe)
+        text=True,  # return streams as text (str) instead of bytes
+        bufsize=1  # send text line-by-line so we see updates immediately
+    )
+
+    _log_subprocess_output(process=process, prefix="llama.cpp")
+
+    process.wait()
+    
+    if process.returncode != 0:
+        raise RuntimeError(f"Conversion failed with exit code {process.returncode}")
+
+
+def run(config: Config) -> None:
+    _convert_to_gguf(config)
+
+
+def main() -> None:
+    try:
+        convert_config = Config()
+        run(convert_config)
+    except Exception:
+        logger.exception("Model conversion to GGUF failed.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

codefinetuner/evaluate/analyze.py

@@ -0,0 +1,198 @@
+import json
+import logging
+from datetime import datetime
+import matplotlib.pyplot as plt
+import numpy as np
+from pathlib import Path
+
+from .config import Config
+
+
+logger = logging.getLogger(__name__)
+
+
+def analyze_metric(config: Config, metric_name: str, higher_is_better: bool) -> dict:
+    base_scores, lora_scores = [], []
+    
+    with open(config.benchmark_evaluation_results_path, 'r') as evaluation_results_file:
+        for line in evaluation_results_file:
+            evaluation_example = json.loads(line)
+            if metric_name == config.codebleu_metric_name and not evaluation_example.get("codebleu_valid", True):
+                continue
+            base_scores.append(evaluation_example[f'base_{metric_name}'])
+            lora_scores.append(evaluation_example[f'lora_{metric_name}'])
+    
+    if not base_scores:
+        logger.error(f"No {metric_name} results found, returning empty stats")
+        return {}
+    
+    base_array_np = np.array(base_scores)
+    lora_array_np = np.array(lora_scores)
+    base_average_np = np.mean(base_array_np)
+    lora_average_np = np.mean(lora_array_np)
+    if higher_is_better:
+        improvement_np = lora_average_np - base_average_np
+    else:
+        improvement_np = base_average_np - lora_average_np
+    
+    logger.info(f"\n=== {metric_name.upper()} SUMMARY ===")
+    logger.info(f"Examples: {len(base_array_np)}")
+    logger.info(f"Base average: {base_average_np:.3f} | LoRA avg: {lora_average_np:.3f}")
+    logger.info(f"Improvement: {improvement_np:+.3f}")
+
+    is_binary = False
+    if (metric_name == config.exact_match_metric_name) or (metric_name == config.line_match_metric_name):
+        is_binary = True
+    
+    metric_stats_np =  {
+        "metric": metric_name,
+        "base_array_np": base_array_np,
+        "lora_array_np": lora_array_np,
+        "base_average_np": base_average_np,
+        "lora_average_np": lora_average_np,
+        "is_binary": is_binary, 
+        "higher_is_better": higher_is_better
+    }
+
+    return metric_stats_np
+
+
+def get_plot_path(plots_dir: Path, metric_name: str) -> Path:
+    """Converts metric names to standardized filenames."""
+    safe_name = metric_name.lower().replace(" ", "_")
+    return plots_dir / f"{safe_name}_plot.png"
+
+
+def plot_metric_and_save(metric_stats_np: dict, metric_name: str, plot_path: Path) -> None:
+    base_array_np = metric_stats_np["base_array_np"]
+    lora_array_np = metric_stats_np["lora_array_np"]
+    base_average_np = metric_stats_np["base_average_np"]
+    lora_average_np = metric_stats_np["lora_average_np"]
+    
+    if metric_stats_np["is_binary"]:
+        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))
+        fig.suptitle(f'{metric_name.title()} Scores', fontsize=16) 
+        
+        ax1.bar(['Base', 'LoRA'], [base_average_np, lora_average_np], color=['steelblue', 'darkorange'])
+        ax1.set_title(f'{metric_name.title()} Success Rate')
+        ax1.set_ylim(0, 1)
+        ax1.set_ylabel('Rate')
+        
+        wins = np.sum((lora_array_np == 1) & (base_array_np == 0))
+        losses = np.sum((lora_array_np == 0) & (base_array_np == 1))
+        ties = len(base_array_np) - wins - losses
+        ax2.bar(['LoRA Wins', 'Ties', 'LoRA Losses'], [wins, ties, losses], color=['green', 'gray', 'red'])
+        ax2.set_title('Example-Level Transitions')
+        ax2.set_ylabel('Count')
+        
+    else:
+        fig, axs = plt.subplots(2, 2, figsize=(14, 10))
+        fig.suptitle(f'{metric_name.title()} Scores', fontsize=16)
+        y_axis_limit = 25
+        
+        # dynamic bounds, prevent outlier examples to screw the plot 
+        max_val = max(np.max(base_array_np), np.max(lora_array_np))
+        use_limit = max_val > y_axis_limit
+        if use_limit:
+            axis_upper_bound = y_axis_limit
+        else:
+            axis_upper_bound = max_val * 1.1
+
+        axs[0, 0].bar(['Base', 'LoRA'], [base_average_np, lora_average_np], color=['steelblue', 'darkorange'])
+        axs[0, 0].set_title('Average Scores')
+        axs[0, 0].set_ylabel('Score')
+        
+        axs[0, 1].boxplot([base_array_np, lora_array_np], tick_labels=['Base', 'LoRA'], patch_artist=True, boxprops=dict(facecolor='lightblue', color='blue'), medianprops=dict(color='red', linewidth=2))
+        axs[0, 1].set_title('Score Distribution')
+        axs[0, 1].set_ylabel('Score')
+        axs[0, 1].set_ylim(0, axis_upper_bound)
+        if use_limit:
+            axs[0, 1].text(0.95, 0.95, f'Note: Values > {y_axis_limit} hidden', transform=axs[0, 1].transAxes, ha='right', va='top', fontsize=10, bbox=dict(facecolor='white', alpha=0.8))
+        
+        axs[1, 0].scatter(base_array_np, lora_array_np, alpha=0.5, color='steelblue')
+        axs[1, 0].plot([0, axis_upper_bound], [0, axis_upper_bound], 'k--', alpha=0.75, label='y=x (Tie)')
+        axs[1, 0].set_title('Base vs LoRA (Per Example)')
+        axs[1, 0].set_xlabel('Base Score')
+        axs[1, 0].set_ylabel('LoRA Score')
+        axs[1, 0].set_xlim(0, axis_upper_bound)
+        axs[1, 0].set_ylim(0, axis_upper_bound)
+        axs[1, 0].legend(loc='upper left')
+        if use_limit:
+            axs[1, 0].text(0.95, 0.05, f'Note: Values > {y_axis_limit} hidden', transform=axs[1, 0].transAxes, ha='right', va='bottom', fontsize=10, bbox=dict(facecolor='white', alpha=0.8))
+        
+        differences = (lora_array_np - base_array_np) if metric_stats_np["higher_is_better"] else (base_array_np - lora_array_np)
+        axs[1, 1].hist(differences, bins=30, color='purple', alpha=0.7, edgecolor='black')
+        axs[1, 1].axvline(0, color='black', linestyle='dashed')
+        axs[1, 1].axvline(np.mean(differences), color='red', linestyle='solid')
+        axs[1, 1].axvline(np.mean(differences), color='red', linestyle='solid', linewidth=2, label='Mean Diff')
+        axs[1, 1].set_title('Improvement Distribution')
+        axs[1, 1].set_xlabel('Difference')
+        axs[1, 1].set_ylabel('Examples')
+        hist_range = (-axis_upper_bound, axis_upper_bound)
+        axs[1, 1].set_xlim(hist_range)
+        axs[1, 1].legend(loc='upper left')
+        if use_limit:
+            axs[1, 1].text(0.95, 0.95, f'Note: |Diff| > {y_axis_limit} hidden', transform=axs[1, 1].transAxes, ha='right', va='top', fontsize=10, bbox=dict(facecolor='white', alpha=0.8))
+
+    plt.tight_layout()
+    plt.savefig(plot_path)
+    plt.close(fig)
+
+
+def save_all_metric_stats(config: Config, all_metric_stats_np: list[dict]) -> None:
+    """Writes all processed metrics to a single summary JSON file."""
+    all_metric_stats = []
+    for stat_np in all_metric_stats_np:
+        stat = {
+            "metric": stat_np["metric"],
+            "base_array": stat_np["base_array_np"].tolist(),
+            "lora_array": stat_np["lora_array_np"].tolist(),
+            "base_average": float(stat_np["base_average_np"]),
+            "lora_average": float(stat_np["lora_average_np"]),
+            "is_binary": stat_np["is_binary"], 
+            "higher_is_better": stat_np["higher_is_better"]
+        }
+        all_metric_stats.append(stat)
+
+    report_content = {
+        "checkpoint": config.trainer_checkpoint,
+        "evaluation_date": datetime.now().isoformat(timespec="seconds"),
+        "all_metric_stats": all_metric_stats 
+    }
+    with open(config.benchmark_analysis_results_path, "w") as report_file:
+        json.dump(report_content, report_file, indent=4)
+    logger.info(f"Analysis results saved to: {config.benchmark_analysis_results_path}")
+
+
+def plot_all_metric_averages_and_save(all_metric_stats_np: dict, plot_path: Path) -> None:
+    number_of_metrics = len(all_metric_stats_np)
+    fig, axes = plt.subplots(2, 3, figsize=(18, 12))
+    axes = axes.flatten()
+    
+    for idx, metric_stats in enumerate(all_metric_stats_np):
+        metric_name = metric_stats["metric"]
+        base_average = metric_stats["base_average_np"]
+        lora_average = metric_stats["lora_average_np"]
+        n_examples = len(metric_stats["base_array_np"])
+        
+        ax = axes[idx]
+        bars = ax.bar(['Base', 'LoRA'], [base_average, lora_average],color=['steelblue', 'darkorange'], alpha=0.8)
+        
+        max_axes_val = max(base_average, lora_average) * 1.1
+        ax.set_ylim(0, max_axes_val)
+        
+        ax.set_ylabel('Score')
+        ax.set_title(f'{metric_name.title()}\nN={n_examples}')
+        
+        for bar, val in zip(bars, [base_average, lora_average]):
+            height = bar.get_height()
+            ax.text(bar.get_x() + bar.get_width()/2., height + max_axes_val*0.02, f'{val:.3f}', ha='center', va='bottom', fontsize=11)
+    
+    for idx in range(number_of_metrics, len(axes)):
+        axes[idx].set_visible(False)
+    
+    plt.suptitle('Base vs LoRA Average Scores', fontsize=14, y=1.02)
+    plt.tight_layout()
+    plt.savefig(plot_path, dpi=300, bbox_inches='tight')
+    plt.close(fig)
+    logger.info(f"All-metrics average plotted and saved to: {plot_path}")

codefinetuner/evaluate/benchmark.py

@@ -0,0 +1,95 @@
+import json
+import logging
+
+from datasets import Features, Sequence, Value, load_dataset
+from transformers import AutoTokenizer
+
+from .config import Config
+
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_fim_parts(config: Config, tokenizer: AutoTokenizer, example_token_ids: list[int]) -> dict:
+    fim_prefix_token_id = tokenizer.convert_tokens_to_ids(config.fim_prefix_token)
+    fim_suffix_token_id = tokenizer.convert_tokens_to_ids(config.fim_suffix_token)
+    fim_middle_token_id = tokenizer.convert_tokens_to_ids(config.fim_middle_token)
+    eos_token_id = tokenizer.convert_tokens_to_ids(config.eos_token)
+
+    try:
+        fim_prefix_token_position = example_token_ids.index(fim_prefix_token_id)
+        fim_suffix_token_position = example_token_ids.index(fim_suffix_token_id)
+        fim_middle_token_position = example_token_ids.index(fim_middle_token_id)
+        eos_token_position = example_token_ids.index(eos_token_id)
+    except ValueError:
+        raise ValueError("Missing FIM or EOS tokens in the sequence.")
+
+    if (fim_prefix_token_position >= fim_suffix_token_position  
+        or fim_suffix_token_position >= fim_middle_token_position
+        or fim_middle_token_position >= eos_token_position):
+        raise ValueError("FIM tokens are in the wrong order.")
+        
+    prefix_token_ids = example_token_ids[fim_prefix_token_position+1 : fim_suffix_token_position]
+    suffix_token_ids = example_token_ids[fim_suffix_token_position+1 : fim_middle_token_position]
+    middle_token_ids = example_token_ids[fim_middle_token_position+1 : eos_token_position]
+
+    prefix = tokenizer.decode(prefix_token_ids, skip_special_tokens=True)
+    suffix = tokenizer.decode(suffix_token_ids, skip_special_tokens=True)
+    middle = tokenizer.decode(middle_token_ids, skip_special_tokens=True)
+
+    return {
+        "example_token_ids": example_token_ids,
+        "prefix_token_ids": prefix_token_ids,
+        "suffix_token_ids": suffix_token_ids,
+        "middle_token_ids": middle_token_ids,
+        "prefix": prefix,
+        "suffix": suffix,
+        "middle": middle
+    }
+
+
+def create_benchmark_dataset(config: Config) -> int:
+    tokenizer = AutoTokenizer.from_pretrained(config.model_name)
+
+    dataset_features = Features({
+        "input_ids": Sequence(Value("int64")),
+        "attention_mask": Sequence(Value("int64")),
+        "labels": Sequence(Value("int64")),
+    })
+
+    test_dataset = load_dataset(
+        "json", 
+        data_files=str(config.test_dataset_path), 
+        features=dataset_features, 
+        streaming=True
+    )["train"]
+    
+    shuffled_dataset = test_dataset.shuffle(
+        buffer_size=config.benchmark_shuffle_buffer_size,
+        seed=config.benchmark_shuffle_seed
+    )
+
+    benchmark_examples = []
+    added_examples_count = 0
+    for idx, example in enumerate(shuffled_dataset):
+        if added_examples_count >= config.benchmark_sample_size:
+            break
+        
+        try:
+            fim_parts = _extract_fim_parts(config, tokenizer, example["input_ids"])
+        except ValueError:
+            logger.warning(f"Could not extract fim parts from example {idx}, skipping") 
+            continue
+
+        if (len(fim_parts["middle_token_ids"]) < config.benchmark_min_fim_middle_tokens   
+            or len(fim_parts["prefix_token_ids"]) == 0):  # skip if no prefix
+            continue
+
+        benchmark_examples.append(fim_parts)
+        added_examples_count += 1
+
+    with open(config.benchmark_dataset_path, 'w', encoding='utf-8') as f:
+        for example in benchmark_examples:
+            f.write(json.dumps(example) + '\n')
+    
+    return len(benchmark_examples)

codefinetuner/evaluate/codebleu_shim.py

@@ -0,0 +1,37 @@
+"""
+This shim module patches CodeBLEU so it uses tree-sitter-language-pack instead
+of importing tree_sitter_<lang> modules directly. This allows CodeBLEU to
+work with the project's existing tree-sitter-language-pack setup.
+(A shim replaces specific internal functions of the originally imported 
+package with custom functions.)
+"""
+
+from typing import Dict, List, Union
+import tree_sitter_language_pack
+import codebleu.utils
+import codebleu.codebleu
+
+
+def shim_get_tree_sitter_language(lang: str):
+    try:
+        return tree_sitter_language_pack.get_language(lang)
+    except Exception as e:
+        raise ImportError(
+            f"Could not load language {lang} from tree_sitter_language_pack: {e}"
+        ) from e
+
+
+# apply shim so all internal CodeBLEU calls use the tree_sitter_language_pack 
+codebleu.utils.get_tree_sitter_language = shim_get_tree_sitter_language
+codebleu.codebleu.get_tree_sitter_language = shim_get_tree_sitter_language
+
+
+def codebleu_score(
+    references: Union[List[str], List[List[str]]],
+    predictions: List[str],
+    lang: str = "python",
+    weights: tuple[float, float, float, float] = (0.25, 0.25, 0.25, 0.25),
+) -> Dict[str, float]:
+    from codebleu import calc_codebleu
+    return calc_codebleu(references, predictions, lang=lang, weights=weights)
+

codefinetuner/evaluate/config.py

@@ -0,0 +1,186 @@
+import logging
+import math
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from typing import Any
+
+import nltk
+import torch
+from omegaconf import OmegaConf, MISSING
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Config:
+    # --- Model (Mandatory Parameters) ---
+    model_name: str = MISSING  
+    fim_prefix_token: str = MISSING
+    fim_suffix_token: str = MISSING 
+    fim_middle_token: str = MISSING 
+    fim_pad_token: str = MISSING 
+    eos_token: str = MISSING 
+    label_pad_token_id: int = MISSING
+
+    # --- Benchmark ---
+    benchmark_sample_size: int = 4
+    benchmark_min_fim_middle_tokens: int = 0 
+    benchmark_shuffle_buffer_size: int = 10000000
+    benchmark_shuffle_seed: int = 42 
+
+    # --- Generation ---
+    generation_max_new_tokens: int = 128 
+    generation_do_sample: bool = False  # note: if set to False temperature and top_p have no effect
+    generation_temperature: float = 0.7
+    generation_top_p: float = 0.95
+
+    # --- Metrics: CodeBLEU ---
+    codebleu_language: str = "c"
+    codebleu_metric_name: str = "codebleu"
+    # CodeBLEU weights (must sum to 1.0)
+    # see: https://arxiv.org/pdf/2009.10297  Section 4.4 for parameter suggestions 0.1, 0.1, 0.4, 0.4
+    codebleu_ngram_weight: float = 0.25  # token-level overlap (standard BLEU) 
+    codebleu_weighted_ngram_weight: float = 0.25  # keyword-level overlap (importance-weighted) 
+    codebleu_syntax_ast_weight: float = 0.25  # structural correctness (abstract syntax tree)
+    codebleu_dataflow_weight: float = 0.25  # logic consistency (variable dependency graph)   
+
+    # --- Metrics: SentenceBLEU ---
+    sentencebleu_metric_name: str = "sentencebleu"
+    # sentence-BLEU weights (must sum to 1.0)
+    sentencebleu_ngram_weight_1: float = 0.25  # 1-gram 
+    sentencebleu_ngram_weight_2: float = 0.25  # 2-gram    
+    sentencebleu_ngram_weight_3: float = 0.25  # 3-gram 
+    sentencebleu_ngram_weight_4: float = 0.25  # 4-gram
+
+    # --- Metrics: Other ---
+    exact_match_metric_name: str = "exact_match"
+    line_match_metric_name: str = "line_match"
+    line_match_number_of_lines: int = 2  
+    perplexity_name: str = "perplexity"
+
+    # --- Execution Controls ---
+    trainer_checkpoint: str = "last"  # "last"->use last checkpoint in  trainer_checkpoints_dir_path
+    plot_only: bool = False  # "skips" generate and evaluate, analyze only
+    benchmark_use_existing_dataset: bool = False
+
+    # --- Hardware Configuration ---
+    device: str = field(init=False)
+    model_dtype: Any = field(init=False)  # # type hint Any because omegaconf does not support torch.dtype 
+    _nltk_initialized: bool = field(init=False, default=False)
+
+    # --- Paths ---
+    workspace_path: Path | None = None 
+    trainer_checkpoints_dir_path: Path = field(init=False)
+    test_dataset_path: Path = field(init=False)  
+    evaluate_outputs_dir_path: Path = field(init=False)
+    benchmark_dataset_path: Path = field(init=False)
+    benchmark_evaluation_results_dir: Path = field(init=False)
+    benchmark_evaluation_results_path: Path = field(init=False)
+    benchmark_analysis_results_path: Path = field(init=False)
+
+    @classmethod
+    def load_from_yaml(cls, yaml_path: Path) -> "Config":
+        if not yaml_path.exists():
+            raise FileNotFoundError(f"Configuration file not found at: {yaml_path}")
+        
+        logger.info(f"Loading configuration from {yaml_path}")
+        config_dict = OmegaConf.structured(cls)
+        try:
+            yaml_file_node = OmegaConf.load(yaml_path)
+        except Exception as e:
+            raise ValueError(f"Failed to load YAML config {yaml_path}") from e
+
+        yaml_file_dict = OmegaConf.to_container(yaml_file_node, resolve=True)
+        yaml_evaluate_dict = yaml_file_dict.get("evaluate", {})
+
+        yaml_evaluate_valid_dict = {}
+        # Filter YAML fields to include only those defined in the Config dataclass.
+        # This prevents OmegaConf from raising an AttributeError when encountering 
+        # global YAML anchors or keys not present in the current Config dataclass. 
+        for field in fields(cls):
+            if field.name in yaml_evaluate_dict:
+                yaml_evaluate_valid_dict[field.name] = yaml_evaluate_dict[field.name]
+        logger.debug(f"Filtered YAML configuration: {yaml_evaluate_valid_dict}")
+
+        merged_config_dict = OmegaConf.merge(config_dict, yaml_evaluate_valid_dict)
+        return OmegaConf.to_object(merged_config_dict)
+
+    def __post_init__(self) -> None:
+        self._setup_device_and_precision()
+        self._setup_paths()
+        self._ensure_output_paths_exist()
+        self._validate_metric_weights()
+        logger.debug("Config initialization complete.")
+
+    def _setup_device_and_precision(self) -> None:
+        if torch.cuda.is_available():
+            self.device = "cuda"
+            self.model_dtype = torch.bfloat16
+        elif torch.backends.mps.is_available():
+            self.device = "mps"
+            self.model_dtype = torch.float16
+        else:
+            self.device = "cpu"
+            self.model_dtype = torch.float32
+        logger.info(f"Execution environment: device={self.device}, dtype={self.model_dtype}")
+        
+    def _validate_metric_weights(self) -> None:
+        codebleu_total_weight = (self.codebleu_ngram_weight + self.codebleu_weighted_ngram_weight + 
+                                 self.codebleu_syntax_ast_weight + self.codebleu_dataflow_weight)
+        if not math.isclose(codebleu_total_weight, 1.0, rel_tol=1e-6):
+            raise ValueError(f"CodeBLEU weights must sum to 1.0, got {codebleu_total_weight}")
+        sentencebleu_total_weight = (self.sentencebleu_ngram_weight_1 + self.sentencebleu_ngram_weight_2 + 
+                                       self.sentencebleu_ngram_weight_3 + self.sentencebleu_ngram_weight_4) 
+        if not math.isclose(sentencebleu_total_weight, 1.0, rel_tol=1e-6):
+            raise ValueError(f"SentenceBLEU weights must sum to 1.0, got {sentencebleu_total_weight}")
+
+    def _setup_paths(self) -> None:
+        if self.workspace_path is None:
+            self.workspace_path = Path.cwd()
+        self.trainer_checkpoints_dir_path = self.workspace_path / "outputs" / "finetune" / "checkpoints"
+        self.test_dataset_path = self.workspace_path / "outputs" / "preprocess" / "results" / "datasets" / "test_dataset.jsonl"
+        self.evaluate_outputs_dir_path = self.workspace_path / "outputs" / "evaluate"
+        self.benchmark_dataset_path = self.evaluate_outputs_dir_path / "datasets" / "benchmark_dataset.jsonl"
+        self.benchmark_evaluation_results_dir = self.evaluate_outputs_dir_path / "results"
+        self.benchmark_evaluation_results_path = self.benchmark_evaluation_results_dir / "evaluation_results.jsonl"
+        self.benchmark_analysis_results_path = self.benchmark_evaluation_results_dir / "analysis_results.json"
+        logger.debug(f"Resolved workspace path to: {self.workspace_path}")
+
+    def _ensure_output_paths_exist(self) -> None:
+        paths = [
+            self.evaluate_outputs_dir_path,
+            self.benchmark_dataset_path,
+            self.benchmark_evaluation_results_dir,
+            self.benchmark_evaluation_results_path,
+            self.benchmark_analysis_results_path,
+        ]
+        for path in paths:
+            if not path.parent.exists():
+                path.parent.mkdir(parents=True, exist_ok=True)
+                logger.debug(f"Created parent directory: {path.parent}")
+            else:
+                logger.debug(f"Parent directory already exists: {path.parent}")
+
+    @property
+    def metric_configs(self) -> list[tuple[str, bool]]:
+        return [
+            (self.sentencebleu_metric_name, True),
+            (self.codebleu_metric_name, True),
+            (self.exact_match_metric_name, True),
+            (self.line_match_metric_name, True),
+            (self.perplexity_name, False),
+        ]
+     
+    def ensure_nltk_initialized(self) -> None:
+        if self._nltk_initialized:
+            return
+
+        logger.info("Initializing NLTK data (punkt, punkt_tab)...")
+        try:
+            nltk.download('punkt', quiet=True)  
+            nltk.download('punkt_tab', quiet=True)
+            self._nltk_initialized = True
+            logger.info("NLTK data initialized successfully.")
+        except Exception:
+            raise RuntimeError(f"NLTK initializerion failed." )

codefinetuner/evaluate/evaluate.py

@@ -0,0 +1,63 @@
+import json
+import logging
+
+from .config import Config
+from .metrics import get_codebleu, get_exact_match, get_line_match, get_sentencebleu
+
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate_and_save(config: Config) -> None:
+    results_path = config.benchmark_evaluation_results_path 
+    temp_path = results_path.with_name(f"{results_path.name}.tmp")
+
+    line_counter = 0
+    try:
+        with results_path.open("r") as evaluation_results_file, \
+             temp_path.open("w") as tmp_file:
+            
+            for line in evaluation_results_file:
+                result = json.loads(line)
+
+                ref = result["reference_middle"] 
+                base_gen = result["base_generated_middle"]
+                lora_gen = result["lora_generated_middle"]
+
+                base_cb, cb_valid = get_codebleu(config, ref, base_gen)
+                lora_cb, _ = get_codebleu(config, ref, lora_gen)
+                
+                base_sb = get_sentencebleu(config, ref, base_gen)
+                lora_sb = get_sentencebleu(config, ref, lora_gen)
+
+                base_em = get_exact_match(ref, base_gen)
+                lora_em = get_exact_match(ref, lora_gen)
+
+                base_lm = get_line_match(config, ref, base_gen)
+                lora_lm = get_line_match(config, ref, lora_gen)
+
+                result.update({
+                    "base_codebleu": base_cb,
+                    "lora_codebleu": lora_cb,
+                    "codebleu_valid": cb_valid,
+                    "base_sentencebleu": base_sb,
+                    "lora_sentencebleu": lora_sb,
+                    "base_exact_match": base_em,
+                    "lora_exact_match": lora_em,
+                    "base_line_match": base_lm,
+                    "lora_line_match": lora_lm,
+                })
+
+                tmp_file.write(json.dumps(result) + "\n")
+                line_counter += 1
+
+                if line_counter % 10 == 0:
+                    logger.info(f"Processed {line_counter} examples...")
+
+        temp_path.replace(results_path)
+        logger.info(f"Successfully evaluated and saved {line_counter} examples to {config.benchmark_evaluation_results_path}.")
+
+    except Exception as e:
+        if temp_path.exists():
+            temp_path.unlink()
+        raise RuntimeError(f"Evaluation failed at example {line_counter}: {e}") from e