mlx_commander 0.2.0__py3-none-any.whl

mlx_commander/__init__.py

@@ -0,0 +1,12 @@
+"""
+mlx_commander - Hugging Face to MLX Dataset Converter
+Convert Hugging Face datasets into Apple MLX fine-tuning formats with interactive TUI and CLI.
+"""
+
+import os
+
+# Set ncurses escape delay to 25ms (default is 1000ms) to ensure instantaneous
+# exit/cancellation on ESC across all curses pickers and dialogs.
+os.environ.setdefault("ESCDELAY", "25")
+
+__version__ = "0.2.0"

mlx_commander/__main__.py

@@ -0,0 +1,10 @@
+"""Top-level package execution entry point.
+
+Allows running MLX Commander via:
+    python3 -m mlx_commander
+"""
+import sys
+from mlx_commander.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

mlx_commander/bin/mac_picker.m

@@ -0,0 +1,69 @@
+#import <Cocoa/Cocoa.h>
+
+int main(int argc, const char * argv[]) {
+    @autoreleasepool {
+        [NSApplication sharedApplication];
+        [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory];
+        [NSApp activateIgnoringOtherApps:YES];
+
+        NSOpenPanel *panel = [NSOpenPanel openPanel];
+        panel.canChooseFiles = YES;
+        panel.canChooseDirectories = YES;
+        panel.allowsMultipleSelection = YES;
+        panel.canCreateDirectories = YES;
+        panel.title = @"Select Dataset (File or Folder)";
+        panel.prompt = @"Select";
+
+        // Argument 1: mode ("both", "folder", "file")
+        if (argc > 1) {
+            NSString *mode = [NSString stringWithUTF8String:argv[1]];
+            if ([mode isEqualToString:@"folder"]) {
+                panel.canChooseFiles = NO;
+                panel.canChooseDirectories = YES;
+                panel.allowsMultipleSelection = NO;
+                panel.title = @"Select Destination Folder";
+            } else if ([mode isEqualToString:@"file"]) {
+                panel.canChooseFiles = YES;
+                panel.canChooseDirectories = NO;
+                panel.allowsMultipleSelection = YES;
+                panel.title = @"Select Dataset File(s)";
+            } else {
+                panel.canChooseFiles = YES;
+                panel.canChooseDirectories = YES;
+                panel.allowsMultipleSelection = YES;
+                panel.title = @"Select Dataset (File, Files, or Folder)";
+            }
+        }
+
+        // Argument 2: initial directory
+        if (argc > 2) {
+            NSString *dirPath = [NSString stringWithUTF8String:argv[2]];
+            BOOL isDir = NO;
+            if ([[NSFileManager defaultManager] fileExistsAtPath:dirPath isDirectory:&isDir]) {
+                if (!isDir) {
+                    dirPath = [dirPath stringByDeletingLastPathComponent];
+                }
+                panel.directoryURL = [NSURL fileURLWithPath:dirPath];
+            }
+        }
+
+        // Argument 3: custom title
+        if (argc > 3) {
+            panel.title = [NSString stringWithUTF8String:argv[3]];
+        }
+
+        [panel makeKeyAndOrderFront:nil];
+        [panel setLevel:NSFloatingWindowLevel];
+
+        NSModalResponse response = [panel runModal];
+        if (response == NSModalResponseOK) {
+            for (NSURL *selectedURL in [panel URLs]) {
+                if (selectedURL) {
+                    printf("%s\n", [[selectedURL path] UTF8String]);
+                }
+            }
+            fflush(stdout);
+        }
+    }
+    return 0;
+}

mlx_commander/cli.py

@@ -0,0 +1,421 @@
+"""
+Command Line Interface (CLI) for mlx_commander converter.
+Dispatches between full-screen Curses TUI, interactive terminal wizard,
+and automated headless conversion.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+# Ensure ncurses escape delay is 25ms to make ESC instantaneous in all TUI pickers
+os.environ.setdefault("ESCDELAY", "25")
+
+import curses
+from pathlib import Path
+from typing import List, Optional
+
+from mlx_commander import __version__
+from mlx_commander.converter import ConversionResult, convert_and_save
+from mlx_commander.formats import (
+    ColumnMapping,
+    MLXFormat,
+    auto_detect_mapping,
+    validate_mapping,
+)
+from mlx_commander.exceptions import MissingDependencyError
+from mlx_commander.loader import load_local_dataset
+from mlx_commander.splitter import SplitConfig, generate_random_seed
+from mlx_commander.tui.app import launch_tui
+from mlx_commander.tui.wizard_fallback import run_interactive_wizard
+
+
+def parse_mapping_arg(mapping_str: str) -> ColumnMapping:
+    """Parse JSON or key=val,key=val string into ColumnMapping."""
+    clean = mapping_str.strip()
+    if clean.startswith("{"):
+        d = json.loads(clean)
+        return ColumnMapping(**d)
+
+    mapping = ColumnMapping()
+    for pair in clean.split(","):
+        if "=" in pair:
+            k, v = pair.split("=", 1)
+            k, v = k.strip(), v.strip()
+            if hasattr(mapping, k):
+                setattr(mapping, k, v)
+            elif k == "prompt":
+                mapping.prompt_col = v
+            elif k == "completion":
+                mapping.completion_col = v
+            elif k == "text":
+                mapping.text_col = v
+            elif k == "messages":
+                mapping.messages_col = v
+            elif k == "user":
+                mapping.user_col = v
+            elif k == "assistant":
+                mapping.assistant_col = v
+            elif k == "system":
+                mapping.system_col = v
+            elif k == "chosen":
+                mapping.chosen_col = v
+            elif k == "rejected":
+                mapping.rejected_col = v
+    return mapping
+
+
+def parse_prefill_state(val: str) -> dict:
+    """Parse prefill state from a JSON string or file path."""
+    clean = val.strip()
+    if clean.startswith("{"):
+        return json.loads(clean)
+    p = Path(clean).expanduser()
+    if p.exists() and p.is_file():
+        with open(p, "r", encoding="utf-8") as f:
+            return json.load(f)
+    try:
+        return json.loads(clean)
+    except Exception as e:
+        raise ValueError(f"Could not parse --prefill-state as JSON or file path: {e}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="mlx_commander",
+        description="Convert Hugging Face datasets into Apple MLX (mlx-lm) format with TUI or CLI.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Launch interactive TUI wizard:
+  mlx_commander
+
+  # Launch line-by-line CLI wizard:
+  mlx_commander --no-tui
+
+  # Direct conversion from CLI:
+  mlx_commander -d ./my_hf_dataset -f prompt_completion -o ./mlx_out \
+         --prompt-col question --completion-col answer \
+         --train 80 --valid 10 --test 10 --seed 42
+
+  # Combine multiple dataset files with schema verification & re-splitting:
+  mlx_commander -d train.jsonl test.jsonl -f prompt_completion -o ./mlx_out \
+         --prompt-col question --completion-col answer
+        """,
+    )
+
+    parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {__version__}")
+
+    # Core conversion flags
+    parser.add_argument(
+        "-d", "--dataset",
+        nargs="+",
+        type=str,
+        help="Path(s) to local Hugging Face dataset folder or data file(s) (.parquet, .arrow, .jsonl, .csv, .tsv, .sqlite, .tar). Multiple files will be verified for schema consistency and merged.",
+    )
+    parser.add_argument(
+        "-f", "--format",
+        type=str,
+        choices=["text", "chat", "prompt_completion", "dpo"],
+        help="Target MLX format: 'text', 'chat', 'prompt_completion', or 'dpo'.",
+    )
+    parser.add_argument(
+        "-o", "--output",
+        type=str,
+        help="Destination directory where train.jsonl, valid.jsonl, test.jsonl will be saved (default: 'mlx_dataset' subfolder in same folder as source dataset).",
+    )
+
+    # Split parameters
+    parser.add_argument("--train", type=float, default=None, help="Train split percentage (e.g. 80.0).")
+    parser.add_argument("--valid", type=float, default=None, help="Validation split percentage (e.g. 10.0).")
+    parser.add_argument("--test", type=float, default=None, help="Test split percentage (e.g. 10.0, or 0 to omit).")
+    parser.add_argument("--seed", type=int, default=None, help="Random seed for reproducible shuffling.")
+    parser.add_argument("--keep-splits", action="store_true", help="Preserve existing splits without re-splitting.")
+
+    # Column mapping flags
+    parser.add_argument("--mapping", type=str, help="Column mapping as JSON string or key=val,key=val.")
+    parser.add_argument("--text-col", type=str, help="Source column for 'text' format.")
+    parser.add_argument("--text-template", type=str, help="Template string for 'text' format (e.g. '{instruction}\\n{output}').")
+    parser.add_argument("--prompt-col", type=str, help="Source column for prompt / question.")
+    parser.add_argument("--completion-col", type=str, help="Source column for completion / answer.")
+    parser.add_argument("--messages-col", type=str, help="Source column containing chat messages list.")
+    parser.add_argument("--user-col", type=str, help="Source column for user turn in chat format.")
+    parser.add_argument("--assistant-col", type=str, help="Source column for assistant turn in chat format.")
+    parser.add_argument("--system-col", type=str, help="Source column for system prompt in chat format.")
+    parser.add_argument("--chosen-col", type=str, help="Source column for chosen response in DPO format.")
+    parser.add_argument("--rejected-col", type=str, help="Source column for rejected response in DPO format.")
+
+    # Agent & Hand-off flags
+    parser.add_argument(
+        "--manifest-file",
+        type=str,
+        default=None,
+        help="Custom file path where machine-readable mlx_manifest.json will be saved.",
+    )
+    parser.add_argument(
+        "--prefill-state",
+        type=str,
+        default=None,
+        help="Pre-populate TUI state from a JSON string or path to JSON file.",
+    )
+    parser.add_argument(
+        "--spawn-terminal",
+        action="store_true",
+        help="Spawn interactive TUI in an external macOS Terminal window (ideal for AI agents & subshells).",
+    )
+    parser.add_argument(
+        "--mcp",
+        action="store_true",
+        help="Launch Model Context Protocol (MCP) server over stdio for Claude Desktop, Cursor, etc.",
+    )
+
+    # UI mode flags
+    parser.add_argument(
+        "--commander", "--tui",
+        action="store_true",
+        dest="commander",
+        help="Launch full-screen persistent MLX Commander TUI dashboard (default in interactive terminal).",
+    )
+    parser.add_argument(
+        "--wizard", "--no-tui", "--cli",
+        action="store_true",
+        dest="wizard",
+        help="Run sequential step-by-step terminal wizard instead of persistent MLX Commander dashboard.",
+    )
+
+    return parser
+
+
+def run_direct_conversion(args: argparse.Namespace) -> ConversionResult:
+    """Perform headless conversion using command-line arguments."""
+    dataset = load_local_dataset(args.dataset)
+    target_format = MLXFormat(args.format.lower())
+
+    # Build mapping
+    if args.mapping:
+        mapping = parse_mapping_arg(args.mapping)
+    else:
+        mapping = auto_detect_mapping(target_format, dataset.columns)
+        if args.text_col:
+            mapping.text_col = args.text_col
+        if args.text_template:
+            mapping.text_template = args.text_template
+        if args.prompt_col:
+            mapping.prompt_col = args.prompt_col
+        if args.completion_col:
+            mapping.completion_col = args.completion_col
+        if args.messages_col:
+            mapping.messages_col = args.messages_col
+        if args.user_col:
+            mapping.user_col = args.user_col
+        if args.assistant_col:
+            mapping.assistant_col = args.assistant_col
+        if args.system_col:
+            mapping.system_col = args.system_col
+        if args.chosen_col:
+            mapping.chosen_col = args.chosen_col
+        if args.rejected_col:
+            mapping.rejected_col = args.rejected_col
+
+    # Validate mapping
+    errs = validate_mapping(target_format, mapping, dataset.columns)
+    if errs:
+        raise ValueError("Mapping error: " + "; ".join(errs))
+
+    # Split config
+    split_config = None
+    if not (args.keep_splits and dataset.is_split):
+        train_p = args.train if args.train is not None else 80.0
+        valid_p = args.valid if args.valid is not None else 10.0
+        test_p = args.test if args.test is not None else (100.0 - train_p - valid_p)
+        seed = args.seed if args.seed is not None else generate_random_seed()
+        split_config = SplitConfig(train_pct=train_p, valid_pct=valid_p, test_pct=test_p, seed=seed)
+        v_errs = split_config.validate()
+        if v_errs:
+            raise ValueError("Split error: " + "; ".join(v_errs))
+
+    out_dir = args.output or str(dataset.default_output_dir)
+    return convert_and_save(
+        dataset=dataset,
+        format_type=target_format,
+        mapping=mapping,
+        output_dir=out_dir,
+        split_config=split_config,
+        use_existing_splits=args.keep_splits,
+        manifest_file=getattr(args, "manifest_file", None),
+    )
+
+
+def is_interactive_tty() -> bool:
+    """Check if stdout and stdin are interactive TTYs with adequate terminfo."""
+    if not sys.stdin.isatty() or not sys.stdout.isatty():
+        return False
+    term = os.environ.get("TERM", "")
+    if not term or term == "dumb":
+        return False
+    return True
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.mcp:
+        from mlx_commander.mcp_server import run_mcp_server
+        return run_mcp_server()
+
+    # Normalize dataset path argument
+    dataset_input = None
+    if args.dataset:
+        if isinstance(args.dataset, list):
+            dataset_input = "\n".join(args.dataset) if len(args.dataset) > 1 else args.dataset[0]
+        else:
+            dataset_input = args.dataset
+
+    # Case 1: Direct Headless Run (dataset and format specified, without explicit UI flags)
+    if args.dataset and args.format and not args.wizard and not args.commander and not args.spawn_terminal:
+        try:
+            result = run_direct_conversion(args)
+            src_desc = f"{len(args.dataset)} files (merged)" if isinstance(args.dataset, list) and len(args.dataset) > 1 else (args.dataset[0] if isinstance(args.dataset, list) else str(args.dataset))
+            print(f"[OK] Successfully converted {src_desc} to {args.format} format in {result.output_dir}")
+            for s_name, path in result.output_files.items():
+                cnt = result.record_counts.get(s_name, 0)
+                print(f"  • {path.name}: {cnt:,} records")
+            if result.manifest_path:
+                print(f"  • Manifest: {result.manifest_path}")
+            print(f"\nMLX Fine-tuning command:\n{result.generate_mlx_lora_command()}\n")
+            return 0
+        except KeyboardInterrupt:
+            print("\nOperation cancelled.")
+            return 130
+        except MissingDependencyError as e:
+            print(f"\nMissing Dependency Error:\n{e.format_name} format requires package '{e.package_name}'.", file=sys.stderr)
+            print(f"\nTo install, run:\n    {e.install_command}", file=sys.stderr)
+            print(f"or install optional extra:\n    {e.pip_extra}\n", file=sys.stderr)
+            return 1
+        except Exception as e:
+            print(f"Error: {e}", file=sys.stderr)
+            return 1
+
+    # Case 2: Sequential wizard explicitly requested via --wizard / --no-tui / --cli
+    if args.wizard:
+        try:
+            run_interactive_wizard(
+                dataset_path=dataset_input,
+                format_arg=args.format,
+                output_dir_arg=args.output,
+                train_pct_arg=args.train,
+                valid_pct_arg=args.valid,
+                test_pct_arg=args.test,
+                seed_arg=args.seed,
+            )
+            return 0
+        except (KeyboardInterrupt, EOFError):
+            print("\nOperation cancelled.")
+            return 130
+        except MissingDependencyError as e:
+            print(f"\nMissing Dependency Error:\n{e.format_name} format requires package '{e.package_name}'.", file=sys.stderr)
+            print(f"\nTo install, run:\n    {e.install_command}", file=sys.stderr)
+            print(f"or install optional extra:\n    {e.pip_extra}\n", file=sys.stderr)
+            return 1
+        except Exception as e:
+            print(f"\nError: {e}", file=sys.stderr)
+            return 1
+
+    # Case 3: Interactive TUI Dashboard (with optional prefill and Terminal Spawner)
+    prefill_dict: Dict[str, Any] = {}
+    if args.prefill_state:
+        try:
+            prefill_dict.update(parse_prefill_state(args.prefill_state))
+        except Exception as e:
+            print(f"Error parsing --prefill-state: {e}", file=sys.stderr)
+            return 1
+
+    if args.format:
+        prefill_dict["format"] = args.format
+    if args.prompt_col:
+        prefill_dict["prompt_col"] = args.prompt_col
+    if args.completion_col:
+        prefill_dict["completion_col"] = args.completion_col
+    if args.text_col:
+        prefill_dict["text_col"] = args.text_col
+    if args.text_template:
+        prefill_dict["text_template"] = args.text_template
+    if args.messages_col:
+        prefill_dict["messages_col"] = args.messages_col
+    if args.user_col:
+        prefill_dict["user_col"] = args.user_col
+    if args.assistant_col:
+        prefill_dict["assistant_col"] = args.assistant_col
+    if args.system_col:
+        prefill_dict["system_col"] = args.system_col
+    if args.chosen_col:
+        prefill_dict["chosen_col"] = args.chosen_col
+    if args.rejected_col:
+        prefill_dict["rejected_col"] = args.rejected_col
+    if args.train is not None:
+        prefill_dict["train"] = args.train
+    if args.valid is not None:
+        prefill_dict["valid"] = args.valid
+    if args.test is not None:
+        prefill_dict["test"] = args.test
+    if args.seed is not None:
+        prefill_dict["seed"] = args.seed
+    if args.output:
+        prefill_dict["output"] = args.output
+    if dataset_input:
+        prefill_dict["dataset"] = dataset_input
+
+    # Check if external terminal window should be spawned
+    from mlx_commander.terminal_spawner import is_macos, spawn_terminal_tui
+    should_spawn = args.spawn_terminal or (not is_interactive_tty() and is_macos())
+
+    if should_spawn:
+        raw_args = list(sys.argv[1:] if argv is None else argv)
+        return spawn_terminal_tui(raw_args, manifest_path=args.manifest_file)
+
+    if not is_interactive_tty():
+        print(
+            "Error: No interactive terminal (TTY) detected.\n"
+            "On macOS, use --spawn-terminal to launch in Terminal.app, or specify --format for headless conversion.",
+            file=sys.stderr,
+        )
+        return 1
+
+    try:
+        result = launch_tui(default_dataset_path=dataset_input, prefill=prefill_dict)
+        if result is not None:
+            if args.manifest_file:
+                result.save_manifest(args.manifest_file)
+            print(f"\n[OK] Successfully converted dataset to {result.format_type.value} format in {result.output_dir}")
+            if result.manifest_path:
+                print(f"  • Manifest: {result.manifest_path}")
+            return 0
+        else:
+            if args.manifest_file:
+                try:
+                    Path(args.manifest_file).parent.mkdir(parents=True, exist_ok=True)
+                    with open(args.manifest_file, "w", encoding="utf-8") as f:
+                        json.dump({"status": "cancelled"}, f, indent=2)
+                except Exception:
+                    pass
+            print("\nOperation cancelled.")
+            return 130
+    except (KeyboardInterrupt, EOFError):
+        print("\nOperation cancelled.")
+        return 130
+    except curses.error as e:
+        print(f"\nTerminal error: {e}", file=sys.stderr)
+        return 1
+    except Exception as e:
+        print(f"\nError: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        print("\nOperation cancelled.")
+        sys.exit(130)

mlx_commander/converter.py

@@ -0,0 +1,177 @@
+"""
+Conversion Engine for writing MLX JSONL datasets.
+Streams converted and mapped records to train.jsonl, valid.jsonl, and test.jsonl.
+"""
+
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Union
+
+from mlx_commander.formats import ColumnMapping, MLXFormat, format_record, validate_mapping
+from mlx_commander.loader import LoadedDataset
+from mlx_commander.splitter import SplitConfig, split_records
+
+
+@dataclass
+class ConversionResult:
+    output_dir: Path
+    format_type: MLXFormat
+    output_files: Dict[str, Path]
+    record_counts: Dict[str, int]
+    file_sizes: Dict[str, int]
+    sample_records: Dict[str, List[Dict[str, Any]]]
+    seed_used: Optional[int]
+    manifest_path: Optional[Path] = None
+    source_path: Optional[str] = None
+
+    def generate_mlx_lora_command(self, model_name: str = "mlx-community/Llama-3.2-3B-Instruct-4bit") -> str:
+        """Generate a ready-to-use mlx_lm.lora command line string."""
+        mask_arg = " --mask-prompt" if self.format_type in (MLXFormat.CHAT, MLXFormat.PROMPT_COMPLETION) else ""
+        return (
+            f"mlx_lm.lora \\\n"
+            f"    --model {model_name} \\\n"
+            f"    --train \\\n"
+            f"    --data {self.output_dir}{mask_arg} \\\n"
+            f"    --iters 600 \\\n"
+            f"    --batch-size 4"
+        )
+
+    def to_manifest_dict(self) -> Dict[str, Any]:
+        """Produce a structured, machine-readable dictionary representing conversion output."""
+        return {
+            "status": "success",
+            "format": self.format_type.value,
+            "source_path": self.source_path or "",
+            "output_dir": str(self.output_dir),
+            "files": {
+                split: {
+                    "path": str(p),
+                    "filename": p.name,
+                    "records": self.record_counts.get(split, 0),
+                    "size_bytes": self.file_sizes.get(split, 0),
+                }
+                for split, p in self.output_files.items()
+            },
+            "splits": self.record_counts,
+            "total_records": sum(self.record_counts.values()),
+            "seed_used": self.seed_used,
+            "mlx_lora_command": self.generate_mlx_lora_command(),
+            "manifest_path": str(self.manifest_path) if self.manifest_path else str(self.output_dir / "mlx_manifest.json"),
+        }
+
+    def save_manifest(self, path: Optional[Union[str, Path]] = None) -> Path:
+        """Write manifest JSON to disk and record manifest_path."""
+        target = Path(path).expanduser().resolve() if path else (self.output_dir / "mlx_manifest.json")
+        target.parent.mkdir(parents=True, exist_ok=True)
+        manifest_data = self.to_manifest_dict()
+        manifest_data["manifest_path"] = str(target)
+        with open(target, "w", encoding="utf-8") as f:
+            json.dump(manifest_data, f, indent=2, ensure_ascii=False)
+        self.manifest_path = target
+        return target
+
+
+def convert_and_save(
+    dataset: LoadedDataset,
+    format_type: MLXFormat,
+    mapping: ColumnMapping,
+    output_dir: Optional[Union[str, Path]] = None,
+    split_config: Optional[SplitConfig] = None,
+    use_existing_splits: bool = False,
+    progress_callback: Optional[Callable[[str, int, int], None]] = None,
+    output_dir_str: Optional[Union[str, Path]] = None,
+    **kwargs: Any,
+) -> ConversionResult:
+    """
+    Convert a loaded dataset to MLX JSONL format and write output files.
+    Supports output_dir as str or Path, as well as output_dir_str for backward compatibility.
+    """
+    target = output_dir if output_dir is not None else output_dir_str
+    if target is None:
+        target = kwargs.get("output_path")
+    if target is None:
+        target = getattr(dataset, "default_output_dir", None)
+    if target is None:
+        raise ValueError("output_dir must be specified for convert_and_save")
+
+    output_dir = Path(target).expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Validate mapping against dataset columns
+    errors = validate_mapping(format_type, mapping, dataset.columns)
+    if errors:
+        raise ValueError("Invalid column mapping: " + "; ".join(errors))
+
+    # Determine split dictionary
+    splits_data: Dict[str, List[Dict[str, Any]]] = {}
+
+    if use_existing_splits and dataset.is_split:
+        for s_name in dataset.split_names:
+            normalized_s = s_name.lower()
+            if normalized_s in ("validation", "val", "dev"):
+                key = "valid"
+            elif normalized_s in ("train", "training"):
+                key = "train"
+            elif normalized_s in ("test", "testing", "eval"):
+                key = "test"
+            else:
+                key = normalized_s
+            splits_data[key] = list(dataset.iter_records(split=s_name))
+    else:
+        # Re-split all records according to SplitConfig
+        all_records = dataset.get_all_records()
+        if split_config is None:
+            # Default 80/10/10 split
+            split_config = SplitConfig(train_pct=80.0, valid_pct=10.0, test_pct=10.0, seed=42)
+        splits_data = split_records(all_records, split_config)
+
+    output_files: Dict[str, Path] = {}
+    record_counts: Dict[str, int] = {}
+    file_sizes: Dict[str, int] = {}
+    samples: Dict[str, List[Dict[str, Any]]] = {}
+
+    for split_name, records in splits_data.items():
+        if not records:
+            continue
+
+        file_name = f"{split_name}.jsonl"
+        target_path = output_dir / file_name
+        output_files[split_name] = target_path
+
+        written_count = 0
+        split_samples: List[Dict[str, Any]] = []
+        total_in_split = len(records)
+
+        with open(target_path, "w", encoding="utf-8") as out_f:
+            for idx, raw_record in enumerate(records, start=1):
+                formatted = format_record(raw_record, format_type, mapping)
+                line = json.dumps(formatted, ensure_ascii=False)
+                out_f.write(line + "\n")
+                written_count += 1
+
+                if len(split_samples) < 3:
+                    split_samples.append(formatted)
+
+                if progress_callback and (idx % 200 == 0 or idx == total_in_split):
+                    progress_callback(split_name, idx, total_in_split)
+
+        record_counts[split_name] = written_count
+        file_sizes[split_name] = target_path.stat().st_size
+        samples[split_name] = split_samples
+
+    res = ConversionResult(
+        output_dir=output_dir,
+        format_type=format_type,
+        output_files=output_files,
+        record_counts=record_counts,
+        file_sizes=file_sizes,
+        sample_records=samples,
+        seed_used=split_config.seed if split_config else None,
+        source_path=getattr(dataset, "source_path", None),
+    )
+
+    manifest_file = kwargs.get("manifest_file")
+    res.save_manifest(manifest_file if manifest_file else None)
+    return res