icdev 0.0.3__py3-none-any.whl

tools/finetune/pair_generator.py

@@ -0,0 +1,832 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Q&A pair generator — auto-generate training pairs from RAG chunks (D-FT-10).
+
+Uses qwen3 (scanner_function) to generate question-answer pairs from
+document chunks or RAG content. Pairs require human review before training.
+
+Pipeline: chunks → qwen3 generates Q&A → store as unapproved examples → human labels
+
+Usage:
+    python tools/finetune/pair_generator.py --generate --dataset-id "ds-xxx" --document-id "ftdoc-xxx" --json
+    python tools/finetune/pair_generator.py --generate-from-rag --dataset-id "ds-xxx" --source-table "research_signals" --json
+    python tools/finetune/pair_generator.py --stats --dataset-id "ds-xxx" --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+DB_PATH = BASE_DIR / "data" / "icdev.db"
+
+
+def _get_db(db_path: Optional[Path] = None) -> sqlite3.Connection:
+    conn = sqlite3.connect(str(db_path or DB_PATH))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    return conn
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _content_hash(system_prompt: str, user_input: str, expected_output: str) -> str:
+    """SHA-256 content hash for dedup."""
+    combined = f"{system_prompt}|{user_input}|{expected_output}"
+    return hashlib.sha256(combined.encode()).hexdigest()[:16]
+
+
+def _load_config() -> Dict[str, Any]:
+    """Load pair generation config from finetune_config.yaml."""
+    config_path = BASE_DIR / "args" / "finetune_config.yaml"
+    if config_path.exists():
+        try:
+            import yaml
+            with open(config_path) as f:
+                cfg = yaml.safe_load(f) or {}
+            return cfg.get("dataset", {})
+        except (ImportError, Exception):
+            pass
+    return {"questions_per_chunk": 3, "auto_generate_pairs": True}
+
+
+# -- LLM-based Pair Generation ------------------------------------------------
+
+
+def _generate_pairs_from_chunk(
+    chunk_content: str,
+    purpose: str = "general",
+    questions_per_chunk: int = 3,
+    system_prompt: str = "",
+) -> List[Dict[str, str]]:
+    """Generate Q&A pairs from a chunk using qwen3 (scanner_function).
+
+    Returns list of {"user_input": ..., "expected_output": ...}.
+    """
+    prompt = _build_generation_prompt(chunk_content, purpose, questions_per_chunk)
+
+    try:
+        from tools.llm.router import LLMRouter
+        from tools.llm.provider import LLMRequest
+
+        router = LLMRouter()
+        request = LLMRequest(
+            messages=[
+                {"role": "system", "content": _PAIR_GEN_SYSTEM_PROMPT},
+                {"role": "user", "content": prompt},
+            ],
+            temperature=0.3,
+            max_tokens=2000,
+        )
+
+        # Use scanner_function path (qwen3 only, no Claude review)
+        response = router.invoke("ft_pair_generation", request)
+
+        if response and response.content:
+            return _parse_pairs_response(response.content, system_prompt)
+    except ImportError:
+        pass
+    except Exception:
+        pass
+
+    # Fallback: template-based generation (air-gap safe, no LLM needed)
+    return _generate_pairs_template(chunk_content, purpose, questions_per_chunk)
+
+
+_PAIR_GEN_SYSTEM_PROMPT = """You are a training data generator for fine-tuning language models.
+Given a text passage, generate high-quality question-answer pairs.
+
+Rules:
+1. Questions should be diverse (factual, analytical, procedural)
+2. Answers should be grounded in the passage content only
+3. Each answer should be 2-5 sentences
+4. Output ONLY valid JSON array format
+
+Output format:
+[
+  {"question": "...", "answer": "..."},
+  {"question": "...", "answer": "..."}
+]"""
+
+
+def _build_generation_prompt(
+    chunk_content: str,
+    purpose: str,
+    count: int,
+) -> str:
+    """Build the generation prompt for qwen3."""
+    purpose_hints = {
+        "general": "Generate diverse questions covering facts, processes, and analysis.",
+        "proposal_drafting": "Generate questions about proposal writing, requirements, deliverables, and compliance.",
+        "compliance_export": "Generate questions about compliance controls, security requirements, and assessment procedures.",
+        "code_generation": "Generate questions about code patterns, architecture decisions, and implementation approaches.",
+        "custom": "Generate diverse questions covering the main topics in the text.",
+    }
+    hint = purpose_hints.get(purpose, purpose_hints["general"])
+
+    return f"""Generate exactly {count} question-answer pairs from this text passage.
+
+{hint}
+
+TEXT PASSAGE:
+---
+{chunk_content[:3000]}
+---
+
+Output ONLY the JSON array. No other text."""
+
+
+def _parse_pairs_response(
+    response_text: str,
+    system_prompt: str = "",
+) -> List[Dict[str, str]]:
+    """Parse LLM response into Q&A pairs."""
+    # Try to extract JSON from response
+    text = response_text.strip()
+
+    # Handle markdown-wrapped JSON
+    if "```json" in text:
+        start = text.index("```json") + 7
+        end = text.index("```", start) if "```" in text[start:] else len(text)
+        text = text[start:end].strip()
+    elif "```" in text:
+        start = text.index("```") + 3
+        end = text.index("```", start) if "```" in text[start:] else len(text)
+        text = text[start:end].strip()
+
+    try:
+        pairs = json.loads(text)
+        if not isinstance(pairs, list):
+            return []
+
+        result = []
+        for pair in pairs:
+            q = pair.get("question", "").strip()
+            a = pair.get("answer", "").strip()
+            if q and a and len(q) > 10 and len(a) > 20:
+                result.append({
+                    "system_prompt": system_prompt,
+                    "user_input": q,
+                    "expected_output": a,
+                })
+        return result
+    except (json.JSONDecodeError, ValueError):
+        return []
+
+
+def _generate_pairs_template(
+    chunk_content: str,
+    purpose: str,
+    count: int,
+) -> List[Dict[str, str]]:
+    """Template-based pair generation (air-gap fallback, no LLM needed).
+
+    Generates simple extractive Q&A pairs from sentences.
+    """
+    sentences = [s.strip() for s in chunk_content.split(".") if len(s.strip()) > 30]
+    pairs = []
+
+    for i, sentence in enumerate(sentences[:count]):
+        # Simple extractive pattern: turn statement into question
+        words = sentence.split()
+        if len(words) < 5:
+            continue
+
+        question = f"What can you tell me about: {' '.join(words[:8])}...?"
+        answer = sentence.strip()
+        if not answer.endswith("."):
+            answer += "."
+
+        pairs.append({
+            "system_prompt": "",
+            "user_input": question,
+            "expected_output": answer,
+        })
+
+    return pairs
+
+
+# -- Pipeline: Generate from Document Chunks -----------------------------------
+
+
+def generate_from_document(
+    dataset_id: str,
+    chunks: List[Dict[str, Any]],
+    purpose: str = "general",
+    questions_per_chunk: int = 3,
+    system_prompt: str = "",
+    source_document_id: str = "",
+    classification: str = "CUI",
+    tenant_id: str = "",
+    project_id: str = "",
+    db_path: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """Generate Q&A pairs from document chunks and add to dataset.
+
+    Args:
+        dataset_id: Target dataset
+        chunks: List of chunk dicts with 'content' and optional 'chunk_id'
+        purpose: Dataset purpose for guiding generation
+        questions_per_chunk: How many Q&A pairs per chunk
+        system_prompt: Optional system prompt for all pairs
+        source_document_id: Document ID for provenance
+    """
+    config = _load_config()
+    qpc = questions_per_chunk or config.get("questions_per_chunk", 3)
+
+    from tools.finetune.dataset_manager import add_example
+
+    total_generated = 0
+    total_added = 0
+    total_dedup = 0
+    errors = []
+
+    for chunk in chunks:
+        content = chunk.get("content", "")
+        chunk_id = chunk.get("chunk_id", "")
+
+        if not content or len(content) < 50:
+            continue
+
+        pairs = _generate_pairs_from_chunk(
+            chunk_content=content,
+            purpose=purpose,
+            questions_per_chunk=qpc,
+            system_prompt=system_prompt,
+        )
+        total_generated += len(pairs)
+
+        for pair in pairs:
+            result = add_example(
+                dataset_id=dataset_id,
+                user_input=pair["user_input"],
+                expected_output=pair["expected_output"],
+                system_prompt=pair.get("system_prompt", system_prompt),
+                source="rag_auto_generated",
+                source_chunk_id=chunk_id,
+                source_document_id=source_document_id,
+                classification=classification,
+                tenant_id=tenant_id,
+                project_id=project_id,
+                db_path=db_path,
+            )
+            if result.get("success"):
+                if result.get("deduplicated"):
+                    total_dedup += 1
+                else:
+                    total_added += 1
+            else:
+                errors.append(result.get("error", "Unknown error"))
+
+    return {
+        "success": True,
+        "dataset_id": dataset_id,
+        "chunks_processed": len(chunks),
+        "pairs_generated": total_generated,
+        "pairs_added": total_added,
+        "pairs_deduplicated": total_dedup,
+        "errors": errors[:10],  # Limit error output
+    }
+
+
+def generate_from_rag_source(
+    dataset_id: str,
+    source_table: str,
+    purpose: str = "general",
+    questions_per_chunk: int = 3,
+    system_prompt: str = "",
+    limit: int = 100,
+    classification: str = "CUI",
+    tenant_id: str = "",
+    project_id: str = "",
+    db_path: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """Generate Q&A pairs from existing RAG chunks.
+
+    Reads chunks from rag_chunks table filtered by source_table.
+    """
+    conn = _get_db(db_path)
+    try:
+        rows = conn.execute(
+            """SELECT chunk_id, content, source_id
+               FROM rag_chunks
+               WHERE source_table = ? AND tier = 'hot'
+               ORDER BY created_at DESC LIMIT ?""",
+            (source_table, limit),
+        ).fetchall()
+
+        if not rows:
+            return {"success": False, "error": f"No RAG chunks found for source: {source_table}"}
+
+        chunks = [
+            {"content": r["content"], "chunk_id": r["chunk_id"]}
+            for r in rows
+        ]
+    except Exception as e:
+        return {"success": False, "error": f"RAG query error: {e}"}
+    finally:
+        conn.close()
+
+    return generate_from_document(
+        dataset_id=dataset_id,
+        chunks=chunks,
+        purpose=purpose,
+        questions_per_chunk=questions_per_chunk,
+        system_prompt=system_prompt,
+        classification=classification,
+        tenant_id=tenant_id,
+        project_id=project_id,
+        db_path=db_path,
+    )
+
+
+def generate_from_govcon_source(
+    dataset_id: str,
+    source_type: str = "approved_drafts",
+    questions_per_chunk: int = 3,
+    system_prompt: str = "You are a federal government proposal writing assistant.",
+    limit: int = 100,
+    classification: str = "CUI // SP-CTI",
+    tenant_id: str = "",
+    project_id: str = "",
+    db_path: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """Generate Q&A pairs from GovProposal sources (Phase 59/60/62 bridge).
+
+    Reads directly from GovProposal tables (not via RAG chunks) to create
+    proposal_drafting training data. Accepted proposals are HITL-validated
+    (D373) and represent the highest quality training data.
+
+    Args:
+        dataset_id: Target fine-tuning dataset.
+        source_type: One of 'approved_drafts', 'knowledge_base', 'shall_statements'.
+        questions_per_chunk: Q&A pairs to generate per source record.
+        system_prompt: System prompt for training pairs.
+        limit: Max records to process.
+
+    Returns:
+        Generation summary with counts.
+    """
+    _GOVCON_QUERIES = {
+        "approved_drafts": {
+            "sql": """SELECT id, draft_content AS content, draft_method, confidence_score
+                      FROM proposal_section_drafts
+                      WHERE status IN ('approved', 'reviewed')
+                      ORDER BY created_at DESC LIMIT ?""",
+            "source_label": "govcon_approved_draft",
+        },
+        "knowledge_base": {
+            "sql": """SELECT id, title || ': ' || content AS content, category, domain
+                      FROM proposal_knowledge_base
+                      WHERE status = 'active'
+                      ORDER BY usage_count DESC LIMIT ?""",
+            "source_label": "govcon_knowledge_base",
+        },
+        "shall_statements": {
+            "sql": """SELECT id, shall_text AS content, domain, priority_level
+                      FROM rfp_shall_statements
+                      ORDER BY created_at DESC LIMIT ?""",
+            "source_label": "govcon_shall_statement",
+        },
+    }
+
+    if source_type not in _GOVCON_QUERIES:
+        return {
+            "success": False,
+            "error": f"Invalid source_type: {source_type}. Valid: {list(_GOVCON_QUERIES.keys())}",
+        }
+
+    query_info = _GOVCON_QUERIES[source_type]
+    conn = _get_db(db_path)
+    try:
+        rows = conn.execute(query_info["sql"], (limit,)).fetchall()
+        if not rows:
+            return {"success": False, "error": f"No records found for source: {source_type}"}
+
+        chunks = [
+            {"content": r["content"], "chunk_id": f"{query_info['source_label']}:{r['id']}"}
+            for r in rows
+            if r["content"] and len(r["content"]) >= 50
+        ]
+    except Exception as e:
+        return {"success": False, "error": f"DB query error: {e}"}
+    finally:
+        conn.close()
+
+    from tools.finetune.dataset_manager import add_example
+
+    total_generated = 0
+    total_added = 0
+    total_dedup = 0
+
+    for chunk in chunks:
+        pairs = _generate_pairs_from_chunk(
+            chunk_content=chunk["content"],
+            purpose="proposal_drafting",
+            questions_per_chunk=questions_per_chunk,
+            system_prompt=system_prompt,
+        )
+        total_generated += len(pairs)
+
+        for pair in pairs:
+            result = add_example(
+                dataset_id=dataset_id,
+                user_input=pair["user_input"],
+                expected_output=pair["expected_output"],
+                system_prompt=pair.get("system_prompt", system_prompt),
+                source=query_info["source_label"],
+                source_chunk_id=chunk["chunk_id"],
+                classification=classification,
+                tenant_id=tenant_id,
+                project_id=project_id,
+                db_path=db_path,
+            )
+            if result.get("success"):
+                if result.get("deduplicated"):
+                    total_dedup += 1
+                else:
+                    total_added += 1
+
+    return {
+        "success": True,
+        "dataset_id": dataset_id,
+        "source_type": source_type,
+        "records_processed": len(chunks),
+        "pairs_generated": total_generated,
+        "pairs_added": total_added,
+        "pairs_deduplicated": total_dedup,
+    }
+
+
+def get_generation_stats(
+    dataset_id: str,
+    db_path: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """Get statistics about auto-generated pairs in a dataset."""
+    conn = _get_db(db_path)
+    try:
+        total = conn.execute(
+            "SELECT COUNT(*) FROM ft_dataset_examples WHERE dataset_id = ? AND source = 'rag_auto_generated'",
+            (dataset_id,),
+        ).fetchone()[0]
+
+        approved = conn.execute(
+            "SELECT COUNT(*) FROM ft_dataset_examples WHERE dataset_id = ? AND source = 'rag_auto_generated' AND approved = 1",
+            (dataset_id,),
+        ).fetchone()[0]
+
+        labeled = conn.execute(
+            """SELECT COUNT(*) FROM ft_dataset_examples
+               WHERE dataset_id = ? AND source = 'rag_auto_generated'
+                 AND (quality_score > 0 OR compliance_score > 0 OR relevance_score > 0)""",
+            (dataset_id,),
+        ).fetchone()[0]
+
+        return {
+            "success": True,
+            "dataset_id": dataset_id,
+            "auto_generated_total": total,
+            "auto_generated_approved": approved,
+            "auto_generated_labeled": labeled,
+            "auto_generated_pending_review": total - labeled,
+            "approval_rate_pct": round(approved / labeled * 100, 1) if labeled > 0 else 0.0,
+        }
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+    finally:
+        conn.close()
+
+
+# -- KARL Pass-Rate Filtering (D-KARL-4) ----------------------------------------
+
+
+def _evaluate_pair_with_model(
+    pair: Dict[str, str],
+    model_function: str = "ft_pair_generation",
+) -> Dict[str, Any]:
+    """Have a model attempt to answer the question, return pass/fail.
+
+    Uses scanner-tier LLM (qwen3.5, zero Claude tokens).
+    """
+    try:
+        from tools.llm.router import LLMRouter
+        from tools.llm.provider import LLMRequest
+
+        router = LLMRouter()
+        request = LLMRequest(
+            messages=[
+                {"role": "system", "content": pair.get("system_prompt", "")},
+                {"role": "user", "content": pair["user_input"]},
+            ],
+            temperature=0.7,  # Slightly varied for multiple runs
+            max_tokens=1024,
+        )
+        response = router.invoke(model_function, request)
+        if response and response.content:
+            return {"success": True, "answer": response.content}
+    except Exception as exc:
+        return {"success": False, "error": str(exc)}
+    return {"success": False, "error": "No response"}
+
+
+def _score_answer_similarity(generated: str, expected: str) -> float:
+    """Simple token-overlap similarity between generated and expected answer."""
+    gen_tokens = set(generated.lower().split())
+    exp_tokens = set(expected.lower().split())
+    if not gen_tokens or not exp_tokens:
+        return 0.0
+    intersection = gen_tokens & exp_tokens
+    # Jaccard similarity
+    return len(intersection) / len(gen_tokens | exp_tokens)
+
+
+def pass_rate_filter(
+    pairs: List[Dict[str, str]],
+    *,
+    num_attempts: int = 3,
+    min_pass_rate: float = 0.33,
+    max_pass_rate: float = 0.90,
+    similarity_threshold: float = 0.3,
+    model_function: str = "ft_pair_generation",
+) -> Dict[str, Any]:
+    """KARL-inspired pass-rate filtering for training pair quality (D-KARL-4).
+
+    KARL's insight: the best training data is neither too easy (all models
+    solve it) nor too hard (no model solves it). The "Goldilocks zone" is
+    partial pass rate — pairs that challenge the model but are solvable.
+
+    For each Q&A pair, runs `num_attempts` independent model evaluations.
+    Keeps pairs where pass_rate is in [min_pass_rate, max_pass_rate].
+
+    Args:
+        pairs: List of {"user_input": ..., "expected_output": ..., "system_prompt": ...}
+        num_attempts: How many model attempts per pair.
+        min_pass_rate: Minimum fraction correct (below = too hard, discard).
+        max_pass_rate: Maximum fraction correct (above = too easy, discard).
+        similarity_threshold: Min token overlap to count as "correct".
+        model_function: LLM router function for evaluation.
+
+    Returns:
+        Dict with filtered pairs, stats, and per-pair pass rates.
+    """
+    filtered_pairs = []
+    too_easy = 0
+    too_hard = 0
+    goldilocks = 0
+    pair_details = []
+
+    for pair in pairs:
+        expected = pair.get("expected_output", "")
+        passes = 0
+
+        for _ in range(num_attempts):
+            result = _evaluate_pair_with_model(pair, model_function)
+            if result.get("success"):
+                similarity = _score_answer_similarity(
+                    result["answer"], expected,
+                )
+                if similarity >= similarity_threshold:
+                    passes += 1
+
+        pass_rate = passes / num_attempts if num_attempts > 0 else 0.0
+        detail = {
+            "user_input": pair["user_input"][:100],
+            "pass_rate": round(pass_rate, 2),
+            "passes": passes,
+            "attempts": num_attempts,
+            "verdict": "goldilocks",
+        }
+
+        if pass_rate > max_pass_rate:
+            detail["verdict"] = "too_easy"
+            too_easy += 1
+        elif pass_rate < min_pass_rate:
+            detail["verdict"] = "too_hard"
+            too_hard += 1
+        else:
+            detail["verdict"] = "goldilocks"
+            goldilocks += 1
+            filtered_pairs.append(pair)
+
+        pair_details.append(detail)
+
+    return {
+        "success": True,
+        "total_input": len(pairs),
+        "goldilocks_count": goldilocks,
+        "too_easy_count": too_easy,
+        "too_hard_count": too_hard,
+        "filtered_pairs": filtered_pairs,
+        "pair_details": pair_details,
+        "filter_params": {
+            "num_attempts": num_attempts,
+            "min_pass_rate": min_pass_rate,
+            "max_pass_rate": max_pass_rate,
+            "similarity_threshold": similarity_threshold,
+        },
+    }
+
+
+def generate_with_pass_rate_filter(
+    dataset_id: str,
+    chunks: List[Dict[str, Any]],
+    *,
+    purpose: str = "general",
+    questions_per_chunk: int = 3,
+    system_prompt: str = "",
+    num_attempts: int = 3,
+    min_pass_rate: float = 0.33,
+    max_pass_rate: float = 0.90,
+    classification: str = "CUI",
+    tenant_id: str = "",
+    project_id: str = "",
+    db_path: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """Generate Q&A pairs with KARL pass-rate filtering (D-KARL-4).
+
+    Two-stage pipeline:
+      1. Generate candidate pairs from chunks (existing pipeline)
+      2. Filter via pass-rate (keep Goldilocks-difficulty pairs only)
+      3. Add survivors to dataset
+
+    This produces higher-quality training data by ensuring pairs are
+    challenging enough to teach the model but not impossibly hard.
+    """
+    # Stage 1: Generate candidates
+    all_candidates = []
+    for chunk in chunks:
+        content = chunk.get("content", "")
+        if not content or len(content) < 50:
+            continue
+        pairs = _generate_pairs_from_chunk(
+            chunk_content=content,
+            purpose=purpose,
+            questions_per_chunk=questions_per_chunk,
+            system_prompt=system_prompt,
+        )
+        all_candidates.extend(pairs)
+
+    if not all_candidates:
+        return {"success": False, "error": "No candidate pairs generated"}
+
+    # Stage 2: Pass-rate filter
+    filter_result = pass_rate_filter(
+        all_candidates,
+        num_attempts=num_attempts,
+        min_pass_rate=min_pass_rate,
+        max_pass_rate=max_pass_rate,
+    )
+
+    # Stage 3: Add filtered pairs to dataset
+    from tools.finetune.dataset_manager import add_example
+
+    total_added = 0
+    total_dedup = 0
+
+    for pair in filter_result["filtered_pairs"]:
+        result = add_example(
+            dataset_id=dataset_id,
+            user_input=pair["user_input"],
+            expected_output=pair["expected_output"],
+            system_prompt=pair.get("system_prompt", system_prompt),
+            source="karl_pass_rate_filtered",
+            classification=classification,
+            tenant_id=tenant_id,
+            project_id=project_id,
+            db_path=db_path,
+        )
+        if result.get("success"):
+            if result.get("deduplicated"):
+                total_dedup += 1
+            else:
+                total_added += 1
+
+    return {
+        "success": True,
+        "dataset_id": dataset_id,
+        "candidates_generated": len(all_candidates),
+        "goldilocks_count": filter_result["goldilocks_count"],
+        "too_easy_count": filter_result["too_easy_count"],
+        "too_hard_count": filter_result["too_hard_count"],
+        "pairs_added": total_added,
+        "pairs_deduplicated": total_dedup,
+        "filter_params": filter_result["filter_params"],
+    }
+
+
+# -- CLI -----------------------------------------------------------------------
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fine-tuning Q&A pair generator")
+    parser.add_argument("--generate", action="store_true", help="Generate from document chunks")
+    parser.add_argument("--generate-from-rag", action="store_true", help="Generate from RAG chunks")
+    parser.add_argument("--generate-from-govcon", action="store_true",
+                        help="Generate from GovProposal sources (approved_drafts, knowledge_base, shall_statements)")
+    parser.add_argument("--generate-filtered", action="store_true",
+                        help="Generate with KARL pass-rate filtering (Goldilocks difficulty)")
+    parser.add_argument("--stats", action="store_true", help="Generation statistics")
+    parser.add_argument("--num-attempts", type=int, default=3,
+                        help="Pass-rate filter: model attempts per pair")
+    parser.add_argument("--min-pass-rate", type=float, default=0.33,
+                        help="Pass-rate filter: min pass rate (below = too hard)")
+    parser.add_argument("--max-pass-rate", type=float, default=0.90,
+                        help="Pass-rate filter: max pass rate (above = too easy)")
+
+    parser.add_argument("--dataset-id", type=str, default="")
+    parser.add_argument("--document-id", type=str, default="")
+    parser.add_argument("--source-table", type=str, default="")
+    parser.add_argument("--source-type", type=str, default="approved_drafts",
+                        help="GovProposal source: approved_drafts, knowledge_base, shall_statements")
+    parser.add_argument("--purpose", type=str, default="general")
+    parser.add_argument("--questions-per-chunk", type=int, default=3)
+    parser.add_argument("--system-prompt", type=str, default="")
+    parser.add_argument("--limit", type=int, default=100)
+    parser.add_argument("--classification", type=str, default="CUI")
+    parser.add_argument("--tenant-id", type=str, default="")
+    parser.add_argument("--project-id", type=str, default="")
+    parser.add_argument("--json", action="store_true")
+
+    args = parser.parse_args()
+
+    if args.generate and args.dataset_id:
+        # For CLI, user provides document ID and we extract chunks from doc_extractor
+        from tools.finetune.doc_extractor import extract_document
+        if args.document_id:
+            # Try to load chunks from a previous extraction
+            result = {"success": False, "error": "Provide --file with --generate to extract and generate pairs"}
+        else:
+            result = {"success": False, "error": "Provide --dataset-id and --document-id or use programmatic API"}
+    elif args.generate_from_rag and args.dataset_id:
+        result = generate_from_rag_source(
+            dataset_id=args.dataset_id,
+            source_table=args.source_table,
+            purpose=args.purpose,
+            questions_per_chunk=args.questions_per_chunk,
+            system_prompt=args.system_prompt,
+            limit=args.limit,
+            classification=args.classification,
+            tenant_id=args.tenant_id,
+            project_id=args.project_id,
+        )
+    elif args.generate_from_govcon and args.dataset_id:
+        result = generate_from_govcon_source(
+            dataset_id=args.dataset_id,
+            source_type=args.source_type,
+            questions_per_chunk=args.questions_per_chunk,
+            system_prompt=args.system_prompt,
+            limit=args.limit,
+            classification=args.classification,
+            tenant_id=args.tenant_id,
+            project_id=args.project_id,
+        )
+    elif args.generate_filtered and args.dataset_id and args.source_table:
+        # Generate with KARL pass-rate filtering from RAG source
+        conn = _get_db()
+        try:
+            rows = conn.execute(
+                """SELECT chunk_id, content FROM rag_chunks
+                   WHERE source_table = ? AND tier = 'hot'
+                   ORDER BY created_at DESC LIMIT ?""",
+                (args.source_table, args.limit),
+            ).fetchall()
+            chunks = [{"content": r["content"], "chunk_id": r["chunk_id"]} for r in rows]
+        except Exception:
+            chunks = []
+        finally:
+            conn.close()
+
+        if chunks:
+            result = generate_with_pass_rate_filter(
+                dataset_id=args.dataset_id,
+                chunks=chunks,
+                purpose=args.purpose,
+                questions_per_chunk=args.questions_per_chunk,
+                system_prompt=args.system_prompt,
+                num_attempts=args.num_attempts,
+                min_pass_rate=args.min_pass_rate,
+                max_pass_rate=args.max_pass_rate,
+                classification=args.classification,
+                tenant_id=args.tenant_id,
+                project_id=args.project_id,
+            )
+        else:
+            result = {"success": False, "error": f"No RAG chunks found for source: {args.source_table}"}
+    elif args.stats and args.dataset_id:
+        result = get_generation_stats(dataset_id=args.dataset_id)
+    else:
+        result = {"success": False, "error": "Specify --generate, --generate-from-rag, --generate-from-govcon, --generate-filtered, or --stats with --dataset-id"}
+
+    print(json.dumps(result, indent=2, default=str))
+
+
+if __name__ == "__main__":
+    main()