icdev 0.0.3__py3-none-any.whl

tools/govcon/draft_orchestrator.py

@@ -0,0 +1,1151 @@
+# CUI // SP-CTI
+# ICDEV Draft Orchestrator — Phase 67 (D-KB-3, D-KB-4) + Phase 68 (D-P68-1..8)
+# RAG-assisted auto-draft generation for proposal sections.
+
+"""
+Draft Orchestrator — Generate proposal section drafts using RAG + LLM.
+
+v1 Pipeline (Phase 67):
+    1. Load section metadata + opportunity context + shall-statements
+    2. Search knowledge base for relevant blocks (hybrid keyword + RAG semantic)
+    3. Fill hardprompts/proposal_draft.md template with context
+    4. Route through LLM (fine-tuned override → two-tier → chain fallback)
+    5. Save draft to proposal_section_drafts with provenance
+
+v2 Pipeline (Phase 68):
+    1. KB Retrieval — search_blocks() + methodology blocks
+    2. Engine Enrichment — enrich_from_engines() (DB keyword queries)
+    3. SMART Solutioning — LLM generates structured SMART plan using ICDEV methodology
+    4. Content Generation — LLM draft with all context + WriteGuard post-check + auto-rewrite
+
+Batch Generation:
+    - Kahn's topological sort on proposal_section_dependencies
+    - Sequential generation with prior_sections_context injection
+    - Background thread with job progress tracking
+
+Usage:
+    python tools/govcon/draft_orchestrator.py --generate --section-id <id> --opportunity-id <id> --json
+    python tools/govcon/draft_orchestrator.py --generate-v2 --section-id <id> --opportunity-id <id> --json
+    python tools/govcon/draft_orchestrator.py --preview --section-id <id> --opportunity-id <id> --json
+    python tools/govcon/draft_orchestrator.py --batch --opportunity-id <id> --json
+"""
+
+import argparse
+import collections
+import json
+import os
+import sqlite3
+import sys
+import threading
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from tools.db.storage import get_connection
+DB_PATH = None  # Storage layer handles path resolution (D-DB-20)
+
+_ROOT = Path(__file__).resolve().parent.parent.parent
+sys.path.insert(0, str(_ROOT))
+
+from tools.govcon.knowledge_base import search_blocks, increment_usage  # noqa: E402
+from tools.govcon.engine_enrichment import enrich_from_engines  # noqa: E402
+
+_DB_PATH = Path(os.environ.get("ICDEV_DB_PATH", str(_ROOT / "data" / "icdev.db")))
+_PROMPT_PATH = _ROOT / "hardprompts" / "proposal_draft.md"
+_METHODOLOGY_PATH = _ROOT / "context" / "icdev_methodology.md"
+
+
+def _get_db():
+    conn = sqlite3.connect(str(_DB_PATH))
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def _now():
+    return datetime.now(timezone.utc).isoformat()
+
+
+# ── Data loaders ─────────────────────────────────────────────────────────
+
+def _load_section(conn, section_id):
+    row = conn.execute(
+        "SELECT * FROM proposal_sections WHERE id = ?", (section_id,)
+    ).fetchone()
+    return dict(row) if row else None
+
+
+def _load_opportunity(conn, opportunity_id):
+    row = conn.execute(
+        "SELECT * FROM proposal_opportunities WHERE id = ?", (opportunity_id,)
+    ).fetchone()
+    return dict(row) if row else None
+
+
+def _load_shall_statements(conn, opportunity_id, limit=20):
+    rows = conn.execute(
+        "SELECT statement_text AS shall_text, domain_category AS domain, "
+        "statement_type AS priority_level "
+        "FROM rfp_shall_statements WHERE proposal_opportunity_id = ? "
+        "ORDER BY extracted_at DESC LIMIT ?",
+        (opportunity_id, limit),
+    ).fetchall()
+    return [dict(r) for r in rows]
+
+
+def _load_prompt_template():
+    if _PROMPT_PATH.exists():
+        return _PROMPT_PATH.read_text(encoding="utf-8")
+    return (
+        "You are a federal government proposal writer for ICDEV. "
+        "Draft the following proposal section.\n\n"
+        "## Opportunity: {opportunity_context}\n"
+        "## Requirements: {shall_statements}\n"
+        "## Reference: {knowledge_blocks}\n"
+        "## Section: {section_title} ({volume_type})\n"
+        "## Engine Enrichments: {engine_enrichments}\n"
+        "## SMART Plan: {smart_plan}\n"
+        "## Prior Sections: {prior_sections}\n"
+        "## Style: {style_hints}\n"
+    )
+
+
+def _load_methodology():
+    if _METHODOLOGY_PATH.exists():
+        text = _METHODOLOGY_PATH.read_text(encoding="utf-8")
+        return text[:3000]
+    return ""
+
+
+# ── Formatters ───────────────────────────────────────────────────────────
+
+def _build_search_query(section, opportunity):
+    parts = []
+    if section:
+        parts.append(section.get("title", ""))
+    if opportunity:
+        parts.append(opportunity.get("title", ""))
+        desc = opportunity.get("description", "")
+        if desc:
+            parts.append(desc[:200])
+    return " ".join(parts)
+
+
+def _format_opportunity_context(opp):
+    if not opp:
+        return "No opportunity context available."
+    lines = [
+        f"Title: {opp.get('title', 'N/A')}",
+        f"Solicitation: {opp.get('solicitation_number', 'N/A')}",
+        f"Agency: {opp.get('agency', 'N/A')}",
+        f"NAICS: {opp.get('naics_code', 'N/A')}",
+        f"Due Date: {opp.get('due_date', 'TBD')}",
+        f"Set-Aside: {opp.get('set_aside_type', 'None')}",
+    ]
+    desc = opp.get("description", "")
+    if desc:
+        lines.append(f"\nDescription: {desc[:500]}")
+    return "\n".join(lines)
+
+
+def _format_shall_statements(shalls):
+    if not shalls:
+        return "No specific shall-statements extracted for this opportunity."
+    lines = []
+    for i, s in enumerate(shalls, 1):
+        priority = s.get("priority_level", "medium")
+        lines.append(f"{i}. [{priority.upper()}] {s['shall_text']}")
+    return "\n".join(lines)
+
+
+def _format_knowledge_blocks(blocks):
+    if not blocks:
+        return "No relevant knowledge base content found."
+    lines = []
+    for i, b in enumerate(blocks, 1):
+        lines.append(f"--- Reference {i}: {b.get('title', 'Untitled')} ---")
+        lines.append(f"Category: {b.get('category', 'N/A')} | Domain: {b.get('domain', 'N/A')}")
+        lines.append(b.get("content", ""))
+        lines.append("")
+    return "\n".join(lines)
+
+
+def _format_engine_enrichments(enrichment: dict) -> str:
+    if not enrichment or enrichment.get("total_results", 0) == 0:
+        return "No engine enrichment data available."
+    lines = [f"Keywords searched: {', '.join(enrichment.get('keywords', [])[:10])}"]
+    for engine in ("innovation", "creative", "research"):
+        items = enrichment.get(engine, [])
+        if items:
+            lines.append(f"\n### {engine.title()} Engine ({len(items)} results)")
+            for item in items:
+                lines.append(f"- **{item.get('title', 'N/A')}** [{item.get('source', '')}]")
+                desc = item.get("description", "")
+                if desc:
+                    lines.append(f"  {desc[:200]}")
+    return "\n".join(lines)
+
+
+def _format_prior_sections(prior_sections: list[dict]) -> str:
+    if not prior_sections:
+        return "This is the first section being drafted."
+    lines = []
+    for ps in prior_sections:
+        title = ps.get("title", "Untitled")
+        content = ps.get("draft_content", "")[:400]
+        lines.append(f"### {title}")
+        lines.append(content)
+        lines.append("")
+    return "\n".join(lines)
+
+
+# ── v1 Pipeline (Phase 67 — preserved as fallback) ──────────────────────
+
+def get_draft_context(section_id, opportunity_id, top_k=5):
+    """Preview what context would be used for draft generation."""
+    conn = _get_db()
+    try:
+        section = _load_section(conn, section_id)
+        opportunity = _load_opportunity(conn, opportunity_id)
+        shalls = _load_shall_statements(conn, opportunity_id)
+    finally:
+        conn.close()
+
+    if not section:
+        return {"status": "error", "message": f"Section {section_id} not found"}
+
+    query = _build_search_query(section, opportunity)
+    volume_type = section.get("volume_type") or "technical"
+    domain_hint = "management" if volume_type == "management" else None
+
+    kb_result = search_blocks(query, domain=domain_hint, top_k=top_k)
+    kb_blocks = kb_result.get("results", []) if kb_result.get("status") == "ok" else []
+
+    template = _load_prompt_template()
+    filled = template.format(
+        opportunity_context=_format_opportunity_context(opportunity),
+        shall_statements=_format_shall_statements(shalls),
+        knowledge_blocks=_format_knowledge_blocks(kb_blocks),
+        section_title=section.get("title", "Untitled Section"),
+        volume_type=volume_type,
+        engine_enrichments="(preview only)",
+        smart_plan="(preview only)",
+        prior_sections="(preview only)",
+        style_hints="(preview only)",
+    )
+
+    return {
+        "status": "ok",
+        "section": section,
+        "opportunity": opportunity,
+        "shall_statements": shalls,
+        "knowledge_blocks": kb_blocks,
+        "knowledge_block_ids": [b["id"] for b in kb_blocks],
+        "prompt_preview": filled[:2000],
+        "prompt_length": len(filled),
+        "classification": "CUI // SP-CTI",
+    }
+
+
+def generate_draft(section_id, opportunity_id, top_k=5, force_regenerate=False):
+    """v1 RAG-assisted draft generation (Phase 67 — kept as fallback)."""
+    conn = _get_db()
+    try:
+        section = _load_section(conn, section_id)
+        opportunity = _load_opportunity(conn, opportunity_id)
+        shalls = _load_shall_statements(conn, opportunity_id)
+    finally:
+        conn.close()
+
+    if not section:
+        return {"status": "error", "message": f"Section {section_id} not found"}
+
+    if not force_regenerate:
+        conn = _get_db()
+        existing = conn.execute(
+            "SELECT id, draft_content FROM proposal_section_drafts "
+            "WHERE section_id = ? AND status != 'rejected'",
+            (section_id,),
+        ).fetchone()
+        conn.close()
+        if existing and existing["draft_content"]:
+            return {
+                "status": "ok",
+                "draft_content": existing["draft_content"],
+                "draft_id": existing["id"],
+                "message": "Existing draft found. Use force_regenerate=True to overwrite.",
+                "was_cached": True,
+            }
+
+    query = _build_search_query(section, opportunity)
+    volume_type = section.get("volume_type") or "technical"
+    domain_hint = "management" if volume_type == "management" else None
+
+    kb_result = search_blocks(query, domain=domain_hint, top_k=top_k)
+    kb_blocks = kb_result.get("results", []) if kb_result.get("status") == "ok" else []
+    kb_ids = [b["id"] for b in kb_blocks]
+
+    template = _load_prompt_template()
+    system_prompt = template.format(
+        opportunity_context=_format_opportunity_context(opportunity),
+        shall_statements=_format_shall_statements(shalls),
+        knowledge_blocks=_format_knowledge_blocks(kb_blocks),
+        section_title=section.get("title", "Untitled Section"),
+        volume_type=volume_type,
+        engine_enrichments="N/A (v1 pipeline)",
+        smart_plan="N/A (v1 pipeline)",
+        prior_sections="N/A (v1 pipeline)",
+        style_hints="N/A (v1 pipeline)",
+    )
+
+    generation_model = "none"
+    draft_content = ""
+
+    try:
+        from tools.llm.router import LLMRouter
+        from tools.llm.provider import LLMRequest
+
+        router = LLMRouter()
+        request = LLMRequest(
+            messages=[{"role": "user", "content": f"Draft the '{section.get('title', '')}' section."}],
+            system_prompt=system_prompt,
+            max_tokens=4096,
+            temperature=0.7,
+            effort="high",
+            classification="CUI // SP-CTI",
+        )
+
+        response = router.invoke("proposal_drafting", request)
+        draft_content = response.content
+        generation_model = response.model_id or response.provider or "unknown"
+
+    except Exception:
+        draft_content = _generate_placeholder_draft(section, opportunity, shalls, kb_blocks)
+        generation_model = "placeholder_no_llm"
+
+    if not draft_content:
+        return {"status": "error", "message": "Draft generation produced empty content"}
+
+    conn = _get_db()
+    try:
+        draft_id = str(uuid.uuid4())
+        now = _now()
+
+        existing = conn.execute(
+            "SELECT id FROM proposal_section_drafts WHERE section_id = ?",
+            (section_id,),
+        ).fetchone()
+
+        if existing:
+            conn.execute(
+                "UPDATE proposal_section_drafts "
+                "SET draft_content = ?, generation_model = ?, "
+                "knowledge_block_ids = ?, status = 'draft', confidence = 0.7 "
+                "WHERE id = ?",
+                (draft_content, generation_model, json.dumps(kb_ids), existing["id"]),
+            )
+            draft_id = existing["id"]
+        else:
+            conn.execute(
+                "INSERT INTO proposal_section_drafts "
+                "(id, section_id, opportunity_id, draft_content, "
+                "generation_model, knowledge_block_ids, status, confidence, "
+                "classification, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, 'draft', 0.7, 'CUI // SP-CTI', ?)",
+                (draft_id, section_id, opportunity_id, draft_content,
+                 generation_model, json.dumps(kb_ids), now),
+            )
+
+        conn.commit()
+    finally:
+        conn.close()
+
+    for block_id in kb_ids:
+        try:
+            increment_usage(block_id)
+        except Exception:
+            pass
+
+    return {
+        "status": "ok",
+        "draft_id": draft_id,
+        "draft_content": draft_content,
+        "generation_model": generation_model,
+        "knowledge_blocks_used": len(kb_ids),
+        "knowledge_block_ids": kb_ids,
+        "shall_statements_count": len(shalls),
+        "draft_method": "rag_assisted",
+        "pipeline_version": "v1",
+        "was_cached": False,
+        "classification": "CUI // SP-CTI",
+    }
+
+
+# ── v2 Pipeline (Phase 68 — Enhanced 4-Step) ────────────────────────────
+
+def generate_draft_v2(section_id, opportunity_id, top_k=5,
+                      force_regenerate=False, job_id=None,
+                      prior_sections_context=None):
+    """Enhanced 4-step pipeline:
+
+    1. KB Retrieval — search_blocks() + methodology blocks
+    2. Engine Enrichment — enrich_from_engines() (DB queries only)
+    3. SMART Solutioning — LLM generates structured SMART plan
+    4. Content Generation — LLM draft + WriteGuard post-check + auto-rewrite
+
+    Args:
+        section_id: Proposal section ID
+        opportunity_id: Opportunity ID
+        top_k: Number of KB blocks to retrieve
+        force_regenerate: Skip existing draft check
+        job_id: Optional batch job ID for progress tracking
+        prior_sections_context: List of prior section dicts for coherence
+
+    Returns:
+        dict with draft_content, pipeline steps, quality score, etc.
+    """
+    steps_log = []
+
+    def _log_step(step_name, detail, status="completed"):
+        entry = {"step": step_name, "detail": detail, "status": status, "ts": _now()}
+        steps_log.append(entry)
+        if job_id:
+            _update_job_step(job_id, section_id, step_name, detail, status)
+
+    # Load context
+    conn = _get_db()
+    try:
+        section = _load_section(conn, section_id)
+        opportunity = _load_opportunity(conn, opportunity_id)
+        shalls = _load_shall_statements(conn, opportunity_id)
+    finally:
+        conn.close()
+
+    if not section:
+        return {"status": "error", "message": f"Section {section_id} not found"}
+
+    section_title = section.get("title", "Untitled Section")
+
+    # Check existing draft
+    if not force_regenerate:
+        conn = _get_db()
+        existing = conn.execute(
+            "SELECT id, draft_content, pipeline_version FROM proposal_section_drafts "
+            "WHERE section_id = ? AND status != 'rejected'",
+            (section_id,),
+        ).fetchone()
+        conn.close()
+        if existing and existing["draft_content"]:
+            return {
+                "status": "ok",
+                "draft_content": existing["draft_content"],
+                "draft_id": existing["id"],
+                "message": "Existing draft found. Use force_regenerate=True to overwrite.",
+                "was_cached": True,
+                "pipeline_version": existing["pipeline_version"] or "v1",
+            }
+
+    # ── Step 1: KB Retrieval ─────────────────────────────────────────────
+    _log_step("kb_retrieval", f"Searching knowledge base for '{section_title}'", "running")
+
+    query = _build_search_query(section, opportunity)
+    volume_type = section.get("volume_type") or "technical"
+    domain_hint = "management" if volume_type == "management" else None
+
+    kb_result = search_blocks(query, domain=domain_hint, top_k=top_k)
+    kb_blocks = kb_result.get("results", []) if kb_result.get("status") == "ok" else []
+    kb_ids = [b["id"] for b in kb_blocks]
+
+    # Also search for methodology blocks
+    meth_result = search_blocks("GOTCHA ATLAS methodology framework", top_k=3)
+    meth_blocks = meth_result.get("results", []) if meth_result.get("status") == "ok" else []
+
+    _log_step("kb_retrieval", f"Found {len(kb_blocks)} KB blocks, {len(meth_blocks)} methodology blocks")
+
+    # ── Step 2: Engine Enrichment ────────────────────────────────────────
+    _log_step("engine_enrichment", "Querying Innovation, Creative, Research engines", "running")
+
+    enrichment = enrich_from_engines(section, opportunity, shalls, top_k=top_k)
+    enrichment_ids = enrichment.get("enrichment_ids", [])
+
+    _log_step("engine_enrichment",
+              f"Found {enrichment.get('total_results', 0)} results across 3 engines")
+
+    # ── Step 3: SMART Solutioning ────────────────────────────────────────
+    _log_step("smart_solutioning", "Generating SMART plan", "running")
+
+    methodology_text = _load_methodology()
+    smart_plan = _generate_smart_plan(
+        section, opportunity, shalls, kb_blocks, meth_blocks,
+        enrichment, methodology_text
+    )
+
+    _log_step("smart_solutioning",
+              f"SMART plan generated ({len(smart_plan)} chars)")
+
+    # ── Step 4: Content Generation + Quality Gate ────────────────────────
+    _log_step("content_generation", "Generating draft content", "running")
+
+    # Build style hints
+    style_hints = (
+        "- Write in active voice, direct and authoritative\n"
+        "- Reference ICDEV frameworks by name when the SMART plan calls for them\n"
+        "- Include measurable outcomes (KPIs, metrics) from the SMART plan\n"
+        "- Maintain terminology consistency with prior sections"
+    )
+
+    # Fill enhanced prompt template
+    template = _load_prompt_template()
+    system_prompt = template.format(
+        opportunity_context=_format_opportunity_context(opportunity),
+        shall_statements=_format_shall_statements(shalls),
+        knowledge_blocks=_format_knowledge_blocks(kb_blocks + meth_blocks),
+        engine_enrichments=_format_engine_enrichments(enrichment),
+        smart_plan=smart_plan,
+        prior_sections=_format_prior_sections(prior_sections_context or []),
+        section_title=section_title,
+        volume_type=volume_type,
+        style_hints=style_hints,
+    )
+
+    generation_model = "none"
+    draft_content = ""
+
+    try:
+        from tools.llm.router import LLMRouter
+        from tools.llm.provider import LLMRequest
+
+        router = LLMRouter()
+        request = LLMRequest(
+            messages=[{"role": "user", "content": f"Draft the '{section_title}' section."}],
+            system_prompt=system_prompt,
+            max_tokens=4096,
+            temperature=0.7,
+            effort="high",
+            classification="CUI // SP-CTI",
+        )
+
+        response = router.invoke("proposal_drafting", request)
+        draft_content = response.content
+        generation_model = response.model_id or response.provider or "unknown"
+
+    except Exception:
+        draft_content = _generate_placeholder_draft_v2(
+            section, opportunity, shalls, kb_blocks, enrichment, smart_plan
+        )
+        generation_model = "placeholder_no_llm"
+
+    if not draft_content:
+        return {"status": "error", "message": "Draft generation produced empty content"}
+
+    _log_step("content_generation",
+              f"Draft generated ({len(draft_content)} chars) via {generation_model}")
+
+    # ── WriteGuard post-check ────────────────────────────────────────────
+    writeguard_score = None
+    writeguard_result_id = None
+    was_auto_rewritten = False
+
+    try:
+        from tools.writeguard.analyzer import analyze as wg_analyze
+
+        wg_result = wg_analyze(draft_content, skip_llm=True)
+        if wg_result and wg_result.get("status") == "ok":
+            writeguard_score = wg_result.get("overall_quality_score", 0)
+            writeguard_result_id = wg_result.get("result_id")
+
+            _log_step("quality_check",
+                      f"WriteGuard score: {writeguard_score}")
+
+            # Auto-rewrite if score < 60
+            if writeguard_score and writeguard_score < 60:
+                _log_step("auto_rewrite", "Quality below 60, attempting rewrite", "running")
+                try:
+                    from tools.writeguard.rewriter import rewrite as wg_rewrite
+
+                    rewrite_result = wg_rewrite(
+                        draft_content,
+                        findings=wg_result.get("findings", []),
+                        skip_llm=True,
+                    )
+                    if rewrite_result and rewrite_result.get("status") == "ok":
+                        rewritten = rewrite_result.get("rewritten_text", "")
+                        if rewritten:
+                            draft_content = rewritten
+                            was_auto_rewritten = True
+                            # Re-check score
+                            wg2 = wg_analyze(draft_content, skip_llm=True)
+                            if wg2 and wg2.get("status") == "ok":
+                                writeguard_score = wg2.get("overall_quality_score", writeguard_score)
+                            _log_step("auto_rewrite",
+                                      f"Rewritten. New score: {writeguard_score}")
+                except Exception:
+                    _log_step("auto_rewrite", "Rewriter unavailable, keeping original", "skipped")
+    except ImportError:
+        _log_step("quality_check", "WriteGuard not available, skipping", "skipped")
+
+    # ── Save draft ───────────────────────────────────────────────────────
+    conn = _get_db()
+    try:
+        draft_id = str(uuid.uuid4())
+        now = _now()
+
+        existing = conn.execute(
+            "SELECT id FROM proposal_section_drafts WHERE section_id = ?",
+            (section_id,),
+        ).fetchone()
+
+        if existing:
+            conn.execute(
+                "UPDATE proposal_section_drafts "
+                "SET draft_content = ?, generation_model = ?, "
+                "knowledge_block_ids = ?, engine_enrichment_ids = ?, "
+                "smart_plan = ?, writeguard_score = ?, writeguard_result_id = ?, "
+                "was_auto_rewritten = ?, pipeline_version = 'v2', "
+                "status = 'draft', confidence = 0.8 "
+                "WHERE id = ?",
+                (draft_content, generation_model,
+                 json.dumps(kb_ids), json.dumps(enrichment_ids),
+                 smart_plan, writeguard_score, writeguard_result_id,
+                 1 if was_auto_rewritten else 0,
+                 existing["id"]),
+            )
+            draft_id = existing["id"]
+        else:
+            conn.execute(
+                "INSERT INTO proposal_section_drafts "
+                "(id, section_id, opportunity_id, draft_content, "
+                "generation_model, knowledge_block_ids, engine_enrichment_ids, "
+                "smart_plan, writeguard_score, writeguard_result_id, "
+                "was_auto_rewritten, pipeline_version, "
+                "status, confidence, classification, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'v2', 'draft', 0.8, 'CUI // SP-CTI', ?)",
+                (draft_id, section_id, opportunity_id, draft_content,
+                 generation_model, json.dumps(kb_ids), json.dumps(enrichment_ids),
+                 smart_plan, writeguard_score, writeguard_result_id,
+                 1 if was_auto_rewritten else 0, now),
+            )
+
+        # Flag needs_review if still low quality after rewrite
+        if writeguard_score is not None and writeguard_score < 60:
+            conn.execute(
+                "UPDATE proposal_section_drafts SET status = 'draft' "
+                "WHERE id = ? AND status = 'draft'",
+                (draft_id,),
+            )
+            _log_step("quality_gate", "Score still < 60 after rewrite. Flagged for human review.")
+
+        conn.commit()
+    finally:
+        conn.close()
+
+    for block_id in kb_ids:
+        try:
+            increment_usage(block_id)
+        except Exception:
+            pass
+
+    return {
+        "status": "ok",
+        "draft_id": draft_id,
+        "draft_content": draft_content,
+        "generation_model": generation_model,
+        "knowledge_blocks_used": kb_blocks,
+        "knowledge_block_ids": kb_ids,
+        "engine_enrichments": enrichment,
+        "smart_plan": smart_plan,
+        "writeguard_score": writeguard_score,
+        "was_auto_rewritten": was_auto_rewritten,
+        "pipeline_version": "v2",
+        "pipeline_steps": steps_log,
+        "shall_statements_count": len(shalls),
+        "was_cached": False,
+        "classification": "CUI // SP-CTI",
+    }
+
+
+# ── SMART Plan Generator ────────────────────────────────────────────────
+
+def _generate_smart_plan(section, opportunity, shalls, kb_blocks,
+                         meth_blocks, enrichment, methodology_text):
+    """Generate a SMART plan using LLM or deterministic fallback."""
+    section_title = section.get("title", "Section") if section else "Section"
+
+    # Build SMART prompt
+    smart_context = [
+        "Generate a SMART solution plan for this proposal section.\n",
+        f"Section: {section_title}",
+        f"Opportunity: {opportunity.get('title', 'N/A') if opportunity else 'N/A'}",
+    ]
+
+    if shalls:
+        smart_context.append("\nRequirements to address:")
+        for i, s in enumerate(shalls[:5], 1):
+            smart_context.append(f"  {i}. {s['shall_text'][:150]}")
+
+    if methodology_text:
+        smart_context.append(f"\nICDEV Methodology Reference:\n{methodology_text[:1500]}")
+
+    if enrichment and enrichment.get("total_results", 0) > 0:
+        smart_context.append("\nEngine insights available:")
+        for engine in ("innovation", "creative", "research"):
+            items = enrichment.get(engine, [])
+            for item in items[:2]:
+                smart_context.append(f"  - [{engine}] {item.get('title', '')}")
+
+    smart_context.append(
+        "\nOutput a SMART plan with these sections:\n"
+        "- **Specific:** What ICDEV tools/frameworks address each requirement?\n"
+        "- **Measurable:** What KPIs/evidence demonstrate compliance?\n"
+        "- **Achievable:** What proven patterns enable delivery?\n"
+        "- **Relevant:** How does this align with the customer's mission?\n"
+        "- **Time-bound:** What is the phased delivery approach?\n"
+    )
+
+    smart_prompt = "\n".join(smart_context)
+
+    # Try LLM
+    try:
+        from tools.llm.router import LLMRouter
+        from tools.llm.provider import LLMRequest
+
+        router = LLMRouter()
+        request = LLMRequest(
+            messages=[{"role": "user", "content": smart_prompt}],
+            system_prompt=(
+                "You are an ICDEV solution architect. Generate a concise SMART plan "
+                "that maps ICDEV frameworks (GOTCHA, ATLAS, RICOAS, ZTA) to the "
+                "proposal requirements. Be specific about tools and deliverables. "
+                "Keep it under 800 words."
+            ),
+            max_tokens=2048,
+            temperature=0.5,
+            effort="high",
+            classification="CUI // SP-CTI",
+        )
+
+        response = router.invoke("proposal_drafting", request)
+        return response.content or _deterministic_smart_plan(section, shalls, kb_blocks)
+
+    except Exception:
+        return _deterministic_smart_plan(section, shalls, kb_blocks)
+
+
+def _deterministic_smart_plan(section, shalls, kb_blocks):
+    """Deterministic SMART plan when LLM is unavailable."""
+    title = section.get("title", "Section") if section else "Section"
+    lines = [
+        f"## SMART Plan: {title}\n",
+        "### Specific",
+        "- ICDEV addresses each requirement using the GOTCHA 6-layer framework",
+        "- Deterministic Python tools execute compliance checks, code generation, and security scans",
+        "- ATLAS workflow (Architect→Trace→Link→Assemble→Stress-test) ensures structured delivery",
+    ]
+
+    if shalls:
+        lines.append("")
+        for i, s in enumerate(shalls[:3], 1):
+            lines.append(f"- Requirement {i}: Addressed via GOTCHA Tools layer + ATLAS Assemble phase")
+
+    lines.extend([
+        "\n### Measurable",
+        "- 100% traceability: every requirement mapped to implementation via digital thread",
+        "- Compliance crosswalk: single NIST 800-53 control auto-populates 8 frameworks",
+        "- Code quality: cyclomatic complexity < 25, zero CAT1 STIG findings",
+        "\n### Achievable",
+        "- Proven patterns from knowledge base with >60 reusable content blocks",
+        "- 11 specialized agents handle domain-specific tasks autonomously",
+        "- Two-tier LLM architecture ensures 95%+ accuracy across multi-step workflows",
+        "\n### Relevant",
+        "- Zero Trust Architecture aligned with DoD 7-pillar strategy",
+        "- CUI markings at generation time (not post-processing)",
+        "- Air-gap safe: all tools run on local SQLite, no external dependencies required",
+        "\n### Time-bound",
+        "- Sprint 1-2: Architecture and requirements decomposition (ATLAS A/T phases)",
+        "- Sprint 3-4: Implementation with TDD (ATLAS L/A phases)",
+        "- Sprint 5: Stress testing, compliance gates, ATO evidence collection (ATLAS S phase)",
+    ])
+
+    return "\n".join(lines)
+
+
+# ── Placeholder v2 ──────────────────────────────────────────────────────
+
+def _generate_placeholder_draft(section, opportunity, shalls, kb_blocks):
+    """v1 placeholder when no LLM is available."""
+    lines = ["CUI // SP-CTI\n"]
+    title = section.get("title", "Section") if section else "Section"
+    lines.append(f"# {title}\n")
+
+    if opportunity:
+        lines.append(f"**Solicitation:** {opportunity.get('solicitation_number', 'N/A')}")
+        lines.append(f"**Agency:** {opportunity.get('agency', 'N/A')}\n")
+
+    if shalls:
+        lines.append("## Requirements Addressed\n")
+        for i, s in enumerate(shalls[:5], 1):
+            lines.append(f"{i}. {s['shall_text'][:200]}")
+        lines.append("")
+
+    if kb_blocks:
+        lines.append("## Relevant Capabilities\n")
+        for b in kb_blocks[:3]:
+            lines.append(f"### {b.get('title', 'Capability')}\n")
+            lines.append(b.get("content", "")[:500])
+            lines.append("")
+
+    lines.append(
+        "\n*[NOTE: This is a placeholder draft assembled from knowledge base content. "
+        "An LLM-generated version will provide better prose when a language model is available.]*"
+    )
+
+    return "\n".join(lines)
+
+
+def _generate_placeholder_draft_v2(section, opportunity, shalls,
+                                    kb_blocks, enrichment, smart_plan):
+    """v2 placeholder with enrichment and SMART plan context."""
+    lines = ["CUI // SP-CTI\n"]
+    title = section.get("title", "Section") if section else "Section"
+    lines.append(f"# {title}\n")
+
+    if opportunity:
+        lines.append(f"**Solicitation:** {opportunity.get('solicitation_number', 'N/A')}")
+        lines.append(f"**Agency:** {opportunity.get('agency', 'N/A')}\n")
+
+    if smart_plan:
+        lines.append("## Solution Approach\n")
+        lines.append(smart_plan[:1000])
+        lines.append("")
+
+    if shalls:
+        lines.append("## Requirements Addressed\n")
+        for i, s in enumerate(shalls[:5], 1):
+            lines.append(f"{i}. {s['shall_text'][:200]}")
+        lines.append("")
+
+    if kb_blocks:
+        lines.append("## Relevant Capabilities\n")
+        for b in kb_blocks[:3]:
+            lines.append(f"### {b.get('title', 'Capability')}\n")
+            lines.append(b.get("content", "")[:500])
+            lines.append("")
+
+    if enrichment and enrichment.get("total_results", 0) > 0:
+        lines.append("## Market & Technical Context\n")
+        for engine in ("innovation", "creative", "research"):
+            for item in enrichment.get(engine, [])[:2]:
+                lines.append(f"- **{item.get('title', '')}**: {item.get('description', '')[:150]}")
+        lines.append("")
+
+    lines.append(
+        "\n*[NOTE: This is an enhanced placeholder draft (v2 pipeline). "
+        "An LLM-generated version will provide better prose when a language model is available.]*"
+    )
+
+    return "\n".join(lines)
+
+
+# ── Batch Generation ─────────────────────────────────────────────────────
+
+def _resolve_generation_order(conn, opportunity_id):
+    """Kahn's topological sort on proposal_section_dependencies.
+
+    Falls back to sort_order if no dependencies exist or cycle detected.
+    """
+    # Get all sections for this opportunity
+    sections = conn.execute(
+        "SELECT ps.id, ps.title, ps.sort_order, ps.volume_id "
+        "FROM proposal_sections ps "
+        "JOIN proposal_volumes pv ON ps.volume_id = pv.id "
+        "WHERE pv.opportunity_id = ? "
+        "ORDER BY pv.volume_number, ps.sort_order",
+        (opportunity_id,),
+    ).fetchall()
+    sections = [dict(s) for s in sections]
+
+    if not sections:
+        return []
+
+    section_ids = {s["id"] for s in sections}
+    section_map = {s["id"]: s for s in sections}
+
+    # Load dependencies
+    deps = conn.execute(
+        "SELECT section_id, depends_on_section_id "
+        "FROM proposal_section_dependencies "
+        "WHERE section_id IN ({})".format(",".join("?" * len(section_ids))),
+        list(section_ids),
+    ).fetchall()
+
+    if not deps:
+        # No dependencies — use sort_order
+        return sections
+
+    # Build adjacency list and in-degree count
+    in_degree = {s["id"]: 0 for s in sections}
+    adjacency = {s["id"]: [] for s in sections}
+
+    for d in deps:
+        src = d["depends_on_section_id"]
+        dst = d["section_id"]
+        if src in section_ids and dst in section_ids:
+            adjacency[src].append(dst)
+            in_degree[dst] = in_degree.get(dst, 0) + 1
+
+    # Kahn's algorithm
+    queue = collections.deque()
+    for sid in in_degree:
+        if in_degree[sid] == 0:
+            queue.append(sid)
+
+    ordered = []
+    while queue:
+        node = queue.popleft()
+        ordered.append(section_map[node])
+        for neighbor in adjacency.get(node, []):
+            in_degree[neighbor] -= 1
+            if in_degree[neighbor] == 0:
+                queue.append(neighbor)
+
+    if len(ordered) != len(sections):
+        # Cycle detected — fallback to sort_order
+        return sections
+
+    return ordered
+
+
+def _update_job_step(job_id, section_id, step_name, detail, status="running"):
+    """Append step progress to draft_generation_jobs."""
+    try:
+        conn = _get_db()
+        row = conn.execute(
+            "SELECT steps_completed FROM draft_generation_jobs WHERE id = ?",
+            (job_id,),
+        ).fetchone()
+        if row:
+            steps = json.loads(row["steps_completed"] or "[]")
+            steps.append({
+                "section_id": section_id,
+                "step": step_name,
+                "detail": detail,
+                "status": status,
+                "ts": _now(),
+            })
+            conn.execute(
+                "UPDATE draft_generation_jobs "
+                "SET steps_completed = ?, current_step = ?, current_section_id = ? "
+                "WHERE id = ?",
+                (json.dumps(steps), step_name, section_id, job_id),
+            )
+            conn.commit()
+        conn.close()
+    except Exception:
+        pass  # Non-critical — don't break generation
+
+
+def generate_batch(opportunity_id, force_regenerate=False):
+    """Create a batch generation job and return job_id.
+
+    Actual generation runs in a background thread.
+    """
+    conn = _get_db()
+
+    # Create job
+    job_id = str(uuid.uuid4())
+    now = _now()
+
+    ordered_sections = _resolve_generation_order(conn, opportunity_id)
+    total = len(ordered_sections)
+
+    if total == 0:
+        conn.close()
+        return {"status": "error", "message": "No sections found for this opportunity"}
+
+    conn.execute(
+        "INSERT INTO draft_generation_jobs "
+        "(id, opportunity_id, job_type, total_sections, completed_sections, "
+        "status, created_at, classification) "
+        "VALUES (?, ?, 'batch', ?, 0, 'pending', ?, 'CUI // SP-CTI')",
+        (job_id, opportunity_id, total, now),
+    )
+    conn.commit()
+    conn.close()
+
+    # Launch background thread
+    thread = threading.Thread(
+        target=_run_batch_generation,
+        args=(job_id, opportunity_id, ordered_sections, force_regenerate),
+        daemon=True,
+    )
+    thread.start()
+
+    return {
+        "status": "ok",
+        "job_id": job_id,
+        "total_sections": total,
+        "section_order": [{"id": s["id"], "title": s["title"]} for s in ordered_sections],
+        "message": "Batch generation started",
+    }
+
+
+def _run_batch_generation(job_id, opportunity_id, ordered_sections, force_regenerate):
+    """Background thread: generate each section sequentially."""
+    conn = _get_db()
+    conn.execute(
+        "UPDATE draft_generation_jobs SET status = 'running', started_at = ? WHERE id = ?",
+        (_now(), job_id),
+    )
+    conn.commit()
+    conn.close()
+
+    prior_sections = []
+    completed = 0
+
+    for section_info in ordered_sections:
+        section_id = section_info["id"]
+        section_title = section_info["title"]
+
+        # Update current section
+        conn = _get_db()
+        conn.execute(
+            "UPDATE draft_generation_jobs "
+            "SET current_section_id = ?, current_section_title = ? "
+            "WHERE id = ?",
+            (section_id, section_title, job_id),
+        )
+        conn.commit()
+        conn.close()
+
+        try:
+            result = generate_draft_v2(
+                section_id=section_id,
+                opportunity_id=opportunity_id,
+                top_k=5,
+                force_regenerate=force_regenerate,
+                job_id=job_id,
+                prior_sections_context=prior_sections[-3:],  # Last 3 sections for context
+            )
+
+            if result.get("status") == "ok" and not result.get("was_cached"):
+                prior_sections.append({
+                    "title": section_title,
+                    "draft_content": result.get("draft_content", "")[:400],
+                })
+
+            completed += 1
+
+        except Exception as e:
+            _update_job_step(job_id, section_id, "error",
+                             f"Generation failed: {str(e)[:200]}", "failed")
+            completed += 1  # Count as completed (failed) to keep progress moving
+
+        # Update completed count
+        conn = _get_db()
+        conn.execute(
+            "UPDATE draft_generation_jobs SET completed_sections = ? WHERE id = ?",
+            (completed, job_id),
+        )
+        conn.commit()
+        conn.close()
+
+    # Mark job complete
+    conn = _get_db()
+    conn.execute(
+        "UPDATE draft_generation_jobs "
+        "SET status = 'completed', completed_at = ?, "
+        "current_step = 'done', current_section_id = NULL "
+        "WHERE id = ?",
+        (_now(), job_id),
+    )
+    conn.commit()
+    conn.close()
+
+
+def get_job_progress(job_id):
+    """Get batch job progress for polling."""
+    conn = _get_db()
+    row = conn.execute(
+        "SELECT * FROM draft_generation_jobs WHERE id = ?", (job_id,),
+    ).fetchone()
+    conn.close()
+
+    if not row:
+        return {"status": "error", "message": f"Job {job_id} not found"}
+
+    job = dict(row)
+    steps = json.loads(job.get("steps_completed") or "[]")
+
+    return {
+        "status": "ok",
+        "job_id": job_id,
+        "job_status": job["status"],
+        "total_sections": job["total_sections"],
+        "completed_sections": job["completed_sections"],
+        "current_section_id": job.get("current_section_id"),
+        "current_section_title": job.get("current_section_title"),
+        "current_step": job.get("current_step"),
+        "steps": steps[-20:],  # Last 20 steps for UI
+        "started_at": job.get("started_at"),
+        "completed_at": job.get("completed_at"),
+        "error_message": job.get("error_message"),
+    }
+
+
+# ── CLI ──────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="Draft Orchestrator — RAG-assisted proposal drafting")
+    parser.add_argument("--generate", action="store_true", help="Generate draft (v1 pipeline)")
+    parser.add_argument("--generate-v2", action="store_true", help="Generate draft (v2 enhanced pipeline)")
+    parser.add_argument("--preview", action="store_true", help="Preview context without generating")
+    parser.add_argument("--batch", action="store_true", help="Batch generate all sections")
+    parser.add_argument("--section-id", type=str, help="Section ID")
+    parser.add_argument("--opportunity-id", type=str, help="Opportunity ID")
+    parser.add_argument("--top-k", type=int, default=5, help="Number of knowledge blocks")
+    parser.add_argument("--force", action="store_true", help="Force regenerate")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    if args.preview:
+        if not args.section_id or not args.opportunity_id:
+            parser.error("--preview requires --section-id and --opportunity-id")
+        result = get_draft_context(args.section_id, args.opportunity_id, args.top_k)
+    elif args.generate:
+        if not args.section_id or not args.opportunity_id:
+            parser.error("--generate requires --section-id and --opportunity-id")
+        result = generate_draft(args.section_id, args.opportunity_id, args.top_k, args.force)
+    elif args.generate_v2:
+        if not args.section_id or not args.opportunity_id:
+            parser.error("--generate-v2 requires --section-id and --opportunity-id")
+        result = generate_draft_v2(args.section_id, args.opportunity_id, args.top_k, args.force)
+    elif args.batch:
+        if not args.opportunity_id:
+            parser.error("--batch requires --opportunity-id")
+        result = generate_batch(args.opportunity_id, args.force)
+    else:
+        parser.print_help()
+        sys.exit(0)
+
+    if args.json:
+        print(json.dumps(result, indent=2, default=str))
+    else:
+        if result.get("status") == "ok":
+            if args.preview:
+                print(f"Knowledge blocks: {len(result.get('knowledge_blocks', []))}")
+                print(f"Shall-statements: {len(result.get('shall_statements', []))}")
+                print(f"Prompt length: {result.get('prompt_length', 0)} chars")
+            elif args.batch:
+                print(f"Batch job started: {result.get('job_id')}")
+                print(f"Total sections: {result.get('total_sections')}")
+            else:
+                content = result.get("draft_content", "")
+                print(f"Draft generated ({len(content)} chars) via {result.get('generation_model', '?')}")
+                print(f"Pipeline: {result.get('pipeline_version', 'v1')}")
+                if result.get("writeguard_score") is not None:
+                    print(f"WriteGuard score: {result.get('writeguard_score')}")
+        else:
+            print(f"Error: {result.get('message')}", file=sys.stderr)
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()