haiku.rag-slim 0.16.0__py3-none-any.whl → 0.24.0__py3-none-any.whl

haiku/rag/graph/research/models.py

@@ -1,149 +1,128 @@
-import uuid
-from enum import Enum
+from typing import TYPE_CHECKING
 
 from pydantic import BaseModel, Field, field_validator
 
+if TYPE_CHECKING:
+    from haiku.rag.store.models import SearchResult
 
-def _deduplicate_list(items: list[str]) -> list[str]:
-    """Remove duplicates while preserving order."""
-    return list(dict.fromkeys(items))
 
+class ResearchPlan(BaseModel):
+    """A structured research plan with sub-questions to explore."""
 
-class InsightStatus(str, Enum):
-    OPEN = "open"
-    VALIDATED = "validated"
-    TENTATIVE = "tentative"
-
-
-class GapSeverity(str, Enum):
-    LOW = "low"
-    MEDIUM = "medium"
-    HIGH = "high"
-
-
-class TrackedRecord(BaseModel):
-    """Base model for tracked entities with sources and metadata."""
-
-    model_config = {"validate_assignment": True}
-
-    id: str = Field(
-        default_factory=lambda: str(uuid.uuid4())[:8],
-        description="Unique identifier for the record",
-    )
-    supporting_sources: list[str] = Field(
-        default_factory=list,
-        description="Source identifiers backing this record",
-    )
-    notes: str | None = Field(
-        default=None,
-        description="Optional elaboration or caveats",
+    sub_questions: list[str] = Field(
+        ...,
+        description="Specific questions to research, phrased as complete questions",
     )
 
-    @field_validator("supporting_sources", mode="before")
+    @field_validator("sub_questions")
     @classmethod
-    def deduplicate_sources(cls, v: list[str]) -> list[str]:
-        """Ensure supporting_sources has no duplicates."""
-        return _deduplicate_list(v) if v else []
+    def validate_sub_questions(cls, v: list[str]) -> list[str]:
+        if len(v) < 1:
+            raise ValueError("Must have at least 1 sub-question")
+        if len(v) > 12:
+            raise ValueError("Cannot have more than 12 sub-questions")
+        return v
 
 
-class InsightRecord(TrackedRecord):
-    """Structured insight with provenance and lifecycle metadata."""
+class Citation(BaseModel):
+    """Resolved citation with full metadata for display/visual grounding."""
 
-    summary: str = Field(description="Concise description of the insight")
-    status: InsightStatus = Field(
-        default=InsightStatus.OPEN,
-        description="Lifecycle status for the insight",
-    )
-    originating_questions: list[str] = Field(
-        default_factory=list,
-        description="Research sub-questions that produced this insight",
-    )
+    document_id: str
+    chunk_id: str
+    document_uri: str
+    document_title: str | None = None
+    page_numbers: list[int] = Field(default_factory=list)
+    headings: list[str] | None = None
+    content: str
 
-    @field_validator("originating_questions", mode="before")
-    @classmethod
-    def deduplicate_questions(cls, v: list[str]) -> list[str]:
-        """Ensure originating_questions has no duplicates."""
-        return _deduplicate_list(v) if v else []
 
+class RawSearchAnswer(BaseModel):
+    """Answer to a search query with chunk references."""
 
-class GapRecord(TrackedRecord):
-    """Structured representation of an identified research gap."""
-
-    description: str = Field(description="Concrete statement of what is missing")
-    severity: GapSeverity = Field(
-        default=GapSeverity.MEDIUM,
-        description="Severity of the gap for answering the main question",
-    )
-    blocking: bool = Field(
-        default=True,
-        description="Whether this gap blocks a confident answer",
-    )
-    resolved: bool = Field(
-        default=False,
-        description="Flag indicating if the gap has been resolved",
-    )
-    resolved_by: list[str] = Field(
+    query: str = Field(..., description="The question that was answered")
+    answer: str = Field(..., description="The answer to the question")
+    cited_chunks: list[str] = Field(
         default_factory=list,
-        description="Insight IDs or notes explaining how the gap was closed",
+        description="IDs of chunks used to form the answer",
+    )
+    confidence: float = Field(
+        default=1.0,
+        description="Confidence score for this answer (0-1)",
+        ge=0.0,
+        le=1.0,
     )
-
-    @field_validator("resolved_by", mode="before")
-    @classmethod
-    def deduplicate_resolved_by(cls, v: list[str]) -> list[str]:
-        """Ensure resolved_by has no duplicates."""
-        return _deduplicate_list(v) if v else []
 
 
-class InsightAnalysis(BaseModel):
-    """Output of the insight aggregation agent."""
+class SearchAnswer(RawSearchAnswer):
+    """Answer to a search query with resolved citations."""
 
-    highlights: list[InsightRecord] = Field(
-        default_factory=list,
-        description="New or updated insights discovered this iteration",
-    )
-    gap_assessments: list[GapRecord] = Field(
-        default_factory=list,
-        description="New or updated gap records based on current evidence",
-    )
-    resolved_gaps: list[str] = Field(
+    citations: list[Citation] = Field(
         default_factory=list,
-        description="Gap identifiers or descriptions considered resolved",
-    )
-    new_questions: list[str] = Field(
-        default_factory=list,
-        max_length=3,
-        description="Up to three follow-up sub-questions to pursue next",
-    )
-    commentary: str = Field(
-        description="Short narrative summary of the incremental findings",
+        description="Resolved citations with full metadata",
     )
 
+    @classmethod
+    def from_raw(
+        cls,
+        raw: RawSearchAnswer,
+        search_results: "list[SearchResult]",
+    ) -> "SearchAnswer":
+        """Create SearchAnswer from RawSearchAnswer with resolved citations."""
+        citations = resolve_citations(raw.cited_chunks, search_results)
+        return cls(
+            query=raw.query,
+            answer=raw.answer,
+            cited_chunks=raw.cited_chunks,
+            confidence=raw.confidence,
+            citations=citations,
+        )
+
+
+def resolve_citations(
+    cited_chunk_ids: list[str],
+    search_results: "list[SearchResult]",
+) -> list[Citation]:
+    """Resolve chunk IDs to full Citation objects with metadata."""
+    by_id = {r.chunk_id: r for r in search_results if r.chunk_id}
+
+    citations = []
+    for chunk_id in cited_chunk_ids:
+        r = by_id.get(chunk_id)
+        if not r:
+            continue
+        citations.append(
+            Citation(
+                document_id=r.document_id or "",
+                chunk_id=chunk_id,
+                document_uri=r.document_uri or "",
+                document_title=r.document_title,
+                page_numbers=r.page_numbers,
+                headings=r.headings,
+                content=r.content,
+            )
+        )
+    return citations
+
 
 class EvaluationResult(BaseModel):
-    """Result of analysis and evaluation."""
+    """Result of research sufficiency evaluation."""
 
-    key_insights: list[str] = Field(
-        description="Main insights extracted from the research so far"
-    )
-    new_questions: list[str] = Field(
-        description="New sub-questions to add to the research (max 3)",
-        max_length=3,
-        default=[],
-    )
-    gaps: list[str] = Field(
-        description="Concrete information gaps that remain", default_factory=list
+    is_sufficient: bool = Field(
+        description="Whether the research is sufficient to answer the original question"
     )
     confidence_score: float = Field(
-        description="Confidence level in the completeness of research (0-1)",
         ge=0.0,
         le=1.0,
-    )
-    is_sufficient: bool = Field(
-        description="Whether the research is sufficient to answer the original question"
+        description="Confidence level in the completeness of research (0-1)",
     )
     reasoning: str = Field(
         description="Explanation of why the research is or isn't complete"
     )
+    new_questions: list[str] = Field(
+        default_factory=list,
+        max_length=3,
+        description="New sub-questions to add to the research (max 3)",
+    )
 
 
 class ResearchReport(BaseModel):

haiku/rag/graph/research/prompts.py

@@ -1,107 +1,115 @@
-INSIGHT_AGENT_PROMPT = """You are the insight aggregation specialist for the
-research workflow.
+PLAN_PROMPT = """You are the research orchestrator for a focused, iterative workflow.
+
+Responsibilities:
+1. Understand and decompose the main question
+2. Propose a minimal, high-leverage plan
+3. Coordinate specialized agents to gather evidence
+4. Iterate based on gaps and new findings
+
+Plan requirements:
+- Produce at most 3 sub_questions that together cover the main question.
+- sub_questions must be a list of plain strings, where each string is a complete
+  question. Do NOT use objects with nested fields like {question, details}.
+- Each sub_question must be a standalone, self-contained query that can run
+  without extra context. Include concrete entities, scope, timeframe, and any
+  qualifiers. Avoid ambiguous pronouns (it/they/this/that).
+- Prioritize the highest-value aspects first; avoid redundancy and overlap.
+- Prefer questions that are likely answerable from the current knowledge base;
+  if coverage is uncertain, make scopes narrower and specific.
+- Order sub_questions by execution priority (most valuable first).
+
+Use the gather_context tool once on the main question before planning."""
+
+SEARCH_PROMPT = """You are a search and question-answering specialist.
+
+Process:
+1. Call search_and_answer with relevant keywords from the question.
+2. Review the results and their relevance scores.
+3. If needed, perform follow-up searches with different keywords (max 3 total).
+4. Provide a concise answer based strictly on the retrieved content.
+
+The search tool returns results like:
+[9bde5847-44c9-400a-8997-0e6b65babf92] (score: 0.85)
+Source: "Document Title" > Section > Subsection
+Type: paragraph
+Content:
+The actual text content here...
+
+[d5a63c82-cb40-439f-9b2e-de7d177829b7] (score: 0.72)
+Source: "Another Document"
+Type: table
+Content:
+| Column 1 | Column 2 |
+...
+
+Each result includes:
+- chunk_id in brackets and relevance score
+- Source: document title and section hierarchy (when available)
+- Type: content type like paragraph, table, code, list_item (when available)
+- Content: the actual text
+
+Output format:
+- query: Echo the question you are answering
+- answer: Your concise answer based on the retrieved content
+- cited_chunks: List of plain strings containing only the chunk UUIDs (not objects)
+- confidence: A score from 0.0 to 1.0 indicating answer confidence
+
+IMPORTANT: Use the EXACT, COMPLETE chunk ID (full UUID). Do NOT truncate IDs.
+
+Guidelines:
+- Base answers strictly on retrieved content - do not use external knowledge.
+- Use the Source and Type metadata to understand context.
+- If multiple results are relevant, synthesize them coherently.
+- If information is insufficient, say so clearly.
+- Be concise and direct; avoid meta commentary about the process.
+- Higher scores indicate more relevant results."""
+
+DECISION_PROMPT = """You are the research evaluator responsible for assessing
+whether gathered evidence sufficiently answers the research question.
 
 Inputs available:
-- Original research question and sub-questions
-- Question–answer pairs with supporting snippets and sources
-- Existing insights and gaps (with status metadata)
+- Original research question
+- Question-answer pairs with supporting sources
+- Previous evaluation (if any)
 
 Tasks:
-1. Extract new or refined insights that advance understanding of the question.
-2. Update gap status, creating new gap entries when necessary and marking
-   resolved ones explicitly.
-3. Suggest up to 3 high-impact follow-up sub_questions that would close the
-   most important remaining gaps.
-
-Output format (map directly to fields):
-- highlights: list of insights with fields {summary, status, supporting_sources,
-  originating_questions, notes}. Use status one of {validated, open, tentative}.
-- gap_assessments: list of gaps with fields {description, severity, blocking,
-  resolved, resolved_by, supporting_sources, notes}. Severity must be one of
-  {low, medium, high}. resolved_by may reference related insight summaries if no
-  stable identifier yet.
-- resolved_gaps: list of identifiers or descriptions for gaps now closed.
-- new_questions: up to 3 standalone, specific sub-questions (no duplicates with
-  existing ones).
-- commentary: 1–3 sentences summarizing what changed this round.
-
-Guidance:
-- Be concise and avoid repeating previously recorded information unless it
-  changed materially.
-- Tie supporting_sources to the evidence used; omit if unavailable.
-- Only propose new sub_questions that directly address remaining gaps.
-- When marking a gap as resolved, ensure the rationale is clear via
-  resolved_by or notes."""
-
-DECISION_AGENT_PROMPT = """You are the research governor responsible for making
-stop/go decisions.
+1. Assess whether the collected evidence answers the original question.
+2. Provide a confidence_score in [0,1] reflecting coverage and evidence quality.
+3. Optionally propose up to 3 new sub-questions if important gaps remain.
 
-Inputs available:
-- Original research question and current plan
-- Full insight ledger with status metadata
-- Up-to-date gap tracker, including resolved indicators
-- Latest insight analysis summary (highlights, gap changes, new questions)
-- Previous evaluation decision (if any)
+Output fields:
+- is_sufficient: true when the question is adequately answered
+- confidence_score: numeric in [0,1]
+- reasoning: brief explanation of the assessment
+- new_questions: list of follow-up questions (max 3), only if needed
 
-Tasks:
-1. Determine whether the collected evidence now answers the original question.
-2. Provide a confidence_score in [0,1] that reflects coverage, evidence quality,
-   and agreement across sources.
-3. List the highest-priority gaps that still block a confident answer. Reference
-   existing gap descriptions rather than inventing new ones.
-4. Optionally propose up to 3 new sub_questions only if they are not already in
-   the current backlog.
-
-Strictness:
-- Only mark research as sufficient when every critical aspect of the main
-  question is addressed with reliable, corroborated evidence.
-- Treat unresolved high-severity or blocking gaps as a hard stop.
-
-Output fields must line up with EvaluationResult:
-- key_insights: concise bullet-ready statements of the most decision-relevant
-  insights (cite status if helpful).
-- new_questions: follow-up sub-questions (max 3) meeting the specificity rules.
-- gaps: list remaining blockers; reuse wording from the tracked gaps when
-  possible to aid downstream reconciliation.
-- confidence_score: numeric in [0,1].
-- is_sufficient: true only when no blocking gaps remain.
-- reasoning: short narrative tying the decision to evidence coverage.
-
-Remember: prefer maintaining continuity with the structured context over
-introducing new terminology."""
-
-SYNTHESIS_AGENT_PROMPT = """You are a synthesis specialist producing the final
-research report.
+Be strict: only mark sufficient when key aspects are addressed with reliable evidence."""
+
+SYNTHESIS_PROMPT = """You are a synthesis specialist producing the final
+research report that directly answers the original question.
 
 Goals:
-1. Synthesize all gathered information into a coherent narrative.
+1. Directly answer the research question using gathered evidence.
 2. Present findings clearly and concisely.
-3. Draw evidence‑based conclusions and recommendations.
+3. Draw evidence-based conclusions and recommendations.
 4. State limitations and uncertainties transparently.
 
 Report guidelines (map to output fields):
-- title: concise (5–12 words), informative.
-- executive_summary: 3–5 sentences summarizing the overall answer.
-- main_findings: 4–8 one‑sentence bullets; each reflects evidence from the
-  research (do not include inline citations or snippet text).
-- conclusions: 2–4 bullets that follow logically from findings.
-- recommendations: 2–5 actionable bullets tied to findings.
-- limitations: 1–3 bullets describing key constraints or uncertainties.
-- sources_summary: 2–4 sentences summarizing sources used and their reliability.
+- title: concise (5-12 words), informative.
+- executive_summary: 3-5 sentences that DIRECTLY ANSWER the original question.
+  Write the actual answer, not a description of what the report contains.
+  BAD: "This report examines the topic and presents findings..."
+  GOOD: "The system requires configuration X and supports features Y and Z..."
+- main_findings: list of plain strings, 4-8 one-sentence bullets reflecting evidence.
+- conclusions: list of plain strings, 2-4 bullets following logically from findings.
+- recommendations: list of plain strings, 2-5 actionable bullets tied to findings.
+- limitations: list of plain strings, 1-3 bullets describing constraints or uncertainties.
+- sources_summary: single string listing sources with document paths and page numbers.
+
+All list fields must contain plain strings only, not objects.
 
 Style:
 - Base all content solely on the collected evidence.
 - Be professional, objective, and specific.
-- Avoid meta commentary and refrain from speculation beyond the evidence."""
-
-PRESEARCH_AGENT_PROMPT = """You are a rapid research surveyor.
-
-Task:
-- Call gather_context once on the main question to obtain relevant text from
-  the knowledge base (KB).
-- Read that context and produce a short natural‑language summary of what the
-  KB appears to contain relative to the question.
-
-Rules:
-- Base the summary strictly on the provided text; do not invent.
-- Output only the summary as plain text (one short paragraph)."""
+- NEVER use meta-commentary like "This report covers..." or "The findings show...".
+  Instead, state the actual information directly."""

haiku/rag/graph/research/state.py

@@ -1,22 +1,29 @@
 import asyncio
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 from pydantic import BaseModel, Field
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.graph.research.dependencies import ResearchContext
-from haiku.rag.graph.research.models import (
-    EvaluationResult,
-    InsightAnalysis,
-    ResearchReport,
-)
+from haiku.rag.graph.research.models import EvaluationResult, ResearchReport
 
 if TYPE_CHECKING:
     from haiku.rag.config.models import AppConfig
     from haiku.rag.graph.agui.emitter import AGUIEmitter
 
 
+class HumanDecision(BaseModel):
+    """Human decision input for interactive research."""
+
+    action: Literal[
+        "search", "synthesize", "modify_questions", "add_questions", "chat", "research"
+    ]
+    questions: list[str] | None = None
+    message: str | None = None
+    research_question: str | None = None
+
+
 @dataclass
 class ResearchDeps:
     """Dependencies for research graph execution."""
@@ -24,14 +31,11 @@ class ResearchDeps:
     client: HaikuRAG
     agui_emitter: "AGUIEmitter[ResearchState, ResearchReport] | None" = None
     semaphore: asyncio.Semaphore | None = None
+    human_input_queue: asyncio.Queue[HumanDecision] | None = None
+    interactive: bool = False
 
     def emit_log(self, message: str, state: "ResearchState | None" = None) -> None:
-        """Emit a log message through AG-UI events.
-
-        Args:
-            message: The message to log
-            state: Optional state to include in state update
-        """
+        """Emit a log message through AG-UI events."""
         if self.agui_emitter:
             self.agui_emitter.log(message)
             if state:
@@ -39,15 +43,12 @@ class ResearchDeps:
 
 
 class ResearchState(BaseModel):
-    """Research graph state model.
-
-    Fully JSON-serializable Pydantic model suitable for AG-UI state synchronization.
-    """
+    """Research graph state model."""
 
     model_config = {"arbitrary_types_allowed": True}
 
     context: ResearchContext = Field(
-        description="Shared research context with questions, insights, and gaps"
+        description="Shared research context with questions and QA responses"
     )
     iterations: int = Field(default=0, description="Current iteration number")
     max_iterations: int = Field(default=3, description="Maximum allowed iterations")
@@ -60,26 +61,33 @@ class ResearchState(BaseModel):
     last_eval: EvaluationResult | None = Field(
         default=None, description="Last evaluation result"
     )
-    last_analysis: InsightAnalysis | None = Field(
-        default=None, description="Last insight analysis"
+    search_filter: str | None = Field(
+        default=None, description="SQL WHERE clause to filter search results"
     )
 
     @classmethod
     def from_config(
-        cls, context: ResearchContext, config: "AppConfig"
+        cls,
+        context: ResearchContext,
+        config: "AppConfig",
+        max_iterations: int | None = None,
+        confidence_threshold: float | None = None,
     ) -> "ResearchState":
         """Create a ResearchState from an AppConfig.
 
         Args:
-            context: The ResearchContext containing the question and settings
-            config: The AppConfig object (uses config.research for state parameters)
-
-        Returns:
-            A configured ResearchState instance
+            context: The ResearchContext containing the question
+            config: The AppConfig object
+            max_iterations: Override max iterations (None uses config default)
+            confidence_threshold: Override threshold (None uses config, 0.0 disables check)
         """
         return cls(
             context=context,
-            max_iterations=config.research.max_iterations,
-            confidence_threshold=config.research.confidence_threshold,
+            max_iterations=max_iterations
+            if max_iterations is not None
+            else config.research.max_iterations,
+            confidence_threshold=confidence_threshold
+            if confidence_threshold is not None
+            else config.research.confidence_threshold,
             max_concurrency=config.research.max_concurrency,
         )

haiku/rag/inspector/__init__.py

@@ -0,0 +1,8 @@
+try:
+    from haiku.rag.inspector.app import run_inspector
+except ImportError as e:
+    raise ImportError(
+        "textual is not installed. Please install it with `pip install 'haiku.rag-slim[inspector]'` or use the full haiku.rag package."
+    ) from e
+
+__all__ = ["run_inspector"]