docent-python 0.1.49a0__tar.gz → 0.1.51a0__tar.gz

{docent_python-0.1.49a0 → docent_python-0.1.51a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docent-python
-Version: 0.1.49a0
+Version: 0.1.51a0
 Summary: Docent SDK
 Project-URL: Homepage, https://github.com/TransluceAI/docent
 Project-URL: Issues, https://github.com/TransluceAI/docent/issues

{docent_python-0.1.49a0 → docent_python-0.1.51a0}/docent/data_models/chat/content.py

@@ -39,15 +39,21 @@ class ContentReasoning(BaseContent):
     Attributes:
         type: Fixed as "reasoning" to identify this content type.
         reasoning: The actual reasoning text.
+        summary: Optional human-readable reasoning summary.
         signature: Optional signature associated with the reasoning.
         redacted: Flag indicating if the reasoning has been redacted.
     """
 
     type: Literal["reasoning"] = "reasoning"  # type: ignore
     reasoning: str
+    summary: str | None = None
     signature: str | None = None
     redacted: bool = False
 
+    @property
+    def display_reasoning(self) -> str:
+        return self.summary if self.redacted and self.summary else self.reasoning
+
 
 # Content type discriminated union
 Content = Annotated[ContentText | ContentReasoning, Discriminator("type")]

docent_python-0.1.51a0/docent/data_models/feedback.py

@@ -0,0 +1,369 @@
+"""Data structures for run-centric feedback elicitation and user context inference."""
+
+import json
+from collections.abc import Iterator
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+from docent.data_models.citation import InlineCitation
+from docent.judges.util.voting import OutputDistribution
+
+
+def _stable_json(value: Any) -> str:
+    return json.dumps(value, sort_keys=True)
+
+
+def _indent_lines(lines: list[str], indent: int) -> list[str]:
+    prefix = " " * max(0, indent)
+    return [f"{prefix}{line}" if line else "" for line in lines]
+
+
+def _tag_block(tag: str, body_lines: list[str], indent: int) -> list[str]:
+    lines = [f"<{tag}>"]
+    if body_lines:
+        lines.extend(_indent_lines(body_lines, indent))
+    else:
+        lines.extend(_indent_lines(["N/A"], indent))
+    lines.append(f"</{tag}>")
+    return lines
+
+
+def _text_or_na(text: str | None) -> str:
+    if text is None:
+        return "N/A"
+    stripped = text.strip()
+    return stripped if stripped else "N/A"
+
+
+def _text_lines_or_na(text: str | None) -> list[str]:
+    return _text_or_na(text).splitlines()
+
+
+def _render_citations_block(citations: list[InlineCitation], indent: int) -> list[str]:
+    citation_payload = [citation.model_dump(mode="json") for citation in citations]
+    citation_text = _stable_json(citation_payload) if citation_payload else "N/A"
+    return _tag_block("Citations", [citation_text], indent)
+
+
+def _render_user_distribution_block(
+    user_distribution: OutputDistribution | None,
+    indent: int,
+) -> list[str]:
+    distribution_text = (
+        _stable_json(user_distribution.model_dump(mode="json"))
+        if user_distribution is not None
+        else "N/A"
+    )
+    return _tag_block("Estimated user distribution p_u", [distribution_text], indent)
+
+
+def _render_user_distribution_reasoning_block(
+    reasoning: str | None,
+    reasoning_citations: list[InlineCitation] | None,
+    indent: int,
+) -> list[str]:
+    body_lines = _text_lines_or_na(reasoning)
+    body_lines.extend(_render_citations_block(reasoning_citations or [], indent))
+    return _tag_block("p_u reasoning", body_lines, indent)
+
+
+class LabelingRequestFocusItem(BaseModel):
+    """Specific rubric-related question the human labeler should inspect."""
+
+    question: str
+    citations: list[InlineCitation] = Field(default_factory=list[InlineCitation])
+    sample_answers: list[str] = Field(default_factory=list[str])
+
+    def to_str(self, indent: int = 0) -> str:
+        """Render focus item in a deterministic LLM-facing format."""
+        lines: list[str] = []
+
+        # Render the question and its citations as one nested block.
+        question_lines = _text_lines_or_na(self.question)
+        question_lines.extend(_render_citations_block(self.citations, indent))
+        lines.extend(_tag_block("Question", question_lines, indent))
+
+        sample_answers_lines = (
+            [
+                f"Answer {sample_idx}: {sample_answer}"
+                for sample_idx, sample_answer in enumerate(self.sample_answers, start=1)
+            ]
+            if self.sample_answers
+            else ["N/A"]
+        )
+        lines.extend(_tag_block("Sample Answers", sample_answers_lines, indent))
+        return "\n".join(lines)
+
+
+class QAPair(BaseModel):
+    """A single review-focus answer captured for one run."""
+
+    # What the user was shown
+    focus_item: LabelingRequestFocusItem
+
+    # Whether the user selected a sample answer or not
+    selected_sample_index: int | None = None
+    is_custom_response: bool = False
+
+    # What the user responded
+    answer: str
+    explanation: str | None = None
+
+    # The user could have skipped this question and provided nothing
+    status: Literal["answered", "skipped"]
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+    def to_str(self, indent: int = 0) -> str:
+        """Render QA pair in a deterministic LLM-facing format."""
+        lines = self.focus_item.to_str(indent=indent).splitlines()
+        lines.append(f"User answer: {_text_or_na(self.answer)}")
+        lines.append(f"User explanation: {_text_or_na(self.explanation)}")
+        return "\n".join(lines)
+
+
+class LabelingRequest(BaseModel):
+    """Structured labeling request shown to the user."""
+
+    title: str
+    review_context: str
+    review_context_citations: list[InlineCitation] = Field(default_factory=list[InlineCitation])
+    review_focus: list[LabelingRequestFocusItem] = Field(
+        default_factory=list[LabelingRequestFocusItem]
+    )
+    user_distribution: OutputDistribution | None = None
+    user_distribution_reasoning: str | None = None
+
+    def to_str(self, indent: int = 0) -> str:
+        """Render labeling request in a deterministic LLM-facing format."""
+        body_lines: list[str] = [f"Title: {_text_or_na(self.title)}"]
+
+        review_context_lines = _text_lines_or_na(self.review_context)
+        review_context_lines.extend(_render_citations_block(self.review_context_citations, indent))
+        body_lines.extend(_tag_block("Review Context", review_context_lines, indent))
+
+        review_focus_lines: list[str] = []
+        if self.review_focus:
+            for focus_idx, focus_item in enumerate(self.review_focus, start=1):
+                focus_lines = focus_item.to_str(indent=indent).splitlines()
+                review_focus_lines.extend(_tag_block(f"Focus {focus_idx}", focus_lines, indent))
+        else:
+            review_focus_lines.append("N/A")
+        body_lines.extend(_tag_block("Review Focus", review_focus_lines, indent))
+
+        body_lines.extend(_render_user_distribution_block(self.user_distribution, indent))
+        body_lines.extend(
+            _render_user_distribution_reasoning_block(
+                self.user_distribution_reasoning,
+                reasoning_citations=None,
+                indent=indent,
+            )
+        )
+
+        lines = _tag_block("Labeling Request", body_lines, indent)
+        return "\n".join(lines)
+
+
+class LabeledRun(BaseModel):
+    """A human label for one agent run."""
+
+    agent_run_id: str
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+    # What the user responded
+    label_value: dict[str, Any]
+    explanation: str | None = None
+
+    def to_str(
+        self,
+        labeling_request: LabelingRequest | None = None,
+        indent: int = 0,
+    ) -> str:
+        """Render user label in a deterministic LLM-facing format."""
+        body_lines = [
+            f"User label: {_stable_json(self.label_value)}",
+            f"User explanation: {_text_or_na(self.explanation)}",
+        ]
+        if labeling_request is None:
+            return "\n".join(_tag_block("Label", body_lines, indent))
+
+        body_lines.extend(
+            _render_user_distribution_block(labeling_request.user_distribution, indent)
+        )
+        body_lines.extend(
+            _render_user_distribution_reasoning_block(
+                labeling_request.user_distribution_reasoning,
+                reasoning_citations=None,
+                indent=indent,
+            )
+        )
+        return "\n".join(_tag_block("Label", body_lines, indent))
+
+
+class AgentRunFeedbackContext(BaseModel):
+    """All feedback collected for a single agent run."""
+
+    agent_run_id: str
+    round: int
+    created_at: datetime = Field(default_factory=datetime.now)
+    last_updated: datetime = Field(default_factory=datetime.now)
+
+    # What the user was shown
+    labeling_request: LabelingRequest
+
+    # What the user responded
+    qa_pairs: list[QAPair] = Field(default_factory=list[QAPair])
+    label: LabeledRun | None = None
+
+    @model_validator(mode="after")
+    def validate_nested_agent_run_ids(self) -> "AgentRunFeedbackContext":
+        """Ensure nested run IDs are consistent with the top-level run ID."""
+        if self.label is not None and self.label.agent_run_id != self.agent_run_id:
+            raise ValueError("label.agent_run_id must match agent_run_id")
+        return self
+
+    def to_str(self, indent: int = 0) -> str:
+        """Render full feedback entry in a deterministic LLM-facing format."""
+        lines = self.labeling_request.to_str(indent=indent).splitlines()
+
+        qa_lines: list[str] = []
+        if not self.qa_pairs:
+            qa_lines.append("N/A")
+        else:
+            for qa_idx, qa_pair in enumerate(self.qa_pairs, start=1):
+                qa_entry_lines = qa_pair.to_str(indent=indent).splitlines()
+                qa_lines.extend(_tag_block(f"QA {qa_idx}", qa_entry_lines, indent))
+        lines.extend(_tag_block("Question Answer Pairs", qa_lines, indent))
+
+        if self.label is None:
+            label_body_lines = [
+                "User label: N/A",
+                "User explanation: N/A",
+            ]
+            label_body_lines.extend(
+                _render_user_distribution_block(self.labeling_request.user_distribution, indent)
+            )
+            label_body_lines.extend(
+                _render_user_distribution_reasoning_block(
+                    self.labeling_request.user_distribution_reasoning,
+                    reasoning_citations=None,
+                    indent=indent,
+                )
+            )
+            lines.extend(_tag_block("Label", label_body_lines, indent))
+        else:
+            lines.extend(
+                self.label.to_str(
+                    labeling_request=self.labeling_request,
+                    indent=indent,
+                ).splitlines()
+            )
+        return "\n".join(lines)
+
+
+class FeedbackContext(BaseModel):
+    """Feedback context returned by the feedback REST API."""
+
+    feedback_context_id: str
+    feedback_session_id: str
+    agent_run_id: str
+    labeling_request: LabelingRequest
+    created_at: datetime
+    updated_at: datetime
+
+
+class FeedbackContextsResponse(BaseModel):
+    """Round-scoped feedback contexts returned by the feedback REST API."""
+
+    current_round: int
+    contexts: list[FeedbackContext] = Field(default_factory=list[FeedbackContext])
+
+
+class UserData(BaseModel):
+    """User Data (U) for user-context inference and downstream evaluation."""
+
+    initial_rubric: str
+    agent_run_feedbacks: list[AgentRunFeedbackContext] = Field(
+        default_factory=lambda: list[AgentRunFeedbackContext]()
+    )
+    created_at: datetime = Field(default_factory=datetime.now)
+    last_updated: datetime = Field(default_factory=datetime.now)
+
+    def upsert_run_feedback(self, agent_run_feedback: AgentRunFeedbackContext) -> None:
+        """Insert or replace feedback for an agent run ID, updating timestamps."""
+        now = datetime.now()
+        upserted_feedback = agent_run_feedback.model_copy(deep=True)
+        upserted_feedback.last_updated = now
+
+        for idx, existing in enumerate(self.agent_run_feedbacks):
+            if existing.agent_run_id != upserted_feedback.agent_run_id:
+                continue
+            upserted_feedback.created_at = existing.created_at
+            self.agent_run_feedbacks[idx] = upserted_feedback
+            self.last_updated = now
+            return
+
+        self.agent_run_feedbacks.append(upserted_feedback)
+        self.last_updated = now
+
+    def validate_against_agreement_keys(self, agreement_keys: set[str]) -> None:
+        """Validate stored labels and p_u outcomes against rubric agreement keys."""
+        for feedback in self.agent_run_feedbacks:
+            run_id = feedback.agent_run_id
+
+            label = feedback.label
+            if label is not None:
+                invalid_label_keys = sorted(set(label.label_value.keys()) - agreement_keys)
+                if invalid_label_keys:
+                    raise ValueError(
+                        "Run "
+                        f"{run_id} has label_value keys outside rubric agreement keys: "
+                        + ", ".join(invalid_label_keys)
+                    )
+
+            user_distribution = feedback.labeling_request.user_distribution
+            if user_distribution is None:
+                continue
+
+            for outcome_idx, outcome in enumerate(user_distribution.outcomes, start=1):
+                invalid_output_keys = sorted(set(outcome.output.keys()) - agreement_keys)
+                if invalid_output_keys:
+                    raise ValueError(
+                        "Run "
+                        f"{run_id} has user_distribution outcome #{outcome_idx} keys outside "
+                        "rubric agreement keys: " + ", ".join(invalid_output_keys)
+                    )
+                for key, value in outcome.output.items():
+                    if isinstance(value, (str, bool, int, float)):
+                        continue
+                    raise ValueError(
+                        "Run "
+                        f"{run_id} has user_distribution outcome #{outcome_idx} non-scalar "
+                        f"value for key '{key}': {type(value).__name__}"
+                    )
+
+    def iter_answered_qa_entries(self) -> Iterator[tuple[AgentRunFeedbackContext, QAPair]]:
+        """Iterate answered QA pairs with their parent run feedback."""
+        for feedback in self.agent_run_feedbacks:
+            for qa_pair in feedback.qa_pairs:
+                if qa_pair.status == "answered":
+                    yield feedback, qa_pair
+
+    def iter_skipped_qa_entries(self) -> Iterator[tuple[AgentRunFeedbackContext, QAPair]]:
+        """Iterate skipped QA pairs with their parent run feedback."""
+        for feedback in self.agent_run_feedbacks:
+            for qa_pair in feedback.qa_pairs:
+                if qa_pair.status == "skipped":
+                    yield feedback, qa_pair
+
+    def iter_labeled_entries(self) -> Iterator[tuple[AgentRunFeedbackContext, LabeledRun]]:
+        """Iterate labeled run entries with their parent run feedback."""
+        for feedback in self.agent_run_feedbacks:
+            if feedback.label is None:
+                continue
+            yield feedback, feedback.label
+
+
+# Backward-compatible alias used by older callers/scripts.
+AgentRunFeedback = AgentRunFeedbackContext

{docent_python-0.1.49a0 → docent_python-0.1.51a0}/docent/data_models/transcript.py

@@ -179,7 +179,7 @@ def format_chat_message(
     if isinstance(message, AssistantMessage) and message.content:
         for content in message.content:
             if isinstance(content, ContentReasoning):
-                cur_content = f"<reasoning>\n{content.reasoning}\n</reasoning>\n"
+                cur_content = f"<reasoning>\n{content.display_reasoning}\n</reasoning>\n"
 
     # Main content text
     cur_content += message.text