agentinputlab 0.1.0a1__py3-none-any.whl

agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py

@@ -0,0 +1,127 @@
+"""Clean-room paper-spec implementation of the Plan-and-Solve PS prompt.
+
+The authors' repository has no source-code license.  This module therefore
+implements the short prompt intervention described in the ACL 2023 paper; it
+does not vendor their experiment harness or claim historical native execution.
+"""
+
+from __future__ import annotations
+
+from agentinputlab.contracts import (
+    ExecutionTrace,
+    FrozenAgentState,
+    InputPatch,
+    InterventionContext,
+    PatchOp,
+    PatchOperation,
+    SemanticFamily,
+    TraceEvent,
+    TracePhase,
+)
+from agentinputlab.contracts.serialization import content_hash
+
+PLAN_AND_SOLVE_UPSTREAM_REVISION = "9d5a176f5b4de1011d426009ab6398dc76046e88"
+PLAN_AND_SOLVE_PROMPT_ID = 201
+PLAN_AND_SOLVE_TRIGGER = (
+    "Let's first understand the problem and devise a plan to solve the problem. "
+    "Then, let's carry out the plan to solve the problem step by step."
+)
+PLAN_AND_SOLVE_PLUS_PROMPT_ID = 302
+PLAN_AND_SOLVE_PLUS_TRIGGER = (
+    "Let's first understand the problem, extract relevant variables and their corresponding "
+    "numerals, and devise a complete plan. Then, let's carry out the plan, calculate "
+    "intermediate variables (pay attention to correct numerical calculation and commonsense), "
+    "solve the problem step by step, and show the answer."
+)
+
+
+def render_plan_and_solve_prompt(
+    question: str,
+    *,
+    trigger: str = PLAN_AND_SOLVE_TRIGGER,
+) -> str:
+    """Render the paper/repository PS input serialization for golden tests."""
+
+    return f"Q:{question}\nA: {trigger}"
+
+
+class PlanAndSolvePSPrompt:
+    """Append the PS instruction to the frozen downstream-agent input."""
+
+    method_id = "plan_and_solve"
+    release_id = "plan.plan_and_solve.ps_prompt"
+    family = SemanticFamily.DECOMPOSE
+    prompt_id = PLAN_AND_SOLVE_PROMPT_ID
+    trigger = PLAN_AND_SOLVE_TRIGGER
+    variant = "ps"
+
+    def propose(self, state: FrozenAgentState, context: InterventionContext) -> InputPatch:
+        rendered = render_plan_and_solve_prompt(state.query, trigger=self.trigger)
+        trace = ExecutionTrace(
+            method_id=self.method_id,
+            release_id=self.release_id,
+            events=(
+                TraceEvent(
+                    TracePhase.PREPARE,
+                    "paper_prompt_spec",
+                    {
+                        "prompt_id": self.prompt_id,
+                        "variant": self.variant,
+                        "trigger": self.trigger,
+                        "rendered_prompt": rendered,
+                        "rendered_prompt_hash": content_hash(rendered),
+                        "upstream_revision_for_audit": PLAN_AND_SOLVE_UPSTREAM_REVISION,
+                        "historical_backend": "text-davinci-003",
+                    },
+                ),
+                TraceEvent(
+                    TracePhase.PROPOSE,
+                    "append_plan_and_solve_instruction",
+                    {
+                        "instruction": self.trigger,
+                        "downstream_execution_required": True,
+                    },
+                ),
+            ),
+            counters={"optimizer_model_calls": 0},
+            warnings=(
+                "clean_room_paper_spec_not_upstream_source_port",
+                "modern_backend_results_are_controlled_model_transfer",
+            ),
+        )
+        return InputPatch(
+            method_id=self.method_id,
+            release_id=self.release_id,
+            source_state_hash=state.state_hash,
+            operations=(
+                PatchOperation(
+                    PatchOp.APPEND_INSTRUCTION,
+                    value=self.trigger,
+                    target="instruction",
+                    rationale=f"ACL 2023 Plan-and-Solve {self.variant.upper()} prompt instruction",
+                ),
+            ),
+            trace=trace,
+            implementation_identity=f"paper_reimplementation:acl2023:{self.variant}_prompt_{self.prompt_id}",
+            profile_kind="controlled_adapter",
+            metadata={
+                "implementation_origin": "clean_room_paper_spec",
+                "prompt_id": self.prompt_id,
+                "variant": self.variant,
+                "historical_native_available": False,
+            },
+        )
+
+
+class PlanAndSolvePSPlusPrompt(PlanAndSolvePSPrompt):
+    """Append the source-fixed PS+ 302 instruction; the caller executes the LLM.
+
+    This input intervention does not execute the original arithmetic answer-
+    extraction pass or replace a benchmark's parser/scorer. Downstream model
+    transfer and task-specific output handling remain explicit caller choices.
+    """
+
+    release_id = "plan.plan_and_solve.ps_plus_prompt"
+    prompt_id = PLAN_AND_SOLVE_PLUS_PROMPT_ID
+    trigger = PLAN_AND_SOLVE_PLUS_TRIGGER
+    variant = "ps_plus"

agentinputlab/algorithms/decompose/self_ask/__init__.py

@@ -0,0 +1,27 @@
+"""MIT-licensed Self-Ask notebook logic, minimally engineered as a loop."""
+
+from .loop import (
+    SELF_ASK_UPSTREAM_REVISION,
+    SelfAskConfig,
+    SelfAskLoop,
+    SelfAskMarkers,
+    SelfAskResult,
+    SelfAskStep,
+    parse_final_answer,
+    parse_followup_question,
+    render_initial_self_ask_prompt,
+)
+from .planner_component import SelfAskPlannerComponent
+
+__all__ = [
+    "SELF_ASK_UPSTREAM_REVISION",
+    "SelfAskConfig",
+    "SelfAskLoop",
+    "SelfAskMarkers",
+    "SelfAskPlannerComponent",
+    "SelfAskResult",
+    "SelfAskStep",
+    "parse_final_answer",
+    "parse_followup_question",
+    "render_initial_self_ask_prompt",
+]

agentinputlab/algorithms/decompose/self_ask/loop.py

@@ -0,0 +1,362 @@
+"""Stateful Self-Ask loop with injected language-model and search backends."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+from agentinputlab.backends.llm.base import GenerationRequest, LLMBackend
+from agentinputlab.backends.retrieval.base import RetrievalResult, Retriever
+from agentinputlab.contracts import (
+    AlgorithmOutputError,
+    BudgetExceededError,
+    CapabilityError,
+    ContextBlock,
+    ExecutionTrace,
+    ExternalDependencyError,
+    FrozenAgentState,
+    InputPatch,
+    InterventionContext,
+    PatchOp,
+    PatchOperation,
+    SemanticFamily,
+    TraceEvent,
+    TracePhase,
+    ValidationError,
+)
+from agentinputlab.contracts.serialization import content_hash
+
+SELF_ASK_UPSTREAM_REVISION = "559e56ec1ba8dc93cf63c4148ec1f4717caf4b60"
+
+
+@dataclass(frozen=True, slots=True)
+class SelfAskMarkers:
+    followup_needed: str = "Are follow up questions needed here:"
+    followup: str = "Follow up:"
+    intermediate_answer: str = "Intermediate answer:"
+    final_answer: str = "So the final answer is:"
+
+
+DEFAULT_SELF_ASK_MARKERS = SelfAskMarkers()
+
+
+@dataclass(frozen=True, slots=True)
+class SelfAskConfig:
+    demonstrations: str
+    max_followups: int = 5
+    retrieval_top_k: int = 1
+    allow_model_search_fallback: bool = True
+    max_tokens_per_call: int = 256
+    temperature: float = 0.0
+
+    def __post_init__(self) -> None:
+        if not self.demonstrations.strip():
+            raise ValidationError(
+                "Self-Ask full_loop requires an explicit frozen demonstration artifact"
+            )
+        if self.max_followups < 0:
+            raise ValidationError("max_followups must be non-negative")
+        if self.retrieval_top_k < 1:
+            raise ValidationError("retrieval_top_k must be positive")
+        if self.max_tokens_per_call < 1:
+            raise ValidationError("max_tokens_per_call must be positive")
+
+
+@dataclass(frozen=True, slots=True)
+class SelfAskStep:
+    index: int
+    followup_question: str
+    intermediate_answer: str
+    answer_source: str
+    document_ids: tuple[str, ...] = ()
+    retrieval_records: tuple[Mapping[str, object], ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class SelfAskResult:
+    transcript: str
+    final_answer: str
+    steps: tuple[SelfAskStep, ...]
+    model_calls: int
+    retrieval_calls: int
+    model_fallbacks: int
+    termination: str = "final_answer_marker"
+
+
+def _last_marker_value(text: str, marker: str) -> str | None:
+    pattern = re.compile(re.escape(marker) + r"\s*(?P<value>[^\n\r]+)")
+    matches = tuple(pattern.finditer(text))
+    if not matches:
+        return None
+    value = matches[-1].group("value").strip()
+    return value or None
+
+
+def parse_followup_question(
+    text: str, markers: SelfAskMarkers = DEFAULT_SELF_ASK_MARKERS
+) -> str | None:
+    return _last_marker_value(text, markers.followup)
+
+
+def parse_final_answer(text: str, markers: SelfAskMarkers = DEFAULT_SELF_ASK_MARKERS) -> str | None:
+    return _last_marker_value(text, markers.final_answer)
+
+
+def render_initial_self_ask_prompt(
+    query: str, config: SelfAskConfig, markers: SelfAskMarkers = DEFAULT_SELF_ASK_MARKERS,
+) -> str:
+    """Serialize the first stage from frozen demonstrations and the current query."""
+    return (
+        f"{config.demonstrations.rstrip()}\n\n"
+        f"Question: {query}\n{markers.followup_needed}"
+    )
+
+
+class SelfAskLoop:
+    method_id = "self_ask"
+    release_id = "plan.self_ask.full_loop"
+    family = SemanticFamily.DECOMPOSE
+
+    def __init__(
+        self,
+        config: SelfAskConfig,
+        *,
+        markers: SelfAskMarkers = DEFAULT_SELF_ASK_MARKERS,
+    ) -> None:
+        self.config = config
+        self.markers = markers
+
+    @staticmethod
+    def _require_backend(value: object) -> LLMBackend:
+        if value is None:
+            raise ExternalDependencyError("Self-Ask requires an injected LLMBackend")
+        if not hasattr(value, "generate") or not hasattr(value, "fingerprint"):
+            raise CapabilityError("Self-Ask backend does not implement LLMBackend")
+        return value  # type: ignore[return-value]
+
+    @staticmethod
+    def _require_retriever(value: object) -> Retriever:
+        if value is None:
+            raise ExternalDependencyError(
+                "Self-Ask full_loop requires a frozen Retriever/search backend"
+            )
+        if not hasattr(value, "retrieve") or not hasattr(value, "fingerprint"):
+            raise CapabilityError("Self-Ask retriever does not implement Retriever")
+        return value  # type: ignore[return-value]
+
+    def run(self, state: FrozenAgentState, context: InterventionContext) -> SelfAskResult:
+        backend = self._require_backend(context.backend)
+        retriever = self._require_retriever(context.retriever)
+        transcript = render_initial_self_ask_prompt(state.query, self.config, self.markers)
+        model_calls = 0
+        retrieval_calls = 0
+        model_fallbacks = 0
+        steps: list[SelfAskStep] = []
+        termination = "final_answer_marker"
+
+        def generate(prompt: str, *, stop: tuple[str, ...] = ()) -> str:
+            nonlocal model_calls
+            if model_calls >= state.budget.max_model_calls:
+                raise BudgetExceededError("Self-Ask exhausted max_model_calls")
+            if state.budget.max_output_tokens < 1:
+                raise BudgetExceededError("Self-Ask has no output-token budget")
+            response = backend.generate(
+                GenerationRequest(
+                    prompt=prompt,
+                    temperature=self.config.temperature,
+                    max_tokens=min(
+                        self.config.max_tokens_per_call,
+                        state.budget.max_output_tokens,
+                    ),
+                    seed=context.seed + model_calls,
+                    stop=stop,
+                    metadata={
+                        "method_id": self.method_id,
+                        "release_id": self.release_id,
+                    },
+                )
+            )
+            model_calls += 1
+            if not isinstance(response.text, str) or not response.text.strip():
+                raise AlgorithmOutputError("Self-Ask backend returned an empty response")
+            return response.text.strip()
+
+        continuation = generate(
+            transcript,
+            stop=(f"\n{self.markers.intermediate_answer}",),
+        )
+        transcript = f"{transcript} {continuation}"
+
+        while True:
+            # Parse only the current continuation.  The frozen demonstrations
+            # intentionally contain final-answer markers of their own.
+            final_answer = parse_final_answer(continuation, self.markers)
+            if final_answer is not None:
+                break
+            followup = parse_followup_question(continuation, self.markers)
+            if followup is None:
+                prompt = f"{transcript}\n{self.markers.final_answer}"
+                final_completion = generate(prompt)
+                transcript = f"{prompt} {final_completion}"
+                final_answer = parse_final_answer(final_completion, self.markers)
+                if final_answer is None:
+                    final_answer = final_completion.splitlines()[0].strip()
+                    termination = "first_line_fallback"
+                if not final_answer:
+                    raise AlgorithmOutputError("Self-Ask could not parse a final answer")
+                break
+            if len(steps) >= self.config.max_followups:
+                raise BudgetExceededError("Self-Ask reached the registered follow-up safety cap")
+            if retrieval_calls >= state.budget.max_retrieval_calls:
+                raise BudgetExceededError("Self-Ask exhausted max_retrieval_calls")
+            results: tuple[RetrievalResult, ...] = tuple(
+                retriever.retrieve(followup, top_k=self.config.retrieval_top_k)
+            )
+            retrieval_calls += 1
+            if results:
+                intermediate = results[0].document.text.strip()
+                source = "retrieval"
+                document_ids = tuple(item.document.doc_id for item in results)
+                retrieval_records = tuple(
+                    {
+                        "document_id": item.document.doc_id,
+                        "corpus_revision": item.document.corpus_revision,
+                        "rank": item.rank,
+                        "score": item.score,
+                    }
+                    for item in results
+                )
+            elif self.config.allow_model_search_fallback:
+                fallback_prompt = f"{transcript}\n{self.markers.intermediate_answer}"
+                intermediate = generate(fallback_prompt, stop=("\n",))
+                model_fallbacks += 1
+                source = "model_fallback"
+                document_ids = ()
+                retrieval_records = ()
+            else:
+                raise AlgorithmOutputError(f"Self-Ask search returned no result for {followup!r}")
+            step = SelfAskStep(
+                index=len(steps),
+                followup_question=followup,
+                intermediate_answer=intermediate,
+                answer_source=source,
+                document_ids=document_ids,
+                retrieval_records=retrieval_records,
+            )
+            steps.append(step)
+            transcript = f"{transcript}\n{self.markers.intermediate_answer} {intermediate}"
+            continuation = generate(
+                transcript,
+                stop=(f"\n{self.markers.intermediate_answer}",),
+            )
+            transcript = f"{transcript}\n{continuation}"
+
+        return SelfAskResult(
+            transcript=transcript,
+            final_answer=final_answer,
+            steps=tuple(steps),
+            model_calls=model_calls,
+            retrieval_calls=retrieval_calls,
+            model_fallbacks=model_fallbacks,
+            termination=termination,
+        )
+
+    def propose(self, state: FrozenAgentState, context: InterventionContext) -> InputPatch:
+        result = self.run(state, context)
+        events = [
+            TraceEvent(
+                TracePhase.PREPARE,
+                "self_ask_configuration",
+                {
+                    "demonstration_hash": content_hash(self.config.demonstrations),
+                    "marker_protocol": {
+                        "followup_needed": self.markers.followup_needed,
+                        "followup": self.markers.followup,
+                        "intermediate_answer": self.markers.intermediate_answer,
+                        "final_answer": self.markers.final_answer,
+                    },
+                    "max_followups": self.config.max_followups,
+                    "retriever": dict(context.retriever.fingerprint()),
+                    "backend": dict(context.backend.fingerprint()),
+                },
+            )
+        ]
+        for step in result.steps:
+            events.append(
+                TraceEvent(
+                    TracePhase.OBSERVE,
+                    "self_ask_step",
+                    {
+                        "index": step.index,
+                        "followup_question": step.followup_question,
+                        "intermediate_answer": step.intermediate_answer,
+                        "answer_source": step.answer_source,
+                        "document_ids": step.document_ids,
+                        "retrieval_records": step.retrieval_records,
+                    },
+                )
+            )
+        events.append(
+            TraceEvent(
+                TracePhase.FINALIZE,
+                "self_ask_final",
+                {
+                    "final_answer": result.final_answer,
+                    "transcript": result.transcript,
+                    "termination": result.termination,
+                },
+            )
+        )
+        trace = ExecutionTrace(
+            method_id=self.method_id,
+            release_id=self.release_id,
+            events=tuple(events),
+            counters={
+                "model_calls": result.model_calls,
+                "retrieval_calls": result.retrieval_calls,
+                "followups": len(result.steps),
+                "model_search_fallbacks": result.model_fallbacks,
+            },
+        )
+        context_block = ContextBlock(
+            text=result.transcript,
+            block_id=f"{state.task_id or state.state_hash}:self_ask",
+            source="self_ask_intermediate_trace",
+            metadata={
+                "method_id": self.method_id,
+                "release_id": self.release_id,
+                "final_answer": result.final_answer,
+            },
+        )
+        return InputPatch(
+            method_id=self.method_id,
+            release_id=self.release_id,
+            source_state_hash=state.state_hash,
+            operations=(
+                PatchOperation(
+                    PatchOp.ADD_CONTEXT,
+                    value=context_block,
+                    target="context",
+                    rationale="materialize Self-Ask intermediate reasoning/search trajectory",
+                ),
+                PatchOperation(
+                    PatchOp.SET_METADATA,
+                    value={
+                        "self_ask_final_answer": result.final_answer,
+                        "self_ask_trace_hash": trace.trace_hash,
+                    },
+                    target="metadata",
+                ),
+            ),
+            trace=trace,
+            implementation_identity=f"minimal_engineering:ofirpress/self-ask@{SELF_ASK_UPSTREAM_REVISION}",
+            profile_kind="controlled_adapter",
+            metadata={
+                "implementation_origin": "official_notebook_minimal_engineering",
+                "full_loop": True,
+                "search_mode": str(
+                    context.run_config.get("search_mode", "frozen_or_user_supplied")
+                ),
+            },
+        )

agentinputlab/algorithms/decompose/self_ask/planner_component.py

@@ -0,0 +1,117 @@
+"""First-continuation component of the existing controlled Self-Ask adapter.
+
+The source prompt, first-call stop and parsing markers are preserved. Search,
+intermediate answers and iterative solver execution are not part of this API.
+"""
+from __future__ import annotations
+
+import re
+
+from agentinputlab.backends.llm import GenerationRequest
+from agentinputlab.contracts import (
+    AlgorithmOutputError,
+    BudgetExceededError,
+    ExecutionTrace,
+    FrozenAgentState,
+    InputPatch,
+    InterventionContext,
+    PatchOp,
+    PatchOperation,
+    SemanticFamily,
+    TraceEvent,
+    TracePhase,
+    ValidationError,
+)
+from agentinputlab.contracts.serialization import content_hash
+
+from .loop import (
+    DEFAULT_SELF_ASK_MARKERS,
+    SELF_ASK_UPSTREAM_REVISION,
+    SelfAskConfig,
+    SelfAskLoop,
+    SelfAskMarkers,
+    parse_final_answer,
+    parse_followup_question,
+    render_initial_self_ask_prompt,
+)
+
+
+class SelfAskPlannerComponent:
+    """Suggest the first native follow-up in one paid, budgeted generation.
+
+    A native ``No`` decision yields an explicit NO_OP. A continuation that
+    already contains the final-answer marker remains visible as such; it is
+    not a searched fact, verified answer or complete Self-Ask trajectory.
+    """
+
+    method_id = "self_ask"
+    release_id = "plan.self_ask.planner_component"
+    family = SemanticFamily.DECOMPOSE
+
+    def __init__(self, config: SelfAskConfig, *, markers: SelfAskMarkers = DEFAULT_SELF_ASK_MARKERS):
+        if not isinstance(config, SelfAskConfig) or not isinstance(markers, SelfAskMarkers):
+            raise ValidationError("Self-Ask planner requires SelfAskConfig and SelfAskMarkers")
+        self.config, self.markers = config, markers
+
+    def propose(self, state: FrozenAgentState, context: InterventionContext) -> InputPatch:
+        if not state.query.strip():
+            raise ValidationError("Self-Ask planner requires a non-empty query")
+        if state.budget.max_model_calls < 1 or state.budget.max_output_tokens < 1:
+            raise BudgetExceededError("Self-Ask planner has no generation budget")
+        backend = SelfAskLoop._require_backend(context.backend)
+        prompt = render_initial_self_ask_prompt(state.query, self.config, self.markers)
+        request = GenerationRequest(prompt=prompt, temperature=self.config.temperature,
+            max_tokens=min(self.config.max_tokens_per_call, state.budget.max_output_tokens),
+            seed=context.seed, stop=(f"\n{self.markers.intermediate_answer}",),
+            metadata={"method_id": self.method_id, "release_id": self.release_id})
+        response = backend.generate(request)
+        continuation = response.text.strip()
+        if not continuation:
+            raise AlgorithmOutputError("Self-Ask planner returned an empty initial continuation")
+        final = parse_final_answer(continuation, self.markers)
+        followup = None if final is not None else parse_followup_question(continuation, self.markers)
+        if final is not None:
+            decision = "initial_final_answer"
+        elif followup is not None:
+            decision = "first_followup"
+        elif re.fullmatch(
+            rf"(?:{re.escape(self.markers.followup_needed)}\s*)?No[.!]?",
+            continuation, flags=re.IGNORECASE,
+        ):
+            decision = "no_followup"
+        else:
+            raise AlgorithmOutputError("Self-Ask planner initial continuation has no supported marker/No decision")
+        operation = (
+            PatchOperation(PatchOp.NO_OP, rationale="Self-Ask initial planner requested no follow-up")
+            if decision == "no_followup" else
+            PatchOperation(PatchOp.APPEND_INSTRUCTION,
+                value="Self-Ask initial planner continuation (not retrieved evidence):\n" + continuation,
+                target="instruction", rationale="materialize only the native first continuation")
+        )
+        counters = {"model_calls": 1, "optimizer_model_calls": 1, "retrieval_calls": 0,
+                    "input_tokens": response.usage.input_tokens, "output_tokens": response.usage.output_tokens}
+        if response.usage.cost_usd is not None:
+            counters["cost_usd"] = response.usage.cost_usd
+        return InputPatch(method_id=self.method_id, release_id=self.release_id,
+            source_state_hash=state.state_hash, operations=(operation,),
+            implementation_identity=f"controlled_self_ask_first_continuation@{SELF_ASK_UPSTREAM_REVISION}",
+            profile_kind="controlled_adapter",
+            trace=ExecutionTrace(method_id=self.method_id, release_id=self.release_id,
+                events=(TraceEvent(TracePhase.PREPARE, "self_ask_initial_request", {
+                    "demonstration_hash": content_hash(self.config.demonstrations),
+                    "prompt_hash": content_hash(prompt), "optimizer_input_projection": "query_only",
+                    "upstream_revision": SELF_ASK_UPSTREAM_REVISION,
+                    "max_tokens": request.max_tokens, "seed": request.seed, "stop": request.stop}),
+                    TraceEvent(TracePhase.PROPOSE, "self_ask_initial_continuation", {
+                        "continuation": continuation, "decision": decision,
+                        "first_followup": followup, "initial_answer_present": final is not None})),
+                counters=counters),
+            metadata={"component_only": True, "full_loop": False,
+                "source_stage": "full_loop_initial_continuation", "planner_decision": decision,
+                "initial_answer_present": final is not None,
+                "search_executed": False, "iterative_solver_executed": False,
+                "optimizer_input_projection": "query_only", "gold_fields_visible_to_optimizer": False,
+                "paper_result_parity_claimed": False})
+
+
+__all__ = ["SelfAskPlannerComponent"]