prompture 1.11.1.dev1__tar.gz → 1.11.2.dev2__tar.gz

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/.env.copy

@@ -9,10 +9,14 @@ AI_PROVIDER=ollama
 # ─── OpenAI ──────────────────────────────────────────────────────────────────
 OPENAI_API_KEY=your-openai-api-key
 OPENAI_MODEL=gpt-4o-mini
+# Optional: only used by explicit BillingClient report requests.
+OPENAI_ADMIN_KEY=
 
 # ─── Claude (Anthropic) ─────────────────────────────────────────────────────
 CLAUDE_API_KEY=your-claude-api-key
 CLAUDE_MODEL=claude-haiku-4-5-20251001
+# Optional organization reporting credential, separate from CLAUDE_API_KEY.
+ANTHROPIC_ADMIN_KEY=
 
 # ─── Ollama (local) ─────────────────────────────────────────────────────────
 OLLAMA_ENDPOINT=http://localhost:11434

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prompture
-Version: 1.11.1.dev1
+Version: 1.11.2.dev2
 Summary: Ask LLMs to return structured JSON and run cross-model tests. API-first.
 Author-email: Juan Denis <juan@vene.co>
 License-Expression: MIT

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/prompture/__init__.py

@@ -11,6 +11,7 @@ except ImportError:  # tukuy not installed
     create_tukuy_backend = None  # type: ignore[assignment]
 from . import eval as eval
 from . import (
+    execution,
     jobs,
     mcp,
     plugins,

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/prompture/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.11.1.dev1'
-__version_tuple__ = version_tuple = (1, 11, 1, 'dev1')
+__version__ = version = '1.11.2.dev2'
+__version_tuple__ = version_tuple = (1, 11, 2, 'dev2')
 
 __commit_id__ = commit_id = None

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/prompture/agents/assistant.py

@@ -464,6 +464,61 @@ class Assistant:
             raw=result,
         )
 
+    # ----- execution strategies -----------------------------------------
+
+    def execution_request(self, prompt: str, **kwargs: Any) -> Any:
+        """Build an :class:`~prompture.execution.types.ExecutionRequest` from this bundle.
+
+        Use it when you want to inspect or adjust the request before running a
+        strategy.  See :func:`prompture.execution.integration.request_from_assistant`
+        for the accepted keyword arguments (``category``, ``output_model``,
+        ``passages``, ``retriever``, ``limits``, ``allowed_tools``, ``variables``).
+        """
+        from ..execution.integration import request_from_assistant
+
+        return request_from_assistant(self, prompt, **kwargs)
+
+    def run_strategy(self, prompt: str, strategy: Any = "direct", **kwargs: Any) -> Any:
+        """Run this assistant's persona/skills/tools through an execution strategy.
+
+        This is an opt-in alternative to :meth:`arun`, not a replacement:
+        :meth:`arun` keeps returning an :class:`AssistantResult` from the agent
+        loop, while this returns an
+        :class:`~prompture.execution.types.ExecutionResult` with validation,
+        evidence, step records and full usage accounting.
+
+        Args:
+            prompt: The task.
+            strategy: A registered strategy name (``"direct"``,
+                ``"retrieve_verify"``, ``"draft_critique"``) or a pre-built
+                :class:`~prompture.execution.strategies.base.ExecutionStrategy`.
+            **kwargs: Forwarded to
+                :func:`~prompture.execution.integration.request_from_assistant`.
+
+        Example::
+
+            result = assistant.run_strategy("Summarise the policy", strategy="direct")
+            print(result.termination.value, result.usage.cost_complete)
+        """
+        from ..execution.integration import resolve_strategy_instance
+
+        request = self.execution_request(prompt, **kwargs)
+        instance = resolve_strategy_instance(strategy)
+        if instance.model is None and not request.model:
+            raise ValueError(
+                f"Assistant {self.name!r} has no model and the strategy has no default; "
+                "set `model=` on one of them."
+            )
+        return instance.run(request)
+
+    async def arun_strategy(self, prompt: str, strategy: Any = "direct", **kwargs: Any) -> Any:
+        """Async counterpart to :meth:`run_strategy`."""
+        from ..execution.integration import resolve_strategy_instance
+
+        request = self.execution_request(prompt, **kwargs)
+        instance = resolve_strategy_instance(strategy)
+        return await instance.arun(request)
+
     # ----- sync convenience --------------------------------------------
 
     def run(self, prompt: str, **kwargs: Any) -> AssistantResult:

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/prompture/agents/async_conversation.py

@@ -597,6 +597,9 @@ class AsyncConversation:
         return msgs
 
     def _accumulate_usage(self, meta: dict[str, Any]) -> None:
+        from ..infra.tracker import accumulate_usage_details
+
+        accumulate_usage_details(self._usage, meta)
         delta_tokens = meta.get("total_tokens", 0)
         delta_cost = meta.get("cost", 0.0)
         prompt_tokens = meta.get("prompt_tokens", 0)
@@ -943,32 +946,73 @@ class AsyncConversation:
             turn_usage: dict[str, Any] = {}
 
             stream = self._driver.generate_messages_with_tools_stream(msgs, tool_defs, merged)
-            async for event in stream:
-                yield event
-                et = getattr(event, "event_type", None)
-                if et == "text_delta":
-                    assistant_text_parts.append(event.text)
-                elif et == "thinking_delta":
-                    assistant_thinking_parts.append(event.text)
-                elif et == "tool_use_stop":
-                    tool_use_id = event.id or f"call_{uuid.uuid4().hex}"
-                    pending_tools.append(
-                        {
-                            "id": tool_use_id,
-                            "name": event.name,
-                            "arguments": event.input,
-                            "truncated": getattr(event, "truncated", False),
-                        }
-                    )
-                    tool_calls_in_turn.append(
-                        {
-                            "id": tool_use_id,
-                            "type": "function",
-                            "function": {"name": event.name, "arguments": json.dumps(event.input)},
-                        }
+            stream_started = time.perf_counter()
+            recorded = False
+            stream_error = None
+            try:
+                async for event in stream:
+                    et = getattr(event, "event_type", None)
+                    if et == "text_delta":
+                        assistant_text_parts.append(event.text)
+                    elif et == "thinking_delta":
+                        assistant_thinking_parts.append(event.text)
+                    elif et == "tool_use_stop":
+                        tool_use_id = event.id or f"call_{uuid.uuid4().hex}"
+                        pending_tools.append(
+                            {
+                                "id": tool_use_id,
+                                "name": event.name,
+                                "arguments": event.input,
+                                "truncated": getattr(event, "truncated", False),
+                            }
+                        )
+                        tool_calls_in_turn.append(
+                            {
+                                "id": tool_use_id,
+                                "type": "function",
+                                "function": {"name": event.name, "arguments": json.dumps(event.input)},
+                            }
+                        )
+                    elif et == "message_stop":
+                        turn_usage = dict(event.usage or {})
+                        self._driver._auto_record_usage(
+                            {"meta": turn_usage}, (time.perf_counter() - stream_started) * 1000
+                        )
+                        self._driver._fire_callback(
+                            "on_response",
+                            {
+                                "meta": turn_usage,
+                                "driver": self._model_name,
+                                "elapsed_ms": (time.perf_counter() - stream_started) * 1000,
+                            },
+                        )
+                        self._accumulate_usage(turn_usage)
+                        recorded = True
+                    yield event
+            except Exception as exc:
+                stream_error = exc
+                self._driver._fire_callback("on_error", {"error": exc, "driver": self._model_name})
+                raise
+            finally:
+                if not recorded:
+                    incomplete = {"usage_complete": False, "cost_status": "unknown"}
+                    self._driver._auto_record_usage(
+                        {"meta": incomplete},
+                        (time.perf_counter() - stream_started) * 1000,
+                        status="error" if stream_error is not None else "incomplete",
+                        error=stream_error,
                     )
-                elif et == "message_stop":
-                    turn_usage = dict(event.usage or {})
+                    if stream_error is None:
+                        self._driver._fire_callback(
+                            "on_response",
+                            {
+                                "meta": incomplete,
+                                "driver": self._model_name,
+                                "status": "incomplete",
+                                "elapsed_ms": (time.perf_counter() - stream_started) * 1000,
+                            },
+                        )
+                    self._accumulate_usage(incomplete)
 
             full_text = "".join(assistant_text_parts)
             full_thinking = "".join(assistant_thinking_parts) or None
@@ -982,8 +1026,6 @@ class AsyncConversation:
 
             self._messages.append(assistant_msg)
             msgs.append(assistant_msg)
-            if turn_usage:
-                self._accumulate_usage(turn_usage)
 
             if not pending_tools:
                 yield TurnComplete(usage=dict(self._usage))
@@ -1237,6 +1279,8 @@ class AsyncConversation:
 
         t0 = time.perf_counter()
         full_text = ""
+        recorded = False
+        stream_error = None
         try:
             async for chunk in self._driver.generate_messages_stream(messages, merged):
                 chunk_type = chunk["type"]
@@ -1261,10 +1305,13 @@ class AsyncConversation:
                             "elapsed_ms": elapsed_ms,
                         },
                     )
+                    self._driver._auto_record_usage({"meta": meta}, elapsed_ms)
+                    recorded = True
                     self._accumulate_usage(meta)
                     if chunk.get("reasoning_content"):
                         self._last_reasoning = chunk["reasoning_content"]
         except Exception as exc:
+            stream_error = exc
             elapsed_ms = (time.perf_counter() - t0) * 1000
             self._driver._fire_callback(
                 "on_error",
@@ -1272,6 +1319,32 @@ class AsyncConversation:
             )
             raise
 
+        finally:
+            if not recorded:
+                elapsed_ms = (time.perf_counter() - t0) * 1000
+                incomplete = {
+                    "usage_complete": False,
+                    "cost_status": "unknown",
+                    "requested_model": merged.get("model", driver_name),
+                }
+                self._driver._auto_record_usage(
+                    {"meta": incomplete},
+                    elapsed_ms,
+                    status="error" if stream_error is not None else "incomplete",
+                    error=stream_error,
+                )
+                if stream_error is None:
+                    self._driver._fire_callback(
+                        "on_response",
+                        {
+                            "meta": incomplete,
+                            "driver": driver_name,
+                            "status": "incomplete",
+                            "elapsed_ms": elapsed_ms,
+                        },
+                    )
+                self._accumulate_usage(incomplete)
+
         self._messages.append(self._assistant_message(full_text))
 
     async def ask_stream(

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/prompture/agents/conversation.py

@@ -601,6 +601,9 @@ class Conversation:
         return msgs
 
     def _accumulate_usage(self, meta: dict[str, Any]) -> None:
+        from ..infra.tracker import accumulate_usage_details
+
+        accumulate_usage_details(self._usage, meta)
         delta_tokens = meta.get("total_tokens", 0)
         delta_cost = meta.get("cost", 0.0)
         prompt_tokens = meta.get("prompt_tokens", 0)
@@ -984,33 +987,74 @@ class Conversation:
             stop_reason: str = "end_turn"
 
             stream = self._driver.generate_messages_with_tools_stream(msgs, tool_defs, merged)
-            for event in stream:
-                yield event
-                et = getattr(event, "event_type", None)
-                if et == "text_delta":
-                    assistant_text_parts.append(event.text)
-                elif et == "thinking_delta":
-                    assistant_thinking_parts.append(event.text)
-                elif et == "tool_use_stop":
-                    tool_use_id = event.id or f"call_{uuid.uuid4().hex}"
-                    pending_tools.append(
-                        {
-                            "id": tool_use_id,
-                            "name": event.name,
-                            "arguments": event.input,
-                            "truncated": getattr(event, "truncated", False),
-                        }
-                    )
-                    tool_calls_in_turn.append(
-                        {
-                            "id": tool_use_id,
-                            "type": "function",
-                            "function": {"name": event.name, "arguments": json.dumps(event.input)},
-                        }
+            stream_started = time.perf_counter()
+            recorded = False
+            stream_error = None
+            try:
+                for event in stream:
+                    et = getattr(event, "event_type", None)
+                    if et == "text_delta":
+                        assistant_text_parts.append(event.text)
+                    elif et == "thinking_delta":
+                        assistant_thinking_parts.append(event.text)
+                    elif et == "tool_use_stop":
+                        tool_use_id = event.id or f"call_{uuid.uuid4().hex}"
+                        pending_tools.append(
+                            {
+                                "id": tool_use_id,
+                                "name": event.name,
+                                "arguments": event.input,
+                                "truncated": getattr(event, "truncated", False),
+                            }
+                        )
+                        tool_calls_in_turn.append(
+                            {
+                                "id": tool_use_id,
+                                "type": "function",
+                                "function": {"name": event.name, "arguments": json.dumps(event.input)},
+                            }
+                        )
+                    elif et == "message_stop":
+                        turn_usage = dict(event.usage or {})
+                        self._driver._auto_record_usage(
+                            {"meta": turn_usage}, (time.perf_counter() - stream_started) * 1000
+                        )
+                        self._driver._fire_callback(
+                            "on_response",
+                            {
+                                "meta": turn_usage,
+                                "driver": self._model_name,
+                                "elapsed_ms": (time.perf_counter() - stream_started) * 1000,
+                            },
+                        )
+                        self._accumulate_usage(turn_usage)
+                        recorded = True
+                        stop_reason = event.stop_reason
+                    yield event
+            except Exception as exc:
+                stream_error = exc
+                self._driver._fire_callback("on_error", {"error": exc, "driver": self._model_name})
+                raise
+            finally:
+                if not recorded:
+                    incomplete = {"usage_complete": False, "cost_status": "unknown"}
+                    self._driver._auto_record_usage(
+                        {"meta": incomplete},
+                        (time.perf_counter() - stream_started) * 1000,
+                        status="error" if stream_error is not None else "incomplete",
+                        error=stream_error,
                     )
-                elif et == "message_stop":
-                    turn_usage = dict(event.usage or {})
-                    stop_reason = event.stop_reason
+                    if stream_error is None:
+                        self._driver._fire_callback(
+                            "on_response",
+                            {
+                                "meta": incomplete,
+                                "driver": self._model_name,
+                                "status": "incomplete",
+                                "elapsed_ms": (time.perf_counter() - stream_started) * 1000,
+                            },
+                        )
+                    self._accumulate_usage(incomplete)
 
             full_text = "".join(assistant_text_parts)
             full_thinking = "".join(assistant_thinking_parts) or None
@@ -1024,8 +1068,6 @@ class Conversation:
 
             self._messages.append(assistant_msg)
             msgs.append(assistant_msg)
-            if turn_usage:
-                self._accumulate_usage(turn_usage)
 
             if not pending_tools:
                 yield TurnComplete(usage=dict(self._usage))
@@ -1274,6 +1316,8 @@ class Conversation:
 
         t0 = time.perf_counter()
         full_text = ""
+        recorded = False
+        stream_error = None
         try:
             for chunk in self._driver.generate_messages_stream(messages, merged):
                 chunk_type = chunk["type"]
@@ -1298,10 +1342,13 @@ class Conversation:
                             "elapsed_ms": elapsed_ms,
                         },
                     )
+                    self._driver._auto_record_usage({"meta": meta}, elapsed_ms)
+                    recorded = True
                     self._accumulate_usage(meta)
                     if chunk.get("reasoning_content"):
                         self._last_reasoning = chunk["reasoning_content"]
         except Exception as exc:
+            stream_error = exc
             elapsed_ms = (time.perf_counter() - t0) * 1000
             self._driver._fire_callback(
                 "on_error",
@@ -1309,6 +1356,32 @@ class Conversation:
             )
             raise
 
+        finally:
+            if not recorded:
+                elapsed_ms = (time.perf_counter() - t0) * 1000
+                incomplete = {
+                    "usage_complete": False,
+                    "cost_status": "unknown",
+                    "requested_model": merged.get("model", driver_name),
+                }
+                self._driver._auto_record_usage(
+                    {"meta": incomplete},
+                    elapsed_ms,
+                    status="error" if stream_error is not None else "incomplete",
+                    error=stream_error,
+                )
+                if stream_error is None:
+                    self._driver._fire_callback(
+                        "on_response",
+                        {
+                            "meta": incomplete,
+                            "driver": driver_name,
+                            "status": "incomplete",
+                            "elapsed_ms": elapsed_ms,
+                        },
+                    )
+                self._accumulate_usage(incomplete)
+
         self._messages.append(self._assistant_message(full_text))
 
     def ask_stream(

{prompture-1.11.1.dev1 → prompture-1.11.2.dev2}/prompture/drivers/_openai_compat_stream.py

@@ -85,7 +85,11 @@ def _process_chunk(
     """
     from ..agents.live_events import TextDelta, ThinkingDelta, ToolInputDelta, ToolUseStart
 
+    for key in ("id", "model", "service_tier"):
+        if isinstance(getattr(chunk, key, None), str):
+            state.setdefault("response_info", {})[key] = getattr(chunk, key)
     if getattr(chunk, "usage", None):
+        state["reported_usage"] = chunk.usage
         state["prompt_tokens"] = chunk.usage.prompt_tokens or 0
         state["completion_tokens"] = chunk.usage.completion_tokens or 0
         state["cached_prompt_tokens"] = _extract_cached_tokens(chunk.usage)
@@ -235,6 +239,7 @@ def iter_openai_compat_live_events(
     *,
     model: str,
     cost_fn: CostFn,
+    meta_fn: Callable[[dict[str, Any]], dict[str, Any]] | None = None,
 ) -> Iterator[Any]:
     """Consume a sync OpenAI-compat ``chat.completions`` stream with tools
     and yield :class:`~prompture.agents.live_events.LiveEvent`.
@@ -259,7 +264,10 @@ def iter_openai_compat_live_events(
         state["completion_tokens"],
         state["cached_prompt_tokens"],
     )
-    yield _build_message_stop(state, model, cost)
+    stop = _build_message_stop(state, model, cost)
+    if meta_fn is not None:
+        stop.usage.update(meta_fn(state))
+    yield stop
 
 
 async def aiter_openai_compat_live_events(
@@ -267,6 +275,7 @@ async def aiter_openai_compat_live_events(
     *,
     model: str,
     cost_fn: AsyncCostFn,
+    meta_fn: Callable[[dict[str, Any]], dict[str, Any]] | None = None,
 ) -> AsyncIterator[Any]:
     """Async sibling of :func:`iter_openai_compat_live_events`.
 
@@ -286,7 +295,10 @@ async def aiter_openai_compat_live_events(
         state["cached_prompt_tokens"],
     )
     cost = await raw_cost if inspect.isawaitable(raw_cost) else raw_cost
-    yield _build_message_stop(state, model, cost)
+    stop = _build_message_stop(state, model, cost)
+    if meta_fn is not None:
+        stop.usage.update(meta_fn(state))
+    yield stop
 
 
 # ----------------------------------------------------------------------
@@ -348,12 +360,28 @@ def stream_openai_compat_tool_call(
     )
     _apply_openai_tool_options(kwargs, options)
 
+    if provider == "openai":
+        from .openai_driver import _apply_openai_reporting_options
+
+        _apply_openai_reporting_options(kwargs, options)
     stream = driver.client.chat.completions.create(**kwargs)
 
     def cost_fn(prompt_tokens: int, completion_tokens: int, cached_tokens: int) -> float:
         return driver._calculate_cost(provider, model, prompt_tokens, completion_tokens, cached_tokens=cached_tokens)
 
-    yield from iter_openai_compat_live_events(stream, model=model, cost_fn=cost_fn)
+    def meta_fn(state: dict[str, Any]) -> dict[str, Any]:
+        from ._usage_reporting import usage_meta
+
+        return usage_meta(
+            driver, provider, model, state.get("reported_usage"), response=state.get("response_info"), options=options
+        )
+
+    yield from iter_openai_compat_live_events(
+        stream,
+        model=model,
+        cost_fn=cost_fn,
+        meta_fn=meta_fn if provider == "openai" else None,
+    )
 
 
 async def astream_openai_compat_tool_call(
@@ -399,12 +427,28 @@ async def astream_openai_compat_tool_call(
     )
     _apply_openai_tool_options(kwargs, options)
 
+    if provider == "openai":
+        from .openai_driver import _apply_openai_reporting_options
+
+        _apply_openai_reporting_options(kwargs, options)
     stream = await driver.client.chat.completions.create(**kwargs)
 
     def cost_fn(prompt_tokens: int, completion_tokens: int, cached_tokens: int) -> float:
         return driver._calculate_cost(provider, model, prompt_tokens, completion_tokens, cached_tokens=cached_tokens)
 
-    async for ev in aiter_openai_compat_live_events(stream, model=model, cost_fn=cost_fn):
+    def meta_fn(state: dict[str, Any]) -> dict[str, Any]:
+        from ._usage_reporting import usage_meta
+
+        return usage_meta(
+            driver, provider, model, state.get("reported_usage"), response=state.get("response_info"), options=options
+        )
+
+    async for ev in aiter_openai_compat_live_events(
+        stream,
+        model=model,
+        cost_fn=cost_fn,
+        meta_fn=meta_fn if provider == "openai" else None,
+    ):
         yield ev