deeptutor-visualize 1.7.0__py3-none-any.whl

deeptutor/agents/visualize/capability.py

@@ -0,0 +1,571 @@
+"""Plugin-driven visualization capability built on the shared chat loop."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from deeptutor.agents._shared.capability_result import emit_capability_result
+from deeptutor.core.capability_protocol import TurnCapability
+from deeptutor.core.context import UnifiedContext
+from deeptutor.core.trace import merge_trace_metadata
+from deeptutor.i18n import StatusI18n
+from deeptutor.runtime.agentic.usage import UsageTracker
+from deeptutor.runtime.bootstrap.builtin_capabilities import builtin_capability_manifest
+from deeptutor.runtime.request_contracts import (
+    VisualizeRequestConfig,
+    validate_visualize_request_config,
+)
+from deeptutor.runtime.stream_bus import StreamBus
+from deeptutor.services.config.capabilities_settings import get_visualize_params
+from deeptutor.visualizers.protocol import (
+    REQUESTED_VISUALIZER_KEY,
+    VISUALIZATION_RESULT_KEY,
+    VISUALIZE_MODE_KEY,
+)
+from deeptutor.visualizers.registry import get_visualizer_registry
+from deeptutor.visualizers.store import VisualizerStoreError
+
+logger = logging.getLogger(__name__)
+
+# Default (English) copy for the tool-calling diagnosis, kept beside the code
+# that emits it. StatusI18n prefers the localized strings in this capability's
+# prompt files and falls back to these.
+_TOOL_CALLING_DISABLED_WARNING = (
+    "Tool calling is disabled for {target}, but a visualization only reaches "
+    "the canvas through the submit_visualization tool call. Enable it in "
+    "Settings → Models → LLM → Capabilities → Tool calling → Supported."
+)
+_NO_PAYLOAD_TOOL_CALLING_DISABLED = (
+    "No visualization was committed: tool calling is disabled for {target}, so "
+    "the model could not call submit_visualization — the only way to hand a "
+    "payload to the canvas. Local servers (LM Studio, Ollama, vLLM, llama.cpp) "
+    "are opted out by default; declare the model as tool-capable in "
+    "Settings → Models → LLM → Capabilities → Tool calling → Supported, "
+    "then run the visualization again."
+)
+_NO_PAYLOAD = (
+    "The visualization agent finished without a valid canvas payload. A "
+    "reasoning model can spend most of its output budget on internal "
+    "thinking before the submit_visualization call is even written, which "
+    "cuts the call off mid-argument. If this repeats, raise Max tokens for "
+    "Visualize in Settings → Capabilities → Visualize."
+)
+
+# Stages exposed in the manifest. The first three cover the text-emitting
+# path (svg/chartjs/mermaid/html); the rest cover the manim subprocess
+# path. A given turn only streams a subset of these.
+_VISUALIZE_STAGES = [
+    "analyzing",
+    "generating",
+    "reviewing",
+    "concept_analysis",
+    "concept_design",
+    "code_generation",
+    "code_retry",
+    "summary",
+    "render_output",
+]
+
+_MANIM_RENDER_TYPES = {"manim_video", "manim_image"}
+
+# Visualizer packages contribute model-facing payload documentation. Keep the
+# reused chat loop's read/grounding surface, but never let package instructions
+# unlock state-changing built-ins such as exec, write_memory, write_note, cron,
+# github or deferred tool loading.
+_VISUALIZE_SAFE_BUILTINS = (
+    "rag",
+    "kb_files",
+    "read_source",
+    "read_memory",
+    "list_notebook",
+    "read_skill",
+    "web_fetch",
+    "ask_user",
+)
+
+
+class VisualizeCapability(TurnCapability):
+    manifest = builtin_capability_manifest("visualize")
+
+    async def run(self, context: UnifiedContext, stream: StreamBus) -> None:
+        request_config = validate_visualize_request_config(context.config_overrides)
+        render_mode = request_config.render_mode
+        i18n = StatusI18n(self.name, context.language, module="visualize")
+        registry = get_visualizer_registry()
+
+        history_context = str(context.metadata.get("conversation_context_text", "") or "").strip()
+        async with stream.stage("analyzing", source=self.name):
+            await stream.progress(
+                i18n.t("analyzing", "Analyzing visualization requirements..."),
+                source=self.name,
+                stage="analyzing",
+            )
+            if render_mode != "auto" and registry.get(render_mode) is None:
+                available = ", ".join(plugin.manifest.id for plugin in registry.installed())
+                raise VisualizerStoreError(
+                    f"visualizer '{render_mode}' is not installed or is disabled. "
+                    f"Available: {available or 'none'}"
+                )
+
+        # Artifact renderers retain their specialized subprocess pipeline.
+        if render_mode in _MANIM_RENDER_TYPES:
+            from deeptutor.services.llm.config import get_llm_config
+
+            llm_config = get_llm_config()
+            await self._run_manim_path(
+                context=context,
+                stream=stream,
+                render_type=render_mode,
+                visualize_config=request_config,
+                history_context=history_context,
+                usage=UsageTracker(model=getattr(llm_config, "model", None)),
+                i18n=i18n,
+            )
+            return
+
+        # The generic capability only marks the turn and owns the final
+        # envelope. Generation, retrieval, repair and tool dispatch all run in
+        # the shared Chat Engine through VisualizationLoopCapability.
+        from deeptutor.agents.chat.agentic_pipeline import AgenticChatPipeline
+
+        # A visualization only reaches the canvas through the
+        # submit_visualization tool call, so a provider that is never handed
+        # tool schemas has no way to commit one: it answers in prose, and the
+        # loop capability's output policy discards that prose. Local
+        # OpenAI-compatible servers (LM Studio, Ollama, vLLM, llama.cpp) are
+        # opted out of native tools by default, which used to surface only as
+        # a bare "no valid canvas payload" error after a full five-round loop.
+        # Say so up front, and keep the finding for the failure message.
+        #
+        # This deliberately does not abort the turn: the DSML fallback still
+        # recovers text-format tool calls from some local DeepSeek
+        # deployments, so a run without native tools can still succeed.
+        tool_calling_disabled_for = self._tool_calling_disabled_for()
+        if tool_calling_disabled_for:
+            await stream.progress(
+                i18n.t(
+                    "tool_calling_disabled",
+                    _TOOL_CALLING_DISABLED_WARNING,
+                    target=tool_calling_disabled_for,
+                ),
+                source=self.name,
+                stage="generating",
+                metadata={"trace_kind": "warning"},
+            )
+
+        context.metadata[VISUALIZE_MODE_KEY] = True
+        context.metadata[REQUESTED_VISUALIZER_KEY] = render_mode
+        context.metadata.pop(VISUALIZATION_RESULT_KEY, None)
+        if context.allowed_builtin_tools is None:
+            context.allowed_builtin_tools = list(_VISUALIZE_SAFE_BUILTINS)
+        else:
+            allowed = set(context.allowed_builtin_tools)
+            context.allowed_builtin_tools = [
+                name for name in _VISUALIZE_SAFE_BUILTINS if name in allowed
+            ]
+        # Read the visualize budget instead of stating it here: a reasoning
+        # model can spend most of one round on chain-of-thought, which truncated
+        # the submit_visualization arguments mid-JSON and left a blank canvas
+        # with no error (#1546). The defaults are the values this call used to
+        # hardcode, so an untouched install behaves exactly as before.
+        try:
+            params = get_visualize_params()
+        except Exception as exc:  # pragma: no cover - defensive config read
+            logger.warning("Failed to load visualize params, using defaults: %s", exc)
+            params = {}
+        pipeline = AgenticChatPipeline(
+            language=context.language,
+            max_rounds=5,
+            temperature=params.get("temperature", 0.15),
+            max_tokens=params.get("max_tokens", 16000),
+            event_source=self.name,
+            event_stage="generating",
+            emit_result=False,
+        )
+        loop_result = await pipeline.run(context, stream)
+        envelope = context.metadata.get(VISUALIZATION_RESULT_KEY)
+        if not isinstance(envelope, dict):
+            if tool_calling_disabled_for:
+                raise RuntimeError(
+                    i18n.t(
+                        "no_payload_tool_calling_disabled",
+                        _NO_PAYLOAD_TOOL_CALLING_DISABLED,
+                        target=tool_calling_disabled_for,
+                    )
+                )
+            raise RuntimeError(i18n.t("no_payload", _NO_PAYLOAD))
+
+        payload = envelope.get("payload") if isinstance(envelope.get("payload"), dict) else {}
+        data = payload.get("data")
+        plugin = registry.get(str(envelope.get("render_type") or ""))
+        language_tag = plugin.manifest.language_tag if plugin is not None else "text"
+        serialized = (
+            data if isinstance(data, str) else json.dumps(data, ensure_ascii=False, indent=2)
+        )
+        content_md = f"```{language_tag}\n{serialized}\n```"
+        await stream.content(content_md, source=self.name, stage="reviewing")
+
+        result = {
+            **envelope,
+            # Legacy compatibility for existing consumers while they migrate
+            # to renderer/payload/presentation.
+            "response": content_md,
+            "code": {"language": language_tag, "content": serialized},
+            "analysis": {
+                "render_type": envelope.get("render_type"),
+                "description": (envelope.get("presentation") or {}).get("description", ""),
+                "engine": "chat_agent_loop",
+                "requested_type": render_mode,
+            },
+            "review": {
+                "changed": False,
+                "review_notes": "Accepted by the installed visualizer validator.",
+            },
+            "engine": "chat_agent_loop",
+            "loop": loop_result,
+        }
+        await emit_capability_result(
+            stream,
+            result,
+            source=self.name,
+            usage=pipeline.usage,
+        )
+
+    @staticmethod
+    def _tool_calling_disabled_for() -> str:
+        """``"binding/model"`` when this turn gets no tool schemas, else ``""``.
+
+        Mirrors the resolution ``AgenticChatPipeline`` performs for the same
+        LLM config, so the warning and the failure message describe the loop
+        that actually ran. Probing must never be why a visualization fails, so
+        an unreadable config reports "enabled" and leaves the generic
+        diagnosis in place.
+        """
+        try:
+            from deeptutor.runtime.agentic.client import can_use_native_tool_calling
+            from deeptutor.services.llm.config import get_llm_config
+
+            llm_config = get_llm_config()
+            binding = str(getattr(llm_config, "binding", None) or "openai")
+            model = str(getattr(llm_config, "model", None) or "")
+            if can_use_native_tool_calling(binding=binding, model=model or None):
+                return ""
+            return f"{binding}/{model}" if model else binding
+        except Exception:  # pragma: no cover - defensive: never block a turn
+            logger.debug("Tool-calling probe unavailable for visualize", exc_info=True)
+            return ""
+
+    async def _run_manim_path(
+        self,
+        *,
+        context: UnifiedContext,
+        stream: StreamBus,
+        render_type: str,
+        visualize_config: VisualizeRequestConfig,
+        history_context: str,
+        usage: UsageTracker | None = None,
+        i18n: StatusI18n | None = None,
+    ) -> None:
+        """
+        Manim sub-pipeline. Mirrors ``MathAnimatorCapability.run`` but emits
+        the final result with ``render_type`` as the discriminator so the
+        unified frontend dispatcher can route to ``MathAnimatorViewer``.
+        """
+        import importlib.util
+        import time
+
+        from deeptutor.modules.catalog import require_module
+
+        require_module("renderer-manim")
+        if importlib.util.find_spec("manim") is None:
+            raise RuntimeError(
+                "Manim rendering requires optional dependencies. "
+                "Install with `pip install 'deeptutor[math-animator]'` "
+                "or `pip install -r requirements/math-animator.txt`."
+            )
+
+        from deeptutor.agents.math_animator.pipeline import MathAnimatorPipeline
+        from deeptutor.agents.math_animator.request_config import MathAnimatorRequestConfig
+        from deeptutor.core.trace import build_trace_metadata, new_call_id
+        from deeptutor.services.llm.config import get_llm_config
+
+        if i18n is None:
+            i18n = StatusI18n(self.name, context.language, module="visualize")
+        output_mode = "image" if render_type == "manim_image" else "video"
+        request_config = MathAnimatorRequestConfig(
+            output_mode=output_mode,  # type: ignore[arg-type]
+            quality=visualize_config.quality,
+            style_hint=visualize_config.style_hint,
+        )
+
+        llm_config = get_llm_config()
+        workspace = context.runtime.workspace
+        pipeline = MathAnimatorPipeline(
+            api_key=llm_config.api_key,
+            base_url=llm_config.base_url,
+            api_version=llm_config.api_version,
+            language=context.language,
+            trace_callback=self._build_trace_bridge(stream, i18n=i18n),
+            workspace_output_dir=workspace.output_dir if workspace else None,
+            workspace_root=workspace.root if workspace else None,
+            workspace_id=workspace.workspace_id if workspace else "",
+        )
+
+        timings: dict[str, float] = {}
+        turn_id = str(
+            context.metadata.get("turn_id", "") or context.session_id or "visualize-manim"
+        )
+        render_call_meta = build_trace_metadata(
+            call_id=new_call_id("manim-render"),
+            phase="render_output",
+            label="Render output",
+            call_kind="math_render_output",
+            trace_role="render",
+            trace_kind="progress",
+            output_mode=request_config.output_mode,
+            quality=request_config.quality,
+        )
+
+        stage_start = time.perf_counter()
+        async with stream.stage("concept_analysis", source=self.name):
+            analysis = await pipeline.run_analysis(
+                user_input=context.user_message,
+                history_context=history_context,
+                request_config=request_config,
+                attachments=context.attachments,
+            )
+        timings["concept_analysis"] = round(time.perf_counter() - stage_start, 3)
+
+        stage_start = time.perf_counter()
+        async with stream.stage("concept_design", source=self.name):
+            design = await pipeline.run_design(
+                user_input=context.user_message,
+                request_config=request_config,
+                analysis=analysis,
+            )
+        timings["concept_design"] = round(time.perf_counter() - stage_start, 3)
+
+        stage_start = time.perf_counter()
+        async with stream.stage("code_generation", source=self.name):
+            generated = await pipeline.run_code_generation(
+                user_input=context.user_message,
+                request_config=request_config,
+                analysis=analysis,
+                design=design,
+            )
+            await stream.progress(
+                message=i18n.t("manim_code_prepared", "Manim code prepared."),
+                source=self.name,
+                stage="code_generation",
+            )
+        timings["code_generation"] = round(time.perf_counter() - stage_start, 3)
+
+        async def _on_retry(retry_attempt) -> None:
+            await stream.progress(
+                message=i18n.t(
+                    "manim_retry",
+                    f"Retry {retry_attempt.attempt}: {retry_attempt.error}",
+                    attempt=retry_attempt.attempt,
+                    error=retry_attempt.error,
+                ),
+                source=self.name,
+                stage="code_retry",
+                metadata={**render_call_meta, "trace_layer": "raw"},
+            )
+
+        async def _on_render_progress(message: str, raw: bool) -> None:
+            await stream.progress(
+                message=message,
+                source=self.name,
+                stage="render_output",
+                metadata={
+                    **render_call_meta,
+                    "trace_layer": "raw" if raw else "summary",
+                },
+            )
+
+        async def _on_retry_status(message: str) -> None:
+            await stream.progress(
+                message=message,
+                source=self.name,
+                stage="code_retry",
+                metadata={"trace_layer": "summary"},
+            )
+
+        stage_start = time.perf_counter()
+        async with stream.stage("code_retry", source=self.name):
+            await stream.progress(
+                message=i18n.t(
+                    "manim_rendering",
+                    (
+                        f"Rendering {request_config.output_mode} "
+                        f"with quality={request_config.quality}."
+                    ),
+                    mode=request_config.output_mode,
+                    quality=request_config.quality,
+                ),
+                source=self.name,
+                stage="code_retry",
+                metadata={**render_call_meta, "call_state": "running"},
+            )
+            final_code, render_result = await pipeline.run_render(
+                turn_id=turn_id,
+                user_input=context.user_message,
+                request_config=request_config,
+                initial_code=generated.code,
+                on_retry=_on_retry,
+                on_render_progress=_on_render_progress,
+                on_retry_status=_on_retry_status,
+            )
+        timings["code_retry"] = round(time.perf_counter() - stage_start, 3)
+
+        stage_start = time.perf_counter()
+        async with stream.stage("summary", source=self.name):
+            summary = await pipeline.run_summary(
+                user_input=context.user_message,
+                request_config=request_config,
+                analysis=analysis,
+                design=design,
+                render_result=render_result,
+            )
+            if summary.summary_text:
+                await stream.content(summary.summary_text, source=self.name, stage="summary")
+        timings["summary"] = round(time.perf_counter() - stage_start, 3)
+
+        async with stream.stage("render_output", source=self.name):
+            artifact_count = len(render_result.artifacts)
+            artifact_key = "manim_artifacts_one" if artifact_count == 1 else "manim_artifacts_many"
+            await stream.progress(
+                message=i18n.t(
+                    artifact_key,
+                    (
+                        f"Prepared {artifact_count} "
+                        f"{'artifact' if artifact_count == 1 else 'artifacts'}."
+                    ),
+                    count=artifact_count,
+                ),
+                source=self.name,
+                stage="render_output",
+                metadata={**render_call_meta, "call_state": "complete"},
+            )
+        timings["render_output"] = 0.0
+        visual_review = getattr(render_result, "visual_review", None)
+        workspace_items = list(getattr(render_result, "workspace_items", []) or [])
+        if workspace_items:
+            await stream.sources(
+                [{"type": "workspace_item", **item} for item in workspace_items],
+                source=self.name,
+                stage="render_output",
+            )
+
+        await emit_capability_result(
+            stream,
+            {
+                "response": summary.summary_text,
+                "render_type": render_type,
+                "summary": summary.model_dump(),
+                "code": {
+                    "language": "python",
+                    "content": final_code,
+                },
+                "output_mode": request_config.output_mode,
+                "artifacts": [artifact.model_dump() for artifact in render_result.artifacts],
+                "workspace_items": workspace_items,
+                "timings": timings,
+                "render": {
+                    "quality": request_config.quality,
+                    "retry_attempts": render_result.retry_attempts,
+                    "retry_history": [item.model_dump() for item in render_result.retry_history],
+                    "source_code_path": render_result.source_code_path,
+                    "visual_review": visual_review.model_dump() if visual_review else None,
+                },
+                "analysis": analysis.model_dump(),
+                "design": design.model_dump(),
+            },
+            source=self.name,
+            usage=usage,
+        )
+
+    def _build_trace_bridge(self, stream: StreamBus, i18n: StatusI18n | None = None):
+        async def _trace_bridge(update: dict[str, Any]) -> None:
+            event = str(update.get("event", "") or "")
+            stage = str(update.get("phase") or update.get("stage") or "analyzing")
+            base_metadata = {
+                key: value
+                for key, value in update.items()
+                if key
+                not in {"event", "state", "response", "chunk", "result", "tool_name", "tool_args"}
+            }
+
+            if event != "llm_call":
+                return
+
+            state = str(update.get("state", "running"))
+            label = str(base_metadata.get("label", "") or stage.replace("_", " ").title())
+            if state == "running":
+                await stream.progress(
+                    message=label,
+                    source=self.name,
+                    stage=stage,
+                    metadata=merge_trace_metadata(
+                        base_metadata,
+                        {"trace_kind": "call_status", "call_state": "running"},
+                    ),
+                )
+                return
+            if state == "streaming":
+                chunk = str(update.get("chunk", "") or "")
+                if chunk:
+                    await stream.thinking(
+                        chunk,
+                        source=self.name,
+                        stage=stage,
+                        metadata=merge_trace_metadata(
+                            base_metadata,
+                            {"trace_kind": "llm_chunk"},
+                        ),
+                    )
+                return
+            if state == "complete":
+                was_streaming = update.get("streaming", False)
+                if not was_streaming:
+                    response = str(update.get("response", "") or "")
+                    if response:
+                        await stream.thinking(
+                            response,
+                            source=self.name,
+                            stage=stage,
+                            metadata=merge_trace_metadata(
+                                base_metadata,
+                                {"trace_kind": "llm_output"},
+                            ),
+                        )
+                await stream.progress(
+                    message=label,
+                    source=self.name,
+                    stage=stage,
+                    metadata=merge_trace_metadata(
+                        base_metadata,
+                        {"trace_kind": "call_status", "call_state": "complete"},
+                    ),
+                )
+                return
+            if state == "error":
+                fallback = (
+                    i18n.t("llm_call_failed", "LLM call failed.")
+                    if i18n is not None
+                    else "LLM call failed."
+                )
+                await stream.error(
+                    str(update.get("response", "") or fallback),
+                    source=self.name,
+                    stage=stage,
+                    metadata=merge_trace_metadata(
+                        base_metadata,
+                        {"trace_kind": "call_status", "call_state": "error"},
+                    ),
+                )
+
+        return _trace_bridge

deeptutor/agents/visualize/models.py

@@ -0,0 +1,120 @@
+"""Data models for the visualize pipeline."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Literal, get_args
+
+from pydantic import BaseModel, Field, model_validator
+
+logger = logging.getLogger(__name__)
+
+RenderType = Literal[
+    "svg",
+    "chartjs",
+    "mermaid",
+    "mindmap",
+    "html",
+    "manim_video",
+    "manim_image",
+]
+
+VisualGenre = Literal[
+    "",
+    "flowchart",
+    "structural",
+    "illustrative",
+    "chart",
+    "stepper",
+    "interactive",
+    "mockup",
+    "art",
+]
+
+
+class VisualizationAnalysis(BaseModel):
+    """Output of the analysis stage."""
+
+    render_type: RenderType = Field(
+        description=(
+            "Render output: raw SVG, a Chart.js configuration, a Mermaid "
+            "diagram, a self-contained interactive HTML page, or a Manim "
+            "animation (video) / storyboard image."
+        ),
+    )
+    description: str = Field(
+        default="",
+        description="High-level description of what the visualization should show.",
+    )
+    data_description: str = Field(
+        default="",
+        description="Description of the data or elements to be visualized.",
+    )
+    chart_type: str = Field(
+        default="",
+        description=(
+            "Chart.js chart type (bar, line, pie, doughnut, radar, etc.) when render_type is chartjs, "
+            "Mermaid diagram type (flowchart, sequenceDiagram, mindmap, classDiagram, stateDiagram, etc.) "
+            "when render_type is mermaid, or a short interaction tag (e.g. 'interactive', 'animation', "
+            "'walkthrough') when render_type is html."
+        ),
+    )
+    visual_elements: list[str] = Field(
+        default_factory=list,
+        description="Key visual elements to include (shapes, labels, axes, colors, etc.).",
+    )
+    rationale: str = Field(
+        default="",
+        description="Why this render_type was chosen over the alternative.",
+    )
+    visual_genre: VisualGenre = Field(
+        default="",
+        description=(
+            "Teaching-oriented sub-type that drives the code-generation style, "
+            "routed on the user's intent (the verb), not the topic (the noun): "
+            "'flowchart'/'structural' for reference maps, 'illustrative' for "
+            "intuition/'how does X work' spatial metaphors, 'stepper' for "
+            "cyclic or staged walkthroughs, 'chart' for quantitative data, "
+            "'interactive'/'mockup'/'art' for the matching HTML/SVG experiences. "
+            "Empty when no sub-type applies."
+        ),
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def _drop_off_enum_values(cls, data: Any) -> Any:
+        """Degrade an invented enum value instead of failing the whole render.
+
+        Models regularly answer with a genre or render type that is not in the
+        prompt's list ("simulation", "diagram"). Both fields are enums, so
+        validation used to abort generation over a label that no downstream
+        stage strictly needs — the genre only selects a code-generation style,
+        and svg is the universal fallback render.
+        """
+        if not isinstance(data, dict):
+            return data
+        genre = data.get("visual_genre")
+        if genre is not None and genre not in get_args(VisualGenre):
+            logger.warning("Discarding unknown visual_genre %r", genre)
+            data = {**data, "visual_genre": ""}
+        render_type = data.get("render_type")
+        if render_type is not None and render_type not in get_args(RenderType):
+            logger.warning("Falling back to svg for unknown render_type %r", render_type)
+            data = {**data, "render_type": "svg"}
+        return data
+
+
+class ReviewResult(BaseModel):
+    """Output of the review / optimization stage."""
+
+    optimized_code: str = Field(
+        description="The final (potentially optimized) visualization code.",
+    )
+    changed: bool = Field(
+        default=False,
+        description="Whether the reviewer made modifications.",
+    )
+    review_notes: str = Field(
+        default="",
+        description="Notes on what was checked or changed.",
+    )