simit 0.1.0__py3-none-any.whl

simit/pipeline/core.py

@@ -0,0 +1,586 @@
+"""The SIMIT synthesis pipeline (Fig. 2 of the paper), written as coroutines.
+
+Every query runs as its own coroutine on a background event loop; their
+model calls are batched by the backend's engine, and rendering runs in the
+worker pool, so many queries (and many candidates per query) progress
+concurrently.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+import re
+import threading
+import time
+from collections import deque
+from typing import Callable, Optional, Sequence
+
+from PIL import Image
+
+from ..backends.base import Backend, GenRequest, ImageRequest, ScoreRequest
+from ..config import SIMITConfig
+from ..skills.base import Realization, Skill
+from ..skills.pool import RenderPool
+from ..skills.routing import build_router_prompt, parse_route, pre_route
+from ..types import Demo, Imagination, ZeroShot
+from ..utils import clean_response, to_pil
+from . import prompts as P
+
+log = logging.getLogger("simit")
+
+
+class Runtime:
+    """A background asyncio loop shared by all pipeline coroutines."""
+
+    def __init__(self):
+        self.loop = asyncio.new_event_loop()
+        self.thread = threading.Thread(target=self.loop.run_forever, daemon=True, name="simit-loop")
+        self.thread.start()
+
+    def run(self, coro):
+        if threading.current_thread() is self.thread:
+            raise RuntimeError("SIMIT's blocking API cannot be called from its own event loop")
+        return asyncio.run_coroutine_threadsafe(coro, self.loop).result()
+
+    def close(self):
+        self.loop.call_soon_threadsafe(self.loop.stop)
+
+
+class SkillContext:
+    """What a ``Skill.realize`` implementation may call."""
+
+    def __init__(self, pipe: "Pipeline", size: int, allow_composite: bool = True):
+        self.pipe, self.size, self.allow_composite = pipe, size, allow_composite
+
+    async def generate(self, prompt: str, max_new_tokens: int = 512, temperature: float = 0.0,
+                       images: Optional[Sequence[Image.Image]] = None) -> str:
+        parts = list(images or []) + [prompt]
+        res = await self.pipe.generate(GenRequest(parts=parts, style="chat", max_new_tokens=max_new_tokens,
+                                                  temperature=temperature), tag="skill_spec")
+        return res.text
+
+    async def render(self, skill: Skill, spec) -> Image.Image:
+        self.pipe._started(f"render:{skill.name}")
+        t0 = time.perf_counter()
+        try:
+            return await asyncio.wrap_future(self.pipe.pool.submit(skill, spec))
+        finally:
+            if self.pipe.trace is not None:
+                self.pipe.trace.append((f"render:{skill.name}", time.perf_counter() - t0, 0))
+
+    async def generate_image(self, prompt: str) -> Optional[Image.Image]:
+        return await self.pipe.generate_image(prompt, self.size)
+
+    async def realize(self, request: str, retries: int = 1, allow_composite: bool = False,
+                      size: Optional[int] = None) -> Optional[Realization]:
+        """Route and realize a sub-request without verification (composite panels)."""
+        skill = await self.pipe.route(request, allow_composite=allow_composite)
+        ctx = SkillContext(self.pipe, size or self.size, allow_composite)
+        try:
+            return await skill.realize(ctx, request, retries=retries)
+        except Exception as e:  # a broken panel must not sink the composite
+            log.debug("sub-realization failed: %s", e)
+            return None
+
+    def log(self, msg: str):
+        log.debug(msg)
+
+
+_SCORE_RE = re.compile(r"SCORE:\s*(\d{1,3})", re.IGNORECASE)
+
+
+def _parse_score(text: str) -> Optional[int]:
+    m = _SCORE_RE.findall(text)
+    return max(0, min(100, int(m[-1]))) if m else None
+
+
+def _norm_q(question: str) -> str:
+    return " ".join(re.sub(r"[^\w\s]", " ", question.lower()).split())
+
+
+def _mean_prob(logprobs) -> Optional[float]:
+    return math.exp(sum(logprobs) / len(logprobs)) if logprobs else None
+
+
+class Pipeline:
+    def __init__(self, backend: Backend, skills: Sequence[Skill], config: SIMITConfig,
+                 image_generator: Optional[Backend] = None, pool: Optional[RenderPool] = None,
+                 runtime: Optional[Runtime] = None):
+        self.backend = backend
+        self.image_generator = image_generator
+        self.config = config
+        self.pool = pool or RenderPool()
+        self.runtime = runtime or Runtime()
+        self.trace: Optional[list] = None   # set to [] to record (stage, seconds, tokens) per model call
+        self.on_call: Optional[Callable[[str], None]] = None   # called with a stage tag as each model call starts
+        self.set_skills(skills)
+
+    # ------------------------------------------------------------ setup
+    @property
+    def can_generate_images(self) -> bool:
+        return self.backend.native_image_generation or self.image_generator is not None
+
+    def set_skills(self, skills: Sequence[Skill]):
+        skills = list(skills)
+        names = [s.name for s in skills]
+        if len(set(names)) != len(names):
+            raise ValueError(f"duplicate skill names: {names}")
+        if not self.can_generate_images:  # structured skills only
+            skills = [s for s in skills if s.name != "natural"]
+        self.skills = skills
+        self.by_name = {s.name: s for s in skills}
+        self.router_prompt = build_router_prompt(skills)
+
+    def option(self, name: str, default):
+        value = getattr(self.config, name)
+        if value is None:
+            value = getattr(self.backend, "pipeline_defaults", {}).get(name, default)
+        return value
+
+    @property
+    def use_skills(self) -> bool:
+        return bool(self.option("use_skills", True)) or "natural" not in self.by_name
+
+    @property
+    def verify_enabled(self) -> bool:
+        return bool(self.option("verify", True))
+
+    @property
+    def verify_think(self) -> bool:
+        v = self.config.verify_think
+        return self.backend.verify_think if v is None else v
+
+    # ------------------------------------------------------- model calls
+    def _started(self, tag: str):
+        if self.on_call is not None:
+            try:
+                self.on_call(tag)
+            except Exception as e:  # a progress callback must not break the pipeline
+                log.debug("on_call failed: %s", e)
+
+    async def generate(self, req: GenRequest, tag: str = "generate"):
+        self._started(tag)
+        t0 = time.perf_counter()
+        res = await asyncio.wrap_future(self.backend.submit_generate(req))
+        if self.trace is not None:
+            self.trace.append((tag, time.perf_counter() - t0, res.num_tokens))
+        return res
+
+    async def score(self, req: ScoreRequest):
+        return await asyncio.wrap_future(self.backend.submit_score(req))
+
+    async def generate_image(self, prompt: str, size: int) -> Optional[Image.Image]:
+        target = self.backend if self.backend.native_image_generation else self.image_generator
+        if target is None:
+            return None
+        self._started("image")
+        try:
+            t0 = time.perf_counter()
+            img = await asyncio.wrap_future(target.submit_image(
+                ImageRequest(prompt=prompt, width=size, height=size, seed=self._next_seed())))
+            if self.trace is not None:
+                self.trace.append(("image", time.perf_counter() - t0, 0))
+            return img
+        except Exception as e:
+            log.debug("image generation failed: %s", e)
+            return None
+
+    def _next_seed(self):
+        if self.config.seed is None:
+            return None
+        self._seed = getattr(self, "_seed", self.config.seed) + 1
+        return self._seed
+
+    # --------------------------------------------------- answering (ICL)
+    async def zero_shot(self, image, question: str, max_new_tokens: int) -> ZeroShot:
+        res = await self.generate(GenRequest(parts=[image, question], style="answer",
+                                             max_new_tokens=max_new_tokens, logprobs=True), tag="zero_shot")
+        return ZeroShot(res.text, _mean_prob(res.token_logprobs), res.token_logprobs or ())
+
+    async def answer(self, image, question: str, demos: Sequence[Demo], max_new_tokens: int) -> str:
+        res = await self.generate(GenRequest(parts=icl_parts(demos, image, question), style="answer",
+                                             max_new_tokens=max_new_tokens), tag="answer")
+        return res.text
+
+    async def confidence(self, image, question: str, answer: str) -> Optional[float]:
+        lp = await self.score(ScoreRequest(parts=[image, question], target=answer, style="answer"))
+        return _mean_prob(lp)
+
+    # ---------------------------------------------------- triplet synthesis
+    @staticmethod
+    def _parse_triplets(text: str, final: bool = True) -> list[tuple[str, str, str]]:
+        text = clean_response(text).replace("**", "")
+        found = []
+        for m in P.TRIPLET_RE.finditer(text):
+            if not final and m.end() >= len(text.rstrip()):
+                break  # may still be growing
+            q, a, d = m.groups()
+            lines = [ln.strip() for ln in q.strip().splitlines() if ln.strip()]
+            if len(lines) > 1 and lines[0].endswith("?"):
+                q = lines[0]  # stray text (often the answer) leaked onto the question's next line
+            sep = d.find("\n---")
+            found.append(tuple(_unbracket(x) for x in (q, a, d[:sep] if sep != -1 else d)))
+        return found
+
+    async def stream_triplets(self, image, k: int, push, max_attempts: int = 3,
+                              temperatures: tuple = (0.2, 0.7), tag: str = "synthesis") -> int:
+        """Like ``triplets`` but hands each triplet to ``push`` (which returns
+        True if it was kept) as soon as it is complete in the decoded stream,
+        so realization starts early; decoding stops once ``k`` were kept."""
+        loop = asyncio.get_running_loop()
+        base = P.TRIPLET_PROMPT if self.config.diversity_prompt else P.TRIPLET_PROMPT_NO_DIVERSITY
+        prompt = base.format(K=k)
+        kept = 0
+        for attempt in range(max_attempts):
+            seen = [0]
+            call: list = []
+
+            def take(found):
+                nonlocal kept
+                for t in found[seen[0]:]:
+                    kept += bool(push(t))
+                seen[0] = max(seen[0], len(found))
+                if kept >= k and call and not call[0].done():
+                    call[0].cancel()  # enough triplets: stop decoding the rest
+
+            req = GenRequest(parts=[image, prompt], style="chat", max_new_tokens=1800,
+                             temperature=temperatures[min(attempt, len(temperatures) - 1)],
+                             on_text=lambda text: loop.call_soon_threadsafe(
+                                 lambda: take(self._parse_triplets(text, final=False))))
+            call.append(asyncio.ensure_future(self.generate(req, tag=tag)))
+            try:
+                res = await call[0]
+            except asyncio.CancelledError:
+                if kept >= k:
+                    return kept
+                raise
+            take(self._parse_triplets(res.text))
+            if kept >= k:
+                return kept
+            prompt = base.format(K=k) + P.FORMAT_REMINDER.format(K=k)
+        return kept
+
+    async def decomposed_triplets(self, image, k: int, push, max_attempts: Optional[int] = None) -> int:
+        """Three short calls per triplet (question -> answer -> description),
+        for models that cannot follow the one-shot K-triplet prompt."""
+        budget = [max_attempts or k * 6]
+        kept = [0]
+        counter = [0]
+
+        def clean(text: str) -> str:
+            text = clean_response(text).strip()
+            if not text:
+                return ""
+            line = text.splitlines()[0].strip().strip('"')
+            line = re.sub(r"^[-*•]\s*", "", line)  # bullets copied from the few-shot list
+            line = re.sub(r"^question:\s*", "", line, flags=re.IGNORECASE)
+            # chat preambles: "Sure, here's the first one: How many ...?"
+            line = re.sub(r"^(sure|okay|ok|here(?:'s| is| are))\b[^:?]{0,60}:\s*", "", line, flags=re.IGNORECASE)
+            return re.sub(r"^\d+[.)]\s*", "", line)
+
+        async def ask(prompt, n):
+            res = await self.generate(GenRequest(parts=[image, prompt], style="chat", max_new_tokens=n,
+                                                 temperature=0.8), tag="synthesis")
+            return res.text
+
+        async def worker():
+            while kept[0] < k and budget[0] > 0:
+                budget[0] -= 1
+                topic = P.DECOMPOSED_TOPICS[counter[0] % len(P.DECOMPOSED_TOPICS)]
+                counter[0] += 1
+                q = clean(await ask(P.DECOMPOSED_QUESTION_PROMPT.format(topic=topic), 40))
+                if (len(q.split()) < 3 or not q.endswith("?") or P.COORD_LIST_RE.match(q)
+                        or P.META_LEAK_RE.search(q)):
+                    continue
+                hint = next((h for pat, h in P.ANSWER_TYPE_HINTS if pat.match(q)), "")
+                a = clean(await ask(P.DECOMPOSED_ANSWER_PROMPT.format(question=q, type_hint=hint), 16))
+                if not a:
+                    continue
+                d = (await ask(P.DECOMPOSED_DESCRIPTION_PROMPT.format(question=q, answer=a), 150)).strip()
+                if len(d) < 20 or P.COORD_LIST_RE.match(d):
+                    continue
+                if kept[0] < k:
+                    kept[0] += bool(push((q, a, d)))
+
+        await asyncio.gather(*[worker() for _ in range(k)])
+        return kept[0]
+
+    async def triplets(self, image, k: int, max_attempts: int = 3,
+                       temperatures: tuple = (0.2, 0.7)) -> list[tuple[str, str, str]]:
+        base = P.TRIPLET_PROMPT if self.config.diversity_prompt else P.TRIPLET_PROMPT_NO_DIVERSITY
+        prompt = base.format(K=k)
+        best: list = []
+        for attempt in range(max_attempts):
+            res = await self.generate(GenRequest(parts=[image, prompt], style="chat", max_new_tokens=1800,
+                                                 temperature=temperatures[min(attempt, len(temperatures) - 1)]),
+                                      tag="synthesis_fresh")
+            found = self._parse_triplets(res.text)
+            if len(found) > len(best):
+                best = found
+            if len(best) >= k:
+                break
+            prompt = base.format(K=k) + P.FORMAT_REMINDER.format(K=k)
+        return best
+
+    # ------------------------------------------------------------- routing
+    async def route(self, description: str, allow_composite: bool = True) -> Skill:
+        if not self.use_skills:
+            return self.by_name["natural"]
+        skills = [s for s in self.skills if allow_composite or s.name != "html_composite"]
+        name = pre_route(description, skills)
+        if name is None:
+            res = await self.generate(GenRequest(parts=[self.router_prompt.replace("{request}", description)],
+                                                 style="chat", max_new_tokens=20), tag="route")
+            default = "figure" if "figure" in self.by_name else skills[0].name
+            name = parse_route(clean_response(res.text), skills, default)
+        return self.by_name[name]
+
+    # -------------------------------------------------------- verification
+    async def critic(self, description: str, image: Image.Image, rubric: str):
+        template = P.CRITIC_STRICT_PROMPT if rubric == "strict" else P.CRITIC_STRUCTURED_PROMPT
+        res = await self.generate(GenRequest(parts=[to_pil(image), template.format(request=description)],
+                                             style="chat", max_new_tokens=self.config.verify_max_new_tokens,
+                                             think=self.verify_think), tag="critic")
+        score = _parse_score(res.text)
+        if score is None and res.raw_text:  # thinking cut short before the answer: use what it wrote
+            score = _parse_score(res.raw_text)
+        if score is None and res.num_tokens >= self.config.verify_max_new_tokens:
+            # A verbose critic ran out of tokens before its verdict: ask for the score line only.
+            critique = (res.text or res.raw_text).strip()
+            follow = (template.format(request=description) + "\n\nYour review so far:\n" + critique[-3000:]
+                      + "\n\nNow output only the final line: 'SCORE: <integer 0-100>'.")
+            res2 = await self.generate(GenRequest(parts=[to_pil(image), follow], style="chat", max_new_tokens=16),
+                                       tag="critic_score")
+            score = _parse_score(res2.text)
+        return score, res.text
+
+    async def revise(self, request: str, critique: str) -> str:
+        res = await self.generate(GenRequest(parts=[P.REVISE_PROMPT.format(request=request, critique=critique)],
+                                             style="chat", max_new_tokens=200, think=self.verify_think), tag="revise")
+        return res.text.strip() or request
+
+    async def realize_verified(self, triplet, single_shot: bool) -> Optional[Demo]:
+        """Route, realize and verify one triplet. The image is always judged
+        against the original description; failed rounds revise the
+        description used for the next generation."""
+        question, answer, description = triplet
+        cfg = self.config
+        skill = await self.route(description)
+        ctx = SkillContext(self, cfg.image_size)
+        retries = 0 if single_shot else cfg.repair_retries
+        rounds = 1 if single_shot else (cfg.verify_rounds if self.verify_enabled else 1)
+        request, failures = description, 0
+        for rnd in range(rounds):
+            try:
+                real = await skill.realize(ctx, request, retries=retries)
+            except Exception as e:
+                log.debug("realization error (%s): %s", skill.name, e)
+                real = None
+            if real is None:
+                failures += 1
+                if failures >= 2:
+                    break
+                continue
+            failures = 0
+            score = None
+            if self.verify_enabled:
+                score, critique = await self.critic(description, real.image, skill.verify_rubric)
+                if score is None or score < cfg.verify_threshold:
+                    if rnd < rounds - 1:
+                        request = await self.revise(request, critique)
+                    continue
+            return Demo(image=to_pil(real.image), question=question, answer=answer, description=description,
+                        skill=real.skill, verify_score=score, spec=real.spec_text)
+        return None
+
+    # -------------------------------------------------------------- imagine
+    async def imagine(self, image, question: str, *, k: Optional[int] = None, fill: bool = False,
+                      max_new_tokens: Optional[int] = None, zero_shot: Optional[ZeroShot] = None,
+                      speculative: bool = False, time_limit: Optional[float] = None,
+                      on_demo: Optional[Callable[[Demo], None]] = None) -> Imagination:
+        """Imagine demonstrations for one query.
+
+        * default: adaptive budget K*(p0) and difficulty filtering (online rule:
+          stop once K* candidates pass DF or K_max candidates exist);
+        * ``k=n``: exactly ``n`` verified demos, no ABA/DF;
+        * ``fill=True``: K_max verified candidates, all kept (tuning cache).
+
+        ``time_limit`` (seconds) stops synthesis and returns the demos accepted
+        so far; ``on_demo`` is called with each accepted demo as it arrives."""
+        cfg = self.config
+        image = to_pil(image)
+        mnt = max_new_tokens or cfg.answer_max_new_tokens
+        zs = zero_shot or await self.zero_shot(image, question, mnt)
+        if fill:
+            target, cap, accept = cfg.k_max, cfg.k_max, (lambda c: True)
+            budget = cfg.k_max
+        elif k is not None:
+            target, cap, accept = k, k, (lambda c: True)
+            budget = k
+        else:
+            budget = cfg.budget(zs.confidence)
+            target, cap, accept = budget, cfg.k_max, cfg.in_band
+        if target <= 0:
+            return Imagination(demos=[], zero_shot=zs, budget=budget)
+        need_conf = fill or (cfg.use_df and k is None)
+        demos, cands, stats = await self._fill_slots(image, target, cap, accept, need_conf, speculative,
+                                                     time_limit, on_demo)
+        return Imagination(demos=demos, zero_shot=zs, budget=budget, candidates=cands, stats=stats)
+
+    async def _fill_slots(self, image, target, cap, accept, need_conf, speculative=False, time_limit=None,
+                          on_demo=None):
+        """Breadth-first slot filling (Appendix G.1): every fresh triplet gets
+        one cheap attempt before any failure gets the full repair budget."""
+        cfg = self.config
+        seen: set = set()
+
+        def novel(triplet) -> bool:
+            key = _norm_q(triplet[0])
+            if key in seen:
+                return False
+            seen.add(key)
+            return True
+
+        queue: deque = deque()
+        arrived = asyncio.Event()
+
+        def push(triplet) -> bool:
+            if novel(triplet):
+                queue.append(triplet)
+                arrived.set()
+                return True
+            return False
+
+        decomposed = self.option("synthesis", "batch") == "decomposed"
+        synth = asyncio.ensure_future(self.decomposed_triplets(image, target, push) if decomposed
+                                      else self.stream_triplets(image, target, push))
+        attempts_left = cfg.attempts_per_slot * target
+        phase1_left = max(target, attempts_left - target)
+        retry_pool: deque = deque()
+        accepted: list[Demo] = []
+        produced: list[Demo] = []
+        running: dict = {}
+        stats = {"attempts": 0, "fresh_triplet_calls": 0, "verified": 0, "df_rejected": 0}
+        loop = asyncio.get_running_loop()
+        deadline = loop.time() + time_limit if time_limit is not None else None
+
+        async def attempt(triplet, single_shot):
+            if triplet is None:
+                # One new triplet at a time (as in the paper), sampled so that it
+                # does not repeat a question this query already has.
+                got: list = []
+                for _ in range(3):
+                    stats["fresh_triplet_calls"] += 1
+                    keep = lambda t: novel(t) and (got.append(t) or True)  # noqa: E731
+                    if decomposed:
+                        await self.decomposed_triplets(image, 1, keep, max_attempts=2)
+                    else:
+                        await self.stream_triplets(image, 1, keep, max_attempts=1, temperatures=(0.7,),
+                                                   tag="synthesis_fresh")
+                    if got:
+                        triplet = got[0]
+                        break
+                else:
+                    return None, None
+            demo = await self.realize_verified(triplet, single_shot)
+            if demo is not None and need_conf:
+                demo.confidence = await self.confidence(demo.image, demo.question, demo.answer)
+            return triplet, demo
+
+        def open_slots():
+            if speculative:  # every candidate that may still be needed, in parallel
+                return cap - len(produced) - len(running)
+            return min(target - len(accepted), cap - len(produced)) - len(running)
+
+        while True:
+            if deadline is not None and loop.time() >= deadline:
+                stats["timed_out"] = True
+                break
+            while open_slots() > 0 and attempts_left > 0:
+                if phase1_left > 0:
+                    if queue:
+                        triplet = queue.popleft()
+                    elif not synth.done():
+                        break  # more triplets are being decoded right now
+                    else:
+                        triplet = None  # request a fresh one
+                    single = True
+                    phase1_left -= 1
+                elif retry_pool:
+                    triplet, single = retry_pool.popleft(), False
+                else:
+                    break
+                attempts_left -= 1
+                stats["attempts"] += 1
+                task = asyncio.ensure_future(attempt(triplet, single))
+                running[task] = triplet
+            waiters = list(running)
+            arrival = None
+            if not synth.done():
+                arrived.clear()
+                arrival = asyncio.ensure_future(arrived.wait())
+                waiters += [synth, arrival]
+            if not waiters:
+                break
+            timeout = max(0.0, deadline - loop.time()) if deadline is not None else None
+            done, _ = await asyncio.wait(waiters, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
+            if arrival is not None and not arrival.done():
+                arrival.cancel()
+            for task in done:
+                if task not in running:
+                    continue
+                running.pop(task)
+                try:
+                    triplet, demo = task.result()
+                except Exception as e:
+                    log.debug("attempt crashed: %s", e)
+                    triplet, demo = None, None
+                if demo is None:
+                    if triplet is not None:
+                        retry_pool.append(triplet)
+                    continue
+                stats["verified"] += 1
+                produced.append(demo)
+                if accept(demo.confidence):
+                    accepted.append(demo)
+                    if on_demo is not None and len(accepted) <= target:
+                        try:
+                            on_demo(demo)
+                        except Exception as e:  # a UI callback must not break synthesis
+                            log.debug("on_demo failed: %s", e)
+                else:
+                    stats["df_rejected"] += 1
+                    # a filtered candidate reopens its slot with a fresh budget
+                    attempts_left += cfg.attempts_per_slot
+                    phase1_left += cfg.attempts_per_slot - 1
+            if len(accepted) >= target or len(produced) >= cap:
+                break
+        for task in running:
+            task.cancel()
+        if not synth.done():
+            synth.cancel()
+        return accepted[:target], produced, stats
+
+
+def _unbracket(field: str) -> str:
+    """Drop the "[...]" the prompt's template uses around a whole field
+    (models sometimes copy it: "Answer: [3]")."""
+    field = field.strip()
+    if len(field) > 2 and field[0] == "[" and field[-1] == "]" and "[" not in field[1:-1] and "]" not in field[1:-1]:
+        return field[1:-1].strip()
+    return field
+
+
+def icl_parts(demos: Sequence[Demo], image, question: str) -> list:
+    """The in-context prompt of Appendix G (SIMIT-ICL)."""
+    image = to_pil(image)
+    if not demos:
+        return [image, question]
+    n = len(demos)
+    parts: list = [f"Here {'is' if n == 1 else 'are'} {n} example{'s' if n > 1 else ''} to help you answer the question:\n"]
+    for i, d in enumerate(demos, 1):
+        parts += [f"\nExample {i}:", to_pil(d.image), f"Question: {d.question}\nAnswer: {d.answer}"]
+    parts += ["\nNow, answer this question:", image, question]
+    return parts