simit 0.1.0__py3-none-any.whl

simit/pipeline/prompts.py

@@ -0,0 +1,167 @@
+"""Prompts of the SIMIT synthesis pipeline (triplet synthesis, critic,
+description revision), unchanged from the paper's code."""
+
+import re
+
+# Triplet synthesis (Appendix G.1). ``{K}`` is the number of triplets requested.
+TRIPLET_PROMPT = (
+    "I will give you an image. "
+    "Generate exactly {K} diverse visual question-answering training examples.\n\n"
+    "Each example must have:\n"
+    "- A visual question (about any visual concept — NOT tied to the given image)\n"
+    "- A short correct answer\n"
+    "- A detailed description of a new image where your question and answer are correct, your description should be detailed and specific, one should be able to visualize the image perfectly from the description\n\n"
+    "DIVERSITY RULES:\n"
+    "- Cover as many different visual reasoning types as possible across the {K} examples:\n"
+    "  counting, color/attribute, spatial position, object recognition, action/activity,\n"
+    "  text/OCR, yes/no judgment, comparison, scene type, etc.\n"
+    "- Use the provided image only as inspiration for difficulty level and visual domain.\n"
+    "  The generated examples do NOT need to match the provided image.\n"
+    "- Answers must be short (1-5 words).\n\n"
+    "Output format (blank line between examples):\n\n"
+    "Question: [your visual question]\n"
+    "Answer: [short correct answer]\n"
+    "Description: [detailed description of a new image where question+answer are correct]\n\n"
+    "Question: [another diverse visual question]\n"
+    "Answer: [short correct answer]\n"
+    "Description: [description for this example]\n\n"
+    "... (exactly {K} Question/Answer/Description triples)"
+)
+
+# Ablation without diversity-encouraging prompting (Table 2, "w/o DP").
+TRIPLET_PROMPT_NO_DIVERSITY = (
+    "I will give you an image. "
+    "Generate exactly {K} visual question-answering training examples.\n\n"
+    "Each example must have:\n"
+    "- A visual question (about any visual concept — NOT tied to the given image)\n"
+    "- A short correct answer\n"
+    "- A detailed description of a new image where your question and answer are correct, your description should be detailed and specific, one should be able to visualize the image perfectly from the description\n\n"
+    "  The generated examples do NOT need to match the provided image.\n"
+    "- Answers must be short (1-5 words).\n\n"
+    "Output format (blank line between examples):\n\n"
+    "Question: [your visual question]\n"
+    "Answer: [short correct answer]\n"
+    "Description: [detailed description of a new image where question+answer are correct]\n\n"
+    "Question: [another visual question]\n"
+    "Answer: [short correct answer]\n"
+    "Description: [description for this example]\n\n"
+    "... (exactly {K} Question/Answer/Description triples)"
+)
+
+FORMAT_REMINDER = (
+    "\n\nIMPORTANT: your previous response did not follow the required output "
+    "format. Output ONLY plain text in exactly this form, nothing else -- no "
+    "markdown bold, no numbering, no preamble or summary:\n\n"
+    "Question: ...\nAnswer: ...\nDescription: ...\n\n"
+    "Question: ...\nAnswer: ...\nDescription: ...\n\n(blank line between each of the {K} examples)"
+)
+
+# Tolerates "Example N:" lines and "1. " numbering between triplets.
+TRIPLET_RE = re.compile(
+    # BAGEL doesn't always follow the literal template -- observed variants
+    # include a leading "Example N:" line and "1. " numbering before
+    # "Question:". Tolerating these in the boundary lookahead (not just the
+    # leading match) matters most: without it, example 1's Description
+    # swallows the rest of the response, since the lookahead never finds a
+    # bare "Question:" to stop at, silently dropping examples 2..K instead
+    # of just losing the formatting on one field. Markdown bold markers
+    # ("**Question:**", "**Question**:", ...) are stripped separately before
+    # matching -- they showed up wrapping the keyword in inconsistent
+    # positions, so stripping "**" outright is far more robust than trying
+    # to enumerate every position they can appear in.
+    r"Question:\s*(.+?)\nAnswer:\s*(.+?)\nDescription:\s*(.+?)"
+    r"(?=\n\n(?:\d+[.)]\s*|Example\s+\d+:?\s*\n)?Question:|\Z)",
+    re.DOTALL | re.IGNORECASE,
+)
+
+# Critic rubrics: strict (natural photos) and structural (rendered visuals).
+CRITIC_STRICT_PROMPT = (
+    'You are a strict quality-control critic scoring an AI-generated image against this brief: "{request}"\n\n'
+    "Go through the brief clause by clause. For each specific, checkable detail "
+    "(counts, colors, species, objects, setting, text, spatial relations), state "
+    "whether the image satisfies it, with no tolerance for approximation. List "
+    "every discrepancy you find, however small.\n\n"
+    "Then assign a single quality SCORE from 0 to 100 using this rubric:\n"
+    "  90-100: every checkable detail in the brief is satisfied, no discrepancies\n"
+    "  70-89:  matches the brief overall, only minor/cosmetic discrepancies "
+    "(e.g. slightly off shade, imprecise count in a busy/cluttered scene)\n"
+    "  40-69:  matches part of the brief but has at least one clear factual "
+    "discrepancy (wrong count, wrong object, wrong text, wrong spatial relation)\n"
+    "  0-39:   mostly or entirely fails to match the brief\n\n"
+    "End your answer with exactly one line: 'SCORE: <integer 0-100>'."
+)
+
+CRITIC_STRUCTURED_PROMPT = (
+    'You are a quality-control critic scoring an AI-generated chart/diagram/figure '
+    'against this brief: "{request}"\n\n'
+    "This image is being generated as training data by first asking a model for a "
+    "detailed description (which may invent specific numbers/labels/statistics), then "
+    "asking a separate model to write code that renders that description. The exact "
+    "numeric values, colors, or specific labels in the brief are NOT required to match "
+    "precisely -- do not penalize the image for a different number, value, or label than "
+    "the brief states. Judge STRUCTURAL correctness only:\n"
+    "- Is it the right TYPE of visualization (e.g. bar chart vs line chart vs pie chart vs "
+    "diagram vs table), matching what the brief describes?\n"
+    "- Does it have the general elements the brief mentions (e.g. axis labels, a title, "
+    "the right number of categories/bars/nodes, a legend if described)?\n"
+    "- Is the image coherent and readable, with no garbled text, broken layout, or "
+    "rendering artifacts?\n\n"
+    "Then assign a single quality SCORE from 0 to 100 using this rubric:\n"
+    "  90-100: right visualization type, all described structural elements present, clean and readable\n"
+    "  70-89:  right visualization type with the described elements, but missing one minor "
+    "structural element (e.g. no legend when one was implied) or slightly cluttered\n"
+    "  40-69:  right general visualization type but missing a clearly-described structural "
+    "element, or somewhat hard to read\n"
+    "  0-39:   wrong visualization type entirely, or broken/garbled/unreadable rendering\n\n"
+    "End your answer with exactly one line: 'SCORE: <integer 0-100>'."
+)
+
+REVISE_PROMPT = (
+    'This image description was used to generate an image: "{request}"\n'
+    'A critic reviewed the generated image and found these problems:\n"{critique}"\n\n'
+    "Rewrite the description, keeping everything that was already correct, but "
+    "adding or changing whatever is needed so the next generation avoids these "
+    "exact problems. Output ONLY the revised description text, nothing else."
+)
+
+
+# Decomposed triplet synthesis for models that only follow short single-turn
+# instructions (used for Lance in the paper's cross-model study): question,
+# answer and description come from three separate calls.
+DECOMPOSED_TOPICS = [
+    "counting", "color or attribute", "spatial position", "object recognition",
+    "action or activity", "text or OCR", "a yes/no judgment", "comparison", "scene type",
+]
+
+DECOMPOSED_QUESTION_PROMPT = (
+    "Here are examples of short visual questions used to test other pictures:\n"
+    "- How many birds are on the wire?\n"
+    "- What color is the umbrella?\n"
+    "- Is the door open or closed?\n\n"
+    "Write ONE new short visual question about {topic}, on a completely "
+    "different topic than this image -- the question does NOT need to match "
+    "this image, use it only as inspiration for difficulty level and domain. "
+    "Output only the question, nothing else."
+)
+DECOMPOSED_ANSWER_PROMPT = (
+    'Give a short 1-5 word answer to this question, as if answering about '
+    'some other picture: "{question}"\n{type_hint}Output only the answer, nothing else.'
+)
+DECOMPOSED_DESCRIPTION_PROMPT = (
+    'Imagine a different picture. In that picture, the question "{question}" '
+    'has the answer "{answer}". Describe that picture in detail: what it '
+    "shows, the setting, colors, and objects."
+)
+
+COORD_LIST_RE = re.compile(r"^[\d.,\[\]\s-]+$")
+
+META_LEAK_RE = re.compile(
+    r"\bquestion (about|is)\b|excellent question|as i mentioned|short visual question|different topic than this image",
+    re.IGNORECASE,
+)
+
+ANSWER_TYPE_HINTS = [
+    (re.compile(r"^how many\b", re.IGNORECASE), "Answer with just a number. "),
+    (re.compile(r"^(what|which) color", re.IGNORECASE), "Answer with just a color name. "),
+    (re.compile(r"^(is|are|do|does|can|will|was|were|has|have)\b", re.IGNORECASE), "Answer with Yes or No. "),
+]

simit/skills/__init__.py

@@ -0,0 +1,6 @@
+"""Skill library: turn image descriptions into images."""
+
+from .base import Realization, Skill, SpecError, extract_fenced_block
+from .builtin import BUILTIN_SKILLS, default_skills
+
+__all__ = ["Skill", "SpecError", "Realization", "extract_fenced_block", "BUILTIN_SKILLS", "default_skills"]

simit/skills/assets/MERMAID_LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 - 2022 Knut Sveidqvist
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.