openai-gabriel 1.0.1__py3-none-any.whl

gabriel/cli/__main__.py

@@ -0,0 +1,60 @@
+"""Command line interface for GABRIEL.
+
+The CLI intentionally exposes a light-touch interface focused on discovery
+and verification.  It is designed to help users confirm the installation,
+identify available helpers, and navigate toward the Python API for day-to-day
+use.  Full task execution is provided by the Python functions rather than the
+CLI so that users can compose prompts, callbacks, and checkpoints in code.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Iterable, List, Optional
+
+from gabriel import __version__
+from gabriel import tasks as _tasks
+
+
+def _iter_task_names() -> Iterable[str]:
+    """Yield the public task helpers exposed by ``gabriel.tasks``."""
+
+    return sorted(_tasks.__all__)
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="gabriel",
+        description=(
+            "GABRIEL provides GPT-powered helpers for social-science analysis. "
+            "Use the Python API for full control; run `gabriel --list-tasks` "
+            "to see the available helpers."
+        ),
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+        help="Show the installed gabriel version and exit.",
+    )
+    parser.add_argument(
+        "--list-tasks",
+        action="store_true",
+        help="List the Python helpers bundled with gabriel.",
+    )
+
+    args = parser.parse_args(argv)
+
+    if args.list_tasks:
+        print("Available helpers (importable from `gabriel`):")
+        for name in _iter_task_names():
+            print(f"- {name}")
+        return 0
+
+    parser.print_help()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

gabriel/core/__init__.py

@@ -0,0 +1,7 @@
+"""Core plumbing components for GABRIEL."""
+
+from .llm_client import LLMClient, OpenAIClient
+from .prompt_template import PromptTemplate
+from .pipeline import Pipeline
+
+__all__ = ["LLMClient", "OpenAIClient", "PromptTemplate", "Pipeline"]

gabriel/core/llm_client.py

@@ -0,0 +1,34 @@
+"""Abstract LLM client interface."""
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+import os
+
+from ..utils.openai_utils import get_all_responses, get_response
+class LLMClient(ABC):
+    """Minimal interface for language model providers."""
+
+    @abstractmethod
+    async def acall(self, messages: List[Dict[str, str]], **kwargs: Any) -> str:
+        """Asynchronously call the LLM with provided messages."""
+        raise NotImplementedError
+
+class OpenAIClient(LLMClient):
+    """Concrete LLM client leveraging :func:`get_response`."""
+
+    def __init__(
+        self, api_key: Optional[str] = None, base_url: Optional[str] = None
+    ) -> None:
+        # ``get_response`` manages a shared client; API key via env or arg
+        if api_key:
+            os.environ.setdefault("OPENAI_API_KEY", api_key)
+        if base_url:
+            os.environ.setdefault("OPENAI_BASE_URL", base_url)
+
+    async def acall(self, messages: List[Dict[str, str]], **kwargs: Any) -> str:
+        prompt = "\n".join(m.get("content", "") for m in messages)
+        responses, _ = await get_response(prompt, **kwargs)
+        return responses[0] if responses else ""
+
+    async def get_all_responses(self, **kwargs: Any):
+        """Convenience wrapper around :func:`get_all_responses`."""
+        return await get_all_responses(**kwargs)

gabriel/core/pipeline.py

@@ -0,0 +1,18 @@
+"""Pipeline primitives."""
+from typing import Iterable, Any
+
+
+class Pipeline:
+    """Very small placeholder for a processing pipeline."""
+
+    def __init__(self, steps: Iterable[Any]):
+        self.steps = list(steps)
+
+    def run(self, data: Any) -> Any:
+        """Run all steps sequentially."""
+        for step in self.steps:
+            if hasattr(step, "fit"):
+                step.fit(data)
+            if hasattr(step, "transform"):
+                data = step.transform(data)
+        return data

gabriel/core/prompt_template.py

@@ -0,0 +1,152 @@
+"""Prompt template utilities."""
+
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+import warnings
+from typing import Dict, Optional, Set
+
+from jinja2 import Environment, PackageLoader, Template, meta
+
+from ..utils.jinja import shuffled, shuffled_dict
+
+
+@dataclass
+class PromptTemplate:
+    """Simple Jinja2-based prompt template."""
+
+    text: str
+    _environment: Environment = field(init=False, repr=False)
+    _template: Template = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        self._environment = Environment(loader=PackageLoader("gabriel", "prompts"))
+        self._environment.filters["shuffled_dict"] = shuffled_dict
+        self._environment.filters["shuffled"] = shuffled
+        self._template = self._environment.from_string(self.text)
+
+    def render(self, **params: Dict[str, str]) -> str:
+        """Render the template with the given parameters."""
+        attrs = params.get("attributes")
+        descs = params.get("descriptions")
+        if isinstance(attrs, list):
+            if isinstance(descs, list) and len(descs) == len(attrs):
+                params["attributes"] = {a: d for a, d in zip(attrs, descs)}
+            else:
+                params["attributes"] = {a: a for a in attrs}
+        return self._template.render(**params)
+
+    # ------------------------------------------------------------------
+    # Loading helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _extract_variables(text: str) -> Set[str]:
+        """Return the set of undeclared variables in ``text``.
+
+        A minimal Jinja environment is used that registers the filters
+        expected by the built-in prompts.  This allows templates using
+        those filters to be parsed without error while ignoring their
+        actual behaviour.
+        """
+
+        env = Environment()
+        env.filters["shuffled_dict"] = lambda x: x
+        env.filters["shuffled"] = lambda x: x
+        ast = env.parse(text)
+        return meta.find_undeclared_variables(ast)
+
+    @classmethod
+    def from_file(
+        cls,
+        path: str,
+        *,
+        reference_filename: Optional[str] = None,
+        package: str = "gabriel.prompts",
+    ) -> "PromptTemplate":
+        """Load a template from ``path`` with optional variable checking.
+
+        If ``reference_filename`` is provided, the custom template is
+        validated to ensure it declares the exact same set of variables
+        as the reference template.  A descriptive ``ValueError`` is
+        raised if there is a mismatch.
+        """
+
+        text = Path(path).read_text(encoding="utf-8")
+        if reference_filename is not None:
+            ref_text = resources.files(package).joinpath(reference_filename).read_text(
+                encoding="utf-8"
+            )
+            vars_custom = cls._extract_variables(text)
+            vars_ref = cls._extract_variables(ref_text)
+            missing = vars_ref - vars_custom
+            extra = vars_custom - vars_ref
+            if missing or extra:
+                parts = []
+                if missing:
+                    parts.append(f"missing variables: {sorted(missing)}")
+                if extra:
+                    parts.append(f"unexpected variables: {sorted(extra)}")
+                msg = "Custom template variable mismatch (" + "; ".join(parts) + ")"
+                required = {v for v in vars_ref if v in {"text", "attributes"}}
+                missing_required = required - vars_custom
+                if missing_required:
+                    raise ValueError(msg)
+                warnings.warn(
+                    msg + "; proceeding because required variables are present.",
+                    UserWarning,
+                    stacklevel=2,
+                )
+        return cls(text)
+
+    @classmethod
+    def from_package(
+        cls,
+        filename: str,
+        package: str = "gabriel.prompts",
+    ) -> "PromptTemplate":
+        """Load a template from the given package file."""
+        text = resources.files(package).joinpath(filename).read_text(encoding="utf-8")
+        return cls(text)
+
+
+def resolve_template(
+    *,
+    template: Optional[PromptTemplate],
+    template_path: Optional[str],
+    reference_filename: str,
+    package: str = "gabriel.prompts",
+) -> PromptTemplate:
+    """Return a prompt template using either an object or a filesystem override.
+
+    Parameters
+    ----------
+    template:
+        Optional :class:`PromptTemplate` instance supplied directly by the caller.
+    template_path:
+        Filesystem path to a custom Jinja2 template.  When provided, the template
+        is validated against ``reference_filename`` to ensure it exposes the same
+        variables as the built-in prompt.
+    reference_filename:
+        Name of the packaged template used as the default for the task.  This is
+        used both as the fallback when no overrides are supplied and as the
+        reference for variable validation when ``template_path`` is set.
+    package:
+        Package containing the built-in prompt templates.  Callers rarely need to
+        override this.
+    """
+
+    if template is not None and template_path is not None:
+        raise ValueError("Provide either template or template_path, not both")
+
+    if template_path is not None:
+        template = PromptTemplate.from_file(
+            template_path,
+            reference_filename=reference_filename,
+            package=package,
+        )
+
+    return template or PromptTemplate.from_package(
+        reference_filename,
+        package=package,
+    )

gabriel/prompts/__init__.py

@@ -0,0 +1 @@
+

gabriel/prompts/bucket_prompt.jinja2

@@ -0,0 +1,113 @@
+Your goal is to step back and propose exactly {{ bucket_count }} broader "bucket" terms that could serve as mutually exclusive groupings to capture many of these raw terms while still being specific.
+Buckets often can be stylistically similar to the raw terms themselves (short phrases); can even be a raw term itself if that term already captures a broader grouping.
+
+Each bucket should substantively and thematically group multiple raw terms.
+Fine if some raw terms don't fit cleanly into any of your final buckets; aim though for your mutually exclusive buckets to jointly cover as many as reasonable.
+Identify umbrella terms that group raw term subsets without losing important distinctions, specificity, and substance.
+
+Below are the raw terms to bucketize.
+BEGIN RAW TERMS
+{{ terms }}
+END RAW TERMS
+
+We want exactly {{ bucket_count }} bucket terms.
+They should together capture as many raw terms as is reasonable, without vagueness and still being mutually exclusive, descriptive, and specific.
+Avoid buckets that are combinations of multiple distinct buckets, like "housing and employment issues" or "family and financial problems".
+Each bucket should be a single coherent concept, specific and distinct from other buckets.
+Ensure bucket terms have specificity and substance.
+
+Really avoid vague buckets that are catch-alls, "miscellaneous", "singleton", or "other" categories.
+If a potential bucket you have in mind is too vague and catch-all, consider a more specific term that captures a narrower grouping with more thematic precision to what those raw terms are about.
+Strongly prefer descriptive, specific, and directional bucket terms (should be clear how a relevant piece of content could have more/less of the attribute).
+No vague, overly broad, trivial, or overly abstract buckets.
+Better to make a too specific than a too vague bucket, even if it means capturing fewer raw terms.
+Final buckets must be meaningful, specific, interesting, and capturing the largest and most important clusters of raw terms.
+
+Ensure mutual exclusivity-each bucket must be clearly distinct from each other bucket.
+Do not select multiple conceptually overlapping buckets.
+Rather think broadly and approach from different angles, noticing subtle hidden creative patterns that are important alongside the obvious ones.
+We want broad coverage not just variations on the same idea, both obvious choices and subtle interesting patterns that one would only spot by scouring the data as you are.
+Bucket diversity, mutual exclusivity, and creative exploration of different angles/subtle patterns are key.
+We wish to identify both obvious and novel features/patterns that are non-obvious ex ante but significant in the actual data, requiring creative thinking to notice.
+
+Final list must be exactly {{ bucket_count }} mutually exclusive, specific, and well constructed bucket terms.
+
+{% if differentiate %}
+Understand that these raw terms are differences that have been assessed as distinguishing features between two pieces of content, so consider bucket terms that maintain that spirit of distinguishing between two classes of content.
+Specifically, realize that "square" and "circle" correspond to two classes of content as references.
+So if raw terms you are grouping mention "circle" entries having an attribute or a "square" entry having an attribute more than "circle" entry, then those reference classes must be in the bucket name.
+Crucial: you must distinguish between the the two classes of content. "square exhibits more X than circle" and "circle exhibits more X than square" do NOT mean the same thing and don't belong in the same bucket.
+But "square exhibits more X" and "circle lacks X" DO mean the same thing and DO belong in the same bucket.
+Ensure you are only bucketizing differences that actually mean the same thing; it is possible to even have both directions of the same attribute as two different buckets, if both versions are prevalent in the raw terms.
+It is essential you mentally separate "square more X than circle" and "circle more X than square" instances regarding the same attribute X.
+Consider each set by itself, and decide separately if each by itself merits a bucket.
+
+Buckets must be clear on which class has more of the attribute (e.g. "square entries have more X than circle entries" or "circle entries lack Y relative to square entries").
+Each bucket term should be a directional statement (e.g. "circle entries showcase stronger communication amongst colleagues than square entries", "square entries use lots of first person perspective while circle entries do not", "square entries invoke more chronic health issues circle entries", "circle entries involve more complications from type 2 diabetes than square entries", "square entries have few direct quotes from evidence relative to circle entries", etc) rather than an "X vs Y" phrase.
+Closely follow the style of these examples; again, no "vs." etc; it must be clear and explicitly stated in bucket terms which group has more/less of the relevant attribute.
+Focus on the most salient and common differences between the two classes of content, specifically as they appear in the raw terms.
+TO BE CLEAR: all your bucket terms must explicitly state "square entries" and "circle entries", like "circle entries have more X than square entries", "square entries lack Y relative to circle ", "square entries exhibit more Z than circle entries", etc.
+All buckets must make explicit references to both classes of entries ("square entries" and "circle entries"), and explicitly claim that one class has more of an attribute than the other.
+Bucket definitions must clearly explain what it means for a square entry to have more of the attribute than a circle entry (or vice versa, depending on the bucket), grounded in examples from the raw terms.
+{% endif %}
+
+{% if voting %}
+You are selecting the {{ bucket_count }} bucket terms by voting.
+The only bucket terms you can choose from are from the following set of bucket candidates:
+BEGIN BUCKET CANDIDATES
+{{ bucket_candidates }}
+END BUCKET CANDIDATES
+
+From these candidates only, choose the buckets (verbatim as they appear in the candidate keys) that best group the raw terms with the desired specificity, substance, and mutual exclusivity.
+{% if selected_buckets %}
+The following bucket terms have **already been selected**; you should **not** select any similar/overlapping buckets from the candidate list:
+
+BEGIN ALREADY FINALIZED BUCKETS
+{{ selected_buckets }}
+END ALREADY FINALIZED BUCKETS
+
+You only need to vote for {{ bucket_count - (selected_buckets|length) }} additional buckets, not the full {{ bucket_count }}.
+This is because the ultimate, final set of the best, mutually exclusive buckets will be the union of the already selected buckets and the additional buckets you vote for.
+Thus, when voting for the {{ bucket_count - (selected_buckets|length) }} additional buckets, ensure they are mutually exclusive both with each other **and** with the already selected buckets above.
+Consider the selected buckets to locked in; your new selections should not overlap with them at all.
+Choose the {{ bucket_count - (selected_buckets|length) }} additional buckets from the candidate list such that the final set of all selected buckets (including the already selected ones) are mutually exclusive from each other and excellent at grouping as many raw terms as reasonable, without losing specificity and substance.
+It is like being GM of an NBA team: you already have some players on the roster, and you need to pick new ones that fit well with the existing ones and make a great team overall.
+These already finalized buckets are your existing players; now focus on identifying the best complementary pieces to add to the team to get the best overall coverage with the final set of buckets.
+It is essential you avoid voting for terms that overlap with the already selected buckets.
+DON'T vote for any bucket that is conceptually already covered by the already selected buckets.
+Strongly prefer new buckets that are orthogonal to existing buckets, explaining a distinct but important pattern in the data.
+If the obvious patterns are already covered, novelty (without triviality) is highly prized.
+Build a full, mutually exclusive set of buckets that together capture the biggest themes present in the raw terms, amongst the totality of all existing and new buckets you are voting for.
+
+Again, the already selected buckets are: {{ selected_buckets.keys() }}
+You must vote for exactly {{ bucket_count - (selected_buckets|length) }} additional buckets.
+    {% else %}
+You must vote for exactly {{ bucket_count }} buckets from the candidate list keys.
+    {% endif %}
+Output JSON only, in following format:
+{
+    "<insert first voted for candidate bucket here verbatim>": "<insert short 1 sentence justification on why this bucket and how it is mutually exclusive from all other selected buckets here>",
+    "<second voted for candidate bucket verbatim>": "...",
+    ...
+}
+{% else %}
+Each bucket definition/justification should concisely (1-3 sentences) explain what it means to fall into that bucket, also mentioning some member raw terms/direct evidence as illustrative examples.
+It should be clear what it means for some relevant content to fall into each bucket or have more/less of it as an attribute.
+Output JSON only, in following format:
+{
+    "<insert bucket here>": "<insert efficient definition of bucket and justification with direct examples, evidence, and reasoning (1-3 sentences)>",
+    "<bucket>": "...",
+    ...
+}
+Avoid using underscores in bucket terms; ensure easy readability (should understand specific concept just by reading the term).
+{% endif %}
+
+{% if additional_instructions %}
+The user has provided the following additional instructions, clarifications, or labelled examples (if these provided, rely heavily on them to calibrate yourself).
+If they conflict with the other instructions, these user instructions take precedence.
+If they specify a specific scope to confine your buckets to, it is essential you comply with that scope, focus closely on it, and not output any buckets outside of it.
+
+BEGIN ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{{ additional_instructions }}
+END ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{% endif %}

gabriel/prompts/classification_prompt.jinja2

@@ -0,0 +1,50 @@
+{% import "snippets.jinja2" as snip %}
+{% if differentiate %}
+{{ snip.pair_entries(modality, entry_square, entry_circle, circle_first=circle_first|default(false)) }}
+You are specifically looking at features that distinguish the two entries from each other.
+When deciding which labels apply, assess whether the label captures a key difference between the two entries.
+Pay close attention to the specific entry class, square or circle.
+Both directions of the label (square over circle and circle over square) are provided on each attribute; only one can be True, the other must be False.
+Default to both being False unless clear and evident that one applies to the provided entries. Often, neither will apply because the attribute is not present, or is equally present, in which case both must be False.
+Again: for each label pair (same attribute, inverted directions), both can be False (default) or one can be True while other is False. Both cannot be True.
+If one label in a pair concerns circle entries having more of an attribute than square entries, then that label applies/returns True only if the circle entry here has that attribute MORE than the square entry here.
+Insufficient simply for attribute to be present; the specific condition that the provided circle entry manifests the attribute more than the provided square entry must be clearly and evidently true, for this label to return True.
+If instead the square entry manifests the attribute more than the circle entry: then the "circle more than square" label of the pair must return False, while the "square more than circle" label of the pair would return True.
+If the attribute is absent from both square and circle, or is equally present in both, then both paired labels must return the default False.
+Triple check your logic and ensure you apply the correct label of each pair (True for one, False for the other), and default to False for both unless clear and evident that one of the pair is True.
+Evaluate each pair separately from other pairs.
+{% else %}
+{{ snip.single_entry(modality, text) }}
+{% endif %}
+
+Task: For each label below, decide whether it applies to the provided content.
+
+BEGIN LABELS
+{{ attributes | shuffled_dict }}
+END LABELS
+Each key is a label. If a definition is provided, use it to anchor judgment; otherwise use your best consistent definition.
+
+Rules (per label):
+- Output True if label applies to the content; False if not
+- Zero, one, or many labels may apply; mark True for all that apply
+- Independently judge each label; no cross-label inference
+
+Output JSON only, in following format:
+{
+    "<insert label name here>": <insert True if this label applies, False if not>,
+    "<label name>": <insert False|True>,
+    ...
+}
+
+Labels you are evaluating are: {{ attributes.keys() | shuffled }}
+Assess EVERY label (False or True); no drops. Use these label names verbatim, with absolutely no modification.
+Same case, same spelling, same punctuation, same formatting.
+
+{% if additional_instructions %}
+The user has provided the following additional instructions, clarifications, or labelled examples (if these provided, rely heavily on them to calibrate your judgment).
+If they conflict with the other instructions, these user instructions take precedence.
+Important: if user has explicitly asked for only the best label, output True for ONLY the one label that best applies, and False for all others.
+BEGIN ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{{ additional_instructions }}
+END ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{% endif %}

gabriel/prompts/codify_prompt.jinja2

@@ -0,0 +1,95 @@
+{% import "snippets.jinja2" as snip %}
+{% if entry_circle is defined %}
+{{ snip.pair_entries(modality, entry_square, entry_circle, circle_first=circle_first|default(false)) }}
+{% else %}
+{{ snip.single_entry(modality, text) }}
+{% endif %}
+{% if modality != "text" %}
+Since these inputs are not text, you should not output verbatim excerpts.
+Instead, describe the relevant parts of the entity being coded, in your own words.
+{% endif %}
+
+Your task is to act as a passage coder, as is done in the social sciences.
+
+{% if categories %}
+You are to carefully read and dissect the text, marking the boundaries of snippets/subsections of the text that belong to the following categories/attributes:
+
+BEGIN CATEGORIES/ATTRIBUTES
+{{ categories }}
+END CATEGORIES/ATTRIBUTES
+Follow the specific definitions of the categories in assigning them to snippets/subsections of the text.
+{% else %}
+Consider the broad topic alluded to in these user instructions:
+BEGIN USER INSTRUCTIONS
+{{ additional_instructions }}
+END USER INSTRUCTIONS
+Consider the specific category labels that fall under the broad topic alluded to in the user instructions.
+For example, if the broad topic is Bible verses, the specific category labels would be specific Bible verses like "John 3:16" or "Psalm 23:1".
+Or if the topic is Supreme Court decisions, the specific category labels would be specific Supreme Court decisions like "Marbury v. Madison" or "Brown v. Board of Education".
+In your output, you would only output as category labels the specific category labels (under the broad topic) that are present in the text and should be marked.
+So, in the example above, if the text alludes to the Bible verse "John 3:16", you would output "John 3:16" as the category label and mark the snippets/subsections that allude to the John 3:16.
+You would not output every single Bible verse; just every Bible verse that is present in the text.
+Be thorough and exhaustive in your output, denoting every specific category label that is applicable in the text and marking every snippet/subsection that is applicable to each category label.
+{% endif %}
+
+You mark the boundaries of the snippets/subsections by verbatim outputting the first 8 words and the last 8 words of the snippet/subsection (more if necessary to ensure uniqueness in programmatically identifying the snippets/subsections).
+If the snippet/subsection is less than 20 words, you should write out the entire snippet/subsection as the beginning excerpt and leave the ending excerpt blank.
+It is crucial that these beginning and ending excerpts are completely verbatim, same exact spelling/mistakes/punctuation/capitalization/line breaks/etc.
+We will programmatically use these excerpts to identify and pull out the whole snippets/subsections that you are marking.
+Be sure that the beginning and ending excerpts are completely verbatim, same exact everything, same exact characters.
+
+Output a JSON in the following format:
+
+{
+    "<insert category name here>":
+        [
+            {
+                "beginning excerpt": "<insert the first 8 words of the snippet/subsection here verbatim (or the entire snippet/subsection if it is less than 20 words)>",
+                "ending excerpt": "<insert the last 8 words of the snippet/subsection here verbatim (or leave blank if the snippet/subsection is less than 20 words)>",
+            },
+            {
+                "beginning excerpt": "<insert the first 8 words of a different snippet/subsection here verbatim (or the entire snippet/subsection if it is less than 20 words)>",
+                "ending excerpt": "<insert the last 8 words of that snippet/subsection here verbatim (or leave blank if the snippet/subsection is less than 20 words)>",
+            },
+            ...
+        ]
+    "<category name>",
+        [
+            {
+                "beginning excerpt": "...",
+                "ending excerpt": "...",
+            },
+            ...
+        ],
+    ...
+}
+
+{% if categories %}
+Again: the categories you are marking are: {{ categories.keys() | shuffled }}
+Ensure you input these category names verbatim into the JSON.
+Exact spelling/capitalization/underscoring/any other formatting of the category names must be preserved.
+Carefully consider the entire text for each and every category; do not skip any (though it is possible that some categories are not present in the text, in which case that category name must still be in the JSON but with an empty list of snippets/subsections).
+{% endif %}
+It is possible and often the case that text will overlap between categories, in which case you should mark the text for each applicable category separately. The same area of text can be marked for multiple categories.
+Often in this case, it is not the exact same snippet/subsection that is being marked for each category, but rather a slightly different snippet/subsection that perhaps starts or ends at a different word, or is a different length.
+You must carefully consider extremely precise boundaries for the snippets/subsections distinctly for each category.
+Be precise with your identified snippets/subsections; don't include any words/chunks of text that are outside the essential boundaries of the snippet/subsection where the category applies.
+Be very thorough. Every single applicable snippet/subsection (include small ones) throughout the text, beginning to end, must be marked.
+This requires meticulosity and attention to detail.
+Patiently find every single snippet/subsection that fits each category, and be willing to mark a large number of snippets/subsections if necessary. 
+Also be willing to mark the same or similar snippets/subsections under multiple categories separately if applicable.
+Don't be lazy and mark giant chunks of text where most of it doesn't fit the category. In this case, only mark narrowly scoped excerpts of the text that fit the category.
+Rigorously consider every single category. Remember that the same or similar snippets/subsections can be marked separately under multiple categories.
+It is absolutely crucial that you look through the entire text. Do not skim through the middle and end; don't only mark snippets/subsections at the beginning. Full, thorough coverage is key.
+Scour every single sentence, phrase, and paragraph of the text. Double check that you have not missed any snippets/subsections and that you have not neglected any portions of the text in your thinking and marking.
+Make extra effort to check each of the beginning, middle, and end of the text to ensure you have not missed any snippets/subsections there. Cover everything, beginning to end.
+It is crucial that you think through each portion of the text carefully and thoroughly, otherwise you will miss snippets/subsections.
+
+{% if additional_instructions %}
+These are additional instructions from the user, follow them carefully. They take precedence over the general instructions above, if there is a conflict.
+In particular, consider any instructions, if provided, on the scope of the text to be marked (should you be marking very specific phrases or full paragraphs/sections?).
+If the user indicates a different scope of how large the snippets/subsections should be/how much context to include, follow their instructions over the general instructions above, if there is a conflict.
+BEGIN ADDITIONAL INSTRUCTIONS
+{{ additional_instructions }}
+END ADDITIONAL INSTRUCTIONS
+{% endif %}

gabriel/prompts/comparison_prompt.jinja2

@@ -0,0 +1,60 @@
+{% import "snippets.jinja2" as snip %}
+{{ snip.pair_entries(modality, entry_square, entry_circle, circle_first=circle_first|default(false)) }}
+
+{% if differentiate %}
+Your task is to list concise, mutually exclusive terms or short phrases that best explain how entry "circle" differs from entry "square".
+These terms should be specific and reflect clear differences observable between these particular entries.
+There can be any number of terms from none up; do not invent terms if there is no meaningful difference.
+Compile all relevant differences; be thorough.
+
+When listing differences, explicitly note whether "square" or "circle" shows more or less of the attribute. For example, say "circle has more X than square" or "square lacks Y whereas circle has some".
+Each extracted term should be a directional statement (e.g. "square exhibits strong communication amongst colleagues, circle does not", "circle uses lots of first person perspective whereas square rarely does", "square involves more chronic health issues than circle", "complications from type 2 diabetes seen in square but not circle", "circle has no direct quotes from evidence, square has some", etc) rather than an "X vs Y" phrase.
+Strong preference for descriptive, highly specific, and directional terms-avoid vague and broad terms, and terms where it is unclear what it means for an entry to have more or less of the term.
+Again: all about differences. An attribute shown in one entry but not the other is a difference. An attribute shown to a greater extent in one entry than the other is a difference.
+
+Shortened example output:
+{
+    "circle mentions renewable energy more than square": "Circle discusses wind and solar incentives extensively (stating 'A' and 'B'), whereas square only briefly references green jobs (focusing instead on 'C').",
+    "square uses historic evidence and anecdotes while circle does not": "Square highlights historical evidence and anecdotes in its argument (citing 'D', 'E'), whereas circle invokes no details from history.",
+    ...
+}
+Stick closely to this "square focuses more on ... than circle", "circle advocates for ... more than square", "square uses ... while circle does not" general style in your term names.
+{% else %}
+Your task is to list concise, mutually exclusive terms or short phrases that best explain how entry "circle" is similar to entry "square".
+These terms should be specific and reflect clear similarities observable between these particular entries.
+There can be any number of terms from none up; do not invent terms if there is no meaningful similarity.
+Compile all relevant similarities; be thorough.
+
+Shortened example output:
+{
+    "focus on renewable energy": "Both entries discuss wind and solar incentives extensively. Entry circle says 'A' and 'B', and entry square shows a similar focus, stating 'C' and 'D'.",
+    "use of historic evidence and anecdotes": "Both passages highlight historical evidence and anecdotes in their arguments. Entry square cites 'E'; entry circle invokes 'F' and 'G'.",
+    ...
+}
+{% endif %}
+
+Don't be vague or generic. Be precise and parsimonious.
+Each term must have a brief justification, directly quoting/using evidence from the entries where relevant.
+No terms, one term, three, six, many terms may apply-don't skip any applicable terms but output zero/few if that is the correct choice.
+Avoid trivial and pedantic terms; only substantive, meaningful terms-zero/fewer terms better than pointless trivialities.
+Ensure your terms are mutually exclusive; each should be a distinct idea and must not overlap with other terms.
+Include both obvious and subtle but relevant observations-think broadly and approach the comparison from different angles.
+Employ creative thinking to assess from many different angles so that you notice non-obvious (but still relevant) points of comparison too.
+ABSOLUTELY no underscores/hyphens in your term names-just natural, parsimonious language, like the examples.
+
+Output JSON only, mapping each term to a 1-3 sentence justification:
+{
+    "<insert term name here>": "<insert direct quotes/evidence and reasoning as justification (1-3 evidence dense sentences)>",
+    "<term>": "...",
+    ...
+}
+
+{% if additional_instructions %}
+The user has provided the following additional instructions, clarifications, or labelled examples (if these provided, rely heavily on them to calibrate yourself).
+If they conflict with the other instructions, these user instructions take precedence.
+If they specify a specific scope to confine your terms to, it is essential you comply with that scope, focus closely on it, and not output any terms outside of it.
+
+BEGIN ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{{ additional_instructions }}
+END ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{% endif %}

gabriel/prompts/deduplicate_prompt.jinja2

@@ -0,0 +1,41 @@
+Consider the following raw terms:
+BEGIN RAW TERMS
+{{ raw_terms }}
+END RAW TERMS
+{% if modality == "text" %}
+Each raw term is a text snippet. You are deduplicating the texts. In your output, use the identifier keys for each text to denote all representative / mapped texts. Only use those identifier keys in the mapping, not the full texts.
+{% endif %}
+
+Deduplicate these raw terms.
+Do this by exhaustively identifying all subsets of raw terms such that all terms in the subset are largely equivalent to each other.
+For each subset, map the raw terms to a single representative term.
+The representative term must be one of the raw terms in the subset - it should be the best, most specific representative of the subset.
+Strongly prefer precise and detailed representative terms; avoid vague, overly broad, or overly abstract representative terms (vague terms are better as simple members of the subset, not representatives).
+Important: all raw terms that are not duplicates should be left alone and not included in output.
+Only output the deduplication map, exhaustively compiling all subsets of largely equivalent raw terms and ignoring the rest.
+
+All terms in the map must be taken from the raw terms verbatim, absolutely no modification.
+Same case, same spelling, same punctuation, same formatting.
+
+Output JSON only, in following format:
+{
+    "<insert representative term here>": ["<insert raw term in corresponding subset here>", "<raw term>", ...],
+    "<representative term>": ["<raw term>", ...],
+    ...
+}
+
+Again: representative term is generally not the broadest catch-all raw term; it is specific, precise, detailed - the term that all other raw terms in the subset are largely equivalent to.
+Include the representative term twice: both as the key AND as member in the list of raw terms in subset.
+Don't force meaningfully distinct terms into the same subset; it is fine if most or all terms are ignored and not deduplicated because they have no duplicate(s).
+If no duplicates present, output an empty JSON.
+Nonetheless, be thorough and exhaustive on all genuine duplicates - even pairs of just two equivalent terms must be deduplicated.
+There may be many such pairs/triples/etc and you must find all that exist.
+
+{% if additional_instructions %}
+The user has provided the following additional instructions, clarifications, or labelled examples (if these provided, rely heavily on them to calibrate yourself).
+If they conflict with the other instructions, these user instructions take precedence.
+
+BEGIN ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{{ additional_instructions }}
+END ADDITIONAL INSTRUCTIONS/CLARIFICATIONS
+{% endif %}