palimpzest 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

palimpzest/prompts/prompt_factory.py

@@ -2,7 +2,6 @@
 
 import base64
 import json
-from string import Formatter
 
 from pydantic import BaseModel
 
@@ -10,137 +9,114 @@ from palimpzest.constants import (
     LLAMA_CONTEXT_TOKENS_LIMIT,
     TOKENS_PER_CHARACTER,
     Cardinality,
+    Modality,
     Model,
     PromptStrategy,
 )
 from palimpzest.core.elements.records import DataRecord
-from palimpzest.core.lib.schemas import AudioBase64, AudioFilepath, ImageBase64, ImageFilepath, ImageURL
+from palimpzest.core.lib.schemas import (
+    AUDIO_FIELD_TYPES,
+    IMAGE_FIELD_TYPES,
+    AudioBase64,
+    AudioFilepath,
+    ImageBase64,
+    ImageFilepath,
+    ImageURL,
+)
 from palimpzest.prompts.convert_prompts import (
-    COT_QA_AUDIO_DISCLAIMER,
-    COT_QA_AUDIO_EXAMPLE_ANSWER,
-    COT_QA_AUDIO_EXAMPLE_CONTEXT,
-    COT_QA_AUDIO_EXAMPLE_INPUT_FIELDS,
-    COT_QA_AUDIO_EXAMPLE_OUTPUT_FIELDS,
-    COT_QA_AUDIO_EXAMPLE_REASONING,
-    COT_QA_AUDIO_JOB_INSTRUCTION,
-    COT_QA_BASE_SYSTEM_PROMPT,
-    COT_QA_BASE_USER_PROMPT,
-    COT_QA_EXAMPLE_ANSWER,
-    COT_QA_EXAMPLE_CONTEXT,
-    COT_QA_EXAMPLE_INPUT_FIELDS,
-    COT_QA_EXAMPLE_OUTPUT_FIELDS,
-    COT_QA_EXAMPLE_REASONING,
-    COT_QA_IMAGE_DISCLAIMER,
-    COT_QA_IMAGE_EXAMPLE_ANSWER,
-    COT_QA_IMAGE_EXAMPLE_CONTEXT,
-    COT_QA_IMAGE_EXAMPLE_INPUT_FIELDS,
-    COT_QA_IMAGE_EXAMPLE_OUTPUT_FIELDS,
-    COT_QA_IMAGE_EXAMPLE_REASONING,
-    COT_QA_IMAGE_JOB_INSTRUCTION,
-    COT_QA_JOB_INSTRUCTION,
-    COT_QA_NO_REASONING_BASE_SYSTEM_PROMPT,
-    COT_QA_NO_REASONING_BASE_USER_PROMPT,
+    MAP_BASE_SYSTEM_PROMPT,
+    MAP_BASE_USER_PROMPT,
+    MAP_NO_REASONING_BASE_SYSTEM_PROMPT,
+    MAP_NO_REASONING_BASE_USER_PROMPT,
 )
-from palimpzest.prompts.critique_and_refine_convert_prompts import (
+from palimpzest.prompts.critique_and_refine_prompts import (
     BASE_CRITIQUE_PROMPT,
     BASE_REFINEMENT_PROMPT,
-    COT_QA_CRITIQUE_CRITERIA,
-    COT_QA_CRITIQUE_FINISH_INSTRUCTION,
-    COT_QA_IMAGE_CRITIQUE_CRITERIA,
-    COT_QA_IMAGE_REFINEMENT_CRITERIA,
-    COT_QA_REFINEMENT_CRITERIA,
-    COT_QA_REFINEMENT_FINISH_INSTRUCTION,
+    FILTER_CRITIQUE_CRITERIA,
+    FILTER_CRITIQUE_FINISH_INSTRUCTION,
+    FILTER_REFINEMENT_CRITERIA,
+    FILTER_REFINEMENT_FINISH_INSTRUCTION,
+    MAP_CRITIQUE_CRITERIA,
+    MAP_CRITIQUE_FINISH_INSTRUCTION,
+    MAP_REFINEMENT_CRITERIA,
+    MAP_REFINEMENT_FINISH_INSTRUCTION,
 )
 from palimpzest.prompts.filter_prompts import (
-    COT_BOOL_AUDIO_DISCLAIMER,
-    COT_BOOL_AUDIO_EXAMPLE_CONTEXT,
-    COT_BOOL_AUDIO_EXAMPLE_FILTER_CONDITION,
-    COT_BOOL_AUDIO_EXAMPLE_INPUT_FIELDS,
-    COT_BOOL_AUDIO_EXAMPLE_REASONING,
-    COT_BOOL_AUDIO_JOB_INSTRUCTION,
-    COT_BOOL_BASE_SYSTEM_PROMPT,
-    COT_BOOL_BASE_USER_PROMPT,
-    COT_BOOL_EXAMPLE_CONTEXT,
-    COT_BOOL_EXAMPLE_FILTER_CONDITION,
-    COT_BOOL_EXAMPLE_INPUT_FIELDS,
-    COT_BOOL_EXAMPLE_REASONING,
-    COT_BOOL_IMAGE_DISCLAIMER,
-    COT_BOOL_IMAGE_EXAMPLE_CONTEXT,
-    COT_BOOL_IMAGE_EXAMPLE_FILTER_CONDITION,
-    COT_BOOL_IMAGE_EXAMPLE_INPUT_FIELDS,
-    COT_BOOL_IMAGE_EXAMPLE_REASONING,
-    COT_BOOL_IMAGE_JOB_INSTRUCTION,
-    COT_BOOL_JOB_INSTRUCTION,
-    COT_BOOL_NO_REASONING_BASE_SYSTEM_PROMPT,
-    COT_BOOL_NO_REASONING_BASE_USER_PROMPT,
+    FILTER_BASE_SYSTEM_PROMPT,
+    FILTER_BASE_USER_PROMPT,
+    FILTER_NO_REASONING_BASE_SYSTEM_PROMPT,
+    FILTER_NO_REASONING_BASE_USER_PROMPT,
 )
 from palimpzest.prompts.join_prompts import (
-    COT_JOIN_AUDIO_DISCLAIMER,
-    COT_JOIN_AUDIO_EXAMPLE_CONTEXT,
-    COT_JOIN_AUDIO_EXAMPLE_INPUT_FIELDS,
-    COT_JOIN_AUDIO_EXAMPLE_JOIN_CONDITION,
-    COT_JOIN_AUDIO_EXAMPLE_REASONING,
-    COT_JOIN_AUDIO_JOB_INSTRUCTION,
-    COT_JOIN_AUDIO_RIGHT_EXAMPLE_CONTEXT,
-    COT_JOIN_AUDIO_RIGHT_EXAMPLE_INPUT_FIELDS,
-    COT_JOIN_BASE_SYSTEM_PROMPT,
-    COT_JOIN_BASE_USER_PROMPT,
-    COT_JOIN_EXAMPLE_CONTEXT,
-    COT_JOIN_EXAMPLE_INPUT_FIELDS,
-    COT_JOIN_EXAMPLE_JOIN_CONDITION,
-    COT_JOIN_EXAMPLE_REASONING,
-    COT_JOIN_IMAGE_DISCLAIMER,
-    COT_JOIN_IMAGE_EXAMPLE_CONTEXT,
-    COT_JOIN_IMAGE_EXAMPLE_INPUT_FIELDS,
-    COT_JOIN_IMAGE_EXAMPLE_JOIN_CONDITION,
-    COT_JOIN_IMAGE_EXAMPLE_REASONING,
-    COT_JOIN_IMAGE_JOB_INSTRUCTION,
-    COT_JOIN_IMAGE_RIGHT_EXAMPLE_CONTEXT,
-    COT_JOIN_IMAGE_RIGHT_EXAMPLE_INPUT_FIELDS,
-    COT_JOIN_JOB_INSTRUCTION,
-    COT_JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,
-    COT_JOIN_NO_REASONING_BASE_USER_PROMPT,
-    COT_JOIN_RIGHT_AUDIO_DISCLAIMER,
-    COT_JOIN_RIGHT_EXAMPLE_CONTEXT,
-    COT_JOIN_RIGHT_EXAMPLE_INPUT_FIELDS,
-    COT_JOIN_RIGHT_IMAGE_DISCLAIMER,
+    JOIN_BASE_SYSTEM_PROMPT,
+    JOIN_BASE_USER_PROMPT,
+    JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,
+    JOIN_NO_REASONING_BASE_USER_PROMPT,
 )
-from palimpzest.prompts.moa_aggregator_convert_prompts import (
-    COT_MOA_AGG_BASE_SYSTEM_PROMPT,
-    COT_MOA_AGG_BASE_USER_PROMPT,
+from palimpzest.prompts.moa_aggregator_prompts import (
+    FILTER_MOA_AGG_BASE_SYSTEM_PROMPT,
+    FILTER_MOA_AGG_BASE_USER_PROMPT,
+    MAP_MOA_AGG_BASE_SYSTEM_PROMPT,
+    MAP_MOA_AGG_BASE_USER_PROMPT,
 )
-from palimpzest.prompts.moa_proposer_convert_prompts import (
-    COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
-    COT_MOA_PROPOSER_BASE_USER_PROMPT,
-    COT_MOA_PROPOSER_EXAMPLE_ANSWER,
-    COT_MOA_PROPOSER_EXAMPLE_CONTEXT,
-    COT_MOA_PROPOSER_EXAMPLE_INPUT_FIELDS,
-    COT_MOA_PROPOSER_EXAMPLE_OUTPUT_FIELDS,
-    COT_MOA_PROPOSER_IMAGE_DISCLAIMER,
-    COT_MOA_PROPOSER_IMAGE_EXAMPLE_ANSWER,
-    COT_MOA_PROPOSER_IMAGE_EXAMPLE_CONTEXT,
-    COT_MOA_PROPOSER_IMAGE_EXAMPLE_INPUT_FIELDS,
-    COT_MOA_PROPOSER_IMAGE_EXAMPLE_OUTPUT_FIELDS,
-    COT_MOA_PROPOSER_IMAGE_JOB_INSTRUCTION,
-    COT_MOA_PROPOSER_JOB_INSTRUCTION,
+from palimpzest.prompts.moa_proposer_prompts import (
+    FILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
+    FILTER_MOA_PROPOSER_BASE_USER_PROMPT,
+    MAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
+    MAP_MOA_PROPOSER_BASE_USER_PROMPT,
 )
 from palimpzest.prompts.split_merge_prompts import (
-    COT_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
-    COT_SPLIT_MERGER_BASE_USER_PROMPT,
+    FILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
+    FILTER_SPLIT_MERGER_BASE_USER_PROMPT,
+    MAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
+    MAP_SPLIT_MERGER_BASE_USER_PROMPT,
 )
 from palimpzest.prompts.split_proposer_prompts import (
-    COT_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
-    COT_SPLIT_PROPOSER_BASE_USER_PROMPT,
-    SPLIT_PROPOSER_EXAMPLE_ANSWER,
-    SPLIT_PROPOSER_EXAMPLE_CONTEXT,
-    SPLIT_PROPOSER_EXAMPLE_INPUT_FIELDS,
-    SPLIT_PROPOSER_EXAMPLE_OUTPUT_FIELDS,
-    SPLIT_PROPOSER_JOB_INSTRUCTION,
+    FILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
+    FILTER_SPLIT_PROPOSER_BASE_USER_PROMPT,
+    MAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
+    MAP_SPLIT_PROPOSER_BASE_USER_PROMPT,
 )
-from palimpzest.prompts.util_phrases import (
+from palimpzest.prompts.utils import (
+    AUDIO_DISCLAIMER,
+    AUDIO_EXAMPLE_ANSWER,
+    AUDIO_EXAMPLE_CONTEXT,
+    AUDIO_EXAMPLE_INPUT_FIELDS,
+    AUDIO_EXAMPLE_OUTPUT_FIELDS,
+    AUDIO_EXAMPLE_REASONING,
+    AUDIO_SENTENCE_EXAMPLE_ANSWER,
     DESC_SECTION,
+    EXAMPLE_FILTER_CONDITION,
+    EXAMPLE_JOIN_CONDITION,
+    FILTER_EXAMPLE_REASONING,
+    FILTER_JOB_INSTRUCTION,
+    IMAGE_DISCLAIMER,
+    IMAGE_EXAMPLE_ANSWER,
+    IMAGE_EXAMPLE_CONTEXT,
+    IMAGE_EXAMPLE_INPUT_FIELDS,
+    IMAGE_EXAMPLE_OUTPUT_FIELDS,
+    IMAGE_EXAMPLE_REASONING,
+    IMAGE_SENTENCE_EXAMPLE_ANSWER,
+    JOIN_EXAMPLE_REASONING,
+    JOIN_JOB_INSTRUCTION,
+    MAP_JOB_INSTRUCTION,
     ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION,
     ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION,
+    PROPOSER_JOB_INSTRUCTION,
+    RIGHT_AUDIO_DISCLAIMER,
+    RIGHT_AUDIO_EXAMPLE_CONTEXT,
+    RIGHT_AUDIO_EXAMPLE_INPUT_FIELDS,
+    RIGHT_IMAGE_DISCLAIMER,
+    RIGHT_IMAGE_EXAMPLE_CONTEXT,
+    RIGHT_IMAGE_EXAMPLE_INPUT_FIELDS,
+    RIGHT_TEXT_EXAMPLE_CONTEXT,
+    RIGHT_TEXT_EXAMPLE_INPUT_FIELDS,
+    TEXT_EXAMPLE_ANSWER,
+    TEXT_EXAMPLE_CONTEXT,
+    TEXT_EXAMPLE_INPUT_FIELDS,
+    TEXT_EXAMPLE_OUTPUT_FIELDS,
+    TEXT_EXAMPLE_REASONING,
+    TEXT_SENTENCE_EXAMPLE_ANSWER,
 )
 
 
@@ -148,62 +124,54 @@ class PromptFactory:
     """Factory class for generating prompts for the Generator given the input(s)."""
 
     BASE_SYSTEM_PROMPT_MAP = {
-        PromptStrategy.COT_BOOL: COT_BOOL_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_BOOL_NO_REASONING: COT_BOOL_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_BOOL_AUDIO_NO_REASONING: COT_BOOL_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_BOOL_IMAGE_NO_REASONING: COT_BOOL_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_JOIN: COT_JOIN_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_JOIN_NO_REASONING: COT_JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_JOIN_AUDIO_NO_REASONING: COT_JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_JOIN_IMAGE_NO_REASONING: COT_JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_QA: COT_QA_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_QA_NO_REASONING: COT_QA_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_QA_AUDIO: COT_QA_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_QA_AUDIO_NO_REASONING: COT_QA_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_QA_CRITIC: None,
-        PromptStrategy.COT_QA_REFINE: None,
-        PromptStrategy.COT_QA_IMAGE: COT_QA_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_QA_IMAGE_NO_REASONING: COT_QA_NO_REASONING_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_QA_IMAGE_CRITIC: None,
-        PromptStrategy.COT_QA_IMAGE_REFINE: None,
-        PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
-        PromptStrategy.COT_MOA_AGG: COT_MOA_AGG_BASE_SYSTEM_PROMPT,
-        PromptStrategy.SPLIT_PROPOSER: COT_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
-        PromptStrategy.SPLIT_MERGER: COT_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
+        # filter system prompts
+        PromptStrategy.FILTER: FILTER_BASE_SYSTEM_PROMPT,
+        PromptStrategy.FILTER_NO_REASONING: FILTER_NO_REASONING_BASE_SYSTEM_PROMPT,
+        PromptStrategy.FILTER_CRITIC: None,
+        PromptStrategy.FILTER_REFINE: None,
+        PromptStrategy.FILTER_MOA_PROPOSER: FILTER_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
+        PromptStrategy.FILTER_MOA_AGG: FILTER_MOA_AGG_BASE_SYSTEM_PROMPT,
+        PromptStrategy.FILTER_SPLIT_PROPOSER: FILTER_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
+        PromptStrategy.FILTER_SPLIT_MERGER: FILTER_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
+
+        # join system prompts
+        PromptStrategy.JOIN: JOIN_BASE_SYSTEM_PROMPT,
+        PromptStrategy.JOIN_NO_REASONING: JOIN_NO_REASONING_BASE_SYSTEM_PROMPT,
+
+        # map system prompts
+        PromptStrategy.MAP: MAP_BASE_SYSTEM_PROMPT,
+        PromptStrategy.MAP_NO_REASONING: MAP_NO_REASONING_BASE_SYSTEM_PROMPT,
+        PromptStrategy.MAP_CRITIC: None,
+        PromptStrategy.MAP_REFINE: None,
+        PromptStrategy.MAP_MOA_PROPOSER: MAP_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
+        PromptStrategy.MAP_MOA_AGG: MAP_MOA_AGG_BASE_SYSTEM_PROMPT,
+        PromptStrategy.MAP_SPLIT_PROPOSER: MAP_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
+        PromptStrategy.MAP_SPLIT_MERGER: MAP_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
     }
     BASE_USER_PROMPT_MAP = {
-        PromptStrategy.COT_BOOL: COT_BOOL_BASE_USER_PROMPT,
-        PromptStrategy.COT_BOOL_NO_REASONING: COT_BOOL_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_BASE_USER_PROMPT,
-        PromptStrategy.COT_BOOL_AUDIO_NO_REASONING: COT_BOOL_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_BASE_USER_PROMPT,
-        PromptStrategy.COT_BOOL_IMAGE_NO_REASONING: COT_BOOL_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_JOIN: COT_JOIN_BASE_USER_PROMPT,
-        PromptStrategy.COT_JOIN_NO_REASONING: COT_JOIN_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_BASE_USER_PROMPT,
-        PromptStrategy.COT_JOIN_AUDIO_NO_REASONING: COT_JOIN_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_BASE_USER_PROMPT,
-        PromptStrategy.COT_JOIN_IMAGE_NO_REASONING: COT_JOIN_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_QA: COT_QA_BASE_USER_PROMPT,
-        PromptStrategy.COT_QA_NO_REASONING: COT_QA_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_QA_AUDIO: COT_QA_BASE_USER_PROMPT,
-        PromptStrategy.COT_QA_AUDIO_NO_REASONING: COT_QA_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_QA_CRITIC: BASE_CRITIQUE_PROMPT,
-        PromptStrategy.COT_QA_REFINE: BASE_REFINEMENT_PROMPT,
-        PromptStrategy.COT_QA_IMAGE: COT_QA_BASE_USER_PROMPT,
-        PromptStrategy.COT_QA_IMAGE_NO_REASONING: COT_QA_NO_REASONING_BASE_USER_PROMPT,
-        PromptStrategy.COT_QA_IMAGE_CRITIC: BASE_CRITIQUE_PROMPT,
-        PromptStrategy.COT_QA_IMAGE_REFINE: BASE_REFINEMENT_PROMPT,
-        PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_BASE_USER_PROMPT,
-        PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_BASE_USER_PROMPT,
-        PromptStrategy.COT_MOA_AGG: COT_MOA_AGG_BASE_USER_PROMPT,
-        PromptStrategy.SPLIT_PROPOSER: COT_SPLIT_PROPOSER_BASE_USER_PROMPT,
-        PromptStrategy.SPLIT_MERGER: COT_SPLIT_MERGER_BASE_USER_PROMPT,
+        # filter user prompts
+        PromptStrategy.FILTER: FILTER_BASE_USER_PROMPT,
+        PromptStrategy.FILTER_NO_REASONING: FILTER_NO_REASONING_BASE_USER_PROMPT,
+        PromptStrategy.FILTER_CRITIC: BASE_CRITIQUE_PROMPT,
+        PromptStrategy.FILTER_REFINE: BASE_REFINEMENT_PROMPT,
+        PromptStrategy.FILTER_MOA_PROPOSER: FILTER_MOA_PROPOSER_BASE_USER_PROMPT,
+        PromptStrategy.FILTER_MOA_AGG: FILTER_MOA_AGG_BASE_USER_PROMPT,
+        PromptStrategy.FILTER_SPLIT_PROPOSER: FILTER_SPLIT_PROPOSER_BASE_USER_PROMPT,
+        PromptStrategy.FILTER_SPLIT_MERGER: FILTER_SPLIT_MERGER_BASE_USER_PROMPT,
+
+        # join user prompts
+        PromptStrategy.JOIN: JOIN_BASE_USER_PROMPT,
+        PromptStrategy.JOIN_NO_REASONING: JOIN_NO_REASONING_BASE_USER_PROMPT,
+
+        # map user prompts
+        PromptStrategy.MAP: MAP_BASE_USER_PROMPT,
+        PromptStrategy.MAP_NO_REASONING: MAP_NO_REASONING_BASE_USER_PROMPT,
+        PromptStrategy.MAP_CRITIC: BASE_CRITIQUE_PROMPT,
+        PromptStrategy.MAP_REFINE: BASE_REFINEMENT_PROMPT,
+        PromptStrategy.MAP_MOA_PROPOSER: MAP_MOA_PROPOSER_BASE_USER_PROMPT,
+        PromptStrategy.MAP_MOA_AGG: MAP_MOA_AGG_BASE_USER_PROMPT,
+        PromptStrategy.MAP_SPLIT_PROPOSER: MAP_SPLIT_PROPOSER_BASE_USER_PROMPT,
+        PromptStrategy.MAP_SPLIT_MERGER: MAP_SPLIT_MERGER_BASE_USER_PROMPT,
     }
 
     def __init__(self, prompt_strategy: PromptStrategy, model: Model, cardinality: Cardinality, desc: str | None = None) -> None:
@@ -277,6 +245,54 @@ class PromptFactory:
         input_fields = [field for field in input_fields if field in candidate.get_field_names()]
         return input_fields
 
+    def _get_input_modalities(self, candidate: DataRecord, input_fields: list[str]) -> set[Modality]:
+        """
+        The list of input modalities for the given input fields.
+
+        Args:
+            candidate (DataRecord): The input record.
+            input_fields (list[str]): The input fields.
+
+        Returns:
+            set[Modality]: The list of input modalities.
+        """
+        input_modalities = []
+        for field_name in input_fields:
+            field_type = candidate.get_field_type(field_name)
+            if field_type.annotation in IMAGE_FIELD_TYPES:
+                input_modalities.append(Modality.IMAGE)
+            elif field_type.annotation in AUDIO_FIELD_TYPES:
+                input_modalities.append(Modality.AUDIO)
+            else:
+                input_modalities.append(Modality.TEXT)
+
+        return set(input_modalities)
+
+    def _get_modalities_str(self, input_modalities: set[Modality]) -> str:
+        """
+        Returns a format string to reflect the input modalities.
+
+        Args:
+            input_modalities (set[Modality]): The input modalities.
+
+        Returns:
+            str: The string to reflect the input modalities.
+        """
+        if input_modalities == {Modality.TEXT}:
+            return "text"
+        elif input_modalities == {Modality.IMAGE}:
+            return "image(s)"
+        elif input_modalities == {Modality.AUDIO}:
+            return "audio"
+        elif input_modalities == {Modality.TEXT, Modality.IMAGE}:
+            return "text and/or image(s)"
+        elif input_modalities == {Modality.TEXT, Modality.AUDIO}:
+            return "text and/or audio"
+        elif input_modalities == {Modality.IMAGE, Modality.AUDIO}:
+            return "image(s) and/or audio"
+        elif input_modalities == {Modality.TEXT, Modality.IMAGE, Modality.AUDIO}:
+            return "text, image(s), and/or audio"
+
     def _get_input_fields_desc(self, candidate: DataRecord, input_fields: list[str]) -> str:
         """
         Returns a multi-line description of each input field for the prompt.
@@ -305,8 +321,8 @@ class PromptFactory:
             str: The output fields description.
         """
         output_fields_desc = ""
-        output_schema: BaseModel = kwargs.get("output_schema")
-        if self.prompt_strategy.is_convert_prompt():
+        output_schema: type[BaseModel] = kwargs.get("output_schema")
+        if self.prompt_strategy.is_map_prompt():
             assert output_schema is not None, "Output schema must be provided for convert prompts."
 
             for field_name in sorted(output_fields):
@@ -324,7 +340,7 @@ class PromptFactory:
             str | None: The filter condition (if applicable).
         """
         filter_condition = kwargs.get("filter_condition")
-        if self.prompt_strategy.is_bool_prompt():
+        if self.prompt_strategy.is_filter_prompt():
             assert filter_condition is not None, "Filter condition must be provided for filter operations."
 
         return filter_condition
@@ -390,7 +406,8 @@ class PromptFactory:
         if self.prompt_strategy.is_moa_aggregator_prompt():
             model_responses = ""
             for idx, model_response in enumerate(kwargs.get("model_responses")):
-                model_responses += f"MODEL RESPONSE {idx + 1}: {model_response}\n"
+                model_responses += f"MODEL RESPONSE {idx + 1}: {model_response.rstrip()}\n\n"
+        model_responses = model_responses.rstrip() if model_responses is not None else None
 
         return model_responses
 
@@ -408,7 +425,8 @@ class PromptFactory:
         if self.prompt_strategy.is_split_merger_prompt():
             chunk_outputs = ""
             for idx, chunk_output in enumerate(kwargs.get("chunk_outputs")):
-                chunk_outputs += f"CHUNK OUTPUT {idx + 1}: {chunk_output}\n"
+                chunk_outputs += f"CHUNK OUTPUT {idx + 1}: {chunk_output.rstrip()}\n\n"
+        chunk_outputs = chunk_outputs.rstrip() if chunk_outputs is not None else None
 
         return chunk_outputs
 
@@ -425,28 +443,33 @@ class PromptFactory:
             else ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION
         )
 
-    def _get_job_instruction(self) -> str | None:
+    def _get_job_instruction(self, input_modalities: set[Modality]) -> str | None:
         """
         Returns the job instruction based on the prompt strategy.
 
+        Args:
+            input_modalities (set[Modality]): The modalities of the input fields.
+
         Returns:
-            str | None: The job instruction (if applicable).
+            str | None: The job instruction.
         """
-        prompt_strategy_to_job_instruction = {
-            PromptStrategy.COT_BOOL: COT_BOOL_JOB_INSTRUCTION,
-            PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_AUDIO_JOB_INSTRUCTION,
-            PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_IMAGE_JOB_INSTRUCTION,
-            PromptStrategy.COT_JOIN: COT_JOIN_JOB_INSTRUCTION,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_JOB_INSTRUCTION,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_JOB_INSTRUCTION,
-            PromptStrategy.COT_QA: COT_QA_JOB_INSTRUCTION,
-            PromptStrategy.COT_QA_AUDIO: COT_QA_AUDIO_JOB_INSTRUCTION,
-            PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_JOB_INSTRUCTION,
-            PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_JOB_INSTRUCTION,
-            PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_JOB_INSTRUCTION,
-            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_JOB_INSTRUCTION,
-        }
-        return prompt_strategy_to_job_instruction.get(self.prompt_strategy)
+        # get the job instruction based on the prompt strategy
+        job_instruction = None
+        if self.prompt_strategy.is_moa_proposer_prompt() or self.prompt_strategy.is_split_proposer_prompt():
+            job_instruction = PROPOSER_JOB_INSTRUCTION
+        elif self.prompt_strategy.is_map_prompt():
+            job_instruction = MAP_JOB_INSTRUCTION
+        elif self.prompt_strategy.is_filter_prompt():
+            job_instruction = FILTER_JOB_INSTRUCTION
+        elif self.prompt_strategy.is_join_prompt():
+            job_instruction = JOIN_JOB_INSTRUCTION
+
+        # format the job instruction based on the input modalities
+        modalities = self._get_modalities_str(input_modalities)
+        if job_instruction is not None:
+            job_instruction = job_instruction.format(modalities=modalities)
+
+        return job_instruction
 
     def _get_desc_section(self) -> str:
         """
@@ -470,9 +493,7 @@ class PromptFactory:
         """
         critique_criteria = None
         if self.prompt_strategy.is_critic_prompt():
-            critique_criteria = (
-                COT_QA_IMAGE_CRITIQUE_CRITERIA if self.prompt_strategy.is_image_prompt() else COT_QA_CRITIQUE_CRITERIA
-            )
+            critique_criteria = MAP_CRITIQUE_CRITERIA if self.prompt_strategy.is_map_prompt() else FILTER_CRITIQUE_CRITERIA
 
         return critique_criteria
 
@@ -485,11 +506,7 @@ class PromptFactory:
         """
         refinement_criteria = None
         if self.prompt_strategy.is_refine_prompt():
-            refinement_criteria = (
-                COT_QA_IMAGE_REFINEMENT_CRITERIA
-                if self.prompt_strategy.is_image_prompt()
-                else COT_QA_REFINEMENT_CRITERIA
-            )
+            refinement_criteria = MAP_REFINEMENT_CRITERIA if self.prompt_strategy.is_map_prompt() else FILTER_REFINEMENT_CRITERIA
 
         return refinement_criteria
 
@@ -502,240 +519,156 @@ class PromptFactory:
         """
         finish_instruction = None
         if self.prompt_strategy.is_critic_prompt():
-            finish_instruction = COT_QA_CRITIQUE_FINISH_INSTRUCTION
+            finish_instruction = MAP_CRITIQUE_FINISH_INSTRUCTION if self.prompt_strategy.is_map_prompt() else FILTER_CRITIQUE_FINISH_INSTRUCTION
         elif self.prompt_strategy.is_refine_prompt():
-            finish_instruction = COT_QA_REFINEMENT_FINISH_INSTRUCTION
+            finish_instruction = MAP_REFINEMENT_FINISH_INSTRUCTION if self.prompt_strategy.is_map_prompt() else FILTER_REFINEMENT_FINISH_INSTRUCTION
 
         return finish_instruction
 
-    def _get_example_input_fields(self) -> str | None:
+    def _get_example_input_fields(self, input_modalities: set[Modality], right: bool = False) -> str:
         """
         Returns the example input fields for the prompt.
 
-        Returns:
-            str | None: The example input fields (if applicable).
-        """
-        prompt_strategy_to_example_input_fields = {
-            PromptStrategy.COT_BOOL: COT_BOOL_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_AUDIO_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_IMAGE_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_JOIN: COT_JOIN_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_QA: COT_QA_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_QA_AUDIO: COT_QA_AUDIO_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_INPUT_FIELDS,
-        }
-
-        return prompt_strategy_to_example_input_fields.get(self.prompt_strategy)
-
-    def _get_right_example_input_fields(self) -> str | None:
-        """
-        Returns the example right input fields for the join prompt.
+        Args:
+            input_modalities (set[Modality]): The modalities of the input fields.
+            right (bool): Whether to return the right input fields for the join prompt.
 
         Returns:
-            str | None: The example right input fields (if applicable).
+            str: The example input fields.
         """
-        prompt_strategy_to_right_example_input_fields = {
-            PromptStrategy.COT_JOIN: COT_JOIN_RIGHT_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_RIGHT_EXAMPLE_INPUT_FIELDS,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_RIGHT_EXAMPLE_INPUT_FIELDS,
+        input_modality_to_example_input_fields = {
+            Modality.TEXT: RIGHT_TEXT_EXAMPLE_INPUT_FIELDS if right else TEXT_EXAMPLE_INPUT_FIELDS,
+            Modality.IMAGE: RIGHT_IMAGE_EXAMPLE_INPUT_FIELDS if right else IMAGE_EXAMPLE_INPUT_FIELDS,
+            Modality.AUDIO: RIGHT_AUDIO_EXAMPLE_INPUT_FIELDS if right else AUDIO_EXAMPLE_INPUT_FIELDS,
         }
 
-        return prompt_strategy_to_right_example_input_fields.get(self.prompt_strategy)
+        example_input_fields = ""
+        for input_modality in input_modalities:
+            example_input_fields += input_modality_to_example_input_fields[input_modality].rstrip()
+        example_input_fields = example_input_fields.lstrip() + "\n"
+
+        return example_input_fields
 
-    def _get_example_output_fields(self) -> str | None:
+    def _get_example_output_fields(self, input_modalities: set[Modality]) -> str:
         """
         Returns the example output fields for the prompt.
 
         Returns:
-            str | None: The example output fields (if applicable).
+            str: The example output fields.
         """
-        prompt_strategy_to_example_output_fields = {
-            PromptStrategy.COT_QA: COT_QA_EXAMPLE_OUTPUT_FIELDS,
-            PromptStrategy.COT_QA_AUDIO: COT_QA_AUDIO_EXAMPLE_OUTPUT_FIELDS,
-            PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_OUTPUT_FIELDS,
-            PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_OUTPUT_FIELDS,
-            PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_OUTPUT_FIELDS,
-            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_OUTPUT_FIELDS,
+        input_modality_to_example_output_fields = {
+            Modality.TEXT: TEXT_EXAMPLE_OUTPUT_FIELDS,
+            Modality.IMAGE: IMAGE_EXAMPLE_OUTPUT_FIELDS,
+            Modality.AUDIO: AUDIO_EXAMPLE_OUTPUT_FIELDS,
         }
 
-        return prompt_strategy_to_example_output_fields.get(self.prompt_strategy)
+        example_output_fields = ""
+        for input_modality in input_modalities:
+            example_output_fields += input_modality_to_example_output_fields[input_modality].rstrip()
+        example_output_fields = example_output_fields.lstrip() + "\n"
 
-    def _get_example_context(self) -> str | None:
+        return example_output_fields
+
+    def _get_example_context(self, input_modalities: set[Modality], right: bool = False) -> str:
         """
         Returns the example context for the prompt.
 
         Returns:
-            str | None: The example context (if applicable).
+            str: The example context.
         """
-        prompt_strategy_to_example_context = {
-            PromptStrategy.COT_BOOL: COT_BOOL_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_AUDIO_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_IMAGE_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_JOIN: COT_JOIN_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_QA: COT_QA_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_QA_AUDIO: COT_QA_AUDIO_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_CONTEXT,
-            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_CONTEXT,
+        input_modality_to_example_context = {
+            Modality.TEXT: RIGHT_TEXT_EXAMPLE_CONTEXT if right else TEXT_EXAMPLE_CONTEXT,
+            Modality.IMAGE: RIGHT_IMAGE_EXAMPLE_CONTEXT if right else IMAGE_EXAMPLE_CONTEXT,
+            Modality.AUDIO: RIGHT_AUDIO_EXAMPLE_CONTEXT if right else AUDIO_EXAMPLE_CONTEXT,
         }
 
-        return prompt_strategy_to_example_context.get(self.prompt_strategy)
-
-    def _get_right_example_context(self) -> str | None:
-        """
-        Returns the right example context for the join prompt.
-
-        Returns:
-            str | None: The right example context (if applicable).
-        """
-        prompt_strategy_to_right_example_context = {
-            PromptStrategy.COT_JOIN: COT_JOIN_RIGHT_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_RIGHT_EXAMPLE_CONTEXT,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_RIGHT_EXAMPLE_CONTEXT,
-        }
+        example_context = ""
+        for input_modality in input_modalities:
+            example_context += input_modality_to_example_context[input_modality].rstrip() + ","
+        example_context = example_context[:-1] + "\n"
 
-        return prompt_strategy_to_right_example_context.get(self.prompt_strategy)
+        return example_context
 
-    def _get_image_disclaimer(self) -> str:
+    def _get_image_disclaimer(self, input_modalities: set[Modality], right: bool = False) -> str:
         """
         Returns the image disclaimer for the prompt. The disclaimer must be an empty string
-        for text prompts.
+        for non-image prompts.
 
         Returns:
             str: The image disclaimer. If this is a text prompt then it is an empty string.
         """
-        prompt_strategy_to_image_disclaimer = {
-            PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_IMAGE_DISCLAIMER,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_DISCLAIMER,
-            PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_DISCLAIMER,
-            PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_DISCLAIMER,
-        }
-
-        return prompt_strategy_to_image_disclaimer.get(self.prompt_strategy, "")
+        image_disclaimer = RIGHT_IMAGE_DISCLAIMER if right else IMAGE_DISCLAIMER
+        return image_disclaimer if Modality.IMAGE in input_modalities else ""
 
-    def _get_audio_disclaimer(self) -> str:
+    def _get_audio_disclaimer(self, input_modalities: set[Modality], right: bool = False) -> str:
         """
         Returns the audio disclaimer for the prompt. The disclaimer must be an empty string
-        for text prompts.
+        for non-audio prompts.
 
         Returns:
             str: The audio disclaimer. If this is a text prompt then it is an empty string.
         """
-        prompt_strategy_to_audio_disclaimer = {
-            PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_AUDIO_DISCLAIMER,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_DISCLAIMER,
-            PromptStrategy.COT_QA_AUDIO: COT_QA_AUDIO_DISCLAIMER,
-        }
-
-        return prompt_strategy_to_audio_disclaimer.get(self.prompt_strategy, "")
-
-    def _get_right_image_disclaimer(self) -> str:
-        """
-        Returns the right image disclaimer for the prompt. The disclaimer must be an empty string
-        for text prompts.
-
-        Returns:
-            str: The right image disclaimer. If this is a text prompt then it is an empty string.
-        """
-        prompt_strategy_to_image_disclaimer = {
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_RIGHT_IMAGE_DISCLAIMER,
-        }
-
-        return prompt_strategy_to_image_disclaimer.get(self.prompt_strategy, "")
-
-    def _get_right_audio_disclaimer(self) -> str:
-        """
-        Returns the right audio disclaimer for the prompt. The disclaimer must be an empty string
-        for text prompts.
-
-        Returns:
-            str: The right audio disclaimer. If this is a text prompt then it is an empty string.
-        """
-        prompt_strategy_to_audio_disclaimer = {
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_RIGHT_AUDIO_DISCLAIMER,
-        }
+        audio_disclaimer = RIGHT_AUDIO_DISCLAIMER if right else AUDIO_DISCLAIMER
+        return audio_disclaimer if Modality.AUDIO in input_modalities else ""
 
-        return prompt_strategy_to_audio_disclaimer.get(self.prompt_strategy, "")
-
-    def _get_example_filter_condition(self) -> str | None:
+    def _get_example_reasoning(self, input_modalities: set[Modality]) -> str:
         """
-        Returns the example filter condition for the prompt.
+        Returns the example reasoning for the prompt.
 
         Returns:
-            str | None: The example filter condition (if applicable).
-        """
-        prompt_strategy_to_example_filter_condition = {
-            PromptStrategy.COT_BOOL: COT_BOOL_EXAMPLE_FILTER_CONDITION,
-            PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_AUDIO_EXAMPLE_FILTER_CONDITION,
-            PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_IMAGE_EXAMPLE_FILTER_CONDITION,
-        }
-
-        return prompt_strategy_to_example_filter_condition.get(self.prompt_strategy)
-
-    def _get_example_join_condition(self) -> str | None:
+            str: The example reasoning.
         """
-        Returns the example join condition for the prompt.
+        if self.prompt_strategy.is_filter_prompt():
+            return FILTER_EXAMPLE_REASONING
+        elif self.prompt_strategy.is_join_prompt():
+            return JOIN_EXAMPLE_REASONING
 
-        Returns:
-            str | None: The example join condition (if applicable).
-        """
-        prompt_strategy_to_example_join_condition = {
-            PromptStrategy.COT_JOIN: COT_JOIN_EXAMPLE_JOIN_CONDITION,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_EXAMPLE_JOIN_CONDITION,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_EXAMPLE_JOIN_CONDITION,
+        input_modality_to_example_reasoning = {
+            Modality.TEXT: TEXT_EXAMPLE_REASONING,
+            Modality.IMAGE: IMAGE_EXAMPLE_REASONING,
+            Modality.AUDIO: AUDIO_EXAMPLE_REASONING,
         }
 
-        return prompt_strategy_to_example_join_condition.get(self.prompt_strategy)
-
-    def _get_example_reasoning(self) -> str | None:
-        """
-        Returns the example reasoning for the prompt.
-
-        Returns:
-            str | None: The example reasoning (if applicable).
-        """
-        prompt_strategy_to_example_reasoning = {
-            PromptStrategy.COT_BOOL: COT_BOOL_EXAMPLE_REASONING,
-            PromptStrategy.COT_BOOL_AUDIO: COT_BOOL_AUDIO_EXAMPLE_REASONING,
-            PromptStrategy.COT_BOOL_IMAGE: COT_BOOL_IMAGE_EXAMPLE_REASONING,
-            PromptStrategy.COT_JOIN: COT_JOIN_EXAMPLE_REASONING,
-            PromptStrategy.COT_JOIN_AUDIO: COT_JOIN_AUDIO_EXAMPLE_REASONING,
-            PromptStrategy.COT_JOIN_IMAGE: COT_JOIN_IMAGE_EXAMPLE_REASONING,
-            PromptStrategy.COT_QA: COT_QA_EXAMPLE_REASONING,
-            PromptStrategy.COT_QA_AUDIO: COT_QA_AUDIO_EXAMPLE_REASONING,
-            PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_REASONING,
-        }
+        example_reasoning = ""
+        for input_modality in input_modalities:
+            example_reasoning += input_modality_to_example_reasoning[input_modality] + " "
+        example_reasoning = example_reasoning.rstrip()
 
-        return prompt_strategy_to_example_reasoning.get(self.prompt_strategy)
+        return example_reasoning
 
-    def _get_example_answer(self) -> str | None:
+    def _get_example_answer(self, input_modalities: set[Modality]) -> str:
         """
         Returns the example answer for the prompt.
 
         Returns:
-            str | None: The example answer (if applicable).
+            str: The example answer.
         """
-        prompt_strategy_to_example_answer = {
-            PromptStrategy.COT_QA: COT_QA_EXAMPLE_ANSWER,
-            PromptStrategy.COT_QA_AUDIO: COT_QA_AUDIO_EXAMPLE_ANSWER,
-            PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_ANSWER,
-            PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_ANSWER,
-            PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_ANSWER,
-            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_ANSWER,
+        use_sentence_answers = self.prompt_strategy.is_split_proposer_prompt() or self.prompt_strategy.is_moa_proposer_prompt()
+        input_modality_to_example_answer = {
+            Modality.TEXT: TEXT_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else TEXT_EXAMPLE_ANSWER,
+            Modality.IMAGE: IMAGE_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else IMAGE_EXAMPLE_ANSWER,
+            Modality.AUDIO: AUDIO_SENTENCE_EXAMPLE_ANSWER if use_sentence_answers else AUDIO_EXAMPLE_ANSWER,
         }
 
-        return prompt_strategy_to_example_answer.get(self.prompt_strategy)
+        example_answer = ""
+        for input_modality in input_modalities:
+            example_answer += input_modality_to_example_answer[input_modality].rstrip()
+            if use_sentence_answers:
+                example_answer += " "
+        example_answer = example_answer + "\n"
+
+        return example_answer
 
     def _get_all_format_kwargs(
-        self, candidate: DataRecord, input_fields: list[str], output_fields: list[str], right_candidate: DataRecord | None, right_input_fields: list[str], **kwargs
+        self,
+        candidate: DataRecord,
+        input_fields: list[str],
+        input_modalities: set[Modality],
+        output_fields: list[str],
+        right_candidate: DataRecord | None,
+        right_input_fields: list[str],
+        right_input_modalities: set[Modality],
+        **kwargs,
     ) -> dict:
         """
         Returns a dictionary containing all the format kwargs for templating the prompts.
@@ -770,26 +703,27 @@ class PromptFactory:
             })
 
         # get format kwargs which depend on the prompt strategy
+        full_input_modalities = input_modalities.union(right_input_modalities)
         prompt_strategy_format_kwargs = {
             "output_format_instruction": self._get_output_format_instruction(),
-            "job_instruction": self._get_job_instruction(),
+            "job_instruction": self._get_job_instruction(full_input_modalities),
             "desc_section": self._get_desc_section(),
             "critique_criteria": self._get_critique_criteria(),
             "refinement_criteria": self._get_refinement_criteria(),
             "finish_instruction": self._get_finish_instruction(),
-            "example_input_fields": self._get_example_input_fields(),
-            "right_example_input_fields": self._get_right_example_input_fields(),
-            "example_output_fields": self._get_example_output_fields(),
-            "example_context": self._get_example_context(),
-            "right_example_context": self._get_right_example_context(),
-            "image_disclaimer": self._get_image_disclaimer(),
-            "audio_disclaimer": self._get_audio_disclaimer(),
-            "right_image_disclaimer": self._get_right_image_disclaimer(),
-            "right_audio_disclaimer": self._get_right_audio_disclaimer(),
-            "example_filter_condition": self._get_example_filter_condition(),
-            "example_join_condition": self._get_example_join_condition(),
-            "example_reasoning": self._get_example_reasoning(),
-            "example_answer": self._get_example_answer(),
+            "example_input_fields": self._get_example_input_fields(input_modalities),
+            "right_example_input_fields": self._get_example_input_fields(right_input_modalities, right=True),
+            "example_output_fields": self._get_example_output_fields(input_modalities),
+            "example_context": self._get_example_context(input_modalities),
+            "right_example_context": self._get_example_context(right_input_modalities, right=True),
+            "image_disclaimer": self._get_image_disclaimer(input_modalities),
+            "audio_disclaimer": self._get_audio_disclaimer(input_modalities),
+            "right_image_disclaimer": self._get_image_disclaimer(right_input_modalities, right=True),
+            "right_audio_disclaimer": self._get_audio_disclaimer(right_input_modalities, right=True),
+            "example_filter_condition": EXAMPLE_FILTER_CONDITION,
+            "example_join_condition": EXAMPLE_JOIN_CONDITION,
+            "example_reasoning": self._get_example_reasoning(input_modalities),
+            "example_answer": self._get_example_answer(input_modalities),
         }
 
         # return all format kwargs
@@ -937,7 +871,7 @@ class PromptFactory:
         # get any audio messages for the chat payload (will be an empty list if no audio fields exist)
         audio_messages = self._create_audio_messages(candidate, input_fields)
 
-        # get any right image messages for the chat payload (will be an empty list if this is not a join image prompt)
+        # get any right image / audio messages for the chat payload (will be an empty list if image / audio not present)
         right_image_messages, right_audio_messages = [], []
         if self.prompt_strategy.is_join_prompt():
             assert right_candidate is not None, "Right candidate must be provided for join prompts."
@@ -951,121 +885,63 @@ class PromptFactory:
                 "Original messages must be provided for critique and refinement operations."
             )
 
-        # TODO: in the future if we support many modalities (e.g. images and audio) in the same prompt,
-        #       then we will need to streamline this logic to handle the many different cases
+        # combine image and audio messages
+        image_audio_messages = image_messages + audio_messages
+        right_image_audio_messages = right_image_messages + right_audio_messages
+        has_image_audio = len(image_audio_messages) > 0
+        has_right_image_audio = len(right_image_audio_messages) > 0
+
         # construct the user messages based on the prompt strategy
         user_messages = []
         if self.prompt_strategy.is_critic_prompt() or self.prompt_strategy.is_refine_prompt():
-            # NOTE: if this critic / refinement prompt is processing images, those images will
-            #       be part of the `original_messages` and will show up in the final chat payload
+            # NOTE: if this critic / refinement prompt is processing images / audio, those images / audio
+            # will be part of the `original_messages` and will show up in the final chat payload
             base_prompt_start, base_prompt_end = base_prompt.split("<<original-prompt-placeholder>>\n")
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)})
             user_messages.extend(original_messages)
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)})
 
-        # image not join
-        elif self.prompt_strategy.is_image_prompt() and not self.prompt_strategy.is_join_prompt():
-            base_prompt = base_prompt.replace("<<audio-placeholder>>", "")
-            base_prompt_start, base_prompt_end = base_prompt.split("<<image-placeholder>>")
+        # handle joins with left and right images / audio
+        elif self.prompt_strategy.is_join_prompt() and has_image_audio and has_right_image_audio:
+            base_prompt_start, base_prompt_rest = base_prompt.split("<<image-audio-placeholder>>")
+            base_prompt_mid, base_prompt_end = base_prompt_rest.split("<<right-image-audio-placeholder>>")
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)})
-            user_messages.extend(image_messages)
+            user_messages.extend(image_audio_messages)
+            user_messages.append({"role": "user", "type": "text", "content": base_prompt_mid.format(**kwargs)})
+            user_messages.extend(right_image_audio_messages)
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)})
 
-        # image join
-        elif self.prompt_strategy.is_image_prompt() and self.prompt_strategy.is_join_prompt():
-            # for join image prompts, we may have two sets of images (one from the left candidate and one from the right candidate)
-            base_prompt = base_prompt.replace("<<audio-placeholder>>", "")
-            base_prompt_start, base_prompt_mid, base_prompt_end = base_prompt.split("<<image-placeholder>>")
+        # handle joins with only left images / audio
+        elif self.prompt_strategy.is_join_prompt() and has_image_audio and not has_right_image_audio:
+            base_prompt = base_prompt.replace("<<right-image-audio-placeholder>>", "")
+            base_prompt_start, base_prompt_end = base_prompt.split("<<image-audio-placeholder>>")
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)})
-            user_messages.extend(image_messages)
-            user_messages.append({"role": "user", "type": "text", "content": base_prompt_mid.format(**kwargs)})
-            user_messages.extend(right_image_messages)
+            user_messages.extend(image_audio_messages)
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)})
 
-        # audio not join
-        elif self.prompt_strategy.is_audio_prompt() and not self.prompt_strategy.is_join_prompt():
-            base_prompt = base_prompt.replace("<<image-placeholder>>", "")
-            base_prompt_start, base_prompt_end = base_prompt.split("<<audio-placeholder>>")
+        # handle joins with only right images / audio
+        elif self.prompt_strategy.is_join_prompt() and not has_image_audio and has_right_image_audio:
+            base_prompt = base_prompt.replace("<<image-audio-placeholder>>", "")
+            base_prompt_start, base_prompt_end = base_prompt.split("<<right-image-audio-placeholder>>")
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)})
-            user_messages.extend(audio_messages)
+            user_messages.extend(right_image_audio_messages)
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)})
 
-        # audio join
-        elif self.prompt_strategy.is_audio_prompt() and self.prompt_strategy.is_join_prompt():
-            # for join image prompts, we may have two sets of images (one from the left candidate and one from the right candidate)
-            base_prompt = base_prompt.replace("<<image-placeholder>>", "")
-            base_prompt_start, base_prompt_mid, base_prompt_end = base_prompt.split("<<audio-placeholder>>")
+        # handle non-joins with images / audio
+        elif not self.prompt_strategy.is_join_prompt() and has_image_audio and not self.prompt_strategy.is_moa_aggregator_prompt():
+            base_prompt_start, base_prompt_end = base_prompt.split("<<image-audio-placeholder>>")
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_start.format(**kwargs)})
-            user_messages.extend(audio_messages)
-            user_messages.append({"role": "user", "type": "text", "content": base_prompt_mid.format(**kwargs)})
-            user_messages.extend(right_audio_messages)
+            user_messages.extend(image_audio_messages)
             user_messages.append({"role": "user", "type": "text", "content": base_prompt_end.format(**kwargs)})
 
+        # handle prompts w/no images or audio
         else:
-            base_prompt = base_prompt.replace("<<image-placeholder>>", "")
-            base_prompt = base_prompt.replace("<<audio-placeholder>>", "")
+            base_prompt = base_prompt.replace("<<image-audio-placeholder>>", "")
+            base_prompt = base_prompt.replace("<<right-image-audio-placeholder>>", "")
             user_messages.append({"role": "user", "type": "text", "content": base_prompt.format(**kwargs)})
 
         return user_messages
 
-    def _process_custom_user_prompt(self, candidate: DataRecord, input_fields: list[str], **kwargs) -> list[dict]:
-        """
-        Processes a custom user prompt provided by the user.
-
-        Args:
-            candidate (DataRecord): The input record.
-            kwargs: The keyword arguments provided by the user.
-
-        Returns:
-            list[dict]: The messages for the chat payload.
-        """
-        # get the user prompt
-        user_prompt: str = kwargs["prompt"]
-
-        # sanity check that we have all the inputs for the user's prompt template
-        prompt_field_names = [fname for _, fname, _, _ in Formatter().parse(user_prompt) if fname]
-        fields_check = all([field in input_fields for field in prompt_field_names])
-        if not fields_check:
-            if sorted(candidate.get_field_names()) != (input_fields):
-                err_msg = (
-                    f"Prompt string has fields which are not in input fields.\n"
-                    f"Prompt fields: {prompt_field_names}\n"
-                    f"Computed fields: {candidate.get_field_names()}\n"
-                    f"Input fields: {input_fields}\n"
-                    f"Be careful that you are not projecting out computed fields. "
-                    f"If you use `depends_on` in your program, make sure it includes the fields you need."
-                )
-            else:
-                err_msg = (
-                    f"Prompt string has fields which are not in input fields.\n"
-                    f"Prompt fields: {prompt_field_names}\n"
-                    f"Input fields: {input_fields}\n"
-                )
-            assert fields_check, err_msg
-
-        # build set of format kwargs
-        format_kwargs = {
-            field_name: "<bytes>"
-            if candidate.get_field_type(field_name).annotation in [bytes, bytes | None]
-            else candidate[field_name]
-            for field_name in input_fields
-        }
-
-        # split prompt on <<image-placeholder>> if it exists
-        if "<<image-placeholder>>" in user_prompt:
-            raise NotImplementedError("Image prompts are not yet supported.")
-
-        prompt_sections = user_prompt.split("<<image-placeholder>>")
-        messages = [{"role": "user", "type": "text", "content": prompt_sections[0].format(**format_kwargs)}]
-
-        # NOTE: this currently assumes that the user can only provide a single <<image-placeholder>>
-        if len(prompt_sections) > 1:
-            image_messages = self._create_image_messages(candidate, input_fields)
-            messages.extend(image_messages)
-            messages.append({"role": "user", "type": "text", "content": prompt_sections[1].format(**format_kwargs)})
-
-        return messages
-
     def create_messages(self, candidate: DataRecord, output_fields: list[str], right_candidate: DataRecord | None = None, **kwargs) -> list[dict]:
         """
         Creates the messages for the chat payload based on the prompt strategy.
@@ -1090,19 +966,15 @@ class PromptFactory:
         input_fields = self._get_input_fields(candidate, **kwargs)
         right_input_fields = [] if right_candidate is None else self._get_input_fields(right_candidate, **kwargs)
 
-        # if the user provides a prompt, we process that prompt into messages and return them
-        if "prompt" in kwargs:
-            messages = []
-            if "system_prompt" in kwargs:
-                messages.append({"role": "system", "type": "text", "content": kwargs["system_prompt"]})
-            messages.extend(self._process_custom_user_prompt(candidate, input_fields, **kwargs))
-            return messages
+        # use input fields to determine the left / right input modalities
+        input_modalities = self._get_input_modalities(candidate, input_fields)
+        right_input_modalities = set() if right_candidate is None else self._get_input_modalities(right_candidate, right_input_fields)
 
         # initialize messages
         messages = []
 
         # compute the full dictionary of format kwargs and add to kwargs
-        format_kwargs = self._get_all_format_kwargs(candidate, input_fields, output_fields, right_candidate, right_input_fields, **kwargs)
+        format_kwargs = self._get_all_format_kwargs(candidate, input_fields, input_modalities, output_fields, right_candidate, right_input_fields, right_input_modalities, **kwargs)
         kwargs = {**kwargs, **format_kwargs}
 
         # generate system message (if applicable)