palimpzest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

palimpzest/core/elements/records.py

@@ -107,7 +107,7 @@ class DataRecord:
 
 
     def __hash__(self):
-        return hash(self.to_json_str())
+        return hash(self.to_json_str(bytes_to_str=True))
 
 
     def __iter__(self):
@@ -131,6 +131,9 @@ class DataRecord:
             cardinality_idx=self.cardinality_idx,
         )
 
+        # copy the passed_operator attribute
+        new_dr.passed_operator = self.passed_operator
+
         # get the set of fields to copy from the parent record
         copy_field_names = project_cols if project_cols is not None else self.get_field_names()
         copy_field_names = [field.split(".")[-1] for field in copy_field_names]
@@ -255,16 +258,16 @@ class DataRecord:
             for record in records
         ])
 
-    def to_json_str(self, include_bytes: bool = True, project_cols: list[str] | None = None):
+    def to_json_str(self, include_bytes: bool = True, bytes_to_str: bool = False, project_cols: list[str] | None = None):
         """Return a JSON representation of this DataRecord"""
-        record_dict = self.to_dict(include_bytes, project_cols)
+        record_dict = self.to_dict(include_bytes, bytes_to_str, project_cols)
         record_dict = {
             field_name: self.schema.field_to_json(field_name, field_value)
             for field_name, field_value in record_dict.items()
         }
         return json.dumps(record_dict, indent=2)
 
-    def to_dict(self, include_bytes: bool = True, project_cols: list[str] | None = None):
+    def to_dict(self, include_bytes: bool = True, bytes_to_str: bool = False, project_cols: list[str] | None = None):
         """Return a dictionary representation of this DataRecord"""
         # TODO(chjun): In case of numpy types, the json.dumps will fail. Convert to native types.
         # Better ways to handle this.
@@ -276,9 +279,16 @@ class DataRecord:
 
         if not include_bytes:
             for k, v in dct.items():
-                if isinstance(v, bytes) or (isinstance(v, list) and len(v) > 0 and isinstance(v[0], bytes)):
+                if isinstance(v, bytes) or (isinstance(v, list) and len(v) > 0 and  any([isinstance(elt, bytes) for elt in v])):
                     dct[k] = "<bytes>"
 
+        if bytes_to_str:
+            for k, v in dct.items():
+                if isinstance(v, bytes):
+                    dct[k] = v.decode("utf-8")
+                elif isinstance(v, list) and len(v) > 0 and any([isinstance(elt, bytes) for elt in v]):
+                    dct[k] = [elt.decode("utf-8") if isinstance(elt, bytes) else elt for elt in v]
+
         return dct
 
 
@@ -290,7 +300,12 @@ class DataRecordSet:
 
     The record_op_stats could be empty if the DataRecordSet is not from executing an operator.
     """
-    def __init__(self, data_records: list[DataRecord], record_op_stats: list[RecordOpStats]):
+    def __init__(
+            self,
+            data_records: list[DataRecord],
+            record_op_stats: list[RecordOpStats],
+            field_to_score_fn: dict[str, str | callable] | None = None,
+        ):
         # check that all data_records are derived from the same parent record
         if len(data_records) > 0:
             parent_id = data_records[0].parent_id
@@ -302,20 +317,27 @@ class DataRecordSet:
         self.data_records = data_records
         self.parent_id = data_records[0].parent_id if len(data_records) > 0 else None
         self.source_idx = data_records[0].source_idx if len(data_records) > 0 else None
+        self.schema = data_records[0].schema if len(data_records) > 0 else None
 
         # set statistics for generating these records
         self.record_op_stats = record_op_stats
 
+        # assign field_to_score_fn if provided
+        self.field_to_score_fn = {} if field_to_score_fn is None else field_to_score_fn
 
-    def __getitem__(self, slice):
-        return self.data_records[slice]
+    def get_total_cost(self) -> float:
+        return sum([record_op_stats.cost_per_record for record_op_stats in self.record_op_stats])
 
+    def get_field_to_score_fn(self) -> dict[str, str | callable]:
+        return self.field_to_score_fn
 
-    def __len__(self):
-        return len(self.data_records)
+    def __getitem__(self, slice) -> DataRecord | list[DataRecord]:
+        return self.data_records[slice]
 
+    def __len__(self) -> int:
+        return len(self.data_records)
 
-    def __iter__(self):
+    def __iter__(self) -> Generator[DataRecord]:
         yield from self.data_records
 
 

palimpzest/core/lib/fields.py

@@ -135,6 +135,7 @@ class ListField(Field):
             "element_type": element_type,
             "is_image_field": element_type.is_image_field,
             "type": list,
+            "_desc": desc,
         }
 
         return type(f"List[{element_type.__name__}]", (Field,), attrs)

palimpzest/core/lib/schemas.py

@@ -348,6 +348,10 @@ class File(Schema):
     filename = StringField(desc="The UNIX-style name of the file")
     contents = BytesField(desc="The contents of the file")
 
+class TextFile(Schema):
+    """A text file is a File that contains only text. No binary data."""
+    filename = StringField(desc="The UNIX-style name of the file")
+    contents = StringField(desc="The contents of the file")
 
 class Number(Schema):
     """Just a number. Often used for aggregates"""
@@ -418,9 +422,6 @@ class PDFFile(File):
     text_contents = StringField(desc="The text-only contents of the PDF")
 
 
-class TextFile(File):
-    """A text file is a File that contains only text. No binary data."""
-
 list_of_numbers = ListField(NumericField)
 class XLSFile(File):
     """An XLS file is a File that contains one or more Excel spreadsheets."""

palimpzest/prompts/moa_proposer_convert_prompts.py

@@ -7,8 +7,6 @@ Be sure to cite information from the context as evidence of why your answers are
 
 You will be provided with a description of each input field and each output field.
 
-{output_format_instruction} Finish your response with a newline character followed by ---
-
 An example is shown below:
 ---
 INPUT FIELDS:
@@ -31,8 +29,6 @@ You will be presented with a context and a set of output fields to generate. You
 Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.
 
 You will be provided with a description of each input field and each output field.
-
-{output_format_instruction} Finish your response with a newline character followed by ---
 ---
 INPUT FIELDS:
 {input_fields_desc}

palimpzest/prompts/prompt_factory.py

@@ -75,6 +75,19 @@ from palimpzest.prompts.moa_proposer_convert_prompts import (
     COT_MOA_PROPOSER_IMAGE_JOB_INSTRUCTION,
     COT_MOA_PROPOSER_JOB_INSTRUCTION,
 )
+from palimpzest.prompts.split_merge_prompts import (
+    COT_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
+    COT_SPLIT_MERGER_BASE_USER_PROMPT,
+)
+from palimpzest.prompts.split_proposer_prompts import (
+    COT_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
+    COT_SPLIT_PROPOSER_BASE_USER_PROMPT,
+    SPLIT_PROPOSER_EXAMPLE_ANSWER,
+    SPLIT_PROPOSER_EXAMPLE_CONTEXT,
+    SPLIT_PROPOSER_EXAMPLE_INPUT_FIELDS,
+    SPLIT_PROPOSER_EXAMPLE_OUTPUT_FIELDS,
+    SPLIT_PROPOSER_JOB_INSTRUCTION,
+)
 from palimpzest.prompts.util_phrases import (
     ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION,
     ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION,
@@ -96,6 +109,8 @@ class PromptFactory:
         PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
         PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT,
         PromptStrategy.COT_MOA_AGG: COT_MOA_AGG_BASE_SYSTEM_PROMPT,
+        PromptStrategy.SPLIT_PROPOSER: COT_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT,
+        PromptStrategy.SPLIT_MERGER: COT_SPLIT_MERGER_BASE_SYSTEM_PROMPT,
     }
     BASE_USER_PROMPT_MAP = {
         PromptStrategy.COT_BOOL: COT_BOOL_BASE_USER_PROMPT,
@@ -109,6 +124,8 @@ class PromptFactory:
         PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_BASE_USER_PROMPT,
         PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_BASE_USER_PROMPT,
         PromptStrategy.COT_MOA_AGG: COT_MOA_AGG_BASE_USER_PROMPT,
+        PromptStrategy.SPLIT_PROPOSER: COT_SPLIT_PROPOSER_BASE_USER_PROMPT,
+        PromptStrategy.SPLIT_MERGER: COT_SPLIT_MERGER_BASE_USER_PROMPT,
     }
 
     def __init__(self, prompt_strategy: PromptStrategy, model: Model, cardinality: Cardinality) -> None:
@@ -145,7 +162,7 @@ class PromptFactory:
             # NOTE: MIXTRAL_LLAMA_CONTEXT_TOKENS_LIMIT is a rough estimate which leaves room for the rest of the prompt text
             while total_context_len * TOKENS_PER_CHARACTER > MIXTRAL_LLAMA_CONTEXT_TOKENS_LIMIT:
                 # sort fields by length
-                field_lengths = [(field, len(value)) for field, value in context.items()]
+                field_lengths = [(field, len(value) if value is not None else 0) for field, value in context.items()]
                 sorted_fields = sorted(field_lengths, key=lambda item: item[1], reverse=True)
 
                 # get field with longest context
@@ -205,11 +222,7 @@ class PromptFactory:
         """
         output_fields_desc = ""
         output_schema: Schema = kwargs.get("output_schema")
-        if (
-            self.prompt_strategy.is_cot_qa_prompt()
-            or self.prompt_strategy.is_moa_proposer_prompt()
-            or self.prompt_strategy.is_moa_aggregator_prompt()
-        ):
+        if self.prompt_strategy.is_convert_prompt():
             assert output_schema is not None, "Output schema must be provided for convert prompts."
 
             field_desc_map = output_schema.field_desc_map()
@@ -227,7 +240,7 @@ class PromptFactory:
             str | None: The filter condition (if applicable).
         """
         filter_condition = kwargs.get("filter_condition")
-        if self.prompt_strategy.is_cot_bool_prompt():
+        if self.prompt_strategy.is_bool_prompt():
             assert filter_condition is not None, "Filter condition must be provided for filter operations."
 
         return filter_condition
@@ -284,6 +297,24 @@ class PromptFactory:
 
         return model_responses
 
+    def _get_chunk_outputs(self, **kwargs) -> str | None:
+        """
+        Returns the chunk outputs for the split-convert.
+
+        Args:
+            kwargs: The keyword arguments provided by the user.
+
+        Returns:
+            str | None: The chunk outputs.
+        """
+        chunk_outputs = None
+        if self.prompt_strategy.is_split_merger_prompt():
+            chunk_outputs = ""
+            for idx, chunk_output in enumerate(kwargs.get("chunk_outputs")):
+                chunk_outputs += f"CHUNK OUTPUT {idx + 1}: {chunk_output}\n"
+
+        return chunk_outputs
+
     def _get_output_format_instruction(self) -> str:
         """
         Returns the output format instruction based on the cardinality.
@@ -311,6 +342,7 @@ class PromptFactory:
             PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_JOB_INSTRUCTION,
             PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_JOB_INSTRUCTION,
             PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_JOB_INSTRUCTION,
+            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_JOB_INSTRUCTION,
         }
         return prompt_strategy_to_job_instruction.get(self.prompt_strategy)
 
@@ -375,6 +407,7 @@ class PromptFactory:
             PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_INPUT_FIELDS,
             PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_INPUT_FIELDS,
             PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_INPUT_FIELDS,
+            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_INPUT_FIELDS,
         }
 
         return prompt_strategy_to_example_input_fields.get(self.prompt_strategy)
@@ -391,6 +424,7 @@ class PromptFactory:
             PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_OUTPUT_FIELDS,
             PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_OUTPUT_FIELDS,
             PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_OUTPUT_FIELDS,
+            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_OUTPUT_FIELDS,
         }
 
         return prompt_strategy_to_example_output_fields.get(self.prompt_strategy)
@@ -409,6 +443,7 @@ class PromptFactory:
             PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_CONTEXT,
             PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_CONTEXT,
             PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_CONTEXT,
+            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_CONTEXT,
         }
 
         return prompt_strategy_to_example_context.get(self.prompt_strategy)
@@ -471,6 +506,7 @@ class PromptFactory:
             PromptStrategy.COT_QA_IMAGE: COT_QA_IMAGE_EXAMPLE_ANSWER,
             PromptStrategy.COT_MOA_PROPOSER: COT_MOA_PROPOSER_EXAMPLE_ANSWER,
             PromptStrategy.COT_MOA_PROPOSER_IMAGE: COT_MOA_PROPOSER_IMAGE_EXAMPLE_ANSWER,
+            PromptStrategy.SPLIT_PROPOSER: SPLIT_PROPOSER_EXAMPLE_ANSWER,
         }
 
         return prompt_strategy_to_example_answer.get(self.prompt_strategy)
@@ -499,6 +535,7 @@ class PromptFactory:
             "original_output": self._get_original_output(**kwargs),
             "critique_output": self._get_critique_output(**kwargs),
             "model_responses": self._get_model_responses(**kwargs),
+            "chunk_outputs": self._get_chunk_outputs(**kwargs),
         }
 
         # get format kwargs which depend on the prompt strategy

palimpzest/prompts/split_merge_prompts.py

@@ -0,0 +1,56 @@
+"""This file contains prompts for SplitConvert aggregator operations."""
+
+### SYSTEM PROMPTS ###
+COT_SPLIT_MERGER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to generate a JSON object.
+You will be presented with one or more outputs produced by a set of models operating on chunks of an input. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values.
+It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates.
+
+You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses.
+
+{output_format_instruction} Finish your response with a newline character followed by ---
+
+An example is shown below:
+---
+CHUNK 1 OUTPUT: the text mentions the scientists "Augusta Ada King, Countess of Lovelace" and "Charles Babbage". It states that King was an English mathematician who worked on Babbage's Analytical Engine.
+
+CHUNK 2 OUTPUT: the text passage mentions the scientist "Charles Babbage", who was a mathematician. Therefore, the name output should be ["Charles Babbage"] and the field_of_study output should be ["Mathematician"].
+
+INPUT FIELDS:
+- text: a text passage describing scientists
+
+OUTPUT FIELDS:
+- name: the list of names for each scientist mentioned in the text
+- field_of_study: a list with the field of study for each scientist
+
+Let's think step-by-step in order to answer the question.
+
+REASONING: Looking at both chunk outputs, they specify that the scientists' formal names are "Augusta Ada King" and "Charles Babbage". Chunk Output 2 indicates that Charles Babbage was a Mathematician and Chunk Output 1 says that Augusta Ada King was an English mathematician. Therefore, the name output should be ["Augusta Ada King", "Charles Babbage"] and the field_of_study output should be ["Mathematician", "Mathematician"].
+
+ANSWER:
+{{
+  "name": ["Augusta Ada King", "Charles Babbage"],
+  "field_of_study": ["Mathematician", "Mathematician"]
+}}
+---
+"""
+
+### USER / INSTANCE-SPECIFIC PROMPTS ###
+COT_SPLIT_MERGER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to generate a JSON object.
+You will be presented with one or more outputs produced by a set of models. Your task is to synthesize these responses into a single, high-quality JSON object which fills in the output fields with the correct values.
+It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased, incorrect, or contain duplicates.
+
+You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the model responses.
+
+{output_format_instruction} Finish your response with a newline character followed by ---
+---
+{chunk_outputs}
+
+INPUT FIELDS:
+{input_fields_desc}
+
+OUTPUT FIELDS:
+{output_fields_desc}
+
+Let's think step-by-step in order to answer the question.
+
+REASONING: """

palimpzest/prompts/split_proposer_prompts.py

@@ -0,0 +1,55 @@
+"""This file contains prompts for SplitConvert operations on text inputs."""
+
+### BASE PROMPTS ###
+COT_SPLIT_PROPOSER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}.
+You will be presented with a context and a set of output fields to generate. Your task is to generate a paragraph or two which describes what you believe is the correct value for each output field.
+Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.
+
+You will be provided with a description of each input field and each output field.
+
+An example is shown below:
+---
+INPUT FIELDS:
+{example_input_fields}
+
+OUTPUT FIELDS:
+{example_output_fields}
+
+CONTEXT:
+{example_context}
+
+Let's think step-by-step in order to answer the question.
+
+ANSWER: {example_answer}
+---
+"""
+
+COT_SPLIT_PROPOSER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}.
+You will be presented with a context and a set of output fields to generate. Your task is to generate a paragraph or two which describes what you believe is the correct value for each output field.
+Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.
+
+You will be provided with a description of each input field and each output field.
+---
+INPUT FIELDS:
+{input_fields_desc}
+
+OUTPUT FIELDS:
+{output_fields_desc}
+
+CONTEXT:
+{context}
+
+Let's think step-by-step in order to answer the question.
+
+ANSWER: """
+
+
+### TEMPLATE INPUTS ###
+SPLIT_PROPOSER_JOB_INSTRUCTION = "produce an answer to a question"
+SPLIT_PROPOSER_EXAMPLE_INPUT_FIELDS = """- text: a text passage describing scientists"""
+SPLIT_PROPOSER_EXAMPLE_OUTPUT_FIELDS = """- name: the list of names for each scientist mentioned in the text
+- field_of_study: a list with the field of study for each scientist"""
+SPLIT_PROPOSER_EXAMPLE_CONTEXT = """{{
+  "text": "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, born December 10, 1815 was an English mathematician and writer chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation."
+}}"""
+SPLIT_PROPOSER_EXAMPLE_ANSWER = """the text passage mentions the scientists "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace" and "Charles Babbage", both of whom were mathematicians. Therefore, the name output should be ["Augusta Ada King", "Charles Babbage"] and the field_of_study output should be ["Mathematician", "Mathematician"]."""