palimpzest 0.8.1__py3-none-any.whl → 0.8.3__py3-none-any.whl

palimpzest/validator/validator.py

@@ -79,8 +79,7 @@ class Validator:
         Compute the quality of the generated output for the given fields and input_record.
         """
         # create prompt factory
-        prompt_strategy = PromptStrategy.COT_QA_IMAGE if op.is_image_conversion() else PromptStrategy.COT_QA
-        factory = PromptFactory(prompt_strategy, Model.o4_MINI, Cardinality.ONE_TO_ONE) # TODO: switch to o4_MINI after merging in dev
+        factory = PromptFactory(PromptStrategy.MAP, Model.o4_MINI, Cardinality.ONE_TO_ONE)
 
         # get the input messages; strip out the system message(s)
         msg_kwargs = {"output_schema": op.output_schema, "project_cols": op.get_input_fields()}
@@ -94,7 +93,7 @@ class Validator:
         score, gen_stats = None, GenerationStats()
         try:
             start_time = time.time()
-            validator_prompt = MAP_IMAGE_VALIDATOR_PROMPT if op.is_image_conversion() else MAP_VALIDATOR_PROMPT
+            validator_prompt = MAP_IMAGE_VALIDATOR_PROMPT if op.is_image_op() else MAP_VALIDATOR_PROMPT
             val_messages = [{"role": "system", "content": validator_prompt}] + input_messages + [{"role": "user", "content": output_message}]
             completion = litellm.completion(model="openai/o4-mini", messages=val_messages)
             completion_text = completion.choices[0].message.content
@@ -116,8 +115,7 @@ class Validator:
         Compute the quality for each record_op_stats object in the given record_set.
         """
         # create prompt factory
-        prompt_strategy = PromptStrategy.COT_QA_IMAGE if op.is_image_conversion() else PromptStrategy.COT_QA
-        factory = PromptFactory(prompt_strategy, Model.o4_MINI, Cardinality.ONE_TO_MANY) # TODO: switch to o4_MINI after merging in dev
+        factory = PromptFactory(PromptStrategy.MAP, Model.o4_MINI, Cardinality.ONE_TO_MANY)
 
         # get the input messages; strip out the system message(s)
         msg_kwargs = {"output_schema": op.output_schema, "project_cols": op.get_input_fields()}
@@ -131,7 +129,7 @@ class Validator:
         score, gen_stats = None, GenerationStats()
         try:
             start_time = time.time()
-            validator_prompt = FLAT_MAP_IMAGE_VALIDATOR_PROMPT if op.is_image_conversion() else FLAT_MAP_VALIDATOR_PROMPT
+            validator_prompt = FLAT_MAP_IMAGE_VALIDATOR_PROMPT if op.is_image_op() else FLAT_MAP_VALIDATOR_PROMPT
             val_messages = [{"role": "system", "content": validator_prompt}] + input_messages + [{"role": "user", "content": output_message}]
             completion = litellm.completion(model="openai/o4-mini", messages=val_messages)
             completion_text = completion.choices[0].message.content
@@ -163,7 +161,7 @@ class Validator:
             validator_op.model = Model.o4_MINI
             try:
                 target_record_set = validator_op(input_record)
-                label = target_record_set[0].passed_operator
+                label = target_record_set[0]._passed_operator
                 self.filter_cache[filter_input_hash] = label
                 score = label == output
                 record_op_stats = target_record_set.record_op_stats[0]
@@ -196,7 +194,7 @@ class Validator:
             validator_op.model = Model.o4_MINI
             try:
                 target_record_set = validator_op([left_input_record], [right_input_record])
-                label = target_record_set[0].passed_operator
+                label = target_record_set[0]._passed_operator
                 self.join_cache[join_input_hash] = label
                 score = label == output
                 record_op_stats = target_record_set.record_op_stats[0]
@@ -227,7 +225,7 @@ class Validator:
         # TODO: retrieve k=25; score each item based on relevance; compute F1
         # TODO: support retrieval over images
         # create prompt factory
-        factory = PromptFactory(PromptStrategy.COT_QA, Model.o4_MINI, Cardinality.ONE_TO_ONE) # TODO: switch to o4_MINI after merging in dev
+        factory = PromptFactory(PromptStrategy.MAP, Model.o4_MINI, Cardinality.ONE_TO_ONE)
 
         # get the input messages; strip out the system message(s)
         msg_kwargs = {"output_schema": op.output_schema, "project_cols": op.get_input_fields()}

{palimpzest-0.8.1.dist-info → palimpzest-0.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: palimpzest
-Version: 0.8.1
+Version: 0.8.3
 Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
 Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
 Project-URL: homepage, https://palimpzest.org
@@ -12,7 +12,7 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: anthropic>=0.55.0
@@ -22,7 +22,7 @@ Requires-Dist: colorama>=0.4.6
 Requires-Dist: datasets>=4.0.0
 Requires-Dist: fastapi~=0.115.0
 Requires-Dist: gradio>=5.26.0
-Requires-Dist: litellm>=1.73.1
+Requires-Dist: litellm>=1.76.1
 Requires-Dist: numpy==2.0.2
 Requires-Dist: openai>=1.0
 Requires-Dist: pandas>=2.1.1
@@ -44,11 +44,6 @@ Requires-Dist: tabulate>=0.9.0
 Requires-Dist: together>=1.5.5
 Requires-Dist: tqdm~=4.66.1
 Requires-Dist: rich[jupyter]>=13.9.2
-Provides-Extra: docs
-Requires-Dist: mkdocs>=1.6.1; extra == "docs"
-Requires-Dist: mkdocs-material>=9.6.3; extra == "docs"
-Requires-Dist: mkdocstrings-python>=1.15.0; extra == "docs"
-Requires-Dist: mkdocs-material[imaging]; extra == "docs"
 Provides-Extra: vllm
 Requires-Dist: vllm>=0.10.1.1; extra == "vllm"
 Dynamic: license-file

palimpzest-0.8.3.dist-info/RECORD

@@ -0,0 +1,95 @@
+palimpzest/__init__.py,sha256=1PzadDDOVMQJKNEYUH0_tw8tQKUYTT31M0vuzTr2Rqk,1694
+palimpzest/constants.py,sha256=7WO9eJEBgERN3XkheS1lsIzrB3ljhT4jRqtRlQa6_lA,23095
+palimpzest/policy.py,sha256=lIvw_C_rmwCH4LZaeNkAuixl8zw9RAW_JcSWSHPjKyc,11628
+palimpzest/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/agents/compute_agents.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/agents/search_agents.py,sha256=t2QMreB5Ph71aoNk5bBtV-0l8im79z-pMAR3JDAySDw,29418
+palimpzest/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/core/models.py,sha256=VNi49i9xn_FxekyYrGPS1-_C_PaGXL8dz-dqjrIOk8g,42477
+palimpzest/core/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/core/data/context.py,sha256=x1xYyu9qW65dvtK_XayIfv_CgsCEPW6Qe0DTiSf9sjU,16207
+palimpzest/core/data/context_manager.py,sha256=8hAKWD2jhFZgghTu7AYgjkvKDsJUPVxq8g4nG0HWvfo,6150
+palimpzest/core/data/dataset.py,sha256=M7SxPXzHsfj-ljy_P3ckcJNqGf4RwNxtZI02q_tmL2M,28178
+palimpzest/core/data/index_dataset.py,sha256=adO67DgzHhA4lBME0-h4SjXfdz9UcNMSDGXTpUdKbgE,1929
+palimpzest/core/data/iter_dataset.py,sha256=K47ajOXsCZV3WhOuDkw3xfiHzn8mXPU976uN3SjaP2U,20507
+palimpzest/core/elements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/core/elements/filters.py,sha256=fU2x0eWDwfP52_5fUmqJXTuhs4H0vvHtPZLdA3IIw8I,1642
+palimpzest/core/elements/groupbysig.py,sha256=oFH5UkZzcR0msAgfQiRQOOvyJ3HaW4Dwr03h7tVOcrM,2324
+palimpzest/core/elements/records.py,sha256=KNY55cN9OuV9Q6apUaUq9W_WyfNremqFlQjClyWR1PU,18518
+palimpzest/core/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/core/lib/schemas.py,sha256=eFH_Lw3UoXLPk_-5Pds5NPRUXKhILPtCkErB26FyKMo,8983
+palimpzest/prompts/__init__.py,sha256=942kdENfPU5mFjIxYm-FusL0FD6LNhoj6cYoSGiUsCI,1628
+palimpzest/prompts/agent_prompts.py,sha256=CUzBVLBiPSw8OShtKp4VTpQwtrNMtcMglo-IZHMvuDM,17459
+palimpzest/prompts/context_search.py,sha256=s3pti4XNRiIyiWzjVNL_NqmqEc31jzSKMF2SlN0Aaf8,357
+palimpzest/prompts/convert_prompts.py,sha256=KJ6x9MqM63mAPT61v3GFFLdRZB-NKSZYolWXszlhDx8,3122
+palimpzest/prompts/critique_and_refine_prompts.py,sha256=hFApYb3e5NdyL5DhX2haxA0cCyhp1WUJsegQfkeg1zM,2857
+palimpzest/prompts/filter_prompts.py,sha256=D-aY3-th1GzEHrVGbKORVN2R7x7coYGjp8FrEN_1UKs,2338
+palimpzest/prompts/join_prompts.py,sha256=z-y4L1cw1O3I_F9DW6MvqeztdQoKDQawX6nK6vQAkdM,2916
+palimpzest/prompts/moa_aggregator_prompts.py,sha256=b5cz4G2oF86LlHOy8vmtxoMcZ9zaZoppKrURHgzCzNU,5248
+palimpzest/prompts/moa_proposer_prompts.py,sha256=yfZYwmCg-Tg9h0H7PJMEuDYPR45EbYnORmVX6cY2vRQ,3125
+palimpzest/prompts/prompt_factory.py,sha256=32GFAfvWOwRHUsAMRDPEiFzaObRK8FeVfGgkG-QKcYs,44187
+palimpzest/prompts/split_merge_prompts.py,sha256=hX-MThmW4VU7rjgm7gb-bpniEMdj25mtp0o8qBeWvIQ,5573
+palimpzest/prompts/split_proposer_prompts.py,sha256=Ucqwfn4FqFk-b9E024EK4e_3_QndTJjggwiwa1x5CQs,3115
+palimpzest/prompts/utils.py,sha256=iFv4nuFRuON-DEAdO2JI-J84ukV8Ev27YYWPLwfk44A,5655
+palimpzest/prompts/validator.py,sha256=pJTZjlt_OiFM3IFOgsJ0jQdayra8iRVrpqENlXI9tQQ,10532
+palimpzest/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/execution/all_sample_execution_strategy.py,sha256=8a8-eKsndo_edCwIamNgcISLQmTzVSv5vmD6Ogl8a6k,14367
+palimpzest/query/execution/execution_strategy.py,sha256=Lg2anGM8I4WBetqduWwnPFE67sfihHJwsu3fQ_sYYJk,18909
+palimpzest/query/execution/execution_strategy_type.py,sha256=vRQBPCQN5_aoyD3TLIeW3VPo15mqF-5RBvEXkENz9FE,987
+palimpzest/query/execution/mab_execution_strategy.py,sha256=LKAi1RWGsIK7ppjWmH8sbAbHjyLA12y4tBn1ycOt35Q,43554
+palimpzest/query/execution/parallel_execution_strategy.py,sha256=roZZy7wLcmAwm_ecYvqSJanRaiox3OoNPuXxvRZ5TXg,15710
+palimpzest/query/execution/single_threaded_execution_strategy.py,sha256=sESji79ytKxth9Tpm02c34Mltw0YiFn4GL5h0MI5Noo,16255
+palimpzest/query/generators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/generators/generators.py,sha256=UldCUEwaiBfpvQDieA-h7SiC8KM76gCABPj-mvqAJus,21500
+palimpzest/query/operators/__init__.py,sha256=yjOdbx3Sm5uVTR5R7fw66F602qBBkguH1ktv48T_JJ8,4264
+palimpzest/query/operators/aggregate.py,sha256=NZ_rmi0YrbNFivbBgKtonrCrK6fZw4h9Pm4lMMI5XVc,11376
+palimpzest/query/operators/compute.py,sha256=X_pWN45smg8L4dV54nOae7dldQGL1nJVlVyJ3ULWSmI,8432
+palimpzest/query/operators/convert.py,sha256=VfrWUFyuZC8fPf7LR7mMfpOjqSfxAuTLUxw-S-pn7hk,16123
+palimpzest/query/operators/critique_and_refine.py,sha256=Q-NhasVoD9meX7g36RPrv3q4R48_8XEU4d3TE46hRJI,8979
+palimpzest/query/operators/distinct.py,sha256=ZTXlIS7IaFRTsWv9RemzCo1JLz25vEma-TB42CV5fJQ,2614
+palimpzest/query/operators/filter.py,sha256=ufREsO2-8CBk4u4fabDBYpEvb806E11EOyW-wuRs4vw,10356
+palimpzest/query/operators/join.py,sha256=79pdA4gEgyXcWirkIRYdWNljrKjDvjtApGXKsroYElA,25591
+palimpzest/query/operators/limit.py,sha256=pdo7WfWY97SW3c-WqZ4SIPw7lHIVbaXPEWqHyK8qkF8,2130
+palimpzest/query/operators/logical.py,sha256=K_dRlNKkda35kQ7gYGsrW9PoFuDPzexpjtDq_FYdhVw,20223
+palimpzest/query/operators/mixture_of_agents.py,sha256=TWdg6XEg2u4TQM4d94gmbYqnK15wC7Q4Cyefp8SA4i8,11547
+palimpzest/query/operators/physical.py,sha256=T32uwehr5t0LXB9wcXZ3Sm41zthmyz86NdRQ9rsqLrE,9505
+palimpzest/query/operators/project.py,sha256=gxbjsHEACCP9wxATH-mw6wOSUi5s13JyWsLqqhAYWXQ,2111
+palimpzest/query/operators/rag.py,sha256=k2vdthQBNb7FEYX7K1UP9fSQfwK09HNwVJu72ExSsCA,20045
+palimpzest/query/operators/retrieve.py,sha256=-OvEWmxwbepGz0w40FpHbqcOHZQ4Bp-MdXFbYTmxdVY,13082
+palimpzest/query/operators/scan.py,sha256=OqCiPRTvTY7SbauNMyFvGT5nRVeRzVsGYSrkoN1Ib_w,7407
+palimpzest/query/operators/search.py,sha256=cQin-Qc9FT7V0Gv3-pxMLbVMjqE6ALe99V0OrQhA6CI,22711
+palimpzest/query/operators/split.py,sha256=oLzwnYb8TNf3XA9TMKEAIw7EIA12wHneaD42BNLIHiI,15043
+palimpzest/query/optimizer/__init__.py,sha256=XsWk_qDh4lvPGhQgsya6D3ZWWHk6UXRUEzHuPHDifx8,2418
+palimpzest/query/optimizer/cost_model.py,sha256=OldPy-TJdfsQbYRoKlb3yWeKbi15jcldTIUS6BTi9T8,12678
+palimpzest/query/optimizer/optimizer.py,sha256=bsH4RhdDncbrGLA9Pq6kZ5CoqjetQm_5Vyl7l48jKpQ,19578
+palimpzest/query/optimizer/optimizer_strategy.py,sha256=9YlNGkqwgX0WaV6y8tKOOHVN8kC8GjDI3DttvGW5SYY,10206
+palimpzest/query/optimizer/optimizer_strategy_type.py,sha256=V-MMHvJdnfZKoUX1xxxwh66q1RjN2FL35IsiT1C62c8,1084
+palimpzest/query/optimizer/plan.py,sha256=VIhN7tWT7EoRE9BKYa1qvvOhX7dEaM-aiobByX0qjzg,22900
+palimpzest/query/optimizer/primitives.py,sha256=jMMVq37y1tWiPU1lSSKQP9OP-mzkpSxSmUeDajRYYOQ,5445
+palimpzest/query/optimizer/rules.py,sha256=rVWIsygEMQmT1_rdHUzsOXC1GYCzLDhiuGjwkszyl-Y,48591
+palimpzest/query/optimizer/tasks.py,sha256=SAoErqxHwoE7nIhLwyH-2YN2nHYG3gR4m4NV7p0U7AE,30028
+palimpzest/query/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/query/processor/config.py,sha256=vHVsgeBnKigacO0QA7bLf5q8pJhFWA2j9-p_no2bmYo,2366
+palimpzest/query/processor/query_processor.py,sha256=T4ffPbnOX23G8FDITzmM7Iw7DUEDWIHnwl8XLYllgjg,6240
+palimpzest/query/processor/query_processor_factory.py,sha256=6w9R1Y8AOV22X8MUf7g2G5Qb15BGEZAXQKbCQJafWJ0,8048
+palimpzest/schemabuilder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/schemabuilder/schema_builder.py,sha256=QraGp66dcD-ej6Y2mER40o86G9JqlBkL7swkJzjUAIY,7968
+palimpzest/tools/README.md,sha256=56_6LPG80uc0CLVhTBP6I1wgIffNv9cyTr0TmVZqmrM,483
+palimpzest/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/tools/allenpdf.py,sha256=fXMOmSDdSSLXDKAPYYJ8k4egtWEBf_Me9Lq9tM3iyoA,1690
+palimpzest/tools/pdfparser.py,sha256=0DOVUZLxYfqjxM8WNEfYcyiXb1qW9BWVIHEB_B_YhWA,9570
+palimpzest/tools/skema_tools.py,sha256=HXUFpjMhbVxZwKKkATeK-FwtlTCawaCbeP-uHntI1Kg,669
+palimpzest/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/utils/env_helpers.py,sha256=n81KzoJ459pRxo7QmJA7duazwWsfoMGTHc71D2LatFk,334
+palimpzest/utils/hash_helpers.py,sha256=3A8dA7SbXTwnnvZvPVNqqMLlVRhCKyKF_bjNNAu3Exk,334
+palimpzest/utils/model_helpers.py,sha256=X6SlMgD5I5Aj_cxaFaoGaaNvOOqTNZVmjj6zbfn63Yk,2476
+palimpzest/utils/progress.py,sha256=7gucyZr82udMDZitrrkAOSKHZVljE3R2wv9nf5gA5TM,20807
+palimpzest/utils/udfs.py,sha256=LjHic54B1az-rKgNLur0wOpaz2ko_UodjLEJrazkxvY,1854
+palimpzest/validator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+palimpzest/validator/validator.py,sha256=dbqpSnis-3u6fpVmRMNwBlx5owwyAXH-sktN-eFoZVU,15995
+palimpzest-0.8.3.dist-info/licenses/LICENSE,sha256=5GUlHy9lr-Py9kvV38FF1m3yy3NqM18fefuE9wkWumo,1079
+palimpzest-0.8.3.dist-info/METADATA,sha256=ncTqIyyVX_1nCCF2X4h3JqSAWJ5CtWcvUYIvywiI8ZE,7048
+palimpzest-0.8.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+palimpzest-0.8.3.dist-info/top_level.txt,sha256=raV06dJUgohefUn3ZyJS2uqp_Y76EOLA9Y2e_fxt8Ew,11
+palimpzest-0.8.3.dist-info/RECORD,,

palimpzest/prompts/critique_and_refine_convert_prompts.py

@@ -1,216 +0,0 @@
-"""This file contains prompts for CriticAndRefineConvert operations."""
-
-### CRITIQUE PROMPT AND CRITERIA ###
-BASE_CRITIQUE_PROMPT = """You are a helpful assistant tasked with critiquing the output of a model based on a given prompt.
-Below is the original user prompt used to generate the output:
-
-ORIGINAL PROMPT:
-<<original-prompt-placeholder>>
-
-Here is the output generated by the model:
-
-OUTPUT:
-{original_output}
-
-Your task is to critique the output based on the following:
-{critique_criteria}
-
-{finish_instruction}
-"""
-
-# BASE_CRITIQUE_PROMPT = """You are a helpful assistant tasked with critiquing the output of a model based on a given prompt.
-# Below is the original user prompt used to generate the output:
-
-# ORIGINAL PROMPT:
-# {user_prompt}
-
-# Here is the output generated by the model:
-
-# OUTPUT:
-# {original_output}
-
-# Your task is to critique the output based on the following:
-# {critique_criteria}
-
-# {finish_instruction}
-# """
-
-# COT_BOOL_CRITIQUE_CRITERIA = """1. Does the output adhere to the required TRUE or FALSE format?
-# 2. Is the reasoning provided logically sound and well-supported by the context?
-# 3. Are there any errors in applying the filter condition to the given context?
-# """
-
-# COT_BOOL_IMAGE_CRITIQUE_CRITERIA = """1. Does the output adhere to the required TRUE or FALSE format?
-# 2. Is the reasoning provided logically sound and well-supported by the image(s) and/or text?
-# 3. Are there any errors in applying the filter condition to the given context?
-# """
-
-COT_QA_CRITIQUE_CRITERIA = """1. Does the JSON object adhere to the required format? Highlight any structural issues.
-2. Are the values of the output fields accurate based on the provided context? If any fields are incorrect or missing, provide specific examples.
-3. Are there any logical errors in reasoning used to derive the output? Provide detailed feedback on potential mistakes.
-"""
-
-COT_QA_IMAGE_CRITIQUE_CRITERIA = """1. Does the JSON object adhere to the required format?
-2. Are the values of the output fields accurate based on the provided image(s) and/or text?
-3. Are there any logical errors in the model's reasoning when extracting information from images and text?
-"""
-
-# COT_BO0L_CRITIQUE_FINISH_INSTRUCTION = """Finish your critique with actionable recommendations for improving the model's reasoning and answer format."""
-COT_QA_CRITIQUE_FINISH_INSTRUCTION = """Finish your critique with actionable recommendations for improving the JSON object."""
-
-
-### REFINEMENT PROMPT AND CRITERIA ###
-BASE_REFINEMENT_PROMPT = """You are a helpful assistant tasked with refining the output of a model based on a critique.
-Below is the original user prompt used to generate the output:
-
-ORIGINAL PROMPT:
-<<original-prompt-placeholder>>
-
-Here is the original output generated by the model:
-
-OUTPUT:
-{original_output}
-
-Here is the critique of the output:
-
-CRITIQUE:
-{critique_output}
-
-Your task is to refine the original output to address the critique. Ensure that:
-{refinement_criteria}
-
-{finish_instruction}
-"""
-
-# BASE_REFINEMENT_PROMPT = """You are a helpful assistant tasked with refining the output of a model based on a critique.
-# Below is the original user prompt used to generate the output:
-
-# ORIGINAL PROMPT:
-# {user_prompt}
-
-# Here is the output generated by the model:
-
-# OUTPUT:
-# {original_output}
-
-# Here is the critique of the output:
-
-# CRITIQUE:
-# {critique_output}
-
-# Your task is to refine the original output to address the critique. Ensure that:
-# {refinement_criteria}
-
-# {finish_instruction}
-# """
-
-# COT_BOOL_REFINEMENT_CRITERIA = """1. The answer adheres to the required TRUE or FALSE format.
-# 2. The reasoning is logically sound and supported by the given context.
-# 3. The filter condition is correctly applied.
-# """
-
-# COT_BOOL_IMAGE_REFINEMENT_CRITERIA = """1. The answer adheres to the required TRUE or FALSE format.
-# 2. The reasoning correctly considers the image(s) and/or text provided in the context.
-# 3. The filter condition is properly applied.
-# """
-
-COT_QA_REFINEMENT_CRITERIA = """1. The answer adheres to the required JSON format specified in the original prompt.
-2. Correctly derives all values for the output fields based on the provided context.
-3. Resolves any logical errors identified in the critique.
-"""
-
-COT_QA_IMAGE_REFINEMENT_CRITERIA = """1. The answer adheres to the required JSON format specified in the original prompt.
-2. Correctly derives all values for the output fields based on the provided image(s) and/or text.
-3. Resolves any logical errors identified in the critique.
-"""
-
-# COT_BO0L_REFINEMENT_FINISH_INSTRUCTION = """Return the improved answer."""
-COT_QA_REFINEMENT_FINISH_INSTRUCTION = """Return the refined JSON object as your final answer."""
-
-
-### PROMPTS FOR MOA + CRITIC; WHICH ARE BEING DEFERRED FOR NOW ###
-# COT_MOA_AGG_BASE_SYSTEM_PROMPT_CRITIQUE = """You are a helpful assistant tasked with reviewing the output of a model based on a given prompt.
-# Below is the original user prompt used to generate the output:
-
-# USER PROMPT:
-# {user_prompt}
-
-# Here is the synthesized JSON object generated by the model:
-
-# OUTPUT:
-# {original_output}
-
-# Your task is to critique the output based on the following:
-# 1. Does the JSON object adhere to the required format?
-# 2. Does the synthesis appropriately combine responses from multiple models, resolving conflicts where necessary?
-# 3. Are there any biases, inaccuracies, or missing information in the final output?
-
-# Finish your critique with actionable recommendations for improving the synthesized response.
-# """
-
-# COT_MOA_AGG_BASE_SYSTEM_PROMPT_REFINEMENT = """You are a helpful assistant tasked with refining the output of a model based on a critique.
-# Below is the original user prompt used to generate the output:
-
-# USER PROMPT:
-# {user_prompt}
-
-# Here is the original synthesized JSON object generated by the model:
-
-# ORIGINAL OUTPUT:
-# {original_output}
-
-# Here is the critique of the output:
-
-# CRITIQUE:
-# {critique_output}
-
-# Your task is to refine the synthesized JSON object based on the critique. Ensure that:
-# 1. The JSON object adheres to the required format.
-# 2. The synthesis properly reconciles different model responses, making informed decisions on conflicts.
-# 3. The final output is accurate, unbiased, and complete.
-
-# Return the improved JSON object.
-# """
-
-# COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT_CRITIQUE = """You are a helpful assistant tasked with reviewing the output of a model based on a given prompt.
-# Below is the original user prompt used to generate the output:
-
-# USER PROMPT:
-# {user_prompt}
-
-# Here is the response generated by the model:
-
-# OUTPUT:
-# {original_output}
-
-# Your task is to critique the output based on the following:
-# 1. Is the response well-structured and does it clearly explain each output field?
-# 2. Are all claims supported by the provided context? Identify any instances of hallucination or missing evidence.
-# 3. Does the response cite specific parts of the context when making claims?
-
-# Finish your critique with actionable recommendations for improving the response.
-# """
-
-# COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT_REFINEMENT = """You are a helpful assistant tasked with refining the output of a model based on a critique.
-# Below is the original user prompt used to generate the output:
-
-# USER PROMPT:
-# {user_prompt}
-
-# Here is the original response generated by the model:
-
-# ORIGINAL OUTPUT:
-# {original_output}
-
-# Here is the critique of the output:
-
-# CRITIQUE:
-# {critique_output}
-
-# Your task is to refine the response based on the critique. Ensure that:
-# 1. The response is well-structured and clearly explains each output field.
-# 2. All claims are directly supported by the provided context.
-# 3. The response explicitly cites relevant parts of the context.
-
-# Return the improved response.
-# """

palimpzest/prompts/moa_proposer_convert_prompts.py

@@ -1,75 +0,0 @@
-"""This file contains prompts for MixtureOfAgentsConvert operations on text inputs."""
-
-### BASE PROMPTS ###
-COT_MOA_PROPOSER_BASE_SYSTEM_PROMPT = """You are a helpful assistant whose job is to {job_instruction}.
-You will be presented with a context and a set of output fields to generate. Your task is to generate a paragraph or two which describes what you believe is the correct value for each output field.
-Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.
-
-You will be provided with a description of each input field and each output field.
-
-An example is shown below:
----
-INPUT FIELDS:
-{example_input_fields}
-
-OUTPUT FIELDS:
-{example_output_fields}
-
-CONTEXT:
-{example_context}{image_disclaimer}
-
-Let's think step-by-step in order to answer the question.
-
-ANSWER: {example_answer}
----
-"""
-
-COT_MOA_PROPOSER_BASE_USER_PROMPT = """You are a helpful assistant whose job is to {job_instruction}.
-You will be presented with a context and a set of output fields to generate. Your task is to generate a paragraph or two which describes what you believe is the correct value for each output field.
-Be sure to cite information from the context as evidence of why your answers are correct. Do not hallucinate evidence.
-{desc_section}
-You will be provided with a description of each input field and each output field.
----
-INPUT FIELDS:
-{input_fields_desc}
-
-OUTPUT FIELDS:
-{output_fields_desc}
-
-CONTEXT:
-{context}<<image-placeholder>>
-
-Let's think step-by-step in order to answer the question.
-
-ANSWER: """
-
-
-### TEMPLATE INPUTS ###
-COT_MOA_PROPOSER_JOB_INSTRUCTION = "produce an answer to a question"
-COT_MOA_PROPOSER_IMAGE_JOB_INSTRUCTION = "analyze input image(s) and/or text in order to produce an answer to a question"
-
-COT_MOA_PROPOSER_EXAMPLE_INPUT_FIELDS = """- text: a text passage describing a scientist
-- birthday: the scientist's birthday"""
-COT_MOA_PROPOSER_IMAGE_EXAMPLE_INPUT_FIELDS = """- image: an image of a scene
-- photographer: the photographer of the image"""
-
-COT_MOA_PROPOSER_EXAMPLE_OUTPUT_FIELDS = """- name: the name of the scientist
-- birth_year: the year the scientist was born"""
-COT_MOA_PROPOSER_IMAGE_EXAMPLE_OUTPUT_FIELDS = """- dog_in_image: true if a dog is in the image and false otherwise
-- person_in_image: true if a person is in the image and false otherwise"""
-
-COT_MOA_PROPOSER_EXAMPLE_CONTEXT = """{{
-  "text": "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, was an English mathematician and writer chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation.",
-  "birthday": "December 10, 1815"
-}}"""
-COT_MOA_PROPOSER_IMAGE_EXAMPLE_CONTEXT = """{{
-  "image": <bytes>,
-  "photographer": "CameraEnthusiast1"
-}}"""
-
-COT_MOA_PROPOSER_IMAGE_DISCLAIMER = """
-\n<image content provided here; assume in this example the image shows a dog and a cat playing>
-"""
-
-COT_MOA_PROPOSER_EXAMPLE_ANSWER = """the text passage mentions the scientist's name as "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace" and the scientist's birthday as "December 10, 1815". Therefore, the name of the scientist is "Augusta Ada King" and the birth year is 1815."""
-COT_MOA_PROPOSER_IMAGE_EXAMPLE_ANSWER = """The image shows a dog playing with a cat, so there is a dog in the image. There is no person in the image."""

palimpzest/prompts/util_phrases.py

@@ -1,19 +0,0 @@
-"""This file contains utility phrases which are templated into many of our prompts."""
-
-### FORMATTING INSTRUCTIONS ###
-ONE_TO_ONE_OUTPUT_FORMAT_INSTRUCTION = "Remember, your answer must be a valid JSON dictionary. The dictionary should only have the specified output fields."
-ONE_TO_MANY_OUTPUT_FORMAT_INSTRUCTION = "Remember, your answer must be a valid JSON list of dictionaries. The list may contain one or more dictionaries, and each dictionary should only have the specified output fields."
-
-### REASONING INSTRUCTION FOR IMAGE PROMPTS ###
-COT_REASONING_INSTRUCTION = """Let's think step-by-step in order to answer the question.
-
-REASONING: """
-
-COT_ANSWER_INSTRUCTION = """Let's think step-by-step in order to answer the question.
-
-ANSWER: """
-
-DESC_SECTION = """
-The user has additionally provided you with this description of the task you need to perform:
-{desc}
-"""

palimpzest/query/operators/critique_and_refine_convert.py

@@ -1,113 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from pydantic.fields import FieldInfo
-
-from palimpzest.constants import MODEL_CARDS, Model, PromptStrategy
-from palimpzest.core.elements.records import DataRecord
-from palimpzest.core.models import GenerationStats, OperatorCostEstimates
-from palimpzest.query.generators.generators import Generator
-from palimpzest.query.operators.convert import LLMConvert
-
-# TYPE DEFINITIONS
-FieldName = str
-
-
-class CriticAndRefineConvert(LLMConvert):
-
-    def __init__(
-        self,
-        critic_model: Model,
-        refine_model: Model,
-        *args,
-        **kwargs,
-    ):
-        super().__init__(*args, **kwargs)
-        self.critic_model = critic_model
-        self.refine_model = refine_model
-        
-        if self.prompt_strategy == PromptStrategy.COT_QA:
-            self.critic_prompt_strategy = PromptStrategy.COT_QA_CRITIC
-            self.refinement_prompt_strategy = PromptStrategy.COT_QA_REFINE
-        elif self.prompt_strategy == PromptStrategy.COT_QA_IMAGE:
-            self.critic_prompt_strategy = PromptStrategy.COT_QA_IMAGE_CRITIC
-            self.refinement_prompt_strategy = PromptStrategy.COT_QA_IMAGE_REFINE
-        else:
-            raise ValueError(f"Unsupported prompt strategy: {self.prompt_strategy}")
-
-        # create generators
-        self.critic_generator = Generator(self.critic_model, self.critic_prompt_strategy, self.reasoning_effort, self.api_base, self.cardinality, self.desc, self.verbose)
-        self.refine_generator = Generator(self.refine_model, self.refinement_prompt_strategy, self.reasoning_effort, self.api_base, self.cardinality, self.desc, self.verbose)
-
-    def __str__(self):
-        op = super().__str__()
-        op += f"    Critic Model: {self.critic_model}\n"
-        op += f"    Critic Prompt Strategy: {self.critic_prompt_strategy}\n"
-        op += f"    Refine Model: {self.refine_model}\n"
-        op += f"    Refinement Prompt Strategy: {self.refinement_prompt_strategy}\n"
-        return op
-
-    def get_id_params(self):
-        id_params = super().get_id_params()
-        id_params = {
-            "critic_model": self.critic_model.value,
-            "refine_model": self.refine_model.value,
-            **id_params,
-        }
-
-        return id_params
-
-    def get_op_params(self):
-        op_params = super().get_op_params()
-        op_params = {
-            "critic_model": self.critic_model,
-            "refine_model": self.refine_model,
-            **op_params,
-        }
-
-        return op_params
-
-    def naive_cost_estimates(self, source_op_cost_estimates: OperatorCostEstimates) -> OperatorCostEstimates:
-        """
-        Currently, we are invoking `self.model`, then critiquing its output with `self.critic_model`, and
-        finally refining the output with `self.refine_model`. Thus, we roughly expect to incur the cost
-        and time of three LLMConverts. In practice, this naive quality estimate will be overwritten by the
-        CostModel's estimate once it executes a few instances of the operator.
-        """
-        # get naive cost estimates for first LLM call and multiply by 3 for now;
-        # of course we should sum individual estimates for each model, but this is a rough estimate
-        # and in practice we will need to revamp our naive cost estimates in the near future
-        naive_op_cost_estimates = 3 * super().naive_cost_estimates(source_op_cost_estimates)
-
-        # for naive setting, estimate quality as quality of refine model
-        model_quality = MODEL_CARDS[self.refine_model.value]["overall"] / 100.0
-        naive_op_cost_estimates.quality = model_quality
-        naive_op_cost_estimates.quality_lower_bound = naive_op_cost_estimates.quality
-        naive_op_cost_estimates.quality_upper_bound = naive_op_cost_estimates.quality
-
-        return naive_op_cost_estimates
-
-    def convert(self, candidate: DataRecord, fields: dict[str, FieldInfo]) -> tuple[dict[FieldName, list[Any]], GenerationStats]:
-        # get input fields
-        input_fields = self.get_input_fields()
-
-        # NOTE: when I merge in the `abacus` branch, I will want to update this to reflect the changes I made to reasoning extraction
-        # execute the initial model
-        original_gen_kwargs = {"project_cols": input_fields, "output_schema": self.output_schema}
-        field_answers, reasoning, original_gen_stats, original_messages = self.generator(candidate, fields, **original_gen_kwargs)
-        original_output = f"REASONING: {reasoning}\nANSWER: {field_answers}\n"
-
-        # execute the critic model
-        critic_gen_kwargs = {"original_output": original_output, "original_messages": original_messages, **original_gen_kwargs}
-        _, reasoning, critic_gen_stats, _ = self.critic_generator(candidate, fields, json_output=False, **critic_gen_kwargs)
-        critique_output = f"CRITIQUE: {reasoning}\n"
-
-        # execute the refinement model
-        refine_gen_kwargs = {"critique_output": critique_output, **critic_gen_kwargs}
-        field_answers, reasoning, refine_gen_stats, _ = self.refine_generator(candidate, fields, **refine_gen_kwargs)
-
-        # compute the total generation stats
-        generation_stats = original_gen_stats + critic_gen_stats + refine_gen_stats
-
-        return field_answers, generation_stats