palimpzest 0.8.5__py3-none-any.whl → 0.8.7__py3-none-any.whl

palimpzest/core/elements/records.py

@@ -11,6 +11,8 @@ from pydantic.fields import FieldInfo
 
 from palimpzest.core.data import context
 from palimpzest.core.lib.schemas import (
+    AUDIO_FIELD_TYPES,
+    IMAGE_FIELD_TYPES,
     AudioBase64,
     AudioFilepath,
     ImageBase64,
@@ -303,9 +305,11 @@ class DataRecord:
             dct = {k: v for k, v in dct.items() if k in project_field_names}
 
         if not include_bytes:
+            bytes_field_types = [bytes, list[bytes], bytes | None, list[bytes] | None, bytes | Any, list[bytes] | Any]
+            bytes_field_types += AUDIO_FIELD_TYPES + IMAGE_FIELD_TYPES
             for k in dct:
                 field_type = self.get_field_type(k)
-                if field_type.annotation in [bytes, AudioBase64, ImageBase64, list[bytes], list[ImageBase64]]:
+                if field_type.annotation in bytes_field_types:
                     dct[k] = "<bytes>"
 
         if bytes_to_str:

palimpzest/core/lib/schemas.py

@@ -33,20 +33,27 @@ IMAGE_LIST_FIELD_TYPES = [
     list[ImageBase64] | None,
     list[ImageFilepath] | None,
     list[ImageURL] | None,
+    list[ImageBase64] | Any,
+    list[ImageFilepath] | Any,
+    list[ImageURL] | Any,
 ]
 IMAGE_FIELD_TYPES = IMAGE_LIST_FIELD_TYPES + [
     ImageBase64, ImageFilepath, ImageURL,
     ImageBase64 | None, ImageFilepath | None, ImageURL | None,
+    ImageBase64 | Any, ImageFilepath | Any, ImageURL | Any,
 ]
 AUDIO_LIST_FIELD_TYPES = [
     list[AudioBase64],
     list[AudioFilepath],
     list[AudioBase64] | None,
     list[AudioFilepath] | None,
+    list[AudioBase64] | Any,
+    list[AudioFilepath] | Any,
 ]
 AUDIO_FIELD_TYPES = AUDIO_LIST_FIELD_TYPES + [
     AudioBase64, AudioFilepath,
     AudioBase64 | None, AudioFilepath | None,
+    AudioBase64 | Any, AudioFilepath | Any,
 ]
 
 

palimpzest/prompts/prompt_factory.py

@@ -2,6 +2,7 @@
 
 import base64
 import json
+from typing import Any
 
 from pydantic import BaseModel
 
@@ -747,14 +748,14 @@ class PromptFactory:
             field_type = candidate.get_field_type(field_name)
 
             # audio filepath (or list of audio filepaths)
-            if field_type.annotation in [AudioFilepath, AudioFilepath | None]:
+            if field_type.annotation in [AudioFilepath, AudioFilepath | None, AudioFilepath | Any]:
                 with open(field_value, "rb") as f:
                     base64_audio_str = base64.b64encode(f.read()).decode("utf-8")
                 audio_content.append(
                     {"type": "input_audio", "input_audio": {"data": base64_audio_str, "format": "wav"}}
                 )
 
-            elif field_type.annotation in [list[AudioFilepath], list[AudioFilepath] | None]:
+            elif field_type.annotation in [list[AudioFilepath], list[AudioFilepath] | None, list[AudioFilepath] | Any]:
                 for audio_filepath in field_value:
                     with open(audio_filepath, "rb") as f:
                         base64_audio_str = base64.b64encode(f.read()).decode("utf-8")
@@ -763,12 +764,12 @@ class PromptFactory:
                     )
 
             # pre-encoded images (or list of pre-encoded images)
-            elif field_type.annotation in [AudioBase64, AudioBase64 | None]:
+            elif field_type.annotation in [AudioBase64, AudioBase64 | None, AudioBase64 | Any]:
                 audio_content.append(
                     {"type": "input_audio", "input_audio": {"data": field_value, "format": "wav"}}
                 )
 
-            elif field_type.annotation in [list[AudioBase64], list[AudioBase64] | None]:
+            elif field_type.annotation in [list[AudioBase64], list[AudioBase64] | None, list[AudioBase64] | Any]:
                 for base64_audio in field_value:
                     audio_content.append(
                         {"type": "input_audio", "input_audio": {"data": base64_audio, "format": "wav"}}
@@ -794,14 +795,14 @@ class PromptFactory:
             field_type = candidate.get_field_type(field_name)
 
             # image filepath (or list of image filepaths)
-            if field_type.annotation in [ImageFilepath, ImageFilepath | None]:
+            if field_type.annotation in [ImageFilepath, ImageFilepath | None, ImageFilepath | Any]:
                 with open(field_value, "rb") as f:
                     base64_image_str = base64.b64encode(f.read()).decode("utf-8")
                 image_content.append(
                     {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image_str}"}}
                 )
 
-            elif field_type.annotation in [list[ImageFilepath], list[ImageFilepath] | None]:
+            elif field_type.annotation in [list[ImageFilepath], list[ImageFilepath] | None, list[ImageFilepath] | Any]:
                 for image_filepath in field_value:
                     with open(image_filepath, "rb") as f:
                         base64_image_str = base64.b64encode(f.read()).decode("utf-8")
@@ -810,20 +811,20 @@ class PromptFactory:
                     )
 
             # image url (or list of image urls)
-            elif field_type.annotation in [ImageURL, ImageURL | None]:
+            elif field_type.annotation in [ImageURL, ImageURL | None, ImageURL | Any]:
                 image_content.append({"type": "image_url", "image_url": {"url": field_value}})
 
-            elif field_type.annotation in [list[ImageURL], list[ImageURL] | None]:
+            elif field_type.annotation in [list[ImageURL], list[ImageURL] | None, list[ImageURL] | Any]:
                 for image_url in field_value:
                     image_content.append({"type": "image_url", "image_url": {"url": image_url}})
 
             # pre-encoded images (or list of pre-encoded images)
-            elif field_type.annotation in [ImageBase64, ImageBase64 | None]:
+            elif field_type.annotation in [ImageBase64, ImageBase64 | None, ImageBase64 | Any]:
                 image_content.append(
                     {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{field_value}"}}
                 )
 
-            elif field_type.annotation in [list[ImageBase64], list[ImageBase64] | None]:
+            elif field_type.annotation in [list[ImageBase64], list[ImageBase64] | None, list[ImageBase64] | Any]:
                 for base64_image in field_value:
                     image_content.append(
                         {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}}

palimpzest/query/execution/mab_execution_strategy.py

@@ -755,7 +755,6 @@ class MABExecutionStrategy(SentinelExecutionStrategy):
 
     def execute_sentinel_plan(self, plan: SentinelPlan, train_dataset: dict[str, Dataset], validator: Validator) -> SentinelPlanStats:
         logger.info(f"Executing plan {plan.plan_id} with {self.max_workers} workers")
-        logger.info(f"Plan Details: {plan}")
 
         # initialize plan stats
         plan_stats = SentinelPlanStats.from_plan(plan)

palimpzest/query/optimizer/optimizer_strategy.py

@@ -58,7 +58,6 @@ class GreedyStrategy(OptimizationStrategy):
     def get_optimal_plans(self, groups: dict, final_group_id: int, policy: Policy, use_final_op_quality: bool) -> list[PhysicalPlan]:
         logger.info(f"Getting greedy optimal plans for final group id: {final_group_id}")
         plans = [self._get_greedy_physical_plan(groups, final_group_id)]
-        logger.info(f"Greedy optimal plans: {plans}")
         logger.info(f"Done getting greedy optimal plans for final group id: {final_group_id}")
 
         return plans
@@ -137,7 +136,6 @@ class ParetoStrategy(OptimizationStrategy):
             optimal_plan = optimal_plan if policy.choose(optimal_plan.plan_cost, plan.plan_cost) else plan
 
         plans = [optimal_plan]
-        logger.info(f"Pareto optimal plans: {plans}")
         logger.info(f"Done getting pareto optimal plans for final group id: {final_group_id}")
         return plans
     
@@ -174,7 +172,6 @@ class SentinelStrategy(OptimizationStrategy):
     def get_optimal_plans(self, groups: dict, final_group_id: int, policy: Policy, use_final_op_quality: bool) -> list[SentinelPlan]:
         logger.info(f"Getting sentinel optimal plans for final group id: {final_group_id}")
         plans = [self._get_sentinel_plan(groups, final_group_id)]
-        logger.info(f"Sentinel optimal plans: {plans}")
         logger.info(f"Done getting sentinel optimal plans for final group id: {final_group_id}")
         return plans
 

palimpzest/query/optimizer/plan.py

@@ -330,12 +330,11 @@ class SentinelPlan(Plan):
 
     def _get_str(self, idx: int = 0, indent: int = 0) -> str:
         indent_str = " " * (indent * 2)
-        plan_str = ""
-        for inner_idx, operator in enumerate(self.operator_set):
-            inner_idx_str = "" if len(self.operator_set) == 1 else f"{inner_idx + 1}."
-            plan_str += f"{indent_str}{idx}.{inner_idx_str} {str(operator)}\n"
-            for subplan in self.subplans:
-                plan_str += subplan._get_str(idx=idx + 1, indent=indent + 1)
+        operator = self.operator_set[0]
+        inner_idx_str = "" if len(self.operator_set) == 1 else f"1 - {len(self.operator_set)}."
+        plan_str = f"{indent_str}{idx}.{inner_idx_str} {str(operator)}\n"
+        for subplan in self.subplans:
+            plan_str += subplan._get_str(idx=idx + 1, indent=indent + 1)
 
         return plan_str
 

palimpzest/validator/validator.py

@@ -2,8 +2,8 @@ import json
 import time
 
 import litellm
-from colorama import Fore, Style
 
+# from colorama import Fore, Style
 from palimpzest.constants import MODEL_CARDS, Cardinality, Model, PromptStrategy
 from palimpzest.core.elements.records import DataRecord
 from palimpzest.core.models import GenerationStats
@@ -87,7 +87,7 @@ class Validator:
         input_messages = [msg for msg in messages if msg["role"] != "system"]
         output = json.dumps(output, indent=2)
         output_message = f"OUTPUT:\n--------\n{output}\n\nEVALUATION: "
-        input_str = '\n'.join(list(map(lambda d: d['content'], input_messages + [{"role": "user", "content": output_message}])))
+        # input_str = '\n'.join(list(map(lambda d: d['content'], input_messages + [{"role": "user", "content": output_message}])))
 
         # invoke the judge
         score, gen_stats = None, GenerationStats()
@@ -98,8 +98,8 @@ class Validator:
             completion = litellm.completion(model=self.model.value, messages=val_messages)
             completion_text = completion.choices[0].message.content
             gen_stats = self._get_gen_stats_from_completion(completion, start_time)
-            print(f"INPUT:\n{input_str}")
-            print(Fore.GREEN + f"{completion_text}\n" + Style.RESET_ALL)
+            # print(f"INPUT:\n{input_str}")
+            # print(Fore.GREEN + f"{completion_text}\n" + Style.RESET_ALL)
 
             # parse the evaluation
             eval_dict: dict = get_json_from_answer(completion_text, self.model, Cardinality.ONE_TO_ONE)
@@ -233,7 +233,7 @@ class Validator:
         input_messages = [msg for msg in messages if msg["role"] != "system"]
         output = json.dumps(output, indent=2)
         output_message = f"OUTPUT:\n--------\n{output}\n\nEVALUATION: "
-        input_str = '\n'.join(list(map(lambda d: d['content'], input_messages + [{"role": "user", "content": output_message}])))
+        # input_str = '\n'.join(list(map(lambda d: d['content'], input_messages + [{"role": "user", "content": output_message}])))
 
         # invoke the judge
         score, gen_stats = None, GenerationStats()
@@ -245,8 +245,8 @@ class Validator:
             completion = litellm.completion(model="openai/o4-mini", messages=val_messages)
             completion_text = completion.choices[0].message.content
             gen_stats = self._get_gen_stats_from_completion(completion, start_time)
-            print(f"INPUT:\n{input_str}")
-            print(Fore.GREEN + f"{completion_text}\n" + Style.RESET_ALL)
+            # print(f"INPUT:\n{input_str}")
+            # print(Fore.GREEN + f"{completion_text}\n" + Style.RESET_ALL)
 
             # parse the evaluation
             eval_dict: dict = get_json_from_answer(completion_text, self.model, Cardinality.ONE_TO_ONE)

{palimpzest-0.8.5.dist-info → palimpzest-0.8.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: palimpzest
-Version: 0.8.5
+Version: 0.8.7
 Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
 Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
 Project-URL: homepage, https://palimpzest.org

{palimpzest-0.8.5.dist-info → palimpzest-0.8.7.dist-info}/RECORD

@@ -15,9 +15,9 @@ palimpzest/core/data/iter_dataset.py,sha256=K47ajOXsCZV3WhOuDkw3xfiHzn8mXPU976uN
 palimpzest/core/elements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 palimpzest/core/elements/filters.py,sha256=fU2x0eWDwfP52_5fUmqJXTuhs4H0vvHtPZLdA3IIw8I,1642
 palimpzest/core/elements/groupbysig.py,sha256=oFH5UkZzcR0msAgfQiRQOOvyJ3HaW4Dwr03h7tVOcrM,2324
-palimpzest/core/elements/records.py,sha256=pqtuSgc-Jm5N57d6jtUXmQx0D-khqjOIQAFZjS1XmNM,17075
+palimpzest/core/elements/records.py,sha256=gmMP3nzf6rGfYzbzwUCuiKxz2SrLYlDNktvktLJVRM0,17263
 palimpzest/core/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/core/lib/schemas.py,sha256=2fzbTZBTssKTl9CFGDEQneXasOwo-PLP2lCqHZn2eng,9318
+palimpzest/core/lib/schemas.py,sha256=8hq12gZQwRNpbPNggJfhSvlGvplJArCthVMSUDMSiKw,9568
 palimpzest/prompts/__init__.py,sha256=942kdENfPU5mFjIxYm-FusL0FD6LNhoj6cYoSGiUsCI,1628
 palimpzest/prompts/agent_prompts.py,sha256=CUzBVLBiPSw8OShtKp4VTpQwtrNMtcMglo-IZHMvuDM,17459
 palimpzest/prompts/context_search.py,sha256=s3pti4XNRiIyiWzjVNL_NqmqEc31jzSKMF2SlN0Aaf8,357
@@ -27,7 +27,7 @@ palimpzest/prompts/filter_prompts.py,sha256=D-aY3-th1GzEHrVGbKORVN2R7x7coYGjp8Fr
 palimpzest/prompts/join_prompts.py,sha256=z-y4L1cw1O3I_F9DW6MvqeztdQoKDQawX6nK6vQAkdM,2916
 palimpzest/prompts/moa_aggregator_prompts.py,sha256=b5cz4G2oF86LlHOy8vmtxoMcZ9zaZoppKrURHgzCzNU,5248
 palimpzest/prompts/moa_proposer_prompts.py,sha256=yfZYwmCg-Tg9h0H7PJMEuDYPR45EbYnORmVX6cY2vRQ,3125
-palimpzest/prompts/prompt_factory.py,sha256=32GFAfvWOwRHUsAMRDPEiFzaObRK8FeVfGgkG-QKcYs,44187
+palimpzest/prompts/prompt_factory.py,sha256=SlpvrbgBtJbtxBafUaIafmXnR-c-e_MYcrCySUv3KfY,44432
 palimpzest/prompts/split_merge_prompts.py,sha256=hX-MThmW4VU7rjgm7gb-bpniEMdj25mtp0o8qBeWvIQ,5573
 palimpzest/prompts/split_proposer_prompts.py,sha256=Ucqwfn4FqFk-b9E024EK4e_3_QndTJjggwiwa1x5CQs,3115
 palimpzest/prompts/utils.py,sha256=iFv4nuFRuON-DEAdO2JI-J84ukV8Ev27YYWPLwfk44A,5655
@@ -37,7 +37,7 @@ palimpzest/query/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 palimpzest/query/execution/all_sample_execution_strategy.py,sha256=8a8-eKsndo_edCwIamNgcISLQmTzVSv5vmD6Ogl8a6k,14367
 palimpzest/query/execution/execution_strategy.py,sha256=XoRVNlJSAgON-NWis9SecFr0B7DlJIm-25u1v5rjvu8,19085
 palimpzest/query/execution/execution_strategy_type.py,sha256=vRQBPCQN5_aoyD3TLIeW3VPo15mqF-5RBvEXkENz9FE,987
-palimpzest/query/execution/mab_execution_strategy.py,sha256=rTiDcDGW1sKgHDMhhvFjHGXnV7L4weYGrGQoeh5q2oE,46337
+palimpzest/query/execution/mab_execution_strategy.py,sha256=YjUZ2qBGvQMVUxi7rQCSU8JKP1RtqhG8Owik8hKB_UU,46292
 palimpzest/query/execution/parallel_execution_strategy.py,sha256=roZZy7wLcmAwm_ecYvqSJanRaiox3OoNPuXxvRZ5TXg,15710
 palimpzest/query/execution/single_threaded_execution_strategy.py,sha256=sESji79ytKxth9Tpm02c34Mltw0YiFn4GL5h0MI5Noo,16255
 palimpzest/query/generators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -63,9 +63,9 @@ palimpzest/query/operators/split.py,sha256=oLzwnYb8TNf3XA9TMKEAIw7EIA12wHneaD42B
 palimpzest/query/optimizer/__init__.py,sha256=COn-okHtnYEyoNFRt3o3SA7jI5Wssx9BUEgfOfP4dOE,2560
 palimpzest/query/optimizer/cost_model.py,sha256=OldPy-TJdfsQbYRoKlb3yWeKbi15jcldTIUS6BTi9T8,12678
 palimpzest/query/optimizer/optimizer.py,sha256=BrhljITlFC5S5euA01pv4dzlqxrtKNEt_0DmhRtcMTk,19966
-palimpzest/query/optimizer/optimizer_strategy.py,sha256=9YlNGkqwgX0WaV6y8tKOOHVN8kC8GjDI3DttvGW5SYY,10206
+palimpzest/query/optimizer/optimizer_strategy.py,sha256=0foDaBHqQehK_zz6IlDEbNIw-44wxY6LO5H1anJi56Y,10042
 palimpzest/query/optimizer/optimizer_strategy_type.py,sha256=V-MMHvJdnfZKoUX1xxxwh66q1RjN2FL35IsiT1C62c8,1084
-palimpzest/query/optimizer/plan.py,sha256=VIhN7tWT7EoRE9BKYa1qvvOhX7dEaM-aiobByX0qjzg,22900
+palimpzest/query/optimizer/plan.py,sha256=NoCUS_lyZ7LFj15_qpZ_cOFHVkCFMcIn8A7EsNeD57c,22849
 palimpzest/query/optimizer/primitives.py,sha256=jMMVq37y1tWiPU1lSSKQP9OP-mzkpSxSmUeDajRYYOQ,5445
 palimpzest/query/optimizer/rules.py,sha256=er8K47L-qdRn0hCra-2PaqxhQEvDwJ7IVzNEszWHJ48,50452
 palimpzest/query/optimizer/tasks.py,sha256=GCRA4rK6Q8dBGj2FnsRJUk3IdKthNQgiK5lFEu7v0mI,30439
@@ -87,9 +87,9 @@ palimpzest/utils/model_helpers.py,sha256=X6SlMgD5I5Aj_cxaFaoGaaNvOOqTNZVmjj6zbfn
 palimpzest/utils/progress.py,sha256=7gucyZr82udMDZitrrkAOSKHZVljE3R2wv9nf5gA5TM,20807
 palimpzest/utils/udfs.py,sha256=LjHic54B1az-rKgNLur0wOpaz2ko_UodjLEJrazkxvY,1854
 palimpzest/validator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/validator/validator.py,sha256=vasnvAzEv9tDNLGz2X7MpMJBpn8MqSNelQSXk3X6MBs,16002
-palimpzest-0.8.5.dist-info/licenses/LICENSE,sha256=5GUlHy9lr-Py9kvV38FF1m3yy3NqM18fefuE9wkWumo,1079
-palimpzest-0.8.5.dist-info/METADATA,sha256=RrnXm-PJIQtSDM99iEfr15NIj37EzyZjkXsElXARf78,7048
-palimpzest-0.8.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-palimpzest-0.8.5.dist-info/top_level.txt,sha256=raV06dJUgohefUn3ZyJS2uqp_Y76EOLA9Y2e_fxt8Ew,11
-palimpzest-0.8.5.dist-info/RECORD,,
+palimpzest/validator/validator.py,sha256=Ixm5cmDf2eaHJoY7eUz0NULklCanC2zcSNtTbmXo-vo,16016
+palimpzest-0.8.7.dist-info/licenses/LICENSE,sha256=5GUlHy9lr-Py9kvV38FF1m3yy3NqM18fefuE9wkWumo,1079
+palimpzest-0.8.7.dist-info/METADATA,sha256=NC6mWNLDZzoBmKe29GR0pwD92TXkg_KMRsVEZYeQKbg,7048
+palimpzest-0.8.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+palimpzest-0.8.7.dist-info/top_level.txt,sha256=raV06dJUgohefUn3ZyJS2uqp_Y76EOLA9Y2e_fxt8Ew,11
+palimpzest-0.8.7.dist-info/RECORD,,