palimpzest 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

palimpzest/core/models.py

@@ -35,12 +35,18 @@ class GenerationStats(BaseModel):
     # typed as a float because GenerationStats may be amortized (i.e. divided) across a number of output records
     total_output_tokens: float = 0.0
 
+    # the total number of input tokens processed by embedding models
+    total_embedding_input_tokens: float = 0.0
+
     # the total cost of processing the input tokens; None if this operation did not use an LLM
     total_input_cost: float = 0.0
 
     # the total cost of processing the output tokens; None if this operation did not use an LLM
     total_output_cost: float = 0.0
 
+    # the total cost of processing input tokens for embedding models
+    total_embedding_cost: float = 0.0
+
     # the total cost of processing the input and output tokens; None if this operation did not use an LLM
     cost_per_record: float = 0.0
 
@@ -68,6 +74,9 @@ class GenerationStats(BaseModel):
             "fn_call_duration_secs",
             "total_llm_calls",
             "total_embedding_llm_calls",
+            "total_embedding_input_tokens",
+            "total_embedding_cost"
+            
         ]:
             setattr(self, model_field, getattr(self, model_field) + getattr(other, model_field))
         return self
@@ -85,6 +94,8 @@ class GenerationStats(BaseModel):
                 "cost_per_record",
                 "total_llm_calls",
                 "total_embedding_llm_calls",
+                "total_embedding_input_tokens",
+                "total_embedding_cost"
             ]
         }
         # dct['raw_answers'] = self.raw_answers + other.raw_answers
@@ -107,6 +118,8 @@ class GenerationStats(BaseModel):
             "fn_call_duration_secs",
             "total_llm_calls",
             "total_embedding_llm_calls",
+            "total_embedding_input_tokens",
+            "total_embedding_cost"
         ]:
             setattr(self, model_field, getattr(self, model_field) / quotient)
         return self
@@ -128,6 +141,8 @@ class GenerationStats(BaseModel):
                 "total_llm_calls",
                 "total_embedding_llm_calls",
                 "cost_per_record",
+                "total_embedding_input_tokens",
+                "total_embedding_cost"
             ]
         }
         dct["model_name"] = self.model_name
@@ -217,6 +232,10 @@ class RecordOpStats(BaseModel):
     # typed as a float because GenerationStats may be amortized (i.e. divided) across a number of output records
     total_output_tokens: float = 0.0
 
+    # the total number of input tokens processed by embedding models
+    # typed as a float because GenerationStats may be amortized (i.e. divided) across a number of output records
+    total_embedding_input_tokens: float = 0.0
+
     # the total cost of processing the input tokens; None if this operation did not use an LLM
     total_input_cost: float = 0.0
 
@@ -278,6 +297,9 @@ class OperatorStats(BaseModel):
     # the total output tokens processed by this operation
     total_output_tokens: int = 0
 
+    #the total embedding input tokens processed by this operation
+    total_embedding_input_tokens: int = 0
+
     # a list of RecordOpStats processed by the operation
     record_op_stats_lst: list[RecordOpStats] = Field(default_factory=list)
 
@@ -309,6 +331,7 @@ class OperatorStats(BaseModel):
             self.total_op_cost += stats.total_op_cost
             self.total_input_tokens += stats.total_input_tokens
             self.total_output_tokens += stats.total_output_tokens
+            self.total_embedding_input_tokens += stats.total_embedding_input_tokens
             self.record_op_stats_lst.extend(stats.record_op_stats_lst)
 
         elif isinstance(stats, RecordOpStats):
@@ -319,6 +342,7 @@ class OperatorStats(BaseModel):
             self.total_op_cost += stats.cost_per_record
             self.total_input_tokens += stats.total_input_tokens
             self.total_output_tokens += stats.total_output_tokens
+            self.total_embedding_input_tokens += stats.total_embedding_input_tokens
 
         else:
             raise TypeError(f"Cannot add {type(stats)} to OperatorStats")
@@ -370,6 +394,9 @@ class BasePlanStats(BaseModel):
     # total output tokens processed by this plan
     total_output_tokens: int = 0
 
+    # total embedding input tokens processed by this plan
+    total_embedding_input_tokens: int = 0
+
     # start time for the plan execution; should be set by calling PlanStats.start()
     start_time: float | None = None
 
@@ -385,6 +412,7 @@ class BasePlanStats(BaseModel):
         self.total_plan_cost = self.sum_op_costs() + self.sum_validation_costs()
         self.total_input_tokens = self.sum_input_tokens() + self.sum_validation_input_tokens()
         self.total_output_tokens = self.sum_output_tokens() + self.sum_validation_output_tokens()
+        self.total_embedding_input_tokens = self.sum_embedding_input_tokens() + self.sum_validation_embedding_input_tokens()
 
     @staticmethod
     @abstractmethod
@@ -415,6 +443,13 @@ class BasePlanStats(BaseModel):
         """
         pass
 
+    @abstractmethod
+    def sum_embedding_input_tokens(self) -> int:
+        """
+        Sum the input embedding tokens processed by all operators in this plan.
+        """
+        pass
+
     @abstractmethod
     def add_record_op_stats(self, unique_full_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None:
         """
@@ -453,6 +488,12 @@ class BasePlanStats(BaseModel):
         Sum the output tokens processed by all validation generations in this plan.
         """
         return sum([gen_stats.total_output_tokens for _, gen_stats in self.validation_gen_stats.items()])
+    
+    def sum_validation_embedding_input_tokens(self) -> int:
+        """
+        Sum the input embedding tokens processed by all validation generations in this plan.
+        """
+        return sum([gen_stats.total_embedding_input_tokens for _, gen_stats in self.validation_gen_stats.items()])
 
     def get_total_cost_so_far(self) -> float:
         """
@@ -501,6 +542,12 @@ class PlanStats(BasePlanStats):
         Sum the output tokens processed by all operators in this plan.
         """
         return sum([op_stats.total_output_tokens for _, op_stats in self.operator_stats.items()])
+    
+    def sum_embedding_input_tokens(self) -> int:
+        """
+        Sum the input embedding tokens processed by all operators in this plan.
+        """
+        return sum([op_stats.total_embedding_input_tokens for _, op_stats in self.operator_stats.items()])
 
     def add_record_op_stats(self, unique_full_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None:
         """
@@ -528,6 +575,7 @@ class PlanStats(BasePlanStats):
         self.total_plan_cost += plan_stats.total_plan_cost
         self.total_input_tokens += plan_stats.total_input_tokens
         self.total_output_tokens += plan_stats.total_output_tokens
+        self.total_embedding_input_tokens += plan_stats.total_embedding_input_tokens
         for unique_full_op_id, op_stats in plan_stats.operator_stats.items():
             if unique_full_op_id in self.operator_stats:
                 self.operator_stats[unique_full_op_id] += op_stats
@@ -539,6 +587,7 @@ class PlanStats(BasePlanStats):
         stats += f"total_plan_cost={self.total_plan_cost} \n"
         stats += f"total_input_tokens={self.total_input_tokens} \n"
         stats += f"total_output_tokens={self.total_output_tokens} \n"
+        stats += f"total_embedding_input_tokens={self.total_embedding_input_tokens} \n"
         for idx, op_stats in enumerate(self.operator_stats.values()):
             stats += f"{idx}. {op_stats.op_name} time={op_stats.total_op_time} cost={op_stats.total_op_cost} \n"
         return stats
@@ -586,6 +635,12 @@ class SentinelPlanStats(BasePlanStats):
         Sum the output tokens processed by all operators in this plan.
         """
         return sum(sum([op_stats.total_output_tokens for _, op_stats in phys_op_stats.items()]) for _, phys_op_stats in self.operator_stats.items())
+    
+    def sum_embedding_input_tokens(self) -> int:
+        """
+        Sum the output tokens processed by all operators in this plan.
+        """
+        return sum(sum([op_stats.total_embedding_input_tokens for _, op_stats in phys_op_stats.items()]) for _, phys_op_stats in self.operator_stats.items())
 
     def add_record_op_stats(self, unique_logical_op_id: str, record_op_stats: RecordOpStats | list[RecordOpStats]) -> None:
         """
@@ -627,6 +682,7 @@ class SentinelPlanStats(BasePlanStats):
         self.total_plan_cost += plan_stats.total_plan_cost
         self.total_input_tokens += plan_stats.total_input_tokens
         self.total_output_tokens += plan_stats.total_output_tokens
+        self.total_embedding_input_tokens += plan_stats.total_embedding_input_tokens
         for unique_logical_op_id, physical_op_stats in plan_stats.operator_stats.items():
             for full_op_id, op_stats in physical_op_stats.items():
                 if unique_logical_op_id in self.operator_stats:
@@ -648,6 +704,7 @@ class SentinelPlanStats(BasePlanStats):
         stats += f"total_plan_cost={self.total_plan_cost} \n"
         stats += f"total_input_tokens={self.total_input_tokens} \n"
         stats += f"total_output_tokens={self.total_output_tokens} \n"
+        stats += f"total_embedding_input_tokens={self.total_embedding_input_tokens} \n"
         for outer_idx, physical_op_stats in enumerate(self.operator_stats.values()):
             total_time = sum([op_stats.total_op_time for op_stats in physical_op_stats.values()])
             total_cost = sum([op_stats.total_op_cost for op_stats in physical_op_stats.values()])
@@ -695,6 +752,9 @@ class ExecutionStats(BaseModel):
     # total number of output tokens processed
     total_output_tokens: int = 0
 
+     # total number of embedding input tokens processed
+    total_embedding_input_tokens: int = 0
+
     # total number of tokens processed
     total_tokens: int = 0
 
@@ -748,7 +808,8 @@ class ExecutionStats(BaseModel):
         # compute the tokens for total execution
         self.total_input_tokens = self.sum_input_tokens()
         self.total_output_tokens = self.sum_output_tokens()
-        self.total_tokens = self.total_input_tokens + self.total_output_tokens
+        self.total_embedding_input_tokens = self.sum_embedding_input_tokens()
+        self.total_tokens = self.total_input_tokens + self.total_output_tokens + self.total_embedding_input_tokens
 
         # compute plan_strs
         self.plan_strs = {plan_id: plan_stats.plan_str for plan_id, plan_stats in self.plan_stats.items()}
@@ -780,6 +841,15 @@ class ExecutionStats(BaseModel):
         sentinel_plan_output_tokens = sum([plan_stats.sum_output_tokens() for _, plan_stats in self.sentinel_plan_stats.items()])
         plan_output_tokens = sum([plan_stats.sum_output_tokens() for _, plan_stats in self.plan_stats.items()])
         return plan_output_tokens + sentinel_plan_output_tokens
+    
+
+    def sum_embedding_input_tokens(self) -> int:
+        """
+        Sum the embedding input tokens processed in this execution
+        """
+        sentinel_plan_embedding_input_tokens = sum([plan_stats.sum_embedding_input_tokens() for _, plan_stats in self.sentinel_plan_stats.items()])
+        plan_embedding_input_tokens = sum([plan_stats.sum_embedding_input_tokens() for _, plan_stats in self.plan_stats.items()])
+        return plan_embedding_input_tokens + sentinel_plan_embedding_input_tokens
 
     def add_plan_stats(self, plan_stats: PlanStats | SentinelPlanStats | list[PlanStats] | list[SentinelPlanStats]) -> None:
         """

palimpzest/prompts/prompt_factory.py

@@ -830,7 +830,7 @@ class PromptFactory:
                 field_type = dr.get_field_type(field_name)
 
                 # audio filepath (or list of audio filepaths)
-                if field_type.annotation in [AudioFilepath, AudioFilepath | None, AudioFilepath | Any]:
+                if field_type.annotation in [AudioFilepath, AudioFilepath | None, AudioFilepath | Any] and field_value is not None:
                     with open(field_value, "rb") as f:
                         base64_audio_str = base64.b64encode(f.read()).decode("utf-8")
                     audio_content.append(
@@ -839,6 +839,8 @@ class PromptFactory:
 
                 elif field_type.annotation in [list[AudioFilepath], list[AudioFilepath] | None, list[AudioFilepath] | Any]:
                     for audio_filepath in field_value:
+                        if audio_filepath is None:
+                            continue
                         with open(audio_filepath, "rb") as f:
                             base64_audio_str = base64.b64encode(f.read()).decode("utf-8")
                         audio_content.append(
@@ -846,13 +848,15 @@ class PromptFactory:
                         )
 
                 # pre-encoded images (or list of pre-encoded images)
-                elif field_type.annotation in [AudioBase64, AudioBase64 | None, AudioBase64 | Any]:
+                elif field_type.annotation in [AudioBase64, AudioBase64 | None, AudioBase64 | Any] and field_value is not None:
                     audio_content.append(
                         {"type": "input_audio", "input_audio": {"data": field_value, "format": "wav"}}
                     )
 
                 elif field_type.annotation in [list[AudioBase64], list[AudioBase64] | None, list[AudioBase64] | Any]:
                     for base64_audio in field_value:
+                        if base64_audio is None:
+                            continue
                         audio_content.append(
                             {"type": "input_audio", "input_audio": {"data": base64_audio, "format": "wav"}}
                         )
@@ -882,7 +886,7 @@ class PromptFactory:
                 field_type = dr.get_field_type(field_name)
 
                 # image filepath (or list of image filepaths)
-                if field_type.annotation in [ImageFilepath, ImageFilepath | None, ImageFilepath | Any]:
+                if field_type.annotation in [ImageFilepath, ImageFilepath | None, ImageFilepath | Any] and field_value is not None:
                     with open(field_value, "rb") as f:
                         base64_image_str = base64.b64encode(f.read()).decode("utf-8")
                     image_content.append(
@@ -891,6 +895,8 @@ class PromptFactory:
 
                 elif field_type.annotation in [list[ImageFilepath], list[ImageFilepath] | None, list[ImageFilepath] | Any]:
                     for image_filepath in field_value:
+                        if image_filepath is None:
+                            continue
                         with open(image_filepath, "rb") as f:
                             base64_image_str = base64.b64encode(f.read()).decode("utf-8")
                         image_content.append(
@@ -898,21 +904,25 @@ class PromptFactory:
                         )
 
                 # image url (or list of image urls)
-                elif field_type.annotation in [ImageURL, ImageURL | None, ImageURL | Any]:
+                elif field_type.annotation in [ImageURL, ImageURL | None, ImageURL | Any] and field_value is not None:
                     image_content.append({"type": "image_url", "image_url": {"url": field_value}})
 
                 elif field_type.annotation in [list[ImageURL], list[ImageURL] | None, list[ImageURL] | Any]:
                     for image_url in field_value:
+                        if image_url is None:
+                            continue
                         image_content.append({"type": "image_url", "image_url": {"url": image_url}})
 
                 # pre-encoded images (or list of pre-encoded images)
-                elif field_type.annotation in [ImageBase64, ImageBase64 | None, ImageBase64 | Any]:
+                elif field_type.annotation in [ImageBase64, ImageBase64 | None, ImageBase64 | Any] and field_value is not None:
                     image_content.append(
                         {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{field_value}"}}
                     )
 
                 elif field_type.annotation in [list[ImageBase64], list[ImageBase64] | None, list[ImageBase64] | Any]:
                     for base64_image in field_value:
+                        if base64_image is None:
+                            continue
                         image_content.append(
                             {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}}
                         )

palimpzest/query/execution/execution_strategy.py

@@ -91,6 +91,7 @@ class SentinelExecutionStrategy(BaseExecutionStrategy, ABC):
         use_final_op_quality: bool = False,
         seed: int = 42,
         exp_name: str | None = None,
+        dont_use_priors: bool = False,
         *args,
         **kwargs,
     ):
@@ -105,6 +106,7 @@ class SentinelExecutionStrategy(BaseExecutionStrategy, ABC):
         self.seed = seed
         self.rng = np.random.default_rng(seed=seed)
         self.exp_name = exp_name
+        self.dont_use_priors = dont_use_priors
 
         # general cache which maps hash(logical_op_id, phys_op_id, hash(input)) --> record_set
         self.cache: dict[int, DataRecordSet] = {}

palimpzest/query/execution/mab_execution_strategy.py

@@ -44,6 +44,7 @@ class OpFrontier:
             seed: int,
             policy: Policy,
             priors: dict | None = None,
+            dont_use_priors: bool = False,
         ):
         # set k and j, which are the initial number of operators in the frontier and the
         # initial number of records to sample for each frontier operator
@@ -51,6 +52,7 @@ class OpFrontier:
         self.j = j
         self.source_indices = source_indices
         self.root_dataset_ids = root_dataset_ids
+        self.dont_use_priors = dont_use_priors
 
         # store the policy that we are optimizing under
         self.policy = policy
@@ -68,6 +70,7 @@ class OpFrontier:
         is_llm_filter = isinstance(sample_op, LLMFilter)
         is_llm_topk = isinstance(sample_op, TopKOp) and isinstance(sample_op.index, Collection)
         self.is_llm_op = is_llm_convert or is_llm_filter or is_llm_topk or self.is_llm_join
+        self.is_llm_convert = is_llm_convert
 
         # get order in which we will sample physical operators for this logical operator
         sample_op_indices = self._get_op_index_order(op_set, seed)
@@ -190,7 +193,9 @@ class OpFrontier:
         Returns a list of indices for the operators in the op_set.
         """
         # if this is not an llm-operator, we simply return the indices in random order
-        if not self.is_llm_op:
+        if not self.is_llm_op or self.dont_use_priors:
+            if self.is_llm_convert:
+                print("Using NO PRIORS for operator sampling order")
             rng = np.random.default_rng(seed=seed)
             op_indices = np.arange(len(op_set))
             rng.shuffle(op_indices)
@@ -198,6 +203,8 @@ class OpFrontier:
 
         # if this is an llm-operator, but we do not have priors, we first compute naive priors
         if self.priors is None or any([op_id not in self.priors for op_id in map(lambda op: op.get_op_id(), op_set)]):
+            if self.is_llm_convert:
+                print("Using NAIVE PRIORS for operator sampling order")
             self.priors = self._compute_naive_priors(op_set)
 
         # NOTE: self.priors is a dictionary with format:
@@ -770,7 +777,7 @@ class MABExecutionStrategy(SentinelExecutionStrategy):
 
                 # if the operator is a non-llm filter which has filtered out records, remove those records from
                 # all downstream operators' full_op_id_to_sources_not_processed
-                if isinstance(op_set[0], NonLLMFilter):
+                if isinstance(op_set[0], NonLLMFilter) and next_unique_logical_op_id is not None:
                     self._remove_filtered_records_from_downstream_ops(topo_idx, plan, op_frontiers, source_indices_to_all_record_sets)
 
         # finalize plan stats
@@ -805,7 +812,7 @@ class MABExecutionStrategy(SentinelExecutionStrategy):
                 assert len(root_dataset_ids) == 1, f"Scan for {sample_op} has {len(root_dataset_ids)} > 1 root dataset ids"
                 root_dataset_id = root_dataset_ids[0]
                 source_indices = dataset_id_to_shuffled_source_indices[root_dataset_id]
-                op_frontiers[unique_logical_op_id] = OpFrontier(op_set, source_unique_logical_op_ids, root_dataset_ids, source_indices, self.k, self.j, self.seed, self.policy, self.priors)
+                op_frontiers[unique_logical_op_id] = OpFrontier(op_set, source_unique_logical_op_ids, root_dataset_ids, source_indices, self.k, self.j, self.seed, self.policy, self.priors, self.dont_use_priors)
             elif isinstance(sample_op, JoinOp):
                 assert len(source_unique_logical_op_ids) == 2, f"Join for {sample_op} has {len(source_unique_logical_op_ids)} != 2 source logical operators"
                 left_source_indices = op_frontiers[source_unique_logical_op_ids[0]].source_indices
@@ -814,10 +821,10 @@ class MABExecutionStrategy(SentinelExecutionStrategy):
                 for left_source_idx in left_source_indices:
                     for right_source_idx in right_source_indices:
                         source_indices.append((left_source_idx, right_source_idx))
-                op_frontiers[unique_logical_op_id] = OpFrontier(op_set, source_unique_logical_op_ids, root_dataset_ids, source_indices, self.k, self.j, self.seed, self.policy, self.priors)
+                op_frontiers[unique_logical_op_id] = OpFrontier(op_set, source_unique_logical_op_ids, root_dataset_ids, source_indices, self.k, self.j, self.seed, self.policy, self.priors, self.dont_use_priors)
             else:
                 source_indices = op_frontiers[source_unique_logical_op_ids[0]].source_indices
-                op_frontiers[unique_logical_op_id] = OpFrontier(op_set, source_unique_logical_op_ids, root_dataset_ids, source_indices, self.k, self.j, self.seed, self.policy, self.priors)
+                op_frontiers[unique_logical_op_id] = OpFrontier(op_set, source_unique_logical_op_ids, root_dataset_ids, source_indices, self.k, self.j, self.seed, self.policy, self.priors, self.dont_use_priors)
 
         # initialize and start the progress manager
         self.progress_manager = create_progress_manager(plan, sample_budget=self.sample_budget, sample_cost_budget=self.sample_cost_budget, progress=self.progress)

palimpzest/query/operators/convert.py

@@ -121,8 +121,10 @@ class ConvertOp(PhysicalOperator, ABC):
                 generated_fields=field_names,
                 total_input_tokens=per_record_stats.total_input_tokens,
                 total_output_tokens=per_record_stats.total_output_tokens,
+                total_embedding_input_tokens=per_record_stats.total_embedding_input_tokens,
                 total_input_cost=per_record_stats.total_input_cost,
                 total_output_cost=per_record_stats.total_output_cost,
+                total_embedding_cost=per_record_stats.total_embedding_cost,
                 llm_call_duration_secs=per_record_stats.llm_call_duration_secs,
                 fn_call_duration_secs=per_record_stats.fn_call_duration_secs,
                 total_llm_calls=per_record_stats.total_llm_calls,

palimpzest/query/operators/filter.py

@@ -89,8 +89,10 @@ class FilterOp(PhysicalOperator, ABC):
             filter_str=self.filter_obj.get_filter_str(),
             total_input_tokens=generation_stats.total_input_tokens,
             total_output_tokens=generation_stats.total_output_tokens,
+            total_embedding_input_tokens=generation_stats.total_embedding_input_tokens,
             total_input_cost=generation_stats.total_input_cost,
             total_output_cost=generation_stats.total_output_cost,
+            total_embedding_cost=generation_stats.total_embedding_cost,
             llm_call_duration_secs=generation_stats.llm_call_duration_secs,
             fn_call_duration_secs=generation_stats.fn_call_duration_secs,
             total_llm_calls=generation_stats.total_llm_calls,

palimpzest/query/operators/join.py

@@ -27,6 +27,25 @@ from palimpzest.query.generators.generators import Generator
 from palimpzest.query.operators.physical import PhysicalOperator
 
 
+class Singleton:
+     def __new__(cls, *args, **kw):
+         if not hasattr(cls, '_instance'):
+             orig = super(Singleton, cls)  # noqa: UP008
+             cls._instance = orig.__new__(cls, *args, **kw)
+         return cls._instance
+
+class Locks(Singleton):
+    model = None
+    clip_lock = threading.Lock()
+    exec_lock = threading.Lock()
+
+    @classmethod
+    def get_model(cls, model_name: str):
+        with cls.clip_lock:
+            if cls.model is None:
+                cls.model = SentenceTransformer(model_name)
+            return cls.model
+
 def compute_similarity(left_embedding: list[float], right_embedding: list[float]) -> float:
     """
     Compute the similarity between two embeddings using cosine similarity.
@@ -357,8 +376,10 @@ class LLMJoin(JoinOp):
             join_condition=self.condition,
             total_input_tokens=generation_stats.total_input_tokens,
             total_output_tokens=generation_stats.total_output_tokens,
+            total_embedding_input_tokens=generation_stats.total_embedding_input_tokens,
             total_input_cost=generation_stats.total_input_cost,
             total_output_cost=generation_stats.total_output_cost,
+            total_embedding_cost=generation_stats.total_embedding_cost,
             llm_call_duration_secs=generation_stats.llm_call_duration_secs,
             fn_call_duration_secs=generation_stats.fn_call_duration_secs,
             total_llm_calls=generation_stats.total_llm_calls,
@@ -487,8 +508,7 @@ class EmbeddingJoin(LLMJoin):
             if field_name.split(".")[-1] in self.get_input_fields()
         ])
         self.embedding_model = Model.TEXT_EMBEDDING_3_SMALL if self.text_only else Model.CLIP_VIT_B_32
-        self.clip_model = None
-        self._lock = threading.Lock()
+        self.locks = Locks()
 
         # keep track of embedding costs that could not be amortized if no output records were produced
         self.residual_embedding_cost = 0.0
@@ -560,28 +580,22 @@ class EmbeddingJoin(LLMJoin):
             quality=quality,
         )
 
-    def _get_clip_model(self):
-        with self._lock:
-            if self.clip_model is None:
-                self.clip_model = SentenceTransformer(self.embedding_model.value)
-            return self.clip_model
-
     def _compute_embeddings(self, candidates: list[DataRecord], input_fields: list[str]) -> tuple[np.ndarray, GenerationStats]:
         # return empty array and empty stats if no candidates  
         if len(candidates) == 0:
             return np.zeros((0, 512)), GenerationStats()
 
         start_time = time.time()
-        total_input_tokens = 0
+        total_embedding_input_tokens = 0
         embeddings = None
         if self.text_only:
             client = OpenAI()
             inputs = [dr.to_json_str(bytes_to_str=True, project_cols=input_fields, sorted=True) for dr in candidates]
             response = client.embeddings.create(input=inputs, model=self.embedding_model.value)
-            total_input_tokens = response.usage.total_tokens
+            total_embedding_input_tokens = response.usage.total_tokens
             embeddings = np.array([item.embedding for item in response.data])
         else:
-            model = self._get_clip_model()
+            model = self.locks.get_model(self.embedding_model.value)
             embeddings = np.zeros((len(candidates), 512))  # CLIP embeddings are 512-dimensional
             num_input_fields_present = 0
             for field in input_fields:
@@ -604,14 +618,16 @@ class EmbeddingJoin(LLMJoin):
 
         # compute cost of embedding(s)
         model_card = MODEL_CARDS[self.embedding_model.value]
-        total_input_cost = model_card["usd_per_input_token"] * total_input_tokens
+        total_embedding_cost = model_card["usd_per_input_token"] * total_embedding_input_tokens
         embedding_gen_stats = GenerationStats(
             model_name=self.embedding_model.value,
-            total_input_tokens=total_input_tokens,
+            total_input_tokens=0.0,
             total_output_tokens=0.0,
-            total_input_cost=total_input_cost,
+            total_embedding_input_tokens=total_embedding_input_tokens,
+            total_input_cost=0.0,
             total_output_cost=0.0,
-            cost_per_record=total_input_cost,
+            total_embedding_cost=total_embedding_cost,
+            cost_per_record=total_embedding_cost,
             llm_call_duration_secs=time.time() - start_time,
             total_llm_calls=1,
             total_embedding_llm_calls=len(candidates),
@@ -623,7 +639,7 @@ class EmbeddingJoin(LLMJoin):
         output_record, output_record_op_stats = super()._process_join_candidate_pair(left_candidate, right_candidate, gen_kwargs)
         return output_record, output_record_op_stats, embedding_sim
 
-    def _process_join_candidate_with_sim(self, left_candidate: DataRecord, right_candidate: DataRecord, passed_operator: bool) -> tuple[DataRecord, RecordOpStats]:
+    def _process_join_candidate_with_sim(self, left_candidate: DataRecord, right_candidate: DataRecord, embedding_sim: float, passed_operator: bool) -> tuple[DataRecord, RecordOpStats]:
         # compute output record and add to output_records
         join_dr = DataRecord.from_join_parents(self.output_schema, left_candidate, right_candidate)
         join_dr._passed_operator = passed_operator
@@ -656,7 +672,7 @@ class EmbeddingJoin(LLMJoin):
             op_details={k: str(v) for k, v in self.get_id_params().items()},
         )
 
-        return join_dr, record_op_stats
+        return join_dr, record_op_stats, embedding_sim
 
     def __call__(self, left_candidates: list[DataRecord], right_candidates: list[DataRecord], final: bool = False) -> tuple[DataRecordSet, int]:
         # get the set of input fields from both records in the join
@@ -690,36 +706,50 @@ class EmbeddingJoin(LLMJoin):
         output_records, output_record_op_stats, num_inputs_processed = [], [], 0
 
         # draw samples until num_samples is reached
-        if self.samples_drawn < self.num_samples:
-            samples_to_draw = min(self.num_samples - self.samples_drawn, len(join_candidates))
-            join_candidate_samples = join_candidates[:samples_to_draw]
-            join_candidates = join_candidates[samples_to_draw:]
-
-            # apply the generator to each pair of candidates
-            with ThreadPoolExecutor(max_workers=self.join_parallelism) as executor:
-                futures = [
-                    executor.submit(self._process_join_candidate_pair, left_candidate, right_candidate, gen_kwargs, embedding_sim)
-                    for left_candidate, right_candidate, embedding_sim in join_candidate_samples
-                ]
-
-                # collect results as they complete
-                for future in as_completed(futures):
-                    self.join_idx += 1
-                    join_output_record, join_output_record_op_stats, embedding_sim = future.result()
-                    output_records.append(join_output_record)
-                    output_record_op_stats.append(join_output_record_op_stats)
-                    print(f"{self.join_idx} JOINED")
-
-                    # update similarity thresholds
-                    records_joined = join_output_record._passed_operator
-                    if not records_joined and embedding_sim > self.max_non_matching_sim:
-                        self.max_non_matching_sim = embedding_sim
-                    if records_joined and embedding_sim < self.min_matching_sim:
-                        self.min_matching_sim = embedding_sim
-
-            # update samples drawn and num_inputs_processed
-            self.samples_drawn += samples_to_draw
-            num_inputs_processed += samples_to_draw
+        with self.locks.exec_lock:
+            if self.samples_drawn < self.num_samples:
+                samples_to_draw = min(self.num_samples - self.samples_drawn, len(join_candidates))
+                join_candidate_samples = join_candidates[:samples_to_draw]
+                join_candidates = join_candidates[samples_to_draw:]
+
+                # apply the generator to each pair of candidates
+                with ThreadPoolExecutor(max_workers=self.join_parallelism) as executor:
+                    futures = [
+                        executor.submit(self._process_join_candidate_pair, left_candidate, right_candidate, gen_kwargs, embedding_sim)
+                        for left_candidate, right_candidate, embedding_sim in join_candidate_samples
+                    ]
+
+                    # collect results as they complete
+                    similarities, joined = [], []
+                    for future in as_completed(futures):
+                        self.join_idx += 1
+                        join_output_record, join_output_record_op_stats, embedding_sim = future.result()
+                        output_records.append(join_output_record)
+                        output_record_op_stats.append(join_output_record_op_stats)
+                        similarities.append(embedding_sim)
+                        joined.append(join_output_record._passed_operator)
+                        print(f"{self.join_idx} JOINED")
+
+                    # sort join results by embedding similarity
+                    sorted_sim_join_tuples = sorted(zip(similarities, joined), key=lambda x: x[0])
+
+                    # compute threshold below which no records joined
+                    for embedding_sim, records_joined in sorted_sim_join_tuples:
+                        if records_joined:
+                            break
+                        if not records_joined and embedding_sim > self.max_non_matching_sim:
+                            self.max_non_matching_sim = embedding_sim
+
+                    # compute threshold above which all records joined
+                    for embedding_sim, records_joined in reversed(sorted_sim_join_tuples):
+                        if not records_joined:
+                            break
+                        if records_joined and embedding_sim < self.min_matching_sim:
+                            self.min_matching_sim = embedding_sim
+
+                # update samples drawn and num_inputs_processed
+                self.samples_drawn += samples_to_draw
+                num_inputs_processed += samples_to_draw
 
         # process remaining candidates based on embedding similarity
         if len(join_candidates) > 0:
@@ -727,43 +757,48 @@ class EmbeddingJoin(LLMJoin):
              with ThreadPoolExecutor(max_workers=self.join_parallelism) as executor:
                 futures = []
                 for left_candidate, right_candidate, embedding_sim in join_candidates:
-                    llm_call_needed = (
-                        self.min_matching_sim == float("inf")
-                        or self.max_non_matching_sim == float("-inf")
-                        or self.min_matching_sim <= embedding_sim <= self.max_non_matching_sim
-                    )
+                    # if the embedding similarity is lower than the threshold below which no records joined,
+                    # then we can skip the LLM call and mark the records as not joined
+                    if embedding_sim < self.max_non_matching_sim:
+                        futures.append(executor.submit(self._process_join_candidate_with_sim, left_candidate, right_candidate, embedding_sim, passed_operator=False))
 
-                    if llm_call_needed:
-                        futures.append(executor.submit(self._process_join_candidate_pair, left_candidate, right_candidate, gen_kwargs, embedding_sim))
+                    # if the embedding similarity is higher than the threshold above which all records joined,
+                    # then we can skip the LLM call and mark the records as joined
+                    elif embedding_sim > self.min_matching_sim:
+                        futures.append(executor.submit(self._process_join_candidate_with_sim, left_candidate, right_candidate, embedding_sim, passed_operator=True))
 
-                    elif embedding_sim < self.min_matching_sim:
-                        self.join_idx += 1
-                        output_record, record_op_stats = self._process_join_candidate_with_sim(left_candidate, right_candidate, passed_operator=False)
-                        output_records.append(output_record)
-                        output_record_op_stats.append(record_op_stats)
-                        print(f"{self.join_idx} SKIPPED (low sim: {embedding_sim:.4f} < {self.min_matching_sim:.4f})")
-
-                    elif embedding_sim > self.max_non_matching_sim:
-                        self.join_idx += 1
-                        output_record, record_op_stats = self._process_join_candidate_with_sim(left_candidate, right_candidate, passed_operator=True)
-                        output_records.append(output_record)
-                        output_record_op_stats.append(record_op_stats)
-                        print(f"{self.join_idx} JOINED (high sim: {embedding_sim:.4f} > {self.max_non_matching_sim:.4f})")
+                    # otherwise, we will process the LLM call
+                    else:
+                        futures.append(executor.submit(self._process_join_candidate_pair, left_candidate, right_candidate, gen_kwargs, embedding_sim))
 
                     num_inputs_processed += 1
 
                 # collect results as they complete
+                similarities, joined = [], []
                 for future in as_completed(futures):
                     self.join_idx += 1
                     join_output_record, join_output_record_op_stats, embedding_sim = future.result()
                     output_records.append(join_output_record)
                     output_record_op_stats.append(join_output_record_op_stats)
+                    similarities.append(embedding_sim)
+                    joined.append(join_output_record._passed_operator)
                     print(f"{self.join_idx} JOINED")
 
-                    # update similarity thresholds
-                    records_joined = join_output_record._passed_operator
+                ### update thresholds if there are llm calls which incrementally squeeze the boundaries ###
+                # sort join results by embedding similarity
+                sorted_sim_join_tuples = sorted(zip(similarities, joined), key=lambda x: x[0])
+
+                # potentially update threshold below which no records joined
+                for embedding_sim, records_joined in sorted_sim_join_tuples:
+                    if records_joined:
+                        break
                     if not records_joined and embedding_sim > self.max_non_matching_sim:
                         self.max_non_matching_sim = embedding_sim
+
+                # potentially update threshold above which all records joined
+                for embedding_sim, records_joined in reversed(sorted_sim_join_tuples):
+                    if not records_joined:
+                        break
                     if records_joined and embedding_sim < self.min_matching_sim:
                         self.min_matching_sim = embedding_sim
 

palimpzest/query/operators/rag.py

@@ -109,15 +109,17 @@ class RAGConvert(LLMConvert):
 
         # compute the generation stats object
         model_card = MODEL_CARDS[model_name]
-        total_input_tokens = response.usage.total_tokens
-        total_input_cost = model_card["usd_per_input_token"] * total_input_tokens
+        total_embedding_input_tokens = response.usage.total_tokens
+        total_embedding_cost = model_card["usd_per_input_token"] * total_embedding_input_tokens
         embed_stats = GenerationStats(
             model_name=model_name,  # NOTE: this should be overwritten by generation model in convert()
-            total_input_tokens=total_input_tokens,
+            total_input_tokens=0.0,
             total_output_tokens=0.0,
-            total_input_cost=total_input_cost,
+            total_embedding_input_tokens=total_embedding_input_tokens,
+            total_input_cost=0.0,
             total_output_cost=0.0,
-            cost_per_record=total_input_cost,
+            total_embedding_cost=total_embedding_cost,
+            cost_per_record=total_embedding_cost,
             llm_call_duration_secs=total_time,
             total_llm_calls=1,
             total_embedding_llm_calls=1,
@@ -156,7 +158,7 @@ class RAGConvert(LLMConvert):
             # skip this field if it is not a string or a list of strings
             is_string_field = field.annotation in [str, str | None, str | Any]
             is_list_string_field = field.annotation in [list[str], list[str] | None, list[str] | Any]
-            if not (is_string_field or is_list_string_field):
+            if not (is_string_field or is_list_string_field) or candidate[field_name] is None:
                 continue
 
             # if this is a list of strings, join the strings
@@ -318,15 +320,17 @@ class RAGFilter(LLMFilter):
 
         # compute the generation stats object
         model_card = MODEL_CARDS[model_name]
-        total_input_tokens = response.usage.total_tokens
-        total_input_cost = model_card["usd_per_input_token"] * total_input_tokens
+        total_embedding_input_tokens = response.usage.total_tokens
+        total_embedding_cost = model_card["usd_per_input_token"] * total_embedding_input_tokens
         embed_stats = GenerationStats(
             model_name=model_name,  # NOTE: this should be overwritten by generation model in filter()
-            total_input_tokens=total_input_tokens,
+            total_input_tokens=0.0,
             total_output_tokens=0.0,
-            total_input_cost=total_input_cost,
+            total_embedding_input_tokens=total_embedding_input_tokens,
+            total_input_cost=0.0,
             total_output_cost=0.0,
-            cost_per_record=total_input_cost,
+            total_embedding_cost=total_embedding_cost,
+            cost_per_record=total_embedding_cost,
             llm_call_duration_secs=total_time,
             total_llm_calls=1,
             total_embedding_llm_calls=1,

palimpzest/query/operators/topk.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import threading
 import time
 from typing import Callable
 
@@ -17,6 +18,24 @@ from palimpzest.core.models import GenerationStats, OperatorCostEstimates, Recor
 from palimpzest.query.operators.physical import PhysicalOperator
 
 
+class Singleton:
+     def __new__(cls, *args, **kw):
+         if not hasattr(cls, '_instance'):
+             orig = super(Singleton, cls)  # noqa: UP008
+             cls._instance = orig.__new__(cls, *args, **kw)
+         return cls._instance
+
+class ClipModel(Singleton):
+    model = None
+    lock = threading.Lock()
+
+    @classmethod
+    def get_model(cls, model_name: str):
+        with cls.lock:
+            if cls.model is None:
+                cls.model = SentenceTransformer(model_name)
+            return cls.model
+
 class TopKOp(PhysicalOperator):
     def __init__(
         self,
@@ -56,6 +75,7 @@ class TopKOp(PhysicalOperator):
         self.output_attrs = output_attrs
         self.search_func = search_func if search_func is not None else self.default_search_func
         self.k = k
+        self.clip_model = ClipModel()
 
     def __str__(self):
         op = super().__str__()
@@ -185,7 +205,6 @@ class TopKOp(PhysicalOperator):
         # construct and return the record set
         return DataRecordSet(drs, record_op_stats_lst)
 
-
     def __call__(self, candidate: DataRecord) -> DataRecordSet:
         start_time = time.time()
 
@@ -209,9 +228,9 @@ class TopKOp(PhysicalOperator):
         inputs, gen_stats = None, GenerationStats()
         if isinstance(self.index, Collection):
             uses_openai_embedding_fcn = isinstance(self.index._embedding_function, OpenAIEmbeddingFunction)
-            uses_sentence_transformer_embedding_fcn = isinstance(self.index._embedding_function, SentenceTransformerEmbeddingFunction)
+            uses_clip_model = isinstance(self.index._embedding_function, SentenceTransformerEmbeddingFunction)
             error_msg = "ChromaDB index must use OpenAI or SentenceTransformer embedding function; see: https://docs.trychroma.com/integrations/embedding-models/openai"
-            assert uses_openai_embedding_fcn or uses_sentence_transformer_embedding_fcn, error_msg
+            assert uses_openai_embedding_fcn or uses_clip_model, error_msg
 
             model_name = self.index._embedding_function.model_name if uses_openai_embedding_fcn else "clip-ViT-B-32"
             err_msg = f"For Chromadb, we currently only support `text-embedding-3-small` and `clip-ViT-B-32`; your index uses: {model_name}"
@@ -228,8 +247,8 @@ class TopKOp(PhysicalOperator):
                     total_input_tokens = response.usage.total_tokens
                     inputs = [item.embedding for item in response.data]
 
-                elif uses_sentence_transformer_embedding_fcn:
-                    model = SentenceTransformer(model_name)
+                elif uses_clip_model:
+                    model = self.clip_model.get_model(model_name)
                     inputs = model.encode(query)
 
                 embed_total_time = time.time() - embed_start_time

palimpzest/query/optimizer/cost_model.py

@@ -105,9 +105,10 @@ class SampleBasedCostModel:
                     "time_per_record": record_op_stats.time_per_record,
                     "quality": record_op_stats.quality,
                     "passed_operator": record_op_stats.passed_operator,
-                    "source_indices": record_op_stats.record_source_indices,  # TODO: remove
-                    "op_details": record_op_stats.op_details,                 # TODO: remove
-                    "answer": record_op_stats.answer,                         # TODO: remove
+                    "source_indices": record_op_stats.record_source_indices,
+                    "op_details": record_op_stats.op_details,
+                    "answer": record_op_stats.answer,
+                    "op_name": record_op_stats.op_name,
                 }
                 execution_record_op_stats.append(record_op_stats_dict)
 
@@ -128,8 +129,12 @@ class SampleBasedCostModel:
                     else physical_op_df.source_indices.apply(tuple).nunique()
                 )
 
-                # compute selectivity
+                # compute selectivity; for filters this may be 1.0 on smalle samples;
+                # always put something slightly less than 1.0 to ensure that filters are pushed down when possible
                 selectivity = physical_op_df.passed_operator.sum() / num_source_records
+                op_name = physical_op_df.op_name.iloc[0].lower()
+                if selectivity == 1.0 and "filter" in op_name:
+                    selectivity -= 1e-3
 
                 # compute quality; if all qualities are None then this will be NaN
                 quality = physical_op_df.quality.mean()

palimpzest/query/processor/config.py

@@ -48,6 +48,7 @@ class QueryProcessorConfig(BaseModel):
     seed: int = Field(default=42)
     exp_name: str | None = Field(default=None)
     priors: dict | None = Field(default=None)
+    dont_use_priors: bool = Field(default=False)
 
     def to_dict(self) -> dict:
         """Convert the config to a dict representation."""

palimpzest/query/processor/query_processor_factory.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from enum import Enum
 
 from palimpzest.core.data.dataset import Dataset
@@ -91,6 +92,27 @@ class QueryProcessorFactory:
         # set the final set of available models in the config
         config.available_models = available_models
 
+        if len(config.available_models) == 0:
+            raise ValueError("No available models found.")
+
+        openai_key = os.getenv("OPENAI_API_KEY")
+        anthropic_key = os.getenv("ANTHROPIC_API_KEY")
+        together_key = os.getenv("TOGETHER_API_KEY")
+        gemini_key = os.getenv("GEMINI_API_KEY")
+        google_key = os.getenv("GOOGLE_API_KEY")
+
+        for model in config.available_models:
+            if model.is_openai_model() and not openai_key:
+                raise ValueError("OPENAI_API_KEY must be set to use OpenAI models.")
+            if model.is_anthropic_model() and not anthropic_key:
+                raise ValueError("ANTHROPIC_API_KEY must be set to use Anthropic models.")
+            if model.is_together_model() and not together_key:
+                raise ValueError("TOGETHER_API_KEY must be set to use Together models.")
+            if model.is_google_model() and not (gemini_key or google_key or config.gemini_credentials_path):
+                raise ValueError("GEMINI_API_KEY, GOOGLE_API_KEY, or gemini_credentials path must be set to use Google Gemini models.")
+            if model.is_vllm_model() and config.api_base is None:
+                raise ValueError("api_base must be set to use vLLM models.")
+
         return config, validator
 
     @classmethod

{palimpzest-1.1.0.dist-info → palimpzest-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: palimpzest
-Version: 1.1.0
+Version: 1.2.0
 Summary: Palimpzest is a system which enables anyone to process AI-powered analytical queries simply by defining them in a declarative language
 Author-email: MIT DSG Semantic Management Lab <michjc@csail.mit.edu>
 Project-URL: homepage, https://palimpzest.org
@@ -31,7 +31,7 @@ Requires-Dist: pillow>=11.3.0
 Requires-Dist: prettytable>=3.9.0
 Requires-Dist: psutil==5.9.5
 Requires-Dist: PyLD>=2.0.4
-Requires-Dist: pyarrow==20.0.0
+Requires-Dist: pyarrow>=20.0.0
 Requires-Dist: pypdf>=5.1.0
 Requires-Dist: pytest-mock>=3.14.0
 Requires-Dist: pyyaml>=6.0.1

{palimpzest-1.1.0.dist-info → palimpzest-1.2.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ palimpzest/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 palimpzest/agents/compute_agents.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 palimpzest/agents/search_agents.py,sha256=t2QMreB5Ph71aoNk5bBtV-0l8im79z-pMAR3JDAySDw,29418
 palimpzest/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/core/models.py,sha256=FKyKW9PqmqpnDGWOINNT6XgBj0raaAskxtdNdFZ4Zyw,42688
+palimpzest/core/models.py,sha256=t4zHPA-Nrz2Mmq2EZfJWU_CsSbzu4LFv6_Wob10MZnc,46110
 palimpzest/core/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 palimpzest/core/data/context.py,sha256=x1xYyu9qW65dvtK_XayIfv_CgsCEPW6Qe0DTiSf9sjU,16207
 palimpzest/core/data/context_manager.py,sha256=8hAKWD2jhFZgghTu7AYgjkvKDsJUPVxq8g4nG0HWvfo,6150
@@ -28,7 +28,7 @@ palimpzest/prompts/filter_prompts.py,sha256=D-aY3-th1GzEHrVGbKORVN2R7x7coYGjp8Fr
 palimpzest/prompts/join_prompts.py,sha256=z-y4L1cw1O3I_F9DW6MvqeztdQoKDQawX6nK6vQAkdM,2916
 palimpzest/prompts/moa_aggregator_prompts.py,sha256=b5cz4G2oF86LlHOy8vmtxoMcZ9zaZoppKrURHgzCzNU,5248
 palimpzest/prompts/moa_proposer_prompts.py,sha256=yfZYwmCg-Tg9h0H7PJMEuDYPR45EbYnORmVX6cY2vRQ,3125
-palimpzest/prompts/prompt_factory.py,sha256=0xj3glD5Y7R7MUsmKxJCOa4q9VeIILDO2IVWz_4huYw,49355
+palimpzest/prompts/prompt_factory.py,sha256=txtCvDI0sv_LEar0iK_E1_mlRMvuwZseM-6BSC9ugUs,49926
 palimpzest/prompts/split_merge_prompts.py,sha256=hX-MThmW4VU7rjgm7gb-bpniEMdj25mtp0o8qBeWvIQ,5573
 palimpzest/prompts/split_proposer_prompts.py,sha256=Ucqwfn4FqFk-b9E024EK4e_3_QndTJjggwiwa1x5CQs,3115
 palimpzest/prompts/utils.py,sha256=Eure2pqm8Ftme9lQlHwFL9EqK3yjH14WQHofnQINce4,7497
@@ -36,9 +36,9 @@ palimpzest/prompts/validator.py,sha256=OxebGjvXNBy0Cq79XI3aPRbongzOdtHH6mQctpbWc
 palimpzest/query/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 palimpzest/query/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 palimpzest/query/execution/all_sample_execution_strategy.py,sha256=d2MO_AmXF_HbV4rUNkFqHsuoBCofU98zQ-D3Q06BXCc,14369
-palimpzest/query/execution/execution_strategy.py,sha256=PkXtO8wZpv6HHLlpxXwcc0t5pPCGafp0L_iVkG9bmXM,19162
+palimpzest/query/execution/execution_strategy.py,sha256=TnSInUlcGZHn2GUpLiIFVgfPpmsNfIKKgElnRt6a6ss,19248
 palimpzest/query/execution/execution_strategy_type.py,sha256=vRQBPCQN5_aoyD3TLIeW3VPo15mqF-5RBvEXkENz9FE,987
-palimpzest/query/execution/mab_execution_strategy.py,sha256=i03LYRhaG2VLia-XSiYbKdlu3hLQZul75xMcRGm065M,47767
+palimpzest/query/execution/mab_execution_strategy.py,sha256=BLRTSQXPeWBlJ_-8GAFHj2fbIY_eoPhuWeDcIdOokcg,48247
 palimpzest/query/execution/parallel_execution_strategy.py,sha256=Di-8d7waE0bev4kNDXEJJqQ0wwQ87_sPV-t5qFtAlPQ,17589
 palimpzest/query/execution/single_threaded_execution_strategy.py,sha256=1rjMel0-AI6KUi_SMNgPPXxMgG5-t9lenLKoYEClgjk,17464
 palimpzest/query/generators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -46,23 +46,23 @@ palimpzest/query/generators/generators.py,sha256=OV2HBZvCQtvhj6rOwti_8dpQX_bqTZ-
 palimpzest/query/operators/__init__.py,sha256=B9zr_VmUs6YRep4fjbj7e0aTM6T9-GrqbY7tKWxEdkc,4734
 palimpzest/query/operators/aggregate.py,sha256=nQ6Zh1DYeqDoIDwkPQDw8QCwW0y52sGC-No6uPSRc8A,27367
 palimpzest/query/operators/compute.py,sha256=X_pWN45smg8L4dV54nOae7dldQGL1nJVlVyJ3ULWSmI,8432
-palimpzest/query/operators/convert.py,sha256=cjUPrSgvZBZXBbrbepIxZMBXjbWWPLuTX4JwLyvVg2U,16050
+palimpzest/query/operators/convert.py,sha256=beJLS-vnVc_VLnngoDKTj-k_Ul7GmDA-yMKM6-mX5Ho,16218
 palimpzest/query/operators/critique_and_refine.py,sha256=Q-NhasVoD9meX7g36RPrv3q4R48_8XEU4d3TE46hRJI,8979
 palimpzest/query/operators/distinct.py,sha256=ZTXlIS7IaFRTsWv9RemzCo1JLz25vEma-TB42CV5fJQ,2614
-palimpzest/query/operators/filter.py,sha256=ufREsO2-8CBk4u4fabDBYpEvb806E11EOyW-wuRs4vw,10356
-palimpzest/query/operators/join.py,sha256=17BGzrxf_fkqhnEzhq-5b0qv2qQTw7z6job5YkBUrZE,36993
+palimpzest/query/operators/filter.py,sha256=h559CweLdcWw_-LPyR2h04pKsm-jVM_Kazif-BXYBFo,10516
+palimpzest/query/operators/join.py,sha256=hxIbSSDMy_5bch7kpYQ-iQS5XEth_T19cTSlXd76gdg,38845
 palimpzest/query/operators/limit.py,sha256=pdo7WfWY97SW3c-WqZ4SIPw7lHIVbaXPEWqHyK8qkF8,2130
 palimpzest/query/operators/logical.py,sha256=OtB82L1X19ibtLx1GIfeXXyO7YfjkFmh3puIUgqKQRE,21160
 palimpzest/query/operators/mixture_of_agents.py,sha256=KC-ZpjtGY28sfwlk2TpduLC_fADj_UASFCaicaKqSFc,11671
 palimpzest/query/operators/physical.py,sha256=0_BfFX9nKuN__440eAfEfApWAoGOunVSCZIQxS4HO2Y,9773
 palimpzest/query/operators/project.py,sha256=gxbjsHEACCP9wxATH-mw6wOSUi5s13JyWsLqqhAYWXQ,2111
-palimpzest/query/operators/rag.py,sha256=CJm83pBapA8HEGfhRnWjqt_ESS6hJAPvPJksRTOGL7M,20124
+palimpzest/query/operators/rag.py,sha256=ZDloc4nC8foI3rTSHQxqduAVPj5LV8xMu_ng0EjDOA0,20409
 palimpzest/query/operators/scan.py,sha256=OqCiPRTvTY7SbauNMyFvGT5nRVeRzVsGYSrkoN1Ib_w,7407
 palimpzest/query/operators/search.py,sha256=cQin-Qc9FT7V0Gv3-pxMLbVMjqE6ALe99V0OrQhA6CI,22711
 palimpzest/query/operators/split.py,sha256=oLzwnYb8TNf3XA9TMKEAIw7EIA12wHneaD42BNLIHiI,15043
-palimpzest/query/operators/topk.py,sha256=92Bu98xc8CMlS9bf1xc0FxcfVuhv6j4x_303Aq1v-U0,13053
+palimpzest/query/operators/topk.py,sha256=MZl83Cu43QmN4skjlfpR8EVFFCgA7sR6PbGgBGWC0tg,13564
 palimpzest/query/optimizer/__init__.py,sha256=v9fSBOL2p3sQew4LrN2DQUPe0WezO328Hr54qBTqrAs,2799
-palimpzest/query/optimizer/cost_model.py,sha256=p7AsR6f4VYdGjrUKPGN_VTErY36GjY90Bsvsys4le2M,12655
+palimpzest/query/optimizer/cost_model.py,sha256=JaxdLuUZuq52BJ52YdW4ChfWptwXsh7Rk7oaPCn_gWc,12956
 palimpzest/query/optimizer/optimizer.py,sha256=ksLkzQ2sVgJFbkxGF3ncF74EsAHZFos8G19xlHQrtJo,20063
 palimpzest/query/optimizer/optimizer_strategy.py,sha256=0foDaBHqQehK_zz6IlDEbNIw-44wxY6LO5H1anJi56Y,10042
 palimpzest/query/optimizer/optimizer_strategy_type.py,sha256=V-MMHvJdnfZKoUX1xxxwh66q1RjN2FL35IsiT1C62c8,1084
@@ -71,9 +71,9 @@ palimpzest/query/optimizer/primitives.py,sha256=jMMVq37y1tWiPU1lSSKQP9OP-mzkpSxS
 palimpzest/query/optimizer/rules.py,sha256=awhe76trskv5Tq5E2QHpUN_YV6jH8INywa0Ige8IIhY,53341
 palimpzest/query/optimizer/tasks.py,sha256=DNJjY2QldfKFWj6INHElMh88dYc36Z5m3wHwbs4jyF4,30455
 palimpzest/query/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-palimpzest/query/processor/config.py,sha256=MkZ776VUk9tIOCZdVyH__H4Z0gO4c8fpehX2Gqywvks,2472
+palimpzest/query/processor/config.py,sha256=8-MpPYHv2SI4dub4MP_gOYSRxO80_ALLuWRxD-F2YOg,2521
 palimpzest/query/processor/query_processor.py,sha256=T4ffPbnOX23G8FDITzmM7Iw7DUEDWIHnwl8XLYllgjg,6240
-palimpzest/query/processor/query_processor_factory.py,sha256=i9L9StqlUi7m1AqZMuYQWhunqOJi3nLK47skhxq9tIA,8317
+palimpzest/query/processor/query_processor_factory.py,sha256=l9f0C0lngOihZDzH0TK9WdKR9CwwgB6IbNZftonSFR0,9576
 palimpzest/schemabuilder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 palimpzest/schemabuilder/schema_builder.py,sha256=QraGp66dcD-ej6Y2mER40o86G9JqlBkL7swkJzjUAIY,7968
 palimpzest/tools/README.md,sha256=56_6LPG80uc0CLVhTBP6I1wgIffNv9cyTr0TmVZqmrM,483
@@ -89,8 +89,8 @@ palimpzest/utils/progress.py,sha256=eHXrTPTCRHjMdK0EjYRUzSxcV6N1lK8TS3Ju_ZlQLhY,
 palimpzest/utils/udfs.py,sha256=LjHic54B1az-rKgNLur0wOpaz2ko_UodjLEJrazkxvY,1854
 palimpzest/validator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 palimpzest/validator/validator.py,sha256=SvjK09zCpGtK0yM0OasvQlSzyq3loy32DyOOKRmYXC0,15977
-palimpzest-1.1.0.dist-info/licenses/LICENSE,sha256=5GUlHy9lr-Py9kvV38FF1m3yy3NqM18fefuE9wkWumo,1079
-palimpzest-1.1.0.dist-info/METADATA,sha256=0AZq33WMFrxkarQADVPv2OFQu7ko38fzhBOtTQjc3Fw,5359
-palimpzest-1.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-palimpzest-1.1.0.dist-info/top_level.txt,sha256=raV06dJUgohefUn3ZyJS2uqp_Y76EOLA9Y2e_fxt8Ew,11
-palimpzest-1.1.0.dist-info/RECORD,,
+palimpzest-1.2.0.dist-info/licenses/LICENSE,sha256=5GUlHy9lr-Py9kvV38FF1m3yy3NqM18fefuE9wkWumo,1079
+palimpzest-1.2.0.dist-info/METADATA,sha256=IKxg8RllEvn6dgboEJVnxdnd5RwYmXFhIL2FHvoYpWw,5359
+palimpzest-1.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+palimpzest-1.2.0.dist-info/top_level.txt,sha256=raV06dJUgohefUn3ZyJS2uqp_Y76EOLA9Y2e_fxt8Ew,11
+palimpzest-1.2.0.dist-info/RECORD,,