palimpzest 0.7.20__py3-none-any.whl → 0.8.0__py3-none-any.whl

palimpzest/prompts/validator.py

@@ -0,0 +1,239 @@
+### MAP ###
+MAP_VALIDATOR_PROMPT = """You are an intelligent judge whose job is to evaluate how successfully an agent executed a given instruction.
+You will be presented with the input(s) provided to the agent followed by the output produced by the agent.
+
+Each output will be a dictionary. The keys will be **output fields** which were computed by the agent.
+
+Your job will be to assign a score of 1.0 to every output field which was computed correctly, and a score of 0.0 to every output field which was computed incorrectly. If the output for a field is a list, you may give a score in between 0.0 and 1.0 representing the fraction of correct items in the list.
+
+Here is an example evaluation:
+
+INPUT MESSAGES:
+---------------
+You are a helpful assistant whose job is to generate a JSON object. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.
+You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.
+
+INPUT FIELDS:
+- text: a text passage describing a scientist
+- birthday: the scientist's birthday
+
+OUTPUT FIELDS:
+- name: the name of the scientist
+- birth_year: the year the scientist was born
+
+CONTEXT:
+{{
+  "text": "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, was an English mathematician and writer chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation.",
+  "birthday": "December 10, 1815"
+}}
+
+OUTPUT:
+--------
+{{
+  "name": "Charles Babbage",
+  "birth_year": 1815
+}}
+
+EVALUATION: {"name": 0.0, "birth_year": 1.0}
+
+Remember, be sure to output your evaluation as a dictionary where each value contains a 0.0 or 1.0 score for each output field (or a score within [0.0, 1.0] for list output fields).
+
+INPUT MESSAGES:
+---------------
+
+"""
+
+MAP_IMAGE_VALIDATOR_PROMPT = """You are an intelligent judge whose job is to evaluate how successfully an agent executed a given instruction.
+You will be presented with the input(s) provided to the agent followed by the output produced by the agent.
+
+Each output will be a dictionary. The keys will be **output fields** which were computed by the agent.
+
+Your job will be to assign a score of 1.0 to every output field which was computed correctly, and a score of 0.0 to every output field which was computed incorrectly. If the output for a field is a list, you may give a score in between 0.0 and 1.0 representing the fraction of correct items in the list.
+
+Here is an example evaluation:
+
+INPUT MESSAGES:
+---------------
+You are a helpful assistant whose job is to analyze input image(s) and/or text in order to produce a JSON object. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.
+You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.
+
+INPUT FIELDS:
+- image: an image of a scene
+- photographer: the photographer of the image
+
+OUTPUT FIELDS:
+- dog_in_image: true if a dog is in the image and false otherwise
+- person_in_image: true if a person is in the image and false otherwise
+
+CONTEXT:
+{{
+  "image": <bytes>,
+  "photographer": "CameraEnthusiast1"
+}}
+<image content provided here; assume in this example the image shows a dog and a cat playing>
+
+OUTPUT:
+--------
+{{
+  "dog_in_image": true,
+  "person_in_image": true
+}}
+
+EVALUATION: {"dog_in_image": 1.0, "person_in_image": 0.0}
+
+Remember, be sure to output your evaluation as a dictionary where each value contains a 0.0 or 1.0 score for each output field (or a score within [0.0, 1.0] for list output fields).
+
+INPUT MESSAGES:
+---------------
+
+"""
+
+
+### FLAT MAP ###
+FLAT_MAP_VALIDATOR_PROMPT = """You are an intelligent judge whose job is to evaluate how successfully an agent executed a given instruction.
+You will be presented with the input(s) provided to the agent followed by the output(s) produced by the agent.
+
+Each output will be a list of dictionaries. The keys of each dictionary will be **output fields** which were computed by the agent.
+
+Your job will be to assign a score of 1.0 to every output field which was computed correctly, and a score of 0.0 to every output field which was computed incorrectly. If the output for a field is a list, you may give a score in between 0.0 and 1.0 representing the fraction of correct items in the list.
+
+Here is an example evaluation:
+
+INPUT MESSAGES:
+---------------
+You are a helpful assistant whose job is to generate a JSON object. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.
+You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.
+
+INPUT FIELDS:
+- text: a text passage describing scientists
+- birthdays: text containing birth dates
+
+OUTPUT FIELDS:
+- name: the name of the scientist
+- birth_year: the year the scientist was born
+
+CONTEXT:
+{{
+  "text": "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, was an English mathematician and writer chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation.",
+  "birthdays": "...Lovelace was born on December 10, 1815, almost exactly 24 years after Babbage's birth on 26 December 1791..."
+}}
+
+OUTPUTS:
+--------
+[
+  {{
+    "name": "Ada Lovelace",
+    "birth_year": 1815
+  }},
+  {{
+    "name": "Charles Babbage",
+    "birth_year": 1790
+  }}
+]
+
+EVALUATION: [{"name": 1.0, "birth_year": 1.0}, {"name": 1.0, "birth_year": 0.0}]
+
+Remember, be sure to output your evaluation as a list of dictionaries where each dictionary contains a 0.0 or 1.0 score for each output field (or a score within [0.0, 1.0] for list output fields).
+
+INPUT MESSAGES:
+---------------
+
+"""
+
+FLAT_MAP_IMAGE_VALIDATOR_PROMPT = """You are an intelligent judge whose job is to evaluate how successfully an agent executed a given instruction.
+You will be presented with the input(s) provided to the agent followed by the output(s) produced by the agent.
+
+Each output will be a list of dictionaries. The keys of each dictionary will be **output fields** which were computed by the agent.
+
+Your job will be to assign a score of 1.0 to every output field which was computed correctly, and a score of 0.0 to every output field which was computed incorrectly. If the output for a field is a list, you may give a score in between 0.0 and 1.0 representing the fraction of correct items in the list.
+
+Here is an example evaluation:
+
+INPUT MESSAGES:
+---------------
+You are a helpful assistant whose job is to analyze input image(s) and/or text in order to produce a JSON object. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.
+You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.
+
+INPUT FIELDS:
+- image: an image of a scene
+- photographer: the photographer of the image
+
+OUTPUT FIELDS:
+- animal: the type of animal in the image
+- animal_is_canine: true if the animal is a canine and false otherwise
+
+CONTEXT:
+{{
+  "image": <bytes>,
+  "photographer": "CameraEnthusiast1"
+}}
+<image content provided here; assume in this example the image shows a dog and a cat playing>
+
+OUTPUT:
+--------
+[
+  {{
+    "animal": "dog",
+    "animal_is_canine": true
+  }},
+  {{
+    "animal": "cat",
+    "animal_is_canine": true
+  }}
+]
+
+EVALUATION: [{"animal": 1.0, "animal_is_canine": 1.0}, {"animal": 1.0, "animal_is_canine": 0.0}]
+
+Remember, be sure to output your evaluation as a list of dictionaries where each dictionary contains a 0.0 or 1.0 score for each output field (or a score within [0.0, 1.0] for list output fields).
+
+INPUT MESSAGES:
+---------------
+
+"""
+
+
+### RETRIEVE
+RETRIEVE_VALIDATOR_PROMPT = """You are an intelligent judge whose job is to evaluate how successfully an agent executed a given instruction.
+You will be presented with the input(s) provided to the agent followed by the output produced by the agent.
+
+Each output will be a dictionary. The keys will be **output fields** which were computed by the agent.
+
+Your job will be to assign a score of 1.0 to every output field which was computed correctly, and a score of 0.0 to every output field which was computed incorrectly. If the output for a field is a list, you may give a score in between 0.0 and 1.0 representing the fraction of correct items in the list.
+
+Here is an example evaluation:
+
+INPUT MESSAGES:
+---------------
+You are a helpful assistant whose job is to generate a JSON object. You will be presented with a context and a set of output fields to generate. Your task is to generate a JSON object which fills in the output fields with the correct values.
+You will be provided with a description of each input field and each output field. All of the fields in the output JSON object can be derived using information from the context.
+
+INPUT FIELDS:
+- text: a text passage describing a scientist
+
+OUTPUT FIELDS:
+- related_scientists: list of scientists who perform similar work as the scientist described in the text
+
+CONTEXT:
+{{
+  "text": "Augusta Ada King, Countess of Lovelace, also known as Ada Lovelace, was an English mathematician and writer chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation.",
+}}
+
+OUTPUT:
+--------
+{{
+  "related_scientists": [
+    "Charles Babbage",
+    "Alan Turing",
+    "Charles Darwin",
+    "John von Neumann",
+  ]
+}}
+
+EVALUATION: {"related_scientists": 0.75}
+
+Remember, be sure to output your evaluation as a dictionary where each value contains a 0.0 or 1.0 score for each output field (or a score within [0.0, 1.0] for list output fields).
+
+INPUT MESSAGES:
+---------------
+
+"""

palimpzest/query/execution/all_sample_execution_strategy.py

@@ -2,13 +2,18 @@ import logging
 
 import numpy as np
 
-from palimpzest.core.data.dataclasses import SentinelPlanStats
+from palimpzest.core.data.dataset import Dataset
 from palimpzest.core.elements.records import DataRecord, DataRecordSet
+from palimpzest.core.models import SentinelPlanStats
 from palimpzest.query.execution.execution_strategy import SentinelExecutionStrategy
+from palimpzest.query.operators.aggregate import AggregateOp
+from palimpzest.query.operators.filter import FilterOp
+from palimpzest.query.operators.join import JoinOp
 from palimpzest.query.operators.physical import PhysicalOperator
-from palimpzest.query.operators.scan import ScanPhysicalOp
+from palimpzest.query.operators.scan import ContextScanOp, ScanPhysicalOp
 from palimpzest.query.optimizer.plan import SentinelPlan
 from palimpzest.utils.progress import create_progress_manager
+from palimpzest.validator.validator import Validator
 
 logger = logging.getLogger(__name__)
 
@@ -21,36 +26,79 @@ class OpSet:
     2. has been sampled fewer than j times
     """
 
-    def __init__(self, op_set: list[PhysicalOperator], source_indices: list[int]):
+    def __init__(self, op_set: list[PhysicalOperator], source_unique_logical_op_ids: list[str], source_indices: list[int]):
         # construct the set of operators
         self.ops = op_set
 
         # store the order in which we will sample the source records
         self.source_indices = source_indices
 
+        # boolean indication of the type of operator in this OpSet
+        sample_op = op_set[0]
+        self.is_scan_op = isinstance(sample_op, (ScanPhysicalOp, ContextScanOp))
+        self.is_filter_op = isinstance(sample_op, FilterOp)
+        self.is_aggregate_op = isinstance(sample_op, AggregateOp)
+        self.is_llm_join = isinstance(sample_op, JoinOp)
+
         # set the initial inputs for this logical operator
-        is_scan_op = isinstance(op_set[0], ScanPhysicalOp)
-        self.source_idx_to_input = {source_idx: [source_idx] for source_idx in self.source_indices} if is_scan_op else {}
+        self.source_indices_to_inputs = {source_unique_logical_op_id: {} for source_unique_logical_op_id in source_unique_logical_op_ids}
+        if self.is_scan_op:
+            self.source_indices_to_inputs["source"] = {source_idx: [int(source_idx.split("-")[-1])] for source_idx in self.source_indices}
 
-    def get_op_input_pairs(self) -> list[PhysicalOperator, DataRecord | int | None]:
+    def get_op_inputs(self) -> list[PhysicalOperator, DataRecord | int | None]:
         """
-        Returns the list of frontier operators and their next input to process. If there are
-        any indices in `source_indices_to_sample` which this operator does not sample on its own, then
-        we also have this frontier process that source_idx's input with its max quality operator.
+        Returns the list of frontier operators and their next input to process.
         """
-        # get the list of (op, source_idx) pairs which this operator needs to execute
-        op_source_idx_pairs = []
+        # if this is an aggregate, run on every input
+        if self.is_aggregate_op:
+            op = self.ops[0]
+            all_inputs = []
+            for _, source_indices_to_inputs in self.source_indices_to_inputs.items():
+                for _, inputs in source_indices_to_inputs.items():
+                    all_inputs.extend(inputs)
+            return [(op, tuple(), all_inputs)]
+
+        # if this is an un-optimized (non-scan, non-join) operator, flatten inputs and run on each one
+        elif not self.is_scan_op and not self.is_llm_join and len(self.ops) == 1:
+            op_inputs = []
+            op = self.ops[0]
+            for _, source_indices_to_inputs in self.source_indices_to_inputs.items():
+                for source_indices, inputs in source_indices_to_inputs.items():
+                    for input in inputs:
+                        op_inputs.append((op, source_indices, input))
+            return op_inputs
+
+        # get the list of (op, source_indices) pairs which this operator needs to execute
+        op_source_indices_pairs = []
         for op in self.ops:
-            # construct list of inputs by looking up the input for the given source_idx
-            for source_idx in self.source_indices:
-                op_source_idx_pairs.append((op, source_idx))
-
-        # fetch the corresponding (op, input) pairs
-        op_input_pairs = []
-        for op, source_idx in op_source_idx_pairs:
-            op_input_pairs.extend([(op, input_record) for input_record in self.source_idx_to_input[source_idx]])
-
-        return op_input_pairs
+            # construct list of inputs by looking up the input for the given source_indices
+            for source_indices in self.source_indices:
+                op_source_indices_pairs.append((op, source_indices))
+
+        # construct the op inputs
+        op_inputs = []
+        if self.is_llm_join:
+            left_source_unique_logical_op_id, right_source_unique_logical_op_id = list(self.source_indices_to_inputs)
+            left_source_indices_to_inputs = self.source_indices_to_inputs[left_source_unique_logical_op_id]
+            right_source_indices_to_inputs = self.source_indices_to_inputs[right_source_unique_logical_op_id]
+            for op, source_indices in op_source_indices_pairs:
+                left_source_indices = source_indices[0]
+                right_source_indices = source_indices[1]
+                left_inputs = left_source_indices_to_inputs.get(left_source_indices, [])
+                right_inputs = right_source_indices_to_inputs.get(right_source_indices, [])
+                if len(left_inputs) > 0 and len(right_inputs) > 0:
+                    op_inputs.append((op, (left_source_indices, right_source_indices), (left_inputs, right_inputs)))
+            return op_inputs
+
+        # if operator is not a join
+        source_unique_logical_op_id = list(self.source_indices_to_inputs)[0]
+        op_inputs = [
+            (op, source_indices, input)
+            for op, source_indices in op_source_indices_pairs
+            for input in self.source_indices_to_inputs[source_unique_logical_op_id].get(source_indices, [])
+        ]
+
+        return op_inputs
 
     def pick_highest_quality_output(self, record_sets: list[DataRecordSet]) -> DataRecordSet:
         # if there's only one operator in the set, we return its record_set
@@ -100,75 +148,64 @@ class OpSet:
             for record in max_quality_record_set:
                 input.append(record if record.passed_operator else None)
 
-            self.source_idx_to_input[source_idx] = input
-
+            self.source_indices_to_inputs[source_idx] = input
 
 class AllSamplingExecutionStrategy(SentinelExecutionStrategy):
 
-    def _get_source_indices(self):
-        """Get the list of source indices which the sentinel plan should execute over."""
-        # create list of all source indices and shuffle it
-        total_num_samples = len(self.val_datasource)
-        source_indices = list(np.arange(total_num_samples))
-
-        return source_indices
-
     def _execute_sentinel_plan(self,
             plan: SentinelPlan,
             op_sets: dict[str, OpSet],
-            expected_outputs: dict[int, dict] | None,
+            validator: Validator,
             plan_stats: SentinelPlanStats,
         ) -> SentinelPlanStats:
         # execute operator sets in sequence
-        for op_idx, (logical_op_id, op_set) in enumerate(plan):
+        for topo_idx, (logical_op_id, _) in enumerate(plan):
+            # compute unique logical op id within plan
+            unique_logical_op_id = f"{topo_idx}-{logical_op_id}"
+
             # get frontier ops and their next input
-            op_input_pairs = op_sets[logical_op_id].get_op_input_pairs()
+            op_inputs = op_sets[logical_op_id].get_op_inputs()
 
-            # break out of the loop if op_input_pairs is empty, as this means all records have been filtered out
-            if len(op_input_pairs) == 0:
+            # break out of the loop if op_inputs is empty, as this means all records have been filtered out
+            if len(op_inputs) == 0:
                 break
 
             # run sampled operators on sampled inputs
-            source_idx_to_record_sets_and_ops, _ = self._execute_op_set(op_input_pairs)
-
-            # FUTURE TODO: have this return the highest quality record set simply based on our posterior (or prior) belief on operator quality
-            # get the target record set for each source_idx
-            source_idx_to_target_record_set = self._get_target_record_sets(logical_op_id, source_idx_to_record_sets_and_ops, expected_outputs)
+            source_indices_to_record_set_tuples, _ = self._execute_op_set(unique_logical_op_id, op_inputs)
 
-            # TODO: make consistent across here and RandomSampling
-            # FUTURE TODO: move this outside of the loop (i.e. assume we only get quality label(s) after executing full program)
             # score the quality of each generated output
-            physical_op_cls = op_set[0].__class__
-            source_idx_to_record_sets = {
-                source_idx: list(map(lambda tup: tup[0], record_sets_and_ops))
-                for source_idx, record_sets_and_ops in source_idx_to_record_sets_and_ops.items()
-            }
-            source_idx_to_record_sets = self._score_quality(physical_op_cls, source_idx_to_record_sets, source_idx_to_target_record_set)
-
-            # flatten the lists of records and record_op_stats
-            all_records, all_record_op_stats = self._flatten_record_sets(source_idx_to_record_sets)
+            source_indices_to_all_record_sets = {
+                    source_indices: [(record_set, op) for record_set, op, _ in record_set_tuples]
+                    for source_indices, record_set_tuples in source_indices_to_record_set_tuples.items()
+                }
+            source_indices_to_all_record_sets, val_gen_stats = self._score_quality(validator, source_indices_to_all_record_sets)
+
+            # remove records that were read from the execution cache before adding to record op stats
+            new_record_op_stats = []
+            for _, record_set_tuples in source_indices_to_record_set_tuples.items():
+                for record_set, _, is_new in record_set_tuples:
+                    if is_new:
+                        new_record_op_stats.extend(record_set.record_op_stats)
 
             # update plan stats
-            plan_stats.add_record_op_stats(all_record_op_stats)
-
-            # add records (which are not filtered) to the cache, if allowed
-            self._add_records_to_cache(logical_op_id, all_records)
-
-            # FUTURE TODO: simply set input based on source_idx_to_target_record_set (b/c we won't have scores computed)
-            # provide the champion record sets as inputs to the next logical operator
-            if op_idx + 1 < len(plan):
-                next_logical_op_id = plan.logical_op_ids[op_idx + 1]
-                op_sets[next_logical_op_id].update_inputs(source_idx_to_record_sets)
-
-        # close the cache
-        self._close_cache(plan.logical_op_ids)
+            plan_stats.add_record_op_stats(unique_logical_op_id, new_record_op_stats)
+            plan_stats.add_validation_gen_stats(unique_logical_op_id, val_gen_stats)
+
+            # provide the best record sets as inputs to the next logical operator
+            next_unique_logical_op_id = plan.get_next_unique_logical_op_id(unique_logical_op_id)
+            if next_unique_logical_op_id is not None:
+                source_indices_to_all_record_sets = {
+                    source_indices: [record_set for record_set, _ in record_set_tuples]
+                    for source_indices, record_set_tuples in source_indices_to_all_record_sets.items()
+                }
+                op_sets[next_unique_logical_op_id].update_inputs(unique_logical_op_id, source_indices_to_all_record_sets)
 
         # finalize plan stats
         plan_stats.finish()
 
         return plan_stats
 
-    def execute_sentinel_plan(self, plan: SentinelPlan, expected_outputs: dict[int, dict] | None):
+    def execute_sentinel_plan(self, plan: SentinelPlan, train_dataset: dict[str, Dataset], validator: Validator): # expected_outputs: dict[int, dict] | None):
         """
         NOTE: this function currently requires us to set k and j properly in order to make
               comparison in our research against the corresponding sample budget in MAB.
@@ -177,8 +214,6 @@ class AllSamplingExecutionStrategy(SentinelExecutionStrategy):
         calls does not perfectly match the sample_budget. This may cause some minor discrepancies with
         the progress manager as a result.
         """
-        # for now, assert that the first operator in the plan is a ScanPhysicalOp
-        assert all(isinstance(op, ScanPhysicalOp) for op in plan.operator_sets[0]), "First operator in physical plan must be a ScanPhysicalOp"
         logger.info(f"Executing plan {plan.plan_id} with {self.max_workers} workers")
         logger.info(f"Plan Details: {plan}")
 
@@ -186,25 +221,48 @@ class AllSamplingExecutionStrategy(SentinelExecutionStrategy):
         plan_stats = SentinelPlanStats.from_plan(plan)
         plan_stats.start()
 
-        # get list of source indices which can be sampled from
-        source_indices = self._get_source_indices()
+        # get lists of source indices
+        dataset_id_to_source_indices = {}
+        for dataset_id, dataset in train_dataset.items():
+            total_num_samples = len(dataset)
+            source_indices = [f"{dataset_id}-{int(idx)}" for idx in np.arange(total_num_samples)]
+            dataset_id_to_source_indices[dataset_id] = source_indices
 
         # initialize set of physical operators for each logical operator
-        op_sets = {
-            logical_op_id: OpSet(op_set, source_indices)
-            for logical_op_id, op_set in plan
-        }
+        op_sets = {}
+        for topo_idx, (logical_op_id, op_set) in enumerate(plan):
+            unique_logical_op_id = f"{topo_idx}-{logical_op_id}"
+            source_unique_logical_op_ids = plan.get_source_unique_logical_op_ids(unique_logical_op_id)
+            sample_op = op_set[0]
+            if isinstance(sample_op, (ScanPhysicalOp, ContextScanOp)):
+                root_dataset_ids = plan.get_root_dataset_ids(unique_logical_op_id)
+                assert len(root_dataset_ids) == 1, f"Scan for {sample_op} has {len(root_dataset_ids)} > 1 root dataset ids"
+                root_dataset_id = root_dataset_ids[0]
+                source_indices = dataset_id_to_source_indices[root_dataset_id]
+                op_sets[unique_logical_op_id] = OpSet(op_set, source_unique_logical_op_ids, source_indices)
+            elif isinstance(sample_op, JoinOp):
+                assert len(source_unique_logical_op_ids) == 2, f"Join for {sample_op} has {len(source_unique_logical_op_ids)} != 2 source logical operators"
+                left_source_indices = op_sets[source_unique_logical_op_ids[0]].source_indices
+                right_source_indices = op_sets[source_unique_logical_op_ids[1]].source_indices
+                source_indices = []
+                for left_source_idx in left_source_indices:
+                    for right_source_idx in right_source_indices:
+                        source_indices.append((left_source_idx, right_source_idx))
+                op_sets[unique_logical_op_id] = OpSet(op_set, source_unique_logical_op_ids, source_indices)
+            else:
+                source_indices = op_sets[source_unique_logical_op_ids[0]].source_indices
+                op_sets[unique_logical_op_id] = OpSet(op_set, source_unique_logical_op_ids, source_indices)
 
         # initialize and start the progress manager
         self.progress_manager = create_progress_manager(plan, sample_budget=self.sample_budget, progress=self.progress)
         self.progress_manager.start()
 
-        # NOTE: we must handle progress manager outside of _exeecute_sentinel_plan to ensure that it is shut down correctly;
+        # NOTE: we must handle progress manager outside of _execute_sentinel_plan to ensure that it is shut down correctly;
         #       if we don't have the `finally:` branch, then program crashes can cause future program runs to fail because
         #       the progress manager cannot get a handle to the console 
         try:
             # execute sentinel plan by sampling records and operators
-            plan_stats = self._execute_sentinel_plan(plan, op_sets, expected_outputs, plan_stats)
+            plan_stats = self._execute_sentinel_plan(plan, op_sets, validator, plan_stats)
 
         finally:
             # finish progress tracking