palimpzest 0.6.3__py3-none-any.whl → 0.7.0__py3-none-any.whl

palimpzest/query/execution/parallel_execution_strategy.py

@@ -1,18 +1,21 @@
+import logging
 import multiprocessing
-import time
 from concurrent.futures import ThreadPoolExecutor, wait
 
 from palimpzest.constants import PARALLEL_EXECUTION_SLEEP_INTERVAL_SECS
-from palimpzest.core.data.dataclasses import OperatorStats, PlanStats
+from palimpzest.core.data.dataclasses import PlanStats
+from palimpzest.core.elements.records import DataRecord, DataRecordSet
 from palimpzest.query.execution.execution_strategy import ExecutionStrategy
 from palimpzest.query.operators.aggregate import AggregateOp
 from palimpzest.query.operators.limit import LimitScanOp
 from palimpzest.query.operators.physical import PhysicalOperator
 from palimpzest.query.operators.scan import ScanPhysicalOp
 from palimpzest.query.optimizer.plan import PhysicalPlan
+from palimpzest.utils.progress import create_progress_manager
 
+logger = logging.getLogger(__name__)
 
-class PipelinedParallelExecutionStrategy(ExecutionStrategy):
+class ParallelExecutionStrategy(ExecutionStrategy):
     """
     A parallel execution strategy that processes data through a pipeline of operators using thread-based parallelism.
     """
@@ -20,12 +23,12 @@ class PipelinedParallelExecutionStrategy(ExecutionStrategy):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.max_workers = (
-            self.get_parallel_max_workers()
+            self._get_parallel_max_workers()
             if self.max_workers is None
             else self.max_workers
         )
 
-    def get_parallel_max_workers(self):
+    def _get_parallel_max_workers(self):
         # for now, return the number of system CPUs;
         # in the future, we may want to consider the models the user has access to
         # and whether or not they will encounter rate-limits. If they will, we should
@@ -35,180 +38,148 @@ class PipelinedParallelExecutionStrategy(ExecutionStrategy):
         # changing the max_workers in response to 429 errors.
         return max(int(0.8 * multiprocessing.cpu_count()), 1)
 
-    def execute_plan(self, plan: PhysicalPlan, num_samples: int | float = float("inf"), plan_workers: int = 1):
-        """Initialize the stats and the execute the plan."""
-        if self.verbose:
-            print("----------------------")
-            print(f"PLAN[{plan.plan_id}] (n={num_samples}):")
-            print(plan)
-            print("---")
-
-        plan_start_time = time.time()
-
-        # initialize plan stats and operator stats
-        plan_stats = PlanStats(plan_id=plan.plan_id, plan_str=str(plan))
-        for op in plan.operators:
-            op_id = op.get_op_id()
-            op_name = op.op_name()
-            op_details = {k: str(v) for k, v in op.get_id_params().items()}
-            plan_stats.operator_stats[op_id] = OperatorStats(op_id=op_id, op_name=op_name, op_details=op_details)
-
-        # initialize list of output records and intermediate variables
+    def _any_queue_not_empty(self, queues: dict[str, list]) -> bool:
+        """Helper function to check if any queue is not empty."""
+        return any(len(queue) > 0 for queue in queues.values())
+
+    def _upstream_ops_finished(self, plan: PhysicalPlan, op_idx: int, input_queues: dict[str, list], future_queues: dict[str, list]) -> bool:
+        """Helper function to check if all upstream operators have finished processing their inputs."""
+        for upstream_op_idx in range(op_idx):
+            upstream_op_id = plan.operators[upstream_op_idx].get_op_id()
+            if len(input_queues[upstream_op_id]) > 0 or len(future_queues[upstream_op_id]) > 0:
+                return False
+
+        return True
+
+    def _process_future_results(self, operator: PhysicalOperator, future_queues: dict[str, list], plan_stats: PlanStats) -> list[DataRecord]:
+        """
+        Helper function which takes an operator, the future queues, and plan stats, and performs
+        the updates to plan stats and progress manager before returning the results from the finished futures.
+        """
+        # get the op_id for the operator
+        op_id = operator.get_op_id()
+
+        # this function is called when the future queue is not empty
+        # and the executor is not busy processing other futures
+        done_futures, not_done_futures = wait(future_queues[op_id], timeout=PARALLEL_EXECUTION_SLEEP_INTERVAL_SECS)
+
+        # add the unfinished futures back to the previous op's future queue
+        future_queues[op_id] = list(not_done_futures)
+
+        # add the finished futures to the input queue for this operator
+        output_records = []
+        for future in done_futures:
+            record_set: DataRecordSet = future.result()
+            records = record_set.data_records
+            record_op_stats = record_set.record_op_stats
+            num_outputs = sum(record.passed_operator for record in records)
+
+            # update the progress manager
+            self.progress_manager.incr(op_id, num_outputs=num_outputs, total_cost=record_set.get_total_cost())
+
+            # update plan stats
+            plan_stats.add_record_op_stats(record_op_stats)
+
+            # add records to the cache
+            self._add_records_to_cache(operator.target_cache_id, records)
+
+            # add records which aren't filtered to the output records
+            output_records.extend([record for record in records if record.passed_operator])
+        
+        return output_records
+
+    def _execute_plan(
+            self,
+            plan: PhysicalPlan,
+            input_queues: dict[str, list],
+            future_queues: dict[str, list],
+            plan_stats: PlanStats,
+        ) -> tuple[list[DataRecord], PlanStats]:
+        # process all of the input records using a thread pool
         output_records = []
-        source_records_scanned = 0
-
-        # initialize data structures to help w/processing DAG
-        processing_queue = []
-        op_id_to_futures_in_flight = {op.get_op_id(): 0 for op in plan.operators}
-        op_id_to_operator = {op.get_op_id(): op for op in plan.operators}
-        op_id_to_prev_operator = {
-            op.get_op_id(): plan.operators[idx - 1] if idx > 0 else None for idx, op in enumerate(plan.operators)
-        }
-        op_id_to_next_operator = {
-            op.get_op_id(): plan.operators[idx + 1] if idx + 1 < len(plan.operators) else None
-            for idx, op in enumerate(plan.operators)
-        }
-        op_id_to_op_idx = {op.get_op_id(): idx for idx, op in enumerate(plan.operators)}
-
-        # get handle to scan operator and pre-compute its op_id and size
-        source_operator = plan.operators[0]
-        assert isinstance(source_operator, ScanPhysicalOp), "First operator in physical plan must be a ScanPhysicalOp"
-        source_op_id = source_operator.get_op_id()
-        datareader_len = len(source_operator.datareader)
-
-        # get limit of final limit operator (if one exists)
-        final_limit = plan.operators[-1].limit if isinstance(plan.operators[-1], LimitScanOp) else None
-
-        # create thread pool w/max workers
-        futures = []
-        current_scan_idx = self.scan_start_idx
-        with ThreadPoolExecutor(max_workers=plan_workers) as executor:
-            # create initial (set of) future(s) to read first source record;
-            futures.append(executor.submit(PhysicalOperator.execute_op_wrapper, source_operator, current_scan_idx))
-            op_id_to_futures_in_flight[source_op_id] += 1
-            current_scan_idx += 1
-
-            # iterate until we have processed all operators on all records or come to an early stopping condition
-            while len(futures) > 0:
-                # get the set of futures that have (and have not) finished in the last PARALLEL_EXECUTION_SLEEP_INTERVAL_SECS
-                done_futures, not_done_futures = wait(futures, timeout=PARALLEL_EXECUTION_SLEEP_INTERVAL_SECS)
-
-                # cast not_done_futures from a set to a list so we can append to it
-                not_done_futures = list(not_done_futures)
-
-                # process finished futures, creating new ones as needed
-                new_futures = []
-                for future in done_futures:
-                    # get the result
-                    record_set, operator, _ = future.result()
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            logger.debug(f"Created thread pool with {self.max_workers} workers")
+
+            # execute the plan until either:
+            # 1. all records have been processed, or
+            # 2. the final limit operation has completed (we break out of the loop if this happens)
+            final_op = plan.operators[-1]
+            while self._any_queue_not_empty(input_queues) or self._any_queue_not_empty(future_queues):
+                for op_idx, operator in enumerate(plan.operators):
                     op_id = operator.get_op_id()
 
-                    # decrement future from mapping of futures in-flight
-                    op_id_to_futures_in_flight[op_id] -= 1
-
-                    # update plan stats
-                    prev_operator = op_id_to_prev_operator[op_id]
-                    plan_stats.operator_stats[op_id].add_record_op_stats(
-                        record_set.record_op_stats,
-                        source_op_id=prev_operator.get_op_id() if prev_operator is not None else None,
-                        plan_id=plan.plan_id,
-                    )
-
-                    # process each record output by the future's operator
-                    for record in record_set:
-                        # skip records which are filtered out
-                        if not getattr(record, "passed_operator", True):
-                            continue
-
-                        # add records (which are not filtered) to the cache, if allowed
-                        if not self.nocache:
-                            # self.datadir.append_cache(operator.target_cache_id, record)
-                            pass
-
-                        # add records to processing queue if there is a next_operator; otherwise add to output_records
-                        next_operator = op_id_to_next_operator[op_id]
-                        if next_operator is not None:
-                            processing_queue.append((next_operator, record))
-                        else:
-                            output_records.append(record)
-
-                    # if this operator was a source scan, update the number of source records scanned
-                    if op_id == source_op_id:
-                        source_records_scanned += len(record_set)
-
-                        # scan next record if we can still draw records from source
-                        if source_records_scanned < num_samples and current_scan_idx < datareader_len:
-                            new_futures.append(executor.submit(PhysicalOperator.execute_op_wrapper, source_operator, current_scan_idx))
-                            op_id_to_futures_in_flight[source_op_id] += 1
-                            current_scan_idx += 1
-
-                    # check early stopping condition based on final limit
-                    if final_limit is not None and len(output_records) >= final_limit:
-                        output_records = output_records[:final_limit]
-                        futures = []
-                        break
-
-                # process all records in the processing queue which are ready to be executed
-                temp_processing_queue = []
-                for operator, candidate in processing_queue:
-                    # if the candidate is not an input to an aggregate, execute it right away
-                    if not isinstance(operator, AggregateOp):
-                        future = executor.submit(PhysicalOperator.execute_op_wrapper, operator, candidate)
-                        new_futures.append(future)
-                        op_id_to_futures_in_flight[operator.get_op_id()] += 1
-
-                    # otherwise, put it back on the queue
-                    else:
-                        temp_processing_queue.append((operator, candidate))
-
-                # any remaining candidates are inputs to aggregate operators; for each aggregate operator
-                # determine if it is ready to execute -- and execute all of its candidates if so
-                processing_queue = []
-                agg_op_ids = set([operator.get_op_id() for operator, _ in temp_processing_queue])
-                for agg_op_id in agg_op_ids:
-                    agg_op_idx = op_id_to_op_idx[agg_op_id]
-
-                    # compute if all upstream operators' processing queues are empty and their in-flight futures are finished
-                    upstream_ops_are_finished = True
-                    for upstream_op_idx in range(agg_op_idx):
-                        upstream_op_id = plan.operators[upstream_op_idx].get_op_id()
-                        upstream_op_id_queue = list(
-                            filter(lambda tup: tup[0].get_op_id() == upstream_op_id, temp_processing_queue)
-                        )
-
-                        upstream_ops_are_finished = (
-                            upstream_ops_are_finished
-                            and len(upstream_op_id_queue) == 0
-                            and op_id_to_futures_in_flight[upstream_op_id] == 0
-                        )
-
-                    # get the subset of candidates for this aggregate operator
-                    candidate_tuples = list(filter(lambda tup: tup[0].get_op_id() == agg_op_id, temp_processing_queue))
-
-                    # execute the operator on the candidates if it's ready
-                    if upstream_ops_are_finished:
-                        operator = op_id_to_operator[agg_op_id]
-                        candidates = list(map(lambda tup: tup[1], candidate_tuples))
-                        future = executor.submit(PhysicalOperator.execute_op_wrapper, operator, candidates)
-                        new_futures.append(future)
-                        op_id_to_futures_in_flight[operator.get_op_id()] += 1
-
-                    # otherwise, add the candidates back to the processing queue
+                    # get any finished futures from the previous operator and add them to the input queue for this operator
+                    if not isinstance(operator, ScanPhysicalOp):
+                        prev_operator = plan.operators[op_idx - 1]
+                        records = self._process_future_results(prev_operator, future_queues, plan_stats)
+                        input_queues[op_id].extend(records)
+
+                    # for the final operator, add any finished futures to the output_records
+                    if operator.get_op_id() == final_op.get_op_id():
+                        records = self._process_future_results(operator, future_queues, plan_stats)
+                        output_records.extend(records)
+
+                    # if this operator does not have enough inputs to execute, then skip it
+                    num_inputs = len(input_queues[op_id])
+                    agg_op_not_ready = isinstance(operator, AggregateOp) and not self._upstream_ops_finished(plan, op_idx, input_queues, future_queues)
+                    if num_inputs == 0 or agg_op_not_ready:
+                        continue
+
+                    # if this operator is an aggregate, process all the records in the input queue
+                    if isinstance(operator, AggregateOp):
+                        input_records = [input_queues[op_id].pop(0) for _ in range(num_inputs)]
+                        future = executor.submit(operator, input_records)
+                        future_queues[op_id].append(future)
+
                     else:
-                        processing_queue.extend(candidate_tuples)
+                        input_record = input_queues[op_id].pop(0)
+                        future = executor.submit(operator, input_record)
+                        future_queues[op_id].append(future)
 
-                # update list of futures
-                not_done_futures.extend(new_futures)
-                futures = not_done_futures
+                # break out of loop if the final operator is a LimitScanOp and we've reached its limit
+                if isinstance(final_op, LimitScanOp) and len(output_records) == final_op.limit:
+                    break
 
-        # if caching was allowed, close the cache
-        if not self.nocache:
-            for _ in plan.operators:
-                # self.datadir.close_cache(operator.target_cache_id)
-                pass
+        # close the cache
+        self._close_cache([op.target_cache_id for op in plan.operators])
 
         # finalize plan stats
-        total_plan_time = time.time() - plan_start_time
-        plan_stats.finalize(total_plan_time)
+        plan_stats.finish()
+
+        return output_records, plan_stats
+
+
+    def execute_plan(self, plan: PhysicalPlan):
+        """Initialize the stats and execute the plan."""
+        # for now, assert that the first operator in the plan is a ScanPhysicalOp
+        assert isinstance(plan.operators[0], ScanPhysicalOp), "First operator in physical plan must be a ScanPhysicalOp"
+        logger.info(f"Executing plan {plan.plan_id} with {self.max_workers} workers")
+        logger.info(f"Plan Details: {plan}")
+
+        # initialize plan stats
+        plan_stats = PlanStats.from_plan(plan)
+        plan_stats.start()
+
+        # initialize input queues and future queues for each operation
+        input_queues = self._create_input_queues(plan)
+        future_queues = {op.get_op_id(): [] for op in plan.operators}
+
+        # initialize and start the progress manager
+        self.progress_manager = create_progress_manager(plan, num_samples=self.num_samples, progress=self.progress)
+        self.progress_manager.start()
+
+        # NOTE: we must handle progress manager outside of _exeecute_plan to ensure that it is shut down correctly;
+        #       if we don't have the `finally:` branch, then program crashes can cause future program runs to fail
+        #       because the progress manager cannot get a handle to the console 
+        try:
+            # execute plan
+            output_records, plan_stats = self._execute_plan(plan, input_queues, future_queues, plan_stats)
+
+        finally:
+            # finish progress tracking
+            self.progress_manager.finish()
+
+        logger.info(f"Done executing plan: {plan.plan_id}")
+        logger.debug(f"Plan stats: (plan_cost={plan_stats.total_plan_cost}, plan_time={plan_stats.total_plan_time})")
 
         return output_records, plan_stats

palimpzest/query/execution/random_sampling_execution_strategy.py

@@ -0,0 +1,240 @@
+import logging
+
+import numpy as np
+
+from palimpzest.core.data.dataclasses import SentinelPlanStats
+from palimpzest.core.elements.records import DataRecord, DataRecordSet
+from palimpzest.query.execution.execution_strategy import SentinelExecutionStrategy
+from palimpzest.query.operators.physical import PhysicalOperator
+from palimpzest.query.operators.scan import ScanPhysicalOp
+from palimpzest.query.optimizer.plan import SentinelPlan
+from palimpzest.utils.progress import create_progress_manager
+
+logger = logging.getLogger(__name__)
+
+class OpSet:
+    """
+    This class represents the set of operators which are currently in the frontier for a given logical operator.
+    Each operator in the frontier is an instance of a PhysicalOperator which either:
+
+    1. lies on the Pareto frontier of the set of sampled operators, or
+    2. has been sampled fewer than j times
+    """
+
+    def __init__(self, op_set: list[PhysicalOperator], source_indices: list[int], k: int, j: int, seed: int):
+        # set k and j, which are the initial number of operators in the frontier and the
+        # initial number of records to sample for each frontier operator
+        self.k = min(k, len(op_set))
+        self.j = min(j, len(source_indices))
+
+        # get order in which we will sample physical operators for this logical operator
+        sample_op_indices = self._get_op_index_order(op_set, seed)
+
+        # construct the set of operators
+        self.ops = [op_set[sample_idx] for sample_idx in sample_op_indices[:self.k]]
+
+        # store the order in which we will sample the source records
+        self.source_indices = source_indices
+
+        # set the initial inputs for this logical operator
+        is_scan_op = isinstance(op_set[0], ScanPhysicalOp)
+        self.source_idx_to_input = {source_idx: [source_idx] for source_idx in self.source_indices} if is_scan_op else {}
+
+
+    def _get_op_index_order(self, op_set: list[PhysicalOperator], seed: int) -> list[int]:
+        """
+        Returns a list of indices for the operators in the op_set.
+        """
+        rng = np.random.default_rng(seed=seed)
+        op_indices = np.arange(len(op_set))
+        rng.shuffle(op_indices)
+        return op_indices
+
+    def get_op_input_pairs(self) -> list[PhysicalOperator, DataRecord | int | None]:
+        """
+        Returns the list of frontier operators and their next input to process. If there are
+        any indices in `source_indices_to_sample` which this operator does not sample on its own, then
+        we also have this frontier process that source_idx's input with its max quality operator.
+        """
+        # get the list of (op, source_idx) pairs which this operator needs to execute
+        op_source_idx_pairs = []
+        for op in self.ops:
+            # construct list of inputs by looking up the input for the given source_idx
+            for sample_idx in range(self.j):
+                source_idx = self.source_indices[sample_idx]
+                op_source_idx_pairs.append((op, source_idx))
+
+        # fetch the corresponding (op, input) pairs
+        op_input_pairs = []
+        for op, source_idx in op_source_idx_pairs:
+            op_input_pairs.extend([(op, input_record) for input_record in self.source_idx_to_input[source_idx]])
+
+        return op_input_pairs
+
+    def pick_highest_quality_output(self, record_sets: list[DataRecordSet]) -> DataRecordSet:
+        # if there's only one operator in the set, we return its record_set
+        if len(record_sets) == 1:
+            return record_sets[0]
+
+        # NOTE: I don't like that this assumes the models are consistent in
+        #       how they order their record outputs for one-to-many converts;
+        #       eventually we can try out more robust schemes to account for
+        #       differences in ordering
+        # aggregate records at each index in the response
+        idx_to_records = {}
+        for record_set in record_sets:
+            for idx in range(len(record_set)):
+                record, record_op_stats = record_set[idx], record_set.record_op_stats[idx]
+                if idx not in idx_to_records:
+                    idx_to_records[idx] = [(record, record_op_stats)]
+                else:
+                    idx_to_records[idx].append((record, record_op_stats))
+
+        # compute highest quality answer at each index
+        out_records = []
+        out_record_op_stats = []
+        for idx in range(len(idx_to_records)):
+            records_lst, record_op_stats_lst = zip(*idx_to_records[idx])
+            max_quality_record, max_quality = records_lst[0], record_op_stats_lst[0].quality
+            max_quality_stats = record_op_stats_lst[0]
+            for record, record_op_stats in zip(records_lst[1:], record_op_stats_lst[1:]):
+                record_quality = record_op_stats.quality
+                if record_quality > max_quality:
+                    max_quality_record = record
+                    max_quality = record_quality
+                    max_quality_stats = record_op_stats
+            out_records.append(max_quality_record)
+            out_record_op_stats.append(max_quality_stats)
+
+        # create and return final DataRecordSet
+        return DataRecordSet(out_records, out_record_op_stats)
+
+    def update_inputs(self, source_idx_to_record_sets: dict[int, DataRecordSet]):
+        """
+        Update the inputs for this logical operator based on the outputs of the previous logical operator.
+        """
+        for source_idx, record_sets in source_idx_to_record_sets.items():
+            input = []
+            max_quality_record_set = self.pick_highest_quality_output(record_sets)
+            for record in max_quality_record_set:
+                input.append(record if record.passed_operator else None)
+
+            self.source_idx_to_input[source_idx] = input
+
+
+class RandomSamplingExecutionStrategy(SentinelExecutionStrategy):
+
+    def _get_source_indices(self):
+        """Get the list of source indices which the sentinel plan should execute over."""
+        # create list of all source indices and shuffle it
+        total_num_samples = len(self.val_datasource)
+        source_indices = list(np.arange(total_num_samples))
+        self.rng.shuffle(source_indices)
+
+        # slice the list of source indices to get the first j indices
+        j = min(self.j, len(source_indices))
+        source_indices = source_indices[:j]
+        
+        return source_indices
+
+    def _execute_sentinel_plan(self,
+            plan: SentinelPlan,
+            op_sets: dict[str, OpSet],
+            expected_outputs: dict[int, dict] | None,
+            plan_stats: SentinelPlanStats,
+        ) -> SentinelPlanStats:
+        # execute operator sets in sequence
+        for op_idx, (logical_op_id, op_set) in enumerate(plan):
+            # get frontier ops and their next input
+            op_input_pairs = op_sets[logical_op_id].get_op_input_pairs()
+
+            # break out of the loop if op_input_pairs is empty, as this means all records have been filtered out
+            if len(op_input_pairs) == 0:
+                break
+
+            # run sampled operators on sampled inputs
+            source_idx_to_record_sets_and_ops, _ = self._execute_op_set(op_input_pairs)
+
+            # FUTURE TODO: have this return the highest quality record set simply based on our posterior (or prior) belief on operator quality
+            # get the target record set for each source_idx
+            source_idx_to_target_record_set = self._get_target_record_sets(logical_op_id, source_idx_to_record_sets_and_ops, expected_outputs)
+
+            # TODO: make consistent across here and RandomSampling
+            # FUTURE TODO: move this outside of the loop (i.e. assume we only get quality label(s) after executing full program)
+            # score the quality of each generated output
+            physical_op_cls = op_set[0].__class__
+            source_idx_to_record_sets = {
+                source_idx: list(map(lambda tup: tup[0], record_sets_and_ops))
+                for source_idx, record_sets_and_ops in source_idx_to_record_sets_and_ops.items()
+            }
+            source_idx_to_record_sets = self._score_quality(physical_op_cls, source_idx_to_record_sets, source_idx_to_target_record_set)
+
+            # flatten the lists of records and record_op_stats
+            all_records, all_record_op_stats = self._flatten_record_sets(source_idx_to_record_sets)
+
+            # update plan stats
+            plan_stats.add_record_op_stats(all_record_op_stats)
+
+            # add records (which are not filtered) to the cache, if allowed
+            self._add_records_to_cache(logical_op_id, all_records)
+
+            # FUTURE TODO: simply set input based on source_idx_to_target_record_set (b/c we won't have scores computed)
+            # provide the champion record sets as inputs to the next logical operator
+            if op_idx + 1 < len(plan):
+                next_logical_op_id = plan.logical_op_ids[op_idx + 1]
+                op_sets[next_logical_op_id].update_inputs(source_idx_to_record_sets)
+
+        # close the cache
+        self._close_cache(plan.logical_op_ids)
+
+        # finalize plan stats
+        plan_stats.finish()
+
+        return plan_stats
+
+    def execute_sentinel_plan(self, plan: SentinelPlan, expected_outputs: dict[int, dict] | None):
+        """
+        NOTE: this function currently requires us to set k and j properly in order to make
+              comparison in our research against the corresponding sample budget in MAB.
+
+        NOTE: the number of samples will slightly exceed the sample_budget if the number of operator
+        calls does not perfectly match the sample_budget. This may cause some minor discrepancies with
+        the progress manager as a result.
+        """
+        # for now, assert that the first operator in the plan is a ScanPhysicalOp
+        assert all(isinstance(op, ScanPhysicalOp) for op in plan.operator_sets[0]), "First operator in physical plan must be a ScanPhysicalOp"
+        logger.info(f"Executing plan {plan.plan_id} with {self.max_workers} workers")
+        logger.info(f"Plan Details: {plan}")
+
+        # initialize plan stats
+        plan_stats = SentinelPlanStats.from_plan(plan)
+        plan_stats.start()
+
+        # get list of source indices which can be sampled from
+        source_indices = self._get_source_indices()
+
+        # initialize set of physical operators for each logical operator
+        op_sets = {
+            logical_op_id: OpSet(op_set, source_indices, self.k, self.j, self.seed)
+            for logical_op_id, op_set in plan
+        }
+
+        # initialize and start the progress manager
+        self.progress_manager = create_progress_manager(plan, sample_budget=self.sample_budget, progress=self.progress)
+        self.progress_manager.start()
+
+        # NOTE: we must handle progress manager outside of _exeecute_sentinel_plan to ensure that it is shut down correctly;
+        #       if we don't have the `finally:` branch, then program crashes can cause future program runs to fail because
+        #       the progress manager cannot get a handle to the console 
+        try:
+            # execute sentinel plan by sampling records and operators
+            plan_stats = self._execute_sentinel_plan(plan, op_sets, expected_outputs, plan_stats)
+
+        finally:
+            # finish progress tracking
+            self.progress_manager.finish()
+
+        logger.info(f"Done executing sentinel plan: {plan.plan_id}")
+        logger.debug(f"Plan stats: (plan_cost={plan_stats.total_plan_cost}, plan_time={plan_stats.total_plan_time})")
+
+        return plan_stats